From fc91b572ecd8d7cbb1aca20b974b4112087b5a38 Mon Sep 17 00:00:00 2001 From: Jaby Date: Tue, 3 Jan 2023 16:34:39 +0100 Subject: [PATCH] Implement LZ4 strip and make tools write errors to err instead of out --- src/Library/Library.code-workspace | 8 +- src/Library/src/BootLoader/gpu_boot.cpp | 2 +- src/Tools/cpp_out/Cargo.toml | 2 +- src/Tools/cpp_out/src/lib.rs | 26 +++-- src/Tools/jaby_engine_fconv/src/main.rs | 4 +- src/Tools/mkoverlay/src/main.rs | 4 +- src/Tools/psxcdgen_ex/src/main.rs | 4 +- .../psxcdgen_ex/src/types/overlay/mod.rs | 6 +- src/Tools/tool_helper/Cargo.toml | 9 +- src/Tools/tool_helper/src/bits.rs | 4 + src/Tools/tool_helper/src/compress.rs | 105 +++++++++++++++++- 11 files changed, 140 insertions(+), 34 deletions(-) diff --git a/src/Library/Library.code-workspace b/src/Library/Library.code-workspace index c34ce892..31e78c26 100644 --- a/src/Library/Library.code-workspace +++ b/src/Library/Library.code-workspace @@ -20,17 +20,13 @@ "label": "make", "type": "shell", "command": "./run_make.bat ${input:target} ${input:build cfg}", - "group": { - "kind": "build" - } + "group": "build" }, { "label": "Build Libary and Tools", "type": "shell", "command": "./build_all.bat ${input:target}", - "group": { - "kind": "build" - }, + "group": "build", "options": { "cwd": "../.." } diff --git a/src/Library/src/BootLoader/gpu_boot.cpp b/src/Library/src/BootLoader/gpu_boot.cpp index f59fb090..77cb7e84 100644 --- a/src/Library/src/BootLoader/gpu_boot.cpp +++ b/src/Library/src/BootLoader/gpu_boot.cpp @@ -17,7 +17,7 @@ namespace JabyEngine { static size_t decompress_logo() { LZ4Decompressor lz4_decomp(reinterpret_cast(&__boot_loader_end)); - const auto [progress, bytes_ready] = lz4_decomp.process(ArrayRange(SplashScreen + 11, (sizeof(SplashScreen) - 11 - 4))); + const auto [progress, bytes_ready] = lz4_decomp.process(ArrayRange(SplashScreen, sizeof(SplashScreen))); switch(progress) { case Progress::InProgress: printf("Decompressing still in progress... %llu\n", bytes_ready); diff --git a/src/Tools/cpp_out/Cargo.toml b/src/Tools/cpp_out/Cargo.toml index f2c8c94b..b0964cec 100644 --- a/src/Tools/cpp_out/Cargo.toml +++ b/src/Tools/cpp_out/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cpp_out" -version = "1.0.0" +version = "1.0.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/src/Tools/cpp_out/src/lib.rs b/src/Tools/cpp_out/src/lib.rs index 99a7d49f..38cab58e 100644 --- a/src/Tools/cpp_out/src/lib.rs +++ b/src/Tools/cpp_out/src/lib.rs @@ -2,6 +2,8 @@ use tool_helper::{Input, Output}; use std::io::Read; pub use tool_helper::{Error, format_if_error}; +const EMPTY_CONTENT_ERROR_STRING:&'static str = "Content is not allowed to be empty"; + pub enum LineFeed { Unix, Windows @@ -32,8 +34,9 @@ const WINDOWS_LINEFEED: &'static str = "\r\n"; const C_DECLARATIONS: FileDeclarations = FileDeclarations{include: "", var_type: "char"}; const CPP_DECLARATIONS: FileDeclarations = FileDeclarations{include: "#include ", var_type: "uint8_t"}; -fn output_bytes(input: Input, output: &mut Output, line_feed: &str) -> Result<(), std::io::Error> { +fn output_bytes(input: Input, output: &mut Output, line_feed: &str) -> Result { let mut byte_line_count = 0; + let mut bytes_written = 0; write!(output, "\t")?; for byte in input.bytes() { @@ -43,6 +46,7 @@ fn output_bytes(input: Input, output: &mut Output, line_feed: &str) -> Result<() write!(output, "{:#04X}", byte?)?; byte_line_count += 1; + bytes_written += 1; if byte_line_count >= 16 { write!(output, ",{}\t", line_feed)?; @@ -50,10 +54,10 @@ fn output_bytes(input: Input, output: &mut Output, line_feed: &str) -> Result<() } } - Ok(()) + Ok(bytes_written) } -fn write_header_file(input: Input, output: &mut Output, file_name: String, declarations: &FileDeclarations, data_name: String, line_feed: &str) -> Result<(), std::io::Error> { +fn write_header_file(input: Input, output: &mut Output, file_name: String, declarations: &FileDeclarations, data_name: String, line_feed: &str) -> Result<(), Error> { let file_name = file_name.to_uppercase().replace('.', "_"); write!(output, "#ifndef __{}__{}", file_name, line_feed)?; @@ -61,19 +65,25 @@ fn write_header_file(input: Input, output: &mut Output, file_name: String, decla write!(output, "{}{}{}", declarations.include, line_feed, line_feed)?; write!(output, "static const {} {}[] = {{{}", declarations.var_type, data_name, line_feed)?; - output_bytes(input, output, line_feed)?; + if output_bytes(input, output, line_feed)? == 0 { + return Err(Error::from_str(EMPTY_CONTENT_ERROR_STRING)); + } write!(output, "{}}};{}", line_feed, line_feed)?; - write!(output, "#endif // !__{}__{}", file_name, line_feed) + write!(output, "#endif // !__{}__{}", file_name, line_feed)?; + Ok(()) } -fn write_source_file(input: Input, output: &mut Output, declarations: &FileDeclarations, data_name: String, line_feed: &str) -> Result<(), std::io::Error> { +fn write_source_file(input: Input, output: &mut Output, declarations: &FileDeclarations, data_name: String, line_feed: &str) -> Result<(), Error> { write!(output, "{}{}{}", declarations.include, line_feed, line_feed)?; write!(output, "const {} {}[] = {{{}", declarations.var_type, data_name, line_feed)?; - output_bytes(input, output, line_feed)?; + if output_bytes(input, output, line_feed)? == 0 { + return Err(Error::from_str(EMPTY_CONTENT_ERROR_STRING)); + } - write!(output, "{}}};", line_feed) + write!(output, "{}}};", line_feed)?; + Ok(()) } pub fn convert(cfg: Configuration, input: Input, mut output: Output) -> Result<(), Error> { diff --git a/src/Tools/jaby_engine_fconv/src/main.rs b/src/Tools/jaby_engine_fconv/src/main.rs index 9449bea9..8db9f729 100644 --- a/src/Tools/jaby_engine_fconv/src/main.rs +++ b/src/Tools/jaby_engine_fconv/src/main.rs @@ -52,7 +52,7 @@ fn run_main() -> Result<(), Error> { // We encoded the file to a temporary buffer and now need to write it if cmd.compress_lz4 { - let buffer = tool_helper::compress::lz4(&buffer, 16)?; + let buffer = tool_helper::compress::psx_default::lz4(&buffer)?; output_file.write(&buffer)?; } @@ -69,7 +69,7 @@ fn run_main() -> Result<(), Error> { fn main() { if let Err(error) = run_main() { - println!("{}", error.text); + eprintln!("{}", error.text); std::process::exit(error.exit_code); } } \ No newline at end of file diff --git a/src/Tools/mkoverlay/src/main.rs b/src/Tools/mkoverlay/src/main.rs index dab87161..315fdfd5 100644 --- a/src/Tools/mkoverlay/src/main.rs +++ b/src/Tools/mkoverlay/src/main.rs @@ -30,11 +30,11 @@ fn main() { Ok(cmd_line) => { match run_main(cmd_line) { Ok(_) => (), - Err(error) => println!("{}", error) + Err(error) => eprintln!("{}", error) } }, Err(error) => { - println!("{}", error); + eprintln!("{}", error); } } } \ No newline at end of file diff --git a/src/Tools/psxcdgen_ex/src/main.rs b/src/Tools/psxcdgen_ex/src/main.rs index 1671f1ab..0f606515 100644 --- a/src/Tools/psxcdgen_ex/src/main.rs +++ b/src/Tools/psxcdgen_ex/src/main.rs @@ -57,11 +57,11 @@ fn main() { Ok(cmd_line) => { match run_main(cmd_line) { Ok(_) => (), - Err(error) => println!("{}", error) + Err(error) => eprintln!("{}", error) } }, Err(error) => { - println!("{}", error); + eprintln!("{}", error); } } } \ No newline at end of file diff --git a/src/Tools/psxcdgen_ex/src/types/overlay/mod.rs b/src/Tools/psxcdgen_ex/src/types/overlay/mod.rs index 759bab03..c1d46a23 100644 --- a/src/Tools/psxcdgen_ex/src/types/overlay/mod.rs +++ b/src/Tools/psxcdgen_ex/src/types/overlay/mod.rs @@ -6,8 +6,6 @@ use tool_helper::{Error, format_if_error, read_file}; pub type LBANameVec = Vec; -const COMPRESSION_LEVEL:u32 = 16; - #[repr(packed)] struct OverlayHeader { _start_adr: u32, @@ -39,7 +37,7 @@ impl LBAEntry { pub fn load_from(file_name: &str, file_path: PathBuf, lba_source: PathBuf) -> Result { let content = load_content(&file_path)?; let lba_names = load_lba_names(lba_source)?; - let content_size = format_if_error!(tool_helper::compress::lz4(&content, COMPRESSION_LEVEL), "Compressing {} failed with \"{error_text}\"", file_path.to_string_lossy())?.len(); + let content_size = format_if_error!(tool_helper::compress::psx_default::lz4(&content), "Compressing {} failed with \"{error_text}\"", file_path.to_string_lossy())?.len(); Ok(File::new_overlay(file_name, content, lba_names, content_size)?) } @@ -66,7 +64,7 @@ pub fn update_content(content: &mut Vec, lba_names: &LBANameVec, file_map: & } } - Ok(tool_helper::compress::lz4(content, COMPRESSION_LEVEL)?) + Ok(tool_helper::compress::psx_default::lz4(content)?) } fn load_content(file_path: &PathBuf) -> Result, Error> { diff --git a/src/Tools/tool_helper/Cargo.toml b/src/Tools/tool_helper/Cargo.toml index 415b0407..80ab4565 100644 --- a/src/Tools/tool_helper/Cargo.toml +++ b/src/Tools/tool_helper/Cargo.toml @@ -1,11 +1,12 @@ [package] name = "tool_helper" -version = "0.6.0" +version = "0.7.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -cdtypes = {path = "../cdtypes"} -lz4 = "*" -paste = "*" \ No newline at end of file +byteorder = "*" +cdtypes = {path = "../cdtypes"} +lz4 = "*" +paste = "*" \ No newline at end of file diff --git a/src/Tools/tool_helper/src/bits.rs b/src/Tools/tool_helper/src/bits.rs index ec926ce7..3f03cf56 100644 --- a/src/Tools/tool_helper/src/bits.rs +++ b/src/Tools/tool_helper/src/bits.rs @@ -27,6 +27,10 @@ macro_rules! create_bit_functions { pub const fn [< get_value_ $type_val >](src: $type_val, range: &BitRange) -> $type_val { (src & [< get_mask_ $type_val >](range)) >> range.start } + + pub const fn [< bit_set_ $type_val >](src: $type_val, bit: usize) -> bool { + src & (1 << bit) != 0 + } } }; } diff --git a/src/Tools/tool_helper/src/compress.rs b/src/Tools/tool_helper/src/compress.rs index 5feb6737..fefc72c8 100644 --- a/src/Tools/tool_helper/src/compress.rs +++ b/src/Tools/tool_helper/src/compress.rs @@ -1,8 +1,11 @@ -use super::Error; -use lz4::EncoderBuilder; +use super::{bits::*, Error}; +use byteorder::{ByteOrder, LittleEndian}; +use lz4::{BlockMode, EncoderBuilder}; -pub fn lz4(data: &Vec, compression_level: u32) -> Result, Error> { - let mut lz4_encoder = EncoderBuilder::new().level(compression_level).checksum(lz4::ContentChecksum::NoChecksum).build(Vec::::new())?; +pub use lz4::BlockSize; + +pub fn lz4(data: &Vec, block_size: BlockSize, compression_level: u32) -> Result, Error> { + let mut lz4_encoder = EncoderBuilder::new().level(compression_level).checksum(lz4::ContentChecksum::NoChecksum).block_size(block_size).block_mode(BlockMode::Linked).build(Vec::::new())?; std::io::copy(&mut&data[..], &mut lz4_encoder)?; let (output, result) = lz4_encoder.finish(); @@ -11,4 +14,98 @@ pub fn lz4(data: &Vec, compression_level: u32) -> Result, Error> { Ok(()) => Ok(output), Err(error) => Err(Error::from_error(error)) } +} + +pub fn strip_lz4(compressed_data: Vec) -> Result, Error> { + fn read_header(raw_data: &[u8]) -> Result<(usize, bool, bool), Error> { + const FRAME_HEADER_MIN_SIZE:usize = 7; + const MAGIC_NUMBER:u32 = 0x184D2204u32; + const BLOCK_CHECKSUM_BIT:usize = 4; + const CONTENT_SIZE_BIT:usize = 3; + const CONTENT_CHECKSUM_BIT:usize = 2; + const DICT_ID_BIT:usize = 0; + + let mut bytes_to_skip = FRAME_HEADER_MIN_SIZE; + if raw_data.len() >= FRAME_HEADER_MIN_SIZE { + let magic_number = LittleEndian::read_u32(raw_data); + if magic_number.to_le() == MAGIC_NUMBER { + let raw_data = &raw_data[std::mem::size_of::()..]; + let flg = raw_data[0]; + let _bd = raw_data[1]; + + if bit_set_u8(flg, CONTENT_SIZE_BIT) { + bytes_to_skip += std::mem::size_of::(); + } + + if bit_set_u8(flg, DICT_ID_BIT) { + bytes_to_skip += std::mem::size_of::(); + } + + if raw_data.len() < bytes_to_skip { + Err(Error::from_text(format!("Can't skip {} bytes of frame header because there are only {} bytes left", bytes_to_skip, raw_data.len()))) + } + + else { + Ok((bytes_to_skip, bit_set_u8(flg, BLOCK_CHECKSUM_BIT), bit_set_u8(flg, CONTENT_CHECKSUM_BIT))) + } + } + + else { + Err(Error::from_text(format!("Magic number needs to be {:#08x} - but {:#08x} was found!", MAGIC_NUMBER, magic_number.to_le()))) + } + } + + else { + Err(Error::from_str("Not enough bytes for minimum header size")) + } + } + + let (bytes_to_skip, has_block_checksum, has_content_checksum) = read_header(&compressed_data)?; + let mut raw_data = &compressed_data[bytes_to_skip..]; + let end_len = { + if has_content_checksum { + std::mem::size_of::()*2 + } + + else { + std::mem::size_of::() + } + }; + let mut new_data = Vec::new(); + + while raw_data.len() > end_len { + let block_size = LittleEndian::read_u32(raw_data); + raw_data = &raw_data[std::mem::size_of::()..]; + + if bit_set_u32(block_size, 31) { + return Err(Error::from_str("Stripping uncompressed data is not supported!")); + } + + let block_size = block_size as usize; + new_data.extend_from_slice(&raw_data[0..block_size]); + raw_data = &raw_data[block_size..]; + + if has_block_checksum { + raw_data = &raw_data[std::mem::size_of::()..]; + } + } + + if LittleEndian::read_u32(raw_data) != 0 { + Err(Error::from_str("EOF is not zero!")) + } + + else { + Ok(new_data) + } +} + +pub mod psx_default { + use super::*; + + const DEFAULT_COMPRESSION_LEVEL:u32 = 16; + const DEFAULT_BLOCK_SIZE:BlockSize = BlockSize::Max4MB; + + pub fn lz4(data: &Vec) -> Result, Error> { + strip_lz4(super::lz4(data, DEFAULT_BLOCK_SIZE, DEFAULT_COMPRESSION_LEVEL)?) + } } \ No newline at end of file