Implement LZ4 strip and make tools write errors to err instead of out

src/Library/Library.code-workspace

@@ -20,17 +20,13 @@
 				"label": "make",
 				"type": "shell",
 				"command": "./run_make.bat ${input:target} ${input:build cfg}",
-				"group": {
+				"group": "build"
-					"kind": "build"
-				}
 			},
 			{
 				"label": "Build Libary and Tools",
 				"type": "shell",
 				"command": "./build_all.bat ${input:target}",
-				"group": {
+				"group": "build",
-					"kind": "build"
-				},
 				"options": {
 					"cwd": "../.."
 				}

src/Library/src/BootLoader/gpu_boot.cpp

@@ -17,7 +17,7 @@ namespace JabyEngine {
         static size_t decompress_logo() {
             LZ4Decompressor lz4_decomp(reinterpret_cast<uint8_t*>(&__boot_loader_end));
 
-            const auto [progress, bytes_ready] = lz4_decomp.process(ArrayRange(SplashScreen + 11, (sizeof(SplashScreen) - 11 - 4)));
+            const auto [progress, bytes_ready] = lz4_decomp.process(ArrayRange(SplashScreen, sizeof(SplashScreen)));
             switch(progress) {
                 case Progress::InProgress:
                     printf("Decompressing still in progress... %llu\n", bytes_ready);

src/Tools/cpp_out/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cpp_out"
-version = "1.0.0"
+version = "1.0.1"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

src/Tools/cpp_out/src/lib.rs

@@ -2,6 +2,8 @@ use tool_helper::{Input, Output};
 use std::io::Read;
 pub use tool_helper::{Error, format_if_error};
 
+const EMPTY_CONTENT_ERROR_STRING:&'static str = "Content is not allowed to be empty";
+
 pub enum LineFeed {
     Unix,
     Windows
@@ -32,8 +34,9 @@ const WINDOWS_LINEFEED: &'static str = "\r\n";
 const   C_DECLARATIONS: FileDeclarations = FileDeclarations{include: "", var_type: "char"};
 const CPP_DECLARATIONS: FileDeclarations = FileDeclarations{include: "#include <stdint.h>", var_type: "uint8_t"};
 
-fn output_bytes(input: Input, output: &mut Output, line_feed: &str) -> Result<(), std::io::Error> {
+fn output_bytes(input: Input, output: &mut Output, line_feed: &str) -> Result<usize, std::io::Error> {
     let mut byte_line_count = 0;
+    let mut bytes_written   = 0;
 
     write!(output, "\t")?;
     for byte in input.bytes() {
@@ -43,6 +46,7 @@ fn output_bytes(input: Input, output: &mut Output, line_feed: &str) -> Result<()
 
         write!(output, "{:#04X}", byte?)?;
         byte_line_count += 1;
+        bytes_written   += 1;
 
         if byte_line_count >= 16 {
             write!(output, ",{}\t", line_feed)?;
@@ -50,10 +54,10 @@ fn output_bytes(input: Input, output: &mut Output, line_feed: &str) -> Result<()
         }
     }
     
-    Ok(())
+    Ok(bytes_written)
 }
 
-fn write_header_file(input: Input, output: &mut Output, file_name: String, declarations: &FileDeclarations, data_name: String, line_feed: &str) -> Result<(), std::io::Error> {
+fn write_header_file(input: Input, output: &mut Output, file_name: String, declarations: &FileDeclarations, data_name: String, line_feed: &str) -> Result<(), Error> {
     let file_name = file_name.to_uppercase().replace('.', "_");
 
     write!(output, "#ifndef __{}__{}", file_name, line_feed)?;
@@ -61,19 +65,25 @@ fn write_header_file(input: Input, output: &mut Output, file_name: String, decla
     write!(output, "{}{}{}", declarations.include, line_feed, line_feed)?;
     write!(output, "static const {} {}[] = {{{}", declarations.var_type, data_name, line_feed)?;
 
-    output_bytes(input, output, line_feed)?;
+    if output_bytes(input, output, line_feed)? == 0 {
+        return Err(Error::from_str(EMPTY_CONTENT_ERROR_STRING));
+    }
 
     write!(output, "{}}};{}", line_feed, line_feed)?;
-    write!(output, "#endif // !__{}__{}", file_name, line_feed)
+    write!(output, "#endif // !__{}__{}", file_name, line_feed)?;
+    Ok(())
 }
 
-fn write_source_file(input: Input, output: &mut Output, declarations: &FileDeclarations, data_name: String, line_feed: &str) -> Result<(), std::io::Error> {
+fn write_source_file(input: Input, output: &mut Output, declarations: &FileDeclarations, data_name: String, line_feed: &str) -> Result<(), Error> {
     write!(output, "{}{}{}", declarations.include, line_feed, line_feed)?;
     write!(output, "const {} {}[] = {{{}", declarations.var_type, data_name, line_feed)?;
 
-    output_bytes(input, output, line_feed)?;
+    if output_bytes(input, output, line_feed)? == 0 {
+        return Err(Error::from_str(EMPTY_CONTENT_ERROR_STRING));
+    }
 
-    write!(output, "{}}};", line_feed)
+    write!(output, "{}}};", line_feed)?;
+    Ok(())
 }
 
 pub fn convert(cfg: Configuration, input: Input, mut output: Output) -> Result<(), Error> {

src/Tools/jaby_engine_fconv/src/main.rs

@@ -52,7 +52,7 @@ fn run_main() -> Result<(), Error> {
 
             // We encoded the file to a temporary buffer and now need to write it
             if cmd.compress_lz4 {
-                let buffer = tool_helper::compress::lz4(&buffer, 16)?;
+                let buffer = tool_helper::compress::psx_default::lz4(&buffer)?;
                 output_file.write(&buffer)?;
             }
 
@@ -69,7 +69,7 @@ fn run_main() -> Result<(), Error> {
 
 fn main() {
     if let Err(error) = run_main() {
-        println!("{}", error.text);
+        eprintln!("{}", error.text);
         std::process::exit(error.exit_code);
     }
 }

src/Tools/mkoverlay/src/main.rs

@@ -30,11 +30,11 @@ fn main() {
         Ok(cmd_line) => {
             match run_main(cmd_line) {
                 Ok(_)      => (),
-                Err(error) => println!("{}", error)
+                Err(error) => eprintln!("{}", error)
             }
         },
         Err(error) => {
-            println!("{}", error);
+            eprintln!("{}", error);
         }
     }
 }

src/Tools/psxcdgen_ex/src/main.rs

@@ -57,11 +57,11 @@ fn main() {
         Ok(cmd_line) => {
             match run_main(cmd_line) {
                 Ok(_)      => (),
-                Err(error) => println!("{}", error)
+                Err(error) => eprintln!("{}", error)
             }
         },
         Err(error) => {
-            println!("{}", error);
+            eprintln!("{}", error);
         }
     }
 }

src/Tools/psxcdgen_ex/src/types/overlay/mod.rs

@@ -6,8 +6,6 @@ use tool_helper::{Error, format_if_error, read_file};
 
 pub type LBANameVec = Vec<String>;
 
-const COMPRESSION_LEVEL:u32 = 16;
-
 #[repr(packed)]
 struct OverlayHeader {
     _start_adr: u32,
@@ -39,7 +37,7 @@ impl LBAEntry {
 pub fn load_from(file_name: &str, file_path: PathBuf, lba_source: PathBuf) -> Result<File, Error> {
     let content      = load_content(&file_path)?;
     let lba_names    = load_lba_names(lba_source)?;
-    let content_size = format_if_error!(tool_helper::compress::lz4(&content, COMPRESSION_LEVEL), "Compressing {} failed with \"{error_text}\"", file_path.to_string_lossy())?.len();
+    let content_size = format_if_error!(tool_helper::compress::psx_default::lz4(&content), "Compressing {} failed with \"{error_text}\"", file_path.to_string_lossy())?.len();
     
     Ok(File::new_overlay(file_name, content, lba_names, content_size)?)
 }
@@ -66,7 +64,7 @@ pub fn update_content(content: &mut Vec<u8>, lba_names: &LBANameVec, file_map: &
         }
     }
 
-    Ok(tool_helper::compress::lz4(content, COMPRESSION_LEVEL)?)
+    Ok(tool_helper::compress::psx_default::lz4(content)?)
 }
 
 fn load_content(file_path: &PathBuf) -> Result<Vec<u8>, Error> {

src/Tools/tool_helper/Cargo.toml

@@ -1,11 +1,12 @@
 [package]
 name = "tool_helper"
-version = "0.6.0"
+version = "0.7.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-cdtypes = {path = "../cdtypes"}
+byteorder  = "*"
-lz4     = "*"
+cdtypes    = {path = "../cdtypes"}
-paste   = "*"
+lz4        = "*"
+paste      = "*"

src/Tools/tool_helper/src/bits.rs

@@ -27,6 +27,10 @@ macro_rules! create_bit_functions {
             pub const fn [< get_value_ $type_val >](src: $type_val, range: &BitRange) -> $type_val {
                 (src & [< get_mask_ $type_val >](range)) >> range.start
             }
+
+            pub const fn [< bit_set_ $type_val >](src: $type_val, bit: usize) -> bool {
+                src & (1 << bit) != 0
+            }
         }
     };
 }

src/Tools/tool_helper/src/compress.rs

@@ -1,8 +1,11 @@
-use super::Error;
+use super::{bits::*, Error};
-use lz4::EncoderBuilder;
+use byteorder::{ByteOrder, LittleEndian};
+use lz4::{BlockMode, EncoderBuilder};
 
-pub fn lz4(data: &Vec<u8>, compression_level: u32) -> Result<Vec<u8>, Error> {
+pub use lz4::BlockSize;
-    let mut lz4_encoder = EncoderBuilder::new().level(compression_level).checksum(lz4::ContentChecksum::NoChecksum).build(Vec::<u8>::new())?;
+
+pub fn lz4(data: &Vec<u8>, block_size: BlockSize, compression_level: u32) -> Result<Vec<u8>, Error> {
+    let mut lz4_encoder = EncoderBuilder::new().level(compression_level).checksum(lz4::ContentChecksum::NoChecksum).block_size(block_size).block_mode(BlockMode::Linked).build(Vec::<u8>::new())?;
     
     std::io::copy(&mut&data[..], &mut lz4_encoder)?;
     let (output, result) = lz4_encoder.finish();
@@ -12,3 +15,97 @@ pub fn lz4(data: &Vec<u8>, compression_level: u32) -> Result<Vec<u8>, Error> {
         Err(error) => Err(Error::from_error(error))
     }
 }
+
+pub fn strip_lz4(compressed_data: Vec<u8>) -> Result<Vec<u8>, Error> {
+    fn read_header(raw_data: &[u8]) -> Result<(usize, bool, bool), Error> {
+        const FRAME_HEADER_MIN_SIZE:usize = 7;
+        const MAGIC_NUMBER:u32            = 0x184D2204u32;
+        const BLOCK_CHECKSUM_BIT:usize    = 4;
+        const CONTENT_SIZE_BIT:usize      = 3;
+        const CONTENT_CHECKSUM_BIT:usize  = 2;
+        const DICT_ID_BIT:usize           = 0;
+        
+        let mut bytes_to_skip = FRAME_HEADER_MIN_SIZE;
+        if raw_data.len() >= FRAME_HEADER_MIN_SIZE {
+            let magic_number = LittleEndian::read_u32(raw_data);
+            if magic_number.to_le() == MAGIC_NUMBER {
+                let raw_data          = &raw_data[std::mem::size_of::<u32>()..];
+                let flg               = raw_data[0];
+                let _bd               = raw_data[1];                
+
+                if bit_set_u8(flg, CONTENT_SIZE_BIT) {
+                    bytes_to_skip += std::mem::size_of::<u64>();
+                }
+
+                if bit_set_u8(flg, DICT_ID_BIT) {
+                    bytes_to_skip += std::mem::size_of::<u32>();
+                }
+
+                if raw_data.len() < bytes_to_skip {
+                    Err(Error::from_text(format!("Can't skip {} bytes of frame header because there are only {} bytes left", bytes_to_skip, raw_data.len())))
+                }
+
+                else {
+                    Ok((bytes_to_skip, bit_set_u8(flg, BLOCK_CHECKSUM_BIT), bit_set_u8(flg, CONTENT_CHECKSUM_BIT)))
+                }
+            }
+
+            else {
+                Err(Error::from_text(format!("Magic number needs to be {:#08x} - but {:#08x} was found!", MAGIC_NUMBER, magic_number.to_le())))
+            }
+        }
+
+        else {
+            Err(Error::from_str("Not enough bytes for minimum header size"))
+        }
+    }
+    
+    let (bytes_to_skip, has_block_checksum, has_content_checksum) = read_header(&compressed_data)?;
+    let mut raw_data                                              = &compressed_data[bytes_to_skip..];
+    let end_len                                                   = {
+        if has_content_checksum {
+            std::mem::size_of::<u32>()*2
+        }
+
+        else {
+            std::mem::size_of::<u32>()
+        }
+    };
+    let mut new_data = Vec::new();
+
+    while raw_data.len() > end_len {
+        let block_size = LittleEndian::read_u32(raw_data);
+        raw_data       = &raw_data[std::mem::size_of::<u32>()..];
+
+        if bit_set_u32(block_size, 31) {
+            return Err(Error::from_str("Stripping uncompressed data is not supported!"));
+        }
+
+        let block_size = block_size as usize;
+        new_data.extend_from_slice(&raw_data[0..block_size]);
+        raw_data = &raw_data[block_size..];
+
+        if has_block_checksum {
+            raw_data = &raw_data[std::mem::size_of::<u32>()..];
+        }
+    }
+
+    if LittleEndian::read_u32(raw_data) != 0 {
+        Err(Error::from_str("EOF is not zero!"))
+    }
+
+    else {
+        Ok(new_data)
+    }
+}
+
+pub mod psx_default {
+    use super::*;
+
+    const DEFAULT_COMPRESSION_LEVEL:u32 = 16;
+    const DEFAULT_BLOCK_SIZE:BlockSize  = BlockSize::Max4MB;
+
+    pub fn lz4(data: &Vec<u8>) -> Result<Vec<u8>, Error> {
+        strip_lz4(super::lz4(data, DEFAULT_BLOCK_SIZE, DEFAULT_COMPRESSION_LEVEL)?)
+    }
+}