Implement LZ4 strip and make tools write errors to err instead of out

src/Tools/tool_helper/Cargo.toml

@@ -1,11 +1,12 @@
 [package]
 name = "tool_helper"
-version = "0.6.0"
+version = "0.7.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-cdtypes = {path = "../cdtypes"}
-lz4     = "*"
-paste   = "*"
+byteorder  = "*"
+cdtypes    = {path = "../cdtypes"}
+lz4        = "*"
+paste      = "*"

src/Tools/tool_helper/src/bits.rs

@@ -27,6 +27,10 @@ macro_rules! create_bit_functions {
             pub const fn [< get_value_ $type_val >](src: $type_val, range: &BitRange) -> $type_val {
                 (src & [< get_mask_ $type_val >](range)) >> range.start
             }
+
+            pub const fn [< bit_set_ $type_val >](src: $type_val, bit: usize) -> bool {
+                src & (1 << bit) != 0
+            }
         }
     };
 }

src/Tools/tool_helper/src/compress.rs

@@ -1,8 +1,11 @@
-use super::Error;
-use lz4::EncoderBuilder;
+use super::{bits::*, Error};
+use byteorder::{ByteOrder, LittleEndian};
+use lz4::{BlockMode, EncoderBuilder};
 
-pub fn lz4(data: &Vec<u8>, compression_level: u32) -> Result<Vec<u8>, Error> {
-    let mut lz4_encoder = EncoderBuilder::new().level(compression_level).checksum(lz4::ContentChecksum::NoChecksum).build(Vec::<u8>::new())?;
+pub use lz4::BlockSize;
+
+pub fn lz4(data: &Vec<u8>, block_size: BlockSize, compression_level: u32) -> Result<Vec<u8>, Error> {
+    let mut lz4_encoder = EncoderBuilder::new().level(compression_level).checksum(lz4::ContentChecksum::NoChecksum).block_size(block_size).block_mode(BlockMode::Linked).build(Vec::<u8>::new())?;
     
     std::io::copy(&mut&data[..], &mut lz4_encoder)?;
     let (output, result) = lz4_encoder.finish();
@@ -11,4 +14,98 @@ pub fn lz4(data: &Vec<u8>, compression_level: u32) -> Result<Vec<u8>, Error> {
         Ok(()) => Ok(output),
         Err(error) => Err(Error::from_error(error))
     }
+}
+
+pub fn strip_lz4(compressed_data: Vec<u8>) -> Result<Vec<u8>, Error> {
+    fn read_header(raw_data: &[u8]) -> Result<(usize, bool, bool), Error> {
+        const FRAME_HEADER_MIN_SIZE:usize = 7;
+        const MAGIC_NUMBER:u32            = 0x184D2204u32;
+        const BLOCK_CHECKSUM_BIT:usize    = 4;
+        const CONTENT_SIZE_BIT:usize      = 3;
+        const CONTENT_CHECKSUM_BIT:usize  = 2;
+        const DICT_ID_BIT:usize           = 0;
+        
+        let mut bytes_to_skip = FRAME_HEADER_MIN_SIZE;
+        if raw_data.len() >= FRAME_HEADER_MIN_SIZE {
+            let magic_number = LittleEndian::read_u32(raw_data);
+            if magic_number.to_le() == MAGIC_NUMBER {
+                let raw_data          = &raw_data[std::mem::size_of::<u32>()..];
+                let flg               = raw_data[0];
+                let _bd               = raw_data[1];                
+
+                if bit_set_u8(flg, CONTENT_SIZE_BIT) {
+                    bytes_to_skip += std::mem::size_of::<u64>();
+                }
+
+                if bit_set_u8(flg, DICT_ID_BIT) {
+                    bytes_to_skip += std::mem::size_of::<u32>();
+                }
+
+                if raw_data.len() < bytes_to_skip {
+                    Err(Error::from_text(format!("Can't skip {} bytes of frame header because there are only {} bytes left", bytes_to_skip, raw_data.len())))
+                }
+
+                else {
+                    Ok((bytes_to_skip, bit_set_u8(flg, BLOCK_CHECKSUM_BIT), bit_set_u8(flg, CONTENT_CHECKSUM_BIT)))
+                }
+            }
+
+            else {
+                Err(Error::from_text(format!("Magic number needs to be {:#08x} - but {:#08x} was found!", MAGIC_NUMBER, magic_number.to_le())))
+            }
+        }
+
+        else {
+            Err(Error::from_str("Not enough bytes for minimum header size"))
+        }
+    }
+    
+    let (bytes_to_skip, has_block_checksum, has_content_checksum) = read_header(&compressed_data)?;
+    let mut raw_data                                              = &compressed_data[bytes_to_skip..];
+    let end_len                                                   = {
+        if has_content_checksum {
+            std::mem::size_of::<u32>()*2
+        }
+
+        else {
+            std::mem::size_of::<u32>()
+        }
+    };
+    let mut new_data = Vec::new();
+
+    while raw_data.len() > end_len {
+        let block_size = LittleEndian::read_u32(raw_data);
+        raw_data       = &raw_data[std::mem::size_of::<u32>()..];
+
+        if bit_set_u32(block_size, 31) {
+            return Err(Error::from_str("Stripping uncompressed data is not supported!"));
+        }
+
+        let block_size = block_size as usize;
+        new_data.extend_from_slice(&raw_data[0..block_size]);
+        raw_data = &raw_data[block_size..];
+
+        if has_block_checksum {
+            raw_data = &raw_data[std::mem::size_of::<u32>()..];
+        }
+    }
+
+    if LittleEndian::read_u32(raw_data) != 0 {
+        Err(Error::from_str("EOF is not zero!"))
+    }
+
+    else {
+        Ok(new_data)
+    }
+}
+
+pub mod psx_default {
+    use super::*;
+
+    const DEFAULT_COMPRESSION_LEVEL:u32 = 16;
+    const DEFAULT_BLOCK_SIZE:BlockSize  = BlockSize::Max4MB;
+
+    pub fn lz4(data: &Vec<u8>) -> Result<Vec<u8>, Error> {
+        strip_lz4(super::lz4(data, DEFAULT_BLOCK_SIZE, DEFAULT_COMPRESSION_LEVEL)?)
+    }
 }