Remove Overlayhader and support loading overlays

include/PSX/File/cd_file_types.hpp

@@ -7,13 +7,15 @@ namespace JabyEngine {
     enum struct CDFileType : uint8_t {
         SimpleTIM = 0,
         CopyTo,
+        Overlay,
     };
 
     struct __no_align CDFile {
         union __no_align Payload {
-            uint32_t  empty;
+            uint32_t  raw;
             SimpleTIM simple_tim;
             CopyTo    copy_to;
+            Overlay   overlay;
         };
 
         uint8_t    rel_lba_idx;
@@ -31,6 +33,10 @@ namespace JabyEngine {
         static constexpr CDFile copy_to(uint8_t rel_lba_idx, uint32_t* dst) {
             return CDFile{.rel_lba_idx = rel_lba_idx, .type = CDFileType::CopyTo, .payload = {.copy_to = CopyTo{dst}}};
         }
+
+        static constexpr CDFile overlay(uint8_t rel_lba_idx, uint32_t* overlay_dst) {
+            return CDFile{.rel_lba_idx = rel_lba_idx, .type = CDFileType::Overlay, .payload = {.overlay = Overlay{overlay_dst}}};
+        }
     };
 }
 #endif //!__JABYENGINE_CD_FILE_TYPES_HPP__

include/PSX/File/file_types.hpp

@@ -42,5 +42,7 @@ namespace JabyEngine {
     struct __no_align CopyTo {
         uint32_t* dst;
     };
+
+    typedef CopyTo Overlay;
 }
 #endif // !__JABYENGINE_FILE_TYPES_HPP__

include/PSX/Overlay/overlay.hpp

@@ -3,11 +3,9 @@
 #include "../AutoLBA/auto_lba.hpp"
 
 namespace JabyEngine {
-    struct __attribute__((packed)) OverlayHeader {
-        void   (*execute)();
-        uint16_t lba_size; //< The size of the OverlayLBA section
-    };
-
     typedef AutoLBAEntry OverlayLBA;
+
+    // Can be used to create dummy values to trick LZ4 into compressing an uncompressed overlay
+    #define __create_dummy_fill(length) static const uint8_t __section(".keep.dummy") __used DummyFilling[length] = {0x0}
 }
 #endif //!__JABYENGINE_OVERLAY__HPP__

include/PSX/Overlay/overlay_declaration.hpp

@@ -4,10 +4,7 @@
 // No include here because this header should be included in a namespace and we don't want multiple namespace definitions of OverlayHeader and OverlayLBA
 
 #include "../AutoLBA/auto_lba_declaration.hpp"
-extern const JabyEngine::OverlayHeader overlay_header;
 
 #define __declare_overlay_header(function, enum_struct) \
-__declare_lba_header(enum_struct); \
+__declare_lba_header(enum_struct)
-[[gnu::used]] \
-const JabyEngine::OverlayHeader __section(".header") overlay_header = {.execute = &function, .lba_size = static_cast<uint16_t>(enum_struct::EndOfRequest)};
 #endif //!__JABYENGINE_OVERLAY_DECLARATION__HPP__

include/PSX/jabyengine_defines.h

@@ -3,7 +3,7 @@
 #include "jabyengine_config.hpp"
 #include "../stddef.h"
 
-#define __keep          __attribute__((used))
+#define __used          __attribute__((used))
 #define __no_align      __attribute__((packed))
 #define __no_inline     __attribute__((noinline))
 #define __always_inline __attribute__((always_inline))

src/Library/src/File/Processor/cd_file_processor.cpp

@@ -4,21 +4,26 @@
 
 namespace JabyEngine {
     static constexpr auto NormalCircularBufferSize   = 5;
+    static constexpr auto DisabledCircularBufferSize = 512;
 
     void CDFileProcessor :: start_cur_job() {
         using CD::internal::FileInfo;
         using CD::internal::SectorBufferAllocator;
             const auto configurate_for = [this](const CDFile& file) {
-                const auto disable_lz4 = [this](uint32_t* work_area, size_t size) -> uint32_t* {
+                const auto disable_lz4 = [this](uint32_t* work_area, size_t size, uint32_t* overwrite_dst = nullptr) -> uint32_t* {
-                    uint8_t* dst_adr = reinterpret_cast<uint8_t*>(this->circular_buffer.setup(reinterpret_cast<CD_IO::DataSector*>(work_area), size));
+                    uint8_t*const dst_adr = reinterpret_cast<uint8_t*>(this->circular_buffer.setup(reinterpret_cast<CD_IO::DataSector*>(work_area), size));
 
                     this->lz4_decomp.disable();
-                    return reinterpret_cast<uint32_t*>(dst_adr);
+                    return overwrite_dst ? overwrite_dst : reinterpret_cast<uint32_t*>(dst_adr);
                 };
 
-                const auto enable_lz4 = [this](uint32_t* work_area, size_t size) -> uint32_t* {
+                const auto enable_lz4 = [this](uint32_t* work_area, size_t size, uint32_t* override_dst = nullptr) -> uint32_t* {
                     uint8_t* dst_adr = reinterpret_cast<uint8_t*>(this->circular_buffer.setup(reinterpret_cast<CD_IO::DataSector*>(work_area), size));
                     
+                    if(override_dst) {
+                        dst_adr = reinterpret_cast<uint8_t*>(override_dst);
+                    }
+
                     this->lz4_decomp.setup(dst_adr);
                     return reinterpret_cast<uint32_t*>(dst_adr);
                 };
@@ -31,6 +36,10 @@ namespace JabyEngine {
                     case CDFileType::CopyTo:
                         this->file_state = FileProcessor::create(disable_lz4(file.payload.copy_to.dst, 512), Nothing());
                         break;
+
+                    case CDFileType::Overlay:
+                        this->file_state = FileProcessor::create(enable_lz4(this->tmp_area, NormalCircularBufferSize, file.payload.overlay.dst), Nothing());
+                        break;
                 }            
             };
 

src/Tools/mkoverlay/src/creator/ldscript.rs

@@ -58,13 +58,23 @@ fn write_section(output: &mut Output, sections: &Vec<OverlaySection>, add_end_na
     for section in sections {
         writeln!(output, "\t\t.{} {{", section.name)?;
             writeln!(output, "\t\t\t__{}_start = .;", section.name)?;
-            section.file_pattern.iter().try_for_each(|patr| writeln!(output, "\t\t\tKEEP({}(.header))", patr))?;
             writeln!(output, "\t\t\t__{}_lbas = .;", section.name)?;
             section.file_pattern.iter().try_for_each(|patr| writeln!(output, "\t\t\tKEEP({}(.header.lbas))", patr))?;
             writeln!(output, "\t\t\t__{}_ctor = .;", section.name)?;
 
-            for section_type in [".text.startup._GLOBAL__*", ".ctors", ".text.*", ".rodata*", ".sdata*", ".data*", ".sbss*", ".bss*"] {
+            for (section_type, keep) in [(".text.startup._GLOBAL__*", false), (".ctors", false), (".text.*", false), (".rodata*", false), (".sdata*", false), (".data*", false), (".sbss*", false), (".bss*", false), (".keep.dummy", true)] {
-                section.file_pattern.iter().try_for_each(|patr| writeln!(output, "\t\t\t{}({})", patr, section_type))?;
+                section.file_pattern.iter().try_for_each(|patr| {
+                    let section_entry = format!("{}({})", patr, section_type);
+                    
+                    write!(output, "\t\t\t")?;
+                    if keep {
+                        writeln!(output, "KEEP({})", section_entry)
+                    }
+                    
+                    else {
+                        writeln!(output, "{}", section_entry)
+                    }
+                })?;
             }
 
             if add_end_name {

src/Tools/psxcdgen_ex/src/types/overlay/mod.rs

@@ -8,18 +8,6 @@ pub type LBANameVec = Vec<String>;
 
 mod main;
 
-#[repr(packed)]
-struct OverlayHeader {
-    _start_adr: u32,
-    lba_count: u16,
-}
-
-impl OverlayHeader {
-    pub fn read_lba_count(&self) -> usize {
-        u16::from_le(self.lba_count) as usize
-    }
-}
-
 #[repr(packed)]
 struct LBAEntry {
     lba:        u16,
@@ -51,7 +39,7 @@ impl LBAEntry {
 }
 
 pub fn load_from(file_name: &str, file_path: PathBuf, lba_source: PathBuf) -> Result<File, Error> {
-    let content      = load_content(&file_path)?;
+    let content      = read_file(&file_path)?;
     let lba_names    = load_lba_names(lba_source)?;
     let content_size = format_if_error!(tool_helper::compress::psx_default::lz4(&content), "Compressing {} failed with \"{error_text}\"", file_path.to_string_lossy())?.len();
     
@@ -63,14 +51,10 @@ pub fn load_for_main(file_name: &str, content: Vec<u8>, lba_source: PathBuf) ->
 }
 
 pub fn update_content(content: &mut Vec<u8>, lba_names: &LBANameVec, file_map: &FileSystemMap, length_func: LengthCalculatorFunction) -> Result<Vec<u8>, Error> {
-    let (lba_header, lba_count) = skip_to_lba_header(content);
+    let lba_header = skip_to_lba_header(content);
-    let  lba_header             = unsafe{std::slice::from_raw_parts_mut(lba_header.as_mut_ptr() as *mut LBAEntry, lba_count)};
+    let lba_header = unsafe{std::slice::from_raw_parts_mut(lba_header.as_mut_ptr() as *mut LBAEntry, lba_names.len())};
     
     for_each_lba_name(lba_names, file_map, length_func, |idx, (lba, bytes)| {
-        if idx >= lba_count {
-            return Err(Error::from_text(format!("Trying to write more LBAs then there is space!")));
-        }
-            
         lba_header[idx].write_entry(lba, bytes)
     })?;
 
@@ -105,34 +89,10 @@ fn for_each_lba_name<F: FnMut(usize, (u16, usize)) -> Result<(), Error>>(lba_nam
     Ok(())
 }
 
-fn load_content(file_path: &PathBuf) -> Result<Vec<u8>, Error> {
+fn skip_to_lba_header(content: &mut Vec<u8>) -> &mut [u8] {
-    let mut content = read_file(&file_path)?;
+    let overlay_header_size = 0;
 
-    if content.len() < std::mem::size_of::<OverlayHeader>() {
+    &mut content[overlay_header_size..]
-        return Err(Error::from_text(format!("Overlay {} has no header!", file_path.to_string_lossy())));
-    }
-
-    let (lba_header, lba_count) = skip_to_lba_header(&mut content);
-    if lba_header.is_empty() {
-        return Err(Error::from_text(format!("LBA header of overlay {} is smaller then {} elements", file_path.to_string_lossy(), lba_count)));
-    }
-
-    let mut count = 0;
-    for byte in lba_header {
-        *byte = count as u8;
-        count += 1;
-    }
-
-    Ok(content)
-}
-
-fn skip_to_lba_header(content: &mut Vec<u8>) -> (&mut [u8], usize) {
-    let overlay_header_size = std::mem::size_of::<OverlayHeader>();
-
-    let lba_count       = unsafe{std::mem::transmute::<&u8, &OverlayHeader>(&content[0])}.read_lba_count();
-    let lba_header_size = lba_count*std::mem::size_of::<LBAEntry>();
-
-    (&mut content[overlay_header_size..(overlay_header_size+lba_header_size)], lba_count)
 }
 
 fn load_lba_names(lba_source: PathBuf) -> Result<LBANameVec, Error> {