Abandon other means of oodle for now

oodle_loader/comp.bin

@@ -0,0 +1,6 @@
+Ì
+In tools and when compressing large inputs in one call, consider using
+        $OodleXLZ_Compress_AsyncAndWait (in the Oodle2 Ext lib) instead to get parallelism. Alternatively,
+        chop the data into small fixed size chunks (we recommend at least 256KiB, i.e. 262144 bytes) and
+        call compress on each of them, which decreases compression ratio but makes for trivial parallel
+        compression and decompression.

oodle_loader/src/lib.rs

@@ -136,10 +136,15 @@ pub enum Error {
     #[error("IO error {0:?}")]
     Io(#[from] std::io::Error),
     #[error("ureq error {0:?}")]
-    Ureq(#[from] ureq::Error),
+    Ureq(Box<ureq::Error>),
     #[error("libloading error {0:?}")]
     LibLoading(#[from] libloading::Error),
 }
+impl From<ureq::Error> for Error {
+    fn from(value: ureq::Error) -> Self {
+        Self::Ureq(value.into())
+    }
+}
 
 fn check_hash(buffer: &[u8]) -> Result<()> {
     use sha2::{Digest, Sha256};

repak/Cargo.toml

@@ -10,10 +10,7 @@ keywords.workspace = true
 [features]
 default = ["compression", "encryption"]
 compression = ["dep:flate2", "dep:zstd"]
-oodle = []
+oodle = ["dep:oodle_loader"]
-oodle_loader = ["dep:oodle_loader"]
-oodle_explicit = ["oodle"]
-oodle_implicit_dynamic = ["dep:oodle_loader", "oodle"]
 encryption = ["dep:aes"]
 
 [dependencies]

repak/src/data.rs

@@ -18,6 +18,7 @@ pub(crate) struct PartialBlock {
     pub(crate) data: Vec<u8>,
 }
 
+#[cfg(feature = "compression")]
 fn get_compression_slot(
     version: Version,
     compression_slots: &mut Vec<Option<Compression>>,
@@ -55,16 +56,19 @@ fn get_compression_slot(
 }
 
 impl PartialEntry {
-    pub(crate) fn into_entry(
+    pub(crate) fn build_entry(
         &self,
         version: Version,
-        compression_slots: &mut Vec<Option<Compression>>,
+        #[allow(unused)] compression_slots: &mut Vec<Option<Compression>>,
         file_offset: u64,
     ) -> Result<Entry> {
+        #[cfg(feature = "compression")]
         let compression_slot = self
             .compression
             .map(|c| get_compression_slot(version, compression_slots, c))
             .transpose()?;
+        #[cfg(not(feature = "compression"))]
+        let compression_slot = None;
 
         let blocks = (!self.blocks.is_empty()).then(|| {
             let entry_size =
@@ -92,7 +96,7 @@ impl PartialEntry {
             uncompressed: self.uncompressed_size,
             compression_slot,
             timestamp: None,
-            hash: Some(self.hash.clone()),
+            hash: Some(self.hash),
             blocks,
             flags: 0,
             compression_block_size: self.compression_block_size,
@@ -101,7 +105,6 @@ impl PartialEntry {
 }
 
 pub(crate) fn build_partial_entry(
-    //version: Version,
     allowed_compression: &[Compression],
     data: &[u8],
 ) -> Result<PartialEntry> {
@@ -138,7 +141,7 @@ pub(crate) fn build_partial_entry(
         }
         None => {
             compression_block_size = 0;
-            hasher.update(&data);
+            hasher.update(data);
             (vec![], uncompressed_size)
         }
     };
@@ -153,6 +156,7 @@ pub(crate) fn build_partial_entry(
     })
 }
 
+#[cfg(feature = "compression")]
 fn compress(compression: Compression, data: &[u8]) -> Result<Vec<u8>> {
     use std::io::Write;
 
@@ -171,18 +175,22 @@ fn compress(compression: Compression, data: &[u8]) -> Result<Vec<u8>> {
         }
         Compression::Zstd => zstd::stream::encode_all(data, 0)?,
         Compression::Oodle => {
-            let mut output = vec![];
+            #[cfg(not(feature = "oodle"))]
-            oodle_loader::oodle()
+            return Err(super::Error::Oodle);
-                .unwrap()
+            #[cfg(feature = "oodle")]
-                .compress(
+            {
-                    data.as_ref(),
+                let mut output = vec![];
-                    &mut output,
+                oodle_loader::oodle()
-                    oodle_loader::Compressor::Mermaid,
+                    .unwrap()
-                    oodle_loader::CompressionLevel::Normal,
+                    .compress(
-                )
+                        data.as_ref(),
-                .unwrap();
+                        &mut output,
-            output
+                        oodle_loader::Compressor::Mermaid,
-            //return Err(Error::Other("writing Oodle compression unsupported".into()))
+                        oodle_loader::CompressionLevel::Normal,
+                    )
+                    .unwrap();
+                output
+            }
         }
     };
 

repak/src/entry.rs

@@ -2,7 +2,6 @@ use crate::{data::build_partial_entry, Error, Hash};
 
 use super::{ext::BoolExt, ext::ReadExt, Compression, Version, VersionMajor};
 use byteorder::{ReadBytesExt, WriteBytesExt, LE};
-use oodle_loader::oodle;
 use std::io;
 
 #[derive(Debug, PartialEq, Clone, Copy)]
@@ -108,10 +107,10 @@ impl Entry {
     ) -> Result<Self, Error> {
         let partial_entry = build_partial_entry(allowed_compression, data)?;
         let stream_position = writer.stream_position()?;
-        let entry = partial_entry.into_entry(version, compression_slots, stream_position)?;
+        let entry = partial_entry.build_entry(version, compression_slots, stream_position)?;
         entry.write(writer, version, crate::entry::EntryLocation::Data)?;
         if partial_entry.blocks.is_empty() {
-            writer.write_all(&data)?;
+            writer.write_all(data)?;
         } else {
             for block in partial_entry.blocks {
                 writer.write_all(&block.data)?;
@@ -341,7 +340,6 @@ impl Entry {
         version: Version,
         compression: &[Option<Compression>],
         #[allow(unused)] key: &super::Key,
-        #[allow(unused)] oodle: &super::Oodle,
         buf: &mut W,
     ) -> Result<(), super::Error> {
         reader.seek(io::SeekFrom::Start(self.offset))?;
@@ -411,10 +409,6 @@ impl Entry {
             }
             #[cfg(feature = "oodle")]
             Some(Compression::Oodle) => {
-                let oodle = match oodle {
-                    crate::Oodle::Some(getter) => getter().map_err(|_| super::Error::OodleFailed),
-                    crate::Oodle::None => Err(super::Error::OodleFailed),
-                }?;
                 let mut decompressed = vec![0; self.uncompressed as usize];
 
                 let mut compress_offset = 0;
@@ -428,7 +422,7 @@ impl Entry {
                             .min(self.uncompressed as usize - compress_offset)
                     };
                     let buffer = &mut data[range];
-                    let out = oodle.decompress(
+                    let out = oodle_loader::oodle()?.decompress(
                         buffer,
                         &mut decompressed[decompress_offset..decompress_offset + decomp],
                     );

repak/src/error.rs

@@ -42,8 +42,9 @@ pub enum Error {
     #[error("found magic of {:#x} instead of {:#x}", .0, super::MAGIC)]
     Magic(u32),
 
-    #[error("pointer to OodleLZ_Decompress was not provided")]
+    #[cfg(feature = "oodle")]
-    OodleFailed,
+    #[error("Oodle loader error: {0}")]
+    OodleFailed(#[from] oodle_loader::Error),
 
     #[error("No entry found at {0}")]
     MissingEntry(String),

repak/src/lib.rs

@@ -10,15 +10,6 @@ pub use {error::*, pak::*};
 
 pub const MAGIC: u32 = 0x5A6F12E1;
 
-#[cfg(feature = "oodle")]
-mod oodle {
-    pub type OodleGetter = fn() -> Result<&'static oodle_loader::Oodle, oodle_loader::Error>;
-    pub type OodleDecompress = fn(comp_buf: &[u8], raw_buf: &mut [u8]) -> i32;
-}
-
-#[cfg(feature = "oodle_loader")]
-pub use oodle_loader;
-
 #[derive(
     Clone,
     Copy,
@@ -145,11 +136,3 @@ impl From<aes::Aes256> for Key {
         Self::Some(value)
     }
 }
-
-#[derive(Debug, Default)]
-pub(crate) enum Oodle {
-    #[cfg(feature = "oodle")]
-    Some(oodle::OodleGetter),
-    #[default]
-    None,
-}

repak/src/pak.rs

@@ -20,7 +20,6 @@ impl std::fmt::Debug for Hash {
 #[derive(Debug)]
 pub struct PakBuilder {
     key: super::Key,
-    oodle: super::Oodle,
     allowed_compression: Vec<Compression>,
 }
 
@@ -34,10 +33,6 @@ impl PakBuilder {
     pub fn new() -> Self {
         Self {
             key: Default::default(),
-            #[cfg(not(feature = "oodle_implicit_dynamic"))]
-            oodle: super::Oodle::None,
-            #[cfg(feature = "oodle_implicit_dynamic")]
-            oodle: super::Oodle::Some(oodle_loader::oodle),
             allowed_compression: Default::default(),
         }
     }
@@ -46,25 +41,20 @@ impl PakBuilder {
         self.key = super::Key::Some(key);
         self
     }
-    #[cfg(feature = "oodle_explicit")]
-    pub fn oodle(mut self, oodle_getter: super::oodle::OodleGetter) -> Self {
-        self.oodle = super::Oodle::Some(oodle_getter);
-        self
-    }
     #[cfg(feature = "compression")]
     pub fn compression(mut self, compression: impl IntoIterator<Item = Compression>) -> Self {
         self.allowed_compression = compression.into_iter().collect();
         self
     }
     pub fn reader<R: Read + Seek>(self, reader: &mut R) -> Result<PakReader, super::Error> {
-        PakReader::new_any_inner(reader, self.key, self.oodle)
+        PakReader::new_any_inner(reader, self.key)
     }
     pub fn reader_with_version<R: Read + Seek>(
         self,
         reader: &mut R,
         version: super::Version,
     ) -> Result<PakReader, super::Error> {
-        PakReader::new_inner(reader, version, self.key, self.oodle)
+        PakReader::new_inner(reader, version, self.key)
     }
     pub fn writer<W: Write + Seek>(
         self,
@@ -88,7 +78,6 @@ impl PakBuilder {
 pub struct PakReader {
     pak: Pak,
     key: super::Key,
-    oodle: super::Oodle,
 }
 
 #[derive(Debug)]
@@ -180,14 +169,13 @@ impl PakReader {
     fn new_any_inner<R: Read + Seek>(
         reader: &mut R,
         key: super::Key,
-        oodle: super::Oodle,
     ) -> Result<Self, super::Error> {
         use std::fmt::Write;
         let mut log = "\n".to_owned();
 
         for ver in Version::iter() {
             match Pak::read(&mut *reader, ver, &key) {
-                Ok(pak) => return Ok(Self { pak, key, oodle }),
+                Ok(pak) => return Ok(Self { pak, key }),
                 Err(err) => writeln!(log, "trying version {} failed: {}", ver, err)?,
             }
         }
@@ -198,9 +186,8 @@ impl PakReader {
         reader: &mut R,
         version: super::Version,
         key: super::Key,
-        oodle: super::Oodle,
     ) -> Result<Self, super::Error> {
-        Pak::read(reader, version, &key).map(|pak| Self { pak, key, oodle })
+        Pak::read(reader, version, &key).map(|pak| Self { pak, key })
     }
 
     pub fn version(&self) -> super::Version {
@@ -241,7 +228,6 @@ impl PakReader {
                 self.pak.version,
                 &self.pak.compression,
                 &self.key,
-                &self.oodle,
                 writer,
             ),
             None => Err(super::Error::MissingEntry(path.to_owned())),
@@ -333,7 +319,7 @@ impl<W: Write + Seek> PakWriter<W> {
                         let stream_position = self.writer.stream_position()?;
                         let (path, data, partial_entry) = message?;
 
-                        let entry = partial_entry.into_entry(
+                        let entry = partial_entry.build_entry(
                             self.pak.version,
                             &mut self.pak.compression,
                             stream_position,
@@ -358,7 +344,7 @@ impl<W: Write + Seek> PakWriter<W> {
                     });
 
                 if let Err(err) = handle.join().unwrap() {
-                    Err(err.into()) // prioritize error from user code
+                    Err(err) // prioritize error from user code
                 } else if let Err(err) = result {
                     Err(err.into()) // user code was successful, check pak writer error
                 } else {

repak/tests/test.rs

@@ -183,12 +183,10 @@ fn test_write(_version: repak::Version, _file_name: &str, bytes: &[u8]) {
         Some(0x205C5A7D),
     );
 
-    pak_writer.parallel(|writer| {
+    for path in pak_reader.files() {
-        for path in pak_reader.files() {
+        let data = pak_reader.get(&path, &mut reader).unwrap();
-            let data = pak_reader.get(&path, &mut reader).unwrap();
+        pak_writer.write_file(&path, data).unwrap();
-            writer.write_file(path, data).unwrap();
+    }
-        }
-    }).unwrap();
 
     assert!(pak_writer.write_index().unwrap().into_inner() == reader.into_inner());
 }

repak_cli/Cargo.toml

@@ -19,7 +19,7 @@ path = "src/main.rs"
 
 [features]
 default = ["oodle"]
-oodle = ["repak/oodle_implicit_dynamic"]
+oodle = ["repak/oodle"]
 
 [dependencies]
 repak = { path = "../repak" }