diff --git a/oodle_loader/comp.bin b/oodle_loader/comp.bin new file mode 100644 index 0000000..d43c5b4 --- /dev/null +++ b/oodle_loader/comp.bin @@ -0,0 +1,6 @@ +Ì +In tools and when compressing large inputs in one call, consider using + $OodleXLZ_Compress_AsyncAndWait (in the Oodle2 Ext lib) instead to get parallelism. Alternatively, + chop the data into small fixed size chunks (we recommend at least 256KiB, i.e. 262144 bytes) and + call compress on each of them, which decreases compression ratio but makes for trivial parallel + compression and decompression. \ No newline at end of file diff --git a/oodle_loader/src/lib.rs b/oodle_loader/src/lib.rs index 753aec8..7fe5f89 100644 --- a/oodle_loader/src/lib.rs +++ b/oodle_loader/src/lib.rs @@ -136,10 +136,15 @@ pub enum Error { #[error("IO error {0:?}")] Io(#[from] std::io::Error), #[error("ureq error {0:?}")] - Ureq(#[from] ureq::Error), + Ureq(Box), #[error("libloading error {0:?}")] LibLoading(#[from] libloading::Error), } +impl From for Error { + fn from(value: ureq::Error) -> Self { + Self::Ureq(value.into()) + } +} fn check_hash(buffer: &[u8]) -> Result<()> { use sha2::{Digest, Sha256}; diff --git a/repak/Cargo.toml b/repak/Cargo.toml index 07fea7e..bb3fcb9 100644 --- a/repak/Cargo.toml +++ b/repak/Cargo.toml @@ -10,10 +10,7 @@ keywords.workspace = true [features] default = ["compression", "encryption"] compression = ["dep:flate2", "dep:zstd"] -oodle = [] -oodle_loader = ["dep:oodle_loader"] -oodle_explicit = ["oodle"] -oodle_implicit_dynamic = ["dep:oodle_loader", "oodle"] +oodle = ["dep:oodle_loader"] encryption = ["dep:aes"] [dependencies] diff --git a/repak/src/data.rs b/repak/src/data.rs index d6a1cca..7832db7 100644 --- a/repak/src/data.rs +++ b/repak/src/data.rs @@ -18,6 +18,7 @@ pub(crate) struct PartialBlock { pub(crate) data: Vec, } +#[cfg(feature = "compression")] fn get_compression_slot( version: Version, compression_slots: &mut Vec>, @@ -55,16 +56,19 @@ fn get_compression_slot( } impl PartialEntry { - pub(crate) fn into_entry( + pub(crate) fn build_entry( &self, version: Version, - compression_slots: &mut Vec>, + #[allow(unused)] compression_slots: &mut Vec>, file_offset: u64, ) -> Result { + #[cfg(feature = "compression")] let compression_slot = self .compression .map(|c| get_compression_slot(version, compression_slots, c)) .transpose()?; + #[cfg(not(feature = "compression"))] + let compression_slot = None; let blocks = (!self.blocks.is_empty()).then(|| { let entry_size = @@ -92,7 +96,7 @@ impl PartialEntry { uncompressed: self.uncompressed_size, compression_slot, timestamp: None, - hash: Some(self.hash.clone()), + hash: Some(self.hash), blocks, flags: 0, compression_block_size: self.compression_block_size, @@ -101,7 +105,6 @@ impl PartialEntry { } pub(crate) fn build_partial_entry( - //version: Version, allowed_compression: &[Compression], data: &[u8], ) -> Result { @@ -138,7 +141,7 @@ pub(crate) fn build_partial_entry( } None => { compression_block_size = 0; - hasher.update(&data); + hasher.update(data); (vec![], uncompressed_size) } }; @@ -153,6 +156,7 @@ pub(crate) fn build_partial_entry( }) } +#[cfg(feature = "compression")] fn compress(compression: Compression, data: &[u8]) -> Result> { use std::io::Write; @@ -171,18 +175,22 @@ fn compress(compression: Compression, data: &[u8]) -> Result> { } Compression::Zstd => zstd::stream::encode_all(data, 0)?, Compression::Oodle => { - let mut output = vec![]; - oodle_loader::oodle() - .unwrap() - .compress( - data.as_ref(), - &mut output, - oodle_loader::Compressor::Mermaid, - oodle_loader::CompressionLevel::Normal, - ) - .unwrap(); - output - //return Err(Error::Other("writing Oodle compression unsupported".into())) + #[cfg(not(feature = "oodle"))] + return Err(super::Error::Oodle); + #[cfg(feature = "oodle")] + { + let mut output = vec![]; + oodle_loader::oodle() + .unwrap() + .compress( + data.as_ref(), + &mut output, + oodle_loader::Compressor::Mermaid, + oodle_loader::CompressionLevel::Normal, + ) + .unwrap(); + output + } } }; diff --git a/repak/src/entry.rs b/repak/src/entry.rs index 6e353e4..e0311a1 100644 --- a/repak/src/entry.rs +++ b/repak/src/entry.rs @@ -2,7 +2,6 @@ use crate::{data::build_partial_entry, Error, Hash}; use super::{ext::BoolExt, ext::ReadExt, Compression, Version, VersionMajor}; use byteorder::{ReadBytesExt, WriteBytesExt, LE}; -use oodle_loader::oodle; use std::io; #[derive(Debug, PartialEq, Clone, Copy)] @@ -108,10 +107,10 @@ impl Entry { ) -> Result { let partial_entry = build_partial_entry(allowed_compression, data)?; let stream_position = writer.stream_position()?; - let entry = partial_entry.into_entry(version, compression_slots, stream_position)?; + let entry = partial_entry.build_entry(version, compression_slots, stream_position)?; entry.write(writer, version, crate::entry::EntryLocation::Data)?; if partial_entry.blocks.is_empty() { - writer.write_all(&data)?; + writer.write_all(data)?; } else { for block in partial_entry.blocks { writer.write_all(&block.data)?; @@ -341,7 +340,6 @@ impl Entry { version: Version, compression: &[Option], #[allow(unused)] key: &super::Key, - #[allow(unused)] oodle: &super::Oodle, buf: &mut W, ) -> Result<(), super::Error> { reader.seek(io::SeekFrom::Start(self.offset))?; @@ -411,10 +409,6 @@ impl Entry { } #[cfg(feature = "oodle")] Some(Compression::Oodle) => { - let oodle = match oodle { - crate::Oodle::Some(getter) => getter().map_err(|_| super::Error::OodleFailed), - crate::Oodle::None => Err(super::Error::OodleFailed), - }?; let mut decompressed = vec![0; self.uncompressed as usize]; let mut compress_offset = 0; @@ -428,7 +422,7 @@ impl Entry { .min(self.uncompressed as usize - compress_offset) }; let buffer = &mut data[range]; - let out = oodle.decompress( + let out = oodle_loader::oodle()?.decompress( buffer, &mut decompressed[decompress_offset..decompress_offset + decomp], ); diff --git a/repak/src/error.rs b/repak/src/error.rs index b60bc92..363dc86 100644 --- a/repak/src/error.rs +++ b/repak/src/error.rs @@ -42,8 +42,9 @@ pub enum Error { #[error("found magic of {:#x} instead of {:#x}", .0, super::MAGIC)] Magic(u32), - #[error("pointer to OodleLZ_Decompress was not provided")] - OodleFailed, + #[cfg(feature = "oodle")] + #[error("Oodle loader error: {0}")] + OodleFailed(#[from] oodle_loader::Error), #[error("No entry found at {0}")] MissingEntry(String), diff --git a/repak/src/lib.rs b/repak/src/lib.rs index 5ff71a8..643381c 100644 --- a/repak/src/lib.rs +++ b/repak/src/lib.rs @@ -10,15 +10,6 @@ pub use {error::*, pak::*}; pub const MAGIC: u32 = 0x5A6F12E1; -#[cfg(feature = "oodle")] -mod oodle { - pub type OodleGetter = fn() -> Result<&'static oodle_loader::Oodle, oodle_loader::Error>; - pub type OodleDecompress = fn(comp_buf: &[u8], raw_buf: &mut [u8]) -> i32; -} - -#[cfg(feature = "oodle_loader")] -pub use oodle_loader; - #[derive( Clone, Copy, @@ -145,11 +136,3 @@ impl From for Key { Self::Some(value) } } - -#[derive(Debug, Default)] -pub(crate) enum Oodle { - #[cfg(feature = "oodle")] - Some(oodle::OodleGetter), - #[default] - None, -} diff --git a/repak/src/pak.rs b/repak/src/pak.rs index 3b4d728..043453f 100644 --- a/repak/src/pak.rs +++ b/repak/src/pak.rs @@ -20,7 +20,6 @@ impl std::fmt::Debug for Hash { #[derive(Debug)] pub struct PakBuilder { key: super::Key, - oodle: super::Oodle, allowed_compression: Vec, } @@ -34,10 +33,6 @@ impl PakBuilder { pub fn new() -> Self { Self { key: Default::default(), - #[cfg(not(feature = "oodle_implicit_dynamic"))] - oodle: super::Oodle::None, - #[cfg(feature = "oodle_implicit_dynamic")] - oodle: super::Oodle::Some(oodle_loader::oodle), allowed_compression: Default::default(), } } @@ -46,25 +41,20 @@ impl PakBuilder { self.key = super::Key::Some(key); self } - #[cfg(feature = "oodle_explicit")] - pub fn oodle(mut self, oodle_getter: super::oodle::OodleGetter) -> Self { - self.oodle = super::Oodle::Some(oodle_getter); - self - } #[cfg(feature = "compression")] pub fn compression(mut self, compression: impl IntoIterator) -> Self { self.allowed_compression = compression.into_iter().collect(); self } pub fn reader(self, reader: &mut R) -> Result { - PakReader::new_any_inner(reader, self.key, self.oodle) + PakReader::new_any_inner(reader, self.key) } pub fn reader_with_version( self, reader: &mut R, version: super::Version, ) -> Result { - PakReader::new_inner(reader, version, self.key, self.oodle) + PakReader::new_inner(reader, version, self.key) } pub fn writer( self, @@ -88,7 +78,6 @@ impl PakBuilder { pub struct PakReader { pak: Pak, key: super::Key, - oodle: super::Oodle, } #[derive(Debug)] @@ -180,14 +169,13 @@ impl PakReader { fn new_any_inner( reader: &mut R, key: super::Key, - oodle: super::Oodle, ) -> Result { use std::fmt::Write; let mut log = "\n".to_owned(); for ver in Version::iter() { match Pak::read(&mut *reader, ver, &key) { - Ok(pak) => return Ok(Self { pak, key, oodle }), + Ok(pak) => return Ok(Self { pak, key }), Err(err) => writeln!(log, "trying version {} failed: {}", ver, err)?, } } @@ -198,9 +186,8 @@ impl PakReader { reader: &mut R, version: super::Version, key: super::Key, - oodle: super::Oodle, ) -> Result { - Pak::read(reader, version, &key).map(|pak| Self { pak, key, oodle }) + Pak::read(reader, version, &key).map(|pak| Self { pak, key }) } pub fn version(&self) -> super::Version { @@ -241,7 +228,6 @@ impl PakReader { self.pak.version, &self.pak.compression, &self.key, - &self.oodle, writer, ), None => Err(super::Error::MissingEntry(path.to_owned())), @@ -333,7 +319,7 @@ impl PakWriter { let stream_position = self.writer.stream_position()?; let (path, data, partial_entry) = message?; - let entry = partial_entry.into_entry( + let entry = partial_entry.build_entry( self.pak.version, &mut self.pak.compression, stream_position, @@ -358,7 +344,7 @@ impl PakWriter { }); if let Err(err) = handle.join().unwrap() { - Err(err.into()) // prioritize error from user code + Err(err) // prioritize error from user code } else if let Err(err) = result { Err(err.into()) // user code was successful, check pak writer error } else { diff --git a/repak/tests/test.rs b/repak/tests/test.rs index 51e6d22..5c63f8c 100644 --- a/repak/tests/test.rs +++ b/repak/tests/test.rs @@ -183,12 +183,10 @@ fn test_write(_version: repak::Version, _file_name: &str, bytes: &[u8]) { Some(0x205C5A7D), ); - pak_writer.parallel(|writer| { - for path in pak_reader.files() { - let data = pak_reader.get(&path, &mut reader).unwrap(); - writer.write_file(path, data).unwrap(); - } - }).unwrap(); + for path in pak_reader.files() { + let data = pak_reader.get(&path, &mut reader).unwrap(); + pak_writer.write_file(&path, data).unwrap(); + } assert!(pak_writer.write_index().unwrap().into_inner() == reader.into_inner()); } diff --git a/repak_cli/Cargo.toml b/repak_cli/Cargo.toml index 7397c92..2de9afb 100644 --- a/repak_cli/Cargo.toml +++ b/repak_cli/Cargo.toml @@ -19,7 +19,7 @@ path = "src/main.rs" [features] default = ["oodle"] -oodle = ["repak/oodle_implicit_dynamic"] +oodle = ["repak/oodle"] [dependencies] repak = { path = "../repak" }