xavo95/repak
1
0
Fork 0
mirror of https://github.com/xavo95/repak.git synced 2025-02-22 15:53:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Abandon other means of oodle for now

Browse source
This commit is contained in:
Truman Kilen 2025-01-18 14:18:21 -06:00
parent c2b5461f25
commit 1d1ad7138d
10 changed files with 55 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
oodle_loader/comp.bin Normal file
View file

@ -0,0 +1,6 @@
Ì
In tools and when compressing large inputs in one call, consider using
$OodleXLZ_Compress_AsyncAndWait (in the Oodle2 Ext lib) instead to get parallelism. Alternatively,
chop the data into small fixed size chunks (we recommend at least 256KiB, i.e. 262144 bytes) and
call compress on each of them, which decreases compression ratio but makes for trivial parallel
compression and decompression.

7
oodle_loader/src/lib.rs
View file

@ -136,10 +136,15 @@ pub enum Error {
#[error("IO error {0:?}")]
Io(#[from] std::io::Error),
#[error("ureq error {0:?}")]
Ureq(#[from] ureq::Error),
Ureq(Box<ureq::Error>),
#[error("libloading error {0:?}")]
LibLoading(#[from] libloading::Error),
}
impl From<ureq::Error> for Error {
fn from(value: ureq::Error) -> Self {
Self::Ureq(value.into())
}
}
fn check_hash(buffer: &[u8]) -> Result<()> {
use sha2::{Digest, Sha256};

5
repak/Cargo.toml
View file

@ -10,10 +10,7 @@ keywords.workspace = true
[features]
default = ["compression", "encryption"]
compression = ["dep:flate2", "dep:zstd"]
oodle = []
oodle_loader = ["dep:oodle_loader"]
oodle_explicit = ["oodle"]
oodle_implicit_dynamic = ["dep:oodle_loader", "oodle"]
oodle = ["dep:oodle_loader"]
encryption = ["dep:aes"]
[dependencies]

42
repak/src/data.rs
View file

@ -18,6 +18,7 @@ pub(crate) struct PartialBlock {
pub(crate) data: Vec<u8>,
}
#[cfg(feature = "compression")]
fn get_compression_slot(
version: Version,
compression_slots: &mut Vec<Option<Compression>>,
@ -55,16 +56,19 @@ fn get_compression_slot(
}
impl PartialEntry {
pub(crate) fn into_entry(
pub(crate) fn build_entry(
&self,
version: Version,
compression_slots: &mut Vec<Option<Compression>>,
#[allow(unused)] compression_slots: &mut Vec<Option<Compression>>,
file_offset: u64,
) -> Result<Entry> {
#[cfg(feature = "compression")]
let compression_slot = self
.compression
.map(|c| get_compression_slot(version, compression_slots, c))
.transpose()?;
#[cfg(not(feature = "compression"))]
let compression_slot = None;
let blocks = (!self.blocks.is_empty()).then(|| {
let entry_size =
@ -92,7 +96,7 @@ impl PartialEntry {
uncompressed: self.uncompressed_size,
compression_slot,
timestamp: None,
hash: Some(self.hash.clone()),
hash: Some(self.hash),
blocks,
flags: 0,
compression_block_size: self.compression_block_size,
@ -101,7 +105,6 @@ impl PartialEntry {
}
pub(crate) fn build_partial_entry(
//version: Version,
allowed_compression: &[Compression],
data: &[u8],
) -> Result<PartialEntry> {
@ -138,7 +141,7 @@ pub(crate) fn build_partial_entry(
}
None => {
compression_block_size = 0;
hasher.update(&data);
hasher.update(data);
(vec![], uncompressed_size)
}
};
@ -153,6 +156,7 @@ pub(crate) fn build_partial_entry(
})
}
#[cfg(feature = "compression")]
fn compress(compression: Compression, data: &[u8]) -> Result<Vec<u8>> {
use std::io::Write;
@ -171,18 +175,22 @@ fn compress(compression: Compression, data: &[u8]) -> Result<Vec<u8>> {
}
Compression::Zstd => zstd::stream::encode_all(data, 0)?,
Compression::Oodle => {
let mut output = vec![];
oodle_loader::oodle()
.unwrap()
.compress(
data.as_ref(),
&mut output,
oodle_loader::Compressor::Mermaid,
oodle_loader::CompressionLevel::Normal,
)
.unwrap();
output
//return Err(Error::Other("writing Oodle compression unsupported".into()))
#[cfg(not(feature = "oodle"))]
return Err(super::Error::Oodle);
#[cfg(feature = "oodle")]
{
let mut output = vec![];
oodle_loader::oodle()
.unwrap()
.compress(
data.as_ref(),
&mut output,
oodle_loader::Compressor::Mermaid,
oodle_loader::CompressionLevel::Normal,
)
.unwrap();
output
}
}
};

12
repak/src/entry.rs
View file

@ -2,7 +2,6 @@ use crate::{data::build_partial_entry, Error, Hash};
use super::{ext::BoolExt, ext::ReadExt, Compression, Version, VersionMajor};
use byteorder::{ReadBytesExt, WriteBytesExt, LE};
use oodle_loader::oodle;
use std::io;
#[derive(Debug, PartialEq, Clone, Copy)]
@ -108,10 +107,10 @@ impl Entry {
) -> Result<Self, Error> {
let partial_entry = build_partial_entry(allowed_compression, data)?;
let stream_position = writer.stream_position()?;
let entry = partial_entry.into_entry(version, compression_slots, stream_position)?;
let entry = partial_entry.build_entry(version, compression_slots, stream_position)?;
entry.write(writer, version, crate::entry::EntryLocation::Data)?;
if partial_entry.blocks.is_empty() {
writer.write_all(&data)?;
writer.write_all(data)?;
} else {
for block in partial_entry.blocks {
writer.write_all(&block.data)?;
@ -341,7 +340,6 @@ impl Entry {
version: Version,
compression: &[Option<Compression>],
#[allow(unused)] key: &super::Key,
#[allow(unused)] oodle: &super::Oodle,
buf: &mut W,
) -> Result<(), super::Error> {
reader.seek(io::SeekFrom::Start(self.offset))?;
@ -411,10 +409,6 @@ impl Entry {
}
#[cfg(feature = "oodle")]
Some(Compression::Oodle) => {
let oodle = match oodle {
crate::Oodle::Some(getter) => getter().map_err(|_| super::Error::OodleFailed),
crate::Oodle::None => Err(super::Error::OodleFailed),
}?;
let mut decompressed = vec![0; self.uncompressed as usize];
let mut compress_offset = 0;
@ -428,7 +422,7 @@ impl Entry {
.min(self.uncompressed as usize - compress_offset)
};
let buffer = &mut data[range];
let out = oodle.decompress(
let out = oodle_loader::oodle()?.decompress(
buffer,
&mut decompressed[decompress_offset..decompress_offset + decomp],
);

5
repak/src/error.rs
View file

@ -42,8 +42,9 @@ pub enum Error {
#[error("found magic of {:#x} instead of {:#x}", .0, super::MAGIC)]
Magic(u32),
#[error("pointer to OodleLZ_Decompress was not provided")]
OodleFailed,
#[cfg(feature = "oodle")]
#[error("Oodle loader error: {0}")]
OodleFailed(#[from] oodle_loader::Error),
#[error("No entry found at {0}")]
MissingEntry(String),

17
repak/src/lib.rs
View file

@ -10,15 +10,6 @@ pub use {error::*, pak::*};
pub const MAGIC: u32 = 0x5A6F12E1;
#[cfg(feature = "oodle")]
mod oodle {
pub type OodleGetter = fn() -> Result<&'static oodle_loader::Oodle, oodle_loader::Error>;
pub type OodleDecompress = fn(comp_buf: &[u8], raw_buf: &mut [u8]) -> i32;
}
#[cfg(feature = "oodle_loader")]
pub use oodle_loader;
#[derive(
Clone,
Copy,
@ -145,11 +136,3 @@ impl From<aes::Aes256> for Key {
Self::Some(value)
}
}
#[derive(Debug, Default)]
pub(crate) enum Oodle {
#[cfg(feature = "oodle")]
Some(oodle::OodleGetter),
#[default]
None,
}

26
repak/src/pak.rs
View file

@ -20,7 +20,6 @@ impl std::fmt::Debug for Hash {
#[derive(Debug)]
pub struct PakBuilder {
key: super::Key,
oodle: super::Oodle,
allowed_compression: Vec<Compression>,
}
@ -34,10 +33,6 @@ impl PakBuilder {
pub fn new() -> Self {
Self {
key: Default::default(),
#[cfg(not(feature = "oodle_implicit_dynamic"))]
oodle: super::Oodle::None,
#[cfg(feature = "oodle_implicit_dynamic")]
oodle: super::Oodle::Some(oodle_loader::oodle),
allowed_compression: Default::default(),
}
}
@ -46,25 +41,20 @@ impl PakBuilder {
self.key = super::Key::Some(key);
self
}
#[cfg(feature = "oodle_explicit")]
pub fn oodle(mut self, oodle_getter: super::oodle::OodleGetter) -> Self {
self.oodle = super::Oodle::Some(oodle_getter);
self
}
#[cfg(feature = "compression")]
pub fn compression(mut self, compression: impl IntoIterator<Item = Compression>) -> Self {
self.allowed_compression = compression.into_iter().collect();
self
}
pub fn reader<R: Read + Seek>(self, reader: &mut R) -> Result<PakReader, super::Error> {
PakReader::new_any_inner(reader, self.key, self.oodle)
PakReader::new_any_inner(reader, self.key)
}
pub fn reader_with_version<R: Read + Seek>(
self,
reader: &mut R,
version: super::Version,
) -> Result<PakReader, super::Error> {
PakReader::new_inner(reader, version, self.key, self.oodle)
PakReader::new_inner(reader, version, self.key)
}
pub fn writer<W: Write + Seek>(
self,
@ -88,7 +78,6 @@ impl PakBuilder {
pub struct PakReader {
pak: Pak,
key: super::Key,
oodle: super::Oodle,
}
#[derive(Debug)]
@ -180,14 +169,13 @@ impl PakReader {
fn new_any_inner<R: Read + Seek>(
reader: &mut R,
key: super::Key,
oodle: super::Oodle,
) -> Result<Self, super::Error> {
use std::fmt::Write;
let mut log = "\n".to_owned();
for ver in Version::iter() {
match Pak::read(&mut *reader, ver, &key) {
Ok(pak) => return Ok(Self { pak, key, oodle }),
Ok(pak) => return Ok(Self { pak, key }),
Err(err) => writeln!(log, "trying version {} failed: {}", ver, err)?,
}
}
@ -198,9 +186,8 @@ impl PakReader {
reader: &mut R,
version: super::Version,
key: super::Key,
oodle: super::Oodle,
) -> Result<Self, super::Error> {
Pak::read(reader, version, &key).map(|pak| Self { pak, key, oodle })
Pak::read(reader, version, &key).map(|pak| Self { pak, key })
}
pub fn version(&self) -> super::Version {
@ -241,7 +228,6 @@ impl PakReader {
self.pak.version,
&self.pak.compression,
&self.key,
&self.oodle,
writer,
),
None => Err(super::Error::MissingEntry(path.to_owned())),
@ -333,7 +319,7 @@ impl<W: Write + Seek> PakWriter<W> {
let stream_position = self.writer.stream_position()?;
let (path, data, partial_entry) = message?;
let entry = partial_entry.into_entry(
let entry = partial_entry.build_entry(
self.pak.version,
&mut self.pak.compression,
stream_position,
@ -358,7 +344,7 @@ impl<W: Write + Seek> PakWriter<W> {
});
if let Err(err) = handle.join().unwrap() {
Err(err.into()) // prioritize error from user code
Err(err) // prioritize error from user code
} else if let Err(err) = result {
Err(err.into()) // user code was successful, check pak writer error
} else {

10
repak/tests/test.rs
View file

@ -183,12 +183,10 @@ fn test_write(_version: repak::Version, _file_name: &str, bytes: &[u8]) {
Some(0x205C5A7D),
);
pak_writer.parallel(|writer| {
for path in pak_reader.files() {
let data = pak_reader.get(&path, &mut reader).unwrap();
writer.write_file(path, data).unwrap();
}
}).unwrap();
for path in pak_reader.files() {
let data = pak_reader.get(&path, &mut reader).unwrap();
pak_writer.write_file(&path, data).unwrap();
}
assert!(pak_writer.write_index().unwrap().into_inner() == reader.into_inner());
}

2
repak_cli/Cargo.toml
View file

@ -19,7 +19,7 @@ path = "src/main.rs"
[features]
default = ["oodle"]
oodle = ["repak/oodle_implicit_dynamic"]
oodle = ["repak/oodle"]
[dependencies]
repak = { path = "../repak" }