From 6680e7871b1b6f74e9939fb421f2dad7e99ad927 Mon Sep 17 00:00:00 2001 From: spuds <71292624+bananaturtlesandwich@users.noreply.github.com> Date: Tue, 19 Sep 2023 22:47:40 +0100 Subject: [PATCH 01/16] remove automatic download --- repak/Cargo.toml | 7 +- repak/src/entry.rs | 182 +++++++++++---------------------------------- repak/src/lib.rs | 20 ++++- 3 files changed, 61 insertions(+), 148 deletions(-) diff --git a/repak/Cargo.toml b/repak/Cargo.toml index 332e236..c149198 100644 --- a/repak/Cargo.toml +++ b/repak/Cargo.toml @@ -9,7 +9,7 @@ edition.workspace = true [features] default = ["compression", "encryption"] compression = ["dep:flate2", "dep:zstd"] -oodle = ["dep:libloading", "dep:ureq", "dep:once_cell", "dep:hex-literal", "dep:hex"] +oodle = [] encryption = ["dep:aes"] [dependencies] @@ -20,11 +20,6 @@ zstd = { version = "0.12", optional = true } thiserror = "1.0" sha1 = "0.10.5" strum = { workspace = true } -libloading = { version = "0.7", optional = true } -ureq = { version = "2.6", optional = true } -once_cell = { version = "1.17", optional = true} -hex-literal = { version = "0.4", optional = true } -hex = { workspace = true, optional = true } [dev-dependencies] base64 = { workspace = true } diff --git a/repak/src/entry.rs b/repak/src/entry.rs index 00b3dfb..dc05d84 100644 --- a/repak/src/entry.rs +++ b/repak/src/entry.rs @@ -375,112 +375,52 @@ impl Entry { } } #[cfg(feature = "oodle")] - Some(Compression::Oodle) => { - #[cfg(not(target_os = "windows"))] - return Err(super::Error::Oodle); + Some(Compression::Oodle) => unsafe { + // #[allow(non_snake_case)] + // #[allow(clippy::type_complexity)] + // let OodleLZ_Decompress = lib.get(b"OodleLZ_Decompress").unwrap(); + let mut decompressed = vec![0; self.uncompressed as usize]; - #[cfg(target_os = "windows")] - unsafe { - use std::ops::Deref; - let lib = match OODLE.deref() { - Ok(lib) => Ok(lib), - Err(e) => Err(super::Error::Other(e.to_string())), - }?; - - /* - let set_printf: libloading::Symbol< - unsafe extern "C" fn( - unsafe extern "C" fn( - i32, - *const std::ffi::c_char, - i32, - *const std::ffi::c_char, - ... - ) -> std::ffi::c_int, - ), - > = lib.get(b"OodleCore_Plugins_SetPrintf").unwrap(); - - pub unsafe extern "C" fn printf( - a: i32, - b: *const std::ffi::c_char, - c: i32, - str: *const std::ffi::c_char, - mut args: ... - ) -> std::ffi::c_int { - use printf_compat::{format, output}; - let mut s = String::new(); - let bytes_written = format(str, args.as_va_list(), output::fmt_write(&mut s)); - print!("[OODLE]: {}", s); - bytes_written - } - - set_printf(printf); - */ - - #[allow(non_snake_case)] - #[allow(clippy::type_complexity)] - let OodleLZ_Decompress: libloading::Symbol< - extern "C" fn( - compBuf: *mut u8, - compBufSize: usize, - rawBuf: *mut u8, - rawLen: usize, - fuzzSafe: u32, - checkCRC: u32, - verbosity: u32, - decBufBase: u64, - decBufSize: usize, - fpCallback: u64, - callbackUserData: u64, - decoderMemory: *mut u8, - decoderMemorySize: usize, - threadPhase: u32, - ) -> i32, - > = lib.get(b"OodleLZ_Decompress").unwrap(); - - let mut decompressed = vec![0; self.uncompressed as usize]; - - let mut compress_offset = 0; - let mut decompress_offset = 0; - let block_count = ranges.len(); - for range in ranges { - let decomp = if block_count == 1 { - self.uncompressed as usize - } else { - (self.compression_block_size as usize) - .min(self.uncompressed as usize - compress_offset) - }; - let buffer = &mut data[range]; - let out = OodleLZ_Decompress( - buffer.as_mut_ptr(), - buffer.len(), - decompressed.as_mut_ptr().offset(decompress_offset), - decomp, - 1, - 1, - 0, //verbose 3 - 0, - 0, - 0, - 0, - std::ptr::null_mut(), - 0, - 3, - ); - if out == 0 { - return Err(super::Error::DecompressionFailed(Compression::Oodle)); - } - compress_offset += self.compression_block_size as usize; - decompress_offset += out as isize; - } - - assert_eq!( - decompress_offset, self.uncompressed as isize, - "Oodle decompression length mismatch" + let mut compress_offset = 0; + let mut decompress_offset = 0; + let block_count = ranges.len(); + for range in ranges { + let decomp = if block_count == 1 { + self.uncompressed as usize + } else { + (self.compression_block_size as usize) + .min(self.uncompressed as usize - compress_offset) + }; + let buffer = &mut data[range]; + let out = OodleLZ_Decompress( + buffer.as_mut_ptr(), + buffer.len(), + decompressed.as_mut_ptr().offset(decompress_offset), + decomp, + 1, + 1, + 0, //verbose 3 + 0, + 0, + 0, + 0, + std::ptr::null_mut(), + 0, + 3, ); - buf.write_all(&decompressed)?; + if out == 0 { + return Err(super::Error::DecompressionFailed(Compression::Oodle)); + } + compress_offset += self.compression_block_size as usize; + decompress_offset += out as isize; } - } + + assert_eq!( + decompress_offset, self.uncompressed as isize, + "Oodle decompression length mismatch" + ); + buf.write_all(&decompressed)?; + }, #[cfg(not(feature = "oodle"))] Some(Compression::Oodle) => return Err(super::Error::Oodle), #[cfg(not(feature = "compression"))] @@ -491,42 +431,6 @@ impl Entry { } } -#[cfg(feature = "oodle")] -use once_cell::sync::Lazy; -#[cfg(feature = "oodle")] -static OODLE: Lazy> = - Lazy::new(|| get_oodle().map_err(|e| e.to_string())); -#[cfg(feature = "oodle")] -static OODLE_HASH: [u8; 20] = hex_literal::hex!("4bcc73614cb8fd2b0bce8d0f91ee5f3202d9d624"); - -#[cfg(feature = "oodle")] -fn get_oodle() -> Result { - use sha1::{Digest, Sha1}; - - let oodle = std::env::current_exe()?.with_file_name("oo2core_9_win64.dll"); - if !oodle.exists() { - let mut data = vec![]; - ureq::get("https://cdn.discordapp.com/attachments/817251677086285848/992648087371792404/oo2core_9_win64.dll") - .call().map_err(Box::new)? - .into_reader().read_to_end(&mut data)?; - - std::fs::write(&oodle, data)?; - } - - let mut hasher = Sha1::new(); - hasher.update(std::fs::read(&oodle)?); - let hash = hasher.finalize(); - (hash[..] == OODLE_HASH).then_some(()).ok_or_else(|| { - super::Error::Other(format!( - "oodle hash mismatch expected: {} got: {} ", - hex::encode(OODLE_HASH), - hex::encode(hash) - )) - })?; - - unsafe { libloading::Library::new(oodle) }.map_err(|_| super::Error::OodleFailed) -} - mod test { #[test] fn test_entry() { diff --git a/repak/src/lib.rs b/repak/src/lib.rs index e6ea5a1..18f942e 100644 --- a/repak/src/lib.rs +++ b/repak/src/lib.rs @@ -7,11 +7,25 @@ mod pak; pub use {error::*, pak::*}; -#[cfg(all(feature = "oodle", not(target_os = "windows")))] -compile_error!("Oodle compression only supported on Windows (or WINE)"); - pub const MAGIC: u32 = 0x5A6F12E1; +type DECOMPRESS = fn( + compBuf: *mut u8, + compBufSize: usize, + rawBuf: *mut u8, + rawLen: usize, + fuzzSafe: u32, + checkCRC: u32, + verbosity: u32, + decBufBase: u64, + decBufSize: usize, + fpCallback: u64, + callbackUserData: u64, + decoderMemory: *mut u8, + decoderMemorySize: usize, + threadPhase: u32, +) -> i32; + #[derive( Clone, Copy, From 1a79469a1140296e1d058ca8fbd3ae550e2424a5 Mon Sep 17 00:00:00 2001 From: spuds <71292624+bananaturtlesandwich@users.noreply.github.com> Date: Tue, 19 Sep 2023 23:11:40 +0100 Subject: [PATCH 02/16] pass function pointer --- repak/src/entry.rs | 11 ++++++----- repak/src/error.rs | 15 ++------------- repak/src/lib.rs | 11 +++++++++++ repak/src/pak.rs | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/repak/src/entry.rs b/repak/src/entry.rs index dc05d84..184dee6 100644 --- a/repak/src/entry.rs +++ b/repak/src/entry.rs @@ -307,6 +307,7 @@ impl Entry { compression: &[Compression], #[allow(unused)] key: &super::Key, buf: &mut W, + #[allow(unused)] oodle: super::Oodle, ) -> Result<(), super::Error> { reader.seek(io::SeekFrom::Start(self.offset))?; Entry::read(reader, version)?; @@ -376,9 +377,9 @@ impl Entry { } #[cfg(feature = "oodle")] Some(Compression::Oodle) => unsafe { - // #[allow(non_snake_case)] - // #[allow(clippy::type_complexity)] - // let OodleLZ_Decompress = lib.get(b"OodleLZ_Decompress").unwrap(); + let super::Oodle::Some(oodle) = oodle else { + return Err(super::Error::OodleFailed); + }; let mut decompressed = vec![0; self.uncompressed as usize]; let mut compress_offset = 0; @@ -392,7 +393,7 @@ impl Entry { .min(self.uncompressed as usize - compress_offset) }; let buffer = &mut data[range]; - let out = OodleLZ_Decompress( + let out = oodle( buffer.as_mut_ptr(), buffer.len(), decompressed.as_mut_ptr().offset(decompress_offset), @@ -415,7 +416,7 @@ impl Entry { decompress_offset += out as isize; } - assert_eq!( + debug_assert_eq!( decompress_offset, self.uncompressed as isize, "Oodle decompression length mismatch" ); diff --git a/repak/src/error.rs b/repak/src/error.rs index 29a7642..f023917 100644 --- a/repak/src/error.rs +++ b/repak/src/error.rs @@ -16,14 +16,7 @@ pub enum Error { #[error("enable the encryption feature to read encrypted paks")] Encryption, - #[cfg_attr( - windows, - error("enable the oodle feature to read Oodle compressed paks") - )] - #[cfg_attr( - not(windows), - error("Oodle compression only supported on Windows (or WINE)") - )] + #[error("enable the oodle feature to read Oodle compressed paks")] Oodle, // std errors @@ -39,10 +32,6 @@ pub enum Error { #[error("utf16 conversion: {0}")] Utf16(#[from] std::string::FromUtf16Error), - #[cfg(feature = "oodle")] - #[error("ureq error: {0}")] - Ureq(#[from] Box), // boxed because ureq::Error is quite large - #[error("bufwriter dereference: {0}")] IntoInner(#[from] std::io::IntoInnerError>>), @@ -53,7 +42,7 @@ pub enum Error { #[error("found magic of {0:#x} instead of {:#x}", super::MAGIC)] Magic(u32), - #[error("Could not load oo2core_9_win64.dll")] + #[error("pointer to OodleLZ_Decompress was not provided")] OodleFailed, #[error("No entry found at {0}")] diff --git a/repak/src/lib.rs b/repak/src/lib.rs index 18f942e..eabf15c 100644 --- a/repak/src/lib.rs +++ b/repak/src/lib.rs @@ -151,3 +151,14 @@ impl From for Key { Self::Some(value) } } + +#[cfg(feature = "oodle")] +pub(crate) enum Oodle<'func> { + Some(&'func DECOMPRESS), + None, +} + +#[cfg(not(feature = "oodle"))] +pub(crate) enum Oodle { + None, +} diff --git a/repak/src/pak.rs b/repak/src/pak.rs index 0452a84..bdaf5b8 100644 --- a/repak/src/pak.rs +++ b/repak/src/pak.rs @@ -180,6 +180,18 @@ impl PakReader { Ok(data) } + #[cfg(feature = "oodle")] + pub fn get_with_oodle( + &self, + path: &str, + reader: &mut R, + oodle: &super::DECOMPRESS, + ) -> Result, super::Error> { + let mut data = Vec::new(); + self.read_file_with_oodle(path, reader, &mut data, oodle)?; + Ok(data) + } + pub fn read_file( &self, path: &str, @@ -193,6 +205,28 @@ impl PakReader { &self.pak.compression, &self.key, writer, + super::Oodle::None, + ), + None => Err(super::Error::MissingEntry(path.to_owned())), + } + } + + #[cfg(feature = "oodle")] + pub fn read_file_with_oodle( + &self, + path: &str, + reader: &mut R, + writer: &mut W, + oodle: &super::DECOMPRESS, + ) -> Result<(), super::Error> { + match self.pak.index.entries().get(path) { + Some(entry) => entry.read_file( + reader, + self.pak.version, + &self.pak.compression, + &self.key, + writer, + super::Oodle::Some(oodle), ), None => Err(super::Error::MissingEntry(path.to_owned())), } From 6c0ef1ce7b4bbdd1079ed919d8043b4dcf69ebd7 Mon Sep 17 00:00:00 2001 From: spuds <71292624+bananaturtlesandwich@users.noreply.github.com> Date: Wed, 20 Sep 2023 08:02:16 +0100 Subject: [PATCH 03/16] download in repak-cli --- Cargo.toml | 2 +- repak/Cargo.toml | 7 ++--- repak/src/lib.rs | 3 +- repak_cli/Cargo.toml | 8 +++++- repak_cli/src/main.rs | 67 ++++++++++++++++++++++++++++++++++++++++--- 5 files changed, 75 insertions(+), 12 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 63d145d..41d7c06 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,8 +12,8 @@ edition = "2021" [workspace.dependencies] aes = "0.8.2" base64 = "0.21.0" -hex = "0.4.3" strum = { version = "0.24", features = ["derive"] } +sha1 = "0.10" # generated by 'cargo dist init' [profile.dist] diff --git a/repak/Cargo.toml b/repak/Cargo.toml index c149198..a7b66c2 100644 --- a/repak/Cargo.toml +++ b/repak/Cargo.toml @@ -14,16 +14,13 @@ encryption = ["dep:aes"] [dependencies] byteorder = "1.4" -aes = { version = "0.8", optional = true } +aes = { workspace = true, optional = true } flate2 = { version = "1.0", optional = true } zstd = { version = "0.12", optional = true } thiserror = "1.0" -sha1 = "0.10.5" +sha1 = { workspace = true } strum = { workspace = true } [dev-dependencies] base64 = { workspace = true } paste = "1.0.11" - -[package.metadata.cargo-all-features] -denylist = ["oodle"] diff --git a/repak/src/lib.rs b/repak/src/lib.rs index eabf15c..9884bfe 100644 --- a/repak/src/lib.rs +++ b/repak/src/lib.rs @@ -9,7 +9,8 @@ pub use {error::*, pak::*}; pub const MAGIC: u32 = 0x5A6F12E1; -type DECOMPRESS = fn( +#[cfg(feature = "oodle")] +pub type DECOMPRESS = fn( compBuf: *mut u8, compBufSize: usize, rawBuf: *mut u8, diff --git a/repak_cli/Cargo.toml b/repak_cli/Cargo.toml index b31338b..b8c9f71 100644 --- a/repak_cli/Cargo.toml +++ b/repak_cli/Cargo.toml @@ -12,6 +12,12 @@ path = "src/main.rs" [target.'cfg(windows)'.dependencies] repak = { path = "../repak", features = ["oodle"] } +sha1 = { workspace = true } +ureq = "2.6" +once_cell = "1.17" +hex-literal = "0.4" +libloading = "0.7" + [target.'cfg(not(windows))'.dependencies] repak = { path = "../repak" } @@ -19,7 +25,7 @@ repak = { path = "../repak" } aes = { workspace = true } base64 = { workspace = true } clap = { version = "4.1.4", features = ["derive"] } -hex = { workspace = true } +hex = "0.4.3" indicatif = { version = "0.17.3", features = ["rayon"] } path-clean = "0.1.0" path-slash = "0.2.1" diff --git a/repak_cli/src/main.rs b/repak_cli/src/main.rs index fcd8b9f..6e798c1 100644 --- a/repak_cli/src/main.rs +++ b/repak_cli/src/main.rs @@ -118,7 +118,7 @@ enum Action { Info(ActionInfo), /// List .pak files List(ActionList), - /// List .pka files and the SHA256 of their contents. Useful for finding differences between paks + /// List .pak files and the SHA256 of their contents. Useful for finding differences between paks HashList(ActionHashList), /// Unpack .pak file Unpack(ActionUnpack), @@ -248,6 +248,12 @@ fn hash_list(aes_key: Option, action: ActionHashList) -> Result<(), let hashes: std::sync::Arc, Vec>>> = Default::default(); + use std::ops::Deref; + let lib = match OODLE.deref() { + Ok(lib) => lib, + Err(e) => return Err(repak::Error::Other(e.clone())), + }; + let oodle_decompress = unsafe { lib.get(b"OodleLZ_Decompress") }.unwrap(); full_paths.par_iter().zip(stripped).try_for_each_init( || (hashes.clone(), File::open(&action.input)), @@ -255,10 +261,11 @@ fn hash_list(aes_key: Option, action: ActionHashList) -> Result<(), use sha2::Digest; let mut hasher = sha2::Sha256::new(); - pak.read_file( + pak.read_file_with_oodle( path, &mut BufReader::new(file.as_ref().unwrap()), &mut hasher, + &oodle_decompress, )?; let hash = hasher.finalize(); hashes @@ -354,6 +361,12 @@ fn unpack(aes_key: Option, action: ActionUnpack) -> Result<(), repa let progress = indicatif::ProgressBar::new(entries.len() as u64) .with_style(indicatif::ProgressStyle::with_template(STYLE).unwrap()); + use std::ops::Deref; + let lib = match OODLE.deref() { + Ok(lib) => lib, + Err(e) => return Err(repak::Error::Other(e.clone())), + }; + let decompress = unsafe { lib.get(b"OodleLZ_Decompress") }.unwrap(); entries.par_iter().try_for_each_init( || (progress.clone(), File::open(input)), |(progress, file), entry| -> Result<(), repak::Error> { @@ -361,13 +374,14 @@ fn unpack(aes_key: Option, action: ActionUnpack) -> Result<(), repa progress.println(format!("unpacking {}", entry.entry_path)); } fs::create_dir_all(&entry.out_dir)?; - pak.read_file( + pak.read_file_with_oodle( &entry.entry_path, &mut BufReader::new( file.as_ref() .map_err(|e| repak::Error::Other(format!("error reading pak: {e}")))?, ), &mut fs::File::create(&entry.out_path)?, + &decompress, )?; progress.inc(1); Ok(()) @@ -463,6 +477,51 @@ fn get(aes_key: Option, args: ActionGet) -> Result<(), repak::Error })?; use std::io::Write; - std::io::stdout().write_all(&pak.get(&file.to_slash_lossy(), &mut reader)?)?; + use std::ops::Deref; + let lib = match OODLE.deref() { + Ok(lib) => lib, + Err(e) => return Err(repak::Error::Other(e.clone())), + }; + std::io::stdout().write_all(&pak.get_with_oodle( + &file.to_slash_lossy(), + &mut reader, + &unsafe { lib.get(b"OodleLZ_Decompress") }.unwrap(), + )?)?; Ok(()) } + +#[cfg(windows)] +use once_cell::sync::Lazy; +#[cfg(windows)] +static OODLE: Lazy> = + Lazy::new(|| get_oodle().map_err(|e| e.to_string())); +#[cfg(windows)] +static OODLE_HASH: [u8; 20] = hex_literal::hex!("4bcc73614cb8fd2b0bce8d0f91ee5f3202d9d624"); + +#[cfg(windows)] +fn get_oodle() -> Result { + use sha1::{Digest, Sha1}; + + let oodle = std::env::current_exe()?.with_file_name("oo2core_9_win64.dll"); + if !oodle.exists() { + let mut data = vec![]; + ureq::get("https://cdn.discordapp.com/attachments/817251677086285848/992648087371792404/oo2core_9_win64.dll") + .call().map_err(|e| repak::Error::Other(e.to_string()))? + .into_reader().read_to_end(&mut data)?; + + std::fs::write(&oodle, data)?; + } + + let mut hasher = Sha1::new(); + hasher.update(std::fs::read(&oodle)?); + let hash = hasher.finalize(); + (hash[..] == OODLE_HASH).then_some(()).ok_or_else(|| { + repak::Error::Other(format!( + "oodle hash mismatch expected: {} got: {} ", + hex::encode(OODLE_HASH), + hex::encode(hash) + )) + })?; + + unsafe { libloading::Library::new(oodle) }.map_err(|_| repak::Error::OodleFailed) +} From 01515310e4867427d76101131f26f8d5e453cbac Mon Sep 17 00:00:00 2001 From: spuds <71292624+bananaturtlesandwich@users.noreply.github.com> Date: Wed, 20 Sep 2023 10:07:47 +0100 Subject: [PATCH 04/16] get oodle decompress function --- repak_cli/src/main.rs | 57 ++++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/repak_cli/src/main.rs b/repak_cli/src/main.rs index 6e798c1..f71fea9 100644 --- a/repak_cli/src/main.rs +++ b/repak_cli/src/main.rs @@ -1,6 +1,8 @@ use std::collections::BTreeMap; use std::fs::{self, File}; use std::io::{self, BufReader, BufWriter}; +#[cfg(windows)] +use std::ops::Deref; use std::path::{Path, PathBuf}; use clap::builder::TypedValueParser; @@ -248,12 +250,8 @@ fn hash_list(aes_key: Option, action: ActionHashList) -> Result<(), let hashes: std::sync::Arc, Vec>>> = Default::default(); - use std::ops::Deref; - let lib = match OODLE.deref() { - Ok(lib) => lib, - Err(e) => return Err(repak::Error::Other(e.clone())), - }; - let oodle_decompress = unsafe { lib.get(b"OodleLZ_Decompress") }.unwrap(); + #[cfg(windows)] + let oodle_decompress = decompress()?; full_paths.par_iter().zip(stripped).try_for_each_init( || (hashes.clone(), File::open(&action.input)), @@ -261,6 +259,13 @@ fn hash_list(aes_key: Option, action: ActionHashList) -> Result<(), use sha2::Digest; let mut hasher = sha2::Sha256::new(); + #[cfg(not(windows))] + pak.read_file( + path, + &mut BufReader::new(file.as_ref().unwrap()), + &mut hasher, + )?; + #[cfg(windows)] pak.read_file_with_oodle( path, &mut BufReader::new(file.as_ref().unwrap()), @@ -361,12 +366,9 @@ fn unpack(aes_key: Option, action: ActionUnpack) -> Result<(), repa let progress = indicatif::ProgressBar::new(entries.len() as u64) .with_style(indicatif::ProgressStyle::with_template(STYLE).unwrap()); - use std::ops::Deref; - let lib = match OODLE.deref() { - Ok(lib) => lib, - Err(e) => return Err(repak::Error::Other(e.clone())), - }; - let decompress = unsafe { lib.get(b"OodleLZ_Decompress") }.unwrap(); + + #[cfg(windows)] + let oodle_decompress = decompress()?; entries.par_iter().try_for_each_init( || (progress.clone(), File::open(input)), |(progress, file), entry| -> Result<(), repak::Error> { @@ -374,6 +376,16 @@ fn unpack(aes_key: Option, action: ActionUnpack) -> Result<(), repa progress.println(format!("unpacking {}", entry.entry_path)); } fs::create_dir_all(&entry.out_dir)?; + #[cfg(not(windows))] + pak.read_file( + &entry.entry_path, + &mut BufReader::new( + file.as_ref() + .map_err(|e| repak::Error::Other(format!("error reading pak: {e}")))?, + ), + &mut fs::File::create(&entry.out_path)?, + )?; + #[cfg(windows)] pak.read_file_with_oodle( &entry.entry_path, &mut BufReader::new( @@ -381,7 +393,7 @@ fn unpack(aes_key: Option, action: ActionUnpack) -> Result<(), repa .map_err(|e| repak::Error::Other(format!("error reading pak: {e}")))?, ), &mut fs::File::create(&entry.out_path)?, - &decompress, + &oodle_decompress, )?; progress.inc(1); Ok(()) @@ -477,17 +489,22 @@ fn get(aes_key: Option, args: ActionGet) -> Result<(), repak::Error })?; use std::io::Write; - use std::ops::Deref; + std::io::stdout().write_all( + #[cfg(not(windows))] + &pak.get(&file.to_slash_lossy(), &mut reader)?, + #[cfg(windows)] + &pak.get_with_oodle(&file.to_slash_lossy(), &mut reader, decompress()?.deref())?, + )?; + Ok(()) +} + +#[cfg(windows)] +fn decompress<'func>() -> Result, repak::Error> { let lib = match OODLE.deref() { Ok(lib) => lib, Err(e) => return Err(repak::Error::Other(e.clone())), }; - std::io::stdout().write_all(&pak.get_with_oodle( - &file.to_slash_lossy(), - &mut reader, - &unsafe { lib.get(b"OodleLZ_Decompress") }.unwrap(), - )?)?; - Ok(()) + Ok(unsafe { lib.get(b"OodleLZ_Decompress") }.unwrap()) } #[cfg(windows)] From 564c4f6a4523f6530f08134ad85155d45950196c Mon Sep 17 00:00:00 2001 From: spuds <71292624+bananaturtlesandwich@users.noreply.github.com> Date: Wed, 20 Sep 2023 13:23:42 +0100 Subject: [PATCH 05/16] make decompress unsafe extern c --- repak/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repak/src/lib.rs b/repak/src/lib.rs index 9884bfe..34e3ace 100644 --- a/repak/src/lib.rs +++ b/repak/src/lib.rs @@ -10,7 +10,7 @@ pub use {error::*, pak::*}; pub const MAGIC: u32 = 0x5A6F12E1; #[cfg(feature = "oodle")] -pub type DECOMPRESS = fn( +pub type DECOMPRESS = unsafe extern "C" fn( compBuf: *mut u8, compBufSize: usize, rawBuf: *mut u8, From e2edd97d6e2be7838ae554e8060ab7489a8bb2fd Mon Sep 17 00:00:00 2001 From: spuds <71292624+bananaturtlesandwich@users.noreply.github.com> Date: Wed, 20 Sep 2023 19:42:48 +0100 Subject: [PATCH 06/16] use builder pattern --- repak/Cargo.toml | 1 + repak/src/entry.rs | 3 +- repak/src/lib.rs | 7 +- repak/src/pak.rs | 167 +++++++++++---------------------------------- 4 files changed, 47 insertions(+), 131 deletions(-) diff --git a/repak/Cargo.toml b/repak/Cargo.toml index a7b66c2..45d5dd4 100644 --- a/repak/Cargo.toml +++ b/repak/Cargo.toml @@ -20,6 +20,7 @@ zstd = { version = "0.12", optional = true } thiserror = "1.0" sha1 = { workspace = true } strum = { workspace = true } +once_cell = "1.18" [dev-dependencies] base64 = { workspace = true } diff --git a/repak/src/entry.rs b/repak/src/entry.rs index 184dee6..475f5a3 100644 --- a/repak/src/entry.rs +++ b/repak/src/entry.rs @@ -307,7 +307,6 @@ impl Entry { compression: &[Compression], #[allow(unused)] key: &super::Key, buf: &mut W, - #[allow(unused)] oodle: super::Oodle, ) -> Result<(), super::Error> { reader.seek(io::SeekFrom::Start(self.offset))?; Entry::read(reader, version)?; @@ -377,7 +376,7 @@ impl Entry { } #[cfg(feature = "oodle")] Some(Compression::Oodle) => unsafe { - let super::Oodle::Some(oodle) = oodle else { + let Some(ref oodle) = super::OODLE else { return Err(super::Error::OodleFailed); }; let mut decompressed = vec![0; self.uncompressed as usize]; diff --git a/repak/src/lib.rs b/repak/src/lib.rs index 34e3ace..887e3bf 100644 --- a/repak/src/lib.rs +++ b/repak/src/lib.rs @@ -10,7 +10,10 @@ pub use {error::*, pak::*}; pub const MAGIC: u32 = 0x5A6F12E1; #[cfg(feature = "oodle")] -pub type DECOMPRESS = unsafe extern "C" fn( +static mut OODLE: Option> = None; + +#[cfg(feature = "oodle")] +pub type Decompress = unsafe extern "C" fn( compBuf: *mut u8, compBufSize: usize, rawBuf: *mut u8, @@ -155,7 +158,7 @@ impl From for Key { #[cfg(feature = "oodle")] pub(crate) enum Oodle<'func> { - Some(&'func DECOMPRESS), + Some(&'func Decompress), None, } diff --git a/repak/src/pak.rs b/repak/src/pak.rs index bdaf5b8..6d35780 100644 --- a/repak/src/pak.rs +++ b/repak/src/pak.rs @@ -4,6 +4,46 @@ use byteorder::{ReadBytesExt, WriteBytesExt, LE}; use std::collections::BTreeMap; use std::io::{self, Read, Seek, Write}; +#[derive(Debug)] +pub struct PakBuilder { + key: super::Key, +} + +impl PakBuilder { + pub fn new() -> Self { + Self { + key: super::Key::None, + } + } + #[cfg(feature = "encryption")] + pub fn key(&mut self, key: aes::Aes256) { + self.key = super::Key::Some(key) + } + #[cfg(feature = "oodle")] + pub fn oodle(&mut self, oodle: fn() -> super::Decompress) { + unsafe { super::OODLE = Some(once_cell::sync::Lazy::new(oodle)) } + } + pub fn reader(self, reader: &mut R) -> Result { + PakReader::new_any_inner(reader, self.key) + } + pub fn reader_with_version( + self, + reader: &mut R, + version: super::Version, + ) -> Result { + PakReader::new_inner(reader, version, self.key) + } + pub fn writer( + self, + writer: W, + version: super::Version, + mount_point: String, + path_hash_seed: Option, + ) -> PakWriter { + PakWriter::new_inner(writer, self.key, version, mount_point, path_hash_seed) + } +} + #[derive(Debug)] pub struct PakReader { pak: Pak, @@ -83,29 +123,6 @@ fn decrypt(key: &super::Key, bytes: &mut [u8]) -> Result<(), super::Error> { } impl PakReader { - pub fn new_any(reader: &mut R) -> Result { - Self::new_any_inner(reader, super::Key::None) - } - - #[cfg(feature = "encryption")] - pub fn new_any_with_key( - reader: &mut R, - key: aes::Aes256, - ) -> Result { - Self::new_any_inner(reader, key.into()) - } - - #[cfg(feature = "encryption")] - pub fn new_any_with_optional_key( - reader: &mut R, - key: Option, - ) -> Result { - match key { - Some(key) => Self::new_any_with_key(reader, key), - None => Self::new_any(reader), - } - } - fn new_any_inner( reader: &mut R, key: super::Key, @@ -122,34 +139,6 @@ impl PakReader { Err(super::Error::UnsupportedOrEncrypted(log)) } - pub fn new( - reader: &mut R, - version: super::Version, - ) -> Result { - Self::new_inner(reader, version, super::Key::None) - } - - #[cfg(feature = "encryption")] - pub fn new_with_key( - reader: &mut R, - version: super::Version, - key: aes::Aes256, - ) -> Result { - Self::new_inner(reader, version, key.into()) - } - - #[cfg(feature = "encryption")] - pub fn new_with_optional_key( - reader: &mut R, - version: super::Version, - key: Option, - ) -> Result { - match key { - Some(key) => Self::new_with_key(reader, version, key), - None => Self::new(reader, version), - } - } - fn new_inner( reader: &mut R, version: super::Version, @@ -180,18 +169,6 @@ impl PakReader { Ok(data) } - #[cfg(feature = "oodle")] - pub fn get_with_oodle( - &self, - path: &str, - reader: &mut R, - oodle: &super::DECOMPRESS, - ) -> Result, super::Error> { - let mut data = Vec::new(); - self.read_file_with_oodle(path, reader, &mut data, oodle)?; - Ok(data) - } - pub fn read_file( &self, path: &str, @@ -205,28 +182,6 @@ impl PakReader { &self.pak.compression, &self.key, writer, - super::Oodle::None, - ), - None => Err(super::Error::MissingEntry(path.to_owned())), - } - } - - #[cfg(feature = "oodle")] - pub fn read_file_with_oodle( - &self, - path: &str, - reader: &mut R, - writer: &mut W, - oodle: &super::DECOMPRESS, - ) -> Result<(), super::Error> { - match self.pak.index.entries().get(path) { - Some(entry) => entry.read_file( - reader, - self.pak.version, - &self.pak.compression, - &self.key, - writer, - super::Oodle::Some(oodle), ), None => Err(super::Error::MissingEntry(path.to_owned())), } @@ -250,48 +205,6 @@ impl PakReader { } impl PakWriter { - pub fn new( - writer: W, - version: Version, - mount_point: String, - path_hash_seed: Option, - ) -> Self { - PakWriter { - pak: Pak::new(version, mount_point, path_hash_seed), - writer, - key: super::Key::None, - } - } - - #[cfg(feature = "encryption")] - pub fn new_with_key( - writer: W, - key: aes::Aes256, - version: Version, - mount_point: String, - path_hash_seed: Option, - ) -> Self { - PakWriter { - pak: Pak::new(version, mount_point, path_hash_seed), - writer, - key: key.into(), - } - } - - #[cfg(feature = "encryption")] - pub fn new_with_optional_key( - writer: W, - key: Option, - version: Version, - mount_point: String, - path_hash_seed: Option, - ) -> Self { - match key { - Some(key) => Self::new_with_key(writer, key, version, mount_point, path_hash_seed), - None => Self::new(writer, version, mount_point, path_hash_seed), - } - } - fn new_inner( writer: W, key: super::Key, From 63396ac9f677f12470ab632b6f4cb097bcd39f83 Mon Sep 17 00:00:00 2001 From: spuds <71292624+bananaturtlesandwich@users.noreply.github.com> Date: Wed, 20 Sep 2023 20:08:11 +0100 Subject: [PATCH 07/16] update repak-cli --- repak_cli/src/main.rs | 90 ++++++++++++++++--------------------------- 1 file changed, 33 insertions(+), 57 deletions(-) diff --git a/repak_cli/src/main.rs b/repak_cli/src/main.rs index f71fea9..e7204e1 100644 --- a/repak_cli/src/main.rs +++ b/repak_cli/src/main.rs @@ -1,8 +1,6 @@ use std::collections::BTreeMap; use std::fs::{self, File}; use std::io::{self, BufReader, BufWriter}; -#[cfg(windows)] -use std::ops::Deref; use std::path::{Path, PathBuf}; use clap::builder::TypedValueParser; @@ -177,10 +175,11 @@ fn main() -> Result<(), repak::Error> { } fn info(aes_key: Option, action: ActionInfo) -> Result<(), repak::Error> { - let pak = repak::PakReader::new_any_with_optional_key( - &mut BufReader::new(File::open(action.input)?), - aes_key, - )?; + let mut builder = repak::PakBuilder::new(); + if let Some(aes_key) = aes_key { + builder.key(aes_key) + } + let pak = builder.reader(&mut BufReader::new(File::open(action.input)?))?; println!("mount point: {}", pak.mount_point()); println!("version: {}", pak.version()); println!("version major: {}", pak.version().version_major()); @@ -191,10 +190,11 @@ fn info(aes_key: Option, action: ActionInfo) -> Result<(), repak::E } fn list(aes_key: Option, action: ActionList) -> Result<(), repak::Error> { - let pak = repak::PakReader::new_any_with_optional_key( - &mut BufReader::new(File::open(action.input)?), - aes_key, - )?; + let mut builder = repak::PakBuilder::new(); + if let Some(aes_key) = aes_key { + builder.key(aes_key) + } + let pak = builder.reader(&mut BufReader::new(File::open(action.input)?))?; let mount_point = PathBuf::from(pak.mount_point()); let prefix = Path::new(&action.strip_prefix); @@ -223,10 +223,11 @@ fn list(aes_key: Option, action: ActionList) -> Result<(), repak::E } fn hash_list(aes_key: Option, action: ActionHashList) -> Result<(), repak::Error> { - let pak = repak::PakReader::new_any_with_optional_key( - &mut BufReader::new(File::open(&action.input)?), - aes_key, - )?; + let mut builder = repak::PakBuilder::new(); + if let Some(aes_key) = aes_key { + builder.key(aes_key) + } + let pak = builder.reader(&mut BufReader::new(File::open(&action.input)?))?; let mount_point = PathBuf::from(pak.mount_point()); let prefix = Path::new(&action.strip_prefix); @@ -250,28 +251,17 @@ fn hash_list(aes_key: Option, action: ActionHashList) -> Result<(), let hashes: std::sync::Arc, Vec>>> = Default::default(); - #[cfg(windows)] - let oodle_decompress = decompress()?; - full_paths.par_iter().zip(stripped).try_for_each_init( || (hashes.clone(), File::open(&action.input)), |(hashes, file), ((_full_path, path), stripped)| -> Result<(), repak::Error> { use sha2::Digest; let mut hasher = sha2::Sha256::new(); - #[cfg(not(windows))] pak.read_file( path, &mut BufReader::new(file.as_ref().unwrap()), &mut hasher, )?; - #[cfg(windows)] - pak.read_file_with_oodle( - path, - &mut BufReader::new(file.as_ref().unwrap()), - &mut hasher, - &oodle_decompress, - )?; let hash = hasher.finalize(); hashes .lock() @@ -292,10 +282,13 @@ const STYLE: &str = "[{elapsed_precise}] [{wide_bar}] {pos}/{len} ({eta})"; fn unpack(aes_key: Option, action: ActionUnpack) -> Result<(), repak::Error> { for input in &action.input { - let pak = repak::PakReader::new_any_with_optional_key( - &mut BufReader::new(File::open(input)?), - aes_key.clone(), - )?; + let mut builder = repak::PakBuilder::new(); + #[cfg(windows)] + builder.oodle(decompress); + if let Some(aes_key) = aes_key.clone() { + builder.key(aes_key) + } + let pak = builder.reader(&mut BufReader::new(File::open(input)?))?; let output = action .output .as_ref() @@ -367,8 +360,6 @@ fn unpack(aes_key: Option, action: ActionUnpack) -> Result<(), repa let progress = indicatif::ProgressBar::new(entries.len() as u64) .with_style(indicatif::ProgressStyle::with_template(STYLE).unwrap()); - #[cfg(windows)] - let oodle_decompress = decompress()?; entries.par_iter().try_for_each_init( || (progress.clone(), File::open(input)), |(progress, file), entry| -> Result<(), repak::Error> { @@ -376,7 +367,6 @@ fn unpack(aes_key: Option, action: ActionUnpack) -> Result<(), repa progress.println(format!("unpacking {}", entry.entry_path)); } fs::create_dir_all(&entry.out_dir)?; - #[cfg(not(windows))] pak.read_file( &entry.entry_path, &mut BufReader::new( @@ -385,16 +375,6 @@ fn unpack(aes_key: Option, action: ActionUnpack) -> Result<(), repa ), &mut fs::File::create(&entry.out_path)?, )?; - #[cfg(windows)] - pak.read_file_with_oodle( - &entry.entry_path, - &mut BufReader::new( - file.as_ref() - .map_err(|e| repak::Error::Other(format!("error reading pak: {e}")))?, - ), - &mut fs::File::create(&entry.out_path)?, - &oodle_decompress, - )?; progress.inc(1); Ok(()) }, @@ -441,9 +421,8 @@ fn pack(args: ActionPack) -> Result<(), repak::Error> { collect_files(&mut paths, input_path)?; paths.sort(); - let mut pak = repak::PakWriter::new_with_optional_key( + let mut pak = repak::PakBuilder::new().writer( BufWriter::new(File::create(&output)?), - None, args.version, args.mount_point, Some(args.path_hash_seed), @@ -476,7 +455,13 @@ fn pack(args: ActionPack) -> Result<(), repak::Error> { fn get(aes_key: Option, args: ActionGet) -> Result<(), repak::Error> { let mut reader = BufReader::new(File::open(&args.input)?); - let pak = repak::PakReader::new_any_with_optional_key(&mut reader, aes_key)?; + let mut builder = repak::PakBuilder::new(); + #[cfg(windows)] + builder.oodle(decompress); + if let Some(aes_key) = aes_key { + builder.key(aes_key) + } + let pak = builder.reader(&mut reader)?; let mount_point = PathBuf::from(pak.mount_point()); let prefix = Path::new(&args.strip_prefix); @@ -489,22 +474,13 @@ fn get(aes_key: Option, args: ActionGet) -> Result<(), repak::Error })?; use std::io::Write; - std::io::stdout().write_all( - #[cfg(not(windows))] - &pak.get(&file.to_slash_lossy(), &mut reader)?, - #[cfg(windows)] - &pak.get_with_oodle(&file.to_slash_lossy(), &mut reader, decompress()?.deref())?, - )?; + std::io::stdout().write_all(&pak.get(&file.to_slash_lossy(), &mut reader)?)?; Ok(()) } #[cfg(windows)] -fn decompress<'func>() -> Result, repak::Error> { - let lib = match OODLE.deref() { - Ok(lib) => lib, - Err(e) => return Err(repak::Error::Other(e.clone())), - }; - Ok(unsafe { lib.get(b"OodleLZ_Decompress") }.unwrap()) +fn decompress() -> repak::Decompress { + *unsafe { OODLE.as_ref().unwrap().get(b"OodleLZ_Decompress") }.unwrap() } #[cfg(windows)] From 6805b5c142ec452ee217b20bff6e8c6e1b8f3abf Mon Sep 17 00:00:00 2001 From: spuds <71292624+bananaturtlesandwich@users.noreply.github.com> Date: Wed, 20 Sep 2023 20:21:15 +0100 Subject: [PATCH 08/16] break get_oodle into separate package --- Cargo.toml | 1 + get_oodle/Cargo.toml | 16 +++++++++++++++ get_oodle/src/lib.rs | 35 +++++++++++++++++++++++++++++++++ repak_cli/Cargo.toml | 8 ++------ repak_cli/src/main.rs | 45 ++----------------------------------------- 5 files changed, 56 insertions(+), 49 deletions(-) create mode 100644 get_oodle/Cargo.toml create mode 100644 get_oodle/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 41d7c06..43472ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ aes = "0.8.2" base64 = "0.21.0" strum = { version = "0.24", features = ["derive"] } sha1 = "0.10" +hex = "0.4" # generated by 'cargo dist init' [profile.dist] diff --git a/get_oodle/Cargo.toml b/get_oodle/Cargo.toml new file mode 100644 index 0000000..7b03cee --- /dev/null +++ b/get_oodle/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "get_oodle" +repository.workspace = true +authors.workspace = true +license.workspace = true +version.workspace = true +edition.workspace = true + +[dependencies] +repak = { path = "../repak" } +sha1 = { workspace = true } +ureq = "2.6" +once_cell = "1.17" +hex-literal = "0.4" +libloading = "0.7" +hex = { workspace = true } \ No newline at end of file diff --git a/get_oodle/src/lib.rs b/get_oodle/src/lib.rs new file mode 100644 index 0000000..d4afc3f --- /dev/null +++ b/get_oodle/src/lib.rs @@ -0,0 +1,35 @@ +pub fn decompress() -> repak::Decompress { + *unsafe { OODLE.as_ref().unwrap().get(b"OodleLZ_Decompress") }.unwrap() +} + +use once_cell::sync::Lazy; +static OODLE: Lazy> = + Lazy::new(|| get_oodle().map_err(|e| e.to_string())); +static OODLE_HASH: [u8; 20] = hex_literal::hex!("4bcc73614cb8fd2b0bce8d0f91ee5f3202d9d624"); + +fn get_oodle() -> Result { + use sha1::{Digest, Sha1}; + + let oodle = std::env::current_exe()?.with_file_name("oo2core_9_win64.dll"); + if !oodle.exists() { + let mut data = vec![]; + ureq::get("https://cdn.discordapp.com/attachments/817251677086285848/992648087371792404/oo2core_9_win64.dll") + .call().map_err(|e| repak::Error::Other(e.to_string()))? + .into_reader().read_to_end(&mut data)?; + + std::fs::write(&oodle, data)?; + } + + let mut hasher = Sha1::new(); + hasher.update(std::fs::read(&oodle)?); + let hash = hasher.finalize(); + (hash[..] == OODLE_HASH).then_some(()).ok_or_else(|| { + repak::Error::Other(format!( + "oodle hash mismatch expected: {} got: {} ", + hex::encode(OODLE_HASH), + hex::encode(hash) + )) + })?; + + unsafe { libloading::Library::new(oodle) }.map_err(|_| repak::Error::OodleFailed) +} diff --git a/repak_cli/Cargo.toml b/repak_cli/Cargo.toml index b8c9f71..f3ab94a 100644 --- a/repak_cli/Cargo.toml +++ b/repak_cli/Cargo.toml @@ -12,20 +12,16 @@ path = "src/main.rs" [target.'cfg(windows)'.dependencies] repak = { path = "../repak", features = ["oodle"] } -sha1 = { workspace = true } -ureq = "2.6" -once_cell = "1.17" -hex-literal = "0.4" -libloading = "0.7" [target.'cfg(not(windows))'.dependencies] repak = { path = "../repak" } [dependencies] +get_oodle = { path = "../get_oodle" } aes = { workspace = true } base64 = { workspace = true } clap = { version = "4.1.4", features = ["derive"] } -hex = "0.4.3" +hex = { workspace = true } indicatif = { version = "0.17.3", features = ["rayon"] } path-clean = "0.1.0" path-slash = "0.2.1" diff --git a/repak_cli/src/main.rs b/repak_cli/src/main.rs index e7204e1..6d876dc 100644 --- a/repak_cli/src/main.rs +++ b/repak_cli/src/main.rs @@ -284,7 +284,7 @@ fn unpack(aes_key: Option, action: ActionUnpack) -> Result<(), repa for input in &action.input { let mut builder = repak::PakBuilder::new(); #[cfg(windows)] - builder.oodle(decompress); + builder.oodle(get_oodle::decompress); if let Some(aes_key) = aes_key.clone() { builder.key(aes_key) } @@ -457,7 +457,7 @@ fn get(aes_key: Option, args: ActionGet) -> Result<(), repak::Error let mut reader = BufReader::new(File::open(&args.input)?); let mut builder = repak::PakBuilder::new(); #[cfg(windows)] - builder.oodle(decompress); + builder.oodle(get_oodle::decompress); if let Some(aes_key) = aes_key { builder.key(aes_key) } @@ -477,44 +477,3 @@ fn get(aes_key: Option, args: ActionGet) -> Result<(), repak::Error std::io::stdout().write_all(&pak.get(&file.to_slash_lossy(), &mut reader)?)?; Ok(()) } - -#[cfg(windows)] -fn decompress() -> repak::Decompress { - *unsafe { OODLE.as_ref().unwrap().get(b"OodleLZ_Decompress") }.unwrap() -} - -#[cfg(windows)] -use once_cell::sync::Lazy; -#[cfg(windows)] -static OODLE: Lazy> = - Lazy::new(|| get_oodle().map_err(|e| e.to_string())); -#[cfg(windows)] -static OODLE_HASH: [u8; 20] = hex_literal::hex!("4bcc73614cb8fd2b0bce8d0f91ee5f3202d9d624"); - -#[cfg(windows)] -fn get_oodle() -> Result { - use sha1::{Digest, Sha1}; - - let oodle = std::env::current_exe()?.with_file_name("oo2core_9_win64.dll"); - if !oodle.exists() { - let mut data = vec![]; - ureq::get("https://cdn.discordapp.com/attachments/817251677086285848/992648087371792404/oo2core_9_win64.dll") - .call().map_err(|e| repak::Error::Other(e.to_string()))? - .into_reader().read_to_end(&mut data)?; - - std::fs::write(&oodle, data)?; - } - - let mut hasher = Sha1::new(); - hasher.update(std::fs::read(&oodle)?); - let hash = hasher.finalize(); - (hash[..] == OODLE_HASH).then_some(()).ok_or_else(|| { - repak::Error::Other(format!( - "oodle hash mismatch expected: {} got: {} ", - hex::encode(OODLE_HASH), - hex::encode(hash) - )) - })?; - - unsafe { libloading::Library::new(oodle) }.map_err(|_| repak::Error::OodleFailed) -} From a15e3fe372a894b8dc96a2111d8f930376ef859f Mon Sep 17 00:00:00 2001 From: spuds <71292624+bananaturtlesandwich@users.noreply.github.com> Date: Wed, 20 Sep 2023 20:28:41 +0100 Subject: [PATCH 09/16] return mutable self --- repak/src/pak.rs | 8 +++++--- repak_cli/src/main.rs | 10 +++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/repak/src/pak.rs b/repak/src/pak.rs index 6d35780..29e7bfa 100644 --- a/repak/src/pak.rs +++ b/repak/src/pak.rs @@ -16,12 +16,14 @@ impl PakBuilder { } } #[cfg(feature = "encryption")] - pub fn key(&mut self, key: aes::Aes256) { - self.key = super::Key::Some(key) + pub fn key(&mut self, key: aes::Aes256) -> &mut Self { + self.key = super::Key::Some(key); + self } #[cfg(feature = "oodle")] - pub fn oodle(&mut self, oodle: fn() -> super::Decompress) { + pub fn oodle(&mut self, oodle: fn() -> super::Decompress) -> &mut Self { unsafe { super::OODLE = Some(once_cell::sync::Lazy::new(oodle)) } + self } pub fn reader(self, reader: &mut R) -> Result { PakReader::new_any_inner(reader, self.key) diff --git a/repak_cli/src/main.rs b/repak_cli/src/main.rs index 6d876dc..c7e5dfc 100644 --- a/repak_cli/src/main.rs +++ b/repak_cli/src/main.rs @@ -177,7 +177,7 @@ fn main() -> Result<(), repak::Error> { fn info(aes_key: Option, action: ActionInfo) -> Result<(), repak::Error> { let mut builder = repak::PakBuilder::new(); if let Some(aes_key) = aes_key { - builder.key(aes_key) + builder.key(aes_key); } let pak = builder.reader(&mut BufReader::new(File::open(action.input)?))?; println!("mount point: {}", pak.mount_point()); @@ -192,7 +192,7 @@ fn info(aes_key: Option, action: ActionInfo) -> Result<(), repak::E fn list(aes_key: Option, action: ActionList) -> Result<(), repak::Error> { let mut builder = repak::PakBuilder::new(); if let Some(aes_key) = aes_key { - builder.key(aes_key) + builder.key(aes_key); } let pak = builder.reader(&mut BufReader::new(File::open(action.input)?))?; @@ -225,7 +225,7 @@ fn list(aes_key: Option, action: ActionList) -> Result<(), repak::E fn hash_list(aes_key: Option, action: ActionHashList) -> Result<(), repak::Error> { let mut builder = repak::PakBuilder::new(); if let Some(aes_key) = aes_key { - builder.key(aes_key) + builder.key(aes_key); } let pak = builder.reader(&mut BufReader::new(File::open(&action.input)?))?; @@ -286,7 +286,7 @@ fn unpack(aes_key: Option, action: ActionUnpack) -> Result<(), repa #[cfg(windows)] builder.oodle(get_oodle::decompress); if let Some(aes_key) = aes_key.clone() { - builder.key(aes_key) + builder.key(aes_key); } let pak = builder.reader(&mut BufReader::new(File::open(input)?))?; let output = action @@ -459,7 +459,7 @@ fn get(aes_key: Option, args: ActionGet) -> Result<(), repak::Error #[cfg(windows)] builder.oodle(get_oodle::decompress); if let Some(aes_key) = aes_key { - builder.key(aes_key) + builder.key(aes_key); } let pak = builder.reader(&mut reader)?; let mount_point = PathBuf::from(pak.mount_point()); From 8798e578f17ccc8968b1a8bee2e802ab8f0fb68a Mon Sep 17 00:00:00 2001 From: spuds <71292624+bananaturtlesandwich@users.noreply.github.com> Date: Wed, 20 Sep 2023 20:36:06 +0100 Subject: [PATCH 10/16] consume in builders --- repak/src/pak.rs | 4 ++-- repak_cli/src/main.rs | 18 +++++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/repak/src/pak.rs b/repak/src/pak.rs index 29e7bfa..7d14c81 100644 --- a/repak/src/pak.rs +++ b/repak/src/pak.rs @@ -16,12 +16,12 @@ impl PakBuilder { } } #[cfg(feature = "encryption")] - pub fn key(&mut self, key: aes::Aes256) -> &mut Self { + pub fn key(mut self, key: aes::Aes256) -> Self { self.key = super::Key::Some(key); self } #[cfg(feature = "oodle")] - pub fn oodle(&mut self, oodle: fn() -> super::Decompress) -> &mut Self { + pub fn oodle(self, oodle: fn() -> super::Decompress) -> Self { unsafe { super::OODLE = Some(once_cell::sync::Lazy::new(oodle)) } self } diff --git a/repak_cli/src/main.rs b/repak_cli/src/main.rs index c7e5dfc..6a6b62f 100644 --- a/repak_cli/src/main.rs +++ b/repak_cli/src/main.rs @@ -177,7 +177,7 @@ fn main() -> Result<(), repak::Error> { fn info(aes_key: Option, action: ActionInfo) -> Result<(), repak::Error> { let mut builder = repak::PakBuilder::new(); if let Some(aes_key) = aes_key { - builder.key(aes_key); + builder = builder.key(aes_key); } let pak = builder.reader(&mut BufReader::new(File::open(action.input)?))?; println!("mount point: {}", pak.mount_point()); @@ -192,7 +192,7 @@ fn info(aes_key: Option, action: ActionInfo) -> Result<(), repak::E fn list(aes_key: Option, action: ActionList) -> Result<(), repak::Error> { let mut builder = repak::PakBuilder::new(); if let Some(aes_key) = aes_key { - builder.key(aes_key); + builder = builder.key(aes_key); } let pak = builder.reader(&mut BufReader::new(File::open(action.input)?))?; @@ -225,7 +225,7 @@ fn list(aes_key: Option, action: ActionList) -> Result<(), repak::E fn hash_list(aes_key: Option, action: ActionHashList) -> Result<(), repak::Error> { let mut builder = repak::PakBuilder::new(); if let Some(aes_key) = aes_key { - builder.key(aes_key); + builder = builder.key(aes_key); } let pak = builder.reader(&mut BufReader::new(File::open(&action.input)?))?; @@ -284,9 +284,11 @@ fn unpack(aes_key: Option, action: ActionUnpack) -> Result<(), repa for input in &action.input { let mut builder = repak::PakBuilder::new(); #[cfg(windows)] - builder.oodle(get_oodle::decompress); + { + builder = builder.oodle(get_oodle::decompress); + } if let Some(aes_key) = aes_key.clone() { - builder.key(aes_key); + builder = builder.key(aes_key); } let pak = builder.reader(&mut BufReader::new(File::open(input)?))?; let output = action @@ -457,9 +459,11 @@ fn get(aes_key: Option, args: ActionGet) -> Result<(), repak::Error let mut reader = BufReader::new(File::open(&args.input)?); let mut builder = repak::PakBuilder::new(); #[cfg(windows)] - builder.oodle(get_oodle::decompress); + { + builder = builder.oodle(get_oodle::decompress); + } if let Some(aes_key) = aes_key { - builder.key(aes_key); + builder = builder.key(aes_key); } let pak = builder.reader(&mut reader)?; let mount_point = PathBuf::from(pak.mount_point()); From 384734b9599a3d99ed080b1b2a229a085d529fae Mon Sep 17 00:00:00 2001 From: spuds <71292624+bananaturtlesandwich@users.noreply.github.com> Date: Wed, 20 Sep 2023 21:08:50 +0100 Subject: [PATCH 11/16] get_oodle needs oodle enabled --- get_oodle/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/get_oodle/Cargo.toml b/get_oodle/Cargo.toml index 7b03cee..d5f4aa9 100644 --- a/get_oodle/Cargo.toml +++ b/get_oodle/Cargo.toml @@ -7,7 +7,7 @@ version.workspace = true edition.workspace = true [dependencies] -repak = { path = "../repak" } +repak = { path = "../repak", features = ["oodle"] } sha1 = { workspace = true } ureq = "2.6" once_cell = "1.17" From 9716e378ad676d5e32472ecce34751608f54afd5 Mon Sep 17 00:00:00 2001 From: spuds <71292624+bananaturtlesandwich@users.noreply.github.com> Date: Wed, 20 Sep 2023 21:13:07 +0100 Subject: [PATCH 12/16] update tests --- repak/tests/test.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/repak/tests/test.rs b/repak/tests/test.rs index ccbf90c..eaa4b53 100644 --- a/repak/tests/test.rs +++ b/repak/tests/test.rs @@ -97,7 +97,10 @@ fn test_read(version: repak::Version, _file_name: &str, bytes: &[u8]) { let len = inner_reader.seek(SeekFrom::End(0)).unwrap(); let mut reader = ReadCounter::new_size(inner_reader, len as usize); - let pak = repak::PakReader::new_any_with_key(&mut reader, key).unwrap(); + let pak = repak::PakBuilder::new() + .key(key) + .reader(&mut reader) + .unwrap(); assert_eq!(pak.mount_point(), "../mount/point/root/"); assert_eq!(pak.version(), version); @@ -160,10 +163,13 @@ fn test_write(_version: repak::Version, _file_name: &str, bytes: &[u8]) { .unwrap(); let mut reader = std::io::Cursor::new(bytes); - let pak_reader = repak::PakReader::new_any_with_key(&mut reader, key).unwrap(); + let pak_reader = repak::PakBuilder::new() + .key(key) + .reader(&mut reader) + .unwrap(); let writer = Cursor::new(vec![]); - let mut pak_writer = repak::PakWriter::new( + let mut pak_writer = repak::PakBuilder::new().writer( writer, pak_reader.version(), pak_reader.mount_point().to_owned(), @@ -189,7 +195,7 @@ fn test_rewrite_index(_version: repak::Version, _file_name: &str, bytes: &[u8]) .unwrap(); let mut buf = std::io::Cursor::new(bytes.to_vec()); - let pak_reader = repak::PakReader::new_any_with_key(&mut buf, key).unwrap(); + let pak_reader = repak::PakBuilder::new().key(key).reader(&mut buf).unwrap(); let rewrite = pak_reader .into_pakwriter(buf) From 02b1ae4264fc8d9febb3fddb5e07e1eb516d0640 Mon Sep 17 00:00:00 2001 From: Truman Kilen Date: Tue, 3 Oct 2023 00:50:58 -0500 Subject: [PATCH 13/16] Implement oodle DLL loader for Linux --- get_oodle/Cargo.toml | 15 +- get_oodle/section.bin | Bin 0 -> 300 bytes get_oodle/src/lib.rs | 388 ++++++++++++++++++++++++++++++++++++++++-- repak/src/entry.rs | 20 +-- repak/src/lib.rs | 21 +-- repak/src/pak.rs | 2 +- repak_cli/Cargo.toml | 7 +- repak_cli/src/main.rs | 12 +- 8 files changed, 393 insertions(+), 72 deletions(-) create mode 100644 get_oodle/section.bin diff --git a/get_oodle/Cargo.toml b/get_oodle/Cargo.toml index d5f4aa9..feb6ec8 100644 --- a/get_oodle/Cargo.toml +++ b/get_oodle/Cargo.toml @@ -6,11 +6,18 @@ license.workspace = true version.workspace = true edition.workspace = true +[target.'cfg(windows)'.dependencies] +libloading = "0.7" + +[target.'cfg(unix)'.dependencies] +object = { version = "0.32.1", default-features = false, features = ["std", "read"] } +libc = "0.2.148" +seq-macro = "0.3.5" + [dependencies] -repak = { path = "../repak", features = ["oodle"] } sha1 = { workspace = true } ureq = "2.6" -once_cell = "1.17" hex-literal = "0.4" -libloading = "0.7" -hex = { workspace = true } \ No newline at end of file +hex = { workspace = true } +anyhow = "1.0.75" +lzma-rs = "0.3.0" diff --git a/get_oodle/section.bin b/get_oodle/section.bin new file mode 100644 index 0000000000000000000000000000000000000000..af4e9ebc58ba1f96931336ce6412496f559c2ca5 GIT binary patch literal 300 zcmaKnEsTRu5QNwDNyG^x5{X12P9Tv;LeOV!H$UPJ@+~5@HtMz0t8RMJyFT=(FBLdMh>;*gh8zV-RH)IQML!SoI8XBq_Xo*$ zS#;IQC*dz2BeUEJE3LA|TJzRfZ-Y%Hjr(wp5o0DynK5U repak::Decompress { - *unsafe { OODLE.as_ref().unwrap().get(b"OodleLZ_Decompress") }.unwrap() +use anyhow::{anyhow, Result}; + +use std::sync::OnceLock; + +type OodleDecompress = fn(comp_buf: &[u8], raw_buf: &mut [u8]) -> i32; + +#[allow(non_camel_case_types)] +type OodleLZ_Decompress = unsafe extern "win64" fn( + compBuf: *const u8, + compBufSize: usize, + rawBuf: *mut u8, + rawLen: usize, + fuzzSafe: u32, + checkCRC: u32, + verbosity: u32, + decBufBase: u64, + decBufSize: usize, + fpCallback: u64, + callbackUserData: u64, + decoderMemory: *mut u8, + decoderMemorySize: usize, + threadPhase: u32, +) -> i32; + +pub fn decompress() -> OodleDecompress { + #[cfg(windows)] + return windows_oodle::decompress_wrapper_windows; + #[cfg(unix)] + return linux_oodle::get_oodle_linux(); +} + +fn call_decompress(comp_buf: &[u8], raw_buf: &mut [u8], decompress: OodleLZ_Decompress) -> i32 { + unsafe { + decompress( + comp_buf.as_ptr(), + comp_buf.len(), + raw_buf.as_mut_ptr(), + raw_buf.len(), + 1, + 1, + 0, + 0, + 0, + 0, + 0, + std::ptr::null_mut(), + 0, + 3, + ) + } } -use once_cell::sync::Lazy; -static OODLE: Lazy> = - Lazy::new(|| get_oodle().map_err(|e| e.to_string())); static OODLE_HASH: [u8; 20] = hex_literal::hex!("4bcc73614cb8fd2b0bce8d0f91ee5f3202d9d624"); -fn get_oodle() -> Result { +fn fetch_oodle() -> Result { use sha1::{Digest, Sha1}; - let oodle = std::env::current_exe()?.with_file_name("oo2core_9_win64.dll"); - if !oodle.exists() { - let mut data = vec![]; - ureq::get("https://cdn.discordapp.com/attachments/817251677086285848/992648087371792404/oo2core_9_win64.dll") - .call().map_err(|e| repak::Error::Other(e.to_string()))? - .into_reader().read_to_end(&mut data)?; + let oodle_path = std::env::current_exe()?.with_file_name("oo2core_9_win64.dll"); + if !oodle_path.exists() { + let mut compressed = vec![]; + ureq::get("https://origin.warframe.com/Tools/Oodle/x64/final/oo2core_9_win64.dll.F2DB01967705B62AECEF3CD3E5A28E4D.lzma") + .call()? + .into_reader().read_to_end(&mut compressed)?; - std::fs::write(&oodle, data)?; + let mut decompressed = vec![]; + lzma_rs::lzma_decompress(&mut std::io::Cursor::new(compressed), &mut decompressed).unwrap(); + + std::fs::write(&oodle_path, decompressed)?; } let mut hasher = Sha1::new(); - hasher.update(std::fs::read(&oodle)?); + hasher.update(std::fs::read(&oodle_path)?); let hash = hasher.finalize(); (hash[..] == OODLE_HASH).then_some(()).ok_or_else(|| { - repak::Error::Other(format!( + anyhow!( "oodle hash mismatch expected: {} got: {} ", hex::encode(OODLE_HASH), hex::encode(hash) - )) + ) })?; - unsafe { libloading::Library::new(oodle) }.map_err(|_| repak::Error::OodleFailed) + Ok(oodle_path) +} + +#[cfg(windows)] +mod windows_oodle { + use super::*; + + use anyhow::Context; + + static DECOMPRESS: OnceLock<(OodleLZ_Decompress, libloading::Library)> = OnceLock::new(); + + pub fn decompress_wrapper_windows(comp_buf: &[u8], raw_buf: &mut [u8]) -> i32 { + let decompress = DECOMPRESS.get_or_init(|| { + let path = fetch_oodle().context("failed to fetch oodle").unwrap(); + + let lib = unsafe { libloading::Library::new(path) } + .context("failed to load oodle") + .unwrap(); + + (*unsafe { lib.get(b"OodleLZ_Decompress") }.unwrap(), lib) + }); + call_decompress(comp_buf, raw_buf, decompress.0) + } +} + +#[cfg(unix)] +mod linux_oodle { + use super::*; + + use anyhow::Result; + use object::pe::{ + ImageNtHeaders64, IMAGE_REL_BASED_DIR64, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ, + IMAGE_SCN_MEM_WRITE, + }; + use object::read::pe::{ImageOptionalHeader, ImageThunkData, PeFile64}; + + use object::{LittleEndian as LE, Object, ObjectSection}; + use std::collections::HashMap; + use std::ffi::{c_void, CStr}; + + #[repr(C)] + struct ThreadInformationBlock { + exception_list: *const c_void, + stack_base: *const c_void, + stack_limit: *const c_void, + sub_system_tib: *const c_void, + fiber_data: *const c_void, + arbitrary_user_pointer: *const c_void, + teb: *const c_void, + } + + const TIB: ThreadInformationBlock = ThreadInformationBlock { + exception_list: std::ptr::null(), + stack_base: std::ptr::null(), + stack_limit: std::ptr::null(), + sub_system_tib: std::ptr::null(), + fiber_data: std::ptr::null(), + arbitrary_user_pointer: std::ptr::null(), + teb: std::ptr::null(), + }; + + static DECOMPRESS: OnceLock = OnceLock::new(); + + fn decompress_wrapper(comp_buf: &[u8], raw_buf: &mut [u8]) -> i32 { + unsafe { + // Set GS register in calling thread + const ARCH_SET_GS: i32 = 0x1001; + libc::syscall(libc::SYS_arch_prctl, ARCH_SET_GS, &TIB); + + // Call actual decompress function + call_decompress(comp_buf, raw_buf, *DECOMPRESS.get().unwrap()) + } + } + + #[allow(non_snake_case)] + mod imports { + use super::*; + + pub unsafe extern "win64" fn OutputDebugStringA(string: *const std::ffi::c_char) { + print!("[OODLE] {}", CStr::from_ptr(string).to_string_lossy()); + } + pub unsafe extern "win64" fn GetProcessHeap() -> *const c_void { + 0x12345678 as *const c_void + } + pub unsafe extern "win64" fn HeapAlloc( + _heap: *const c_void, + flags: i32, + size: usize, + ) -> *const c_void { + assert_eq!(0, flags); + libc::malloc(size) + } + pub unsafe extern "win64" fn HeapFree( + _heap: *const c_void, + _flags: i32, + ptr: *mut c_void, + ) -> bool { + libc::free(ptr); + true + } + pub unsafe extern "win64" fn memset( + ptr: *mut c_void, + value: i32, + num: usize, + ) -> *const c_void { + libc::memset(ptr, value, num) + } + pub unsafe extern "win64" fn memmove( + destination: *mut c_void, + source: *const c_void, + num: usize, + ) -> *const c_void { + libc::memmove(destination, source, num) + } + pub unsafe extern "win64" fn memcpy( + destination: *mut c_void, + source: *const c_void, + num: usize, + ) -> *const c_void { + libc::memcpy(destination, source, num) + } + } + + // Create some unique function pointers to use for unimplemented imports + const DEBUG_FNS: [*const fn(); 100] = gen_debug_fns(); + static mut DEBUG_NAMES: [&str; 100] = [""; 100]; + const fn gen_debug_fns() -> [*const fn(); 100] { + fn log() { + unimplemented!("import {:?}", unsafe { DEBUG_NAMES[I] }); + } + let mut array = [std::ptr::null(); 100]; + seq_macro::seq!(N in 0..100 { + array[N] = log:: as *const fn(); + }); + array + } + + pub fn get_oodle_linux() -> OodleDecompress { + DECOMPRESS.get_or_init(|| get_decompress_inner().unwrap()); + decompress_wrapper + } + + fn get_decompress_inner() -> Result { + fetch_oodle().ok(); + let oodle = std::env::current_exe() + .unwrap() + .with_file_name("oo2core_9_win64.dll"); + let dll = std::fs::read(&oodle)?; + + let obj_file = PeFile64::parse(&*dll)?; + + let size = obj_file.nt_headers().optional_header.size_of_image() as usize; + let header_size = obj_file.nt_headers().optional_header.size_of_headers() as usize; + + let image_base = obj_file.relative_address_base() as usize; + + // Create map + let mmap = unsafe { + std::slice::from_raw_parts_mut( + libc::mmap( + std::ptr::null_mut(), + size, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, + -1, + 0, + ) as *mut u8, + size, + ) + }; + + let map_base = mmap.as_ptr(); + + // Copy header to map + mmap[0..header_size].copy_from_slice(&dll[0..header_size]); + unsafe { + assert_eq!( + 0, + libc::mprotect( + mmap.as_mut_ptr() as *mut c_void, + header_size, + libc::PROT_READ + ) + ); + } + + // Copy section data to map + for section in obj_file.sections() { + let address = section.address() as usize; + let data = section.data()?; + mmap[(address - image_base)..(address - image_base + data.len())] + .copy_from_slice(section.data()?); + } + + // Apply relocations + let sections = obj_file.section_table(); + let mut blocks = obj_file + .data_directories() + .relocation_blocks(&*dll, §ions)? + .unwrap(); + + while let Some(block) = blocks.next()? { + let block_address = block.virtual_address(); + let block_data = sections.pe_data_at(&*dll, block_address).map(object::Bytes); + for reloc in block { + let offset = (reloc.virtual_address - block_address) as usize; + match reloc.typ { + IMAGE_REL_BASED_DIR64 => { + let addend = block_data + .and_then(|data| data.read_at::>(offset).ok()) + .map(|addend| addend.get(LE)); + if let Some(addend) = addend { + mmap[reloc.virtual_address as usize + ..reloc.virtual_address as usize + 8] + .copy_from_slice(&u64::to_le_bytes( + addend - image_base as u64 + map_base as u64, + )); + } + } + _ => unimplemented!(), + } + } + } + + // Fix up imports + let import_table = obj_file.import_table()?.unwrap(); + let mut import_descs = import_table.descriptors()?; + + let mut i = 0; + while let Some(import_desc) = import_descs.next()? { + let mut thunks = import_table.thunks(import_desc.original_first_thunk.get(LE))?; + + let mut address = import_desc.first_thunk.get(LE) as usize; + while let Some(thunk) = thunks.next::()? { + let (_hint, name) = import_table.hint_name(thunk.address())?; + let name = String::from_utf8_lossy(name).to_string(); + + use imports::*; + + let fn_addr = match name.as_str() { + "OutputDebugStringA" => OutputDebugStringA as usize, + "GetProcessHeap" => GetProcessHeap as usize, + "HeapAlloc" => HeapAlloc as usize, + "HeapFree" => HeapFree as usize, + "memset" => memset as usize, + "memcpy" => memcpy as usize, + "memmove" => memmove as usize, + _ => { + unsafe { DEBUG_NAMES[i] = name.leak() } + let a = DEBUG_FNS[i] as usize; + i += 1; + a + } + }; + + mmap[address..address + 8].copy_from_slice(&usize::to_le_bytes(fn_addr)); + + address += 8; + } + } + + // Build export table + let mut exports = HashMap::new(); + for export in obj_file.exports()? { + let name = String::from_utf8_lossy(export.name()); + let address = export.address() - image_base as u64 + map_base as u64; + exports.insert(name, address as *const c_void); + } + + // Fix section permissions + for section in obj_file.sections() { + let address = section.address() as usize; + let data = section.data()?; + let size = data.len(); + + let mut permissions = 0; + + let flags = match section.flags() { + object::SectionFlags::Coff { characteristics } => characteristics, + _ => unreachable!(), + }; + + if 0 != flags & IMAGE_SCN_MEM_READ { + permissions |= libc::PROT_READ; + } + if 0 != flags & IMAGE_SCN_MEM_WRITE { + permissions |= libc::PROT_WRITE; + } + if 0 != flags & IMAGE_SCN_MEM_EXECUTE { + permissions |= libc::PROT_EXEC; + } + + unsafe { + assert_eq!( + 0, + libc::mprotect( + mmap.as_mut_ptr().add(address - image_base) as *mut c_void, + size, + permissions + ) + ); + } + } + + // Break things! + Ok(unsafe { std::mem::transmute(exports["OodleLZ_Decompress"]) }) + } } diff --git a/repak/src/entry.rs b/repak/src/entry.rs index 475f5a3..2d7af3b 100644 --- a/repak/src/entry.rs +++ b/repak/src/entry.rs @@ -393,30 +393,18 @@ impl Entry { }; let buffer = &mut data[range]; let out = oodle( - buffer.as_mut_ptr(), - buffer.len(), - decompressed.as_mut_ptr().offset(decompress_offset), - decomp, - 1, - 1, - 0, //verbose 3 - 0, - 0, - 0, - 0, - std::ptr::null_mut(), - 0, - 3, + buffer, + &mut decompressed[decompress_offset..decompress_offset + decomp], ); if out == 0 { return Err(super::Error::DecompressionFailed(Compression::Oodle)); } compress_offset += self.compression_block_size as usize; - decompress_offset += out as isize; + decompress_offset += out as usize; } debug_assert_eq!( - decompress_offset, self.uncompressed as isize, + decompress_offset, self.uncompressed as usize, "Oodle decompression length mismatch" ); buf.write_all(&decompressed)?; diff --git a/repak/src/lib.rs b/repak/src/lib.rs index 887e3bf..37ec9a5 100644 --- a/repak/src/lib.rs +++ b/repak/src/lib.rs @@ -10,25 +10,10 @@ pub use {error::*, pak::*}; pub const MAGIC: u32 = 0x5A6F12E1; #[cfg(feature = "oodle")] -static mut OODLE: Option> = None; +static mut OODLE: Option> = None; #[cfg(feature = "oodle")] -pub type Decompress = unsafe extern "C" fn( - compBuf: *mut u8, - compBufSize: usize, - rawBuf: *mut u8, - rawLen: usize, - fuzzSafe: u32, - checkCRC: u32, - verbosity: u32, - decBufBase: u64, - decBufSize: usize, - fpCallback: u64, - callbackUserData: u64, - decoderMemory: *mut u8, - decoderMemorySize: usize, - threadPhase: u32, -) -> i32; +type OodleDecompress = fn(comp_buf: &[u8], raw_buf: &mut [u8]) -> i32; #[derive( Clone, @@ -158,7 +143,7 @@ impl From for Key { #[cfg(feature = "oodle")] pub(crate) enum Oodle<'func> { - Some(&'func Decompress), + Some(&'func OodleDecompress), None, } diff --git a/repak/src/pak.rs b/repak/src/pak.rs index 7d14c81..ccbf49b 100644 --- a/repak/src/pak.rs +++ b/repak/src/pak.rs @@ -21,7 +21,7 @@ impl PakBuilder { self } #[cfg(feature = "oodle")] - pub fn oodle(self, oodle: fn() -> super::Decompress) -> Self { + pub fn oodle(self, oodle: fn() -> super::OodleDecompress) -> Self { unsafe { super::OODLE = Some(once_cell::sync::Lazy::new(oodle)) } self } diff --git a/repak_cli/Cargo.toml b/repak_cli/Cargo.toml index f3ab94a..5156f5e 100644 --- a/repak_cli/Cargo.toml +++ b/repak_cli/Cargo.toml @@ -10,14 +10,9 @@ edition.workspace = true name = "repak" path = "src/main.rs" -[target.'cfg(windows)'.dependencies] -repak = { path = "../repak", features = ["oodle"] } - -[target.'cfg(not(windows))'.dependencies] -repak = { path = "../repak" } - [dependencies] get_oodle = { path = "../get_oodle" } +repak = { path = "../repak", features = ["oodle"] } aes = { workspace = true } base64 = { workspace = true } clap = { version = "4.1.4", features = ["derive"] } diff --git a/repak_cli/src/main.rs b/repak_cli/src/main.rs index 6a6b62f..f824a3d 100644 --- a/repak_cli/src/main.rs +++ b/repak_cli/src/main.rs @@ -282,11 +282,7 @@ const STYLE: &str = "[{elapsed_precise}] [{wide_bar}] {pos}/{len} ({eta})"; fn unpack(aes_key: Option, action: ActionUnpack) -> Result<(), repak::Error> { for input in &action.input { - let mut builder = repak::PakBuilder::new(); - #[cfg(windows)] - { - builder = builder.oodle(get_oodle::decompress); - } + let mut builder = repak::PakBuilder::new().oodle(get_oodle::decompress); if let Some(aes_key) = aes_key.clone() { builder = builder.key(aes_key); } @@ -457,11 +453,7 @@ fn pack(args: ActionPack) -> Result<(), repak::Error> { fn get(aes_key: Option, args: ActionGet) -> Result<(), repak::Error> { let mut reader = BufReader::new(File::open(&args.input)?); - let mut builder = repak::PakBuilder::new(); - #[cfg(windows)] - { - builder = builder.oodle(get_oodle::decompress); - } + let mut builder = repak::PakBuilder::new().oodle(get_oodle::decompress); if let Some(aes_key) = aes_key { builder = builder.key(aes_key); } From cb7743e3471bb3c9a4b3b91af51e295195b0cf53 Mon Sep 17 00:00:00 2001 From: Truman Kilen Date: Sat, 16 Dec 2023 16:09:28 -0600 Subject: [PATCH 14/16] Get oodle URL from index --- get_oodle/src/lib.rs | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/get_oodle/src/lib.rs b/get_oodle/src/lib.rs index e38f1fa..f0db0c3 100644 --- a/get_oodle/src/lib.rs +++ b/get_oodle/src/lib.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Context, Result}; use std::sync::OnceLock; @@ -51,16 +51,34 @@ fn call_decompress(comp_buf: &[u8], raw_buf: &mut [u8], decompress: OodleLZ_Deco } static OODLE_HASH: [u8; 20] = hex_literal::hex!("4bcc73614cb8fd2b0bce8d0f91ee5f3202d9d624"); +static OODLE_DLL_NAME: &str = "oo2core_9_win64.dll"; fn fetch_oodle() -> Result { use sha1::{Digest, Sha1}; - let oodle_path = std::env::current_exe()?.with_file_name("oo2core_9_win64.dll"); + let oodle_path = std::env::current_exe()?.with_file_name(OODLE_DLL_NAME); + if !oodle_path.exists() { let mut compressed = vec![]; - ureq::get("https://origin.warframe.com/Tools/Oodle/x64/final/oo2core_9_win64.dll.F2DB01967705B62AECEF3CD3E5A28E4D.lzma") + ureq::get("https://origin.warframe.com/origin/50F7040A/index.txt.lzma") .call()? - .into_reader().read_to_end(&mut compressed)?; + .into_reader() + .read_to_end(&mut compressed)?; + + let mut decompressed = vec![]; + lzma_rs::lzma_decompress(&mut std::io::Cursor::new(compressed), &mut decompressed).unwrap(); + let index = String::from_utf8(decompressed)?; + let line = index + .lines() + .find(|l| l.contains(OODLE_DLL_NAME)) + .with_context(|| format!("{OODLE_DLL_NAME} not found in index"))?; + let path = line.split_once(',').context("failed to parse index")?.0; + + let mut compressed = vec![]; + ureq::get(&format!("https://content.warframe.com{path}")) + .call()? + .into_reader() + .read_to_end(&mut compressed)?; let mut decompressed = vec![]; lzma_rs::lzma_decompress(&mut std::io::Cursor::new(compressed), &mut decompressed).unwrap(); @@ -222,11 +240,11 @@ mod linux_oodle { } fn get_decompress_inner() -> Result { - fetch_oodle().ok(); + fetch_oodle()?; let oodle = std::env::current_exe() .unwrap() - .with_file_name("oo2core_9_win64.dll"); - let dll = std::fs::read(&oodle)?; + .with_file_name(OODLE_DLL_NAME); + let dll = std::fs::read(oodle)?; let obj_file = PeFile64::parse(&*dll)?; From 95b4f35b0d9eec24277b9ce6e0d709892689bbac Mon Sep 17 00:00:00 2001 From: Truman Kilen Date: Sat, 16 Dec 2023 16:23:56 -0600 Subject: [PATCH 15/16] Rename get_oodle -> oodle_loader --- {get_oodle => oodle_loader}/Cargo.toml | 2 +- {get_oodle => oodle_loader}/section.bin | Bin {get_oodle => oodle_loader}/src/lib.rs | 4 ++-- repak_cli/Cargo.toml | 2 +- repak_cli/src/main.rs | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) rename {get_oodle => oodle_loader}/Cargo.toml (95%) rename {get_oodle => oodle_loader}/section.bin (100%) rename {get_oodle => oodle_loader}/src/lib.rs (99%) diff --git a/get_oodle/Cargo.toml b/oodle_loader/Cargo.toml similarity index 95% rename from get_oodle/Cargo.toml rename to oodle_loader/Cargo.toml index feb6ec8..188882d 100644 --- a/get_oodle/Cargo.toml +++ b/oodle_loader/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "get_oodle" +name = "oodle_loader" repository.workspace = true authors.workspace = true license.workspace = true diff --git a/get_oodle/section.bin b/oodle_loader/section.bin similarity index 100% rename from get_oodle/section.bin rename to oodle_loader/section.bin diff --git a/get_oodle/src/lib.rs b/oodle_loader/src/lib.rs similarity index 99% rename from get_oodle/src/lib.rs rename to oodle_loader/src/lib.rs index f0db0c3..58f2165 100644 --- a/get_oodle/src/lib.rs +++ b/oodle_loader/src/lib.rs @@ -26,7 +26,7 @@ pub fn decompress() -> OodleDecompress { #[cfg(windows)] return windows_oodle::decompress_wrapper_windows; #[cfg(unix)] - return linux_oodle::get_oodle_linux(); + return linux_oodle::oodle_loader_linux(); } fn call_decompress(comp_buf: &[u8], raw_buf: &mut [u8], decompress: OodleLZ_Decompress) -> i32 { @@ -234,7 +234,7 @@ mod linux_oodle { array } - pub fn get_oodle_linux() -> OodleDecompress { + pub fn oodle_loader_linux() -> OodleDecompress { DECOMPRESS.get_or_init(|| get_decompress_inner().unwrap()); decompress_wrapper } diff --git a/repak_cli/Cargo.toml b/repak_cli/Cargo.toml index 5156f5e..f084cfb 100644 --- a/repak_cli/Cargo.toml +++ b/repak_cli/Cargo.toml @@ -11,7 +11,7 @@ name = "repak" path = "src/main.rs" [dependencies] -get_oodle = { path = "../get_oodle" } +oodle_loader = { path = "../oodle_loader" } repak = { path = "../repak", features = ["oodle"] } aes = { workspace = true } base64 = { workspace = true } diff --git a/repak_cli/src/main.rs b/repak_cli/src/main.rs index f824a3d..43ef79e 100644 --- a/repak_cli/src/main.rs +++ b/repak_cli/src/main.rs @@ -282,7 +282,7 @@ const STYLE: &str = "[{elapsed_precise}] [{wide_bar}] {pos}/{len} ({eta})"; fn unpack(aes_key: Option, action: ActionUnpack) -> Result<(), repak::Error> { for input in &action.input { - let mut builder = repak::PakBuilder::new().oodle(get_oodle::decompress); + let mut builder = repak::PakBuilder::new().oodle(oodle_loader::decompress); if let Some(aes_key) = aes_key.clone() { builder = builder.key(aes_key); } @@ -453,7 +453,7 @@ fn pack(args: ActionPack) -> Result<(), repak::Error> { fn get(aes_key: Option, args: ActionGet) -> Result<(), repak::Error> { let mut reader = BufReader::new(File::open(&args.input)?); - let mut builder = repak::PakBuilder::new().oodle(get_oodle::decompress); + let mut builder = repak::PakBuilder::new().oodle(oodle_loader::decompress); if let Some(aes_key) = aes_key { builder = builder.key(aes_key); } From c84be2776305d82b7a83db54cbbe1762f792ac4d Mon Sep 17 00:00:00 2001 From: Truman Kilen Date: Sat, 16 Dec 2023 18:47:31 -0600 Subject: [PATCH 16/16] Don't store oodle as a global --- oodle_loader/src/lib.rs | 6 +++--- repak/Cargo.toml | 3 ++- repak/src/entry.rs | 12 +++++++----- repak/src/lib.rs | 23 ++++++++++++----------- repak/src/pak.rs | 23 +++++++++++++---------- 5 files changed, 37 insertions(+), 30 deletions(-) diff --git a/oodle_loader/src/lib.rs b/oodle_loader/src/lib.rs index 58f2165..314848e 100644 --- a/oodle_loader/src/lib.rs +++ b/oodle_loader/src/lib.rs @@ -22,11 +22,11 @@ type OodleLZ_Decompress = unsafe extern "win64" fn( threadPhase: u32, ) -> i32; -pub fn decompress() -> OodleDecompress { +pub fn decompress() -> Result> { #[cfg(windows)] - return windows_oodle::decompress_wrapper_windows; + return Ok(windows_oodle::decompress_wrapper_windows); #[cfg(unix)] - return linux_oodle::oodle_loader_linux(); + return Ok(linux_oodle::oodle_loader_linux()); } fn call_decompress(comp_buf: &[u8], raw_buf: &mut [u8], decompress: OodleLZ_Decompress) -> i32 { diff --git a/repak/Cargo.toml b/repak/Cargo.toml index 45d5dd4..38fce10 100644 --- a/repak/Cargo.toml +++ b/repak/Cargo.toml @@ -10,6 +10,7 @@ edition.workspace = true default = ["compression", "encryption"] compression = ["dep:flate2", "dep:zstd"] oodle = [] +oodle_loader = ["dep:oodle_loader"] encryption = ["dep:aes"] [dependencies] @@ -17,10 +18,10 @@ byteorder = "1.4" aes = { workspace = true, optional = true } flate2 = { version = "1.0", optional = true } zstd = { version = "0.12", optional = true } +oodle_loader = { path = "../oodle_loader", optional = true} thiserror = "1.0" sha1 = { workspace = true } strum = { workspace = true } -once_cell = "1.18" [dev-dependencies] base64 = { workspace = true } diff --git a/repak/src/entry.rs b/repak/src/entry.rs index 2d7af3b..04e47f4 100644 --- a/repak/src/entry.rs +++ b/repak/src/entry.rs @@ -306,6 +306,7 @@ impl Entry { version: Version, compression: &[Compression], #[allow(unused)] key: &super::Key, + #[allow(unused)] oodle: &super::Oodle, buf: &mut W, ) -> Result<(), super::Error> { reader.seek(io::SeekFrom::Start(self.offset))?; @@ -375,10 +376,11 @@ impl Entry { } } #[cfg(feature = "oodle")] - Some(Compression::Oodle) => unsafe { - let Some(ref oodle) = super::OODLE else { - return Err(super::Error::OodleFailed); - }; + Some(Compression::Oodle) => { + let oodle = match oodle { + crate::Oodle::Some(getter) => getter().map_err(|_| super::Error::OodleFailed), + crate::Oodle::None => Err(super::Error::OodleFailed), + }?; let mut decompressed = vec![0; self.uncompressed as usize]; let mut compress_offset = 0; @@ -408,7 +410,7 @@ impl Entry { "Oodle decompression length mismatch" ); buf.write_all(&decompressed)?; - }, + } #[cfg(not(feature = "oodle"))] Some(Compression::Oodle) => return Err(super::Error::Oodle), #[cfg(not(feature = "compression"))] diff --git a/repak/src/lib.rs b/repak/src/lib.rs index 37ec9a5..159c06e 100644 --- a/repak/src/lib.rs +++ b/repak/src/lib.rs @@ -10,10 +10,13 @@ pub use {error::*, pak::*}; pub const MAGIC: u32 = 0x5A6F12E1; #[cfg(feature = "oodle")] -static mut OODLE: Option> = None; +mod oodle { + pub type OodleGetter = fn() -> Result>; + pub type OodleDecompress = fn(comp_buf: &[u8], raw_buf: &mut [u8]) -> i32; +} -#[cfg(feature = "oodle")] -type OodleDecompress = fn(comp_buf: &[u8], raw_buf: &mut [u8]) -> i32; +#[cfg(feature = "oodle_loader")] +pub use oodle_loader; #[derive( Clone, @@ -127,10 +130,11 @@ pub enum Compression { } #[allow(clippy::large_enum_variant)] -#[derive(Debug)] +#[derive(Debug, Default)] pub(crate) enum Key { #[cfg(feature = "encryption")] Some(aes::Aes256), + #[default] None, } @@ -141,13 +145,10 @@ impl From for Key { } } -#[cfg(feature = "oodle")] -pub(crate) enum Oodle<'func> { - Some(&'func OodleDecompress), - None, -} - -#[cfg(not(feature = "oodle"))] +#[derive(Debug, Default)] pub(crate) enum Oodle { + #[cfg(feature = "oodle")] + Some(oodle::OodleGetter), + #[default] None, } diff --git a/repak/src/pak.rs b/repak/src/pak.rs index ccbf49b..dac7b85 100644 --- a/repak/src/pak.rs +++ b/repak/src/pak.rs @@ -4,16 +4,15 @@ use byteorder::{ReadBytesExt, WriteBytesExt, LE}; use std::collections::BTreeMap; use std::io::{self, Read, Seek, Write}; -#[derive(Debug)] +#[derive(Debug, Default)] pub struct PakBuilder { key: super::Key, + oodle: super::Oodle, } impl PakBuilder { pub fn new() -> Self { - Self { - key: super::Key::None, - } + Self::default() } #[cfg(feature = "encryption")] pub fn key(mut self, key: aes::Aes256) -> Self { @@ -21,19 +20,19 @@ impl PakBuilder { self } #[cfg(feature = "oodle")] - pub fn oodle(self, oodle: fn() -> super::OodleDecompress) -> Self { - unsafe { super::OODLE = Some(once_cell::sync::Lazy::new(oodle)) } + pub fn oodle(mut self, oodle_getter: super::oodle::OodleGetter) -> Self { + self.oodle = super::Oodle::Some(oodle_getter); self } pub fn reader(self, reader: &mut R) -> Result { - PakReader::new_any_inner(reader, self.key) + PakReader::new_any_inner(reader, self.key, self.oodle) } pub fn reader_with_version( self, reader: &mut R, version: super::Version, ) -> Result { - PakReader::new_inner(reader, version, self.key) + PakReader::new_inner(reader, version, self.key, self.oodle) } pub fn writer( self, @@ -50,6 +49,7 @@ impl PakBuilder { pub struct PakReader { pak: Pak, key: super::Key, + oodle: super::Oodle, } #[derive(Debug)] @@ -128,13 +128,14 @@ impl PakReader { fn new_any_inner( reader: &mut R, key: super::Key, + oodle: super::Oodle, ) -> Result { use std::fmt::Write; let mut log = "\n".to_owned(); for ver in Version::iter() { match Pak::read(&mut *reader, ver, &key) { - Ok(pak) => return Ok(Self { pak, key }), + Ok(pak) => return Ok(Self { pak, key, oodle }), Err(err) => writeln!(log, "trying version {} failed: {}", ver, err)?, } } @@ -145,8 +146,9 @@ impl PakReader { reader: &mut R, version: super::Version, key: super::Key, + oodle: super::Oodle, ) -> Result { - Pak::read(reader, version, &key).map(|pak| Self { pak, key }) + Pak::read(reader, version, &key).map(|pak| Self { pak, key, oodle }) } pub fn version(&self) -> super::Version { @@ -183,6 +185,7 @@ impl PakReader { self.pak.version, &self.pak.compression, &self.key, + &self.oodle, writer, ), None => Err(super::Error::MissingEntry(path.to_owned())),