From 02b1ae4264fc8d9febb3fddb5e07e1eb516d0640 Mon Sep 17 00:00:00 2001 From: Truman Kilen Date: Tue, 3 Oct 2023 00:50:58 -0500 Subject: [PATCH] Implement oodle DLL loader for Linux --- get_oodle/Cargo.toml | 15 +- get_oodle/section.bin | Bin 0 -> 300 bytes get_oodle/src/lib.rs | 388 ++++++++++++++++++++++++++++++++++++++++-- repak/src/entry.rs | 20 +-- repak/src/lib.rs | 21 +-- repak/src/pak.rs | 2 +- repak_cli/Cargo.toml | 7 +- repak_cli/src/main.rs | 12 +- 8 files changed, 393 insertions(+), 72 deletions(-) create mode 100644 get_oodle/section.bin diff --git a/get_oodle/Cargo.toml b/get_oodle/Cargo.toml index d5f4aa9..feb6ec8 100644 --- a/get_oodle/Cargo.toml +++ b/get_oodle/Cargo.toml @@ -6,11 +6,18 @@ license.workspace = true version.workspace = true edition.workspace = true +[target.'cfg(windows)'.dependencies] +libloading = "0.7" + +[target.'cfg(unix)'.dependencies] +object = { version = "0.32.1", default-features = false, features = ["std", "read"] } +libc = "0.2.148" +seq-macro = "0.3.5" + [dependencies] -repak = { path = "../repak", features = ["oodle"] } sha1 = { workspace = true } ureq = "2.6" -once_cell = "1.17" hex-literal = "0.4" -libloading = "0.7" -hex = { workspace = true } \ No newline at end of file +hex = { workspace = true } +anyhow = "1.0.75" +lzma-rs = "0.3.0" diff --git a/get_oodle/section.bin b/get_oodle/section.bin new file mode 100644 index 0000000000000000000000000000000000000000..af4e9ebc58ba1f96931336ce6412496f559c2ca5 GIT binary patch literal 300 zcmaKnEsTRu5QNwDNyG^x5{X12P9Tv;LeOV!H$UPJ@+~5@HtMz0t8RMJyFT=(FBLdMh>;*gh8zV-RH)IQML!SoI8XBq_Xo*$ zS#;IQC*dz2BeUEJE3LA|TJzRfZ-Y%Hjr(wp5o0DynK5U repak::Decompress { - *unsafe { OODLE.as_ref().unwrap().get(b"OodleLZ_Decompress") }.unwrap() +use anyhow::{anyhow, Result}; + +use std::sync::OnceLock; + +type OodleDecompress = fn(comp_buf: &[u8], raw_buf: &mut [u8]) -> i32; + +#[allow(non_camel_case_types)] +type OodleLZ_Decompress = unsafe extern "win64" fn( + compBuf: *const u8, + compBufSize: usize, + rawBuf: *mut u8, + rawLen: usize, + fuzzSafe: u32, + checkCRC: u32, + verbosity: u32, + decBufBase: u64, + decBufSize: usize, + fpCallback: u64, + callbackUserData: u64, + decoderMemory: *mut u8, + decoderMemorySize: usize, + threadPhase: u32, +) -> i32; + +pub fn decompress() -> OodleDecompress { + #[cfg(windows)] + return windows_oodle::decompress_wrapper_windows; + #[cfg(unix)] + return linux_oodle::get_oodle_linux(); +} + +fn call_decompress(comp_buf: &[u8], raw_buf: &mut [u8], decompress: OodleLZ_Decompress) -> i32 { + unsafe { + decompress( + comp_buf.as_ptr(), + comp_buf.len(), + raw_buf.as_mut_ptr(), + raw_buf.len(), + 1, + 1, + 0, + 0, + 0, + 0, + 0, + std::ptr::null_mut(), + 0, + 3, + ) + } } -use once_cell::sync::Lazy; -static OODLE: Lazy> = - Lazy::new(|| get_oodle().map_err(|e| e.to_string())); static OODLE_HASH: [u8; 20] = hex_literal::hex!("4bcc73614cb8fd2b0bce8d0f91ee5f3202d9d624"); -fn get_oodle() -> Result { +fn fetch_oodle() -> Result { use sha1::{Digest, Sha1}; - let oodle = std::env::current_exe()?.with_file_name("oo2core_9_win64.dll"); - if !oodle.exists() { - let mut data = vec![]; - ureq::get("https://cdn.discordapp.com/attachments/817251677086285848/992648087371792404/oo2core_9_win64.dll") - .call().map_err(|e| repak::Error::Other(e.to_string()))? - .into_reader().read_to_end(&mut data)?; + let oodle_path = std::env::current_exe()?.with_file_name("oo2core_9_win64.dll"); + if !oodle_path.exists() { + let mut compressed = vec![]; + ureq::get("https://origin.warframe.com/Tools/Oodle/x64/final/oo2core_9_win64.dll.F2DB01967705B62AECEF3CD3E5A28E4D.lzma") + .call()? + .into_reader().read_to_end(&mut compressed)?; - std::fs::write(&oodle, data)?; + let mut decompressed = vec![]; + lzma_rs::lzma_decompress(&mut std::io::Cursor::new(compressed), &mut decompressed).unwrap(); + + std::fs::write(&oodle_path, decompressed)?; } let mut hasher = Sha1::new(); - hasher.update(std::fs::read(&oodle)?); + hasher.update(std::fs::read(&oodle_path)?); let hash = hasher.finalize(); (hash[..] == OODLE_HASH).then_some(()).ok_or_else(|| { - repak::Error::Other(format!( + anyhow!( "oodle hash mismatch expected: {} got: {} ", hex::encode(OODLE_HASH), hex::encode(hash) - )) + ) })?; - unsafe { libloading::Library::new(oodle) }.map_err(|_| repak::Error::OodleFailed) + Ok(oodle_path) +} + +#[cfg(windows)] +mod windows_oodle { + use super::*; + + use anyhow::Context; + + static DECOMPRESS: OnceLock<(OodleLZ_Decompress, libloading::Library)> = OnceLock::new(); + + pub fn decompress_wrapper_windows(comp_buf: &[u8], raw_buf: &mut [u8]) -> i32 { + let decompress = DECOMPRESS.get_or_init(|| { + let path = fetch_oodle().context("failed to fetch oodle").unwrap(); + + let lib = unsafe { libloading::Library::new(path) } + .context("failed to load oodle") + .unwrap(); + + (*unsafe { lib.get(b"OodleLZ_Decompress") }.unwrap(), lib) + }); + call_decompress(comp_buf, raw_buf, decompress.0) + } +} + +#[cfg(unix)] +mod linux_oodle { + use super::*; + + use anyhow::Result; + use object::pe::{ + ImageNtHeaders64, IMAGE_REL_BASED_DIR64, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ, + IMAGE_SCN_MEM_WRITE, + }; + use object::read::pe::{ImageOptionalHeader, ImageThunkData, PeFile64}; + + use object::{LittleEndian as LE, Object, ObjectSection}; + use std::collections::HashMap; + use std::ffi::{c_void, CStr}; + + #[repr(C)] + struct ThreadInformationBlock { + exception_list: *const c_void, + stack_base: *const c_void, + stack_limit: *const c_void, + sub_system_tib: *const c_void, + fiber_data: *const c_void, + arbitrary_user_pointer: *const c_void, + teb: *const c_void, + } + + const TIB: ThreadInformationBlock = ThreadInformationBlock { + exception_list: std::ptr::null(), + stack_base: std::ptr::null(), + stack_limit: std::ptr::null(), + sub_system_tib: std::ptr::null(), + fiber_data: std::ptr::null(), + arbitrary_user_pointer: std::ptr::null(), + teb: std::ptr::null(), + }; + + static DECOMPRESS: OnceLock = OnceLock::new(); + + fn decompress_wrapper(comp_buf: &[u8], raw_buf: &mut [u8]) -> i32 { + unsafe { + // Set GS register in calling thread + const ARCH_SET_GS: i32 = 0x1001; + libc::syscall(libc::SYS_arch_prctl, ARCH_SET_GS, &TIB); + + // Call actual decompress function + call_decompress(comp_buf, raw_buf, *DECOMPRESS.get().unwrap()) + } + } + + #[allow(non_snake_case)] + mod imports { + use super::*; + + pub unsafe extern "win64" fn OutputDebugStringA(string: *const std::ffi::c_char) { + print!("[OODLE] {}", CStr::from_ptr(string).to_string_lossy()); + } + pub unsafe extern "win64" fn GetProcessHeap() -> *const c_void { + 0x12345678 as *const c_void + } + pub unsafe extern "win64" fn HeapAlloc( + _heap: *const c_void, + flags: i32, + size: usize, + ) -> *const c_void { + assert_eq!(0, flags); + libc::malloc(size) + } + pub unsafe extern "win64" fn HeapFree( + _heap: *const c_void, + _flags: i32, + ptr: *mut c_void, + ) -> bool { + libc::free(ptr); + true + } + pub unsafe extern "win64" fn memset( + ptr: *mut c_void, + value: i32, + num: usize, + ) -> *const c_void { + libc::memset(ptr, value, num) + } + pub unsafe extern "win64" fn memmove( + destination: *mut c_void, + source: *const c_void, + num: usize, + ) -> *const c_void { + libc::memmove(destination, source, num) + } + pub unsafe extern "win64" fn memcpy( + destination: *mut c_void, + source: *const c_void, + num: usize, + ) -> *const c_void { + libc::memcpy(destination, source, num) + } + } + + // Create some unique function pointers to use for unimplemented imports + const DEBUG_FNS: [*const fn(); 100] = gen_debug_fns(); + static mut DEBUG_NAMES: [&str; 100] = [""; 100]; + const fn gen_debug_fns() -> [*const fn(); 100] { + fn log() { + unimplemented!("import {:?}", unsafe { DEBUG_NAMES[I] }); + } + let mut array = [std::ptr::null(); 100]; + seq_macro::seq!(N in 0..100 { + array[N] = log:: as *const fn(); + }); + array + } + + pub fn get_oodle_linux() -> OodleDecompress { + DECOMPRESS.get_or_init(|| get_decompress_inner().unwrap()); + decompress_wrapper + } + + fn get_decompress_inner() -> Result { + fetch_oodle().ok(); + let oodle = std::env::current_exe() + .unwrap() + .with_file_name("oo2core_9_win64.dll"); + let dll = std::fs::read(&oodle)?; + + let obj_file = PeFile64::parse(&*dll)?; + + let size = obj_file.nt_headers().optional_header.size_of_image() as usize; + let header_size = obj_file.nt_headers().optional_header.size_of_headers() as usize; + + let image_base = obj_file.relative_address_base() as usize; + + // Create map + let mmap = unsafe { + std::slice::from_raw_parts_mut( + libc::mmap( + std::ptr::null_mut(), + size, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, + -1, + 0, + ) as *mut u8, + size, + ) + }; + + let map_base = mmap.as_ptr(); + + // Copy header to map + mmap[0..header_size].copy_from_slice(&dll[0..header_size]); + unsafe { + assert_eq!( + 0, + libc::mprotect( + mmap.as_mut_ptr() as *mut c_void, + header_size, + libc::PROT_READ + ) + ); + } + + // Copy section data to map + for section in obj_file.sections() { + let address = section.address() as usize; + let data = section.data()?; + mmap[(address - image_base)..(address - image_base + data.len())] + .copy_from_slice(section.data()?); + } + + // Apply relocations + let sections = obj_file.section_table(); + let mut blocks = obj_file + .data_directories() + .relocation_blocks(&*dll, §ions)? + .unwrap(); + + while let Some(block) = blocks.next()? { + let block_address = block.virtual_address(); + let block_data = sections.pe_data_at(&*dll, block_address).map(object::Bytes); + for reloc in block { + let offset = (reloc.virtual_address - block_address) as usize; + match reloc.typ { + IMAGE_REL_BASED_DIR64 => { + let addend = block_data + .and_then(|data| data.read_at::>(offset).ok()) + .map(|addend| addend.get(LE)); + if let Some(addend) = addend { + mmap[reloc.virtual_address as usize + ..reloc.virtual_address as usize + 8] + .copy_from_slice(&u64::to_le_bytes( + addend - image_base as u64 + map_base as u64, + )); + } + } + _ => unimplemented!(), + } + } + } + + // Fix up imports + let import_table = obj_file.import_table()?.unwrap(); + let mut import_descs = import_table.descriptors()?; + + let mut i = 0; + while let Some(import_desc) = import_descs.next()? { + let mut thunks = import_table.thunks(import_desc.original_first_thunk.get(LE))?; + + let mut address = import_desc.first_thunk.get(LE) as usize; + while let Some(thunk) = thunks.next::()? { + let (_hint, name) = import_table.hint_name(thunk.address())?; + let name = String::from_utf8_lossy(name).to_string(); + + use imports::*; + + let fn_addr = match name.as_str() { + "OutputDebugStringA" => OutputDebugStringA as usize, + "GetProcessHeap" => GetProcessHeap as usize, + "HeapAlloc" => HeapAlloc as usize, + "HeapFree" => HeapFree as usize, + "memset" => memset as usize, + "memcpy" => memcpy as usize, + "memmove" => memmove as usize, + _ => { + unsafe { DEBUG_NAMES[i] = name.leak() } + let a = DEBUG_FNS[i] as usize; + i += 1; + a + } + }; + + mmap[address..address + 8].copy_from_slice(&usize::to_le_bytes(fn_addr)); + + address += 8; + } + } + + // Build export table + let mut exports = HashMap::new(); + for export in obj_file.exports()? { + let name = String::from_utf8_lossy(export.name()); + let address = export.address() - image_base as u64 + map_base as u64; + exports.insert(name, address as *const c_void); + } + + // Fix section permissions + for section in obj_file.sections() { + let address = section.address() as usize; + let data = section.data()?; + let size = data.len(); + + let mut permissions = 0; + + let flags = match section.flags() { + object::SectionFlags::Coff { characteristics } => characteristics, + _ => unreachable!(), + }; + + if 0 != flags & IMAGE_SCN_MEM_READ { + permissions |= libc::PROT_READ; + } + if 0 != flags & IMAGE_SCN_MEM_WRITE { + permissions |= libc::PROT_WRITE; + } + if 0 != flags & IMAGE_SCN_MEM_EXECUTE { + permissions |= libc::PROT_EXEC; + } + + unsafe { + assert_eq!( + 0, + libc::mprotect( + mmap.as_mut_ptr().add(address - image_base) as *mut c_void, + size, + permissions + ) + ); + } + } + + // Break things! + Ok(unsafe { std::mem::transmute(exports["OodleLZ_Decompress"]) }) + } } diff --git a/repak/src/entry.rs b/repak/src/entry.rs index 475f5a3..2d7af3b 100644 --- a/repak/src/entry.rs +++ b/repak/src/entry.rs @@ -393,30 +393,18 @@ impl Entry { }; let buffer = &mut data[range]; let out = oodle( - buffer.as_mut_ptr(), - buffer.len(), - decompressed.as_mut_ptr().offset(decompress_offset), - decomp, - 1, - 1, - 0, //verbose 3 - 0, - 0, - 0, - 0, - std::ptr::null_mut(), - 0, - 3, + buffer, + &mut decompressed[decompress_offset..decompress_offset + decomp], ); if out == 0 { return Err(super::Error::DecompressionFailed(Compression::Oodle)); } compress_offset += self.compression_block_size as usize; - decompress_offset += out as isize; + decompress_offset += out as usize; } debug_assert_eq!( - decompress_offset, self.uncompressed as isize, + decompress_offset, self.uncompressed as usize, "Oodle decompression length mismatch" ); buf.write_all(&decompressed)?; diff --git a/repak/src/lib.rs b/repak/src/lib.rs index 887e3bf..37ec9a5 100644 --- a/repak/src/lib.rs +++ b/repak/src/lib.rs @@ -10,25 +10,10 @@ pub use {error::*, pak::*}; pub const MAGIC: u32 = 0x5A6F12E1; #[cfg(feature = "oodle")] -static mut OODLE: Option> = None; +static mut OODLE: Option> = None; #[cfg(feature = "oodle")] -pub type Decompress = unsafe extern "C" fn( - compBuf: *mut u8, - compBufSize: usize, - rawBuf: *mut u8, - rawLen: usize, - fuzzSafe: u32, - checkCRC: u32, - verbosity: u32, - decBufBase: u64, - decBufSize: usize, - fpCallback: u64, - callbackUserData: u64, - decoderMemory: *mut u8, - decoderMemorySize: usize, - threadPhase: u32, -) -> i32; +type OodleDecompress = fn(comp_buf: &[u8], raw_buf: &mut [u8]) -> i32; #[derive( Clone, @@ -158,7 +143,7 @@ impl From for Key { #[cfg(feature = "oodle")] pub(crate) enum Oodle<'func> { - Some(&'func Decompress), + Some(&'func OodleDecompress), None, } diff --git a/repak/src/pak.rs b/repak/src/pak.rs index 7d14c81..ccbf49b 100644 --- a/repak/src/pak.rs +++ b/repak/src/pak.rs @@ -21,7 +21,7 @@ impl PakBuilder { self } #[cfg(feature = "oodle")] - pub fn oodle(self, oodle: fn() -> super::Decompress) -> Self { + pub fn oodle(self, oodle: fn() -> super::OodleDecompress) -> Self { unsafe { super::OODLE = Some(once_cell::sync::Lazy::new(oodle)) } self } diff --git a/repak_cli/Cargo.toml b/repak_cli/Cargo.toml index f3ab94a..5156f5e 100644 --- a/repak_cli/Cargo.toml +++ b/repak_cli/Cargo.toml @@ -10,14 +10,9 @@ edition.workspace = true name = "repak" path = "src/main.rs" -[target.'cfg(windows)'.dependencies] -repak = { path = "../repak", features = ["oodle"] } - -[target.'cfg(not(windows))'.dependencies] -repak = { path = "../repak" } - [dependencies] get_oodle = { path = "../get_oodle" } +repak = { path = "../repak", features = ["oodle"] } aes = { workspace = true } base64 = { workspace = true } clap = { version = "4.1.4", features = ["derive"] } diff --git a/repak_cli/src/main.rs b/repak_cli/src/main.rs index 6a6b62f..f824a3d 100644 --- a/repak_cli/src/main.rs +++ b/repak_cli/src/main.rs @@ -282,11 +282,7 @@ const STYLE: &str = "[{elapsed_precise}] [{wide_bar}] {pos}/{len} ({eta})"; fn unpack(aes_key: Option, action: ActionUnpack) -> Result<(), repak::Error> { for input in &action.input { - let mut builder = repak::PakBuilder::new(); - #[cfg(windows)] - { - builder = builder.oodle(get_oodle::decompress); - } + let mut builder = repak::PakBuilder::new().oodle(get_oodle::decompress); if let Some(aes_key) = aes_key.clone() { builder = builder.key(aes_key); } @@ -457,11 +453,7 @@ fn pack(args: ActionPack) -> Result<(), repak::Error> { fn get(aes_key: Option, args: ActionGet) -> Result<(), repak::Error> { let mut reader = BufReader::new(File::open(&args.input)?); - let mut builder = repak::PakBuilder::new(); - #[cfg(windows)] - { - builder = builder.oodle(get_oodle::decompress); - } + let mut builder = repak::PakBuilder::new().oodle(get_oodle::decompress); if let Some(aes_key) = aes_key { builder = builder.key(aes_key); }