Add tests for v5, v7-v9, v11

.gitignore

@@ -1,2 +1,3 @@
 *target/
-*.lock
+*.lock
+/tests/config.sh

Cargo.toml

@@ -19,3 +19,6 @@ flate2 = "1.0"
 hashbrown = "0.13"
 thiserror = "1.0"
 base64 = "0.21.0"
+
+[dev-dependencies]
+paste = "1.0.11"

src/lib.rs

@@ -14,16 +14,18 @@ pub const MAGIC: u32 = 0x5A6F12E1;
     Clone, Copy, PartialEq, Eq, PartialOrd, Debug, strum::Display, strum::FromRepr, strum::EnumIter,
 )]
 pub enum Version {
-    Unknown,               // unknown (mostly just for padding)
+    Unknown,               // v0 unknown (mostly just for padding)
-    Initial,               // initial specification
+    Initial,               // v1 initial specification
-    NoTimestamps,          // timestamps removed
+    NoTimestamps,          // v2 timestamps removed
-    CompressionEncryption, // compression and encryption support
+    CompressionEncryption, // v3 compression and encryption support
-    IndexEncryption,       // index encryption support
+    IndexEncryption,       // v4 index encryption support
-    RelativeChunkOffsets,  // offsets are relative to header
+    RelativeChunkOffsets,  // v5 offsets are relative to header
-    DeleteRecords,         // record deletion support
+    DeleteRecords,         // v6 record deletion support
-    EncryptionKeyGuid,     // include key GUID
+    EncryptionKeyGuid,     // v7 include key GUID
-    FNameBasedCompression, // compression names included
+    FNameBasedCompression, // v8 compression names included
-    FrozenIndex,           // frozen index byte included
+    FrozenIndex,           // v9 frozen index byte included
+    PathHashIndex,         // v10
+    Fnv64BugFix,           // v11
 }
 
 // strum shouldn't need to be installed by users

tests/config.example.sh

@@ -0,0 +1,5 @@
+UNREAL_4_20=ue/4.20/Engine/Binaries/Linux/UnrealPak
+UNREAL_4_21=ue/4.21/Engine/Binaries/Linux/UnrealPak
+UNREAL_4_23=ue/4.23/Engine/Binaries/Linux/UnrealPak
+UNREAL_4_25=ue/4.25/Engine/Binaries/Linux/UnrealPak
+UNREAL_4_27=ue/4.27/Engine/Binaries/Linux/UnrealPak

tests/crypto.json

@@ -0,0 +1,27 @@
+{
+   "$types":{
+      "UnrealBuildTool.EncryptionAndSigning+CryptoSettings, UnrealBuildTool, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null":"1",
+      "UnrealBuildTool.EncryptionAndSigning+EncryptionKey, UnrealBuildTool, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null":"2",
+      "UnrealBuildTool.EncryptionAndSigning+SigningKeyPair, UnrealBuildTool, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null":"3",
+      "UnrealBuildTool.EncryptionAndSigning+SigningKey, UnrealBuildTool, Version=4.0.0.0, Culture=neutral, PublicKeyToken=null":"4"
+   },
+   "$type":"1",
+   "EncryptionKey":{
+      "$type":"2",
+      "Name":"key",
+      "Guid":"00000000000000000000000000000000",
+      "Key":"lNJbw660IOC+kU7cnVQ1oeqrXyhk4J6UAZrCBbcnp94="
+   },
+   "SigningKey": null,
+   "bEnablePakSigning":true,
+   "bEnablePakIndexEncryption":true,
+   "bEnablePakIniEncryption":true,
+   "bEnablePakUAssetEncryption":true,
+   "bEnablePakFullAssetEncryption":false,
+   "bDataCryptoRequired":true,
+   "PakEncryptionRequired":true,
+   "PakSigningRequired":true,
+   "SecondaryEncryptionKeys":[
+
+   ]
+}

tests/generate.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+set -a
+. config.sh
+set +a
+
+unreal-version-for() {
+  case "$1" in
+    5)
+      echo -n "$UNREAL_4_20"
+      ;;
+    7)
+      echo -n "$UNREAL_4_21"
+      ;;
+    8)
+      echo -n "$UNREAL_4_23"
+      ;;
+    9)
+      echo -n "$UNREAL_4_25"
+      ;;
+    11)
+      echo -n "$UNREAL_4_27"
+      ;;
+  esac
+}
+
+generate() {
+  rm -r packs && mkdir packs
+  _version=(5 7 8 9 11)
+  _compress=("" "-compress")
+  _encrypt=("" "-encrypt")
+  _encryptindex=("" "-encryptindex")
+  echo "\"$(realpath "pack/*")\" \"../mount/point/\"" > input.txt
+  for version in "${_version[@]}"; do
+    for compress in "${_compress[@]}"; do
+      for encrypt in "${_encrypt[@]}"; do
+        for encryptindex in "${_encryptindex[@]}"; do
+          name="$version$compress$encrypt$encryptindex"
+          "$(unreal-version-for "$version")" "$(realpath "packs/pack_v${name//-/_}.pak")" -Create="$(realpath input.txt)" -cryptokeys="$(realpath crypto.json)" ${compress:+"$compress"} ${encrypt:+"$encrypt"} ${encryptindex:+"$encryptindex"} &
+        done
+      done
+    done
+  done
+  wait
+  rm input.txt
+}
+
+if [ $# -eq 0 ]; then
+  generate
+else
+  "$@"
+fi

tests/pack/root/directory/nested.txt

@@ -0,0 +1 @@
+Proin urna leo, placerat non tristique sed, commodo sit amet enim. Nam aliquet metus et turpis semper tempus. Aliquam vitae dolor aliquam, elementum augue non, molestie nisi. Maecenas aliquet sagittis elit, id elementum magna dictum sed. Vivamus nulla nulla, aliquet et magna ut, tempus ultrices diam. Donec posuere fringilla feugiat. Etiam imperdiet neque nec mollis ornare. Fusce mollis neque risus, ac molestie ligula sagittis vel. Nam tempus et ante eget egestas. Curabitur porta placerat nisi ut vehicula. Nunc suscipit lacinia leo nec tincidunt. Phasellus blandit arcu non pulvinar mollis.

tests/pack/root/test.txt

@@ -0,0 +1 @@
+Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

tests/test.rs

@@ -0,0 +1,95 @@
+fn load_pak(
+    bytes: &[u8],
+    key: Option<String>,
+) -> Result<unpak::Pak<std::io::Cursor<&[u8]>>, unpak::Error> {
+    use aes::cipher::KeyInit;
+    use base64::{engine::general_purpose, Engine as _};
+    let key = key
+        .map(|k| {
+            general_purpose::STANDARD
+                .decode(k)
+                .as_ref()
+                .map_err(|_| unpak::Error::Base64)
+                .and_then(|bytes| {
+                    aes::Aes256Dec::new_from_slice(bytes).map_err(|_| unpak::Error::Aes)
+                })
+        })
+        .transpose()?;
+
+    for ver in unpak::Version::iter() {
+        match unpak::Pak::new(std::io::Cursor::new(bytes), ver, key.clone()) {
+            Ok(pak) => {
+                return Ok(pak);
+            }
+            _ => {
+                continue;
+            }
+        }
+    }
+    Err(unpak::Error::Other("version unsupported"))
+}
+
+static AES_KEY: &str = "lNJbw660IOC+kU7cnVQ1oeqrXyhk4J6UAZrCBbcnp94=";
+
+use paste::paste;
+
+macro_rules! matrix_test {
+    ( ($($version:literal $exp_version:expr),* $(,)?), $compress:tt, $encrypt:tt, $encryptindex:tt ) => {
+        $( compress!($version, $exp_version, $compress, $encrypt, $encryptindex); )*
+    };
+}
+
+macro_rules! compress {
+    ( $version:literal, $exp_version:expr, ($($compress:literal),* $(,)?), $encrypt:tt, $encryptindex:tt ) => {
+        $( encrypt!($version, $exp_version, $compress, $encrypt, $encryptindex); )*
+    };
+}
+
+macro_rules! encrypt {
+    ( $version:literal, $exp_version:expr, $compress:literal, ($($encrypt:literal),* $(,)?), $encryptindex:tt ) => {
+        $( encryptindex!($version, $exp_version, $compress, $encrypt, $encryptindex); )*
+    };
+}
+
+macro_rules! encryptindex {
+    ( $version:literal, $exp_version:expr, $compress:literal, $encrypt:literal, ($($encryptindex:literal),* $(,)?) ) => {
+        $(
+            paste! {
+                #[test]
+                fn [< test_version_ $version $compress $encrypt $encryptindex >]() {
+                    let mut pak = load_pak(include_bytes!(concat!("packs/pack_", $version, $compress, $encrypt, $encryptindex, ".pak")), Some(AES_KEY.to_string())).unwrap();
+                    assert_eq!(pak.version(), $exp_version);
+                    use std::collections::HashSet;
+                    let files: HashSet<String> = HashSet::from_iter(pak.files());
+                    assert_eq!(files, HashSet::from_iter(vec!["test.txt", "test.png", "zeros.bin", "directory/nested.txt"].into_iter().map(String::from)));
+
+                    for file in files {
+                        let mut buf = vec![];
+                        let mut writer = std::io::Cursor::new(&mut buf);
+                        pak.read(&file, &mut writer).unwrap();
+                        match file.as_str() {
+                            "test.txt" => assert_eq!(buf, include_bytes!("pack/root/test.txt"), "test.txt incorrect contents"),
+                            "test.png" => assert_eq!(buf, include_bytes!("pack/root/test.png"), "test.png incorrect contents"),
+                            "zeros.bin" => assert_eq!(buf, include_bytes!("pack/root/zeros.bin"), "zeros.bin incorrect contents"),
+                            "directory/nested.txt" => assert_eq!(buf, include_bytes!("pack/root/directory/nested.txt"), "nested.txt incorrect contents"),
+                            name => panic!("unrecognized file {}", name)
+                        }
+                    }
+                }
+            }
+        )*
+    };
+}
+
+matrix_test!(
+    (
+        "v5" unpak::Version::RelativeChunkOffsets,
+        "v7" unpak::Version::EncryptionKeyGuid,
+        "v8" unpak::Version::FNameBasedCompression,
+        "v9" unpak::Version::FrozenIndex,
+        "v11" unpak::Version::Fnv64BugFix,
+    ),
+    ("", "_compress"),
+    ("", "_encrypt"),
+    ("", "_encryptindex")
+);