From 44580b0d71948d571e979194877fcf4f8ee928fc Mon Sep 17 00:00:00 2001 From: Luke Street Date: Tue, 18 Jun 2024 21:04:59 -0600 Subject: [PATCH] Add `dat check` command for verifying collections --- Cargo.lock | 13 +- nod/src/io/wia.rs | 12 +- nodtool/Cargo.toml | 5 + nodtool/assets/gc-non-redump.dat | 645 ++++++++++++++++++ nodtool/assets/gc-npdp.dat | 155 +++++ .../assets/{redump-gc.dat => gc-redump.dat} | 149 ++-- .../assets/{redump-wii.dat => wii-redump.dat} | 0 nodtool/build.rs | 58 +- nodtool/src/cmd/convert.rs | 13 +- nodtool/src/cmd/dat.rs | 231 +++++++ nodtool/src/cmd/extract.rs | 6 +- nodtool/src/cmd/info.rs | 4 +- nodtool/src/cmd/mod.rs | 1 + nodtool/src/cmd/verify.rs | 13 +- nodtool/src/lib.rs | 18 +- nodtool/src/util/redump.rs | 173 ++++- 16 files changed, 1368 insertions(+), 128 deletions(-) create mode 100644 nodtool/assets/gc-non-redump.dat create mode 100644 nodtool/assets/gc-npdp.dat rename nodtool/assets/{redump-gc.dat => gc-redump.dat} (98%) rename nodtool/assets/{redump-wii.dat => wii-redump.dat} (100%) create mode 100644 nodtool/src/cmd/dat.rs diff --git a/Cargo.lock b/Cargo.lock index 7613af6..c8d986e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -457,6 +457,7 @@ dependencies = [ "tracing-subscriber", "xxhash-rust", "zerocopy", + "zstd", ] [[package]] @@ -1006,27 +1007,27 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" +checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.0.0" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" +checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.9+zstd.1.5.5" +version = "2.0.11+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +checksum = "75652c55c0b6f3e6f12eb786fe1bc960396bf05a1eb3bf1f3691c3610ac2e6d4" dependencies = [ "cc", "pkg-config", diff --git a/nod/src/io/wia.rs b/nod/src/io/wia.rs index cdb0f4d..99fd144 100644 --- a/nod/src/io/wia.rs +++ b/nod/src/io/wia.rs @@ -161,7 +161,7 @@ pub struct WIADisc { /// /// RVZ only: /// > This is signed (instead of unsigned) to support negative compression levels in - /// [Zstandard](WIACompression::Zstandard) (RVZ only). + /// > [Zstandard](WIACompression::Zstandard) (RVZ only). pub compression_level: I32, /// The size of the chunks that data is divided into. /// @@ -170,13 +170,13 @@ pub struct WIADisc { /// /// RVZ only: /// > Chunk sizes smaller than 2 MiB are supported. The following applies when using a chunk size - /// smaller than 2 MiB: + /// > smaller than 2 MiB: /// > - The chunk size must be at least 32 KiB and must be a power of two. (Just like with WIA, - /// sizes larger than 2 MiB do not have to be a power of two, they just have to be an integer - /// multiple of 2 MiB.) + /// > sizes larger than 2 MiB do not have to be a power of two, they just have to be an integer + /// > multiple of 2 MiB.) /// > - For Wii partition data, each chunk contains one [WIAExceptionList] which contains - /// exceptions for that chunk (and no other chunks). Offset 0 refers to the first hash of the - /// current chunk, not the first hash of the full 2 MiB of data. + /// > exceptions for that chunk (and no other chunks). Offset 0 refers to the first hash of the + /// > current chunk, not the first hash of the full 2 MiB of data. pub chunk_size: U32, /// The first 0x80 bytes of the disc image. pub disc_head: [u8; DISC_HEAD_SIZE], diff --git a/nodtool/Cargo.toml b/nodtool/Cargo.toml index 5c5b82c..c983367 100644 --- a/nodtool/Cargo.toml +++ b/nodtool/Cargo.toml @@ -25,11 +25,14 @@ base16ct = "0.2.0" crc32fast = "1.4.0" digest = "0.10.7" enable-ansi-support = "0.2.1" +hex = { version = "0.4.3", features = ["serde"] } indicatif = "0.17.8" itertools = "0.12.1" log = "0.4.20" md-5 = "0.10.6" nod = { path = "../nod" } +quick-xml = { version = "0.31.0", features = ["serialize"] } +serde = { version = "1.0.197", features = ["derive"] } sha1 = "0.10.6" size = "0.4.1" supports-color = "3.0.0" @@ -38,9 +41,11 @@ tracing-attributes = "0.1.27" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } xxhash-rust = { version = "0.8.10", features = ["xxh64"] } zerocopy = { version = "0.7.32", features = ["alloc", "derive"] } +zstd = "0.13.1" [build-dependencies] hex = { version = "0.4.3", features = ["serde"] } quick-xml = { version = "0.31.0", features = ["serialize"] } serde = { version = "1.0.197", features = ["derive"] } zerocopy = { version = "0.7.32", features = ["alloc", "derive"] } +zstd = "0.13.1" diff --git a/nodtool/assets/gc-non-redump.dat b/nodtool/assets/gc-non-redump.dat new file mode 100644 index 0000000..8e1c2ae --- /dev/null +++ b/nodtool/assets/gc-non-redump.dat @@ -0,0 +1,645 @@ + + +
+ 160 + Non-Redump - Nintendo - Nintendo GameCube + Non-Redump - Nintendo - Nintendo GameCube + Non-Redump + 20240602-041318 + bikerspade, Gefflon, Hiccup, NovaAurora, rarenight, relax, Seventy7, togemet2 + No-Intro + https://www.no-intro.org + +
+ + Asterix & Obelix XXL (Europe) (Beta) (2004-02-25) + + + + ATV - Quad Power Racing 2 (Europe) (Beta) (2002-10-11) + + + + Baldur's Gate - Dark Alliance (USA) (Beta) (2002-10-14) + + + + BattleBots (USA) (Proto) + + + + Games + Biohazard 4 (Japan) (Beta) (2004-09-08) + + + + Games + Biohazard 4 (Japan) (Beta) (2004-09-10) + + + + Diag Ver 3.1.1 (Japan) + + + + Disney's Hide & Sneak (Japan) (Beta) (2003-12-01) + + + + Dodger (Unknown) (Tech Demo) (Compiled) + + + + Dr. Muto (Europe) (Beta) (2003-02-12) + + + + Freestyle Street Soccer (USA) (Beta) (2003-11-24) + + + + Demos + Gekkan Nintendo Tentou Demo 2002.4.4 (Japan) + + + + Gladius (USA) (Beta) (2003-05-21) + + + + Killer7 (Europe) (Disc 2) (Beta) + + + + Killer7 (Europe) (Disc 1) (Beta) + + + + Killer7 (Europe) (Disc 2) (Beta) (2005-05-11) + + + + Killer7 (Europe) (Disc 1) (Beta) (2005-05-11) + + + + Killer7 (Japan) (Disc 2) (Beta) (2005-06-03) + + + + Killer7 (Japan) (Disc 1) (Beta) (2005-06-03) + + + + Major League Baseball 2K6 (USA) (Beta) (2006-05-18) + + + + Games + Preproduction + Mega Man X - Command Mission (USA) (Beta) + + + + Metal Gear Solid - The Twin Snakes (USA) (Disc 2) (Beta) + + + + Metal Gear Solid - The Twin Snakes (USA) (Disc 1) (Beta) + + + + NDDEMO (Unknown) (v1.0.8) (Tech Demo) (Compiled) + + + + NDDEMO (Unknown) (v1.0.7) (Tech Demo) (Loose) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Need for Speed - Underground (Europe) (Beta) (2003-09-10) + + + + Games + Need for Speed - Underground (Europe) (PZHP) [b] + + + + P.N.03 (USA) (Beta) (2003-05-08) + + + + Applications + Pokemon Distributing Machine (USA) (v2.0.1) [b] + + + + Applications + Pokemon Distributing Machine (USA) (v1.0.1) [b] + + + + Radio Allergy (USA) (Proto) (2007-03-23) + + + + Rayman Arena (USA) (En,Fr,Es) (Beta) (2002-03-12) + + + + Games + Resident Evil 4 (Europe) (En,Fr,De,Es,It) (Disc 1) (Beta) (2005-01-17) + + + + Resident Evil 4 (USA) (Disc 1) (Beta) (2004-11-25) + + + + Games + Resident Evil 4 (USA) (Disc 2) (Beta) (2004-11-25) + + + + Games + Resident Evil 4 (USA) (Disc 1) (Beta) (2004-09-27) + + + + Games + Resident Evil 4 (Europe) (En,Fr,De,Es,It) (Disc 2) (Beta) (2005-01-17) + + + + Games + Resident Evil 4 (USA) (Beta) (E3 Ver.) (2004-04-13) + + + + Smashing Drive (USA) (Beta) (2001-11-05) + + + + Sonic Adventure 2 - Battle (USA) (Beta) + + + + Sonic Adventure 2 - Battle (Japan) (Beta) (2001-11-29) + + + + Sonic Adventure DX (Japan) (Beta) + + + + Sonic Adventure DX - Director's Cut (USA) (Beta) + + + + Sonic Heroes (USA) (Beta) (2003-10-08) + + + + Sonic Heroes (USA) (Beta) (2003-11-18) + + + + Sonic Heroes (USA) (Beta) (E3) + + + + Sonic Mega Collection (USA) (Beta) (2005-08-15) + + + + Super Mario Sunshine (World) (En,Ja,Fr,De,Es,It) (Super Mario 3D All-Stars) + + + + WWE Day of Reckoning 2 (Europe) (Beta) + + + + XGIII - Extreme G Racing (Europe) (Beta) (2002-02-19) [b] + + + + Zapper (USA) (Beta) (2002-09-12) + + + \ No newline at end of file diff --git a/nodtool/assets/gc-npdp.dat b/nodtool/assets/gc-npdp.dat new file mode 100644 index 0000000..ccf2260 --- /dev/null +++ b/nodtool/assets/gc-npdp.dat @@ -0,0 +1,155 @@ + + +
+ 261 + Nintendo - Nintendo GameCube (NPDP Carts) + Nintendo - Nintendo GameCube (NPDP Carts) + 20240104-124921 + Hiccup, NovaAurora, relax, togemet2 + No-Intro + https://www.no-intro.org + +
+ + 2 Games in 1 - Nickelodeon Bob L'eponge - Le Film & Nickelodeon Bob L'eponge - La Bataille de Bikini Bottom (France) (Disc 2) (Proto) + + + + 2 Games in 1 - Nickelodeon Bob L'eponge - Le Film & Nickelodeon Bob L'eponge - La Bataille de Bikini Bottom (France) (Disc 1) (Proto) + + + + 2 Games in 1 - Nickelodeon SpongeBob SquarePants - The Movie & Nickelodeon SpongeBob SquarePants - Battle for Bikini Bottom (Europe) (Disc 2) (Proto) (2005-11-10) + + + + 2 Games in 1 - Nickelodeon SpongeBob SquarePants - The Movie & Nickelodeon SpongeBob SquarePants - Battle for Bikini Bottom (Europe) (Disc 1) (Proto) (2005-11-10) + + + + Games + Preproduction + Batman Begins (Europe) (Beta) (2005-05-01) + + + + Games + Preproduction + Crash Bandicoot - The Wrath of Cortex (USA) (Beta) (2002-07-24) + + + + Games + Preproduction + Crash Bandicoot - The Wrath of Cortex (USA) (Beta) (2001-12-17) + + + + Disney-Pixar Mr. Incredible (Japan) (Beta) (2004-09-20) + + + + Disney-Pixar Mr. Incredible (Japan) (Beta) (2004-09-13) + + + + Disney-Pixar Mr. Incredible - Kyouteki Underminer Toujou (Japan) (Beta) (2005-09-15) + + + + Disney-Pixar Mr. Incredible - Kyouteki Underminer Toujou (Japan) (Beta) (2005-10-21) + + + + Disney-Pixar Ratatouille (USA) (Beta) (Later) + + + + Disney-Pixar Ratatouille (USA) (Beta) (2006-01-18) + + + + Disney-Pixar The Incredibles (USA) (Beta) (2004-09-12) + + + + Disney-Pixar The Incredibles (USA) (Beta) (2004-07-19) + + + + Disney-Pixar The Incredibles - Rise of the Underminer (Europe) (Beta) (2005-09-15) + + + + Gun (USA) (Beta) (2004-04-07) + + + + Games + Preproduction + Ice Age 2 - The Meltdown (Europe) (Beta) + + + + Games + Preproduction + Ice Age 2 - The Meltdown (Europe) (En,Fr,De,Nl) (Beta) (2005-12-12) + + + + kpaddemo (Unknown) + + + + Memory Card Utility Program (Japan) (v1.1.0) + + + + Memory Card Utility Program (USA) (v1.1.0) + + + + Memory Card Utility Program (Unknown) (v1.0.4) (Alt) + + + + Memory Card Utility Program (Unknown) (v1.0.4) + + + + Metroid Prime 3 - Corruption (USA) (Demo) (Kiosk, E3 2006) + + + + Pirates of the Caribbean (USA) (Proto) (2004-04-05) + + + + Shadow the Hedgehog - Trial Version (Japan) (Demo) (2005-10-31) + + + + Sonic Heroes (Japan) (Beta) (2003-10-30) + + + + Sonic Heroes (USA) (Beta) + + + + Sonic Riders (USA) (Beta) + + + + Summoner - The Prophecy (USA) (Beta) (Rev 1121B) + + + + Summoner - The Prophecy (USA) (Beta) (Rev 1121A) + + + + There Is No Program on This Disc (Unknown) + + + \ No newline at end of file diff --git a/nodtool/assets/redump-gc.dat b/nodtool/assets/gc-redump.dat similarity index 98% rename from nodtool/assets/redump-gc.dat rename to nodtool/assets/gc-redump.dat index e60babe..be3258e 100644 --- a/nodtool/assets/redump-gc.dat +++ b/nodtool/assets/gc-redump.dat @@ -3,9 +3,9 @@
Nintendo - GameCube - Nintendo - GameCube - Discs (1989) (2024-02-13 01-56-25) - 2024-02-13 01-56-25 - 2024-02-13 01-56-25 + Nintendo - GameCube - Discs (1992) (2024-06-02 00-38-06) + 2024-06-02 00-38-06 + 2024-06-02 00-38-06 redump.org redump.org http://redump.org/ @@ -5376,7 +5376,7 @@ - Games + Bonus Discs Ribbit King Plus! (USA) (Bonus Disc) @@ -9100,10 +9100,10 @@ Action Replay for GameCube (Europe) (En,Fr,De,Es,It,Pt) (Unl) (v1.14b) - + Coverdiscs - Cube CD 01 (20) (Europe) (Unl) - + Cube CD 01 (20) (UK) (Unl) + Games @@ -9120,50 +9120,50 @@ Action Replay Ultimate Cheats fuer Pokemon Colosseum (Germany) (Unl) - + Coverdiscs - Cube CD 02 (21) (Europe) (Unl) - + Cube CD 02 (21) (UK) (Unl) + - + Coverdiscs - Cube CD 03 (22) (Europe) (Unl) - + Cube CD 03 (22) (UK) (Unl) + - + Coverdiscs - Cube CD 04 (23) (Europe) (Unl) - + Cube CD 04 (23) (UK) (Unl) + - + Coverdiscs - Cube CD 05 (24) (Europe) (Unl) - + Cube CD 05 (24) (UK) (Unl) + - + Coverdiscs - Cube CD 06 (25) (Europe) (Unl) - + Cube CD 06 (25) (UK) (Unl) + - + Coverdiscs - Cube CD 07 (26) (Europe) (Unl) - + Cube CD 07 (26) (UK) (Unl) + - + Coverdiscs - Cube CD 08 (27) (Europe) (Unl) - + Cube CD 08 (27) (UK) (Unl) + - + Coverdiscs - Cube CD 09 (28) (Europe) (Unl) - + Cube CD 09 (28) (UK) (Unl) + - + Coverdiscs - Cube CD 10 (29) (Europe) (Unl) - + Cube CD 10 (29) (UK) (Unl) + Applications @@ -9175,25 +9175,25 @@ Nickelodeon SpongeBob SquarePants - Creatuur van de Krokante Krab (Netherlands) - + Coverdiscs - Cube CD 11 (30) (Europe) (Unl) - + Cube CD 11 (30) (UK) (Unl) + - + Coverdiscs - Cube CD 12 (31) (Europe) (Unl) - + Cube CD 12 (31) (UK) (Unl) + - + Coverdiscs - Cube CD 13 (32) (Europe) (Unl) - + Cube CD 13 (32) (UK) (Unl) + - + Coverdiscs - Cube CD 15 (34) (Europe) (Unl) - + Cube CD 15 (34) (UK) (Unl) + Applications @@ -9225,30 +9225,30 @@ 2002 FIFA World Cup (France) - + Coverdiscs - Cube CD 16 (35) (Europe) (Unl) - + Cube CD 16 (35) (UK) (Unl) + - + Coverdiscs - Cube CD 17 (36) (Europe) (Unl) - + Cube CD 17 (36) (UK) (Unl) + - + Coverdiscs - Cube CD 18 (37) (Europe) (Unl) - + Cube CD 18 (37) (UK) (Unl) + - + Coverdiscs - Cube CD 19 (38) (Europe) (Unl) - + Cube CD 19 (38) (UK) (Unl) + - + Coverdiscs - Cube CD 20 (40) (Europe) (Unl) - + Cube CD 20 (40) (UK) (Unl) + Games @@ -9475,10 +9475,10 @@ Gekkan Nintendo Tentou Demo 2003.10.1 (Japan) (Rev 1) - + Coverdiscs - Cube CD 14 (33) (Europe) (Unl) - + Cube CD 14 (33) (UK) (Unl) + Applications @@ -9570,10 +9570,10 @@ Gekkan Nintendo Tentou Demo 2006.8.1 (Japan) - + Coverdiscs - Cube CD 20 (39) (Europe) (Unl) - + Cube CD 20 (39) (UK) (Unl) + Applications @@ -9955,4 +9955,19 @@ WWE WrestleMania X8 (Japan) (Taikenban) + + Applications + Karat GC-you Pro Action Replay - Best Price! Sokukouryaku GC-you Vol. 1 (Japan) (Unl) + + + + Applications + Karat GC-you Pro Action Replay PAR GC-you (Japan) (Unl) (v1.07) + + + + Preproduction + Evolution Worlds (USA) (Beta) + + diff --git a/nodtool/assets/redump-wii.dat b/nodtool/assets/wii-redump.dat similarity index 100% rename from nodtool/assets/redump-wii.dat rename to nodtool/assets/wii-redump.dat diff --git a/nodtool/build.rs b/nodtool/build.rs index 0ada74d..def2bf7 100644 --- a/nodtool/build.rs +++ b/nodtool/build.rs @@ -40,48 +40,71 @@ fn main() { let out_dir = env::var("OUT_DIR").unwrap(); let dest_path = Path::new(&out_dir).join("parsed-dats.bin"); - let mut f = BufWriter::new(File::create(dest_path).unwrap()); + let out_file = File::create(dest_path).expect("Failed to open out file"); + let mut out = zstd::Encoder::new(BufWriter::new(out_file), zstd::zstd_safe::max_c_level()) + .expect("Failed to create zstd encoder"); // Parse dat files let mut entries = Vec::<(GameEntry, String)>::new(); - for path in ["assets/redump-gc.dat", "assets/redump-wii.dat"] { + for path in [ + "assets/gc-non-redump.dat", + "assets/gc-npdp.dat", + "assets/gc-redump.dat", + "assets/wii-redump.dat", + ] { println!("cargo:rustc-rerun-if-changed={}", path); let file = BufReader::new(File::open(path).expect("Failed to open dat file")); let dat: DatFile = quick_xml::de::from_reader(file).expect("Failed to parse dat file"); - entries.extend(dat.games.into_iter().map(|game| { - ( + entries.extend(dat.games.into_iter().filter_map(|game| { + if game.roms.len() != 1 { + return None; + } + let rom = &game.roms[0]; + Some(( GameEntry { string_table_offset: 0, - crc32: u32::from_be_bytes(game.rom.crc32), - md5: game.rom.md5, - sha1: game.rom.sha1, - sectors: game.rom.size.div_ceil(0x8000) as u32, + crc32: u32::from_be_bytes(rom.crc32), + md5: rom.md5, + sha1: rom.sha1, + sectors: rom.size.div_ceil(0x8000) as u32, }, game.name, - ) + )) })); } // Sort by CRC32 entries.sort_by_key(|(entry, _)| entry.crc32); + // Calculate total size and store in zstd header + let entries_size = entries.len() * size_of::(); + let string_table_size = entries.iter().map(|(_, name)| name.len() + 4).sum::(); + let total_size = size_of::
() + entries_size + string_table_size; + out.set_pledged_src_size(Some(total_size as u64)).unwrap(); + out.include_contentsize(true).unwrap(); + // Write game entries let header = Header { entry_count: entries.len() as u32, entry_size: size_of::() as u32 }; - f.write_all(header.as_bytes()).unwrap(); + out.write_all(header.as_bytes()).unwrap(); let mut string_table_offset = 0u32; for (entry, name) in &mut entries { entry.string_table_offset = string_table_offset; - f.write_all(entry.as_bytes()).unwrap(); + out.write_all(entry.as_bytes()).unwrap(); string_table_offset += name.as_bytes().len() as u32 + 4; } // Write string table for (_, name) in &entries { - f.write_all(&(name.len() as u32).to_le_bytes()).unwrap(); - f.write_all(name.as_bytes()).unwrap(); + out.write_all(&(name.len() as u32).to_le_bytes()).unwrap(); + out.write_all(name.as_bytes()).unwrap(); } - f.flush().unwrap(); + + // Finalize + out.finish() + .expect("Failed to finish zstd encoder") + .flush() + .expect("Failed to flush output file"); } #[derive(Clone, Debug, Deserialize)] @@ -94,11 +117,16 @@ struct DatFile { struct DatGame { #[serde(rename = "@name")] name: String, - rom: DatGameRom, + // #[serde(rename = "category", default)] + // categories: Vec, + #[serde(rename = "rom")] + roms: Vec, } #[derive(Clone, Debug, Deserialize)] struct DatGameRom { + // #[serde(rename = "@name")] + // name: String, #[serde(rename = "@size")] size: u64, #[serde(rename = "@crc", deserialize_with = "deserialize_hex")] diff --git a/nodtool/src/cmd/convert.rs b/nodtool/src/cmd/convert.rs index 74decdd..991a03a 100644 --- a/nodtool/src/cmd/convert.rs +++ b/nodtool/src/cmd/convert.rs @@ -2,12 +2,12 @@ use std::path::PathBuf; use argp::FromArgs; -use crate::util::shared::convert_and_verify; +use crate::util::{redump, shared::convert_and_verify}; #[derive(FromArgs, Debug)] /// Converts a disc image to ISO. #[argp(subcommand, name = "convert")] -pub struct ConvertArgs { +pub struct Args { #[argp(positional)] /// path to disc image file: PathBuf, @@ -17,8 +17,15 @@ pub struct ConvertArgs { #[argp(switch)] /// enable MD5 hashing (slower) md5: bool, + #[argp(option, short = 'd')] + /// path to DAT file(s) for verification (optional) + dat: Vec, } -pub fn convert(args: ConvertArgs) -> nod::Result<()> { +pub fn run(args: Args) -> nod::Result<()> { + if !args.dat.is_empty() { + println!("Loading dat files..."); + redump::load_dats(args.dat.iter().map(PathBuf::as_ref))?; + } convert_and_verify(&args.file, Some(&args.out), args.md5) } diff --git a/nodtool/src/cmd/dat.rs b/nodtool/src/cmd/dat.rs new file mode 100644 index 0000000..e38de37 --- /dev/null +++ b/nodtool/src/cmd/dat.rs @@ -0,0 +1,231 @@ +use std::{ + cmp::min, + collections::BTreeMap, + fmt, + io::Read, + path::{Path, PathBuf}, + sync::{mpsc::sync_channel, Arc}, + thread, +}; + +use argp::FromArgs; +use indicatif::{ProgressBar, ProgressState, ProgressStyle}; +use nod::{Disc, OpenOptions, Result, ResultContext}; +use zerocopy::FromZeroes; + +use crate::util::{ + digest::{digest_thread, DigestResult}, + redump, + redump::GameResult, +}; + +#[derive(FromArgs, Debug)] +/// Commands related to DAT files. +#[argp(subcommand, name = "dat")] +pub struct Args { + #[argp(subcommand)] + command: SubCommand, +} + +#[derive(FromArgs, Debug)] +#[argp(subcommand)] +pub enum SubCommand { + Check(CheckArgs), +} + +#[derive(FromArgs, Debug)] +/// Verify a collection of disc images against DAT files. +#[argp(subcommand, name = "check")] +pub struct CheckArgs { + #[argp(positional)] + /// disc image directory + dir: PathBuf, + #[argp(option, short = 'd')] + /// path to DAT file(s) + dat: Vec, + #[argp(switch)] + /// rename files to match DAT entries + rename: bool, + #[argp(switch)] + /// don't use embedded hashes if available + full_verify: bool, +} + +pub fn run(args: Args) -> Result<()> { + match args.command { + SubCommand::Check(c_args) => check(c_args), + } +} + +fn check(args: CheckArgs) -> Result<()> { + if !args.dat.is_empty() { + println!("Loading dat files..."); + redump::load_dats(args.dat.iter().map(PathBuf::as_ref))?; + } + let mut disc_results = BTreeMap::::new(); + let mut rename_map = BTreeMap::::new(); + for entry in std::fs::read_dir(&args.dir).context("Opening ROM directory")? { + let entry = entry.context("Reading ROM directory entry")?; + let path = entry.path(); + if path.is_file() { + let name = entry.file_name().to_string_lossy().to_string(); + match load_disc(&path, &name, args.full_verify) { + Ok(hashes) => { + let redump_entry = redump::find_by_crc32(hashes.crc32); + if let Some(entry) = &redump_entry { + let mut full_match = true; + if entry.sha1 != hashes.sha1 { + full_match = false; + } + if full_match { + println!("{}: ✅ {}", name, entry.name); + } else { + println!("{}: ❓ {} (partial match)", name, entry.name); + } + if entry.name != path.file_stem().unwrap() { + let file_name = if let Some(ext) = path.extension() { + format!("{}.{}", entry.name, ext.to_string_lossy()) + } else { + entry.name.to_string() + }; + rename_map.insert(path.clone(), path.with_file_name(file_name)); + } + disc_results.insert(hashes.crc32, DiscResult { + name, + // hashes, + redump_entry: Some(entry.clone()), + matched: full_match, + }); + } else { + println!("{}: ❌ Not found", name); + disc_results.insert(hashes.crc32, DiscResult { + name, + // hashes, + redump_entry: None, + matched: false, + }); + } + } + Err(e) => println!("{}: ❌ Error: {}", name, e), + } + } + } + println!(); + let mut matched_count = 0usize; + let mut missing_count = 0usize; + let mut mismatch_count = 0usize; + let mut total_count = 0usize; + let mut extra_count = 0usize; + for entry in redump::EntryIter::new() { + if let Some(result) = disc_results.get(&entry.crc32) { + if result.matched { + matched_count += 1; + } else { + println!("❓ Mismatched: {}", entry.name); + mismatch_count += 1; + } + } else { + println!("❌ Missing: {}", entry.name); + missing_count += 1; + } + total_count += 1; + } + for result in disc_results.values() { + if !result.matched && result.redump_entry.is_none() { + println!("❓ Unmatched: {}", result.name); + extra_count += 1; + } + } + println!( + "Matched: {}, Missing: {}, Mismatched: {}, Total: {}", + matched_count, missing_count, mismatch_count, total_count + ); + println!("Unmatched: {}", extra_count); + + if args.rename && !rename_map.is_empty() { + println!("\nRenaming files..."); + for (old_path, new_path) in rename_map { + println!("{} -> {}", old_path.display(), new_path.display()); + std::fs::rename(&old_path, &new_path).context("Renaming file")?; + } + } + Ok(()) +} + +struct DiscResult { + pub name: String, + // pub hashes: DiscHashes, + pub redump_entry: Option>, + pub matched: bool, +} + +struct DiscHashes { + pub crc32: u32, + pub sha1: [u8; 20], +} + +fn load_disc(path: &Path, name: &str, full_verify: bool) -> Result { + let mut disc = Disc::new_with_options(path, &OpenOptions { + rebuild_encryption: true, + validate_hashes: false, + })?; + let disc_size = disc.disc_size(); + if !full_verify { + let meta = disc.meta(); + if let (Some(crc32), Some(sha1)) = (meta.crc32, meta.sha1) { + return Ok(DiscHashes { crc32, sha1 }); + } + } + + let pb = ProgressBar::new(disc_size).with_message(format!("{}:", name)); + pb.set_style(ProgressStyle::with_template("{msg} {spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({bytes_per_sec}, {eta})") + .unwrap() + .with_key("eta", |state: &ProgressState, w: &mut dyn fmt::Write| { + write!(w, "{:.1}s", state.eta().as_secs_f64()).unwrap() + }) + .progress_chars("#>-")); + + const BUFFER_SIZE: usize = 1015808; // LCM(0x8000, 0x7C00) + let digest_threads = [digest_thread::(), digest_thread::()]; + + let (w_tx, w_rx) = sync_channel::>(1); + let w_thread = thread::spawn(move || { + let mut total_written = 0u64; + while let Ok(data) = w_rx.recv() { + total_written += data.len() as u64; + pb.set_position(total_written); + } + pb.finish_and_clear(); + }); + + let mut total_read = 0u64; + let mut buf = ::new_box_slice_zeroed(BUFFER_SIZE); + while total_read < disc_size { + let read = min(BUFFER_SIZE as u64, disc_size - total_read) as usize; + disc.read_exact(&mut buf[..read]).with_context(|| { + format!("Reading {} bytes at disc offset {}", BUFFER_SIZE, total_read) + })?; + + let arc = Arc::<[u8]>::from(&buf[..read]); + for (tx, _) in &digest_threads { + tx.send(arc.clone()).map_err(|_| "Sending data to hash thread")?; + } + w_tx.send(arc).map_err(|_| "Sending data to write thread")?; + total_read += read as u64; + } + drop(w_tx); // Close channel + w_thread.join().unwrap(); + + let mut crc32 = None; + let mut sha1 = None; + for (tx, handle) in digest_threads { + drop(tx); // Close channel + match handle.join().unwrap() { + DigestResult::Crc32(v) => crc32 = Some(v), + DigestResult::Sha1(v) => sha1 = Some(v), + _ => {} + } + } + + Ok(DiscHashes { crc32: crc32.unwrap(), sha1: sha1.unwrap() }) +} diff --git a/nodtool/src/cmd/extract.rs b/nodtool/src/cmd/extract.rs index 78738c7..99824a9 100644 --- a/nodtool/src/cmd/extract.rs +++ b/nodtool/src/cmd/extract.rs @@ -19,9 +19,9 @@ use zerocopy::AsBytes; use crate::util::{display, has_extension}; #[derive(FromArgs, Debug)] -/// Extract a disc image. +/// Extracts a disc image. #[argp(subcommand, name = "extract")] -pub struct ExtractArgs { +pub struct Args { #[argp(positional)] /// Path to disc image file: PathBuf, @@ -40,7 +40,7 @@ pub struct ExtractArgs { partition: Option, } -pub fn extract(args: ExtractArgs) -> nod::Result<()> { +pub fn run(args: Args) -> nod::Result<()> { let output_dir: PathBuf; if let Some(dir) = args.out { output_dir = dir; diff --git a/nodtool/src/cmd/info.rs b/nodtool/src/cmd/info.rs index 983193e..4015bd0 100644 --- a/nodtool/src/cmd/info.rs +++ b/nodtool/src/cmd/info.rs @@ -9,13 +9,13 @@ use crate::util::{display, shared::print_header}; #[derive(FromArgs, Debug)] /// Displays information about disc images. #[argp(subcommand, name = "info")] -pub struct InfoArgs { +pub struct Args { #[argp(positional)] /// Path to disc image(s) file: Vec, } -pub fn info(args: InfoArgs) -> nod::Result<()> { +pub fn run(args: Args) -> nod::Result<()> { for file in &args.file { info_file(file)?; } diff --git a/nodtool/src/cmd/mod.rs b/nodtool/src/cmd/mod.rs index ae9653f..d8e9a9f 100644 --- a/nodtool/src/cmd/mod.rs +++ b/nodtool/src/cmd/mod.rs @@ -1,4 +1,5 @@ pub mod convert; +pub mod dat; pub mod extract; pub mod info; pub mod verify; diff --git a/nodtool/src/cmd/verify.rs b/nodtool/src/cmd/verify.rs index f00a2f3..c1e35bc 100644 --- a/nodtool/src/cmd/verify.rs +++ b/nodtool/src/cmd/verify.rs @@ -2,21 +2,28 @@ use std::path::PathBuf; use argp::FromArgs; -use crate::util::shared::convert_and_verify; +use crate::util::{redump, shared::convert_and_verify}; #[derive(FromArgs, Debug)] /// Verifies disc images. #[argp(subcommand, name = "verify")] -pub struct VerifyArgs { +pub struct Args { #[argp(positional)] /// path to disc image(s) file: Vec, #[argp(switch)] /// enable MD5 hashing (slower) md5: bool, + #[argp(option, short = 'd')] + /// path to DAT file(s) for verification (optional) + dat: Vec, } -pub fn verify(args: VerifyArgs) -> nod::Result<()> { +pub fn run(args: Args) -> nod::Result<()> { + if !args.dat.is_empty() { + println!("Loading dat files..."); + redump::load_dats(args.dat.iter().map(PathBuf::as_ref))?; + } for file in &args.file { convert_and_verify(file, None, args.md5)?; println!(); diff --git a/nodtool/src/lib.rs b/nodtool/src/lib.rs index 16d3375..c20d907 100644 --- a/nodtool/src/lib.rs +++ b/nodtool/src/lib.rs @@ -6,17 +6,19 @@ pub(crate) mod util; #[derive(FromArgs, Debug)] #[argp(subcommand)] pub enum SubCommand { - Info(cmd::info::InfoArgs), - Extract(cmd::extract::ExtractArgs), - Convert(cmd::convert::ConvertArgs), - Verify(cmd::verify::VerifyArgs), + Dat(cmd::dat::Args), + Info(cmd::info::Args), + Extract(cmd::extract::Args), + Convert(cmd::convert::Args), + Verify(cmd::verify::Args), } pub fn run(command: SubCommand) -> nod::Result<()> { match command { - SubCommand::Info(c_args) => cmd::info::info(c_args), - SubCommand::Convert(c_args) => cmd::convert::convert(c_args), - SubCommand::Extract(c_args) => cmd::extract::extract(c_args), - SubCommand::Verify(c_args) => cmd::verify::verify(c_args), + SubCommand::Dat(c_args) => cmd::dat::run(c_args), + SubCommand::Info(c_args) => cmd::info::run(c_args), + SubCommand::Convert(c_args) => cmd::convert::run(c_args), + SubCommand::Extract(c_args) => cmd::extract::run(c_args), + SubCommand::Verify(c_args) => cmd::verify::run(c_args), } } diff --git a/nodtool/src/util/redump.rs b/nodtool/src/util/redump.rs index c839758..bc07033 100644 --- a/nodtool/src/util/redump.rs +++ b/nodtool/src/util/redump.rs @@ -1,25 +1,70 @@ -use std::{mem::size_of, str}; +use std::{ + fs::File, + io::{BufReader, Cursor, Write}, + mem::size_of, + path::Path, + str, + sync::OnceLock, +}; -use nod::array_ref; -use zerocopy::{FromBytes, FromZeroes}; +use hex::deserialize as deserialize_hex; +use nod::{array_ref, Result}; +use serde::Deserialize; +use zerocopy::{AsBytes, FromBytes, FromZeroes}; #[derive(Clone, Debug)] -pub struct GameResult { - pub name: &'static str, +pub struct GameResult<'a> { + pub name: &'a str, pub crc32: u32, pub md5: [u8; 16], pub sha1: [u8; 20], } -pub fn find_by_crc32(crc32: u32) -> Option { - let header: &Header = Header::ref_from_prefix(&DATA.0).unwrap(); +pub struct EntryIter<'a> { + data: &'a [u8], + index: usize, +} + +impl EntryIter<'static> { + pub fn new() -> EntryIter<'static> { Self { data: loaded_data(), index: 0 } } +} + +impl<'a> Iterator for EntryIter<'a> { + type Item = GameResult<'a>; + + fn next(&mut self) -> Option { + let header: &Header = Header::ref_from_prefix(self.data).unwrap(); + assert_eq!(header.entry_size as usize, size_of::()); + if self.index >= header.entry_count as usize { + return None; + } + + let entries_size = header.entry_count as usize * size_of::(); + let entries: &[GameEntry] = GameEntry::slice_from( + &self.data[size_of::
()..size_of::
() + entries_size], + ) + .unwrap(); + let string_table: &[u8] = &self.data[size_of::
() + entries_size..]; + + let entry = &entries[self.index]; + let offset = entry.string_table_offset as usize; + let name_size = u32::from_ne_bytes(*array_ref![string_table, offset, 4]) as usize; + let name = str::from_utf8(&string_table[offset + 4..offset + 4 + name_size]).unwrap(); + self.index += 1; + Some(GameResult { name, crc32: entry.crc32, md5: entry.md5, sha1: entry.sha1 }) + } +} + +pub fn find_by_crc32(crc32: u32) -> Option> { + let data = loaded_data(); + let header: &Header = Header::ref_from_prefix(data).unwrap(); assert_eq!(header.entry_size as usize, size_of::()); let entries_size = header.entry_count as usize * size_of::(); let entries: &[GameEntry] = - GameEntry::slice_from(&DATA.0[size_of::
()..size_of::
() + entries_size]) + GameEntry::slice_from(&data[size_of::
()..size_of::
() + entries_size]) .unwrap(); - let string_table: &[u8] = &DATA.0[size_of::
() + entries_size..]; + let string_table: &[u8] = &data[size_of::
() + entries_size..]; // Binary search by CRC32 let index = entries.binary_search_by_key(&crc32, |entry| entry.crc32).ok()?; @@ -32,14 +77,82 @@ pub fn find_by_crc32(crc32: u32) -> Option { Some(GameResult { name, crc32: entry.crc32, md5: entry.md5, sha1: entry.sha1 }) } -#[repr(C, align(4))] -struct Aligned(T); +const BUILTIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/parsed-dats.bin")); +static LOADED: OnceLock> = OnceLock::new(); -const DATA: &Aligned<[u8]> = - &Aligned(*include_bytes!(concat!(env!("OUT_DIR"), "/parsed-dats.bin"))); +fn loaded_data() -> &'static [u8] { + LOADED + .get_or_init(|| { + let size = zstd::zstd_safe::get_frame_content_size(BUILTIN).unwrap().unwrap() as usize; + let mut out = ::new_box_slice_zeroed(size); + let out_size = zstd::bulk::Decompressor::new() + .unwrap() + .decompress_to_buffer(BUILTIN, out.as_mut()) + .unwrap(); + debug_assert_eq!(out_size, size); + out + }) + .as_ref() +} + +pub fn load_dats<'a>(paths: impl Iterator) -> Result<()> { + // Parse dat files + let mut entries = Vec::<(GameEntry, String)>::new(); + for path in paths { + let file = BufReader::new(File::open(path).expect("Failed to open dat file")); + let dat: DatFile = quick_xml::de::from_reader(file).expect("Failed to parse dat file"); + entries.extend(dat.games.into_iter().filter_map(|game| { + if game.roms.len() != 1 { + return None; + } + let rom = &game.roms[0]; + Some(( + GameEntry { + string_table_offset: 0, + crc32: u32::from_be_bytes(rom.crc32), + md5: rom.md5, + sha1: rom.sha1, + sectors: rom.size.div_ceil(0x8000) as u32, + }, + game.name, + )) + })); + } + + // Sort by CRC32 + entries.sort_by_key(|(entry, _)| entry.crc32); + + // Calculate total size + let entries_size = entries.len() * size_of::(); + let string_table_size = entries.iter().map(|(_, name)| name.len() + 4).sum::(); + let total_size = size_of::
() + entries_size + string_table_size; + let mut result = ::new_box_slice_zeroed(total_size); + let mut out = Cursor::new(result.as_mut()); + + // Write game entries + let header = + Header { entry_count: entries.len() as u32, entry_size: size_of::() as u32 }; + out.write_all(header.as_bytes()).unwrap(); + let mut string_table_offset = 0u32; + for (entry, name) in &mut entries { + entry.string_table_offset = string_table_offset; + out.write_all(entry.as_bytes()).unwrap(); + string_table_offset += name.as_bytes().len() as u32 + 4; + } + + // Write string table + for (_, name) in &entries { + out.write_all(&(name.len() as u32).to_le_bytes()).unwrap(); + out.write_all(name.as_bytes()).unwrap(); + } + + // Finalize + assert_eq!(out.position() as usize, total_size); + LOADED.set(result).map_err(|_| nod::Error::Other("dats already loaded".to_string())) +} // Keep in sync with build.rs -#[derive(Clone, Debug, FromBytes, FromZeroes)] +#[derive(Clone, Debug, AsBytes, FromBytes, FromZeroes)] #[repr(C, align(4))] struct Header { entry_count: u32, @@ -47,7 +160,7 @@ struct Header { } // Keep in sync with build.rs -#[derive(Clone, Debug, FromBytes, FromZeroes)] +#[derive(Clone, Debug, AsBytes, FromBytes, FromZeroes)] #[repr(C, align(4))] struct GameEntry { crc32: u32, @@ -56,3 +169,33 @@ struct GameEntry { md5: [u8; 16], sha1: [u8; 20], } + +#[derive(Clone, Debug, Deserialize)] +struct DatFile { + #[serde(rename = "game")] + games: Vec, +} + +#[derive(Clone, Debug, Deserialize)] +struct DatGame { + #[serde(rename = "@name")] + name: String, + // #[serde(rename = "category", default)] + // categories: Vec, + #[serde(rename = "rom")] + roms: Vec, +} + +#[derive(Clone, Debug, Deserialize)] +struct DatGameRom { + // #[serde(rename = "@name")] + // name: String, + #[serde(rename = "@size")] + size: u64, + #[serde(rename = "@crc", deserialize_with = "deserialize_hex")] + crc32: [u8; 4], + #[serde(rename = "@md5", deserialize_with = "deserialize_hex")] + md5: [u8; 16], + #[serde(rename = "@sha1", deserialize_with = "deserialize_hex")] + sha1: [u8; 20], +}