diff --git a/Cargo.lock b/Cargo.lock
index 7613af6..c8d986e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -457,6 +457,7 @@ dependencies = [
"tracing-subscriber",
"xxhash-rust",
"zerocopy",
+ "zstd",
]
[[package]]
@@ -1006,27 +1007,27 @@ dependencies = [
[[package]]
name = "zstd"
-version = "0.13.0"
+version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110"
+checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
-version = "7.0.0"
+version = "7.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e"
+checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
-version = "2.0.9+zstd.1.5.5"
+version = "2.0.11+zstd.1.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656"
+checksum = "75652c55c0b6f3e6f12eb786fe1bc960396bf05a1eb3bf1f3691c3610ac2e6d4"
dependencies = [
"cc",
"pkg-config",
diff --git a/nod/src/io/wia.rs b/nod/src/io/wia.rs
index cdb0f4d..99fd144 100644
--- a/nod/src/io/wia.rs
+++ b/nod/src/io/wia.rs
@@ -161,7 +161,7 @@ pub struct WIADisc {
///
/// RVZ only:
/// > This is signed (instead of unsigned) to support negative compression levels in
- /// [Zstandard](WIACompression::Zstandard) (RVZ only).
+ /// > [Zstandard](WIACompression::Zstandard) (RVZ only).
pub compression_level: I32,
/// The size of the chunks that data is divided into.
///
@@ -170,13 +170,13 @@ pub struct WIADisc {
///
/// RVZ only:
/// > Chunk sizes smaller than 2 MiB are supported. The following applies when using a chunk size
- /// smaller than 2 MiB:
+ /// > smaller than 2 MiB:
/// > - The chunk size must be at least 32 KiB and must be a power of two. (Just like with WIA,
- /// sizes larger than 2 MiB do not have to be a power of two, they just have to be an integer
- /// multiple of 2 MiB.)
+ /// > sizes larger than 2 MiB do not have to be a power of two, they just have to be an integer
+ /// > multiple of 2 MiB.)
/// > - For Wii partition data, each chunk contains one [WIAExceptionList] which contains
- /// exceptions for that chunk (and no other chunks). Offset 0 refers to the first hash of the
- /// current chunk, not the first hash of the full 2 MiB of data.
+ /// > exceptions for that chunk (and no other chunks). Offset 0 refers to the first hash of the
+ /// > current chunk, not the first hash of the full 2 MiB of data.
pub chunk_size: U32,
/// The first 0x80 bytes of the disc image.
pub disc_head: [u8; DISC_HEAD_SIZE],
diff --git a/nodtool/Cargo.toml b/nodtool/Cargo.toml
index 5c5b82c..c983367 100644
--- a/nodtool/Cargo.toml
+++ b/nodtool/Cargo.toml
@@ -25,11 +25,14 @@ base16ct = "0.2.0"
crc32fast = "1.4.0"
digest = "0.10.7"
enable-ansi-support = "0.2.1"
+hex = { version = "0.4.3", features = ["serde"] }
indicatif = "0.17.8"
itertools = "0.12.1"
log = "0.4.20"
md-5 = "0.10.6"
nod = { path = "../nod" }
+quick-xml = { version = "0.31.0", features = ["serialize"] }
+serde = { version = "1.0.197", features = ["derive"] }
sha1 = "0.10.6"
size = "0.4.1"
supports-color = "3.0.0"
@@ -38,9 +41,11 @@ tracing-attributes = "0.1.27"
tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
xxhash-rust = { version = "0.8.10", features = ["xxh64"] }
zerocopy = { version = "0.7.32", features = ["alloc", "derive"] }
+zstd = "0.13.1"
[build-dependencies]
hex = { version = "0.4.3", features = ["serde"] }
quick-xml = { version = "0.31.0", features = ["serialize"] }
serde = { version = "1.0.197", features = ["derive"] }
zerocopy = { version = "0.7.32", features = ["alloc", "derive"] }
+zstd = "0.13.1"
diff --git a/nodtool/assets/gc-non-redump.dat b/nodtool/assets/gc-non-redump.dat
new file mode 100644
index 0000000..8e1c2ae
--- /dev/null
+++ b/nodtool/assets/gc-non-redump.dat
@@ -0,0 +1,645 @@
+
+
+
+ 160
+ Non-Redump - Nintendo - Nintendo GameCube
+ Non-Redump - Nintendo - Nintendo GameCube
+ Non-Redump
+ 20240602-041318
+ bikerspade, Gefflon, Hiccup, NovaAurora, rarenight, relax, Seventy7, togemet2
+ No-Intro
+ https://www.no-intro.org
+
+
+
+ Asterix & Obelix XXL (Europe) (Beta) (2004-02-25)
+
+
+
+ ATV - Quad Power Racing 2 (Europe) (Beta) (2002-10-11)
+
+
+
+ Baldur's Gate - Dark Alliance (USA) (Beta) (2002-10-14)
+
+
+
+ BattleBots (USA) (Proto)
+
+
+
+ Games
+ Biohazard 4 (Japan) (Beta) (2004-09-08)
+
+
+
+ Games
+ Biohazard 4 (Japan) (Beta) (2004-09-10)
+
+
+
+ Diag Ver 3.1.1 (Japan)
+
+
+
+ Disney's Hide & Sneak (Japan) (Beta) (2003-12-01)
+
+
+
+ Dodger (Unknown) (Tech Demo) (Compiled)
+
+
+
+ Dr. Muto (Europe) (Beta) (2003-02-12)
+
+
+
+ Freestyle Street Soccer (USA) (Beta) (2003-11-24)
+
+
+
+ Demos
+ Gekkan Nintendo Tentou Demo 2002.4.4 (Japan)
+
+
+
+ Gladius (USA) (Beta) (2003-05-21)
+
+
+
+ Killer7 (Europe) (Disc 2) (Beta)
+
+
+
+ Killer7 (Europe) (Disc 1) (Beta)
+
+
+
+ Killer7 (Europe) (Disc 2) (Beta) (2005-05-11)
+
+
+
+ Killer7 (Europe) (Disc 1) (Beta) (2005-05-11)
+
+
+
+ Killer7 (Japan) (Disc 2) (Beta) (2005-06-03)
+
+
+
+ Killer7 (Japan) (Disc 1) (Beta) (2005-06-03)
+
+
+
+ Major League Baseball 2K6 (USA) (Beta) (2006-05-18)
+
+
+
+ Games
+ Preproduction
+ Mega Man X - Command Mission (USA) (Beta)
+
+
+
+ Metal Gear Solid - The Twin Snakes (USA) (Disc 2) (Beta)
+
+
+
+ Metal Gear Solid - The Twin Snakes (USA) (Disc 1) (Beta)
+
+
+
+ NDDEMO (Unknown) (v1.0.8) (Tech Demo) (Compiled)
+
+
+
+ NDDEMO (Unknown) (v1.0.7) (Tech Demo) (Loose)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Need for Speed - Underground (Europe) (Beta) (2003-09-10)
+
+
+
+ Games
+ Need for Speed - Underground (Europe) (PZHP) [b]
+
+
+
+ P.N.03 (USA) (Beta) (2003-05-08)
+
+
+
+ Applications
+ Pokemon Distributing Machine (USA) (v2.0.1) [b]
+
+
+
+ Applications
+ Pokemon Distributing Machine (USA) (v1.0.1) [b]
+
+
+
+ Radio Allergy (USA) (Proto) (2007-03-23)
+
+
+
+ Rayman Arena (USA) (En,Fr,Es) (Beta) (2002-03-12)
+
+
+
+ Games
+ Resident Evil 4 (Europe) (En,Fr,De,Es,It) (Disc 1) (Beta) (2005-01-17)
+
+
+
+ Resident Evil 4 (USA) (Disc 1) (Beta) (2004-11-25)
+
+
+
+ Games
+ Resident Evil 4 (USA) (Disc 2) (Beta) (2004-11-25)
+
+
+
+ Games
+ Resident Evil 4 (USA) (Disc 1) (Beta) (2004-09-27)
+
+
+
+ Games
+ Resident Evil 4 (Europe) (En,Fr,De,Es,It) (Disc 2) (Beta) (2005-01-17)
+
+
+
+ Games
+ Resident Evil 4 (USA) (Beta) (E3 Ver.) (2004-04-13)
+
+
+
+ Smashing Drive (USA) (Beta) (2001-11-05)
+
+
+
+ Sonic Adventure 2 - Battle (USA) (Beta)
+
+
+
+ Sonic Adventure 2 - Battle (Japan) (Beta) (2001-11-29)
+
+
+
+ Sonic Adventure DX (Japan) (Beta)
+
+
+
+ Sonic Adventure DX - Director's Cut (USA) (Beta)
+
+
+
+ Sonic Heroes (USA) (Beta) (2003-10-08)
+
+
+
+ Sonic Heroes (USA) (Beta) (2003-11-18)
+
+
+
+ Sonic Heroes (USA) (Beta) (E3)
+
+
+
+ Sonic Mega Collection (USA) (Beta) (2005-08-15)
+
+
+
+ Super Mario Sunshine (World) (En,Ja,Fr,De,Es,It) (Super Mario 3D All-Stars)
+
+
+
+ WWE Day of Reckoning 2 (Europe) (Beta)
+
+
+
+ XGIII - Extreme G Racing (Europe) (Beta) (2002-02-19) [b]
+
+
+
+ Zapper (USA) (Beta) (2002-09-12)
+
+
+
\ No newline at end of file
diff --git a/nodtool/assets/gc-npdp.dat b/nodtool/assets/gc-npdp.dat
new file mode 100644
index 0000000..ccf2260
--- /dev/null
+++ b/nodtool/assets/gc-npdp.dat
@@ -0,0 +1,155 @@
+
+
+
+ 261
+ Nintendo - Nintendo GameCube (NPDP Carts)
+ Nintendo - Nintendo GameCube (NPDP Carts)
+ 20240104-124921
+ Hiccup, NovaAurora, relax, togemet2
+ No-Intro
+ https://www.no-intro.org
+
+
+
+ 2 Games in 1 - Nickelodeon Bob L'eponge - Le Film & Nickelodeon Bob L'eponge - La Bataille de Bikini Bottom (France) (Disc 2) (Proto)
+
+
+
+ 2 Games in 1 - Nickelodeon Bob L'eponge - Le Film & Nickelodeon Bob L'eponge - La Bataille de Bikini Bottom (France) (Disc 1) (Proto)
+
+
+
+ 2 Games in 1 - Nickelodeon SpongeBob SquarePants - The Movie & Nickelodeon SpongeBob SquarePants - Battle for Bikini Bottom (Europe) (Disc 2) (Proto) (2005-11-10)
+
+
+
+ 2 Games in 1 - Nickelodeon SpongeBob SquarePants - The Movie & Nickelodeon SpongeBob SquarePants - Battle for Bikini Bottom (Europe) (Disc 1) (Proto) (2005-11-10)
+
+
+
+ Games
+ Preproduction
+ Batman Begins (Europe) (Beta) (2005-05-01)
+
+
+
+ Games
+ Preproduction
+ Crash Bandicoot - The Wrath of Cortex (USA) (Beta) (2002-07-24)
+
+
+
+ Games
+ Preproduction
+ Crash Bandicoot - The Wrath of Cortex (USA) (Beta) (2001-12-17)
+
+
+
+ Disney-Pixar Mr. Incredible (Japan) (Beta) (2004-09-20)
+
+
+
+ Disney-Pixar Mr. Incredible (Japan) (Beta) (2004-09-13)
+
+
+
+ Disney-Pixar Mr. Incredible - Kyouteki Underminer Toujou (Japan) (Beta) (2005-09-15)
+
+
+
+ Disney-Pixar Mr. Incredible - Kyouteki Underminer Toujou (Japan) (Beta) (2005-10-21)
+
+
+
+ Disney-Pixar Ratatouille (USA) (Beta) (Later)
+
+
+
+ Disney-Pixar Ratatouille (USA) (Beta) (2006-01-18)
+
+
+
+ Disney-Pixar The Incredibles (USA) (Beta) (2004-09-12)
+
+
+
+ Disney-Pixar The Incredibles (USA) (Beta) (2004-07-19)
+
+
+
+ Disney-Pixar The Incredibles - Rise of the Underminer (Europe) (Beta) (2005-09-15)
+
+
+
+ Gun (USA) (Beta) (2004-04-07)
+
+
+
+ Games
+ Preproduction
+ Ice Age 2 - The Meltdown (Europe) (Beta)
+
+
+
+ Games
+ Preproduction
+ Ice Age 2 - The Meltdown (Europe) (En,Fr,De,Nl) (Beta) (2005-12-12)
+
+
+
+ kpaddemo (Unknown)
+
+
+
+ Memory Card Utility Program (Japan) (v1.1.0)
+
+
+
+ Memory Card Utility Program (USA) (v1.1.0)
+
+
+
+ Memory Card Utility Program (Unknown) (v1.0.4) (Alt)
+
+
+
+ Memory Card Utility Program (Unknown) (v1.0.4)
+
+
+
+ Metroid Prime 3 - Corruption (USA) (Demo) (Kiosk, E3 2006)
+
+
+
+ Pirates of the Caribbean (USA) (Proto) (2004-04-05)
+
+
+
+ Shadow the Hedgehog - Trial Version (Japan) (Demo) (2005-10-31)
+
+
+
+ Sonic Heroes (Japan) (Beta) (2003-10-30)
+
+
+
+ Sonic Heroes (USA) (Beta)
+
+
+
+ Sonic Riders (USA) (Beta)
+
+
+
+ Summoner - The Prophecy (USA) (Beta) (Rev 1121B)
+
+
+
+ Summoner - The Prophecy (USA) (Beta) (Rev 1121A)
+
+
+
+ There Is No Program on This Disc (Unknown)
+
+
+
\ No newline at end of file
diff --git a/nodtool/assets/redump-gc.dat b/nodtool/assets/gc-redump.dat
similarity index 98%
rename from nodtool/assets/redump-gc.dat
rename to nodtool/assets/gc-redump.dat
index e60babe..be3258e 100644
--- a/nodtool/assets/redump-gc.dat
+++ b/nodtool/assets/gc-redump.dat
@@ -3,9 +3,9 @@
Nintendo - GameCube
- Nintendo - GameCube - Discs (1989) (2024-02-13 01-56-25)
- 2024-02-13 01-56-25
- 2024-02-13 01-56-25
+ Nintendo - GameCube - Discs (1992) (2024-06-02 00-38-06)
+ 2024-06-02 00-38-06
+ 2024-06-02 00-38-06
redump.org
redump.org
http://redump.org/
@@ -5376,7 +5376,7 @@
- Games
+ Bonus Discs
Ribbit King Plus! (USA) (Bonus Disc)
@@ -9100,10 +9100,10 @@
Action Replay for GameCube (Europe) (En,Fr,De,Es,It,Pt) (Unl) (v1.14b)
-
+
Coverdiscs
- Cube CD 01 (20) (Europe) (Unl)
-
+ Cube CD 01 (20) (UK) (Unl)
+
Games
@@ -9120,50 +9120,50 @@
Action Replay Ultimate Cheats fuer Pokemon Colosseum (Germany) (Unl)
-
+
Coverdiscs
- Cube CD 02 (21) (Europe) (Unl)
-
+ Cube CD 02 (21) (UK) (Unl)
+
-
+
Coverdiscs
- Cube CD 03 (22) (Europe) (Unl)
-
+ Cube CD 03 (22) (UK) (Unl)
+
-
+
Coverdiscs
- Cube CD 04 (23) (Europe) (Unl)
-
+ Cube CD 04 (23) (UK) (Unl)
+
-
+
Coverdiscs
- Cube CD 05 (24) (Europe) (Unl)
-
+ Cube CD 05 (24) (UK) (Unl)
+
-
+
Coverdiscs
- Cube CD 06 (25) (Europe) (Unl)
-
+ Cube CD 06 (25) (UK) (Unl)
+
-
+
Coverdiscs
- Cube CD 07 (26) (Europe) (Unl)
-
+ Cube CD 07 (26) (UK) (Unl)
+
-
+
Coverdiscs
- Cube CD 08 (27) (Europe) (Unl)
-
+ Cube CD 08 (27) (UK) (Unl)
+
-
+
Coverdiscs
- Cube CD 09 (28) (Europe) (Unl)
-
+ Cube CD 09 (28) (UK) (Unl)
+
-
+
Coverdiscs
- Cube CD 10 (29) (Europe) (Unl)
-
+ Cube CD 10 (29) (UK) (Unl)
+
Applications
@@ -9175,25 +9175,25 @@
Nickelodeon SpongeBob SquarePants - Creatuur van de Krokante Krab (Netherlands)
-
+
Coverdiscs
- Cube CD 11 (30) (Europe) (Unl)
-
+ Cube CD 11 (30) (UK) (Unl)
+
-
+
Coverdiscs
- Cube CD 12 (31) (Europe) (Unl)
-
+ Cube CD 12 (31) (UK) (Unl)
+
-
+
Coverdiscs
- Cube CD 13 (32) (Europe) (Unl)
-
+ Cube CD 13 (32) (UK) (Unl)
+
-
+
Coverdiscs
- Cube CD 15 (34) (Europe) (Unl)
-
+ Cube CD 15 (34) (UK) (Unl)
+
Applications
@@ -9225,30 +9225,30 @@
2002 FIFA World Cup (France)
-
+
Coverdiscs
- Cube CD 16 (35) (Europe) (Unl)
-
+ Cube CD 16 (35) (UK) (Unl)
+
-
+
Coverdiscs
- Cube CD 17 (36) (Europe) (Unl)
-
+ Cube CD 17 (36) (UK) (Unl)
+
-
+
Coverdiscs
- Cube CD 18 (37) (Europe) (Unl)
-
+ Cube CD 18 (37) (UK) (Unl)
+
-
+
Coverdiscs
- Cube CD 19 (38) (Europe) (Unl)
-
+ Cube CD 19 (38) (UK) (Unl)
+
-
+
Coverdiscs
- Cube CD 20 (40) (Europe) (Unl)
-
+ Cube CD 20 (40) (UK) (Unl)
+
Games
@@ -9475,10 +9475,10 @@
Gekkan Nintendo Tentou Demo 2003.10.1 (Japan) (Rev 1)
-
+
Coverdiscs
- Cube CD 14 (33) (Europe) (Unl)
-
+ Cube CD 14 (33) (UK) (Unl)
+
Applications
@@ -9570,10 +9570,10 @@
Gekkan Nintendo Tentou Demo 2006.8.1 (Japan)
-
+
Coverdiscs
- Cube CD 20 (39) (Europe) (Unl)
-
+ Cube CD 20 (39) (UK) (Unl)
+
Applications
@@ -9955,4 +9955,19 @@
WWE WrestleMania X8 (Japan) (Taikenban)
+
+ Applications
+ Karat GC-you Pro Action Replay - Best Price! Sokukouryaku GC-you Vol. 1 (Japan) (Unl)
+
+
+
+ Applications
+ Karat GC-you Pro Action Replay PAR GC-you (Japan) (Unl) (v1.07)
+
+
+
+ Preproduction
+ Evolution Worlds (USA) (Beta)
+
+
diff --git a/nodtool/assets/redump-wii.dat b/nodtool/assets/wii-redump.dat
similarity index 100%
rename from nodtool/assets/redump-wii.dat
rename to nodtool/assets/wii-redump.dat
diff --git a/nodtool/build.rs b/nodtool/build.rs
index 0ada74d..def2bf7 100644
--- a/nodtool/build.rs
+++ b/nodtool/build.rs
@@ -40,48 +40,71 @@ fn main() {
let out_dir = env::var("OUT_DIR").unwrap();
let dest_path = Path::new(&out_dir).join("parsed-dats.bin");
- let mut f = BufWriter::new(File::create(dest_path).unwrap());
+ let out_file = File::create(dest_path).expect("Failed to open out file");
+ let mut out = zstd::Encoder::new(BufWriter::new(out_file), zstd::zstd_safe::max_c_level())
+ .expect("Failed to create zstd encoder");
// Parse dat files
let mut entries = Vec::<(GameEntry, String)>::new();
- for path in ["assets/redump-gc.dat", "assets/redump-wii.dat"] {
+ for path in [
+ "assets/gc-non-redump.dat",
+ "assets/gc-npdp.dat",
+ "assets/gc-redump.dat",
+ "assets/wii-redump.dat",
+ ] {
println!("cargo:rustc-rerun-if-changed={}", path);
let file = BufReader::new(File::open(path).expect("Failed to open dat file"));
let dat: DatFile = quick_xml::de::from_reader(file).expect("Failed to parse dat file");
- entries.extend(dat.games.into_iter().map(|game| {
- (
+ entries.extend(dat.games.into_iter().filter_map(|game| {
+ if game.roms.len() != 1 {
+ return None;
+ }
+ let rom = &game.roms[0];
+ Some((
GameEntry {
string_table_offset: 0,
- crc32: u32::from_be_bytes(game.rom.crc32),
- md5: game.rom.md5,
- sha1: game.rom.sha1,
- sectors: game.rom.size.div_ceil(0x8000) as u32,
+ crc32: u32::from_be_bytes(rom.crc32),
+ md5: rom.md5,
+ sha1: rom.sha1,
+ sectors: rom.size.div_ceil(0x8000) as u32,
},
game.name,
- )
+ ))
}));
}
// Sort by CRC32
entries.sort_by_key(|(entry, _)| entry.crc32);
+ // Calculate total size and store in zstd header
+ let entries_size = entries.len() * size_of::();
+ let string_table_size = entries.iter().map(|(_, name)| name.len() + 4).sum::();
+ let total_size = size_of::() + entries_size + string_table_size;
+ out.set_pledged_src_size(Some(total_size as u64)).unwrap();
+ out.include_contentsize(true).unwrap();
+
// Write game entries
let header =
Header { entry_count: entries.len() as u32, entry_size: size_of::() as u32 };
- f.write_all(header.as_bytes()).unwrap();
+ out.write_all(header.as_bytes()).unwrap();
let mut string_table_offset = 0u32;
for (entry, name) in &mut entries {
entry.string_table_offset = string_table_offset;
- f.write_all(entry.as_bytes()).unwrap();
+ out.write_all(entry.as_bytes()).unwrap();
string_table_offset += name.as_bytes().len() as u32 + 4;
}
// Write string table
for (_, name) in &entries {
- f.write_all(&(name.len() as u32).to_le_bytes()).unwrap();
- f.write_all(name.as_bytes()).unwrap();
+ out.write_all(&(name.len() as u32).to_le_bytes()).unwrap();
+ out.write_all(name.as_bytes()).unwrap();
}
- f.flush().unwrap();
+
+ // Finalize
+ out.finish()
+ .expect("Failed to finish zstd encoder")
+ .flush()
+ .expect("Failed to flush output file");
}
#[derive(Clone, Debug, Deserialize)]
@@ -94,11 +117,16 @@ struct DatFile {
struct DatGame {
#[serde(rename = "@name")]
name: String,
- rom: DatGameRom,
+ // #[serde(rename = "category", default)]
+ // categories: Vec,
+ #[serde(rename = "rom")]
+ roms: Vec,
}
#[derive(Clone, Debug, Deserialize)]
struct DatGameRom {
+ // #[serde(rename = "@name")]
+ // name: String,
#[serde(rename = "@size")]
size: u64,
#[serde(rename = "@crc", deserialize_with = "deserialize_hex")]
diff --git a/nodtool/src/cmd/convert.rs b/nodtool/src/cmd/convert.rs
index 74decdd..991a03a 100644
--- a/nodtool/src/cmd/convert.rs
+++ b/nodtool/src/cmd/convert.rs
@@ -2,12 +2,12 @@ use std::path::PathBuf;
use argp::FromArgs;
-use crate::util::shared::convert_and_verify;
+use crate::util::{redump, shared::convert_and_verify};
#[derive(FromArgs, Debug)]
/// Converts a disc image to ISO.
#[argp(subcommand, name = "convert")]
-pub struct ConvertArgs {
+pub struct Args {
#[argp(positional)]
/// path to disc image
file: PathBuf,
@@ -17,8 +17,15 @@ pub struct ConvertArgs {
#[argp(switch)]
/// enable MD5 hashing (slower)
md5: bool,
+ #[argp(option, short = 'd')]
+ /// path to DAT file(s) for verification (optional)
+ dat: Vec,
}
-pub fn convert(args: ConvertArgs) -> nod::Result<()> {
+pub fn run(args: Args) -> nod::Result<()> {
+ if !args.dat.is_empty() {
+ println!("Loading dat files...");
+ redump::load_dats(args.dat.iter().map(PathBuf::as_ref))?;
+ }
convert_and_verify(&args.file, Some(&args.out), args.md5)
}
diff --git a/nodtool/src/cmd/dat.rs b/nodtool/src/cmd/dat.rs
new file mode 100644
index 0000000..e38de37
--- /dev/null
+++ b/nodtool/src/cmd/dat.rs
@@ -0,0 +1,231 @@
+use std::{
+ cmp::min,
+ collections::BTreeMap,
+ fmt,
+ io::Read,
+ path::{Path, PathBuf},
+ sync::{mpsc::sync_channel, Arc},
+ thread,
+};
+
+use argp::FromArgs;
+use indicatif::{ProgressBar, ProgressState, ProgressStyle};
+use nod::{Disc, OpenOptions, Result, ResultContext};
+use zerocopy::FromZeroes;
+
+use crate::util::{
+ digest::{digest_thread, DigestResult},
+ redump,
+ redump::GameResult,
+};
+
+#[derive(FromArgs, Debug)]
+/// Commands related to DAT files.
+#[argp(subcommand, name = "dat")]
+pub struct Args {
+ #[argp(subcommand)]
+ command: SubCommand,
+}
+
+#[derive(FromArgs, Debug)]
+#[argp(subcommand)]
+pub enum SubCommand {
+ Check(CheckArgs),
+}
+
+#[derive(FromArgs, Debug)]
+/// Verify a collection of disc images against DAT files.
+#[argp(subcommand, name = "check")]
+pub struct CheckArgs {
+ #[argp(positional)]
+ /// disc image directory
+ dir: PathBuf,
+ #[argp(option, short = 'd')]
+ /// path to DAT file(s)
+ dat: Vec,
+ #[argp(switch)]
+ /// rename files to match DAT entries
+ rename: bool,
+ #[argp(switch)]
+ /// don't use embedded hashes if available
+ full_verify: bool,
+}
+
+pub fn run(args: Args) -> Result<()> {
+ match args.command {
+ SubCommand::Check(c_args) => check(c_args),
+ }
+}
+
+fn check(args: CheckArgs) -> Result<()> {
+ if !args.dat.is_empty() {
+ println!("Loading dat files...");
+ redump::load_dats(args.dat.iter().map(PathBuf::as_ref))?;
+ }
+ let mut disc_results = BTreeMap::::new();
+ let mut rename_map = BTreeMap::::new();
+ for entry in std::fs::read_dir(&args.dir).context("Opening ROM directory")? {
+ let entry = entry.context("Reading ROM directory entry")?;
+ let path = entry.path();
+ if path.is_file() {
+ let name = entry.file_name().to_string_lossy().to_string();
+ match load_disc(&path, &name, args.full_verify) {
+ Ok(hashes) => {
+ let redump_entry = redump::find_by_crc32(hashes.crc32);
+ if let Some(entry) = &redump_entry {
+ let mut full_match = true;
+ if entry.sha1 != hashes.sha1 {
+ full_match = false;
+ }
+ if full_match {
+ println!("{}: ✅ {}", name, entry.name);
+ } else {
+ println!("{}: ❓ {} (partial match)", name, entry.name);
+ }
+ if entry.name != path.file_stem().unwrap() {
+ let file_name = if let Some(ext) = path.extension() {
+ format!("{}.{}", entry.name, ext.to_string_lossy())
+ } else {
+ entry.name.to_string()
+ };
+ rename_map.insert(path.clone(), path.with_file_name(file_name));
+ }
+ disc_results.insert(hashes.crc32, DiscResult {
+ name,
+ // hashes,
+ redump_entry: Some(entry.clone()),
+ matched: full_match,
+ });
+ } else {
+ println!("{}: ❌ Not found", name);
+ disc_results.insert(hashes.crc32, DiscResult {
+ name,
+ // hashes,
+ redump_entry: None,
+ matched: false,
+ });
+ }
+ }
+ Err(e) => println!("{}: ❌ Error: {}", name, e),
+ }
+ }
+ }
+ println!();
+ let mut matched_count = 0usize;
+ let mut missing_count = 0usize;
+ let mut mismatch_count = 0usize;
+ let mut total_count = 0usize;
+ let mut extra_count = 0usize;
+ for entry in redump::EntryIter::new() {
+ if let Some(result) = disc_results.get(&entry.crc32) {
+ if result.matched {
+ matched_count += 1;
+ } else {
+ println!("❓ Mismatched: {}", entry.name);
+ mismatch_count += 1;
+ }
+ } else {
+ println!("❌ Missing: {}", entry.name);
+ missing_count += 1;
+ }
+ total_count += 1;
+ }
+ for result in disc_results.values() {
+ if !result.matched && result.redump_entry.is_none() {
+ println!("❓ Unmatched: {}", result.name);
+ extra_count += 1;
+ }
+ }
+ println!(
+ "Matched: {}, Missing: {}, Mismatched: {}, Total: {}",
+ matched_count, missing_count, mismatch_count, total_count
+ );
+ println!("Unmatched: {}", extra_count);
+
+ if args.rename && !rename_map.is_empty() {
+ println!("\nRenaming files...");
+ for (old_path, new_path) in rename_map {
+ println!("{} -> {}", old_path.display(), new_path.display());
+ std::fs::rename(&old_path, &new_path).context("Renaming file")?;
+ }
+ }
+ Ok(())
+}
+
+struct DiscResult {
+ pub name: String,
+ // pub hashes: DiscHashes,
+ pub redump_entry: Option>,
+ pub matched: bool,
+}
+
+struct DiscHashes {
+ pub crc32: u32,
+ pub sha1: [u8; 20],
+}
+
+fn load_disc(path: &Path, name: &str, full_verify: bool) -> Result {
+ let mut disc = Disc::new_with_options(path, &OpenOptions {
+ rebuild_encryption: true,
+ validate_hashes: false,
+ })?;
+ let disc_size = disc.disc_size();
+ if !full_verify {
+ let meta = disc.meta();
+ if let (Some(crc32), Some(sha1)) = (meta.crc32, meta.sha1) {
+ return Ok(DiscHashes { crc32, sha1 });
+ }
+ }
+
+ let pb = ProgressBar::new(disc_size).with_message(format!("{}:", name));
+ pb.set_style(ProgressStyle::with_template("{msg} {spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({bytes_per_sec}, {eta})")
+ .unwrap()
+ .with_key("eta", |state: &ProgressState, w: &mut dyn fmt::Write| {
+ write!(w, "{:.1}s", state.eta().as_secs_f64()).unwrap()
+ })
+ .progress_chars("#>-"));
+
+ const BUFFER_SIZE: usize = 1015808; // LCM(0x8000, 0x7C00)
+ let digest_threads = [digest_thread::(), digest_thread::()];
+
+ let (w_tx, w_rx) = sync_channel::>(1);
+ let w_thread = thread::spawn(move || {
+ let mut total_written = 0u64;
+ while let Ok(data) = w_rx.recv() {
+ total_written += data.len() as u64;
+ pb.set_position(total_written);
+ }
+ pb.finish_and_clear();
+ });
+
+ let mut total_read = 0u64;
+ let mut buf = ::new_box_slice_zeroed(BUFFER_SIZE);
+ while total_read < disc_size {
+ let read = min(BUFFER_SIZE as u64, disc_size - total_read) as usize;
+ disc.read_exact(&mut buf[..read]).with_context(|| {
+ format!("Reading {} bytes at disc offset {}", BUFFER_SIZE, total_read)
+ })?;
+
+ let arc = Arc::<[u8]>::from(&buf[..read]);
+ for (tx, _) in &digest_threads {
+ tx.send(arc.clone()).map_err(|_| "Sending data to hash thread")?;
+ }
+ w_tx.send(arc).map_err(|_| "Sending data to write thread")?;
+ total_read += read as u64;
+ }
+ drop(w_tx); // Close channel
+ w_thread.join().unwrap();
+
+ let mut crc32 = None;
+ let mut sha1 = None;
+ for (tx, handle) in digest_threads {
+ drop(tx); // Close channel
+ match handle.join().unwrap() {
+ DigestResult::Crc32(v) => crc32 = Some(v),
+ DigestResult::Sha1(v) => sha1 = Some(v),
+ _ => {}
+ }
+ }
+
+ Ok(DiscHashes { crc32: crc32.unwrap(), sha1: sha1.unwrap() })
+}
diff --git a/nodtool/src/cmd/extract.rs b/nodtool/src/cmd/extract.rs
index 78738c7..99824a9 100644
--- a/nodtool/src/cmd/extract.rs
+++ b/nodtool/src/cmd/extract.rs
@@ -19,9 +19,9 @@ use zerocopy::AsBytes;
use crate::util::{display, has_extension};
#[derive(FromArgs, Debug)]
-/// Extract a disc image.
+/// Extracts a disc image.
#[argp(subcommand, name = "extract")]
-pub struct ExtractArgs {
+pub struct Args {
#[argp(positional)]
/// Path to disc image
file: PathBuf,
@@ -40,7 +40,7 @@ pub struct ExtractArgs {
partition: Option,
}
-pub fn extract(args: ExtractArgs) -> nod::Result<()> {
+pub fn run(args: Args) -> nod::Result<()> {
let output_dir: PathBuf;
if let Some(dir) = args.out {
output_dir = dir;
diff --git a/nodtool/src/cmd/info.rs b/nodtool/src/cmd/info.rs
index 983193e..4015bd0 100644
--- a/nodtool/src/cmd/info.rs
+++ b/nodtool/src/cmd/info.rs
@@ -9,13 +9,13 @@ use crate::util::{display, shared::print_header};
#[derive(FromArgs, Debug)]
/// Displays information about disc images.
#[argp(subcommand, name = "info")]
-pub struct InfoArgs {
+pub struct Args {
#[argp(positional)]
/// Path to disc image(s)
file: Vec,
}
-pub fn info(args: InfoArgs) -> nod::Result<()> {
+pub fn run(args: Args) -> nod::Result<()> {
for file in &args.file {
info_file(file)?;
}
diff --git a/nodtool/src/cmd/mod.rs b/nodtool/src/cmd/mod.rs
index ae9653f..d8e9a9f 100644
--- a/nodtool/src/cmd/mod.rs
+++ b/nodtool/src/cmd/mod.rs
@@ -1,4 +1,5 @@
pub mod convert;
+pub mod dat;
pub mod extract;
pub mod info;
pub mod verify;
diff --git a/nodtool/src/cmd/verify.rs b/nodtool/src/cmd/verify.rs
index f00a2f3..c1e35bc 100644
--- a/nodtool/src/cmd/verify.rs
+++ b/nodtool/src/cmd/verify.rs
@@ -2,21 +2,28 @@ use std::path::PathBuf;
use argp::FromArgs;
-use crate::util::shared::convert_and_verify;
+use crate::util::{redump, shared::convert_and_verify};
#[derive(FromArgs, Debug)]
/// Verifies disc images.
#[argp(subcommand, name = "verify")]
-pub struct VerifyArgs {
+pub struct Args {
#[argp(positional)]
/// path to disc image(s)
file: Vec,
#[argp(switch)]
/// enable MD5 hashing (slower)
md5: bool,
+ #[argp(option, short = 'd')]
+ /// path to DAT file(s) for verification (optional)
+ dat: Vec,
}
-pub fn verify(args: VerifyArgs) -> nod::Result<()> {
+pub fn run(args: Args) -> nod::Result<()> {
+ if !args.dat.is_empty() {
+ println!("Loading dat files...");
+ redump::load_dats(args.dat.iter().map(PathBuf::as_ref))?;
+ }
for file in &args.file {
convert_and_verify(file, None, args.md5)?;
println!();
diff --git a/nodtool/src/lib.rs b/nodtool/src/lib.rs
index 16d3375..c20d907 100644
--- a/nodtool/src/lib.rs
+++ b/nodtool/src/lib.rs
@@ -6,17 +6,19 @@ pub(crate) mod util;
#[derive(FromArgs, Debug)]
#[argp(subcommand)]
pub enum SubCommand {
- Info(cmd::info::InfoArgs),
- Extract(cmd::extract::ExtractArgs),
- Convert(cmd::convert::ConvertArgs),
- Verify(cmd::verify::VerifyArgs),
+ Dat(cmd::dat::Args),
+ Info(cmd::info::Args),
+ Extract(cmd::extract::Args),
+ Convert(cmd::convert::Args),
+ Verify(cmd::verify::Args),
}
pub fn run(command: SubCommand) -> nod::Result<()> {
match command {
- SubCommand::Info(c_args) => cmd::info::info(c_args),
- SubCommand::Convert(c_args) => cmd::convert::convert(c_args),
- SubCommand::Extract(c_args) => cmd::extract::extract(c_args),
- SubCommand::Verify(c_args) => cmd::verify::verify(c_args),
+ SubCommand::Dat(c_args) => cmd::dat::run(c_args),
+ SubCommand::Info(c_args) => cmd::info::run(c_args),
+ SubCommand::Convert(c_args) => cmd::convert::run(c_args),
+ SubCommand::Extract(c_args) => cmd::extract::run(c_args),
+ SubCommand::Verify(c_args) => cmd::verify::run(c_args),
}
}
diff --git a/nodtool/src/util/redump.rs b/nodtool/src/util/redump.rs
index c839758..bc07033 100644
--- a/nodtool/src/util/redump.rs
+++ b/nodtool/src/util/redump.rs
@@ -1,25 +1,70 @@
-use std::{mem::size_of, str};
+use std::{
+ fs::File,
+ io::{BufReader, Cursor, Write},
+ mem::size_of,
+ path::Path,
+ str,
+ sync::OnceLock,
+};
-use nod::array_ref;
-use zerocopy::{FromBytes, FromZeroes};
+use hex::deserialize as deserialize_hex;
+use nod::{array_ref, Result};
+use serde::Deserialize;
+use zerocopy::{AsBytes, FromBytes, FromZeroes};
#[derive(Clone, Debug)]
-pub struct GameResult {
- pub name: &'static str,
+pub struct GameResult<'a> {
+ pub name: &'a str,
pub crc32: u32,
pub md5: [u8; 16],
pub sha1: [u8; 20],
}
-pub fn find_by_crc32(crc32: u32) -> Option {
- let header: &Header = Header::ref_from_prefix(&DATA.0).unwrap();
+pub struct EntryIter<'a> {
+ data: &'a [u8],
+ index: usize,
+}
+
+impl EntryIter<'static> {
+ pub fn new() -> EntryIter<'static> { Self { data: loaded_data(), index: 0 } }
+}
+
+impl<'a> Iterator for EntryIter<'a> {
+ type Item = GameResult<'a>;
+
+ fn next(&mut self) -> Option {
+ let header: &Header = Header::ref_from_prefix(self.data).unwrap();
+ assert_eq!(header.entry_size as usize, size_of::());
+ if self.index >= header.entry_count as usize {
+ return None;
+ }
+
+ let entries_size = header.entry_count as usize * size_of::();
+ let entries: &[GameEntry] = GameEntry::slice_from(
+ &self.data[size_of::()..size_of::() + entries_size],
+ )
+ .unwrap();
+ let string_table: &[u8] = &self.data[size_of::() + entries_size..];
+
+ let entry = &entries[self.index];
+ let offset = entry.string_table_offset as usize;
+ let name_size = u32::from_ne_bytes(*array_ref![string_table, offset, 4]) as usize;
+ let name = str::from_utf8(&string_table[offset + 4..offset + 4 + name_size]).unwrap();
+ self.index += 1;
+ Some(GameResult { name, crc32: entry.crc32, md5: entry.md5, sha1: entry.sha1 })
+ }
+}
+
+pub fn find_by_crc32(crc32: u32) -> Option> {
+ let data = loaded_data();
+ let header: &Header = Header::ref_from_prefix(data).unwrap();
assert_eq!(header.entry_size as usize, size_of::());
let entries_size = header.entry_count as usize * size_of::();
let entries: &[GameEntry] =
- GameEntry::slice_from(&DATA.0[size_of::()..size_of::() + entries_size])
+ GameEntry::slice_from(&data[size_of::()..size_of::() + entries_size])
.unwrap();
- let string_table: &[u8] = &DATA.0[size_of::() + entries_size..];
+ let string_table: &[u8] = &data[size_of::() + entries_size..];
// Binary search by CRC32
let index = entries.binary_search_by_key(&crc32, |entry| entry.crc32).ok()?;
@@ -32,14 +77,82 @@ pub fn find_by_crc32(crc32: u32) -> Option {
Some(GameResult { name, crc32: entry.crc32, md5: entry.md5, sha1: entry.sha1 })
}
-#[repr(C, align(4))]
-struct Aligned(T);
+const BUILTIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/parsed-dats.bin"));
+static LOADED: OnceLock> = OnceLock::new();
-const DATA: &Aligned<[u8]> =
- &Aligned(*include_bytes!(concat!(env!("OUT_DIR"), "/parsed-dats.bin")));
+fn loaded_data() -> &'static [u8] {
+ LOADED
+ .get_or_init(|| {
+ let size = zstd::zstd_safe::get_frame_content_size(BUILTIN).unwrap().unwrap() as usize;
+ let mut out = ::new_box_slice_zeroed(size);
+ let out_size = zstd::bulk::Decompressor::new()
+ .unwrap()
+ .decompress_to_buffer(BUILTIN, out.as_mut())
+ .unwrap();
+ debug_assert_eq!(out_size, size);
+ out
+ })
+ .as_ref()
+}
+
+pub fn load_dats<'a>(paths: impl Iterator- ) -> Result<()> {
+ // Parse dat files
+ let mut entries = Vec::<(GameEntry, String)>::new();
+ for path in paths {
+ let file = BufReader::new(File::open(path).expect("Failed to open dat file"));
+ let dat: DatFile = quick_xml::de::from_reader(file).expect("Failed to parse dat file");
+ entries.extend(dat.games.into_iter().filter_map(|game| {
+ if game.roms.len() != 1 {
+ return None;
+ }
+ let rom = &game.roms[0];
+ Some((
+ GameEntry {
+ string_table_offset: 0,
+ crc32: u32::from_be_bytes(rom.crc32),
+ md5: rom.md5,
+ sha1: rom.sha1,
+ sectors: rom.size.div_ceil(0x8000) as u32,
+ },
+ game.name,
+ ))
+ }));
+ }
+
+ // Sort by CRC32
+ entries.sort_by_key(|(entry, _)| entry.crc32);
+
+ // Calculate total size
+ let entries_size = entries.len() * size_of::();
+ let string_table_size = entries.iter().map(|(_, name)| name.len() + 4).sum::();
+ let total_size = size_of::() + entries_size + string_table_size;
+ let mut result = ::new_box_slice_zeroed(total_size);
+ let mut out = Cursor::new(result.as_mut());
+
+ // Write game entries
+ let header =
+ Header { entry_count: entries.len() as u32, entry_size: size_of::() as u32 };
+ out.write_all(header.as_bytes()).unwrap();
+ let mut string_table_offset = 0u32;
+ for (entry, name) in &mut entries {
+ entry.string_table_offset = string_table_offset;
+ out.write_all(entry.as_bytes()).unwrap();
+ string_table_offset += name.as_bytes().len() as u32 + 4;
+ }
+
+ // Write string table
+ for (_, name) in &entries {
+ out.write_all(&(name.len() as u32).to_le_bytes()).unwrap();
+ out.write_all(name.as_bytes()).unwrap();
+ }
+
+ // Finalize
+ assert_eq!(out.position() as usize, total_size);
+ LOADED.set(result).map_err(|_| nod::Error::Other("dats already loaded".to_string()))
+}
// Keep in sync with build.rs
-#[derive(Clone, Debug, FromBytes, FromZeroes)]
+#[derive(Clone, Debug, AsBytes, FromBytes, FromZeroes)]
#[repr(C, align(4))]
struct Header {
entry_count: u32,
@@ -47,7 +160,7 @@ struct Header {
}
// Keep in sync with build.rs
-#[derive(Clone, Debug, FromBytes, FromZeroes)]
+#[derive(Clone, Debug, AsBytes, FromBytes, FromZeroes)]
#[repr(C, align(4))]
struct GameEntry {
crc32: u32,
@@ -56,3 +169,33 @@ struct GameEntry {
md5: [u8; 16],
sha1: [u8; 20],
}
+
+#[derive(Clone, Debug, Deserialize)]
+struct DatFile {
+ #[serde(rename = "game")]
+ games: Vec,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+struct DatGame {
+ #[serde(rename = "@name")]
+ name: String,
+ // #[serde(rename = "category", default)]
+ // categories: Vec,
+ #[serde(rename = "rom")]
+ roms: Vec,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+struct DatGameRom {
+ // #[serde(rename = "@name")]
+ // name: String,
+ #[serde(rename = "@size")]
+ size: u64,
+ #[serde(rename = "@crc", deserialize_with = "deserialize_hex")]
+ crc32: [u8; 4],
+ #[serde(rename = "@md5", deserialize_with = "deserialize_hex")]
+ md5: [u8; 16],
+ #[serde(rename = "@sha1", deserialize_with = "deserialize_hex")]
+ sha1: [u8; 20],
+}