diff --git a/Cargo.lock b/Cargo.lock index e851ff4..072f148 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "aes" version = "0.8.4" @@ -394,10 +400,20 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +[[package]] +name = "miniz_oxide" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +dependencies = [ + "adler", +] + [[package]] name = "nod" version = "0.2.0" dependencies = [ + "adler", "aes", "base16ct", "bzip2", @@ -408,6 +424,7 @@ dependencies = [ "itertools", "liblzma", "log", + "miniz_oxide", "rayon", "sha1", "thiserror", diff --git a/README.md b/README.md index 2fcb510..d1c8b90 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# nod-rs [![Build Status]][actions] [![Latest Version]][crates.io] [![Api Rustdoc]][rustdoc] ![Rust Version] +# nod [![Build Status]][actions] [![Latest Version]][crates.io] [![Api Rustdoc]][rustdoc] ![Rust Version] [Build Status]: https://github.com/encounter/nod-rs/actions/workflows/build.yaml/badge.svg [actions]: https://github.com/encounter/nod-rs/actions @@ -8,7 +8,7 @@ [rustdoc]: https://docs.rs/nod [Rust Version]: https://img.shields.io/badge/rust-1.73+-blue.svg?maxAge=3600 -Library for traversing & reading GameCube and Wii disc images. +Library for traversing & reading Nintendo Optical Disc (GameCube and Wii) images. Originally based on the C++ library [nod](https://github.com/AxioDL/nod), but does not currently support authoring. @@ -19,6 +19,7 @@ Currently supported file formats: - WBFS (+ NKit 2 lossless) - CISO (+ NKit 2 lossless) - NFS (Wii U VC) +- GCZ ## CLI tool diff --git a/nod/Cargo.toml b/nod/Cargo.toml index 65e8029..431f5b3 100644 --- a/nod/Cargo.toml +++ b/nod/Cargo.toml @@ -15,13 +15,15 @@ keywords = ["gamecube", "wii", "iso", "wbfs", "rvz"] categories = ["command-line-utilities", "parser-implementations"] [features] -default = ["compress-bzip2", "compress-lzma", "compress-zstd"] +default = ["compress-bzip2", "compress-lzma", "compress-zlib", "compress-zstd"] asm = ["sha1/asm"] compress-bzip2 = ["bzip2"] compress-lzma = ["liblzma"] +compress-zlib = ["adler", "miniz_oxide"] compress-zstd = ["zstd"] [dependencies] +adler = { version = "1.0.2", optional = true } aes = "0.8.4" base16ct = "0.2.0" bzip2 = { version = "0.4.4", features = ["static"], optional = true } @@ -32,6 +34,7 @@ encoding_rs = "0.8.33" itertools = "0.12.1" liblzma = { version = "0.2.3", features = ["static"], optional = true } log = "0.4.20" +miniz_oxide = { version = "0.7.2", optional = true } rayon = "1.8.1" sha1 = "0.10.6" thiserror = "1.0.57" diff --git a/nod/src/disc/gcn.rs b/nod/src/disc/gcn.rs index aa8d83c..cbc3efd 100644 --- a/nod/src/disc/gcn.rs +++ b/nod/src/disc/gcn.rs @@ -21,7 +21,7 @@ use crate::{ pub struct PartitionGC { io: Box, - block: Option, + block: Block, block_buf: Box<[u8]>, block_idx: u32, sector_buf: Box<[u8; SECTOR_SIZE]>, @@ -34,7 +34,7 @@ impl Clone for PartitionGC { fn clone(&self) -> Self { Self { io: self.io.clone(), - block: None, + block: Block::default(), block_buf: ::new_box_slice_zeroed(self.block_buf.len()), block_idx: u32::MAX, sector_buf: <[u8; SECTOR_SIZE]>::new_box_zeroed(), @@ -50,7 +50,7 @@ impl PartitionGC { let block_size = inner.block_size(); Ok(Box::new(Self { io: inner, - block: None, + block: Block::default(), block_buf: ::new_box_slice_zeroed(block_size as usize), block_idx: u32::MAX, sector_buf: <[u8; SECTOR_SIZE]>::new_box_zeroed(), @@ -76,10 +76,7 @@ impl Read for PartitionGC { // Copy sector if necessary if sector != self.sector { - let Some(block) = &self.block else { - return Ok(0); - }; - block.copy_raw( + self.block.copy_raw( self.sector_buf.as_mut(), self.block_buf.as_ref(), block_idx, diff --git a/nod/src/disc/hashes.rs b/nod/src/disc/hashes.rs index c8ba3fb..6d54ac8 100644 --- a/nod/src/disc/hashes.rs +++ b/nod/src/disc/hashes.rs @@ -196,7 +196,7 @@ pub fn rebuild_hashes(reader: &mut DiscReader) -> Result<()> { } #[inline] -fn hash_bytes(buf: &[u8]) -> HashBytes { +pub fn hash_bytes(buf: &[u8]) -> HashBytes { let mut hasher = Sha1::new(); hasher.update(buf); hasher.finalize().into() diff --git a/nod/src/disc/reader.rs b/nod/src/disc/reader.rs index fb647ab..9ea5211 100644 --- a/nod/src/disc/reader.rs +++ b/nod/src/disc/reader.rs @@ -27,7 +27,7 @@ pub enum EncryptionMode { pub struct DiscReader { io: Box, - block: Option, + block: Block, block_buf: Box<[u8]>, block_idx: u32, sector_buf: Box<[u8; SECTOR_SIZE]>, @@ -43,7 +43,7 @@ impl Clone for DiscReader { fn clone(&self) -> Self { Self { io: self.io.clone(), - block: None, + block: Block::default(), block_buf: ::new_box_slice_zeroed(self.block_buf.len()), block_idx: u32::MAX, sector_buf: <[u8; SECTOR_SIZE]>::new_box_zeroed(), @@ -63,7 +63,7 @@ impl DiscReader { let meta = inner.meta(); let mut reader = Self { io: inner, - block: None, + block: Block::default(), block_buf: ::new_box_slice_zeroed(block_size as usize), block_idx: u32::MAX, sector_buf: <[u8; SECTOR_SIZE]>::new_box_zeroed(), @@ -92,7 +92,7 @@ impl DiscReader { } pub fn reset(&mut self) { - self.block = None; + self.block = Block::default(); self.block_buf.fill(0); self.block_idx = u32::MAX; self.sector_buf.fill(0); @@ -171,19 +171,16 @@ impl Read for DiscReader { // Read new sector into buffer if abs_sector != self.sector_idx { - let Some(block) = &self.block else { - return Ok(0); - }; if let Some(partition) = partition { match self.mode { - EncryptionMode::Decrypted => block.decrypt( + EncryptionMode::Decrypted => self.block.decrypt( self.sector_buf.as_mut(), self.block_buf.as_ref(), block_idx, abs_sector, partition, )?, - EncryptionMode::Encrypted => block.encrypt( + EncryptionMode::Encrypted => self.block.encrypt( self.sector_buf.as_mut(), self.block_buf.as_ref(), block_idx, @@ -192,7 +189,7 @@ impl Read for DiscReader { )?, } } else { - block.copy_raw( + self.block.copy_raw( self.sector_buf.as_mut(), self.block_buf.as_ref(), block_idx, diff --git a/nod/src/disc/wii.rs b/nod/src/disc/wii.rs index 13b4470..56354f7 100644 --- a/nod/src/disc/wii.rs +++ b/nod/src/disc/wii.rs @@ -233,7 +233,7 @@ impl WiiPartitionHeader { pub struct PartitionWii { io: Box, partition: PartitionInfo, - block: Option, + block: Block, block_buf: Box<[u8]>, block_idx: u32, sector_buf: Box<[u8; SECTOR_SIZE]>, @@ -250,7 +250,7 @@ impl Clone for PartitionWii { Self { io: self.io.clone(), partition: self.partition.clone(), - block: None, + block: Block::default(), block_buf: ::new_box_slice_zeroed(self.block_buf.len()), block_idx: u32::MAX, sector_buf: <[u8; SECTOR_SIZE]>::new_box_zeroed(), @@ -296,7 +296,7 @@ impl PartitionWii { Ok(Box::new(Self { io: reader.into_inner(), partition: partition.clone(), - block: None, + block: Block::default(), block_buf: ::new_box_slice_zeroed(block_size as usize), block_idx: u32::MAX, sector_buf: <[u8; SECTOR_SIZE]>::new_box_zeroed(), @@ -328,10 +328,7 @@ impl Read for PartitionWii { // Decrypt sector if necessary if sector != self.sector { - let Some(block) = &self.block else { - return Ok(0); - }; - block.decrypt( + self.block.decrypt( self.sector_buf.as_mut(), self.block_buf.as_ref(), block_idx, diff --git a/nod/src/io/block.rs b/nod/src/io/block.rs index eb800f8..f000f3a 100644 --- a/nod/src/io/block.rs +++ b/nod/src/io/block.rs @@ -10,7 +10,7 @@ use crate::{ wii::{WiiPartitionHeader, HASHES_SIZE, SECTOR_DATA_SIZE}, SECTOR_SIZE, }, - io::{aes_decrypt, aes_encrypt, ciso, iso, nfs, wbfs, wia, KeyBytes, MagicBytes}, + io::{aes_decrypt, aes_encrypt, KeyBytes, MagicBytes}, util::{lfg::LaggedFibonacci, read::read_from}, DiscHeader, DiscMeta, Error, PartitionHeader, PartitionKind, Result, ResultContext, }; @@ -18,15 +18,59 @@ use crate::{ /// Block I/O trait for reading disc images. pub trait BlockIO: DynClone + Send + Sync { /// Reads a block from the disc image. + fn read_block_internal( + &mut self, + out: &mut [u8], + block: u32, + partition: Option<&PartitionInfo>, + ) -> io::Result; + + /// Reads a full block from the disc image, combining smaller blocks if necessary. fn read_block( &mut self, out: &mut [u8], block: u32, partition: Option<&PartitionInfo>, - ) -> io::Result>; + ) -> io::Result { + let block_size_internal = self.block_size_internal(); + let block_size = self.block_size(); + if block_size_internal == block_size { + self.read_block_internal(out, block, partition) + } else { + let mut offset = 0usize; + let mut result = None; + let mut block_idx = + ((block as u64 * block_size as u64) / block_size_internal as u64) as u32; + while offset < block_size as usize { + let block = self.read_block_internal( + &mut out[offset..offset + block_size_internal as usize], + block_idx, + partition, + )?; + if result.is_none() { + result = Some(block); + } else if result != Some(block) { + if block == Block::Zero { + out[offset..offset + block_size_internal as usize].fill(0); + } else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Inconsistent block types in split block", + )); + } + } + offset += block_size_internal as usize; + block_idx += 1; + } + Ok(result.unwrap_or_default()) + } + } - /// The format's block size in bytes. Must be a multiple of the sector size (0x8000). - fn block_size(&self) -> u32; + /// The format's block size in bytes. Can be smaller than the sector size (0x8000). + fn block_size_internal(&self) -> u32; + + /// The block size used for processing. Must be a multiple of the sector size (0x8000). + fn block_size(&self) -> u32 { self.block_size_internal().max(SECTOR_SIZE as u32) } /// Returns extra metadata included in the disc file format, if any. fn meta(&self) -> DiscMeta; @@ -54,16 +98,41 @@ pub fn open(filename: &Path) -> Result> { read_from(&mut file) .with_context(|| format!("Reading magic bytes from {}", filename.display()))? }; - match magic { - ciso::CISO_MAGIC => Ok(ciso::DiscIOCISO::new(path)?), - nfs::NFS_MAGIC => match path.parent() { - Some(parent) if parent.is_dir() => Ok(nfs::DiscIONFS::new(path.parent().unwrap())?), - _ => Err(Error::DiscFormat("Failed to locate NFS parent directory".to_string())), + let io: Box = match magic { + crate::io::ciso::CISO_MAGIC => crate::io::ciso::DiscIOCISO::new(path)?, + #[cfg(feature = "compress-zlib")] + crate::io::gcz::GCZ_MAGIC => crate::io::gcz::DiscIOGCZ::new(path)?, + crate::io::nfs::NFS_MAGIC => match path.parent() { + Some(parent) if parent.is_dir() => { + crate::io::nfs::DiscIONFS::new(path.parent().unwrap())? + } + _ => { + return Err(Error::DiscFormat("Failed to locate NFS parent directory".to_string())); + } }, - wbfs::WBFS_MAGIC => Ok(wbfs::DiscIOWBFS::new(path)?), - wia::WIA_MAGIC | wia::RVZ_MAGIC => Ok(wia::DiscIOWIA::new(path)?), - _ => Ok(iso::DiscIOISO::new(path)?), + crate::io::wbfs::WBFS_MAGIC => crate::io::wbfs::DiscIOWBFS::new(path)?, + crate::io::wia::WIA_MAGIC | crate::io::wia::RVZ_MAGIC => { + crate::io::wia::DiscIOWIA::new(path)? + } + _ => crate::io::iso::DiscIOISO::new(path)?, + }; + if io.block_size_internal() < SECTOR_SIZE as u32 + && SECTOR_SIZE as u32 % io.block_size_internal() != 0 + { + return Err(Error::DiscFormat(format!( + "Sector size {} is not divisible by block size {}", + SECTOR_SIZE, + io.block_size_internal(), + ))); } + if io.block_size() % SECTOR_SIZE as u32 != 0 { + return Err(Error::DiscFormat(format!( + "Block size {} is not a multiple of sector size {}", + io.block_size(), + SECTOR_SIZE + ))); + } + Ok(io) } #[derive(Debug, Clone)] @@ -80,7 +149,7 @@ pub struct PartitionInfo { pub hash_table: Option, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum Block { /// Raw data or encrypted Wii partition data Raw, @@ -92,6 +161,7 @@ pub enum Block { /// Wii partition junk data Junk, /// All zeroes + #[default] Zero, } diff --git a/nod/src/io/ciso.rs b/nod/src/io/ciso.rs index ed0edd8..3a089bf 100644 --- a/nod/src/io/ciso.rs +++ b/nod/src/io/ciso.rs @@ -23,17 +23,18 @@ use crate::{ pub const CISO_MAGIC: MagicBytes = *b"CISO"; pub const CISO_MAP_SIZE: usize = SECTOR_SIZE - 8; +/// CISO header (little endian) #[derive(Clone, Debug, PartialEq, FromBytes, FromZeroes, AsBytes)] #[repr(C, align(4))] struct CISOHeader { magic: MagicBytes, - // little endian block_size: U32, block_present: [u8; CISO_MAP_SIZE], } static_assert!(size_of::() == SECTOR_SIZE); +#[derive(Clone)] pub struct DiscIOCISO { inner: SplitFileReader, header: CISOHeader, @@ -41,17 +42,6 @@ pub struct DiscIOCISO { nkit_header: Option, } -impl Clone for DiscIOCISO { - fn clone(&self) -> Self { - Self { - inner: self.inner.clone(), - header: self.header.clone(), - block_map: self.block_map, - nkit_header: self.nkit_header.clone(), - } - } -} - impl DiscIOCISO { pub fn new(filename: &Path) -> Result> { let mut inner = SplitFileReader::new(filename)?; @@ -97,15 +87,15 @@ impl DiscIOCISO { } impl BlockIO for DiscIOCISO { - fn read_block( + fn read_block_internal( &mut self, out: &mut [u8], block: u32, _partition: Option<&PartitionInfo>, - ) -> io::Result> { + ) -> io::Result { if block >= CISO_MAP_SIZE as u32 { // Out of bounds - return Ok(None); + return Ok(Block::Zero); } // Find the block in the map @@ -113,11 +103,11 @@ impl BlockIO for DiscIOCISO { if phys_block == u16::MAX { // Check if block is junk data if self.nkit_header.as_ref().is_some_and(|h| h.is_junk_block(block).unwrap_or(false)) { - return Ok(Some(Block::Junk)); + return Ok(Block::Junk); }; // Otherwise, read zeroes - return Ok(Some(Block::Zero)); + return Ok(Block::Zero); } // Read block @@ -125,10 +115,10 @@ impl BlockIO for DiscIOCISO { + phys_block as u64 * self.header.block_size.get() as u64; self.inner.seek(SeekFrom::Start(file_offset))?; self.inner.read_exact(out)?; - Ok(Some(Block::Raw)) + Ok(Block::Raw) } - fn block_size(&self) -> u32 { self.header.block_size.get() } + fn block_size_internal(&self) -> u32 { self.header.block_size.get() } fn meta(&self) -> DiscMeta { let mut result = DiscMeta { diff --git a/nod/src/io/gcz.rs b/nod/src/io/gcz.rs new file mode 100644 index 0000000..b72fffc --- /dev/null +++ b/nod/src/io/gcz.rs @@ -0,0 +1,192 @@ +use std::{ + io, + io::{Read, Seek, SeekFrom}, + mem::size_of, + path::Path, +}; + +use adler::adler32_slice; +use miniz_oxide::{inflate, inflate::core::inflate_flags}; +use zerocopy::{little_endian::*, AsBytes, FromBytes, FromZeroes}; +use zstd::zstd_safe::WriteBuf; + +use crate::{ + io::{ + block::{Block, BlockIO}, + split::SplitFileReader, + MagicBytes, + }, + static_assert, + util::read::{read_box_slice, read_from}, + Compression, DiscMeta, Error, Format, PartitionInfo, Result, ResultContext, +}; + +pub const GCZ_MAGIC: MagicBytes = [0x01, 0xC0, 0x0B, 0xB1]; + +/// GCZ header (little endian) +#[derive(Clone, Debug, PartialEq, FromBytes, FromZeroes, AsBytes)] +#[repr(C, align(4))] +struct GCZHeader { + magic: MagicBytes, + disc_type: U32, + compressed_size: U64, + disc_size: U64, + block_size: U32, + block_count: U32, +} + +static_assert!(size_of::() == 32); + +pub struct DiscIOGCZ { + inner: SplitFileReader, + header: GCZHeader, + block_map: Box<[U64]>, + block_hashes: Box<[U32]>, + block_buf: Box<[u8]>, + data_offset: u64, +} + +impl Clone for DiscIOGCZ { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + header: self.header.clone(), + block_map: self.block_map.clone(), + block_hashes: self.block_hashes.clone(), + block_buf: ::new_box_slice_zeroed(self.block_buf.len()), + data_offset: self.data_offset, + } + } +} + +impl DiscIOGCZ { + pub fn new(filename: &Path) -> Result> { + let mut inner = SplitFileReader::new(filename)?; + + // Read header + let header: GCZHeader = read_from(&mut inner).context("Reading GCZ header")?; + if header.magic != GCZ_MAGIC { + return Err(Error::DiscFormat("Invalid GCZ magic".to_string())); + } + + // Read block map and hashes + let block_count = header.block_count.get(); + let block_map = + read_box_slice(&mut inner, block_count as usize).context("Reading GCZ block map")?; + let block_hashes = + read_box_slice(&mut inner, block_count as usize).context("Reading GCZ block hashes")?; + + // header + block_count * (u64 + u32) + let data_offset = size_of::() as u64 + block_count as u64 * 12; + + // Reset reader + inner.reset(); + let block_buf = ::new_box_slice_zeroed(header.block_size.get() as usize); + Ok(Box::new(Self { inner, header, block_map, block_hashes, block_buf, data_offset })) + } +} + +impl BlockIO for DiscIOGCZ { + fn read_block_internal( + &mut self, + out: &mut [u8], + block: u32, + _partition: Option<&PartitionInfo>, + ) -> io::Result { + if block >= self.header.block_count.get() { + // Out of bounds + return Ok(Block::Zero); + } + + // Find block offset and size + let mut file_offset = self.block_map[block as usize].get(); + let mut compressed = true; + if file_offset & (1 << 63) != 0 { + file_offset &= !(1 << 63); + compressed = false; + } + let compressed_size = + ((self.block_map.get(block as usize + 1).unwrap_or(&self.header.compressed_size).get() + & !(1 << 63)) + - file_offset) as usize; + if compressed_size > self.block_buf.len() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "Compressed block size exceeds block size: {} > {}", + compressed_size, + self.block_buf.len() + ), + )); + } else if !compressed && compressed_size != self.block_buf.len() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "Uncompressed block size does not match block size: {} != {}", + compressed_size, + self.block_buf.len() + ), + )); + } + + // Read block + self.inner.seek(SeekFrom::Start(self.data_offset + file_offset))?; + self.inner.read_exact(&mut self.block_buf[..compressed_size])?; + + // Verify block checksum + let checksum = adler32_slice(&self.block_buf[..compressed_size]); + let expected_checksum = self.block_hashes[block as usize].get(); + if checksum != expected_checksum { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "Block checksum mismatch: {:#010x} != {:#010x}", + checksum, expected_checksum + ), + )); + } + + if compressed { + // Decompress block + let mut decompressor = inflate::core::DecompressorOxide::new(); + let input = &self.block_buf[..compressed_size]; + let (status, in_size, out_size) = inflate::core::decompress( + &mut decompressor, + input, + out, + 0, + inflate_flags::TINFL_FLAG_PARSE_ZLIB_HEADER + | inflate_flags::TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF, + ); + if status != inflate::TINFLStatus::Done + || in_size != compressed_size + || out_size != self.block_buf.len() + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "Deflate decompression failed: {:?} (in: {}, out: {})", + status, in_size, out_size + ), + )); + } + } else { + // Copy uncompressed block + out.copy_from_slice(self.block_buf.as_slice()); + } + Ok(Block::Raw) + } + + fn block_size_internal(&self) -> u32 { self.header.block_size.get() } + + fn meta(&self) -> DiscMeta { + DiscMeta { + format: Format::Gcz, + compression: Compression::Deflate, + block_size: Some(self.header.block_size.get()), + lossless: true, + disc_size: Some(self.header.disc_size.get()), + ..Default::default() + } + } +} diff --git a/nod/src/io/iso.rs b/nod/src/io/iso.rs index 3215b83..167a816 100644 --- a/nod/src/io/iso.rs +++ b/nod/src/io/iso.rs @@ -1,6 +1,6 @@ use std::{ io, - io::{Read, Seek}, + io::{Read, Seek, SeekFrom}, path::Path, }; @@ -11,7 +11,7 @@ use crate::{ split::SplitFileReader, Format, }, - DiscMeta, Error, Result, + DiscMeta, Result, }; #[derive(Clone)] @@ -22,34 +22,37 @@ pub struct DiscIOISO { impl DiscIOISO { pub fn new(filename: &Path) -> Result> { let inner = SplitFileReader::new(filename)?; - if inner.len() % SECTOR_SIZE as u64 != 0 { - return Err(Error::DiscFormat( - "ISO size is not a multiple of sector size (0x8000 bytes)".to_string(), - )); - } Ok(Box::new(Self { inner })) } } impl BlockIO for DiscIOISO { - fn read_block( + fn read_block_internal( &mut self, out: &mut [u8], block: u32, _partition: Option<&PartitionInfo>, - ) -> io::Result> { + ) -> io::Result { let offset = block as u64 * SECTOR_SIZE as u64; - if offset >= self.inner.len() { + let total_size = self.inner.len(); + if offset >= total_size { // End of file - return Ok(None); + return Ok(Block::Zero); } - self.inner.seek(io::SeekFrom::Start(offset))?; - self.inner.read_exact(out)?; - Ok(Some(Block::Raw)) + self.inner.seek(SeekFrom::Start(offset))?; + if offset + SECTOR_SIZE as u64 > total_size { + // If the last block is not a full sector, fill the rest with zeroes + let read = (total_size - offset) as usize; + self.inner.read_exact(&mut out[..read])?; + out[read..].fill(0); + } else { + self.inner.read_exact(out)?; + } + Ok(Block::Raw) } - fn block_size(&self) -> u32 { SECTOR_SIZE as u32 } + fn block_size_internal(&self) -> u32 { SECTOR_SIZE as u32 } fn meta(&self) -> DiscMeta { DiscMeta { diff --git a/nod/src/io/mod.rs b/nod/src/io/mod.rs index 441d08a..4029d05 100644 --- a/nod/src/io/mod.rs +++ b/nod/src/io/mod.rs @@ -4,6 +4,8 @@ use std::fmt; pub(crate) mod block; pub(crate) mod ciso; +#[cfg(feature = "compress-zlib")] +pub(crate) mod gcz; pub(crate) mod iso; pub(crate) mod nfs; pub(crate) mod nkit; @@ -27,6 +29,8 @@ pub enum Format { Iso, /// CISO Ciso, + /// GCZ + Gcz, /// NFS (Wii U VC) Nfs, /// RVZ @@ -42,6 +46,7 @@ impl fmt::Display for Format { match self { Format::Iso => write!(f, "ISO"), Format::Ciso => write!(f, "CISO"), + Format::Gcz => write!(f, "GCZ"), Format::Nfs => write!(f, "NFS"), Format::Rvz => write!(f, "RVZ"), Format::Wbfs => write!(f, "WBFS"), @@ -55,14 +60,16 @@ pub enum Compression { /// No compression #[default] None, - /// Purge (WIA only) - Purge, /// BZIP2 Bzip2, + /// Deflate (GCZ only) + Deflate, /// LZMA Lzma, /// LZMA2 Lzma2, + /// Purge (WIA only) + Purge, /// Zstandard Zstandard, } @@ -71,10 +78,11 @@ impl fmt::Display for Compression { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Compression::None => write!(f, "None"), - Compression::Purge => write!(f, "Purge"), Compression::Bzip2 => write!(f, "BZIP2"), + Compression::Deflate => write!(f, "Deflate"), Compression::Lzma => write!(f, "LZMA"), Compression::Lzma2 => write!(f, "LZMA2"), + Compression::Purge => write!(f, "Purge"), Compression::Zstandard => write!(f, "Zstandard"), } } @@ -108,6 +116,7 @@ pub struct DiscMeta { } /// Encrypts data in-place using AES-128-CBC with the given key and IV. +#[inline(always)] pub(crate) fn aes_encrypt(key: &KeyBytes, iv: KeyBytes, data: &mut [u8]) { use aes::cipher::{block_padding::NoPadding, BlockEncryptMut, KeyIvInit}; >::new(key.into(), &aes::Block::from(iv)) @@ -116,6 +125,7 @@ pub(crate) fn aes_encrypt(key: &KeyBytes, iv: KeyBytes, data: &mut [u8]) { } /// Decrypts data in-place using AES-128-CBC with the given key and IV. +#[inline(always)] pub(crate) fn aes_decrypt(key: &KeyBytes, iv: KeyBytes, data: &mut [u8]) { use aes::cipher::{block_padding::NoPadding, BlockDecryptMut, KeyIvInit}; >::new(key.into(), &aes::Block::from(iv)) diff --git a/nod/src/io/nfs.rs b/nod/src/io/nfs.rs index 32cf0dc..e48aaac 100644 --- a/nod/src/io/nfs.rs +++ b/nod/src/io/nfs.rs @@ -83,6 +83,7 @@ impl NFSHeader { } } +#[derive(Clone)] pub struct DiscIONFS { inner: SplitFileReader, header: NFSHeader, @@ -91,18 +92,6 @@ pub struct DiscIONFS { key: KeyBytes, } -impl Clone for DiscIONFS { - fn clone(&self) -> Self { - Self { - inner: self.inner.clone(), - header: self.header.clone(), - raw_size: self.raw_size, - disc_size: self.disc_size, - key: self.key, - } - } -} - impl DiscIONFS { pub fn new(directory: &Path) -> Result> { let mut disc_io = Box::new(Self { @@ -118,17 +107,17 @@ impl DiscIONFS { } impl BlockIO for DiscIONFS { - fn read_block( + fn read_block_internal( &mut self, out: &mut [u8], sector: u32, partition: Option<&PartitionInfo>, - ) -> io::Result> { + ) -> io::Result { // Calculate physical sector let phys_sector = self.header.phys_sector(sector); if phys_sector == u32::MAX { // Logical zero sector - return Ok(Some(Block::Zero)); + return Ok(Block::Zero); } // Read sector @@ -146,13 +135,13 @@ impl BlockIO for DiscIONFS { aes_decrypt(&self.key, iv, out); if partition.is_some() { - Ok(Some(Block::PartDecrypted { has_hashes: true })) + Ok(Block::PartDecrypted { has_hashes: true }) } else { - Ok(Some(Block::Raw)) + Ok(Block::Raw) } } - fn block_size(&self) -> u32 { SECTOR_SIZE as u32 } + fn block_size_internal(&self) -> u32 { SECTOR_SIZE as u32 } fn meta(&self) -> DiscMeta { DiscMeta { format: Format::Nfs, decrypted: true, ..Default::default() } diff --git a/nod/src/io/wbfs.rs b/nod/src/io/wbfs.rs index ffd9110..8981649 100644 --- a/nod/src/io/wbfs.rs +++ b/nod/src/io/wbfs.rs @@ -109,30 +109,33 @@ impl DiscIOWBFS { } impl BlockIO for DiscIOWBFS { - fn read_block( + fn read_block_internal( &mut self, out: &mut [u8], block: u32, _partition: Option<&PartitionInfo>, - ) -> io::Result> { + ) -> io::Result { let block_size = self.header.block_size(); if block >= self.header.max_blocks() { - return Ok(None); + return Ok(Block::Zero); } // Check if block is junk data - if self.nkit_header.as_ref().is_some_and(|h| h.is_junk_block(block).unwrap_or(false)) { - return Ok(Some(Block::Junk)); + if self.nkit_header.as_ref().and_then(|h| h.is_junk_block(block)).unwrap_or(false) { + return Ok(Block::Junk); } // Read block let block_start = block_size as u64 * self.block_table[block as usize].get() as u64; + if block_start == 0 { + return Ok(Block::Zero); + } self.inner.seek(SeekFrom::Start(block_start))?; self.inner.read_exact(out)?; - Ok(Some(Block::Raw)) + Ok(Block::Raw) } - fn block_size(&self) -> u32 { self.header.block_size() } + fn block_size_internal(&self) -> u32 { self.header.block_size() } fn meta(&self) -> DiscMeta { let mut result = DiscMeta { diff --git a/nod/src/io/wia.rs b/nod/src/io/wia.rs index 37251aa..fabeeff 100644 --- a/nod/src/io/wia.rs +++ b/nod/src/io/wia.rs @@ -5,11 +5,11 @@ use std::{ path::Path, }; -use sha1::{Digest, Sha1}; use zerocopy::{big_endian::*, AsBytes, FromBytes, FromZeroes}; use crate::{ disc::{ + hashes::hash_bytes, wii::{HASHES_SIZE, SECTOR_DATA_SIZE}, SECTOR_SIZE, }, @@ -533,13 +533,6 @@ impl Clone for DiscIOWIA { } } -#[inline] -fn hash_bytes(buf: &[u8]) -> HashBytes { - let mut hasher = Sha1::new(); - hasher.update(buf); - hasher.finalize().into() -} - fn verify_hash(buf: &[u8], expected: &HashBytes) -> Result<()> { let out = hash_bytes(buf); if out != *expected { @@ -686,12 +679,12 @@ where } impl BlockIO for DiscIOWIA { - fn read_block( + fn read_block_internal( &mut self, out: &mut [u8], sector: u32, partition: Option<&PartitionInfo>, - ) -> io::Result> { + ) -> io::Result { let mut chunk_size = self.disc.chunk_size.get(); let sectors_per_chunk = chunk_size / SECTOR_SIZE as u32; let disc_offset = sector as u64 * SECTOR_SIZE as u64; @@ -792,7 +785,7 @@ impl BlockIO for DiscIOWIA { // Special case for all-zero data if group.data_size() == 0 { self.exception_lists.clear(); - return Ok(Some(Block::Zero)); + return Ok(Block::Zero); } // Read group data if necessary @@ -875,17 +868,17 @@ impl BlockIO for DiscIOWIA { &self.group_data[sector_data_start..sector_data_start + SECTOR_DATA_SIZE]; out[..HASHES_SIZE].fill(0); out[HASHES_SIZE..SECTOR_SIZE].copy_from_slice(sector_data); - Ok(Some(Block::PartDecrypted { has_hashes: false })) + Ok(Block::PartDecrypted { has_hashes: false }) } else { let sector_data_start = group_sector as usize * SECTOR_SIZE; out.copy_from_slice( &self.group_data[sector_data_start..sector_data_start + SECTOR_SIZE], ); - Ok(Some(Block::Raw)) + Ok(Block::Raw) } } - fn block_size(&self) -> u32 { + fn block_size_internal(&self) -> u32 { // WIA/RVZ chunks aren't always the full size, so we'll consider the // block size to be one sector, and handle the complexity ourselves. SECTOR_SIZE as u32 diff --git a/nod/src/util/mod.rs b/nod/src/util/mod.rs index 69f60d9..5d558b0 100644 --- a/nod/src/util/mod.rs +++ b/nod/src/util/mod.rs @@ -17,7 +17,7 @@ where T: Div + Rem + Copy { #[macro_export] macro_rules! array_ref { ($slice:expr, $offset:expr, $size:expr) => {{ - #[inline] + #[inline(always)] fn to_array(slice: &[T]) -> &[T; $size] { unsafe { &*(slice.as_ptr() as *const [_; $size]) } } @@ -29,7 +29,7 @@ macro_rules! array_ref { #[macro_export] macro_rules! array_ref_mut { ($slice:expr, $offset:expr, $size:expr) => {{ - #[inline] + #[inline(always)] fn to_array(slice: &mut [T]) -> &mut [T; $size] { unsafe { &mut *(slice.as_ptr() as *mut [_; $size]) } } diff --git a/nodtool/build.rs b/nodtool/build.rs index a7b77fd..0ada74d 100644 --- a/nodtool/build.rs +++ b/nodtool/build.rs @@ -45,6 +45,7 @@ fn main() { // Parse dat files let mut entries = Vec::<(GameEntry, String)>::new(); for path in ["assets/redump-gc.dat", "assets/redump-wii.dat"] { + println!("cargo:rustc-rerun-if-changed={}", path); let file = BufReader::new(File::open(path).expect("Failed to open dat file")); let dat: DatFile = quick_xml::de::from_reader(file).expect("Failed to parse dat file"); entries.extend(dat.games.into_iter().map(|game| { diff --git a/nodtool/src/main.rs b/nodtool/src/main.rs index 19be358..c2a6393 100644 --- a/nodtool/src/main.rs +++ b/nodtool/src/main.rs @@ -12,7 +12,7 @@ use std::{ fs::File, io, io::{BufWriter, Read, Write}, - path::{Path, PathBuf}, + path::{Path, PathBuf, MAIN_SEPARATOR}, str::FromStr, sync::{mpsc::sync_channel, Arc}, thread, @@ -27,7 +27,7 @@ use nod::{ Compression, Disc, DiscHeader, DiscMeta, Fst, Node, OpenOptions, PartitionBase, PartitionKind, PartitionMeta, Result, ResultContext, SECTOR_SIZE, }; -use size::Size; +use size::{Base, Size}; use supports_color::Stream; use tracing::level_filters::LevelFilter; use tracing_subscriber::EnvFilter; @@ -64,12 +64,12 @@ enum SubCommand { } #[derive(FromArgs, Debug)] -/// Displays information about a disc image. +/// Displays information about disc images. #[argp(subcommand, name = "info")] struct InfoArgs { #[argp(positional)] - /// path to disc image - file: PathBuf, + /// Path to disc image(s) + file: Vec, } #[derive(FromArgs, Debug)] @@ -77,17 +77,21 @@ struct InfoArgs { #[argp(subcommand, name = "extract")] struct ExtractArgs { #[argp(positional)] - /// path to disc image + /// Path to disc image file: PathBuf, #[argp(positional)] - /// output directory (optional) + /// Output directory (optional) out: Option, #[argp(switch, short = 'q')] - /// quiet output + /// Quiet output quiet: bool, #[argp(switch, short = 'h')] - /// validate disc hashes (Wii only) + /// Validate data hashes (Wii only) validate: bool, + #[argp(option, short = 'p')] + /// Partition to extract (default: data) + /// Options: all, data, update, channel, or a partition index + partition: Option, } #[derive(FromArgs, Debug)] @@ -206,7 +210,7 @@ fn main() { let mut result = Ok(()); if let Some(dir) = &args.chdir { result = env::set_current_dir(dir).map_err(|e| { - nod::Error::Io(format!("Failed to change working directory to '{}'", dir.display()), e) + nod::Error::Io(format!("Failed to change working directory to '{}'", display(dir)), e) }); } result = result.and_then(|_| match args.command { @@ -253,7 +257,15 @@ fn print_header(header: &DiscHeader, meta: &DiscMeta) { } fn info(args: InfoArgs) -> Result<()> { - let disc = Disc::new_with_options(args.file, &OpenOptions { + for file in &args.file { + info_file(file)?; + } + Ok(()) +} + +fn info_file(path: &Path) -> Result<()> { + log::info!("Loading {}", display(path)); + let disc = Disc::new_with_options(path, &OpenOptions { rebuild_encryption: false, validate_hashes: false, })?; @@ -327,6 +339,7 @@ fn info(args: InfoArgs) -> Result<()> { header.wii_magic.get() ); } + println!(); Ok(()) } @@ -343,7 +356,7 @@ fn verify(args: VerifyArgs) -> Result<()> { } fn convert_and_verify(in_file: &Path, out_file: Option<&Path>, md5: bool) -> Result<()> { - println!("Loading {}", in_file.display()); + println!("Loading {}", display(in_file)); let mut disc = Disc::new_with_options(in_file, &OpenOptions { rebuild_encryption: true, validate_hashes: false, @@ -357,7 +370,7 @@ fn convert_and_verify(in_file: &Path, out_file: Option<&Path>, md5: bool) -> Res let mut file = if let Some(out_file) = out_file { Some( File::create(out_file) - .with_context(|| format!("Creating file {}", out_file.display()))?, + .with_context(|| format!("Creating file {}", display(out_file)))?, ) } else { None @@ -432,7 +445,7 @@ fn convert_and_verify(in_file: &Path, out_file: Option<&Path>, md5: bool) -> Res println!(); if let Some(path) = out_file { - println!("Wrote {} to {}", Size::from_bytes(total_read), path.display()); + println!("Wrote {} to {}", Size::from_bytes(total_read), display(path)); } println!(); @@ -450,11 +463,7 @@ fn convert_and_verify(in_file: &Path, out_file: Option<&Path>, md5: bool) -> Res } } - let redump_entry = if let (Some(crc32), Some(sha1)) = (crc32, sha1) { - redump::find_by_hashes(crc32, sha1) - } else { - None - }; + let redump_entry = crc32.and_then(redump::find_by_crc32); let expected_crc32 = meta.crc32.or(redump_entry.as_ref().map(|e| e.crc32)); let expected_md5 = meta.md5.or(redump_entry.as_ref().map(|e| e.md5)); let expected_sha1 = meta.sha1.or(redump_entry.as_ref().map(|e| e.sha1)); @@ -475,7 +484,22 @@ fn convert_and_verify(in_file: &Path, out_file: Option<&Path>, md5: bool) -> Res } if let Some(entry) = &redump_entry { - println!("Redump: {} ✅", entry.name); + let mut full_match = true; + if let Some(md5) = md5 { + if entry.md5 != md5 { + full_match = false; + } + } + if let Some(sha1) = sha1 { + if entry.sha1 != sha1 { + full_match = false; + } + } + if full_match { + println!("Redump: {} ✅", entry.name); + } else { + println!("Redump: {} ❓ (partial match)", entry.name); + } } else { println!("Redump: Not found ❌"); } @@ -520,12 +544,49 @@ fn extract(args: ExtractArgs) -> Result<()> { validate_hashes: args.validate, })?; let is_wii = disc.header().is_wii(); - let mut partition = disc.open_partition_kind(PartitionKind::Data)?; + if let Some(partition) = args.partition { + if partition.eq_ignore_ascii_case("all") { + for info in disc.partitions() { + let mut out_dir = output_dir.clone(); + out_dir.push(info.kind.dir_name().as_ref()); + let mut partition = disc.open_partition(info.index)?; + extract_partition(partition.as_mut(), &out_dir, is_wii, args.quiet)?; + } + } else if partition.eq_ignore_ascii_case("data") { + let mut partition = disc.open_partition_kind(PartitionKind::Data)?; + extract_partition(partition.as_mut(), &output_dir, is_wii, args.quiet)?; + } else if partition.eq_ignore_ascii_case("update") { + let mut partition = disc.open_partition_kind(PartitionKind::Update)?; + extract_partition(partition.as_mut(), &output_dir, is_wii, args.quiet)?; + } else if partition.eq_ignore_ascii_case("channel") { + let mut partition = disc.open_partition_kind(PartitionKind::Channel)?; + extract_partition(partition.as_mut(), &output_dir, is_wii, args.quiet)?; + } else { + let idx = partition.parse::().map_err(|_| "Invalid partition index")?; + let mut partition = disc.open_partition(idx)?; + extract_partition(partition.as_mut(), &output_dir, is_wii, args.quiet)?; + } + } else { + let mut partition = disc.open_partition_kind(PartitionKind::Data)?; + extract_partition(partition.as_mut(), &output_dir, is_wii, args.quiet)?; + } + Ok(()) +} + +fn extract_partition( + partition: &mut dyn PartitionBase, + out_dir: &Path, + is_wii: bool, + quiet: bool, +) -> Result<()> { let meta = partition.meta()?; - extract_sys_files(meta.as_ref(), &output_dir.join("sys"), args.quiet)?; + extract_sys_files(meta.as_ref(), out_dir, quiet)?; // Extract FST - let files_dir = output_dir.join("files"); + let files_dir = out_dir.join("files"); + fs::create_dir_all(&files_dir) + .with_context(|| format!("Creating directory {}", display(&files_dir)))?; + let fst = Fst::new(&meta.raw_fst)?; let mut path_segments = Vec::<(Cow, usize)>::new(); for (idx, node, name) in fst.iter() { @@ -548,28 +609,47 @@ fn extract(args: ExtractArgs) -> Result<()> { fs::create_dir_all(files_dir.join(&path)) .with_context(|| format!("Creating directory {}", path))?; } else { - extract_node(node, partition.as_mut(), &files_dir, &path, is_wii, args.quiet)?; + extract_node(node, partition, &files_dir, &path, is_wii, quiet)?; } } Ok(()) } fn extract_sys_files(data: &PartitionMeta, out_dir: &Path, quiet: bool) -> Result<()> { - fs::create_dir_all(out_dir) - .with_context(|| format!("Creating output directory {}", out_dir.display()))?; - extract_file(data.raw_boot.as_ref(), &out_dir.join("boot.bin"), quiet)?; - extract_file(data.raw_bi2.as_ref(), &out_dir.join("bi2.bin"), quiet)?; - extract_file(data.raw_apploader.as_ref(), &out_dir.join("apploader.img"), quiet)?; - extract_file(data.raw_fst.as_ref(), &out_dir.join("fst.bin"), quiet)?; - extract_file(data.raw_dol.as_ref(), &out_dir.join("main.dol"), quiet)?; + let sys_dir = out_dir.join("sys"); + fs::create_dir_all(&sys_dir) + .with_context(|| format!("Creating directory {}", display(&sys_dir)))?; + extract_file(data.raw_boot.as_ref(), &sys_dir.join("boot.bin"), quiet)?; + extract_file(data.raw_bi2.as_ref(), &sys_dir.join("bi2.bin"), quiet)?; + extract_file(data.raw_apploader.as_ref(), &sys_dir.join("apploader.img"), quiet)?; + extract_file(data.raw_fst.as_ref(), &sys_dir.join("fst.bin"), quiet)?; + extract_file(data.raw_dol.as_ref(), &sys_dir.join("main.dol"), quiet)?; + + // Wii files + if let Some(ticket) = data.raw_ticket.as_deref() { + extract_file(ticket, &out_dir.join("ticket.bin"), quiet)?; + } + if let Some(tmd) = data.raw_tmd.as_deref() { + extract_file(tmd, &out_dir.join("tmd.bin"), quiet)?; + } + if let Some(cert_chain) = data.raw_cert_chain.as_deref() { + extract_file(cert_chain, &out_dir.join("cert.bin"), quiet)?; + } + if let Some(h3_table) = data.raw_h3_table.as_deref() { + extract_file(h3_table, &out_dir.join("h3.bin"), quiet)?; + } Ok(()) } fn extract_file(bytes: &[u8], out_path: &Path, quiet: bool) -> Result<()> { if !quiet { - println!("Extracting {} (size: {})", out_path.display(), Size::from_bytes(bytes.len())); + println!( + "Extracting {} (size: {})", + display(out_path), + Size::from_bytes(bytes.len()).format().with_base(Base::Base10) + ); } - fs::write(out_path, bytes).with_context(|| format!("Writing file {}", out_path.display()))?; + fs::write(out_path, bytes).with_context(|| format!("Writing file {}", display(out_path)))?; Ok(()) } @@ -585,12 +665,12 @@ fn extract_node( if !quiet { println!( "Extracting {} (size: {})", - file_path.display(), - Size::from_bytes(node.length(is_wii)) + display(&file_path), + Size::from_bytes(node.length(is_wii)).format().with_base(Base::Base10) ); } let file = File::create(&file_path) - .with_context(|| format!("Creating file {}", file_path.display()))?; + .with_context(|| format!("Creating file {}", display(&file_path)))?; let mut w = BufWriter::with_capacity(partition.ideal_buffer_size(), file); let mut r = partition.open_file(node).with_context(|| { format!( @@ -600,7 +680,33 @@ fn extract_node( node.length(is_wii) ) })?; - io::copy(&mut r, &mut w).with_context(|| format!("Extracting file {}", file_path.display()))?; - w.flush().with_context(|| format!("Flushing file {}", file_path.display()))?; + io::copy(&mut r, &mut w).with_context(|| format!("Extracting file {}", display(&file_path)))?; + w.flush().with_context(|| format!("Flushing file {}", display(&file_path)))?; Ok(()) } + +fn display(path: &Path) -> PathDisplay { PathDisplay { path } } + +struct PathDisplay<'a> { + path: &'a Path, +} + +impl<'a> fmt::Display for PathDisplay<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use fmt::Write; + let mut first = true; + for segment in self.path.iter() { + let segment_str = segment.to_string_lossy(); + if segment_str == "." { + continue; + } + if first { + first = false; + } else { + f.write_char(MAIN_SEPARATOR)?; + } + f.write_str(&segment_str)?; + } + Ok(()) + } +} diff --git a/nodtool/src/redump.rs b/nodtool/src/redump.rs index fc47f77..93e9ad8 100644 --- a/nodtool/src/redump.rs +++ b/nodtool/src/redump.rs @@ -12,7 +12,7 @@ pub struct GameResult { pub size: u64, } -pub fn find_by_hashes(crc32: u32, sha1: [u8; 20]) -> Option { +pub fn find_by_crc32(crc32: u32) -> Option { let header: &Header = Header::ref_from_prefix(&DATA.0).unwrap(); assert_eq!(header.entry_size as usize, size_of::()); @@ -25,13 +25,8 @@ pub fn find_by_hashes(crc32: u32, sha1: [u8; 20]) -> Option { // Binary search by CRC32 let index = entries.binary_search_by_key(&crc32, |entry| entry.crc32).ok()?; - // Verify SHA-1 - let entry = &entries[index]; - if entry.sha1 != sha1 { - return None; - } - // Parse the entry + let entry = &entries[index]; let offset = entry.string_table_offset as usize; let name_size = u32::from_ne_bytes(*array_ref![string_table, offset, 4]) as usize; let name = str::from_utf8(&string_table[offset + 4..offset + 4 + name_size]).unwrap();