From fb3542f4459b38e0598f3e810337bd3a90334bfc Mon Sep 17 00:00:00 2001 From: Luke Street Date: Tue, 4 Mar 2025 22:54:09 -0700 Subject: [PATCH] Rename PartitionHeader -> BootHeader & various fixes --- nod/src/build/gc.rs | 81 ++++++------- nod/src/common.rs | 27 +++-- nod/src/disc/direct.rs | 14 +++ nod/src/disc/gcn.rs | 58 +++++---- nod/src/disc/hashes.rs | 9 +- nod/src/disc/mod.rs | 34 ++++-- nod/src/disc/preloader.rs | 140 ++++++++++++++++------ nod/src/disc/reader.rs | 199 ++++++++++++++++--------------- nod/src/disc/wii.rs | 1 + nod/src/io/block.rs | 3 +- nod/src/io/tgc.rs | 48 ++++---- nod/src/io/wia.rs | 88 ++++++++------ nod/src/lib.rs | 33 ++++- nod/src/read.rs | 34 ++++-- nod/src/util/compress.rs | 32 ++++- nod/src/util/mod.rs | 30 ++++- nodtool/Cargo.toml | 7 +- nodtool/assets/gc-non-redump.dat | 8 +- nodtool/assets/gc-redump.dat | 135 ++++++++++++++------- nodtool/assets/wii-redump.dat | 67 +++++++---- nodtool/src/cmd/gen.rs | 42 +++---- nodtool/src/main.rs | 6 +- 22 files changed, 694 insertions(+), 402 deletions(-) diff --git a/nod/src/build/gc.rs b/nod/src/build/gc.rs index 069c0aa..d616b87 100644 --- a/nod/src/build/gc.rs +++ b/nod/src/build/gc.rs @@ -11,11 +11,11 @@ use zerocopy::{FromZeros, IntoBytes}; use crate::{ disc::{ fst::{Fst, FstBuilder}, - DiscHeader, PartitionHeader, BI2_SIZE, BOOT_SIZE, GCN_MAGIC, MINI_DVD_SIZE, SECTOR_SIZE, + BootHeader, DiscHeader, BI2_SIZE, BOOT_SIZE, GCN_MAGIC, MINI_DVD_SIZE, SECTOR_SIZE, WII_MAGIC, }, read::DiscStream, - util::{align_up_64, array_ref, array_ref_mut, lfg::LaggedFibonacci}, + util::{array_ref, array_ref_mut, lfg::LaggedFibonacci, Align}, Error, Result, ResultContext, }; @@ -33,7 +33,7 @@ pub struct FileInfo { pub struct GCPartitionBuilder { disc_header: Box, - partition_header: Box, + boot_header: Box, user_files: Vec, overrides: PartitionOverrides, junk_files: Vec, @@ -97,7 +97,7 @@ impl GCPartitionBuilder { } Self { disc_header, - partition_header: PartitionHeader::new_box_zeroed().unwrap(), + boot_header: BootHeader::new_box_zeroed().unwrap(), user_files: Vec::new(), overrides, junk_files: Vec::new(), @@ -110,8 +110,8 @@ impl GCPartitionBuilder { } #[inline] - pub fn set_partition_header(&mut self, partition_header: Box) { - self.partition_header = partition_header; + pub fn set_boot_header(&mut self, boot_header: Box) { + self.boot_header = boot_header; } pub fn add_file(&mut self, info: FileInfo) -> Result<()> { @@ -139,8 +139,8 @@ impl GCPartitionBuilder { layout.locate_sys_files(sys_file_callback)?; layout.apply_overrides(&self.overrides)?; let write_info = layout.layout_files()?; - let disc_size = layout.partition_header.user_offset.get() as u64 - + layout.partition_header.user_size.get() as u64; + let disc_size = + layout.boot_header.user_offset.get() as u64 + layout.boot_header.user_size.get() as u64; let junk_id = layout.junk_id(); Ok(GCPartitionWriter::new(write_info, disc_size, junk_id, self.disc_header.disc_num)) } @@ -148,7 +148,7 @@ impl GCPartitionBuilder { struct GCPartitionLayout { disc_header: Box, - partition_header: Box, + boot_header: Box, user_files: Vec, apploader_file: Option, dol_file: Option, @@ -162,7 +162,7 @@ impl GCPartitionLayout { fn new(builder: &GCPartitionBuilder) -> Self { GCPartitionLayout { disc_header: builder.disc_header.clone(), - partition_header: builder.partition_header.clone(), + boot_header: builder.boot_header.clone(), user_files: builder.user_files.clone(), apploader_file: None, dol_file: None, @@ -194,9 +194,7 @@ impl GCPartitionLayout { ))); } self.disc_header.as_mut_bytes().copy_from_slice(&data[..size_of::()]); - self.partition_header - .as_mut_bytes() - .copy_from_slice(&data[size_of::()..]); + self.boot_header.as_mut_bytes().copy_from_slice(&data[size_of::()..]); *handled = true; continue; } @@ -228,7 +226,7 @@ impl GCPartitionLayout { // Locate other system files let is_wii = self.disc_header.is_wii(); for (info, handled) in self.user_files.iter().zip(handled.iter_mut()) { - let dol_offset = self.partition_header.dol_offset(is_wii); + let dol_offset = self.boot_header.dol_offset(is_wii); if (dol_offset != 0 && info.offset == Some(dol_offset)) || info.name == "sys/main.dol" { let mut info = info.clone(); if info.alignment.is_none() { @@ -239,7 +237,7 @@ impl GCPartitionLayout { continue; } - let fst_offset = self.partition_header.fst_offset(is_wii); + let fst_offset = self.boot_header.fst_offset(is_wii); if (fst_offset != 0 && info.offset == Some(fst_offset)) || info.name == "sys/fst.bin" { let mut data = Vec::with_capacity(info.size as usize); file_callback(&mut data, &info.name) @@ -293,7 +291,7 @@ impl GCPartitionLayout { if let Some(audio_stream_buf_size) = overrides.audio_stream_buf_size { self.disc_header.audio_stream_buf_size = audio_stream_buf_size; } - let set_bi2 = self.raw_bi2.is_none() && overrides.region.is_some(); + let set_bi2 = self.raw_bi2.is_none() || overrides.region.is_some(); let raw_bi2 = self.raw_bi2.get_or_insert_with(|| { <[u8]>::new_box_zeroed_with_elems(BI2_SIZE).expect("Failed to allocate BI2") }); @@ -383,11 +381,11 @@ impl GCPartitionLayout { } } let raw_fst = builder.finalize(); - if raw_fst.len() != self.partition_header.fst_size(is_wii) as usize { + if raw_fst.len() != self.boot_header.fst_size(is_wii) as usize { return Err(Error::Other(format!( "FST size mismatch: {} != {}", raw_fst.len(), - self.partition_header.fst_size(is_wii) + self.boot_header.fst_size(is_wii) ))); } Ok(Arc::from(raw_fst)) @@ -411,7 +409,7 @@ impl GCPartitionLayout { let mut boot = <[u8]>::new_box_zeroed_with_elems(BOOT_SIZE)?; boot[..size_of::()].copy_from_slice(self.disc_header.as_bytes()); - boot[size_of::()..].copy_from_slice(self.partition_header.as_bytes()); + boot[size_of::()..].copy_from_slice(self.boot_header.as_bytes()); write_info.push(WriteInfo { kind: WriteKind::Static(Arc::from(boot), "[BOOT]"), size: BOOT_SIZE as u64, @@ -433,21 +431,21 @@ impl GCPartitionLayout { // Update DOL and FST offsets if not set let is_wii = self.disc_header.is_wii(); - let mut dol_offset = self.partition_header.dol_offset(is_wii); + let mut dol_offset = self.boot_header.dol_offset(is_wii); if dol_offset == 0 { - dol_offset = align_up_64(last_offset, dol_file.alignment.unwrap() as u64); - self.partition_header.set_dol_offset(dol_offset, is_wii); + dol_offset = last_offset.align_up(dol_file.alignment.unwrap() as u64); + self.boot_header.set_dol_offset(dol_offset, is_wii); } - let mut fst_offset = self.partition_header.fst_offset(is_wii); + let mut fst_offset = self.boot_header.fst_offset(is_wii); if fst_offset == 0 { // TODO handle DOL in user data - fst_offset = align_up_64(dol_offset + dol_file.size, 128); - self.partition_header.set_fst_offset(fst_offset, is_wii); + fst_offset = (dol_offset + dol_file.size).align_up(128); + self.boot_header.set_fst_offset(fst_offset, is_wii); } let fst_size = self.calculate_fst_size()?; - self.partition_header.set_fst_size(fst_size, is_wii); - if self.partition_header.fst_max_size(is_wii) < fst_size { - self.partition_header.set_fst_max_size(fst_size, is_wii); + self.boot_header.set_fst_size(fst_size, is_wii); + if self.boot_header.fst_max_size(is_wii) < fst_size { + self.boot_header.set_fst_max_size(fst_size, is_wii); } if dol_offset < fst_offset { @@ -474,10 +472,10 @@ impl GCPartitionLayout { let mut last_offset = self.layout_system_data(&mut system_write_info)?; // Layout user data - let mut user_offset = self.partition_header.user_offset.get() as u64; + let mut user_offset = self.boot_header.user_offset.get() as u64; if user_offset == 0 { - user_offset = align_up_64(last_offset, SECTOR_SIZE as u64); - self.partition_header.user_offset.set(user_offset as u32); + user_offset = last_offset.align_up(SECTOR_SIZE as u64); + self.boot_header.user_offset.set(user_offset as u32); } else if user_offset < last_offset { return Err(Error::Other(format!( "User offset {:#X} is before FST {:#X}", @@ -488,7 +486,7 @@ impl GCPartitionLayout { for info in &self.user_files { let offset = info .offset - .unwrap_or_else(|| align_up_64(last_offset, info.alignment.unwrap_or(32) as u64)); + .unwrap_or_else(|| last_offset.align_up(info.alignment.unwrap_or(32) as u64)); write_info.push(WriteInfo { kind: WriteKind::File(info.name.clone()), offset, @@ -504,7 +502,7 @@ impl GCPartitionLayout { write_info.push(WriteInfo { kind: WriteKind::Static(fst_data, "[FST]"), size: fst_size, - offset: self.partition_header.fst_offset(is_wii), + offset: self.boot_header.fst_offset(is_wii), }); // Add system files to write info write_info.extend(system_write_info); @@ -512,17 +510,17 @@ impl GCPartitionLayout { sort_files(&mut write_info)?; // Update user size if not set - if self.partition_header.user_size.get() == 0 { + if self.boot_header.user_size.get() == 0 { let user_end = if self.disc_header.is_wii() { - align_up_64(last_offset, SECTOR_SIZE as u64) + last_offset.align_up(SECTOR_SIZE as u64) } else { MINI_DVD_SIZE }; - self.partition_header.user_size.set((user_end - user_offset) as u32); + self.boot_header.user_size.set((user_end - user_offset) as u32); } // Insert junk data - let write_info = insert_junk_data(write_info, &self.partition_header); + let write_info = insert_junk_data(write_info, &self.boot_header); Ok(write_info) } @@ -534,11 +532,11 @@ impl GCPartitionLayout { pub(crate) fn insert_junk_data( write_info: Vec, - partition_header: &PartitionHeader, + boot_header: &BootHeader, ) -> Vec { let mut new_write_info = Vec::with_capacity(write_info.len()); - let fst_end = partition_header.fst_offset(false) + partition_header.fst_size(false); + let fst_end = boot_header.fst_offset(false) + boot_header.fst_size(false); let file_gap = find_file_gap(&write_info, fst_end); let mut last_file_end = 0; for info in write_info { @@ -550,7 +548,7 @@ pub(crate) fn insert_junk_data( // FST, and the junk data in between the inner and outer rim files. This attempts to // determine the correct alignment, but is not 100% accurate. let junk_start = if file_gap == Some(last_file_end) { - align_up_64(last_file_end, 4) + last_file_end.align_up(4) } else { aligned_end }; @@ -565,8 +563,7 @@ pub(crate) fn insert_junk_data( new_write_info.push(info); } let aligned_end = gcm_align(last_file_end); - let user_end = - partition_header.user_offset.get() as u64 + partition_header.user_size.get() as u64; + let user_end = boot_header.user_offset.get() as u64 + boot_header.user_size.get() as u64; if aligned_end < user_end && aligned_end >= fst_end { new_write_info.push(WriteInfo { kind: WriteKind::Junk, diff --git a/nod/src/common.rs b/nod/src/common.rs index 826b9c3..fa97f06 100644 --- a/nod/src/common.rs +++ b/nod/src/common.rs @@ -6,8 +6,10 @@ use zerocopy::FromBytes; use crate::{ disc::{ - fst::Fst, wii::WiiPartitionHeader, DiscHeader, PartitionHeader, BOOT_SIZE, SECTOR_SIZE, + fst::Fst, wii::WiiPartitionHeader, BootHeader, DebugHeader, DiscHeader, BB2_OFFSET, + BOOT_SIZE, SECTOR_SIZE, }, + util::array_ref, Error, Result, }; @@ -308,7 +310,7 @@ pub struct PartitionInfo { pub has_encryption: bool, /// Whether the partition data hashes are present pub has_hashes: bool, - /// Disc and partition header (boot.bin) + /// Disc and boot header (boot.bin) pub raw_boot: Arc<[u8; BOOT_SIZE]>, /// File system table (fst.bin), or `None` if partition is invalid pub raw_fst: Option>, @@ -330,15 +332,26 @@ impl PartitionInfo { /// A view into the disc header. #[inline] pub fn disc_header(&self) -> &DiscHeader { - DiscHeader::ref_from_bytes(&self.raw_boot[..size_of::()]) + DiscHeader::ref_from_bytes(array_ref![self.raw_boot, 0, size_of::()]) .expect("Invalid disc header alignment") } - /// A view into the partition header. + /// A view into the debug header. #[inline] - pub fn partition_header(&self) -> &PartitionHeader { - PartitionHeader::ref_from_bytes(&self.raw_boot[size_of::()..]) - .expect("Invalid partition header alignment") + pub fn debug_header(&self) -> &DebugHeader { + DebugHeader::ref_from_bytes(array_ref![ + self.raw_boot, + size_of::(), + size_of::() + ]) + .expect("Invalid debug header alignment") + } + + /// A view into the boot header. + #[inline] + pub fn boot_header(&self) -> &BootHeader { + BootHeader::ref_from_bytes(array_ref![self.raw_boot, BB2_OFFSET, size_of::()]) + .expect("Invalid boot header alignment") } /// A view into the file system table (FST). diff --git a/nod/src/disc/direct.rs b/nod/src/disc/direct.rs index ae5da16..2fc6dcf 100644 --- a/nod/src/disc/direct.rs +++ b/nod/src/disc/direct.rs @@ -14,6 +14,7 @@ use crate::{ Result, }; +#[derive(Clone)] pub enum DirectDiscReaderMode { Raw, Partition { disc_header: Arc, data_start_sector: u32, key: KeyBytes }, @@ -31,6 +32,19 @@ pub struct DirectDiscReader { mode: DirectDiscReaderMode, } +impl Clone for DirectDiscReader { + fn clone(&self) -> Self { + Self { + io: self.io.clone(), + block: Block::default(), + block_buf: <[u8]>::new_box_zeroed_with_elems(self.block_buf.len()).unwrap(), + block_decrypted: false, + pos: 0, + mode: self.mode.clone(), + } + } +} + impl DirectDiscReader { pub fn new(inner: Box) -> Result> { let block_size = inner.block_size() as usize; diff --git a/nod/src/disc/gcn.rs b/nod/src/disc/gcn.rs index 7a1553b..7e29149 100644 --- a/nod/src/disc/gcn.rs +++ b/nod/src/disc/gcn.rs @@ -1,6 +1,6 @@ use std::{ io, - io::{BufRead, Read, Seek, SeekFrom}, + io::{BufRead, Seek, SeekFrom}, mem::size_of, sync::Arc, }; @@ -10,11 +10,11 @@ use zerocopy::{FromBytes, FromZeros, IntoBytes}; use crate::{ disc::{ preloader::{fetch_sector_group, Preloader, SectorGroup, SectorGroupRequest}, - ApploaderHeader, DiscHeader, DolHeader, PartitionHeader, BI2_SIZE, BOOT_SIZE, - SECTOR_GROUP_SIZE, SECTOR_SIZE, + ApploaderHeader, BootHeader, DolHeader, BB2_OFFSET, BI2_SIZE, BOOT_SIZE, SECTOR_GROUP_SIZE, + SECTOR_SIZE, }, io::block::BlockReader, - read::{PartitionEncryption, PartitionMeta, PartitionReader}, + read::{DiscStream, PartitionEncryption, PartitionMeta, PartitionReader}, util::{ impl_read_for_bufread, read::{read_arc, read_arc_slice, read_from}, @@ -69,12 +69,12 @@ impl BufRead for PartitionReaderGC { let abs_sector = (self.pos / SECTOR_SIZE as u64) as u32; let group_idx = abs_sector / 64; - let abs_group_sector = group_idx * 64; let max_groups = self.disc_size.div_ceil(SECTOR_GROUP_SIZE as u64) as u32; let request = SectorGroupRequest { group_idx, partition_idx: None, mode: PartitionEncryption::Original, + force_rehash: false, }; // Load sector group @@ -82,7 +82,7 @@ impl BufRead for PartitionReaderGC { fetch_sector_group(request, max_groups, &mut self.sector_group, &self.preloader)?; // Calculate the number of consecutive sectors in the group - let group_sector = abs_sector - abs_group_sector; + let group_sector = abs_sector - sector_group.start_sector; let consecutive_sectors = sector_group.consecutive_sectors(group_sector); if consecutive_sectors == 0 { return Ok(&[]); @@ -90,7 +90,7 @@ impl BufRead for PartitionReaderGC { let num_sectors = group_sector + consecutive_sectors; // Read from sector group buffer - let group_start = abs_group_sector as u64 * SECTOR_SIZE as u64; + let group_start = sector_group.start_sector as u64 * SECTOR_SIZE as u64; let offset = (self.pos - group_start) as usize; let end = (num_sectors as u64 * SECTOR_SIZE as u64).min(self.disc_size - group_start) as usize; @@ -131,12 +131,14 @@ impl PartitionReader for PartitionReaderGC { } pub(crate) fn read_dol( - reader: &mut dyn PartitionReader, - partition_header: &PartitionHeader, + // TODO: replace with &dyn mut DiscStream when trait_upcasting is stabilized + // https://github.com/rust-lang/rust/issues/65991 + reader: &mut (impl DiscStream + ?Sized), + boot_header: &BootHeader, is_wii: bool, ) -> Result> { reader - .seek(SeekFrom::Start(partition_header.dol_offset(is_wii))) + .seek(SeekFrom::Start(boot_header.dol_offset(is_wii))) .context("Seeking to DOL offset")?; let dol_header: DolHeader = read_from(reader).context("Reading DOL header")?; let dol_size = (dol_header.text_offs.iter().zip(&dol_header.text_sizes)) @@ -150,30 +152,28 @@ pub(crate) fn read_dol( Ok(Arc::from(raw_dol)) } -pub(crate) fn read_fst( - reader: &mut R, - partition_header: &PartitionHeader, +pub(crate) fn read_fst( + // TODO: replace with &dyn mut DiscStream when trait_upcasting is stabilized + // https://github.com/rust-lang/rust/issues/65991 + reader: &mut (impl DiscStream + ?Sized), + boot_header: &BootHeader, is_wii: bool, -) -> Result> -where - R: Read + Seek + ?Sized, -{ +) -> Result> { reader - .seek(SeekFrom::Start(partition_header.fst_offset(is_wii))) + .seek(SeekFrom::Start(boot_header.fst_offset(is_wii))) .context("Seeking to FST offset")?; - let raw_fst: Arc<[u8]> = read_arc_slice(reader, partition_header.fst_size(is_wii) as usize) + let raw_fst: Arc<[u8]> = read_arc_slice(reader, boot_header.fst_size(is_wii) as usize) .with_context(|| { format!( "Reading partition FST (offset {}, size {})", - partition_header.fst_offset(is_wii), - partition_header.fst_size(is_wii) + boot_header.fst_offset(is_wii), + boot_header.fst_size(is_wii) ) })?; Ok(raw_fst) } -pub(crate) fn read_apploader(reader: &mut R) -> Result> -where R: Read + Seek + ?Sized { +pub(crate) fn read_apploader(reader: &mut dyn DiscStream) -> Result> { reader .seek(SeekFrom::Start(BOOT_SIZE as u64 + BI2_SIZE as u64)) .context("Seeking to apploader offset")?; @@ -190,14 +190,10 @@ where R: Read + Seek + ?Sized { Ok(Arc::from(raw_apploader)) } -pub(crate) fn read_part_meta( - reader: &mut dyn PartitionReader, - is_wii: bool, -) -> Result { +pub(crate) fn read_part_meta(reader: &mut dyn DiscStream, is_wii: bool) -> Result { // boot.bin let raw_boot: Arc<[u8; BOOT_SIZE]> = read_arc(reader).context("Reading boot.bin")?; - let partition_header = - PartitionHeader::ref_from_bytes(&raw_boot[size_of::()..]).unwrap(); + let boot_header = BootHeader::ref_from_bytes(&raw_boot[BB2_OFFSET..]).unwrap(); // bi2.bin let raw_bi2: Arc<[u8; BI2_SIZE]> = read_arc(reader).context("Reading bi2.bin")?; @@ -206,10 +202,10 @@ pub(crate) fn read_part_meta( let raw_apploader = read_apploader(reader)?; // fst.bin - let raw_fst = read_fst(reader, partition_header, is_wii)?; + let raw_fst = read_fst(reader, boot_header, is_wii)?; // main.dol - let raw_dol = read_dol(reader, partition_header, is_wii)?; + let raw_dol = read_dol(reader, boot_header, is_wii)?; Ok(PartitionMeta { raw_boot, diff --git a/nod/src/disc/hashes.rs b/nod/src/disc/hashes.rs index 52268ca..433b35f 100644 --- a/nod/src/disc/hashes.rs +++ b/nod/src/disc/hashes.rs @@ -42,7 +42,10 @@ impl GroupHashes { pub const NUM_H0_HASHES: usize = SECTOR_DATA_SIZE / HASHES_SIZE; #[instrument(skip_all)] -pub fn hash_sector_group(sector_group: &[u8; SECTOR_GROUP_SIZE]) -> Box { +pub fn hash_sector_group( + sector_group: &[u8; SECTOR_GROUP_SIZE], + ignore_existing: bool, +) -> Box { let mut result = GroupHashes::new_box_zeroed().unwrap(); for (h2_index, h2_hash) in result.h2_hashes.iter_mut().enumerate() { let out_h1_hashes = array_ref_mut![result.h1_hashes, h2_index * 8, 8]; @@ -50,7 +53,9 @@ pub fn hash_sector_group(sector_group: &[u8; SECTOR_GROUP_SIZE]) -> Box() + size_of::(); +/// Offset in bytes of the boot block within a disc partition. +pub const BB2_OFFSET: usize = 0x420; -/// Size in bytes of the debug and region information. (bi2.bin) +/// Size in bytes of the disc header, debug block and boot block. (boot.bin) +pub const BOOT_SIZE: usize = 0x440; + +/// Size in bytes of the DVD Boot Info (debug and region information, bi2.bin) pub const BI2_SIZE: usize = 0x2000; /// The size of a single-layer MiniDVD. (1.4 GB) @@ -114,20 +117,28 @@ impl DiscHeader { pub fn has_partition_encryption(&self) -> bool { self.no_partition_encryption == 0 } } -/// A header describing the contents of a disc partition. +/// The debug block of a disc partition. /// -/// **GameCube**: Always follows the disc header. -/// -/// **Wii**: Follows the disc header within each partition. +/// Located at offset 0x400 (following the disc header) within each partition. #[derive(Clone, Debug, PartialEq, FromBytes, IntoBytes, Immutable, KnownLayout)] #[repr(C, align(4))] -pub struct PartitionHeader { +pub struct DebugHeader { /// Debug monitor offset pub debug_mon_offset: U32, /// Debug monitor load address pub debug_load_address: U32, /// Padding _pad1: [u8; 0x18], +} + +static_assert!(size_of::() == 0x20); + +/// The boot block (BB2) of a disc partition. +/// +/// Located at offset 0x420 (following the debug block) within each partition. +#[derive(Clone, Debug, PartialEq, FromBytes, IntoBytes, Immutable, KnownLayout)] +#[repr(C, align(4))] +pub struct BootHeader { /// Offset to main DOL (Wii: >> 2) pub dol_offset: U32, /// Offset to file system table (Wii: >> 2) @@ -146,9 +157,12 @@ pub struct PartitionHeader { _pad2: [u8; 4], } -static_assert!(size_of::() == 0x40); +static_assert!(size_of::() == 0x20); +static_assert!( + size_of::() + size_of::() + size_of::() == BOOT_SIZE +); -impl PartitionHeader { +impl BootHeader { /// Offset within the partition to the main DOL. #[inline] pub fn dol_offset(&self, is_wii: bool) -> u64 { diff --git a/nod/src/disc/preloader.rs b/nod/src/disc/preloader.rs index e793bd9..f3e6977 100644 --- a/nod/src/disc/preloader.rs +++ b/nod/src/disc/preloader.rs @@ -1,5 +1,6 @@ use std::{ collections::HashMap, + fmt::{Display, Formatter}, io, num::NonZeroUsize, sync::{Arc, Mutex}, @@ -19,7 +20,9 @@ use zerocopy::FromZeros; use crate::{ common::PartitionInfo, disc::{ - hashes::hash_sector_group, wii::HASHES_SIZE, DiscHeader, SECTOR_GROUP_SIZE, SECTOR_SIZE, + hashes::{hash_sector_group, GroupHashes}, + wii::HASHES_SIZE, + DiscHeader, SECTOR_GROUP_SIZE, SECTOR_SIZE, }, io::{ block::{Block, BlockKind, BlockReader}, @@ -30,6 +33,7 @@ use crate::{ aes::{decrypt_sector, encrypt_sector}, array_ref_mut, }, + IoResultContext, }; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -37,14 +41,27 @@ pub struct SectorGroupRequest { pub group_idx: u32, pub partition_idx: Option, pub mode: PartitionEncryption, + pub force_rehash: bool, +} + +impl Display for SectorGroupRequest { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self.partition_idx { + Some(idx) => write!(f, "Partition {} group {}", idx, self.group_idx), + None => write!(f, "Group {}", self.group_idx), + } + } } #[derive(Clone)] pub struct SectorGroup { pub request: SectorGroupRequest, + pub start_sector: u32, pub data: Bytes, pub sector_bitmap: u64, pub io_duration: Option, + #[allow(unused)] // TODO WIA hash exceptions + pub group_hashes: Option>, } impl SectorGroup { @@ -208,9 +225,12 @@ fn preloader_thread( let end = Instant::now(); last_request_end = Some(end); let req_time = end - start; - stat_tx + if stat_tx .send(PreloaderThreadStats { thread_id, wait_time, req_time, io_time }) - .expect("Failed to send preloader stats"); + .is_err() + { + break; + } } }) .expect("Failed to spawn preloader thread") @@ -329,6 +349,18 @@ impl Clone for SectorGroupLoader { } } +#[derive(Default)] +struct LoadedSectorGroup { + /// Start sector of the group + start_sector: u32, + /// Bitmap of sectors that were read + sector_bitmap: u64, + /// Total duration of I/O operations + io_duration: Option, + /// Calculated sector group hashes + group_hashes: Option>, +} + impl SectorGroupLoader { pub fn new( io: Box, @@ -344,13 +376,21 @@ impl SectorGroupLoader { let mut sector_group_buf = BytesMut::zeroed(SECTOR_GROUP_SIZE); let out = array_ref_mut![sector_group_buf, 0, SECTOR_GROUP_SIZE]; - let (sector_bitmap, io_duration) = if request.partition_idx.is_some() { - self.load_partition_group(request, out)? - } else { - self.load_raw_group(request, out)? - }; + let LoadedSectorGroup { start_sector, sector_bitmap, io_duration, group_hashes } = + if request.partition_idx.is_some() { + self.load_partition_group(request, out)? + } else { + self.load_raw_group(request, out)? + }; - Ok(SectorGroup { request, data: sector_group_buf.freeze(), sector_bitmap, io_duration }) + Ok(SectorGroup { + request, + start_sector, + data: sector_group_buf.freeze(), + sector_bitmap, + io_duration, + group_hashes, + }) } /// Load a sector group from a partition. @@ -360,16 +400,16 @@ impl SectorGroupLoader { &mut self, request: SectorGroupRequest, sector_group_buf: &mut [u8; SECTOR_GROUP_SIZE], - ) -> io::Result<(u64, Option)> { + ) -> io::Result { let Some(partition) = request.partition_idx.and_then(|idx| self.partitions.get(idx as usize)) else { - return Ok((0, None)); + return Ok(LoadedSectorGroup::default()); }; let abs_group_sector = partition.data_start_sector + request.group_idx * 64; if abs_group_sector >= partition.data_end_sector { - return Ok((0, None)); + return Ok(LoadedSectorGroup::default()); } // Bitmap of sectors that were read @@ -382,6 +422,8 @@ impl SectorGroupLoader { let mut hash_exceptions = Vec::::new(); // Total duration of I/O operations let mut io_duration = None; + // Calculated sector group hashes + let mut group_hashes = None; // Read sector group for sector in 0..64 { @@ -397,7 +439,10 @@ impl SectorGroupLoader { // Read new block if !self.block.contains(abs_sector) { - self.block = self.io.read_block(self.block_buf.as_mut(), abs_sector)?; + self.block = self + .io + .read_block(self.block_buf.as_mut(), abs_sector) + .io_with_context(|| format!("Reading block for sector {abs_sector}"))?; if let Some(duration) = self.block.io_duration { *io_duration.get_or_insert_with(Duration::default) += duration; } @@ -408,16 +453,21 @@ impl SectorGroupLoader { } // Add hash exceptions - self.block.append_hash_exceptions(abs_sector, sector, &mut hash_exceptions)?; + self.block + .append_hash_exceptions(abs_sector, sector, &mut hash_exceptions) + .io_with_context(|| format!("Appending hash exceptions for sector {abs_sector}"))?; // Read new sector into buffer - let (encrypted, has_hashes) = self.block.copy_sector( - sector_data, - self.block_buf.as_mut(), - abs_sector, - partition.disc_header(), - Some(partition), - )?; + let (encrypted, has_hashes) = self + .block + .copy_sector( + sector_data, + self.block_buf.as_mut(), + abs_sector, + partition.disc_header(), + Some(partition), + ) + .io_with_context(|| format!("Copying sector {abs_sector} from block"))?; if !encrypted { decrypted_sectors |= 1 << sector; } @@ -428,7 +478,9 @@ impl SectorGroupLoader { } // Recover hashes - if request.mode != PartitionEncryption::ForceDecryptedNoHashes && hash_recovery_sectors != 0 + if request.force_rehash + || (request.mode != PartitionEncryption::ForceDecryptedNoHashes + && hash_recovery_sectors != 0) { // Decrypt any encrypted sectors if decrypted_sectors != u64::MAX { @@ -443,16 +495,19 @@ impl SectorGroupLoader { } // Recover hashes - let group_hashes = hash_sector_group(sector_group_buf); + let hashes = hash_sector_group(sector_group_buf, request.force_rehash); // Apply hashes for sector in 0..64 { let sector_data = array_ref_mut![sector_group_buf, sector * SECTOR_SIZE, SECTOR_SIZE]; if (hash_recovery_sectors >> sector) & 1 == 1 { - group_hashes.apply(sector_data, sector); + hashes.apply(sector_data, sector); } } + + // Persist hashes + group_hashes = Some(Arc::from(hashes)); } // Apply hash exceptions @@ -508,7 +563,12 @@ impl SectorGroupLoader { } } - Ok((sector_bitmap, io_duration)) + Ok(LoadedSectorGroup { + start_sector: abs_group_sector, + sector_bitmap, + io_duration, + group_hashes, + }) } /// Loads a non-partition sector group. @@ -516,7 +576,7 @@ impl SectorGroupLoader { &mut self, request: SectorGroupRequest, sector_group_buf: &mut [u8; SECTOR_GROUP_SIZE], - ) -> io::Result<(u64, Option)> { + ) -> io::Result { let abs_group_sector = request.group_idx * 64; // Bitmap of sectors that were read @@ -534,7 +594,10 @@ impl SectorGroupLoader { // Read new block if !self.block.contains(abs_sector) { - self.block = self.io.read_block(self.block_buf.as_mut(), abs_sector)?; + self.block = self + .io + .read_block(self.block_buf.as_mut(), abs_sector) + .io_with_context(|| format!("Reading block for sector {abs_sector}"))?; if let Some(duration) = self.block.io_duration { *io_duration.get_or_insert_with(Duration::default) += duration; } @@ -544,17 +607,24 @@ impl SectorGroupLoader { } // Read new sector into buffer - self.block.copy_sector( - sector_data, - self.block_buf.as_mut(), - abs_sector, - self.disc_header.as_ref(), - None, - )?; + self.block + .copy_sector( + sector_data, + self.block_buf.as_mut(), + abs_sector, + self.disc_header.as_ref(), + None, + ) + .io_with_context(|| format!("Copying sector {abs_sector} from block"))?; sector_bitmap |= 1 << sector; } - Ok((sector_bitmap, io_duration)) + Ok(LoadedSectorGroup { + start_sector: abs_group_sector, + sector_bitmap, + io_duration, + group_hashes: None, + }) } } diff --git a/nod/src/disc/reader.rs b/nod/src/disc/reader.rs index e6a533c..4384550 100644 --- a/nod/src/disc/reader.rs +++ b/nod/src/disc/reader.rs @@ -5,6 +5,7 @@ use std::{ }; use bytes::Bytes; +use polonius_the_crab::{polonius, polonius_return}; use tracing::warn; use zerocopy::{FromBytes, IntoBytes}; @@ -21,13 +22,13 @@ use crate::{ PartitionReaderWii, WiiPartEntry, WiiPartGroup, WiiPartitionHeader, REGION_OFFSET, REGION_SIZE, WII_PART_GROUP_OFF, }, - DiscHeader, PartitionHeader, BOOT_SIZE, DL_DVD_SIZE, MINI_DVD_SIZE, SECTOR_GROUP_SIZE, - SECTOR_SIZE, SL_DVD_SIZE, + BootHeader, DiscHeader, BB2_OFFSET, BOOT_SIZE, DL_DVD_SIZE, MINI_DVD_SIZE, + SECTOR_GROUP_SIZE, SECTOR_SIZE, SL_DVD_SIZE, }, io::block::BlockReader, read::{DiscMeta, DiscOptions, PartitionEncryption, PartitionOptions, PartitionReader}, util::{ - impl_read_for_bufread, + array_ref, impl_read_for_bufread, read::{read_arc, read_from, read_vec}, }, Error, Result, ResultContext, @@ -210,15 +211,18 @@ impl DiscReader { } } + /// A reference to the disc's boot header (BB2) for GameCube discs. + /// For Wii discs, use the boot header from the appropriate [PartitionInfo]. #[inline] - pub fn partition_header(&self) -> Option<&PartitionHeader> { + pub fn boot_header(&self) -> Option<&BootHeader> { match &self.disc_data { DiscReaderData::GameCube { .. } => Some( - PartitionHeader::ref_from_bytes( - &self.raw_boot[size_of::() - ..size_of::() + size_of::()], - ) - .expect("Invalid partition header alignment"), + BootHeader::ref_from_bytes(array_ref![ + self.raw_boot, + BB2_OFFSET, + size_of::() + ]) + .expect("Invalid boot header alignment"), ), _ => None, } @@ -306,48 +310,62 @@ impl DiscReader { } } - pub fn fill_buf_internal(&mut self) -> io::Result { - if self.pos >= self.size { - return Ok(Bytes::new()); - } - - // Read from modified disc header - if self.pos < size_of::() as u64 { - if let Some(alt_disc_header) = &self.alt_disc_header { - return Ok(Bytes::copy_from_slice( - &alt_disc_header.as_bytes()[self.pos as usize..], - )); - } - } - - // Build sector group request - let abs_sector = (self.pos / SECTOR_SIZE as u64) as u32; - let (request, abs_group_sector, max_groups) = if let Some(partition) = + pub fn load_sector_group( + &mut self, + abs_sector: u32, + force_rehash: bool, + ) -> io::Result<(&SectorGroup, bool)> { + let (request, max_groups) = if let Some(partition) = self.orig_partitions().iter().find(|part| part.data_contains_sector(abs_sector)) { let group_idx = (abs_sector - partition.data_start_sector) / 64; - let abs_group_sector = partition.data_start_sector + group_idx * 64; let max_groups = (partition.data_end_sector - partition.data_start_sector).div_ceil(64); let request = SectorGroupRequest { group_idx, partition_idx: Some(partition.index as u8), mode: self.mode, + force_rehash, }; - (request, abs_group_sector, max_groups) + (request, max_groups) } else { let group_idx = abs_sector / 64; - let abs_group_sector = group_idx * 64; let max_groups = self.size.div_ceil(SECTOR_GROUP_SIZE as u64) as u32; - let request = SectorGroupRequest { group_idx, partition_idx: None, mode: self.mode }; - (request, abs_group_sector, max_groups) + let request = SectorGroupRequest { + group_idx, + partition_idx: None, + mode: self.mode, + force_rehash, + }; + (request, max_groups) }; // Load sector group - let (sector_group, _updated) = + let (sector_group, updated) = fetch_sector_group(request, max_groups, &mut self.sector_group, &self.preloader)?; + Ok((sector_group, updated)) + } + + pub fn fill_buf_internal(&mut self) -> io::Result { + let pos = self.pos; + let size = self.size; + if pos >= size { + return Ok(Bytes::new()); + } + + // Read from modified disc header + if pos < size_of::() as u64 { + if let Some(alt_disc_header) = &self.alt_disc_header { + return Ok(Bytes::copy_from_slice(&alt_disc_header.as_bytes()[pos as usize..])); + } + } + + // Load sector group + let abs_sector = (pos / SECTOR_SIZE as u64) as u32; + let (sector_group, _updated) = self.load_sector_group(abs_sector, false)?; + // Calculate the number of consecutive sectors in the group - let group_sector = abs_sector - abs_group_sector; + let group_sector = abs_sector - sector_group.start_sector; let consecutive_sectors = sector_group.consecutive_sectors(group_sector); if consecutive_sectors == 0 { return Ok(Bytes::new()); @@ -355,54 +373,37 @@ impl DiscReader { let num_sectors = group_sector + consecutive_sectors; // Read from sector group buffer - let group_start = abs_group_sector as u64 * SECTOR_SIZE as u64; - let offset = (self.pos - group_start) as usize; - let end = (num_sectors as u64 * SECTOR_SIZE as u64).min(self.size - group_start) as usize; + let group_start = sector_group.start_sector as u64 * SECTOR_SIZE as u64; + let offset = (pos - group_start) as usize; + let end = (num_sectors as u64 * SECTOR_SIZE as u64).min(size - group_start) as usize; Ok(sector_group.data.slice(offset..end)) } } impl BufRead for DiscReader { fn fill_buf(&mut self) -> io::Result<&[u8]> { - if self.pos >= self.size { + let pos = self.pos; + let size = self.size; + if pos >= size { return Ok(&[]); } - // Read from modified disc header - if self.pos < size_of::() as u64 { - if let Some(alt_disc_header) = &self.alt_disc_header { - return Ok(&alt_disc_header.as_bytes()[self.pos as usize..]); + let mut this = self; + polonius!(|this| -> io::Result<&'polonius [u8]> { + // Read from modified disc header + if pos < size_of::() as u64 { + if let Some(alt_disc_header) = &this.alt_disc_header { + polonius_return!(Ok(&alt_disc_header.as_bytes()[pos as usize..])); + } } - } - - // Build sector group request - let abs_sector = (self.pos / SECTOR_SIZE as u64) as u32; - let (request, abs_group_sector, max_groups) = if let Some(partition) = - self.orig_partitions().iter().find(|part| part.data_contains_sector(abs_sector)) - { - let group_idx = (abs_sector - partition.data_start_sector) / 64; - let abs_group_sector = partition.data_start_sector + group_idx * 64; - let max_groups = (partition.data_end_sector - partition.data_start_sector).div_ceil(64); - let request = SectorGroupRequest { - group_idx, - partition_idx: Some(partition.index as u8), - mode: self.mode, - }; - (request, abs_group_sector, max_groups) - } else { - let group_idx = abs_sector / 64; - let abs_group_sector = group_idx * 64; - let max_groups = self.size.div_ceil(SECTOR_GROUP_SIZE as u64) as u32; - let request = SectorGroupRequest { group_idx, partition_idx: None, mode: self.mode }; - (request, abs_group_sector, max_groups) - }; + }); // Load sector group - let (sector_group, _updated) = - fetch_sector_group(request, max_groups, &mut self.sector_group, &self.preloader)?; + let abs_sector = (pos / SECTOR_SIZE as u64) as u32; + let (sector_group, _updated) = this.load_sector_group(abs_sector, false)?; // Calculate the number of consecutive sectors in the group - let group_sector = abs_sector - abs_group_sector; + let group_sector = abs_sector - sector_group.start_sector; let consecutive_sectors = sector_group.consecutive_sectors(group_sector); if consecutive_sectors == 0 { return Ok(&[]); @@ -410,9 +411,9 @@ impl BufRead for DiscReader { let num_sectors = group_sector + consecutive_sectors; // Read from sector group buffer - let group_start = abs_group_sector as u64 * SECTOR_SIZE as u64; - let offset = (self.pos - group_start) as usize; - let end = (num_sectors as u64 * SECTOR_SIZE as u64).min(self.size - group_start) as usize; + let group_start = sector_group.start_sector as u64 * SECTOR_SIZE as u64; + let offset = (pos - group_start) as usize; + let end = (num_sectors as u64 * SECTOR_SIZE as u64).min(size - group_start) as usize; Ok(§or_group.data[offset..end]) } @@ -490,37 +491,43 @@ fn read_partition_info( data_start_sector, key, }); - let raw_boot: Arc<[u8; BOOT_SIZE]> = - read_arc(reader).context("Reading partition headers")?; + let raw_boot: Arc<[u8; BOOT_SIZE]> = read_arc(reader).context("Reading boot data")?; let partition_disc_header = - DiscHeader::ref_from_bytes(&raw_boot[..size_of::()]) + DiscHeader::ref_from_bytes(array_ref![raw_boot, 0, size_of::()]) .expect("Invalid disc header alignment"); - let partition_header = - PartitionHeader::ref_from_bytes(&raw_boot[size_of::()..]) - .expect("Invalid partition header alignment"); + let boot_header = BootHeader::ref_from_bytes(&raw_boot[BB2_OFFSET..]) + .expect("Invalid boot header alignment"); let raw_fst = if partition_disc_header.is_wii() { - let raw_fst = read_fst(reader, partition_header, true)?; - let fst = Fst::new(&raw_fst)?; - let max_fst_offset = fst - .nodes - .iter() - .filter_map(|n| match n.kind() { - NodeKind::File => Some(n.offset(true) + n.length() as u64), - _ => None, - }) - .max() - .unwrap_or(0); - if max_fst_offset > data_size { - if data_size == 0 { - // Guess data size for decrypted partitions - data_end_sector = max_fst_offset.div_ceil(SECTOR_SIZE as u64) as u32; - } else { - return Err(Error::DiscFormat(format!( - "Partition {group_idx}:{part_idx} FST exceeds data size", - ))); + let raw_fst = read_fst(reader, boot_header, true)?; + match Fst::new(&raw_fst) { + Ok(fst) => { + let max_fst_offset = fst + .nodes + .iter() + .filter_map(|n| match n.kind() { + NodeKind::File => Some(n.offset(true) + n.length() as u64), + _ => None, + }) + .max() + .unwrap_or(0); + if max_fst_offset > data_size { + if data_size == 0 { + // Guess data size for decrypted partitions + data_end_sector = + max_fst_offset.div_ceil(SECTOR_SIZE as u64) as u32; + } else { + return Err(Error::DiscFormat(format!( + "Partition {group_idx}:{part_idx} FST exceeds data size", + ))); + } + } + Some(raw_fst) + } + Err(e) => { + warn!("Partition {group_idx}:{part_idx} FST is not valid: {e}"); + None } } - Some(raw_fst) } else { warn!("Partition {group_idx}:{part_idx} is not valid"); None diff --git a/nod/src/disc/wii.rs b/nod/src/disc/wii.rs index 8d34808..643e131 100644 --- a/nod/src/disc/wii.rs +++ b/nod/src/disc/wii.rs @@ -378,6 +378,7 @@ impl BufRead for PartitionReaderWii { } else { PartitionEncryption::ForceDecryptedNoHashes }, + force_rehash: false, }; // Load sector group diff --git a/nod/src/io/block.rs b/nod/src/io/block.rs index 336cc06..171b826 100644 --- a/nod/src/io/block.rs +++ b/nod/src/io/block.rs @@ -107,7 +107,8 @@ pub const WBFS_MAGIC: MagicBytes = *b"WBFS"; pub const WIA_MAGIC: MagicBytes = *b"WIA\x01"; pub const RVZ_MAGIC: MagicBytes = *b"RVZ\x01"; -pub fn detect(stream: &mut R) -> io::Result> { +pub fn detect(stream: &mut R) -> io::Result> +where R: Read + ?Sized { let data: [u8; 0x20] = match read_from(stream) { Ok(magic) => magic, Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None), diff --git a/nod/src/io/tgc.rs b/nod/src/io/tgc.rs index 822237c..958cd61 100644 --- a/nod/src/io/tgc.rs +++ b/nod/src/io/tgc.rs @@ -15,14 +15,14 @@ use crate::{ gcn::{read_dol, read_fst}, reader::DiscReader, writer::{DataCallback, DiscWriter}, - DiscHeader, PartitionHeader, SECTOR_SIZE, + BootHeader, DiscHeader, BB2_OFFSET, SECTOR_SIZE, }, io::block::{Block, BlockKind, BlockReader, TGC_MAGIC}, read::{DiscMeta, DiscStream, PartitionOptions, PartitionReader}, util::{ - align_up_32, array_ref, + array_ref, read::{read_arc, read_arc_slice, read_from, read_with_zero_fill}, - static_assert, + static_assert, Align, }, write::{DiscFinalization, DiscWriterWeight, FormatOptions, ProcessOptions}, Error, Result, ResultContext, @@ -82,19 +82,21 @@ impl BlockReaderTGC { } let disc_size = (header.gcm_files_start.get() + header.user_size.get()) as u64; - // Read disc header and partition header + // Read GCM header inner .seek(SeekFrom::Start(header.header_offset.get() as u64)) .context("Seeking to GCM header")?; let raw_header = read_arc::<[u8; GCM_HEADER_SIZE], _>(inner.as_mut()).context("Reading GCM header")?; - let (disc_header, remain) = DiscHeader::ref_from_prefix(raw_header.as_ref()) - .expect("Invalid disc header alignment"); + let disc_header = + DiscHeader::ref_from_bytes(array_ref![raw_header, 0, size_of::()]) + .expect("Invalid disc header alignment"); let disc_header = disc_header.clone(); - let (partition_header, _) = - PartitionHeader::ref_from_prefix(remain).expect("Invalid partition header alignment"); - let partition_header = partition_header.clone(); + let boot_header = + BootHeader::ref_from_bytes(array_ref![raw_header, BB2_OFFSET, size_of::()]) + .expect("Invalid boot header alignment"); + let boot_header = boot_header.clone(); // Read DOL inner.seek(SeekFrom::Start(header.dol_offset.get() as u64)).context("Seeking to DOL")?; @@ -116,12 +118,12 @@ impl BlockReaderTGC { write_info.push(WriteInfo { kind: WriteKind::Static(raw_dol, "sys/main.dol"), size: header.dol_size.get() as u64, - offset: partition_header.dol_offset(false), + offset: boot_header.dol_offset(false), }); write_info.push(WriteInfo { kind: WriteKind::Static(raw_fst.clone(), "sys/fst.bin"), size: header.fst_size.get() as u64, - offset: partition_header.fst_offset(false), + offset: boot_header.fst_offset(false), }); // Collect files @@ -136,7 +138,7 @@ impl BlockReaderTGC { }); } write_info.sort_unstable_by(|a, b| a.offset.cmp(&b.offset).then(a.size.cmp(&b.size))); - let write_info = insert_junk_data(write_info, &partition_header); + let write_info = insert_junk_data(write_info, &boot_header); let file_callback = FileCallbackTGC::new(inner, raw_fst, header); let disc_id = *array_ref![disc_header.game_id, 0, 4]; @@ -225,14 +227,13 @@ impl DiscWriterTGC { // Read GCM header let mut raw_header = <[u8; GCM_HEADER_SIZE]>::new_box_zeroed()?; inner.read_exact(raw_header.as_mut()).context("Reading GCM header")?; - let (_, remain) = DiscHeader::ref_from_prefix(raw_header.as_ref()) - .expect("Invalid disc header alignment"); - let (partition_header, _) = - PartitionHeader::ref_from_prefix(remain).expect("Invalid partition header alignment"); + let boot_header = + BootHeader::ref_from_bytes(array_ref![raw_header, BB2_OFFSET, size_of::()]) + .expect("Invalid boot header alignment"); // Read DOL - let raw_dol = read_dol(inner.as_mut(), partition_header, false)?; - let raw_fst = read_fst(inner.as_mut(), partition_header, false)?; + let raw_dol = read_dol(inner.as_mut(), boot_header, false)?; + let raw_fst = read_fst(inner.as_mut(), boot_header, false)?; // Parse FST let fst = Fst::new(&raw_fst)?; @@ -249,11 +250,10 @@ impl DiscWriterTGC { // Layout system files let gcm_header_offset = SECTOR_SIZE as u32; let fst_offset = gcm_header_offset + GCM_HEADER_SIZE as u32; - let dol_offset = align_up_32(fst_offset + partition_header.fst_size.get(), 32); + let dol_offset = (fst_offset + boot_header.fst_size.get()).align_up(32); let user_size = - partition_header.user_offset.get() + partition_header.user_size.get() - gcm_files_start; - let user_end = - align_up_32(dol_offset + raw_dol.len() as u32 + user_size, SECTOR_SIZE as u32); + boot_header.user_offset.get() + boot_header.user_size.get() - gcm_files_start; + let user_end = (dol_offset + raw_dol.len() as u32 + user_size).align_up(SECTOR_SIZE as u32); let user_offset = user_end - user_size; let header = TGCHeader { @@ -262,8 +262,8 @@ impl DiscWriterTGC { header_offset: gcm_header_offset.into(), header_size: (GCM_HEADER_SIZE as u32).into(), fst_offset: fst_offset.into(), - fst_size: partition_header.fst_size, - fst_max_size: partition_header.fst_max_size, + fst_size: boot_header.fst_size, + fst_max_size: boot_header.fst_max_size, dol_offset: dol_offset.into(), dol_size: (raw_dol.len() as u32).into(), user_offset: user_offset.into(), diff --git a/nod/src/io/wia.rs b/nod/src/io/wia.rs index 7c42a48..37be8fe 100644 --- a/nod/src/io/wia.rs +++ b/nod/src/io/wia.rs @@ -19,7 +19,7 @@ use crate::{ reader::DiscReader, wii::{HASHES_SIZE, SECTOR_DATA_SIZE}, writer::{par_process, read_block, BlockProcessor, BlockResult, DataCallback, DiscWriter}, - DiscHeader, PartitionHeader, SECTOR_SIZE, + BootHeader, DiscHeader, SECTOR_SIZE, }, io::{ block::{Block, BlockKind, BlockReader, RVZ_MAGIC, WIA_MAGIC}, @@ -28,15 +28,15 @@ use crate::{ read::{DiscMeta, DiscStream}, util::{ aes::decrypt_sector_data_b2b, - align_up_32, align_up_64, array_ref, array_ref_mut, + array_ref, array_ref_mut, compress::{Compressor, DecompressionKind, Decompressor}, digest::{sha1_hash, xxh64_hash, DigestManager}, lfg::{LaggedFibonacci, SEED_SIZE, SEED_SIZE_BYTES}, read::{read_arc_slice, read_from, read_vec}, - static_assert, + static_assert, Align, }, write::{DiscFinalization, DiscWriterWeight, FormatOptions, ProcessOptions}, - Error, Result, ResultContext, + Error, IoResultContext, Result, ResultContext, }; const WIA_VERSION: u32 = 0x01000000; @@ -395,7 +395,7 @@ pub struct WIARawData { } impl WIARawData { - pub fn start_offset(&self) -> u64 { self.raw_data_offset.get() & !(SECTOR_SIZE as u64 - 1) } + pub fn start_offset(&self) -> u64 { self.raw_data_offset.get().align_down(SECTOR_SIZE as u64) } pub fn start_sector(&self) -> u32 { (self.start_offset() / SECTOR_SIZE as u64) as u32 } @@ -457,19 +457,21 @@ pub struct RVZGroup { pub rvz_packed_size: U32, } +const COMPRESSED_BIT: u32 = 1 << 31; + impl RVZGroup { #[inline] - pub fn data_size(&self) -> u32 { self.data_size_and_flag.get() & 0x7FFFFFFF } + pub fn data_size(&self) -> u32 { self.data_size_and_flag.get() & !COMPRESSED_BIT } #[inline] - pub fn is_compressed(&self) -> bool { self.data_size_and_flag.get() & 0x80000000 != 0 } + pub fn is_compressed(&self) -> bool { self.data_size_and_flag.get() & COMPRESSED_BIT != 0 } } impl From<&WIAGroup> for RVZGroup { fn from(value: &WIAGroup) -> Self { Self { data_offset: value.data_offset, - data_size_and_flag: U32::new(value.data_size.get() | 0x80000000), + data_size_and_flag: U32::new(value.data_size.get() | COMPRESSED_BIT), rvz_packed_size: U32::new(0), } } @@ -886,23 +888,40 @@ impl BlockReader for BlockReaderWIA { || !group.is_compressed(); let mut exception_lists = vec![]; if info.in_partition && uncompressed_exception_lists { - exception_lists = read_exception_lists(&mut group_data, chunk_size, true)?; + exception_lists = read_exception_lists(&mut group_data, chunk_size, true) + .io_context("Reading uncompressed exception lists")?; } let mut decompressed = if group.is_compressed() { - let mut decompressed = BytesMut::zeroed(chunk_size as usize); - let len = self.decompressor.decompress(group_data.as_ref(), decompressed.as_mut())?; + let max_decompressed_size = + self.decompressor.get_content_size(group_data.as_ref())?.unwrap_or_else(|| { + if info.in_partition && !uncompressed_exception_lists { + // Add room for potential hash exceptions. See [WIAExceptionList] for details on the + // maximum number of exceptions. + chunk_size as usize + + (2 + 3328 * size_of::()) + * (chunk_size as usize).div_ceil(0x200000) + } else { + chunk_size as usize + } + }); + let mut decompressed = BytesMut::zeroed(max_decompressed_size); + let len = self + .decompressor + .decompress(group_data.as_ref(), decompressed.as_mut()) + .io_context("Decompressing group data")?; decompressed.truncate(len); decompressed.freeze() } else { group_data }; if info.in_partition && !uncompressed_exception_lists { - exception_lists = read_exception_lists(&mut decompressed, chunk_size, false)?; + exception_lists = read_exception_lists(&mut decompressed, chunk_size, false) + .io_context("Reading compressed exception lists")?; } if group.rvz_packed_size.get() > 0 { // Decode RVZ packed data - rvz_unpack(&mut decompressed, out, &info)?; + rvz_unpack(&mut decompressed, out, &info).io_context("Unpacking RVZ group data")?; } else { // Read and decompress data if decompressed.remaining() != info.size as usize { @@ -975,9 +994,9 @@ fn rvz_unpack(data: &mut impl Buf, out: &mut [u8], info: &GroupInfo) -> io::Resu while data.remaining() >= 4 { let size = data.get_u32(); let remain = out.len() - read; - if size & 0x80000000 != 0 { + if size & COMPRESSED_BIT != 0 { // Junk data - let size = size & 0x7FFFFFFF; + let size = size & !COMPRESSED_BIT; if size as usize > remain { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -1057,14 +1076,13 @@ impl JunkInfo { start_sector: u32, end_sector: u32, disc_header: &DiscHeader, - partition_header: Option<&PartitionHeader>, + boot_header: Option<&BootHeader>, fst: Option, ) -> Self { let is_wii = disc_header.is_wii(); let mut file_ends = BTreeSet::new(); - if let Some(partition_header) = partition_header { - file_ends - .insert(partition_header.fst_offset(is_wii) + partition_header.fst_size(is_wii)); + if let Some(boot_header) = boot_header { + file_ends.insert(boot_header.fst_offset(is_wii) + boot_header.fst_size(is_wii)); } if let Some(fst) = fst { for entry in fst.nodes.iter().filter(|n| n.is_file()) { @@ -1145,7 +1163,7 @@ impl BlockProcessor for BlockProcessorWIA { }; let uncompressed_size = - align_up_32(hash_exception_data.len() as u32, 4) + group_data.len() as u32; + (hash_exception_data.len() as u32).align_up(4) + group_data.len() as u32; if hash_exception_data.as_ref().iter().all(|&b| b == 0) && group_data.as_ref().iter().all(|&b| b == 0) { @@ -1167,7 +1185,7 @@ impl BlockProcessor for BlockProcessorWIA { if is_rvz { if let Some(packed_data) = self.try_rvz_pack(group_data.as_ref(), &info) { meta.data_size = - align_up_32(hash_exception_data.len() as u32, 4) + packed_data.len() as u32; + (hash_exception_data.len() as u32).align_up(4) + packed_data.len() as u32; meta.rvz_packed_size = packed_data.len() as u32; group_data = packed_data; } @@ -1185,9 +1203,9 @@ impl BlockProcessor for BlockProcessorWIA { let compressed_size = self.compressor.buffer.len() as u32; // For WIA, we must always store compressed data. // For RVZ, only store compressed data if it's smaller than uncompressed. - if !is_rvz || align_up_32(compressed_size, 4) < meta.data_size { + if !is_rvz || compressed_size.align_up(4) < meta.data_size { // Align resulting block data to 4 bytes - let mut buf = BytesMut::zeroed(align_up_32(compressed_size, 4) as usize); + let mut buf = BytesMut::zeroed(compressed_size.align_up(4) as usize); buf[..compressed_size as usize].copy_from_slice(&self.compressor.buffer); meta.is_compressed = true; // Data size does not include end alignment @@ -1214,9 +1232,9 @@ impl BlockProcessor for BlockProcessorWIA { } // Store uncompressed group, aligned to 4 bytes after hash exceptions and at the end - let mut buf = BytesMut::zeroed(align_up_32(meta.data_size, 4) as usize); + let mut buf = BytesMut::zeroed(meta.data_size.align_up(4) as usize); buf[..hash_exception_data.len()].copy_from_slice(hash_exception_data.as_ref()); - let offset = align_up_32(hash_exception_data.len() as u32, 4) as usize; + let offset = (hash_exception_data.len() as u32).align_up(4) as usize; buf[offset..offset + group_data.len()].copy_from_slice(group_data.as_ref()); meta.data_hash = xxh64_hash(buf.as_ref()); Ok(BlockResult { block_idx: group_idx, disc_data, block_data: buf.freeze(), meta }) @@ -1336,7 +1354,7 @@ impl BlockProcessorWIA { sector, ); } - *array_ref_mut![out, offset, 4] = (len as u32 | 0x80000000).to_be_bytes(); + *array_ref_mut![out, offset, 4] = (len as u32 | COMPRESSED_BIT).to_be_bytes(); array_ref_mut![out, offset + 4, SEED_SIZE_BYTES].copy_from_slice(seed_out.as_bytes()); } @@ -1528,13 +1546,11 @@ impl DiscWriterWIA { let partition_data_start = partition.data_start_sector as u64 * SECTOR_SIZE as u64; let partition_end = partition.data_end_sector as u64 * SECTOR_SIZE as u64; - let partition_header = partition.partition_header(); - let management_data_end = align_up_64( - partition_offset_to_raw( - partition_header.fst_offset(true) + partition_header.fst_size(true), - ), - 0x200000, // Align to 2 MiB - ); + let boot_header = partition.boot_header(); + let management_data_end = + partition_offset_to_raw(boot_header.fst_offset(true) + boot_header.fst_size(true)) + // Align to 2 MiB + .align_up(0x200000); let management_end_sector = ((partition_data_start + management_data_end) .min(partition_end) / SECTOR_SIZE as u64) as u32; @@ -1629,7 +1645,7 @@ impl DiscWriterWIA { partition.data_start_sector, partition.data_end_sector, partition.disc_header(), - Some(partition.partition_header()), + Some(partition.boot_header()), partition.fst(), )); } @@ -1637,7 +1653,7 @@ impl DiscWriterWIA { 0, disc_size.div_ceil(SECTOR_SIZE as u64) as u32, disc_header, - inner.partition_header(), + inner.boot_header(), inner.fst(), )); @@ -1713,7 +1729,7 @@ impl DiscWriter for DiscWriterWIA { groups[group_idx as usize] = RVZGroup { data_offset: data_offset.into(), data_size_and_flag: (group.meta.data_size - | if group.meta.is_compressed { 0x80000000 } else { 0 }) + | if group.meta.is_compressed { COMPRESSED_BIT } else { 0 }) .into(), rvz_packed_size: group.meta.rvz_packed_size.into(), }; diff --git a/nod/src/lib.rs b/nod/src/lib.rs index 05deea0..ba750c1 100644 --- a/nod/src/lib.rs +++ b/nod/src/lib.rs @@ -160,7 +160,7 @@ pub enum Error { #[error("disc format error: {0}")] DiscFormat(String), /// A general I/O error. - #[error("I/O error: {0}")] + #[error("{0}")] Io(String, #[source] std::io::Error), /// An unknown error. #[error("error: {0}")] @@ -225,3 +225,34 @@ where E: ErrorContext self.map_err(|e| e.context(f())) } } + +pub(crate) trait IoErrorContext { + fn io_context(self, context: impl Into) -> std::io::Error; +} + +impl IoErrorContext for std::io::Error { + #[inline] + fn io_context(self, context: impl Into) -> std::io::Error { + std::io::Error::new(self.kind(), self.context(context)) + } +} + +pub(crate) trait IoResultContext { + fn io_context(self, context: impl Into) -> std::io::Result; + + fn io_with_context(self, f: F) -> std::io::Result + where F: FnOnce() -> String; +} + +impl IoResultContext for std::io::Result { + #[inline] + fn io_context(self, context: impl Into) -> std::io::Result { + self.map_err(|e| e.io_context(context)) + } + + #[inline] + fn io_with_context(self, f: F) -> std::io::Result + where F: FnOnce() -> String { + self.map_err(|e| e.io_context(f())) + } +} diff --git a/nod/src/read.rs b/nod/src/read.rs index 1aa14bc..ee1cea5 100644 --- a/nod/src/read.rs +++ b/nod/src/read.rs @@ -14,10 +14,11 @@ use crate::{ disc::{ fst::{Fst, Node}, wii::{ContentMetadata, Ticket, TmdHeader, H3_TABLE_SIZE, REGION_SIZE}, - ApploaderHeader, DiscHeader, DolHeader, PartitionHeader, BI2_SIZE, BOOT_SIZE, + ApploaderHeader, BootHeader, DebugHeader, DiscHeader, DolHeader, BB2_OFFSET, BI2_SIZE, + BOOT_SIZE, }, io::block, - util::WindowedReader, + util::{array_ref, WindowedReader}, Result, }; @@ -198,7 +199,7 @@ pub struct DiscMeta { } /// An open disc partition. -pub trait PartitionReader: DynClone + BufRead + Seek + Send + Sync { +pub trait PartitionReader: BufRead + DiscStream { /// Whether this is a Wii partition. (GameCube otherwise) fn is_wii(&self) -> bool; @@ -306,7 +307,7 @@ dyn_clone::clone_trait_object!(PartitionReader); /// Extra disc partition data. (DOL, FST, etc.) #[derive(Clone, Debug)] pub struct PartitionMeta { - /// Disc and partition header (boot.bin) + /// Disc and boot header (boot.bin) pub raw_boot: Arc<[u8; BOOT_SIZE]>, /// Debug and region information (bi2.bin) pub raw_bi2: Arc<[u8; BI2_SIZE]>, @@ -329,16 +330,27 @@ pub struct PartitionMeta { impl PartitionMeta { /// A view into the disc header. #[inline] - pub fn header(&self) -> &DiscHeader { - DiscHeader::ref_from_bytes(&self.raw_boot[..size_of::()]) - .expect("Invalid header alignment") + pub fn disc_header(&self) -> &DiscHeader { + DiscHeader::ref_from_bytes(array_ref![self.raw_boot, 0, size_of::()]) + .expect("Invalid disc header alignment") } - /// A view into the partition header. + /// A view into the debug header. #[inline] - pub fn partition_header(&self) -> &PartitionHeader { - PartitionHeader::ref_from_bytes(&self.raw_boot[size_of::()..]) - .expect("Invalid partition header alignment") + pub fn debug_header(&self) -> &DebugHeader { + DebugHeader::ref_from_bytes(array_ref![ + self.raw_boot, + size_of::(), + size_of::() + ]) + .expect("Invalid debug header alignment") + } + + /// A view into the boot header. + #[inline] + pub fn boot_header(&self) -> &BootHeader { + BootHeader::ref_from_bytes(array_ref![self.raw_boot, BB2_OFFSET, size_of::()]) + .expect("Invalid boot header alignment") } /// A view into the apploader header. diff --git a/nod/src/util/compress.rs b/nod/src/util/compress.rs index 7763112..428d4b8 100644 --- a/nod/src/util/compress.rs +++ b/nod/src/util/compress.rs @@ -10,6 +10,7 @@ use crate::{ pub struct Decompressor { pub kind: DecompressionKind, + #[allow(unused)] // if compression features are disabled pub cache: DecompressorCache, } @@ -38,7 +39,13 @@ impl Decompressor { pub fn decompress(&mut self, buf: &[u8], out: &mut [u8]) -> io::Result { match &self.kind { DecompressionKind::None => { - out.copy_from_slice(buf); + if buf.len() > out.len() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Decompressed data too large: {} > {}", buf.len(), out.len()), + )); + } + out[..buf.len()].copy_from_slice(buf); Ok(buf.len()) } #[cfg(feature = "compress-zlib")] @@ -130,6 +137,18 @@ impl Decompressor { } } } + + pub fn get_content_size(&self, buf: &[u8]) -> io::Result> { + match &self.kind { + DecompressionKind::None => Ok(Some(buf.len())), + #[cfg(feature = "compress-zstd")] + DecompressionKind::Zstandard => zstd_safe::get_frame_content_size(buf) + .map(|n| n.map(|n| n as usize)) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e.to_string())), + #[allow(unreachable_patterns)] // if compression features are disabled + _ => Ok(None), + } + } } #[derive(Debug, Clone)] @@ -227,6 +246,14 @@ impl Compressor { pub fn compress(&mut self, buf: &[u8]) -> io::Result { self.buffer.clear(); match self.kind { + Compression::None => { + if self.buffer.capacity() >= buf.len() { + self.buffer.extend_from_slice(buf); + Ok(true) + } else { + Ok(false) + } + } #[cfg(feature = "compress-zlib")] Compression::Deflate(level) => { let compressor = match &mut self.cache { @@ -288,6 +315,8 @@ impl Compressor { _ => { let mut ctx = zstd_safe::CCtx::create(); ctx.init(level as i32).map_err(zstd_util::map_error_code)?; + ctx.set_parameter(zstd_safe::CParameter::ContentSizeFlag(true)) + .map_err(zstd_util::map_error_code)?; self.cache = CompressorCache::Zstandard(ctx); match &mut self.cache { CompressorCache::Zstandard(compressor) => compressor, @@ -302,6 +331,7 @@ impl Compressor { Err(e) => Err(zstd_util::map_error_code(e)), } } + #[allow(unreachable_patterns)] // if compression is disabled _ => Err(io::Error::new( io::ErrorKind::Other, format!("Unsupported compression: {:?}", self.kind), diff --git a/nod/src/util/mod.rs b/nod/src/util/mod.rs index 9610905..3971f17 100644 --- a/nod/src/util/mod.rs +++ b/nod/src/util/mod.rs @@ -127,18 +127,36 @@ where T: Div + Rem + Copy { (x / y, x % y) } -#[inline] -pub(crate) fn align_up_32(n: u32, align: u32) -> u32 { (n + align - 1) & !(align - 1) } +pub(crate) trait Align { + fn align_up(self, align: Self) -> Self; -#[inline] -pub(crate) fn align_up_64(n: u64, align: u64) -> u64 { (n + align - 1) & !(align - 1) } + fn align_down(self, align: Self) -> Self; +} + +macro_rules! impl_align { + ($ty:ident) => { + impl Align for $ty { + #[inline(always)] + fn align_up(self, align: Self) -> Self { (self + (align - 1)) & !(align - 1) } + + #[inline(always)] + fn align_down(self, align: Self) -> Self { self & !(align - 1) } + } + }; +} + +impl_align!(u8); +impl_align!(u16); +impl_align!(u32); +impl_align!(u64); +impl_align!(usize); /// Creates a fixed-size array reference from a slice. macro_rules! array_ref { ($slice:expr, $offset:expr, $size:expr) => {{ #[inline(always)] fn to_array(slice: &[T]) -> &[T; $size] { - unsafe { &*(slice.as_ptr() as *const [_; $size]) } + unsafe { &*(slice as *const [T] as *const [T; $size]) } } to_array(&$slice[$offset..$offset + $size]) }}; @@ -150,7 +168,7 @@ macro_rules! array_ref_mut { ($slice:expr, $offset:expr, $size:expr) => {{ #[inline(always)] fn to_array(slice: &mut [T]) -> &mut [T; $size] { - unsafe { &mut *(slice.as_ptr() as *mut [_; $size]) } + unsafe { &mut *(slice as *mut [T] as *mut [T; $size]) } } to_array(&mut $slice[$offset..$offset + $size]) }}; diff --git a/nodtool/Cargo.toml b/nodtool/Cargo.toml index 2b85a28..4396dd0 100644 --- a/nodtool/Cargo.toml +++ b/nodtool/Cargo.toml @@ -16,6 +16,11 @@ categories = ["command-line-utilities", "parser-implementations"] build = "build.rs" [features] +default = ["compress-bzip2", "compress-lzma", "compress-zlib", "compress-zstd"] +compress-bzip2 = ["nod/compress-bzip2"] +compress-lzma = ["nod/compress-lzma"] +compress-zlib = ["nod/compress-zlib"] +compress-zstd = ["nod/compress-zstd"] openssl = ["nod/openssl"] openssl-vendored = ["nod/openssl-vendored"] tracy = ["dep:tracing-tracy"] @@ -28,7 +33,7 @@ enable-ansi-support = "0.2" hex = { version = "0.4", features = ["serde"] } indicatif = "0.17" md-5 = { workspace = true } -nod = { version = "2.0.0-alpha", path = "../nod" } +nod = { version = "2.0.0-alpha", path = "../nod", default-features = false } num_cpus = "1.16" quick-xml = { version = "0.37", features = ["serialize"] } serde = { version = "1.0", features = ["derive"] } diff --git a/nodtool/assets/gc-non-redump.dat b/nodtool/assets/gc-non-redump.dat index 8e1c2ae..2f22542 100644 --- a/nodtool/assets/gc-non-redump.dat +++ b/nodtool/assets/gc-non-redump.dat @@ -5,7 +5,7 @@ Non-Redump - Nintendo - Nintendo GameCube Non-Redump - Nintendo - Nintendo GameCube Non-Redump - 20240602-041318 + 20240904-155318 bikerspade, Gefflon, Hiccup, NovaAurora, rarenight, relax, Seventy7, togemet2 No-Intro https://www.no-intro.org @@ -94,12 +94,6 @@ Major League Baseball 2K6 (USA) (Beta) (2006-05-18) - - Games - Preproduction - Mega Man X - Command Mission (USA) (Beta) - - Metal Gear Solid - The Twin Snakes (USA) (Disc 2) (Beta) diff --git a/nodtool/assets/gc-redump.dat b/nodtool/assets/gc-redump.dat index be3258e..8d64d17 100644 --- a/nodtool/assets/gc-redump.dat +++ b/nodtool/assets/gc-redump.dat @@ -3,9 +3,9 @@
Nintendo - GameCube - Nintendo - GameCube - Discs (1992) (2024-06-02 00-38-06) - 2024-06-02 00-38-06 - 2024-06-02 00-38-06 + Nintendo - GameCube - Discs (2001) (2024-12-01 23-54-46) + 2024-12-01 23-54-46 + 2024-12-01 23-54-46 redump.org redump.org http://redump.org/ @@ -2085,10 +2085,10 @@ Nickelodeon Tak 2 - The Staff of Dreams (USA) - + Demos - Nintendo GameCube Preview Disc - May 2003 (USA) - + Nintendo GameCube Preview Disc - May 2003 (USA, Canada) + Demos @@ -2395,10 +2395,10 @@ Need for Speed - Underground (USA) - + Games - Nickelodeon SpongeBob SquarePants - The Movie (USA) (Rev 1) - + Nickelodeon The SpongeBob SquarePants Movie (USA) (Rev 1) + Games @@ -2760,15 +2760,15 @@ Sonic Adventure DX - Director's Cut (Europe) (En,Ja,Fr,De,Es) (Rev 1) - + Games - GoldenEye - Rogue Agent (France) (Disc 1) - + GoldenEye - Au Service du Mal (France) (Disc 1) + - + Games - GoldenEye - Rogue Agent (France) (Disc 2) - + GoldenEye - Au Service du Mal (France) (Disc 2) + Games @@ -3265,10 +3265,10 @@ Mega Man X - Command Mission (USA) - + Games - Skies of Arcadia Legends (USA) - + Skies of Arcadia - Legends (USA) + Games @@ -4380,10 +4380,10 @@ World Soccer Winning Eleven 6 - Final Evolution (Japan) - + Games - Skies of Arcadia Legends (Europe) (En,Fr,De,Es) - + Skies of Arcadia - Legends (Europe) (En,Fr,De,Es) + Games @@ -6555,10 +6555,10 @@ 2002 FIFA World Cup (Europe) (Fr,Nl) - + Games - Nickelodeon SpongeBob SquarePants - The Movie (Europe) (Fr,Nl) - + Nickelodeon The SpongeBob SquarePants Movie (Europe) (Fr,Nl) + Games @@ -7070,10 +7070,10 @@ Nickelodeon SpongeBob SquarePants - Creature from the Krusty Krab (Europe) - + Games - Nickelodeon SpongeBob SquarePants - The Movie (Europe) - + Nickelodeon The SpongeBob SquarePants Movie (Europe) + Games @@ -7205,10 +7205,10 @@ Gekkan Nintendo Tentou Demo 2002.7.1 (Japan) - + Demos - Gekkan Nintendo Tentou Demo 2002.7.10 (Japan) - + Gekkan Nintendo Tentou Demo 2002.7.10 Zoukan-gou (Japan) + Demos @@ -7725,10 +7725,10 @@ Family Stadium 2003 (Japan) - + Games - Gakuen Toshi Vara Noir Roses (Japan) - + Gakuen Toshi Varanoir - Roses (Japan) + Games @@ -8660,10 +8660,10 @@ SSX on Tour (Europe) (En,Fr,De) - + Games - 2 Games in 1 - Bob L'eponge - Le Film + Tak 2 - Le Sceptre des Reves (France) (Disc 1) (Bob L'eponge - Le Film) - + 2 Games in 1 - Bob L'eponge - Le Film + Tak 2 - Le Sceptre des Reves (France) (Fr,Nl) (Disc 1) (Bob L'eponge - Le Film) + Games @@ -9325,10 +9325,10 @@ Disney-Pixar Nemo-reul Chajaseo (Korea) - + Applications - Action Replay for GameCube (USA) (En,Fr,De,Es,It,Pt) (Unl) (v1.08) - + Action Replay for GameCube (USA, Europe) (En,Fr,De,Es,It,Pt) (Unl) (v1.08) + Games @@ -9480,10 +9480,10 @@ Cube CD 14 (33) (UK) (Unl) - + Applications - Action Replay Max (Europe) (En,Fr,De,Es,It,Pt) (Unl) - + Action Replay Max (USA, Europe) (En,Fr,De,Es,It,Pt) (Unl) + Demos @@ -9880,10 +9880,10 @@ FIFA Soccer 07 (Latin America) - + Preproduction - Pickles (USA) (Proto) - + Pickles (USA) (Proto 1) + Preproduction @@ -9970,4 +9970,49 @@ Evolution Worlds (USA) (Beta) + + Preproduction + Pickles (USA) (Proto 2) + + + + Applications + Advance Connector GC-you (Japan) (Unl) + + + + Games + Xeno Crisis (Japan) (En,Ja,Fr,De,Es,It,Nl,Pt) (Unl) + + + + Demos + Gekkan Nintendo Tentou Demo 2002.8.1 (Japan) + + + + Games + NBA Live 2003 (France) + + + + Applications + Advance Game Port (USA) (Unl) (Rev 2) + + + + Applications + SD Media Launcher for GameCube & Wii (Japan) (Unl) + + + + Preproduction + Mega Man X - Command Mission (USA) (Beta) (2004-07-23) + + + + Demos + Mario Party 4 Event-you Disc (Japan) + + diff --git a/nodtool/assets/wii-redump.dat b/nodtool/assets/wii-redump.dat index f31edb0..93e3921 100644 --- a/nodtool/assets/wii-redump.dat +++ b/nodtool/assets/wii-redump.dat @@ -3,9 +3,9 @@
Nintendo - Wii - Nintendo - Wii - Discs (3770) (2024-02-13 21-52-18) - 2024-02-13 21-52-18 - 2024-02-13 21-52-18 + Nintendo - Wii - Discs (3775) (2024-11-29 20-05-00) + 2024-11-29 20-05-00 + 2024-11-29 20-05-00 redump.org redump.org http://redump.org/ @@ -400,10 +400,10 @@ Wii Music (USA) (En,Fr,Es) - + Games - Trauma Center - New Blood (USA) - + Trauma Center - New Blood (USA) (En,Ja) + Games @@ -13390,10 +13390,10 @@ Nickelodeon Dora the Explorer - Dora Saves the Snow Princess (Europe) (En,Fr,Nl) - + Games - Food Network - Cook or Be Cooked (USA) - + Food Network - Cook or Be Cooked! (USA) + Games @@ -17300,10 +17300,10 @@ Horrid Henry - Missions of Mischief (Europe) (En,Fr,De,Es,It) - + Games - I'm a Celebrity...Get Me Out of Here! (Europe) - + I'm a Celebrity...Get Me Out of Here! (UK) + Games @@ -17945,10 +17945,10 @@ Transformers - The Game (Korea) - + Applications - FreeLoader for Nintendo Wii (USA) (Unl) - + FreeLoader for Nintendo Wii (USA) (Unl) (Rev 1) + Applications @@ -18450,10 +18450,10 @@ Fishing Master World Tour (USA) (Beta) (2008-11-14) - + Preproduction - Food Network - Cook or Be Cooked (USA) (Beta) (2009-07-20) - + Food Network - Cook or Be Cooked! (USA) (Beta) (2009-07-20) + Preproduction @@ -18810,10 +18810,10 @@ Lara Croft Tomb Raider - Anniversary (USA) (Beta) (2007-09-20) - + Preproduction - Trauma Center - New Blood (USA) (Beta) (2007-10-02) - + Trauma Center - New Blood (USA) (En,Ja) (Beta) (2007-10-02) + Preproduction @@ -18860,4 +18860,29 @@ Yu-Gi-Oh! 5D's - Duel Transer (USA) (Beta) (2010-07-15) + + Games + Deca Sporta - Wiiro Jeulgineun Sports 10 Jongmok! (Korea) + + + + Demos + Wii Fit (Japan) (Taikenban) + + + + Applications + FreeLoader for Nintendo Wii (USA) (Unl) + + + + Demos + Food Network - Cook or Be Cooked! (USA) (Demo) + + + + Games + Disney Sing It (Russia) (En,Ru) + + diff --git a/nodtool/src/cmd/gen.rs b/nodtool/src/cmd/gen.rs index 38a0c1b..bbdaccd 100644 --- a/nodtool/src/cmd/gen.rs +++ b/nodtool/src/cmd/gen.rs @@ -13,7 +13,8 @@ use nod::{ build::gc::{FileCallback, FileInfo, GCPartitionBuilder, PartitionOverrides}, common::PartitionKind, disc::{ - fst::Fst, DiscHeader, PartitionHeader, BI2_SIZE, BOOT_SIZE, MINI_DVD_SIZE, SECTOR_SIZE, + fst::Fst, BootHeader, DiscHeader, BB2_OFFSET, BI2_SIZE, BOOT_SIZE, MINI_DVD_SIZE, + SECTOR_SIZE, }, read::{ DiscOptions, DiscReader, PartitionEncryption, PartitionMeta, PartitionOptions, @@ -114,11 +115,12 @@ pub fn run(args: Args) -> nod::Result<()> { // Build metadata let mut file_infos = Vec::new(); let boot_data: Box<[u8; BOOT_SIZE]> = read_fixed(&boot_path)?; - let header = DiscHeader::ref_from_bytes(&boot_data[..size_of::()]) + let header = DiscHeader::ref_from_bytes(array_ref![boot_data, 0, size_of::()]) .expect("Failed to read disc header"); let junk_id = get_junk_id(header); - let partition_header = PartitionHeader::ref_from_bytes(&boot_data[size_of::()..]) - .expect("Failed to read partition header"); + let boot_header = + BootHeader::ref_from_bytes(array_ref![boot_data, BB2_OFFSET, size_of::()]) + .expect("Failed to read boot header"); let fst_path = args.dir.join("sys/fst.bin"); let fst_data = read_all(&fst_path)?; let fst = Fst::new(&fst_data).expect("Failed to parse FST"); @@ -138,8 +140,8 @@ pub fn run(args: Args) -> nod::Result<()> { offset: BOOT_SIZE as u64 + BI2_SIZE as u64, length: apploader_size, }); - let fst_offset = partition_header.fst_offset(false); - let dol_offset = partition_header.dol_offset(false); + let fst_offset = boot_header.fst_offset(false); + let dol_offset = boot_header.dol_offset(false); if dol_offset < fst_offset { file_infos.push(FileWriteInfo { name: "sys/main.dol".to_string(), @@ -162,7 +164,7 @@ pub fn run(args: Args) -> nod::Result<()> { return Err(nod::Error::DiscFormat("DOL not found in FST".to_string())); } } - let fst_size = partition_header.fst_size(false); + let fst_size = boot_header.fst_size(false); file_infos.push(FileWriteInfo { name: "sys/fst.bin".to_string(), offset: fst_offset, @@ -213,21 +215,15 @@ pub fn run(args: Args) -> nod::Result<()> { let mut out = File::create(&args.out) .with_context(|| format!("Failed to create {}", args.out.display()))?; info!("Writing disc image to {} ({} files)", args.out.display(), file_infos.len()); - let crc = write_files( - &mut out, - &file_infos, - header, - partition_header, - junk_id, - |out, name| match name { + let crc = + write_files(&mut out, &file_infos, header, boot_header, junk_id, |out, name| match name { "sys/boot.bin" => out.write_all(boot_data.as_ref()), "sys/fst.bin" => out.write_all(fst_data.as_ref()), path => { let mut in_file = File::open(args.dir.join(path))?; io::copy(&mut in_file, out).map(|_| ()) } - }, - )?; + })?; out.flush().context("Failed to flush output file")?; info!("Generated disc image in {:?} (CRC32: {:08X})", start.elapsed(), crc); let redump_entry = redump::find_by_crc32(crc); @@ -265,14 +261,14 @@ fn write_files( w: &mut W, file_infos: &[FileWriteInfo], header: &DiscHeader, - partition_header: &PartitionHeader, + boot_header: &BootHeader, junk_id: Option<[u8; 4]>, mut callback: impl FnMut(&mut HashStream<&mut W>, &str) -> io::Result<()>, ) -> nod::Result where W: Write + ?Sized, { - let fst_end = partition_header.fst_offset(false) + partition_header.fst_size(false); + let fst_end = boot_header.fst_offset(false) + boot_header.fst_size(false); let file_gap = find_file_gap(file_infos, fst_end); let mut lfg = LaggedFibonacci::default(); let mut out = HashStream::new(w); @@ -461,9 +457,9 @@ fn in_memory_test( // Build metadata let mut file_infos = Vec::new(); - let header = meta.header(); + let header = meta.disc_header(); let junk_id = get_junk_id(header); - let partition_header = meta.partition_header(); + let boot_header = meta.boot_header(); let fst = meta.fst()?; file_infos.push(FileWriteInfo { @@ -481,8 +477,8 @@ fn in_memory_test( offset: BOOT_SIZE as u64 + BI2_SIZE as u64, length: meta.raw_apploader.len() as u64, }); - let fst_offset = partition_header.fst_offset(false); - let dol_offset = partition_header.dol_offset(false); + let fst_offset = boot_header.fst_offset(false); + let dol_offset = boot_header.dol_offset(false); if dol_offset < fst_offset { file_infos.push(FileWriteInfo { name: "sys/main.dol".to_string(), @@ -505,7 +501,7 @@ fn in_memory_test( return Err(nod::Error::Other("DOL not found in FST".to_string())); } } - let fst_size = partition_header.fst_size(false); + let fst_size = boot_header.fst_size(false); file_infos.push(FileWriteInfo { name: "sys/fst.bin".to_string(), offset: fst_offset, diff --git a/nodtool/src/main.rs b/nodtool/src/main.rs index 729be24..2efe9d7 100644 --- a/nodtool/src/main.rs +++ b/nodtool/src/main.rs @@ -145,8 +145,10 @@ fn main() { result = result.and_then(|_| run(args.command)); if let Err(e) = result { eprintln!("Failed: {}", e); - if let Some(source) = e.source() { - eprintln!("Caused by: {}", source); + let mut source = e.source(); + while let Some(e) = source { + eprintln!("Caused by: {}", e); + source = e.source(); } std::process::exit(1); }