encounter
/
nod-rs
mirror of https://github.com/encounter/nod-rs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add GCZ support, nodtool extract --partition, various fixes

This commit is contained in:
Luke Street 2024-02-22 19:58:01 -07:00
parent 60b3004999
commit 7e6d880792
19 changed files with 528 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
Cargo.lock generated
View File

@ -2,6 +2,12 @@
# It is not intended for manual editing.
version = 3
[[package]]
name = "adler"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "aes"
version = "0.8.4"
@ -394,10 +400,20 @@ version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
[[package]]
name = "miniz_oxide"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7"
dependencies = [
"adler",
]
[[package]]
name = "nod"
version = "0.2.0"
dependencies = [
"adler",
"aes",
"base16ct",
"bzip2",
@ -408,6 +424,7 @@ dependencies = [
"itertools",
"liblzma",
"log",
"miniz_oxide",
"rayon",
"sha1",
"thiserror",

5
README.md
View File

@ -1,4 +1,4 @@
# nod-rs [![Build Status]][actions] [![Latest Version]][crates.io] [![Api Rustdoc]][rustdoc] ![Rust Version]
# nod [![Build Status]][actions] [![Latest Version]][crates.io] [![Api Rustdoc]][rustdoc] ![Rust Version]
[Build Status]: https://github.com/encounter/nod-rs/actions/workflows/build.yaml/badge.svg
[actions]: https://github.com/encounter/nod-rs/actions
@ -8,7 +8,7 @@
[rustdoc]: https://docs.rs/nod
[Rust Version]: https://img.shields.io/badge/rust-1.73+-blue.svg?maxAge=3600
Library for traversing & reading GameCube and Wii disc images.
Library for traversing & reading Nintendo Optical Disc (GameCube and Wii) images.
Originally based on the C++ library [nod](https://github.com/AxioDL/nod),
but does not currently support authoring.
@ -19,6 +19,7 @@ Currently supported file formats:
- WBFS (+ NKit 2 lossless)
- CISO (+ NKit 2 lossless)
- NFS (Wii U VC)
- GCZ
## CLI tool

5
nod/Cargo.toml
View File

@ -15,13 +15,15 @@ keywords = ["gamecube", "wii", "iso", "wbfs", "rvz"]
categories = ["command-line-utilities", "parser-implementations"]
[features]
default = ["compress-bzip2", "compress-lzma", "compress-zstd"]
default = ["compress-bzip2", "compress-lzma", "compress-zlib", "compress-zstd"]
asm = ["sha1/asm"]
compress-bzip2 = ["bzip2"]
compress-lzma = ["liblzma"]
compress-zlib = ["adler", "miniz_oxide"]
compress-zstd = ["zstd"]
[dependencies]
adler = { version = "1.0.2", optional = true }
aes = "0.8.4"
base16ct = "0.2.0"
bzip2 = { version = "0.4.4", features = ["static"], optional = true }
@ -32,6 +34,7 @@ encoding_rs = "0.8.33"
itertools = "0.12.1"
liblzma = { version = "0.2.3", features = ["static"], optional = true }
log = "0.4.20"
miniz_oxide = { version = "0.7.2", optional = true }
rayon = "1.8.1"
sha1 = "0.10.6"
thiserror = "1.0.57"

11
nod/src/disc/gcn.rs
View File

@ -21,7 +21,7 @@ use crate::{
pub struct PartitionGC {
io: Box<dyn BlockIO>,
block: Option<Block>,
block: Block,
block_buf: Box<[u8]>,
block_idx: u32,
sector_buf: Box<[u8; SECTOR_SIZE]>,
@ -34,7 +34,7 @@ impl Clone for PartitionGC {
fn clone(&self) -> Self {
Self {
io: self.io.clone(),
block: None,
block: Block::default(),
block_buf: <u8>::new_box_slice_zeroed(self.block_buf.len()),
block_idx: u32::MAX,
sector_buf: <[u8; SECTOR_SIZE]>::new_box_zeroed(),
@ -50,7 +50,7 @@ impl PartitionGC {
let block_size = inner.block_size();
Ok(Box::new(Self {
io: inner,
block: None,
block: Block::default(),
block_buf: <u8>::new_box_slice_zeroed(block_size as usize),
block_idx: u32::MAX,
sector_buf: <[u8; SECTOR_SIZE]>::new_box_zeroed(),
@ -76,10 +76,7 @@ impl Read for PartitionGC {
// Copy sector if necessary
if sector != self.sector {
let Some(block) = &self.block else {
return Ok(0);
};
block.copy_raw(
self.block.copy_raw(
self.sector_buf.as_mut(),
self.block_buf.as_ref(),
block_idx,

2
nod/src/disc/hashes.rs
View File

@ -196,7 +196,7 @@ pub fn rebuild_hashes(reader: &mut DiscReader) -> Result<()> {
}
#[inline]
fn hash_bytes(buf: &[u8]) -> HashBytes {
pub fn hash_bytes(buf: &[u8]) -> HashBytes {
let mut hasher = Sha1::new();
hasher.update(buf);
hasher.finalize().into()

17
nod/src/disc/reader.rs
View File

@ -27,7 +27,7 @@ pub enum EncryptionMode {
pub struct DiscReader {
io: Box<dyn BlockIO>,
block: Option<Block>,
block: Block,
block_buf: Box<[u8]>,
block_idx: u32,
sector_buf: Box<[u8; SECTOR_SIZE]>,
@ -43,7 +43,7 @@ impl Clone for DiscReader {
fn clone(&self) -> Self {
Self {
io: self.io.clone(),
block: None,
block: Block::default(),
block_buf: <u8>::new_box_slice_zeroed(self.block_buf.len()),
block_idx: u32::MAX,
sector_buf: <[u8; SECTOR_SIZE]>::new_box_zeroed(),
@ -63,7 +63,7 @@ impl DiscReader {
let meta = inner.meta();
let mut reader = Self {
io: inner,
block: None,
block: Block::default(),
block_buf: <u8>::new_box_slice_zeroed(block_size as usize),
block_idx: u32::MAX,
sector_buf: <[u8; SECTOR_SIZE]>::new_box_zeroed(),
@ -92,7 +92,7 @@ impl DiscReader {
}
pub fn reset(&mut self) {
self.block = None;
self.block = Block::default();
self.block_buf.fill(0);
self.block_idx = u32::MAX;
self.sector_buf.fill(0);
@ -171,19 +171,16 @@ impl Read for DiscReader {
// Read new sector into buffer
if abs_sector != self.sector_idx {
let Some(block) = &self.block else {
return Ok(0);
};
if let Some(partition) = partition {
match self.mode {
EncryptionMode::Decrypted => block.decrypt(
EncryptionMode::Decrypted => self.block.decrypt(
self.sector_buf.as_mut(),
self.block_buf.as_ref(),
block_idx,
abs_sector,
partition,
)?,
EncryptionMode::Encrypted => block.encrypt(
EncryptionMode::Encrypted => self.block.encrypt(
self.sector_buf.as_mut(),
self.block_buf.as_ref(),
block_idx,
@ -192,7 +189,7 @@ impl Read for DiscReader {
)?,
}
} else {
block.copy_raw(
self.block.copy_raw(
self.sector_buf.as_mut(),
self.block_buf.as_ref(),
block_idx,

11
nod/src/disc/wii.rs
View File

@ -233,7 +233,7 @@ impl WiiPartitionHeader {
pub struct PartitionWii {
io: Box<dyn BlockIO>,
partition: PartitionInfo,
block: Option<Block>,
block: Block,
block_buf: Box<[u8]>,
block_idx: u32,
sector_buf: Box<[u8; SECTOR_SIZE]>,
@ -250,7 +250,7 @@ impl Clone for PartitionWii {
Self {
io: self.io.clone(),
partition: self.partition.clone(),
block: None,
block: Block::default(),
block_buf: <u8>::new_box_slice_zeroed(self.block_buf.len()),
block_idx: u32::MAX,
sector_buf: <[u8; SECTOR_SIZE]>::new_box_zeroed(),
@ -296,7 +296,7 @@ impl PartitionWii {
Ok(Box::new(Self {
io: reader.into_inner(),
partition: partition.clone(),
block: None,
block: Block::default(),
block_buf: <u8>::new_box_slice_zeroed(block_size as usize),
block_idx: u32::MAX,
sector_buf: <[u8; SECTOR_SIZE]>::new_box_zeroed(),
@ -328,10 +328,7 @@ impl Read for PartitionWii {
// Decrypt sector if necessary
if sector != self.sector {
let Some(block) = &self.block else {
return Ok(0);
};
block.decrypt(
self.block.decrypt(
self.sector_buf.as_mut(),
self.block_buf.as_ref(),
block_idx,

96
nod/src/io/block.rs
View File

@ -10,7 +10,7 @@ use crate::{
wii::{WiiPartitionHeader, HASHES_SIZE, SECTOR_DATA_SIZE},
SECTOR_SIZE,
},
io::{aes_decrypt, aes_encrypt, ciso, iso, nfs, wbfs, wia, KeyBytes, MagicBytes},
io::{aes_decrypt, aes_encrypt, KeyBytes, MagicBytes},
util::{lfg::LaggedFibonacci, read::read_from},
DiscHeader, DiscMeta, Error, PartitionHeader, PartitionKind, Result, ResultContext,
};
@ -18,15 +18,59 @@ use crate::{
/// Block I/O trait for reading disc images.
pub trait BlockIO: DynClone + Send + Sync {
/// Reads a block from the disc image.
fn read_block_internal(
&mut self,
out: &mut [u8],
block: u32,
partition: Option<&PartitionInfo>,
) -> io::Result<Block>;
/// Reads a full block from the disc image, combining smaller blocks if necessary.
fn read_block(
&mut self,
out: &mut [u8],
block: u32,
partition: Option<&PartitionInfo>,
) -> io::Result<Option<Block>>;
) -> io::Result<Block> {
let block_size_internal = self.block_size_internal();
let block_size = self.block_size();
if block_size_internal == block_size {
self.read_block_internal(out, block, partition)
} else {
let mut offset = 0usize;
let mut result = None;
let mut block_idx =
((block as u64 * block_size as u64) / block_size_internal as u64) as u32;
while offset < block_size as usize {
let block = self.read_block_internal(
&mut out[offset..offset + block_size_internal as usize],
block_idx,
partition,
)?;
if result.is_none() {
result = Some(block);
} else if result != Some(block) {
if block == Block::Zero {
out[offset..offset + block_size_internal as usize].fill(0);
} else {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"Inconsistent block types in split block",
));
}
}
offset += block_size_internal as usize;
block_idx += 1;
}
Ok(result.unwrap_or_default())
}
}
/// The format's block size in bytes. Must be a multiple of the sector size (0x8000).
fn block_size(&self) -> u32;
/// The format's block size in bytes. Can be smaller than the sector size (0x8000).
fn block_size_internal(&self) -> u32;
/// The block size used for processing. Must be a multiple of the sector size (0x8000).
fn block_size(&self) -> u32 { self.block_size_internal().max(SECTOR_SIZE as u32) }
/// Returns extra metadata included in the disc file format, if any.
fn meta(&self) -> DiscMeta;
@ -54,16 +98,41 @@ pub fn open(filename: &Path) -> Result<Box<dyn BlockIO>> {
read_from(&mut file)
.with_context(|| format!("Reading magic bytes from {}", filename.display()))?
};
match magic {
ciso::CISO_MAGIC => Ok(ciso::DiscIOCISO::new(path)?),
nfs::NFS_MAGIC => match path.parent() {
Some(parent) if parent.is_dir() => Ok(nfs::DiscIONFS::new(path.parent().unwrap())?),
_ => Err(Error::DiscFormat("Failed to locate NFS parent directory".to_string())),
let io: Box<dyn BlockIO> = match magic {
crate::io::ciso::CISO_MAGIC => crate::io::ciso::DiscIOCISO::new(path)?,
#[cfg(feature = "compress-zlib")]
crate::io::gcz::GCZ_MAGIC => crate::io::gcz::DiscIOGCZ::new(path)?,
crate::io::nfs::NFS_MAGIC => match path.parent() {
Some(parent) if parent.is_dir() => {
crate::io::nfs::DiscIONFS::new(path.parent().unwrap())?
}
_ => {
return Err(Error::DiscFormat("Failed to locate NFS parent directory".to_string()));
}
},
wbfs::WBFS_MAGIC => Ok(wbfs::DiscIOWBFS::new(path)?),
wia::WIA_MAGIC | wia::RVZ_MAGIC => Ok(wia::DiscIOWIA::new(path)?),
_ => Ok(iso::DiscIOISO::new(path)?),
crate::io::wbfs::WBFS_MAGIC => crate::io::wbfs::DiscIOWBFS::new(path)?,
crate::io::wia::WIA_MAGIC | crate::io::wia::RVZ_MAGIC => {
crate::io::wia::DiscIOWIA::new(path)?
}
_ => crate::io::iso::DiscIOISO::new(path)?,
};
if io.block_size_internal() < SECTOR_SIZE as u32
&& SECTOR_SIZE as u32 % io.block_size_internal() != 0
{
return Err(Error::DiscFormat(format!(
"Sector size {} is not divisible by block size {}",
SECTOR_SIZE,
io.block_size_internal(),
)));
}
if io.block_size() % SECTOR_SIZE as u32 != 0 {
return Err(Error::DiscFormat(format!(
"Block size {} is not a multiple of sector size {}",
io.block_size(),
SECTOR_SIZE
)));
}
Ok(io)
}
#[derive(Debug, Clone)]
@ -80,7 +149,7 @@ pub struct PartitionInfo {
pub hash_table: Option<HashTable>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum Block {
/// Raw data or encrypted Wii partition data
Raw,
@ -92,6 +161,7 @@ pub enum Block {
/// Wii partition junk data
Junk,
/// All zeroes
#[default]
Zero,
}

28
nod/src/io/ciso.rs
View File

@ -23,17 +23,18 @@ use crate::{
pub const CISO_MAGIC: MagicBytes = *b"CISO";
pub const CISO_MAP_SIZE: usize = SECTOR_SIZE - 8;
/// CISO header (little endian)
#[derive(Clone, Debug, PartialEq, FromBytes, FromZeroes, AsBytes)]
#[repr(C, align(4))]
struct CISOHeader {
magic: MagicBytes,
// little endian
block_size: U32,
block_present: [u8; CISO_MAP_SIZE],
}
static_assert!(size_of::<CISOHeader>() == SECTOR_SIZE);
#[derive(Clone)]
pub struct DiscIOCISO {
inner: SplitFileReader,
header: CISOHeader,
@ -41,17 +42,6 @@ pub struct DiscIOCISO {
nkit_header: Option<NKitHeader>,
}
impl Clone for DiscIOCISO {
fn clone(&self) -> Self {
Self {
inner: self.inner.clone(),
header: self.header.clone(),
block_map: self.block_map,
nkit_header: self.nkit_header.clone(),
}
}
}
impl DiscIOCISO {
pub fn new(filename: &Path) -> Result<Box<Self>> {
let mut inner = SplitFileReader::new(filename)?;
@ -97,15 +87,15 @@ impl DiscIOCISO {
}
impl BlockIO for DiscIOCISO {
fn read_block(
fn read_block_internal(
&mut self,
out: &mut [u8],
block: u32,
_partition: Option<&PartitionInfo>,
) -> io::Result<Option<Block>> {
) -> io::Result<Block> {
if block >= CISO_MAP_SIZE as u32 {
// Out of bounds
return Ok(None);
return Ok(Block::Zero);
}
// Find the block in the map
@ -113,11 +103,11 @@ impl BlockIO for DiscIOCISO {
if phys_block == u16::MAX {
// Check if block is junk data
if self.nkit_header.as_ref().is_some_and(|h| h.is_junk_block(block).unwrap_or(false)) {
return Ok(Some(Block::Junk));
return Ok(Block::Junk);
};
// Otherwise, read zeroes
return Ok(Some(Block::Zero));
return Ok(Block::Zero);
}
// Read block
@ -125,10 +115,10 @@ impl BlockIO for DiscIOCISO {
+ phys_block as u64 * self.header.block_size.get() as u64;
self.inner.seek(SeekFrom::Start(file_offset))?;
self.inner.read_exact(out)?;
Ok(Some(Block::Raw))
Ok(Block::Raw)
}
fn block_size(&self) -> u32 { self.header.block_size.get() }
fn block_size_internal(&self) -> u32 { self.header.block_size.get() }
fn meta(&self) -> DiscMeta {
let mut result = DiscMeta {

192
nod/src/io/gcz.rs Normal file
View File

@ -0,0 +1,192 @@
use std::{
io,
io::{Read, Seek, SeekFrom},
mem::size_of,
path::Path,
};
use adler::adler32_slice;
use miniz_oxide::{inflate, inflate::core::inflate_flags};
use zerocopy::{little_endian::*, AsBytes, FromBytes, FromZeroes};
use zstd::zstd_safe::WriteBuf;
use crate::{
io::{
block::{Block, BlockIO},
split::SplitFileReader,
MagicBytes,
},
static_assert,
util::read::{read_box_slice, read_from},
Compression, DiscMeta, Error, Format, PartitionInfo, Result, ResultContext,
};
pub const GCZ_MAGIC: MagicBytes = [0x01, 0xC0, 0x0B, 0xB1];
/// GCZ header (little endian)
#[derive(Clone, Debug, PartialEq, FromBytes, FromZeroes, AsBytes)]
#[repr(C, align(4))]
struct GCZHeader {
magic: MagicBytes,
disc_type: U32,
compressed_size: U64,
disc_size: U64,
block_size: U32,
block_count: U32,
}
static_assert!(size_of::<GCZHeader>() == 32);
pub struct DiscIOGCZ {
inner: SplitFileReader,
header: GCZHeader,
block_map: Box<[U64]>,
block_hashes: Box<[U32]>,
block_buf: Box<[u8]>,
data_offset: u64,
}
impl Clone for DiscIOGCZ {
fn clone(&self) -> Self {
Self {
inner: self.inner.clone(),
header: self.header.clone(),
block_map: self.block_map.clone(),
block_hashes: self.block_hashes.clone(),
block_buf: <u8>::new_box_slice_zeroed(self.block_buf.len()),
data_offset: self.data_offset,
}
}
}
impl DiscIOGCZ {
pub fn new(filename: &Path) -> Result<Box<Self>> {
let mut inner = SplitFileReader::new(filename)?;
// Read header
let header: GCZHeader = read_from(&mut inner).context("Reading GCZ header")?;
if header.magic != GCZ_MAGIC {
return Err(Error::DiscFormat("Invalid GCZ magic".to_string()));
}
// Read block map and hashes
let block_count = header.block_count.get();
let block_map =
read_box_slice(&mut inner, block_count as usize).context("Reading GCZ block map")?;
let block_hashes =
read_box_slice(&mut inner, block_count as usize).context("Reading GCZ block hashes")?;
// header + block_count * (u64 + u32)
let data_offset = size_of::<GCZHeader>() as u64 + block_count as u64 * 12;
// Reset reader
inner.reset();
let block_buf = <u8>::new_box_slice_zeroed(header.block_size.get() as usize);
Ok(Box::new(Self { inner, header, block_map, block_hashes, block_buf, data_offset }))
}
}
impl BlockIO for DiscIOGCZ {
fn read_block_internal(
&mut self,
out: &mut [u8],
block: u32,
_partition: Option<&PartitionInfo>,
) -> io::Result<Block> {
if block >= self.header.block_count.get() {
// Out of bounds
return Ok(Block::Zero);
}
// Find block offset and size
let mut file_offset = self.block_map[block as usize].get();
let mut compressed = true;
if file_offset & (1 << 63) != 0 {
file_offset &= !(1 << 63);
compressed = false;
}
let compressed_size =
((self.block_map.get(block as usize + 1).unwrap_or(&self.header.compressed_size).get()
& !(1 << 63))
- file_offset) as usize;
if compressed_size > self.block_buf.len() {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"Compressed block size exceeds block size: {} > {}",
compressed_size,
self.block_buf.len()
),
));
} else if !compressed && compressed_size != self.block_buf.len() {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"Uncompressed block size does not match block size: {} != {}",
compressed_size,
self.block_buf.len()
),
));
}
// Read block
self.inner.seek(SeekFrom::Start(self.data_offset + file_offset))?;
self.inner.read_exact(&mut self.block_buf[..compressed_size])?;
// Verify block checksum
let checksum = adler32_slice(&self.block_buf[..compressed_size]);
let expected_checksum = self.block_hashes[block as usize].get();
if checksum != expected_checksum {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"Block checksum mismatch: {:#010x} != {:#010x}",
checksum, expected_checksum
),
));
}
if compressed {
// Decompress block
let mut decompressor = inflate::core::DecompressorOxide::new();
let input = &self.block_buf[..compressed_size];
let (status, in_size, out_size) = inflate::core::decompress(
&mut decompressor,
input,
out,
0,
inflate_flags::TINFL_FLAG_PARSE_ZLIB_HEADER
| inflate_flags::TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF,
);
if status != inflate::TINFLStatus::Done
|| in_size != compressed_size
|| out_size != self.block_buf.len()
{
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"Deflate decompression failed: {:?} (in: {}, out: {})",
status, in_size, out_size
),
));
}
} else {
// Copy uncompressed block
out.copy_from_slice(self.block_buf.as_slice());
}
Ok(Block::Raw)
}
fn block_size_internal(&self) -> u32 { self.header.block_size.get() }
fn meta(&self) -> DiscMeta {
DiscMeta {
format: Format::Gcz,
compression: Compression::Deflate,
block_size: Some(self.header.block_size.get()),
lossless: true,
disc_size: Some(self.header.disc_size.get()),
..Default::default()
}
}
}

33
nod/src/io/iso.rs
View File

@ -1,6 +1,6 @@
use std::{
io,
io::{Read, Seek},
io::{Read, Seek, SeekFrom},
path::Path,
};
@ -11,7 +11,7 @@ use crate::{
split::SplitFileReader,
Format,
},
DiscMeta, Error, Result,
DiscMeta, Result,
};
#[derive(Clone)]
@ -22,34 +22,37 @@ pub struct DiscIOISO {
impl DiscIOISO {
pub fn new(filename: &Path) -> Result<Box<Self>> {
let inner = SplitFileReader::new(filename)?;
if inner.len() % SECTOR_SIZE as u64 != 0 {
return Err(Error::DiscFormat(
"ISO size is not a multiple of sector size (0x8000 bytes)".to_string(),
));
}
Ok(Box::new(Self { inner }))
}
}
impl BlockIO for DiscIOISO {
fn read_block(
fn read_block_internal(
&mut self,
out: &mut [u8],
block: u32,
_partition: Option<&PartitionInfo>,
) -> io::Result<Option<Block>> {
) -> io::Result<Block> {
let offset = block as u64 * SECTOR_SIZE as u64;
if offset >= self.inner.len() {
let total_size = self.inner.len();
if offset >= total_size {
// End of file
return Ok(None);
return Ok(Block::Zero);
}
self.inner.seek(io::SeekFrom::Start(offset))?;
self.inner.read_exact(out)?;
Ok(Some(Block::Raw))
self.inner.seek(SeekFrom::Start(offset))?;
if offset + SECTOR_SIZE as u64 > total_size {
// If the last block is not a full sector, fill the rest with zeroes
let read = (total_size - offset) as usize;
self.inner.read_exact(&mut out[..read])?;
out[read..].fill(0);
} else {
self.inner.read_exact(out)?;
}
Ok(Block::Raw)
}
fn block_size(&self) -> u32 { SECTOR_SIZE as u32 }
fn block_size_internal(&self) -> u32 { SECTOR_SIZE as u32 }
fn meta(&self) -> DiscMeta {
DiscMeta {

16
nod/src/io/mod.rs
View File

@ -4,6 +4,8 @@ use std::fmt;
pub(crate) mod block;
pub(crate) mod ciso;
#[cfg(feature = "compress-zlib")]
pub(crate) mod gcz;
pub(crate) mod iso;
pub(crate) mod nfs;
pub(crate) mod nkit;
@ -27,6 +29,8 @@ pub enum Format {
Iso,
/// CISO
Ciso,
/// GCZ
Gcz,
/// NFS (Wii U VC)
Nfs,
/// RVZ
@ -42,6 +46,7 @@ impl fmt::Display for Format {
match self {
Format::Iso => write!(f, "ISO"),
Format::Ciso => write!(f, "CISO"),
Format::Gcz => write!(f, "GCZ"),
Format::Nfs => write!(f, "NFS"),
Format::Rvz => write!(f, "RVZ"),
Format::Wbfs => write!(f, "WBFS"),
@ -55,14 +60,16 @@ pub enum Compression {
/// No compression
#[default]
None,
/// Purge (WIA only)
Purge,
/// BZIP2
Bzip2,
/// Deflate (GCZ only)
Deflate,
/// LZMA
Lzma,
/// LZMA2
Lzma2,
/// Purge (WIA only)
Purge,
/// Zstandard
Zstandard,
}
@ -71,10 +78,11 @@ impl fmt::Display for Compression {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Compression::None => write!(f, "None"),
Compression::Purge => write!(f, "Purge"),
Compression::Bzip2 => write!(f, "BZIP2"),
Compression::Deflate => write!(f, "Deflate"),
Compression::Lzma => write!(f, "LZMA"),
Compression::Lzma2 => write!(f, "LZMA2"),
Compression::Purge => write!(f, "Purge"),
Compression::Zstandard => write!(f, "Zstandard"),
}
}
@ -108,6 +116,7 @@ pub struct DiscMeta {
}
/// Encrypts data in-place using AES-128-CBC with the given key and IV.
#[inline(always)]
pub(crate) fn aes_encrypt(key: &KeyBytes, iv: KeyBytes, data: &mut [u8]) {
use aes::cipher::{block_padding::NoPadding, BlockEncryptMut, KeyIvInit};
<cbc::Encryptor<aes::Aes128>>::new(key.into(), &aes::Block::from(iv))
@ -116,6 +125,7 @@ pub(crate) fn aes_encrypt(key: &KeyBytes, iv: KeyBytes, data: &mut [u8]) {
}
/// Decrypts data in-place using AES-128-CBC with the given key and IV.
#[inline(always)]
pub(crate) fn aes_decrypt(key: &KeyBytes, iv: KeyBytes, data: &mut [u8]) {
use aes::cipher::{block_padding::NoPadding, BlockDecryptMut, KeyIvInit};
<cbc::Decryptor<aes::Aes128>>::new(key.into(), &aes::Block::from(iv))

25
nod/src/io/nfs.rs
View File

@ -83,6 +83,7 @@ impl NFSHeader {
}
}
#[derive(Clone)]
pub struct DiscIONFS {
inner: SplitFileReader,
header: NFSHeader,
@ -91,18 +92,6 @@ pub struct DiscIONFS {
key: KeyBytes,
}
impl Clone for DiscIONFS {
fn clone(&self) -> Self {
Self {
inner: self.inner.clone(),
header: self.header.clone(),
raw_size: self.raw_size,
disc_size: self.disc_size,
key: self.key,
}
}
}
impl DiscIONFS {
pub fn new(directory: &Path) -> Result<Box<Self>> {
let mut disc_io = Box::new(Self {
@ -118,17 +107,17 @@ impl DiscIONFS {
}
impl BlockIO for DiscIONFS {
fn read_block(
fn read_block_internal(
&mut self,
out: &mut [u8],
sector: u32,
partition: Option<&PartitionInfo>,
) -> io::Result<Option<Block>> {
) -> io::Result<Block> {
// Calculate physical sector
let phys_sector = self.header.phys_sector(sector);
if phys_sector == u32::MAX {
// Logical zero sector
return Ok(Some(Block::Zero));
return Ok(Block::Zero);
}
// Read sector
@ -146,13 +135,13 @@ impl BlockIO for DiscIONFS {
aes_decrypt(&self.key, iv, out);
if partition.is_some() {
Ok(Some(Block::PartDecrypted { has_hashes: true }))
Ok(Block::PartDecrypted { has_hashes: true })
} else {
Ok(Some(Block::Raw))
Ok(Block::Raw)
}
}
fn block_size(&self) -> u32 { SECTOR_SIZE as u32 }
fn block_size_internal(&self) -> u32 { SECTOR_SIZE as u32 }
fn meta(&self) -> DiscMeta {
DiscMeta { format: Format::Nfs, decrypted: true, ..Default::default() }

17
nod/src/io/wbfs.rs
View File

@ -109,30 +109,33 @@ impl DiscIOWBFS {
}
impl BlockIO for DiscIOWBFS {
fn read_block(
fn read_block_internal(
&mut self,
out: &mut [u8],
block: u32,
_partition: Option<&PartitionInfo>,
) -> io::Result<Option<Block>> {
) -> io::Result<Block> {
let block_size = self.header.block_size();
if block >= self.header.max_blocks() {
return Ok(None);
return Ok(Block::Zero);
}
// Check if block is junk data
if self.nkit_header.as_ref().is_some_and(|h| h.is_junk_block(block).unwrap_or(false)) {
return Ok(Some(Block::Junk));
if self.nkit_header.as_ref().and_then(|h| h.is_junk_block(block)).unwrap_or(false) {
return Ok(Block::Junk);
}
// Read block
let block_start = block_size as u64 * self.block_table[block as usize].get() as u64;
if block_start == 0 {
return Ok(Block::Zero);
}
self.inner.seek(SeekFrom::Start(block_start))?;
self.inner.read_exact(out)?;
Ok(Some(Block::Raw))
Ok(Block::Raw)
}
fn block_size(&self) -> u32 { self.header.block_size() }
fn block_size_internal(&self) -> u32 { self.header.block_size() }
fn meta(&self) -> DiscMeta {
let mut result = DiscMeta {

21
nod/src/io/wia.rs
View File

@ -5,11 +5,11 @@ use std::{
path::Path,
};
use sha1::{Digest, Sha1};
use zerocopy::{big_endian::*, AsBytes, FromBytes, FromZeroes};
use crate::{
disc::{
hashes::hash_bytes,
wii::{HASHES_SIZE, SECTOR_DATA_SIZE},
SECTOR_SIZE,
},
@ -533,13 +533,6 @@ impl Clone for DiscIOWIA {
}
}
#[inline]
fn hash_bytes(buf: &[u8]) -> HashBytes {
let mut hasher = Sha1::new();
hasher.update(buf);
hasher.finalize().into()
}
fn verify_hash(buf: &[u8], expected: &HashBytes) -> Result<()> {
let out = hash_bytes(buf);
if out != *expected {
@ -686,12 +679,12 @@ where
}
impl BlockIO for DiscIOWIA {
fn read_block(
fn read_block_internal(
&mut self,
out: &mut [u8],
sector: u32,
partition: Option<&PartitionInfo>,
) -> io::Result<Option<Block>> {
) -> io::Result<Block> {
let mut chunk_size = self.disc.chunk_size.get();
let sectors_per_chunk = chunk_size / SECTOR_SIZE as u32;
let disc_offset = sector as u64 * SECTOR_SIZE as u64;
@ -792,7 +785,7 @@ impl BlockIO for DiscIOWIA {
// Special case for all-zero data
if group.data_size() == 0 {
self.exception_lists.clear();
return Ok(Some(Block::Zero));
return Ok(Block::Zero);
}
// Read group data if necessary
@ -875,17 +868,17 @@ impl BlockIO for DiscIOWIA {
&self.group_data[sector_data_start..sector_data_start + SECTOR_DATA_SIZE];
out[..HASHES_SIZE].fill(0);
out[HASHES_SIZE..SECTOR_SIZE].copy_from_slice(sector_data);
Ok(Some(Block::PartDecrypted { has_hashes: false }))
Ok(Block::PartDecrypted { has_hashes: false })
} else {
let sector_data_start = group_sector as usize * SECTOR_SIZE;
out.copy_from_slice(
&self.group_data[sector_data_start..sector_data_start + SECTOR_SIZE],
);
Ok(Some(Block::Raw))
Ok(Block::Raw)
}
}
fn block_size(&self) -> u32 {
fn block_size_internal(&self) -> u32 {
// WIA/RVZ chunks aren't always the full size, so we'll consider the
// block size to be one sector, and handle the complexity ourselves.
SECTOR_SIZE as u32

4
nod/src/util/mod.rs
View File

@ -17,7 +17,7 @@ where T: Div<Output = T> + Rem<Output = T> + Copy {
#[macro_export]
macro_rules! array_ref {
($slice:expr, $offset:expr, $size:expr) => {{
#[inline]
#[inline(always)]
fn to_array<T>(slice: &[T]) -> &[T; $size] {
unsafe { &*(slice.as_ptr() as *const [_; $size]) }
}
@ -29,7 +29,7 @@ macro_rules! array_ref {
#[macro_export]
macro_rules! array_ref_mut {
($slice:expr, $offset:expr, $size:expr) => {{
#[inline]
#[inline(always)]
fn to_array<T>(slice: &mut [T]) -> &mut [T; $size] {
unsafe { &mut *(slice.as_ptr() as *mut [_; $size]) }
}

1
nodtool/build.rs
View File

@ -45,6 +45,7 @@ fn main() {
// Parse dat files
let mut entries = Vec::<(GameEntry, String)>::new();
for path in ["assets/redump-gc.dat", "assets/redump-wii.dat"] {
println!("cargo:rustc-rerun-if-changed={}", path);
let file = BufReader::new(File::open(path).expect("Failed to open dat file"));
let dat: DatFile = quick_xml::de::from_reader(file).expect("Failed to parse dat file");
entries.extend(dat.games.into_iter().map(|game| {

182
nodtool/src/main.rs
View File

@ -12,7 +12,7 @@ use std::{
fs::File,
io,
io::{BufWriter, Read, Write},
path::{Path, PathBuf},
path::{Path, PathBuf, MAIN_SEPARATOR},
str::FromStr,
sync::{mpsc::sync_channel, Arc},
thread,
@ -27,7 +27,7 @@ use nod::{
Compression, Disc, DiscHeader, DiscMeta, Fst, Node, OpenOptions, PartitionBase, PartitionKind,
PartitionMeta, Result, ResultContext, SECTOR_SIZE,
};
use size::Size;
use size::{Base, Size};
use supports_color::Stream;
use tracing::level_filters::LevelFilter;
use tracing_subscriber::EnvFilter;
@ -64,12 +64,12 @@ enum SubCommand {
}
#[derive(FromArgs, Debug)]
/// Displays information about a disc image.
/// Displays information about disc images.
#[argp(subcommand, name = "info")]
struct InfoArgs {
#[argp(positional)]
/// path to disc image
file: PathBuf,
/// Path to disc image(s)
file: Vec<PathBuf>,
}
#[derive(FromArgs, Debug)]
@ -77,17 +77,21 @@ struct InfoArgs {
#[argp(subcommand, name = "extract")]
struct ExtractArgs {
#[argp(positional)]
/// path to disc image
/// Path to disc image
file: PathBuf,
#[argp(positional)]
/// output directory (optional)
/// Output directory (optional)
out: Option<PathBuf>,
#[argp(switch, short = 'q')]
/// quiet output
/// Quiet output
quiet: bool,
#[argp(switch, short = 'h')]
/// validate disc hashes (Wii only)
/// Validate data hashes (Wii only)
validate: bool,
#[argp(option, short = 'p')]
/// Partition to extract (default: data)
/// Options: all, data, update, channel, or a partition index
partition: Option<String>,
}
#[derive(FromArgs, Debug)]
@ -206,7 +210,7 @@ fn main() {
let mut result = Ok(());
if let Some(dir) = &args.chdir {
result = env::set_current_dir(dir).map_err(|e| {
nod::Error::Io(format!("Failed to change working directory to '{}'", dir.display()), e)
nod::Error::Io(format!("Failed to change working directory to '{}'", display(dir)), e)
});
}
result = result.and_then(|_| match args.command {
@ -253,7 +257,15 @@ fn print_header(header: &DiscHeader, meta: &DiscMeta) {
}
fn info(args: InfoArgs) -> Result<()> {
let disc = Disc::new_with_options(args.file, &OpenOptions {
for file in &args.file {
info_file(file)?;
}
Ok(())
}
fn info_file(path: &Path) -> Result<()> {
log::info!("Loading {}", display(path));
let disc = Disc::new_with_options(path, &OpenOptions {
rebuild_encryption: false,
validate_hashes: false,
})?;
@ -327,6 +339,7 @@ fn info(args: InfoArgs) -> Result<()> {
header.wii_magic.get()
);
}
println!();
Ok(())
}
@ -343,7 +356,7 @@ fn verify(args: VerifyArgs) -> Result<()> {
}
fn convert_and_verify(in_file: &Path, out_file: Option<&Path>, md5: bool) -> Result<()> {
println!("Loading {}", in_file.display());
println!("Loading {}", display(in_file));
let mut disc = Disc::new_with_options(in_file, &OpenOptions {
rebuild_encryption: true,
validate_hashes: false,
@ -357,7 +370,7 @@ fn convert_and_verify(in_file: &Path, out_file: Option<&Path>, md5: bool) -> Res
let mut file = if let Some(out_file) = out_file {
Some(
File::create(out_file)
.with_context(|| format!("Creating file {}", out_file.display()))?,
.with_context(|| format!("Creating file {}", display(out_file)))?,
)
} else {
None
@ -432,7 +445,7 @@ fn convert_and_verify(in_file: &Path, out_file: Option<&Path>, md5: bool) -> Res
println!();
if let Some(path) = out_file {
println!("Wrote {} to {}", Size::from_bytes(total_read), path.display());
println!("Wrote {} to {}", Size::from_bytes(total_read), display(path));
}
println!();
@ -450,11 +463,7 @@ fn convert_and_verify(in_file: &Path, out_file: Option<&Path>, md5: bool) -> Res
}
}
let redump_entry = if let (Some(crc32), Some(sha1)) = (crc32, sha1) {
redump::find_by_hashes(crc32, sha1)
} else {
None
};
let redump_entry = crc32.and_then(redump::find_by_crc32);
let expected_crc32 = meta.crc32.or(redump_entry.as_ref().map(|e| e.crc32));
let expected_md5 = meta.md5.or(redump_entry.as_ref().map(|e| e.md5));
let expected_sha1 = meta.sha1.or(redump_entry.as_ref().map(|e| e.sha1));
@ -475,7 +484,22 @@ fn convert_and_verify(in_file: &Path, out_file: Option<&Path>, md5: bool) -> Res
}
if let Some(entry) = &redump_entry {
println!("Redump: {} ✅", entry.name);
let mut full_match = true;
if let Some(md5) = md5 {
if entry.md5 != md5 {
full_match = false;
}
}
if let Some(sha1) = sha1 {
if entry.sha1 != sha1 {
full_match = false;
}
}
if full_match {
println!("Redump: {} ✅", entry.name);
} else {
println!("Redump: {} ❓ (partial match)", entry.name);
}
} else {
println!("Redump: Not found ❌");
}
@ -520,12 +544,49 @@ fn extract(args: ExtractArgs) -> Result<()> {
validate_hashes: args.validate,
})?;
let is_wii = disc.header().is_wii();
let mut partition = disc.open_partition_kind(PartitionKind::Data)?;
if let Some(partition) = args.partition {
if partition.eq_ignore_ascii_case("all") {
for info in disc.partitions() {
let mut out_dir = output_dir.clone();
out_dir.push(info.kind.dir_name().as_ref());
let mut partition = disc.open_partition(info.index)?;
extract_partition(partition.as_mut(), &out_dir, is_wii, args.quiet)?;
}
} else if partition.eq_ignore_ascii_case("data") {
let mut partition = disc.open_partition_kind(PartitionKind::Data)?;
extract_partition(partition.as_mut(), &output_dir, is_wii, args.quiet)?;
} else if partition.eq_ignore_ascii_case("update") {
let mut partition = disc.open_partition_kind(PartitionKind::Update)?;
extract_partition(partition.as_mut(), &output_dir, is_wii, args.quiet)?;
} else if partition.eq_ignore_ascii_case("channel") {
let mut partition = disc.open_partition_kind(PartitionKind::Channel)?;
extract_partition(partition.as_mut(), &output_dir, is_wii, args.quiet)?;
} else {
let idx = partition.parse::<usize>().map_err(|_| "Invalid partition index")?;
let mut partition = disc.open_partition(idx)?;
extract_partition(partition.as_mut(), &output_dir, is_wii, args.quiet)?;
}
} else {
let mut partition = disc.open_partition_kind(PartitionKind::Data)?;
extract_partition(partition.as_mut(), &output_dir, is_wii, args.quiet)?;
}
Ok(())
}
fn extract_partition(
partition: &mut dyn PartitionBase,
out_dir: &Path,
is_wii: bool,
quiet: bool,
) -> Result<()> {
let meta = partition.meta()?;
extract_sys_files(meta.as_ref(), &output_dir.join("sys"), args.quiet)?;
extract_sys_files(meta.as_ref(), out_dir, quiet)?;
// Extract FST
let files_dir = output_dir.join("files");
let files_dir = out_dir.join("files");
fs::create_dir_all(&files_dir)
.with_context(|| format!("Creating directory {}", display(&files_dir)))?;
let fst = Fst::new(&meta.raw_fst)?;
let mut path_segments = Vec::<(Cow<str>, usize)>::new();
for (idx, node, name) in fst.iter() {
@ -548,28 +609,47 @@ fn extract(args: ExtractArgs) -> Result<()> {
fs::create_dir_all(files_dir.join(&path))
.with_context(|| format!("Creating directory {}", path))?;
} else {
extract_node(node, partition.as_mut(), &files_dir, &path, is_wii, args.quiet)?;
extract_node(node, partition, &files_dir, &path, is_wii, quiet)?;
}
}
Ok(())
}
fn extract_sys_files(data: &PartitionMeta, out_dir: &Path, quiet: bool) -> Result<()> {
fs::create_dir_all(out_dir)
.with_context(|| format!("Creating output directory {}", out_dir.display()))?;
extract_file(data.raw_boot.as_ref(), &out_dir.join("boot.bin"), quiet)?;
extract_file(data.raw_bi2.as_ref(), &out_dir.join("bi2.bin"), quiet)?;
extract_file(data.raw_apploader.as_ref(), &out_dir.join("apploader.img"), quiet)?;
extract_file(data.raw_fst.as_ref(), &out_dir.join("fst.bin"), quiet)?;
extract_file(data.raw_dol.as_ref(), &out_dir.join("main.dol"), quiet)?;
let sys_dir = out_dir.join("sys");
fs::create_dir_all(&sys_dir)
.with_context(|| format!("Creating directory {}", display(&sys_dir)))?;
extract_file(data.raw_boot.as_ref(), &sys_dir.join("boot.bin"), quiet)?;
extract_file(data.raw_bi2.as_ref(), &sys_dir.join("bi2.bin"), quiet)?;
extract_file(data.raw_apploader.as_ref(), &sys_dir.join("apploader.img"), quiet)?;
extract_file(data.raw_fst.as_ref(), &sys_dir.join("fst.bin"), quiet)?;
extract_file(data.raw_dol.as_ref(), &sys_dir.join("main.dol"), quiet)?;
// Wii files
if let Some(ticket) = data.raw_ticket.as_deref() {
extract_file(ticket, &out_dir.join("ticket.bin"), quiet)?;
}
if let Some(tmd) = data.raw_tmd.as_deref() {
extract_file(tmd, &out_dir.join("tmd.bin"), quiet)?;
}
if let Some(cert_chain) = data.raw_cert_chain.as_deref() {
extract_file(cert_chain, &out_dir.join("cert.bin"), quiet)?;
}
if let Some(h3_table) = data.raw_h3_table.as_deref() {
extract_file(h3_table, &out_dir.join("h3.bin"), quiet)?;
}
Ok(())
}
fn extract_file(bytes: &[u8], out_path: &Path, quiet: bool) -> Result<()> {
if !quiet {
println!("Extracting {} (size: {})", out_path.display(), Size::from_bytes(bytes.len()));
println!(
"Extracting {} (size: {})",
display(out_path),
Size::from_bytes(bytes.len()).format().with_base(Base::Base10)
);
}
fs::write(out_path, bytes).with_context(|| format!("Writing file {}", out_path.display()))?;
fs::write(out_path, bytes).with_context(|| format!("Writing file {}", display(out_path)))?;
Ok(())
}
@ -585,12 +665,12 @@ fn extract_node(
if !quiet {
println!(
"Extracting {} (size: {})",
file_path.display(),
Size::from_bytes(node.length(is_wii))
display(&file_path),
Size::from_bytes(node.length(is_wii)).format().with_base(Base::Base10)
);
}
let file = File::create(&file_path)
.with_context(|| format!("Creating file {}", file_path.display()))?;
.with_context(|| format!("Creating file {}", display(&file_path)))?;
let mut w = BufWriter::with_capacity(partition.ideal_buffer_size(), file);
let mut r = partition.open_file(node).with_context(|| {
format!(
@ -600,7 +680,33 @@ fn extract_node(
node.length(is_wii)
)
})?;
io::copy(&mut r, &mut w).with_context(|| format!("Extracting file {}", file_path.display()))?;
w.flush().with_context(|| format!("Flushing file {}", file_path.display()))?;
io::copy(&mut r, &mut w).with_context(|| format!("Extracting file {}", display(&file_path)))?;
w.flush().with_context(|| format!("Flushing file {}", display(&file_path)))?;
Ok(())
}
fn display(path: &Path) -> PathDisplay { PathDisplay { path } }
struct PathDisplay<'a> {
path: &'a Path,
}
impl<'a> fmt::Display for PathDisplay<'a> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
use fmt::Write;
let mut first = true;
for segment in self.path.iter() {
let segment_str = segment.to_string_lossy();
if segment_str == "." {
continue;
}
if first {
first = false;
} else {
f.write_char(MAIN_SEPARATOR)?;
}
f.write_str(&segment_str)?;
}
Ok(())
}
}

9
nodtool/src/redump.rs
View File

@ -12,7 +12,7 @@ pub struct GameResult {
pub size: u64,
}
pub fn find_by_hashes(crc32: u32, sha1: [u8; 20]) -> Option<GameResult> {
pub fn find_by_crc32(crc32: u32) -> Option<GameResult> {
let header: &Header = Header::ref_from_prefix(&DATA.0).unwrap();
assert_eq!(header.entry_size as usize, size_of::<GameEntry>());
@ -25,13 +25,8 @@ pub fn find_by_hashes(crc32: u32, sha1: [u8; 20]) -> Option<GameResult> {
// Binary search by CRC32
let index = entries.binary_search_by_key(&crc32, |entry| entry.crc32).ok()?;
// Verify SHA-1
let entry = &entries[index];
if entry.sha1 != sha1 {
return None;
}
// Parse the entry
let entry = &entries[index];
let offset = entry.string_table_offset as usize;
let name_size = u32::from_ne_bytes(*array_ref![string_table, offset, 4]) as usize;
let name = str::from_utf8(&string_table[offset + 4..offset + 4 + name_size]).unwrap();