diff --git a/Cargo.lock b/Cargo.lock index ddd0584..f39dc07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,7 +43,7 @@ dependencies = [ "cfg-if", "once_cell", "version_check", - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -414,6 +414,7 @@ dependencies = [ "crossterm", "cwdemangle", "cwextab 1.0.2", + "dyn-clone", "enable-ansi-support", "filetime", "fixedbitset 0.5.7", @@ -454,7 +455,7 @@ dependencies = [ "tracing-attributes", "tracing-subscriber", "xxhash-rust", - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -919,9 +920,9 @@ dependencies = [ [[package]] name = "nod" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69a1349ff4dfc0757d9b0537c6f7ef777ed414c183db59ae1e5faec4c998b8f1" +checksum = "75b9bd092c2ebed932654aa6de256e39d2156ce8c87ace31191f8086f6d22f02" dependencies = [ "adler", "aes", @@ -938,15 +939,15 @@ dependencies = [ "rayon", "sha1", "thiserror", - "zerocopy", + "zerocopy 0.8.0", "zstd", ] [[package]] name = "nodtool" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f97653db7343722bc0a13001dfc9ec83491246dcf113ee5aa2fdefd0f37447" +checksum = "598b0c24bb98d0094d37e8dc8d83bb600d857c9e42a475806ac1667ec52afbb3" dependencies = [ "argp", "base16ct", @@ -969,7 +970,7 @@ dependencies = [ "tracing-attributes", "tracing-subscriber", "xxhash-rust", - "zerocopy", + "zerocopy 0.8.0", "zstd", ] @@ -2160,7 +2161,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df7885ffcb82507a0f213c593e77c5f13d12cb96588d4e835ad7e9423ba034db" +dependencies = [ + "zerocopy-derive 0.8.0", ] [[package]] @@ -2174,6 +2184,17 @@ dependencies = [ "syn 2.0.79", ] +[[package]] +name = "zerocopy-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "930ad75608219e8ffdb8962a5433cb2b30064c7ccb564d3b76c2963390b1e435" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "zstd" version = "0.13.2" diff --git a/Cargo.toml b/Cargo.toml index 73089c4..5eab5be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ publish = false repository = "https://github.com/encounter/decomp-toolkit" readme = "README.md" categories = ["command-line-utilities"] -rust-version = "1.73.0" +rust-version = "1.80.0" [[bin]] name = "dtk" @@ -33,6 +33,7 @@ base64 = "0.22" crossterm = "0.28" cwdemangle = "1.0" cwextab = "1.0.2" +dyn-clone = "1.0" enable-ansi-support = "0.2" filetime = "0.2" fixedbitset = "0.5" @@ -47,7 +48,7 @@ memchr = "2.7" memmap2 = "0.9" multimap = "0.10" nintendo-lz = "0.1" -nodtool = "1.3" +nodtool = "1.4" #nodtool = { path = "../nod-rs/nodtool" } num_enum = "0.7" objdiff-core = { version = "2.1", features = ["ppc"] } diff --git a/src/cmd/alf.rs b/src/cmd/alf.rs index 8356e3e..4efd5a1 100644 --- a/src/cmd/alf.rs +++ b/src/cmd/alf.rs @@ -10,9 +10,10 @@ use crate::{ cmd, util::{ alf::AlfFile, - file::{buf_writer, map_file}, + file::buf_writer, reader::{Endian, FromReader}, }, + vfs::open_path, }; #[derive(FromArgs, PartialEq, Debug)] @@ -60,9 +61,8 @@ pub fn run(args: Args) -> Result<()> { fn hashes(args: HashesArgs) -> Result<()> { let alf_file = { - let file = map_file(&args.alf_file)?; - let mut reader = file.as_reader(); - AlfFile::from_reader(&mut reader, Endian::Little)? + let mut file = open_path(&args.alf_file, true)?; + AlfFile::from_reader(file.as_mut(), Endian::Little)? }; let mut w: Box = if let Some(output) = args.output { Box::new(buf_writer(output)?) diff --git a/src/cmd/ar.rs b/src/cmd/ar.rs index e61fcf8..42b0283 100644 --- a/src/cmd/ar.rs +++ b/src/cmd/ar.rs @@ -9,7 +9,10 @@ use anyhow::{anyhow, bail, Context, Result}; use argp::FromArgs; use object::{Object, ObjectSymbol, SymbolScope}; -use crate::util::file::{buf_writer, map_file, map_file_basic, process_rsp}; +use crate::{ + util::file::{buf_writer, process_rsp}, + vfs::open_path, +}; #[derive(FromArgs, PartialEq, Debug)] /// Commands for processing static libraries. @@ -80,8 +83,8 @@ fn create(args: CreateArgs) -> Result<()> { Entry::Vacant(e) => e.insert(Vec::new()), Entry::Occupied(_) => bail!("Duplicate file name '{path_str}'"), }; - let file = map_file_basic(path)?; - let obj = object::File::parse(file.as_slice())?; + let mut file = open_path(path, false)?; + let obj = object::File::parse(file.map()?)?; for symbol in obj.symbols() { if symbol.scope() == SymbolScope::Dynamic { entries.push(symbol.name_bytes()?.to_vec()); @@ -126,8 +129,8 @@ fn extract(args: ExtractArgs) -> Result<()> { println!("Extracting {} to {}", path.display(), out_dir.display()); } - let file = map_file(path)?; - let mut archive = ar::Archive::new(file.as_slice()); + let mut file = open_path(path, false)?; + let mut archive = ar::Archive::new(file.map()?); while let Some(entry) = archive.next_entry() { let mut entry = entry.with_context(|| format!("Processing entry in {}", path.display()))?; diff --git a/src/cmd/dol.rs b/src/cmd/dol.rs index 87e157f..ae24f5d 100644 --- a/src/cmd/dol.rs +++ b/src/cmd/dol.rs @@ -5,7 +5,7 @@ use std::{ ffi::OsStr, fs, fs::DirBuilder, - io::{Cursor, Write}, + io::{Cursor, Seek, Write}, mem::take, path::{Path, PathBuf}, time::Instant, @@ -48,10 +48,7 @@ use crate::{ diff::{calc_diff_ranges, print_diff, process_code}, dol::process_dol, elf::{process_elf, write_elf}, - file::{ - buf_reader, buf_writer, map_file, map_file_basic, touch, verify_hash, FileIterator, - FileReadInfo, - }, + file::{buf_writer, touch, verify_hash, FileIterator, FileReadInfo}, lcf::{asm_path_for_unit, generate_ldscript, obj_path_for_unit}, map::apply_map_file, rel::{process_rel, process_rel_header, update_rel_section_alignment}, @@ -59,6 +56,7 @@ use crate::{ split::{is_linker_generated_object, split_obj, update_splits}, IntoCow, ToCow, }, + vfs::{open_fs, open_path, open_path_fs, ArchiveKind, Vfs, VfsFile}, }; #[derive(FromArgs, PartialEq, Debug)] @@ -236,6 +234,9 @@ pub struct ProjectConfig { /// Marks all emitted symbols as "exported" to prevent the linker from removing them. #[serde(default = "bool_true", skip_serializing_if = "is_true")] pub export_all: bool, + /// Optional base path for all object files. + #[serde(default, skip_serializing_if = "is_default")] + pub object_base: Option, } impl Default for ProjectConfig { @@ -254,6 +255,7 @@ impl Default for ProjectConfig { symbols_known: false, fill_gaps: true, export_all: true, + object_base: None, } } } @@ -483,8 +485,8 @@ fn apply_selfile(obj: &mut ObjInfo, buf: &[u8]) -> Result<()> { pub fn info(args: InfoArgs) -> Result<()> { let mut obj = { - let file = map_file(&args.dol_file)?; - process_dol(file.as_slice(), "")? + let mut file = open_path(&args.dol_file, true)?; + process_dol(file.map()?, "")? }; apply_signatures(&mut obj)?; @@ -502,8 +504,8 @@ pub fn info(args: InfoArgs) -> Result<()> { apply_signatures_post(&mut obj)?; if let Some(selfile) = &args.selfile { - let file = map_file(selfile)?; - apply_selfile(&mut obj, file.as_slice())?; + let mut file = open_path(selfile, true)?; + apply_selfile(&mut obj, file.map()?)?; } println!("{}:", obj.name); @@ -787,16 +789,18 @@ struct AnalyzeResult { splits_cache: Option, } -fn load_analyze_dol(config: &ProjectConfig) -> Result { - log::debug!("Loading {}", config.base.object.display()); +fn load_analyze_dol(config: &ProjectConfig, object_base: &ObjectBase) -> Result { + let object_path = object_base.join(&config.base.object); + log::debug!("Loading {}", object_path.display()); let mut obj = { - let file = map_file(&config.base.object)?; + let mut file = object_base.open(&config.base.object)?; + let data = file.map()?; if let Some(hash_str) = &config.base.hash { - verify_hash(file.as_slice(), hash_str)?; + verify_hash(data, hash_str)?; } - process_dol(file.as_slice(), config.base.name().as_ref())? + process_dol(data, config.base.name().as_ref())? }; - let mut dep = vec![config.base.object.clone()]; + let mut dep = vec![object_path]; if let Some(comment_version) = config.mw_comment_version { obj.mw_comment = Some(MWComment::new(comment_version)?); @@ -843,11 +847,12 @@ fn load_analyze_dol(config: &ProjectConfig) -> Result { if let Some(selfile) = &config.selfile { log::info!("Loading {}", selfile.display()); - let file = map_file(selfile)?; + let mut file = open_path(selfile, true)?; + let data = file.map()?; if let Some(hash) = &config.selfile_hash { - verify_hash(file.as_slice(), hash)?; + verify_hash(data, hash)?; } - apply_selfile(&mut obj, file.as_slice())?; + apply_selfile(&mut obj, data)?; dep.push(selfile.clone()); } @@ -1004,12 +1009,11 @@ fn split_write_obj( fn write_if_changed(path: &Path, contents: &[u8]) -> Result<()> { if path.is_file() { - let old_file = map_file_basic(path)?; + let mut old_file = open_path(path, true)?; + let old_data = old_file.map()?; // If the file is the same size, check if the contents are the same // Avoid writing if unchanged, since it will update the file's mtime - if old_file.len() == contents.len() as u64 - && xxh3_64(old_file.as_slice()) == xxh3_64(contents) - { + if old_data.len() == contents.len() && xxh3_64(old_data) == xxh3_64(contents) { return Ok(()); } } @@ -1018,20 +1022,26 @@ fn write_if_changed(path: &Path, contents: &[u8]) -> Result<()> { Ok(()) } -fn load_analyze_rel(config: &ProjectConfig, module_config: &ModuleConfig) -> Result { - debug!("Loading {}", module_config.object.display()); - let file = map_file(&module_config.object)?; +fn load_analyze_rel( + config: &ProjectConfig, + object_base: &ObjectBase, + module_config: &ModuleConfig, +) -> Result { + let object_path = object_base.join(&module_config.object); + debug!("Loading {}", object_path.display()); + let mut file = object_base.open(&module_config.object)?; + let data = file.map()?; if let Some(hash_str) = &module_config.hash { - verify_hash(file.as_slice(), hash_str)?; + verify_hash(data, hash_str)?; } let (header, mut module_obj) = - process_rel(&mut Cursor::new(file.as_slice()), module_config.name().as_ref())?; + process_rel(&mut Cursor::new(data), module_config.name().as_ref())?; if let Some(comment_version) = config.mw_comment_version { module_obj.mw_comment = Some(MWComment::new(comment_version)?); } - let mut dep = vec![module_config.object.clone()]; + let mut dep = vec![object_path]; if let Some(map_path) = &module_config.map { apply_map_file(map_path, &mut module_obj, None, None)?; dep.push(map_path.clone()); @@ -1082,22 +1092,24 @@ fn load_analyze_rel(config: &ProjectConfig, module_config: &ModuleConfig) -> Res fn split(args: SplitArgs) -> Result<()> { if let Some(jobs) = args.jobs { - rayon::ThreadPoolBuilder::new().num_threads(jobs).build_global().unwrap(); + rayon::ThreadPoolBuilder::new().num_threads(jobs).build_global()?; } let command_start = Instant::now(); info!("Loading {}", args.config.display()); let mut config: ProjectConfig = { - let mut config_file = buf_reader(&args.config)?; - serde_yaml::from_reader(&mut config_file)? + let mut config_file = open_path(&args.config, true)?; + serde_yaml::from_reader(config_file.as_mut())? }; + let object_base = find_object_base(&config)?; for module_config in config.modules.iter_mut() { - let file = map_file(&module_config.object)?; + let mut file = object_base.open(&module_config.object)?; + let mut data = file.map()?; if let Some(hash_str) = &module_config.hash { - verify_hash(file.as_slice(), hash_str)?; + verify_hash(data, hash_str)?; } else { - module_config.hash = Some(file_sha1_string(&mut file.as_reader())?); + module_config.hash = Some(file_sha1_string(&mut data)?); } } @@ -1121,7 +1133,7 @@ fn split(args: SplitArgs) -> Result<()> { s.spawn(|_| { let _span = info_span!("module", name = %config.base.name()).entered(); dol_result = - Some(load_analyze_dol(&config).with_context(|| { + Some(load_analyze_dol(&config, &object_base).with_context(|| { format!("While loading object '{}'", config.base.file_name()) })); }); @@ -1133,7 +1145,7 @@ fn split(args: SplitArgs) -> Result<()> { .par_iter() .map(|module_config| { let _span = info_span!("module", name = %module_config.name()).entered(); - load_analyze_rel(&config, module_config).with_context(|| { + load_analyze_rel(&config, &object_base, module_config).with_context(|| { format!("While loading object '{}'", module_config.file_name()) }) }) @@ -1538,16 +1550,18 @@ fn symbol_name_fuzzy_eq(a: &ObjSymbol, b: &ObjSymbol) -> bool { fn diff(args: DiffArgs) -> Result<()> { log::info!("Loading {}", args.config.display()); - let mut config_file = buf_reader(&args.config)?; - let config: ProjectConfig = serde_yaml::from_reader(&mut config_file)?; + let mut config_file = open_path(&args.config, true)?; + let config: ProjectConfig = serde_yaml::from_reader(config_file.as_mut())?; + let object_base = find_object_base(&config)?; - log::info!("Loading {}", config.base.object.display()); + log::info!("Loading {}", object_base.join(&config.base.object).display()); let mut obj = { - let file = map_file(&config.base.object)?; + let mut file = object_base.open(&config.base.object)?; + let data = file.map()?; if let Some(hash_str) = &config.base.hash { - verify_hash(file.as_slice(), hash_str)?; + verify_hash(data, hash_str)?; } - process_dol(file.as_slice(), config.base.name().as_ref())? + process_dol(data, config.base.name().as_ref())? }; if let Some(symbols_path) = &config.base.symbols { @@ -1717,16 +1731,18 @@ fn diff(args: DiffArgs) -> Result<()> { fn apply(args: ApplyArgs) -> Result<()> { log::info!("Loading {}", args.config.display()); - let mut config_file = buf_reader(&args.config)?; - let config: ProjectConfig = serde_yaml::from_reader(&mut config_file)?; + let mut config_file = open_path(&args.config, true)?; + let config: ProjectConfig = serde_yaml::from_reader(config_file.as_mut())?; + let object_base = find_object_base(&config)?; - log::info!("Loading {}", config.base.object.display()); + log::info!("Loading {}", object_base.join(&config.base.object).display()); let mut obj = { - let file = map_file(&config.base.object)?; + let mut file = object_base.open(&config.base.object)?; + let data = file.map()?; if let Some(hash_str) = &config.base.hash { - verify_hash(file.as_slice(), hash_str)?; + verify_hash(data, hash_str)?; } - process_dol(file.as_slice(), config.base.name().as_ref())? + process_dol(data, config.base.name().as_ref())? }; let Some(symbols_path) = &config.base.symbols else { @@ -1881,30 +1897,31 @@ fn config(args: ConfigArgs) -> Result<()> { let mut config = ProjectConfig::default(); let mut modules = Vec::<(u32, ModuleConfig)>::new(); for result in FileIterator::new(&args.objects)? { - let (path, entry) = result?; + let (path, mut entry) = result?; log::info!("Loading {}", path.display()); match path.extension() { Some(ext) if ext.eq_ignore_ascii_case(OsStr::new("dol")) => { config.base.object = path; - config.base.hash = Some(file_sha1_string(&mut entry.as_reader())?); + config.base.hash = Some(file_sha1_string(&mut entry)?); } Some(ext) if ext.eq_ignore_ascii_case(OsStr::new("rel")) => { - let header = process_rel_header(&mut entry.as_reader())?; + let header = process_rel_header(&mut entry)?; + entry.rewind()?; modules.push((header.module_id, ModuleConfig { object: path, - hash: Some(file_sha1_string(&mut entry.as_reader())?), + hash: Some(file_sha1_string(&mut entry)?), ..Default::default() })); } Some(ext) if ext.eq_ignore_ascii_case(OsStr::new("sel")) => { config.selfile = Some(path); - config.selfile_hash = Some(file_sha1_string(&mut entry.as_reader())?); + config.selfile_hash = Some(file_sha1_string(&mut entry)?); } Some(ext) if ext.eq_ignore_ascii_case(OsStr::new("rso")) => { config.modules.push(ModuleConfig { object: path, - hash: Some(file_sha1_string(&mut entry.as_reader())?), + hash: Some(file_sha1_string(&mut entry)?), ..Default::default() }); } @@ -1975,3 +1992,52 @@ fn apply_add_relocations(obj: &mut ObjInfo, relocations: &[AddRelocationConfig]) } Ok(()) } + +pub enum ObjectBase { + None, + Directory(PathBuf), + Vfs(PathBuf, Box), +} + +impl ObjectBase { + pub fn join(&self, path: &Path) -> PathBuf { + match self { + ObjectBase::None => path.to_path_buf(), + ObjectBase::Directory(base) => base.join(path), + ObjectBase::Vfs(base, _) => { + PathBuf::from(format!("{}:{}", base.display(), path.display())) + } + } + } + + pub fn open(&self, path: &Path) -> Result> { + match self { + ObjectBase::None => open_path(path, true), + ObjectBase::Directory(base) => open_path(&base.join(path), true), + ObjectBase::Vfs(vfs_path, vfs) => open_path_fs(vfs.clone(), path, true) + .with_context(|| format!("Using disc image {}", vfs_path.display())), + } + } +} + +pub fn find_object_base(config: &ProjectConfig) -> Result { + if let Some(base) = &config.object_base { + // Search for disc images in the object base directory + for result in base.read_dir()? { + let entry = result?; + if entry.file_type()?.is_file() { + let path = entry.path(); + let mut file = open_path(&path, false)?; + let format = nodtool::nod::Disc::detect(file.as_mut())?; + if format.is_some() { + file.rewind()?; + log::info!("Using disc image {}", path.display()); + let fs = open_fs(file, ArchiveKind::Disc)?; + return Ok(ObjectBase::Vfs(path, fs)); + } + } + } + return Ok(ObjectBase::Directory(base.clone())); + } + Ok(ObjectBase::None) +} diff --git a/src/cmd/dwarf.rs b/src/cmd/dwarf.rs index 4311024..a213f1a 100644 --- a/src/cmd/dwarf.rs +++ b/src/cmd/dwarf.rs @@ -16,12 +16,15 @@ use syntect::{ parsing::{ParseState, ScopeStack, SyntaxReference, SyntaxSet}, }; -use crate::util::{ - dwarf::{ - process_compile_unit, process_cu_tag, process_overlay_branch, read_debug_section, - should_skip_tag, tag_type_string, AttributeKind, TagKind, +use crate::{ + util::{ + dwarf::{ + process_compile_unit, process_cu_tag, process_overlay_branch, read_debug_section, + should_skip_tag, tag_type_string, AttributeKind, TagKind, + }, + file::buf_writer, }, - file::{buf_writer, map_file}, + vfs::open_path, }; #[derive(FromArgs, PartialEq, Debug)] @@ -73,8 +76,8 @@ fn dump(args: DumpArgs) -> Result<()> { let theme = theme_set.themes.get("Solarized (dark)").context("Failed to load theme")?; let syntax = syntax_set.find_syntax_by_name("C++").context("Failed to find syntax")?.clone(); - let file = map_file(&args.in_file)?; - let buf = file.as_slice(); + let mut file = open_path(&args.in_file, true)?; + let buf = file.map()?; if buf.starts_with(b"!\n") { let mut archive = ar::Archive::new(buf); while let Some(result) = archive.next_entry() { diff --git a/src/cmd/elf2dol.rs b/src/cmd/elf2dol.rs index 7304d11..352ae68 100644 --- a/src/cmd/elf2dol.rs +++ b/src/cmd/elf2dol.rs @@ -7,7 +7,7 @@ use anyhow::{anyhow, bail, ensure, Result}; use argp::FromArgs; use object::{Architecture, Endianness, Object, ObjectKind, ObjectSection, SectionKind}; -use crate::util::file::{buf_writer, map_file}; +use crate::{util::file::buf_writer, vfs::open_path}; #[derive(FromArgs, PartialEq, Eq, Debug)] /// Converts an ELF file to a DOL file. @@ -46,8 +46,8 @@ const MAX_TEXT_SECTIONS: usize = 7; const MAX_DATA_SECTIONS: usize = 11; pub fn run(args: Args) -> Result<()> { - let file = map_file(&args.elf_file)?; - let obj_file = object::read::File::parse(file.as_slice())?; + let mut file = open_path(&args.elf_file, true)?; + let obj_file = object::read::File::parse(file.map()?)?; match obj_file.architecture() { Architecture::PowerPc => {} arch => bail!("Unexpected architecture: {arch:?}"), diff --git a/src/cmd/map.rs b/src/cmd/map.rs index 2a332fb..cde20b4 100644 --- a/src/cmd/map.rs +++ b/src/cmd/map.rs @@ -5,11 +5,13 @@ use argp::FromArgs; use cwdemangle::{demangle, DemangleOptions}; use tracing::error; -use crate::util::{ - config::{write_splits_file, write_symbols_file}, - file::map_file, - map::{create_obj, process_map, SymbolEntry, SymbolRef}, - split::update_splits, +use crate::{ + util::{ + config::{write_splits_file, write_symbols_file}, + map::{create_obj, process_map, SymbolEntry, SymbolRef}, + split::update_splits, + }, + vfs::open_path, }; #[derive(FromArgs, PartialEq, Debug)] @@ -73,8 +75,8 @@ pub fn run(args: Args) -> Result<()> { } fn entries(args: EntriesArgs) -> Result<()> { - let file = map_file(&args.map_file)?; - let entries = process_map(&mut file.as_reader(), None, None)?; + let mut file = open_path(&args.map_file, true)?; + let entries = process_map(file.as_mut(), None, None)?; match entries.unit_entries.get_vec(&args.unit) { Some(vec) => { println!("Entries for {}:", args.unit); @@ -104,9 +106,9 @@ fn entries(args: EntriesArgs) -> Result<()> { } fn symbol(args: SymbolArgs) -> Result<()> { - let file = map_file(&args.map_file)?; + let mut file = open_path(&args.map_file, true)?; log::info!("Processing map..."); - let entries = process_map(&mut file.as_reader(), None, None)?; + let entries = process_map(file.as_mut(), None, None)?; log::info!("Done!"); let mut opt_ref: Option<(String, SymbolEntry)> = None; @@ -179,9 +181,9 @@ fn symbol(args: SymbolArgs) -> Result<()> { } fn config(args: ConfigArgs) -> Result<()> { - let file = map_file(&args.map_file)?; + let mut file = open_path(&args.map_file, true)?; log::info!("Processing map..."); - let entries = process_map(&mut file.as_reader(), None, None)?; + let entries = process_map(file.as_mut(), None, None)?; let mut obj = create_obj(&entries)?; if let Err(e) = update_splits(&mut obj, None, false) { error!("Failed to update splits: {}", e) diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs index 46e1ab3..beb82cf 100644 --- a/src/cmd/mod.rs +++ b/src/cmd/mod.rs @@ -14,5 +14,6 @@ pub mod rel; pub mod rso; pub mod shasum; pub mod u8_arc; +pub mod vfs; pub mod yay0; pub mod yaz0; diff --git a/src/cmd/nlzss.rs b/src/cmd/nlzss.rs index 62246fb..a4af753 100644 --- a/src/cmd/nlzss.rs +++ b/src/cmd/nlzss.rs @@ -3,9 +3,9 @@ use std::{fs, path::PathBuf}; use anyhow::{anyhow, Context, Result}; use argp::FromArgs; -use crate::util::{ - file::{open_file, process_rsp}, - IntoCow, ToCow, +use crate::{ + util::{file::process_rsp, IntoCow, ToCow}, + vfs::open_path, }; #[derive(FromArgs, PartialEq, Debug)] @@ -45,7 +45,8 @@ fn decompress(args: DecompressArgs) -> Result<()> { let files = process_rsp(&args.files)?; let single_file = files.len() == 1; for path in files { - let data = nintendo_lz::decompress(&mut open_file(&path)?) + let mut file = open_path(&path, false)?; + let data = nintendo_lz::decompress(&mut file) .map_err(|e| anyhow!("Failed to decompress '{}' with NLZSS: {}", path.display(), e))?; let out_path = if let Some(output) = &args.output { if single_file { diff --git a/src/cmd/rarc.rs b/src/cmd/rarc.rs index 3208dd5..f050d6b 100644 --- a/src/cmd/rarc.rs +++ b/src/cmd/rarc.rs @@ -1,11 +1,11 @@ -use std::{fs, fs::DirBuilder, path::PathBuf}; +use std::path::PathBuf; -use anyhow::{Context, Result}; +use anyhow::{anyhow, Result}; use argp::FromArgs; -use crate::util::{ - file::{decompress_if_needed, map_file}, - rarc::{Node, RarcReader}, +use crate::{ + util::rarc::{RarcNodeKind, RarcView}, + vfs::open_path, }; #[derive(FromArgs, PartialEq, Debug)] @@ -55,71 +55,42 @@ pub fn run(args: Args) -> Result<()> { } fn list(args: ListArgs) -> Result<()> { - let file = map_file(&args.file)?; - let rarc = RarcReader::new(&mut file.as_reader()) - .with_context(|| format!("Failed to process RARC file '{}'", args.file.display()))?; - - let mut current_path = PathBuf::new(); - for node in rarc.nodes() { - match node { - Node::DirectoryBegin { name } => { - current_path.push(name.name); - } - Node::DirectoryEnd { name: _ } => { - current_path.pop(); - } - Node::File { name, offset, size } => { - let path = current_path.join(name.name); - println!("{}: {} bytes, offset {:#X}", path.display(), size, offset); - } - Node::CurrentDirectory => {} - Node::ParentDirectory => {} - } - } + let mut file = open_path(&args.file, true)?; + let view = RarcView::new(file.map()?).map_err(|e| anyhow!(e))?; + test(&view, "")?; + test(&view, "/")?; + test(&view, "//")?; + test(&view, "/rels")?; + test(&view, "/rels/")?; + test(&view, "/rels/amem")?; + test(&view, "/rels/amem/")?; + test(&view, "/rels/mmem")?; + test(&view, "/rels/mmem/../mmem")?; + test(&view, "/rels/amem/d_a_am.rel")?; + test(&view, "//amem/d_a_am.rel")?; + test(&view, "amem/d_a_am.rel")?; + test(&view, "amem/d_a_am.rel/")?; + test(&view, "mmem/d_a_obj_pirateship.rel")?; + test(&view, "mmem//d_a_obj_pirateship.rel")?; + test(&view, "mmem/da_obj_pirateship.rel")?; Ok(()) } -fn extract(args: ExtractArgs) -> Result<()> { - let file = map_file(&args.file)?; - let rarc = RarcReader::new(&mut file.as_reader()) - .with_context(|| format!("Failed to process RARC file '{}'", args.file.display()))?; - - let mut current_path = PathBuf::new(); - for node in rarc.nodes() { - match node { - Node::DirectoryBegin { name } => { - current_path.push(name.name); - } - Node::DirectoryEnd { name: _ } => { - current_path.pop(); - } - Node::File { name, offset, size } => { - let file_data = decompress_if_needed( - &file.as_slice()[offset as usize..offset as usize + size as usize], - )?; - let file_path = current_path.join(&name.name); - let output_path = args - .output - .as_ref() - .map(|p| p.join(&file_path)) - .unwrap_or_else(|| file_path.clone()); - if !args.quiet { - println!( - "Extracting {} to {} ({} bytes)", - file_path.display(), - output_path.display(), - size - ); - } - if let Some(parent) = output_path.parent() { - DirBuilder::new().recursive(true).create(parent)?; - } - fs::write(&output_path, file_data) - .with_context(|| format!("Failed to write file '{}'", output_path.display()))?; - } - Node::CurrentDirectory => {} - Node::ParentDirectory => {} - } - } +fn test(view: &RarcView, path: &str) -> Result<()> { + let option = view.find(path); + let data = if let Some(RarcNodeKind::File(_, node)) = option { + view.get_data(node).map_err(|e| anyhow!(e))? + } else { + &[] + }; + let vec = data.iter().cloned().take(4).collect::>(); + println!("{:?}: {:?} (len: {:?})", path, option, vec.as_slice()); + // if let Some(RarcNodeKind::Directory(_, dir)) = option { + // for node in view.children(dir) { + // println!("Child: {:?} ({:?})", node, view.get_string(node.name_offset())); + // } + // } Ok(()) } + +fn extract(_args: ExtractArgs) -> Result<()> { todo!() } diff --git a/src/cmd/rel.rs b/src/cmd/rel.rs index d8c24ba..9eba279 100644 --- a/src/cmd/rel.rs +++ b/src/cmd/rel.rs @@ -1,7 +1,7 @@ use std::{ collections::{btree_map, BTreeMap}, fs, - io::Write, + io::{Cursor, Write}, path::PathBuf, time::Instant, }; @@ -27,13 +27,13 @@ use crate::{ tracker::Tracker, }, array_ref_mut, - cmd::dol::{ModuleConfig, ProjectConfig}, + cmd::dol::{find_object_base, ModuleConfig, ObjectBase, ProjectConfig}, obj::{ObjInfo, ObjReloc, ObjRelocKind, ObjSection, ObjSectionKind, ObjSymbol}, util::{ config::{is_auto_symbol, read_splits_sections, SectionDef}, dol::process_dol, elf::{to_obj_reloc_kind, write_elf}, - file::{buf_reader, buf_writer, map_file, process_rsp, verify_hash, FileIterator}, + file::{buf_writer, process_rsp, verify_hash, FileIterator}, nested::NestedMap, rel::{ print_relocations, process_rel, process_rel_header, process_rel_sections, write_rel, @@ -41,6 +41,7 @@ use crate::{ }, IntoCow, ToCow, }, + vfs::open_path, }; #[derive(FromArgs, PartialEq, Debug)] @@ -162,12 +163,13 @@ fn match_section_index( // }) } -fn load_rel(module_config: &ModuleConfig) -> Result { - let file = map_file(&module_config.object)?; +fn load_rel(module_config: &ModuleConfig, object_base: &ObjectBase) -> Result { + let mut file = object_base.open(&module_config.object)?; + let data = file.map()?; if let Some(hash_str) = &module_config.hash { - verify_hash(file.as_slice(), hash_str)?; + verify_hash(data, hash_str)?; } - let mut reader = file.as_reader(); + let mut reader = Cursor::new(data); let header = process_rel_header(&mut reader)?; let sections = process_rel_sections(&mut reader, &header)?; let section_defs = if let Some(splits_path) = &module_config.splits { @@ -261,15 +263,19 @@ fn make(args: MakeArgs) -> Result<()> { let mut existing_headers = BTreeMap::::new(); let mut name_to_module_id = FxHashMap::::default(); if let Some(config_path) = &args.config { - let config: ProjectConfig = serde_yaml::from_reader(&mut buf_reader(config_path)?)?; + let config: ProjectConfig = { + let mut file = open_path(config_path, true)?; + serde_yaml::from_reader(file.as_mut())? + }; + let object_base = find_object_base(&config)?; for module_config in &config.modules { let module_name = module_config.name(); if !args.names.is_empty() && !args.names.iter().any(|n| n == &module_name) { continue; } let _span = info_span!("module", name = %module_name).entered(); - let info = load_rel(module_config).with_context(|| { - format!("While loading REL '{}'", module_config.object.display()) + let info = load_rel(module_config, &object_base).with_context(|| { + format!("While loading REL '{}'", object_base.join(&module_config.object).display()) })?; name_to_module_id.insert(module_name.to_string(), info.0.module_id); match existing_headers.entry(info.0.module_id) { @@ -287,9 +293,9 @@ fn make(args: MakeArgs) -> Result<()> { } // Load all modules - let files = paths.iter().map(map_file).collect::>>()?; + let mut files = paths.iter().map(|p| open_path(p, true)).collect::>>()?; let modules = files - .par_iter() + .par_iter_mut() .enumerate() .zip(&paths) .map(|((idx, file), path)| { @@ -301,7 +307,7 @@ fn make(args: MakeArgs) -> Result<()> { .and_then(|n| name_to_module_id.get(n)) .copied() .unwrap_or(idx as u32); - load_obj(file.as_slice()) + load_obj(file.map()?) .map(|o| LoadedModule { module_id, file: o, path: path.clone() }) .with_context(|| format!("Failed to load '{}'", path.display())) }) @@ -399,8 +405,8 @@ fn make(args: MakeArgs) -> Result<()> { } fn info(args: InfoArgs) -> Result<()> { - let file = map_file(args.rel_file)?; - let (header, mut module_obj) = process_rel(&mut file.as_reader(), "")?; + let mut file = open_path(&args.rel_file, true)?; + let (header, mut module_obj) = process_rel(file.as_mut(), "")?; let mut state = AnalyzerState::default(); state.detect_functions(&module_obj)?; @@ -458,7 +464,7 @@ fn info(args: InfoArgs) -> Result<()> { if args.relocations { println!("\nRelocations:"); println!(" [Source] section:address RelocType -> [Target] module:section:address"); - print_relocations(&mut file.as_reader(), &header)?; + print_relocations(file.as_mut(), &header)?; } Ok(()) } @@ -469,9 +475,9 @@ const fn align32(x: u32) -> u32 { (x + 31) & !31 } fn merge(args: MergeArgs) -> Result<()> { log::info!("Loading {}", args.dol_file.display()); let mut obj = { - let file = map_file(&args.dol_file)?; + let mut file = open_path(&args.dol_file, true)?; let name = args.dol_file.file_stem().map(|s| s.to_string_lossy()).unwrap_or_default(); - process_dol(file.as_slice(), name.as_ref())? + process_dol(file.map()?, name.as_ref())? }; log::info!("Performing signature analysis"); @@ -481,10 +487,10 @@ fn merge(args: MergeArgs) -> Result<()> { let mut processed = 0; let mut module_map = BTreeMap::::new(); for result in FileIterator::new(&args.rel_files)? { - let (path, entry) = result?; + let (path, mut entry) = result?; log::info!("Loading {}", path.display()); let name = path.file_stem().map(|s| s.to_string_lossy()).unwrap_or_default(); - let (_, obj) = process_rel(&mut entry.as_reader(), name.as_ref())?; + let (_, obj) = process_rel(&mut entry, name.as_ref())?; match module_map.entry(obj.module_id) { btree_map::Entry::Vacant(e) => e.insert(obj), btree_map::Entry::Occupied(_) => bail!("Duplicate module ID {}", obj.module_id), diff --git a/src/cmd/rso.rs b/src/cmd/rso.rs index 6875c7d..2bd606c 100644 --- a/src/cmd/rso.rs +++ b/src/cmd/rso.rs @@ -11,13 +11,16 @@ use object::{ SymbolIndex, SymbolKind, SymbolSection, }; -use crate::util::{ - file::{buf_reader, buf_writer, map_file}, - reader::{Endian, ToWriter}, - rso::{ - process_rso, symbol_hash, RsoHeader, RsoRelocation, RsoSectionHeader, RsoSymbol, - RSO_SECTION_NAMES, +use crate::{ + util::{ + file::buf_writer, + reader::{Endian, ToWriter}, + rso::{ + process_rso, symbol_hash, RsoHeader, RsoRelocation, RsoSectionHeader, RsoSymbol, + RSO_SECTION_NAMES, + }, }, + vfs::open_path, }; #[derive(FromArgs, PartialEq, Debug)] @@ -74,8 +77,8 @@ pub fn run(args: Args) -> Result<()> { fn info(args: InfoArgs) -> Result<()> { let rso = { - let file = map_file(args.rso_file)?; - let obj = process_rso(&mut file.as_reader())?; + let mut file = open_path(&args.rso_file, true)?; + let obj = process_rso(file.as_mut())?; #[allow(clippy::let_and_return)] obj }; @@ -84,8 +87,8 @@ fn info(args: InfoArgs) -> Result<()> { } fn make(args: MakeArgs) -> Result<()> { - let file = map_file(&args.input)?; - let obj_file = object::read::File::parse(file.as_slice())?; + let mut file = open_path(&args.input, true)?; + let obj_file = object::read::File::parse(file.map()?)?; match obj_file.architecture() { Architecture::PowerPc => {} arch => bail!("Unexpected architecture: {arch:?}"), @@ -97,9 +100,9 @@ fn make(args: MakeArgs) -> Result<()> { None => args.input.display().to_string(), }; - let symbols_to_export = match args.export { + let symbols_to_export = match &args.export { Some(export_file_path) => { - let export_file_reader = buf_reader(export_file_path)?; + let export_file_reader = open_path(export_file_path, true)?; export_file_reader.lines().map_while(Result::ok).collect() } None => vec![], diff --git a/src/cmd/shasum.rs b/src/cmd/shasum.rs index e600dbb..065b494 100644 --- a/src/cmd/shasum.rs +++ b/src/cmd/shasum.rs @@ -1,6 +1,6 @@ use std::{ fs::File, - io::{stdout, BufRead, BufReader, Read, Write}, + io::{stdout, BufRead, Read, Write}, path::{Path, PathBuf}, }; @@ -10,7 +10,10 @@ use owo_colors::{OwoColorize, Stream}; use path_slash::PathExt; use sha1::{Digest, Sha1}; -use crate::util::file::{buf_writer, open_file, process_rsp, touch}; +use crate::{ + util::file::{buf_writer, process_rsp, touch}, + vfs::open_path, +}; #[derive(FromArgs, PartialEq, Eq, Debug)] /// Print or check SHA1 (160-bit) checksums. @@ -36,8 +39,8 @@ const DEFAULT_BUF_SIZE: usize = 8192; pub fn run(args: Args) -> Result<()> { if args.check { for path in process_rsp(&args.files)? { - let file = open_file(&path)?; - check(&args, &mut BufReader::new(file))?; + let mut file = open_path(&path, false)?; + check(&args, file.as_mut())?; } if let Some(out_path) = &args.output { touch(out_path) @@ -53,8 +56,8 @@ pub fn run(args: Args) -> Result<()> { Box::new(stdout()) }; for path in process_rsp(&args.files)? { - let mut file = open_file(&path)?; - hash(w.as_mut(), &mut file, &path)? + let mut file = open_path(&path, false)?; + hash(w.as_mut(), file.as_mut(), &path)? } } Ok(()) diff --git a/src/cmd/u8_arc.rs b/src/cmd/u8_arc.rs index ecadb02..d0964cf 100644 --- a/src/cmd/u8_arc.rs +++ b/src/cmd/u8_arc.rs @@ -4,9 +4,12 @@ use anyhow::{anyhow, Context, Result}; use argp::FromArgs; use itertools::Itertools; -use crate::util::{ - file::{decompress_if_needed, map_file}, - u8_arc::{U8Node, U8View}, +use crate::{ + util::{ + file::decompress_if_needed, + u8_arc::{U8Node, U8View}, + }, + vfs::open_path, }; #[derive(FromArgs, PartialEq, Debug)] @@ -56,8 +59,8 @@ pub fn run(args: Args) -> Result<()> { } fn list(args: ListArgs) -> Result<()> { - let file = map_file(&args.file)?; - let view = U8View::new(file.as_slice()) + let mut file = open_path(&args.file, true)?; + let view = U8View::new(file.map()?) .map_err(|e| anyhow!("Failed to open U8 file '{}': {}", args.file.display(), e))?; visit_files(&view, |_, node, path| { println!("{}: {} bytes, offset {:#X}", path, node.length(), node.offset()); @@ -66,15 +69,15 @@ fn list(args: ListArgs) -> Result<()> { } fn extract(args: ExtractArgs) -> Result<()> { - let file = map_file(&args.file)?; - let view = U8View::new(file.as_slice()) + let mut file = open_path(&args.file, true)?; + let data = file.map()?; + let view = U8View::new(data) .map_err(|e| anyhow!("Failed to open U8 file '{}': {}", args.file.display(), e))?; visit_files(&view, |_, node, path| { let offset = node.offset(); let size = node.length(); - let file_data = decompress_if_needed( - &file.as_slice()[offset as usize..offset as usize + size as usize], - )?; + let file_data = + decompress_if_needed(&data[offset as usize..offset as usize + size as usize])?; let output_path = args .output .as_ref() @@ -94,7 +97,7 @@ fn extract(args: ExtractArgs) -> Result<()> { fn visit_files( view: &U8View, - mut visitor: impl FnMut(usize, &U8Node, String) -> Result<()>, + mut visitor: impl FnMut(usize, U8Node, String) -> Result<()>, ) -> Result<()> { let mut path_segments = Vec::<(Cow, usize)>::new(); for (idx, node, name) in view.iter() { diff --git a/src/cmd/vfs.rs b/src/cmd/vfs.rs new file mode 100644 index 0000000..605f21b --- /dev/null +++ b/src/cmd/vfs.rs @@ -0,0 +1,121 @@ +use std::{fs::File, io, io::Write, path::PathBuf}; + +use anyhow::{anyhow, bail}; +use argp::FromArgs; +use nodtool::nod::ResultContext; + +use crate::vfs::{decompress_file, detect, open_fs, FileFormat, StdFs, Vfs, VfsFileType}; + +#[derive(FromArgs, PartialEq, Debug)] +/// Commands for interacting with discs and containers. +#[argp(subcommand, name = "vfs")] +pub struct Args { + #[argp(subcommand)] + command: SubCommand, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argp(subcommand)] +enum SubCommand { + Ls(LsArgs), + Cp(CpArgs), +} + +#[derive(FromArgs, PartialEq, Eq, Debug)] +/// List files in a directory or container. +#[argp(subcommand, name = "ls")] +pub struct LsArgs { + #[argp(positional)] + /// Path to the container. + path: PathBuf, +} + +#[derive(FromArgs, PartialEq, Eq, Debug)] +/// Copy files from a container. +#[argp(subcommand, name = "cp")] +pub struct CpArgs { + #[argp(positional)] + /// Source path(s) and destination path. + paths: Vec, +} + +pub fn run(args: Args) -> anyhow::Result<()> { + match args.command { + SubCommand::Ls(args) => ls(args), + SubCommand::Cp(args) => cp(args), + } +} + +fn find(path: &str) -> anyhow::Result<(Box, &str)> { + let mut split = path.split(':'); + let mut fs: Box = Box::new(StdFs); + let mut path = split.next().unwrap(); + for next in split { + let mut file = fs.open(path)?; + match detect(file.as_mut())? { + FileFormat::Archive(kind) => { + fs = open_fs(file, kind)?; + path = next; + } + _ => bail!("'{}' is not a container", path), + } + } + Ok((fs, path)) +} + +fn ls(args: LsArgs) -> anyhow::Result<()> { + let str = args.path.to_str().ok_or_else(|| anyhow!("Path is not valid UTF-8"))?; + let (mut fs, mut path) = find(str)?; + let metadata = fs.metadata(path)?; + if metadata.is_file() { + let mut file = fs.open(path)?; + match detect(file.as_mut())? { + FileFormat::Archive(kind) => { + fs = open_fs(file, kind)?; + path = ""; + } + _ => bail!("'{}' is not a directory", path), + } + } + let entries = fs.read_dir(path)?; + for entry in entries { + println!("{}", entry); + } + Ok(()) +} + +fn cp(mut args: CpArgs) -> anyhow::Result<()> { + if args.paths.len() < 2 { + bail!("Both source and destination paths must be provided"); + } + let dest = args.paths.pop().unwrap(); + let dest_is_dir = args.paths.len() > 1 || dest.metadata().ok().is_some_and(|m| m.is_dir()); + for path in args.paths { + let str = path.to_str().ok_or_else(|| anyhow!("Path is not valid UTF-8"))?; + let (mut fs, path) = find(str)?; + let metadata = fs.metadata(path)?; + match metadata.file_type { + VfsFileType::File => { + let mut file = fs.open(path)?; + if let FileFormat::Compressed(kind) = detect(file.as_mut())? { + file = decompress_file(file, kind)?; + } + let dest = if dest_is_dir { + let name = path.rsplit('/').next().unwrap(); + dest.join(name) + } else { + dest.clone() + }; + let mut dest_file = File::create(&dest) + .with_context(|| format!("Failed to create file {}", dest.display()))?; + io::copy(file.as_mut(), &mut dest_file) + .with_context(|| format!("Failed to write file {}", dest.display()))?; + dest_file + .flush() + .with_context(|| format!("Failed to flush file {}", dest.display()))?; + } + VfsFileType::Directory => bail!("Cannot copy directory"), + } + } + Ok(()) +} diff --git a/src/cmd/yay0.rs b/src/cmd/yay0.rs index 6369573..030515a 100644 --- a/src/cmd/yay0.rs +++ b/src/cmd/yay0.rs @@ -3,10 +3,13 @@ use std::{fs, path::PathBuf}; use anyhow::{Context, Result}; use argp::FromArgs; -use crate::util::{ - file::{map_file_basic, process_rsp}, - ncompress::{compress_yay0, decompress_yay0}, - IntoCow, ToCow, +use crate::{ + util::{ + file::process_rsp, + ncompress::{compress_yay0, decompress_yay0}, + IntoCow, ToCow, + }, + vfs::open_path, }; #[derive(FromArgs, PartialEq, Debug)] @@ -62,8 +65,8 @@ fn compress(args: CompressArgs) -> Result<()> { let single_file = files.len() == 1; for path in files { let data = { - let file = map_file_basic(&path)?; - compress_yay0(file.as_slice()) + let mut file = open_path(&path, false)?; + compress_yay0(file.map()?) }; let out_path = if let Some(output) = &args.output { if single_file { @@ -85,8 +88,8 @@ fn decompress(args: DecompressArgs) -> Result<()> { let single_file = files.len() == 1; for path in files { let data = { - let file = map_file_basic(&path)?; - decompress_yay0(file.as_slice()) + let mut file = open_path(&path, true)?; + decompress_yay0(file.map()?) .with_context(|| format!("Failed to decompress '{}' using Yay0", path.display()))? }; let out_path = if let Some(output) = &args.output { diff --git a/src/cmd/yaz0.rs b/src/cmd/yaz0.rs index b21a0cc..7e60e1a 100644 --- a/src/cmd/yaz0.rs +++ b/src/cmd/yaz0.rs @@ -3,10 +3,13 @@ use std::{fs, path::PathBuf}; use anyhow::{Context, Result}; use argp::FromArgs; -use crate::util::{ - file::{map_file_basic, process_rsp}, - ncompress::{compress_yaz0, decompress_yaz0}, - IntoCow, ToCow, +use crate::{ + util::{ + file::process_rsp, + ncompress::{compress_yaz0, decompress_yaz0}, + IntoCow, ToCow, + }, + vfs::open_path, }; #[derive(FromArgs, PartialEq, Debug)] @@ -62,8 +65,8 @@ fn compress(args: CompressArgs) -> Result<()> { let single_file = files.len() == 1; for path in files { let data = { - let file = map_file_basic(&path)?; - compress_yaz0(file.as_slice()) + let mut file = open_path(&path, false)?; + compress_yaz0(file.map()?) }; let out_path = if let Some(output) = &args.output { if single_file { @@ -85,8 +88,8 @@ fn decompress(args: DecompressArgs) -> Result<()> { let single_file = files.len() == 1; for path in files { let data = { - let file = map_file_basic(&path)?; - decompress_yaz0(file.as_slice()) + let mut file = open_path(&path, false)?; + decompress_yaz0(file.map()?) .with_context(|| format!("Failed to decompress '{}' using Yaz0", path.display()))? }; let out_path = if let Some(output) = &args.output { diff --git a/src/main.rs b/src/main.rs index 44c8af8..dfe2d21 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,6 +12,7 @@ pub mod argp_version; pub mod cmd; pub mod obj; pub mod util; +pub mod vfs; // musl's allocator is very slow, so use mimalloc when targeting musl. // Otherwise, use the system allocator to avoid extra code size. @@ -102,6 +103,7 @@ enum SubCommand { Rso(cmd::rso::Args), Shasum(cmd::shasum::Args), U8(cmd::u8_arc::Args), + Vfs(cmd::vfs::Args), Yay0(cmd::yay0::Args), Yaz0(cmd::yaz0::Args), } @@ -177,6 +179,7 @@ fn main() { SubCommand::Rso(c_args) => cmd::rso::run(c_args), SubCommand::Shasum(c_args) => cmd::shasum::run(c_args), SubCommand::U8(c_args) => cmd::u8_arc::run(c_args), + SubCommand::Vfs(c_args) => cmd::vfs::run(c_args), SubCommand::Yay0(c_args) => cmd::yay0::run(c_args), SubCommand::Yaz0(c_args) => cmd::yaz0::run(c_args), }); diff --git a/src/util/config.rs b/src/util/config.rs index 3ea38ef..508f63c 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -21,9 +21,10 @@ use crate::{ ObjSymbolFlags, ObjSymbolKind, ObjUnit, }, util::{ - file::{buf_writer, map_file, FileReadInfo}, + file::{buf_writer, FileReadInfo}, split::default_section_align, }, + vfs::open_path, }; pub fn parse_u32(s: &str) -> Result { @@ -46,10 +47,11 @@ pub fn parse_i32(s: &str) -> Result { pub fn apply_symbols_file

(path: P, obj: &mut ObjInfo) -> Result> where P: AsRef { - Ok(if path.as_ref().is_file() { - let file = map_file(path)?; - let cached = FileReadInfo::new(&file)?; - for result in file.as_reader().lines() { + let path = path.as_ref(); + Ok(if path.is_file() { + let mut file = open_path(path, true)?; + let cached = FileReadInfo::new(file.as_mut())?; + for result in file.lines() { let line = match result { Ok(line) => line, Err(e) => bail!("Failed to process symbols file: {e:?}"), @@ -206,8 +208,8 @@ where // Check file mtime let path = path.as_ref(); let new_mtime = fs::metadata(path).ok().map(|m| FileTime::from_last_modification_time(&m)); - if let Some(new_mtime) = new_mtime { - if new_mtime != cached_file.mtime { + if let (Some(new_mtime), Some(old_mtime)) = (new_mtime, cached_file.mtime) { + if new_mtime != old_mtime { // File changed, don't write warn!(path = %path.display(), "File changed since read, not updating"); return Ok(()); @@ -625,10 +627,11 @@ enum SplitState { pub fn apply_splits_file

(path: P, obj: &mut ObjInfo) -> Result> where P: AsRef { - Ok(if path.as_ref().is_file() { - let file = map_file(path)?; - let cached = FileReadInfo::new(&file)?; - apply_splits(&mut file.as_reader(), obj)?; + let path = path.as_ref(); + Ok(if path.is_file() { + let mut file = open_path(path, true)?; + let cached = FileReadInfo::new(file.as_mut())?; + apply_splits(file.as_mut(), obj)?; Some(cached) } else { None @@ -737,14 +740,14 @@ where R: BufRead + ?Sized { pub fn read_splits_sections

(path: P) -> Result>> where P: AsRef { - if !path.as_ref().is_file() { + let path = path.as_ref(); + if !path.is_file() { return Ok(None); } - let file = map_file(path)?; - let r = file.as_reader(); + let file = open_path(path, true)?; let mut sections = Vec::new(); let mut state = SplitState::None; - for result in r.lines() { + for result in file.lines() { let line = match result { Ok(line) => line, Err(e) => return Err(e.into()), diff --git a/src/util/dep.rs b/src/util/dep.rs index 244b10c..ee8c241 100644 --- a/src/util/dep.rs +++ b/src/util/dep.rs @@ -4,37 +4,38 @@ use std::{ }; use itertools::Itertools; -use path_slash::PathBufExt; - -use crate::util::file::split_path; pub struct DepFile { - pub name: PathBuf, - pub dependencies: Vec, + pub name: String, + pub dependencies: Vec, +} + +fn normalize_path(path: &Path) -> String { + let path = path.to_string_lossy().replace('\\', "/"); + path.split_once(':').map(|(p, _)| p.to_string()).unwrap_or(path) } impl DepFile { - pub fn new(name: PathBuf) -> Self { Self { name, dependencies: vec![] } } + pub fn new(name: PathBuf) -> Self { + Self { name: name.to_string_lossy().into_owned(), dependencies: vec![] } + } pub fn push

(&mut self, dependency: P) where P: AsRef { - let path = split_path(dependency.as_ref()) - .map(|(p, _)| p) - .unwrap_or_else(|_| dependency.as_ref().to_path_buf()); + let path = dependency.as_ref().to_string_lossy().replace('\\', "/"); + let path = path.split_once(':').map(|(p, _)| p.to_string()).unwrap_or(path); self.dependencies.push(path); } pub fn extend(&mut self, dependencies: Vec) { - self.dependencies.extend(dependencies.iter().map(|dependency| { - split_path(dependency).map(|(p, _)| p).unwrap_or_else(|_| dependency.clone()) - })); + self.dependencies.extend(dependencies.iter().map(|dependency| normalize_path(dependency))); } pub fn write(&self, w: &mut W) -> std::io::Result<()> where W: Write + ?Sized { - write!(w, "{}:", self.name.to_slash_lossy())?; + write!(w, "{}:", self.name)?; for dep in self.dependencies.iter().unique() { - write!(w, " \\\n {}", dep.to_slash_lossy().replace(' ', "\\ "))?; + write!(w, " \\\n {}", dep.replace(' ', "\\ "))?; } Ok(()) } diff --git a/src/util/elf.rs b/src/util/elf.rs index cc35248..64ef9a3 100644 --- a/src/util/elf.rs +++ b/src/util/elf.rs @@ -29,9 +29,9 @@ use crate::{ }, util::{ comment::{CommentSym, MWComment}, - file::map_file, reader::{Endian, FromReader, ToWriter}, }, + vfs::open_path, }; pub const SHT_MWCATS: u32 = SHT_LOUSER + 0x4A2A82C2; @@ -47,8 +47,9 @@ enum BoundaryState { pub fn process_elf

(path: P) -> Result where P: AsRef { - let file = map_file(path)?; - let obj_file = object::read::File::parse(file.as_slice())?; + let path = path.as_ref(); + let mut file = open_path(path, true)?; + let obj_file = object::read::File::parse(file.map()?)?; let architecture = match obj_file.architecture() { Architecture::PowerPc => ObjArchitecture::PowerPc, arch => bail!("Unexpected architecture: {arch:?}"), diff --git a/src/util/file.rs b/src/util/file.rs index 659ab9a..845df5d 100644 --- a/src/util/file.rs +++ b/src/util/file.rs @@ -1,15 +1,13 @@ use std::{ - ffi::OsStr, fs::{DirBuilder, File, OpenOptions}, - io::{BufRead, BufReader, BufWriter, Cursor, Read, Seek, SeekFrom}, - path::{Component, Path, PathBuf}, + io, + io::{BufRead, BufWriter, Read, Seek, SeekFrom}, + path::{Path, PathBuf}, }; -use anyhow::{anyhow, bail, Context, Result}; +use anyhow::{anyhow, Context, Result}; use filetime::{set_file_mtime, FileTime}; -use memmap2::{Mmap, MmapOptions}; use path_slash::PathBufExt; -use rarc::RarcReader; use sha1::{Digest, Sha1}; use xxhash_rust::xxh3::xxh3_64; @@ -17,194 +15,11 @@ use crate::{ array_ref, util::{ ncompress::{decompress_yay0, decompress_yaz0, YAY0_MAGIC, YAZ0_MAGIC}, - rarc, - rarc::{Node, RARC_MAGIC}, - take_seek::{TakeSeek, TakeSeekExt}, - u8_arc::{U8View, U8_MAGIC}, Bytes, }, + vfs::{open_path, VfsFile}, }; -pub struct MappedFile { - mmap: Mmap, - mtime: FileTime, - offset: u64, - len: u64, -} - -impl MappedFile { - pub fn as_reader(&self) -> Cursor<&[u8]> { Cursor::new(self.as_slice()) } - - pub fn as_slice(&self) -> &[u8] { - &self.mmap[self.offset as usize..self.offset as usize + self.len as usize] - } - - pub fn len(&self) -> u64 { self.len } - - pub fn is_empty(&self) -> bool { self.len == 0 } - - pub fn into_inner(self) -> Mmap { self.mmap } -} - -pub fn split_path

(path: P) -> Result<(PathBuf, Option)> -where P: AsRef { - let mut base_path = PathBuf::new(); - let mut sub_path: Option = None; - for component in path.as_ref().components() { - if let Component::Normal(str) = component { - let str = str.to_str().ok_or(anyhow!("Path is not valid UTF-8"))?; - if let Some((a, b)) = str.split_once(':') { - base_path.push(a); - sub_path = Some(PathBuf::from(b)); - continue; - } - } - if let Some(sub_path) = &mut sub_path { - sub_path.push(component); - } else { - base_path.push(component); - } - } - Ok((base_path, sub_path)) -} - -/// Opens a memory mapped file, and decompresses it if needed. -pub fn map_file

(path: P) -> Result -where P: AsRef { - let (base_path, sub_path) = split_path(path.as_ref())?; - let file = File::open(&base_path) - .with_context(|| format!("Failed to open file '{}'", base_path.display()))?; - let mtime = FileTime::from_last_modification_time(&file.metadata()?); - let mmap = unsafe { MmapOptions::new().map(&file) } - .with_context(|| format!("Failed to mmap file: '{}'", base_path.display()))?; - let (offset, len) = if let Some(sub_path) = sub_path { - if sub_path.as_os_str() == OsStr::new("nlzss") { - return Ok(FileEntry::Buffer( - nintendo_lz::decompress(&mut mmap.as_ref()) - .map_err(|e| { - anyhow!( - "Failed to decompress '{}' with NLZSS: {}", - path.as_ref().display(), - e - ) - })? - .into_boxed_slice(), - mtime, - )); - } else if sub_path.as_os_str() == OsStr::new("yaz0") { - return Ok(FileEntry::Buffer( - decompress_yaz0(mmap.as_ref()).with_context(|| { - format!("Failed to decompress '{}' with Yaz0", path.as_ref().display()) - })?, - mtime, - )); - } else if sub_path.as_os_str() == OsStr::new("yay0") { - return Ok(FileEntry::Buffer( - decompress_yay0(mmap.as_ref()).with_context(|| { - format!("Failed to decompress '{}' with Yay0", path.as_ref().display()) - })?, - mtime, - )); - } - - let buf = mmap.as_ref(); - match *array_ref!(buf, 0, 4) { - RARC_MAGIC => { - let rarc = RarcReader::new(&mut Cursor::new(mmap.as_ref())).with_context(|| { - format!("Failed to open '{}' as RARC archive", base_path.display()) - })?; - let (offset, size) = rarc.find_file(&sub_path)?.ok_or_else(|| { - anyhow!("File '{}' not found in '{}'", sub_path.display(), base_path.display()) - })?; - (offset, size as u64) - } - U8_MAGIC => { - let arc = U8View::new(buf).map_err(|e| { - anyhow!("Failed to open '{}' as U8 archive: {}", base_path.display(), e) - })?; - let (_, node) = arc.find(sub_path.to_slash_lossy().as_ref()).ok_or_else(|| { - anyhow!("File '{}' not found in '{}'", sub_path.display(), base_path.display()) - })?; - (node.offset() as u64, node.length() as u64) - } - _ => bail!("Couldn't detect archive type for '{}'", path.as_ref().display()), - } - } else { - (0, mmap.len() as u64) - }; - let map = MappedFile { mmap, mtime, offset, len }; - let buf = map.as_slice(); - // Auto-detect compression if there's a magic number. - if buf.len() > 4 { - match *array_ref!(buf, 0, 4) { - YAZ0_MAGIC => { - return Ok(FileEntry::Buffer( - decompress_yaz0(buf).with_context(|| { - format!("Failed to decompress '{}' with Yaz0", path.as_ref().display()) - })?, - mtime, - )); - } - YAY0_MAGIC => { - return Ok(FileEntry::Buffer( - decompress_yay0(buf).with_context(|| { - format!("Failed to decompress '{}' with Yay0", path.as_ref().display()) - })?, - mtime, - )); - } - _ => {} - } - } - Ok(FileEntry::MappedFile(map)) -} - -/// Opens a memory mapped file without decompression or archive handling. -pub fn map_file_basic

(path: P) -> Result -where P: AsRef { - let path = path.as_ref(); - let file = - File::open(path).with_context(|| format!("Failed to open file '{}'", path.display()))?; - let mtime = FileTime::from_last_modification_time(&file.metadata()?); - let mmap = unsafe { MmapOptions::new().map(&file) } - .with_context(|| format!("Failed to mmap file: '{}'", path.display()))?; - let len = mmap.len() as u64; - Ok(FileEntry::MappedFile(MappedFile { mmap, mtime, offset: 0, len })) -} - -pub type OpenedFile = TakeSeek; - -/// Opens a file (not memory mapped). No decompression is performed. -pub fn open_file

(path: P) -> Result -where P: AsRef { - let (base_path, sub_path) = split_path(path)?; - let mut file = File::open(&base_path) - .with_context(|| format!("Failed to open file '{}'", base_path.display()))?; - let (offset, size) = if let Some(sub_path) = sub_path { - let rarc = RarcReader::new(&mut BufReader::new(&file)) - .with_context(|| format!("Failed to read RARC '{}'", base_path.display()))?; - rarc.find_file(&sub_path)?.map(|(o, s)| (o, s as u64)).ok_or_else(|| { - anyhow!("File '{}' not found in '{}'", sub_path.display(), base_path.display()) - })? - } else { - (0, file.seek(SeekFrom::End(0))?) - }; - file.seek(SeekFrom::Start(offset))?; - Ok(file.take_seek(size)) -} - -pub trait Reader: BufRead + Seek {} - -impl Reader for Cursor<&[u8]> {} - -/// Creates a buffered reader around a file (not memory mapped). -pub fn buf_reader

(path: P) -> Result> -where P: AsRef { - let file = File::open(&path) - .with_context(|| format!("Failed to open file '{}'", path.as_ref().display()))?; - Ok(BufReader::new(file)) -} - /// Creates a buffered writer around a file (not memory mapped). pub fn buf_writer

(path: P) -> Result> where P: AsRef { @@ -217,18 +32,18 @@ where P: AsRef { } /// Reads a string with known size at the specified offset. -pub fn read_string(reader: &mut R, off: u64, size: usize) -> Result +pub fn read_string(reader: &mut R, off: u64, size: usize) -> io::Result where R: Read + Seek + ?Sized { let mut data = vec![0u8; size]; let pos = reader.stream_position()?; reader.seek(SeekFrom::Start(off))?; reader.read_exact(&mut data)?; reader.seek(SeekFrom::Start(pos))?; - Ok(String::from_utf8(data)?) + String::from_utf8(data).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) } /// Reads a zero-terminated string at the specified offset. -pub fn read_c_string(reader: &mut R, off: u64) -> Result +pub fn read_c_string(reader: &mut R, off: u64) -> io::Result where R: Read + Seek + ?Sized { let pos = reader.stream_position()?; reader.seek(SeekFrom::Start(off))?; @@ -252,8 +67,9 @@ pub fn process_rsp(files: &[PathBuf]) -> Result> { let path_str = path.to_str().ok_or_else(|| anyhow!("'{}' is not valid UTF-8", path.display()))?; if let Some(rsp_file) = path_str.strip_prefix('@') { - let reader = buf_reader(rsp_file)?; - for result in reader.lines() { + let rsp_path = Path::new(rsp_file); + let file = open_path(rsp_path, true)?; + for result in file.lines() { let line = result?; if !line.is_empty() { out.push(PathBuf::from_slash(line)); @@ -270,120 +86,20 @@ pub fn process_rsp(files: &[PathBuf]) -> Result> { Ok(out) } -/// Iterator over files in a RARC archive. -struct RarcIterator { - file: MappedFile, - base_path: PathBuf, - paths: Vec<(PathBuf, u64, u32)>, - index: usize, -} - -impl RarcIterator { - pub fn new(file: MappedFile, base_path: &Path) -> Result { - let reader = RarcReader::new(&mut file.as_reader())?; - let paths = Self::collect_paths(&reader, base_path); - Ok(Self { file, base_path: base_path.to_owned(), paths, index: 0 }) - } - - fn collect_paths(reader: &RarcReader, base_path: &Path) -> Vec<(PathBuf, u64, u32)> { - let mut current_path = PathBuf::new(); - let mut paths = vec![]; - for node in reader.nodes() { - match node { - Node::DirectoryBegin { name } => { - current_path.push(name.name); - } - Node::DirectoryEnd { name: _ } => { - current_path.pop(); - } - Node::File { name, offset, size } => { - let path = base_path.join(¤t_path).join(name.name); - paths.push((path, offset, size)); - } - Node::CurrentDirectory => {} - Node::ParentDirectory => {} - } - } - paths - } -} - -impl Iterator for RarcIterator { - type Item = Result<(PathBuf, Box<[u8]>)>; - - fn next(&mut self) -> Option { - if self.index >= self.paths.len() { - return None; - } - - let (path, off, size) = self.paths[self.index].clone(); - self.index += 1; - - let slice = &self.file.as_slice()[off as usize..off as usize + size as usize]; - match decompress_if_needed(slice) { - Ok(buf) => Some(Ok((path, buf.into_owned()))), - Err(e) => Some(Err(e)), - } - } -} - -/// A file entry, either a memory mapped file or an owned buffer. -pub enum FileEntry { - MappedFile(MappedFile), - Buffer(Box<[u8]>, FileTime), -} - -impl FileEntry { - /// Creates a reader for the file. - pub fn as_reader(&self) -> Cursor<&[u8]> { - match self { - Self::MappedFile(file) => file.as_reader(), - Self::Buffer(slice, _) => Cursor::new(slice), - } - } - - pub fn as_slice(&self) -> &[u8] { - match self { - Self::MappedFile(file) => file.as_slice(), - Self::Buffer(slice, _) => slice, - } - } - - pub fn len(&self) -> u64 { - match self { - Self::MappedFile(file) => file.len(), - Self::Buffer(slice, _) => slice.len() as u64, - } - } - - pub fn is_empty(&self) -> bool { - match self { - Self::MappedFile(file) => file.is_empty(), - Self::Buffer(slice, _) => slice.is_empty(), - } - } - - pub fn mtime(&self) -> FileTime { - match self { - Self::MappedFile(file) => file.mtime, - Self::Buffer(_, mtime) => *mtime, - } - } -} - /// Information about a file when it was read. /// Used to determine if a file has changed since it was read (mtime) /// and if it needs to be written (hash). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct FileReadInfo { - pub mtime: FileTime, + pub mtime: Option, pub hash: u64, } impl FileReadInfo { - pub fn new(entry: &FileEntry) -> Result { - let hash = xxh3_64(entry.as_slice()); - Ok(Self { mtime: entry.mtime(), hash }) + pub fn new(entry: &mut dyn VfsFile) -> Result { + let hash = xxh3_64(entry.map()?); + let metadata = entry.metadata()?; + Ok(Self { mtime: metadata.mtime, hash }) } } @@ -393,104 +109,34 @@ impl FileReadInfo { pub struct FileIterator { paths: Vec, index: usize, - rarc: Option, } impl FileIterator { pub fn new(paths: &[PathBuf]) -> Result { - Ok(Self { paths: process_rsp(paths)?, index: 0, rarc: None }) + Ok(Self { paths: process_rsp(paths)?, index: 0 }) } - fn next_rarc(&mut self) -> Option> { - if let Some(rarc) = &mut self.rarc { - match rarc.next() { - Some(Ok((path, buf))) => { - let mut path_str = rarc.base_path.as_os_str().to_os_string(); - path_str.push(OsStr::new(":")); - path_str.push(path.as_os_str()); - return Some(Ok((path, FileEntry::Buffer(buf, rarc.file.mtime)))); - } - Some(Err(err)) => return Some(Err(err)), - None => self.rarc = None, - } - } - None - } - - fn next_path(&mut self) -> Option> { + fn next_path(&mut self) -> Option)>> { if self.index >= self.paths.len() { return None; } let path = self.paths[self.index].clone(); self.index += 1; - match map_file(&path) { - Ok(FileEntry::MappedFile(map)) => self.handle_file(map, path), - Ok(entry) => Some(Ok((path, entry))), - Err(err) => Some(Err(err)), + match open_path(&path, true) { + Ok(file) => Some(Ok((path, file))), + Err(e) => Some(Err(e)), } } - - fn handle_file( - &mut self, - file: MappedFile, - path: PathBuf, - ) -> Option> { - let buf = file.as_slice(); - if buf.len() <= 4 { - return Some(Ok((path, FileEntry::MappedFile(file)))); - } - - match *array_ref!(buf, 0, 4) { - YAZ0_MAGIC => self.handle_yaz0(file, path), - YAY0_MAGIC => self.handle_yay0(file, path), - RARC_MAGIC => self.handle_rarc(file, path), - _ => Some(Ok((path, FileEntry::MappedFile(file)))), - } - } - - fn handle_yaz0( - &mut self, - file: MappedFile, - path: PathBuf, - ) -> Option> { - Some(match decompress_yaz0(file.as_slice()) { - Ok(buf) => Ok((path, FileEntry::Buffer(buf, file.mtime))), - Err(e) => Err(e), - }) - } - - fn handle_yay0( - &mut self, - file: MappedFile, - path: PathBuf, - ) -> Option> { - Some(match decompress_yay0(file.as_slice()) { - Ok(buf) => Ok((path, FileEntry::Buffer(buf, file.mtime))), - Err(e) => Err(e), - }) - } - - fn handle_rarc( - &mut self, - file: MappedFile, - path: PathBuf, - ) -> Option> { - self.rarc = match RarcIterator::new(file, &path) { - Ok(iter) => Some(iter), - Err(e) => return Some(Err(e)), - }; - self.next() - } } impl Iterator for FileIterator { - type Item = Result<(PathBuf, FileEntry)>; + type Item = Result<(PathBuf, Box)>; - fn next(&mut self) -> Option { self.next_rarc().or_else(|| self.next_path()) } + fn next(&mut self) -> Option { self.next_path() } } -pub fn touch

(path: P) -> std::io::Result<()> +pub fn touch

(path: P) -> io::Result<()> where P: AsRef { if path.as_ref().exists() { set_file_mtime(path, FileTime::now()) diff --git a/src/util/map.rs b/src/util/map.rs index 6dcd8ee..a964331 100644 --- a/src/util/map.rs +++ b/src/util/map.rs @@ -23,9 +23,9 @@ use crate::{ ObjSections, ObjSplit, ObjSymbol, ObjSymbolFlagSet, ObjSymbolFlags, ObjSymbolKind, ObjSymbols, ObjUnit, }, - util::{file::map_file, nested::NestedVec}, + util::nested::NestedVec, + vfs::open_path, }; - #[derive(Debug, Copy, Clone, Eq, PartialEq)] pub enum SymbolKind { Function, @@ -722,8 +722,8 @@ pub fn apply_map_file

( where P: AsRef, { - let file = map_file(&path)?; - let info = process_map(&mut file.as_reader(), common_bss_start, mw_comment_version)?; + let mut file = open_path(path.as_ref(), true)?; + let info = process_map(file.as_mut(), common_bss_start, mw_comment_version)?; apply_map(info, obj) } diff --git a/src/util/rarc.rs b/src/util/rarc.rs index 29fae5f..66d9896 100644 --- a/src/util/rarc.rs +++ b/src/util/rarc.rs @@ -1,425 +1,292 @@ -// Source: https://github.com/Julgodis/picori/blob/650da9f4fe6050b39b80d5360416591c748058d5/src/rarc.rs -// License: MIT -// Modified to use `std::io::Cursor<&[u8]>` and project's FromReader trait -use std::{ - collections::HashMap, - fmt::Display, - hash::{Hash, Hasher}, - io, - io::{Read, Seek, SeekFrom}, - path::{Component, Path, PathBuf}, -}; +use std::{borrow::Cow, ffi::CStr}; -use anyhow::{anyhow, bail, ensure, Result}; +use zerocopy::{big_endian::*, AsBytes, FromBytes, FromZeroes}; -use crate::util::{ - file::read_c_string, - reader::{struct_size, Endian, FromReader}, -}; - -#[derive(Debug, Clone)] -pub struct NamedHash { - pub name: String, - pub hash: u16, -} - -impl Display for NamedHash { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.name) - } -} - -impl Hash for NamedHash { - fn hash(&self, state: &mut H) - where H: Hasher { - self.hash.hash(state); - } -} - -impl PartialEq for NamedHash { - fn eq(&self, other: &Self) -> bool { - if self.hash == other.hash { - self.name == other.name - } else { - false - } - } -} - -impl Eq for NamedHash {} - -#[derive(Debug, Clone)] -enum RarcDirectory { - File { - /// Name of the file. - name: NamedHash, - /// Offset of the file in the RARC file. This offset is relative to the start of the RARC file. - offset: u64, - /// Size of the file. - size: u32, - }, - Folder { - /// Name of the folder. - name: NamedHash, - }, - CurrentFolder, - ParentFolder, -} - -#[derive(Debug, Clone)] -struct RarcNode { - /// Index of first directory. - pub index: u32, - /// Number of directories. - pub count: u32, -} - -pub struct RarcReader { - directories: Vec, - nodes: HashMap, - root_node: NamedHash, -} +use crate::static_assert; pub const RARC_MAGIC: [u8; 4] = *b"RARC"; -struct RarcHeader { +#[derive(Copy, Clone, Debug, PartialEq, FromBytes, FromZeroes, AsBytes)] +#[repr(C, align(4))] +pub struct RarcHeader { + /// Magic identifier. (Always "RARC") magic: [u8; 4], - _file_length: u32, - header_length: u32, - file_offset: u32, - _file_length_2: u32, - _unk0: u32, - _unk1: u32, - _unk2: u32, - node_count: u32, - node_offset: u32, - directory_count: u32, - directory_offset: u32, - string_table_length: u32, - string_table_offset: u32, - _file_count: u16, - _unk3: u16, - _unk4: u32, + /// Length of the RARC file. + file_len: U32, + /// Length of the header. (Always 32) + header_len: U32, + /// Start of the file data, relative to the end of the file header. + data_offset: U32, + /// Length of the file data. + data_len: U32, + _unk1: U32, + _unk2: U32, + _unk3: U32, } -impl FromReader for RarcHeader { - type Args = (); +static_assert!(size_of::() == 0x20); - const STATIC_SIZE: usize = struct_size([ - 4, // magic - u32::STATIC_SIZE, // file_length - u32::STATIC_SIZE, // header_length - u32::STATIC_SIZE, // file_offset - u32::STATIC_SIZE, // file_length - u32::STATIC_SIZE, // unk0 - u32::STATIC_SIZE, // unk1 - u32::STATIC_SIZE, // unk2 - u32::STATIC_SIZE, // node_count - u32::STATIC_SIZE, // node_offset - u32::STATIC_SIZE, // directory_count - u32::STATIC_SIZE, // directory_offset - u32::STATIC_SIZE, // string_table_length - u32::STATIC_SIZE, // string_table_offset - u16::STATIC_SIZE, // file_count - u16::STATIC_SIZE, // unk3 - u32::STATIC_SIZE, // unk4 - ]); +impl RarcHeader { + /// Length of the RARC file. + pub fn file_len(&self) -> u32 { self.file_len.get() } - fn from_reader_args(reader: &mut R, e: Endian, _args: Self::Args) -> io::Result - where R: Read + Seek + ?Sized { - let header = Self { - magic: <[u8; 4]>::from_reader(reader, e)?, - _file_length: u32::from_reader(reader, e)?, - header_length: u32::from_reader(reader, e)?, - file_offset: u32::from_reader(reader, e)?, - _file_length_2: u32::from_reader(reader, e)?, - _unk0: u32::from_reader(reader, e)?, - _unk1: u32::from_reader(reader, e)?, - _unk2: u32::from_reader(reader, e)?, - node_count: u32::from_reader(reader, e)?, - node_offset: u32::from_reader(reader, e)?, - directory_count: u32::from_reader(reader, e)?, - directory_offset: u32::from_reader(reader, e)?, - string_table_length: u32::from_reader(reader, e)?, - string_table_offset: u32::from_reader(reader, e)?, - _file_count: u16::from_reader(reader, e)?, - _unk3: u16::from_reader(reader, e)?, - _unk4: u32::from_reader(reader, e)?, + /// Length of the header. + pub fn header_len(&self) -> u32 { self.header_len.get() } + + /// Start of the file data, relative to the end of the file header. + pub fn data_offset(&self) -> u32 { self.data_offset.get() } + + /// Length of the file data. + pub fn data_len(&self) -> u32 { self.data_len.get() } +} + +#[derive(Copy, Clone, Debug, PartialEq, FromBytes, FromZeroes, AsBytes)] +#[repr(C, align(4))] +struct RarcInfo { + /// Number of directories in the directory table. + directory_count: U32, + /// Offset to the start of the directory table, relative to the end of the file header. + directory_offset: U32, + /// Number of nodes in the node table. + node_count: U32, + /// Offset to the start of the node table, relative to the end of the file header. + node_offset: U32, + /// Length of the string table. + string_table_len: U32, + /// Offset to the start of the string table, relative to the end of the file header. + string_table_offset: U32, + /// Number of files in the node table. + _file_count: U16, + _unk4: U16, + _unk5: U32, +} + +static_assert!(size_of::() == 0x20); + +#[derive(Copy, Clone, Debug, PartialEq, FromBytes, FromZeroes, AsBytes)] +#[repr(C, align(4))] +pub struct RarcNode { + /// Index of the node. (0xFFFF for directories) + index: U16, + /// Hash of the node name. + name_hash: U16, + /// Unknown. (0x200 for folders, 0x1100 for files) + _unk0: U16, + /// Offset in the string table to the node name. + name_offset: U16, + /// Files: Offset in the data to the file data. + /// Directories: Index of the directory in the directory table. + data_offset: U32, + /// Files: Length of the data. + /// Directories: Unknown. Always 16. + data_length: U32, + _unk1: U32, +} + +static_assert!(size_of::() == 0x14); + +impl RarcNode { + /// Whether the node is a file. + pub fn is_file(&self) -> bool { self.index.get() != 0xFFFF } + + /// Whether the node is a directory. + pub fn is_dir(&self) -> bool { self.index.get() == 0xFFFF } + + /// Offset in the string table to the node name. + pub fn name_offset(&self) -> u32 { self.name_offset.get() as u32 } + + /// Files: Offset in the data to the file data. + /// Directories: Index of the directory in the directory table. + pub fn data_offset(&self) -> u32 { self.data_offset.get() } + + /// Files: Length of the data. + /// Directories: Unknown. Always 16. + pub fn data_length(&self) -> u32 { self.data_length.get() } +} + +#[derive(Copy, Clone, Debug, PartialEq, FromBytes, FromZeroes, AsBytes)] +#[repr(C, align(4))] +pub struct RarcDirectory { + /// Identifier of the directory. + identifier: [u8; 4], + /// Offset in the string table to the directory name. + name_offset: U32, + /// Hash of the directory name. + name_hash: U16, + /// Number of nodes in the directory. + count: U16, + /// Index of the first node in the directory. + index: U32, +} + +static_assert!(size_of::() == 0x10); + +impl RarcDirectory { + /// Offset in the string table to the directory name. + pub fn name_offset(&self) -> u32 { self.name_offset.get() } + + /// Index of the first node in the directory. + pub fn node_index(&self) -> u32 { self.index.get() } + + /// Number of nodes in the directory. + pub fn node_count(&self) -> u16 { self.count.get() } +} + +/// A view into a RARC archive. +pub struct RarcView<'a> { + /// The RARC archive header. + pub header: &'a RarcHeader, + /// The directories in the RARC archive. + pub directories: &'a [RarcDirectory], + /// The nodes in the RARC archive. + pub nodes: &'a [RarcNode], + /// The string table containing all file and directory names. + pub string_table: &'a [u8], + /// The file data. + pub data: &'a [u8], +} + +impl<'a> RarcView<'a> { + /// Create a new RARC view from a buffer. + pub fn new(buf: &'a [u8]) -> Result { + let Some(header) = RarcHeader::ref_from_prefix(buf) else { + return Err("Buffer not large enough for RARC header"); }; if header.magic != RARC_MAGIC { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("invalid RARC magic: {:?}", header.magic), - )); + return Err("RARC magic mismatch"); } - if header.node_count >= 0x10000 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("invalid node count: {}", header.node_count), - )); + if header.header_len.get() as usize != size_of::() { + return Err("RARC header size mismatch"); } - if header.directory_count >= 0x10000 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("invalid directory count: {}", header.directory_count), - )); + + // All offsets are relative to the _end_ of the header, so we can + // just trim the header from the buffer and use the offsets as is. + let buf = &buf[size_of::()..]; + let Some(info) = RarcInfo::ref_from_prefix(buf) else { + return Err("Buffer not large enough for RARC info"); + }; + + let directory_table_offset = info.directory_offset.get() as usize; + let directory_table_size = info.directory_count.get() as usize * size_of::(); + let directories_buf = buf + .get(directory_table_offset..directory_table_offset + directory_table_size) + .ok_or("RARC directory table out of bounds")?; + let directories = + RarcDirectory::slice_from(directories_buf).ok_or("RARC directory table not aligned")?; + if directories.is_empty() || directories[0].identifier != *b"ROOT" { + return Err("RARC root directory not found"); } - Ok(header) + + let node_table_offset = info.node_offset.get() as usize; + let node_table_size = info.node_count.get() as usize * size_of::(); + let nodes_buf = buf + .get(node_table_offset..node_table_offset + node_table_size) + .ok_or("RARC node table out of bounds")?; + let nodes = RarcNode::slice_from(nodes_buf).ok_or("RARC node table not aligned")?; + + let string_table_offset = info.string_table_offset.get() as usize; + let string_table_size = info.string_table_len.get() as usize; + let string_table = buf + .get(string_table_offset..string_table_offset + string_table_size) + .ok_or("RARC string table out of bounds")?; + + let data_offset = header.data_offset.get() as usize; + let data_size = header.data_len.get() as usize; + let data = + buf.get(data_offset..data_offset + data_size).ok_or("RARC file data out of bounds")?; + + Ok(Self { header, directories, nodes, string_table, data }) } -} -struct RarcFileNode { - index: u16, - name_hash: u16, - _unk0: u16, // 0x200 for folders, 0x1100 for files - name_offset: u16, - data_offset: u32, - data_length: u32, - _unk1: u32, -} - -impl FromReader for RarcFileNode { - type Args = (); - - const STATIC_SIZE: usize = struct_size([ - u16::STATIC_SIZE, // index - u16::STATIC_SIZE, // name_hash - u16::STATIC_SIZE, // unk0 - u16::STATIC_SIZE, // name_offset - u32::STATIC_SIZE, // data_offset - u32::STATIC_SIZE, // data_length - u32::STATIC_SIZE, // unk1 - ]); - - fn from_reader_args(reader: &mut R, e: Endian, _args: Self::Args) -> io::Result - where R: Read + Seek + ?Sized { - Ok(Self { - index: u16::from_reader(reader, e)?, - name_hash: u16::from_reader(reader, e)?, - _unk0: u16::from_reader(reader, e)?, - name_offset: u16::from_reader(reader, e)?, - data_offset: u32::from_reader(reader, e)?, - data_length: u32::from_reader(reader, e)?, - _unk1: u32::from_reader(reader, e)?, - }) + /// Get a string from the string table at the given offset. + pub fn get_string(&self, offset: u32) -> Result, String> { + let name_buf = self.string_table.get(offset as usize..).ok_or_else(|| { + format!( + "RARC: name offset {} out of bounds (string table size: {})", + offset, + self.string_table.len() + ) + })?; + let c_string = CStr::from_bytes_until_nul(name_buf) + .map_err(|_| format!("RARC: name at offset {} not null-terminated", offset))?; + Ok(c_string.to_string_lossy()) } -} -struct RarcDirectoryNode { - _identifier: u32, - name_offset: u32, - name_hash: u16, - count: u16, - index: u32, -} - -impl FromReader for RarcDirectoryNode { - type Args = (); - - const STATIC_SIZE: usize = struct_size([ - u32::STATIC_SIZE, // identifier - u32::STATIC_SIZE, // name_offset - u16::STATIC_SIZE, // name_hash - u16::STATIC_SIZE, // count - u32::STATIC_SIZE, // index - ]); - - fn from_reader_args(reader: &mut R, e: Endian, _args: Self::Args) -> io::Result - where R: Read + Seek + ?Sized { - Ok(Self { - _identifier: u32::from_reader(reader, e)?, - name_offset: u32::from_reader(reader, e)?, - name_hash: u16::from_reader(reader, e)?, - count: u16::from_reader(reader, e)?, - index: u32::from_reader(reader, e)?, - }) + /// Get the data for a file node. + pub fn get_data(&self, node: RarcNode) -> Result<&[u8], &'static str> { + if node.is_dir() { + return Err("Cannot get data for a directory node"); + } + let offset = node.data_offset.get() as usize; + let size = node.data_length.get() as usize; + self.data.get(offset..offset + size).ok_or("RARC file data out of bounds") } -} -impl RarcReader { - /// Creates a new RARC reader. - pub fn new(reader: &mut R) -> Result - where R: Read + Seek + ?Sized { - let base = reader.stream_position()?; - let header = RarcHeader::from_reader(reader, Endian::Big)?; + /// Finds a particular file or directory by path. + pub fn find(&self, path: &str) -> Option { + let mut split = path.split('/'); + let mut current = next_non_empty(&mut split); - let base = base + header.header_length as u64; - let directory_base = base + header.directory_offset as u64; - let data_base = base + header.file_offset as u64; - let mut directories = Vec::with_capacity(header.directory_count as usize); - for i in 0..header.directory_count { - reader.seek(SeekFrom::Start(directory_base + 20 * i as u64))?; - let node = RarcFileNode::from_reader(reader, Endian::Big)?; - - let name = { - let offset = header.string_table_offset as u64; - let offset = offset + node.name_offset as u64; - ensure!( - (node.name_offset as u32) < header.string_table_length, - "invalid string table offset" - ); - read_c_string(reader, base + offset) - }?; - - if node.index == 0xFFFF { - if name == "." { - directories.push(RarcDirectory::CurrentFolder); - } else if name == ".." { - directories.push(RarcDirectory::ParentFolder); - } else { - directories.push(RarcDirectory::Folder { - name: NamedHash { name, hash: node.name_hash }, - }); - } - } else { - directories.push(RarcDirectory::File { - name: NamedHash { name, hash: node.name_hash }, - offset: data_base + node.data_offset as u64, - size: node.data_length, - }); + let mut dir_idx = 0; + let mut dir = self.directories[dir_idx]; + // Allow matching the root directory by name optionally + if let Ok(root_name) = self.get_string(dir.name_offset()) { + if root_name.eq_ignore_ascii_case(current) { + current = next_non_empty(&mut split); } } - - let node_base = base + header.node_offset as u64; - let mut root_node: Option = None; - let mut nodes = HashMap::with_capacity(header.node_count as usize); - for i in 0..header.node_count { - reader.seek(SeekFrom::Start(node_base + 16 * i as u64))?; - let node = RarcDirectoryNode::from_reader(reader, Endian::Big)?; - - ensure!(node.index < header.directory_count, "first directory index out of bounds"); - - let last_index = node.index.checked_add(node.count as u32); - ensure!( - last_index.is_some() && last_index.unwrap() <= header.directory_count, - "last directory index out of bounds" - ); - - let name = { - let offset = header.string_table_offset as u64; - let offset = offset + node.name_offset as u64; - ensure!( - node.name_offset < header.string_table_length, - "invalid string table offset" - ); - read_c_string(reader, base + offset) - }?; - - // FIXME: this assumes that the root node is the first node in the list - if root_node.is_none() { - root_node = Some(NamedHash { name: name.clone(), hash: node.name_hash }); - } - - let name = NamedHash { name, hash: node.name_hash }; - nodes.insert(name.clone(), RarcNode { index: node.index, count: node.count as u32 }); + if current.is_empty() { + return Some(RarcNodeKind::Directory(dir_idx, dir)); } - if let Some(root_node) = root_node { - Ok(Self { directories, nodes, root_node }) - } else { - Err(anyhow!("no root node")) - } - } - - /// Get a iterator over the nodes in the RARC file. - pub fn nodes(&self) -> Nodes<'_> { - let root_node = self.root_node.clone(); - Nodes { parent: self, stack: vec![NodeState::Begin(root_node)] } - } - - /// Find a file in the RARC file. - pub fn find_file

(&self, path: P) -> Result> - where P: AsRef { - let mut cmp_path = PathBuf::new(); - for component in path.as_ref().components() { - match component { - Component::Normal(name) => cmp_path.push(name.to_ascii_lowercase()), - Component::RootDir => {} - component => bail!("Invalid path component: {:?}", component), - } - } - - let mut current_path = PathBuf::new(); - for node in self.nodes() { - match node { - Node::DirectoryBegin { name } => { - current_path.push(name.name.to_ascii_lowercase()); - } - Node::DirectoryEnd { name: _ } => { - current_path.pop(); - } - Node::File { name, offset, size } => { - if current_path.join(name.name.to_ascii_lowercase()) == cmp_path { - return Ok(Some((offset, size))); - } - } - Node::CurrentDirectory => {} - Node::ParentDirectory => {} - } - } - Ok(None) - } -} - -/// A node in an RARC file. -pub enum Node { - /// A directory that has been entered. - DirectoryBegin { name: NamedHash }, - /// A directory that has been exited. - DirectoryEnd { name: NamedHash }, - /// A file in the current directory. - File { name: NamedHash, offset: u64, size: u32 }, - /// The current directory. This is equivalent to ".". - CurrentDirectory, - /// The parent directory. This is equivalent to "..". - ParentDirectory, -} - -enum NodeState { - Begin(NamedHash), - End(NamedHash), - File(NamedHash, u32), -} - -/// An iterator over the nodes in an RARC file. -pub struct Nodes<'parent> { - parent: &'parent RarcReader, - stack: Vec, -} - -impl<'parent> Iterator for Nodes<'parent> { - type Item = Node; - - fn next(&mut self) -> Option { - match self.stack.pop()? { - NodeState::Begin(name) => { - self.stack.push(NodeState::File(name.clone(), 0)); - Some(Node::DirectoryBegin { name }) - } - NodeState::End(name) => Some(Node::DirectoryEnd { name }), - NodeState::File(name, index) => { - if let Some(node) = self.parent.nodes.get(&name) { - if index + 1 >= node.count { - self.stack.push(NodeState::End(name.clone())); + let mut idx = dir.index.get() as usize; + while idx < dir.index.get() as usize + dir.count.get() as usize { + let node = self.nodes.get(idx).copied()?; + let Ok(name) = self.get_string(node.name_offset()) else { + idx += 1; + continue; + }; + if name.eq_ignore_ascii_case(current) { + current = next_non_empty(&mut split); + if node.is_dir() { + dir_idx = node.data_offset.get() as usize; + dir = self.directories.get(dir_idx).cloned()?; + idx = dir.index.get() as usize; + if current.is_empty() { + return Some(RarcNodeKind::Directory(dir_idx, dir)); } else { - self.stack.push(NodeState::File(name.clone(), index + 1)); - } - let directory = &self.parent.directories[(node.index + index) as usize]; - match directory { - RarcDirectory::CurrentFolder => Some(Node::CurrentDirectory), - RarcDirectory::ParentFolder => Some(Node::ParentDirectory), - RarcDirectory::Folder { name } => { - self.stack.push(NodeState::Begin(name.clone())); - self.next() - } - RarcDirectory::File { name, offset, size } => { - Some(Node::File { name: name.clone(), offset: *offset, size: *size }) - } + continue; } } else { - None + return Some(RarcNodeKind::File(idx, node)); } } + idx += 1; + } + + None + } + + /// Get the children of a directory. + pub fn children(&self, dir: RarcDirectory) -> &[RarcNode] { + let start = dir.node_index() as usize; + let end = start + dir.node_count() as usize; + self.nodes.get(start..end).unwrap_or(&[]) + } +} + +#[derive(Debug)] +pub enum RarcNodeKind { + File(usize, RarcNode), + Directory(usize, RarcDirectory), +} + +fn next_non_empty<'a>(iter: &mut impl Iterator) -> &'a str { + loop { + match iter.next() { + Some("") => continue, + Some(next) => break next, + None => break "", } } } diff --git a/src/util/u8_arc.rs b/src/util/u8_arc.rs index c805502..8478703 100644 --- a/src/util/u8_arc.rs +++ b/src/util/u8_arc.rs @@ -32,7 +32,7 @@ pub enum U8NodeKind { } /// An individual file system node. -#[derive(Clone, Debug, PartialEq, FromBytes, FromZeroes, AsBytes)] +#[derive(Copy, Clone, Debug, PartialEq, FromBytes, FromZeroes, AsBytes)] #[repr(C, align(4))] pub struct U8Node { kind: u8, @@ -121,7 +121,7 @@ impl<'a> U8View<'a> { pub fn iter(&self) -> U8Iter { U8Iter { inner: self, idx: 1 } } /// Get the name of a node. - pub fn get_name(&self, node: &U8Node) -> Result, String> { + pub fn get_name(&self, node: U8Node) -> Result, String> { let name_buf = self.string_table.get(node.name_offset() as usize..).ok_or_else(|| { format!( "U8: name offset {} out of bounds (string table size: {})", @@ -136,22 +136,29 @@ impl<'a> U8View<'a> { } /// Finds a particular file or directory by path. - pub fn find(&self, path: &str) -> Option<(usize, &U8Node)> { - let mut split = path.trim_matches('/').split('/'); - let mut current = split.next()?; + pub fn find(&self, path: &str) -> Option<(usize, U8Node)> { + let mut split = path.split('/'); + let mut current = next_non_empty(&mut split); + if current.is_empty() { + return Some((0, self.nodes[0])); + } + let mut idx = 1; let mut stop_at = None; - while let Some(node) = self.nodes.get(idx) { - if self.get_name(node).as_ref().map_or(false, |name| name.eq_ignore_ascii_case(current)) - { - if let Some(next) = split.next() { - current = next; - } else { + while let Some(node) = self.nodes.get(idx).copied() { + if self.get_name(node).map_or(false, |name| name.eq_ignore_ascii_case(current)) { + current = next_non_empty(&mut split); + if current.is_empty() { return Some((idx, node)); } - // Descend into directory - idx += 1; - stop_at = Some(node.length() as usize + idx); + if node.is_dir() { + // Descend into directory + idx += 1; + stop_at = Some(node.length() as usize + idx); + } else { + // Not a directory + break; + } } else if node.is_dir() { // Skip directory idx = node.length() as usize; @@ -169,6 +176,16 @@ impl<'a> U8View<'a> { } } +fn next_non_empty<'a>(iter: &mut impl Iterator) -> &'a str { + loop { + match iter.next() { + Some("") => continue, + Some(next) => break next, + None => break "", + } + } +} + /// Iterator over the nodes in a U8 archive. pub struct U8Iter<'a> { inner: &'a U8View<'a>, @@ -176,11 +193,11 @@ pub struct U8Iter<'a> { } impl<'a> Iterator for U8Iter<'a> { - type Item = (usize, &'a U8Node, Result, String>); + type Item = (usize, U8Node, Result, String>); fn next(&mut self) -> Option { let idx = self.idx; - let node = self.inner.nodes.get(idx)?; + let node = self.inner.nodes.get(idx).copied()?; let name = self.inner.get_name(node); self.idx += 1; Some((idx, node, name)) diff --git a/src/vfs/common.rs b/src/vfs/common.rs new file mode 100644 index 0000000..249cad0 --- /dev/null +++ b/src/vfs/common.rs @@ -0,0 +1,130 @@ +use std::{ + io, + io::{BufRead, Cursor, Read, Seek, SeekFrom}, + sync::Arc, +}; + +use filetime::FileTime; + +use super::{DiscStream, VfsFileType, VfsMetadata}; +use crate::vfs::VfsFile; + +#[derive(Clone)] +pub struct StaticFile { + inner: Cursor>, + mtime: Option, +} + +impl StaticFile { + pub fn new(data: Arc<[u8]>, mtime: Option) -> Self { + Self { inner: Cursor::new(data), mtime } + } +} + +impl BufRead for StaticFile { + fn fill_buf(&mut self) -> io::Result<&[u8]> { self.inner.fill_buf() } + + fn consume(&mut self, amt: usize) { self.inner.consume(amt) } +} + +impl Read for StaticFile { + fn read(&mut self, buf: &mut [u8]) -> io::Result { self.inner.read(buf) } +} + +impl Seek for StaticFile { + fn seek(&mut self, pos: SeekFrom) -> io::Result { self.inner.seek(pos) } +} + +impl VfsFile for StaticFile { + fn map(&mut self) -> io::Result<&[u8]> { Ok(self.inner.get_ref()) } + + fn metadata(&mut self) -> io::Result { + Ok(VfsMetadata { + file_type: VfsFileType::File, + len: self.inner.get_ref().len() as u64, + mtime: self.mtime, + }) + } + + fn into_disc_stream(self: Box) -> Box { self } +} + +#[derive(Clone)] +pub struct WindowedFile { + base: Box, + pos: u64, + begin: u64, + end: u64, +} + +impl WindowedFile { + pub fn new(mut base: Box, offset: u64, size: u64) -> io::Result { + base.seek(SeekFrom::Start(offset))?; + Ok(Self { base, pos: offset, begin: offset, end: offset + size }) + } + + #[inline] + pub fn len(&self) -> u64 { self.end - self.begin } +} + +impl BufRead for WindowedFile { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + let buf = self.base.fill_buf()?; + let remaining = self.end.saturating_sub(self.pos); + Ok(&buf[..buf.len().min(remaining as usize)]) + } + + fn consume(&mut self, amt: usize) { + let remaining = self.end.saturating_sub(self.pos); + let amt = amt.min(remaining as usize); + self.base.consume(amt); + self.pos += amt as u64; + } +} + +impl Read for WindowedFile { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let remaining = self.end.saturating_sub(self.pos); + if remaining == 0 { + return Ok(0); + } + let len = buf.len().min(remaining as usize); + self.base.read(&mut buf[..len]) + } +} + +impl Seek for WindowedFile { + #[inline] + fn seek(&mut self, pos: SeekFrom) -> io::Result { + let mut pos = match pos { + SeekFrom::Start(p) => self.begin + p, + SeekFrom::End(p) => self.end.saturating_add_signed(p), + SeekFrom::Current(p) => self.pos.saturating_add_signed(p), + }; + if pos < self.begin { + pos = self.begin; + } else if pos > self.end { + pos = self.end; + } + let result = self.base.seek(SeekFrom::Start(pos))?; + self.pos = result; + Ok(result - self.begin) + } + + #[inline] + fn stream_position(&mut self) -> io::Result { Ok(self.pos) } +} + +impl VfsFile for WindowedFile { + fn map(&mut self) -> io::Result<&[u8]> { + let buf = self.base.map()?; + Ok(&buf[self.pos as usize..self.end as usize]) + } + + fn metadata(&mut self) -> io::Result { + let metadata = self.base.metadata()?; + Ok(VfsMetadata { file_type: VfsFileType::File, len: self.len(), mtime: metadata.mtime }) + } + + fn into_disc_stream(self: Box) -> Box { self } +} diff --git a/src/vfs/disc.rs b/src/vfs/disc.rs new file mode 100644 index 0000000..3ef075b --- /dev/null +++ b/src/vfs/disc.rs @@ -0,0 +1,297 @@ +use std::{ + io, + io::{BufRead, Cursor, Read, Seek, SeekFrom}, + sync::Arc, +}; + +use filetime::FileTime; +use nodtool::{ + nod, + nod::{DiscStream, Fst, NodeKind, OwnedFileStream, PartitionBase, PartitionMeta}, +}; + +use super::{StaticFile, Vfs, VfsError, VfsFile, VfsFileType, VfsMetadata, VfsResult}; + +#[derive(Clone)] +pub struct DiscFs { + base: Box, + meta: Box, + mtime: Option, +} + +enum DiscNode<'a> { + None, + Root, + Sys, + Node(Fst<'a>, usize, nod::Node), + Static(&'a [u8]), +} + +impl DiscFs { + pub fn new(mut base: Box, mtime: Option) -> io::Result { + let meta = base.meta().map_err(nod_to_io_error)?; + Ok(Self { base, meta, mtime }) + } + + fn find(&self, path: &str) -> VfsResult { + let path = path.trim_matches('/'); + let mut split = path.split('/'); + let Some(segment) = split.next() else { + return Ok(DiscNode::Root); + }; + if segment.is_empty() { + return Ok(DiscNode::Root); + } + if segment.eq_ignore_ascii_case("files") { + let fst = Fst::new(&self.meta.raw_fst)?; + if split.next().is_none() { + let root = fst.nodes[0]; + return Ok(DiscNode::Node(fst, 0, root)); + } + let remainder = &path[segment.len() + 1..]; + match fst.find(remainder) { + Some((idx, node)) => Ok(DiscNode::Node(fst, idx, node)), + None => Ok(DiscNode::None), + } + } else if segment.eq_ignore_ascii_case("sys") { + let Some(segment) = split.next() else { + return Ok(DiscNode::Sys); + }; + // No directories in sys + if split.next().is_some() { + return Ok(DiscNode::None); + } + match segment.to_ascii_lowercase().as_str() { + "" => Ok(DiscNode::Sys), + "boot.bin" => Ok(DiscNode::Static(self.meta.raw_boot.as_slice())), + "bi2.bin" => Ok(DiscNode::Static(self.meta.raw_bi2.as_slice())), + "apploader.bin" => Ok(DiscNode::Static(self.meta.raw_apploader.as_ref())), + "fst.bin" => Ok(DiscNode::Static(self.meta.raw_fst.as_ref())), + "main.dol" => Ok(DiscNode::Static(self.meta.raw_dol.as_ref())), + "ticket.bin" => { + Ok(DiscNode::Static(self.meta.raw_ticket.as_deref().ok_or(VfsError::NotFound)?)) + } + "tmd.bin" => { + Ok(DiscNode::Static(self.meta.raw_tmd.as_deref().ok_or(VfsError::NotFound)?)) + } + "cert.bin" => Ok(DiscNode::Static( + self.meta.raw_cert_chain.as_deref().ok_or(VfsError::NotFound)?, + )), + "h3.bin" => Ok(DiscNode::Static( + self.meta.raw_h3_table.as_deref().ok_or(VfsError::NotFound)?, + )), + _ => Ok(DiscNode::None), + } + } else { + return Ok(DiscNode::None); + } + } +} + +impl Vfs for DiscFs { + fn open(&mut self, path: &str) -> VfsResult> { + match self.find(path)? { + DiscNode::None => Err(VfsError::NotFound), + DiscNode::Root => Err(VfsError::DirectoryExists), + DiscNode::Sys => Err(VfsError::DirectoryExists), + DiscNode::Node(_, _, node) => match node.kind() { + NodeKind::File => { + if node.length() > 2048 { + let file = self.base.clone().into_open_file(node)?; + Ok(Box::new(DiscFile::new(file, self.mtime))) + } else { + let len = node.length() as usize; + let mut file = self.base.open_file(node)?; + let mut data = vec![0u8; len]; + file.read_exact(&mut data)?; + Ok(Box::new(StaticFile::new(Arc::from(data.as_slice()), self.mtime))) + } + } + NodeKind::Directory => Err(VfsError::FileExists), + NodeKind::Invalid => Err(VfsError::from("FST: Invalid node kind")), + }, + DiscNode::Static(data) => Ok(Box::new(StaticFile::new(Arc::from(data), self.mtime))), + } + } + + fn exists(&mut self, path: &str) -> VfsResult { + Ok(!matches!(self.find(path)?, DiscNode::None)) + } + + fn read_dir(&mut self, path: &str) -> VfsResult> { + match self.find(path)? { + DiscNode::None => Err(VfsError::NotFound), + DiscNode::Root => Ok(vec!["files".to_string(), "sys".to_string()]), + DiscNode::Sys => { + let mut sys = vec![ + "boot.bin".to_string(), + "bi2.bin".to_string(), + "apploader.bin".to_string(), + "fst.bin".to_string(), + "main.dol".to_string(), + ]; + if self.meta.raw_ticket.is_some() { + sys.push("ticket.bin".to_string()); + } + if self.meta.raw_tmd.is_some() { + sys.push("tmd.bin".to_string()); + } + if self.meta.raw_cert_chain.is_some() { + sys.push("cert.bin".to_string()); + } + if self.meta.raw_h3_table.is_some() { + sys.push("h3.bin".to_string()); + } + Ok(sys) + } + DiscNode::Node(fst, idx, node) => { + match node.kind() { + NodeKind::File => return Err(VfsError::FileExists), + NodeKind::Directory => {} + NodeKind::Invalid => return Err(VfsError::from("FST: Invalid node kind")), + } + let mut entries = Vec::new(); + let mut idx = idx + 1; + let end = node.length() as usize; + while idx < end { + let child = fst + .nodes + .get(idx) + .copied() + .ok_or_else(|| VfsError::from("FST: Node index out of bounds"))?; + entries.push(fst.get_name(child)?.to_string()); + if child.is_dir() { + idx = child.length() as usize; + } else { + idx += 1; + } + } + Ok(entries) + } + DiscNode::Static(_) => Err(VfsError::FileExists), + } + } + + fn metadata(&mut self, path: &str) -> VfsResult { + match self.find(path)? { + DiscNode::None => Err(VfsError::NotFound), + DiscNode::Root | DiscNode::Sys => { + Ok(VfsMetadata { file_type: VfsFileType::Directory, len: 0, mtime: self.mtime }) + } + DiscNode::Node(_, _, node) => { + let (file_type, len) = match node.kind() { + NodeKind::File => (VfsFileType::File, node.length()), + NodeKind::Directory => (VfsFileType::Directory, 0), + NodeKind::Invalid => return Err(VfsError::from("FST: Invalid node kind")), + }; + Ok(VfsMetadata { file_type, len, mtime: self.mtime }) + } + DiscNode::Static(data) => Ok(VfsMetadata { + file_type: VfsFileType::File, + len: data.len() as u64, + mtime: self.mtime, + }), + } + } +} + +#[derive(Clone)] +enum DiscFileInner { + Stream(OwnedFileStream), + Mapped(Cursor>), +} + +#[derive(Clone)] +struct DiscFile { + inner: DiscFileInner, + mtime: Option, +} + +impl DiscFile { + pub fn new(file: OwnedFileStream, mtime: Option) -> Self { + Self { inner: DiscFileInner::Stream(file), mtime } + } + + fn convert_to_mapped(&mut self) { + match &mut self.inner { + DiscFileInner::Stream(stream) => { + let pos = stream.stream_position().unwrap(); + stream.seek(SeekFrom::Start(0)).unwrap(); + let mut data = vec![0u8; stream.len() as usize]; + stream.read_exact(&mut data).unwrap(); + let mut cursor = Cursor::new(Arc::from(data.as_slice())); + cursor.set_position(pos); + self.inner = DiscFileInner::Mapped(cursor); + } + DiscFileInner::Mapped(_) => {} + }; + } +} + +impl BufRead for DiscFile { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + match &mut self.inner { + DiscFileInner::Stream(stream) => stream.fill_buf(), + DiscFileInner::Mapped(data) => data.fill_buf(), + } + } + + fn consume(&mut self, amt: usize) { + match &mut self.inner { + DiscFileInner::Stream(stream) => stream.consume(amt), + DiscFileInner::Mapped(data) => data.consume(amt), + } + } +} + +impl Read for DiscFile { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match &mut self.inner { + DiscFileInner::Stream(stream) => stream.read(buf), + DiscFileInner::Mapped(data) => data.read(buf), + } + } +} + +impl Seek for DiscFile { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + match &mut self.inner { + DiscFileInner::Stream(stream) => stream.seek(pos), + DiscFileInner::Mapped(data) => data.seek(pos), + } + } +} + +impl VfsFile for DiscFile { + fn map(&mut self) -> io::Result<&[u8]> { + self.convert_to_mapped(); + match &mut self.inner { + DiscFileInner::Stream(_) => unreachable!(), + DiscFileInner::Mapped(data) => Ok(data.get_ref()), + } + } + + fn metadata(&mut self) -> io::Result { + match &mut self.inner { + DiscFileInner::Stream(stream) => Ok(VfsMetadata { + file_type: VfsFileType::File, + len: stream.len(), + mtime: self.mtime, + }), + DiscFileInner::Mapped(data) => Ok(VfsMetadata { + file_type: VfsFileType::File, + len: data.get_ref().len() as u64, + mtime: self.mtime, + }), + } + } + + fn into_disc_stream(self: Box) -> Box { self } +} + +pub fn nod_to_io_error(e: nod::Error) -> io::Error { + match e { + nod::Error::Io(msg, e) => io::Error::new(e.kind(), format!("{}: {}", msg, e)), + e => io::Error::new(io::ErrorKind::InvalidData, e), + } +} diff --git a/src/vfs/mod.rs b/src/vfs/mod.rs new file mode 100644 index 0000000..eae1f0c --- /dev/null +++ b/src/vfs/mod.rs @@ -0,0 +1,289 @@ +mod common; +mod disc; +mod rarc; +mod std_fs; +mod u8_arc; + +use std::{ + error::Error, + fmt::{Debug, Display, Formatter}, + io, + io::{BufRead, Read, Seek, SeekFrom}, + path::Path, + sync::Arc, +}; + +use anyhow::{anyhow, Context}; +use common::{StaticFile, WindowedFile}; +use disc::{nod_to_io_error, DiscFs}; +use dyn_clone::DynClone; +use filetime::FileTime; +use nodtool::{nod, nod::DiscStream}; +use rarc::RarcFs; +pub use std_fs::StdFs; +use u8_arc::U8Fs; + +use crate::util::{ + ncompress::{YAY0_MAGIC, YAZ0_MAGIC}, + rarc::RARC_MAGIC, + u8_arc::U8_MAGIC, +}; + +pub trait Vfs: DynClone + Send + Sync { + fn open(&mut self, path: &str) -> VfsResult>; + + fn exists(&mut self, path: &str) -> VfsResult; + + fn read_dir(&mut self, path: &str) -> VfsResult>; + + fn metadata(&mut self, path: &str) -> VfsResult; +} + +dyn_clone::clone_trait_object!(Vfs); + +pub trait VfsFile: DiscStream + BufRead { + fn map(&mut self) -> io::Result<&[u8]>; + + fn metadata(&mut self) -> io::Result; + + fn into_disc_stream(self: Box) -> Box; +} + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] +pub enum VfsFileType { + File, + Directory, +} + +pub struct VfsMetadata { + pub file_type: VfsFileType, + pub len: u64, + pub mtime: Option, +} + +impl VfsMetadata { + pub fn is_file(&self) -> bool { self.file_type == VfsFileType::File } + + pub fn is_dir(&self) -> bool { self.file_type == VfsFileType::Directory } +} + +dyn_clone::clone_trait_object!(VfsFile); + +#[derive(Debug)] +pub enum VfsError { + NotFound, + IoError(io::Error), + Other(String), + FileExists, + DirectoryExists, +} + +pub type VfsResult = Result; + +impl From for VfsError { + fn from(e: io::Error) -> Self { VfsError::IoError(e) } +} + +impl From for VfsError { + fn from(e: String) -> Self { VfsError::Other(e) } +} + +impl From<&str> for VfsError { + fn from(e: &str) -> Self { VfsError::Other(e.to_string()) } +} + +impl Display for VfsError { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + match self { + VfsError::NotFound => write!(f, "File or directory not found"), + VfsError::IoError(e) => write!(f, "{}", e), + VfsError::Other(e) => write!(f, "{}", e), + VfsError::FileExists => write!(f, "File already exists"), + VfsError::DirectoryExists => write!(f, "Directory already exists"), + } + } +} + +impl Error for VfsError {} + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] +pub enum FileFormat { + Regular, + Compressed(CompressionKind), + Archive(ArchiveKind), +} + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] +pub enum CompressionKind { + Yay0, + Yaz0, + Nlzss, +} + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] +pub enum ArchiveKind { + Rarc, + U8, + Disc, +} + +pub fn detect(file: &mut R) -> io::Result +where R: Read + Seek + ?Sized { + file.seek(SeekFrom::Start(0))?; + let mut magic = [0u8; 4]; + match file.read_exact(&mut magic) { + Ok(_) => {} + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(FileFormat::Regular), + Err(e) => return Err(e), + } + file.seek_relative(-4)?; + match magic { + YAY0_MAGIC => Ok(FileFormat::Compressed(CompressionKind::Yay0)), + YAZ0_MAGIC => Ok(FileFormat::Compressed(CompressionKind::Yaz0)), + RARC_MAGIC => Ok(FileFormat::Archive(ArchiveKind::Rarc)), + U8_MAGIC => Ok(FileFormat::Archive(ArchiveKind::U8)), + _ => { + let format = nod::Disc::detect(file)?; + file.seek(SeekFrom::Start(0))?; + match format { + Some(_) => Ok(FileFormat::Archive(ArchiveKind::Disc)), + None => Ok(FileFormat::Regular), + } + } + } +} + +pub fn open_path(path: &Path, auto_decompress: bool) -> anyhow::Result> { + open_path_fs(Box::new(StdFs), path, auto_decompress) +} + +pub fn open_path_fs( + mut fs: Box, + path: &Path, + auto_decompress: bool, +) -> anyhow::Result> { + let str = path.to_str().ok_or_else(|| anyhow!("Path is not valid UTF-8"))?; + let mut split = str.split(':').peekable(); + let mut within = String::new(); + loop { + let path = split.next().unwrap(); + let mut file = fs + .open(path) + .with_context(|| format!("Failed to open {}", format_path(path, &within)))?; + match detect(file.as_mut()).with_context(|| { + format!("Failed to detect file type for {}", format_path(path, &within)) + })? { + FileFormat::Regular => { + return match split.next() { + None => Ok(file), + Some(segment) => { + if split.next().is_some() { + return Err(anyhow!( + "{} is not an archive", + format_path(path, &within) + )); + } + match segment { + "nlzss" => Ok(decompress_file(file, CompressionKind::Nlzss) + .with_context(|| { + format!( + "Failed to decompress {} with NLZSS", + format_path(path, &within) + ) + })?), + "yay0" => Ok(decompress_file(file, CompressionKind::Yay0) + .with_context(|| { + format!( + "Failed to decompress {} with Yay0", + format_path(path, &within) + ) + })?), + "yaz0" => Ok(decompress_file(file, CompressionKind::Yaz0) + .with_context(|| { + format!( + "Failed to decompress {} with Yaz0", + format_path(path, &within) + ) + })?), + _ => Err(anyhow!("{} is not an archive", format_path(path, &within))), + } + } + } + } + FileFormat::Compressed(kind) => { + return if split.peek().is_none() { + if auto_decompress { + Ok(decompress_file(file, kind).with_context(|| { + format!("Failed to decompress {}", format_path(path, &within)) + })?) + } else { + Ok(file) + } + } else { + Err(anyhow!("{} is not an archive", format_path(path, &within))) + }; + } + FileFormat::Archive(kind) => { + if split.peek().is_none() { + return Ok(file); + } else { + fs = open_fs(file, kind).with_context(|| { + format!("Failed to open container {}", format_path(path, &within)) + })?; + if !within.is_empty() { + within.push(':'); + } + within.push_str(path); + } + } + } + } +} + +pub fn open_fs(mut file: Box, kind: ArchiveKind) -> io::Result> { + let metadata = file.metadata()?; + match kind { + ArchiveKind::Rarc => Ok(Box::new(RarcFs::new(file)?)), + ArchiveKind::U8 => Ok(Box::new(U8Fs::new(file)?)), + ArchiveKind::Disc => { + let disc = nod::Disc::new_stream(file.into_disc_stream()).map_err(nod_to_io_error)?; + let partition = + disc.open_partition_kind(nod::PartitionKind::Data).map_err(nod_to_io_error)?; + Ok(Box::new(DiscFs::new(partition, metadata.mtime)?)) + } + } +} + +pub fn decompress_file( + mut file: Box, + kind: CompressionKind, +) -> io::Result> { + let metadata = file.metadata()?; + match kind { + CompressionKind::Yay0 => { + let data = file.map()?; + let result = orthrus_ncompress::yay0::Yay0::decompress_from(data) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e.to_string()))?; + Ok(Box::new(StaticFile::new(Arc::from(result), metadata.mtime))) + } + CompressionKind::Yaz0 => { + let data = file.map()?; + let result = orthrus_ncompress::yaz0::Yaz0::decompress_from(data) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e.to_string()))?; + Ok(Box::new(StaticFile::new(Arc::from(result), metadata.mtime))) + } + CompressionKind::Nlzss => { + let result = nintendo_lz::decompress(&mut file) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e.to_string()))?; + Ok(Box::new(StaticFile::new(Arc::from(result.as_slice()), metadata.mtime))) + } + } +} + +fn format_path(path: &str, within: &str) -> String { + if within.is_empty() { + format!("'{}'", path) + } else { + format!("'{}' (within '{}')", path, within) + } +} diff --git a/src/vfs/rarc.rs b/src/vfs/rarc.rs new file mode 100644 index 0000000..2e88458 --- /dev/null +++ b/src/vfs/rarc.rs @@ -0,0 +1,76 @@ +use std::io; + +use super::{Vfs, VfsError, VfsFile, VfsFileType, VfsMetadata, VfsResult, WindowedFile}; +use crate::util::rarc::{RarcNodeKind, RarcView}; + +#[derive(Clone)] +pub struct RarcFs { + file: Box, +} + +impl RarcFs { + pub fn new(file: Box) -> io::Result { Ok(Self { file }) } + + fn view(&mut self) -> io::Result { + let data = self.file.map()?; + RarcView::new(data).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) + } +} + +impl Vfs for RarcFs { + fn open(&mut self, path: &str) -> VfsResult> { + let view = self.view()?; + match view.find(path) { + Some(RarcNodeKind::File(_, node)) => { + let offset = view.header.header_len() as u64 + + view.header.data_offset() as u64 + + node.data_offset() as u64; + let len = node.data_length() as u64; + let file = WindowedFile::new(self.file.clone(), offset, len)?; + Ok(Box::new(file)) + } + Some(RarcNodeKind::Directory(_, _)) => Err(VfsError::DirectoryExists), + None => Err(VfsError::NotFound), + } + } + + fn exists(&mut self, path: &str) -> VfsResult { + let view = self.view()?; + Ok(view.find(path).is_some()) + } + + fn read_dir(&mut self, path: &str) -> VfsResult> { + let view = self.view()?; + match view.find(path) { + Some(RarcNodeKind::Directory(_, dir)) => { + let mut entries = Vec::new(); + for node in view.children(dir) { + let name = view.get_string(node.name_offset())?; + if name == "." || name == ".." { + continue; + } + entries.push(name.to_string()); + } + Ok(entries) + } + Some(RarcNodeKind::File(_, _)) => Err(VfsError::FileExists), + None => Err(VfsError::NotFound), + } + } + + fn metadata(&mut self, path: &str) -> VfsResult { + let metadata = self.file.metadata()?; + let view = self.view()?; + match view.find(path) { + Some(RarcNodeKind::File(_, node)) => Ok(VfsMetadata { + file_type: VfsFileType::File, + len: node.data_length() as u64, + mtime: metadata.mtime, + }), + Some(RarcNodeKind::Directory(_, _)) => { + Ok(VfsMetadata { file_type: VfsFileType::Directory, len: 0, mtime: metadata.mtime }) + } + None => Err(VfsError::NotFound), + } + } +} diff --git a/src/vfs/std_fs.rs b/src/vfs/std_fs.rs new file mode 100644 index 0000000..0364e41 --- /dev/null +++ b/src/vfs/std_fs.rs @@ -0,0 +1,106 @@ +use std::{ + io, + io::{BufRead, BufReader, Read, Seek, SeekFrom}, + path::{Path, PathBuf}, +}; + +use filetime::FileTime; + +use super::{DiscStream, Vfs, VfsFile, VfsFileType, VfsMetadata, VfsResult}; + +#[derive(Clone)] +pub struct StdFs; + +impl Vfs for StdFs { + fn open(&mut self, path: &str) -> VfsResult> { + let mut file = StdFile::new(PathBuf::from(path)); + file.file()?; // Open the file now to check for errors + Ok(Box::new(file)) + } + + fn exists(&mut self, path: &str) -> VfsResult { Ok(Path::new(path).exists()) } + + fn read_dir(&mut self, path: &str) -> VfsResult> { + let entries = std::fs::read_dir(path)? + .map(|entry| entry.map(|e| e.file_name().to_string_lossy().into_owned())) + .collect::, _>>()?; + Ok(entries) + } + + fn metadata(&mut self, path: &str) -> VfsResult { + let metadata = std::fs::metadata(path)?; + Ok(VfsMetadata { + file_type: if metadata.is_dir() { VfsFileType::Directory } else { VfsFileType::File }, + len: metadata.len(), + mtime: Some(FileTime::from_last_modification_time(&metadata)), + }) + } +} + +pub struct StdFile { + path: PathBuf, + file: Option>, + mmap: Option, +} + +impl Clone for StdFile { + #[inline] + fn clone(&self) -> Self { Self { path: self.path.clone(), file: None, mmap: None } } +} + +impl StdFile { + #[inline] + pub fn new(path: PathBuf) -> Self { StdFile { path, file: None, mmap: None } } + + pub fn file(&mut self) -> io::Result<&mut BufReader> { + if self.file.is_none() { + self.file = Some(BufReader::new(std::fs::File::open(&self.path)?)); + } + Ok(self.file.as_mut().unwrap()) + } +} + +impl BufRead for StdFile { + #[inline] + fn fill_buf(&mut self) -> io::Result<&[u8]> { self.file()?.fill_buf() } + + #[inline] + fn consume(&mut self, amt: usize) { + if let Ok(file) = self.file() { + file.consume(amt); + } + } +} + +impl Read for StdFile { + #[inline] + fn read(&mut self, buf: &mut [u8]) -> io::Result { self.file()?.read(buf) } +} + +impl Seek for StdFile { + #[inline] + fn seek(&mut self, pos: SeekFrom) -> io::Result { self.file()?.seek(pos) } +} + +impl VfsFile for StdFile { + fn map(&mut self) -> io::Result<&[u8]> { + if self.file.is_none() { + self.file = Some(BufReader::new(std::fs::File::open(&self.path)?)); + } + if self.mmap.is_none() { + self.mmap = Some(unsafe { memmap2::Mmap::map(self.file.as_ref().unwrap().get_ref())? }); + } + Ok(self.mmap.as_ref().unwrap()) + } + + fn metadata(&mut self) -> io::Result { + let metadata = std::fs::metadata(&self.path)?; + Ok(VfsMetadata { + file_type: if metadata.is_dir() { VfsFileType::Directory } else { VfsFileType::File }, + len: metadata.len(), + mtime: Some(FileTime::from_last_modification_time(&metadata)), + }) + } + + fn into_disc_stream(self: Box) -> Box { self } +} diff --git a/src/vfs/u8_arc.rs b/src/vfs/u8_arc.rs new file mode 100644 index 0000000..044a61e --- /dev/null +++ b/src/vfs/u8_arc.rs @@ -0,0 +1,89 @@ +use std::io; + +use super::{Vfs, VfsError, VfsFile, VfsFileType, VfsMetadata, VfsResult, WindowedFile}; +use crate::util::u8_arc::{U8NodeKind, U8View}; + +#[derive(Clone)] +pub struct U8Fs { + file: Box, +} + +impl U8Fs { + pub fn new(file: Box) -> io::Result { Ok(Self { file }) } + + fn view(&mut self) -> io::Result { + let data = self.file.map()?; + U8View::new(data).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) + } +} + +impl Vfs for U8Fs { + fn open(&mut self, path: &str) -> VfsResult> { + let view = self.view()?; + match view.find(path) { + Some((_, node)) => match node.kind() { + U8NodeKind::File => { + let offset = node.offset() as u64; + let len = node.length() as u64; + let file = WindowedFile::new(self.file.clone(), offset, len)?; + Ok(Box::new(file)) + } + U8NodeKind::Directory => Err(VfsError::DirectoryExists), + U8NodeKind::Invalid => Err(VfsError::from("U8: Invalid node kind")), + }, + None => Err(VfsError::NotFound), + } + } + + fn exists(&mut self, path: &str) -> VfsResult { + let view = self.view()?; + Ok(view.find(path).is_some()) + } + + fn read_dir(&mut self, path: &str) -> VfsResult> { + let view = self.view()?; + match view.find(path) { + Some((idx, node)) => match node.kind() { + U8NodeKind::File => Err(VfsError::FileExists), + U8NodeKind::Directory => { + let mut entries = Vec::new(); + let mut idx = idx + 1; + let end = node.length() as usize; + while idx < end { + let child = view.nodes.get(idx).copied().ok_or(VfsError::NotFound)?; + entries.push(view.get_name(child)?.to_string()); + if child.is_dir() { + idx = child.length() as usize; + } else { + idx += 1; + } + } + Ok(entries) + } + U8NodeKind::Invalid => Err(VfsError::from("U8: Invalid node kind")), + }, + None => Err(VfsError::NotFound), + } + } + + fn metadata(&mut self, path: &str) -> VfsResult { + let metdata = self.file.metadata()?; + let view = self.view()?; + match view.find(path) { + Some((_, node)) => match node.kind() { + U8NodeKind::File => Ok(VfsMetadata { + file_type: VfsFileType::File, + len: node.length() as u64, + mtime: metdata.mtime, + }), + U8NodeKind::Directory => Ok(VfsMetadata { + file_type: VfsFileType::Directory, + len: 0, + mtime: metdata.mtime, + }), + U8NodeKind::Invalid => Err(VfsError::from("U8: Invalid node kind")), + }, + None => Err(VfsError::NotFound), + } + } +}