diff --git a/assets/ldscript.lcf b/assets/ldscript.lcf index 7cfb705..24a3695 100644 --- a/assets/ldscript.lcf +++ b/assets/ldscript.lcf @@ -19,11 +19,6 @@ SECTIONS __ArenaHi = $ARENAHI; } -FORCEFILES -{ - $FORCEFILES -} - FORCEACTIVE { $FORCEACTIVE diff --git a/assets/ldscript_partial.lcf b/assets/ldscript_partial.lcf index 25b04e8..6afa87e 100644 --- a/assets/ldscript_partial.lcf +++ b/assets/ldscript_partial.lcf @@ -19,8 +19,3 @@ FORCEACTIVE _epilog $FORCEACTIVE } - -FORCEFILES -{ - $FORCEFILES -} diff --git a/src/analysis/objects.rs b/src/analysis/objects.rs index 16b8a93..6a84b6f 100644 --- a/src/analysis/objects.rs +++ b/src/analysis/objects.rs @@ -14,7 +14,7 @@ pub fn detect_objects(obj: &mut ObjInfo) -> Result<()> { let mut replace_symbols = vec![]; for (idx, symbol) in obj.symbols.for_section(section_index) { let mut symbol = symbol.clone(); - if is_linker_generated_label(&symbol.name) { + if is_linker_generated_label(&symbol.name) || symbol.name.starts_with("..") { continue; } let expected_size = match symbol.data_kind { @@ -124,6 +124,11 @@ pub fn detect_strings(obj: &mut ObjInfo) -> Result<()> { .for_section(section_index) .filter(|(_, sym)| sym.data_kind == ObjDataKind::Unknown) { + if symbol.name.starts_with("@stringBase") { + symbols_set.push((symbol_idx, ObjDataKind::StringTable, symbol.size as usize)); + continue; + } + let data = section.symbol_data(symbol)?; match is_string(data) { StringResult::None => {} diff --git a/src/analysis/pass.rs b/src/analysis/pass.rs index bb8e09d..0b16f87 100644 --- a/src/analysis/pass.rs +++ b/src/analysis/pass.rs @@ -33,7 +33,7 @@ impl AnalysisPass for FindTRKInterruptVectorTable { if data.starts_with(TRK_TABLE_HEADER.as_bytes()) && data[TRK_TABLE_HEADER.as_bytes().len()] == 0 { - log::info!("Found gTRKInterruptVectorTable @ {:#010X}", start); + log::debug!("Found gTRKInterruptVectorTable @ {:#010X}", start); state.known_symbols.insert(start, ObjSymbol { name: "gTRKInterruptVectorTable".to_string(), demangled_name: None, diff --git a/src/analysis/signatures.rs b/src/analysis/signatures.rs index d0b06fc..93d6cfc 100644 --- a/src/analysis/signatures.rs +++ b/src/analysis/signatures.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, bail, Result}; +use anyhow::{anyhow, Result}; use crate::{ analysis::{ @@ -272,21 +272,21 @@ fn apply_ctors_signatures(obj: &mut ObjInfo) -> Result<()> { } fn apply_dtors_signatures(obj: &mut ObjInfo) -> Result<()> { - let Some((_, symbol)) = obj.symbols.by_name("_dtors")? else { - for symbol in obj.symbols.iter() { - println!("{:?} {:#010X} {}", symbol.section, symbol.address, symbol.name); - } - bail!("Missing _dtors symbol"); - // return Ok(()); - }; - let dtors_section_index = - symbol.section.ok_or_else(|| anyhow!("Missing _dtors symbol section"))?; - let dtors_section = &obj.sections[dtors_section_index]; + let (dtors_section_index, dtors_section) = + if let Some((_, symbol)) = obj.symbols.by_name("_dtors")? { + let section_index = + symbol.section.ok_or_else(|| anyhow!("Missing _dtors symbol section"))?; + (section_index, &obj.sections[section_index]) + } else if let Some((section_index, section)) = obj.sections.by_name(".dtors")? { + (section_index, section) + } else { + return Ok(()); + }; // __destroy_global_chain_reference + null pointer if dtors_section.size < 8 { return Ok(()); } - let address = symbol.address; + let address = dtors_section.address; let dgc_target = read_address(obj, dtors_section, address as u32).ok(); let fce_target = read_address(obj, dtors_section, address as u32 + 4).ok(); let mut found_dgc = false; @@ -319,7 +319,7 @@ fn apply_dtors_signatures(obj: &mut ObjInfo) -> Result<()> { )?; found_dgc = true; } else { - log::warn!("Failed to match __destroy_global_chain signature ({:#010X})", dgc_target); + log::debug!("Failed to match __destroy_global_chain signature ({:#010X})", dgc_target); } } @@ -445,3 +445,40 @@ pub fn apply_signatures_post(obj: &mut ObjInfo) -> Result<()> { } Ok(()) } + +/// Create _ctors and _dtors symbols if missing +pub fn update_ctors_dtors(obj: &mut ObjInfo) -> Result<()> { + if obj.symbols.by_name("_ctors")?.is_none() { + if let Some((section_index, section)) = obj.sections.by_name(".ctors")? { + obj.symbols.add_direct(ObjSymbol { + name: "_ctors".to_string(), + demangled_name: None, + address: section.address, + section: Some(section_index), + size: 0, + size_known: true, + flags: ObjSymbolFlagSet(ObjSymbolFlags::Global.into()), + kind: ObjSymbolKind::Unknown, + align: None, + data_kind: Default::default(), + })?; + } + } + if obj.symbols.by_name("_dtors")?.is_none() { + if let Some((section_index, section)) = obj.sections.by_name(".dtors")? { + obj.symbols.add_direct(ObjSymbol { + name: "_dtors".to_string(), + demangled_name: None, + address: section.address, + section: Some(section_index), + size: 0, + size_known: true, + flags: ObjSymbolFlagSet(ObjSymbolFlags::Global.into()), + kind: ObjSymbolKind::Unknown, + align: None, + data_kind: Default::default(), + })?; + } + } + Ok(()) +} diff --git a/src/analysis/tracker.rs b/src/analysis/tracker.rs index bd36094..4cd8d92 100644 --- a/src/analysis/tracker.rs +++ b/src/analysis/tracker.rs @@ -433,7 +433,9 @@ impl Tracker { || self.sda2_base == Some(addr) || self.sda_base == Some(addr) { - return Some(SectionAddress::new(usize::MAX, addr)); + let section_index = + obj.sections.at_address(addr).ok().map(|(idx, _)| idx).unwrap_or(usize::MAX); + return Some(SectionAddress::new(section_index, addr)); } // if addr > 0x80000000 && addr < 0x80003100 { // return true; diff --git a/src/cmd/ar.rs b/src/cmd/ar.rs index 27eeb4f..afccfb8 100644 --- a/src/cmd/ar.rs +++ b/src/cmd/ar.rs @@ -60,8 +60,8 @@ fn create(args: CreateArgs) -> Result<()> { Entry::Vacant(e) => e.insert(Vec::new()), Entry::Occupied(_) => bail!("Duplicate file name '{path_str}'"), }; - let mmap = map_file(path)?; - let obj = object::File::parse(&*mmap)?; + let file = map_file(path)?; + let obj = object::File::parse(file.as_slice())?; for symbol in obj.symbols() { if symbol.scope() == SymbolScope::Dynamic { entries.push(symbol.name_bytes()?.to_vec()); diff --git a/src/cmd/dol.rs b/src/cmd/dol.rs index 36e19da..37d99f9 100644 --- a/src/cmd/dol.rs +++ b/src/cmd/dol.rs @@ -1,8 +1,10 @@ use std::{ borrow::Cow, + cmp::min, collections::{btree_map::Entry, hash_map, BTreeMap, HashMap}, + ffi::OsStr, fs, - fs::{DirBuilder, File}, + fs::DirBuilder, io::Write, mem::take, path::{Path, PathBuf}, @@ -24,12 +26,13 @@ use crate::{ AnalysisPass, FindRelCtorsDtors, FindRelRodataData, FindSaveRestSleds, FindTRKInterruptVectorTable, }, - signatures::{apply_signatures, apply_signatures_post}, + signatures::{apply_signatures, apply_signatures_post, update_ctors_dtors}, tracker::Tracker, }, + cmd::shasum::file_sha1_string, obj::{ - best_match_for_reloc, ObjDataKind, ObjInfo, ObjReloc, ObjRelocKind, ObjSectionKind, - ObjSymbol, ObjSymbolFlagSet, ObjSymbolFlags, ObjSymbolKind, ObjSymbolScope, SymbolIndex, + ObjDataKind, ObjInfo, ObjReloc, ObjRelocKind, ObjSectionKind, ObjSymbol, ObjSymbolFlagSet, + ObjSymbolFlags, ObjSymbolKind, ObjSymbolScope, SymbolIndex, }, util::{ asm::write_asm, @@ -41,10 +44,13 @@ use crate::{ dep::DepFile, dol::process_dol, elf::{process_elf, write_elf}, - file::{buf_writer, decompress_if_needed, map_file, touch, verify_hash, Reader}, + file::{ + buf_reader, buf_writer, decompress_if_needed, map_file, touch, verify_hash, + FileIterator, Reader, + }, lcf::{asm_path_for_unit, generate_ldscript, obj_path_for_unit}, map::apply_map_file, - rel::process_rel, + rel::{process_rel, process_rel_header}, rso::{process_rso, DOL_SECTION_ABS, DOL_SECTION_NAMES}, split::{is_linker_generated_object, split_obj, update_splits}, IntoCow, ToCow, @@ -66,6 +72,7 @@ enum SubCommand { Split(SplitArgs), Diff(DiffArgs), Apply(ApplyArgs), + Config(ConfigArgs), } #[derive(FromArgs, PartialEq, Eq, Debug)] @@ -128,6 +135,18 @@ pub struct ApplyArgs { map_file: PathBuf, } +#[derive(FromArgs, PartialEq, Eq, Debug)] +/// Generates a project configuration file from a DOL (& RELs). +#[argp(subcommand, name = "config")] +pub struct ConfigArgs { + #[argp(positional)] + /// object files + objects: Vec, + #[argp(option, short = 'o')] + /// output config YAML file + out_file: PathBuf, +} + #[inline] fn bool_true() -> bool { true } @@ -195,15 +214,21 @@ pub struct ProjectConfig { pub detect_strings: bool, #[serde(default = "bool_true")] pub write_asm: bool, - /// Adds all objects to FORCEFILES in the linker script. - #[serde(default)] - pub auto_force_files: bool, /// Specifies the start of the common BSS section. pub common_start: Option, + /// Disables all analysis passes that yield new symbols, + /// and instead assumes that all symbols are known. + #[serde(default)] + pub symbols_known: bool, + /// Fills gaps between symbols with + #[serde(default = "bool_true")] + pub fill_gaps: bool, } #[derive(Serialize, Deserialize, Debug, Clone)] pub struct ModuleConfig { + /// Object name. If not specified, the file name without extension will be used. + pub name: Option, #[serde(with = "path_slash_serde")] pub object: PathBuf, pub hash: Option, @@ -213,6 +238,9 @@ pub struct ModuleConfig { pub symbols: Option, #[serde(with = "path_slash_serde_option", default)] pub map: Option, + /// Forces the given symbols to be active in the linker script. + #[serde(default)] + pub force_active: Vec, } impl ModuleConfig { @@ -231,7 +259,9 @@ impl ModuleConfig { } } - pub fn name(&self) -> Cow<'_, str> { self.file_prefix() } + pub fn name(&self) -> Cow<'_, str> { + self.name.as_ref().map(|n| n.as_str().to_cow()).unwrap_or_else(|| self.file_prefix()) + } } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -264,12 +294,12 @@ pub fn run(args: Args) -> Result<()> { SubCommand::Split(c_args) => split(c_args), SubCommand::Diff(c_args) => diff(c_args), SubCommand::Apply(c_args) => apply(c_args), + SubCommand::Config(c_args) => config(c_args), } } -fn apply_selfile(obj: &mut ObjInfo, selfile: &Path) -> Result<()> { - log::info!("Loading {}", selfile.display()); - let rso = process_rso(selfile)?; +fn apply_selfile(obj: &mut ObjInfo, buf: &[u8]) -> Result<()> { + let rso = process_rso(&mut Reader::new(buf))?; for symbol in rso.symbols.iter() { let dol_section_index = match symbol.section { Some(section) => section, @@ -347,12 +377,16 @@ fn apply_selfile(obj: &mut ObjInfo, selfile: &Path) -> Result<()> { } fn info(args: InfoArgs) -> Result<()> { - let mut obj = process_dol(&args.dol_file)?; + let mut obj = { + let file = map_file(&args.dol_file)?; + let data = decompress_if_needed(file.as_slice())?; + process_dol(data.as_ref(), "")? + }; apply_signatures(&mut obj)?; let mut state = AnalyzerState::default(); state.detect_functions(&obj)?; - log::info!("Discovered {} functions", state.function_slices.len()); + log::debug!("Discovered {} functions", state.function_slices.len()); FindTRKInterruptVectorTable::execute(&mut state, &obj)?; FindSaveRestSleds::execute(&mut state, &obj)?; @@ -361,7 +395,9 @@ fn info(args: InfoArgs) -> Result<()> { apply_signatures_post(&mut obj)?; if let Some(selfile) = &args.selfile { - apply_selfile(&mut obj, selfile)?; + let file = map_file(selfile)?; + let data = decompress_if_needed(file.as_slice())?; + apply_selfile(&mut obj, data.as_ref())?; } println!("{}:", obj.name); @@ -434,14 +470,10 @@ fn update_symbols(obj: &mut ObjInfo, modules: &ModuleMap<'_>) -> Result<()> { .get_elf_index(rel_reloc.target_section as usize) .ok_or_else(|| anyhow!("Failed to locate REL section {}", rel_reloc.target_section))?; - let target_symbols = obj - .symbols - .at_section_address(target_section_index, rel_reloc.addend) - .filter(|(_, s)| s.referenced_by(rel_reloc.kind)) - .collect_vec(); - let target_symbol = best_match_for_reloc(target_symbols, rel_reloc.kind); - - if let Some((symbol_index, symbol)) = target_symbol { + if let Some((symbol_index, symbol)) = obj.symbols.for_relocation( + SectionAddress::new(target_section_index, rel_reloc.addend), + rel_reloc.kind, + )? { // Update symbol log::trace!( "Found symbol in section {} at {:#010X}: {}", @@ -524,16 +556,15 @@ fn create_relocations(obj: &mut ObjInfo, modules: &ModuleMap<'_>, dol_obj: &ObjI )? }; - let target_symbols = target_obj - .symbols - .at_section_address(target_section_index, rel_reloc.addend) - .filter(|(_, s)| s.referenced_by(rel_reloc.kind)) - .collect_vec(); - let Some((symbol_index, symbol)) = best_match_for_reloc(target_symbols, rel_reloc.kind) + let Some((symbol_index, symbol)) = target_obj.symbols.for_relocation( + SectionAddress::new(target_section_index, rel_reloc.addend), + rel_reloc.kind, + )? else { bail!( - "Couldn't find module {} symbol in section {} at {:#010X}", + "Couldn't find module {} ({}) symbol in section {} at {:#010X}", rel_reloc.module_id, + target_obj.name, rel_reloc.target_section, rel_reloc.addend ); @@ -625,11 +656,15 @@ fn resolve_external_relocations( type AnalyzeResult = (ObjInfo, Vec); fn load_analyze_dol(config: &ProjectConfig) -> Result { - // log::info!("Loading {}", config.object.display()); - if let Some(hash_str) = &config.base.hash { - verify_hash(&config.base.object, hash_str)?; - } - let mut obj = process_dol(&config.base.object)?; + log::debug!("Loading {}", config.base.object.display()); + let mut obj = { + let file = map_file(&config.base.object)?; + let data = decompress_if_needed(file.as_slice())?; + if let Some(hash_str) = &config.base.hash { + verify_hash(data.as_ref(), hash_str)?; + } + process_dol(data.as_ref(), config.base.name().as_ref())? + }; let mut dep = vec![config.base.object.clone()]; if let Some(comment_version) = config.mw_comment_version { @@ -651,29 +686,38 @@ fn load_analyze_dol(config: &ProjectConfig) -> Result { dep.push(symbols_path.clone()); } - // TODO move before symbols? - debug!("Performing signature analysis"); - apply_signatures(&mut obj)?; + if !config.symbols_known { + // TODO move before symbols? + debug!("Performing signature analysis"); + apply_signatures(&mut obj)?; - if !config.quick_analysis { - let mut state = AnalyzerState::default(); - debug!("Detecting function boundaries"); - state.detect_functions(&obj)?; + if !config.quick_analysis { + let mut state = AnalyzerState::default(); + debug!("Detecting function boundaries"); + state.detect_functions(&obj)?; - FindTRKInterruptVectorTable::execute(&mut state, &obj)?; - FindSaveRestSleds::execute(&mut state, &obj)?; - state.apply(&mut obj)?; + FindTRKInterruptVectorTable::execute(&mut state, &obj)?; + FindSaveRestSleds::execute(&mut state, &obj)?; + state.apply(&mut obj)?; + } + + apply_signatures_post(&mut obj)?; } - apply_signatures_post(&mut obj)?; - if let Some(selfile) = &config.selfile { + log::info!("Loading {}", selfile.display()); + let file = map_file(selfile)?; + let data = decompress_if_needed(file.as_slice())?; if let Some(hash) = &config.selfile_hash { - verify_hash(selfile, hash)?; + verify_hash(data.as_ref(), hash)?; } - apply_selfile(&mut obj, selfile)?; + apply_selfile(&mut obj, data.as_ref())?; dep.push(selfile.clone()); } + + // Create _ctors and _dtors symbols if missing + update_ctors_dtors(&mut obj)?; + Ok((obj, dep)) } @@ -691,7 +735,7 @@ fn split_write_obj( debug!("Applying relocations"); tracker.apply(obj, false)?; - if config.detect_objects { + if !config.symbols_known && config.detect_objects { debug!("Detecting object boundaries"); detect_objects(obj)?; } @@ -702,7 +746,11 @@ fn split_write_obj( } debug!("Adjusting splits"); - update_splits(obj, if obj.module_id == 0 { config.common_start } else { None })?; + update_splits( + obj, + if obj.module_id == 0 { config.common_start } else { None }, + config.fill_gaps, + )?; if !no_update { debug!("Writing configuration"); @@ -741,29 +789,32 @@ fn split_write_obj( } // Generate ldscript.lcf - fs::write(&out_config.ldscript, generate_ldscript(obj, config.auto_force_files)?)?; + fs::write(&out_config.ldscript, generate_ldscript(obj, &module_config.force_active)?)?; - debug!("Writing disassembly"); - let asm_dir = out_dir.join("asm"); - for (unit, split_obj) in obj.link_order.iter().zip(&split_objs) { - let out_path = asm_dir.join(asm_path_for_unit(&unit.name)); + if config.write_asm { + debug!("Writing disassembly"); + let asm_dir = out_dir.join("asm"); + for (unit, split_obj) in obj.link_order.iter().zip(&split_objs) { + let out_path = asm_dir.join(asm_path_for_unit(&unit.name)); - let mut w = buf_writer(&out_path)?; - write_asm(&mut w, split_obj) - .with_context(|| format!("Failed to write {}", out_path.display()))?; - w.flush()?; + let mut w = buf_writer(&out_path)?; + write_asm(&mut w, split_obj) + .with_context(|| format!("Failed to write {}", out_path.display()))?; + w.flush()?; + } } Ok(out_config) } fn load_analyze_rel(config: &ProjectConfig, module_config: &ModuleConfig) -> Result { debug!("Loading {}", module_config.object.display()); - if let Some(hash_str) = &module_config.hash { - verify_hash(&module_config.object, hash_str)?; - } let map = map_file(&module_config.object)?; - let buf = decompress_if_needed(&map)?; - let (_, mut module_obj) = process_rel(&mut Reader::new(buf.as_ref()))?; + let buf = decompress_if_needed(map.as_slice())?; + if let Some(hash_str) = &module_config.hash { + verify_hash(buf.as_ref(), hash_str)?; + } + let (_, mut module_obj) = + process_rel(&mut Reader::new(buf.as_ref()), module_config.name().as_ref())?; if let Some(comment_version) = config.mw_comment_version { module_obj.mw_comment = Some(MWComment::new(comment_version)?); @@ -785,16 +836,22 @@ fn load_analyze_rel(config: &ProjectConfig, module_config: &ModuleConfig) -> Res dep.push(symbols_path.clone()); } - debug!("Analyzing module {}", module_obj.module_id); - if !config.quick_analysis { - let mut state = AnalyzerState::default(); - state.detect_functions(&module_obj)?; - FindRelCtorsDtors::execute(&mut state, &module_obj)?; - FindRelRodataData::execute(&mut state, &module_obj)?; - state.apply(&mut module_obj)?; + if !config.symbols_known { + debug!("Analyzing module {}", module_obj.module_id); + if !config.quick_analysis { + let mut state = AnalyzerState::default(); + state.detect_functions(&module_obj)?; + FindRelCtorsDtors::execute(&mut state, &module_obj)?; + FindRelRodataData::execute(&mut state, &module_obj)?; + state.apply(&mut module_obj)?; + } + apply_signatures(&mut module_obj)?; + apply_signatures_post(&mut module_obj)?; } - apply_signatures(&mut module_obj)?; - apply_signatures_post(&mut module_obj)?; + + // Create _ctors and _dtors symbols if missing + update_ctors_dtors(&mut module_obj)?; + Ok((module_obj, dep)) } @@ -805,18 +862,32 @@ fn split(args: SplitArgs) -> Result<()> { let command_start = Instant::now(); info!("Loading {}", args.config.display()); - let mut config_file = File::open(&args.config) - .with_context(|| format!("Failed to open config file '{}'", args.config.display()))?; - let config: ProjectConfig = serde_yaml::from_reader(&mut config_file)?; + let mut config: ProjectConfig = { + let mut config_file = buf_reader(&args.config)?; + serde_yaml::from_reader(&mut config_file)? + }; + + for module_config in config.modules.iter_mut() { + let file = map_file(&module_config.object)?; + let buf = decompress_if_needed(file.as_slice())?; + if let Some(hash_str) = &module_config.hash { + verify_hash(buf.as_ref(), hash_str)?; + } else { + module_config.hash = Some(file_sha1_string(&mut Reader::new(buf.as_ref()))?); + } + } let out_config_path = args.out_dir.join("config.json"); let mut dep = DepFile::new(out_config_path.clone()); let module_count = config.modules.len() + 1; + let num_threads = min(rayon::current_num_threads(), module_count); info!( - "Loading and analyzing {} modules (using {} threads)", + "Loading and analyzing {} module{} (using {} thread{})", module_count, - rayon::current_num_threads() + if module_count == 1 { "" } else { "s" }, + num_threads, + if num_threads == 1 { "" } else { "s" } ); let mut dol_result: Option> = None; let mut modules_result: Option>> = None; @@ -871,11 +942,13 @@ fn split(args: SplitArgs) -> Result<()> { let module_ids = modules.keys().cloned().collect_vec(); // Create any missing symbols (referenced from other modules) and set FORCEACTIVE - update_symbols(&mut obj, &modules)?; - for &module_id in &module_ids { - let (module_config, mut module_obj) = modules.remove(&module_id).unwrap(); - update_symbols(&mut module_obj, &modules)?; - modules.insert(module_id, (module_config, module_obj)); + if !config.symbols_known { + update_symbols(&mut obj, &modules)?; + for &module_id in &module_ids { + let (module_config, mut module_obj) = modules.remove(&module_id).unwrap(); + update_symbols(&mut module_obj, &modules)?; + modules.insert(module_id, (module_config, module_obj)); + } } // Create relocations to symbols in other modules @@ -1135,12 +1208,18 @@ fn validate>(obj: &ObjInfo, elf_file: P, state: &AnalyzerState) - fn diff(args: DiffArgs) -> Result<()> { log::info!("Loading {}", args.config.display()); - let mut config_file = File::open(&args.config) - .with_context(|| format!("Failed to open config file '{}'", args.config.display()))?; + let mut config_file = buf_reader(&args.config)?; let config: ProjectConfig = serde_yaml::from_reader(&mut config_file)?; log::info!("Loading {}", config.base.object.display()); - let mut obj = process_dol(&config.base.object)?; + let mut obj = { + let file = map_file(&config.base.object)?; + let data = decompress_if_needed(file.as_slice())?; + if let Some(hash_str) = &config.base.hash { + verify_hash(data.as_ref(), hash_str)?; + } + process_dol(data.as_ref(), config.base.name().as_ref())? + }; if let Some(symbols_path) = &config.base.symbols { apply_symbols_file(symbols_path, &mut obj)?; @@ -1267,6 +1346,8 @@ fn diff(args: DiffArgs) -> Result<()> { orig_sym.size, orig_sym.address ); + log::error!("Original: {}", hex::encode_upper(orig_data)); + log::error!("Linked: {}", hex::encode_upper(linked_data)); return Ok(()); } } @@ -1277,12 +1358,18 @@ fn diff(args: DiffArgs) -> Result<()> { fn apply(args: ApplyArgs) -> Result<()> { log::info!("Loading {}", args.config.display()); - let mut config_file = File::open(&args.config) - .with_context(|| format!("Failed to open config file '{}'", args.config.display()))?; + let mut config_file = buf_reader(&args.config)?; let config: ProjectConfig = serde_yaml::from_reader(&mut config_file)?; log::info!("Loading {}", config.base.object.display()); - let mut obj = process_dol(&config.base.object)?; + let mut obj = { + let file = map_file(&config.base.object)?; + let data = decompress_if_needed(file.as_slice())?; + if let Some(hash_str) = &config.base.hash { + verify_hash(data.as_ref(), hash_str)?; + } + process_dol(data.as_ref(), config.base.name().as_ref())? + }; if let Some(symbols_path) = &config.base.symbols { if !apply_symbols_file(symbols_path, &mut obj)? { @@ -1429,3 +1516,75 @@ fn apply(args: ApplyArgs) -> Result<()> { Ok(()) } + +fn config(args: ConfigArgs) -> Result<()> { + let mut config = ProjectConfig { + base: ModuleConfig { + name: None, + object: Default::default(), + hash: None, + splits: None, + symbols: None, + map: None, + force_active: vec![], + }, + selfile: None, + selfile_hash: None, + mw_comment_version: None, + quick_analysis: false, + modules: vec![], + detect_objects: true, + detect_strings: true, + write_asm: true, + common_start: None, + symbols_known: false, + fill_gaps: true, + }; + + let mut modules = BTreeMap::::new(); + for result in FileIterator::new(&args.objects)? { + let (path, entry) = result?; + log::info!("Loading {}", path.display()); + + match path.extension() { + Some(ext) if ext.eq_ignore_ascii_case(OsStr::new("dol")) => { + config.base.object = path; + config.base.hash = Some(file_sha1_string(&mut entry.as_reader())?); + } + Some(ext) if ext.eq_ignore_ascii_case(OsStr::new("rel")) => { + let header = process_rel_header(&mut entry.as_reader())?; + modules.insert(header.module_id, ModuleConfig { + name: None, + object: path, + hash: Some(file_sha1_string(&mut entry.as_reader())?), + splits: None, + symbols: None, + map: None, + force_active: vec![], + }); + } + Some(ext) if ext.eq_ignore_ascii_case(OsStr::new("sel")) => { + config.selfile = Some(path); + config.selfile_hash = Some(file_sha1_string(&mut entry.as_reader())?); + } + Some(ext) if ext.eq_ignore_ascii_case(OsStr::new("rso")) => { + config.modules.push(ModuleConfig { + name: None, + object: path, + hash: Some(file_sha1_string(&mut entry.as_reader())?), + splits: None, + symbols: None, + map: None, + force_active: vec![], + }); + } + _ => bail!("Unknown file extension: '{}'", path.display()), + } + } + config.modules.extend(modules.into_values()); + + let mut out = buf_writer(&args.out_file)?; + serde_yaml::to_writer(&mut out, &config)?; + out.flush()?; + Ok(()) +} diff --git a/src/cmd/dwarf.rs b/src/cmd/dwarf.rs index 50c2e81..2bb19e5 100644 --- a/src/cmd/dwarf.rs +++ b/src/cmd/dwarf.rs @@ -49,9 +49,10 @@ pub fn run(args: Args) -> Result<()> { } fn dump(args: DumpArgs) -> Result<()> { - let mmap = map_file(&args.in_file)?; - if mmap.starts_with(b"!\n") { - let mut archive = ar::Archive::new(&*mmap); + let file = map_file(&args.in_file)?; + let buf = file.as_slice(); + if buf.starts_with(b"!\n") { + let mut archive = ar::Archive::new(buf); while let Some(result) = archive.next_entry() { let mut e = match result { Ok(e) => e, @@ -85,7 +86,7 @@ fn dump(args: DumpArgs) -> Result<()> { } } } else { - let obj_file = object::read::File::parse(&*mmap)?; + let obj_file = object::read::File::parse(buf)?; let debug_section = obj_file .section_by_name(".debug") .ok_or_else(|| anyhow!("Failed to locate .debug section"))?; diff --git a/src/cmd/elf2dol.rs b/src/cmd/elf2dol.rs index d4b5c7d..78b100f 100644 --- a/src/cmd/elf2dol.rs +++ b/src/cmd/elf2dol.rs @@ -43,8 +43,8 @@ const MAX_TEXT_SECTIONS: usize = 7; const MAX_DATA_SECTIONS: usize = 11; pub fn run(args: Args) -> Result<()> { - let map = map_file(&args.elf_file)?; - let obj_file = object::read::File::parse(&*map)?; + let file = map_file(&args.elf_file)?; + let obj_file = object::read::File::parse(file.as_slice())?; match obj_file.architecture() { Architecture::PowerPc => {} arch => bail!("Unexpected architecture: {arch:?}"), diff --git a/src/cmd/map.rs b/src/cmd/map.rs index 1c3d5a2..d0b8d37 100644 --- a/src/cmd/map.rs +++ b/src/cmd/map.rs @@ -6,7 +6,7 @@ use argp::FromArgs; use cwdemangle::{demangle, DemangleOptions}; use crate::util::{ - file::{map_file, map_reader}, + file::map_file, map::{process_map, SymbolEntry, SymbolRef}, }; @@ -90,8 +90,8 @@ pub fn run(args: Args) -> Result<()> { } fn entries(args: EntriesArgs) -> Result<()> { - let map = map_file(&args.map_file)?; - let entries = process_map(map_reader(&map))?; + let file = map_file(&args.map_file)?; + let entries = process_map(&mut file.as_reader())?; match entries.unit_entries.get_vec(&args.unit) { Some(vec) => { for symbol_ref in vec { @@ -108,8 +108,8 @@ fn entries(args: EntriesArgs) -> Result<()> { } fn symbol(args: SymbolArgs) -> Result<()> { - let map = map_file(&args.map_file)?; - let entries = process_map(map_reader(&map))?; + let file = map_file(&args.map_file)?; + let entries = process_map(&mut file.as_reader())?; let opt_ref: Option<(SymbolRef, SymbolEntry)> = None; _ = entries; @@ -165,8 +165,8 @@ fn symbol(args: SymbolArgs) -> Result<()> { } fn order(args: OrderArgs) -> Result<()> { - let map = map_file(&args.map_file)?; - let entries = process_map(map_reader(&map))?; + let file = map_file(&args.map_file)?; + let entries = process_map(&mut file.as_reader())?; _ = entries; // TODO @@ -179,8 +179,8 @@ fn order(args: OrderArgs) -> Result<()> { } fn slices(args: SlicesArgs) -> Result<()> { - let map = map_file(&args.map_file)?; - let entries = process_map(map_reader(&map))?; + let file = map_file(&args.map_file)?; + let entries = process_map(&mut file.as_reader())?; _ = entries; // TODO @@ -213,8 +213,8 @@ fn slices(args: SlicesArgs) -> Result<()> { } fn symbols(args: SymbolsArgs) -> Result<()> { - let map = map_file(&args.map_file)?; - let entries = process_map(map_reader(&map))?; + let file = map_file(&args.map_file)?; + let entries = process_map(&mut file.as_reader())?; _ = entries; // TODO diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs index 40fe441..c76b9c0 100644 --- a/src/cmd/mod.rs +++ b/src/cmd/mod.rs @@ -6,6 +6,8 @@ pub mod elf; pub mod elf2dol; pub mod map; pub mod metroidbuildinfo; +pub mod rarc; pub mod rel; pub mod rso; pub mod shasum; +pub mod yaz0; diff --git a/src/cmd/rarc.rs b/src/cmd/rarc.rs new file mode 100644 index 0000000..378ca76 --- /dev/null +++ b/src/cmd/rarc.rs @@ -0,0 +1,109 @@ +use std::{fs, fs::DirBuilder, path::PathBuf}; + +use anyhow::{Context, Result}; +use argp::FromArgs; + +use crate::util::{ + file::{decompress_if_needed, map_file}, + rarc::{Node, RarcReader}, +}; + +#[derive(FromArgs, PartialEq, Debug)] +/// Commands for processing RSO files. +#[argp(subcommand, name = "rarc")] +pub struct Args { + #[argp(subcommand)] + command: SubCommand, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argp(subcommand)] +enum SubCommand { + List(ListArgs), + Extract(ExtractArgs), +} + +#[derive(FromArgs, PartialEq, Eq, Debug)] +/// Views RARC file information. +#[argp(subcommand, name = "list")] +pub struct ListArgs { + #[argp(positional)] + /// RARC file + file: PathBuf, +} + +#[derive(FromArgs, PartialEq, Eq, Debug)] +/// Extracts RARC file contents. +#[argp(subcommand, name = "extract")] +pub struct ExtractArgs { + #[argp(positional)] + /// RARC file + file: PathBuf, + #[argp(option, short = 'o')] + /// output directory + output: Option, +} + +pub fn run(args: Args) -> Result<()> { + match args.command { + SubCommand::List(c_args) => list(c_args), + SubCommand::Extract(c_args) => extract(c_args), + } +} + +fn list(args: ListArgs) -> Result<()> { + let file = map_file(&args.file)?; + let rarc = RarcReader::new(&mut file.as_reader())?; + + let mut current_path = PathBuf::new(); + for node in rarc.nodes() { + match node { + Node::DirectoryBegin { name } => { + current_path.push(name.name); + } + Node::DirectoryEnd { name: _ } => { + current_path.pop(); + } + Node::File { name, offset, size } => { + let path = current_path.join(name.name); + println!("{}: {} bytes, offset {:#X}", path.display(), size, offset); + } + Node::CurrentDirectory => {} + Node::ParentDirectory => {} + } + } + Ok(()) +} + +fn extract(args: ExtractArgs) -> Result<()> { + let file = map_file(&args.file)?; + let rarc = RarcReader::new(&mut file.as_reader())?; + + let mut current_path = PathBuf::new(); + for node in rarc.nodes() { + match node { + Node::DirectoryBegin { name } => { + current_path.push(name.name); + } + Node::DirectoryEnd { name: _ } => { + current_path.pop(); + } + Node::File { name, offset, size } => { + let file_data = decompress_if_needed( + &file.as_slice()[offset as usize..offset as usize + size as usize], + )?; + let file_path = current_path.join(&name.name); + let output_path = + args.output.as_ref().map(|p| p.join(&file_path)).unwrap_or_else(|| file_path); + if let Some(parent) = output_path.parent() { + DirBuilder::new().recursive(true).create(parent)?; + } + fs::write(&output_path, file_data) + .with_context(|| format!("Failed to write file '{}'", output_path.display()))?; + } + Node::CurrentDirectory => {} + Node::ParentDirectory => {} + } + } + Ok(()) +} diff --git a/src/cmd/rel.rs b/src/cmd/rel.rs index f0b97fa..3a6c7d0 100644 --- a/src/cmd/rel.rs +++ b/src/cmd/rel.rs @@ -93,6 +93,9 @@ pub struct MakeArgs { #[argp(option, short = 'c')] /// (optional) project configuration file config: Option, + #[argp(switch, short = 'w')] + /// disable warnings + no_warn: bool, } pub fn run(args: Args) -> Result<()> { @@ -121,26 +124,27 @@ fn make(args: MakeArgs) -> Result<()> { if let Some(config_path) = &args.config { let config: ProjectConfig = serde_yaml::from_reader(&mut buf_reader(config_path)?)?; for module_config in &config.modules { - if let Some(hash_str) = &module_config.hash { - verify_hash(&module_config.object, hash_str)?; - } let map = map_file(&module_config.object)?; - let buf = decompress_if_needed(&map)?; + let buf = decompress_if_needed(map.as_slice())?; + if let Some(hash_str) = &module_config.hash { + verify_hash(buf.as_ref(), hash_str)?; + } let header = process_rel_header(&mut Reader::new(buf.as_ref()))?; existing_headers.insert(header.module_id, header); } } - let files = process_rsp(&args.files)?; - info!("Loading {} modules", files.len()); + let paths = process_rsp(&args.files)?; + info!("Loading {} modules", paths.len()); // Load all modules - let handles = files.iter().map(map_file).collect::>>()?; - let modules = handles + let files = paths.iter().map(map_file).collect::>>()?; + let modules = files .par_iter() - .zip(&files) - .map(|(map, path)| { - load_obj(map).with_context(|| format!("Failed to load '{}'", path.display())) + .zip(&paths) + .map(|(file, path)| { + load_obj(file.as_slice()) + .with_context(|| format!("Failed to load '{}'", path.display())) }) .collect::>>()?; @@ -194,7 +198,7 @@ fn make(args: MakeArgs) -> Result<()> { address: address as u32, module_id: target_module_id as u32, target_section: target_symbol.section_index().unwrap().0 as u8, - addend: target_symbol.address() as u32, + addend: (target_symbol.address() as i64 + reloc.addend()) as u32, }); } } @@ -211,7 +215,7 @@ fn make(args: MakeArgs) -> Result<()> { // Write RELs let start = Instant::now(); for (((module_id, module), path), relocations) in - modules.iter().enumerate().zip(&files).skip(1).zip(relocations) + modules.iter().enumerate().zip(&paths).skip(1).zip(relocations) { let name = path.file_stem().unwrap_or(OsStr::new("[unknown]")).to_str().unwrap_or("[invalid]"); @@ -224,6 +228,7 @@ fn make(args: MakeArgs) -> Result<()> { align: None, bss_align: None, section_count: None, + quiet: args.no_warn, }; if let Some(existing_module) = existing_headers.get(&(module_id as u32)) { info.version = existing_module.version; @@ -248,9 +253,9 @@ fn make(args: MakeArgs) -> Result<()> { } fn info(args: InfoArgs) -> Result<()> { - let map = map_file(args.rel_file)?; - let buf = decompress_if_needed(&map)?; - let (header, mut module_obj) = process_rel(&mut Reader::new(buf.as_ref()))?; + let file = map_file(args.rel_file)?; + let buf = decompress_if_needed(file.as_slice())?; + let (header, mut module_obj) = process_rel(&mut Reader::new(buf.as_ref()), "")?; let mut state = AnalyzerState::default(); state.detect_functions(&module_obj)?; @@ -312,7 +317,12 @@ const fn align32(x: u32) -> u32 { (x + 31) & !31 } fn merge(args: MergeArgs) -> Result<()> { log::info!("Loading {}", args.dol_file.display()); - let mut obj = process_dol(&args.dol_file)?; + let mut obj = { + let file = map_file(&args.dol_file)?; + let buf = decompress_if_needed(file.as_slice())?; + let name = args.dol_file.file_stem().map(|s| s.to_string_lossy()).unwrap_or_default(); + process_dol(buf.as_ref(), name.as_ref())? + }; log::info!("Performing signature analysis"); apply_signatures(&mut obj)?; @@ -323,7 +333,8 @@ fn merge(args: MergeArgs) -> Result<()> { for result in FileIterator::new(&args.rel_files)? { let (path, entry) = result?; log::info!("Loading {}", path.display()); - let (_, obj) = process_rel(&mut entry.as_reader())?; + let name = path.file_stem().map(|s| s.to_string_lossy()).unwrap_or_default(); + let (_, obj) = process_rel(&mut entry.as_reader(), name.as_ref())?; match module_map.entry(obj.module_id) { btree_map::Entry::Vacant(e) => e.insert(obj), btree_map::Entry::Occupied(_) => bail!("Duplicate module ID {}", obj.module_id), diff --git a/src/cmd/rso.rs b/src/cmd/rso.rs index da05c08..ea3bd99 100644 --- a/src/cmd/rso.rs +++ b/src/cmd/rso.rs @@ -1,9 +1,12 @@ use std::path::PathBuf; -use anyhow::Result; +use anyhow::{Context, Result}; use argp::FromArgs; -use crate::util::rso::process_rso; +use crate::util::{ + file::{decompress_if_needed, map_file, Reader}, + rso::process_rso, +}; #[derive(FromArgs, PartialEq, Debug)] /// Commands for processing RSO files. @@ -35,7 +38,12 @@ pub fn run(args: Args) -> Result<()> { } fn info(args: InfoArgs) -> Result<()> { - let rso = process_rso(args.rso_file)?; + let rso = { + let file = map_file(&args.rso_file)?; + let data = decompress_if_needed(file.as_slice()) + .with_context(|| format!("Failed to decompress '{}'", args.rso_file.display()))?; + process_rso(&mut Reader::new(data.as_ref()))? + }; println!("Read RSO module {}", rso.name); Ok(()) } diff --git a/src/cmd/shasum.rs b/src/cmd/shasum.rs index 5c7ddf3..9796bab 100644 --- a/src/cmd/shasum.rs +++ b/src/cmd/shasum.rs @@ -9,7 +9,7 @@ use argp::FromArgs; use owo_colors::OwoColorize; use sha1::{Digest, Sha1}; -use crate::util::file::{process_rsp, touch}; +use crate::util::file::{open_file, process_rsp, touch}; #[derive(FromArgs, PartialEq, Eq, Debug)] /// Print or check SHA1 (160-bit) checksums. @@ -24,18 +24,20 @@ pub struct Args { #[argp(option, short = 'o')] /// touch output file on successful check output: Option, + #[argp(switch, short = 'q')] + /// only print failures and a summary + quiet: bool, } const DEFAULT_BUF_SIZE: usize = 8192; pub fn run(args: Args) -> Result<()> { for path in process_rsp(&args.files)? { - let file = File::open(&path) - .with_context(|| format!("Failed to open file '{}'", path.display()))?; + let mut file = open_file(&path)?; if args.check { - check(file)? + check(&args, &mut BufReader::new(file))? } else { - hash(file, &path)? + hash(&mut file, &path)? } } if let Some(out_path) = args.output { @@ -45,8 +47,8 @@ pub fn run(args: Args) -> Result<()> { Ok(()) } -fn check(file: File) -> Result<()> { - let reader = BufReader::new(file); +fn check(args: &Args, reader: &mut R) -> Result<()> { + let mut matches = 0usize; let mut mismatches = 0usize; for line in reader.lines() { let line = match line { @@ -63,16 +65,23 @@ fn check(file: File) -> Result<()> { hex::decode_to_slice(hash, &mut hash_bytes) .with_context(|| format!("Invalid line: {line}"))?; - let file = - File::open(file_name).with_context(|| format!("Failed to open file '{file_name}'"))?; - let found_hash = file_sha1(file)?; + let found_hash = file_sha1( + &mut File::open(file_name) + .with_context(|| format!("Failed to open file '{file_name}'"))?, + )?; if hash_bytes == found_hash.as_ref() { - println!("{}: {}", file_name, "OK".green()); + if !args.quiet { + println!("{}: {}", file_name, "OK".green()); + } + matches += 1; } else { println!("{}: {}", file_name, "FAILED".red()); mismatches += 1; } } + if args.quiet && matches > 0 { + println!("{} files {}", matches, "OK".green()); + } if mismatches != 0 { eprintln!( "{}", @@ -83,8 +92,8 @@ fn check(file: File) -> Result<()> { Ok(()) } -fn hash(file: File, path: &Path) -> Result<()> { - let hash = file_sha1(file)?; +fn hash(reader: &mut R, path: &Path) -> Result<()> { + let hash = file_sha1(reader)?; let mut hash_buf = [0u8; 40]; let hash_str = base16ct::lower::encode_str(&hash, &mut hash_buf) .map_err(|e| anyhow!("Failed to encode hash: {e}"))?; @@ -92,14 +101,22 @@ fn hash(file: File, path: &Path) -> Result<()> { Ok(()) } -pub fn file_sha1(mut file: File) -> Result> { +pub fn file_sha1(reader: &mut R) -> Result> { let mut buf = [0u8; DEFAULT_BUF_SIZE]; let mut hasher = Sha1::new(); Ok(loop { - let read = file.read(&mut buf).context("File read failed")?; + let read = reader.read(&mut buf).context("File read failed")?; if read == 0 { break hasher.finalize(); } hasher.update(&buf[0..read]); }) } + +pub fn file_sha1_string(reader: &mut R) -> Result { + let hash = file_sha1(reader)?; + let mut hash_buf = [0u8; 40]; + let hash_str = base16ct::lower::encode_str(&hash, &mut hash_buf) + .map_err(|e| anyhow!("Failed to encode hash: {e}"))?; + Ok(hash_str.to_string()) +} diff --git a/src/cmd/yaz0.rs b/src/cmd/yaz0.rs new file mode 100644 index 0000000..ee60575 --- /dev/null +++ b/src/cmd/yaz0.rs @@ -0,0 +1,55 @@ +use std::{fs, path::PathBuf}; + +use anyhow::{Context, Result}; +use argp::FromArgs; + +use crate::util::{ + file::{decompress_reader, open_file, process_rsp}, + IntoCow, ToCow, +}; + +#[derive(FromArgs, PartialEq, Debug)] +/// Commands for processing YAZ0-compressed files. +#[argp(subcommand, name = "yaz0")] +pub struct Args { + #[argp(subcommand)] + command: SubCommand, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argp(subcommand)] +enum SubCommand { + Decompress(DecompressArgs), +} + +#[derive(FromArgs, PartialEq, Eq, Debug)] +/// Decompresses YAZ0-compressed files. +#[argp(subcommand, name = "decompress")] +pub struct DecompressArgs { + #[argp(positional)] + /// YAZ0-compressed files + files: Vec, + #[argp(option, short = 'o')] + /// Output directory. If not specified, decompresses in-place. + output: Option, +} + +pub fn run(args: Args) -> Result<()> { + match args.command { + SubCommand::Decompress(args) => decompress(args), + } +} + +fn decompress(args: DecompressArgs) -> Result<()> { + for path in process_rsp(&args.files)? { + let data = decompress_reader(&mut open_file(&path)?)?; + let out_path = if let Some(output) = &args.output { + output.join(path.file_name().unwrap()).into_cow() + } else { + path.as_path().to_cow() + }; + fs::write(out_path.as_ref(), data) + .with_context(|| format!("Failed to write '{}'", out_path.display()))?; + } + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index b932e47..c4b885d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -82,9 +82,11 @@ enum SubCommand { Elf2Dol(cmd::elf2dol::Args), // Map(cmd::map::Args), MetroidBuildInfo(cmd::metroidbuildinfo::Args), + Rarc(cmd::rarc::Args), Rel(cmd::rel::Args), Rso(cmd::rso::Args), Shasum(cmd::shasum::Args), + Yaz0(cmd::yaz0::Args), } fn main() { @@ -127,9 +129,11 @@ fn main() { SubCommand::Elf2Dol(c_args) => cmd::elf2dol::run(c_args), // SubCommand::Map(c_args) => cmd::map::run(c_args), SubCommand::MetroidBuildInfo(c_args) => cmd::metroidbuildinfo::run(c_args), + SubCommand::Rarc(c_args) => cmd::rarc::run(c_args), SubCommand::Rel(c_args) => cmd::rel::run(c_args), SubCommand::Rso(c_args) => cmd::rso::run(c_args), SubCommand::Shasum(c_args) => cmd::shasum::run(c_args), + SubCommand::Yaz0(c_args) => cmd::yaz0::run(c_args), }); if let Err(e) = result { eprintln!("Failed: {e:?}"); diff --git a/src/obj/sections.rs b/src/obj/sections.rs index 8609bcc..03e3bfd 100644 --- a/src/obj/sections.rs +++ b/src/obj/sections.rs @@ -172,6 +172,9 @@ impl ObjSection { #[inline] pub fn symbol_data(&self, symbol: &ObjSymbol) -> Result<&[u8]> { + if symbol.size == 0 { + return Ok(&[]); + } self.data_range(symbol.address as u32, symbol.address as u32 + symbol.size as u32) } diff --git a/src/obj/symbols.rs b/src/obj/symbols.rs index 8dfbcd4..4405285 100644 --- a/src/obj/symbols.rs +++ b/src/obj/symbols.rs @@ -329,7 +329,13 @@ impl ObjSymbols { self.at_section_address(section_idx, addr) .filter(|(_, sym)| sym.kind == kind) .at_most_one() - .map_err(|_| anyhow!("Multiple symbols of kind {:?} at address {:#010X}", kind, addr)) + .map_err(|e| { + let symbols = e.map(|(_, s)| s).collect_vec(); + for symbol in symbols { + log::error!("{:?}", symbol); + } + anyhow!("Multiple symbols of kind {:?} at address {:#010X}", kind, addr) + }) } // Iterate over all in address ascending order, excluding ABS symbols diff --git a/src/util/comment.rs b/src/util/comment.rs index 25b6ccf..13044d5 100644 --- a/src/util/comment.rs +++ b/src/util/comment.rs @@ -198,8 +198,7 @@ impl CommentSym { let mut active_flags = 0; if symbol.flags.is_force_active() || (force_active - && matches!(symbol.kind, ObjSymbolKind::Function | ObjSymbolKind::Object) - && symbol.flags.is_global()) + && matches!(symbol.kind, ObjSymbolKind::Function | ObjSymbolKind::Object)) { active_flags |= 0x8; } diff --git a/src/util/config.rs b/src/util/config.rs index 0a23e6d..eac7f77 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -16,17 +16,24 @@ use crate::{ ObjDataKind, ObjInfo, ObjKind, ObjSectionKind, ObjSplit, ObjSymbol, ObjSymbolFlagSet, ObjSymbolFlags, ObjSymbolKind, ObjUnit, }, - util::file::{buf_writer, map_file, map_reader}, + util::{ + file::{buf_writer, map_file}, + split::default_section_align, + }, }; fn parse_hex(s: &str) -> Result { - u32::from_str_radix(s.trim_start_matches("0x"), 16) + if s.starts_with("0x") { + u32::from_str_radix(s.trim_start_matches("0x"), 16) + } else { + s.parse::() + } } pub fn apply_symbols_file>(path: P, obj: &mut ObjInfo) -> Result { Ok(if path.as_ref().is_file() { - let map = map_file(path)?; - for result in map_reader(&map).lines() { + let file = map_file(path)?; + for result in file.as_reader().lines() { let line = match result { Ok(line) => line, Err(e) => bail!("Failed to process symbols file: {e:?}"), @@ -80,6 +87,10 @@ pub fn parse_symbol_line(line: &str, obj: &mut ObjInfo) -> Result(w: &mut W, obj: &ObjInfo, symbol: &ObjSymbol) -> Resul write!(w, " scope:{scope}")?; } if let Some(align) = symbol.align { - write!(w, " align:{align:#X}")?; + write!(w, " align:{align}")?; } if let Some(kind) = symbol_data_kind_to_str(symbol.data_kind) { write!(w, " data:{kind}")?; @@ -342,9 +353,11 @@ pub fn write_splits(w: &mut W, obj: &ObjInfo, all: bool) -> Result<()> split_iter.peek().map(|&(_, _, addr, _)| addr).unwrap_or(0) }; write!(w, "\t{:<11} start:{:#010X} end:{:#010X}", section.name, addr, end)?; - // if let Some(align) = split.align { - // write!(w, " align:{}", align)?; - // } + if let Some(align) = split.align { + if align != default_section_align(section) as u32 { + write!(w, " align:{}", align)?; + } + } if split.common { write!(w, " common")?; } @@ -446,7 +459,7 @@ fn parse_section_line(captures: Captures, state: &SplitState) -> Result { - section.align = Some(u32::from_str(value)?); + section.align = Some(parse_hex(value)?); } _ => bail!("Unknown section attribute '{attr}'"), } @@ -475,7 +488,7 @@ fn parse_section_line(captures: Captures, state: &SplitState) -> Result start = Some(parse_hex(value)?), "end" => end = Some(parse_hex(value)?), - "align" => section.align = Some(u32::from_str(value)?), + "align" => section.align = Some(parse_hex(value)?), "rename" => section.rename = Some(value.to_string()), _ => bail!("Unknown split attribute '{attr}'"), } @@ -509,8 +522,8 @@ enum SplitState { pub fn apply_splits_file>(path: P, obj: &mut ObjInfo) -> Result { Ok(if path.as_ref().is_file() { - let map = map_file(path)?; - apply_splits(map_reader(&map), obj)?; + let file = map_file(path)?; + apply_splits(file.as_reader(), obj)?; true } else { false @@ -587,8 +600,10 @@ pub fn apply_splits(r: R, obj: &mut ObjInfo) -> Result<()> { } }?; let section = obj.sections.get_mut(section_index).unwrap(); + let section_end = (section.address + section.size) as u32; ensure!( - section.contains_range(start..end), + section.contains_range(start..end) + || (start == section_end && end == section_end), "Section {} ({:#010X}..{:#010X}) does not contain range {:#010X}..{:#010X}", name, section.address, diff --git a/src/util/dep.rs b/src/util/dep.rs index ae02c83..8524f65 100644 --- a/src/util/dep.rs +++ b/src/util/dep.rs @@ -1,7 +1,13 @@ -use std::{io::Write, path::PathBuf}; +use std::{ + io::Write, + path::{Path, PathBuf}, +}; +use itertools::Itertools; use path_slash::PathBufExt; +use crate::util::file::split_path; + pub struct DepFile { pub name: PathBuf, pub dependencies: Vec, @@ -10,13 +16,22 @@ pub struct DepFile { impl DepFile { pub fn new(name: PathBuf) -> Self { Self { name, dependencies: vec![] } } - pub fn push(&mut self, dependency: PathBuf) { self.dependencies.push(dependency); } + pub fn push>(&mut self, dependency: P) { + let path = split_path(dependency.as_ref()) + .map(|(p, _)| p) + .unwrap_or_else(|_| dependency.as_ref().to_path_buf()); + self.dependencies.push(path); + } - pub fn extend(&mut self, dependencies: Vec) { self.dependencies.extend(dependencies); } + pub fn extend(&mut self, dependencies: Vec) { + self.dependencies.extend(dependencies.iter().map(|dependency| { + split_path(dependency).map(|(p, _)| p).unwrap_or_else(|_| dependency.clone()) + })); + } pub fn write(&self, w: &mut W) -> std::io::Result<()> { write!(w, "{}:", self.name.to_slash_lossy())?; - for dep in &self.dependencies { + for dep in self.dependencies.iter().unique() { write!(w, " \\\n {}", dep.to_slash_lossy())?; } Ok(()) diff --git a/src/util/dol.rs b/src/util/dol.rs index d4399ec..5955c5e 100644 --- a/src/util/dol.rs +++ b/src/util/dol.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, path::Path}; +use std::collections::BTreeMap; use anyhow::{anyhow, bail, ensure, Result}; use dol::{Dol, DolSection, DolSectionType}; @@ -9,7 +9,7 @@ use crate::{ ObjArchitecture, ObjInfo, ObjKind, ObjSection, ObjSectionKind, ObjSymbol, ObjSymbolFlagSet, ObjSymbolFlags, ObjSymbolKind, }, - util::file::{map_file, map_reader}, + util::file::Reader, }; const MAX_TEXT_SECTIONS: usize = 7; @@ -22,17 +22,8 @@ fn read_u32(dol: &Dol, addr: u32) -> Result { Ok(u32::from_be_bytes(dol.virtual_data_at(addr, 4)?.try_into()?)) } -pub fn process_dol>(path: P) -> Result { - let name = path - .as_ref() - .file_name() - .and_then(|filename| filename.to_str()) - .unwrap_or_default() - .to_string(); - let dol = { - let mmap = map_file(path)?; - Dol::read_from(map_reader(&mmap))? - }; +pub fn process_dol(buf: &[u8], name: &str) -> Result { + let dol = Dol::read_from(Reader::new(buf))?; // Locate _rom_copy_info let first_rom_section = dol @@ -331,8 +322,13 @@ pub fn process_dol>(path: P) -> Result { } // Create object - let mut obj = - ObjInfo::new(ObjKind::Executable, ObjArchitecture::PowerPc, name, vec![], sections); + let mut obj = ObjInfo::new( + ObjKind::Executable, + ObjArchitecture::PowerPc, + name.to_string(), + vec![], + sections, + ); obj.entry = Some(dol.header.entry_point as u64); // Generate _rom_copy_info symbol diff --git a/src/util/elf.rs b/src/util/elf.rs index 8761bea..9f99c3f 100644 --- a/src/util/elf.rs +++ b/src/util/elf.rs @@ -41,8 +41,8 @@ enum BoundaryState { } pub fn process_elf>(path: P) -> Result { - let mmap = map_file(path)?; - let obj_file = object::read::File::parse(&*mmap)?; + let file = map_file(path)?; + let obj_file = object::read::File::parse(file.as_slice())?; let architecture = match obj_file.architecture() { Architecture::PowerPc => ObjArchitecture::PowerPc, arch => bail!("Unexpected architecture: {arch:?}"), diff --git a/src/util/file.rs b/src/util/file.rs index e4c20fb..f2a31f2 100644 --- a/src/util/file.rs +++ b/src/util/file.rs @@ -1,36 +1,105 @@ use std::{ borrow::Cow, + ffi::OsStr, fs::{DirBuilder, File, OpenOptions}, - io::{BufRead, BufReader, BufWriter, Cursor, Read}, - path::{Path, PathBuf}, + io::{BufRead, BufReader, BufWriter, Cursor, Read, Seek, SeekFrom}, + path::{Component, Path, PathBuf}, }; use anyhow::{anyhow, Context, Result}; +use binrw::io::{TakeSeek, TakeSeekExt}; use byteorder::ReadBytesExt; use filetime::{set_file_mtime, FileTime}; use memmap2::{Mmap, MmapOptions}; use path_slash::PathBufExt; +use sha1::{Digest, Sha1}; -use crate::{ - cmd::shasum::file_sha1, - util::{rarc, rarc::Node, yaz0, IntoCow, ToCow}, -}; +use crate::util::{rarc, rarc::Node, yaz0, IntoCow, ToCow}; + +pub struct MappedFile { + mmap: Mmap, + offset: u64, + len: u64, +} + +impl MappedFile { + pub fn new(mmap: Mmap, offset: u64, len: u64) -> Self { Self { mmap, offset, len } } + + pub fn as_reader(&self) -> Reader { Reader::new(self.as_slice()) } + + pub fn as_slice(&self) -> &[u8] { + &self.mmap[self.offset as usize..self.offset as usize + self.len as usize] + } + + pub fn len(&self) -> u64 { self.len } + + pub fn is_empty(&self) -> bool { self.len == 0 } + + pub fn into_inner(self) -> Mmap { self.mmap } +} + +pub fn split_path>(path: P) -> Result<(PathBuf, Option)> { + let mut base_path = PathBuf::new(); + let mut sub_path: Option = None; + for component in path.as_ref().components() { + if let Component::Normal(str) = component { + let str = str.to_str().ok_or(anyhow!("Path is not valid UTF-8"))?; + if let Some((a, b)) = str.split_once(':') { + base_path.push(a); + sub_path = Some(PathBuf::from(b)); + continue; + } + } + if let Some(sub_path) = &mut sub_path { + sub_path.push(component); + } else { + base_path.push(component); + } + } + Ok((base_path, sub_path)) +} /// Opens a memory mapped file. -pub fn map_file>(path: P) -> Result { - let file = File::open(&path) - .with_context(|| format!("Failed to open file '{}'", path.as_ref().display()))?; - let map = unsafe { MmapOptions::new().map(&file) } - .with_context(|| format!("Failed to mmap file: '{}'", path.as_ref().display()))?; - Ok(map) +pub fn map_file>(path: P) -> Result { + let (base_path, sub_path) = split_path(path)?; + let file = File::open(&base_path) + .with_context(|| format!("Failed to open file '{}'", base_path.display()))?; + let mmap = unsafe { MmapOptions::new().map(&file) } + .with_context(|| format!("Failed to mmap file: '{}'", base_path.display()))?; + let (offset, len) = if let Some(sub_path) = sub_path { + let rarc = rarc::RarcReader::new(&mut Reader::new(&*mmap)) + .with_context(|| format!("Failed to read RARC '{}'", base_path.display()))?; + rarc.find_file(&sub_path)?.map(|(o, s)| (o, s as u64)).ok_or_else(|| { + anyhow!("File '{}' not found in '{}'", sub_path.display(), base_path.display()) + })? + } else { + (0, mmap.len() as u64) + }; + Ok(MappedFile { mmap, offset, len }) +} + +pub type OpenedFile = TakeSeek; + +/// Opens a file (not memory mapped). +pub fn open_file>(path: P) -> Result { + let (base_path, sub_path) = split_path(path)?; + let mut file = File::open(&base_path) + .with_context(|| format!("Failed to open file '{}'", base_path.display()))?; + let (offset, size) = if let Some(sub_path) = sub_path { + let rarc = rarc::RarcReader::new(&mut BufReader::new(&file)) + .with_context(|| format!("Failed to read RARC '{}'", base_path.display()))?; + rarc.find_file(&sub_path)?.map(|(o, s)| (o, s as u64)).ok_or_else(|| { + anyhow!("File '{}' not found in '{}'", sub_path.display(), base_path.display()) + })? + } else { + (0, file.seek(SeekFrom::End(0))?) + }; + file.seek(SeekFrom::Start(offset))?; + Ok(file.take_seek(size)) } pub type Reader<'a> = Cursor<&'a [u8]>; -/// Creates a reader for the memory mapped file. -#[inline] -pub fn map_reader(mmap: &Mmap) -> Reader { Reader::new(&**mmap) } - /// Creates a buffered reader around a file (not memory mapped). pub fn buf_reader>(path: P) -> Result> { let file = File::open(&path) @@ -49,19 +118,19 @@ pub fn buf_writer>(path: P) -> Result> { } /// Reads a string with known size at the specified offset. -pub fn read_string(reader: &mut Reader, off: u64, size: usize) -> Result { +pub fn read_string(reader: &mut R, off: u64, size: usize) -> Result { let mut data = vec![0u8; size]; - let pos = reader.position(); - reader.set_position(off); + let pos = reader.stream_position()?; + reader.seek(SeekFrom::Start(off))?; reader.read_exact(&mut data)?; - reader.set_position(pos); + reader.seek(SeekFrom::Start(pos))?; Ok(String::from_utf8(data)?) } /// Reads a zero-terminated string at the specified offset. -pub fn read_c_string(reader: &mut Reader, off: u64) -> Result { - let pos = reader.position(); - reader.set_position(off); +pub fn read_c_string(reader: &mut R, off: u64) -> Result { + let pos = reader.stream_position()?; + reader.seek(SeekFrom::Start(off))?; let mut s = String::new(); loop { let b = reader.read_u8()?; @@ -70,7 +139,7 @@ pub fn read_c_string(reader: &mut Reader, off: u64) -> Result { } s.push(b as char); } - reader.set_position(pos); + reader.seek(SeekFrom::Start(pos))?; Ok(s) } @@ -102,15 +171,16 @@ pub fn process_rsp(files: &[PathBuf]) -> Result> { /// Iterator over files in a RARC archive. struct RarcIterator { file: Mmap, + base_path: PathBuf, paths: Vec<(PathBuf, u64, u32)>, index: usize, } impl RarcIterator { pub fn new(file: Mmap, base_path: &Path) -> Result { - let reader = rarc::RarcReader::new(map_reader(&file))?; + let reader = rarc::RarcReader::new(&mut Reader::new(&*file))?; let paths = Self::collect_paths(&reader, base_path); - Ok(Self { file, paths, index: 0 }) + Ok(Self { file, base_path: base_path.to_owned(), paths, index: 0 }) } fn collect_paths(reader: &rarc::RarcReader, base_path: &Path) -> Vec<(PathBuf, u64, u32)> { @@ -157,7 +227,7 @@ impl Iterator for RarcIterator { /// A file entry, either a memory mapped file or an owned buffer. pub enum FileEntry { - Map(Mmap), + MappedFile(MappedFile), Buffer(Vec), } @@ -165,7 +235,7 @@ impl FileEntry { /// Creates a reader for the file. pub fn as_reader(&self) -> Reader { match self { - Self::Map(map) => map_reader(map), + Self::MappedFile(file) => file.as_reader(), Self::Buffer(slice) => Reader::new(slice), } } @@ -188,7 +258,12 @@ impl FileIterator { fn next_rarc(&mut self) -> Option> { if let Some(rarc) = &mut self.rarc { match rarc.next() { - Some(Ok((path, buf))) => return Some(Ok((path, FileEntry::Buffer(buf)))), + Some(Ok((path, buf))) => { + let mut path_str = rarc.base_path.as_os_str().to_os_string(); + path_str.push(OsStr::new(":")); + path_str.push(path.as_os_str()); + return Some(Ok((path, FileEntry::Buffer(buf)))); + } Some(Err(err)) => return Some(Err(err)), None => self.rarc = None, } @@ -209,20 +284,29 @@ impl FileIterator { } } - fn handle_file(&mut self, map: Mmap, path: PathBuf) -> Option> { - if map.len() <= 4 { - return Some(Ok((path, FileEntry::Map(map)))); + fn handle_file( + &mut self, + file: MappedFile, + path: PathBuf, + ) -> Option> { + let buf = file.as_slice(); + if buf.len() <= 4 { + return Some(Ok((path, FileEntry::MappedFile(file)))); } - match &map[0..4] { - b"Yaz0" => self.handle_yaz0(map, path), - b"RARC" => self.handle_rarc(map, path), - _ => Some(Ok((path, FileEntry::Map(map)))), + match &buf[0..4] { + b"Yaz0" => self.handle_yaz0(file.as_reader(), path), + b"RARC" => self.handle_rarc(file.into_inner(), path), + _ => Some(Ok((path, FileEntry::MappedFile(file)))), } } - fn handle_yaz0(&mut self, map: Mmap, path: PathBuf) -> Option> { - Some(match yaz0::decompress_file(&mut map_reader(&map)) { + fn handle_yaz0( + &mut self, + mut reader: Reader, + path: PathBuf, + ) -> Option> { + Some(match yaz0::decompress_file(&mut reader) { Ok(buf) => Ok((path, FileEntry::Buffer(buf))), Err(e) => Err(e), }) @@ -262,20 +346,38 @@ pub fn decompress_if_needed(buf: &[u8]) -> Result> { }) } -pub fn verify_hash>(path: P, hash_str: &str) -> Result<()> { - let mut hash_bytes = [0u8; 20]; - hex::decode_to_slice(hash_str, &mut hash_bytes) - .with_context(|| format!("Invalid SHA-1 '{hash_str}'"))?; - let file = File::open(path.as_ref()) - .with_context(|| format!("Failed to open file '{}'", path.as_ref().display()))?; - let found_hash = file_sha1(file)?; - if found_hash.as_ref() == hash_bytes { +pub fn decompress_reader(reader: &mut R) -> Result> { + let mut magic = [0u8; 4]; + if reader.read_exact(&mut magic).is_err() { + reader.seek(SeekFrom::Start(0))?; + let mut buf = vec![]; + reader.read_to_end(&mut buf)?; + return Ok(buf); + } + Ok(if magic == *b"Yaz0" { + reader.seek(SeekFrom::Start(0))?; + yaz0::decompress_file(reader)? + } else { + let mut buf = magic.to_vec(); + reader.read_to_end(&mut buf)?; + buf + }) +} + +pub fn verify_hash(buf: &[u8], expected_str: &str) -> Result<()> { + let mut expected_bytes = [0u8; 20]; + hex::decode_to_slice(expected_str, &mut expected_bytes) + .with_context(|| format!("Invalid SHA-1 '{expected_str}'"))?; + let mut hasher = Sha1::new(); + hasher.update(buf); + let hash_bytes = hasher.finalize(); + if hash_bytes.as_ref() == expected_bytes { Ok(()) } else { Err(anyhow!( "Hash mismatch: expected {}, but was {}", - hex::encode(hash_bytes), - hex::encode(found_hash) + hex::encode(expected_bytes), + hex::encode(hash_bytes) )) } } diff --git a/src/util/lcf.rs b/src/util/lcf.rs index 8c22ff1..9affd26 100644 --- a/src/util/lcf.rs +++ b/src/util/lcf.rs @@ -9,9 +9,9 @@ use crate::obj::{ObjInfo, ObjKind}; #[inline] const fn align_up(value: u32, align: u32) -> u32 { (value + (align - 1)) & !(align - 1) } -pub fn generate_ldscript(obj: &ObjInfo, auto_force_files: bool) -> Result { +pub fn generate_ldscript(obj: &ObjInfo, force_active: &[String]) -> Result { if obj.kind == ObjKind::Relocatable { - return generate_ldscript_partial(obj, auto_force_files); + return generate_ldscript_partial(obj, force_active); } let origin = obj.sections.iter().map(|(_, s)| s.address).min().unwrap(); @@ -54,7 +54,7 @@ pub fn generate_ldscript(obj: &ObjInfo, auto_force_files: bool) -> Result Result Result Result { +pub fn generate_ldscript_partial(obj: &ObjInfo, force_active: &[String]) -> Result { let section_defs = obj.sections.iter().map(|(_, s)| format!("{} :{{}}", s.name)).join("\n "); @@ -92,7 +87,7 @@ pub fn generate_ldscript_partial(obj: &ObjInfo, auto_force_files: bool) -> Resul force_files.push(obj_path.file_name().unwrap().to_str().unwrap().to_string()); } - let mut force_active = vec![]; + let mut force_active = force_active.to_vec(); for symbol in obj.symbols.iter() { if symbol.flags.is_force_active() && symbol.flags.is_global() && !symbol.flags.is_no_write() { @@ -100,14 +95,9 @@ pub fn generate_ldscript_partial(obj: &ObjInfo, auto_force_files: bool) -> Resul } } - let mut out = include_str!("../../assets/ldscript_partial.lcf") + let out = include_str!("../../assets/ldscript_partial.lcf") .replace("$SECTIONS", §ion_defs) .replace("$FORCEACTIVE", &force_active.join("\n ")); - out = if auto_force_files { - out.replace("$FORCEFILES", &force_files.join("\n ")) - } else { - out.replace("$FORCEFILES", "") - }; Ok(out) } diff --git a/src/util/map.rs b/src/util/map.rs index 4d80db9..d7ca168 100644 --- a/src/util/map.rs +++ b/src/util/map.rs @@ -1,7 +1,7 @@ #![allow(dead_code)] #![allow(unused_mut)] use std::{ - collections::{btree_map, BTreeMap, HashMap}, + collections::{BTreeMap, HashMap}, hash::Hash, io::BufRead, mem::replace, @@ -10,13 +10,14 @@ use std::{ use anyhow::{anyhow, bail, ensure, Error, Result}; use cwdemangle::{demangle, DemangleOptions}; +use flagset::FlagSet; use multimap::MultiMap; use once_cell::sync::Lazy; use regex::{Captures, Regex}; use crate::{ obj::{ObjInfo, ObjKind, ObjSplit, ObjSymbol, ObjSymbolFlagSet, ObjSymbolFlags, ObjSymbolKind}, - util::file::{map_file, map_reader}, + util::{file::map_file, nested::NestedVec}, }; #[derive(Debug, Copy, Clone, Eq, PartialEq)] @@ -35,7 +36,7 @@ pub enum SymbolVisibility { Weak, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Eq, PartialEq)] pub struct SymbolEntry { pub name: String, pub demangled: Option, @@ -254,6 +255,35 @@ impl StateMachine { Ok(()) } + fn finalize(&mut self) -> Result<()> { + // If we didn't find a link map, guess symbol visibility + if !self.has_link_map { + let mut symbol_occurrences = HashMap::::new(); + for symbol in self.result.section_symbols.values().flat_map(|v| v.values().flatten()) { + if symbol.visibility != SymbolVisibility::Local { + *symbol_occurrences.entry(symbol.name.clone()).or_default() += 1; + } + } + + for symbol in + self.result.section_symbols.values_mut().flat_map(|v| v.values_mut().flatten()) + { + if symbol.visibility == SymbolVisibility::Unknown { + if symbol.name.starts_with('.') // ...rodata.0 + || symbol.name.starts_with('@') // @123 + || symbol.name.starts_with("__sinit") + || symbol_occurrences.get(&symbol.name).cloned().unwrap_or(0) > 1 + { + symbol.visibility = SymbolVisibility::Local; + } else { + symbol.visibility = SymbolVisibility::Global; + } + } + } + } + Ok(()) + } + fn process_link_map_entry( captures: Captures, state: &mut LinkMapState, @@ -427,8 +457,7 @@ impl StateMachine { let address = u32::from_str_radix(captures["addr"].trim(), 16)?; let size = u32::from_str_radix(captures["size"].trim(), 16)?; - let align = - captures.name("align").and_then(|m| u32::from_str_radix(m.as_str().trim(), 16).ok()); + let align = captures.name("align").and_then(|m| m.as_str().trim().parse::().ok()); if state.current_unit.as_ref() != Some(&tu) || sym_name == state.current_section { state.current_unit = Some(tu.clone()); @@ -461,8 +490,7 @@ impl StateMachine { } else { SymbolVisibility::Unknown }; - let kind = if sym_name.starts_with('.') { - visibility = SymbolVisibility::Local; + let kind = if sym_name == state.current_section { SymbolKind::Section } else if size > 0 { if is_code_section(&state.current_section) { @@ -484,12 +512,7 @@ impl StateMachine { align, } }; - match state.symbols.entry(address) { - btree_map::Entry::Occupied(e) => e.into_mut().push(entry), - btree_map::Entry::Vacant(e) => { - e.insert(vec![entry]); - } - } + state.symbols.nested_push(address, entry); Ok(()) } @@ -540,7 +563,7 @@ impl StateMachine { } } -pub fn process_map(reader: R) -> Result { +pub fn process_map(reader: &mut R) -> Result { let mut sm = StateMachine { state: ProcessMapState::None, result: Default::default(), @@ -554,12 +577,13 @@ pub fn process_map(reader: R) -> Result { } let state = replace(&mut sm.state, ProcessMapState::None); sm.end_state(state)?; + sm.finalize()?; Ok(sm.result) } pub fn apply_map_file>(path: P, obj: &mut ObjInfo) -> Result<()> { let file = map_file(&path)?; - let info = process_map(map_reader(&file))?; + let info = process_map(&mut file.as_reader())?; apply_map(&info, obj) } @@ -622,7 +646,7 @@ pub fn apply_map(result: &MapInfo, obj: &mut ObjInfo) -> Result<()> { .map(|(addr, _)| *addr) .unwrap_or_else(|| (section.address + section.size) as u32); section.splits.push(*addr, ObjSplit { - unit: unit.clone(), + unit: unit.replace(' ', "/"), end: next, align: None, common: false, @@ -637,6 +661,16 @@ pub fn apply_map(result: &MapInfo, obj: &mut ObjInfo) -> Result<()> { fn add_symbol(obj: &mut ObjInfo, symbol_entry: &SymbolEntry, section: Option) -> Result<()> { let demangled_name = demangle(&symbol_entry.name, &DemangleOptions::default()); + let mut flags: FlagSet = match symbol_entry.visibility { + SymbolVisibility::Unknown => Default::default(), + SymbolVisibility::Global => ObjSymbolFlags::Global.into(), + SymbolVisibility::Local => ObjSymbolFlags::Local.into(), + SymbolVisibility::Weak => ObjSymbolFlags::Weak.into(), + }; + // TODO move somewhere common + if symbol_entry.name.starts_with("..") { + flags |= ObjSymbolFlags::ForceActive; + } obj.add_symbol( ObjSymbol { name: symbol_entry.name.clone(), @@ -645,12 +679,7 @@ fn add_symbol(obj: &mut ObjInfo, symbol_entry: &SymbolEntry, section: Option Default::default(), - SymbolVisibility::Global => ObjSymbolFlags::Global.into(), - SymbolVisibility::Local => ObjSymbolFlags::Local.into(), - SymbolVisibility::Weak => ObjSymbolFlags::Weak.into(), - }), + flags: ObjSymbolFlagSet(flags), kind: match symbol_entry.kind { SymbolKind::Function => ObjSymbolKind::Function, SymbolKind::Object => ObjSymbolKind::Object, diff --git a/src/util/rarc.rs b/src/util/rarc.rs index b801dc5..6099839 100644 --- a/src/util/rarc.rs +++ b/src/util/rarc.rs @@ -1,12 +1,17 @@ // Source: https://github.com/Julgodis/picori/blob/650da9f4fe6050b39b80d5360416591c748058d5/src/rarc.rs // License: MIT // Modified to use `std::io::Cursor<&[u8]>` and `byteorder` -use std::{collections::HashMap, fmt::Display}; +use std::{ + collections::HashMap, + fmt::Display, + io::{Read, Seek, SeekFrom}, + path::{Component, Path, PathBuf}, +}; -use anyhow::{anyhow, ensure, Result}; +use anyhow::{anyhow, bail, ensure, Result}; use byteorder::{BigEndian, LittleEndian, ReadBytesExt}; -use crate::util::file::{read_c_string, Reader}; +use crate::util::file::read_c_string; #[derive(Debug, Clone)] pub struct NamedHash { @@ -62,17 +67,16 @@ struct RarcNode { pub count: u32, } -pub struct RarcReader<'a> { - reader: Reader<'a>, +pub struct RarcReader { directories: Vec, nodes: HashMap, root_node: NamedHash, } -impl<'a> RarcReader<'a> { +impl RarcReader { /// Creates a new RARC reader. - pub fn new(mut reader: Reader<'a>) -> Result { - let base = reader.position(); + pub fn new(reader: &mut R) -> Result { + let base = reader.stream_position()?; let magic = reader.read_u32::()?; let _file_length = reader.read_u32::()?; @@ -101,7 +105,7 @@ impl<'a> RarcReader<'a> { let data_base = base + file_offset as u64; let mut directories = Vec::with_capacity(directory_count as usize); for i in 0..directory_count { - reader.set_position(directory_base + 20 * i as u64); + reader.seek(SeekFrom::Start(directory_base + 20 * i as u64))?; let index = reader.read_u16::()?; let name_hash = reader.read_u16::()?; let _ = reader.read_u16::()?; // 0x200 for folders, 0x1100 for files @@ -114,7 +118,7 @@ impl<'a> RarcReader<'a> { let offset = string_table_offset as u64; let offset = offset + name_offset as u64; ensure!((name_offset as u32) < string_table_length, "invalid string table offset"); - read_c_string(&mut reader, base + offset) + read_c_string(reader, base + offset) }?; if index == 0xFFFF { @@ -139,7 +143,7 @@ impl<'a> RarcReader<'a> { let mut root_node: Option = None; let mut nodes = HashMap::with_capacity(node_count as usize); for i in 0..node_count { - reader.set_position(node_base + 16 * i as u64); + reader.seek(SeekFrom::Start(node_base + 16 * i as u64))?; let _identifier = reader.read_u32::()?; let name_offset = reader.read_u32::()?; let name_hash = reader.read_u16::()?; @@ -158,7 +162,7 @@ impl<'a> RarcReader<'a> { let offset = string_table_offset as u64; let offset = offset + name_offset as u64; ensure!(name_offset < string_table_length, "invalid string table offset"); - read_c_string(&mut reader, base + offset) + read_c_string(reader, base + offset) }?; // FIXME: this assumes that the root node is the first node in the list @@ -171,23 +175,49 @@ impl<'a> RarcReader<'a> { } if let Some(root_node) = root_node { - Ok(Self { reader, directories, nodes, root_node }) + Ok(Self { directories, nodes, root_node }) } else { Err(anyhow!("no root node")) } } - /// Get the data for a file. - pub fn file_data(&mut self, offset: u64, size: u32) -> Result<&'a [u8]> { - ensure!(offset + size as u64 <= self.reader.get_ref().len() as u64, "out of bounds"); - Ok(&self.reader.get_ref()[offset as usize..offset as usize + size as usize]) - } - /// Get a iterator over the nodes in the RARC file. - pub fn nodes(&self) -> Nodes<'_, '_> { + pub fn nodes(&self) -> Nodes<'_> { let root_node = self.root_node.clone(); Nodes { parent: self, stack: vec![NodeState::Begin(root_node)] } } + + /// Find a file in the RARC file. + pub fn find_file>(&self, path: P) -> Result> { + let mut cmp_path = PathBuf::new(); + for component in path.as_ref().components() { + match component { + Component::Normal(name) => cmp_path.push(name.to_ascii_lowercase()), + Component::RootDir => {} + component => bail!("Invalid path component: {:?}", component), + } + } + + let mut current_path = PathBuf::new(); + for node in self.nodes() { + match node { + Node::DirectoryBegin { name } => { + current_path.push(name.name.to_ascii_lowercase()); + } + Node::DirectoryEnd { name: _ } => { + current_path.pop(); + } + Node::File { name, offset, size } => { + if current_path.join(name.name.to_ascii_lowercase()) == cmp_path { + return Ok(Some((offset, size))); + } + } + Node::CurrentDirectory => {} + Node::ParentDirectory => {} + } + } + Ok(None) + } } /// A node in an RARC file. @@ -211,12 +241,12 @@ enum NodeState { } /// An iterator over the nodes in an RARC file. -pub struct Nodes<'parent, 'a> { - parent: &'parent RarcReader<'a>, +pub struct Nodes<'parent> { + parent: &'parent RarcReader, stack: Vec, } -impl<'parent, 'a> Iterator for Nodes<'parent, 'a> { +impl<'parent> Iterator for Nodes<'parent> { type Item = Node; fn next(&mut self) -> Option { diff --git a/src/util/rel.rs b/src/util/rel.rs index ab1a512..bd2b4b6 100644 --- a/src/util/rel.rs +++ b/src/util/rel.rs @@ -1,6 +1,6 @@ use std::{ cmp::Ordering, - io::{Read, Seek, Write}, + io::{Read, Seek, SeekFrom, Write}, }; use anyhow::{anyhow, bail, ensure, Context, Result}; @@ -15,7 +15,7 @@ use crate::{ ObjArchitecture, ObjInfo, ObjKind, ObjRelocKind, ObjSection, ObjSectionKind, ObjSymbol, ObjSymbolFlagSet, ObjSymbolFlags, ObjSymbolKind, }, - util::{file::Reader, IntoCow}, + util::IntoCow, }; /// Do not relocate anything, but accumulate the offset field for the next relocation offset calculation. @@ -136,14 +136,14 @@ struct RelRelocRaw { addend: u32, } -pub fn process_rel_header(reader: &mut Reader) -> Result { +pub fn process_rel_header(reader: &mut R) -> Result { RelHeader::read_be(reader).context("Failed to read REL header") } -pub fn process_rel(reader: &mut Reader) -> Result<(RelHeader, ObjInfo)> { +pub fn process_rel(reader: &mut R, name: &str) -> Result<(RelHeader, ObjInfo)> { let header = process_rel_header(reader)?; let mut sections = Vec::with_capacity(header.num_sections as usize); - reader.set_position(header.section_info_offset as u64); + reader.seek(SeekFrom::Start(header.section_info_offset as u64))?; let mut found_text = false; let mut total_bss_size = 0; for idx in 0..header.num_sections { @@ -158,13 +158,13 @@ pub fn process_rel(reader: &mut Reader) -> Result<(RelHeader, ObjInfo)> { let data = if offset == 0 { vec![] } else { - let position = reader.position(); - reader.set_position(offset as u64); + let position = reader.stream_position()?; + reader.seek(SeekFrom::Start(offset as u64))?; let mut data = vec![0u8; size as usize]; reader.read_exact(&mut data).with_context(|| { format!("Failed to read REL section {} data with size {:#X}", idx, size) })?; - reader.set_position(position); + reader.seek(SeekFrom::Start(position))?; data }; @@ -236,8 +236,8 @@ pub fn process_rel(reader: &mut Reader) -> Result<(RelHeader, ObjInfo)> { let mut unresolved_relocations = Vec::new(); let mut imp_idx = 0; let imp_end = (header.imp_offset + header.imp_size) as u64; - reader.set_position(header.imp_offset as u64); - while reader.position() < imp_end { + reader.seek(SeekFrom::Start(header.imp_offset as u64))?; + while reader.stream_position()? < imp_end { let import = RelImport::read_be(reader)?; if imp_idx == 0 { @@ -261,8 +261,8 @@ pub fn process_rel(reader: &mut Reader) -> Result<(RelHeader, ObjInfo)> { } } - let position = reader.position(); - reader.set_position(import.offset as u64); + let position = reader.stream_position()?; + reader.seek(SeekFrom::Start(import.offset as u64))?; let mut address = 0u32; let mut section = u8::MAX; loop { @@ -306,14 +306,14 @@ pub fn process_rel(reader: &mut Reader) -> Result<(RelHeader, ObjInfo)> { }; unresolved_relocations.push(reloc); } - reader.set_position(position); + reader.seek(SeekFrom::Start(position))?; } log::debug!("Read REL ID {}", header.module_id); let mut obj = ObjInfo::new( ObjKind::Relocatable, ObjArchitecture::PowerPc, - String::new(), + name.to_string(), symbols, sections, ); @@ -400,6 +400,8 @@ pub struct RelWriteInfo { /// Override the number of sections in the file. /// Useful for matching RELs that included debug sections. pub section_count: Option, + /// If true, don't print warnings about overriding values. + pub quiet: bool, } const PERMITTED_SECTIONS: [&str; 7] = @@ -463,20 +465,20 @@ pub fn write_rel( // Apply overrides if let Some(section_count) = info.section_count { - if section_count != num_sections as usize { + if section_count != num_sections as usize && !info.quiet { warn!(from = num_sections, to = section_count, "Overriding section count"); } num_sections = section_count as u32; } if info.version >= 2 { if let Some(align_override) = info.align { - if align_override != align { + if align_override != align && !info.quiet { warn!(from = align, to = align_override, "Overriding alignment"); } align = align_override; } if let Some(bss_align_override) = info.bss_align { - if bss_align_override != bss_align { + if bss_align_override != bss_align && !info.quiet { warn!(from = bss_align, to = bss_align_override, "Overriding BSS alignment"); } bss_align = bss_align_override; diff --git a/src/util/rso.rs b/src/util/rso.rs index 05eb122..1aa6e34 100644 --- a/src/util/rso.rs +++ b/src/util/rso.rs @@ -1,4 +1,4 @@ -use std::{io::Read, path::Path}; +use std::io::{Read, Seek, SeekFrom}; use anyhow::{anyhow, ensure, Result}; use byteorder::{BigEndian, ReadBytesExt}; @@ -9,7 +9,7 @@ use crate::{ ObjArchitecture, ObjInfo, ObjKind, ObjSection, ObjSectionKind, ObjSymbol, ObjSymbolFlagSet, ObjSymbolFlags, ObjSymbolKind, }, - util::file::{map_file, map_reader, read_c_string, read_string}, + util::file::{read_c_string, read_string}, }; /// For RSO references to the DOL, the sections are hardcoded. @@ -32,10 +32,7 @@ pub const DOL_SECTION_NAMES: [Option<&str>; 14] = [ /// ABS symbol section index. pub const DOL_SECTION_ABS: u32 = 65521; -pub fn process_rso>(path: P) -> Result { - let mmap = map_file(path)?; - let mut reader = map_reader(&mmap); - +pub fn process_rso(reader: &mut R) -> Result { ensure!(reader.read_u32::()? == 0, "Expected 'next' to be 0"); ensure!(reader.read_u32::()? == 0, "Expected 'prev' to be 0"); let num_sections = reader.read_u32::()?; @@ -64,7 +61,7 @@ pub fn process_rso>(path: P) -> Result { let import_table_name_offset = reader.read_u32::()?; let mut sections = Vec::with_capacity(num_sections as usize); - reader.set_position(section_info_offset as u64); + reader.seek(SeekFrom::Start(section_info_offset as u64))?; let mut total_bss_size = 0; for idx in 0..num_sections { let offset = reader.read_u32::()?; @@ -79,11 +76,11 @@ pub fn process_rso>(path: P) -> Result { let data = if offset == 0 { vec![] } else { - let position = reader.position(); - reader.set_position(offset as u64); + let position = reader.stream_position()?; + reader.seek(SeekFrom::Start(offset as u64))?; let mut data = vec![0u8; size as usize]; reader.read_exact(&mut data)?; - reader.set_position(position); + reader.seek(SeekFrom::Start(position))?; data }; @@ -148,8 +145,8 @@ pub fn process_rso>(path: P) -> Result { add_symbol(epilog_section, epilog_offset, "_epilog")?; add_symbol(unresolved_section, unresolved_offset, "_unresolved")?; - reader.set_position(external_rel_offset as u64); - while reader.position() < (external_rel_offset + external_rel_size) as u64 { + reader.seek(SeekFrom::Start(external_rel_offset as u64))?; + while reader.stream_position()? < (external_rel_offset + external_rel_size) as u64 { let offset = reader.read_u32::()?; let id_and_type = reader.read_u32::()?; let id = (id_and_type & 0xFFFFFF00) >> 8; @@ -164,10 +161,10 @@ pub fn process_rso>(path: P) -> Result { ); } - reader.set_position(export_table_offset as u64); - while reader.position() < (export_table_offset + export_table_size) as u64 { + reader.seek(SeekFrom::Start(export_table_offset as u64))?; + while reader.stream_position()? < (export_table_offset + export_table_size) as u64 { let name_off = reader.read_u32::()?; - let name = read_c_string(&mut reader, (export_table_name_offset + name_off) as u64)?; + let name = read_c_string(reader, (export_table_name_offset + name_off) as u64)?; let sym_off = reader.read_u32::()?; let section_idx = reader.read_u32::()?; let hash_n = reader.read_u32::()?; @@ -207,10 +204,10 @@ pub fn process_rso>(path: P) -> Result { data_kind: Default::default(), }); } - reader.set_position(import_table_offset as u64); - while reader.position() < (import_table_offset + import_table_size) as u64 { + reader.seek(SeekFrom::Start(import_table_offset as u64))?; + while reader.stream_position()? < (import_table_offset + import_table_size) as u64 { let name_off = reader.read_u32::()?; - let name = read_c_string(&mut reader, (import_table_name_offset + name_off) as u64)?; + let name = read_c_string(reader, (import_table_name_offset + name_off) as u64)?; let sym_off = reader.read_u32::()?; let section_idx = reader.read_u32::()?; log::debug!("Import: {}, sym off: {}, section: {}", name, sym_off, section_idx); @@ -218,7 +215,7 @@ pub fn process_rso>(path: P) -> Result { let name = match name_offset { 0 => String::new(), - _ => read_string(&mut reader, name_offset as u64, name_size as usize)?, + _ => read_string(reader, name_offset as u64, name_size as usize)?, }; let obj = ObjInfo::new(ObjKind::Relocatable, ObjArchitecture::PowerPc, name, symbols, sections); diff --git a/src/util/split.rs b/src/util/split.rs index 7271362..633b6b8 100644 --- a/src/util/split.rs +++ b/src/util/split.rs @@ -1,5 +1,5 @@ use std::{ - cmp::{min, Ordering}, + cmp::{max, min, Ordering}, collections::{BTreeMap, HashMap, HashSet}, }; @@ -629,7 +629,7 @@ const fn align_up(value: u32, align: u32) -> u32 { (value + (align - 1)) & !(ali /// - Creating splits for gaps between existing splits /// - Resolving a new object link order #[instrument(level = "debug", skip(obj))] -pub fn update_splits(obj: &mut ObjInfo, common_start: Option) -> Result<()> { +pub fn update_splits(obj: &mut ObjInfo, common_start: Option, fill_gaps: bool) -> Result<()> { // Create splits for extab and extabindex entries if let Some((section_index, section)) = obj.sections.by_name("extabindex")? { let start = SectionAddress::new(section_index, section.address as u32); @@ -663,8 +663,10 @@ pub fn update_splits(obj: &mut ObjInfo, common_start: Option) -> Result<()> // Ensure splits don't overlap symbols or each other validate_splits(obj)?; - // Add symbols to beginning of any split that doesn't start with a symbol - add_padding_symbols(obj)?; + if fill_gaps { + // Add symbols to beginning of any split that doesn't start with a symbol + add_padding_symbols(obj)?; + } // Resolve link order obj.link_order = resolve_link_order(obj)?; @@ -779,7 +781,9 @@ pub fn split_obj(obj: &ObjInfo) -> Result> { vec![], ); if let Some(comment_version) = unit.comment_version { - split_obj.mw_comment = Some(MWComment::new(comment_version)?); + if comment_version > 0 { + split_obj.mw_comment = Some(MWComment::new(comment_version)?); + } } else { split_obj.mw_comment = obj.mw_comment.clone(); } @@ -833,8 +837,20 @@ pub fn split_obj(obj: &ObjInfo) -> Result> { .ok_or_else(|| anyhow!("Unit '{}' not in link order", split.unit))?; // Calculate & verify section alignment - let mut align = - split.align.map(u64::from).unwrap_or_else(|| default_section_align(section)); + let mut align = split.align.unwrap_or_else(|| { + let default_align = default_section_align(section) as u32; + max( + // Maximum alignment of any symbol in this split + obj.symbols + .for_section_range(section_index, current_address.address..file_end.address) + .filter(|&(_, s)| s.size_known && s.size > 0) + .filter_map(|(_, s)| s.align) + .max() + .unwrap_or(default_align), + default_align, + ) + }) as u64; + if current_address & (align as u32 - 1) != 0 { log::warn!( "Alignment for {} {} expected {}, but starts at {:#010X}", @@ -877,7 +893,7 @@ pub fn split_obj(obj: &ObjInfo) -> Result> { let mut comm_addr = current_address; for (symbol_idx, symbol) in obj .symbols - .for_section_range(section_index, current_address.address..file_end.address) + .for_section_range(section_index, current_address.address..=file_end.address) .filter(|&(_, s)| { s.section == Some(section_index) && !is_linker_generated_label(&s.name) }) @@ -886,6 +902,15 @@ pub fn split_obj(obj: &ObjInfo) -> Result> { continue; // should never happen? } + // TODO hack for gTRKInterruptVectorTableEnd + if (symbol.address == file_end.address as u64 + && symbol.name != "gTRKInterruptVectorTableEnd") + || (symbol.address == current_address.address as u64 + && symbol.name == "gTRKInterruptVectorTableEnd") + { + continue; + } + if split.common && symbol.address as u32 > comm_addr.address { // HACK: Add padding for common bug file.symbols.add_direct(ObjSymbol { @@ -907,7 +932,7 @@ pub fn split_obj(obj: &ObjInfo) -> Result> { name: symbol.name.clone(), demangled_name: symbol.demangled_name.clone(), address: if split.common { - 4 + symbol.align.unwrap_or(4) as u64 } else { symbol.address - current_address.address as u64 }, @@ -920,7 +945,7 @@ pub fn split_obj(obj: &ObjInfo) -> Result> { symbol.flags }, kind: symbol.kind, - align: if split.common { Some(4) } else { symbol.align }, + align: symbol.align, data_kind: symbol.data_kind, })?); } @@ -1081,7 +1106,7 @@ pub fn default_section_align(section: &ObjSection) -> u64 { ObjSectionKind::Code => 4, _ => match section.name.as_str() { ".ctors" | ".dtors" | "extab" | "extabindex" => 4, - ".sbss" => 4, // ? + ".sbss" => 8, // ? _ => 8, }, }