encounter/decomp-toolkit
1
0
Fork 0
mirror of https://github.com/encounter/decomp-toolkit.git synced 2025-12-14 07:36:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improvements to REL & map support

Browse Source 
- Fix symbols.txt align attribute
- Fully support nested RARC files & transparent Yaz0 decompression
- Guess symbol visibility for maps without link map
- Add module name config
- Add manual force_active config
- Quiet option for shasum
- `symbols_known` and `fill_gaps` config
- Allow disabling .comment generation per-unit (`comment:0`)
- Various minor fixes
- Add `rarc` and `yaz0` commands
This commit is contained in:
Luke Street
2023-09-05 17:22:22 -04:00
parent f9f7fb2e1e
commit e3857d3212
32 changed files with 975 additions and 366 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/cmd/ar.rs
View File

@@ -60,8 +60,8 @@ fn create(args: CreateArgs) -> Result<()> {
Entry::Vacant(e) => e.insert(Vec::new()),
Entry::Occupied(_) => bail!("Duplicate file name '{path_str}'"),
};
let mmap = map_file(path)?;
let obj = object::File::parse(&*mmap)?;
let file = map_file(path)?;
let obj = object::File::parse(file.as_slice())?;
for symbol in obj.symbols() {
if symbol.scope() == SymbolScope::Dynamic {
entries.push(symbol.name_bytes()?.to_vec());

341
src/cmd/dol.rs
View File

@@ -1,8 +1,10 @@
use std::{
borrow::Cow,
cmp::min,
collections::{btree_map::Entry, hash_map, BTreeMap, HashMap},
ffi::OsStr,
fs,
fs::{DirBuilder, File},
fs::DirBuilder,
io::Write,
mem::take,
path::{Path, PathBuf},
@@ -24,12 +26,13 @@ use crate::{
AnalysisPass, FindRelCtorsDtors, FindRelRodataData, FindSaveRestSleds,
FindTRKInterruptVectorTable,
},
signatures::{apply_signatures, apply_signatures_post},
signatures::{apply_signatures, apply_signatures_post, update_ctors_dtors},
tracker::Tracker,
},
cmd::shasum::file_sha1_string,
obj::{
best_match_for_reloc, ObjDataKind, ObjInfo, ObjReloc, ObjRelocKind, ObjSectionKind,
ObjSymbol, ObjSymbolFlagSet, ObjSymbolFlags, ObjSymbolKind, ObjSymbolScope, SymbolIndex,
ObjDataKind, ObjInfo, ObjReloc, ObjRelocKind, ObjSectionKind, ObjSymbol, ObjSymbolFlagSet,
ObjSymbolFlags, ObjSymbolKind, ObjSymbolScope, SymbolIndex,
},
util::{
asm::write_asm,
@@ -41,10 +44,13 @@ use crate::{
dep::DepFile,
dol::process_dol,
elf::{process_elf, write_elf},
file::{buf_writer, decompress_if_needed, map_file, touch, verify_hash, Reader},
file::{
buf_reader, buf_writer, decompress_if_needed, map_file, touch, verify_hash,
FileIterator, Reader,
},
lcf::{asm_path_for_unit, generate_ldscript, obj_path_for_unit},
map::apply_map_file,
rel::process_rel,
rel::{process_rel, process_rel_header},
rso::{process_rso, DOL_SECTION_ABS, DOL_SECTION_NAMES},
split::{is_linker_generated_object, split_obj, update_splits},
IntoCow, ToCow,
@@ -66,6 +72,7 @@ enum SubCommand {
Split(SplitArgs),
Diff(DiffArgs),
Apply(ApplyArgs),
Config(ConfigArgs),
}
#[derive(FromArgs, PartialEq, Eq, Debug)]
@@ -128,6 +135,18 @@ pub struct ApplyArgs {
map_file: PathBuf,
}
#[derive(FromArgs, PartialEq, Eq, Debug)]
/// Generates a project configuration file from a DOL (& RELs).
#[argp(subcommand, name = "config")]
pub struct ConfigArgs {
#[argp(positional)]
/// object files
objects: Vec<PathBuf>,
#[argp(option, short = 'o')]
/// output config YAML file
out_file: PathBuf,
}
#[inline]
fn bool_true() -> bool { true }
@@ -195,15 +214,21 @@ pub struct ProjectConfig {
pub detect_strings: bool,
#[serde(default = "bool_true")]
pub write_asm: bool,
/// Adds all objects to FORCEFILES in the linker script.
#[serde(default)]
pub auto_force_files: bool,
/// Specifies the start of the common BSS section.
pub common_start: Option<u32>,
/// Disables all analysis passes that yield new symbols,
/// and instead assumes that all symbols are known.
#[serde(default)]
pub symbols_known: bool,
/// Fills gaps between symbols with
#[serde(default = "bool_true")]
pub fill_gaps: bool,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct ModuleConfig {
/// Object name. If not specified, the file name without extension will be used.
pub name: Option<String>,
#[serde(with = "path_slash_serde")]
pub object: PathBuf,
pub hash: Option<String>,
@@ -213,6 +238,9 @@ pub struct ModuleConfig {
pub symbols: Option<PathBuf>,
#[serde(with = "path_slash_serde_option", default)]
pub map: Option<PathBuf>,
/// Forces the given symbols to be active in the linker script.
#[serde(default)]
pub force_active: Vec<String>,
}
impl ModuleConfig {
@@ -231,7 +259,9 @@ impl ModuleConfig {
}
}
pub fn name(&self) -> Cow<'_, str> { self.file_prefix() }
pub fn name(&self) -> Cow<'_, str> {
self.name.as_ref().map(|n| n.as_str().to_cow()).unwrap_or_else(|| self.file_prefix())
}
}
#[derive(Serialize, Deserialize, Debug, Clone)]
@@ -264,12 +294,12 @@ pub fn run(args: Args) -> Result<()> {
SubCommand::Split(c_args) => split(c_args),
SubCommand::Diff(c_args) => diff(c_args),
SubCommand::Apply(c_args) => apply(c_args),
SubCommand::Config(c_args) => config(c_args),
}
}
fn apply_selfile(obj: &mut ObjInfo, selfile: &Path) -> Result<()> {
log::info!("Loading {}", selfile.display());
let rso = process_rso(selfile)?;
fn apply_selfile(obj: &mut ObjInfo, buf: &[u8]) -> Result<()> {
let rso = process_rso(&mut Reader::new(buf))?;
for symbol in rso.symbols.iter() {
let dol_section_index = match symbol.section {
Some(section) => section,
@@ -347,12 +377,16 @@ fn apply_selfile(obj: &mut ObjInfo, selfile: &Path) -> Result<()> {
}
fn info(args: InfoArgs) -> Result<()> {
let mut obj = process_dol(&args.dol_file)?;
let mut obj = {
let file = map_file(&args.dol_file)?;
let data = decompress_if_needed(file.as_slice())?;
process_dol(data.as_ref(), "")?
};
apply_signatures(&mut obj)?;
let mut state = AnalyzerState::default();
state.detect_functions(&obj)?;
log::info!("Discovered {} functions", state.function_slices.len());
log::debug!("Discovered {} functions", state.function_slices.len());
FindTRKInterruptVectorTable::execute(&mut state, &obj)?;
FindSaveRestSleds::execute(&mut state, &obj)?;
@@ -361,7 +395,9 @@ fn info(args: InfoArgs) -> Result<()> {
apply_signatures_post(&mut obj)?;
if let Some(selfile) = &args.selfile {
apply_selfile(&mut obj, selfile)?;
let file = map_file(selfile)?;
let data = decompress_if_needed(file.as_slice())?;
apply_selfile(&mut obj, data.as_ref())?;
}
println!("{}:", obj.name);
@@ -434,14 +470,10 @@ fn update_symbols(obj: &mut ObjInfo, modules: &ModuleMap<'_>) -> Result<()> {
.get_elf_index(rel_reloc.target_section as usize)
.ok_or_else(|| anyhow!("Failed to locate REL section {}", rel_reloc.target_section))?;
let target_symbols = obj
.symbols
.at_section_address(target_section_index, rel_reloc.addend)
.filter(|(_, s)| s.referenced_by(rel_reloc.kind))
.collect_vec();
let target_symbol = best_match_for_reloc(target_symbols, rel_reloc.kind);
if let Some((symbol_index, symbol)) = target_symbol {
if let Some((symbol_index, symbol)) = obj.symbols.for_relocation(
SectionAddress::new(target_section_index, rel_reloc.addend),
rel_reloc.kind,
)? {
// Update symbol
log::trace!(
"Found symbol in section {} at {:#010X}: {}",
@@ -524,16 +556,15 @@ fn create_relocations(obj: &mut ObjInfo, modules: &ModuleMap<'_>, dol_obj: &ObjI
)?
};
let target_symbols = target_obj
.symbols
.at_section_address(target_section_index, rel_reloc.addend)
.filter(|(_, s)| s.referenced_by(rel_reloc.kind))
.collect_vec();
let Some((symbol_index, symbol)) = best_match_for_reloc(target_symbols, rel_reloc.kind)
let Some((symbol_index, symbol)) = target_obj.symbols.for_relocation(
SectionAddress::new(target_section_index, rel_reloc.addend),
rel_reloc.kind,
)?
else {
bail!(
"Couldn't find module {} symbol in section {} at {:#010X}",
"Couldn't find module {} ({}) symbol in section {} at {:#010X}",
rel_reloc.module_id,
target_obj.name,
rel_reloc.target_section,
rel_reloc.addend
);
@@ -625,11 +656,15 @@ fn resolve_external_relocations(
type AnalyzeResult = (ObjInfo, Vec<PathBuf>);
fn load_analyze_dol(config: &ProjectConfig) -> Result<AnalyzeResult> {
// log::info!("Loading {}", config.object.display());
if let Some(hash_str) = &config.base.hash {
verify_hash(&config.base.object, hash_str)?;
}
let mut obj = process_dol(&config.base.object)?;
log::debug!("Loading {}", config.base.object.display());
let mut obj = {
let file = map_file(&config.base.object)?;
let data = decompress_if_needed(file.as_slice())?;
if let Some(hash_str) = &config.base.hash {
verify_hash(data.as_ref(), hash_str)?;
}
process_dol(data.as_ref(), config.base.name().as_ref())?
};
let mut dep = vec![config.base.object.clone()];
if let Some(comment_version) = config.mw_comment_version {
@@ -651,29 +686,38 @@ fn load_analyze_dol(config: &ProjectConfig) -> Result<AnalyzeResult> {
dep.push(symbols_path.clone());
}
// TODO move before symbols?
debug!("Performing signature analysis");
apply_signatures(&mut obj)?;
if !config.symbols_known {
// TODO move before symbols?
debug!("Performing signature analysis");
apply_signatures(&mut obj)?;
if !config.quick_analysis {
let mut state = AnalyzerState::default();
debug!("Detecting function boundaries");
state.detect_functions(&obj)?;
if !config.quick_analysis {
let mut state = AnalyzerState::default();
debug!("Detecting function boundaries");
state.detect_functions(&obj)?;
FindTRKInterruptVectorTable::execute(&mut state, &obj)?;
FindSaveRestSleds::execute(&mut state, &obj)?;
state.apply(&mut obj)?;
FindTRKInterruptVectorTable::execute(&mut state, &obj)?;
FindSaveRestSleds::execute(&mut state, &obj)?;
state.apply(&mut obj)?;
}
apply_signatures_post(&mut obj)?;
}
apply_signatures_post(&mut obj)?;
if let Some(selfile) = &config.selfile {
log::info!("Loading {}", selfile.display());
let file = map_file(selfile)?;
let data = decompress_if_needed(file.as_slice())?;
if let Some(hash) = &config.selfile_hash {
verify_hash(selfile, hash)?;
verify_hash(data.as_ref(), hash)?;
}
apply_selfile(&mut obj, selfile)?;
apply_selfile(&mut obj, data.as_ref())?;
dep.push(selfile.clone());
}
// Create _ctors and _dtors symbols if missing
update_ctors_dtors(&mut obj)?;
Ok((obj, dep))
}
@@ -691,7 +735,7 @@ fn split_write_obj(
debug!("Applying relocations");
tracker.apply(obj, false)?;
if config.detect_objects {
if !config.symbols_known && config.detect_objects {
debug!("Detecting object boundaries");
detect_objects(obj)?;
}
@@ -702,7 +746,11 @@ fn split_write_obj(
}
debug!("Adjusting splits");
update_splits(obj, if obj.module_id == 0 { config.common_start } else { None })?;
update_splits(
obj,
if obj.module_id == 0 { config.common_start } else { None },
config.fill_gaps,
)?;
if !no_update {
debug!("Writing configuration");
@@ -741,29 +789,32 @@ fn split_write_obj(
}
// Generate ldscript.lcf
fs::write(&out_config.ldscript, generate_ldscript(obj, config.auto_force_files)?)?;
fs::write(&out_config.ldscript, generate_ldscript(obj, &module_config.force_active)?)?;
debug!("Writing disassembly");
let asm_dir = out_dir.join("asm");
for (unit, split_obj) in obj.link_order.iter().zip(&split_objs) {
let out_path = asm_dir.join(asm_path_for_unit(&unit.name));
if config.write_asm {
debug!("Writing disassembly");
let asm_dir = out_dir.join("asm");
for (unit, split_obj) in obj.link_order.iter().zip(&split_objs) {
let out_path = asm_dir.join(asm_path_for_unit(&unit.name));
let mut w = buf_writer(&out_path)?;
write_asm(&mut w, split_obj)
.with_context(|| format!("Failed to write {}", out_path.display()))?;
w.flush()?;
let mut w = buf_writer(&out_path)?;
write_asm(&mut w, split_obj)
.with_context(|| format!("Failed to write {}", out_path.display()))?;
w.flush()?;
}
}
Ok(out_config)
}
fn load_analyze_rel(config: &ProjectConfig, module_config: &ModuleConfig) -> Result<AnalyzeResult> {
debug!("Loading {}", module_config.object.display());
if let Some(hash_str) = &module_config.hash {
verify_hash(&module_config.object, hash_str)?;
}
let map = map_file(&module_config.object)?;
let buf = decompress_if_needed(&map)?;
let (_, mut module_obj) = process_rel(&mut Reader::new(buf.as_ref()))?;
let buf = decompress_if_needed(map.as_slice())?;
if let Some(hash_str) = &module_config.hash {
verify_hash(buf.as_ref(), hash_str)?;
}
let (_, mut module_obj) =
process_rel(&mut Reader::new(buf.as_ref()), module_config.name().as_ref())?;
if let Some(comment_version) = config.mw_comment_version {
module_obj.mw_comment = Some(MWComment::new(comment_version)?);
@@ -785,16 +836,22 @@ fn load_analyze_rel(config: &ProjectConfig, module_config: &ModuleConfig) -> Res
dep.push(symbols_path.clone());
}
debug!("Analyzing module {}", module_obj.module_id);
if !config.quick_analysis {
let mut state = AnalyzerState::default();
state.detect_functions(&module_obj)?;
FindRelCtorsDtors::execute(&mut state, &module_obj)?;
FindRelRodataData::execute(&mut state, &module_obj)?;
state.apply(&mut module_obj)?;
if !config.symbols_known {
debug!("Analyzing module {}", module_obj.module_id);
if !config.quick_analysis {
let mut state = AnalyzerState::default();
state.detect_functions(&module_obj)?;
FindRelCtorsDtors::execute(&mut state, &module_obj)?;
FindRelRodataData::execute(&mut state, &module_obj)?;
state.apply(&mut module_obj)?;
}
apply_signatures(&mut module_obj)?;
apply_signatures_post(&mut module_obj)?;
}
apply_signatures(&mut module_obj)?;
apply_signatures_post(&mut module_obj)?;
// Create _ctors and _dtors symbols if missing
update_ctors_dtors(&mut module_obj)?;
Ok((module_obj, dep))
}
@@ -805,18 +862,32 @@ fn split(args: SplitArgs) -> Result<()> {
let command_start = Instant::now();
info!("Loading {}", args.config.display());
let mut config_file = File::open(&args.config)
.with_context(|| format!("Failed to open config file '{}'", args.config.display()))?;
let config: ProjectConfig = serde_yaml::from_reader(&mut config_file)?;
let mut config: ProjectConfig = {
let mut config_file = buf_reader(&args.config)?;
serde_yaml::from_reader(&mut config_file)?
};
for module_config in config.modules.iter_mut() {
let file = map_file(&module_config.object)?;
let buf = decompress_if_needed(file.as_slice())?;
if let Some(hash_str) = &module_config.hash {
verify_hash(buf.as_ref(), hash_str)?;
} else {
module_config.hash = Some(file_sha1_string(&mut Reader::new(buf.as_ref()))?);
}
}
let out_config_path = args.out_dir.join("config.json");
let mut dep = DepFile::new(out_config_path.clone());
let module_count = config.modules.len() + 1;
let num_threads = min(rayon::current_num_threads(), module_count);
info!(
"Loading and analyzing {} modules (using {} threads)",
"Loading and analyzing {} module{} (using {} thread{})",
module_count,
rayon::current_num_threads()
if module_count == 1 { "" } else { "s" },
num_threads,
if num_threads == 1 { "" } else { "s" }
);
let mut dol_result: Option<Result<AnalyzeResult>> = None;
let mut modules_result: Option<Result<Vec<AnalyzeResult>>> = None;
@@ -871,11 +942,13 @@ fn split(args: SplitArgs) -> Result<()> {
let module_ids = modules.keys().cloned().collect_vec();
// Create any missing symbols (referenced from other modules) and set FORCEACTIVE
update_symbols(&mut obj, &modules)?;
for &module_id in &module_ids {
let (module_config, mut module_obj) = modules.remove(&module_id).unwrap();
update_symbols(&mut module_obj, &modules)?;
modules.insert(module_id, (module_config, module_obj));
if !config.symbols_known {
update_symbols(&mut obj, &modules)?;
for &module_id in &module_ids {
let (module_config, mut module_obj) = modules.remove(&module_id).unwrap();
update_symbols(&mut module_obj, &modules)?;
modules.insert(module_id, (module_config, module_obj));
}
}
// Create relocations to symbols in other modules
@@ -1135,12 +1208,18 @@ fn validate<P: AsRef<Path>>(obj: &ObjInfo, elf_file: P, state: &AnalyzerState) -
fn diff(args: DiffArgs) -> Result<()> {
log::info!("Loading {}", args.config.display());
let mut config_file = File::open(&args.config)
.with_context(|| format!("Failed to open config file '{}'", args.config.display()))?;
let mut config_file = buf_reader(&args.config)?;
let config: ProjectConfig = serde_yaml::from_reader(&mut config_file)?;
log::info!("Loading {}", config.base.object.display());
let mut obj = process_dol(&config.base.object)?;
let mut obj = {
let file = map_file(&config.base.object)?;
let data = decompress_if_needed(file.as_slice())?;
if let Some(hash_str) = &config.base.hash {
verify_hash(data.as_ref(), hash_str)?;
}
process_dol(data.as_ref(), config.base.name().as_ref())?
};
if let Some(symbols_path) = &config.base.symbols {
apply_symbols_file(symbols_path, &mut obj)?;
@@ -1267,6 +1346,8 @@ fn diff(args: DiffArgs) -> Result<()> {
orig_sym.size,
orig_sym.address
);
log::error!("Original: {}", hex::encode_upper(orig_data));
log::error!("Linked: {}", hex::encode_upper(linked_data));
return Ok(());
}
}
@@ -1277,12 +1358,18 @@ fn diff(args: DiffArgs) -> Result<()> {
fn apply(args: ApplyArgs) -> Result<()> {
log::info!("Loading {}", args.config.display());
let mut config_file = File::open(&args.config)
.with_context(|| format!("Failed to open config file '{}'", args.config.display()))?;
let mut config_file = buf_reader(&args.config)?;
let config: ProjectConfig = serde_yaml::from_reader(&mut config_file)?;
log::info!("Loading {}", config.base.object.display());
let mut obj = process_dol(&config.base.object)?;
let mut obj = {
let file = map_file(&config.base.object)?;
let data = decompress_if_needed(file.as_slice())?;
if let Some(hash_str) = &config.base.hash {
verify_hash(data.as_ref(), hash_str)?;
}
process_dol(data.as_ref(), config.base.name().as_ref())?
};
if let Some(symbols_path) = &config.base.symbols {
if !apply_symbols_file(symbols_path, &mut obj)? {
@@ -1429,3 +1516,75 @@ fn apply(args: ApplyArgs) -> Result<()> {
Ok(())
}
fn config(args: ConfigArgs) -> Result<()> {
let mut config = ProjectConfig {
base: ModuleConfig {
name: None,
object: Default::default(),
hash: None,
splits: None,
symbols: None,
map: None,
force_active: vec![],
},
selfile: None,
selfile_hash: None,
mw_comment_version: None,
quick_analysis: false,
modules: vec![],
detect_objects: true,
detect_strings: true,
write_asm: true,
common_start: None,
symbols_known: false,
fill_gaps: true,
};
let mut modules = BTreeMap::<u32, ModuleConfig>::new();
for result in FileIterator::new(&args.objects)? {
let (path, entry) = result?;
log::info!("Loading {}", path.display());
match path.extension() {
Some(ext) if ext.eq_ignore_ascii_case(OsStr::new("dol")) => {
config.base.object = path;
config.base.hash = Some(file_sha1_string(&mut entry.as_reader())?);
}
Some(ext) if ext.eq_ignore_ascii_case(OsStr::new("rel")) => {
let header = process_rel_header(&mut entry.as_reader())?;
modules.insert(header.module_id, ModuleConfig {
name: None,
object: path,
hash: Some(file_sha1_string(&mut entry.as_reader())?),
splits: None,
symbols: None,
map: None,
force_active: vec![],
});
}
Some(ext) if ext.eq_ignore_ascii_case(OsStr::new("sel")) => {
config.selfile = Some(path);
config.selfile_hash = Some(file_sha1_string(&mut entry.as_reader())?);
}
Some(ext) if ext.eq_ignore_ascii_case(OsStr::new("rso")) => {
config.modules.push(ModuleConfig {
name: None,
object: path,
hash: Some(file_sha1_string(&mut entry.as_reader())?),
splits: None,
symbols: None,
map: None,
force_active: vec![],
});
}
_ => bail!("Unknown file extension: '{}'", path.display()),
}
}
config.modules.extend(modules.into_values());
let mut out = buf_writer(&args.out_file)?;
serde_yaml::to_writer(&mut out, &config)?;
out.flush()?;
Ok(())
}

9
src/cmd/dwarf.rs
View File

@@ -49,9 +49,10 @@ pub fn run(args: Args) -> Result<()> {
}
fn dump(args: DumpArgs) -> Result<()> {
let mmap = map_file(&args.in_file)?;
if mmap.starts_with(b"!<arch>\n") {
let mut archive = ar::Archive::new(&*mmap);
let file = map_file(&args.in_file)?;
let buf = file.as_slice();
if buf.starts_with(b"!<arch>\n") {
let mut archive = ar::Archive::new(buf);
while let Some(result) = archive.next_entry() {
let mut e = match result {
Ok(e) => e,
@@ -85,7 +86,7 @@ fn dump(args: DumpArgs) -> Result<()> {
}
}
} else {
let obj_file = object::read::File::parse(&*mmap)?;
let obj_file = object::read::File::parse(buf)?;
let debug_section = obj_file
.section_by_name(".debug")
.ok_or_else(|| anyhow!("Failed to locate .debug section"))?;

4
src/cmd/elf2dol.rs
View File

@@ -43,8 +43,8 @@ const MAX_TEXT_SECTIONS: usize = 7;
const MAX_DATA_SECTIONS: usize = 11;
pub fn run(args: Args) -> Result<()> {
let map = map_file(&args.elf_file)?;
let obj_file = object::read::File::parse(&*map)?;
let file = map_file(&args.elf_file)?;
let obj_file = object::read::File::parse(file.as_slice())?;
match obj_file.architecture() {
Architecture::PowerPc => {}
arch => bail!("Unexpected architecture: {arch:?}"),

22
src/cmd/map.rs
View File

@@ -6,7 +6,7 @@ use argp::FromArgs;
use cwdemangle::{demangle, DemangleOptions};
use crate::util::{
file::{map_file, map_reader},
file::map_file,
map::{process_map, SymbolEntry, SymbolRef},
};
@@ -90,8 +90,8 @@ pub fn run(args: Args) -> Result<()> {
}
fn entries(args: EntriesArgs) -> Result<()> {
let map = map_file(&args.map_file)?;
let entries = process_map(map_reader(&map))?;
let file = map_file(&args.map_file)?;
let entries = process_map(&mut file.as_reader())?;
match entries.unit_entries.get_vec(&args.unit) {
Some(vec) => {
for symbol_ref in vec {
@@ -108,8 +108,8 @@ fn entries(args: EntriesArgs) -> Result<()> {
}
fn symbol(args: SymbolArgs) -> Result<()> {
let map = map_file(&args.map_file)?;
let entries = process_map(map_reader(&map))?;
let file = map_file(&args.map_file)?;
let entries = process_map(&mut file.as_reader())?;
let opt_ref: Option<(SymbolRef, SymbolEntry)> = None;
_ = entries;
@@ -165,8 +165,8 @@ fn symbol(args: SymbolArgs) -> Result<()> {
}
fn order(args: OrderArgs) -> Result<()> {
let map = map_file(&args.map_file)?;
let entries = process_map(map_reader(&map))?;
let file = map_file(&args.map_file)?;
let entries = process_map(&mut file.as_reader())?;
_ = entries;
// TODO
@@ -179,8 +179,8 @@ fn order(args: OrderArgs) -> Result<()> {
}
fn slices(args: SlicesArgs) -> Result<()> {
let map = map_file(&args.map_file)?;
let entries = process_map(map_reader(&map))?;
let file = map_file(&args.map_file)?;
let entries = process_map(&mut file.as_reader())?;
_ = entries;
// TODO
@@ -213,8 +213,8 @@ fn slices(args: SlicesArgs) -> Result<()> {
}
fn symbols(args: SymbolsArgs) -> Result<()> {
let map = map_file(&args.map_file)?;
let entries = process_map(map_reader(&map))?;
let file = map_file(&args.map_file)?;
let entries = process_map(&mut file.as_reader())?;
_ = entries;
// TODO

2
src/cmd/mod.rs
View File

@@ -6,6 +6,8 @@ pub mod elf;
pub mod elf2dol;
pub mod map;
pub mod metroidbuildinfo;
pub mod rarc;
pub mod rel;
pub mod rso;
pub mod shasum;
pub mod yaz0;

109
src/cmd/rarc.rs Normal file
View File

@@ -0,0 +1,109 @@
use std::{fs, fs::DirBuilder, path::PathBuf};
use anyhow::{Context, Result};
use argp::FromArgs;
use crate::util::{
file::{decompress_if_needed, map_file},
rarc::{Node, RarcReader},
};
#[derive(FromArgs, PartialEq, Debug)]
/// Commands for processing RSO files.
#[argp(subcommand, name = "rarc")]
pub struct Args {
#[argp(subcommand)]
command: SubCommand,
}
#[derive(FromArgs, PartialEq, Debug)]
#[argp(subcommand)]
enum SubCommand {
List(ListArgs),
Extract(ExtractArgs),
}
#[derive(FromArgs, PartialEq, Eq, Debug)]
/// Views RARC file information.
#[argp(subcommand, name = "list")]
pub struct ListArgs {
#[argp(positional)]
/// RARC file
file: PathBuf,
}
#[derive(FromArgs, PartialEq, Eq, Debug)]
/// Extracts RARC file contents.
#[argp(subcommand, name = "extract")]
pub struct ExtractArgs {
#[argp(positional)]
/// RARC file
file: PathBuf,
#[argp(option, short = 'o')]
/// output directory
output: Option<PathBuf>,
}
pub fn run(args: Args) -> Result<()> {
match args.command {
SubCommand::List(c_args) => list(c_args),
SubCommand::Extract(c_args) => extract(c_args),
}
}
fn list(args: ListArgs) -> Result<()> {
let file = map_file(&args.file)?;
let rarc = RarcReader::new(&mut file.as_reader())?;
let mut current_path = PathBuf::new();
for node in rarc.nodes() {
match node {
Node::DirectoryBegin { name } => {
current_path.push(name.name);
}
Node::DirectoryEnd { name: _ } => {
current_path.pop();
}
Node::File { name, offset, size } => {
let path = current_path.join(name.name);
println!("{}: {} bytes, offset {:#X}", path.display(), size, offset);
}
Node::CurrentDirectory => {}
Node::ParentDirectory => {}
}
}
Ok(())
}
fn extract(args: ExtractArgs) -> Result<()> {
let file = map_file(&args.file)?;
let rarc = RarcReader::new(&mut file.as_reader())?;
let mut current_path = PathBuf::new();
for node in rarc.nodes() {
match node {
Node::DirectoryBegin { name } => {
current_path.push(name.name);
}
Node::DirectoryEnd { name: _ } => {
current_path.pop();
}
Node::File { name, offset, size } => {
let file_data = decompress_if_needed(
&file.as_slice()[offset as usize..offset as usize + size as usize],
)?;
let file_path = current_path.join(&name.name);
let output_path =
args.output.as_ref().map(|p| p.join(&file_path)).unwrap_or_else(|| file_path);
if let Some(parent) = output_path.parent() {
DirBuilder::new().recursive(true).create(parent)?;
}
fs::write(&output_path, file_data)
.with_context(|| format!("Failed to write file '{}'", output_path.display()))?;
}
Node::CurrentDirectory => {}
Node::ParentDirectory => {}
}
}
Ok(())
}

47
src/cmd/rel.rs
View File

@@ -93,6 +93,9 @@ pub struct MakeArgs {
#[argp(option, short = 'c')]
/// (optional) project configuration file
config: Option<PathBuf>,
#[argp(switch, short = 'w')]
/// disable warnings
no_warn: bool,
}
pub fn run(args: Args) -> Result<()> {
@@ -121,26 +124,27 @@ fn make(args: MakeArgs) -> Result<()> {
if let Some(config_path) = &args.config {
let config: ProjectConfig = serde_yaml::from_reader(&mut buf_reader(config_path)?)?;
for module_config in &config.modules {
if let Some(hash_str) = &module_config.hash {
verify_hash(&module_config.object, hash_str)?;
}
let map = map_file(&module_config.object)?;
let buf = decompress_if_needed(&map)?;
let buf = decompress_if_needed(map.as_slice())?;
if let Some(hash_str) = &module_config.hash {
verify_hash(buf.as_ref(), hash_str)?;
}
let header = process_rel_header(&mut Reader::new(buf.as_ref()))?;
existing_headers.insert(header.module_id, header);
}
}
let files = process_rsp(&args.files)?;
info!("Loading {} modules", files.len());
let paths = process_rsp(&args.files)?;
info!("Loading {} modules", paths.len());
// Load all modules
let handles = files.iter().map(map_file).collect::<Result<Vec<_>>>()?;
let modules = handles
let files = paths.iter().map(map_file).collect::<Result<Vec<_>>>()?;
let modules = files
.par_iter()
.zip(&files)
.map(|(map, path)| {
load_obj(map).with_context(|| format!("Failed to load '{}'", path.display()))
.zip(&paths)
.map(|(file, path)| {
load_obj(file.as_slice())
.with_context(|| format!("Failed to load '{}'", path.display()))
})
.collect::<Result<Vec<_>>>()?;
@@ -194,7 +198,7 @@ fn make(args: MakeArgs) -> Result<()> {
address: address as u32,
module_id: target_module_id as u32,
target_section: target_symbol.section_index().unwrap().0 as u8,
addend: target_symbol.address() as u32,
addend: (target_symbol.address() as i64 + reloc.addend()) as u32,
});
}
}
@@ -211,7 +215,7 @@ fn make(args: MakeArgs) -> Result<()> {
// Write RELs
let start = Instant::now();
for (((module_id, module), path), relocations) in
modules.iter().enumerate().zip(&files).skip(1).zip(relocations)
modules.iter().enumerate().zip(&paths).skip(1).zip(relocations)
{
let name =
path.file_stem().unwrap_or(OsStr::new("[unknown]")).to_str().unwrap_or("[invalid]");
@@ -224,6 +228,7 @@ fn make(args: MakeArgs) -> Result<()> {
align: None,
bss_align: None,
section_count: None,
quiet: args.no_warn,
};
if let Some(existing_module) = existing_headers.get(&(module_id as u32)) {
info.version = existing_module.version;
@@ -248,9 +253,9 @@ fn make(args: MakeArgs) -> Result<()> {
}
fn info(args: InfoArgs) -> Result<()> {
let map = map_file(args.rel_file)?;
let buf = decompress_if_needed(&map)?;
let (header, mut module_obj) = process_rel(&mut Reader::new(buf.as_ref()))?;
let file = map_file(args.rel_file)?;
let buf = decompress_if_needed(file.as_slice())?;
let (header, mut module_obj) = process_rel(&mut Reader::new(buf.as_ref()), "")?;
let mut state = AnalyzerState::default();
state.detect_functions(&module_obj)?;
@@ -312,7 +317,12 @@ const fn align32(x: u32) -> u32 { (x + 31) & !31 }
fn merge(args: MergeArgs) -> Result<()> {
log::info!("Loading {}", args.dol_file.display());
let mut obj = process_dol(&args.dol_file)?;
let mut obj = {
let file = map_file(&args.dol_file)?;
let buf = decompress_if_needed(file.as_slice())?;
let name = args.dol_file.file_stem().map(|s| s.to_string_lossy()).unwrap_or_default();
process_dol(buf.as_ref(), name.as_ref())?
};
log::info!("Performing signature analysis");
apply_signatures(&mut obj)?;
@@ -323,7 +333,8 @@ fn merge(args: MergeArgs) -> Result<()> {
for result in FileIterator::new(&args.rel_files)? {
let (path, entry) = result?;
log::info!("Loading {}", path.display());
let (_, obj) = process_rel(&mut entry.as_reader())?;
let name = path.file_stem().map(|s| s.to_string_lossy()).unwrap_or_default();
let (_, obj) = process_rel(&mut entry.as_reader(), name.as_ref())?;
match module_map.entry(obj.module_id) {
btree_map::Entry::Vacant(e) => e.insert(obj),
btree_map::Entry::Occupied(_) => bail!("Duplicate module ID {}", obj.module_id),

14
src/cmd/rso.rs
View File

@@ -1,9 +1,12 @@
use std::path::PathBuf;
use anyhow::Result;
use anyhow::{Context, Result};
use argp::FromArgs;
use crate::util::rso::process_rso;
use crate::util::{
file::{decompress_if_needed, map_file, Reader},
rso::process_rso,
};
#[derive(FromArgs, PartialEq, Debug)]
/// Commands for processing RSO files.
@@ -35,7 +38,12 @@ pub fn run(args: Args) -> Result<()> {
}
fn info(args: InfoArgs) -> Result<()> {
let rso = process_rso(args.rso_file)?;
let rso = {
let file = map_file(&args.rso_file)?;
let data = decompress_if_needed(file.as_slice())
.with_context(|| format!("Failed to decompress '{}'", args.rso_file.display()))?;
process_rso(&mut Reader::new(data.as_ref()))?
};
println!("Read RSO module {}", rso.name);
Ok(())
}

47
src/cmd/shasum.rs
View File

@@ -9,7 +9,7 @@ use argp::FromArgs;
use owo_colors::OwoColorize;
use sha1::{Digest, Sha1};
use crate::util::file::{process_rsp, touch};
use crate::util::file::{open_file, process_rsp, touch};
#[derive(FromArgs, PartialEq, Eq, Debug)]
/// Print or check SHA1 (160-bit) checksums.
@@ -24,18 +24,20 @@ pub struct Args {
#[argp(option, short = 'o')]
/// touch output file on successful check
output: Option<PathBuf>,
#[argp(switch, short = 'q')]
/// only print failures and a summary
quiet: bool,
}
const DEFAULT_BUF_SIZE: usize = 8192;
pub fn run(args: Args) -> Result<()> {
for path in process_rsp(&args.files)? {
let file = File::open(&path)
.with_context(|| format!("Failed to open file '{}'", path.display()))?;
let mut file = open_file(&path)?;
if args.check {
check(file)?
check(&args, &mut BufReader::new(file))?
} else {
hash(file, &path)?
hash(&mut file, &path)?
}
}
if let Some(out_path) = args.output {
@@ -45,8 +47,8 @@ pub fn run(args: Args) -> Result<()> {
Ok(())
}
fn check(file: File) -> Result<()> {
let reader = BufReader::new(file);
fn check<R: BufRead>(args: &Args, reader: &mut R) -> Result<()> {
let mut matches = 0usize;
let mut mismatches = 0usize;
for line in reader.lines() {
let line = match line {
@@ -63,16 +65,23 @@ fn check(file: File) -> Result<()> {
hex::decode_to_slice(hash, &mut hash_bytes)
.with_context(|| format!("Invalid line: {line}"))?;
let file =
File::open(file_name).with_context(|| format!("Failed to open file '{file_name}'"))?;
let found_hash = file_sha1(file)?;
let found_hash = file_sha1(
&mut File::open(file_name)
.with_context(|| format!("Failed to open file '{file_name}'"))?,
)?;
if hash_bytes == found_hash.as_ref() {
println!("{}: {}", file_name, "OK".green());
if !args.quiet {
println!("{}: {}", file_name, "OK".green());
}
matches += 1;
} else {
println!("{}: {}", file_name, "FAILED".red());
mismatches += 1;
}
}
if args.quiet && matches > 0 {
println!("{} files {}", matches, "OK".green());
}
if mismatches != 0 {
eprintln!(
"{}",
@@ -83,8 +92,8 @@ fn check(file: File) -> Result<()> {
Ok(())
}
fn hash(file: File, path: &Path) -> Result<()> {
let hash = file_sha1(file)?;
fn hash<R: Read>(reader: &mut R, path: &Path) -> Result<()> {
let hash = file_sha1(reader)?;
let mut hash_buf = [0u8; 40];
let hash_str = base16ct::lower::encode_str(&hash, &mut hash_buf)
.map_err(|e| anyhow!("Failed to encode hash: {e}"))?;
@@ -92,14 +101,22 @@ fn hash(file: File, path: &Path) -> Result<()> {
Ok(())
}
pub fn file_sha1(mut file: File) -> Result<sha1::digest::Output<Sha1>> {
pub fn file_sha1<R: Read>(reader: &mut R) -> Result<sha1::digest::Output<Sha1>> {
let mut buf = [0u8; DEFAULT_BUF_SIZE];
let mut hasher = Sha1::new();
Ok(loop {
let read = file.read(&mut buf).context("File read failed")?;
let read = reader.read(&mut buf).context("File read failed")?;
if read == 0 {
break hasher.finalize();
}
hasher.update(&buf[0..read]);
})
}
pub fn file_sha1_string<R: Read>(reader: &mut R) -> Result<String> {
let hash = file_sha1(reader)?;
let mut hash_buf = [0u8; 40];
let hash_str = base16ct::lower::encode_str(&hash, &mut hash_buf)
.map_err(|e| anyhow!("Failed to encode hash: {e}"))?;
Ok(hash_str.to_string())
}

55
src/cmd/yaz0.rs Normal file
View File

@@ -0,0 +1,55 @@
use std::{fs, path::PathBuf};
use anyhow::{Context, Result};
use argp::FromArgs;
use crate::util::{
file::{decompress_reader, open_file, process_rsp},
IntoCow, ToCow,
};
#[derive(FromArgs, PartialEq, Debug)]
/// Commands for processing YAZ0-compressed files.
#[argp(subcommand, name = "yaz0")]
pub struct Args {
#[argp(subcommand)]
command: SubCommand,
}
#[derive(FromArgs, PartialEq, Debug)]
#[argp(subcommand)]
enum SubCommand {
Decompress(DecompressArgs),
}
#[derive(FromArgs, PartialEq, Eq, Debug)]
/// Decompresses YAZ0-compressed files.
#[argp(subcommand, name = "decompress")]
pub struct DecompressArgs {
#[argp(positional)]
/// YAZ0-compressed files
files: Vec<PathBuf>,
#[argp(option, short = 'o')]
/// Output directory. If not specified, decompresses in-place.
output: Option<PathBuf>,
}
pub fn run(args: Args) -> Result<()> {
match args.command {
SubCommand::Decompress(args) => decompress(args),
}
}
fn decompress(args: DecompressArgs) -> Result<()> {
for path in process_rsp(&args.files)? {
let data = decompress_reader(&mut open_file(&path)?)?;
let out_path = if let Some(output) = &args.output {
output.join(path.file_name().unwrap()).into_cow()
} else {
path.as_path().to_cow()
};
fs::write(out_path.as_ref(), data)
.with_context(|| format!("Failed to write '{}'", out_path.display()))?;
}
Ok(())
}