From a046bf9a66ff0d11507110706c97e7a24f6a1241 Mon Sep 17 00:00:00 2001 From: Luke Street Date: Sun, 6 Aug 2023 15:46:51 -0400 Subject: [PATCH] Updates for build system integration - Use a config file as input to `dol split` - Add depfile output - Adjust splits config writing --- Cargo.lock | 14 +++++- Cargo.toml | 3 +- README.md | 4 +- src/cmd/dol.rs | 117 +++++++++++++++++++++++++++++++++------------ src/obj/mod.rs | 6 +++ src/obj/split.rs | 3 +- src/util/config.rs | 10 +++- src/util/dep.rs | 20 ++++++++ src/util/mod.rs | 1 + 9 files changed, 141 insertions(+), 37 deletions(-) create mode 100644 src/util/dep.rs diff --git a/Cargo.lock b/Cargo.lock index cf9447c..801bfd8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -214,7 +214,7 @@ dependencies = [ [[package]] name = "decomp-toolkit" -version = "0.3.0" +version = "0.3.2" dependencies = [ "anyhow", "ar", @@ -245,6 +245,7 @@ dependencies = [ "regex", "rmp-serde", "serde", + "serde_json", "serde_repr", "serde_yaml", "sha-1", @@ -773,6 +774,17 @@ dependencies = [ "syn 2.0.23", ] +[[package]] +name = "serde_json" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_repr" version = "0.1.14" diff --git a/Cargo.toml b/Cargo.toml index e2cd53e..a19ce6f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ name = "decomp-toolkit" description = "Yet another GameCube/Wii decompilation toolkit." authors = ["Luke Street "] license = "MIT OR Apache-2.0" -version = "0.3.1" +version = "0.3.2" edition = "2021" publish = false build = "build.rs" @@ -49,6 +49,7 @@ petgraph = "0.6.3" ppc750cl = { git = "https://github.com/encounter/ppc750cl", rev = "5f6e991bf495388c4104f188d2e90c79da9f78de" } regex = "1.9.0" serde = "1.0.166" +serde_json = "1.0.104" serde_repr = "0.1.14" serde_yaml = "0.9.22" sha-1 = "0.10.1" diff --git a/README.md b/README.md index 27db309..f440b51 100644 --- a/README.md +++ b/README.md @@ -269,9 +269,11 @@ $ dtk dol info input.dol Analyzes and splits a DOL file into relocatable objects based on user configuration. ```shell -$ dtk dol split input.dol target -s config/symbols.txt -p config/splits.txt +$ dtk dol split config.yml target ``` +TODO: document configuration file + ### dwarf dump Dumps DWARF 1.1 information from an ELF file. (Does **not** support DWARF 2+) diff --git a/src/cmd/dol.rs b/src/cmd/dol.rs index 658e69e..9070f65 100644 --- a/src/cmd/dol.rs +++ b/src/cmd/dol.rs @@ -8,6 +8,7 @@ use std::{ use anyhow::{anyhow, bail, Context, Result}; use argp::FromArgs; +use serde::{Deserialize, Serialize}; use crate::{ analysis::{ @@ -24,6 +25,7 @@ use crate::{ util::{ asm::write_asm, config::{apply_splits, parse_symbol_line, write_splits, write_symbols}, + dep::DepFile, dol::process_dol, elf::{process_elf, write_elf}, file::{map_file, map_reader, touch}, @@ -60,25 +62,45 @@ pub struct InfoArgs { #[argp(subcommand, name = "split")] pub struct SplitArgs { #[argp(positional)] - /// input file - in_file: PathBuf, + /// input configuration file + config: PathBuf, #[argp(positional)] /// output directory out_dir: PathBuf, - #[argp(option, short = 's')] - /// path to symbols file - symbols_file: Option, - #[argp(option, short = 'p')] - /// path to splits file - splits_file: Option, - #[argp(option, short = 'e')] - /// ELF file to validate against (debugging only) - elf_file: Option, #[argp(switch)] /// skip updating splits & symbol files (for build systems) no_update: bool, } +#[inline] +fn bool_true() -> bool { true } + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ProjectConfig { + pub object: PathBuf, + pub splits: Option, + pub symbols: Option, + // Analysis options + #[serde(default = "bool_true")] + pub detect_objects: bool, + #[serde(default = "bool_true")] + pub detect_strings: bool, + #[serde(default = "bool_true")] + pub write_asm: bool, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct OutputUnit { + pub object: PathBuf, + pub name: String, + pub autogenerated: bool, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +pub struct OutputConfig { + pub units: Vec, +} + pub fn run(args: Args) -> Result<()> { match args.command { SubCommand::Info(c_args) => info(c_args), @@ -128,10 +150,20 @@ fn info(args: InfoArgs) -> Result<()> { } fn split(args: SplitArgs) -> Result<()> { - log::info!("Loading {}", args.in_file.display()); - let mut obj = process_dol(&args.in_file)?; + log::info!("Loading {}", args.config.display()); + let mut config_file = File::open(&args.config) + .with_context(|| format!("Failed to open config file '{}'", args.config.display()))?; + let config: ProjectConfig = serde_yaml::from_reader(&mut config_file)?; - if let Some(splits_path) = &args.splits_file { + let out_config_path = args.out_dir.join("config.json"); + let mut dep = DepFile::new(out_config_path.clone()); + + log::info!("Loading {}", config.object.display()); + let mut obj = process_dol(&config.object)?; + dep.push(config.object.clone()); + + if let Some(splits_path) = &config.splits { + dep.push(splits_path.clone()); if splits_path.is_file() { let map = map_file(splits_path)?; apply_splits(map_reader(&map), &mut obj)?; @@ -140,7 +172,8 @@ fn split(args: SplitArgs) -> Result<()> { let mut state = AnalyzerState::default(); - if let Some(symbols_path) = &args.symbols_file { + if let Some(symbols_path) = &config.symbols { + dep.push(symbols_path.clone()); if symbols_path.is_file() { let map = map_file(symbols_path)?; for result in map_reader(&map).lines() { @@ -176,14 +209,21 @@ fn split(args: SplitArgs) -> Result<()> { log::info!("Applying relocations"); tracker.apply(&mut obj, false)?; - log::info!("Detecting object boundaries"); - detect_object_boundaries(&mut obj)?; + if config.detect_objects { + log::info!("Detecting object boundaries"); + detect_object_boundaries(&mut obj)?; + } - log::info!("Detecting strings"); - detect_strings(&mut obj)?; + if config.detect_strings { + log::info!("Detecting strings"); + detect_strings(&mut obj)?; + } + + log::info!("Adjusting splits"); + update_splits(&mut obj)?; if !args.no_update { - if let Some(symbols_path) = &args.symbols_file { + if let Some(symbols_path) = &config.symbols { let mut symbols_writer = BufWriter::new( File::create(symbols_path) .with_context(|| format!("Failed to create '{}'", symbols_path.display()))?, @@ -191,7 +231,7 @@ fn split(args: SplitArgs) -> Result<()> { write_symbols(&mut symbols_writer, &obj)?; } - if let Some(splits_path) = &args.splits_file { + if let Some(splits_path) = &config.splits { let mut splits_writer = BufWriter::new( File::create(splits_path) .with_context(|| format!("Failed to create '{}'", splits_path.display()))?, @@ -200,9 +240,6 @@ fn split(args: SplitArgs) -> Result<()> { } } - log::info!("Adjusting splits"); - update_splits(&mut obj)?; - log::info!("Splitting {} objects", obj.link_order.len()); let split_objs = split_obj(&obj)?; @@ -224,13 +261,17 @@ fn split(args: SplitArgs) -> Result<()> { }; } - let mut units_file = BufWriter::new(File::create(args.out_dir.join("units.txt"))?); + let mut out_config = OutputConfig::default(); for unit in &obj.link_order { let object = file_map .get(unit) .ok_or_else(|| anyhow!("Failed to find object file for unit '{unit}'"))?; let out_path = obj_dir.join(obj_path_for_unit(unit)); - writeln!(units_file, "{}:{}", out_path.display(), unit)?; + out_config.units.push(OutputUnit { + object: out_path.clone(), + name: unit.clone(), + autogenerated: false, + }); if let Some(parent) = out_path.parent() { DirBuilder::new().recursive(true).create(parent)?; } @@ -239,7 +280,11 @@ fn split(args: SplitArgs) -> Result<()> { file.write_all(object)?; file.flush()?; } - units_file.flush()?; + { + let mut out_file = BufWriter::new(File::create(&out_config_path)?); + serde_json::to_writer_pretty(&mut out_file, &out_config)?; + out_file.flush()?; + } // Generate ldscript.lcf fs::write(args.out_dir.join("ldscript.lcf"), generate_ldscript(&obj)?)?; @@ -256,14 +301,26 @@ fn split(args: SplitArgs) -> Result<()> { w.flush()?; } - // (debugging) validate against ELF - if let Some(file) = &args.elf_file { - validate(&obj, file, &state)?; + // Write dep file + { + let dep_path = args.out_dir.join("dep"); + let mut dep_file = BufWriter::new( + File::create(&dep_path) + .with_context(|| format!("Failed to create dep file '{}'", dep_path.display()))?, + ); + dep.write(&mut dep_file)?; + dep_file.flush()?; } + // (debugging) validate against ELF + // if let Some(file) = &args.elf_file { + // validate(&obj, file, &state)?; + // } + Ok(()) } +#[allow(dead_code)] fn validate>(obj: &ObjInfo, elf_file: P, state: &AnalyzerState) -> Result<()> { let real_obj = process_elf(elf_file)?; for real_section in &real_obj.sections { diff --git a/src/obj/mod.rs b/src/obj/mod.rs index 9ba696a..9e834ca 100644 --- a/src/obj/mod.rs +++ b/src/obj/mod.rs @@ -764,6 +764,12 @@ impl ObjInfo { self.splits.entry(address).or_default().push(split); Ok(()) } + + pub fn is_unit_autogenerated(&self, unit: &str) -> bool { + self.splits_for_range(..) + .filter(|(_, split)| split.unit == unit) + .all(|(_, split)| split.autogenerated) + } } impl ObjSection { diff --git a/src/obj/split.rs b/src/obj/split.rs index 7c403c1..6f069d7 100644 --- a/src/obj/split.rs +++ b/src/obj/split.rs @@ -402,8 +402,7 @@ fn resolve_link_order(obj: &ObjInfo) -> Result> { unit_to_index_map.insert(split.unit.clone(), NodeIndex::new(0)); } for (unit, index) in unit_to_index_map.iter_mut() { - let new_index = graph.add_node(unit.clone()); - *index = new_index; + *index = graph.add_node(unit.clone()); } for section in &obj.sections { diff --git a/src/util/config.rs b/src/util/config.rs index ffe0b23..9ac40f0 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -244,7 +244,14 @@ fn symbol_data_kind_from_str(s: &str) -> Option { } pub fn write_splits(w: &mut W, obj: &ObjInfo) -> Result<()> { - for unit in &obj.link_order { + let mut begin = true; + for unit in obj.link_order.iter().filter(|unit| !obj.is_unit_autogenerated(unit)) { + if begin { + begin = false; + } else { + writeln!(w)?; + } + writeln!(w, "{}:", unit)?; let mut split_iter = obj.splits_for_range(..).peekable(); while let Some((addr, split)) = split_iter.next() { @@ -260,7 +267,6 @@ pub fn write_splits(w: &mut W, obj: &ObjInfo) -> Result<()> { writeln!(w, "\t{:<11} start:{:#010X} end:{:#010X}", section.name, addr, end)?; // align:{} } - writeln!(w)?; } Ok(()) } diff --git a/src/util/dep.rs b/src/util/dep.rs new file mode 100644 index 0000000..51e9fe5 --- /dev/null +++ b/src/util/dep.rs @@ -0,0 +1,20 @@ +use std::{io::Write, path::PathBuf}; + +pub struct DepFile { + pub name: PathBuf, + pub dependencies: Vec, +} + +impl DepFile { + pub fn new(name: PathBuf) -> Self { Self { name, dependencies: vec![] } } + + pub fn push(&mut self, dependency: PathBuf) { self.dependencies.push(dependency); } + + pub fn write(&self, w: &mut W) -> std::io::Result<()> { + write!(w, "{}:", self.name.display())?; + for dep in &self.dependencies { + write!(w, " \\\n {}", dep.display())?; + } + Ok(()) + } +} diff --git a/src/util/mod.rs b/src/util/mod.rs index 5b4296d..1d40707 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,6 +1,7 @@ pub mod asm; pub mod comment; pub mod config; +pub mod dep; pub mod dol; pub mod dwarf; pub mod elf;