encounter
/
decomp-toolkit
mirror of https://github.com/encounter/decomp-toolkit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Updates for build system integration 

- Use a config file as input to `dol split`
- Add depfile output
- Adjust splits config writing
This commit is contained in:
Luke Street 2023-08-06 15:46:51 -04:00
parent 4b6e317d0b
commit a046bf9a66
9 changed files with 141 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Cargo.lock generated
View File

@ -214,7 +214,7 @@ dependencies = [
[[package]]
name = "decomp-toolkit"
version = "0.3.0"
version = "0.3.2"
dependencies = [
"anyhow",
"ar",
@ -245,6 +245,7 @@ dependencies = [
"regex",
"rmp-serde",
"serde",
"serde_json",
"serde_repr",
"serde_yaml",
"sha-1",
@ -773,6 +774,17 @@ dependencies = [
"syn 2.0.23",
]
[[package]]
name = "serde_json"
version = "1.0.104"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c"
dependencies = [
"itoa",
"ryu",
"serde",
]
[[package]]
name = "serde_repr"
version = "0.1.14"

3
Cargo.toml
View File

@ -3,7 +3,7 @@ name = "decomp-toolkit"
description = "Yet another GameCube/Wii decompilation toolkit."
authors = ["Luke Street <luke@street.dev>"]
license = "MIT OR Apache-2.0"
version = "0.3.1"
version = "0.3.2"
edition = "2021"
publish = false
build = "build.rs"
@ -49,6 +49,7 @@ petgraph = "0.6.3"
ppc750cl = { git = "https://github.com/encounter/ppc750cl", rev = "5f6e991bf495388c4104f188d2e90c79da9f78de" }
regex = "1.9.0"
serde = "1.0.166"
serde_json = "1.0.104"
serde_repr = "0.1.14"
serde_yaml = "0.9.22"
sha-1 = "0.10.1"

4
README.md
View File

@ -269,9 +269,11 @@ $ dtk dol info input.dol
Analyzes and splits a DOL file into relocatable objects based on user configuration.
```shell
$ dtk dol split input.dol target -s config/symbols.txt -p config/splits.txt
$ dtk dol split config.yml target
```
TODO: document configuration file
### dwarf dump
Dumps DWARF 1.1 information from an ELF file. (Does **not** support DWARF 2+)

109
src/cmd/dol.rs
View File

@ -8,6 +8,7 @@ use std::{
use anyhow::{anyhow, bail, Context, Result};
use argp::FromArgs;
use serde::{Deserialize, Serialize};
use crate::{
analysis::{
@ -24,6 +25,7 @@ use crate::{
util::{
asm::write_asm,
config::{apply_splits, parse_symbol_line, write_splits, write_symbols},
dep::DepFile,
dol::process_dol,
elf::{process_elf, write_elf},
file::{map_file, map_reader, touch},
@ -60,25 +62,45 @@ pub struct InfoArgs {
#[argp(subcommand, name = "split")]
pub struct SplitArgs {
#[argp(positional)]
/// input file
in_file: PathBuf,
/// input configuration file
config: PathBuf,
#[argp(positional)]
/// output directory
out_dir: PathBuf,
#[argp(option, short = 's')]
/// path to symbols file
symbols_file: Option<PathBuf>,
#[argp(option, short = 'p')]
/// path to splits file
splits_file: Option<PathBuf>,
#[argp(option, short = 'e')]
/// ELF file to validate against (debugging only)
elf_file: Option<PathBuf>,
#[argp(switch)]
/// skip updating splits & symbol files (for build systems)
no_update: bool,
}
#[inline]
fn bool_true() -> bool { true }
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct ProjectConfig {
pub object: PathBuf,
pub splits: Option<PathBuf>,
pub symbols: Option<PathBuf>,
// Analysis options
#[serde(default = "bool_true")]
pub detect_objects: bool,
#[serde(default = "bool_true")]
pub detect_strings: bool,
#[serde(default = "bool_true")]
pub write_asm: bool,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct OutputUnit {
pub object: PathBuf,
pub name: String,
pub autogenerated: bool,
}
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
pub struct OutputConfig {
pub units: Vec<OutputUnit>,
}
pub fn run(args: Args) -> Result<()> {
match args.command {
SubCommand::Info(c_args) => info(c_args),
@ -128,10 +150,20 @@ fn info(args: InfoArgs) -> Result<()> {
}
fn split(args: SplitArgs) -> Result<()> {
log::info!("Loading {}", args.in_file.display());
let mut obj = process_dol(&args.in_file)?;
log::info!("Loading {}", args.config.display());
let mut config_file = File::open(&args.config)
.with_context(|| format!("Failed to open config file '{}'", args.config.display()))?;
let config: ProjectConfig = serde_yaml::from_reader(&mut config_file)?;
if let Some(splits_path) = &args.splits_file {
let out_config_path = args.out_dir.join("config.json");
let mut dep = DepFile::new(out_config_path.clone());
log::info!("Loading {}", config.object.display());
let mut obj = process_dol(&config.object)?;
dep.push(config.object.clone());
if let Some(splits_path) = &config.splits {
dep.push(splits_path.clone());
if splits_path.is_file() {
let map = map_file(splits_path)?;
apply_splits(map_reader(&map), &mut obj)?;
@ -140,7 +172,8 @@ fn split(args: SplitArgs) -> Result<()> {
let mut state = AnalyzerState::default();
if let Some(symbols_path) = &args.symbols_file {
if let Some(symbols_path) = &config.symbols {
dep.push(symbols_path.clone());
if symbols_path.is_file() {
let map = map_file(symbols_path)?;
for result in map_reader(&map).lines() {
@ -176,14 +209,21 @@ fn split(args: SplitArgs) -> Result<()> {
log::info!("Applying relocations");
tracker.apply(&mut obj, false)?;
if config.detect_objects {
log::info!("Detecting object boundaries");
detect_object_boundaries(&mut obj)?;
}
if config.detect_strings {
log::info!("Detecting strings");
detect_strings(&mut obj)?;
}
log::info!("Adjusting splits");
update_splits(&mut obj)?;
if !args.no_update {
if let Some(symbols_path) = &args.symbols_file {
if let Some(symbols_path) = &config.symbols {
let mut symbols_writer = BufWriter::new(
File::create(symbols_path)
.with_context(|| format!("Failed to create '{}'", symbols_path.display()))?,
@ -191,7 +231,7 @@ fn split(args: SplitArgs) -> Result<()> {
write_symbols(&mut symbols_writer, &obj)?;
}
if let Some(splits_path) = &args.splits_file {
if let Some(splits_path) = &config.splits {
let mut splits_writer = BufWriter::new(
File::create(splits_path)
.with_context(|| format!("Failed to create '{}'", splits_path.display()))?,
@ -200,9 +240,6 @@ fn split(args: SplitArgs) -> Result<()> {
}
}
log::info!("Adjusting splits");
update_splits(&mut obj)?;
log::info!("Splitting {} objects", obj.link_order.len());
let split_objs = split_obj(&obj)?;
@ -224,13 +261,17 @@ fn split(args: SplitArgs) -> Result<()> {
};
}
let mut units_file = BufWriter::new(File::create(args.out_dir.join("units.txt"))?);
let mut out_config = OutputConfig::default();
for unit in &obj.link_order {
let object = file_map
.get(unit)
.ok_or_else(|| anyhow!("Failed to find object file for unit '{unit}'"))?;
let out_path = obj_dir.join(obj_path_for_unit(unit));
writeln!(units_file, "{}:{}", out_path.display(), unit)?;
out_config.units.push(OutputUnit {
object: out_path.clone(),
name: unit.clone(),
autogenerated: false,
});
if let Some(parent) = out_path.parent() {
DirBuilder::new().recursive(true).create(parent)?;
}
@ -239,7 +280,11 @@ fn split(args: SplitArgs) -> Result<()> {
file.write_all(object)?;
file.flush()?;
}
units_file.flush()?;
{
let mut out_file = BufWriter::new(File::create(&out_config_path)?);
serde_json::to_writer_pretty(&mut out_file, &out_config)?;
out_file.flush()?;
}
// Generate ldscript.lcf
fs::write(args.out_dir.join("ldscript.lcf"), generate_ldscript(&obj)?)?;
@ -256,14 +301,26 @@ fn split(args: SplitArgs) -> Result<()> {
w.flush()?;
}
// (debugging) validate against ELF
if let Some(file) = &args.elf_file {
validate(&obj, file, &state)?;
// Write dep file
{
let dep_path = args.out_dir.join("dep");
let mut dep_file = BufWriter::new(
File::create(&dep_path)
.with_context(|| format!("Failed to create dep file '{}'", dep_path.display()))?,
);
dep.write(&mut dep_file)?;
dep_file.flush()?;
}
// (debugging) validate against ELF
// if let Some(file) = &args.elf_file {
// validate(&obj, file, &state)?;
// }
Ok(())
}
#[allow(dead_code)]
fn validate<P: AsRef<Path>>(obj: &ObjInfo, elf_file: P, state: &AnalyzerState) -> Result<()> {
let real_obj = process_elf(elf_file)?;
for real_section in &real_obj.sections {

6
src/obj/mod.rs
View File

@ -764,6 +764,12 @@ impl ObjInfo {
self.splits.entry(address).or_default().push(split);
Ok(())
}
pub fn is_unit_autogenerated(&self, unit: &str) -> bool {
self.splits_for_range(..)
.filter(|(_, split)| split.unit == unit)
.all(|(_, split)| split.autogenerated)
}
}
impl ObjSection {

3
src/obj/split.rs
View File

@ -402,8 +402,7 @@ fn resolve_link_order(obj: &ObjInfo) -> Result<Vec<String>> {
unit_to_index_map.insert(split.unit.clone(), NodeIndex::new(0));
}
for (unit, index) in unit_to_index_map.iter_mut() {
let new_index = graph.add_node(unit.clone());
*index = new_index;
*index = graph.add_node(unit.clone());
}
for section in &obj.sections {

10
src/util/config.rs
View File

@ -244,7 +244,14 @@ fn symbol_data_kind_from_str(s: &str) -> Option<ObjDataKind> {
}
pub fn write_splits<W: Write>(w: &mut W, obj: &ObjInfo) -> Result<()> {
for unit in &obj.link_order {
let mut begin = true;
for unit in obj.link_order.iter().filter(|unit| !obj.is_unit_autogenerated(unit)) {
if begin {
begin = false;
} else {
writeln!(w)?;
}
writeln!(w, "{}:", unit)?;
let mut split_iter = obj.splits_for_range(..).peekable();
while let Some((addr, split)) = split_iter.next() {
@ -260,7 +267,6 @@ pub fn write_splits<W: Write>(w: &mut W, obj: &ObjInfo) -> Result<()> {
writeln!(w, "\t{:<11} start:{:#010X} end:{:#010X}", section.name, addr, end)?;
// align:{}
}
writeln!(w)?;
}
Ok(())
}

20
src/util/dep.rs Normal file
View File

@ -0,0 +1,20 @@
use std::{io::Write, path::PathBuf};
pub struct DepFile {
pub name: PathBuf,
pub dependencies: Vec<PathBuf>,
}
impl DepFile {
pub fn new(name: PathBuf) -> Self { Self { name, dependencies: vec![] } }
pub fn push(&mut self, dependency: PathBuf) { self.dependencies.push(dependency); }
pub fn write<W: Write>(&self, w: &mut W) -> std::io::Result<()> {
write!(w, "{}:", self.name.display())?;
for dep in &self.dependencies {
write!(w, " \\\n {}", dep.display())?;
}
Ok(())
}
}

1
src/util/mod.rs
View File

@ -1,6 +1,7 @@
pub mod asm;
pub mod comment;
pub mod config;
pub mod dep;
pub mod dol;
pub mod dwarf;
pub mod elf;