Smarter configuration updates

- Avoid overwriting `symbols.txt` or `splits.txt` if the file was modified since it was read or if the file's contents didn't change.
- Remove `binrw` and `byteorder` dependencies, moving to `FromReader`/`ToWriter` traits.
- Migrate generic bounds to `where` clauses.
- Remove unused `build.rs` logic.
This commit is contained in:
Luke Street 2023-11-07 23:21:59 -05:00
parent c452b74666
commit ec4caf5000
29 changed files with 1817 additions and 1334 deletions

76
Cargo.lock generated
View File

@ -112,12 +112,6 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "array-init"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc"
[[package]]
name = "atty"
version = "0.2.14"
@ -171,30 +165,6 @@ dependencies = [
"serde",
]
[[package]]
name = "binrw"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e8318fda24dc135cdd838f57a2b5ccb6e8f04ff6b6c65528c4bd9b5fcdc5cf6"
dependencies = [
"array-init",
"binrw_derive",
"bytemuck",
]
[[package]]
name = "binrw_derive"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db0832bed83248115532dfb25af54fae1c83d67a2e4e3e2f591c13062e372e7e"
dependencies = [
"either",
"owo-colors",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "bitflags"
version = "1.3.2"
@ -216,12 +186,6 @@ dependencies = [
"generic-array",
]
[[package]]
name = "bytemuck"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6"
[[package]]
name = "byteorder"
version = "1.5.0"
@ -330,15 +294,13 @@ dependencies = [
[[package]]
name = "decomp-toolkit"
version = "0.5.7"
version = "0.5.8"
dependencies = [
"anyhow",
"ar",
"argp",
"base16ct",
"base64",
"binrw",
"byteorder",
"cwdemangle",
"dol",
"enable-ansi-support",
@ -364,7 +326,6 @@ dependencies = [
"ppc750cl",
"rayon",
"regex",
"rmp-serde",
"rustc-hash",
"serde",
"serde_json",
@ -376,6 +337,7 @@ dependencies = [
"tracing",
"tracing-attributes",
"tracing-subscriber",
"xxhash-rust",
]
[[package]]
@ -745,12 +707,6 @@ dependencies = [
"supports-color 1.3.1",
]
[[package]]
name = "paste"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c"
[[package]]
name = "path-slash"
version = "0.2.1"
@ -896,28 +852,6 @@ version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
[[package]]
name = "rmp"
version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44519172358fd6d58656c86ab8e7fbc9e1490c3e8f14d35ed78ca0dd07403c9f"
dependencies = [
"byteorder",
"num-traits",
"paste",
]
[[package]]
name = "rmp-serde"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bffea85eea980d8a74453e5d02a8d93028f3c34725de143085a844ebe953258a"
dependencies = [
"byteorder",
"rmp",
"serde",
]
[[package]]
name = "rustc-demangle"
version = "0.1.23"
@ -1389,3 +1323,9 @@ name = "windows_x86_64_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
[[package]]
name = "xxhash-rust"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9828b178da53440fa9c766a3d2f73f7cf5d0ac1fe3980c1e5018d899fd19e07b"

View File

@ -3,10 +3,9 @@ name = "decomp-toolkit"
description = "Yet another GameCube/Wii decompilation toolkit."
authors = ["Luke Street <luke@street.dev>"]
license = "MIT OR Apache-2.0"
version = "0.5.7"
version = "0.5.8"
edition = "2021"
publish = false
build = "build.rs"
repository = "https://github.com/encounter/decomp-toolkit"
readme = "README.md"
categories = ["command-line-utilities"]
@ -27,8 +26,6 @@ ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "write_symbol_tab
argp = "0.3.0"
base16ct = "0.2.0"
base64 = "0.21.4"
binrw = "0.12.0"
byteorder = "1.5.0"
cwdemangle = "0.1.6"
dol = { git = "https://github.com/encounter/ppc750cl", rev = "4a2bbbc6f84dcb76255ab6f3595a8d4a0ce96618" }
enable-ansi-support = "0.2.1"
@ -50,7 +47,7 @@ object = { version = "0.32.1", features = ["read_core", "std", "elf", "write_std
once_cell = "1.18.0"
owo-colors = { version = "3.5.0", features = ["supports-colors"] }
path-slash = "0.2.1"
petgraph = "0.6.4"
petgraph = { version = "0.6.4", default-features = false }
ppc750cl = { git = "https://github.com/encounter/ppc750cl", rev = "4a2bbbc6f84dcb76255ab6f3595a8d4a0ce96618" }
rayon = "1.8.0"
regex = "1.9.6"
@ -65,13 +62,4 @@ supports-color = "2.1.0"
tracing = "0.1.37"
tracing-attributes = "0.1.26"
tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
[build-dependencies]
anyhow = { version = "1.0.75", features = ["backtrace"] }
base64 = "0.21.4"
flagset = { version = "0.4.4", features = ["serde"] }
serde = "1.0.188"
serde_repr = "0.1.16"
serde_yaml = "0.9.25"
rmp-serde = "1.1.2"
flate2 = "1.0.27"
xxhash-rust = { version = "0.8.7", features = ["xxh3"] }

470
build.rs
View File

@ -1,467 +1,9 @@
use std::{collections::HashMap, env, fs, path::PathBuf};
use anyhow::{anyhow, bail, Context, Result};
use base64::{engine::general_purpose::STANDARD, Engine};
use flagset::{flags, FlagSet};
use flate2::{write::GzEncoder, Compression};
use serde::{Deserialize, Serialize};
use serde_repr::{Deserialize_repr, Serialize_repr};
flags! {
#[repr(u8)]
#[derive(Deserialize_repr, Serialize_repr)]
pub enum ObjSymbolFlags: u8 {
Global,
Local,
Weak,
Common,
Hidden,
}
}
#[derive(Debug, Copy, Clone, Default, Eq, PartialEq, Serialize, Deserialize)]
pub struct ObjSymbolFlagSet(pub FlagSet<ObjSymbolFlags>);
#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)]
pub enum ObjSymbolKind {
Unknown,
Function,
Object,
Section,
}
#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)]
pub enum ObjRelocKind {
Absolute,
PpcAddr16Hi,
PpcAddr16Ha,
PpcAddr16Lo,
PpcRel24,
PpcRel14,
PpcEmbSda21,
}
#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
pub struct OutSymbol {
pub kind: ObjSymbolKind,
pub name: String,
pub size: u32,
pub flags: ObjSymbolFlagSet,
pub section: Option<String>,
}
#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
pub struct OutReloc {
pub offset: u32,
pub kind: ObjRelocKind,
pub symbol: usize,
pub addend: i32,
}
#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
pub struct FunctionSignature {
pub symbol: usize,
pub hash: String,
pub signature: String,
pub symbols: Vec<OutSymbol>,
pub relocations: Vec<OutReloc>,
}
#[derive(Debug, Copy, Clone, Eq, PartialEq, Deserialize_repr, Serialize_repr)]
#[repr(u8)]
pub enum SigSymbolKind {
Unknown = 0,
Function = 1,
Object = 2,
}
#[derive(Debug, Copy, Clone, Eq, PartialEq, Deserialize_repr, Serialize_repr)]
#[repr(u8)]
pub enum SigSection {
Init = 0,
Extab = 1,
Extabindex = 2,
Text = 3,
Ctors = 4,
Dtors = 5,
Rodata = 6,
Data = 7,
Bss = 8,
Sdata = 9,
Sbss = 10,
Sdata2 = 11,
Sbss2 = 12,
Dbgtext = 13,
Unknown = 255,
}
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
#[repr(u8)]
pub enum SigSymbolFlag {
Global = 1 << 0,
Local = 1 << 1,
Weak = 1 << 2,
Common = 1 << 3,
Hidden = 1 << 4,
}
#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
pub struct SigSymbol {
pub kind: SigSymbolKind,
pub name: String,
pub size: u32,
pub flags: u8, // SigSymbolFlag
pub section: SigSection,
}
#[derive(Debug, Copy, Clone, Eq, PartialEq, Deserialize_repr, Serialize_repr)]
#[repr(u8)]
pub enum SigRelocKind {
Absolute = 0,
PpcAddr16Hi = 1,
PpcAddr16Ha = 2,
PpcAddr16Lo = 3,
PpcRel24 = 4,
PpcRel14 = 5,
PpcEmbSda21 = 6,
}
#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
pub struct SigReloc {
pub offset: u32,
pub symbol: usize,
pub kind: SigRelocKind,
pub addend: i32,
}
#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
pub struct Sig {
pub symbol: usize,
pub data: Vec<u8>,
pub relocations: Vec<SigReloc>,
pub search: bool,
}
#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
pub struct Output {
pub symbols: Vec<SigSymbol>,
pub signatures: HashMap<String, Sig>,
}
pub fn parse_yml(sig_str: &str) -> Result<Vec<FunctionSignature>> {
Ok(serde_yaml::from_str(sig_str)?)
}
const SIGNATURES: &[(&str, bool)] = &[
("__start", false),
("__init_registers", false),
("__init_hardware", false),
("__init_data", false),
("__set_debug_bba", false),
("__OSPSInit", false),
("__OSFPRInit", false),
("__OSCacheInit", false),
("DMAErrorHandler", false),
("DBInit", false),
("OSInit", false),
("__OSThreadInit", false),
("__OSInitIPCBuffer", false),
("EXIInit", false),
("EXIGetID", false),
("exit", false),
("_ExitProcess", false),
("__fini_cpp", false),
("__fini_cpp_exceptions", false),
("__destroy_global_chain", false),
("__init_cpp", false),
("__init_cpp_exceptions", false),
("InitMetroTRK", false),
("InitMetroTRKCommTable", false),
("OSExceptionInit", false),
("OSDefaultExceptionHandler", false),
("__OSUnhandledException", false),
("OSDisableScheduler", false),
("__OSReschedule", false),
("__OSInitSystemCall", false),
("OSInitAlarm", false),
("__OSInitAlarm", false),
// TODO aliases
// ("__OSEVStart", false),
// ("__OSDBINTSTART", false),
// ("__OSDBJUMPSTART", false),
("SIInit", false),
("SIGetType", false),
("SISetSamplingRate", false),
("SISetXY", false),
("VIGetTvFormat", false),
("DVDInit", false),
("DVDSetAutoFatalMessaging", false),
("OSSetArenaLo", false),
("OSSetArenaHi", false),
("OSSetMEM1ArenaLo", false),
("OSSetMEM1ArenaHi", false),
("OSSetMEM2ArenaLo", false),
("OSSetMEM2ArenaHi", false),
("__OSInitAudioSystem", false),
("__OSInitMemoryProtection", false),
// ("BATConfig", false), TODO
("ReportOSInfo", false),
("__check_pad3", false),
("OSResetSystem", false),
("OSReturnToMenu", false),
("__OSReturnToMenu", false),
("__OSShutdownDevices", false),
("__OSInitSram", false),
("__OSSyncSram", false),
("__OSGetExceptionHandler", false),
("OSRegisterResetFunction", false),
("OSRegisterShutdownFunction", false),
("DecrementerExceptionHandler", false),
("DecrementerExceptionCallback", false),
("__OSInterruptInit", false),
("__OSContextInit", false),
("OSSwitchFPUContext", false),
("OSReport", false),
("TRK_main", false),
("TRKNubWelcome", false),
("TRKInitializeNub", false),
("TRKInitializeIntDrivenUART", false),
("TRKEXICallBack", false),
("TRKLoadContext", false),
("TRKInterruptHandler", false),
("TRKExceptionHandler", false),
("TRKSaveExtended1Block", false),
("TRKNubMainLoop", false),
("TRKTargetContinue", false),
("TRKSwapAndGo", false),
("TRKRestoreExtended1Block", false),
("TRKInterruptHandlerEnableInterrupts", false),
("memset", false),
("__msl_runtime_constraint_violation_s", false),
("ClearArena", false),
("IPCCltInit", false),
("__OSInitSTM", false),
("IOS_Open", false),
("__ios_Ipc2", false),
("IPCiProfQueueReq", false),
("SCInit", false),
("SCReloadConfFileAsync", false),
("NANDPrivateOpenAsync", false),
("nandIsInitialized", false),
("nandOpen", false),
("nandGenerateAbsPath", false),
("nandGetHeadToken", false),
("ISFS_OpenAsync", false),
("nandConvertErrorCode", false),
("NANDLoggingAddMessageAsync", false),
("__NANDPrintErrorMessage", false),
("__OSInitNet", false),
("__DVDCheckDevice", false),
("__OSInitPlayTime", false),
("__OSStartPlayRecord", false),
("NANDInit", false),
("ISFS_OpenLib", false),
("ESP_GetTitleId", false),
("NANDSetAutoErrorMessaging", false),
("__DVDFSInit", false),
("__DVDClearWaitingQueue", false),
("__DVDInitWA", false),
("__DVDLowSetWAType", false),
("__fstLoad", false),
("DVDReset", false),
("DVDLowReset", false),
("DVDReadDiskID", false),
("stateReady", false),
("DVDLowWaitCoverClose", false),
("__DVDStoreErrorCode", false),
("DVDLowStopMotor", false),
("DVDGetDriveStatus", false),
("printf", false),
("sprintf", false),
("vprintf", false),
("vsprintf", false),
("vsnprintf", false),
("__pformatter", false),
("longlong2str", false),
("__mod2u", false),
("__FileWrite", false),
("fwrite", false),
("__fwrite", false),
("__stdio_atexit", false),
("__StringWrite", false),
("RSOStaticLocateObject", true),
];
fn main() -> Result<()> {
let output = std::process::Command::new("git").args(["rev-parse", "HEAD"]).output()?;
let rev = String::from_utf8(output.stdout)?;
fn main() {
let output = std::process::Command::new("git")
.args(["rev-parse", "HEAD"])
.output()
.expect("Failed to execute git");
let rev = String::from_utf8(output.stdout).expect("Failed to parse git output");
println!("cargo:rustc-env=GIT_COMMIT_SHA={rev}");
println!("cargo:rustc-rerun-if-changed=.git/HEAD");
let mut symbols = Vec::<SigSymbol>::new();
let mut out = HashMap::<String, Sig>::new();
let in_dir = PathBuf::from("assets/signatures");
for &(name, search) in SIGNATURES {
let path = in_dir.join(format!("{name}.yml"));
println!("cargo:rustc-rerun-if-changed={}", path.display());
let str = fs::read_to_string(&path)
.with_context(|| format!("Failed to open '{}'", path.display()))?;
apply_sig(&str, &mut symbols, &mut out, search)?;
}
let mut encoder = GzEncoder::new(Vec::new(), Compression::best());
rmp_serde::encode::write(&mut encoder, &Output { symbols, signatures: out })?;
let compressed = encoder.finish()?;
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
fs::write(out_dir.join("signatures.bin"), compressed)?;
Ok(())
}
fn apply_sig(
sig_str: &str,
symbols: &mut Vec<SigSymbol>,
out: &mut HashMap<String, Sig>,
search: bool,
) -> Result<()> {
let data = parse_yml(sig_str)?;
for in_sig in data {
let in_sym = &in_sig.symbols[in_sig.symbol];
let mut out_sig = Sig {
symbol: add_symbol(symbols, in_sym)?,
data: STANDARD.decode(&in_sig.signature)?,
relocations: vec![],
search,
};
for in_reloc in &in_sig.relocations {
out_sig.relocations.push(SigReloc {
offset: in_reloc.offset,
symbol: add_symbol(symbols, &in_sig.symbols[in_reloc.symbol])?,
kind: to_sig_reloc_kind(in_reloc.kind)?,
addend: in_reloc.addend,
});
}
out.insert(in_sym.name.clone(), out_sig);
}
Ok(())
}
fn to_sym_section(s: Option<&str>) -> Result<SigSection> {
match s {
None => Ok(SigSection::Unknown),
Some(".init") => Ok(SigSection::Init),
Some("extab") => Ok(SigSection::Extab),
Some("extabindex") => Ok(SigSection::Extabindex),
Some(".text") => Ok(SigSection::Text),
Some(".ctors") => Ok(SigSection::Ctors),
Some(".dtors") => Ok(SigSection::Dtors),
Some(".rodata") => Ok(SigSection::Rodata),
Some(".data") => Ok(SigSection::Data),
Some(".bss") => Ok(SigSection::Bss),
Some(".sdata") => Ok(SigSection::Sdata),
Some(".sbss") => Ok(SigSection::Sbss),
Some(".sdata2") => Ok(SigSection::Sdata2),
Some(".sbss2") => Ok(SigSection::Sbss2),
Some(".dbgtext") => Ok(SigSection::Dbgtext),
Some(section) => Err(anyhow!("Unknown section {}", section)),
}
}
fn to_sig_symbol_kind(kind: ObjSymbolKind) -> Result<SigSymbolKind> {
match kind {
ObjSymbolKind::Unknown => Ok(SigSymbolKind::Unknown),
ObjSymbolKind::Function => Ok(SigSymbolKind::Function),
ObjSymbolKind::Object => Ok(SigSymbolKind::Object),
ObjSymbolKind::Section => Err(anyhow!("Section symbols unsupported")),
}
}
fn to_sig_symbol_flags(flags: ObjSymbolFlagSet) -> Result<u8> {
let mut out = 0;
for flag in flags.0 {
match flag {
ObjSymbolFlags::Global => {
out |= SigSymbolFlag::Global as u8;
}
ObjSymbolFlags::Local => {
out |= SigSymbolFlag::Local as u8;
}
ObjSymbolFlags::Weak => {
out |= SigSymbolFlag::Weak as u8;
}
ObjSymbolFlags::Common => {
out |= SigSymbolFlag::Common as u8;
}
ObjSymbolFlags::Hidden => {
out |= SigSymbolFlag::Hidden as u8;
}
}
}
Ok(out)
}
fn to_sig_reloc_kind(kind: ObjRelocKind) -> Result<SigRelocKind> {
match kind {
ObjRelocKind::Absolute => Ok(SigRelocKind::Absolute),
ObjRelocKind::PpcAddr16Hi => Ok(SigRelocKind::PpcAddr16Hi),
ObjRelocKind::PpcAddr16Ha => Ok(SigRelocKind::PpcAddr16Ha),
ObjRelocKind::PpcAddr16Lo => Ok(SigRelocKind::PpcAddr16Lo),
ObjRelocKind::PpcRel24 => Ok(SigRelocKind::PpcRel24),
ObjRelocKind::PpcRel14 => Ok(SigRelocKind::PpcRel14),
ObjRelocKind::PpcEmbSda21 => Ok(SigRelocKind::PpcEmbSda21),
}
}
fn add_symbol(symbols: &mut Vec<SigSymbol>, in_sym: &OutSymbol) -> Result<usize> {
let sig_section = to_sym_section(in_sym.section.as_deref())?;
let sig_symbol_kind = to_sig_symbol_kind(in_sym.kind)?;
let sig_symbol_flags = to_sig_symbol_flags(in_sym.flags)?;
if let Some((idx, existing)) = symbols.iter_mut().enumerate().find(|(_, sym)| {
sym.kind == sig_symbol_kind && sym.size == in_sym.size && sym.name == in_sym.name
}) {
if existing.kind != sig_symbol_kind {
bail!(
"Mismatched types for {}: {:?} != {:?}",
in_sym.name,
sig_symbol_kind,
existing.kind
);
}
if existing.section != sig_section {
if existing.section == SigSection::Unknown || sig_section == SigSection::Unknown {
existing.section = SigSection::Unknown;
} else {
eprintln!(
"Mismatched sections for {}: {:?} != {:?}",
in_sym.name, sig_section, existing.section
);
existing.section = SigSection::Unknown;
}
}
if existing.size != in_sym.size {
bail!("Mismatched size for {}: {} != {}", in_sym.name, in_sym.size, existing.size);
}
if existing.flags != sig_symbol_flags {
if (existing.flags & (SigSymbolFlag::Weak as u8) != 0
&& sig_symbol_flags & (SigSymbolFlag::Weak as u8) == 0)
|| (sig_symbol_flags & (SigSymbolFlag::Weak as u8) != 0
&& existing.flags & (SigSymbolFlag::Weak as u8) == 0)
{
existing.flags |= SigSymbolFlag::Weak as u8;
} else {
eprintln!(
"Mismatched flags for {}: {} != {}",
in_sym.name, sig_symbol_flags, existing.flags
);
}
}
return Ok(idx);
}
let idx = symbols.len();
symbols.push(SigSymbol {
kind: sig_symbol_kind,
name: in_sym.name.clone(),
size: in_sym.size,
flags: sig_symbol_flags,
section: sig_section,
});
Ok(idx)
}

View File

@ -6,8 +6,11 @@ use std::ffi::OsStr;
use argp::{parser::ParseGlobalOptions, EarlyExit, FromArgs, TopLevelCommand};
struct ArgsOrVersion<T: FromArgs>(T);
struct ArgsOrVersion<T>(T)
where T: FromArgs;
impl<T> TopLevelCommand for ArgsOrVersion<T> where T: FromArgs {}
impl<T> FromArgs for ArgsOrVersion<T>
where T: FromArgs
{

View File

@ -44,7 +44,7 @@ use crate::{
dep::DepFile,
dol::process_dol,
elf::{process_elf, write_elf},
file::{buf_reader, buf_writer, map_file, touch, verify_hash, FileIterator},
file::{buf_reader, buf_writer, map_file, touch, verify_hash, FileIterator, FileReadInfo},
lcf::{asm_path_for_unit, generate_ldscript, obj_path_for_unit},
map::apply_map_file,
rel::{process_rel, process_rel_header, update_rel_section_alignment},
@ -145,7 +145,10 @@ fn bool_true() -> bool { true }
fn is_true(b: &bool) -> bool { *b }
#[inline]
fn is_default<T: Default + PartialEq>(t: &T) -> bool { t == &T::default() }
fn is_default<T>(t: &T) -> bool
where T: Default + PartialEq {
t == &T::default()
}
mod path_slash_serde {
use std::path::PathBuf;
@ -453,7 +456,14 @@ pub fn info(args: InfoArgs) -> Result<()> {
Ok(())
}
type ModuleMap<'a> = BTreeMap<u32, (&'a ModuleConfig, ObjInfo)>;
struct ModuleInfo<'a> {
obj: ObjInfo,
config: &'a ModuleConfig,
symbols_cache: Option<FileReadInfo>,
splits_cache: Option<FileReadInfo>,
}
type ModuleMap<'a> = BTreeMap<u32, ModuleInfo<'a>>;
fn update_symbols(obj: &mut ObjInfo, modules: &ModuleMap<'_>, create_symbols: bool) -> Result<()> {
log::debug!("Updating symbols for module {}", obj.module_id);
@ -463,8 +473,8 @@ fn update_symbols(obj: &mut ObjInfo, modules: &ModuleMap<'_>, create_symbols: bo
.unresolved_relocations
.iter()
.map(|r| (obj.module_id, r))
.chain(modules.iter().flat_map(|(_, (_, obj))| {
obj.unresolved_relocations.iter().map(|r| (obj.module_id, r))
.chain(modules.iter().flat_map(|(_, info)| {
info.obj.unresolved_relocations.iter().map(|r| (info.obj.module_id, r))
}))
.filter(|(_, r)| r.module_id == obj.module_id)
{
@ -565,7 +575,7 @@ fn create_relocations(obj: &mut ObjInfo, modules: &ModuleMap<'_>, dol_obj: &ObjI
&modules
.get(&rel_reloc.module_id)
.ok_or_else(|| anyhow!("Failed to locate module {}", rel_reloc.module_id))?
.1
.obj
};
let (target_section_index, _target_section) = if rel_reloc.module_id == 0 {
@ -646,7 +656,7 @@ fn resolve_external_relocations(
.ok_or_else(|| {
anyhow!("Failed to locate module {}", reloc.module.unwrap())
})?
.1
.obj
};
let target_symbol = &target_obj.symbols[reloc.target_symbol];
@ -670,7 +680,12 @@ fn resolve_external_relocations(
Ok(())
}
type AnalyzeResult = (ObjInfo, Vec<PathBuf>);
struct AnalyzeResult {
obj: ObjInfo,
dep: Vec<PathBuf>,
symbols_cache: Option<FileReadInfo>,
splits_cache: Option<FileReadInfo>,
}
fn load_analyze_dol(config: &ProjectConfig) -> Result<AnalyzeResult> {
log::debug!("Loading {}", config.base.object.display());
@ -692,15 +707,19 @@ fn load_analyze_dol(config: &ProjectConfig) -> Result<AnalyzeResult> {
dep.push(map_path.clone());
}
if let Some(splits_path) = &config.base.splits {
apply_splits_file(splits_path, &mut obj)?;
let splits_cache = if let Some(splits_path) = &config.base.splits {
dep.push(splits_path.clone());
}
apply_splits_file(splits_path, &mut obj)?
} else {
None
};
if let Some(symbols_path) = &config.base.symbols {
apply_symbols_file(symbols_path, &mut obj)?;
let symbols_cache = if let Some(symbols_path) = &config.base.symbols {
dep.push(symbols_path.clone());
}
apply_symbols_file(symbols_path, &mut obj)?
} else {
None
};
if !config.symbols_known {
// TODO move before symbols?
@ -732,72 +751,73 @@ fn load_analyze_dol(config: &ProjectConfig) -> Result<AnalyzeResult> {
// Create _ctors and _dtors symbols if missing
update_ctors_dtors(&mut obj)?;
Ok((obj, dep))
Ok(AnalyzeResult { obj, dep, symbols_cache, splits_cache })
}
fn split_write_obj(
obj: &mut ObjInfo,
module: &mut ModuleInfo,
config: &ProjectConfig,
module_config: &ModuleConfig,
out_dir: &Path,
no_update: bool,
) -> Result<OutputModule> {
debug!("Performing relocation analysis");
let mut tracker = Tracker::new(obj);
tracker.process(obj)?;
let mut tracker = Tracker::new(&module.obj);
tracker.process(&module.obj)?;
debug!("Applying relocations");
tracker.apply(obj, false)?;
tracker.apply(&mut module.obj, false)?;
if !config.symbols_known && config.detect_objects {
debug!("Detecting object boundaries");
detect_objects(obj)?;
detect_objects(&mut module.obj)?;
}
if config.detect_strings {
debug!("Detecting strings");
detect_strings(obj)?;
detect_strings(&mut module.obj)?;
}
debug!("Adjusting splits");
let module_id = module.obj.module_id;
update_splits(
obj,
if obj.module_id == 0 { config.common_start } else { None },
&mut module.obj,
if module_id == 0 { config.common_start } else { None },
config.fill_gaps,
)?;
if !no_update {
debug!("Writing configuration");
if let Some(symbols_path) = &module_config.symbols {
write_symbols_file(symbols_path, obj)?;
if let Some(symbols_path) = &module.config.symbols {
write_symbols_file(symbols_path, &module.obj, module.symbols_cache)?;
}
if let Some(splits_path) = &module_config.splits {
write_splits_file(splits_path, obj, false)?;
if let Some(splits_path) = &module.config.splits {
write_splits_file(splits_path, &module.obj, false, module.splits_cache)?;
}
}
debug!("Splitting {} objects", obj.link_order.len());
let split_objs = split_obj(obj)?;
debug!("Splitting {} objects", module.obj.link_order.len());
let split_objs = split_obj(&module.obj)?;
debug!("Writing object files");
let obj_dir = out_dir.join("obj");
let entry = if obj.kind == ObjKind::Executable {
obj.entry.and_then(|e| {
let (section_index, _) = obj.sections.at_address(e as u32).ok()?;
let symbols = obj.symbols.at_section_address(section_index, e as u32).collect_vec();
let entry = if module.obj.kind == ObjKind::Executable {
module.obj.entry.and_then(|e| {
let (section_index, _) = module.obj.sections.at_address(e as u32).ok()?;
let symbols =
module.obj.symbols.at_section_address(section_index, e as u32).collect_vec();
best_match_for_reloc(symbols, ObjRelocKind::PpcRel24).map(|(_, s)| s.name.clone())
})
} else {
obj.symbols.by_name("_prolog")?.map(|(_, s)| s.name.clone())
module.obj.symbols.by_name("_prolog")?.map(|(_, s)| s.name.clone())
};
let mut out_config = OutputModule {
name: module_config.name().to_string(),
module_id: obj.module_id,
name: module.config.name().to_string(),
module_id,
ldscript: out_dir.join("ldscript.lcf"),
units: Vec::with_capacity(split_objs.len()),
entry,
};
for (unit, split_obj) in obj.link_order.iter().zip(&split_objs) {
for (unit, split_obj) in module.obj.link_order.iter().zip(&split_objs) {
let out_obj = write_elf(split_obj)?;
let out_path = obj_dir.join(obj_path_for_unit(&unit.name));
out_config.units.push(OutputUnit {
@ -815,7 +835,7 @@ fn split_write_obj(
}
// Generate ldscript.lcf
let ldscript_template = if let Some(template) = &module_config.ldscript_template {
let ldscript_template = if let Some(template) = &module.config.ldscript_template {
Some(fs::read_to_string(template).with_context(|| {
format!("Failed to read linker script template '{}'", template.display())
})?)
@ -824,13 +844,13 @@ fn split_write_obj(
};
fs::write(
&out_config.ldscript,
generate_ldscript(obj, ldscript_template.as_deref(), &module_config.force_active)?,
generate_ldscript(&module.obj, ldscript_template.as_deref(), &module.config.force_active)?,
)?;
if config.write_asm {
debug!("Writing disassembly");
let asm_dir = out_dir.join("asm");
for (unit, split_obj) in obj.link_order.iter().zip(&split_objs) {
for (unit, split_obj) in module.obj.link_order.iter().zip(&split_objs) {
let out_path = asm_dir.join(asm_path_for_unit(&unit.name));
let mut w = buf_writer(&out_path)?;
@ -861,15 +881,19 @@ fn load_analyze_rel(config: &ProjectConfig, module_config: &ModuleConfig) -> Res
dep.push(map_path.clone());
}
if let Some(splits_path) = &module_config.splits {
apply_splits_file(splits_path, &mut module_obj)?;
let splits_cache = if let Some(splits_path) = &module_config.splits {
dep.push(splits_path.clone());
}
apply_splits_file(splits_path, &mut module_obj)?
} else {
None
};
if let Some(symbols_path) = &module_config.symbols {
apply_symbols_file(symbols_path, &mut module_obj)?;
let symbols_cache = if let Some(symbols_path) = &module_config.symbols {
dep.push(symbols_path.clone());
}
apply_symbols_file(symbols_path, &mut module_obj)?
} else {
None
};
if !config.symbols_known {
debug!("Analyzing module {}", module_obj.module_id);
@ -890,7 +914,7 @@ fn load_analyze_rel(config: &ProjectConfig, module_config: &ModuleConfig) -> Res
// Determine REL section alignment
update_rel_section_alignment(&mut module_obj, &header)?;
Ok((module_obj, dep))
Ok(AnalyzeResult { obj: module_obj, dep, symbols_cache, splits_cache })
}
fn split(args: SplitArgs) -> Result<()> {
@ -955,17 +979,30 @@ fn split(args: SplitArgs) -> Result<()> {
});
});
let duration = start.elapsed();
let (mut obj, dep_v) = dol_result.unwrap()?;
let mut function_count = obj.symbols.by_kind(ObjSymbolKind::Function).count();
dep.extend(dep_v);
let mut dol = {
let result = dol_result.unwrap()?;
dep.extend(result.dep);
ModuleInfo {
obj: result.obj,
config: &config.base,
symbols_cache: result.symbols_cache,
splits_cache: result.splits_cache,
}
};
let mut function_count = dol.obj.symbols.by_kind(ObjSymbolKind::Function).count();
let mut modules = BTreeMap::<u32, (&ModuleConfig, ObjInfo)>::new();
for (idx, (module_obj, dep_v)) in modules_result.unwrap()?.into_iter().enumerate() {
function_count += module_obj.symbols.by_kind(ObjSymbolKind::Function).count();
dep.extend(dep_v);
match modules.entry(module_obj.module_id) {
Entry::Vacant(e) => e.insert((&config.modules[idx], module_obj)),
Entry::Occupied(_) => bail!("Duplicate module ID {}", obj.module_id),
let mut modules = BTreeMap::<u32, ModuleInfo<'_>>::new();
for (idx, result) in modules_result.unwrap()?.into_iter().enumerate() {
function_count += result.obj.symbols.by_kind(ObjSymbolKind::Function).count();
dep.extend(result.dep);
match modules.entry(result.obj.module_id) {
Entry::Vacant(e) => e.insert(ModuleInfo {
obj: result.obj,
config: &config.modules[idx],
symbols_cache: result.symbols_cache,
splits_cache: result.splits_cache,
}),
Entry::Occupied(_) => bail!("Duplicate module ID {}", result.obj.module_id),
};
}
info!(
@ -979,26 +1016,26 @@ fn split(args: SplitArgs) -> Result<()> {
let module_ids = modules.keys().cloned().collect_vec();
// Create any missing symbols (referenced from other modules) and set FORCEACTIVE
update_symbols(&mut obj, &modules, !config.symbols_known)?;
update_symbols(&mut dol.obj, &modules, !config.symbols_known)?;
for &module_id in &module_ids {
let (module_config, mut module_obj) = modules.remove(&module_id).unwrap();
update_symbols(&mut module_obj, &modules, !config.symbols_known)?;
modules.insert(module_id, (module_config, module_obj));
let mut module = modules.remove(&module_id).unwrap();
update_symbols(&mut module.obj, &modules, !config.symbols_known)?;
modules.insert(module_id, module);
}
// Create relocations to symbols in other modules
for &module_id in &module_ids {
let (module_config, mut module_obj) = modules.remove(&module_id).unwrap();
create_relocations(&mut module_obj, &modules, &obj)?;
modules.insert(module_id, (module_config, module_obj));
let mut module = modules.remove(&module_id).unwrap();
create_relocations(&mut module.obj, &modules, &dol.obj)?;
modules.insert(module_id, module);
}
// Replace external relocations with internal ones, creating extern symbols
resolve_external_relocations(&mut obj, &modules, None)?;
resolve_external_relocations(&mut dol.obj, &modules, None)?;
for &module_id in &module_ids {
let (module_config, mut module_obj) = modules.remove(&module_id).unwrap();
resolve_external_relocations(&mut module_obj, &modules, Some(&obj))?;
modules.insert(module_id, (module_config, module_obj));
let mut module = modules.remove(&module_id).unwrap();
resolve_external_relocations(&mut module.obj, &modules, Some(&dol.obj))?;
modules.insert(module_id, module);
}
}
@ -1017,16 +1054,17 @@ fn split(args: SplitArgs) -> Result<()> {
// DOL
s.spawn(|_| {
let _span =
info_span!("module", name = %config.base.name(), id = obj.module_id).entered();
info_span!("module", name = %config.base.name(), id = dol.obj.module_id).entered();
dol_result = Some(
split_write_obj(&mut obj, &config, &config.base, &args.out_dir, args.no_update)
.with_context(|| {
split_write_obj(&mut dol, &config, &args.out_dir, args.no_update).with_context(
|| {
format!(
"While processing object '{}' (module ID {})",
config.base.file_name(),
obj.module_id
dol.obj.module_id
)
}),
},
),
);
});
// Modules
@ -1034,25 +1072,20 @@ fn split(args: SplitArgs) -> Result<()> {
modules_result = Some(
modules
.par_iter_mut()
.map(|(&module_id, (module_config, module_obj))| {
.map(|(&module_id, module)| {
let _span =
info_span!("module", name = %module_config.name(), id = module_id)
info_span!("module", name = %module.config.name(), id = module_id)
.entered();
let out_dir = args.out_dir.join(module_config.name().as_ref());
split_write_obj(
module_obj,
&config,
module_config,
&out_dir,
args.no_update,
let out_dir = args.out_dir.join(module.config.name().as_ref());
split_write_obj(module, &config, &out_dir, args.no_update).with_context(
|| {
format!(
"While processing object '{}' (module ID {})",
module.config.file_name(),
module_id
)
},
)
.with_context(|| {
format!(
"While processing object '{}' (module ID {})",
module_config.file_name(),
module_id
)
})
})
.collect(),
);
@ -1101,7 +1134,8 @@ fn split(args: SplitArgs) -> Result<()> {
}
#[allow(dead_code)]
fn validate<P: AsRef<Path>>(obj: &ObjInfo, elf_file: P, state: &AnalyzerState) -> Result<()> {
fn validate<P>(obj: &ObjInfo, elf_file: P, state: &AnalyzerState) -> Result<()>
where P: AsRef<Path> {
let real_obj = process_elf(elf_file)?;
for (section_index, real_section) in real_obj.sections.iter() {
let obj_section = match obj.sections.get(section_index) {
@ -1410,13 +1444,12 @@ fn apply(args: ApplyArgs) -> Result<()> {
process_dol(file.as_slice(), config.base.name().as_ref())?
};
if let Some(symbols_path) = &config.base.symbols {
if !apply_symbols_file(symbols_path, &mut obj)? {
bail!("Symbols file '{}' does not exist", symbols_path.display());
}
} else {
let Some(symbols_path) = &config.base.symbols else {
bail!("No symbols file specified in config");
}
};
let Some(symbols_cache) = apply_symbols_file(symbols_path, &mut obj)? else {
bail!("Symbols file '{}' does not exist", symbols_path.display());
};
log::info!("Loading {}", args.elf_file.display());
let linked_obj = process_elf(&args.elf_file)?;
@ -1550,7 +1583,7 @@ fn apply(args: ApplyArgs) -> Result<()> {
}
}
write_symbols_file(config.base.symbols.as_ref().unwrap(), &obj)?;
write_symbols_file(config.base.symbols.as_ref().unwrap(), &obj, Some(symbols_cache))?;
Ok(())
}

View File

@ -101,11 +101,14 @@ fn dump(args: DumpArgs) -> Result<()> {
Ok(())
}
fn dump_debug_section<W: Write>(
fn dump_debug_section<W>(
w: &mut W,
obj_file: &object::File<'_>,
debug_section: Section,
) -> Result<()> {
) -> Result<()>
where
W: Write + ?Sized,
{
let mut data = debug_section.uncompressed_data()?.into_owned();
// Apply relocations to data

View File

@ -19,10 +19,11 @@ use crate::{
obj::ObjKind,
util::{
asm::write_asm,
comment::{read_comment_sym, MWComment},
comment::{CommentSym, MWComment},
config::{write_splits_file, write_symbols_file},
elf::{process_elf, write_elf},
file::{buf_writer, process_rsp},
reader::{Endian, FromReader},
signatures::{compare_signature, generate_signature, FunctionSignature},
split::split_obj,
IntoCow, ToCow,
@ -136,8 +137,8 @@ fn config(args: ConfigArgs) -> Result<()> {
let obj = process_elf(&args.in_file)?;
DirBuilder::new().recursive(true).create(&args.out_dir)?;
write_symbols_file(args.out_dir.join("symbols.txt"), &obj)?;
write_splits_file(args.out_dir.join("splits.txt"), &obj, false)?;
write_symbols_file(args.out_dir.join("symbols.txt"), &obj, None)?;
write_splits_file(args.out_dir.join("splits.txt"), &obj, false, None)?;
Ok(())
}
@ -545,8 +546,8 @@ fn info(args: InfoArgs) -> Result<()> {
let data = comment_section.uncompressed_data()?;
if !data.is_empty() {
let mut reader = Cursor::new(&*data);
let header =
MWComment::parse_header(&mut reader).context("While reading .comment section")?;
let header = MWComment::from_reader(&mut reader, Endian::Big)
.context("While reading .comment section")?;
println!("\nMetrowerks metadata (.comment):");
println!("\tVersion: {}", header.version);
println!(
@ -577,7 +578,7 @@ fn info(args: InfoArgs) -> Result<()> {
println!("\tUnsafe global reg vars: {}", header.unsafe_global_reg_vars);
println!("\n{: >10} | {: <6} | {: <6} | {: <10}", "Align", "Vis", "Active", "Symbol");
for symbol in in_file.symbols() {
let comment_sym = read_comment_sym(&mut reader)?;
let comment_sym = CommentSym::from_reader(&mut reader, Endian::Big)?;
if symbol.is_definition() {
println!(
"{: >10} | {: <#6X} | {: <#6X} | {: <10}",

View File

@ -150,7 +150,8 @@ const fn align32(x: u32) -> u32 { (x + 31) & !31 }
const ZERO_BUF: [u8; 32] = [0u8; 32];
#[inline]
fn write_aligned<T: Write>(out: &mut T, bytes: &[u8], aligned_size: u32) -> std::io::Result<()> {
fn write_aligned<T>(out: &mut T, bytes: &[u8], aligned_size: u32) -> std::io::Result<()>
where T: Write + ?Sized {
out.write_all(bytes)?;
let padding = aligned_size - bytes.len() as u32;
if padding > 0 {

View File

@ -1,7 +1,6 @@
#![allow(clippy::needless_borrow)]
use std::path::PathBuf;
use anyhow::{bail, Result};
use anyhow::{bail, ensure, Result};
use argp::FromArgs;
use cwdemangle::{demangle, DemangleOptions};
@ -23,9 +22,6 @@ pub struct Args {
enum SubCommand {
Entries(EntriesArgs),
Symbol(SymbolArgs),
Order(OrderArgs),
Slices(SlicesArgs),
Symbols(SymbolsArgs),
}
#[derive(FromArgs, PartialEq, Eq, Debug)]
@ -52,40 +48,10 @@ pub struct SymbolArgs {
symbol: String,
}
#[derive(FromArgs, PartialEq, Eq, Debug)]
/// Attempts to resolve global link order.
#[argp(subcommand, name = "order")]
pub struct OrderArgs {
#[argp(positional)]
/// path to input map
map_file: PathBuf,
}
#[derive(FromArgs, PartialEq, Eq, Debug)]
/// Emits a slices.yml for ppcdis. (WIP)
#[argp(subcommand, name = "slices")]
pub struct SlicesArgs {
#[argp(positional)]
/// path to input map
map_file: PathBuf,
}
#[derive(FromArgs, PartialEq, Eq, Debug)]
/// Emits a symbols.yml for ppcdis. (WIP)
#[argp(subcommand, name = "symbols")]
pub struct SymbolsArgs {
#[argp(positional)]
/// path to input map
map_file: PathBuf,
}
pub fn run(args: Args) -> Result<()> {
match args.command {
SubCommand::Entries(c_args) => entries(c_args),
SubCommand::Symbol(c_args) => symbol(c_args),
SubCommand::Order(c_args) => order(c_args),
SubCommand::Slices(c_args) => slices(c_args),
SubCommand::Symbols(c_args) => symbols(c_args),
}
}