From 195379968c6a4edf5a3e4a26916bdfc0ccc1de96 Mon Sep 17 00:00:00 2001 From: Luke Street Date: Tue, 3 Sep 2024 00:59:06 -0600 Subject: [PATCH] Support for progress categories & linked stats --- objdiff-cli/src/cmd/report.rs | 35 +++++-- objdiff-core/Cargo.toml | 9 +- objdiff-core/protos/proto_descriptor.bin | Bin 15535 -> 17154 bytes objdiff-core/protos/report.proto | 25 +++++ objdiff-core/src/bindings/mod.rs | 1 + objdiff-core/src/bindings/report.rs | 124 ++++++++++++++++++++--- objdiff-core/src/config/mod.rs | 38 +++++++ objdiff-core/src/lib.rs | 7 +- objdiff-core/src/util.rs | 2 +- objdiff-gui/src/views/config.rs | 8 +- 10 files changed, 218 insertions(+), 31 deletions(-) diff --git a/objdiff-cli/src/cmd/report.rs b/objdiff-cli/src/cmd/report.rs index 9b8f191..0c7b563 100644 --- a/objdiff-cli/src/cmd/report.rs +++ b/objdiff-cli/src/cmd/report.rs @@ -11,7 +11,8 @@ use argp::FromArgs; use objdiff_core::{ bindings::report::{ ChangeItem, ChangeItemInfo, ChangeUnit, Changes, ChangesInput, Measures, Report, - ReportItem, ReportItemMetadata, ReportUnit, ReportUnitMetadata, + ReportCategory, ReportItem, ReportItemMetadata, ReportUnit, ReportUnitMetadata, + REPORT_VERSION, }, config::ProjectObject, diff, obj, @@ -129,7 +130,17 @@ fn generate(args: GenerateArgs) -> Result<()> { units = vec.into_iter().flatten().collect(); } let measures = units.iter().flat_map(|u| u.measures.into_iter()).collect(); - let report = Report { measures: Some(measures), units }; + let mut categories = Vec::new(); + for category in &project.progress_categories { + categories.push(ReportCategory { + id: category.id.clone(), + name: category.name.clone(), + measures: Some(Default::default()), + }); + } + let mut report = + Report { measures: Some(measures), units, version: REPORT_VERSION, categories }; + report.calculate_progress_categories(); let duration = start.elapsed(); info!("Report generated in {}.{:03}s", duration.as_secs(), duration.subsec_millis()); write_output(&report, args.output.as_deref(), output_format)?; @@ -145,7 +156,7 @@ fn report_object( ) -> Result> { object.resolve_paths(project_dir, target_dir, base_dir); match (&object.target_path, &object.base_path) { - (None, Some(_)) if object.complete != Some(true) => { + (None, Some(_)) if !object.complete().unwrap_or(false) => { warn!("Skipping object without target: {}", object.name()); return Ok(None); } @@ -173,13 +184,19 @@ fn report_object( let result = diff::diff_objs(&config, target.as_ref(), base.as_ref(), None)?; let metadata = ReportUnitMetadata { - complete: object.complete, + complete: object.complete(), module_name: target .as_ref() .and_then(|o| o.split_meta.as_ref()) .and_then(|m| m.module_name.clone()), module_id: target.as_ref().and_then(|o| o.split_meta.as_ref()).and_then(|m| m.module_id), - source_path: None, // TODO + source_path: object.metadata.as_ref().and_then(|m| m.source_path.clone()), + progress_categories: object + .metadata + .as_ref() + .and_then(|m| m.progress_categories.clone()) + .unwrap_or_default(), + auto_generated: object.metadata.as_ref().and_then(|m| m.auto_generated), }; let mut measures = Measures::default(); let mut sections = vec![]; @@ -191,7 +208,7 @@ fn report_object( let section_match_percent = section_diff.match_percent.unwrap_or_else(|| { // Support cases where we don't have a target object, // assume complete means 100% match - if object.complete == Some(true) { + if object.complete().unwrap_or(false) { 100.0 } else { 0.0 @@ -233,7 +250,7 @@ fn report_object( let match_percent = symbol_diff.match_percent.unwrap_or_else(|| { // Support cases where we don't have a target object, // assume complete means 100% match - if object.complete == Some(true) { + if object.complete().unwrap_or(false) { 100.0 } else { 0.0 @@ -259,6 +276,10 @@ fn report_object( measures.total_functions += 1; } } + if metadata.complete.unwrap_or(false) { + measures.complete_code = measures.total_code; + measures.complete_data = measures.total_data; + } measures.calc_fuzzy_match_percent(); measures.calc_matched_percent(); Ok(Some(ReportUnit { diff --git a/objdiff-core/Cargo.toml b/objdiff-core/Cargo.toml index 1014403..8caf4aa 100644 --- a/objdiff-core/Cargo.toml +++ b/objdiff-core/Cargo.toml @@ -15,7 +15,7 @@ A local diffing tool for decompilation projects. crate-type = ["cdylib", "rlib"] [features] -all = ["config", "dwarf", "mips", "ppc", "x86", "arm"] +all = ["config", "dwarf", "mips", "ppc", "x86", "arm", "bindings"] any-arch = [] # Implicit, used to check if any arch is enabled config = ["globset", "semver", "serde_json", "serde_yaml"] dwarf = ["gimli"] @@ -23,7 +23,8 @@ mips = ["any-arch", "rabbitizer"] ppc = ["any-arch", "cwdemangle", "cwextab", "ppc750cl"] x86 = ["any-arch", "cpp_demangle", "iced-x86", "msvc-demangler"] arm = ["any-arch", "cpp_demangle", "unarm", "arm-attr"] -wasm = ["serde_json", "console_error_panic_hook", "console_log"] +bindings = ["serde_json", "prost", "pbjson"] +wasm = ["bindings", "console_error_panic_hook", "console_log"] [dependencies] anyhow = "1.0.82" @@ -34,8 +35,8 @@ log = "0.4.21" memmap2 = "0.9.4" num-traits = "0.2.18" object = { version = "0.35.0", features = ["read_core", "std", "elf", "pe"], default-features = false } -pbjson = "0.7.0" -prost = "0.13.1" +pbjson = { version = "0.7.0", optional = true } +prost = { version = "0.13.1", optional = true } serde = { version = "1", features = ["derive"] } similar = { version = "2.5.0", default-features = false } strum = { version = "0.26.2", features = ["derive"] } diff --git a/objdiff-core/protos/proto_descriptor.bin b/objdiff-core/protos/proto_descriptor.bin index 46866c0324f0bb9d9c505e66293158f04a01b580..4551eea4e9c0fea4e8491f2714fcf73e87b9380f 100644 GIT binary patch delta 4019 zcmaJ^TTffr74|;o*x1{ckj~jd|<%#p{z1FwZ z{?=Z5nXeBXf6iVt*v8-9lTCKI_AeF>f6rOnd${ZD$)9z`WioT+^#l8W9f=MeTdQpE zY&lhDWv#OA#G^LrwTiXg;w`L+LEhwHWwf&6?5;WGYP=T4MzO)$B;$E^>jLrmYIQXp zyGT5ZHBoqznXG)~x{J(Yt(n{Zz~o=sc5))lYxbSpy;7wdH(?zsdQePozP4I*?pJn8 z&R*QI&EDu>t9;edeCL+Cd{BJDp#8_ZuITJkcB|g5sN+UyJ#?QX&R;LJw=&}!Lw?%v%SCaOH|*?I?nCpvl{eA}BArN4cw0Iq5*2;C zZuMccvU1-kJG)49Jsv{5@NKIYqxBEGW!cg>(J*zHUs}0n{(7oMX>iu zhmKt;+aDiPojuNLd5~O#OuWLiPUEHvku(|>Ray;8r*A=egoluAw0b%yYO18VN$8e92 zB~bK48`Ob;K+)6u9p0nNv6Yd(&1Q}=zBvnFE!w2b8Cf(ZJp|`WYnrE&vmwlWqbX5u zyBFQojpM~NaFBkj6Eh4D^+)P8kpZIqhDP45h)i*0W{pgu$)4AD+djYB9-V0k82JcU zW}{a$oeA=6qSHel&-V0k4v`FD{C+&eTK}wB= z1@8d6Ng9mH?SXU-pU7!k=L7_^ah(znxWSK zNT63a;T)4_g{1vMM}B%WHDMYC>N)BA(orV$t-~IeCYxK8B0yl8Y-{Hk#RM=Vt5>@~ zL04kRr!YY=r<&86!X&Tu4&JXQ;5IFv45j4h2)}6>7Ga;p0?l;PeIKv~n(5YV zO#>y(=}aH$O-|PBz>ioWDIbh9wyxR_%cUx=8SWYR+zu!BG=f>A@r-xdM zWdxD!f}cr{0{GPmB9n!B_kzL_nJnOfUc?grgQiN z*P`wW|0ccgkR`HocYB{&w8BkH{@za1OU zj#TAkJye3Ky_Umnm4V>6j4D%a1PR8u(c`A@*|+2e%SOWNS{rgg7n!GZJL7g+3f664`Pk5wgL}i^!k{nfzejjokTU1 zTv^_|-YS<1pwbDRfDnscuBvV5r+QA%Yv=`*B3aP#w--*KS=urVPIPG9m zWFS#kSk=H!Ro(7!b9JQ9^ym&iee8GhkE=2f#Oh& zf5VE1;Aep^n4BGsH$-6d87?~g1c~4f)-l6WNsh*FX@Sua#|*M)h~F*83}R}C;dIQ9 z*O+p~=3{208F2vw&`%h~89fpa^tN@%Xr9fvGsrc< z^Mz$UyC4`(8Twc6g7K80fi(oK;HL~v zS@H%s10#W_UV#B&{nRTkAgG^u1?HsiGbR^D2i3uWk>)cmO+cjijNyl!ON}cGm~chJ zF1VgCh)Ct-xWv{Oli!S9*IZQGvvV#g?%6pPg+9Z(q^YT#XDn?i5jmmfBw~k5kB4qj zX`i1nQE8uh;=vY7&pq(~l`q5_hImmC`jX-Qq+gE>HsD_k>q~}ToHRhAq}vPm&#|E) zd_uAp3=c9L6A*?k7+z!zLHL59(Xz@9n=j>^@k}mC1HgLeNexI_p45QwdkLvO4q=*A HD?|SVp+_im delta 2514 zcmX|@&rcgi6vw@@YkNJ;57;;^u>r4*4Kc2ZgE5XVF*t-0DM3_95tSl@oYVqD!9YO> zAsVG9x%M!pR;tpfr&fy6i2MQlCwlCUV^6*1TJ`(JkGbvF^WJCPe&@|wfBn*WbNuTy zUwl_n*Zq`uXP-~{?<|qoRj)F;=f4jPTH;Sjz0dxn-skG)bHo2Skw1oI*b4>%FV$4R zw-V=%3%}ViCxv5q&OjieUYBxL2HYz$D;>uY0nbjUze~fTDIw)08Kj>q{1&}-p=_x5 z2XaJ*V=1jlxAM2sCfo|)x9D{awyiiC^52oY{wilk03zXEua@`XM%DJ8#0#i^4 zBkJ37s)WZNo`tKkp%69_xqP9(8tLhkqdEyodG$|u7)ew>p38xv6|No%NqJO%Qzr?} zE~v&twjqRY@NAL<5Cwh*tN~Go5AwbOh(bCe^BMtBR6k7QfUtom8iWl*(I9Lfit!X9 zY|6?E$$~~8ZA`tc4Jf}Nn<#{|!8pK-MF%($XJ(HL4qqr>#`2>$3gOtEQw~g9iKKd4 zi>YUoOCD`mb`$a*HnGXniu%4X+rT3>sPG0*u(%Svs*zAgs`T_-C~Qzk_RA@az*1Eu zCntfhfT$XT1w=L4%SmK-)$3M_NFb_yTHe+O7+zD~OilnnhSv;208ujtGQ8$>n+B0U z)X+3ljew{tt6Bzv46hr64Mg1_= z7KgI(jz(ZQ(w?LR%--*4|6`1|eM&i+An z$c6OKSTQgvY$X!ag<%b_l`B_q@iwmKv;+ITNeBgIn}<9UWn#$ZbUjGxh-{+I|M+m{>A_}a=S%-!qjlgv-DvHsw?FUpkgF$m z$KYvlcNm|+BKRHt0ziS^;V*zocYp8V3($bXetU)!1ZR)0HP(RH<7*8C%pR_F0cRQU zoWsCZs&{rME~V{65O~Pv8KLt$GP4a5J09_wLLu~s&lC#GBR zTpAQgJC5Ga) Result { if data.is_empty() { bail!(std::io::Error::from(std::io::ErrorKind::UnexpectedEof)); } - if data[0] == b'{' { + let report = if data[0] == b'{' { // Load as JSON - Self::from_json(data).map_err(anyhow::Error::new) + Self::from_json(data)? } else { // Load as binary protobuf - Self::decode(data).map_err(anyhow::Error::new) - } + Self::decode(data)? + }; + Ok(report) } fn from_json(bytes: &[u8]) -> Result { @@ -37,6 +42,81 @@ impl Report { } } } + + pub fn migrate(&mut self) -> Result<()> { + if self.version == 0 { + self.migrate_v0()?; + } + if self.version != REPORT_VERSION { + bail!("Unsupported report version: {}", self.version); + } + Ok(()) + } + + fn migrate_v0(&mut self) -> Result<()> { + let Some(measures) = &mut self.measures else { + bail!("Missing measures in report"); + }; + for unit in &mut self.units { + let Some(unit_measures) = &mut unit.measures else { + bail!("Missing measures in report unit"); + }; + let Some(metadata) = &mut unit.metadata else { + bail!("Missing metadata in report unit"); + }; + if metadata.module_name.is_some() || metadata.module_id.is_some() { + metadata.progress_categories = vec!["modules".to_string()]; + } else { + metadata.progress_categories = vec!["dol".to_string()]; + } + if metadata.complete.unwrap_or(false) { + unit_measures.complete_code = unit_measures.total_code; + unit_measures.complete_data = unit_measures.total_data; + unit_measures.complete_code_percent = 100.0; + unit_measures.complete_data_percent = 100.0; + } else { + unit_measures.complete_code = 0; + unit_measures.complete_data = 0; + unit_measures.complete_code_percent = 0.0; + unit_measures.complete_data_percent = 0.0; + } + measures.complete_code += unit_measures.complete_code; + measures.complete_data += unit_measures.complete_data; + } + measures.calc_matched_percent(); + self.version = 1; + Ok(()) + } + + pub fn calculate_progress_categories(&mut self) { + for unit in &self.units { + let Some(metadata) = unit.metadata.as_ref() else { + continue; + }; + let Some(measures) = unit.measures.as_ref() else { + continue; + }; + for category_id in &metadata.progress_categories { + let category = match self.categories.iter_mut().find(|c| &c.id == category_id) { + Some(category) => category, + None => { + self.categories.push(ReportCategory { + id: category_id.clone(), + name: String::new(), + measures: Some(Default::default()), + }); + self.categories.last_mut().unwrap() + } + }; + *category.measures.get_or_insert_with(Default::default) += *measures; + } + } + for category in &mut self.categories { + let measures = category.measures.get_or_insert_with(Default::default); + measures.calc_fuzzy_match_percent(); + measures.calc_matched_percent(); + } + } } impl Measures { @@ -66,6 +146,16 @@ impl Measures { } else { self.matched_functions as f32 / self.total_functions as f32 * 100.0 }; + self.complete_code_percent = if self.total_code == 0 { + 100.0 + } else { + self.complete_code as f32 / self.total_code as f32 * 100.0 + }; + self.complete_data_percent = if self.total_data == 0 { + 100.0 + } else { + self.complete_data as f32 / self.total_data as f32 * 100.0 + }; } } @@ -75,19 +165,27 @@ impl From<&ReportItem> for ChangeItemInfo { } } +impl AddAssign for Measures { + fn add_assign(&mut self, other: Self) { + self.fuzzy_match_percent += other.fuzzy_match_percent * other.total_code as f32; + self.total_code += other.total_code; + self.matched_code += other.matched_code; + self.total_data += other.total_data; + self.matched_data += other.matched_data; + self.total_functions += other.total_functions; + self.matched_functions += other.matched_functions; + self.complete_code += other.complete_code; + self.complete_data += other.complete_data; + } +} + /// Allows [collect](Iterator::collect) to be used on an iterator of [Measures]. impl FromIterator for Measures { fn from_iter(iter: T) -> Self where T: IntoIterator { let mut measures = Measures::default(); for other in iter { - measures.fuzzy_match_percent += other.fuzzy_match_percent * other.total_code as f32; - measures.total_code += other.total_code; - measures.matched_code += other.matched_code; - measures.total_data += other.total_data; - measures.matched_data += other.matched_data; - measures.total_functions += other.total_functions; - measures.matched_functions += other.matched_functions; + measures += other; } measures.calc_fuzzy_match_percent(); measures.calc_matched_percent(); @@ -125,8 +223,10 @@ impl From for Report { total_functions: value.total_functions, matched_functions: value.matched_functions, matched_functions_percent: value.matched_functions_percent, + ..Default::default() }), - units: value.units.into_iter().map(ReportUnit::from).collect(), + units: value.units.into_iter().map(ReportUnit::from).collect::>(), + ..Default::default() } } } diff --git a/objdiff-core/src/config/mod.rs b/objdiff-core/src/config/mod.rs index f1fd1fb..6a89986 100644 --- a/objdiff-core/src/config/mod.rs +++ b/objdiff-core/src/config/mod.rs @@ -31,6 +31,8 @@ pub struct ProjectConfig { pub watch_patterns: Option>, #[serde(default, alias = "units")] pub objects: Vec, + #[serde(default)] + pub progress_categories: Vec, } #[derive(Default, Clone, serde::Deserialize)] @@ -44,11 +46,37 @@ pub struct ProjectObject { #[serde(default)] pub base_path: Option, #[serde(default)] + #[deprecated(note = "Use metadata.reverse_fn_order")] pub reverse_fn_order: Option, #[serde(default)] + #[deprecated(note = "Use metadata.complete")] pub complete: Option, #[serde(default)] pub scratch: Option, + #[serde(default)] + pub metadata: Option, +} + +#[derive(Default, Clone, serde::Deserialize)] +pub struct ProjectObjectMetadata { + #[serde(default)] + pub complete: Option, + #[serde(default)] + pub reverse_fn_order: Option, + #[serde(default)] + pub source_path: Option, + #[serde(default)] + pub progress_categories: Option>, + #[serde(default)] + pub auto_generated: Option, +} + +#[derive(Default, Clone, serde::Deserialize)] +pub struct ProjectProgressCategory { + #[serde(default)] + pub id: String, + #[serde(default)] + pub name: String, } impl ProjectObject { @@ -82,6 +110,16 @@ impl ProjectObject { self.base_path = Some(project_dir.join(path)); } } + + pub fn complete(&self) -> Option { + #[allow(deprecated)] + self.metadata.as_ref().and_then(|m| m.complete).or(self.complete) + } + + pub fn reverse_fn_order(&self) -> Option { + #[allow(deprecated)] + self.metadata.as_ref().and_then(|m| m.reverse_fn_order).or(self.reverse_fn_order) + } } #[derive(Default, Clone, Eq, PartialEq, serde::Deserialize, serde::Serialize)] diff --git a/objdiff-core/src/lib.rs b/objdiff-core/src/lib.rs index 48f1e72..02df5a4 100644 --- a/objdiff-core/src/lib.rs +++ b/objdiff-core/src/lib.rs @@ -1,10 +1,11 @@ +#[cfg(feature = "any-arch")] pub mod arch; +#[cfg(feature = "bindings")] pub mod bindings; #[cfg(feature = "config")] pub mod config; +#[cfg(feature = "any-arch")] pub mod diff; +#[cfg(feature = "any-arch")] pub mod obj; pub mod util; - -#[cfg(not(feature = "any-arch"))] -compile_error!("At least one architecture feature must be enabled."); diff --git a/objdiff-core/src/util.rs b/objdiff-core/src/util.rs index 5f99e94..e46d136 100644 --- a/objdiff-core/src/util.rs +++ b/objdiff-core/src/util.rs @@ -9,7 +9,7 @@ use num_traits::PrimInt; use object::{Endian, Object}; // https://stackoverflow.com/questions/44711012/how-do-i-format-a-signed-integer-to-a-sign-aware-hexadecimal-representation -pub(crate) struct ReallySigned(pub(crate) N); +pub struct ReallySigned(pub(crate) N); impl LowerHex for ReallySigned { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { diff --git a/objdiff-gui/src/views/config.rs b/objdiff-gui/src/views/config.rs index f432f5a..812595c 100644 --- a/objdiff-gui/src/views/config.rs +++ b/objdiff-gui/src/views/config.rs @@ -365,7 +365,7 @@ fn display_object( let selected = matches!(selected_obj, Some(obj) if obj.name == object_name); let color = if selected { appearance.emphasized_text_color - } else if let Some(complete) = object.complete { + } else if let Some(complete) = object.complete() { if complete { appearance.insert_color } else { @@ -392,8 +392,8 @@ fn display_object( name: object_name.to_string(), target_path: object.target_path.clone(), base_path: object.base_path.clone(), - reverse_fn_order: object.reverse_fn_order, - complete: object.complete, + reverse_fn_order: object.reverse_fn_order(), + complete: object.complete(), scratch: object.scratch.clone(), }); } @@ -470,7 +470,7 @@ fn filter_node( if (search.is_empty() || name.to_ascii_lowercase().contains(search)) && (!filter_diffable || (object.base_path.is_some() && object.target_path.is_some())) - && (!filter_incomplete || matches!(object.complete, None | Some(false))) + && (!filter_incomplete || matches!(object.complete(), None | Some(false))) { Some(node.clone()) } else {