More updates to report types

objdiff-cli/protos/report.proto

@@ -2,84 +2,134 @@ syntax = "proto3";
 
 package objdiff.report;
 
-message Report {
+// Progress info for a report or unit
+message Measures {
+  // Overall match percent, including partially matched functions and data
   float fuzzy_match_percent = 1;
+  // Total size of code in bytes
   uint64 total_code = 2;
+  // Fully matched code size in bytes
   uint64 matched_code = 3;
+  // Fully matched code percent
   float matched_code_percent = 4;
+  // Total size of data in bytes
   uint64 total_data = 5;
+  // Fully matched data size in bytes
   uint64 matched_data = 6;
+  // Fully matched data percent
   float matched_data_percent = 7;
+  // Total number of functions
   uint32 total_functions = 8;
+  // Fully matched functions
   uint32 matched_functions = 9;
+  // Fully matched functions percent
   float matched_functions_percent = 10;
-  repeated ReportUnit units = 11;
 }
 
+// Project progress report
+message Report {
+  // Overall progress info
+  Measures measures = 1;
+  // Units within this report
+  repeated ReportUnit units = 2;
+}
+
+// A unit of the report (usually a translation unit)
 message ReportUnit {
+  // The name of the unit
   string name = 1;
-  float fuzzy_match_percent = 2;
+  // Progress info for this unit
-  uint64 total_code = 3;
+  Measures measures = 2;
-  uint64 matched_code = 4;
+  // Sections within this unit
-  uint64 total_data = 5;
+  repeated ReportItem sections = 3;
-  uint64 matched_data = 6;
+  // Functions within this unit
-  uint32 total_functions = 7;
+  repeated ReportItem functions = 4;
-  uint32 matched_functions = 8;
+  // Extra metadata for this unit
-  optional bool complete = 9;
+  optional ReportUnitMetadata metadata = 5;
-  optional string module_name = 10;
-  optional uint32 module_id = 11;
-  repeated ReportItem sections = 12;
-  repeated ReportItem functions = 13;
 }
 
+// Extra metadata for a unit
+message ReportUnitMetadata {
+  // Whether this unit is marked as complete (or "linked")
+  optional bool complete = 1;
+  // The name of the module this unit belongs to
+  optional string module_name = 2;
+  // The ID of the module this unit belongs to
+  optional uint32 module_id = 3;
+  // The path to the source file of this unit
+  optional string source_path = 4;
+}
+
+// A section or function within a unit
 message ReportItem {
+  // The name of the item
   string name = 1;
+  // The size of the item in bytes
   uint64 size = 2;
+  // The overall match percent for this item
   float fuzzy_match_percent = 3;
-  optional string demangled_name = 4;
+  // Extra metadata for this item
-  optional uint64 address = 5;
+  optional ReportItemMetadata metadata = 4;
 }
 
-// Used as stdin for the changes command
+// Extra metadata for an item
+message ReportItemMetadata {
+  // The demangled name of the function
+  optional string demangled_name = 1;
+  // The virtual address of the function or section
+  optional uint64 virtual_address = 2;
+}
+
+// A pair of reports to compare and generate changes
 message ChangesInput {
+  // The previous report
   Report from = 1;
+  // The current report
   Report to = 2;
 }
 
+// Changes between two reports
 message Changes {
-  ChangeInfo from = 1;
+  // The progress info for the previous report
-  ChangeInfo to = 2;
+  Measures from = 1;
+  // The progress info for the current report
+  Measures to = 2;
+  // Units that changed
   repeated ChangeUnit units = 3;
 }
 
-message ChangeInfo {
+// A changed unit
-  float fuzzy_match_percent = 1;
-  uint64 total_code = 2;
-  uint64 matched_code = 3;
-  float matched_code_percent = 4;
-  uint64 total_data = 5;
-  uint64 matched_data = 6;
-  float matched_data_percent = 7;
-  uint32 total_functions = 8;
-  uint32 matched_functions = 9;
-  float matched_functions_percent = 10;
-}
-
 message ChangeUnit {
+  // The name of the unit
   string name = 1;
-  optional ChangeInfo from = 2;
+  // The previous progress info (omitted if new)
-  optional ChangeInfo to = 3;
+  optional Measures from = 2;
+  // The current progress info (omitted if removed)
+  optional Measures to = 3;
+  // Sections that changed
   repeated ChangeItem sections = 4;
+  // Functions that changed
   repeated ChangeItem functions = 5;
+  // Extra metadata for this unit
+  optional ReportUnitMetadata metadata = 6;
 }
 
+// A changed section or function
 message ChangeItem {
+  // The name of the item
   string name = 1;
+  // The previous progress info (omitted if new)
   optional ChangeItemInfo from = 2;
+  // The current progress info (omitted if removed)
   optional ChangeItemInfo to = 3;
+  // Extra metadata for this item
+  optional ReportItemMetadata metadata = 4;
 }
 
+// Progress info for a section or function
 message ChangeItemInfo {
+  // The overall match percent for this item
   float fuzzy_match_percent = 1;
+  // The size of the item in bytes
   uint64 size = 2;
 }

objdiff-cli/src/cmd/report.rs

@@ -19,8 +19,8 @@ use rayon::iter::{IntoParallelRefMutIterator, ParallelIterator};
 use tracing::{info, warn};
 
 use crate::util::report::{
-    ChangeInfo, ChangeItem, ChangeItemInfo, ChangeUnit, Changes, ChangesInput, Report, ReportItem,
+    ChangeItem, ChangeItemInfo, ChangeUnit, Changes, ChangesInput, Measures, Report, ReportItem,
-    ReportUnit,
+    ReportItemMetadata, ReportUnit, ReportUnitMetadata,
 };
 
 #[derive(FromArgs, PartialEq, Debug)]
@@ -90,7 +90,7 @@ impl OutputFormat {
     fn from_str(s: &str) -> Result<Self> {
         match s {
             "json" => Ok(Self::Json),
-            "binpb" | "proto" | "protobuf" => Ok(Self::Proto),
+            "binpb" | "pb" | "proto" | "protobuf" => Ok(Self::Proto),
             _ => bail!("Invalid output format: {}", s),
         }
     }
@@ -117,7 +117,7 @@ fn generate(args: GenerateArgs) -> Result<()> {
     );
 
     let start = Instant::now();
-    let mut report = Report::default();
+    let mut units = vec![];
     let mut existing_functions: HashSet<String> = HashSet::new();
     if args.deduplicate {
         // If deduplicating, we need to run single-threaded
@@ -129,11 +129,11 @@ fn generate(args: GenerateArgs) -> Result<()> {
                 project.base_dir.as_deref(),
                 Some(&mut existing_functions),
             )? {
-                report.units.push(unit);
+                units.push(unit);
             }
         }
     } else {
-        let units = project
+        let vec = project
             .objects
             .par_iter_mut()
             .map(|object| {
@@ -146,38 +146,10 @@ fn generate(args: GenerateArgs) -> Result<()> {
                 )
             })
             .collect::<Result<Vec<Option<ReportUnit>>>>()?;
-        report.units = units.into_iter().flatten().collect();
+        units = vec.into_iter().flatten().collect();
     }
-    for unit in &report.units {
+    let measures = units.iter().flat_map(|u| u.measures.into_iter()).collect();
-        report.fuzzy_match_percent += unit.fuzzy_match_percent * unit.total_code as f32;
+    let report = Report { measures: Some(measures), units };
-        report.total_code += unit.total_code;
-        report.matched_code += unit.matched_code;
-        report.total_data += unit.total_data;
-        report.matched_data += unit.matched_data;
-        report.total_functions += unit.total_functions;
-        report.matched_functions += unit.matched_functions;
-    }
-    if report.total_code == 0 {
-        report.fuzzy_match_percent = 100.0;
-    } else {
-        report.fuzzy_match_percent /= report.total_code as f32;
-    }
-
-    report.matched_code_percent = if report.total_code == 0 {
-        100.0
-    } else {
-        report.matched_code as f32 / report.total_code as f32 * 100.0
-    };
-    report.matched_data_percent = if report.total_data == 0 {
-        100.0
-    } else {
-        report.matched_data as f32 / report.total_data as f32 * 100.0
-    };
-    report.matched_functions_percent = if report.total_functions == 0 {
-        100.0
-    } else {
-        report.matched_functions as f32 / report.total_functions as f32 * 100.0
-    };
     let duration = start.elapsed();
     info!("Report generated in {}.{:03}s", duration.as_secs(), duration.subsec_millis());
     write_output(&report, args.output.as_deref(), output_format)?;
@@ -258,18 +230,21 @@ fn report_object(
         })
         .transpose()?;
     let result = diff::diff_objs(&config, target.as_ref(), base.as_ref(), None)?;
-    let mut unit = ReportUnit {
+
-        name: object.name().to_string(),
+    let metadata = ReportUnitMetadata {
         complete: object.complete,
         module_name: target
             .as_ref()
             .and_then(|o| o.split_meta.as_ref())
             .and_then(|m| m.module_name.clone()),
         module_id: target.as_ref().and_then(|o| o.split_meta.as_ref()).and_then(|m| m.module_id),
-        ..Default::default()
+        source_path: None, // TODO
     };
-    let obj = target.as_ref().or(base.as_ref()).unwrap();
+    let mut measures = Measures::default();
+    let mut sections = vec![];
+    let mut functions = vec![];
 
+    let obj = target.as_ref().or(base.as_ref()).unwrap();
     let obj_diff = result.left.as_ref().or(result.right.as_ref()).unwrap();
     for (section, section_diff) in obj.sections.iter().zip(&obj_diff.sections) {
         let section_match_percent = section_diff.match_percent.unwrap_or_else(|| {
@@ -281,19 +256,21 @@ fn report_object(
                 0.0
             }
         });
-        unit.sections.push(ReportItem {
+        sections.push(ReportItem {
             name: section.name.clone(),
-            demangled_name: None,
             fuzzy_match_percent: section_match_percent,
             size: section.size,
-            address: section.virtual_address,
+            metadata: Some(ReportItemMetadata {
+                demangled_name: None,
+                virtual_address: section.virtual_address,
+            }),
         });
 
         match section.kind {
             ObjSectionKind::Data | ObjSectionKind::Bss => {
-                unit.total_data += section.size;
+                measures.total_data += section.size;
                 if section_match_percent == 100.0 {
-                    unit.matched_data += section.size;
+                    measures.matched_data += section.size;
                 }
                 continue;
             }
@@ -321,76 +298,35 @@ fn report_object(
                     0.0
                 }
             });
-            unit.fuzzy_match_percent += match_percent * symbol.size as f32;
+            measures.fuzzy_match_percent += match_percent * symbol.size as f32;
-            unit.total_code += symbol.size;
+            measures.total_code += symbol.size;
             if match_percent == 100.0 {
-                unit.matched_code += symbol.size;
+                measures.matched_code += symbol.size;
             }
-            unit.functions.push(ReportItem {
+            functions.push(ReportItem {
                 name: symbol.name.clone(),
-                demangled_name: symbol.demangled_name.clone(),
                 size: symbol.size,
                 fuzzy_match_percent: match_percent,
-                address: symbol.virtual_address,
+                metadata: Some(ReportItemMetadata {
+                    demangled_name: symbol.demangled_name.clone(),
+                    virtual_address: symbol.virtual_address,
+                }),
             });
             if match_percent == 100.0 {
-                unit.matched_functions += 1;
+                measures.matched_functions += 1;
             }
-            unit.total_functions += 1;
+            measures.total_functions += 1;
-        }
-    }
-    if unit.total_code == 0 {
-        unit.fuzzy_match_percent = 100.0;
-    } else {
-        unit.fuzzy_match_percent /= unit.total_code as f32;
-    }
-    Ok(Some(unit))
-}
-
-impl From<&Report> for ChangeInfo {
-    fn from(report: &Report) -> Self {
-        Self {
-            fuzzy_match_percent: report.fuzzy_match_percent,
-            total_code: report.total_code,
-            matched_code: report.matched_code,
-            matched_code_percent: report.matched_code_percent,
-            total_data: report.total_data,
-            matched_data: report.matched_data,
-            matched_data_percent: report.matched_data_percent,
-            total_functions: report.total_functions,
-            matched_functions: report.matched_functions,
-            matched_functions_percent: report.matched_functions_percent,
-        }
-    }
-}
-
-impl From<&ReportUnit> for ChangeInfo {
-    fn from(value: &ReportUnit) -> Self {
-        Self {
-            fuzzy_match_percent: value.fuzzy_match_percent,
-            total_code: value.total_code,
-            matched_code: value.matched_code,
-            matched_code_percent: if value.total_code == 0 {
-                100.0
-            } else {
-                value.matched_code as f32 / value.total_code as f32 * 100.0
-            },
-            total_data: value.total_data,
-            matched_data: value.matched_data,
-            matched_data_percent: if value.total_data == 0 {
-                100.0
-            } else {
-                value.matched_data as f32 / value.total_data as f32 * 100.0
-            },
-            total_functions: value.total_functions,
-            matched_functions: value.matched_functions,
-            matched_functions_percent: if value.total_functions == 0 {
-                100.0
-            } else {
-                value.matched_functions as f32 / value.total_functions as f32 * 100.0
-            },
         }
     }
+    measures.calc_fuzzy_match_percent();
+    measures.calc_matched_percent();
+    Ok(Some(ReportUnit {
+        name: object.name().to_string(),
+        measures: Some(measures),
+        sections,
+        functions,
+        metadata: Some(metadata),
+    }))
 }
 
 impl From<&ReportItem> for ChangeItemInfo {
@@ -417,25 +353,25 @@ fn changes(args: ChangesArgs) -> Result<()> {
         let current = read_report(&args.current)?;
         (previous, current)
     };
-    let mut changes = Changes {
+    let mut changes = Changes { from: previous.measures, to: current.measures, units: vec![] };
-        from: Some(ChangeInfo::from(&previous)),
-        to: Some(ChangeInfo::from(&current)),
-        units: vec![],
-    };
     for prev_unit in &previous.units {
         let curr_unit = current.units.iter().find(|u| u.name == prev_unit.name);
         let sections = process_items(prev_unit, curr_unit, |u| &u.sections);
         let functions = process_items(prev_unit, curr_unit, |u| &u.functions);
 
-        let prev_unit_info = ChangeInfo::from(prev_unit);
+        let prev_measures = prev_unit.measures;
-        let curr_unit_info = curr_unit.map(ChangeInfo::from);
+        let curr_measures = curr_unit.and_then(|u| u.measures);
-        if !functions.is_empty() || !matches!(&curr_unit_info, Some(v) if v == &prev_unit_info) {
+        if !functions.is_empty() || prev_measures != curr_measures {
             changes.units.push(ChangeUnit {
                 name: prev_unit.name.clone(),
-                from: Some(prev_unit_info),
+                from: prev_measures,
-                to: curr_unit_info,
+                to: curr_measures,
                 sections,
                 functions,
+                metadata: curr_unit
+                    .as_ref()
+                    .and_then(|u| u.metadata.clone())
+                    .or_else(|| prev_unit.metadata.clone()),
             });
         }
     }
@@ -444,9 +380,10 @@ fn changes(args: ChangesArgs) -> Result<()> {
             changes.units.push(ChangeUnit {
                 name: curr_unit.name.clone(),
                 from: None,
-                to: Some(ChangeInfo::from(curr_unit)),
+                to: curr_unit.measures,
                 sections: process_new_items(&curr_unit.sections),
                 functions: process_new_items(&curr_unit.functions),
+                metadata: curr_unit.metadata.clone(),
             });
         }
     }
@@ -473,6 +410,7 @@ fn process_items<F: Fn(&ReportUnit) -> &Vec<ReportItem>>(
                         name: prev_func.name.clone(),
                         from: Some(prev_func_info),
                         to: Some(curr_func_info),
+                        metadata: curr_func.as_ref().unwrap().metadata.clone(),
                     });
                 }
             } else {
@@ -480,6 +418,7 @@ fn process_items<F: Fn(&ReportUnit) -> &Vec<ReportItem>>(
                     name: prev_func.name.clone(),
                     from: Some(prev_func_info),
                     to: None,
+                    metadata: prev_func.metadata.clone(),
                 });
             }
         }
@@ -489,6 +428,7 @@ fn process_items<F: Fn(&ReportUnit) -> &Vec<ReportItem>>(
                     name: curr_func.name.clone(),
                     from: None,
                     to: Some(ChangeItemInfo::from(curr_func)),
+                    metadata: curr_func.metadata.clone(),
                 });
             }
         }
@@ -498,6 +438,7 @@ fn process_items<F: Fn(&ReportUnit) -> &Vec<ReportItem>>(
                 name: prev_func.name.clone(),
                 from: Some(ChangeItemInfo::from(prev_func)),
                 to: None,
+                metadata: prev_func.metadata.clone(),
             });
         }
     }
@@ -507,7 +448,12 @@ fn process_items<F: Fn(&ReportUnit) -> &Vec<ReportItem>>(
 fn process_new_items(items: &[ReportItem]) -> Vec<ChangeItem> {
     items
         .iter()
-        .map(|f| ChangeItem { name: f.name.clone(), from: None, to: Some(ChangeItemInfo::from(f)) })
+        .map(|item| ChangeItem {
+            name: item.name.clone(),
+            from: None,
+            to: Some(ChangeItemInfo::from(item)),
+            metadata: item.metadata.clone(),
+        })
         .collect()
 }
 

objdiff-cli/src/util/report.rs

@@ -39,6 +39,56 @@ impl Report {
     }
 }
 
+impl Measures {
+    /// Average the fuzzy match percentage over total code bytes.
+    pub fn calc_fuzzy_match_percent(&mut self) {
+        if self.total_code == 0 {
+            self.fuzzy_match_percent = 100.0;
+        } else {
+            self.fuzzy_match_percent /= self.total_code as f32;
+        }
+    }
+
+    /// Calculate the percentage of matched code, data, and functions.
+    pub fn calc_matched_percent(&mut self) {
+        self.matched_code_percent = if self.total_code == 0 {
+            100.0
+        } else {
+            self.matched_code as f32 / self.total_code as f32 * 100.0
+        };
+        self.matched_data_percent = if self.total_data == 0 {
+            100.0
+        } else {
+            self.matched_data as f32 / self.total_data as f32 * 100.0
+        };
+        self.matched_functions_percent = if self.total_functions == 0 {
+            100.0
+        } else {
+            self.matched_functions as f32 / self.total_functions as f32 * 100.0
+        };
+    }
+}
+
+/// Allows [collect](Iterator::collect) to be used on an iterator of [Measures].
+impl FromIterator<Measures> for Measures {
+    fn from_iter<T>(iter: T) -> Self
+    where T: IntoIterator<Item = Measures> {
+        let mut measures = Measures::default();
+        for other in iter {
+            measures.fuzzy_match_percent += other.fuzzy_match_percent * other.total_code as f32;
+            measures.total_code += other.total_code;
+            measures.matched_code += other.matched_code;
+            measures.total_data += other.total_data;
+            measures.matched_data += other.matched_data;
+            measures.total_functions += other.total_functions;
+            measures.matched_functions += other.matched_functions;
+        }
+        measures.calc_fuzzy_match_percent();
+        measures.calc_matched_percent();
+        measures
+    }
+}
+
 // Older JSON report types
 #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
 struct LegacyReport {
@@ -58,6 +108,7 @@ struct LegacyReport {
 impl From<LegacyReport> for Report {
     fn from(value: LegacyReport) -> Self {
         Self {
+            measures: Some(Measures {
                 fuzzy_match_percent: value.fuzzy_match_percent,
                 total_code: value.total_code,
                 matched_code: value.matched_code,
@@ -68,6 +119,7 @@ impl From<LegacyReport> for Report {
                 total_functions: value.total_functions,
                 matched_functions: value.matched_functions,
                 matched_functions_percent: value.matched_functions_percent,
+            }),
             units: value.units.into_iter().map(ReportUnit::from).collect(),
         }
     }
@@ -95,8 +147,7 @@ struct LegacyReportUnit {
 
 impl From<LegacyReportUnit> for ReportUnit {
     fn from(value: LegacyReportUnit) -> Self {
-        Self {
+        let mut measures = Measures {
-            name: value.name.clone(),
             fuzzy_match_percent: value.fuzzy_match_percent,
             total_code: value.total_code,
             matched_code: value.matched_code,
@@ -104,11 +155,20 @@ impl From<LegacyReportUnit> for ReportUnit {
             matched_data: value.matched_data,
             total_functions: value.total_functions,
             matched_functions: value.matched_functions,
+            ..Default::default()
+        };
+        measures.calc_matched_percent();
+        Self {
+            name: value.name.clone(),
+            measures: Some(measures),
+            sections: value.sections.into_iter().map(ReportItem::from).collect(),
+            functions: value.functions.into_iter().map(ReportItem::from).collect(),
+            metadata: Some(ReportUnitMetadata {
                 complete: value.complete,
                 module_name: value.module_name.clone(),
                 module_id: value.module_id,
-            sections: value.sections.into_iter().map(ReportItem::from).collect(),
+                ..Default::default()
-            functions: value.functions.into_iter().map(ReportItem::from).collect(),
+            }),
         }
     }
 }
@@ -133,10 +193,12 @@ impl From<LegacyReportItem> for ReportItem {
     fn from(value: LegacyReportItem) -> Self {
         Self {
             name: value.name,
-            demangled_name: value.demangled_name,
-            address: value.address,
             size: value.size,
             fuzzy_match_percent: value.fuzzy_match_percent,
+            metadata: Some(ReportItemMetadata {
+                demangled_name: value.demangled_name,
+                virtual_address: value.address,
+            }),
         }
     }
 }