encounter/objdiff
1
0
Fork 0
mirror of https://github.com/encounter/objdiff.git synced 2025-12-11 06:27:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support protobuf format for reports

Browse Source 
This migrates to using protobuf to
define the "report" and "changes"
formats in objdiff-cli.

The JSON output now uses the Proto3
"JSON Mapping", which is slightly
incompatible with the existing JSON
format. Mainly, 64-bit numbers are
represented as strings, and addresses
are decimal strings instead of hex.

However, the older JSON format is
still accepted by "report changes"
to ease migration.
This commit is contained in:
Luke Street
2024-08-16 00:52:24 -06:00
parent cf937b0be9
commit cad9b70632
10 changed files with 526 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

254
objdiff-cli/src/cmd/report.rs
View File

@@ -1,7 +1,8 @@
use std::{
collections::HashSet,
fs::File,
io::{BufReader, BufWriter, Write},
io::{BufWriter, Read, Write},
ops::DerefMut,
path::{Path, PathBuf},
time::Instant,
};
@@ -13,9 +14,15 @@ use objdiff_core::{
diff, obj,
obj::{ObjSectionKind, ObjSymbolFlags},
};
use prost::Message;
use rayon::iter::{IntoParallelRefMutIterator, ParallelIterator};
use tracing::{info, warn};
use crate::util::report::{
ChangeInfo, ChangeItem, ChangeItemInfo, ChangeUnit, Changes, ChangesInput, Report, ReportItem,
ReportUnit,
};
#[derive(FromArgs, PartialEq, Debug)]
/// Commands for processing NVIDIA Shield TV alf files.
#[argp(subcommand, name = "report")]
@@ -39,11 +46,14 @@ pub struct GenerateArgs {
/// Project directory
project: Option<PathBuf>,
#[argp(option, short = 'o')]
/// Output JSON file
/// Output file
output: Option<PathBuf>,
#[argp(switch, short = 'd')]
/// Deduplicate global and weak symbols (runs single-threaded)
deduplicate: bool,
#[argp(option, short = 'f')]
/// Output format (json or proto, default json)
format: Option<String>,
}
#[derive(FromArgs, PartialEq, Debug)]
@@ -51,65 +61,17 @@ pub struct GenerateArgs {
#[argp(subcommand, name = "changes")]
pub struct ChangesArgs {
#[argp(positional)]
/// Previous report JSON file
/// Previous report file
previous: PathBuf,
#[argp(positional)]
/// Current report JSON file
/// Current report file
current: PathBuf,
#[argp(option, short = 'o')]
/// Output JSON file
/// Output file
output: Option<PathBuf>,
}
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
struct Report {
fuzzy_match_percent: f32,
total_code: u64,
matched_code: u64,
matched_code_percent: f32,
total_data: u64,
matched_data: u64,
matched_data_percent: f32,
total_functions: u32,
matched_functions: u32,
matched_functions_percent: f32,
units: Vec<ReportUnit>,
}
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
struct ReportUnit {
name: String,
fuzzy_match_percent: f32,
total_code: u64,
matched_code: u64,
total_data: u64,
matched_data: u64,
total_functions: u32,
matched_functions: u32,
#[serde(skip_serializing_if = "Option::is_none")]
complete: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
module_name: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
module_id: Option<u32>,
sections: Vec<ReportItem>,
functions: Vec<ReportItem>,
}
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
struct ReportItem {
name: String,
#[serde(skip_serializing_if = "Option::is_none")]
demangled_name: Option<String>,
#[serde(
default,
skip_serializing_if = "Option::is_none",
serialize_with = "serialize_hex",
deserialize_with = "deserialize_hex"
)]
address: Option<u64>,
size: u64,
fuzzy_match_percent: f32,
#[argp(option, short = 'f')]
/// Output format (json or proto, default json)
format: Option<String>,
}
pub fn run(args: Args) -> Result<()> {
@@ -119,7 +81,28 @@ pub fn run(args: Args) -> Result<()> {
}
}
enum OutputFormat {
Json,
Proto,
}
impl OutputFormat {
fn from_str(s: &str) -> Result<Self> {
match s {
"json" => Ok(Self::Json),
"binpb" | "proto" | "protobuf" => Ok(Self::Proto),
_ => bail!("Invalid output format: {}", s),
}
}
}
fn generate(args: GenerateArgs) -> Result<()> {
let output_format = if let Some(format) = &args.format {
OutputFormat::from_str(format)?
} else {
OutputFormat::Json
};
let project_dir = args.project.as_deref().unwrap_or_else(|| Path::new("."));
info!("Loading project {}", project_dir.display());
@@ -197,17 +180,46 @@ fn generate(args: GenerateArgs) -> Result<()> {
};
let duration = start.elapsed();
info!("Report generated in {}.{:03}s", duration.as_secs(), duration.subsec_millis());
if let Some(output) = &args.output {
write_output(&report, args.output.as_deref(), output_format)?;
Ok(())
}
fn write_output<T>(input: &T, output: Option<&Path>, format: OutputFormat) -> Result<()>
where T: serde::Serialize + prost::Message {
if let Some(output) = output {
info!("Writing to {}", output.display());
let mut output = BufWriter::new(
File::create(output)
.with_context(|| format!("Failed to create file {}", output.display()))?,
);
serde_json::to_writer_pretty(&mut output, &report)?;
output.flush()?;
let file = File::options()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(output)
.with_context(|| format!("Failed to create file {}", output.display()))?;
match format {
OutputFormat::Json => {
let mut output = BufWriter::new(file);
serde_json::to_writer_pretty(&mut output, input)
.context("Failed to write output file")?;
output.flush().context("Failed to flush output file")?;
}
OutputFormat::Proto => {
file.set_len(input.encoded_len() as u64)?;
let map =
unsafe { memmap2::Mmap::map(&file) }.context("Failed to map output file")?;
let mut output = map.make_mut().context("Failed to remap output file")?;
input.encode(&mut output.deref_mut()).context("Failed to encode output")?;
}
}
} else {
serde_json::to_writer_pretty(std::io::stdout(), &report)?;
}
match format {
OutputFormat::Json => {
serde_json::to_writer_pretty(std::io::stdout(), input)?;
}
OutputFormat::Proto => {
std::io::stdout().write_all(&input.encode_to_vec())?;
}
}
};
Ok(())
}
@@ -335,27 +347,6 @@ fn report_object(
Ok(Some(unit))
}
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
struct Changes {
from: ChangeInfo,
to: ChangeInfo,
units: Vec<ChangeUnit>,
}
#[derive(Debug, Clone, Default, PartialEq, serde::Serialize, serde::Deserialize)]
struct ChangeInfo {
fuzzy_match_percent: f32,
total_code: u64,
matched_code: u64,
matched_code_percent: f32,
total_data: u64,
matched_data: u64,
matched_data_percent: f32,
total_functions: u32,
matched_functions: u32,
matched_functions_percent: f32,
}
impl From<&Report> for ChangeInfo {
fn from(report: &Report) -> Self {
Self {
@@ -402,28 +393,6 @@ impl From<&ReportUnit> for ChangeInfo {
}
}
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
struct ChangeUnit {
name: String,
from: Option<ChangeInfo>,
to: Option<ChangeInfo>,
sections: Vec<ChangeItem>,
functions: Vec<ChangeItem>,
}
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
struct ChangeItem {
name: String,
from: Option<ChangeItemInfo>,
to: Option<ChangeItemInfo>,
}
#[derive(Debug, Clone, Default, PartialEq, serde::Serialize, serde::Deserialize)]
struct ChangeItemInfo {
fuzzy_match_percent: f32,
size: u64,
}
impl From<&ReportItem> for ChangeItemInfo {
fn from(value: &ReportItem) -> Self {
Self { fuzzy_match_percent: value.fuzzy_match_percent, size: value.size }
@@ -431,11 +400,26 @@ impl From<&ReportItem> for ChangeItemInfo {
}
fn changes(args: ChangesArgs) -> Result<()> {
let previous = read_report(&args.previous)?;
let current = read_report(&args.current)?;
let output_format = if let Some(format) = &args.format {
OutputFormat::from_str(format)?
} else {
OutputFormat::Json
};
let (previous, current) = if args.previous == Path::new("-") && args.current == Path::new("-") {
// Special case for comparing two reports from stdin
let mut data = vec![];
std::io::stdin().read_to_end(&mut data)?;
let input = ChangesInput::decode(data.as_slice())?;
(input.from.unwrap(), input.to.unwrap())
} else {
let previous = read_report(&args.previous)?;
let current = read_report(&args.current)?;
(previous, current)
};
let mut changes = Changes {
from: ChangeInfo::from(&previous),
to: ChangeInfo::from(&current),
from: Some(ChangeInfo::from(&previous)),
to: Some(ChangeInfo::from(&current)),
units: vec![],
};
for prev_unit in &previous.units {
@@ -466,17 +450,7 @@ fn changes(args: ChangesArgs) -> Result<()> {
});
}
}
if let Some(output) = &args.output {
info!("Writing to {}", output.display());
let mut output = BufWriter::new(
File::create(output)
.with_context(|| format!("Failed to create file {}", output.display()))?,
);
serde_json::to_writer_pretty(&mut output, &changes)?;
output.flush()?;
} else {
serde_json::to_writer_pretty(std::io::stdout(), &changes)?;
}
write_output(&changes, args.output.as_deref(), output_format)?;
Ok(())
}
@@ -538,30 +512,14 @@ fn process_new_items(items: &[ReportItem]) -> Vec<ChangeItem> {
}
fn read_report(path: &Path) -> Result<Report> {
serde_json::from_reader(BufReader::new(
File::open(path).with_context(|| format!("Failed to open {}", path.display()))?,
))
.with_context(|| format!("Failed to read report {}", path.display()))
}
fn serialize_hex<S>(x: &Option<u64>, s: S) -> Result<S::Ok, S::Error>
where S: serde::Serializer {
if let Some(x) = x {
s.serialize_str(&format!("{:#x}", x))
} else {
s.serialize_none()
}
}
fn deserialize_hex<'de, D>(d: D) -> Result<Option<u64>, D::Error>
where D: serde::Deserializer<'de> {
use serde::Deserialize;
let s = String::deserialize(d)?;
if s.is_empty() {
Ok(None)
} else if !s.starts_with("0x") {
Err(serde::de::Error::custom("expected hex string"))
} else {
u64::from_str_radix(&s[2..], 16).map(Some).map_err(serde::de::Error::custom)
if path == Path::new("-") {
let mut data = vec![];
std::io::stdin().read_to_end(&mut data)?;
return Report::parse(&data).with_context(|| "Failed to load report from stdin");
}
let file = File::open(path).with_context(|| format!("Failed to open {}", path.display()))?;
let mmap = unsafe { memmap2::Mmap::map(&file) }
.with_context(|| format!("Failed to map {}", path.display()))?;
Report::parse(mmap.as_ref())
.with_context(|| format!("Failed to load report {}", path.display()))
}

2
objdiff-cli/src/main.rs
View File

@@ -96,7 +96,7 @@ fn main() {
// Try to enable ANSI support on Windows.
let _ = enable_ansi_support();
// Disable isatty check for supports-color. (e.g. when used with ninja)
env::set_var("IGNORE_IS_TERMINAL", "1");
unsafe { env::set_var("IGNORE_IS_TERMINAL", "1") };
supports_color::on(Stream::Stdout).is_some_and(|c| c.has_basic)
};

1
objdiff-cli/src/util/mod.rs
View File

@@ -1 +1,2 @@
pub mod report;
pub mod term;

164
objdiff-cli/src/util/report.rs Normal file
View File

@@ -0,0 +1,164 @@
use anyhow::{bail, Result};
use prost::Message;
use serde_json::error::Category;
// Protobuf report types
include!(concat!(env!("OUT_DIR"), "/objdiff.report.rs"));
include!(concat!(env!("OUT_DIR"), "/objdiff.report.serde.rs"));
impl Report {
pub fn parse(data: &[u8]) -> Result<Self> {
if data.is_empty() {
bail!(std::io::Error::from(std::io::ErrorKind::UnexpectedEof));
}
if data[0] == b'{' {
// Load as JSON
Self::from_json(data).map_err(anyhow::Error::new)
} else {
// Load as binary protobuf
Self::decode(data).map_err(anyhow::Error::new)
}
}
fn from_json(bytes: &[u8]) -> Result<Self, serde_json::Error> {
match serde_json::from_slice::<Self>(bytes) {
Ok(report) => Ok(report),
Err(e) => {
match e.classify() {
Category::Io | Category::Eof | Category::Syntax => Err(e),
Category::Data => {
// Try to load as legacy report
match serde_json::from_slice::<LegacyReport>(bytes) {
Ok(legacy_report) => Ok(Report::from(legacy_report)),
Err(_) => Err(e),
}
}
}
}
}
}
}
// Older JSON report types
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
struct LegacyReport {
fuzzy_match_percent: f32,
total_code: u64,
matched_code: u64,
matched_code_percent: f32,
total_data: u64,
matched_data: u64,
matched_data_percent: f32,
total_functions: u32,
matched_functions: u32,
matched_functions_percent: f32,
units: Vec<LegacyReportUnit>,
}
impl From<LegacyReport> for Report {
fn from(value: LegacyReport) -> Self {
Self {
fuzzy_match_percent: value.fuzzy_match_percent,
total_code: value.total_code,
matched_code: value.matched_code,
matched_code_percent: value.matched_code_percent,
total_data: value.total_data,
matched_data: value.matched_data,
matched_data_percent: value.matched_data_percent,
total_functions: value.total_functions,
matched_functions: value.matched_functions,
matched_functions_percent: value.matched_functions_percent,
units: value.units.into_iter().map(ReportUnit::from).collect(),
}
}
}
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
struct LegacyReportUnit {
name: String,
fuzzy_match_percent: f32,
total_code: u64,
matched_code: u64,
total_data: u64,
matched_data: u64,
total_functions: u32,
matched_functions: u32,
#[serde(skip_serializing_if = "Option::is_none")]
complete: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
module_name: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
module_id: Option<u32>,
sections: Vec<LegacyReportItem>,
functions: Vec<LegacyReportItem>,
}
impl From<LegacyReportUnit> for ReportUnit {
fn from(value: LegacyReportUnit) -> Self {
Self {
name: value.name.clone(),
fuzzy_match_percent: value.fuzzy_match_percent,
total_code: value.total_code,
matched_code: value.matched_code,
total_data: value.total_data,
matched_data: value.matched_data,
total_functions: value.total_functions,
matched_functions: value.matched_functions,
complete: value.complete,
module_name: value.module_name.clone(),
module_id: value.module_id,
sections: value.sections.into_iter().map(ReportItem::from).collect(),
functions: value.functions.into_iter().map(ReportItem::from).collect(),
}
}
}
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
struct LegacyReportItem {
name: String,
#[serde(skip_serializing_if = "Option::is_none")]
demangled_name: Option<String>,
#[serde(
default,
skip_serializing_if = "Option::is_none",
serialize_with = "serialize_hex",
deserialize_with = "deserialize_hex"
)]
address: Option<u64>,
size: u64,
fuzzy_match_percent: f32,
}
impl From<LegacyReportItem> for ReportItem {
fn from(value: LegacyReportItem) -> Self {
Self {
name: value.name,
demangled_name: value.demangled_name,
address: value.address,
size: value.size,
fuzzy_match_percent: value.fuzzy_match_percent,
}
}
}
fn serialize_hex<S>(x: &Option<u64>, s: S) -> Result<S::Ok, S::Error>
where S: serde::Serializer {
if let Some(x) = x {
s.serialize_str(&format!("{:#x}", x))
} else {
s.serialize_none()
}
}
fn deserialize_hex<'de, D>(d: D) -> Result<Option<u64>, D::Error>
where D: serde::Deserializer<'de> {
use serde::Deserialize;
let s = String::deserialize(d)?;
if s.is_empty() {
Ok(None)
} else if !s.starts_with("0x") {
Err(serde::de::Error::custom("expected hex string"))
} else {
u64::from_str_radix(&s[2..], 16).map(Some).map_err(serde::de::Error::custom)
}
}