Implement progress report regression testing (#50)

* Implement progress report regression testing

* Rename to "changes"

* chmod+x again

* Add [...] when truncating long symbols

* Make `ninja baseline` always be rerun, even if file times are older

* Make `ninja changes` also always be rerun, ignoring file times

tools/changes_fmt.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+
+from argparse import ArgumentParser
+import os
+import json
+from pathlib import Path
+from typing import Tuple
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+root_dir = os.path.abspath(os.path.join(script_dir, ".."))
+
+
+UNIT_KEYS_TO_DIFF = [
+    "fuzzy_match_percent",
+    "matched_code_percent",
+    "matched_data_percent",
+    "complete_code_percent",
+    "complete_data_percent",
+]
+
+FUNCTION_KEYS_TO_DIFF = [
+    "fuzzy_match_percent",
+]
+
+type Change = Tuple[str, str, float, float]
+
+
+def get_changes(changes_file: str) -> list[Change]:
+    changes_file = os.path.relpath(changes_file, root_dir)
+    with open(changes_file, "r") as f:
+        changes_json = json.load(f)
+
+    regressions = []
+    progressions = []
+
+    def diff_key(object_name: str, object: dict, key: str):
+        from_value = object.get("from", {}).get(key, 0.0)
+        to_value = object.get("to", {}).get(key, 0.0)
+        key = key.removesuffix("_percent")
+        change = (object_name, key, from_value, to_value)
+        if from_value > to_value:
+            regressions.append(change)
+        elif to_value > from_value:
+            progressions.append(change)
+
+    for key in UNIT_KEYS_TO_DIFF:
+        diff_key(None, changes_json, key)
+
+    for unit in changes_json.get("units", []):
+        unit_name = unit["name"]
+        for key in UNIT_KEYS_TO_DIFF:
+            diff_key(unit_name, unit, key)
+        # Ignore sections
+        for func in unit.get("functions", []):
+            func_name = func["name"]
+            for key in FUNCTION_KEYS_TO_DIFF:
+                diff_key(func_name, func, key)
+
+    return regressions, progressions
+
+
+def generate_changes_plaintext(changes: list[Change]) -> str:
+    if len(changes) == 0:
+        return ""
+
+    table_total_width = 136
+    percents_max_len = 7 + 4 + 7
+    key_max_len = max(len(key) for _, key, _, _ in changes)
+    name_max_len = max(len(name or "Total") for name, _, _, _ in changes)
+    max_width_for_name_col = table_total_width - 3 - key_max_len - 3 - percents_max_len
+    name_max_len = min(max_width_for_name_col, name_max_len)
+
+    out_lines = []
+    for name, key, from_value, to_value in changes:
+        if name is None:
+            name = "Total"
+        if len(name) > name_max_len:
+            name = name[: name_max_len - len("[...]")] + "[...]"
+        out_lines.append(
+            f"{name:>{name_max_len}} | {key:<{key_max_len}} | {from_value:6.2f}% -> {to_value:5.2f}%"
+        )
+
+    return "\n".join(out_lines)
+
+
+def generate_changes_markdown(changes: list[Change], description: str) -> str:
+    if len(changes) == 0:
+        return ""
+
+    out_lines = []
+    name_max_len = 100
+
+    out_lines.append("<details>")
+    out_lines.append(
+        f"<summary>Detected {len(changes)} {description} compared to the base:</summary>"
+    )
+    out_lines.append("")  # Must include a blank line before a table
+    out_lines.append("| Name | Type | Before | After |")
+    out_lines.append("| ---- | ---- | ------ | ----- |")
+
+    for name, key, from_value, to_value in changes:
+        if name is None:
+            name = "Total"
+        else:
+            if len(name) > name_max_len:
+                name = name[: name_max_len - len("...")] + "..."
+            name = f"`{name}`"  # Surround with backticks
+        key = key.replace("_", " ").capitalize()
+        out_lines.append(f"| {name} | {key} | {from_value:.2f}% | {to_value:.2f}% |")
+
+    out_lines.append("</details>")
+
+    return "\n".join(out_lines)
+
+
+def main():
+    parser = ArgumentParser(description="Format objdiff-cli report changes.")
+    parser.add_argument(
+        "report_changes_file",
+        type=Path,
+        help="""path to the JSON file containing the changes, generated by objdiff-cli.""",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=Path,
+        help="""Output file (prints to console if unspecified)""",
+    )
+    parser.add_argument(
+        "--all",
+        action="store_true",
+        help="""Includes progressions as well.""",
+    )
+    args = parser.parse_args()
+
+    regressions, progressions = get_changes(args.report_changes_file)
+
+    if args.output:
+        markdown_output = generate_changes_markdown(regressions, "regressions")
+        if args.all:
+            markdown_output += generate_changes_markdown(progressions, "progressions")
+        with open(args.output, "w", encoding="utf-8") as f:
+            f.write(markdown_output)
+    else:
+        if args.all:
+            changes = progressions + regressions
+        else:
+            changes = regressions
+        text_output = generate_changes_plaintext(changes)
+        print(text_output)
+
+
+if __name__ == "__main__":
+    main()

tools/project.py

@@ -1215,6 +1215,81 @@ def generate_build_ninja(
             order_only="post-build",
         )
 
+        n.comment("Phony edge that will always be considered dirty by ninja.")
+        n.comment(
+            "This can be used as an implicit to a target that should always be rerun, ignoring file modified times."
+        )
+        n.build(
+            outputs="always",
+            rule="phony",
+        )
+        n.newline()
+
+        ###
+        # Regression test progress reports
+        ###
+        report_baseline_path = build_path / "baseline.json"
+        report_changes_path = build_path / "report_changes.json"
+        changes_fmt = config.tools_dir / "changes_fmt.py"
+        regressions_md = build_path / "regressions.md"
+        n.comment(
+            "Create a baseline progress report for later match regression testing"
+        )
+        n.build(
+            outputs=report_baseline_path,
+            rule="report",
+            implicit=[objdiff, "all_source", "always"],
+            order_only="post-build",
+        )
+        n.build(
+            outputs="baseline",
+            rule="phony",
+            inputs=report_baseline_path,
+        )
+        n.comment("Check for any match regressions against the baseline")
+        n.comment("Will fail if no baseline has been created")
+        n.rule(
+            name="report_changes",
+            command=f"{objdiff} report changes --format json-pretty {report_baseline_path} $in -o $out",
+            description="CHANGES",
+        )
+        n.build(
+            outputs=report_changes_path,
+            rule="report_changes",
+            inputs=report_path,
+            implicit=[objdiff, "always"],
+        )
+        n.rule(
+            name="changes_fmt",
+            command=f"$python {changes_fmt} $args $in",
+            description="CHANGESFMT",
+        )
+        n.build(
+            outputs="changes",
+            rule="changes_fmt",
+            inputs=report_changes_path,
+            implicit=changes_fmt,
+        )
+        n.build(
+            outputs="changes_all",
+            rule="changes_fmt",
+            inputs=report_changes_path,
+            implicit=changes_fmt,
+            variables={"args": "--all"},
+        )
+        n.rule(
+            name="changes_md",
+            command=f"$python {changes_fmt} $in -o $out",
+            description="CHANGESFMT $out",
+        )
+        n.build(
+            outputs=regressions_md,
+            rule="changes_md",
+            inputs=report_changes_path,
+            implicit=changes_fmt,
+        )
+        n.newline()
+
         ###
         # Helper tools
         ###