mirror of
https://github.com/encounter/dtk-template.git
synced 2025-12-17 00:47:23 +00:00
Implement progress report regression testing (#50)
* Implement progress report regression testing * Rename to "changes" * chmod+x again * Add [...] when truncating long symbols * Make `ninja baseline` always be rerun, even if file times are older * Make `ninja changes` also always be rerun, ignoring file times
This commit is contained in:
LagoLunatic
GitHub
154
tools/changes_fmt.py
Executable file
154
tools/changes_fmt.py
Executable file
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from argparse import ArgumentParser
|
||||
import os
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Tuple
|
||||
|
||||
script_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
root_dir = os.path.abspath(os.path.join(script_dir, ".."))
|
||||
|
||||
|
||||
UNIT_KEYS_TO_DIFF = [
|
||||
"fuzzy_match_percent",
|
||||
"matched_code_percent",
|
||||
"matched_data_percent",
|
||||
"complete_code_percent",
|
||||
"complete_data_percent",
|
||||
]
|
||||
|
||||
FUNCTION_KEYS_TO_DIFF = [
|
||||
"fuzzy_match_percent",
|
||||
]
|
||||
|
||||
type Change = Tuple[str, str, float, float]
|
||||
|
||||
|
||||
def get_changes(changes_file: str) -> list[Change]:
|
||||
changes_file = os.path.relpath(changes_file, root_dir)
|
||||
with open(changes_file, "r") as f:
|
||||
changes_json = json.load(f)
|
||||
|
||||
regressions = []
|
||||
progressions = []
|
||||
|
||||
def diff_key(object_name: str, object: dict, key: str):
|
||||
from_value = object.get("from", {}).get(key, 0.0)
|
||||
to_value = object.get("to", {}).get(key, 0.0)
|
||||
key = key.removesuffix("_percent")
|
||||
change = (object_name, key, from_value, to_value)
|
||||
if from_value > to_value:
|
||||
regressions.append(change)
|
||||
elif to_value > from_value:
|
||||
progressions.append(change)
|
||||
|
||||
for key in UNIT_KEYS_TO_DIFF:
|
||||
diff_key(None, changes_json, key)
|
||||
|
||||
for unit in changes_json.get("units", []):
|
||||
unit_name = unit["name"]
|
||||
for key in UNIT_KEYS_TO_DIFF:
|
||||
diff_key(unit_name, unit, key)
|
||||
# Ignore sections
|
||||
for func in unit.get("functions", []):
|
||||
func_name = func["name"]
|
||||
for key in FUNCTION_KEYS_TO_DIFF:
|
||||
diff_key(func_name, func, key)
|
||||
|
||||
return regressions, progressions
|
||||
|
||||
|
||||
def generate_changes_plaintext(changes: list[Change]) -> str:
|
||||
if len(changes) == 0:
|
||||
return ""
|
||||
|
||||
table_total_width = 136
|
||||
percents_max_len = 7 + 4 + 7
|
||||
key_max_len = max(len(key) for _, key, _, _ in changes)
|
||||
name_max_len = max(len(name or "Total") for name, _, _, _ in changes)
|
||||
max_width_for_name_col = table_total_width - 3 - key_max_len - 3 - percents_max_len
|
||||
name_max_len = min(max_width_for_name_col, name_max_len)
|
||||
|
||||
out_lines = []
|
||||
for name, key, from_value, to_value in changes:
|
||||
if name is None:
|
||||
name = "Total"
|
||||
if len(name) > name_max_len:
|
||||
name = name[: name_max_len - len("[...]")] + "[...]"
|
||||
out_lines.append(
|
||||
f"{name:>{name_max_len}} | {key:<{key_max_len}} | {from_value:6.2f}% -> {to_value:5.2f}%"
|
||||
)
|
||||
|
||||
return "\n".join(out_lines)
|
||||
|
||||
|
||||
def generate_changes_markdown(changes: list[Change], description: str) -> str:
|
||||
if len(changes) == 0:
|
||||
return ""
|
||||
|
||||
out_lines = []
|
||||
name_max_len = 100
|
||||
|
||||
out_lines.append("<details>")
|
||||
out_lines.append(
|
||||
f"<summary>Detected {len(changes)} {description} compared to the base:</summary>"
|
||||
)
|
||||
out_lines.append("") # Must include a blank line before a table
|
||||
out_lines.append("| Name | Type | Before | After |")
|
||||
out_lines.append("| ---- | ---- | ------ | ----- |")
|
||||
|
||||
for name, key, from_value, to_value in changes:
|
||||
if name is None:
|
||||
name = "Total"
|
||||
else:
|
||||
if len(name) > name_max_len:
|
||||
name = name[: name_max_len - len("...")] + "..."
|
||||
name = f"`{name}`" # Surround with backticks
|
||||
key = key.replace("_", " ").capitalize()
|
||||
out_lines.append(f"| {name} | {key} | {from_value:.2f}% | {to_value:.2f}% |")
|
||||
|
||||
out_lines.append("</details>")
|
||||
|
||||
return "\n".join(out_lines)
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser(description="Format objdiff-cli report changes.")
|
||||
parser.add_argument(
|
||||
"report_changes_file",
|
||||
type=Path,
|
||||
help="""path to the JSON file containing the changes, generated by objdiff-cli.""",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
type=Path,
|
||||
help="""Output file (prints to console if unspecified)""",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--all",
|
||||
action="store_true",
|
||||
help="""Includes progressions as well.""",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
regressions, progressions = get_changes(args.report_changes_file)
|
||||
|
||||
if args.output:
|
||||
markdown_output = generate_changes_markdown(regressions, "regressions")
|
||||
if args.all:
|
||||
markdown_output += generate_changes_markdown(progressions, "progressions")
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
f.write(markdown_output)
|
||||
else:
|
||||
if args.all:
|
||||
changes = progressions + regressions
|
||||
else:
|
||||
changes = regressions
|
||||
text_output = generate_changes_plaintext(changes)
|
||||
print(text_output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1215,6 +1215,81 @@ def generate_build_ninja(
|
||||
order_only="post-build",
|
||||
)
|
||||
|
||||
n.comment("Phony edge that will always be considered dirty by ninja.")
|
||||
n.comment(
|
||||
"This can be used as an implicit to a target that should always be rerun, ignoring file modified times."
|
||||
)
|
||||
n.build(
|
||||
outputs="always",
|
||||
rule="phony",
|
||||
)
|
||||
n.newline()
|
||||
|
||||
###
|
||||
# Regression test progress reports
|
||||
###
|
||||
report_baseline_path = build_path / "baseline.json"
|
||||
report_changes_path = build_path / "report_changes.json"
|
||||
changes_fmt = config.tools_dir / "changes_fmt.py"
|
||||
regressions_md = build_path / "regressions.md"
|
||||
n.comment(
|
||||
"Create a baseline progress report for later match regression testing"
|
||||
)
|
||||
n.build(
|
||||
outputs=report_baseline_path,
|
||||
rule="report",
|
||||
implicit=[objdiff, "all_source", "always"],
|
||||
order_only="post-build",
|
||||
)
|
||||
n.build(
|
||||
outputs="baseline",
|
||||
rule="phony",
|
||||
inputs=report_baseline_path,
|
||||
)
|
||||
n.comment("Check for any match regressions against the baseline")
|
||||
n.comment("Will fail if no baseline has been created")
|
||||
n.rule(
|
||||
name="report_changes",
|
||||
command=f"{objdiff} report changes --format json-pretty {report_baseline_path} $in -o $out",
|
||||
description="CHANGES",
|
||||
)
|
||||
n.build(
|
||||
outputs=report_changes_path,
|
||||
rule="report_changes",
|
||||
inputs=report_path,
|
||||
implicit=[objdiff, "always"],
|
||||
)
|
||||
n.rule(
|
||||
name="changes_fmt",
|
||||
command=f"$python {changes_fmt} $args $in",
|
||||
description="CHANGESFMT",
|
||||
)
|
||||
n.build(
|
||||
outputs="changes",
|
||||
rule="changes_fmt",
|
||||
inputs=report_changes_path,
|
||||
implicit=changes_fmt,
|
||||
)
|
||||
n.build(
|
||||
outputs="changes_all",
|
||||
rule="changes_fmt",
|
||||
inputs=report_changes_path,
|
||||
implicit=changes_fmt,
|
||||
variables={"args": "--all"},
|
||||
)
|
||||
n.rule(
|
||||
name="changes_md",
|
||||
command=f"$python {changes_fmt} $in -o $out",
|
||||
description="CHANGESFMT $out",
|
||||
)
|
||||
n.build(
|
||||
outputs=regressions_md,
|
||||
rule="changes_md",
|
||||
inputs=report_changes_path,
|
||||
implicit=changes_fmt,
|
||||
)
|
||||
n.newline()
|
||||
|
||||
###
|
||||
# Helper tools
|
||||
###
|
||||
|
||||
Reference in New Issue
Block a user