mirror of
https://github.com/boykopovar/AnyPS5.git
synced 2026-10-18 23:07:07 -05:00
179 lines
9.5 KiB
Python
179 lines
9.5 KiB
Python
import argparse
|
|
import json
|
|
import math
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
CONTEXT = ("gpu", "driver", "os", "resolution")
|
|
PERCENTILES = ("p50", "p90", "p99", "max")
|
|
|
|
|
|
def number(value, name):
|
|
try:
|
|
valid = type(value) in (int, float) and math.isfinite(value) and value >= 0
|
|
except OverflowError:
|
|
valid = False
|
|
if not valid:
|
|
raise ValueError(f"{name} must be a finite, non-negative number")
|
|
return value
|
|
|
|
|
|
def validate(summary):
|
|
for key in ("commit", "trace", *CONTEXT):
|
|
if not isinstance(summary.get(key), str):
|
|
raise ValueError(f"{key} must be a string")
|
|
if type(summary.get("trace_dropped")) is not bool:
|
|
raise ValueError("trace_dropped must be a boolean")
|
|
if "dirty" not in summary or (summary["dirty"] is not None and type(summary["dirty"]) is not bool):
|
|
raise ValueError("dirty must be a boolean or null")
|
|
env = summary.get("env")
|
|
if not isinstance(env, dict) or any(not isinstance(v, str) for v in env.values()):
|
|
raise ValueError("env must map names to strings")
|
|
window = summary.get("window")
|
|
if not isinstance(window, dict) or not isinstance(window.get("spec"), str):
|
|
raise ValueError("window must contain a string spec")
|
|
if number(window.get("ms"), "window.ms") == 0:
|
|
raise ValueError("window.ms must be positive")
|
|
if type(window.get("first_present")) is not int or window["first_present"] < 0:
|
|
raise ValueError("window.first_present must be a non-negative integer")
|
|
if type(summary.get("presents")) is not int or summary["presents"] <= 0:
|
|
raise ValueError("presents must be a positive integer")
|
|
number(summary["presents"], "presents")
|
|
frames = summary.get("frame_ms")
|
|
if not isinstance(frames, dict):
|
|
raise ValueError("frame_ms must be an object")
|
|
values = [number(frames.get(key), f"frame_ms.{key}") for key in PERCENTILES]
|
|
if values != sorted(values):
|
|
raise ValueError("frame_ms must satisfy p50 <= p90 <= p99 <= max")
|
|
number(summary.get("gpu_busy_ms_per_present"), "gpu_busy_ms_per_present")
|
|
phases = summary.get("top_phases_ms_per_frame")
|
|
if not isinstance(phases, dict):
|
|
raise ValueError("top_phases_ms_per_frame must be an object")
|
|
for name, value in phases.items():
|
|
number(value, f"top_phases_ms_per_frame.{name}")
|
|
return summary
|
|
|
|
|
|
def load_summary(path, trace=None):
|
|
records = []
|
|
with Path(path).open(encoding="utf-8-sig") as source:
|
|
for line_number, line in enumerate(source, 1):
|
|
if not line.strip():
|
|
continue
|
|
try:
|
|
record = json.loads(line)
|
|
if not isinstance(record, dict):
|
|
raise ValueError("summary must be an object")
|
|
except ValueError as error:
|
|
raise ValueError(f"{path}:{line_number}: {error}") from error
|
|
if trace is None or record.get("trace") == trace:
|
|
records.append((line_number, record))
|
|
if len(records) != 1:
|
|
selection = f" matching trace {trace!r}" if trace is not None else ""
|
|
raise ValueError(f"{path}: expected one summary{selection}, found {len(records)}; "
|
|
"select a unique trace with --baseline-trace/--candidate-trace "
|
|
"or provide a file containing one summary line")
|
|
line_number, record = records[0]
|
|
try:
|
|
return validate(record)
|
|
except ValueError as error:
|
|
raise ValueError(f"{path}:{line_number}: {error}") from error
|
|
|
|
|
|
def delta(baseline, candidate):
|
|
change = candidate - baseline if baseline is not None and candidate is not None else None
|
|
percent = change / baseline * 100 if change is not None and baseline else None
|
|
if change is not None and not math.isfinite(change):
|
|
raise ValueError("metric difference is outside the finite numeric range")
|
|
if percent is not None and not math.isfinite(percent):
|
|
raise ValueError("metric percentage change is outside the finite numeric range")
|
|
return {"baseline": baseline, "candidate": candidate, "change": change, "change_pct": percent}
|
|
|
|
|
|
def compare(baseline, candidate):
|
|
metrics = {f"frame_ms.{key}": delta(baseline["frame_ms"][key], candidate["frame_ms"][key])
|
|
for key in PERCENTILES}
|
|
metrics["mean_frame_ms"] = delta(baseline["window"]["ms"] / baseline["presents"],
|
|
candidate["window"]["ms"] / candidate["presents"])
|
|
metrics["gpu_busy_ms_per_present"] = delta(baseline["gpu_busy_ms_per_present"], candidate["gpu_busy_ms_per_present"])
|
|
before_phases = baseline["top_phases_ms_per_frame"]
|
|
after_phases = candidate["top_phases_ms_per_frame"]
|
|
phases = {name: delta(before_phases.get(name), after_phases.get(name))
|
|
for name in sorted(before_phases.keys() | after_phases.keys())}
|
|
contexts = []
|
|
for summary in (baseline, candidate):
|
|
context = {key: summary[key] for key in CONTEXT}
|
|
context.update({f"window.{key}": summary["window"][key] for key in ("spec", "first_present")})
|
|
context.update({f"env.{key}": value for key, value in summary["env"].items()})
|
|
contexts.append(context)
|
|
differences = {key: {"baseline": contexts[0].get(key), "candidate": contexts[1].get(key)}
|
|
for key in sorted(contexts[0].keys() | contexts[1].keys())
|
|
if contexts[0].get(key) != contexts[1].get(key)}
|
|
notes = ["Compare captures from the same machine, title and scene; summaries do not record the CPU, title or scene.",
|
|
"Changes are candidate minus baseline. A negative frame-time change means a shorter frame interval.",
|
|
"A missing top phase is unreported, not zero; its change is unavailable."]
|
|
if differences:
|
|
notes.append("Recorded configuration differs; these measurements do not isolate the code change.")
|
|
for label, summary in (("baseline", baseline), ("candidate", candidate)):
|
|
if summary["trace_dropped"]:
|
|
notes.append(f"{label}: trace events were dropped; the event trace omits some of the work included in phase totals.")
|
|
if summary["dirty"] is not False:
|
|
notes.append(f"{label}: source tree is dirty or its state is unknown; the commit alone does not identify the build.")
|
|
missing = [key for key in CONTEXT if not summary[key]]
|
|
if missing:
|
|
notes.append(f"{label}: missing configuration: {', '.join(missing)}.")
|
|
return {"baseline": baseline, "candidate": candidate, "configuration_differences": differences,
|
|
"metrics_ms": metrics, "top_phases_ms_per_frame": phases, "notes": notes}
|
|
|
|
|
|
def cell(value):
|
|
text = json.dumps(value, ensure_ascii=False) if not isinstance(value, str) else value
|
|
return text.replace("&", "&").replace("<", "<").replace(">", ">").replace("|", "|").replace("\r", " ").replace("\n", " ")
|
|
|
|
|
|
def markdown(report):
|
|
lines = ["# Performance comparison", "", "| Capture | Commit | Trace | Presents | Window (ms) |",
|
|
"| --- | --- | --- | ---: | ---: |"]
|
|
for label in ("baseline", "candidate"):
|
|
summary = report[label]
|
|
lines.append(f"| {label} | {cell(summary['commit'])} | {cell(summary['trace'])} | "
|
|
f"{summary['presents']} | {summary['window']['ms']:.3f} |")
|
|
lines.extend(["", *[f"- {note}" for note in report["notes"]]])
|
|
if report["configuration_differences"]:
|
|
lines.extend(["", "## Configuration differences", "", "| Setting | Baseline | Candidate |", "| --- | --- | --- |"])
|
|
for key, values in report["configuration_differences"].items():
|
|
lines.append(f"| {cell(key)} | {cell(values['baseline'])} | {cell(values['candidate'])} |")
|
|
for title, key in (("Frame and GPU times (ms)", "metrics_ms"), ("Top phases (ms per frame)", "top_phases_ms_per_frame")):
|
|
lines.extend(["", f"## {title}", "", "| Metric | Baseline | Candidate | Change | Change (%) |",
|
|
"| --- | ---: | ---: | ---: | ---: |"])
|
|
for name, values in report[key].items():
|
|
columns = ["n/a" if values[field] is None else format(values[field], "+.3f" if field.startswith("change") else ".3f")
|
|
for field in ("baseline", "candidate", "change", "change_pct")]
|
|
lines.append(f"| {cell(name)} | {' | '.join(columns)} |")
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def main(argv=None):
|
|
for stream in (sys.stdout, sys.stderr):
|
|
if hasattr(stream, "reconfigure"):
|
|
stream.reconfigure(encoding="utf-8")
|
|
parser = argparse.ArgumentParser(description="Compare two aps5-summary.jsonl captures. "
|
|
"Reports numerical differences, not statistical significance or an automatic pass/fail verdict.")
|
|
parser.add_argument("baseline", type=Path, help="Baseline summary JSONL file")
|
|
parser.add_argument("candidate", type=Path, help="Candidate summary JSONL file")
|
|
parser.add_argument("--baseline-trace", help="Exact trace field to select in the baseline file")
|
|
parser.add_argument("--candidate-trace", help="Exact trace field to select in the candidate file")
|
|
parser.add_argument("--json", action="store_true", help="Write JSON instead of Markdown")
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
report = compare(load_summary(args.baseline, args.baseline_trace), load_summary(args.candidate, args.candidate_trace))
|
|
output = json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False) + "\n" if args.json else markdown(report)
|
|
except (OSError, ValueError, OverflowError) as error:
|
|
parser.exit(2, f"{parser.prog}: {error}\n")
|
|
sys.stdout.write(output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|