diff --git a/tools/performance_compare.py b/tools/performance_compare.py index bbbf7c949d..426a9957f7 100644 --- a/tools/performance_compare.py +++ b/tools/performance_compare.py @@ -162,12 +162,16 @@ def main(argv=None): "Reports numerical differences, not statistical significance or an automatic pass/fail verdict.") parser.add_argument("baseline", type=Path, help="Baseline summary JSONL file") parser.add_argument("candidate", type=Path, help="Candidate summary JSONL file") + parser.add_argument("--require-matching-context", action="store_true", + help="reject differences in recorded configuration; this does not check the title, scene or CPU") parser.add_argument("--baseline-trace", help="Exact trace field to select in the baseline file") parser.add_argument("--candidate-trace", help="Exact trace field to select in the candidate file") parser.add_argument("--json", action="store_true", help="Write JSON instead of Markdown") args = parser.parse_args(argv) try: report = compare(load_summary(args.baseline, args.baseline_trace), load_summary(args.candidate, args.candidate_trace)) + if args.require_matching_context and report["configuration_differences"]: + raise ValueError("recorded configuration differs: " + ", ".join(report["configuration_differences"])) output = json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False) + "\n" if args.json else markdown(report) except (OSError, ValueError, OverflowError) as error: parser.exit(2, f"{parser.prog}: {error}\n") diff --git a/tools/tests/test_performance_compare.py b/tools/tests/test_performance_compare.py index 4b27dd7159..3fdb89675e 100644 --- a/tools/tests/test_performance_compare.py +++ b/tools/tests/test_performance_compare.py @@ -201,6 +201,32 @@ class PerformanceCompareTests(unittest.TestCase): self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(json.loads(result.stdout)["baseline"], self.baseline) + def test_matching_context_can_be_required_without_a_timing_verdict(self): + self.write(self.before, self.baseline) + self.candidate["frame_ms"]["max"] = 100.0 + self.write(self.after, self.candidate) + result = self.run_cli("--require-matching-context", "--json") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout)["metrics_ms"]["frame_ms.max"]["change"], 60.0) + for field in ("gpu", "driver", "os", "resolution"): + with self.subTest(field=field): + candidate = copy.deepcopy(self.candidate) + candidate[field] += " changed" + self.write(self.after, candidate) + result = self.run_cli("--require-matching-context") + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn(field, result.stderr) + self.assertEqual(result.stdout, "") + self.assertEqual(self.run_cli().returncode, 0) + for key, value in (("window", dict(self.candidate["window"], first_present=900)), + ("env", {"APS5_TRACE_FRAMES": "900:4"})): + with self.subTest(key=key): + candidate = copy.deepcopy(self.candidate) + candidate[key] = value + self.write(self.after, candidate) + self.assertEqual(self.run_cli("--require-matching-context").returncode, 2) + + if __name__ == "__main__": unittest.main()