feat: require matching recorded context in performance comparisons

tools/performance_compare.py

@@ -162,12 +162,16 @@ def main(argv=None):
                                      "Reports numerical differences, not statistical significance or an automatic pass/fail verdict.")
     parser.add_argument("baseline", type=Path, help="Baseline summary JSONL file")
     parser.add_argument("candidate", type=Path, help="Candidate summary JSONL file")
+    parser.add_argument("--require-matching-context", action="store_true",
+                        help="reject differences in recorded configuration; this does not check the title, scene or CPU")
     parser.add_argument("--baseline-trace", help="Exact trace field to select in the baseline file")
     parser.add_argument("--candidate-trace", help="Exact trace field to select in the candidate file")
     parser.add_argument("--json", action="store_true", help="Write JSON instead of Markdown")
     args = parser.parse_args(argv)
     try:
         report = compare(load_summary(args.baseline, args.baseline_trace), load_summary(args.candidate, args.candidate_trace))
+        if args.require_matching_context and report["configuration_differences"]:
+            raise ValueError("recorded configuration differs: " + ", ".join(report["configuration_differences"]))
         output = json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False) + "\n" if args.json else markdown(report)
     except (OSError, ValueError, OverflowError) as error:
         parser.exit(2, f"{parser.prog}: {error}\n")

tools/tests/test_performance_compare.py

@@ -201,6 +201,32 @@ class PerformanceCompareTests(unittest.TestCase):
         self.assertEqual(result.returncode, 0, result.stderr)
         self.assertEqual(json.loads(result.stdout)["baseline"], self.baseline)
 
+    def test_matching_context_can_be_required_without_a_timing_verdict(self):
+        self.write(self.before, self.baseline)
+        self.candidate["frame_ms"]["max"] = 100.0
+        self.write(self.after, self.candidate)
+        result = self.run_cli("--require-matching-context", "--json")
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(json.loads(result.stdout)["metrics_ms"]["frame_ms.max"]["change"], 60.0)
+        for field in ("gpu", "driver", "os", "resolution"):
+            with self.subTest(field=field):
+                candidate = copy.deepcopy(self.candidate)
+                candidate[field] += " changed"
+                self.write(self.after, candidate)
+                result = self.run_cli("--require-matching-context")
+                self.assertEqual(result.returncode, 2, result.stderr)
+                self.assertIn(field, result.stderr)
+                self.assertEqual(result.stdout, "")
+                self.assertEqual(self.run_cli().returncode, 0)
+        for key, value in (("window", dict(self.candidate["window"], first_present=900)),
+                           ("env", {"APS5_TRACE_FRAMES": "900:4"})):
+            with self.subTest(key=key):
+                candidate = copy.deepcopy(self.candidate)
+                candidate[key] = value
+                self.write(self.after, candidate)
+                self.assertEqual(self.run_cli("--require-matching-context").returncode, 2)
+
+
 
 if __name__ == "__main__":
     unittest.main()