feat(evaluation): gate transition behavior

docs/active-evidence-experiment.md

@@ -142,8 +142,14 @@ power. Pilot data must size a separate confirmation study.
 
 Labels, actual costs, fresh splits and capture completeness require independent
 verification. Consequently even a numerical pass always returns
-`fieldAcceptance: not_established` and `promotionAuthorized: false`. Event counts,
-transition delay and stationary occupant sensitivity need separate field analysis.
+`fieldAcceptance: not_established` and `promotionAuthorized: false`. The paired
+transition gate requires empty-to-occupied and occupied-to-empty events in at
+least two rooms. Room-clustered upper confidence bounds must show no increase in
+interval-aligned delay, unresolved transitions or false-occupied episodes per
+empty hour. Unknown predictions extend delay; a transition that never resolves
+within its truth run is retained with that run's full duration. `overallGate`
+passes only when both duration and transition gates pass. Stationary occupant
+sensitivity and sub-interval acquisition latency still require field analysis.
 Replay cannot compare an action for which no eligible measurement was captured.
 
 ## Receipts and rollback

docs/validation/active-transition-validation-20261013.json

@@ -0,0 +1,66 @@
+{
+  "schemaVersion": 1,
+  "candidate": "active-transition-metrics-v1",
+  "decision": "INCONCLUSIVE",
+  "decisionReason": "Local frozen acceptance passed; hosted checks at the proposed commit were not yet available when this receipt was created.",
+  "baseline": {
+    "commit": "56b0085b0bb21cf26fc4792e65cb3b42c16d2888",
+    "tree": "b8510c010ed1ca8100699cd6ec23d4722fed68f6",
+    "sourceSha256": "1f343aeec73c88c379f7225dd8ac310f8940bf5dda23ac63413f4ab475b767cf"
+  },
+  "candidateFiles": {
+    "scripts/csi_training/active_evaluation.py": "ddafb74b32c14ba23e9a4091329eecb3550c040c9b62879126451471a5124018",
+    "scripts/tests/test_csi_training_active_evaluation.py": "6927c7368fb23b85b80467eb7d1eecd95cb0c0beb68aa1f54aacab78a8f5bd70",
+    "docs/active-evidence-experiment.md": "23d19ac9f5332c2e0a0c11433a99009237dc7fcfcf6eb2151123e2f22136267f"
+  },
+  "hypothesis": "Room-clustered transition delay, unresolved-transition, and false-occupied-episode deltas close the duration-only evaluator gap without allowing unknown predictions to hide delayed resolution.",
+  "data": {
+    "evidence": "SYNTHETIC",
+    "frozenWorkload": {
+      "rooms": 100,
+      "intervalsPerRoom": 100,
+      "resamples": 100,
+      "timingIterationsPerArm": 30,
+      "seed": 42
+    },
+    "splitPolicy": "Not applicable to the deterministic synthetic regression and performance workload; no model fitting or final-test payload access occurred."
+  },
+  "acceptance": {
+    "transitionSemantics": "Both transition directions in at least two rooms; upper 95% room-bootstrap bounds must be no greater than zero for candidate-minus-baseline delay, unresolved fraction, and false episodes per empty hour.",
+    "runtime": "Candidate p95 must be no greater than max(1.5 times baseline, baseline plus 1 ms).",
+    "memory": "Candidate traced peak must be no greater than baseline plus 524288 bytes."
+  },
+  "tests": [
+    {
+      "command": "PYTHONPATH=scripts:/home/ruv/projects/zenodo-intake-deps /home/ruv/projects/ruview-training-env/bin/python -m unittest discover -s scripts/tests -p 'test_csi_training*.py'",
+      "result": "PASS",
+      "count": 87,
+      "seconds": 9.485
+    }
+  ],
+  "performance": {
+    "baselineP95Ms": 62.387706,
+    "candidateP95Ms": 91.731155,
+    "latencyLimitMs": 93.581559,
+    "baselinePeakBytes": 80368,
+    "candidatePeakBytes": 169488,
+    "memoryLimitBytes": 604656,
+    "result": "PASS"
+  },
+  "safety": {
+    "fieldAcceptance": "not_established",
+    "promotionAuthorized": false,
+    "labelProvenanceVerified": false,
+    "unknownPredictionsExtendDelay": true,
+    "unresolvedTransitionsRetained": true
+  },
+  "cost": "Unknown incremental infrastructure cost; existing authorized CPU only, no paid provisioning or training.",
+  "rollback": "Revert the candidate commit; the input schema and existing numericalGate field remain unchanged.",
+  "missingEvidence": [
+    "Hosted checks at the exact proposed commit",
+    "Authenticated field labels and capture provenance",
+    "Fresh multi-room confirmation",
+    "Sub-interval acquisition and actuation latency"
+  ],
+  "nextAction": "Publish to the existing draft branch with an expected-head lease, verify exact-head hosted checks, and keep promotion disabled."
+}

scripts/csi_training/active_evaluation.py

@@ -50,6 +50,80 @@ def _id(value):
     return value
 
 
+TRANSITIONS = ("empty_to_occupied", "occupied_to_empty")
+SIDES = ("baseline", "candidate")
+
+
+def _new_transition_totals():
+    return {"empty_seconds": 0.0,
+            "baseline_false_episodes": 0, "candidate_false_episodes": 0,
+            "empty_to_occupied": [0, 0.0, 0.0, 0, 0],
+            "occupied_to_empty": [0, 0.0, 0.0, 0, 0]}
+
+
+def _collect_transitions(total, intervals):
+    previous_truth = None
+    previous_predictions = {}
+    for start, end, truth, baseline, candidate in intervals:
+        if truth == "empty":
+            total["empty_seconds"] += (end-start)/1000
+        for side, prediction in zip(SIDES, (baseline, candidate)):
+            if (truth == "empty" and prediction == "occupied"
+                    and (previous_truth != "empty"
+                         or previous_predictions.get(side) != "occupied")):
+                total[f"{side}_false_episodes"] += 1
+            previous_predictions[side] = prediction
+        previous_truth = truth
+
+    run_start = 0
+    while run_start < len(intervals):
+        truth = intervals[run_start][2]
+        run_stop = run_start + 1
+        while run_stop < len(intervals) and intervals[run_stop][2] == truth:
+            run_stop += 1
+        if run_start:
+            direction = f"{intervals[run_start-1][2]}_to_{truth}"
+            values = total[direction]
+            values[0] += 1
+            start_ms = intervals[run_start][0]
+            end_ms = intervals[run_stop-1][1]
+            for side_index, tuple_index in enumerate((3, 4)):
+                matched = next((row[0] for row in intervals[run_start:run_stop]
+                                if row[tuple_index] == truth), None)
+                if matched is None:
+                    matched = end_ms
+                    values[3+side_index] += 1
+                values[1+side_index] += matched-start_ms
+        run_start = run_stop
+
+
+def _summarize_transitions(values):
+    empty = sum(row["empty_seconds"] for row in values)
+    if empty == 0:
+        return None
+    baseline_episodes = sum(row["baseline_false_episodes"] for row in values)
+    candidate_episodes = sum(row["candidate_false_episodes"] for row in values)
+    result = {
+        "empty_seconds": empty,
+        "baseline_false_episodes_per_empty_hour": baseline_episodes/empty*3600,
+        "candidate_false_episodes_per_empty_hour": candidate_episodes/empty*3600,
+        "false_episode_delta_per_empty_hour": (candidate_episodes-baseline_episodes)/empty*3600,
+    }
+    for direction in TRANSITIONS:
+        totals = [sum(row[direction][i] for row in values) for i in range(5)]
+        events, baseline_delay, candidate_delay, baseline_unresolved, candidate_unresolved = totals
+        result[direction] = None if events == 0 else {
+            "events": int(events),
+            "baseline_mean_delay_ms": baseline_delay/events,
+            "candidate_mean_delay_ms": candidate_delay/events,
+            "delay_delta_ms": (candidate_delay-baseline_delay)/events,
+            "baseline_unresolved_fraction": baseline_unresolved/events,
+            "candidate_unresolved_fraction": candidate_unresolved/events,
+            "unresolved_fraction_delta": (candidate_unresolved-baseline_unresolved)/events,
+        }
+    return result
+
+
 def evaluate(document, resamples=1000, seed=42):
     exact(document, ("schemaVersion", "evidence", "sessions"))
     if type(document["schemaVersion"]) is not int or document["schemaVersion"] != 1 or document["evidence"] not in ("SYNTHETIC", "MEASURED"):
@@ -59,7 +133,7 @@ def evaluate(document, resamples=1000, seed=42):
         raise ValueError("invalid session count")
     if type(resamples) is not int or not 100 <= resamples <= 10000 or type(seed) is not int or not 0 <= seed < 2**32:
         raise ValueError("invalid bootstrap settings")
-    rooms, identities, interval_count = {}, set(), 0
+    rooms, transition_rooms, identities, interval_count = {}, {}, set(), 0
     for session in sessions:
         exact(session, ("room", "session", "budget", "intervals"))
         room, sid = _id(session["room"]), _id(session["session"])
@@ -92,6 +166,8 @@ def evaluate(document, resamples=1000, seed=42):
             raise ValueError("too many intervals")
         # empty, occupied, total; per policy false, hit, unknown, unknown-empty, unknown-occupied
         totals = rooms.setdefault(room, [0.] * 13)
+        transition_totals = transition_rooms.setdefault(room, _new_transition_totals())
+        validated_intervals = []
         previous = None
         for interval in intervals:
             exact(interval, ("start_ms", "end_ms", "truth", "baseline", "candidate"))
@@ -116,6 +192,9 @@ def evaluate(document, resamples=1000, seed=42):
                 totals[offset+2] += duration * (prediction == "unknown")
                 totals[offset+3] += duration * (prediction == "unknown" and not occupied)
                 totals[offset+4] += duration * (prediction == "unknown" and occupied)
+            validated_intervals.append((start, end, interval["truth"],
+                                        interval["baseline"], interval["candidate"]))
+        _collect_transitions(transition_totals, validated_intervals)
 
     def summarize(values):
         total = [sum(row[i] for row in values) for i in range(13)]
@@ -155,6 +234,51 @@ def evaluate(document, resamples=1000, seed=42):
     eci, oci = interval(unknown_empty), interval(unknown_occupied)
     evaluable = len(rooms) >= 2 and len(reduction) >= .95 * resamples and point["false_duration_relative_reduction"] is not None
     passes = bool(evaluable and point["false_duration_relative_reduction"] >= .20 and rci[0] > 0 and dci[0] >= -.01 and eci[1] <= .01 and oci[1] <= .01)
+    numerical_gate = "pass" if passes else "fail" if evaluable else "inconclusive"
+
+    transition_point = _summarize_transitions(list(transition_rooms.values()))
+    transition_clusters = list(transition_rooms.values())
+    transition_values = {direction: {"delay": [], "unresolved": []}
+                         for direction in TRANSITIONS}
+    false_episodes = []
+    transition_valid = 0
+    for _ in range(resamples):
+        sample = _summarize_transitions([
+            transition_clusters[rng.randrange(len(transition_clusters))]
+            for _ in transition_clusters])
+        if sample is not None and all(sample[direction] is not None
+                                      for direction in TRANSITIONS):
+            transition_valid += 1
+            false_episodes.append(sample["false_episode_delta_per_empty_hour"])
+            for direction in TRANSITIONS:
+                transition_values[direction]["delay"].append(
+                    sample[direction]["delay_delta_ms"])
+                transition_values[direction]["unresolved"].append(
+                    sample[direction]["unresolved_fraction_delta"])
+    transition_intervals = {
+        direction: {
+            "delay_delta_ms_interval": interval(transition_values[direction]["delay"]),
+            "unresolved_fraction_delta_interval": interval(
+                transition_values[direction]["unresolved"]),
+        } for direction in TRANSITIONS}
+    false_episode_interval = interval(false_episodes)
+    transition_room_coverage = {
+        direction: sum(row[direction][0] > 0 for row in transition_clusters)
+        for direction in TRANSITIONS}
+    transition_evaluable = bool(
+        transition_point is not None
+        and all(transition_room_coverage[direction] >= 2 for direction in TRANSITIONS)
+        and transition_valid >= .95*resamples)
+    transition_passes = bool(
+        transition_evaluable and false_episode_interval[1] <= 0
+        and all(transition_intervals[direction]["delay_delta_ms_interval"][1] <= 0
+                and transition_intervals[direction]["unresolved_fraction_delta_interval"][1] <= 0
+                for direction in TRANSITIONS))
+    transition_gate = ("pass" if transition_passes else
+                       "fail" if transition_evaluable else "inconclusive")
+    overall_gate = ("fail" if "fail" in (numerical_gate, transition_gate) else
+                    "pass" if numerical_gate == transition_gate == "pass" else
+                    "inconclusive")
     return {"schemaVersion": 1, "inputDeclaredEvidence": document["evidence"],
             "labelProvenanceVerified": False, "promotionAuthorized": False,
             "rooms": len(rooms), "sessions": len(sessions), "intervals": interval_count,
@@ -163,9 +287,16 @@ def evaluate(document, resamples=1000, seed=42):
             "reduction_interval": rci, "recall_delta_interval": dci,
             "recall_valid": len(recall), "unknown_empty_delta_interval": eci,
             "unknown_occupied_delta_interval": oci},
-            "numericalGate": "pass" if passes else "fail" if evaluable else "inconclusive",
+            "numericalGate": numerical_gate,
+            "transitionMetrics": transition_point,
+            "transitionBootstrap": {"cluster": "room", "seed": seed,
+                "resamples": resamples, "valid": transition_valid,
+                "confidence": .95, "room_coverage": transition_room_coverage,
+                "false_episode_delta_per_empty_hour_interval": false_episode_interval,
+                **transition_intervals},
+            "transitionGate": transition_gate, "overallGate": overall_gate,
             "fieldAcceptance": "not_established",
             "limitations": ["Declared labels and budget costs are not independently authenticated",
                             "Fresh room splits and capture completeness require external review",
                             "Bootstrap intervals on few rooms may be unstable; size a confirmation study",
-                            "Event counts and detection delay require separate transition evaluation"]}
+                            "Transition timing is interval-aligned and excludes sub-interval, acquisition and actuation latency"]}

scripts/tests/test_csi_training_active_evaluation.py

@@ -18,6 +18,27 @@ def fixture():
     return dict(schemaVersion=1, evidence="SYNTHETIC", sessions=sessions)
 
 
+def transition_fixture():
+    budget = dict(packets=10, airtime_ms=20, energy_mj=30,
+                  available_sensors=["wifi", "radar"])
+    sessions = []
+    for i in range(4):
+        sessions.append(dict(room=f"room{i}", session=f"transition{i}",
+                             budget=dict(baseline=budget.copy(),
+                                         candidate=budget.copy()), intervals=[
+            dict(start_ms=0, end_ms=1000, truth="empty",
+                 baseline="occupied", candidate="empty"),
+            dict(start_ms=1000, end_ms=2000, truth="occupied",
+                 baseline="unknown", candidate="occupied"),
+            dict(start_ms=2000, end_ms=3000, truth="occupied",
+                 baseline="occupied", candidate="occupied"),
+            dict(start_ms=3000, end_ms=4000, truth="empty",
+                 baseline="occupied", candidate="empty"),
+            dict(start_ms=4000, end_ms=5000, truth="empty",
+                 baseline="empty", candidate="empty")]))
+    return dict(schemaVersion=1, evidence="SYNTHETIC", sessions=sessions)
+
+
 class ActiveEvaluationTests(unittest.TestCase):
     def test_duration_and_gate_never_authorize_promotion(self):
         result = evaluate(fixture())
@@ -59,6 +80,81 @@ class ActiveEvaluationTests(unittest.TestCase):
         self.assertEqual(result["bootstrap"]["recall_valid"], 1000)
         self.assertEqual(result["bootstrap"]["recall_delta_interval"], [0, 0])
 
+    def test_transition_gate_rewards_faster_resolution_in_both_directions(self):
+        result = evaluate(transition_fixture())
+        self.assertEqual(result["numericalGate"], "pass")
+        self.assertEqual(result["transitionGate"], "pass")
+        self.assertEqual(result["overallGate"], "pass")
+        self.assertEqual(
+            result["transitionMetrics"]["empty_to_occupied"]["delay_delta_ms"],
+            -1000)
+        self.assertEqual(
+            result["transitionMetrics"]["occupied_to_empty"]["delay_delta_ms"],
+            -1000)
+        self.assertFalse(result["promotionAuthorized"])
+        self.assertEqual(result["fieldAcceptance"], "not_established")
+
+    def test_equal_duration_cannot_hide_later_occupied_transition(self):
+        doc = transition_fixture()
+        for session in doc["sessions"]:
+            session["intervals"][1].update(baseline="occupied",
+                                            candidate="unknown")
+            session["intervals"][2].update(baseline="unknown",
+                                            candidate="occupied")
+        result = evaluate(doc)
+        self.assertEqual(result["numericalGate"], "pass")
+        self.assertGreater(
+            result["transitionMetrics"]["empty_to_occupied"]["delay_delta_ms"],
+            0)
+        self.assertEqual(result["transitionGate"], "fail")
+        self.assertEqual(result["overallGate"], "fail")
+
+    def test_unresolved_unknown_is_counted_and_fails(self):
+        doc = transition_fixture()
+        for session in doc["sessions"]:
+            session["intervals"][1]["candidate"] = "unknown"
+            session["intervals"][2]["candidate"] = "unknown"
+        result = evaluate(doc)
+        metrics = result["transitionMetrics"]["empty_to_occupied"]
+        self.assertEqual(metrics["candidate_unresolved_fraction"], 1)
+        self.assertEqual(metrics["candidate_mean_delay_ms"], 2000)
+        self.assertEqual(result["transitionGate"], "fail")
+
+    def test_false_duration_fragmentation_cannot_hide_extra_episodes(self):
+        doc = transition_fixture()
+        for session in doc["sessions"]:
+            session["intervals"] = [
+                dict(start_ms=0, end_ms=1000, truth="empty",
+                     baseline="empty", candidate="empty"),
+                dict(start_ms=1000, end_ms=2000, truth="occupied",
+                     baseline="occupied", candidate="occupied"),
+                dict(start_ms=2000, end_ms=2500, truth="empty",
+                     baseline="occupied", candidate="occupied"),
+                dict(start_ms=2500, end_ms=3000, truth="empty",
+                     baseline="occupied", candidate="empty"),
+                dict(start_ms=3000, end_ms=3500, truth="empty",
+                     baseline="empty", candidate="occupied"),
+                dict(start_ms=3500, end_ms=4000, truth="empty",
+                     baseline="empty", candidate="empty")]
+        result = evaluate(doc)
+        self.assertEqual(
+            result["metrics"]["baseline_false_seconds_per_empty_hour"],
+            result["metrics"]["candidate_false_seconds_per_empty_hour"])
+        self.assertGreater(
+            result["transitionMetrics"]["false_episode_delta_per_empty_hour"],
+            0)
+        self.assertEqual(result["transitionGate"], "fail")
+
+    def test_transition_gate_requires_two_rooms_and_both_directions(self):
+        one_room = transition_fixture()
+        one_room["sessions"] = one_room["sessions"][:1]
+        self.assertEqual(evaluate(one_room)["transitionGate"], "inconclusive")
+        one_direction = transition_fixture()
+        for session in one_direction["sessions"]:
+            session["intervals"] = session["intervals"][:3]
+        self.assertEqual(
+            evaluate(one_direction)["transitionGate"], "inconclusive")
+
     def test_room_clustering_keeps_sessions_together(self):
         doc = fixture()
         doc["sessions"][1]["room"] = "room0"