diff --git a/docs/active-evidence-experiment.md b/docs/active-evidence-experiment.md index 4ef3ad3d1..02c7bb4c4 100644 --- a/docs/active-evidence-experiment.md +++ b/docs/active-evidence-experiment.md @@ -142,8 +142,14 @@ power. Pilot data must size a separate confirmation study. Labels, actual costs, fresh splits and capture completeness require independent verification. Consequently even a numerical pass always returns -`fieldAcceptance: not_established` and `promotionAuthorized: false`. Event counts, -transition delay and stationary occupant sensitivity need separate field analysis. +`fieldAcceptance: not_established` and `promotionAuthorized: false`. The paired +transition gate requires empty-to-occupied and occupied-to-empty events in at +least two rooms. Room-clustered upper confidence bounds must show no increase in +interval-aligned delay, unresolved transitions or false-occupied episodes per +empty hour. Unknown predictions extend delay; a transition that never resolves +within its truth run is retained with that run's full duration. `overallGate` +passes only when both duration and transition gates pass. Stationary occupant +sensitivity and sub-interval acquisition latency still require field analysis. Replay cannot compare an action for which no eligible measurement was captured. ## Receipts and rollback diff --git a/docs/validation/active-transition-validation-20261013.json b/docs/validation/active-transition-validation-20261013.json new file mode 100644 index 000000000..401760492 --- /dev/null +++ b/docs/validation/active-transition-validation-20261013.json @@ -0,0 +1,66 @@ +{ + "schemaVersion": 1, + "candidate": "active-transition-metrics-v1", + "decision": "INCONCLUSIVE", + "decisionReason": "Local frozen acceptance passed; hosted checks at the proposed commit were not yet available when this receipt was created.", + "baseline": { + "commit": "56b0085b0bb21cf26fc4792e65cb3b42c16d2888", + "tree": "b8510c010ed1ca8100699cd6ec23d4722fed68f6", + "sourceSha256": "1f343aeec73c88c379f7225dd8ac310f8940bf5dda23ac63413f4ab475b767cf" + }, + "candidateFiles": { + "scripts/csi_training/active_evaluation.py": "ddafb74b32c14ba23e9a4091329eecb3550c040c9b62879126451471a5124018", + "scripts/tests/test_csi_training_active_evaluation.py": "6927c7368fb23b85b80467eb7d1eecd95cb0c0beb68aa1f54aacab78a8f5bd70", + "docs/active-evidence-experiment.md": "23d19ac9f5332c2e0a0c11433a99009237dc7fcfcf6eb2151123e2f22136267f" + }, + "hypothesis": "Room-clustered transition delay, unresolved-transition, and false-occupied-episode deltas close the duration-only evaluator gap without allowing unknown predictions to hide delayed resolution.", + "data": { + "evidence": "SYNTHETIC", + "frozenWorkload": { + "rooms": 100, + "intervalsPerRoom": 100, + "resamples": 100, + "timingIterationsPerArm": 30, + "seed": 42 + }, + "splitPolicy": "Not applicable to the deterministic synthetic regression and performance workload; no model fitting or final-test payload access occurred." + }, + "acceptance": { + "transitionSemantics": "Both transition directions in at least two rooms; upper 95% room-bootstrap bounds must be no greater than zero for candidate-minus-baseline delay, unresolved fraction, and false episodes per empty hour.", + "runtime": "Candidate p95 must be no greater than max(1.5 times baseline, baseline plus 1 ms).", + "memory": "Candidate traced peak must be no greater than baseline plus 524288 bytes." + }, + "tests": [ + { + "command": "PYTHONPATH=scripts:/home/ruv/projects/zenodo-intake-deps /home/ruv/projects/ruview-training-env/bin/python -m unittest discover -s scripts/tests -p 'test_csi_training*.py'", + "result": "PASS", + "count": 87, + "seconds": 9.485 + } + ], + "performance": { + "baselineP95Ms": 62.387706, + "candidateP95Ms": 91.731155, + "latencyLimitMs": 93.581559, + "baselinePeakBytes": 80368, + "candidatePeakBytes": 169488, + "memoryLimitBytes": 604656, + "result": "PASS" + }, + "safety": { + "fieldAcceptance": "not_established", + "promotionAuthorized": false, + "labelProvenanceVerified": false, + "unknownPredictionsExtendDelay": true, + "unresolvedTransitionsRetained": true + }, + "cost": "Unknown incremental infrastructure cost; existing authorized CPU only, no paid provisioning or training.", + "rollback": "Revert the candidate commit; the input schema and existing numericalGate field remain unchanged.", + "missingEvidence": [ + "Hosted checks at the exact proposed commit", + "Authenticated field labels and capture provenance", + "Fresh multi-room confirmation", + "Sub-interval acquisition and actuation latency" + ], + "nextAction": "Publish to the existing draft branch with an expected-head lease, verify exact-head hosted checks, and keep promotion disabled." +} diff --git a/scripts/csi_training/active_evaluation.py b/scripts/csi_training/active_evaluation.py index 3a809a0bd..9a908f544 100644 --- a/scripts/csi_training/active_evaluation.py +++ b/scripts/csi_training/active_evaluation.py @@ -50,6 +50,80 @@ def _id(value): return value +TRANSITIONS = ("empty_to_occupied", "occupied_to_empty") +SIDES = ("baseline", "candidate") + + +def _new_transition_totals(): + return {"empty_seconds": 0.0, + "baseline_false_episodes": 0, "candidate_false_episodes": 0, + "empty_to_occupied": [0, 0.0, 0.0, 0, 0], + "occupied_to_empty": [0, 0.0, 0.0, 0, 0]} + + +def _collect_transitions(total, intervals): + previous_truth = None + previous_predictions = {} + for start, end, truth, baseline, candidate in intervals: + if truth == "empty": + total["empty_seconds"] += (end-start)/1000 + for side, prediction in zip(SIDES, (baseline, candidate)): + if (truth == "empty" and prediction == "occupied" + and (previous_truth != "empty" + or previous_predictions.get(side) != "occupied")): + total[f"{side}_false_episodes"] += 1 + previous_predictions[side] = prediction + previous_truth = truth + + run_start = 0 + while run_start < len(intervals): + truth = intervals[run_start][2] + run_stop = run_start + 1 + while run_stop < len(intervals) and intervals[run_stop][2] == truth: + run_stop += 1 + if run_start: + direction = f"{intervals[run_start-1][2]}_to_{truth}" + values = total[direction] + values[0] += 1 + start_ms = intervals[run_start][0] + end_ms = intervals[run_stop-1][1] + for side_index, tuple_index in enumerate((3, 4)): + matched = next((row[0] for row in intervals[run_start:run_stop] + if row[tuple_index] == truth), None) + if matched is None: + matched = end_ms + values[3+side_index] += 1 + values[1+side_index] += matched-start_ms + run_start = run_stop + + +def _summarize_transitions(values): + empty = sum(row["empty_seconds"] for row in values) + if empty == 0: + return None + baseline_episodes = sum(row["baseline_false_episodes"] for row in values) + candidate_episodes = sum(row["candidate_false_episodes"] for row in values) + result = { + "empty_seconds": empty, + "baseline_false_episodes_per_empty_hour": baseline_episodes/empty*3600, + "candidate_false_episodes_per_empty_hour": candidate_episodes/empty*3600, + "false_episode_delta_per_empty_hour": (candidate_episodes-baseline_episodes)/empty*3600, + } + for direction in TRANSITIONS: + totals = [sum(row[direction][i] for row in values) for i in range(5)] + events, baseline_delay, candidate_delay, baseline_unresolved, candidate_unresolved = totals + result[direction] = None if events == 0 else { + "events": int(events), + "baseline_mean_delay_ms": baseline_delay/events, + "candidate_mean_delay_ms": candidate_delay/events, + "delay_delta_ms": (candidate_delay-baseline_delay)/events, + "baseline_unresolved_fraction": baseline_unresolved/events, + "candidate_unresolved_fraction": candidate_unresolved/events, + "unresolved_fraction_delta": (candidate_unresolved-baseline_unresolved)/events, + } + return result + + def evaluate(document, resamples=1000, seed=42): exact(document, ("schemaVersion", "evidence", "sessions")) if type(document["schemaVersion"]) is not int or document["schemaVersion"] != 1 or document["evidence"] not in ("SYNTHETIC", "MEASURED"): @@ -59,7 +133,7 @@ def evaluate(document, resamples=1000, seed=42): raise ValueError("invalid session count") if type(resamples) is not int or not 100 <= resamples <= 10000 or type(seed) is not int or not 0 <= seed < 2**32: raise ValueError("invalid bootstrap settings") - rooms, identities, interval_count = {}, set(), 0 + rooms, transition_rooms, identities, interval_count = {}, {}, set(), 0 for session in sessions: exact(session, ("room", "session", "budget", "intervals")) room, sid = _id(session["room"]), _id(session["session"]) @@ -92,6 +166,8 @@ def evaluate(document, resamples=1000, seed=42): raise ValueError("too many intervals") # empty, occupied, total; per policy false, hit, unknown, unknown-empty, unknown-occupied totals = rooms.setdefault(room, [0.] * 13) + transition_totals = transition_rooms.setdefault(room, _new_transition_totals()) + validated_intervals = [] previous = None for interval in intervals: exact(interval, ("start_ms", "end_ms", "truth", "baseline", "candidate")) @@ -116,6 +192,9 @@ def evaluate(document, resamples=1000, seed=42): totals[offset+2] += duration * (prediction == "unknown") totals[offset+3] += duration * (prediction == "unknown" and not occupied) totals[offset+4] += duration * (prediction == "unknown" and occupied) + validated_intervals.append((start, end, interval["truth"], + interval["baseline"], interval["candidate"])) + _collect_transitions(transition_totals, validated_intervals) def summarize(values): total = [sum(row[i] for row in values) for i in range(13)] @@ -155,6 +234,51 @@ def evaluate(document, resamples=1000, seed=42): eci, oci = interval(unknown_empty), interval(unknown_occupied) evaluable = len(rooms) >= 2 and len(reduction) >= .95 * resamples and point["false_duration_relative_reduction"] is not None passes = bool(evaluable and point["false_duration_relative_reduction"] >= .20 and rci[0] > 0 and dci[0] >= -.01 and eci[1] <= .01 and oci[1] <= .01) + numerical_gate = "pass" if passes else "fail" if evaluable else "inconclusive" + + transition_point = _summarize_transitions(list(transition_rooms.values())) + transition_clusters = list(transition_rooms.values()) + transition_values = {direction: {"delay": [], "unresolved": []} + for direction in TRANSITIONS} + false_episodes = [] + transition_valid = 0 + for _ in range(resamples): + sample = _summarize_transitions([ + transition_clusters[rng.randrange(len(transition_clusters))] + for _ in transition_clusters]) + if sample is not None and all(sample[direction] is not None + for direction in TRANSITIONS): + transition_valid += 1 + false_episodes.append(sample["false_episode_delta_per_empty_hour"]) + for direction in TRANSITIONS: + transition_values[direction]["delay"].append( + sample[direction]["delay_delta_ms"]) + transition_values[direction]["unresolved"].append( + sample[direction]["unresolved_fraction_delta"]) + transition_intervals = { + direction: { + "delay_delta_ms_interval": interval(transition_values[direction]["delay"]), + "unresolved_fraction_delta_interval": interval( + transition_values[direction]["unresolved"]), + } for direction in TRANSITIONS} + false_episode_interval = interval(false_episodes) + transition_room_coverage = { + direction: sum(row[direction][0] > 0 for row in transition_clusters) + for direction in TRANSITIONS} + transition_evaluable = bool( + transition_point is not None + and all(transition_room_coverage[direction] >= 2 for direction in TRANSITIONS) + and transition_valid >= .95*resamples) + transition_passes = bool( + transition_evaluable and false_episode_interval[1] <= 0 + and all(transition_intervals[direction]["delay_delta_ms_interval"][1] <= 0 + and transition_intervals[direction]["unresolved_fraction_delta_interval"][1] <= 0 + for direction in TRANSITIONS)) + transition_gate = ("pass" if transition_passes else + "fail" if transition_evaluable else "inconclusive") + overall_gate = ("fail" if "fail" in (numerical_gate, transition_gate) else + "pass" if numerical_gate == transition_gate == "pass" else + "inconclusive") return {"schemaVersion": 1, "inputDeclaredEvidence": document["evidence"], "labelProvenanceVerified": False, "promotionAuthorized": False, "rooms": len(rooms), "sessions": len(sessions), "intervals": interval_count, @@ -163,9 +287,16 @@ def evaluate(document, resamples=1000, seed=42): "reduction_interval": rci, "recall_delta_interval": dci, "recall_valid": len(recall), "unknown_empty_delta_interval": eci, "unknown_occupied_delta_interval": oci}, - "numericalGate": "pass" if passes else "fail" if evaluable else "inconclusive", + "numericalGate": numerical_gate, + "transitionMetrics": transition_point, + "transitionBootstrap": {"cluster": "room", "seed": seed, + "resamples": resamples, "valid": transition_valid, + "confidence": .95, "room_coverage": transition_room_coverage, + "false_episode_delta_per_empty_hour_interval": false_episode_interval, + **transition_intervals}, + "transitionGate": transition_gate, "overallGate": overall_gate, "fieldAcceptance": "not_established", "limitations": ["Declared labels and budget costs are not independently authenticated", "Fresh room splits and capture completeness require external review", "Bootstrap intervals on few rooms may be unstable; size a confirmation study", - "Event counts and detection delay require separate transition evaluation"]} + "Transition timing is interval-aligned and excludes sub-interval, acquisition and actuation latency"]} diff --git a/scripts/tests/test_csi_training_active_evaluation.py b/scripts/tests/test_csi_training_active_evaluation.py index 9b911eba4..fb4167258 100644 --- a/scripts/tests/test_csi_training_active_evaluation.py +++ b/scripts/tests/test_csi_training_active_evaluation.py @@ -18,6 +18,27 @@ def fixture(): return dict(schemaVersion=1, evidence="SYNTHETIC", sessions=sessions) +def transition_fixture(): + budget = dict(packets=10, airtime_ms=20, energy_mj=30, + available_sensors=["wifi", "radar"]) + sessions = [] + for i in range(4): + sessions.append(dict(room=f"room{i}", session=f"transition{i}", + budget=dict(baseline=budget.copy(), + candidate=budget.copy()), intervals=[ + dict(start_ms=0, end_ms=1000, truth="empty", + baseline="occupied", candidate="empty"), + dict(start_ms=1000, end_ms=2000, truth="occupied", + baseline="unknown", candidate="occupied"), + dict(start_ms=2000, end_ms=3000, truth="occupied", + baseline="occupied", candidate="occupied"), + dict(start_ms=3000, end_ms=4000, truth="empty", + baseline="occupied", candidate="empty"), + dict(start_ms=4000, end_ms=5000, truth="empty", + baseline="empty", candidate="empty")])) + return dict(schemaVersion=1, evidence="SYNTHETIC", sessions=sessions) + + class ActiveEvaluationTests(unittest.TestCase): def test_duration_and_gate_never_authorize_promotion(self): result = evaluate(fixture()) @@ -59,6 +80,81 @@ class ActiveEvaluationTests(unittest.TestCase): self.assertEqual(result["bootstrap"]["recall_valid"], 1000) self.assertEqual(result["bootstrap"]["recall_delta_interval"], [0, 0]) + def test_transition_gate_rewards_faster_resolution_in_both_directions(self): + result = evaluate(transition_fixture()) + self.assertEqual(result["numericalGate"], "pass") + self.assertEqual(result["transitionGate"], "pass") + self.assertEqual(result["overallGate"], "pass") + self.assertEqual( + result["transitionMetrics"]["empty_to_occupied"]["delay_delta_ms"], + -1000) + self.assertEqual( + result["transitionMetrics"]["occupied_to_empty"]["delay_delta_ms"], + -1000) + self.assertFalse(result["promotionAuthorized"]) + self.assertEqual(result["fieldAcceptance"], "not_established") + + def test_equal_duration_cannot_hide_later_occupied_transition(self): + doc = transition_fixture() + for session in doc["sessions"]: + session["intervals"][1].update(baseline="occupied", + candidate="unknown") + session["intervals"][2].update(baseline="unknown", + candidate="occupied") + result = evaluate(doc) + self.assertEqual(result["numericalGate"], "pass") + self.assertGreater( + result["transitionMetrics"]["empty_to_occupied"]["delay_delta_ms"], + 0) + self.assertEqual(result["transitionGate"], "fail") + self.assertEqual(result["overallGate"], "fail") + + def test_unresolved_unknown_is_counted_and_fails(self): + doc = transition_fixture() + for session in doc["sessions"]: + session["intervals"][1]["candidate"] = "unknown" + session["intervals"][2]["candidate"] = "unknown" + result = evaluate(doc) + metrics = result["transitionMetrics"]["empty_to_occupied"] + self.assertEqual(metrics["candidate_unresolved_fraction"], 1) + self.assertEqual(metrics["candidate_mean_delay_ms"], 2000) + self.assertEqual(result["transitionGate"], "fail") + + def test_false_duration_fragmentation_cannot_hide_extra_episodes(self): + doc = transition_fixture() + for session in doc["sessions"]: + session["intervals"] = [ + dict(start_ms=0, end_ms=1000, truth="empty", + baseline="empty", candidate="empty"), + dict(start_ms=1000, end_ms=2000, truth="occupied", + baseline="occupied", candidate="occupied"), + dict(start_ms=2000, end_ms=2500, truth="empty", + baseline="occupied", candidate="occupied"), + dict(start_ms=2500, end_ms=3000, truth="empty", + baseline="occupied", candidate="empty"), + dict(start_ms=3000, end_ms=3500, truth="empty", + baseline="empty", candidate="occupied"), + dict(start_ms=3500, end_ms=4000, truth="empty", + baseline="empty", candidate="empty")] + result = evaluate(doc) + self.assertEqual( + result["metrics"]["baseline_false_seconds_per_empty_hour"], + result["metrics"]["candidate_false_seconds_per_empty_hour"]) + self.assertGreater( + result["transitionMetrics"]["false_episode_delta_per_empty_hour"], + 0) + self.assertEqual(result["transitionGate"], "fail") + + def test_transition_gate_requires_two_rooms_and_both_directions(self): + one_room = transition_fixture() + one_room["sessions"] = one_room["sessions"][:1] + self.assertEqual(evaluate(one_room)["transitionGate"], "inconclusive") + one_direction = transition_fixture() + for session in one_direction["sessions"]: + session["intervals"] = session["intervals"][:3] + self.assertEqual( + evaluate(one_direction)["transitionGate"], "inconclusive") + def test_room_clustering_keeps_sessions_together(self): doc = fixture() doc["sessions"][1]["room"] = "room0"