fix(calibration): learn held out empty residuals

docs/adr/ADR-355-local-empty-room-bootstrap-prior.md

@@ -54,6 +54,14 @@ evidence or numeric heart and breathing rates. Explicit calibration replaces
 it with fresh statistics. Cancelling an unfinished capture restores a valid
 prior. Administrator scoped reset removes it.
 
+A restored image may receive one operator confirmed held out empty refinement.
+The server measures 12 fresh, source bound runtime residuals at one second
+spacing and requires zero stale samples and zero numeric vital samples. It
+persists only the scalar residuals. The new boundary is the larger of the
+existing threshold and 110 percent of the held out 95th percentile, capped at
+125 percent of the prior threshold. The one use marker persists in the model,
+so repeated requests cannot ratchet the threshold upward.
+
 The status endpoint reports the learned runtime window, residual threshold,
 reference count, current background conformance, and the negative only
 authority boundary. A restored model never resumes collection and never
@@ -94,6 +102,9 @@ occupied sequences. Empty data is never relabeled as human training.
 7. `POST /api/v1/calibration/start?source_node_id=N` selects a live source for
    deterministic single link capture. A missing or stale selection fails
    without disabling the active bootstrap prior.
+8. `POST /api/v1/calibration/bootstrap/refine?confirmed_empty=true&source_node_id=N`
+   performs the bounded one time held out refinement. It requires administrator
+   authority when transport authentication is enabled.
 
 ## Measured installation evidence
 
@@ -118,3 +129,6 @@ vitals. Twelve fresh empty samples must contain no numeric vital signs.
 Starting a new room calibration must replace the prior and begin at frame zero.
 When several radios are live, selecting Node 5 must leave Nodes 3 and 7 out of
 the source set for the entire capture.
+After restart, a confirmed empty holdout may refine the restored boundary once.
+The stored snapshot must reject a second refinement and must continue to deny
+calibrated evidence and numeric vital authority.

v2/crates/wifi-densepose-sensing-server/src/main.rs

@@ -6264,6 +6264,13 @@ struct CalibrationStartQuery {
     source_node_id: Option<u8>,
 }
 
+#[derive(Debug, Default, Deserialize)]
+struct BootstrapRefineQuery {
+    #[serde(default)]
+    confirmed_empty: bool,
+    source_node_id: Option<u8>,
+}
+
 async fn calibration_start(
     State(state): State<SharedState>,
     Query(query): Query<CalibrationStartQuery>,
@@ -6401,6 +6408,7 @@ async fn calibration_status(State(state): State<SharedState>) -> Json<serde_json
             "significant_eigenvalue_p95": modes.baseline_runtime_eigenvalue_count,
             "residual_energy_threshold": modes.empty_room_residual_energy_threshold,
             "residual_reference_window_count": modes.empty_room_residual_energy_reference.len(),
+            "residual_refinement_count": modes.empty_room_residual_refinement_count,
             "raw_calibration_frames_persisted": false,
         }))
     });
@@ -6611,6 +6619,227 @@ async fn calibration_promote_bootstrap(
     }))
 }
 
+/// Measure one separate operator-confirmed empty holdout and make a bounded,
+/// single-use adjustment to a restored bootstrap residual boundary. Only
+/// scalar residual energies enter the persisted image; raw CSI never does.
+async fn calibration_refine_bootstrap(
+    State(state): State<SharedState>,
+    Query(query): Query<BootstrapRefineQuery>,
+) -> Json<serde_json::Value> {
+    if !query.confirmed_empty {
+        return Json(serde_json::json!({
+            "success": false,
+            "error_code": "empty_confirmation_required",
+            "error": "Explicit empty-room confirmation is required for bootstrap refinement.",
+        }));
+    }
+
+    let (expected_model_id, expected_digest, source_node_id) = {
+        let s = state.read().await;
+        let Some(metadata) = s.bootstrap_baseline.as_ref().filter(|_| s.bootstrap_baseline_active)
+        else {
+            return Json(serde_json::json!({
+                "success": false,
+                "error_code": "bootstrap_not_active",
+                "error": "A restored startup baseline must be active before refinement.",
+            }));
+        };
+        let [source_node_id] = metadata.source_node_ids.as_slice() else {
+            return Json(serde_json::json!({
+                "success": false,
+                "error_code": "bootstrap_source_invalid",
+                "error": "Bootstrap refinement requires exactly one bound source node.",
+            }));
+        };
+        if query
+            .source_node_id
+            .is_some_and(|requested| requested != *source_node_id)
+        {
+            return Json(serde_json::json!({
+                "success": false,
+                "error_code": "bootstrap_source_mismatch",
+                "error": "The requested source does not match the stored bootstrap source.",
+            }));
+        }
+        (
+            metadata.source_model_id.clone(),
+            metadata.content_sha256.clone(),
+            *source_node_id,
+        )
+    };
+
+    let mut residuals = Vec::with_capacity(BOOTSTRAP_VALIDATION_SAMPLES);
+    let mut stale_sample_count = 0usize;
+    let mut vital_sign_sample_count = 0usize;
+    let mut last_sequence = None;
+    let mut next_sample_at = tokio::time::Instant::now();
+    for _ in 0..BOOTSTRAP_VALIDATION_SAMPLES {
+        tokio::time::sleep_until(next_sample_at).await;
+        let interval_deadline = tokio::time::Instant::now()
+            + Duration::from_millis(BOOTSTRAP_VALIDATION_SAMPLE_TIMEOUT_MS);
+        let mut observed = None;
+        while tokio::time::Instant::now() < interval_deadline {
+            observed = {
+                let s = state.read().await;
+                let identity_matches = s.bootstrap_baseline_active
+                    && s.bootstrap_baseline.as_ref().is_some_and(|metadata| {
+                        metadata.source_model_id == expected_model_id
+                            && metadata.content_sha256 == expected_digest
+                            && metadata.source_node_ids.as_slice() == [source_node_id]
+                    });
+                if !identity_matches {
+                    None
+                } else {
+                    s.node_states.get(&source_node_id).and_then(|node| {
+                        let sequence = node.latest_csi_sequence?;
+                        let fresh = node.last_frame_time.is_some_and(|seen| {
+                            std::time::Instant::now().saturating_duration_since(seen)
+                                < ESP32_OFFLINE_TIMEOUT
+                        }) && Some(sequence) != last_sequence;
+                        if !fresh {
+                            return None;
+                        }
+                        let model = s.field_model.as_ref()?;
+                        let recent_frames: Vec<Vec<f64>> =
+                            node.frame_history.iter().cloned().collect();
+                        let residual = model.empty_room_match(&recent_frames)?.residual_energy;
+                        let vital_signs_absent = s
+                            .latest_update
+                            .as_ref()
+                            .is_some_and(|update| update.vital_signs.is_none());
+                        Some((sequence, residual, vital_signs_absent))
+                    })
+                }
+            };
+            if observed.is_some() {
+                break;
+            }
+            tokio::time::sleep(Duration::from_millis(50)).await;
+        }
+        if let Some((sequence, residual, vital_signs_absent)) = observed {
+            last_sequence = Some(sequence);
+            residuals.push(residual);
+            if !vital_signs_absent {
+                vital_sign_sample_count += 1;
+            }
+        } else {
+            stale_sample_count += 1;
+        }
+        next_sample_at = tokio::time::Instant::now()
+            + Duration::from_millis(BOOTSTRAP_VALIDATION_MIN_SPACING_MS);
+    }
+
+    if residuals.len() != BOOTSTRAP_VALIDATION_SAMPLES
+        || stale_sample_count != 0
+        || vital_sign_sample_count != 0
+    {
+        return Json(serde_json::json!({
+            "success": false,
+            "error_code": "bootstrap_refinement_validation_failed",
+            "error": "The fresh empty-room refinement gate did not pass. The stored baseline was not changed.",
+            "validation": {
+                "sample_count": residuals.len(),
+                "stale_sample_count": stale_sample_count,
+                "vital_sign_sample_count": vital_sign_sample_count,
+            },
+        }));
+    }
+
+    let mut s = state.write().await;
+    let identity_matches = s.bootstrap_baseline_active
+        && s.bootstrap_baseline.as_ref().is_some_and(|metadata| {
+            metadata.source_model_id == expected_model_id
+                && metadata.content_sha256 == expected_digest
+                && metadata.source_node_ids.as_slice() == [source_node_id]
+        });
+    if !identity_matches {
+        return Json(serde_json::json!({
+            "success": false,
+            "error_code": "bootstrap_changed",
+            "error": "The bootstrap identity changed during refinement. The stored baseline was not changed.",
+        }));
+    }
+    let candidate_snapshot = match s
+        .field_model
+        .as_ref()
+        .ok_or(wifi_densepose_signal::ruvsense::field_model::FieldModelError::NotCalibrated)
+        .and_then(|model| model.export_snapshot())
+    {
+        Ok(snapshot) => snapshot,
+        Err(_) => {
+            return Json(serde_json::json!({
+                "success": false,
+                "error_code": "bootstrap_refinement_rejected",
+                "error": "The bootstrap model rejected this bounded refinement.",
+            }));
+        }
+    };
+    let now_us = chrono::Utc::now().timestamp_micros() as u64;
+    let mut candidate = match FieldModel::from_snapshot(candidate_snapshot, now_us) {
+        Ok(model) => model,
+        Err(_) => {
+            return Json(serde_json::json!({
+                "success": false,
+                "error_code": "bootstrap_refinement_rejected",
+                "error": "The bootstrap model rejected this bounded refinement.",
+            }));
+        }
+    };
+    let refinement = match candidate.refine_empty_room_residual_boundary(&residuals) {
+        Ok(refinement) => refinement,
+        Err(_) => {
+            return Json(serde_json::json!({
+                "success": false,
+                "error_code": "bootstrap_refinement_rejected",
+                "error": "The bootstrap model rejected this bounded refinement.",
+            }));
+        }
+    };
+    let Some(installation_id) = s.installation_id.clone() else {
+        return Json(serde_json::json!({
+            "success": false,
+            "error_code": "installation_id_required",
+            "error": "A stable installation identity is required to store a local startup baseline.",
+        }));
+    };
+    let path = bootstrap_baseline::path_in(&s.data_dir);
+    let created_at_unix_ms = chrono::Utc::now().timestamp_millis() as u64;
+    let stored = match bootstrap_baseline::store(
+        &path,
+        &installation_id,
+        &[source_node_id],
+        &expected_model_id,
+        created_at_unix_ms,
+        &candidate,
+    ) {
+        Ok(metadata) => metadata,
+        Err(error) => {
+            return error_response::internal_error_json("bootstrap baseline refinement store", error)
+        }
+    };
+    s.field_model = Some(candidate);
+    s.bootstrap_baseline = Some(stored.clone());
+    Json(serde_json::json!({
+        "success": true,
+        "message": "The local empty-room startup baseline was refined.",
+        "validation": {
+            "sample_count": residuals.len(),
+            "stale_sample_count": stale_sample_count,
+            "vital_sign_sample_count": vital_sign_sample_count,
+        },
+        "refinement": {
+            "threshold_before": refinement.threshold_before,
+            "threshold_after": refinement.threshold_after,
+            "held_out_p95": refinement.held_out_p95,
+            "held_out_sample_count": refinement.held_out_sample_count,
+            "reference_window_count": refinement.reference_window_count,
+        },
+        "bootstrap_baseline": stored,
+        "calibrated_evidence_authorized": false,
+        "numeric_vitals_authorized": false,
+    }))
+}
+
 /// Cancel only an unfinished capture. Completed model deletion remains on the
 /// administrator scoped reset route.
 async fn calibration_cancel(State(state): State<SharedState>) -> Json<serde_json::Value> {
@@ -10054,6 +10283,10 @@ async fn main() {
             "/api/v1/calibration/bootstrap/promote",
             post(calibration_promote_bootstrap),
         )
+        .route(
+            "/api/v1/calibration/bootstrap/refine",
+            post(calibration_refine_bootstrap),
+        )
         .route("/api/v1/calibration/cancel", post(calibration_cancel))
         .route("/api/v1/calibration/status", get(calibration_status))
         .route("/api/v1/calibration/reset", post(calibration_reset))

v2/crates/wifi-densepose-signal/src/ruvsense/field_model.rs

@@ -57,6 +57,10 @@ pub const MIN_CALIBRATION_FRAMES: usize =
     RUNTIME_OCCUPANCY_WINDOW * MIN_BACKGROUND_REFERENCE_WINDOWS;
 const CALIBRATION_BACKGROUND_WINDOWS_MAX: usize = 512;
 const EMPTY_ROOM_RESIDUAL_MARGIN: f64 = 1.5;
+const EMPTY_ROOM_HELD_OUT_MARGIN: f64 = 1.10;
+const EMPTY_ROOM_REFINEMENT_MAX_LIFT: f64 = 1.25;
+const EMPTY_ROOM_REFINEMENT_MIN_SAMPLES: usize = 10;
+const EMPTY_ROOM_REFINEMENT_MAX_SAMPLES: usize = 64;
 
 // ---------------------------------------------------------------------------
 // Error types
@@ -362,6 +366,11 @@ pub struct FieldNormalMode {
     /// an empty room model into positive person evidence.
     #[serde(default)]
     pub empty_room_residual_energy_reference: Vec<f64>,
+    /// Number of operator-confirmed held-out empty refinements. Version one
+    /// permits one bounded refinement so repeated calls cannot ratchet the
+    /// background boundary upward until a weak occupant is suppressed.
+    #[serde(default)]
+    pub empty_room_residual_refinement_count: u8,
 }
 
 /// A bounded comparison between one runtime window and the learned empty room
@@ -377,6 +386,16 @@ pub struct EmptyRoomMatch {
     pub reference_window_count: usize,
 }
 
+/// Receipt for one bounded, operator-confirmed empty-room refinement.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct EmptyRoomRefinement {
+    pub threshold_before: f64,
+    pub threshold_after: f64,
+    pub held_out_p95: f64,
+    pub held_out_sample_count: usize,
+    pub reference_window_count: usize,
+}
+
 /// Body perturbation extracted from a CSI observation.
 ///
 /// After subtracting the baseline and projecting out environmental
@@ -692,6 +711,73 @@ impl FieldModel {
             .and_then(|modes| modes.empty_room_residual_energy_threshold)
     }
 
+    /// Refine a completed single-link empty-room boundary with scalar residuals
+    /// measured during a separate operator-confirmed empty holdout.
+    ///
+    /// This never ingests or persists raw CSI. The lift is capped at 25 percent
+    /// and may happen only once per snapshot, preventing repeated calls from
+    /// silently erasing weak occupied evidence.
+    pub fn refine_empty_room_residual_boundary(
+        &mut self,
+        held_out_residuals: &[f64],
+    ) -> Result<EmptyRoomRefinement, FieldModelError> {
+        if self.config.n_links != 1 || self.status != CalibrationStatus::Fresh {
+            return Err(FieldModelError::NotCalibrated);
+        }
+        if !(EMPTY_ROOM_REFINEMENT_MIN_SAMPLES..=EMPTY_ROOM_REFINEMENT_MAX_SAMPLES)
+            .contains(&held_out_residuals.len())
+            || held_out_residuals
+                .iter()
+                .any(|value| !value.is_finite() || *value < 0.0)
+        {
+            return Err(FieldModelError::InvalidConfig(
+                "held-out empty residuals are missing, excessive, or non-finite".into(),
+            ));
+        }
+        let modes = self.modes.as_mut().ok_or(FieldModelError::NotCalibrated)?;
+        if modes.empty_room_residual_refinement_count != 0 {
+            return Err(FieldModelError::InvalidConfig(
+                "the empty-room boundary already has its one permitted held-out refinement"
+                    .into(),
+            ));
+        }
+        if modes.empty_room_residual_energy_reference.len() + held_out_residuals.len()
+            > CALIBRATION_BACKGROUND_WINDOWS_MAX
+        {
+            return Err(FieldModelError::InvalidConfig(
+                "the empty-room residual reference limit would be exceeded".into(),
+            ));
+        }
+        let threshold_before = modes
+            .empty_room_residual_energy_threshold
+            .ok_or(FieldModelError::NotCalibrated)?
+            .max(EMPTY_ROOM_RESIDUAL_ENERGY_MAX);
+        let held_out_p95 = percentile_95(held_out_residuals.to_vec())
+            .ok_or(FieldModelError::NotCalibrated)?;
+        let proposed = held_out_p95 * EMPTY_ROOM_HELD_OUT_MARGIN;
+        let threshold_after = threshold_before
+            .max(proposed.min(threshold_before * EMPTY_ROOM_REFINEMENT_MAX_LIFT));
+
+        modes
+            .empty_room_residual_energy_reference
+            .extend_from_slice(held_out_residuals);
+        modes
+            .empty_room_residual_energy_reference
+            .sort_by(|left, right| {
+                left.partial_cmp(right).unwrap_or(std::cmp::Ordering::Equal)
+            });
+        modes.empty_room_residual_energy_threshold = Some(threshold_after);
+        modes.empty_room_residual_refinement_count = 1;
+
+        Ok(EmptyRoomRefinement {
+            threshold_before,
+            threshold_after,
+            held_out_p95,
+            held_out_sample_count: held_out_residuals.len(),
+            reference_window_count: modes.empty_room_residual_energy_reference.len(),
+        })
+    }
+
     /// Export aggregate model state suitable for local restart persistence.
     /// Raw calibration observations and Welford accumulators are not exported.
     pub fn export_snapshot(&self) -> Result<FieldModelSnapshotV1, FieldModelError> {
@@ -762,6 +848,7 @@ impl FieldModel {
                 .is_none_or(|threshold| threshold.is_finite() && threshold >= 0.0)
             && modes.empty_room_residual_energy_reference.len()
                 <= CALIBRATION_BACKGROUND_WINDOWS_MAX
+            && modes.empty_room_residual_refinement_count <= 1
             && modes
                 .empty_room_residual_energy_reference
                 .iter()
@@ -1118,6 +1205,7 @@ impl FieldModel {
             baseline_runtime_window_size: None,
             empty_room_residual_energy_threshold: None,
             empty_room_residual_energy_reference: Vec::new(),
+            empty_room_residual_refinement_count: 0,
         };
 
         // The full calibration covariance and a fifty frame runtime covariance
@@ -2066,6 +2154,44 @@ mod tests {
             .is_some_and(|score| (0.5..=1.0).contains(&score)));
     }
 
+    #[test]
+    fn held_out_empty_refinement_is_bounded_and_single_use() {
+        let config = FieldModelConfig {
+            n_links: 1,
+            n_subcarriers: 8,
+            n_modes: 2,
+            min_calibration_frames: 100,
+            min_calibration_duration_s: 0.0,
+            baseline_expiry_s: 86_400.0,
+        };
+        let mut model = FieldModel::new(config).unwrap();
+        for frame_index in 0..1_000 {
+            let phase = frame_index as f64 * 0.071;
+            let frame = (0..8)
+                .map(|index| 20.0 + index as f64 * 0.1 + (phase + index as f64).sin())
+                .collect();
+            model.feed_calibration(&[frame]).unwrap();
+        }
+        model.finalize_calibration(1_000_000, 0).unwrap();
+
+        let before = model.empty_room_residual_energy_threshold().unwrap();
+        let residuals = vec![before * 1.4; 12];
+        let receipt = model
+            .refine_empty_room_residual_boundary(&residuals)
+            .unwrap();
+        assert_eq!(receipt.threshold_before, before);
+        assert!(receipt.threshold_after > before);
+        assert!(receipt.threshold_after <= before * EMPTY_ROOM_REFINEMENT_MAX_LIFT);
+        assert_eq!(receipt.held_out_sample_count, 12);
+        assert_eq!(
+            model.modes().unwrap().empty_room_residual_refinement_count,
+            1
+        );
+        assert!(model
+            .refine_empty_room_residual_boundary(&residuals)
+            .is_err());
+    }
+
     #[cfg(feature = "eigenvalue")]
     #[test]
     fn runtime_reference_rejects_shifted_held_out_window() {