diff --git a/benchmarks/wiflow-dynamics/STATE.md b/benchmarks/wiflow-dynamics/STATE.md index 56386e479..9869ea64b 100644 --- a/benchmarks/wiflow-dynamics/STATE.md +++ b/benchmarks/wiflow-dynamics/STATE.md @@ -62,20 +62,51 @@ All numbers below are **MEASURED by this work**; the 0.9609 baseline is **CITED* oracle gap. The bound is unchanged (+2.9 pts, t = +22.4): the information is still there. -11. **The unifying mechanism, now MEASURED rather than hypothesised.** At K=9, MC-EIG - selects probe sets with **lower predicted variance** (2.91e-4 vs random's 3.45e-4) - but **higher actual squared error** (2.78e-4 vs 2.52e-4). Error/variance ratio — - 1.0 if calibrated — is **0.96 for EIG vs 0.73 for random**. - **Expected-information-gain selection under a miscalibrated predictive - distribution is an overconfidence-seeking criterion.** EIG hunts for the sets the - model *claims* will shrink its entropy most; when the variance head is - miscalibrated, those are disproportionately the sets where its confidence is - unearned. This explains the whole arc: **every rung failed its calibration - criterion, and EIG's failure is downstream of that.** Calibration is not a - nice-to-have for information-gain probing — it is a **precondition**. - The actionable next step is therefore *not* a better EIG estimator but a - **calibrated** predictive (low-rank + diagonal covariance, heavier tails, or - post-hoc recalibration), with EIG re-tested only once coverage passes. +11. **CORRECTED mechanism (my first reading of my own diagnostic was backwards).** + At K=9, MC-EIG picks sets with lower predicted variance (2.91e-4 vs random's + 3.45e-4) but higher actual squared error (2.78e-4 vs 2.52e-4). Error/variance + ratio — **1.0 if calibrated** — is **0.96 for EIG, 0.73 for random**. + I first read this as "EIG seeks overconfidence". **That is wrong.** 0.96 is the + *well-calibrated* end; 0.73 means the variance head assigns **falsely high** + variance to the sets random happens to pick — sets that are actually accurate. + So EIG is not hunting unearned confidence; it is **avoiding sets the head is + over-cautious about, and those are precisely the accurate ones**. The failure is + variance-head miscalibration in the *under*-confident direction corrupting EIG's + ranking. + **The unifying claim survives the correction** — miscalibration in *either* + direction breaks the ranking EIG depends on, so **calibration is a precondition + for information-gain probing, not a nice-to-have**, and every rung here failed its + calibration criterion. But "overconfidence-seeking" is retracted. + **Caveat on strength of evidence:** this two-number comparison comes from absolute + predicted variance on the *selected* sets, whereas EIG selects on *entropy + reduction from a baseline*. It is **suggestive, not a direct test**. The direct + test is item 12. + Actionable next step is therefore *not* a better EIG estimator but a **calibrated** + predictive (low-rank + diagonal covariance, heavier tails, or post-hoc temperature + scaling), with EIG re-tested only once coverage passes. + +12. **THE DIRECT MEASUREMENT, which supersedes both readings in item 11.** Across all + 27 single-probe candidates, per sample, correlating EIG's **predicted** entropy + reduction against the **realised** squared-error reduction: + **mean per-sample Pearson r = +0.039**; fraction of samples with r > 0 = **0.543** + (chance = 0.5). + **EIG's predicted gain is essentially uncorrelated with realised benefit on this + problem.** That, and not any story about over- or under-confidence, is why EIG + performs like random selection — and why its one systematic tendency (preferring + low predicted variance) is, if anything, mildly harmful. + - **MEASURED:** the criterion is uninformative here (r ≈ 0.04). + - **INFERRED, not measured:** that variance-head miscalibration is *why* it is + uninformative. That remains the leading explanation — every rung failed its + calibration criterion — but the causal link is an inference. The clean test is + the pre-registered next experiment: post-hoc temperature-scale the variance head + on val, then re-run EIG and re-measure this same correlation. If r rises with + calibration, the link is established; if it does not, the criterion is + inadequate for a different reason. + +13. **Robustness caveat:** rung 4 is **single-seed (2026)**. Its headline verdict is + unlikely to flip (t = −9.07 against no-probe), but the diagnostic ratios in item 11 + and the correlation in item 12 have not been replicated across seeds. Rung 3 + taught exactly this lesson. Treat items 11–12 as one-seed measurements. **Nothing here is SOTA-beating, and nothing is comparable to the 96.09% full-CSI number** — different input abstraction (540 raw channels x 20 frames → 270 band means) diff --git a/benchmarks/wiflow-dynamics/harness/src/bin/rung4.rs b/benchmarks/wiflow-dynamics/harness/src/bin/rung4.rs index e090584d6..379c6e251 100644 --- a/benchmarks/wiflow-dynamics/harness/src/bin/rung4.rs +++ b/benchmarks/wiflow-dynamics/harness/src/bin/rung4.rs @@ -276,6 +276,11 @@ fn main() { })); } + // DIRECT test of the selection criterion: across the 27 single-probe candidates, + // does EIG's PREDICTED entropy reduction track the REALISED error reduction? + // This is the measurement item 11's two-number comparison could only gesture at. + let signal = eig_signal_quality(&inf, z, gt, &eval_pairs, MC_SAMPLES, seed ^ 0x5160, &device); + let verdict = if any_beats_noprobe && any_beats_random { "PASS (single seed) -- needs >=3-seed sign consistency before any claim" } else { @@ -297,6 +302,20 @@ fn main() { "mpjpe": no_probe.mpjpe, "coverage": no_probe.coverage }, "degeneracy_guard": check_constant_pose(&no_probe.preds, eval_pairs.len()), "policies": rows, + "eig_signal_quality_K1": { + "question": "across the 27 single-probe candidates, does predicted \ + entropy reduction correlate with realised error reduction?", + "mean_per_sample_pearson_r": signal.0, + "fraction_of_samples_with_positive_r": signal.1, + "interpretation": if signal.0 > 0.1 { + "EIG's signal carries real information about benefit" + } else if signal.0 < -0.1 { + "EIG's signal is ANTI-correlated with benefit" + } else { + "EIG's predicted gain is essentially UNCORRELATED with realised \ + benefit -- the criterion is uninformative on this problem" + } + }, "preregistered": { "1_beats_no_probe_by_2se": any_beats_noprobe, "2_beats_random_by_2se": any_beats_random, @@ -456,6 +475,99 @@ fn mc_eig_greedy( out } +/// Across all 27 single-probe candidates, correlate PREDICTED entropy reduction with +/// REALISED squared-error reduction, per sample. Returns (mean Pearson r, fraction of +/// samples with r > 0). +fn eig_signal_quality( + m: &HeteroV2, + z: &[f32], + gt: &[f32], + pairs: &[usize], + samples: usize, + seed: u64, + d: &NdArrayDevice, +) -> (f64, f64) { + let n = pairs.len(); + let mut rng = Rng(seed); + let empty = vec![Vec::new(); n]; + let y: Vec = pairs.iter().flat_map(|&t| (0..POSE_DIM).map(move |i| gt[(t + 1) * POSE_DIM + i])).collect(); + + let (zt0, mn0, mk0) = planes(z, pairs, &empty, None); + let (pmu0, plv0) = m.pose( + to_t::(zt0.clone(), n, BAND_DIM, d), + to_t::(mn0.clone(), n, BAND_DIM, d), + to_t::(mk0.clone(), n, BAND_DIM, d), + ); + let mu0: Vec = pmu0.into_data().to_vec().unwrap(); + let lv0: Vec = plv0.into_data().to_vec().unwrap(); + let h_before: Vec = + (0..n).map(|i| 0.5 * (0..POSE_DIM).map(|q| lv0[i * POSE_DIM + q] as f64).sum::()).collect(); + let err0: Vec = (0..n) + .map(|i| (0..POSE_DIM).map(|q| { let j = i * POSE_DIM + q; ((mu0[j] - y[j]) as f64).powi(2) }).sum()) + .collect(); + let (omu, olv) = m.observation( + to_t::(zt0, n, BAND_DIM, d), + to_t::(mn0, n, BAND_DIM, d), + to_t::(mk0, n, BAND_DIM, d), + ); + let omud: Vec = omu.into_data().to_vec().unwrap(); + let olvd: Vec = olv.into_data().to_vec().unwrap(); + + let mut pred = vec![vec![0.0f64; NUM_PROBE_GROUPS]; n]; + let mut real = vec![vec![0.0f64; NUM_PROBE_GROUPS]; n]; + for cand in 0..NUM_PROBE_GROUPS { + let gs: Vec> = (0..n).map(|_| vec![cand]).collect(); + // predicted: MC-averaged post-probe entropy using SAMPLED values + let mut h_after = vec![0.0f64; n]; + for _s in 0..samples { + let mut vals = vec![0.0f32; n * BAND_DIM]; + for i in 0..n { + for q in 0..BAND_DIM { + vals[i * BAND_DIM + q] = z[(pairs[i] + 1) * BAND_DIM + q]; + } + for q in group_dims(cand) { + let j = i * BAND_DIM + q; + vals[j] = (omud[j] as f64 + (olvd[j] as f64 * 0.5).exp() * rng.normal()) as f32; + } + } + let (zt, mn, mk) = planes(z, pairs, &gs, Some(&vals)); + let (_, plv) = m.pose( + to_t::(zt, n, BAND_DIM, d), to_t::(mn, n, BAND_DIM, d), to_t::(mk, n, BAND_DIM, d)); + let lv: Vec = plv.into_data().to_vec().unwrap(); + for i in 0..n { + h_after[i] += 0.5 * (0..POSE_DIM).map(|q| lv[i * POSE_DIM + q] as f64).sum::(); + } + } + // realised: TRUE values revealed + let (zt, mn, mk) = planes(z, pairs, &gs, None); + let (pmu, _) = m.pose( + to_t::(zt, n, BAND_DIM, d), to_t::(mn, n, BAND_DIM, d), to_t::(mk, n, BAND_DIM, d)); + let mud: Vec = pmu.into_data().to_vec().unwrap(); + for i in 0..n { + pred[i][cand] = h_before[i] - h_after[i] / samples as f64; + let e: f64 = (0..POSE_DIM) + .map(|q| { let j = i * POSE_DIM + q; ((mud[j] - y[j]) as f64).powi(2) }).sum(); + real[i][cand] = err0[i] - e; + } + } + + let mut rs = Vec::with_capacity(n); + for i in 0..n { + let (a, b) = (&pred[i], &real[i]); + let k = NUM_PROBE_GROUPS as f64; + let (ma, mb) = (a.iter().sum::() / k, b.iter().sum::() / k); + let cov: f64 = a.iter().zip(b).map(|(x, y)| (x - ma) * (y - mb)).sum(); + let va: f64 = a.iter().map(|x| (x - ma).powi(2)).sum::().sqrt(); + let vb: f64 = b.iter().map(|x| (x - mb).powi(2)).sum::().sqrt(); + if va > 1e-12 && vb > 1e-12 { + rs.push(cov / (va * vb)); + } + } + let mean = rs.iter().sum::() / rs.len() as f64; + let pos = rs.iter().filter(|r| **r > 0.0).count() as f64 / rs.len() as f64; + (mean, pos) +} + /// Mean predicted pose variance and mean actual squared error under a given policy. fn confidence_vs_error( m: &HeteroV2, diff --git a/benchmarks/wiflow-dynamics/results/rung4.json b/benchmarks/wiflow-dynamics/results/rung4.json index d0369ff01..67b80877c 100644 --- a/benchmarks/wiflow-dynamics/results/rung4.json +++ b/benchmarks/wiflow-dynamics/results/rung4.json @@ -4,6 +4,12 @@ "pred_std": 0.021679363445033534, "threshold": 0.0001 }, + "eig_signal_quality_K1": { + "fraction_of_samples_with_positive_r": 0.5428, + "interpretation": "EIG's predicted gain is essentially UNCORRELATED with realised benefit -- the criterion is uninformative on this problem", + "mean_per_sample_pearson_r": 0.038657429144175144, + "question": "across the 27 single-probe candidates, does predicted entropy reduction correlate with realised error reduction?" + }, "eval_pairs": 5000, "mc_samples": 4, "measured_by": "harness/src/bin/rung4.rs, this run",