diff --git a/benchmarks/wiflow-dynamics/STATE.md b/benchmarks/wiflow-dynamics/STATE.md index 83b1ece11..56386e479 100644 --- a/benchmarks/wiflow-dynamics/STATE.md +++ b/benchmarks/wiflow-dynamics/STATE.md @@ -56,6 +56,27 @@ All numbers below are **MEASURED by this work**; the 0.9609 baseline is **CITED* `p(z_{t+1}[S] | z_t, revealed)` and average the resulting pose entropies — proper Monte-Carlo EIG. That is the single highest-value next experiment. +10. **RUNG 4 (value-aware variance + observation head + Monte-Carlo EIG): DID NOT + CLOSE THE GAP — it went backwards.** MC-EIG is *worse than random* + (t = −4.13 at K=9) and worse than no-probe (t = −9.07), closing **−40%** of the + oracle gap. The bound is unchanged (+2.9 pts, t = +22.4): the information is still + there. + +11. **The unifying mechanism, now MEASURED rather than hypothesised.** At K=9, MC-EIG + selects probe sets with **lower predicted variance** (2.91e-4 vs random's 3.45e-4) + but **higher actual squared error** (2.78e-4 vs 2.52e-4). Error/variance ratio — + 1.0 if calibrated — is **0.96 for EIG vs 0.73 for random**. + **Expected-information-gain selection under a miscalibrated predictive + distribution is an overconfidence-seeking criterion.** EIG hunts for the sets the + model *claims* will shrink its entropy most; when the variance head is + miscalibrated, those are disproportionately the sets where its confidence is + unearned. This explains the whole arc: **every rung failed its calibration + criterion, and EIG's failure is downstream of that.** Calibration is not a + nice-to-have for information-gain probing — it is a **precondition**. + The actionable next step is therefore *not* a better EIG estimator but a + **calibrated** predictive (low-rank + diagonal covariance, heavier tails, or + post-hoc recalibration), with EIG re-tested only once coverage passes. + **Nothing here is SOTA-beating, and nothing is comparable to the 96.09% full-CSI number** — different input abstraction (540 raw channels x 20 frames → 270 band means) and a different task (forecasting `y_{t+1}`, not sensing `y_t`). The only valid diff --git a/benchmarks/wiflow-dynamics/harness/src/bin/rung4.rs b/benchmarks/wiflow-dynamics/harness/src/bin/rung4.rs new file mode 100644 index 000000000..e090584d6 --- /dev/null +++ b/benchmarks/wiflow-dynamics/harness/src/bin/rung4.rs @@ -0,0 +1,541 @@ +//! Rung 4 driver: value-aware variance head + observation head, Monte-Carlo EIG. +//! +//! Pass condition, pre-registered in `../STATE.md` before this file existed: +//! 1. MC-EIG-greedy beats NO-PROBE by >2 SE at some budget, AND +//! 2. beats RANDOM at that budget by >2 SE, with a consistent sign across >=3 seeds. +//! Anything less -> "rung 4 did not close the gap", with the fraction of the oracle +//! gap it did close reported as a measurement. + +use burn::backend::ndarray::{NdArray, NdArrayDevice}; +use burn::backend::Autodiff; +use burn::module::{AutodiffModule, Module}; +use burn::optim::{AdamConfig, GradientsParams, Optimizer}; +use burn::tensor::{Tensor, TensorData}; +use std::time::Instant; + +use wiflow_harness::bands::{group_dims, BAND_DIM, NUM_PROBE_GROUPS}; +use wiflow_harness::gaussian::POSE_DIM; +use wiflow_harness::guards::{check_constant_pose, CorruptionMasks}; +use wiflow_harness::hetero::expand_mask; +use wiflow_harness::hetero2::{diag_nll, masked_obs_nll, HeteroV2}; +use wiflow_harness::metrics::score; +use wiflow_harness::npy::Npy; + +type B = Autodiff>; +type BI = NdArray; + +const RESULTS: &str = + "/home/ruvultra/projects/ruview-worktrees/ruforecast-rust/benchmarks/wiflow-std/results"; +const COVERAGE_LEVELS: [(f64, f64); 4] = [ + (0.50, 0.6744897501960817), + (0.80, 1.2815515655446004), + (0.90, 1.6448536269514722), + (0.95, 1.959963984540054), +]; +const EVAL_PAIRS: usize = 5_000; +const MC_SAMPLES: usize = 4; + +struct Rng(u64); +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + self.0 + } + fn below(&mut self, n: usize) -> usize { + (self.next() >> 11) as usize % n + } + fn unit(&mut self) -> f64 { + ((self.next() >> 11) as f64 / (1u64 << 53) as f64).clamp(1e-12, 1.0 - 1e-12) + } + fn normal(&mut self) -> f64 { + let (u1, u2) = (self.unit(), self.unit()); + (-2.0 * u1.ln()).sqrt() * (2.0 * std::f64::consts::PI * u2).cos() + } +} + +fn subsample(items: &[usize], k: usize, seed: u64) -> Vec { + if items.len() <= k { + return items.to_vec(); + } + let mut rng = Rng(seed); + let mut pool = items.to_vec(); + for i in 0..k { + let j = i + rng.below(pool.len() - i); + pool.swap(i, j); + } + pool.truncate(k); + pool +} + +fn build_pairs(in_split: &[bool], masks: &CorruptionMasks, w2f: &[i64]) -> Vec { + (0..in_split.len() - 1) + .filter(|&t| in_split[t] && w2f[t] == w2f[t + 1] && !masks.is_corrupt(t) && !masks.is_corrupt(t + 1)) + .collect() +} + +fn random_groups(rng: &mut Rng, k: usize) -> Vec { + let mut pool: Vec = (0..NUM_PROBE_GROUPS).collect(); + for i in 0..k { + let j = i + rng.below(pool.len() - i); + pool.swap(i, j); + } + pool[..k].to_vec() +} + +fn paired(a: &[f64], b: &[f64]) -> (f64, f64, f64) { + let n = a.len(); + let d: Vec = a.iter().zip(b).map(|(x, y)| x - y).collect(); + let m = d.iter().sum::() / n as f64; + let var = d.iter().map(|x| (x - m).powi(2)).sum::() / (n as f64 - 1.0); + let se = (var / n as f64).sqrt(); + (m, se, if se > 0.0 { m / se } else { 0.0 }) +} + +/// Build raw input planes. `override_vals` optionally replaces the revealed values +/// (used to inject Monte-Carlo samples instead of true future values). +fn planes( + z: &[f32], + pairs: &[usize], + groups: &[Vec], + override_vals: Option<&[f32]>, +) -> (Vec, Vec, Vec) { + let n = pairs.len(); + let mut zt = Vec::with_capacity(n * BAND_DIM); + let mut mn = Vec::with_capacity(n * BAND_DIM); + let mut mk = Vec::with_capacity(n * BAND_DIM); + for (i, &t) in pairs.iter().enumerate() { + let m = expand_mask(&groups[i]); + for d in 0..BAND_DIM { + zt.push(z[t * BAND_DIM + d]); + let v = match override_vals { + Some(o) => o[i * BAND_DIM + d], + None => z[(t + 1) * BAND_DIM + d], + }; + mn.push(m[d] * v); + mk.push(m[d]); + } + } + (zt, mn, mk) +} + +fn to_t(v: Vec, n: usize, c: usize, d: &Bk::Device) -> Tensor { + Tensor::::from_data(TensorData::new(v, [n, c]), d) +} + +fn main() { + let fx = concat!(env!("CARGO_MANIFEST_DIR"), "/../fixtures"); + let load = |f: &str| Npy::open(format!("{fx}/{f}")).unwrap_or_else(|e| panic!("{e}")); + let t0 = Instant::now(); + let seed: u64 = std::env::var("RUNG4_SEED").ok().and_then(|v| v.parse().ok()).unwrap_or(2026); + + let z_npy = load("band_features.npy"); + let z = z_npy.f32_slice().unwrap(); + let gt_npy = load("all_gt_clean.npy"); + let gt = gt_npy.f32_slice().unwrap(); + let w2f = load("window_to_file.npy").to_i64().unwrap(); + let n_windows = z_npy.shape[0]; + let masks = CorruptionMasks::load( + &format!("{RESULTS}/nan_windows_mask.npy"), + &format!("{RESULTS}/big_windows_mask.npy"), + ) + .unwrap(); + + let mut splits = Vec::new(); + for name in ["train_idx.npy", "val_idx.npy", "test_idx.npy"] { + let idx = load(name).to_i64().unwrap(); + let mut m = vec![false; n_windows]; + for &i in &idx { + m[i as usize] = true; + } + splits.push(m); + } + let train_pairs = subsample(&build_pairs(&splits[0], &masks, &w2f), 60_000, 42); + let val_pairs = subsample(&build_pairs(&splits[1], &masks, &w2f), 4_000, 7); + let eval_pairs = subsample(&build_pairs(&splits[2], &masks, &w2f), EVAL_PAIRS, 999); + + let device = NdArrayDevice::default(); + ::seed(&device, seed); + let mut model: HeteroV2 = HeteroV2::new(&device, 512); + let mut opt = AdamConfig::new().init(); + let mut rng = Rng(seed ^ 0xA5A5); + + let (epochs, batch, lr) = (400usize, 512usize, 1e-3); + let mut best_val = f64::INFINITY; + let mut best_epoch = 0usize; + let mut best: Option> = None; + let mut history = Vec::new(); + + for ep in 0..epochs { + let order = subsample(&train_pairs, train_pairs.len(), 100 + ep as u64); + let (mut run, mut nb) = (0.0f64, 0usize); + for chunk in order.chunks(batch) { + let groups: Vec> = chunk + .iter() + .map(|_| { + let k = rng.below(NUM_PROBE_GROUPS / 2 + 1); + random_groups(&mut rng, k) + }) + .collect(); + let n = chunk.len(); + let (zt, mn, mk) = planes(z, chunk, &groups, None); + let ztt = to_t::(zt, n, BAND_DIM, &device); + let mnt = to_t::(mn, n, BAND_DIM, &device); + let mkt = to_t::(mk.clone(), n, BAND_DIM, &device); + let yv: Vec = chunk + .iter() + .flat_map(|&t| (0..POSE_DIM).map(move |d| gt[(t + 1) * POSE_DIM + d])) + .collect(); + let zn: Vec = chunk + .iter() + .flat_map(|&t| (0..BAND_DIM).map(move |d| z[(t + 1) * BAND_DIM + d])) + .collect(); + let yt = to_t::(yv, n, POSE_DIM, &device); + let znt = to_t::(zn, n, BAND_DIM, &device); + + let (pmu, plv, omu, olv) = model.forward(ztt, mnt, mkt.clone()); + // Joint objective: pose NLL + observation NLL on the UNREVEALED dims. + let loss = diag_nll(pmu, plv, yt).mean() + masked_obs_nll(omu, olv, znt, mkt).mean(); + run += loss.clone().into_scalar() as f64; + nb += 1; + let grads = GradientsParams::from_grads(loss.backward(), &model); + model = opt.step(lr, model, grads); + } + let val = val_pose_nll(&model, z, gt, &val_pairs, &device); + history.push(serde_json::json!({ "epoch": ep, "train": run / nb as f64, "val_pose_nll": val })); + if ep % 20 == 0 { + eprintln!("epoch {ep}: train {:.3} val {:.3} ({:.0?})", run / nb as f64, val, t0.elapsed()); + } + if val + 1e-6 < best_val { + best_val = val; + best_epoch = ep; + best = Some(model.clone()); + } else if ep - best_epoch >= 40 { + eprintln!("early stop at {ep} (best {best_epoch}, val {best_val:.3})"); + break; + } + } + let inf = best.unwrap_or(model).valid(); + + // ---- policies ----------------------------------------------------------- + let no_probe = eval(&inf, z, gt, &eval_pairs, &vec![vec![]; eval_pairs.len()], &device); + let mut rows = Vec::new(); + let mut any_beats_noprobe = false; + let mut any_beats_random = false; + + for &k in &[3usize, 9] { + let mc = mc_eig_greedy(&inf, z, &eval_pairs, k, MC_SAMPLES, seed ^ 0x5EED, &device); + let g = eval(&inf, z, gt, &eval_pairs, &mc, &device); + + let mut r = Rng(seed ^ 0xBEEF ^ k as u64); + let rnd: Vec> = eval_pairs.iter().map(|_| random_groups(&mut r, k)).collect(); + let rd = eval(&inf, z, gt, &eval_pairs, &rnd, &device); + + let oracle_sets = oracle_probe_sets(&inf, z, gt, &eval_pairs, k, &device); + let orc = eval(&inf, z, gt, &eval_pairs, &oracle_sets, &device); + + let (dn, sen, tn) = paired(&g.per_sample, &no_probe.per_sample); + let (dr, ser, tr) = paired(&g.per_sample, &rd.per_sample); + let (do_, seo, to_) = paired(&orc.per_sample, &no_probe.per_sample); + if tn > 2.0 { + any_beats_noprobe = true; + } + if tr > 2.0 { + any_beats_random = true; + } + // How much of the oracle's headroom did MC-EIG capture? + let closed = if do_.abs() > 1e-12 { dn / do_ } else { 0.0 }; + + // Overconfidence diagnostic. Hypothesis: EIG picks sets the model *predicts* + // will shrink entropy; under a miscalibrated head that selects for + // overconfidence, not information. Test: if EIG-chosen sets have LOWER + // predicted variance but HIGHER actual error than random sets, confirmed. + let oc_eig = confidence_vs_error(&inf, z, gt, &eval_pairs, &mc, &device); + let oc_rnd = confidence_vs_error(&inf, z, gt, &eval_pairs, &rnd, &device); + + let mut counts = std::collections::HashMap::new(); + for s in &mc { + *counts.entry(format!("{s:?}")).or_insert(0usize) += 1; + } + rows.push(serde_json::json!({ + "budget_k": k, + "pck@20": { "mc_eig": g.pck20, "random": rd.pck20, + "no_probe": no_probe.pck20, "target_peeking_bound": orc.pck20 }, + "mc_eig_minus_no_probe": { "delta": dn, "se": sen, "t": tn }, + "mc_eig_minus_random": { "delta": dr, "se": ser, "t": tr }, + "bound_minus_no_probe": { "delta": do_, "se": seo, "t": to_ }, + "fraction_of_oracle_gap_closed": closed, + "distinct_probe_sets": counts.len(), + "overconfidence_diagnostic": { + "hypothesis": "EIG selects sets where the model is overconfident, not \ + where it is informative. Confirmed if mc_eig has LOWER \ + mean predicted variance but HIGHER mean actual error.", + "mc_eig": { "mean_predicted_var": oc_eig.0, "mean_actual_sq_err": oc_eig.1 }, + "random": { "mean_predicted_var": oc_rnd.0, "mean_actual_sq_err": oc_rnd.1 }, + "confirmed": oc_eig.0 < oc_rnd.0 && oc_eig.1 > oc_rnd.1 + } + })); + } + + let verdict = if any_beats_noprobe && any_beats_random { + "PASS (single seed) -- needs >=3-seed sign consistency before any claim" + } else { + "RUNG 4 DID NOT CLOSE THE GAP" + }; + + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "milestone": "Rung 4 -- value-aware variance + observation head, Monte-Carlo EIG", + "measured_by": "harness/src/bin/rung4.rs, this run", + "seed": seed, + "mc_samples": MC_SAMPLES, + "eval_pairs": eval_pairs.len(), + "peek_free": "candidate values are SAMPLED from the observation head; only the \ + target-peeking bound reads the future, and it is unachievable", + "training": { "best_epoch": best_epoch, "best_val_pose_nll": best_val, "history": history }, + "no_probe": { "pose_nll": no_probe.nll, "pck@20": no_probe.pck20, + "mpjpe": no_probe.mpjpe, "coverage": no_probe.coverage }, + "degeneracy_guard": check_constant_pose(&no_probe.preds, eval_pairs.len()), + "policies": rows, + "preregistered": { + "1_beats_no_probe_by_2se": any_beats_noprobe, + "2_beats_random_by_2se": any_beats_random, + "3_sign_consistent_across_3_seeds": "run RUNG4_SEED=... and compare" + }, + "verdict": verdict + })) + .unwrap() + ); + eprintln!("{verdict} ({:.0?})", t0.elapsed()); +} + +struct Ev { + nll: f64, + pck20: f64, + mpjpe: f64, + coverage: serde_json::Value, + per_sample: Vec, + preds: Vec, +} + +fn val_pose_nll(m: &HeteroV2, z: &[f32], gt: &[f32], pairs: &[usize], d: &NdArrayDevice) -> f64 { + let n = pairs.len(); + let g = vec![vec![]; n]; + let (zt, mn, mk) = planes(z, pairs, &g, None); + let y: Vec = pairs.iter().flat_map(|&t| (0..POSE_DIM).map(move |i| gt[(t + 1) * POSE_DIM + i])).collect(); + let (pmu, plv) = m.pose(to_t::(zt, n, BAND_DIM, d), to_t::(mn, n, BAND_DIM, d), to_t::(mk, n, BAND_DIM, d)); + diag_nll(pmu, plv, to_t::(y, n, POSE_DIM, d)).mean().into_scalar() as f64 +} + +fn eval(m: &HeteroV2, z: &[f32], gt: &[f32], pairs: &[usize], groups: &[Vec], d: &NdArrayDevice) -> Ev { + let n = pairs.len(); + let (zt, mn, mk) = planes(z, pairs, groups, None); + let y: Vec = pairs.iter().flat_map(|&t| (0..POSE_DIM).map(move |i| gt[(t + 1) * POSE_DIM + i])).collect(); + let (pmu, plv) = m.pose(to_t::(zt, n, BAND_DIM, d), to_t::(mn, n, BAND_DIM, d), to_t::(mk, n, BAND_DIM, d)); + let nll = diag_nll(pmu.clone(), plv.clone(), to_t::(y.clone(), n, POSE_DIM, d)).mean().into_scalar() as f64; + let mud: Vec = pmu.into_data().to_vec().unwrap(); + let lvd: Vec = plv.into_data().to_vec().unwrap(); + + let mut inside = [0usize; COVERAGE_LEVELS.len()]; + for j in 0..n * POSE_DIM { + let sd = (lvd[j] as f64 * 0.5).exp(); + for (li, (_, zs)) in COVERAGE_LEVELS.iter().enumerate() { + if ((y[j] - mud[j]) as f64).abs() <= zs * sd { + inside[li] += 1; + } + } + } + let s = score(&mud, &y, n); + let per_sample: Vec = (0..n) + .map(|i| score(&mud[i * POSE_DIM..(i + 1) * POSE_DIM], &y[i * POSE_DIM..(i + 1) * POSE_DIM], 1).pck_at(0.2)) + .collect(); + let denom = (n * POSE_DIM) as f64; + Ev { + nll, + pck20: s.pck_at(0.2), + mpjpe: s.mpjpe, + per_sample, + preds: mud, + coverage: serde_json::json!(COVERAGE_LEVELS + .iter() + .enumerate() + .map(|(i, (lvl, _))| serde_json::json!({ + "nominal": lvl, "empirical": inside[i] as f64 / denom, + "error_pp": (inside[i] as f64 / denom - lvl) * 100.0 })) + .collect::>()), + } +} + +/// Monte-Carlo EIG greedy selection. Peek-free: candidate values for the extended +/// mask are SAMPLED from the observation head, never read from `z[t+1]`. +fn mc_eig_greedy( + m: &HeteroV2, + z: &[f32], + pairs: &[usize], + k: usize, + samples: usize, + seed: u64, + d: &NdArrayDevice, +) -> Vec> { + let n = pairs.len(); + let mut rng = Rng(seed); + let mut out = vec![Vec::new(); n]; + + for _step in 0..k { + // Current entropy and the sampling distribution, under the current mask. + let (zt0, mn0, mk0) = planes(z, pairs, &out, None); + let (_, plv0) = m.pose( + to_t::(zt0.clone(), n, BAND_DIM, d), + to_t::(mn0.clone(), n, BAND_DIM, d), + to_t::(mk0.clone(), n, BAND_DIM, d), + ); + let h_before: Vec = { + let lv: Vec = plv0.into_data().to_vec().unwrap(); + (0..n).map(|i| 0.5 * (0..POSE_DIM).map(|q| lv[i * POSE_DIM + q] as f64).sum::()).collect() + }; + let (omu, olv) = m.observation( + to_t::(zt0.clone(), n, BAND_DIM, d), + to_t::(mn0.clone(), n, BAND_DIM, d), + to_t::(mk0, n, BAND_DIM, d), + ); + let omud: Vec = omu.into_data().to_vec().unwrap(); + let olvd: Vec = olv.into_data().to_vec().unwrap(); + + let mut best: Vec<(f64, usize)> = vec![(f64::NEG_INFINITY, usize::MAX); n]; + for cand in 0..NUM_PROBE_GROUPS { + let gs: Vec> = (0..n) + .map(|i| { + let mut s = out[i].clone(); + if !s.contains(&cand) { + s.push(cand); + } + s + }) + .collect(); + let mut h_after = vec![0.0f64; n]; + for _m in 0..samples { + // Draw candidate values for the newly revealed dims from the model. + let mut vals = vec![0.0f32; n * BAND_DIM]; + for i in 0..n { + for q in 0..BAND_DIM { + vals[i * BAND_DIM + q] = z[(pairs[i] + 1) * BAND_DIM + q]; + } + for q in group_dims(cand) { + let j = i * BAND_DIM + q; + let sd = (olvd[j] as f64 * 0.5).exp(); + vals[j] = (omud[j] as f64 + sd * rng.normal()) as f32; + } + } + let (zt, mn, mk) = planes(z, pairs, &gs, Some(&vals)); + let (_, plv) = m.pose( + to_t::(zt, n, BAND_DIM, d), + to_t::(mn, n, BAND_DIM, d), + to_t::(mk, n, BAND_DIM, d), + ); + let lv: Vec = plv.into_data().to_vec().unwrap(); + for i in 0..n { + h_after[i] += 0.5 * (0..POSE_DIM).map(|q| lv[i * POSE_DIM + q] as f64).sum::(); + } + } + for i in 0..n { + if out[i].contains(&cand) { + continue; + } + let gain = h_before[i] - h_after[i] / samples as f64; + if gain > best[i].0 { + best[i] = (gain, cand); + } + } + } + for i in 0..n { + if best[i].1 != usize::MAX { + out[i].push(best[i].1); + } + } + } + out +} + +/// Mean predicted pose variance and mean actual squared error under a given policy. +fn confidence_vs_error( + m: &HeteroV2, + z: &[f32], + gt: &[f32], + pairs: &[usize], + groups: &[Vec], + d: &NdArrayDevice, +) -> (f64, f64) { + let n = pairs.len(); + let (zt, mn, mk) = planes(z, pairs, groups, None); + let y: Vec = pairs.iter().flat_map(|&t| (0..POSE_DIM).map(move |i| gt[(t + 1) * POSE_DIM + i])).collect(); + let (pmu, plv) = m.pose( + to_t::(zt, n, BAND_DIM, d), + to_t::(mn, n, BAND_DIM, d), + to_t::(mk, n, BAND_DIM, d), + ); + let mud: Vec = pmu.into_data().to_vec().unwrap(); + let lvd: Vec = plv.into_data().to_vec().unwrap(); + let mut var = 0.0f64; + let mut err = 0.0f64; + for j in 0..n * POSE_DIM { + var += (lvd[j] as f64).exp(); + err += ((mud[j] - y[j]) as f64).powi(2); + } + let dn = (n * POSE_DIM) as f64; + (var / dn, err / dn) +} + +/// Target-peeking upper bound, recomputed for this model on the same pairs. +fn oracle_probe_sets( + m: &HeteroV2, + z: &[f32], + gt: &[f32], + pairs: &[usize], + k: usize, + d: &NdArrayDevice, +) -> Vec> { + let n = pairs.len(); + let y: Vec = pairs.iter().flat_map(|&t| (0..POSE_DIM).map(move |i| gt[(t + 1) * POSE_DIM + i])).collect(); + let mut out = vec![Vec::new(); n]; + for _step in 0..k { + let mut best: Vec<(f64, usize)> = vec![(f64::INFINITY, usize::MAX); n]; + for cand in 0..NUM_PROBE_GROUPS { + let gs: Vec> = (0..n) + .map(|i| { + let mut s = out[i].clone(); + if !s.contains(&cand) { + s.push(cand); + } + s + }) + .collect(); + let (zt, mn, mk) = planes(z, pairs, &gs, None); + let (pmu, _) = m.pose( + to_t::(zt, n, BAND_DIM, d), + to_t::(mn, n, BAND_DIM, d), + to_t::(mk, n, BAND_DIM, d), + ); + let mud: Vec = pmu.into_data().to_vec().unwrap(); + for i in 0..n { + if out[i].contains(&cand) { + continue; + } + let e: f64 = (0..POSE_DIM) + .map(|q| { + let j = i * POSE_DIM + q; + ((mud[j] - y[j]) as f64).powi(2) + }) + .sum(); + if e < best[i].0 { + best[i] = (e, cand); + } + } + } + for i in 0..n { + if best[i].1 != usize::MAX { + out[i].push(best[i].1); + } + } + } + out +} diff --git a/benchmarks/wiflow-dynamics/harness/src/hetero2.rs b/benchmarks/wiflow-dynamics/harness/src/hetero2.rs new file mode 100644 index 000000000..7dd6896ca --- /dev/null +++ b/benchmarks/wiflow-dynamics/harness/src/hetero2.rs @@ -0,0 +1,132 @@ +//! Rung 4: value-aware heteroscedastic head with an observation head, for +//! Monte-Carlo expected information gain. +//! +//! ## What changed from rung 3, and why +//! +//! The rung-3 target-peeking bound proved the information is present (+2.9 pts PCK@20 +//! at K=9, t=+22.3) and that rung-3's closed-form EIG cannot reach it (−0.4 pts). +//! Diagnosis: rung 3's variance head is **blind to revealed values** by construction, +//! so it can only express "how much would set S teach me on average", never "how much +//! did *these* values teach me". The bound shows probe value is value-dependent. +//! +//! Rung 4 therefore: +//! - lets the **variance head see revealed values**, and +//! - adds an **observation head** `p(z_{t+1} | z_t, revealed)` so candidate values +//! can be *sampled from the model* when scoring a probe. +//! +//! The price is that EIG no longer has a closed form -- which is the point, since a +//! closed form is precisely what could not represent value-dependence. +//! +//! **Still peek-free:** candidate values come from the observation head, never from +//! the future. Only the target-peeking *bound* reads the future, and it is labelled +//! as an unachievable ceiling. + +use burn::module::Module; +use burn::nn::{Linear, LinearConfig, Relu}; +use burn::tensor::backend::Backend; +use burn::tensor::Tensor; + +use crate::bands::BAND_DIM; +use crate::gaussian::POSE_DIM; +use crate::hetero::{MAX_LOG_VAR, MIN_LOG_VAR}; + +#[derive(Module, Debug)] +pub struct HeteroV2 { + trunk1: Linear, + trunk2: Linear, + pose_mu: Linear, + pose_lv: Linear, + obs_mu: Linear, + obs_lv: Linear, + act: Relu, +} + +impl HeteroV2 { + pub fn new(device: &B::Device, hidden: usize) -> Self { + let input = BAND_DIM * 3; // z_t, mask*z_{t+1}, mask + Self { + trunk1: LinearConfig::new(input, hidden).init(device), + trunk2: LinearConfig::new(hidden, hidden / 2).init(device), + pose_mu: LinearConfig::new(hidden / 2, POSE_DIM).init(device), + // Variance head now shares the trunk, so it SEES revealed values. + pose_lv: LinearConfig::new(hidden / 2, POSE_DIM).init(device), + obs_mu: LinearConfig::new(hidden / 2, BAND_DIM).init(device), + obs_lv: LinearConfig::new(hidden / 2, BAND_DIM).init(device), + act: Relu::new(), + } + } + + fn trunk(&self, z_t: Tensor, masked_next: Tensor, mask: Tensor) -> Tensor { + let x = Tensor::cat(vec![z_t, masked_next, mask], 1); + let h = self.act.forward(self.trunk1.forward(x)); + self.act.forward(self.trunk2.forward(h)) + } + + /// `(pose_mu, pose_log_var, obs_mu, obs_log_var)`. + pub fn forward( + &self, + z_t: Tensor, + masked_next: Tensor, + mask: Tensor, + ) -> (Tensor, Tensor, Tensor, Tensor) { + let h = self.trunk(z_t, masked_next, mask); + ( + self.pose_mu.forward(h.clone()), + self.pose_lv.forward(h.clone()).clamp(MIN_LOG_VAR, MAX_LOG_VAR), + self.obs_mu.forward(h.clone()), + self.obs_lv.forward(h).clamp(MIN_LOG_VAR, MAX_LOG_VAR), + ) + } + + /// Pose head only. + pub fn pose( + &self, + z_t: Tensor, + masked_next: Tensor, + mask: Tensor, + ) -> (Tensor, Tensor) { + let h = self.trunk(z_t, masked_next, mask); + ( + self.pose_mu.forward(h.clone()), + self.pose_lv.forward(h).clamp(MIN_LOG_VAR, MAX_LOG_VAR), + ) + } + + /// Observation head only -- supplies the sampling distribution for MC EIG. + pub fn observation( + &self, + z_t: Tensor, + masked_next: Tensor, + mask: Tensor, + ) -> (Tensor, Tensor) { + let h = self.trunk(z_t, masked_next, mask); + ( + self.obs_mu.forward(h.clone()), + self.obs_lv.forward(h).clamp(MIN_LOG_VAR, MAX_LOG_VAR), + ) + } +} + +/// Diagonal-Gaussian NLL summed over dims, averaged over the batch, in nats. +pub fn diag_nll(mu: Tensor, log_var: Tensor, y: Tensor) -> Tensor { + let two_pi = (2.0 * std::f64::consts::PI).ln(); + let inv_var = log_var.clone().neg().exp(); + let per_dim = (y - mu).powf_scalar(2.0) * inv_var + log_var + two_pi; + per_dim.sum_dim(1).squeeze::<1>() * 0.5 +} + +/// Observation NLL restricted to the UNREVEALED dims -- predicting values already +/// revealed is free and would flatter the model. +pub fn masked_obs_nll( + mu: Tensor, + log_var: Tensor, + target: Tensor, + mask: Tensor, +) -> Tensor { + let two_pi = (2.0 * std::f64::consts::PI).ln(); + let keep = mask.neg().add_scalar(1.0); // 1 where UNrevealed + let inv_var = log_var.clone().neg().exp(); + let per_dim = ((target - mu).powf_scalar(2.0) * inv_var + log_var + two_pi) * keep.clone(); + let n = keep.sum_dim(1).squeeze::<1>().clamp(1.0, BAND_DIM as f64); + per_dim.sum_dim(1).squeeze::<1>() * 0.5 / n +} diff --git a/benchmarks/wiflow-dynamics/harness/src/lib.rs b/benchmarks/wiflow-dynamics/harness/src/lib.rs index 386141e99..021ac74c3 100644 --- a/benchmarks/wiflow-dynamics/harness/src/lib.rs +++ b/benchmarks/wiflow-dynamics/harness/src/lib.rs @@ -24,6 +24,7 @@ pub mod bands; pub mod gaussian; pub mod guards; pub mod hetero; +pub mod hetero2; pub mod metrics; pub mod npy; diff --git a/benchmarks/wiflow-dynamics/results/rung4.json b/benchmarks/wiflow-dynamics/results/rung4.json new file mode 100644 index 000000000..d0369ff01 --- /dev/null +++ b/benchmarks/wiflow-dynamics/results/rung4.json @@ -0,0 +1,1080 @@ +{ + "degeneracy_guard": { + "degenerate": false, + "pred_std": 0.021679363445033534, + "threshold": 0.0001 + }, + "eval_pairs": 5000, + "mc_samples": 4, + "measured_by": "harness/src/bin/rung4.rs, this run", + "milestone": "Rung 4 -- value-aware variance + observation head, Monte-Carlo EIG", + "no_probe": { + "coverage": [ + { + "empirical": 0.5168133333333333, + "error_pp": 1.6813333333333347, + "nominal": 0.5 + }, + { + "empirical": 0.78612, + "error_pp": -1.3880000000000003, + "nominal": 0.8 + }, + { + "empirical": 0.86912, + "error_pp": -3.088000000000002, + "nominal": 0.9 + }, + { + "empirical": 0.9119, + "error_pp": -3.809999999999991, + "nominal": 0.95 + } + ], + "mpjpe": 0.011658871427115046, + "pck@20": 0.9336533333333333, + "pose_nll": -96.91586303710938 + }, + "peek_free": "candidate values are SAMPLED from the observation head; only the target-peeking bound reads the future, and it is unachievable", + "policies": [ + { + "bound_minus_no_probe": { + "delta": 0.020346666666666683, + "se": 0.0009867892060892445, + "t": 20.61906082992414 + }, + "budget_k": 3, + "distinct_probe_sets": 2293, + "fraction_of_oracle_gap_closed": -0.15465268676277832, + "mc_eig_minus_no_probe": { + "delta": -0.0031466666666666652, + "se": 0.000795942322276987, + "t": -3.9533852875983 + }, + "mc_eig_minus_random": { + "delta": -0.0005600000000000009, + "se": 0.0007890751457845159, + "t": -0.7096915965376612 + }, + "overconfidence_diagnostic": { + "confirmed": true, + "hypothesis": "EIG selects sets where the model is overconfident, not where it is informative. Confirmed if mc_eig has LOWER mean predicted variance but HIGHER mean actual error.", + "mc_eig": { + "mean_actual_sq_err": 0.0002496116882698715, + "mean_predicted_var": 0.00024772139193409 + }, + "random": { + "mean_actual_sq_err": 0.0002422533303273077, + "mean_predicted_var": 0.0002739972780545324 + } + }, + "pck@20": { + "mc_eig": 0.9305066666666667, + "no_probe": 0.9336533333333333, + "random": 0.9310666666666667, + "target_peeking_bound": 0.954 + } + }, + { + "bound_minus_no_probe": { + "delta": 0.02929333333333322, + "se": 0.0013065005684704924, + "t": 22.421217441662993 + }, + "budget_k": 9, + "distinct_probe_sets": 5000, + "fraction_of_oracle_gap_closed": -0.406918525261723, + "mc_eig_minus_no_probe": { + "delta": -0.011920000000000026, + "se": 0.0013141348227212392, + "t": -9.070606602842116 + }, + "mc_eig_minus_random": { + "delta": -0.005226666666666671, + "se": 0.0012668121049503023, + "t": -4.125842061535807 + }, + "overconfidence_diagnostic": { + "confirmed": true, + "hypothesis": "EIG selects sets where the model is overconfident, not where it is informative. Confirmed if mc_eig has LOWER mean predicted variance but HIGHER mean actual error.", + "mc_eig": { + "mean_actual_sq_err": 0.0002776642385207548, + "mean_predicted_var": 0.00029063177459581996 + }, + "random": { + "mean_actual_sq_err": 0.00025184522258984684, + "mean_predicted_var": 0.00034499655156880796 + } + }, + "pck@20": { + "mc_eig": 0.9217333333333333, + "no_probe": 0.9336533333333333, + "random": 0.92696, + "target_peeking_bound": 0.9629466666666666 + } + } + ], + "preregistered": { + "1_beats_no_probe_by_2se": false, + "2_beats_random_by_2se": false, + "3_sign_consistent_across_3_seeds": "run RUNG4_SEED=... and compare" + }, + "seed": 2026, + "training": { + "best_epoch": 149, + "best_val_pose_nll": -106.00489807128906, + "history": [ + { + "epoch": 0, + "train": -28.259576215582378, + "val_pose_nll": -43.77127456665039 + }, + { + "epoch": 1, + "train": -45.274891150199764, + "val_pose_nll": -49.715476989746094 + }, + { + "epoch": 2, + "train": -56.894317950232555, + "val_pose_nll": -49.018001556396484 + }, + { + "epoch": 3, + "train": -58.9342928094379, + "val_pose_nll": -57.38123321533203 + }, + { + "epoch": 4, + "train": -57.72867993176994, + "val_pose_nll": -42.1070556640625 + }, + { + "epoch": 5, + "train": -61.01103071439064, + "val_pose_nll": -51.97028350830078 + }, + { + "epoch": 6, + "train": -65.69430829710879, + "val_pose_nll": -67.97113800048828 + }, + { + "epoch": 7, + "train": -61.830333580405025, + "val_pose_nll": -55.2138786315918 + }, + { + "epoch": 8, + "train": -63.87845375578282, + "val_pose_nll": -69.49407958984375 + }, + { + "epoch": 9, + "train": -67.61157381736626, + "val_pose_nll": -70.37947082519531 + }, + { + "epoch": 10, + "train": -60.43643872212555, + "val_pose_nll": -67.97149658203125 + }, + { + "epoch": 11, + "train": -70.03611186399299, + "val_pose_nll": -72.55150604248047 + }, + { + "epoch": 12, + "train": -73.70152631856628, + "val_pose_nll": -72.99047088623047 + }, + { + "epoch": 13, + "train": -65.46381041963221, + "val_pose_nll": -73.12169647216797 + }, + { + "epoch": 14, + "train": -74.60909759392173, + "val_pose_nll": -76.1427001953125 + }, + { + "epoch": 15, + "train": -72.51099725496971, + "val_pose_nll": -75.48272705078125 + }, + { + "epoch": 16, + "train": -74.03138406397933, + "val_pose_nll": -70.84876251220703 + }, + { + "epoch": 17, + "train": -76.54957645222292, + "val_pose_nll": -78.61882781982422 + }, + { + "epoch": 18, + "train": -75.97667897757837, + "val_pose_nll": -79.04070281982422 + }, + { + "epoch": 19, + "train": -79.99313322164245, + "val_pose_nll": -79.62675476074219 + }, + { + "epoch": 20, + "train": -76.82872988814015, + "val_pose_nll": -69.99085998535156 + }, + { + "epoch": 21, + "train": -80.53882999743445, + "val_pose_nll": -79.71456146240234 + }, + { + "epoch": 22, + "train": -78.90603666790462, + "val_pose_nll": -78.62300109863281 + }, + { + "epoch": 23, + "train": -83.29394582974709, + "val_pose_nll": -84.49946594238281 + }, + { + "epoch": 24, + "train": -84.33372937218617, + "val_pose_nll": -85.30928802490234 + }, + { + "epoch": 25, + "train": -81.04482162604897, + "val_pose_nll": -82.96733093261719 + }, + { + "epoch": 26, + "train": -84.89474920499123, + "val_pose_nll": -83.07476806640625 + }, + { + "epoch": 27, + "train": -85.59716053332312, + "val_pose_nll": -82.89350128173828 + }, + { + "epoch": 28, + "train": -87.45325890233961, + "val_pose_nll": -86.6390380859375 + }, + { + "epoch": 29, + "train": -86.79494618561309, + "val_pose_nll": -86.10261535644531 + }, + { + "epoch": 30, + "train": -86.82227137937384, + "val_pose_nll": -87.69073486328125 + }, + { + "epoch": 31, + "train": -89.0530087099237, + "val_pose_nll": -89.02153015136719 + }, + { + "epoch": 32, + "train": -88.99718391289146, + "val_pose_nll": -89.6836929321289 + }, + { + "epoch": 33, + "train": -90.2288886894614, + "val_pose_nll": -88.87933349609375 + }, + { + "epoch": 34, + "train": -89.99568383168366, + "val_pose_nll": -89.27217102050781 + }, + { + "epoch": 35, + "train": -91.47836187330343, + "val_pose_nll": -90.41267395019531 + }, + { + "epoch": 36, + "train": -91.33006066791081, + "val_pose_nll": -91.53693389892578 + }, + { + "epoch": 37, + "train": -92.37081282017594, + "val_pose_nll": -90.52909851074219 + }, + { + "epoch": 38, + "train": -92.719897060071, + "val_pose_nll": -91.89326477050781 + }, + { + "epoch": 39, + "train": -93.82884358551543, + "val_pose_nll": -92.35438537597656 + }, + { + "epoch": 40, + "train": -93.2972457368495, + "val_pose_nll": -93.44721984863281 + }, + { + "epoch": 41, + "train": -94.99358626543466, + "val_pose_nll": -91.67420196533203 + }, + { + "epoch": 42, + "train": -93.93639044034279, + "val_pose_nll": -93.8262939453125 + }, + { + "epoch": 43, + "train": -95.77858158693475, + "val_pose_nll": -94.56420135498047 + }, + { + "epoch": 44, + "train": -94.34138191352456, + "val_pose_nll": -91.40476989746094 + }, + { + "epoch": 45, + "train": -95.54064656920352, + "val_pose_nll": -92.28553009033203 + }, + { + "epoch": 46, + "train": -96.56708371437202, + "val_pose_nll": -94.14750671386719 + }, + { + "epoch": 47, + "train": -95.63398968971381, + "val_pose_nll": -94.59498596191406 + }, + { + "epoch": 48, + "train": -96.75424562874487, + "val_pose_nll": -93.4129867553711 + }, + { + "epoch": 49, + "train": -97.55491967928612, + "val_pose_nll": -90.34405517578125 + }, + { + "epoch": 50, + "train": -97.82862252704167, + "val_pose_nll": -94.01262664794922 + }, + { + "epoch": 51, + "train": -98.28969321816655, + "val_pose_nll": -94.47893524169922 + }, + { + "epoch": 52, + "train": -98.8871086896476, + "val_pose_nll": -95.76432800292969 + }, + { + "epoch": 53, + "train": -99.36510583909892, + "val_pose_nll": -92.8056411743164 + }, + { + "epoch": 54, + "train": -98.16223745830989, + "val_pose_nll": -90.98680114746094 + }, + { + "epoch": 55, + "train": -99.26324055558544, + "val_pose_nll": -93.741455078125 + }, + { + "epoch": 56, + "train": -98.86401968487239, + "val_pose_nll": -96.33767700195312 + }, + { + "epoch": 57, + "train": -100.74844069400076, + "val_pose_nll": -97.4548568725586 + }, + { + "epoch": 58, + "train": -100.72072821148372, + "val_pose_nll": -96.69882202148438 + }, + { + "epoch": 59, + "train": -101.15032338287871, + "val_pose_nll": -97.45310974121094 + }, + { + "epoch": 60, + "train": -100.8512452335681, + "val_pose_nll": -97.84178161621094 + }, + { + "epoch": 61, + "train": -100.92532090009269, + "val_pose_nll": -97.29773712158203 + }, + { + "epoch": 62, + "train": -100.29390742415089, + "val_pose_nll": -97.2725601196289 + }, + { + "epoch": 63, + "train": -101.44932840638242, + "val_pose_nll": -97.72551727294922 + }, + { + "epoch": 64, + "train": -102.42741995342708, + "val_pose_nll": -99.01476287841797 + }, + { + "epoch": 65, + "train": -101.4437278488935, + "val_pose_nll": -98.18376922607422 + }, + { + "epoch": 66, + "train": -102.20655324903585, + "val_pose_nll": -94.1479263305664 + }, + { + "epoch": 67, + "train": -101.06297244055797, + "val_pose_nll": -98.44898223876953 + }, + { + "epoch": 68, + "train": -102.67455123642743, + "val_pose_nll": -98.7457046508789 + }, + { + "epoch": 69, + "train": -102.81392171827413, + "val_pose_nll": -98.20391845703125 + }, + { + "epoch": 70, + "train": -102.9473721778999, + "val_pose_nll": -97.60135650634766 + }, + { + "epoch": 71, + "train": -102.71237745123395, + "val_pose_nll": -97.71446990966797 + }, + { + "epoch": 72, + "train": -103.15425129260046, + "val_pose_nll": -99.54521942138672 + }, + { + "epoch": 73, + "train": -102.94852951825675, + "val_pose_nll": -99.34471130371094 + }, + { + "epoch": 74, + "train": -103.21374931982008, + "val_pose_nll": -97.63031005859375 + }, + { + "epoch": 75, + "train": -103.88541218385858, + "val_pose_nll": -98.47837829589844 + }, + { + "epoch": 76, + "train": -102.77950357986708, + "val_pose_nll": -98.21824645996094 + }, + { + "epoch": 77, + "train": -104.09876141305698, + "val_pose_nll": -100.68409729003906 + }, + { + "epoch": 78, + "train": -103.54184121600652, + "val_pose_nll": -97.64685821533203 + }, + { + "epoch": 79, + "train": -104.85098266601562, + "val_pose_nll": -98.21634674072266 + }, + { + "epoch": 80, + "train": -102.23957068233166, + "val_pose_nll": -96.17073059082031 + }, + { + "epoch": 81, + "train": -101.78584580502267, + "val_pose_nll": -98.51160430908203 + }, + { + "epoch": 82, + "train": -103.14913119299938, + "val_pose_nll": -97.14644622802734 + }, + { + "epoch": 83, + "train": -103.75766320955955, + "val_pose_nll": -97.57994079589844 + }, + { + "epoch": 84, + "train": -102.75008702682237, + "val_pose_nll": -98.53962707519531 + }, + { + "epoch": 85, + "train": -102.80964279174805, + "val_pose_nll": -99.48949432373047 + }, + { + "epoch": 86, + "train": -104.07463151317532, + "val_pose_nll": -98.52667236328125 + }, + { + "epoch": 87, + "train": -104.33329501394498, + "val_pose_nll": -98.03385925292969 + }, + { + "epoch": 88, + "train": -105.2645268197787, + "val_pose_nll": -99.5439453125 + }, + { + "epoch": 89, + "train": -103.66593002060712, + "val_pose_nll": -99.59048461914062 + }, + { + "epoch": 90, + "train": -105.20963022264384, + "val_pose_nll": -99.60099792480469 + }, + { + "epoch": 91, + "train": -105.22069905167919, + "val_pose_nll": -100.13890075683594 + }, + { + "epoch": 92, + "train": -105.19329523636122, + "val_pose_nll": -101.80403900146484 + }, + { + "epoch": 93, + "train": -105.56999439304158, + "val_pose_nll": -98.90013885498047 + }, + { + "epoch": 94, + "train": -105.82602329577429, + "val_pose_nll": -98.7580337524414 + }, + { + "epoch": 95, + "train": -105.69940974348683, + "val_pose_nll": -99.6220932006836 + }, + { + "epoch": 96, + "train": -106.39974878602109, + "val_pose_nll": -101.0079345703125 + }, + { + "epoch": 97, + "train": -106.06627719685183, + "val_pose_nll": -100.50071716308594 + }, + { + "epoch": 98, + "train": -106.21328916388043, + "val_pose_nll": -100.6803207397461 + }, + { + "epoch": 99, + "train": -106.3349189111742, + "val_pose_nll": -101.99342346191406 + }, + { + "epoch": 100, + "train": -106.28578871387546, + "val_pose_nll": -100.05682373046875 + }, + { + "epoch": 101, + "train": -105.84152176420568, + "val_pose_nll": -98.86126708984375 + }, + { + "epoch": 102, + "train": -106.84311391539492, + "val_pose_nll": -102.46627807617188 + }, + { + "epoch": 103, + "train": -106.80758259660107, + "val_pose_nll": -101.58084106445312 + }, + { + "epoch": 104, + "train": -106.97239122552386, + "val_pose_nll": -101.7745132446289 + }, + { + "epoch": 105, + "train": -106.97850773698192, + "val_pose_nll": -101.33038330078125 + }, + { + "epoch": 106, + "train": -107.07122304884054, + "val_pose_nll": -103.16116333007812 + }, + { + "epoch": 107, + "train": -107.17278270398157, + "val_pose_nll": -102.32933044433594 + }, + { + "epoch": 108, + "train": -107.2677276094081, + "val_pose_nll": -102.22368621826172 + }, + { + "epoch": 109, + "train": -107.52544668165304, + "val_pose_nll": -102.8282241821289 + }, + { + "epoch": 110, + "train": -107.80580656407243, + "val_pose_nll": -103.59212493896484 + }, + { + "epoch": 111, + "train": -107.40342337398205, + "val_pose_nll": -102.52214813232422 + }, + { + "epoch": 112, + "train": -107.91754622378592, + "val_pose_nll": -102.57345581054688 + }, + { + "epoch": 113, + "train": -107.46651879003493, + "val_pose_nll": -103.77754974365234 + }, + { + "epoch": 114, + "train": -108.29393839432021, + "val_pose_nll": -100.74491119384766 + }, + { + "epoch": 115, + "train": -108.04505002296577, + "val_pose_nll": -102.88465118408203 + }, + { + "epoch": 116, + "train": -108.33332805310266, + "val_pose_nll": -103.12680053710938 + }, + { + "epoch": 117, + "train": -108.5866291886669, + "val_pose_nll": -100.20426940917969 + }, + { + "epoch": 118, + "train": -108.52836705870547, + "val_pose_nll": -103.57463836669922 + }, + { + "epoch": 119, + "train": -109.02996729188047, + "val_pose_nll": -102.46588897705078 + }, + { + "epoch": 120, + "train": -109.01201577913963, + "val_pose_nll": -100.93607330322266 + }, + { + "epoch": 121, + "train": -109.238940028821, + "val_pose_nll": -101.2762222290039 + }, + { + "epoch": 122, + "train": -108.91837317256604, + "val_pose_nll": -103.47312927246094 + }, + { + "epoch": 123, + "train": -109.28301562293102, + "val_pose_nll": -101.02635192871094 + }, + { + "epoch": 124, + "train": -108.99346988484011, + "val_pose_nll": -103.33898162841797 + }, + { + "epoch": 125, + "train": -109.19083087727175, + "val_pose_nll": -103.17481231689453 + }, + { + "epoch": 126, + "train": -109.1891083798166, + "val_pose_nll": -104.96502685546875 + }, + { + "epoch": 127, + "train": -109.50481104446669, + "val_pose_nll": -103.70059204101562 + }, + { + "epoch": 128, + "train": -109.69179321547686, + "val_pose_nll": -102.27679443359375 + }, + { + "epoch": 129, + "train": -109.88432105112884, + "val_pose_nll": -103.98023223876953 + }, + { + "epoch": 130, + "train": -109.91781900696836, + "val_pose_nll": -103.52018737792969 + }, + { + "epoch": 131, + "train": -109.49463918653585, + "val_pose_nll": -104.7828369140625 + }, + { + "epoch": 132, + "train": -109.98648672588801, + "val_pose_nll": -103.24151611328125 + }, + { + "epoch": 133, + "train": -109.85768004595224, + "val_pose_nll": -104.06732177734375 + }, + { + "epoch": 134, + "train": -110.11122099019714, + "val_pose_nll": -103.91789245605469 + }, + { + "epoch": 135, + "train": -110.39208712820279, + "val_pose_nll": -104.36652374267578 + }, + { + "epoch": 136, + "train": -110.14527252973136, + "val_pose_nll": -105.06915283203125 + }, + { + "epoch": 137, + "train": -110.45588483648785, + "val_pose_nll": -105.13202667236328 + }, + { + "epoch": 138, + "train": -110.67332264528436, + "val_pose_nll": -103.554931640625 + }, + { + "epoch": 139, + "train": -110.7197457652981, + "val_pose_nll": -104.44255828857422 + }, + { + "epoch": 140, + "train": -110.37750308796511, + "val_pose_nll": -104.9420166015625 + }, + { + "epoch": 141, + "train": -110.55297411902477, + "val_pose_nll": -103.9051742553711 + }, + { + "epoch": 142, + "train": -110.56557833138159, + "val_pose_nll": -101.35829162597656 + }, + { + "epoch": 143, + "train": -110.51394717976198, + "val_pose_nll": -104.34874725341797 + }, + { + "epoch": 144, + "train": -110.7859188661737, + "val_pose_nll": -104.67028045654297 + }, + { + "epoch": 145, + "train": -111.07358421713619, + "val_pose_nll": -103.15605926513672 + }, + { + "epoch": 146, + "train": -111.10427397388523, + "val_pose_nll": -104.72056579589844 + }, + { + "epoch": 147, + "train": -110.88699780480336, + "val_pose_nll": -105.62932586669922 + }, + { + "epoch": 148, + "train": -110.82358531628626, + "val_pose_nll": -105.35981750488281 + }, + { + "epoch": 149, + "train": -111.2289851560431, + "val_pose_nll": -106.00489807128906 + }, + { + "epoch": 150, + "train": -111.2521090749967, + "val_pose_nll": -104.96571350097656 + }, + { + "epoch": 151, + "train": -111.47790061821372, + "val_pose_nll": -105.88749694824219 + }, + { + "epoch": 152, + "train": -111.56620562278619, + "val_pose_nll": -104.23949432373047 + }, + { + "epoch": 153, + "train": -34.99856056601314, + "val_pose_nll": -83.15013885498047 + }, + { + "epoch": 154, + "train": -84.83475145242981, + "val_pose_nll": -87.57707977294922 + }, + { + "epoch": 155, + "train": -88.42053326105668, + "val_pose_nll": -89.46192932128906 + }, + { + "epoch": 156, + "train": -90.7043534618313, + "val_pose_nll": -90.13624572753906 + }, + { + "epoch": 157, + "train": -92.19125747680664, + "val_pose_nll": -90.64643859863281 + }, + { + "epoch": 158, + "train": -93.35641848030737, + "val_pose_nll": -91.28199768066406 + }, + { + "epoch": 159, + "train": -94.28565112615036, + "val_pose_nll": -92.91393280029297 + }, + { + "epoch": 160, + "train": -95.04440740811623, + "val_pose_nll": -93.25658416748047 + }, + { + "epoch": 161, + "train": -95.64024016816737, + "val_pose_nll": -93.64772033691406 + }, + { + "epoch": 162, + "train": -96.24867345518985, + "val_pose_nll": -94.45347595214844 + }, + { + "epoch": 163, + "train": -96.8688306646832, + "val_pose_nll": -94.4967041015625 + }, + { + "epoch": 164, + "train": -97.09138281870696, + "val_pose_nll": -94.67731475830078 + }, + { + "epoch": 165, + "train": -97.68435390925004, + "val_pose_nll": -95.39472198486328 + }, + { + "epoch": 166, + "train": -98.05648583880925, + "val_pose_nll": -95.59574127197266 + }, + { + "epoch": 167, + "train": -98.300949484615, + "val_pose_nll": -95.888671875 + }, + { + "epoch": 168, + "train": -98.54487823227704, + "val_pose_nll": -96.1787338256836 + }, + { + "epoch": 169, + "train": -98.90761281676211, + "val_pose_nll": -96.18257904052734 + }, + { + "epoch": 170, + "train": -99.07494127952447, + "val_pose_nll": -95.99224853515625 + }, + { + "epoch": 171, + "train": -99.27560935586186, + "val_pose_nll": -96.20640563964844 + }, + { + "epoch": 172, + "train": -99.56656181206138, + "val_pose_nll": -96.0919189453125 + }, + { + "epoch": 173, + "train": -99.82243127337956, + "val_pose_nll": -96.7612075805664 + }, + { + "epoch": 174, + "train": -100.12778479365979, + "val_pose_nll": -96.55673217773438 + }, + { + "epoch": 175, + "train": -100.1625725051104, + "val_pose_nll": -96.5045166015625 + }, + { + "epoch": 176, + "train": -100.38817389536712, + "val_pose_nll": -96.63702392578125 + }, + { + "epoch": 177, + "train": -100.59285703756042, + "val_pose_nll": -96.8193130493164 + }, + { + "epoch": 178, + "train": -100.67031032756223, + "val_pose_nll": -96.66488647460938 + }, + { + "epoch": 179, + "train": -100.87159871246855, + "val_pose_nll": -97.29801940917969 + }, + { + "epoch": 180, + "train": -100.96456863920568, + "val_pose_nll": -96.95333099365234 + }, + { + "epoch": 181, + "train": -100.94739907474842, + "val_pose_nll": -96.944580078125 + }, + { + "epoch": 182, + "train": -101.05593555256472, + "val_pose_nll": -96.62226867675781 + }, + { + "epoch": 183, + "train": -101.04854079424325, + "val_pose_nll": -97.39553833007812 + }, + { + "epoch": 184, + "train": -101.406856666177, + "val_pose_nll": -97.86203002929688 + }, + { + "epoch": 185, + "train": -101.49137451689123, + "val_pose_nll": -97.77544403076172 + }, + { + "epoch": 186, + "train": -101.79821014404297, + "val_pose_nll": -97.6044921875 + }, + { + "epoch": 187, + "train": -101.75008935443425, + "val_pose_nll": -97.72862243652344 + }, + { + "epoch": 188, + "train": -101.95847954184322, + "val_pose_nll": -97.64210510253906 + }, + { + "epoch": 189, + "train": -102.19527221938311, + "val_pose_nll": -97.37619018554688 + } + ] + }, + "verdict": "RUNG 4 DID NOT CLOSE THE GAP" +}