From b681313e900baa3d56a84ad0a6affeacaefb407b Mon Sep 17 00:00:00 2001 From: ruv Date: Tue, 8 Sep 2026 23:50:56 -0400 Subject: [PATCH] feat(wiflow-dynamics): target-peeking bound at rung 3 resolves the open question Runs the discriminating experiment: does an oracle that selects probes using the TRUE y_t+1 beat not probing at all, at rung 3? Answer: YES, decisively, and against the intuition. K=3 oracle 0.9620 vs no-probe 0.9416 (+0.0204, t=+19.5) K=9 oracle 0.9705 vs no-probe 0.9416 (+0.0290, t=+22.3) So the marginal information in z_t+1 bands is NOT absent. Persistence R2 0.871 suggested it might be; that inference was wrong. Probing can be worth ~+2.9 pts at K=9. EIG-driven selection cannot find it -- it loses 0.4 pts instead of gaining 2.9, a ~3.3 pt gap to the bound. Mechanism, and it is a design choice made here: the variance head is deliberately blind to revealed VALUES (it sees only z_t and the mask), which is what kept EIG closed-form and peek-free. The bound shows probe value is value-dependent, not merely mask-dependent -- the oracle exploits which values actually came back, and a mask-only variance head structurally cannot represent that. The fix is the observation head the original design constraint called for and which was traded away for tractability: sample candidate z_t+1[S] from p(z_t+1[S] | z_t, revealed) and average the resulting pose entropies (Monte-Carlo EIG). That is the next experiment. Co-Authored-By: RuFlo Claude-Session: https://claude.ai/code/session_01JJnDvpYYk2rTvjkudQUoCk --- benchmarks/wiflow-dynamics/STATE.md | 29 +- .../wiflow-dynamics/harness/src/bin/rung3.rs | 87 + benchmarks/wiflow-dynamics/results/rung3.json | 30 + .../results/rung3_seed555.json | 1466 +++++++++++++++++ 4 files changed, 1605 insertions(+), 7 deletions(-) diff --git a/benchmarks/wiflow-dynamics/STATE.md b/benchmarks/wiflow-dynamics/STATE.md index 2b73bc414..0f31f9d74 100644 --- a/benchmarks/wiflow-dynamics/STATE.md +++ b/benchmarks/wiflow-dynamics/STATE.md @@ -33,13 +33,28 @@ All numbers below are **MEASURED by this work**; the 0.9609 baseline is **CITED* speculated: diagonalising rung 2's covariance gives NLL −81.92 vs rung 3's ≈−98, so heteroscedasticity does pay on marginals; rung 2 won on joint NLL via its full 30×30 covariance. Next: low-rank + diagonal head, heavier-tailed predictive. -8. **The open question this leaves** (do not resolve by assertion): is the missing - information absent, or merely unreachable by this architecture? Persistence R² - 0.871 says 87% of `z_{t+1}` is already implied by `z_t`, which argues *absent*. - Rung 2's target-peeking bound argued *some exists*. The discriminating experiment - is a target-peeking bound **at rung 3**: if even an oracle cannot beat no-probe, - the marginal information is not there and the primitive's value must come from a - different observation definition than time-averaged bands. +8. **The open question is RESOLVED, and against the intuition.** I ran the + discriminating experiment — a target-peeking bound **at rung 3** (n=5,000 pairs): + + | K | oracle | no-probe | delta | SE | t | + |---|---|---|---|---|---| + | 3 | 0.9620 | 0.9416 | **+0.0204** | 0.00104 | **+19.5** | + | 9 | 0.9705 | 0.9416 | **+0.0290** | 0.00130 | **+22.3** | + + **The information EXISTS — abundantly.** Persistence R² 0.871 suggested it might be + absent; that inference was wrong, and measuring beat asserting. Probing *can* be + worth ~+2.9 pts at K=9. **EIG-driven selection simply cannot find it** (it loses + 0.4 pts instead of gaining 2.9 — a ~3.3 pt gap to the bound). + +9. **Why EIG cannot reach it — the mechanism, and it is a design choice I made.** My + variance head is deliberately blind to revealed *values* (only `z_t` and the mask), + which is what kept EIG closed-form and peek-free. But this bound shows the value of + a probe here is **value-dependent**, not merely mask-dependent: the oracle exploits + *which values actually came back*, and a mask-only variance head structurally cannot + represent that. **The fix is the observation head the original design constraint + called for and I traded away for tractability:** sample candidate `z_{t+1}[S]` from + `p(z_{t+1}[S] | z_t, revealed)` and average the resulting pose entropies — proper + Monte-Carlo EIG. That is the single highest-value next experiment. **Nothing here is SOTA-beating, and nothing is comparable to the 96.09% full-CSI number** — different input abstraction (540 raw channels x 20 frames → 270 band means) diff --git a/benchmarks/wiflow-dynamics/harness/src/bin/rung3.rs b/benchmarks/wiflow-dynamics/harness/src/bin/rung3.rs index 8f61b361b..f2d2bc881 100644 --- a/benchmarks/wiflow-dynamics/harness/src/bin/rung3.rs +++ b/benchmarks/wiflow-dynamics/harness/src/bin/rung3.rs @@ -77,6 +77,57 @@ fn random_groups(rng: &mut Rng, k: usize) -> Vec { pool[..k].to_vec() } +/// Per-sample greedy probe selection using the TRUE y_{t+1} -- a target-peeking upper +/// bound on every policy, achievable by none. +fn oracle_probe_sets( + model: &HeteroModel, + z: &[f32], + gt: &[f32], + pairs: &[usize], + k: usize, + device: &NdArrayDevice, +) -> Vec> { + let mut out = vec![Vec::new(); pairs.len()]; + for _step in 0..k { + let mut best: Vec<(f64, usize)> = vec![(f64::INFINITY, usize::MAX); pairs.len()]; + for cand in 0..NUM_PROBE_GROUPS { + let gs: Vec> = (0..pairs.len()) + .map(|i| { + let mut set = out[i].clone(); + if !set.contains(&cand) { + set.push(cand); + } + set + }) + .collect(); + let (zt, mn, mk, yy) = make_batch::(z, gt, pairs, &gs, device); + let (mu, _) = model.forward(zt, mn, mk); + let mud: Vec = mu.into_data().to_vec().unwrap(); + let yd: Vec = yy.into_data().to_vec().unwrap(); + for i in 0..pairs.len() { + if out[i].contains(&cand) { + continue; + } + let err: f64 = (0..POSE_DIM) + .map(|d| { + let j = i * POSE_DIM + d; + ((mud[j] - yd[j]) as f64).powi(2) + }) + .sum(); + if err < best[i].0 { + best[i] = (err, cand); + } + } + } + for i in 0..pairs.len() { + if best[i].1 != usize::MAX { + out[i].push(best[i].1); + } + } + } + out +} + /// Assemble one training/eval batch. fn make_batch( z: &[f32], @@ -248,6 +299,30 @@ fn main() { })); } + // ---- discriminating experiment: target-peeking bound AT RUNG 3 ---------- + // Open question this resolves: is the marginal information ABSENT, or merely + // unreachable by the EIG criterion / this architecture? An oracle that selects + // probes using the TRUE y_{t+1} upper-bounds every policy. If even IT cannot beat + // not probing, the information is not there for this observation definition. + // Subsampled to 5,000 pairs for cost; no-probe is rescored on the SAME pairs. + let oracle_pairs = subsample(&test_pairs, 5_000, 999); + let oracle_noprobe = evaluate(&inf, z, gt, &oracle_pairs, &vec![vec![]; oracle_pairs.len()], &device); + let mut oracle_rows = Vec::new(); + for &k in &[3usize, 9] { + let sets = oracle_probe_sets(&inf, z, gt, &oracle_pairs, k, &device); + let o = evaluate(&inf, z, gt, &oracle_pairs, &sets, &device); + let (d, se, t) = paired(&o.per_sample, &oracle_noprobe.per_sample); + oracle_rows.push(serde_json::json!({ + "budget_k": k, + "oracle_pck@20": o.pck20, + "no_probe_pck@20": oracle_noprobe.pck20, + "oracle_minus_noprobe": { "delta": d, "se": se, "t": t, "beats_by_2se": t > 2.0 } + })); + } + let oracle_beats_noprobe = oracle_rows + .iter() + .any(|r| r["oracle_minus_noprobe"]["beats_by_2se"].as_bool().unwrap_or(false)); + let cov_ok = no_probe .coverage .as_array() @@ -286,6 +361,18 @@ fn main() { "3_eig_beats_random_by_2se": eig_ok, "rung2_best_pck_for_reference": RUNG2_BEST_PCK }, + "target_peeking_bound_at_rung3": { + "pairs": oracle_pairs.len(), + "rows": oracle_rows, + "oracle_beats_no_probe_by_2se": oracle_beats_noprobe, + "interpretation": if oracle_beats_noprobe { + "Information EXISTS but the EIG criterion / architecture cannot reach it." + } else { + "Even a target-peeking oracle cannot beat not probing: the marginal \ + information in time-averaged z_{t+1} bands is ABSENT, not merely \ + unreachable. The primitive needs a different observation definition." + } + }, "does_probing_help_at_all": { "any_budget_where_eig_greedy_beats_no_probe_by_2se": any_probe_beats_noprobe, "no_probe_pck@20": no_probe.pck20, diff --git a/benchmarks/wiflow-dynamics/results/rung3.json b/benchmarks/wiflow-dynamics/results/rung3.json index d4a1884a9..292ec79fb 100644 --- a/benchmarks/wiflow-dynamics/results/rung3.json +++ b/benchmarks/wiflow-dynamics/results/rung3.json @@ -15,6 +15,7 @@ "mpjpe": 0.02603940523099254, "pck@20": 0.75565 }, + "init_seed": 2026, "milestone": "Rung 3 -- neural heteroscedastic head. MEASURED by harness/src/bin/rung3.rs, this run.", "no_probe": { "coverage": [ @@ -243,6 +244,35 @@ "3_eig_beats_random_by_2se": false, "rung2_best_pck_for_reference": 0.8208 }, + "target_peeking_bound_at_rung3": { + "interpretation": "Information EXISTS but the EIG criterion / architecture cannot reach it.", + "oracle_beats_no_probe_by_2se": true, + "pairs": 5000, + "rows": [ + { + "budget_k": 3, + "no_probe_pck@20": 0.9416, + "oracle_minus_noprobe": { + "beats_by_2se": true, + "delta": 0.02036000000000004, + "se": 0.0010449355175440225, + "t": 19.484455890496886 + }, + "oracle_pck@20": 0.96196 + }, + { + "budget_k": 9, + "no_probe_pck@20": 0.9416, + "oracle_minus_noprobe": { + "beats_by_2se": true, + "delta": 0.028946666666666586, + "se": 0.0012961647253593282, + "t": 22.33255241430974 + }, + "oracle_pck@20": 0.9705466666666667 + } + ] + }, "training": { "batch": 512, "epochs": 400, diff --git a/benchmarks/wiflow-dynamics/results/rung3_seed555.json b/benchmarks/wiflow-dynamics/results/rung3_seed555.json index e69de29bb..a49136a70 100644 --- a/benchmarks/wiflow-dynamics/results/rung3_seed555.json +++ b/benchmarks/wiflow-dynamics/results/rung3_seed555.json @@ -0,0 +1,1466 @@ +{ + "backend": "burn-ndarray (CPU), by design -- see STATE.md rung-3 pre-registration", + "beats_bar": true, + "degeneracy_guard": { + "degenerate": false, + "pred_std": 0.019570846775524266, + "threshold": 0.0001 + }, + "does_probing_help_at_all": { + "any_budget_where_eig_greedy_beats_no_probe_by_2se": false, + "no_probe_pck@20": 0.92852, + "note": "If false, EIG is selecting the LEAST HARMFUL probes rather than helpful ones -- revealing time-averaged z_{t+1} bands adds no measurable information about y_{t+1} beyond z_t." + }, + "honesty_bar_same_pairs": { + "mpjpe": 0.02603940523099254, + "pck@20": 0.75565 + }, + "init_seed": 555, + "milestone": "Rung 3 -- neural heteroscedastic head. MEASURED by harness/src/bin/rung3.rs, this run.", + "no_probe": { + "coverage": [ + { + "empirical": 0.51233, + "error_pp": 1.2329999999999952, + "nominal": 0.5 + }, + { + "empirical": 0.7754916666666667, + "error_pp": -2.4508333333333354, + "nominal": 0.8 + }, + { + "empirical": 0.8600816666666666, + "error_pp": -3.991833333333339, + "nominal": 0.9 + }, + { + "empirical": 0.90534, + "error_pp": -4.465999999999992, + "nominal": 0.95 + } + ], + "mpjpe": 0.011192764338044928, + "pck@20": 0.92852, + "pose_nll": -95.5315109375 + }, + "policies": [ + { + "adaptivity": { + "distinct_probe_sets": 27, + "modal_set_fraction": 0.17615, + "note": "rung 2 had exactly 1 distinct set by construction (homoscedastic)" + }, + "budget_k": 1, + "paired_greedy_minus_fixed": { + "delta": -0.00022000000000000012, + "se": 0.00024458698526680025, + "t": -0.8994754964579159 + }, + "paired_greedy_minus_random": { + "beats_by_2se": false, + "delta": 0.000306666666666666, + "se": 0.00023189602054323868, + "t": 1.3224317776056267 + }, + "pck@20": { + "eig_greedy": 0.9277833333333333, + "fixed": 0.9280033333333333, + "random": 0.9274766666666666 + }, + "pose_nll": { + "eig_greedy": -93.05366953125, + "fixed": -95.9821046875, + "random": -95.1911984375 + }, + "vs_no_probe": { + "fixed_minus_noprobe": { + "delta": -0.0005166666666666665, + "se": 0.00020165927669900212, + "t": -2.562077357035483 + }, + "greedy_minus_noprobe": { + "delta": -0.0007366666666666659, + "se": 0.00017958913247108788, + "t": -4.101955705951539 + }, + "note": "positive t means probing BEATS not probing at all", + "random_minus_noprobe": { + "delta": -0.0010433333333333317, + "se": 0.0002102459000795203, + "t": -4.96244318171587 + } + } + }, + { + "adaptivity": { + "distinct_probe_sets": 5295, + "modal_set_fraction": 0.0045, + "note": "rung 2 had exactly 1 distinct set by construction (homoscedastic)" + }, + "budget_k": 3, + "paired_greedy_minus_fixed": { + "delta": -0.0003133333333333332, + "se": 0.00034192403809126025, + "t": -0.9163828758061867 + }, + "paired_greedy_minus_random": { + "beats_by_2se": false, + "delta": 0.00006666666666666648, + "se": 0.0003330745532820336, + "t": 0.20015538866523955 + }, + "pck@20": { + "eig_greedy": 0.92595, + "fixed": 0.9262633333333333, + "random": 0.9258833333333333 + }, + "pose_nll": { + "eig_greedy": -88.32709609375, + "fixed": -94.88389375, + "random": -94.71509140625 + }, + "vs_no_probe": { + "fixed_minus_noprobe": { + "delta": -0.0022566666666666672, + "se": 0.0003048952802938618, + "t": -7.401448341514714 + }, + "greedy_minus_noprobe": { + "delta": -0.002570000000000007, + "se": 0.00027619681450941644, + "t": -9.304958873493407 + }, + "note": "positive t means probing BEATS not probing at all", + "random_minus_noprobe": { + "delta": -0.0026366666666666795, + "se": 0.00029344890630749306, + "t": -8.98509624671705 + } + } + }, + { + "adaptivity": { + "distinct_probe_sets": 19371, + "modal_set_fraction": 0.0003, + "note": "rung 2 had exactly 1 distinct set by construction (homoscedastic)" + }, + "budget_k": 6, + "paired_greedy_minus_fixed": { + "delta": -0.0024333333333333255, + "se": 0.0004018579310055311, + "t": -6.055207936906024 + }, + "paired_greedy_minus_random": { + "beats_by_2se": false, + "delta": -0.00043666666666666707, + "se": 0.0004041022532017477, + "t": -1.0805845877049878 + }, + "pck@20": { + "eig_greedy": 0.9234633333333333, + "fixed": 0.9258966666666667, + "random": 0.9239 + }, + "pose_nll": { + "eig_greedy": -81.70555625, + "fixed": -95.6310359375, + "random": -93.9361234375 + }, + "vs_no_probe": { + "fixed_minus_noprobe": { + "delta": -0.002623333333333347, + "se": 0.0003509128679329913, + "t": -7.475739914542793 + }, + "greedy_minus_noprobe": { + "delta": -0.005056666666666684, + "se": 0.0003752120745281432, + "t": -13.476822868842413 + }, + "note": "positive t means probing BEATS not probing at all", + "random_minus_noprobe": { + "delta": -0.004620000000000014, + "se": 0.00039393525317436433, + "t": -11.727815580788098 + } + } + }, + { + "adaptivity": { + "distinct_probe_sets": 19882, + "modal_set_fraction": 0.0002, + "note": "rung 2 had exactly 1 distinct set by construction (homoscedastic)" + }, + "budget_k": 9, + "paired_greedy_minus_fixed": { + "delta": 0.0034966666666666805, + "se": 0.0005016097664919804, + "t": 6.970890322014063 + }, + "paired_greedy_minus_random": { + "beats_by_2se": false, + "delta": -0.00012333333333333266, + "se": 0.00042824910764449907, + "t": -0.2879943732088636 + }, + "pck@20": { + "eig_greedy": 0.92149, + "fixed": 0.9179933333333333, + "random": 0.9216133333333333 + }, + "pose_nll": { + "eig_greedy": -77.0064125, + "fixed": -94.7542765625, + "random": -93.60820546875 + }, + "vs_no_probe": { + "fixed_minus_noprobe": { + "delta": -0.01052666666666663, + "se": 0.000557422142420839, + "t": -18.884550622532096 + }, + "greedy_minus_noprobe": { + "delta": -0.007029999999999997, + "se": 0.00045776690972399917, + "t": -15.3571607092321 + }, + "note": "positive t means probing BEATS not probing at all", + "random_minus_noprobe": { + "delta": -0.006906666666666686, + "se": 0.0004469680551845675, + "t": -15.452260148244154 + } + } + } + ], + "preregistered_criteria": { + "1_beats_rung2_nll": { + "pass": false, + "rung2": -113.29, + "rung3": -95.5315109375 + }, + "2_coverage_within_3pp_all_levels": false, + "3_eig_beats_random_by_2se": false, + "rung2_best_pck_for_reference": 0.8208 + }, + "training": { + "batch": 512, + "epochs": 400, + "history": [ + { + "epoch": 0, + "train_nll": -44.75524725954411, + "val_nll": -58.6608772277832 + }, + { + "epoch": 1, + "train_nll": -60.232638601529395, + "val_nll": -60.66302490234375 + }, + { + "epoch": 2, + "train_nll": -62.48863446510444, + "val_nll": -65.78565979003906 + }, + { + "epoch": 3, + "train_nll": -54.90721541744168, + "val_nll": -66.58590698242188 + }, + { + "epoch": 4, + "train_nll": -65.78330861107777, + "val_nll": -55.496891021728516 + }, + { + "epoch": 5, + "train_nll": -66.8672034700038, + "val_nll": -71.92774200439453 + }, + { + "epoch": 6, + "train_nll": -72.52238936343436, + "val_nll": -72.13646697998047 + }, + { + "epoch": 7, + "train_nll": -67.27725284382448, + "val_nll": -66.91412353515625 + }, + { + "epoch": 8, + "train_nll": -72.40604610766394, + "val_nll": -77.6010971069336 + }, + { + "epoch": 9, + "train_nll": -77.3317473460052, + "val_nll": -79.3506851196289 + }, + { + "epoch": 10, + "train_nll": -76.00977703676385, + "val_nll": -72.75188446044922 + }, + { + "epoch": 11, + "train_nll": -79.52962493896484, + "val_nll": -82.1740493774414 + }, + { + "epoch": 12, + "train_nll": -81.97309856091516, + "val_nll": -83.31145477294922 + }, + { + "epoch": 13, + "train_nll": -84.56798152600305, + "val_nll": -86.04080963134766 + }, + { + "epoch": 14, + "train_nll": -86.39321375701387, + "val_nll": -82.85487365722656 + }, + { + "epoch": 15, + "train_nll": -87.12269514698093, + "val_nll": -83.70095825195312 + }, + { + "epoch": 16, + "train_nll": -88.40414144225039, + "val_nll": -87.31120300292969 + }, + { + "epoch": 17, + "train_nll": -89.12903853594247, + "val_nll": -86.86592102050781 + }, + { + "epoch": 18, + "train_nll": -90.2981656607935, + "val_nll": -90.41956329345703 + }, + { + "epoch": 19, + "train_nll": -91.5277382802155, + "val_nll": -91.47872161865234 + }, + { + "epoch": 20, + "train_nll": -92.54211923631571, + "val_nll": -92.71356201171875 + }, + { + "epoch": 21, + "train_nll": -93.18119934857901, + "val_nll": -91.87022399902344 + }, + { + "epoch": 22, + "train_nll": -92.99715882640774, + "val_nll": -93.47647094726562 + }, + { + "epoch": 23, + "train_nll": -93.96326427136438, + "val_nll": -94.4481201171875 + }, + { + "epoch": 24, + "train_nll": -95.13767138982223, + "val_nll": -94.05738830566406 + }, + { + "epoch": 25, + "train_nll": -95.08041077953274, + "val_nll": -94.24312591552734 + }, + { + "epoch": 26, + "train_nll": -96.10938728461831, + "val_nll": -94.42205810546875 + }, + { + "epoch": 27, + "train_nll": -96.47613590046511, + "val_nll": -94.260009765625 + }, + { + "epoch": 28, + "train_nll": -96.82682224855584, + "val_nll": -95.23513793945312 + }, + { + "epoch": 29, + "train_nll": -97.32469817339364, + "val_nll": -95.68099975585938 + }, + { + "epoch": 30, + "train_nll": -97.73646364373676, + "val_nll": -96.78202819824219 + }, + { + "epoch": 31, + "train_nll": -98.4531881041446, + "val_nll": -96.28004455566406 + }, + { + "epoch": 32, + "train_nll": -98.4306449243578, + "val_nll": -96.55000305175781 + }, + { + "epoch": 33, + "train_nll": -99.61792102102505, + "val_nll": -95.72620391845703 + }, + { + "epoch": 34, + "train_nll": -99.82060034800384, + "val_nll": -97.72047424316406 + }, + { + "epoch": 35, + "train_nll": -99.95714982889466, + "val_nll": -97.9013900756836 + }, + { + "epoch": 36, + "train_nll": -99.03055837598897, + "val_nll": -95.71492004394531 + }, + { + "epoch": 37, + "train_nll": -100.41408648733365, + "val_nll": -96.56998443603516 + }, + { + "epoch": 38, + "train_nll": -100.97216305490268, + "val_nll": -98.91101837158203 + }, + { + "epoch": 39, + "train_nll": -100.96322819338006, + "val_nll": -98.13298797607422 + }, + { + "epoch": 40, + "train_nll": -101.72641780012745, + "val_nll": -98.57696533203125 + }, + { + "epoch": 41, + "train_nll": -102.1529019889185, + "val_nll": -99.27925109863281 + }, + { + "epoch": 42, + "train_nll": -102.56035930827512, + "val_nll": -97.93634796142578 + }, + { + "epoch": 43, + "train_nll": -102.48529582912639, + "val_nll": -97.79959869384766 + }, + { + "epoch": 44, + "train_nll": -102.87260113732289, + "val_nll": -99.21287536621094 + }, + { + "epoch": 45, + "train_nll": -102.83411129450394, + "val_nll": -99.33049774169922 + }, + { + "epoch": 46, + "train_nll": -103.34355868323375, + "val_nll": -98.89375305175781 + }, + { + "epoch": 47, + "train_nll": -103.62040122080657, + "val_nll": -99.35640716552734 + }, + { + "epoch": 48, + "train_nll": -103.64027437113099, + "val_nll": -100.5289306640625 + }, + { + "epoch": 49, + "train_nll": -104.15023603277692, + "val_nll": -100.2470474243164 + }, + { + "epoch": 50, + "train_nll": -104.06754438755875, + "val_nll": -98.56566619873047 + }, + { + "epoch": 51, + "train_nll": -104.80470321138026, + "val_nll": -100.08606719970703 + }, + { + "epoch": 52, + "train_nll": -104.21006419294972, + "val_nll": -101.80541229248047 + }, + { + "epoch": 53, + "train_nll": -104.86875902596167, + "val_nll": -100.4998550415039 + }, + { + "epoch": 54, + "train_nll": -105.48608210935431, + "val_nll": -100.28878021240234 + }, + { + "epoch": 55, + "train_nll": -105.29973259618727, + "val_nll": -100.44265747070312 + }, + { + "epoch": 56, + "train_nll": -105.62340862468137, + "val_nll": -101.45698547363281 + }, + { + "epoch": 57, + "train_nll": -106.12231335397495, + "val_nll": -101.30706024169922 + }, + { + "epoch": 58, + "train_nll": -105.87922209400242, + "val_nll": -100.6968765258789 + }, + { + "epoch": 59, + "train_nll": -106.009113635047, + "val_nll": -100.34480285644531 + }, + { + "epoch": 60, + "train_nll": -106.21915558637198, + "val_nll": -100.04981994628906 + }, + { + "epoch": 61, + "train_nll": -106.56784452018091, + "val_nll": -101.69368743896484 + }, + { + "epoch": 62, + "train_nll": -106.4651272014036, + "val_nll": -101.9579086303711 + }, + { + "epoch": 63, + "train_nll": -106.62391947083555, + "val_nll": -101.05320739746094 + }, + { + "epoch": 64, + "train_nll": -106.91341975583869, + "val_nll": -98.40308380126953 + }, + { + "epoch": 65, + "train_nll": -106.8320298922264, + "val_nll": -102.27741241455078 + }, + { + "epoch": 66, + "train_nll": -107.15446944155936, + "val_nll": -101.96881866455078 + }, + { + "epoch": 67, + "train_nll": -107.62087133375265, + "val_nll": -99.56864929199219 + }, + { + "epoch": 68, + "train_nll": -107.37915077856032, + "val_nll": -102.31095123291016 + }, + { + "epoch": 69, + "train_nll": -107.74447909856246, + "val_nll": -101.7404556274414 + }, + { + "epoch": 70, + "train_nll": -107.59743318719379, + "val_nll": -102.33624267578125 + }, + { + "epoch": 71, + "train_nll": -107.19404039544574, + "val_nll": -102.40620422363281 + }, + { + "epoch": 72, + "train_nll": -107.84323456327795, + "val_nll": -99.17826080322266 + }, + { + "epoch": 73, + "train_nll": -108.09300341848599, + "val_nll": -101.45121765136719 + }, + { + "epoch": 74, + "train_nll": -108.60587743985451, + "val_nll": -102.3740234375 + }, + { + "epoch": 75, + "train_nll": -107.68735911482472, + "val_nll": -102.88545989990234 + }, + { + "epoch": 76, + "train_nll": -108.28155381800765, + "val_nll": -102.9321060180664 + }, + { + "epoch": 77, + "train_nll": -108.62885342614125, + "val_nll": -101.07453155517578 + }, + { + "epoch": 78, + "train_nll": -108.55020775229244, + "val_nll": -101.39221954345703 + }, + { + "epoch": 79, + "train_nll": -109.1453464314089, + "val_nll": -103.17879486083984 + }, + { + "epoch": 80, + "train_nll": -109.0518607446703, + "val_nll": -102.42135620117188 + }, + { + "epoch": 81, + "train_nll": -108.39021178423349, + "val_nll": -102.5831527709961 + }, + { + "epoch": 82, + "train_nll": -109.19751551999884, + "val_nll": -103.70649719238281 + }, + { + "epoch": 83, + "train_nll": -109.25096123905506, + "val_nll": -102.22742462158203 + }, + { + "epoch": 84, + "train_nll": -109.4087138418424, + "val_nll": -102.27227020263672 + }, + { + "epoch": 85, + "train_nll": -109.51811308779959, + "val_nll": -99.95498657226562 + }, + { + "epoch": 86, + "train_nll": -109.79969638889119, + "val_nll": -102.35853576660156 + }, + { + "epoch": 87, + "train_nll": -109.98338304939917, + "val_nll": -101.10443115234375 + }, + { + "epoch": 88, + "train_nll": -110.06830234850867, + "val_nll": -103.6484146118164 + }, + { + "epoch": 89, + "train_nll": -110.33343124389648, + "val_nll": -104.70040130615234 + }, + { + "epoch": 90, + "train_nll": -109.69162129547637, + "val_nll": -103.36809539794922 + }, + { + "epoch": 91, + "train_nll": -109.93176457033319, + "val_nll": -104.73323059082031 + }, + { + "epoch": 92, + "train_nll": -110.34858878184173, + "val_nll": -103.74706268310547 + }, + { + "epoch": 93, + "train_nll": -110.32575549109508, + "val_nll": -105.18922424316406 + }, + { + "epoch": 94, + "train_nll": -110.64072612180549, + "val_nll": -102.24856567382812 + }, + { + "epoch": 95, + "train_nll": -110.46905944306971, + "val_nll": -101.76087188720703 + }, + { + "epoch": 96, + "train_nll": -110.64281722246591, + "val_nll": -103.39238739013672 + }, + { + "epoch": 97, + "train_nll": -110.83693158424506, + "val_nll": -103.89152526855469 + }, + { + "epoch": 98, + "train_nll": -110.75430724580409, + "val_nll": -104.10822296142578 + }, + { + "epoch": 99, + "train_nll": -110.0846692747989, + "val_nll": -105.05195617675781 + }, + { + "epoch": 100, + "train_nll": -110.63985979759087, + "val_nll": -104.3856430053711 + }, + { + "epoch": 101, + "train_nll": -110.50178644212626, + "val_nll": -105.32987213134766 + }, + { + "epoch": 102, + "train_nll": -109.9723224963172, + "val_nll": -104.49998474121094 + }, + { + "epoch": 103, + "train_nll": -110.57878086930614, + "val_nll": -103.95135498046875 + }, + { + "epoch": 104, + "train_nll": -110.98598402637546, + "val_nll": -104.60916900634766 + }, + { + "epoch": 105, + "train_nll": -111.41322915028718, + "val_nll": -104.44657897949219 + }, + { + "epoch": 106, + "train_nll": -111.07434424707445, + "val_nll": -104.78563690185547 + }, + { + "epoch": 107, + "train_nll": -110.34965444015245, + "val_nll": -105.08504486083984 + }, + { + "epoch": 108, + "train_nll": -109.87568612826072, + "val_nll": -105.36251068115234 + }, + { + "epoch": 109, + "train_nll": -111.4406646469892, + "val_nll": -105.31153106689453 + }, + { + "epoch": 110, + "train_nll": -111.46745242102672, + "val_nll": -104.36237335205078 + }, + { + "epoch": 111, + "train_nll": -111.25952542838404, + "val_nll": -104.77354431152344 + }, + { + "epoch": 112, + "train_nll": -111.85990776449947, + "val_nll": -104.2896499633789 + }, + { + "epoch": 113, + "train_nll": -111.56892045877747, + "val_nll": -103.65894317626953 + }, + { + "epoch": 114, + "train_nll": -111.51308518749173, + "val_nll": -103.87348175048828 + }, + { + "epoch": 115, + "train_nll": -111.74832334356793, + "val_nll": -104.21019744873047 + }, + { + "epoch": 116, + "train_nll": -111.51107057474427, + "val_nll": -104.92137145996094 + }, + { + "epoch": 117, + "train_nll": -111.44726452584995, + "val_nll": -104.13706970214844 + }, + { + "epoch": 118, + "train_nll": -112.42342615935762, + "val_nll": -104.58004760742188 + }, + { + "epoch": 119, + "train_nll": -111.94822537696967, + "val_nll": -103.05516815185547 + }, + { + "epoch": 120, + "train_nll": -111.77384864677818, + "val_nll": -102.6239013671875 + }, + { + "epoch": 121, + "train_nll": -112.6112842236535, + "val_nll": -102.97506713867188 + }, + { + "epoch": 122, + "train_nll": -112.93646602307336, + "val_nll": -104.49773406982422 + }, + { + "epoch": 123, + "train_nll": -112.0794865236444, + "val_nll": -103.80167388916016 + }, + { + "epoch": 124, + "train_nll": -112.68619343385858, + "val_nll": -104.95011901855469 + }, + { + "epoch": 125, + "train_nll": -112.18325527643754, + "val_nll": -103.75450134277344 + }, + { + "epoch": 126, + "train_nll": -112.48614456694006, + "val_nll": -103.62815856933594 + }, + { + "epoch": 127, + "train_nll": -112.35637483758441, + "val_nll": -104.87614440917969 + }, + { + "epoch": 128, + "train_nll": -112.48490957486428, + "val_nll": -103.56392669677734 + }, + { + "epoch": 129, + "train_nll": -113.14856726436291, + "val_nll": -104.55048370361328 + }, + { + "epoch": 130, + "train_nll": -113.06073547621905, + "val_nll": -100.65689086914062 + }, + { + "epoch": 131, + "train_nll": -113.00205153125827, + "val_nll": -103.33881378173828 + }, + { + "epoch": 132, + "train_nll": -113.20558638491873, + "val_nll": -103.28209686279297 + }, + { + "epoch": 133, + "train_nll": -113.05968507669739, + "val_nll": -104.22142028808594 + }, + { + "epoch": 134, + "train_nll": -113.26546058008226, + "val_nll": -103.89044189453125 + }, + { + "epoch": 135, + "train_nll": -112.94521357649464, + "val_nll": -96.12338256835938 + }, + { + "epoch": 136, + "train_nll": -112.19490988779876, + "val_nll": -102.70348358154297 + }, + { + "epoch": 137, + "train_nll": -113.32693694809736, + "val_nll": -104.40049743652344 + }, + { + "epoch": 138, + "train_nll": -113.31024156990698, + "val_nll": -102.85181427001953 + }, + { + "epoch": 139, + "train_nll": -112.88634141825013, + "val_nll": -101.83453369140625 + }, + { + "epoch": 140, + "train_nll": -113.31213805635096, + "val_nll": -105.1564712524414 + }, + { + "epoch": 141, + "train_nll": -113.25876578638109, + "val_nll": -104.04113006591797 + }, + { + "epoch": 142, + "train_nll": -113.3799600762836, + "val_nll": -103.69353485107422 + }, + { + "epoch": 143, + "train_nll": -113.79654447911149, + "val_nll": -105.81382751464844 + }, + { + "epoch": 144, + "train_nll": -113.01444399558892, + "val_nll": -103.6999740600586 + }, + { + "epoch": 145, + "train_nll": -112.89139395245051, + "val_nll": -102.76947021484375 + }, + { + "epoch": 146, + "train_nll": -113.41546469219661, + "val_nll": -102.15895080566406 + }, + { + "epoch": 147, + "train_nll": -113.84737422102589, + "val_nll": -102.3918228149414 + }, + { + "epoch": 148, + "train_nll": -113.58308287798349, + "val_nll": -103.30076599121094 + }, + { + "epoch": 149, + "train_nll": -113.15464317192466, + "val_nll": -105.37236022949219 + }, + { + "epoch": 150, + "train_nll": -113.90122688422768, + "val_nll": -100.81761932373047 + }, + { + "epoch": 151, + "train_nll": -114.11784459776797, + "val_nll": -104.76131439208984 + }, + { + "epoch": 152, + "train_nll": -113.70953996302718, + "val_nll": -102.22447967529297 + }, + { + "epoch": 153, + "train_nll": -114.42151215116857, + "val_nll": -102.08045196533203 + }, + { + "epoch": 154, + "train_nll": -114.1128811593783, + "val_nll": -103.5024185180664 + }, + { + "epoch": 155, + "train_nll": -114.08247433678578, + "val_nll": -102.74663543701172 + }, + { + "epoch": 156, + "train_nll": -113.76487602621822, + "val_nll": -103.02719116210938 + }, + { + "epoch": 157, + "train_nll": -113.22274482856362, + "val_nll": -99.8022689819336 + }, + { + "epoch": 158, + "train_nll": -114.23007518962278, + "val_nll": -104.72696685791016 + }, + { + "epoch": 159, + "train_nll": -114.43971064939338, + "val_nll": -103.66254425048828 + }, + { + "epoch": 160, + "train_nll": -114.41508005433164, + "val_nll": -105.04067993164062 + }, + { + "epoch": 161, + "train_nll": -114.82752363560563, + "val_nll": -105.95853424072266 + }, + { + "epoch": 162, + "train_nll": -113.95678038516287, + "val_nll": -104.22061157226562 + }, + { + "epoch": 163, + "train_nll": -114.14919610750877, + "val_nll": -98.52300262451172 + }, + { + "epoch": 164, + "train_nll": -114.46664448107703, + "val_nll": -103.95291900634766 + }, + { + "epoch": 165, + "train_nll": -114.92235177250232, + "val_nll": -102.9900131225586 + }, + { + "epoch": 166, + "train_nll": -114.70924914085259, + "val_nll": -100.95809936523438 + }, + { + "epoch": 167, + "train_nll": -114.66868222770044, + "val_nll": -101.88765716552734 + }, + { + "epoch": 168, + "train_nll": -114.6690395161257, + "val_nll": -99.5974349975586 + }, + { + "epoch": 169, + "train_nll": -115.08220245878576, + "val_nll": -104.25914001464844 + }, + { + "epoch": 170, + "train_nll": -114.46588716668597, + "val_nll": -102.04439544677734 + }, + { + "epoch": 171, + "train_nll": -115.272126731226, + "val_nll": -102.18499755859375 + }, + { + "epoch": 172, + "train_nll": -114.7891310998949, + "val_nll": -99.10076904296875 + }, + { + "epoch": 173, + "train_nll": -115.05374145507812, + "val_nll": -102.51744079589844 + }, + { + "epoch": 174, + "train_nll": -115.12156179395772, + "val_nll": -100.69406127929688 + }, + { + "epoch": 175, + "train_nll": -114.76918947898736, + "val_nll": -104.75127410888672 + }, + { + "epoch": 176, + "train_nll": -114.83725208347127, + "val_nll": -98.48625946044922 + }, + { + "epoch": 177, + "train_nll": -114.57205374766204, + "val_nll": -104.92448425292969 + }, + { + "epoch": 178, + "train_nll": -115.206353203725, + "val_nll": -98.95789337158203 + }, + { + "epoch": 179, + "train_nll": -114.77692943508342, + "val_nll": -102.57540893554688 + }, + { + "epoch": 180, + "train_nll": -115.51545114032292, + "val_nll": -101.24449920654297 + }, + { + "epoch": 181, + "train_nll": -115.29241387318757, + "val_nll": -101.1468276977539 + }, + { + "epoch": 182, + "train_nll": -115.39828963198904, + "val_nll": -104.80030059814453 + }, + { + "epoch": 183, + "train_nll": -114.57855405645856, + "val_nll": -103.95492553710938 + }, + { + "epoch": 184, + "train_nll": -114.44387410050732, + "val_nll": -102.74073791503906 + }, + { + "epoch": 185, + "train_nll": -114.88871920310845, + "val_nll": -102.67888641357422 + }, + { + "epoch": 186, + "train_nll": -115.3263931274414, + "val_nll": -103.49161529541016 + }, + { + "epoch": 187, + "train_nll": -115.17695869833736, + "val_nll": -100.53548431396484 + }, + { + "epoch": 188, + "train_nll": -114.88156308966168, + "val_nll": -103.64216613769531 + }, + { + "epoch": 189, + "train_nll": -115.56180288023867, + "val_nll": -105.17689514160156 + }, + { + "epoch": 190, + "train_nll": -115.30808232194286, + "val_nll": -103.37601470947266 + }, + { + "epoch": 191, + "train_nll": -115.60743571135957, + "val_nll": -100.63972473144531 + }, + { + "epoch": 192, + "train_nll": -115.48586984408104, + "val_nll": -102.24610137939453 + }, + { + "epoch": 193, + "train_nll": -115.04452430595786, + "val_nll": -98.49396514892578 + }, + { + "epoch": 194, + "train_nll": -115.52874025247864, + "val_nll": -101.64741516113281 + }, + { + "epoch": 195, + "train_nll": -115.19232462220273, + "val_nll": -102.73018646240234 + }, + { + "epoch": 196, + "train_nll": -115.19094389576023, + "val_nll": -103.96497344970703 + }, + { + "epoch": 197, + "train_nll": -115.02374720169325, + "val_nll": -103.07281494140625 + }, + { + "epoch": 198, + "train_nll": -115.62432007870432, + "val_nll": -100.60071563720703 + }, + { + "epoch": 199, + "train_nll": -114.57906522589215, + "val_nll": -99.90306854248047 + }, + { + "epoch": 200, + "train_nll": -112.58974831791248, + "val_nll": -101.28856658935547 + }, + { + "epoch": 201, + "train_nll": -114.43804983365334, + "val_nll": -106.05755615234375 + }, + { + "epoch": 202, + "train_nll": -114.8920752315198, + "val_nll": -101.4579086303711 + }, + { + "epoch": 203, + "train_nll": -114.48546193009716, + "val_nll": -103.65264129638672 + }, + { + "epoch": 204, + "train_nll": -115.20319379386255, + "val_nll": -103.05632781982422 + }, + { + "epoch": 205, + "train_nll": -114.98763275146484, + "val_nll": -102.1039810180664 + }, + { + "epoch": 206, + "train_nll": -114.798505815409, + "val_nll": -101.08912658691406 + }, + { + "epoch": 207, + "train_nll": -115.06620652796859, + "val_nll": -99.91535949707031 + }, + { + "epoch": 208, + "train_nll": -115.08696520530572, + "val_nll": -98.95572662353516 + }, + { + "epoch": 209, + "train_nll": -115.76493441048315, + "val_nll": -100.9600601196289 + }, + { + "epoch": 210, + "train_nll": -115.11889312226893, + "val_nll": -101.96162414550781 + }, + { + "epoch": 211, + "train_nll": -115.75148799055714, + "val_nll": -98.65922546386719 + }, + { + "epoch": 212, + "train_nll": -115.91373437137928, + "val_nll": -102.70689392089844 + }, + { + "epoch": 213, + "train_nll": -115.90137203669144, + "val_nll": -101.23546600341797 + }, + { + "epoch": 214, + "train_nll": -116.07362514431193, + "val_nll": -99.11397552490234 + }, + { + "epoch": 215, + "train_nll": -115.33931130877995, + "val_nll": -99.15640258789062 + }, + { + "epoch": 216, + "train_nll": -115.57459737486758, + "val_nll": -100.9776382446289 + }, + { + "epoch": 217, + "train_nll": -115.80363218663102, + "val_nll": -98.0738754272461 + }, + { + "epoch": 218, + "train_nll": -116.15489377813824, + "val_nll": -102.25245666503906 + }, + { + "epoch": 219, + "train_nll": -115.7092084076445, + "val_nll": -98.26142120361328 + }, + { + "epoch": 220, + "train_nll": -116.20000438366907, + "val_nll": -101.79987335205078 + }, + { + "epoch": 221, + "train_nll": -116.01920596623825, + "val_nll": -95.8310775756836 + }, + { + "epoch": 222, + "train_nll": -115.43420067480055, + "val_nll": -93.26185607910156 + }, + { + "epoch": 223, + "train_nll": -116.4039329916744, + "val_nll": -101.56938934326172 + }, + { + "epoch": 224, + "train_nll": -116.19471462702347, + "val_nll": -100.76290130615234 + }, + { + "epoch": 225, + "train_nll": -116.75976045252914, + "val_nll": -96.49715423583984 + }, + { + "epoch": 226, + "train_nll": -115.72973897901632, + "val_nll": -95.14779663085938 + }, + { + "epoch": 227, + "train_nll": -116.01968461376126, + "val_nll": -100.88912200927734 + }, + { + "epoch": 228, + "train_nll": -116.41616278179622, + "val_nll": -100.97676849365234 + }, + { + "epoch": 229, + "train_nll": -116.13256719556905, + "val_nll": -99.65129852294922 + }, + { + "epoch": 230, + "train_nll": -116.23272776199599, + "val_nll": -100.40963745117188 + }, + { + "epoch": 231, + "train_nll": -116.20800516160868, + "val_nll": -98.6461410522461 + }, + { + "epoch": 232, + "train_nll": -116.57351341894118, + "val_nll": -97.03370666503906 + }, + { + "epoch": 233, + "train_nll": -116.692653009447, + "val_nll": -96.86309051513672 + }, + { + "epoch": 234, + "train_nll": -116.46692592814817, + "val_nll": -95.94799041748047 + }, + { + "epoch": 235, + "train_nll": -115.71954028889284, + "val_nll": -101.70023345947266 + }, + { + "epoch": 236, + "train_nll": -116.57146602565959, + "val_nll": -101.89581298828125 + }, + { + "epoch": 237, + "train_nll": -116.65548822435282, + "val_nll": -102.81192016601562 + }, + { + "epoch": 238, + "train_nll": -116.87748401447878, + "val_nll": -98.25392150878906 + }, + { + "epoch": 239, + "train_nll": -115.77684260222871, + "val_nll": -99.8815689086914 + }, + { + "epoch": 240, + "train_nll": -116.92119727700444, + "val_nll": -101.2697982788086 + }, + { + "epoch": 241, + "train_nll": -116.34516292507365, + "val_nll": -100.06326293945312 + } + ], + "lr": 0.001, + "train_pairs": 60000 + }, + "verdict": "RUNG 3 DID NOT HELP -- at least one pre-registered criterion was missed" +}