diff --git a/.github/workflows/csi-encoder-training.yml b/.github/workflows/csi-encoder-training.yml index 034ae1216..211ea79cc 100644 --- a/.github/workflows/csi-encoder-training.yml +++ b/.github/workflows/csi-encoder-training.yml @@ -8,6 +8,7 @@ on: - 'scripts/train-espfi.py' - 'scripts/improve-espfi.py' - 'scripts/improve-espfi-kernel.py' + - 'scripts/improve-espfi-blend.py' - 'scripts/requirements-espfi-training.txt' - 'scripts/requirements-csi-training.txt' - 'scripts/tests/test_csi_training*.py' diff --git a/docs/espfi-model-optimization.md b/docs/espfi-model-optimization.md index b7f313de1..b9e220efc 100644 --- a/docs/espfi-model-optimization.md +++ b/docs/espfi-model-optimization.md @@ -284,3 +284,52 @@ are retained; the replay is verification, not independent accuracy evidence. Decision: retain the small spectral model and archive the nonlinear model as an accuracy experiment. No deployed model is replaced, no final-test arrays were evaluated, and neither candidate establishes SOTA. + + +## Rejected blend and lossless seed encoding + +A frozen five-weight blend combined the preceding spectral ridge and RFF +candidates using raw regression scores. Training-only mean fold macro F1 +selected equal weights (0.323484), but development macro F1 fell to 0.348261, +with activities 3 and 7 still at zero F1. Accuracy was 30/70. The reproduced +controls matched the previous development scores exactly: spectral 0.366065 +and RFF 0.383999. The blend is rejected; its complete receipt remains available. +No other development weights were evaluated or substituted after selection. + +```sh +python scripts/improve-espfi-blend.py \\ + --manifest /private/espfi/prepared/manifest.json \\ + --output /private/espfi/blend-001 +``` + +The separate seed encoding changes only the representation of the fixed RFF +projection. It stores 1,967 fitted and preprocessing scalars, then regenerates +60,032 random-map scalars from seed 42 at load. Both regenerated arrays must +match the exact export hashes before inference. Unexpected generator behavior, +tampered model bytes, invalid shapes and nonfinite state fail closed. + +```python +from csi_training.espfi_kernel import KernelBundle +from csi_training.espfi_seeded import export_seeded, SeededKernelBundle +parent = KernelBundle('/private/espfi/kernel-001-verified') +export_seeded(parent.state, '/private/espfi/seeded-001') +model = SeededKernelBundle('/private/espfi/seeded-001') +logits = model.predict(amplitudes) +``` + +MEASURED on the ruOS CPU executor: weight bytes decreased from 496,488 to +16,088 (96.76%); required metadata adds 913 bytes. All 70 development logits +were exactly identical, maximum absolute error zero. Median loading time over +30 loads was 1.948925 ms. Runtime state remains 61,999 float64 scalars, so this +is a storage and distribution improvement, not RAM or sensing speed reduction. +The separate parity receipt binds parent model digest and dataset provenance. +The RFF candidate still has two zero-F1 activities and is not deployment ready. + +This round used a different CPU host. In a paired 500-call comparison on that +host, float64 spectral, RFF and blend p50 latencies were 0.352430, 0.443975 and +0.804585 ms. These timings cannot be compared directly with prior-host timings +or with the prior float32 spectral deployment artifact. + +The earlier security workflow fails on preexisting Rust dependency alerts in +unchanged `v2/Cargo.lock`; the Python checks pass. That remains a release gate, +alongside independent room/person confirmation. No SOTA or promotion claim. diff --git a/scripts/csi_training/espfi_blend.py b/scripts/csi_training/espfi_blend.py new file mode 100644 index 000000000..7e62fa6b7 --- /dev/null +++ b/scripts/csi_training/espfi_blend.py @@ -0,0 +1,123 @@ +"""Frozen five-weight CSI ensemble experiment. Development evidence only.""" +import argparse +import json +from pathlib import Path +import time +import numpy as np +from safetensors.numpy import save_file, load_file +from .espfi import load_development +from .espfi_generalize import features, folds +from .espfi_kernel import fit, predict_features +from .espfi_train import scores, digest, write_json + +WEIGHTS = (0., .25, .5, .75, 1.) +COMPONENTS = ( + {'family': 'relative_spectral', 'kind': 'linear', 'alpha': 10.}, + {'family': 'relative_combined', 'kind': 'rff', 'dimensions': 128, + 'gamma_multiplier': .1, 'alpha': .01}) +CONTRACT = {'weights': list(WEIGHTS), 'components': list(COMPONENTS), + 'seed': 42, 'selection': 'mean training-fold macro F1 then smaller RFF weight', + 'mix': '(1-w)*spectral_scores + w*RFF_scores; no softmax', + 'test_access': False, 'promotionAuthorized': False} + +def select_weight(fold_predictions, labels): + results = [] + for weight in WEIGHTS: + evaluations = [scores(y, ((1-weight)*a + weight*b).argmax(1), 7) + for (a, b), y in zip(fold_predictions, labels)] + results.append({'weight': weight, 'folds': evaluations, + 'mean_macro_f1': float(np.mean([e['macro_f1'] for e in evaluations]))}) + return max(results, key=lambda r: (r['mean_macro_f1'], -r['weight'])), results + +def infer(states, weight, amplitudes): + result = None + for index, coefficient in ((0, 1-weight), (1, weight)): + if coefficient: + value = coefficient * predict_features(states[index], features(amplitudes, COMPONENTS[index]['family'])) + result = value if result is None else result + value + if result is None or not np.isfinite(result).all(): + raise ValueError('Invalid ensemble output') + return result + +def run(parts, provenance, output): + if set(parts) != {'train', 'validation'}: + raise ValueError('Development partitions only') + root = Path(output) + root.mkdir(parents=True, exist_ok=False) + write_json(root/'status.json', {'status': 'running', 'promotionAuthorized': False}) + write_json(root/'contract.json', CONTRACT) + start = time.monotonic() + try: + train, dev = parts['train'], parts['validation'] + y = train['y'] + fold_ids = folds(train['identity'], y) + cache = [features(train['x'], c['family']) for c in COMPONENTS] + predictions, labels = [], [] + for fit_ids, eval_ids in fold_ids: + pair = [] + for x, candidate in zip(cache, COMPONENTS): + state = fit(x[fit_ids], y[fit_ids], candidate) + pair.append(predict_features(state, x[eval_ids])) + predictions.append(pair) + labels.append(y[eval_ids]) + selected, results = select_weight(predictions, labels) + write_json(root/'selection.json', {'selected': selected, 'candidates': results}) + # Selection is frozen before using any external development signal. + weight = selected['weight'] + states = [fit(x, y, c) for x, c in zip(cache, COMPONENTS)] + active = [i for i, coefficient in ((0, 1-weight), (1, weight)) if coefficient] + tensors = {str(i)+'__'+k: np.ascontiguousarray(v) + for i in active for k, v in states[i].items()} + save_file(tensors, str(root/'model.safetensors')) + restored = load_file(str(root/'model.safetensors')) + restored_states = [{k.split('__', 1)[1]: v for k, v in restored.items() + if k.startswith(str(i)+'__')} for i in range(2)] + before = infer(states, weight, dev['x']) + after = infer(restored_states, weight, dev['x']) + if not np.array_equal(before, after): + raise ValueError('Export parity failed') + metrics = {name: scores(dev['y'], infer(states, w, dev['x']).argmax(1), 7) + for name, w in (('spectral_control', 0.), ('rff_control', 1.), ('selected', weight))} + timings = {name: [] for name in metrics} + choices = [('spectral_control', 0.), ('rff_control', 1.), ('selected', weight)] + for _ in range(20): + for _, w in choices: + infer(states, w, dev['x'][:1]) + for repeat in range(500): + order = choices[repeat % 3:] + choices[:repeat % 3] + for name, w in order: + tick = time.perf_counter_ns() + infer(states, w, dev['x'][:1]) + timings[name].append((time.perf_counter_ns()-tick)/1e6) + report = {'status': 'complete', 'promotionAuthorized': False, + 'evidence': provenance.get('evidence', 'MEASURED'), + 'contract': CONTRACT, 'selected': selected, 'candidates': results, + 'development': metrics, 'provenance': provenance, + 'model_sha256': digest(root/'model.safetensors'), + 'model_bytes': (root/'model.safetensors').stat().st_size, + 'implementation_sha256': digest(__file__), + 'dependency_sha256': {name: digest(Path(__file__).with_name(name)) + for name in ('espfi_kernel.py', 'espfi_generalize.py', 'espfi.py', 'espfi_train.py')}, + 'reload_exact': True, 'active_components': active, + 'latency': {name: {'p50_ms': float(np.percentile(t, 50)), + 'p95_ms': float(np.percentile(t, 95))} for name, t in timings.items()}, + 'timing_scope': '500 rotated CPU batch1 calls including feature extraction and checks; excludes intake and temporal pooling', + 'elapsed_seconds': time.monotonic()-start, + 'limitations': ['Repeatedly used development set; not independent confirmation', + 'No final test evaluation or SOTA evidence', + 'Controls reproduced in float64 on this executor; not prior float32 deployment timing']} + write_json(root/'metrics.json', report) + write_json(root/'status.json', {'status': 'complete', 'promotionAuthorized': False}) + return report + except Exception: + write_json(root/'status.json', {'status': 'failed', 'promotionAuthorized': False}) + raise + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--manifest', required=True) + parser.add_argument('--output', required=True) + args = parser.parse_args() + parts, provenance = load_development(args.manifest) + report = run(parts, provenance, args.output) + print(json.dumps({k: report[k] for k in ('selected', 'development', 'model_bytes', 'latency')}, indent=2)) diff --git a/scripts/csi_training/espfi_seeded.py b/scripts/csi_training/espfi_seeded.py new file mode 100644 index 000000000..bbe53d697 --- /dev/null +++ b/scripts/csi_training/espfi_seeded.py @@ -0,0 +1,72 @@ +"""Lossless seed encoding for the fixed 128-dimensional CSI RFF candidate.""" +import hashlib +import json +from pathlib import Path +import numpy as np +from safetensors.numpy import load_file, save_file +from .espfi_generalize import features +from .espfi_kernel import predict_features +from .espfi_train import digest, write_json + +SPEC = {'family': 'relative_combined', 'dimensions': 128, 'gamma_multiplier': .1, + 'alpha': .01, 'seed': 42, 'input_features': 468} +SHAPES = {'feature_mean': (468,), 'feature_scale': (468,), 'output_mean': (128,), + 'weights': (128, 7), 'bias': (7,)} +def random_map(): + rng = np.random.default_rng(42) + return {'projection': rng.normal(0, np.sqrt(2*.1/468), (468, 128)), + 'phase': rng.uniform(0, 2*np.pi, 128)} +def tensor_hash(value): + return hashlib.sha256(np.ascontiguousarray(value).tobytes()).hexdigest() +def validate(state): + if set(state) != set(SHAPES) or any( + state[k].shape != shape or state[k].dtype != np.float64 or + not np.isfinite(state[k]).all() for k, shape in SHAPES.items() + ) or not (state['feature_scale'] > 0).all(): + raise ValueError('Invalid compact state') + +def export_seeded(state, output): + generated = random_map() + if set(state) != set(SHAPES) | set(generated) or any( + not np.array_equal(state[k], generated[k]) for k in generated + ): + raise ValueError('Projection does not match frozen seed recipe') + compact = {k: np.ascontiguousarray(state[k]) for k in SHAPES} + validate(compact) + root = Path(output) + root.mkdir(exist_ok=False, parents=True) + save_file(compact, str(root/'model.safetensors')) + report = {'status': 'complete', 'promotionAuthorized': False, 'spec': SPEC, + 'model_sha256': digest(root/'model.safetensors'), + 'model_bytes': (root/'model.safetensors').stat().st_size, + 'random_map_sha256': {k: tensor_hash(v) for k,v in generated.items()}, + 'numpy_version': np.__version__, 'implementation_sha256': digest(__file__), + 'feature_implementation_sha256': digest(Path(__file__).with_name('espfi_generalize.py')), + 'stored_scalars': sum(v.size for v in compact.values()), + 'runtime_scalars': sum(v.size for v in state.values()), + 'scope': 'Lossless on-disk encoding only; projection regenerated at load, runtime memory unchanged'} + write_json(root/'metadata.json', report) + return report + +class SeededKernelBundle: + def __init__(self, directory): + root = Path(directory) + for name in ('metadata.json', 'model.safetensors'): + p = root/name + if p.is_symlink() or not p.is_file() or p.stat().st_size > 65536: + raise ValueError('Invalid bounded compact artifact') + report = json.loads((root/'metadata.json').read_text()) + if report.get('status') != 'complete' or report.get('promotionAuthorized') is not False or report.get('spec') != SPEC: + raise ValueError('Invalid compact contract') + if digest(root/'model.safetensors') != report.get('model_sha256'): + raise ValueError('Compact model hash mismatch') + state = load_file(str(root/'model.safetensors')) + validate(state) + generated = random_map() + if {k:tensor_hash(v) for k,v in generated.items()} != report.get('random_map_sha256'): + raise ValueError('Runtime random generator differs from export') + state.update(generated) + self.state = state + + def predict(self, amplitudes): + return predict_features(self.state, features(amplitudes, SPEC['family'])) diff --git a/scripts/improve-espfi-blend.py b/scripts/improve-espfi-blend.py new file mode 100644 index 000000000..7e83d2268 --- /dev/null +++ b/scripts/improve-espfi-blend.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +"""Run the frozen five-weight development blend with bounded CPU threads.""" +import os +for name in ('OPENBLAS_NUM_THREADS', 'OMP_NUM_THREADS', 'MKL_NUM_THREADS'): + os.environ[name] = '1' +from csi_training.espfi_blend import main +if __name__ == '__main__': + main() diff --git a/scripts/tests/test_espfi_blend.py b/scripts/tests/test_espfi_blend.py new file mode 100644 index 000000000..a1380feec --- /dev/null +++ b/scripts/tests/test_espfi_blend.py @@ -0,0 +1,43 @@ +"""Synthetic integrity checks, not sensing accuracy evidence.""" +import copy +import json +from pathlib import Path +import sys +import tempfile +import unittest +import numpy as np +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from csi_training.espfi_blend import run, infer, select_weight +from test_espfi_generalize import fixture + +class BlendTests(unittest.TestCase): + def test_tie_retains_spectral_endpoint(self): + y = np.arange(7) + logits = np.eye(7) + selected, results = select_weight([(logits, logits)], [y]) + self.assertEqual(selected['weight'], 0.) + self.assertEqual(len(results), 5) + + def test_unused_endpoint_is_not_executed(self): + from unittest.mock import patch + logits = np.ones((1, 7)) + with patch('csi_training.espfi_blend.features', return_value=np.ones((1, 468))), patch('csi_training.espfi_blend.predict_features', return_value=logits) as predict: + np.testing.assert_array_equal(infer([{}, None], 0., None), logits) + predict.assert_called_once_with({}, unittest.mock.ANY) + + def test_development_cannot_change_selection_or_model(self): + parts, provenance = fixture() + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + first = run(parts, provenance, root/'first') + changed = copy.deepcopy(parts) + changed['validation']['x'] *= 1.25 + changed['validation']['y'] = (changed['validation']['y'] + 1) % 7 + second = run(changed, provenance, root/'second') + self.assertEqual(first['selected'], second['selected']) + self.assertEqual(first['model_sha256'], second['model_sha256']) + self.assertTrue(first['reload_exact']) + self.assertFalse(first['promotionAuthorized']) + self.assertEqual(json.loads((root/'first/status.json').read_text())['status'], 'complete') + with self.assertRaisesRegex(ValueError, 'Development partitions'): + run(dict(parts, test=parts['validation']), provenance, root/'invalid') diff --git a/scripts/tests/test_espfi_seeded.py b/scripts/tests/test_espfi_seeded.py new file mode 100644 index 000000000..ef14bca5b --- /dev/null +++ b/scripts/tests/test_espfi_seeded.py @@ -0,0 +1,36 @@ +"""Lossless seeded-map integrity tests.""" +from pathlib import Path +import sys +import tempfile +import unittest +import numpy as np +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from csi_training.espfi_kernel import fit, predict_features +from csi_training.espfi_generalize import features +from csi_training.espfi_seeded import SPEC, export_seeded, SeededKernelBundle + +class SeededTests(unittest.TestCase): + def test_exact_export_and_tamper_rejection(self): + amplitudes = np.exp(np.random.default_rng(2).normal(size=(14,52,190))) + x = features(amplitudes, SPEC['family']) + state = fit(x, np.arange(14)%7, dict(SPEC, kind='rff')) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp)/'model' + report = export_seeded(state, path) + loaded = SeededKernelBundle(path) + np.testing.assert_array_equal(predict_features(state,x), loaded.predict(amplitudes)) + self.assertLess(report['model_bytes'], 17000) + self.assertEqual(report['stored_scalars'], 1967) + self.assertEqual(report['runtime_scalars'], 61999) + with (path/'model.safetensors').open('ab') as f: + f.write(b'bad') + with self.assertRaisesRegex(ValueError,'hash mismatch'): + SeededKernelBundle(path) + + def test_wrong_projection_rejected(self): + x = np.random.default_rng(1).normal(size=(14,468)) + state = fit(x,np.arange(14)%7,dict(SPEC,kind='rff')) + state['projection'][0,0] += .1 + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaisesRegex(ValueError,'seed recipe'): + export_seeded(state,Path(tmp)/'model')