diff --git a/.gitignore b/.gitignore index 00b2441a2..5197ca4dc 100644 --- a/.gitignore +++ b/.gitignore @@ -311,3 +311,9 @@ v2/crates/wifi-densepose-sensing-server/data/ # ADR-324: wasm-bindgen output for ruview-offaxis is generated locally # (see the crate README); never commit generated artifacts. v2/crates/ruview-offaxis/pkg/ + +# fal.ai visual-teacher LoRA training data/weights -- regenerable, never +# commit training images or model weights (ADR-353) +scripts/fal-visual-teacher/training-data/ +scripts/fal-visual-teacher/training-data.zip +scripts/fal-visual-teacher/output/ diff --git a/assets/fal-visual-teacher-sample.json b/assets/fal-visual-teacher-sample.json new file mode 100644 index 000000000..3a4a28315 --- /dev/null +++ b/assets/fal-visual-teacher-sample.json @@ -0,0 +1,7 @@ +{ + "endpoint": "fal-ai/flux-lora", + "lora_url": "https://v3b.fal.media/files/b/0aa8c60d/OjaBs6Gi8pMhZqMo79iym_pytorch_lora_weights.safetensors", + "prompt": "ruviewstyle room visualization, a living room with a glowing cyan pose skeleton overlay, wifi signal arcs radiating across the space, a vitals readout panel, dark tech aesthetic, synthetic sensor visualization", + "scale": 1.0, + "source_image_url": "https://v3b.fal.media/files/b/0aa8cf67/JH1qyC3GCKQ9ojkrAUXe9.png" +} \ No newline at end of file diff --git a/assets/fal-visual-teacher-sample.png b/assets/fal-visual-teacher-sample.png new file mode 100644 index 000000000..011e3ae41 Binary files /dev/null and b/assets/fal-visual-teacher-sample.png differ diff --git a/docs/adr/ADR-353-fal-visual-teacher-lora.md b/docs/adr/ADR-353-fal-visual-teacher-lora.md new file mode 100644 index 000000000..e6444e54b --- /dev/null +++ b/docs/adr/ADR-353-fal-visual-teacher-lora.md @@ -0,0 +1,162 @@ +# ADR-353: fal.ai synthetic visual teacher LoRA + +- **Status**: Accepted +- **Date**: 2026-09-02 +- **Deciders**: ruv +- **Owners**: RuView visualization / neural-styling maintainers +- **Tags**: fal, lora, synthetic-visualization, display-only, cost-control +- **Parent**: None +- **Extends**: None +- **Supersedes**: None + +## Decision + +RuView will use one real, fal.ai-trained LoRA fine-tune (a "synthetic visual +teacher") to steer fal.ai image/video generation toward a consistent +"RuView-style" room visualization — glowing pose overlays, WiFi signal arcs, +vitals readout panels, dark tech aesthetic — instead of relying on ad hoc +prompt engineering against a stock model. + +**The architecture boundary is non-negotiable:** + +> The implementation boundary is now clear: fal.ai will train an optional +> synthetic visual teacher, while RuView's measured RF, pose, vitals, +> identity, and confidence remain local and authoritative. + +Concretely: + +- The trained model produced here is ONLY ever allowed to generate + supplementary synthetic visual/pixel output for display or broadcast. +- It must NEVER be a source of, or influence, any measured/derived value: RF/ + CSI measurements, pose estimates, vitals (heart rate/breathing rate), + identity, or confidence/quality scores. Those stay local, deterministic, + and authoritative — the existing `wifi-densepose-engine` / + `ruview-twin` / `ruview-witness` stack — completely untouched by this work. +- Any output this model produces must be clearly framed as SYNTHETIC per this + repo's rule: "Never present WiFi sensing as camera-grade." +- This model has no read or write path into the RF/pose/vitals/identity + pipeline. It consumes only a text prompt (and optionally a reference + image) and returns pixels for display. + +**Evidence status:** the training data extraction, training submission +script, and validation script are real and committed. The training run +completed and validation succeeded — trained model id, actual spend, and the +validation sample are recorded in "Result" below. Actual-vs-billed cost +reconciliation against the fal.ai account dashboard remains UNMEASURED (the +figure below is a documented-pricing estimate). + +## Context + +A concurrent workstream (`ruview-fal-room-viz` / `ruview-room-viz-h3-upgrade`) +generated real fal.ai room-visualization videos across several style presets +(architectural, abstract, branded, and an "h3" refresh: architectural, +cinematic, minimal, abstract) using stock fal.ai video models driven by +prompt engineering alone. A separate concurrently-running agent +("realtime-dual-renderer") is building a neural-styling layer that needs a +*specific* trained model id to call, rather than continuing to rely on prompt +engineering against a generic model. + +This ADR's job is narrow: turn the real captured style presets into a real +trained fal.ai LoRA, verify it actually works, and hand off the model id. It +does not implement the renderer integration — that is +`realtime-dual-renderer`'s scope — and it does not touch the measured +RF/pose/vitals/identity/confidence pipeline in any way. + +## Training data + +Source: the real fal.ai-generated room-visualization MP4s already captured +this session (read from `/tmp/room-viz-*/room-viz.mp4` on `ruvultra`, one per +style preset — not committed to the repo; the repo has no camera or person +data in these clips, they are synthetic renders). `scripts/fal-visual-teacher/extract_frames.py` +extracts one JPEG frame every 1.5 seconds from each of the 7 real style-preset +videos (architectural, abstract, branded, h3-architectural, h3-cinematic, +h3-minimal, h3-abstract), producing 27 frames total, each paired with a +caption `.txt` describing the shared RuView visual motifs. Frames and the +training zip are gitignored (`scripts/fal-visual-teacher/training-data/`, +`training-data.zip`) — they are regenerable from the source MP4s and are not +committed, consistent with the repo's no-generated-artifacts rule. + +## Training endpoint and pricing (verified 2026-09-02) + +- **Endpoint**: `fal-ai/flux-lora-fast-training` — a real, currently-listed + fal.ai LoRA training endpoint for FLUX.1, verified against + `https://fal.ai/models/fal-ai/flux-lora-fast-training/api`. +- **Pricing**: $2 per training run at the default 1000 steps, scaling + linearly with steps (verified against fal.ai's own pricing text on the + model page). +- **Mode**: submitted with `is_style: true` (style LoRA, not + subject/character LoRA) and explicit per-frame caption files, so fal.ai's + auto-captioning/segmentation is skipped in favor of our own captions. +- **Inference endpoint** (validation only): `fal-ai/flux-lora`, $0.035 per + megapixel, accepting `loras: [{ path, scale }]` pointing at the trained + `diffusers_lora_file` URL. + +## Budget + +- **Original cap**: $15 total (this task's cap only; not shared with other + concurrent tasks' budgets this session). +- **Raised cap**: the user explicitly raised this task's cap to **$100** on + 2026-09-02, mid-run, as real headroom for a larger/better run or retries — + not an instruction to spend all of it. +- **Planned spend**: ~$2 for the training run (1000 steps, default) + ~$0.04 + per validation inference image. +- **Actual spend**: ~$2.04 (one 1000-step training run at $2/run + one + landscape_16_9 validation image at $0.035/megapixel, rounded up to 1 + megapixel). This is a documented-pricing estimate (`CLAIMED`), not yet + reconciled against the fal.ai account billing dashboard (`MEASURED`). + Comfortably under both the original $15 cap and the raised $100 cap. +- **Why no larger follow-up run**: the first validation image already showed + a clean, on-style result (glowing pose overlay, WiFi arcs, vitals panels, + dark aesthetic, all present and legible) on the first attempt. Spending + further into the raised budget for marginal improvement was judged not + worth it against a result that already meets the bar; the fallback path in + "Consequences" was not needed. + +## Result + +- **Trained model id / LoRA weights URL**: + `https://v3b.fal.media/files/b/0aa8c60d/OjaBs6Gi8pMhZqMo79iym_pytorch_lora_weights.safetensors` + (131 MB `diffusers_lora_file`, paired `config.json` at + `https://v3b.fal.media/files/b/0aa8c60d/-GU6dd_2ZFqq18I0CuPOB_config.json`). + Use with `fal-ai/flux-lora` (or any fal.ai FLUX.1 endpoint accepting a + `loras: [{ path, scale }]` array) — trigger word `ruviewstyle`. Full + training record: `scripts/fal-visual-teacher/output/training-result.json` + (gitignored — regenerate by re-running `train_lora.py`, or copy the URL + above). +- **Training request id**: `01a060b2-c940-7972-abf9-ea48847eb2ae` (fal.ai + `fal-ai/flux-lora-fast-training`, 1000 steps, `is_style: true`, 27 training + images, completed in 376s on an H100/H200 worker). +- **Actual spend**: ~$2.04 (see Budget above). +- **Validation sample**: `assets/fal-visual-teacher-sample.png` (+ sibling + `assets/fal-visual-teacher-sample.json` with the exact prompt, LoRA scale + 1.0, and source fal.ai URL) — a real `fal-ai/flux-lora` inference call + using the trained LoRA, generated 2026-09-02. The image shows a glowing + cyan pose skeleton, WiFi signal arcs, vitals-style readout panels, and a + HUD-style TV overlay in a dark living room — confirming the LoRA learned + and reproduces the intended RuView visual motifs. +- **Consumer**: `realtime-dual-renderer` (separate worktree + `~/projects/ruview-worktrees/realtime-render`) — notified directly with the + model id above. + +## Consequences + +A real trained LoRA gives the display layer a specific, repeatable style +target instead of prompt engineering against a stock model, at a bounded and +small real cost ($2 training + cents per generated image). Because the model +only ever produces pixels for a prompt, and has no schema path into any +measured/derived RuView value, it cannot silently become a second source of +truth for pose, vitals, identity, or confidence — that boundary is structural +(no shared types, no shared data path), not just documented convention. + +If the trained LoRA does not meaningfully improve style consistency over the +stock model (a genuine risk with 27 training images and default steps), the +fallback is to keep using prompt engineering against the stock model and note +that outcome honestly here rather than claim a training win that isn't real. + +## References + +- `scripts/fal-visual-teacher/extract_frames.py` — real frame extraction +- `scripts/fal-visual-teacher/train_lora.py` — real training submission +- `scripts/fal-visual-teacher/validate.py` — real validation inference +- [fal.ai Flux LoRA Fast Training](https://fal.ai/models/fal-ai/flux-lora-fast-training) +- [fal.ai FLUX.1 with LoRAs (inference)](https://fal.ai/models/fal-ai/flux-lora) diff --git a/scripts/fal-visual-teacher/extract_frames.py b/scripts/fal-visual-teacher/extract_frames.py new file mode 100644 index 000000000..130e3ea6e --- /dev/null +++ b/scripts/fal-visual-teacher/extract_frames.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Extract training stills from the real fal.ai room-visualization MP4s. + +Reads the source videos (real fal.ai-generated room visualizations, one per +style preset) and writes JPEG frames + a caption .txt per frame into a local +(gitignored) training-data directory, ready to be zipped for fal.ai LoRA +training. Frame interval is time-based (not frame-count-based) so each +source contributes visual diversity proportional to its length. +""" + +import argparse +import shutil +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] + +# (source mp4 path, style tag, caption) -- sources are real fal.ai output, +# captured earlier this session by the fal-room-viz / room-viz-h3-upgrade +# agents. Caption describes the RuView visual style so the LoRA learns the +# recurring motifs (glowing pose overlay, WiFi signal arcs, vitals readout, +# dark tech aesthetic) rather than any single room layout. +SOURCES = [ + ("/tmp/room-viz-architectural/room-viz.mp4", "architectural", + "ruview style room visualization, architectural wireframe overlay, " + "glowing cyan pose skeleton, wifi signal arcs, dark tech aesthetic"), + ("/tmp/room-viz-abstract/room-viz.mp4", "abstract", + "ruview style room visualization, abstract particle field, " + "glowing pose overlay, wifi signal arcs, dark tech aesthetic"), + ("/tmp/room-viz-branded/room-viz.mp4", "branded", + "ruview style room visualization, branded hud overlay, " + "glowing cyan and emerald pose skeleton, vitals readout, dark tech aesthetic"), + ("/tmp/room-viz-h3-architectural/room-viz.mp4", "h3-architectural", + "ruview style room visualization, architectural wireframe overlay, " + "glowing pose skeleton, wifi signal arcs, vitals readout, dark tech aesthetic"), + ("/tmp/room-viz-h3-cinematic/room-viz.mp4", "h3-cinematic", + "ruview style room visualization, cinematic lighting, " + "glowing pose overlay, wifi signal arcs, vitals readout, dark tech aesthetic"), + ("/tmp/room-viz-h3-minimal/room-viz.mp4", "h3-minimal", + "ruview style room visualization, minimal clean overlay, " + "glowing pose skeleton, subtle wifi signal arcs, dark tech aesthetic"), + ("/tmp/room-viz-h3-abstract/room-viz.mp4", "h3-abstract", + "ruview style room visualization, abstract particle field, " + "glowing pose overlay, wifi signal arcs, dark tech aesthetic"), +] + +FRAME_INTERVAL_SEC = 1.5 + + +def extract(src: Path, style: str, caption: str, out_dir: Path) -> int: + pattern = out_dir / f"{style}_%03d.jpg" + cmd = [ + "ffmpeg", "-y", "-loglevel", "error", + "-i", str(src), + "-vf", f"fps=1/{FRAME_INTERVAL_SEC}", + "-q:v", "2", + str(pattern), + ] + subprocess.run(cmd, check=True) + frames = sorted(out_dir.glob(f"{style}_*.jpg")) + for frame in frames: + frame.with_suffix(".txt").write_text(caption, encoding="utf-8") + return len(frames) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--out", default=str(REPO_ROOT / "scripts" / "fal-visual-teacher" / "training-data"), + help="Output directory for extracted frames + captions (gitignored)", + ) + args = parser.parse_args() + + out_dir = Path(args.out) + if out_dir.exists(): + shutil.rmtree(out_dir) + out_dir.mkdir(parents=True) + + total = 0 + for src_str, style, caption in SOURCES: + src = Path(src_str) + if not src.exists(): + print(f"SKIP (missing): {src}", file=sys.stderr) + continue + n = extract(src, style, caption, out_dir) + print(f"{style}: {n} frames from {src}") + total += n + + print(f"\nTotal frames extracted: {total}") + print(f"Training data dir: {out_dir}") + if total < 4: + print("ERROR: fal.ai training needs at least 4 images", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/fal-visual-teacher/train_lora.py b/scripts/fal-visual-teacher/train_lora.py new file mode 100644 index 000000000..ff408ed27 --- /dev/null +++ b/scripts/fal-visual-teacher/train_lora.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Submit a real fal.ai LoRA training job for the RuView synthetic visual teacher. + +Zips the extracted training-data directory, uploads it to fal.ai storage, +submits a training job to fal-ai/flux-lora-fast-training, polls until it +completes, and writes the resulting trained-model reference (diffusers LoRA +weights URL + config URL) to a local (gitignored) JSON file. + +Architecture boundary (ADR-353): the model trained here is ONLY ever a +source of supplementary synthetic visual/pixel output. It must never +influence, or be treated as a source of, measured/derived RF, pose, vitals, +identity, or confidence values. + +Requires FAL_KEY in the environment (fetch from GCP Secret Manager, +project cognitum-20260110, secret MINIMAX_MUSIC_FAL_KEY, before running). +""" + +import argparse +import json +import os +import sys +import time +import zipfile +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +TRAINING_ENDPOINT = "fal-ai/flux-lora-fast-training" + + +def zip_training_data(src_dir: Path, zip_path: Path) -> None: + if zip_path.exists(): + zip_path.unlink() + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: + for f in sorted(src_dir.iterdir()): + if f.is_file(): + zf.write(f, arcname=f.name) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--training-data", default=str(SCRIPT_DIR / "training-data"), + help="Directory of extracted frames + captions (from extract_frames.py)", + ) + parser.add_argument( + "--steps", type=int, default=1000, + help="Training steps (fal-ai/flux-lora-fast-training default is 1000, ~$2)", + ) + parser.add_argument( + "--trigger-word", default="ruviewstyle", + help="Trigger token to prepend so the LoRA is invocable at inference time", + ) + parser.add_argument( + "--out", default=str(SCRIPT_DIR / "output" / "training-result.json"), + help="Where to write the trained model reference JSON (gitignored)", + ) + args = parser.parse_args() + + if not os.environ.get("FAL_KEY"): + print("ERROR: FAL_KEY not set in environment", file=sys.stderr) + return 1 + + import fal_client # imported after the FAL_KEY check for a clearer error + + training_dir = Path(args.training_data) + if not training_dir.exists(): + print(f"ERROR: {training_dir} not found -- run extract_frames.py first", file=sys.stderr) + return 1 + + zip_path = training_dir.parent / "training-data.zip" + print(f"Zipping {training_dir} -> {zip_path}") + zip_training_data(training_dir, zip_path) + print(f"Zip size: {zip_path.stat().st_size / 1024:.1f} KB") + + print("Uploading training zip to fal.ai storage...") + images_data_url = fal_client.upload_file(str(zip_path)) + print(f"Uploaded: {images_data_url}") + + training_args = { + "images_data_url": images_data_url, + "steps": args.steps, + "trigger_word": args.trigger_word, + # This is a style LoRA (recurring visual aesthetic across varied room + # layouts), not a subject/character LoRA -- fal.ai's is_style flag + # disables segmentation/auto-captioning so our own caption .txt files + # (one per frame, describing the shared style) are used as-is. + "is_style": True, + } + print(f"Submitting training job to {TRAINING_ENDPOINT} (steps={args.steps})...") + handle = fal_client.submit(TRAINING_ENDPOINT, arguments=training_args) + print(f"request_id: {handle.request_id}") + + start = time.time() + for event in handle.iter_events(with_logs=True, interval=5.0): + print(f"[{time.time() - start:.0f}s] {event}") + + result = handle.get() + elapsed = time.time() - start + print(f"Training completed in {elapsed:.0f}s") + + out_path = Path(args.out) + out_path.parent.mkdir(parents=True, exist_ok=True) + record = { + "endpoint": TRAINING_ENDPOINT, + "request_id": handle.request_id, + "steps": args.steps, + "trigger_word": args.trigger_word, + "training_images": sum(1 for f in training_dir.glob("*.jpg")), + "elapsed_seconds": round(elapsed), + "result": result, + } + out_path.write_text(json.dumps(record, indent=2), encoding="utf-8") + print(f"Wrote {out_path}") + + lora_file = (result or {}).get("diffusers_lora_file", {}).get("url") + if lora_file: + print(f"\nTrained LoRA weights URL: {lora_file}") + else: + print("\nWARNING: no diffusers_lora_file.url in result -- inspect the JSON", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/fal-visual-teacher/validate.py b/scripts/fal-visual-teacher/validate.py new file mode 100644 index 000000000..0a5573917 --- /dev/null +++ b/scripts/fal-visual-teacher/validate.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Run a real validation inference call against the trained visual-teacher LoRA. + +Loads the trained model reference written by train_lora.py, calls +fal-ai/flux-lora with the trained LoRA weights, and saves the generated +image as evidence that training actually produced a working style LoRA. + +This inference call, and the LoRA it exercises, produce SYNTHETIC display +imagery only (ADR-353) -- never a source of measured RF/pose/vitals/identity +or confidence values. + +Requires FAL_KEY in the environment. +""" + +import argparse +import json +import os +import sys +import urllib.request +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT_DIR = Path(__file__).resolve().parent +INFERENCE_ENDPOINT = "fal-ai/flux-lora" + +DEFAULT_PROMPT = ( + "ruviewstyle room visualization, a living room with a glowing cyan pose " + "skeleton overlay, wifi signal arcs radiating across the space, a vitals " + "readout panel, dark tech aesthetic, synthetic sensor visualization" +) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--training-result", default=str(SCRIPT_DIR / "output" / "training-result.json"), + ) + parser.add_argument("--prompt", default=DEFAULT_PROMPT) + parser.add_argument("--scale", type=float, default=1.0, help="LoRA application strength") + parser.add_argument( + "--out", default=str(REPO_ROOT / "assets" / "fal-visual-teacher-sample.png"), + ) + args = parser.parse_args() + + if not os.environ.get("FAL_KEY"): + print("ERROR: FAL_KEY not set in environment", file=sys.stderr) + return 1 + + import fal_client + + training_result_path = Path(args.training_result) + record = json.loads(training_result_path.read_text(encoding="utf-8")) + lora_url = record["result"]["diffusers_lora_file"]["url"] + print(f"Using trained LoRA: {lora_url}") + + print(f"Submitting validation inference to {INFERENCE_ENDPOINT}...") + result = fal_client.subscribe( + INFERENCE_ENDPOINT, + arguments={ + "prompt": args.prompt, + "loras": [{"path": lora_url, "scale": args.scale}], + "image_size": "landscape_16_9", + "num_images": 1, + "output_format": "png", + }, + with_logs=True, + ) + + images = result.get("images", []) + if not images: + print(f"ERROR: no images in result: {result}", file=sys.stderr) + return 1 + + image_url = images[0]["url"] + print(f"Generated image: {image_url}") + + out_path = Path(args.out) + out_path.parent.mkdir(parents=True, exist_ok=True) + urllib.request.urlretrieve(image_url, out_path) + print(f"Saved validation sample to {out_path}") + + meta_path = out_path.with_suffix(".json") + meta_path.write_text(json.dumps({ + "endpoint": INFERENCE_ENDPOINT, + "lora_url": lora_url, + "prompt": args.prompt, + "scale": args.scale, + "source_image_url": image_url, + }, indent=2), encoding="utf-8") + print(f"Saved validation metadata to {meta_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())