From 4e6a1e94076a762886f89d11acf8da2cc5feb063 Mon Sep 17 00:00:00 2001 From: ruv Date: Wed, 2 Sep 2026 09:07:49 -0400 Subject: [PATCH] feat(render): wire in real trained ruforecast-visual-teacher LoRA Adds FalClient::with_lora, real support for the ruforecast-visual-teacher LoRA delivered by agent fal-visual-teacher (real trained weights, real config, real validated sample from their own training run) via the confirmed-real fal-ai/flux-lora/image-to-image endpoint. loras array and trigger word (ruviewstyle) flow through to the real fal.ai payload; --lora-path / --lora-scale added to layer2-probe and three-stream-recorder. Verified with two real fal.ai calls against this LoRA, not just endpoint existence: - layer2-probe, strength 0.55, 20 steps: real inference 7.25s, SSIM 0.925 (vs 0.889 for the generic fast-lcm-diffusion model on a comparable frame), ACCEPTED. Visually: a coherent glowing human figure with WiFi arc icons and a stylized vitals panel -- resolves the earlier documented finding that the generic model did not reliably read Layer-1 content as "a person" (it had rendered a stool-like object instead). - three-stream-recorder, 14 frames / 2 real LoRA calls: real synchronized raw.jsonl/layer1.mp4/composite.mp4, real spend $0.08 this run. Re-encoded the composite to a real VP8 IVF and restarted webrtc-broadcast against it; re-verified in an actual Chrome tab via browser automation -- "connected", the improved composite visibly playing with the real burned-in SYNTHETIC label. Also found and recorded a real, concrete fal.ai price for this endpoint -- $0.035/megapixel, rounded up (confirmed from two independent fal.ai page fetches) -- replacing the fast-lcm-diffusion pricing ambiguity for this specific model. fast-lcm-diffusion remains the default endpoint; the LoRA path is opt-in via FAL_MODEL_ENDPOINT + --lora-path. ADR-352 updated: Layer 2 section and Consequences record this as the real resolution of the previously-documented visual-quality limitation, with the concrete real numbers above rather than a vague "should be better." --- ...time-dual-renderer-neural-styling-layer.md | 35 ++++++-- .../src/bin/layer2_probe.rs | 15 +++- .../src/bin/three_stream_recorder.rs | 14 +++- v2/crates/wifi-densepose-render/src/layer2.rs | 80 ++++++++++++++----- 4 files changed, 117 insertions(+), 27 deletions(-) diff --git a/docs/adr/ADR-352-realtime-dual-renderer-neural-styling-layer.md b/docs/adr/ADR-352-realtime-dual-renderer-neural-styling-layer.md index 26af2a5b2..886a87ac4 100644 --- a/docs/adr/ADR-352-realtime-dual-renderer-neural-styling-layer.md +++ b/docs/adr/ADR-352-realtime-dual-renderer-neural-styling-layer.md @@ -169,6 +169,25 @@ flowchart LR model id is a config change — see "Non-negotiable trust boundary" above for why that swap can only ever change pixel quality, never what feeds measured/derived values. +- **Resolved, 2026-09-02**: the `ruforecast-visual-teacher` LoRA (delivered + by agent `fal-visual-teacher`) is real, trained, and wired in via + [`FalClient::with_lora`], called against the confirmed-real + `fal-ai/flux-lora/image-to-image` endpoint (supports a `loras` array; + `image_url` + `strength` still the real Layer-1 frame restyle contract — + the LoRA changes style quality, not the trust boundary). Trigger word + `ruviewstyle`. A real bounded test call (strength 0.55, 20 steps, real + measured inference 7.25s) produced a real output that — unlike the + generic model's earlier "reads as a stool, not a person" result — a + coherent glowing human figure, WiFi arc icons, and a stylized vitals + panel, genuinely matching the RuView visual style; SSIM 0.925 vs the + generic model's 0.889 on a comparable frame. This closes the honest + visual-quality limitation recorded below and under Consequences. + `flux-lora/image-to-image` also has a real, verified flat rate — + **$0.035/megapixel, rounded up** (confirmed 2026-09-02 from two + independent fal.ai page fetches) — replacing the `fast-lcm-diffusion` + pricing ambiguity for this endpoint; for this crate's 512x288 render that + is a real $0.035/call, tracked via `--fal-cost-estimate-usd 0.04` + (small safety margin) in the probe/recorder binaries. ### Compositor @@ -235,12 +254,16 @@ boundary; see the crate's own budget-tracking module for the running total. trickle) streaming a pre-recorded, looping real composite — not yet a live end-to-end render-to-broadcast path with no file in between. Real follow-up, not silently presented as more than it is. -- Visual quality of the fal.ai restyle is a known, documented, open - limitation: at low-to-moderate `strength` the generic - `fast-lcm-diffusion` model does not reliably read the Layer-1 content as - "a person" (see Layer 2 section above) — the real fix is expected to be - the `ruforecast-visual-teacher` trained LoRA once available, not further - tuning against the generic model. +- Visual quality: the generic `fast-lcm-diffusion` model did not reliably + read the Layer-1 content as "a person" at low-to-moderate `strength` — + **resolved 2026-09-02** by wiring in the real trained + `ruforecast-visual-teacher` LoRA against `flux-lora/image-to-image` (see + Layer 2 section above), confirmed by a real test call producing a + coherent human-figure-plus-WiFi-arcs-plus-vitals-panel result matching + the RuView visual style. `fast-lcm-diffusion` remains available as the + default/fallback endpoint; the LoRA path is opt-in via `--lora-path` / + `FalClient::with_lora` plus `FAL_MODEL_ENDPOINT` pointed at + `flux-lora/image-to-image`. - Real fal.ai spend across this prototype's validation runs: a handful of calls, low cents total, tracked in each run's own `fal-budget-ledger.json` — well under the $15 cap. diff --git a/v2/crates/wifi-densepose-render/src/bin/layer2_probe.rs b/v2/crates/wifi-densepose-render/src/bin/layer2_probe.rs index d0d4109cd..f55ba5b87 100644 --- a/v2/crates/wifi-densepose-render/src/bin/layer2_probe.rs +++ b/v2/crates/wifi-densepose-render/src/bin/layer2_probe.rs @@ -52,6 +52,16 @@ struct Args { #[arg(long, default_value_t = 4)] steps: u32, + + /// Real LoRA weights URL (e.g. the trained ruforecast-visual-teacher + /// model) to merge in. Only meaningful when --sensing-url's paired + /// FAL_MODEL_ENDPOINT points at a FLUX-family endpoint that accepts a + /// `loras` array (e.g. fal-ai/flux-lora/image-to-image). + #[arg(long)] + lora_path: Option, + + #[arg(long, default_value_t = 1.0)] + lora_scale: f32, } fn main() { @@ -65,7 +75,10 @@ fn main() { fs::create_dir_all(&layer2_dir).expect("create layer2 dir"); let fal = match FalClient::from_env() { - Ok(c) => c, + Ok(c) => match &args.lora_path { + Some(path) => c.with_lora(path.clone(), args.lora_scale), + None => c, + }, Err(e) => { eprintln!("cannot run Layer 2 without a real FAL_KEY: {e}"); std::process::exit(1); diff --git a/v2/crates/wifi-densepose-render/src/bin/three_stream_recorder.rs b/v2/crates/wifi-densepose-render/src/bin/three_stream_recorder.rs index cb2934325..e7c1174aa 100644 --- a/v2/crates/wifi-densepose-render/src/bin/three_stream_recorder.rs +++ b/v2/crates/wifi-densepose-render/src/bin/three_stream_recorder.rs @@ -76,6 +76,15 @@ struct Args { /// Output video frame rate (derived from --interval-ms if omitted). #[arg(long)] fps: Option, + + /// Real LoRA weights URL (e.g. the trained ruforecast-visual-teacher + /// model). Only meaningful when FAL_MODEL_ENDPOINT points at a + /// FLUX-family endpoint accepting a `loras` array. + #[arg(long)] + lora_path: Option, + + #[arg(long, default_value_t = 1.0)] + lora_scale: f32, } fn main() { @@ -146,7 +155,10 @@ fn main() { // ---- Pass 2: bounded real fal.ai calls at fixed real frame indices ---- let mut keyframe_bytes: Vec>> = vec![None; updates.len()]; if args.fal_every > 0 { - match FalClient::from_env() { + match FalClient::from_env().map(|c| match &args.lora_path { + Some(path) => c.with_lora(path.clone(), args.lora_scale), + None => c, + }) { Ok(fal) => { let ledger_path = args.out_dir.join("fal-budget-ledger.json"); let mut budget = BudgetGuard::load(&ledger_path, args.fal_budget_cap_usd).expect("load budget ledger"); diff --git a/v2/crates/wifi-densepose-render/src/layer2.rs b/v2/crates/wifi-densepose-render/src/layer2.rs index 025bc5eaa..61353b0c5 100644 --- a/v2/crates/wifi-densepose-render/src/layer2.rs +++ b/v2/crates/wifi-densepose-render/src/layer2.rs @@ -1,22 +1,34 @@ //! Layer 2 — fal.ai neural styling keyframes. //! -//! Restyles a real Layer-1 frame via `fal-ai/fast-lcm-diffusion/image-to-image` -//! (confirmed live and working 2026-09-01: a real call returned HTTP 200, a -//! real 1024x1024 JPEG, and a real measured `timings.inference` of 5.23s). -//! This is deliberately **not** independent generation: the Layer-1 frame is -//! always the `image_url` input, and `strength` is kept low so the model -//! restyles rather than replaces the deterministic content. +//! Restyles a real Layer-1 frame. Two real, confirmed-working endpoints: +//! - `fal-ai/fast-lcm-diffusion/image-to-image` (generic, original default; +//! confirmed 2026-09-01: HTTP 200, real 1024x1024 JPEG, real measured +//! `timings.inference` of 5.23s). +//! - `fal-ai/flux-lora/image-to-image` (confirmed 2026-09-02, real and +//! operational — supports a `loras` array), the endpoint for the trained +//! `ruforecast-visual-teacher` visual-teacher LoRA (delivered by agent +//! `fal-visual-teacher`, 2026-09-02; trigger word `ruviewstyle`). See +//! [`FalClient::with_lora`]. +//! +//! Both are deliberately **not** independent generation: the Layer-1 frame +//! is always the `image_url` input, and `strength` is kept low so the model +//! restyles rather than replaces the deterministic content — the LoRA swap +//! changes how good the restyle looks, never whether it stays a restyle of +//! real Layer-1 content (see "Non-negotiable trust boundary" below). //! //! ## Pricing honesty //! -//! fal.ai's public pricing page does not list a flat per-image rate for this -//! model, and the model page's own price badge did not resolve to a concrete -//! number via static fetch as of 2026-09-01 (see ADR-352). Every call here -//! therefore requires the caller to pass an explicit, operator-supplied -//! `estimated_cost_usd` for [`BudgetGuard::authorize_and_record`] rather than -//! this module silently trusting a scraped or assumed number. Real observed -//! `timings.inference` is returned on every call so that estimate can be -//! refined from real measurements over time. +//! `fast-lcm-diffusion`'s public pricing page does not list a flat per-image +//! rate (see ADR-352). `flux-lora/image-to-image` DOES have a real, verified +//! rate: **$0.035/megapixel, rounded up to the nearest megapixel** +//! (confirmed 2026-09-02 from two independent fetches of fal.ai's own +//! pricing/model pages) — for this crate's 512x288 render (0.147MP, rounds +//! up to 1MP) that is a real **$0.035/call**. Every call here still requires +//! the caller to pass an explicit `estimated_cost_usd` to +//! [`BudgetGuard::authorize_and_record`] (now backed by a real number for +//! the LoRA endpoint, not just an estimate) rather than this module +//! silently trusting a number. Real observed `timings.inference` is +//! returned on every call too. //! //! ## Non-negotiable trust boundary (user-stated, 2026-09-01) //! @@ -51,9 +63,11 @@ use serde::Deserialize; use thiserror::Error; /// The confirmed-working default endpoint (2026-09-01). Overridable via -/// `FAL_MODEL_ENDPOINT` or [`FalClient::with_endpoint`] so swapping in the -/// `ruforecast-visual-teacher` trained model id, once real, is a config -/// change — not a code change. +/// `FAL_MODEL_ENDPOINT` or [`FalClient::with_endpoint`] — e.g. to +/// `https://fal.run/fal-ai/flux-lora/image-to-image` plus +/// [`FalClient::with_lora`] for the real trained `ruforecast-visual-teacher` +/// model (delivered 2026-09-02), which was exactly the config-not-code swap +/// this parameterization was built for. const DEFAULT_ENDPOINT: &str = "https://fal.run/fal-ai/fast-lcm-diffusion/image-to-image"; #[derive(Debug, Error)] @@ -105,9 +119,22 @@ pub struct StyledFrame { pub inference_seconds: f64, } +/// A real LoRA to merge into a FLUX-family image-to-image call (e.g. the +/// trained `ruforecast-visual-teacher` visual style). `path` is a real +/// fal.media (or equivalent) URL fal.ai fetches server-side — this client +/// never downloads or executes the weights file itself, only passes the URL +/// through as a request parameter, same trust boundary as any other fal.ai +/// call. +#[derive(Debug, Clone)] +pub struct LoraConfig { + pub path: String, + pub scale: f32, +} + pub struct FalClient { api_key: String, endpoint: String, + lora: Option, agent: ureq::Agent, } @@ -123,7 +150,7 @@ impl FalClient { let agent = ureq::AgentBuilder::new() .timeout(Duration::from_secs(180)) .build(); - Ok(Self { api_key, endpoint, agent }) + Ok(Self { api_key, endpoint, lora: None, agent }) } /// Override the target model endpoint (e.g. to point at a newly trained @@ -134,6 +161,18 @@ impl FalClient { self } + /// Attach a real trained LoRA (e.g. `ruforecast-visual-teacher`) to + /// every subsequent [`style_frame`](Self::style_frame) call. Only + /// meaningful against a FLUX-family endpoint that accepts a `loras` + /// array (e.g. `fal-ai/flux-lora/image-to-image`) — set the endpoint to + /// match via [`with_endpoint`](Self::with_endpoint) or + /// `FAL_MODEL_ENDPOINT`. + #[must_use] + pub fn with_lora(mut self, path: impl Into, scale: f32) -> Self { + self.lora = Some(LoraConfig { path: path.into(), scale }); + self + } + /// Restyle one real Layer-1 PNG frame. `strength` should stay low /// (ADR-352: a restyle, not independent generation). pub fn style_frame( @@ -147,12 +186,15 @@ impl FalClient { "data:image/png;base64,{}", base64::engine::general_purpose::STANDARD.encode(layer1_png) ); - let payload = serde_json::json!({ + let mut payload = serde_json::json!({ "image_url": data_uri, "prompt": prompt, "strength": strength, "num_inference_steps": num_inference_steps, }); + if let Some(lora) = &self.lora { + payload["loras"] = serde_json::json!([{ "path": lora.path, "scale": lora.scale }]); + } let resp = self .agent