diff --git a/refract/CHANGELOG.md b/refract/CHANGELOG.md index 603d179..102c844 100644 --- a/refract/CHANGELOG.md +++ b/refract/CHANGELOG.md @@ -5,10 +5,21 @@ matrix result that motivated or validated the change. --- -## v0.3.2.1 — vLLM and SGLang backends production (2026-05-02) +## v0.3.2.1 — vLLM and SGLang backends production + extra-flags escape hatch (2026-05-02) ### What changed +- **`REFRACT_LLAMA_EXTRA_FLAGS` env var** — appended to every + `llama-cli` / `llama-completion` / `llama-perplexity` subprocess. + Lets users on constrained VRAM run REFRACT against large MoE + models without forking the runner (e.g. + `REFRACT_LLAMA_EXTRA_FLAGS="-ngl 28 -ncmoe 32"` on a 12 GB 3060 + running Qwen3.6-35B-A3B). llama.cpp's last-wins rule means the + user's `-ngl` overrides REFRACT's default `-ngl 99`. Parsed with + shlex so quoted args work as on the command line. Reported by + AJ on X (2026-05-02). + + - **vLLM backend rewritten from skeleton to working implementation** (`refract/backends/vllm.py`). Cached in-process LLM with evict-on-key-change for memory-pressured deployments (hybrid models diff --git a/refract/PITFALLS.md b/refract/PITFALLS.md index 8c27a28..ac74b29 100644 --- a/refract/PITFALLS.md +++ b/refract/PITFALLS.md @@ -5,6 +5,26 @@ discoverable so first-time users don't repeat the discovery. ## Setup +### CUDA OOM on consumer GPUs running large MoE models + +REFRACT defaults to `-ngl 99` (all layers on GPU). Consumer cards (12 +GB 3060, 16 GB 4060 Ti, etc.) running large MoE models like +Qwen3.6-35B-A3B will OOM out of the box. The fix is to pass extra +llama.cpp flags via `REFRACT_LLAMA_EXTRA_FLAGS`: + +```bash +# 12 GB GPU + Qwen3.6-35B-A3B +export REFRACT_LLAMA_EXTRA_FLAGS="-ngl 28 -ncmoe 32" +python3 -m refract.cli score --backend llamacpp ... +``` + +The env var is appended to every `llama-cli` / `llama-completion` / +`llama-perplexity` invocation. llama.cpp uses last-wins on repeated +flags, so `-ngl 28` in `REFRACT_LLAMA_EXTRA_FLAGS` overrides the +default `-ngl 99`. + +Reported by AJ on a 3060 (2026-05-02). Added in v0.3.2.1. + ### Wrong default `--axis-a` in v0.1.x / v0.2.x The `gtm` axis used `llama-tokenize` to retokenize the candidate's diff --git a/refract/QUICKSTART.md b/refract/QUICKSTART.md index 4642052..54a2777 100644 --- a/refract/QUICKSTART.md +++ b/refract/QUICKSTART.md @@ -129,6 +129,35 @@ checkout, you need: explicit paths (CI-friendly). - The prompts JSONL ships at `refract/prompts/v0.1.jsonl`. +## Constrained VRAM? Pass extra llama.cpp flags + +REFRACT defaults to `-ngl 99` (all layers on GPU) for the llama.cpp +backend. Consumer-card users running large MoE models (e.g. +Qwen3.6-35B-A3B on a 12 GB 3060) won't fit that — they need +`-ncmoe N` to offload some MoE expert layers to CPU. + +Pass any extra llama.cpp flags via `REFRACT_LLAMA_EXTRA_FLAGS`: + +```bash +# 12 GB consumer GPU running Qwen3.6-35B-A3B with MoE offload +export REFRACT_LLAMA_EXTRA_FLAGS="-ngl 28 -ncmoe 32" +python3 -m refract.cli score --backend llamacpp --model /path/to/model.gguf ... +``` + +The flags get appended to every `llama-cli`, `llama-completion`, and +`llama-perplexity` invocation **after** REFRACT's own. llama.cpp uses +last-wins for repeated flags, so `REFRACT_LLAMA_EXTRA_FLAGS="-ngl 28 +-ncmoe 32"` overrides the default `-ngl 99`. Parsed with `shlex` so +quoted args work the same as on the command line. + +Confirmed working scenarios: +- Consumer 12 GB GPU + 35B-A3B MoE: `-ngl 28 -ncmoe 32` +- CPU-only fallback: `-ngl 0` +- Tensor split across multiple GPUs: `-ts 1,1` + +If a flag REFRACT doesn't recognize trips up its own subprocess, +open an issue with the failing command line and we'll plumb it. + ## Step 2 — preflight (~30 seconds) ```bash diff --git a/refract/runner.py b/refract/runner.py index a1d028d..29dcc50 100644 --- a/refract/runner.py +++ b/refract/runner.py @@ -32,6 +32,25 @@ DEFAULT_BIN_DIR = Path( ) +def _llama_extra_flags() -> list[str]: + """Extra flags appended to every llama-cli / llama-completion / + llama-perplexity / llama-tokenize invocation. + + Set via the ``REFRACT_LLAMA_EXTRA_FLAGS`` env var. Useful for users on + constrained VRAM running large models (e.g. ``-ncmoe 32`` to offload + MoE expert layers to CPU on a 12 GB consumer GPU running Qwen3.6-35B-A3B). + + Parsed with shlex so quoted args work the same as on the command line. + Empty / unset returns []. The flags are appended **after** REFRACT's own + -ngl / -c / etc., so a user's ``REFRACT_LLAMA_EXTRA_FLAGS="-ngl 28 -ncmoe 32"`` + will override REFRACT's default ``-ngl 99`` (last wins in llama.cpp). + """ + raw = os.environ.get("REFRACT_LLAMA_EXTRA_FLAGS", "").strip() + if not raw: + return [] + return shlex.split(raw) + + # v0.3.1: backend dispatch. CLI sets this via set_active_backend() based on # --backend flag or REFRACT_BACKEND env. When set, the legacy run_completion # / run_completion_trajectory functions delegate to the backend so MLX or @@ -279,6 +298,7 @@ def run_completion( if system: cmd.extend(["-sys", system]) cmd.extend(kv.cli_args()) + cmd.extend(_llama_extra_flags()) env = os.environ.copy() env.update(kv.env()) @@ -346,6 +366,7 @@ def run_perplexity_kld_base( "--kl-divergence-base", str(base_path), ] cmd.extend(kv.cli_args()) + cmd.extend(_llama_extra_flags()) env = os.environ.copy() env.update(kv.env()) @@ -394,6 +415,7 @@ def run_perplexity_kld( "--kl-divergence-base", str(base_path), ] cmd.extend(kv.cli_args()) + cmd.extend(_llama_extra_flags()) env = os.environ.copy() env.update(kv.env()) @@ -593,6 +615,7 @@ def run_completion_trajectory( if system: cmd.extend(["-sys", system]) cmd.extend(kv.cli_args()) + cmd.extend(_llama_extra_flags()) env = os.environ.copy() env.update(kv.env())