diff --git a/refract/CHANGELOG.md b/refract/CHANGELOG.md
index 102c844..dffa9e5 100644
--- a/refract/CHANGELOG.md
+++ b/refract/CHANGELOG.md
@@ -5,10 +5,31 @@ matrix result that motivated or validated the change.
---
-## v0.3.2.1 — vLLM and SGLang backends production + extra-flags escape hatch (2026-05-02)
+## v0.3.2.1 — vLLM and SGLang backends production + extra-flags escape hatch + skip-axis fix (2026-05-02)
### What changed
+- **Engine name + version surfaced in HTML report.** The `Run details →
+ Environment` card now shows `backend` (one of llama.cpp / MLX / vLLM /
+ SGLang) plus engine-specific version fields (llama.cpp commit, mlx-lm
+ version, vLLM version, SGLang URL, served model id) when the active
+ backend populates them. Previously a report didn't say which engine
+ ran it, which made cross-engine comparison reports ambiguous in the
+ inbox.
+
+- **`--skip-gtm` / `--skip-kld` no longer inflate the composite to 100.**
+ Previously the CLI fed a stub `score=100.0` to `composite_score`,
+ which made a single real KLD axis of 99.47 produce a misleading
+ composite of 99.74 EXCELLENT. The skipped axis now contributes
+ `None` to `composite_score()` and is excluded from the harmonic
+ mean; report renderers (text + HTML) display "n/a (skipped)"
+ instead of "100 EXCELLENT" for the skipped axis. The harmonic-mean
+ axis count in the text report ("harmonic mean of N axes") now
+ reflects the real number, not 2-or-more. Reported by AJ on X
+ (2026-05-02). Regression tests added in `test_score.py`.
+
+
+
- **`REFRACT_LLAMA_EXTRA_FLAGS` env var** — appended to every
`llama-cli` / `llama-completion` / `llama-perplexity` subprocess.
Lets users on constrained VRAM run REFRACT against large MoE
diff --git a/refract/cli.py b/refract/cli.py
index a4263bc..9e8bc38 100644
--- a/refract/cli.py
+++ b/refract/cli.py
@@ -504,8 +504,13 @@ def _run_score(args) -> int:
print(f" PLAD score: {plad.score:.2f}")
# ---- Composite --------------------------------------------------------
+ # Skipped axes contribute None to the composite (dropped before the
+ # harmonic mean) rather than a stub 100, which would inflate the
+ # composite and read as EXCELLENT in the report.
+ gtm_for_composite = None if args.skip_gtm else gtm.score
+ kld_for_composite = None if args.skip_kld else kld.score
composite = composite_score(
- gtm.score, kld.score,
+ gtm_for_composite, kld_for_composite,
rniah_score=rniah.score if rniah else None,
plad_score=plad.score if plad else None,
floor_score=floor_score,
diff --git a/refract/report.py b/refract/report.py
index cd476e4..5064329 100644
--- a/refract/report.py
+++ b/refract/report.py
@@ -87,8 +87,21 @@ def _axis_label(name: str) -> str:
}.get(name, name)
-def _axis_line(name: str, score: float, bar_width: int = 40) -> str:
- """One report line per axis: label, score, bar, band, prose."""
+def _axis_line(name: str, score: Optional[float], bar_width: int = 40) -> str:
+ """One report line per axis: label, score, bar, band, prose.
+
+ When ``score is None`` the axis was skipped via ``--skip-gtm`` /
+ ``--skip-kld``; render it explicitly as ``n/a`` so a reader doesn't
+ interpret a missing axis as a perfect 100.
+ """
+ if score is None:
+ empty_bar = " " * bar_width
+ skipped = _c("33", "skipped ")
+ return (
+ f" {_axis_label(name)}: n/a "
+ f"{empty_bar} {skipped} "
+ f"{_AXIS_PROSE.get(name, '')}"
+ )
b = band(score)
band_str = _c(_band_color(b), f"{b:<9}")
return (
@@ -152,9 +165,14 @@ def text_report(
# Composite
band_str = _c(_band_color(composite.band), composite.band)
# Count axes that contributed to the harmonic mean so the gloss is honest
- # about what produced the number (2 axes for cheap mode, up to 4 for full).
- n_axes = 2 + (1 if composite.rniah_score is not None else 0) \
- + (1 if composite.plad_score is not None else 0)
+ # about what produced the number. Skipped axes (--skip-gtm / --skip-kld)
+ # contribute None to composite_score and are excluded here too.
+ n_axes = (
+ (1 if composite.gtm_score is not None else 0)
+ + (1 if composite.kld_score is not None else 0)
+ + (1 if composite.rniah_score is not None else 0)
+ + (1 if composite.plad_score is not None else 0)
+ )
lines.append(
_c("1", f" REFRACT score : {composite.composite:6.2f} "
f"{_bar(composite.composite, bar_width)} {band_str}")
@@ -195,39 +213,50 @@ def text_report(
lines.append(chunk)
lines.append("-" * 72)
- # GTM diagnostics
- lines.append(" GTM diagnostics")
- lines.append(f" prompts : {gtm.n_prompts}")
- lines.append(f" tokens decoded each : {gtm.n_tokens_each}")
- lines.append(f" full match rate : {gtm.full_match_rate*100:5.1f} %")
- if gtm.median_first_divergence is not None:
+ # GTM / Trajectory diagnostics — suppress if axis was skipped, since
+ # the diagnostic block on a stub result reads as "0 prompts, all matched"
+ # which is misleading.
+ if composite.gtm_score is not None:
+ lines.append(" GTM diagnostics")
+ lines.append(f" prompts : {gtm.n_prompts}")
+ lines.append(f" tokens decoded each : {gtm.n_tokens_each}")
+ lines.append(f" full match rate : {gtm.full_match_rate*100:5.1f} %")
+ if gtm.median_first_divergence is not None:
+ lines.append(
+ f" median first divergence : token {gtm.median_first_divergence}"
+ )
+ else:
+ lines.append(" median first divergence : (all matched)")
lines.append(
- f" median first divergence : token {gtm.median_first_divergence}"
+ f" mean prefix agreement : {gtm.mean_prefix_agreement_length:5.1f} tokens"
)
+ lines.append(
+ f" mean cand / ref length : {gtm.mean_cand_length:5.1f} / "
+ f"{gtm.mean_ref_length:5.1f} tokens"
+ )
+ if gtm.notes:
+ for n in gtm.notes:
+ lines.append(_c("33", f" NOTE: {n}"))
else:
- lines.append(" median first divergence : (all matched)")
- lines.append(
- f" mean prefix agreement : {gtm.mean_prefix_agreement_length:5.1f} tokens"
- )
- lines.append(
- f" mean cand / ref length : {gtm.mean_cand_length:5.1f} / "
- f"{gtm.mean_ref_length:5.1f} tokens"
- )
- if gtm.notes:
- for n in gtm.notes:
- lines.append(_c("33", f" NOTE: {n}"))
+ lines.append(_c("33",
+ " GTM/Trajectory: SKIPPED via --skip-gtm; not measured."))
# KLD diagnostics
- lines.append("")
- lines.append(" KLD diagnostics")
- lines.append(f" chunks x ctx : {kld.chunks} x {kld.ctx}")
- lines.append(f" mean KLD (nats) : {kld.mean_kld:.6f}")
- if kld.ppl is not None:
- lines.append(f" candidate PPL : {kld.ppl:.4f}")
- if kld.rms_dp_pct is not None:
- lines.append(f" RMS Δp (vs reference) : {kld.rms_dp_pct:.2f} %")
- if kld.same_topp_pct is not None:
- lines.append(f" same top-p (vs reference) : {kld.same_topp_pct:.2f} %")
+ if composite.kld_score is not None:
+ lines.append("")
+ lines.append(" KLD diagnostics")
+ lines.append(f" chunks x ctx : {kld.chunks} x {kld.ctx}")
+ lines.append(f" mean KLD (nats) : {kld.mean_kld:.6f}")
+ if kld.ppl is not None:
+ lines.append(f" candidate PPL : {kld.ppl:.4f}")
+ if kld.rms_dp_pct is not None:
+ lines.append(f" RMS Δp (vs reference) : {kld.rms_dp_pct:.2f} %")
+ if kld.same_topp_pct is not None:
+ lines.append(f" same top-p (vs reference) : {kld.same_topp_pct:.2f} %")
+ else:
+ lines.append("")
+ lines.append(_c("33",
+ " KLD: SKIPPED via --skip-kld; not measured."))
# R-NIAH diagnostics
if rniah is not None:
@@ -307,15 +336,19 @@ def json_report(
# `composite_detail` for diagnostics.
composite_scalar = composite_dict.pop("composite")
composite_band = composite_dict.pop("band")
+ def _band_or_skipped(s):
+ return band(s) if s is not None else "skipped"
axes_block: dict = {
"gtm": {
**gtm_dict,
- "band": band(composite.gtm_score),
+ "band": _band_or_skipped(composite.gtm_score),
+ "skipped": composite.gtm_score is None,
"description": _AXIS_PROSE["gtm"],
},
"kld": {
**asdict(kld),
- "band": band(composite.kld_score),
+ "band": _band_or_skipped(composite.kld_score),
+ "skipped": composite.kld_score is None,
"description": _AXIS_PROSE["kld"],
},
}
diff --git a/refract/report_html.py b/refract/report_html.py
index f641337..c987229 100644
--- a/refract/report_html.py
+++ b/refract/report_html.py
@@ -290,8 +290,29 @@ def _axis_letter_chip(letter: str) -> str:
return f'{_esc(letter)}'
-def _stat_block(name: str, score: float, *, is_composite: bool = False,
+def _stat_block(name: str, score: Optional[float], *, is_composite: bool = False,
low_confidence: bool = False) -> str:
+ # When score is None the axis was skipped (--skip-gtm / --skip-kld).
+ # Render explicit "n/a" instead of a fake 100/EXCELLENT.
+ if score is None and not is_composite:
+ letter = _AXIS_LETTER.get(name, "")
+ short = _AXIS_SHORT.get(name, name)
+ label_html = (
+ f'{_esc(letter)}'
+ f'{_esc(short)}
'
+ )
+ value_html = (
+ f''
+ )
+ return (
+ f''
+ f'{label_html}{value_html}'
+ f'
{_badge("", "Skipped")}
'
+ f'
{"".join(items)}
'
-def _axis_row(name: str, score: float, *, low_confidence: bool = False) -> str:
+def _axis_row(name: str, score: Optional[float], *, low_confidence: bool = False) -> str:
+ if score is None:
+ # Skipped axis: explicit n/a row instead of fake 100/EXCELLENT.
+ empty_meter = ''
+ return (
+ f''
+ f'
{_axis_letter_chip(_AXIS_LETTER.get(name, ""))}'
+ f'{_esc(_AXIS_SHORT.get(name, name))}
'
+ f'
{_esc(_AXIS_FULL.get(name, name))}'
+ f'skipped via --skip-{name}; not measured
'
+ f'
n/a
'
+ f'
{_badge("", "Skipped")}
'
+ f'{empty_meter}'
+ f'
".join(parts) if parts else spec
- env_dl = [
+ env_dl = []
+ em = env_meta or {}
+ backend_name = em.get("backend")
+ if backend_name:
+ # Map internal backend name to a friendlier display label.
+ backend_display = {
+ "llamacpp": "llama.cpp",
+ "mlx": "MLX (Apple Silicon)",
+ "vllm": "vLLM",
+ "sglang": "SGLang",
+ }.get(backend_name, backend_name)
+ env_dl.append(_kv_pair("backend", backend_display))
+ # Engine-specific version + commit fields (whichever the active backend
+ # populated). Surfaced because "which engine?" is the first question
+ # readers ask when a report lands in their inbox.
+ for key, label in (
+ ("llama_cpp_commit", "llama.cpp commit"),
+ ("llama_cpp_version", "llama.cpp version"),
+ ("mlx_lm_version", "mlx-lm version"),
+ ("mlx_version", "mlx version"),
+ ("vllm_version", "vLLM version"),
+ ("sglang_url", "SGLang URL"),
+ ("served_model_id", "served model"),
+ ):
+ if em.get(key):
+ env_dl.append(_kv_pair(label, em[key]))
+ env_dl.extend([
f"reference{_split_kv(_esc(reference_label))}",
f"candidate{_split_kv(_esc(candidate_label))}",
- ]
+ ])
return (
f''
@@ -1161,7 +1223,8 @@ def html_report(
Run details
model · hardware · environment
- {_run_details(model_meta, hw_meta, reference_label, candidate_label)}
+ {_run_details(model_meta, hw_meta, reference_label, candidate_label,
+ env_meta=(raw_json or {}).get("environment"))}
diff --git a/refract/score.py b/refract/score.py
index 9498737..bd1d2b1 100644
--- a/refract/score.py
+++ b/refract/score.py
@@ -53,8 +53,8 @@ def band(score: float) -> str:
def interpret_pattern(
*,
- gtm_score: float,
- kld_score: float,
+ gtm_score: Optional[float],
+ kld_score: Optional[float],
rniah_score: Optional[float] = None,
plad_score: Optional[float] = None,
) -> list[str]:
@@ -153,8 +153,8 @@ class CompositeScore:
composite: float # 0–100 (harmonic_mean of scored axes)
band: str # EXCELLENT / PASS / DEGRADED / FAIL
- gtm_score: float # 0–100 (axis A: gtm or trajectory)
- kld_score: float # 0–100 (axis B)
+ gtm_score: Optional[float] # 0–100 (axis A); None if --skip-gtm
+ kld_score: Optional[float] # 0–100 (axis B); None if --skip-kld
rniah_score: Optional[float] = None # 0–100 (axis C; v0.2)
plad_score: Optional[float] = None # 0–100 (axis D; v0.2)
floor_score: Optional[float] = None # measured floor (ref vs ref)
@@ -164,8 +164,8 @@ class CompositeScore:
def composite_score(
- gtm_score: float,
- kld_score: float,
+ gtm_score: Optional[float],
+ kld_score: Optional[float],
rniah_score: Optional[float] = None,
plad_score: Optional[float] = None,
floor_score: Optional[float] = None,
@@ -175,14 +175,20 @@ def composite_score(
Axes that weren't run (passed as ``None``) are dropped before
aggregation. v0.1 callers passing only gtm + kld get a 2-axis
harmonic mean (unchanged behaviour); v0.2 callers passing all four
- get a 4-axis harmonic mean.
+ get a 4-axis harmonic mean. v0.3.2.1+: gtm and kld are also
+ Optional so that ``--skip-gtm`` / ``--skip-kld`` exclude their axis
+ from the composite rather than feeding a stub 100.
"""
- axes = [gtm_score, kld_score]
+ axes: list[float] = []
+ if gtm_score is not None:
+ axes.append(gtm_score)
+ if kld_score is not None:
+ axes.append(kld_score)
if rniah_score is not None:
axes.append(rniah_score)
if plad_score is not None:
axes.append(plad_score)
- composite = harmonic_mean(axes)
+ composite = harmonic_mean(axes) if axes else 0.0
floor_ok: Optional[bool] = None
notes: list[str] = []
if floor_score is not None:
diff --git a/refract/tests/test_score.py b/refract/tests/test_score.py
index 2a49ff2..4bde3f2 100644
--- a/refract/tests/test_score.py
+++ b/refract/tests/test_score.py
@@ -176,3 +176,85 @@ def test_interpret_two_axis_call_omits_optional():
def test_min_floor_is_99_5():
assert MIN_FLOOR == 99.5
+
+
+# --- skip-axis composite: regression for false-100 reporting -------------
+
+
+def test_composite_skip_gtm_not_inflated_to_100():
+ """v0.3.2.1: --skip-gtm passes gtm_score=None to composite_score, which
+ must drop it from the harmonic mean rather than treat it as a 100 stub.
+
+ Regression: AJ reported (2026-05-02) that --skip-gtm produced a 99.74
+ EXCELLENT composite from a single real KLD score of 99.47, because
+ the previous wiring fed a stub gtm=100."""
+ cs = composite_score(gtm_score=None, kld_score=99.47)
+ assert abs(cs.composite - 99.47) < 0.01
+ assert cs.gtm_score is None
+ assert cs.kld_score == 99.47
+
+
+def test_composite_skip_kld_not_inflated_to_100():
+ cs = composite_score(gtm_score=62.40, kld_score=None)
+ assert abs(cs.composite - 62.40) < 0.01
+ assert cs.gtm_score == 62.40
+ assert cs.kld_score is None
+
+
+def test_composite_skip_both_returns_zero():
+ """If both A and B are skipped and no rniah/plad, don't crash and
+ don't claim 100. Return 0 with FAIL band."""
+ cs = composite_score(gtm_score=None, kld_score=None)
+ assert cs.composite == 0.0
+ assert cs.band == "FAIL"
+
+
+def test_composite_skip_gtm_with_rniah_plad():
+ """Skip-gtm + KLD + R-NIAH + PLAD = harmonic mean of 3 axes only."""
+ cs = composite_score(
+ gtm_score=None, kld_score=99.0, rniah_score=100.0, plad_score=90.0,
+ )
+ expected = 3 / (1/99 + 1/100 + 1/90)
+ assert abs(cs.composite - expected) < 0.01
+
+
+def test_interpret_pattern_handles_skipped_gtm():
+ """interpret_pattern shouldn't blow up on None gtm/kld."""
+ notes = interpret_pattern(
+ gtm_score=None, kld_score=99.0, rniah_score=100.0, plad_score=95.0,
+ )
+ assert isinstance(notes, list)
+
+
+def test_text_report_skip_gtm_renders_n_a_and_one_axis_count():
+ """text_report on a skipped axis emits 'n/a' / 'skipped' (not 100/EXCELLENT)
+ and counts the harmonic-mean axes correctly."""
+ from refract.axes.gtm import GTMResult
+ from refract.axes.kld import KLDResult
+ from refract.report import text_report
+ gtm_stub = GTMResult(
+ score=100.0, full_match_rate=1.0,
+ median_first_divergence=None,
+ mean_prefix_agreement_length=0.0,
+ mean_cand_length=0.0, mean_ref_length=0.0,
+ n_prompts=0, n_tokens_each=0, per_prompt=[],
+ )
+ kld_real = KLDResult(
+ score=99.47, mean_kld=0.0053, ppl=None,
+ rms_dp_pct=None, same_topp_pct=None,
+ base_path="", chunks=32, ctx=512,
+ is_self_reference=False,
+ )
+ cs = composite_score(gtm_score=None, kld_score=99.47)
+ out = text_report(
+ model="m.gguf",
+ reference_label="ctk=f16,ctv=f16",
+ candidate_label="ctk=q8_0,ctv=q8_0",
+ composite=cs, gtm=gtm_stub, kld=kld_real,
+ )
+ import re as _re
+ plain = _re.sub(r"\x1b\[[0-9;]*m", "", out)
+ assert "n/a" in plain or "skipped" in plain.lower()
+ assert "99.47" in plain
+ # gloss must reflect actual axis count after skip, not the old hardcoded 2
+ assert "harmonic mean of 1 axes" in plain