feat(core): add eval CLI and report outputs (#405)

2026-10-22 11:02:59 -05:00 · 2026-07-19 00:02:29 +08:00
1 parent cedbd796e6
commit 4fad8c19d3
@@ -72,8 +72,8 @@ jobs:
      - name: Smoke-test that the built entry points import and the CLI runs
        working-directory: packages/core
        run: |
-          node -e "Promise.all([import('./dist/index.js'), import('./dist/observability/index.js'), import('./dist/observability/file.js'), import('./dist/acp.js'), import('./dist/process.js'), import('./dist/mcp.js'), import('./dist/ai-sdk.js'), import('./dist/eval/index.js')]).then(() => console.log('entry points import OK')).catch((e) => { console.error(e); process.exit(1); })"
-          node dist/cli/oma.js help > /dev/null
+          node -e "Promise.all([import('./dist/index.js'), import('./dist/observability/index.js'), import('./dist/observability/file.js'), import('./dist/acp.js'), import('./dist/process.js'), import('./dist/mcp.js'), import('./dist/ai-sdk.js'), import('./dist/eval/index.js'), import('./dist/eval/file.js')]).then(() => console.log('entry points import OK')).catch((e) => { console.error(e); process.exit(1); })"
+          node dist/cli/oma.js help | grep -q "oma eval run"
          echo "CLI help runs OK"
      - name: Assert the create-oma-app template pins the current core version
        run: |
@@ -99,7 +99,7 @@ jobs:
            exit 1
          fi
          missing=""
-          for f in dist/index.js dist/index.d.ts dist/observability/index.js dist/observability/index.d.ts dist/observability/file.js dist/observability/file.d.ts dist/cli/oma.js dist/acp.js dist/acp.d.ts dist/process.js dist/process.d.ts dist/mcp.js dist/mcp.d.ts dist/ai-sdk.js dist/ai-sdk.d.ts dist/eval/index.js dist/eval/index.d.ts; do
+          for f in dist/index.js dist/index.d.ts dist/observability/index.js dist/observability/index.d.ts dist/observability/file.js dist/observability/file.d.ts dist/cli/oma.js dist/acp.js dist/acp.d.ts dist/process.js dist/process.d.ts dist/mcp.js dist/mcp.d.ts dist/ai-sdk.js dist/ai-sdk.d.ts dist/eval/index.js dist/eval/index.d.ts dist/eval/file.js dist/eval/file.d.ts; do
            echo "$paths" | grep -qxF "$f" || missing="$missing $f"
          done
          if [ -n "$missing" ]; then
@@ -1,6 +1,6 @@
 # Command-line interface (`oma`)

-The package ships a small binary **`oma`** that exposes the same primitives as the TypeScript API: `runTeam`, `runTasks`, single-run dashboard export, plus a static provider reference. It is meant for **shell scripts and CI** (JSON on stdout, stable exit codes).
+The package ships a small binary **`oma`** that exposes the same primitives as the TypeScript API: `runTeam`, `runTasks`, offline EvalSet execution, single-run dashboard export, plus a static provider reference. It is meant for **shell scripts and CI** (JSON on stdout, stable exit codes).

 It does **not** provide an interactive REPL, working-directory injection into tools, human approval gates, or session persistence. Those stay in application code.

@@ -90,6 +90,60 @@ Runs **`OpenMultiAgent.runTasks(team, tasks)`** with a fixed task list (no coord

 Global flags: [`--pretty`](#output-flags), [`--include-messages`](#output-flags).

+### `oma eval run`
+
+Runs a versioned EvalSet against a user-supplied target and writes one or more
+offline reports without invoking a built-in gate:
+
+```bash
+oma eval run --set ./evals/greetings.json --target ./evals/target.mjs \
+  [--scorers ./evals/scorers.mjs] \
+  [--repeats 3] [--concurrency 2] [--tags smoke,regression] \
+  [--report json] [--report markdown] [--report junit] \
+  [--out ./eval-results] [--meta prompt_version=v2] [--pretty]
+```
+
+`--set` and `--target` are required. The EvalSet file is parsed as JSON and
+validated by the same `defineEvalSet()` contract used by the TypeScript API.
+`--repeats`, `--concurrency`, and `--tags` override the matching runner
+options. Repeat `--meta key=value` to attach string metadata to every record.
+
+The target path is dynamically imported as an ES module. Its default export is
+either an `EvalTarget` function or an object containing `{ target, scorers? }`:
+
+```js
+const target = async (input) => ({ output: String(input).toUpperCase() })
+
+const exact = {
+  name: 'exact',
+  score({ output, evalCase }) {
+    const pass = output === evalCase.expected
+    return { score: pass ? 1 : 0, pass }
+  },
+}
+
+export default { target, scorers: [exact] }
+```
+
+When `--scorers` is set, that ES module must default-export a `Scorer[]`.
+Explicit scorers are appended to any embedded scorers. Every scorer name must
+be unique, and an evaluation with no scorers is a usage error. Dynamic import
+executes the supplied modules with the current process permissions; only load
+code you trust. The CLI does not sandbox target or scorer modules.
+
+Repeat `--report` to request any combination of `json`, `markdown`, and
+`junit`; JSON is the default. The output root defaults to `./eval-results`.
+Every invocation writes into `<out>/<evalRunId>/`, using `report.json`,
+`report.md`, and `report.junit.xml` respectively. JSON is the authoritative
+`EvalRunReport` representation. Markdown contains human-readable aggregates
+and failure details. JUnit maps `pass: false` to `<failure>` and target/scorer
+errors to `<error>`.
+
+A completed evaluation exits 0 even when it contains low or failing scores.
+It exits 1 only when every selected case/repeat target invocation failed. File,
+module, argument, and contract errors exit 2. Score gates and baseline
+comparison are not part of `oma eval run`.
+
 ### `oma provider`

 Read-only helper for wiring JSON configs and env vars.
@@ -256,6 +310,25 @@ Every invocation prints **one JSON document** to stdout, followed by a newline.
 }
 ```

+**Successful offline evaluation**
+
+```json
+{
+  "command": "eval",
+  "subcommand": "run",
+  "evalRunId": "eval_run_...",
+  "caseCount": 2,
+  "repeats": 1,
+  "targetErrors": 0,
+  "scorers": [
+    { "name": "exact", "avg": 1, "passRate": 1, "errorCount": 0 }
+  ],
+  "reports": {
+    "json": "/workspace/eval-results/eval_run_.../report.json"
+  }
+}
+```
+
 **Errors (usage, validation, I/O, runtime)**

 ```json
@@ -290,9 +363,9 @@ separate progress stream; for live telemetry use the TypeScript API with

 | Code | Meaning |
 |------|---------|
-| **0** | Success: `run`/`task` finished with `success === true`, dashboard export completed, or help / `provider` completed normally. |
-| **1** | Run finished but **`success === false`** (agent or task failure as reported by the library). |
-| **2** | Usage, validation, readable JSON errors, or file access issues (e.g. missing file). |
+| **0** | Success: `run`/`task` succeeded; dashboard export completed; eval completed without every target failing; or help / `provider` completed normally. Low eval scores alone still exit 0. |
+| **1** | `run`/`task` reported failure, or every selected eval target invocation failed. |
+| **2** | Usage, validation, readable JSON errors, module-load errors, or file access issues (e.g. missing file). |
 | **3** | Unexpected error, including typical LLM/API failures surfaced as thrown errors. |

 In scripts:
@@ -314,6 +387,7 @@ esac

 - Long options only: `--goal`, `--team`, `--file`, etc.
 - Values may be attached with `=`: `--team=./team.json`.
+- `oma eval run` accepts repeated `--report` and `--meta` options in either attached or separate-value form.
 - Boolean-style flags (`--pretty`, `--include-messages`) take no value; if the next token does not start with `--`, it is treated as the value of the previous option (standard `getopt`-style pairing).

 ---
@@ -115,6 +115,67 @@ Agent and team targets convert non-string input with `String(input)` and use it

 Record metadata merges in this order, with later values winning: case metadata, `runEvalSet()` metadata, then metadata echoed by a convenience target (including its configuration fingerprint).

+## Load EvalSets and write reports
+
+File I/O is isolated in the Node-only `@open-multi-agent/core/eval/file`
+subpath. The root package and `@open-multi-agent/core/eval` do not import this
+entry point.
+
+```ts
+import {
+  loadEvalSet,
+  writeEvalReport,
+} from '@open-multi-agent/core/eval/file'
+
+const setFromJson = await loadEvalSet('./evals/greetings.json')
+const fileReport = await runEvalSet(setFromJson, target, { scorers: [exact] })
+
+await writeEvalReport(fileReport, { format: 'json', path: './report.json' })
+await writeEvalReport(fileReport, { format: 'markdown', path: './report.md' })
+await writeEvalReport(fileReport, { format: 'junit', path: './report.junit.xml' })
+```
+
+`loadEvalSet()` parses JSON, applies the same validation and deep freezing as
+`defineEvalSet()`, and includes the resolved file path plus the first schema
+issue in validation errors. `writeEvalReport()` creates parent directories as
+needed and supports:
+
+- `json`: the authoritative, pretty-printed `EvalRunReport` representation.
+- `markdown`: metadata, scorer and tag aggregates, failed samples, and totals
+  for human review. Long failure reasons are truncated.
+- `junit`: one testcase per record. `pass: false` becomes `<failure>`;
+  `scorer_error` and `target_error` become `<error>`; records without `pass`
+  and without an error are successful testcases. XML names and messages are
+  fully escaped.
+
+## Run evaluations from the CLI
+
+After building or installing the package, a no-network target can be evaluated
+from a shell or CI job:
+
+```bash
+oma eval run --set ./evals/greetings.json --target ./evals/target.mjs \
+  --report json --report junit --out ./eval-results \
+  --meta prompt_version=v2
+```
+
+The target module must default-export an `EvalTarget` function or
+`{ target, scorers? }`. An optional `--scorers` module default-exports a
+`Scorer[]`; scorer names must be unique across both sources. The CLI dynamically
+imports and executes these user modules with the current process permissions,
+so they must be trusted.
+
+Reports are written below `<out>/<evalRunId>/`; `--out` defaults to
+`./eval-results`. `--report` is repeatable and defaults to JSON. `--meta
+key=value` is also repeatable and all values are strings. See
+[the CLI reference](cli.md#oma-eval-run) for the complete argument and exit-code
+contract.
+
+Low scores and `pass: false` records do not change the exit code in this
+command. A normal completed evaluation exits 0, every selected target failing
+exits 1, and usage/file/module errors exit 2. Baseline comparison and quality
+gates are intentionally separate from this command.
+
 ## Payload privacy

 EvalSet cases may contain private user data. `storePayloads` therefore defaults to `'none'`, so records contain scores, reasons, metadata, and run references but no input/output snapshots. `'redacted'` serializes each payload field, caps it at 8 KiB, and applies OMA's existing secret redaction. `'full'` keeps the serialized text without redaction but still applies the 8 KiB cap; opt into it only for data you are prepared to retain. A model-based judge necessarily sends the evaluated output to the configured judge model regardless of record payload storage.
@@ -53,6 +53,10 @@
      "types": "./dist/eval/index.d.ts",
      "import": "./dist/eval/index.js"
    },
+    "./eval/file": {
+      "types": "./dist/eval/file.d.ts",
+      "import": "./dist/eval/file.js"
+    },
    "./package.json": "./package.json"
  },
  "scripts": {
@@ -14,10 +14,14 @@
 import { mkdir, stat, writeFile } from 'node:fs/promises'
 import { readFileSync } from 'node:fs'
 import { dirname, join, resolve } from 'node:path'
-import { fileURLToPath } from 'node:url'
+import { fileURLToPath, pathToFileURL } from 'node:url'

 import { OpenMultiAgent } from '../orchestrator/orchestrator.js'
 import { renderRunViewer } from '../dashboard/render-run-viewer.js'
+import { loadEvalSet, writeEvalReport, type EvalReportFormat } from '../eval/file.js'
+import { runEvalSet } from '../eval/runner.js'
+import type { Scorer } from '../eval/scorer.js'
+import type { EvalTarget } from '../eval/target.js'
 import { materializeRun } from '../observability/materialize.js'
 import type { TraceRecord } from '../observability/records.js'
 import type { StoredRun } from '../observability/store.js'
@@ -226,11 +230,217 @@ export function parseArgs(argv: string[]): {
  return { _, flags, kv }
 }

-function getOpt(kv: Map<string, string>, flags: Set<string>, key: string): string | undefined {
+function getOpt(
+  kv: ReadonlyMap<string, string>,
+  flags: ReadonlySet<string>,
+  key: string,
+): string | undefined {
  if (flags.has(key)) return ''
  return kv.get(key)
 }

+function getRepeatedOpts(args: readonly string[], key: string): readonly string[] {
+  const option = `--${key}`
+  const values: string[] = []
+  for (let index = 0; index < args.length; index += 1) {
+    const token = args[index]!
+    if (token === '--') break
+    if (token === option) {
+      const value = args[index + 1]
+      if (value === undefined || value.startsWith('--')) {
+        throw new OmaValidationError(`${option} requires a value`)
+      }
+      values.push(value)
+      index += 1
+    } else if (token.startsWith(`${option}=`)) {
+      values.push(token.slice(option.length + 1))
+    }
+  }
+  return values
+}
+
+function positiveIntegerOption(value: string | undefined, option: string): number | undefined {
+  if (value === undefined) return undefined
+  if (!/^\d+$/.test(value) || Number(value) < 1) {
+    throw new OmaValidationError(`${option} must be a positive integer`)
+  }
+  return Number(value)
+}
+
+function evalMetadata(values: readonly string[]): Readonly<Record<string, string>> {
+  const metadata: Record<string, string> = {}
+  for (const entry of values) {
+    const separator = entry.indexOf('=')
+    if (separator < 1) throw new OmaValidationError('--meta must use key=value')
+    metadata[entry.slice(0, separator)] = entry.slice(separator + 1)
+  }
+  return Object.freeze(metadata)
+}
+
+function evalReportFormats(values: readonly string[]): readonly EvalReportFormat[] {
+  const requested = values.length === 0 ? ['json'] : values
+  const formats: EvalReportFormat[] = []
+  for (const value of requested) {
+    if (value !== 'json' && value !== 'markdown' && value !== 'junit') {
+      throw new OmaValidationError('--report must be one of json, markdown, or junit')
+    }
+    if (!formats.includes(value)) formats.push(value)
+  }
+  return formats
+}
+
+function asScorers(value: unknown, label: string): readonly Scorer[] {
+  if (!Array.isArray(value)) throw new OmaValidationError(`${label} must default-export a Scorer[]`)
+  for (const scorer of value) {
+    if (!isObject(scorer) || typeof scorer['name'] !== 'string' || scorer['name'].trim().length === 0
+      || typeof scorer['score'] !== 'function') {
+      throw new OmaValidationError(`${label} contains an invalid Scorer`)
+    }
+  }
+  return value as unknown as readonly Scorer[]
+}
+
+async function importDefault(modulePath: string, label: string): Promise<unknown> {
+  const absolutePath = resolve(modulePath)
+  try {
+    const imported = await import(pathToFileURL(absolutePath).href) as { readonly default?: unknown }
+    if (!('default' in imported)) {
+      throw new Error('module has no default export')
+    }
+    return imported.default
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error)
+    throw new OmaValidationError(`Unable to load ${label} module ${absolutePath}: ${message}`)
+  }
+}
+
+async function loadEvalTarget(modulePath: string): Promise<{
+  readonly target: EvalTarget
+  readonly scorers: readonly Scorer[]
+}> {
+  const exported = await importDefault(modulePath, 'target')
+  if (typeof exported === 'function') {
+    return { target: exported as EvalTarget, scorers: [] }
+  }
+  if (!isObject(exported) || typeof exported['target'] !== 'function') {
+    throw new OmaValidationError(
+      'Target module must default-export an EvalTarget function or { target, scorers? }',
+    )
+  }
+  const scorers = exported['scorers'] === undefined
+    ? []
+    : asScorers(exported['scorers'], 'target module scorers')
+  return { target: exported['target'] as EvalTarget, scorers }
+}
+
+function mergeEvalScorers(
+  embedded: readonly Scorer[],
+  explicit: readonly Scorer[],
+): readonly Scorer[] {
+  const merged: Scorer[] = []
+  const names = new Set<string>()
+  for (const scorer of [...embedded, ...explicit]) {
+    if (names.has(scorer.name)) {
+      throw new OmaValidationError(`Duplicate scorer name: ${scorer.name}`)
+    }
+    names.add(scorer.name)
+    merged.push(scorer)
+  }
+  if (merged.length === 0) throw new OmaValidationError('At least one scorer is required')
+  return merged
+}
+
+const EVAL_REPORT_FILENAMES: Readonly<Record<EvalReportFormat, string>> = {
+  json: 'report.json',
+  markdown: 'report.md',
+  junit: 'report.junit.xml',
+}
+
+export async function runEvalCli(argv: {
+  readonly _: readonly string[]
+  readonly flags: ReadonlySet<string>
+  readonly kv: ReadonlyMap<string, string>
+}): Promise<{ readonly exit: number; readonly summary: Readonly<Record<string, unknown>> }> {
+  const setPath = getOpt(argv.kv, argv.flags, 'set')
+  const targetPath = getOpt(argv.kv, argv.flags, 'target')
+  const scorersPath = getOpt(argv.kv, argv.flags, 'scorers')
+  const tags = getOpt(argv.kv, argv.flags, 'tags')
+  const out = getOpt(argv.kv, argv.flags, 'out')
+  if (!setPath || !targetPath) {
+    throw new OmaValidationError('oma eval run requires --set and --target')
+  }
+  if (scorersPath === '') throw new OmaValidationError('--scorers requires a non-empty path')
+  if (tags === '') throw new OmaValidationError('--tags requires a comma-separated value')
+  if (out === '') throw new OmaValidationError('--out requires a non-empty path')
+
+  let set
+  try {
+    set = await loadEvalSet(setPath)
+  } catch (error) {
+    const code = (error as NodeJS.ErrnoException).code
+    if (code === 'ENOENT' || code === 'EACCES') throw error
+    throw new OmaValidationError(error instanceof Error ? error.message : String(error))
+  }
+  const targetModule = await loadEvalTarget(targetPath)
+  const explicitScorers = scorersPath === undefined
+    ? []
+    : asScorers(await importDefault(scorersPath, 'scorers'), 'Scorers module')
+  const scorers = mergeEvalScorers(targetModule.scorers, explicitScorers)
+  const repeats = positiveIntegerOption(
+    getOpt(argv.kv, argv.flags, 'repeats'),
+    '--repeats',
+  )
+  const concurrency = positiveIntegerOption(
+    getOpt(argv.kv, argv.flags, 'concurrency'),
+    '--concurrency',
+  )
+  const filterTags = tags === undefined
+    ? undefined
+    : tags.split(',').map((tag) => tag.trim()).filter((tag) => tag.length > 0)
+  if (tags !== undefined && filterTags?.length === 0) {
+    throw new OmaValidationError('--tags requires at least one non-empty tag')
+  }
+  const metadata = evalMetadata(getRepeatedOpts(argv._, 'meta'))
+  const formats = evalReportFormats(getRepeatedOpts(argv._, 'report'))
+  const report = await runEvalSet(set, targetModule.target, {
+    scorers,
+    ...(repeats !== undefined ? { repeats } : {}),
+    ...(concurrency !== undefined ? { concurrency } : {}),
+    ...(filterTags !== undefined ? { filterTags } : {}),
+    metadata,
+  })
+
+  const outputDirectory = resolve(out ?? 'eval-results', report.evalRunId)
+  const reports = Object.fromEntries(formats.map((format) => [
+    format,
+    join(outputDirectory, EVAL_REPORT_FILENAMES[format]),
+  ])) as Partial<Record<EvalReportFormat, string>>
+  await Promise.all(formats.map((format) =>
+    writeEvalReport(report, { format, path: reports[format]! })))
+  const sampleCount = report.caseCount * report.repeats
+  return {
+    exit: sampleCount > 0 && report.totals.targetErrors === sampleCount
+      ? EXIT.RUN_FAILED
+      : EXIT.SUCCESS,
+    summary: {
+      command: 'eval',
+      subcommand: 'run',
+      evalRunId: report.evalRunId,
+      caseCount: report.caseCount,
+      repeats: report.repeats,
+      targetErrors: report.totals.targetErrors,
+      scorers: report.aggregates.map((aggregate) => ({
+        name: aggregate.scorer.name,
+        ...(aggregate.scorer.version !== undefined ? { version: aggregate.scorer.version } : {}),
+        avg: aggregate.avg,
+        ...(aggregate.passRate !== undefined ? { passRate: aggregate.passRate } : {}),
+        errorCount: aggregate.errorCount,
+      })),
+      reports,
+    },
+  }
+}
+
 function readJson(path: string): unknown {
  const abs = resolve(path)
  const raw = readFileSync(abs, 'utf8')
@@ -380,6 +590,9 @@ function help(): string {
    '  oma run --goal <text> --team <team.json> [--orchestrator <orch.json>] [--coordinator <coord.json>]',
    '  oma task --file <tasks.json> [--team <team.json>]',
    '  oma dashboard --trace-store <traces.ndjson> --run-id <id> [--output <run.html>]',
+    '  oma eval run --set <evalset.json> --target <target.mjs> [--scorers <scorers.mjs>]',
+    '               [--repeats <n>] [--concurrency <n>] [--tags <a,b>]',
+    '               [--report <json|markdown|junit>]... [--out <dir>] [--meta key=value]...',
    '  oma provider [list | template <provider>]',
    '',
    'Flags:',
@@ -394,7 +607,7 @@ function help(): string {
    'tasks.json: { "team": TeamConfig, "tasks": [ ... ], "orchestrator"?: { ... } }.',
    '  Optional --team overrides the embedded team object.',
    '',
-    'Exit codes: 0 success, 1 run failed, 2 usage/validation, 3 internal',
+    'Exit codes: 0 success, 1 run/all eval targets failed, 2 usage/validation, 3 internal',
  ].join('\n')
 }

@@ -598,6 +811,23 @@ async function main(): Promise<number> {
  const jsonOpts: CliJsonOptions = { pretty, includeMessages }

  try {
+    if (cmd === 'eval') {
+      if (argv._[1] !== 'run') {
+        printJson({
+          error: {
+            kind: 'usage',
+            message: argv._[1] === undefined
+              ? 'usage: oma eval run --set <evalset.json> --target <target.mjs>'
+              : `unknown eval subcommand: ${argv._[1]}`,
+          },
+        }, pretty)
+        return EXIT.USAGE
+      }
+      const evaluated = await runEvalCli(argv)
+      printJson(evaluated.summary, pretty)
+      return evaluated.exit
+    }
+
    if (cmd === 'dashboard') {
      const traceStore = getOpt(argv.kv, argv.flags, 'trace-store')
      const runId = getOpt(argv.kv, argv.flags, 'run-id')
@@ -0,0 +1,211 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises'
+import { dirname, resolve } from 'node:path'
+
+import { ZodError } from 'zod'
+
+import { defineEvalSet, type EvalSet } from './evalset.js'
+import type { EvalRunReport, ScorerAggregate } from './report.js'
+import type { EvalRecord } from './record.js'
+
+export type EvalReportFormat = 'json' | 'markdown' | 'junit'
+
+const REASON_MAX_CHARS = 200
+
+function truncateReason(value: string): string {
+  if (value.length <= REASON_MAX_CHARS) return value
+  return `${value.slice(0, REASON_MAX_CHARS - 1)}…`
+}
+
+function firstZodIssue(error: ZodError): string {
+  const issue = error.issues[0]
+  if (issue === undefined) return error.message
+  const path = issue.path.length === 0 ? '' : `${issue.path.join('.')}: `
+  return `${path}${issue.message}`
+}
+
+/** Load, validate, defensively copy, and freeze an EvalSet JSON file. */
+export async function loadEvalSet(filePath: string): Promise<EvalSet> {
+  const absolutePath = resolve(filePath)
+  const raw = await readFile(absolutePath, 'utf8')
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(raw) as unknown
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error)
+    throw new Error(`Invalid EvalSet JSON in ${absolutePath}: ${message}`)
+  }
+
+  try {
+    return defineEvalSet(parsed as EvalSet)
+  } catch (error) {
+    const message = error instanceof ZodError
+      ? firstZodIssue(error)
+      : error instanceof Error ? error.message : String(error)
+    throw new Error(`Invalid EvalSet in ${absolutePath}: ${message}`)
+  }
+}
+
+function markdownCell(value: unknown): string {
+  if (value === undefined) return '—'
+  const text = typeof value === 'string' ? value : JSON.stringify(value)
+  return text.replaceAll('\\', '\\\\').replaceAll('|', '\\|').replaceAll('\n', '<br>')
+}
+
+function metric(value: number | undefined): string {
+  if (value === undefined) return '—'
+  return Number(value.toFixed(6)).toString()
+}
+
+function passRate(value: number | undefined): string {
+  return value === undefined ? '—' : `${metric(value * 100)}%`
+}
+
+function aggregateRow(aggregate: ScorerAggregate, tag?: string): string {
+  return [
+    tag,
+    aggregate.scorer.name,
+    aggregate.scorer.version,
+    aggregate.scoredCount,
+    metric(aggregate.avg),
+    metric(aggregate.p50),
+    metric(aggregate.p95),
+    metric(aggregate.min),
+    metric(aggregate.max),
+    passRate(aggregate.passRate),
+    aggregate.errorCount,
+  ].map(markdownCell).join(' | ')
+}
+
+function recordFailureReason(record: EvalRecord): string {
+  if (record.status === 'scored') return truncateReason(record.reason ?? 'Score did not pass.')
+  return truncateReason(record.error?.message ?? record.reason ?? record.status)
+}
+
+function markdownReport(report: EvalRunReport): string {
+  const lines = [
+    `# Evaluation Report: ${report.evalSet.name}@${report.evalSet.version}`,
+    '',
+    `- Eval run: \`${report.evalRunId}\``,
+    `- Started: ${new Date(report.startedAtUnixMs).toISOString()}`,
+    `- Duration: ${report.durationMs} ms`,
+    `- Cases: ${report.caseCount}`,
+    `- Repeats: ${report.repeats}`,
+    `- Aborted: ${report.aborted === true ? 'yes' : 'no'}`,
+    '',
+    '## Metadata',
+    '',
+    '```json',
+    JSON.stringify(report.metadata, null, 2),
+    '```',
+    '',
+    '## Scorer aggregates',
+    '',
+    '| Scorer | Version | Scored | Avg | P50 | P95 | Min | Max | Pass rate | Errors |',
+    '| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |',
+    ...report.aggregates.map((aggregate) => `| ${aggregateRow(aggregate).split(' | ').slice(1).join(' | ')} |`),
+  ]
+
+  const byTag = report.aggregates.flatMap((aggregate) =>
+    Object.entries(aggregate.byTag ?? {}).map(([tag, tagged]) => ({ tag, aggregate: tagged })))
+  if (byTag.length > 0) {
+    lines.push(
+      '',
+      '## Aggregates by tag',
+      '',
+      '| Tag | Scorer | Version | Scored | Avg | P50 | P95 | Min | Max | Pass rate | Errors |',
+      '| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |',
+      ...byTag.map(({ tag, aggregate }) => `| ${aggregateRow(aggregate, tag)} |`),
+    )
+  }
+
+  const failures = report.records.filter((record) =>
+    record.pass === false || record.status === 'scorer_error' || record.status === 'target_error')
+  lines.push('', '## Failed samples', '')
+  if (failures.length === 0) {
+    lines.push('None.')
+  } else {
+    lines.push(
+      '| Case | Repeat | Scorer | Status | Reason |',
+      '| --- | ---: | --- | --- | --- |',
+      ...failures.map((record) => `| ${[
+        record.caseId,
+        record.repeat,
+        record.scorer.name,
+        record.status,
+        recordFailureReason(record),
+      ].map(markdownCell).join(' | ')} |`),
+    )
+  }
+
+  lines.push(
+    '',
+    '## Totals',
+    '',
+    `- Target errors: ${report.totals.targetErrors}`,
+    `- Tokens: ${report.totals.tokens === undefined
+      ? '—'
+      : `${report.totals.tokens.input_tokens} input, ${report.totals.tokens.output_tokens} output`}`,
+    `- Costs: ${report.totals.costs === undefined || report.totals.costs.length === 0
+      ? '—'
+      : report.totals.costs.map((cost) => `${cost.amount} ${cost.currency}`).join(', ')}`,
+    '',
+  )
+  return lines.join('\n')
+}
+
+function xmlEscape(value: unknown): string {
+  return String(value)
+    .replaceAll('&', '&amp;')
+    .replaceAll('<', '&lt;')
+    .replaceAll('>', '&gt;')
+    .replaceAll('"', '&quot;')
+    .replaceAll("'", '&apos;')
+}
+
+function seconds(milliseconds: number | undefined): string {
+  return ((milliseconds ?? 0) / 1_000).toFixed(3)
+}
+
+function junitTestcase(report: EvalRunReport, record: EvalRecord): string {
+  const name = `${record.caseId ?? '_unknown'}#r${record.repeat ?? 0} · ${record.scorer.name}`
+  const start = `  <testcase name="${xmlEscape(name)}" classname="${xmlEscape(report.evalSet.name)}" time="${seconds(record.usage?.durationMs)}"`
+  if (record.status === 'scorer_error' || record.status === 'target_error') {
+    const reason = recordFailureReason(record)
+    return `${start}>\n    <error message="${xmlEscape(reason)}">${xmlEscape(reason)}</error>\n  </testcase>`
+  }
+  if (record.pass === false) {
+    const reason = recordFailureReason(record)
+    return `${start}>\n    <failure message="${xmlEscape(reason)}">${xmlEscape(reason)}</failure>\n  </testcase>`
+  }
+  return `${start} />`
+}
+
+function junitReport(report: EvalRunReport): string {
+  const failures = report.records.filter((record) =>
+    record.status !== 'scorer_error' && record.status !== 'target_error' && record.pass === false).length
+  const errors = report.records.filter((record) =>
+    record.status === 'scorer_error' || record.status === 'target_error').length
+  return [
+    '<?xml version="1.0" encoding="UTF-8"?>',
+    `<testsuite name="${xmlEscape(`${report.evalSet.name}@${report.evalSet.version}`)}" tests="${report.records.length}" failures="${failures}" errors="${errors}" time="${seconds(report.durationMs)}">`,
+    ...report.records.map((record) => junitTestcase(report, record)),
+    '</testsuite>',
+    '',
+  ].join('\n')
+}
+
+function serializeReport(report: EvalRunReport, format: EvalReportFormat): string {
+  if (format === 'json') return JSON.stringify(report, null, 2)
+  if (format === 'markdown') return markdownReport(report)
+  return junitReport(report)
+}
+
+/** Write an EvalRunReport in its authoritative JSON, Markdown, or JUnit form. */
+export async function writeEvalReport(
+  report: EvalRunReport,
+  options: { readonly format: EvalReportFormat; readonly path: string },
+): Promise<void> {
+  const outputPath = resolve(options.path)
+  await mkdir(dirname(outputPath), { recursive: true })
+  await writeFile(outputPath, serializeReport(report, options.format), 'utf8')
+}
@@ -0,0 +1,181 @@
+import { readFile, realpath } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { mkdtempSync } from 'node:fs'
+import { spawnSync } from 'node:child_process'
+
+import { describe, expect, it } from 'vitest'
+
+const cliSource = fileURLToPath(new URL('../src/cli/oma.ts', import.meta.url))
+const tsxBinary = fileURLToPath(new URL('../../../node_modules/.bin/tsx', import.meta.url))
+const fixtures = fileURLToPath(new URL('./fixtures/eval/', import.meta.url))
+const setPath = join(fixtures, 'set.json')
+
+interface CliRun {
+  readonly status: number | null
+  readonly stdout: string
+  readonly stderr: string
+}
+
+function temporaryDirectory(): string {
+  return mkdtempSync(join(tmpdir(), 'oma-eval-cli-'))
+}
+
+function runCli(args: readonly string[], cwd = temporaryDirectory()): CliRun {
+  const result = spawnSync(tsxBinary, [cliSource, ...args], {
+    cwd,
+    encoding: 'utf8',
+    env: { ...process.env, NO_COLOR: '1' },
+  })
+  return {
+    status: result.status,
+    stdout: result.stdout,
+    stderr: result.stderr,
+  }
+}
+
+function output(run: CliRun): Record<string, unknown> {
+  return JSON.parse(run.stdout) as Record<string, unknown>
+}
+
+describe('oma eval run', () => {
+  it('loads an object target, merges metadata, and writes every requested report', async () => {
+    const out = temporaryDirectory()
+    const run = runCli([
+      'eval', 'run',
+      '--set', setPath,
+      '--target', join(fixtures, 'target.mjs'),
+      '--repeats', '2',
+      '--concurrency', '1',
+      '--tags', 'upper',
+      '--report', 'json',
+      '--report=markdown',
+      '--report', 'junit',
+      '--out', out,
+      '--meta', 'prompt_version=v2',
+      '--meta=release=canary',
+    ])
+
+    expect(run.status, run.stderr).toBe(0)
+    const summary = output(run)
+    expect(summary).toMatchObject({
+      command: 'eval',
+      subcommand: 'run',
+      caseCount: 2,
+      repeats: 2,
+      targetErrors: 0,
+      scorers: [{ name: 'exact', version: '1', avg: 1, passRate: 1, errorCount: 0 }],
+    })
+    const reports = summary['reports'] as Record<string, string>
+    expect(reports['json']).toBe(join(out, String(summary['evalRunId']), 'report.json'))
+    expect(reports['markdown']).toBe(join(out, String(summary['evalRunId']), 'report.md'))
+    expect(reports['junit']).toBe(join(out, String(summary['evalRunId']), 'report.junit.xml'))
+
+    const report = JSON.parse(await readFile(reports['json']!, 'utf8')) as Record<string, unknown>
+    expect(report['schemaVersion']).toBe(1)
+    expect(report['metadata']).toEqual({ prompt_version: 'v2', release: 'canary' })
+    expect((report['records'] as unknown[])).toHaveLength(4)
+    expect(await readFile(reports['markdown']!, 'utf8')).toContain('# Evaluation Report: uppercase@1.0.0')
+    expect(await readFile(reports['junit']!, 'utf8')).toContain('tests="4" failures="0" errors="0"')
+  })
+
+  it('accepts a function target plus a separate scorer module and uses the default output root', async () => {
+    const cwd = temporaryDirectory()
+    const run = runCli([
+      'eval', 'run',
+      '--set', setPath,
+      '--target', join(fixtures, 'target-function.mjs'),
+      '--scorers', join(fixtures, 'scorers.mjs'),
+    ], cwd)
+
+    expect(run.status, run.stderr).toBe(0)
+    const summary = output(run)
+    const reports = summary['reports'] as Record<string, string>
+    expect(Object.keys(reports)).toEqual(['json'])
+    expect(reports['json']).toBe(join(
+      await realpath(cwd),
+      'eval-results',
+      String(summary['evalRunId']),
+      'report.json',
+    ))
+    expect(JSON.parse(await readFile(reports['json']!, 'utf8'))).toMatchObject({ schemaVersion: 1 })
+  })
+
+  it('returns usage for duplicate scorer names and for no scorers', () => {
+    const duplicate = runCli([
+      'eval', 'run',
+      '--set', setPath,
+      '--target', join(fixtures, 'target.mjs'),
+      '--scorers', join(fixtures, 'scorers.mjs'),
+    ])
+    expect(duplicate.status).toBe(2)
+    expect(output(duplicate)).toMatchObject({
+      error: { kind: 'validation', message: 'Duplicate scorer name: exact' },
+    })
+
+    const missing = runCli([
+      'eval', 'run',
+      '--set', setPath,
+      '--target', join(fixtures, 'target-function.mjs'),
+    ])
+    expect(missing.status).toBe(2)
+    expect(output(missing)).toMatchObject({
+      error: { kind: 'validation', message: 'At least one scorer is required' },
+    })
+  })
+
+  it('maps missing set and target-module load failures to usage exit code 2', () => {
+    const missingSet = runCli([
+      'eval', 'run',
+      '--set', join(fixtures, 'missing.json'),
+      '--target', join(fixtures, 'target.mjs'),
+    ])
+    expect(missingSet.status).toBe(2)
+    expect(output(missingSet)).toMatchObject({ error: { kind: 'io' } })
+
+    const missingTarget = runCli([
+      'eval', 'run',
+      '--set', setPath,
+      '--target', join(fixtures, 'missing-target.mjs'),
+    ])
+    expect(missingTarget.status).toBe(2)
+    expect(output(missingTarget)).toMatchObject({ error: { kind: 'validation' } })
+  })
+
+  it('returns 0 for partial target failure and 1 only when every target fails', () => {
+    const partial = runCli([
+      'eval', 'run',
+      '--set', setPath,
+      '--target', join(fixtures, 'target-partial-failure.mjs'),
+      '--out', temporaryDirectory(),
+    ])
+    expect(partial.status, partial.stderr).toBe(0)
+    expect(output(partial)).toMatchObject({ targetErrors: 1 })
+
+    const all = runCli([
+      'eval', 'run',
+      '--set', setPath,
+      '--target', join(fixtures, 'target-all-failure.mjs'),
+      '--out', temporaryDirectory(),
+    ])
+    expect(all.status, all.stderr).toBe(1)
+    expect(output(all)).toMatchObject({ targetErrors: 2 })
+  })
+})
+
+describe('oma eval help and routing', () => {
+  it('documents eval run in help', () => {
+    const run = runCli(['help'])
+    expect(run.status).toBe(0)
+    expect(run.stdout).toContain('oma eval run --set <evalset.json> --target <target.mjs>')
+  })
+
+  it('returns usage for unknown eval subcommands', () => {
+    const run = runCli(['eval', 'xyz'])
+    expect(run.status).toBe(2)
+    expect(output(run)).toEqual({
+      error: { kind: 'usage', message: 'unknown eval subcommand: xyz' },
+    })
+  })
+})
@@ -0,0 +1,233 @@
+import { mkdtemp, readFile, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join, resolve } from 'node:path'
+
+import { describe, expect, it } from 'vitest'
+
+import { loadEvalSet, writeEvalReport } from '../src/eval/file.js'
+import type { EvalRunReport } from '../src/eval/report.js'
+
+async function temporaryDirectory(): Promise<string> {
+  return mkdtemp(join(tmpdir(), 'oma-eval-file-'))
+}
+
+function reportFixture(): EvalRunReport {
+  return {
+    schemaVersion: 1,
+    evalRunId: 'eval-run-1',
+    startedAtUnixMs: Date.UTC(2026, 6, 18),
+    durationMs: 2_500,
+    evalSet: { name: 'set<&"\'', version: '1.0.0' },
+    metadata: { prompt_version: 'v2' },
+    caseCount: 4,
+    repeats: 1,
+    records: [
+      {
+        schemaVersion: 1,
+        recordId: 'record-1',
+        evalRunId: 'eval-run-1',
+        source: 'offline',
+        timestampUnixMs: 1,
+        evalSet: { name: 'set<&"\'', version: '1.0.0' },
+        caseId: 'passing<&"\'',
+        repeat: 1,
+        scorer: { name: 'exact<&"\'' },
+        status: 'scored',
+        score: 1,
+        metadata: {},
+        usage: { durationMs: 100 },
+      },
+      {
+        schemaVersion: 1,
+        recordId: 'record-2',
+        evalRunId: 'eval-run-1',
+        source: 'offline',
+        timestampUnixMs: 2,
+        evalSet: { name: 'set<&"\'', version: '1.0.0' },
+        caseId: 'failed',
+        repeat: 1,
+        scorer: { name: 'exact<&"\'' },
+        status: 'scored',
+        score: 0,
+        pass: false,
+        reason: 'mismatch <&"\'',
+        metadata: {},
+        usage: { durationMs: 200 },
+      },
+      {
+        schemaVersion: 1,
+        recordId: 'record-3',
+        evalRunId: 'eval-run-1',
+        source: 'offline',
+        timestampUnixMs: 3,
+        evalSet: { name: 'set<&"\'', version: '1.0.0' },
+        caseId: 'scorer-error',
+        repeat: 1,
+        scorer: { name: 'exact<&"\'' },
+        status: 'scorer_error',
+        metadata: {},
+        error: {
+          kind: 'unknown',
+          name: 'Error',
+          message: 'judge <&"\' failed',
+        },
+      },
+      {
+        schemaVersion: 1,
+        recordId: 'record-4',
+        evalRunId: 'eval-run-1',
+        source: 'offline',
+        timestampUnixMs: 4,
+        evalSet: { name: 'set<&"\'', version: '1.0.0' },
+        caseId: 'target-error',
+        repeat: 1,
+        scorer: { name: '_target' },
+        status: 'target_error',
+        metadata: {},
+        error: {
+          kind: 'unknown',
+          name: 'Error',
+          message: 'target failed',
+        },
+      },
+    ],
+    aggregates: [
+      {
+        scorer: { name: 'exact<&"\'', version: '1' },
+        scoredCount: 2,
+        errorCount: 1,
+        avg: 0.5,
+        p50: 0,
+        p95: 1,
+        min: 0,
+        max: 1,
+        passRate: 0.5,
+        byTag: {
+          edge: {
+            scorer: { name: 'exact<&"\'', version: '1' },
+            scoredCount: 1,
+            errorCount: 1,
+            avg: 0,
+            p50: 0,
+            p95: 0,
+            min: 0,
+            max: 0,
+            passRate: 0,
+          },
+        },
+      },
+    ],
+    totals: {
+      tokens: { input_tokens: 10, output_tokens: 5 },
+      costs: [{ amount: 0.01, currency: 'USD' }],
+      targetErrors: 1,
+    },
+  }
+}
+
+describe('loadEvalSet', () => {
+  it('loads a valid file through defineEvalSet and deeply freezes the result', async () => {
+    const directory = await temporaryDirectory()
+    const path = join(directory, 'set.json')
+    await writeFile(path, JSON.stringify({
+      name: 'fixture',
+      version: '1',
+      cases: [{ id: 'a', input: 'hello', tags: ['smoke'] }],
+    }), 'utf8')
+
+    const set = await loadEvalSet(path)
+
+    expect(set.name).toBe('fixture')
+    expect(Object.isFrozen(set)).toBe(true)
+    expect(Object.isFrozen(set.cases)).toBe(true)
+    expect(Object.isFrozen(set.cases[0])).toBe(true)
+  })
+
+  it('reports the path for invalid JSON', async () => {
+    const directory = await temporaryDirectory()
+    const path = join(directory, 'broken.json')
+    await writeFile(path, '{', 'utf8')
+
+    await expect(loadEvalSet(path)).rejects.toThrow(`Invalid EvalSet JSON in ${resolve(path)}`)
+  })
+
+  it('reports the path and first validation issue', async () => {
+    const directory = await temporaryDirectory()
+    const path = join(directory, 'invalid.json')
+    await writeFile(path, JSON.stringify({ name: 'fixture', version: '', cases: [] }), 'utf8')
+
+    await expect(loadEvalSet(path)).rejects.toThrow(`Invalid EvalSet in ${resolve(path)}: version:`)
+  })
+
+  it('reports duplicate case ids with the file path', async () => {
+    const directory = await temporaryDirectory()
+    const path = join(directory, 'duplicate.json')
+    await writeFile(path, JSON.stringify({
+      name: 'fixture',
+      version: '1',
+      cases: [{ id: 'a', input: 1 }, { id: 'a', input: 2 }],
+    }), 'utf8')
+
+    await expect(loadEvalSet(path)).rejects.toThrow(
+      `Invalid EvalSet in ${resolve(path)}: EvalSet case id "a" must be unique.`,
+    )
+  })
+})
+
+describe('writeEvalReport', () => {
+  it('writes authoritative JSON that round-trips without changes', async () => {
+    const report = reportFixture()
+    const path = join(await temporaryDirectory(), 'nested', 'report.json')
+
+    await writeEvalReport(report, { format: 'json', path })
+
+    const raw = await readFile(path, 'utf8')
+    expect(raw).toBe(JSON.stringify(report, null, 2))
+    expect(JSON.parse(raw)).toEqual(report)
+  })
+
+  it('writes Markdown aggregates, tag breakdowns, failure details, and totals', async () => {
+    const path = join(await temporaryDirectory(), 'report.md')
+
+    await writeEvalReport(reportFixture(), { format: 'markdown', path })
+
+    const markdown = await readFile(path, 'utf8')
+    expect(markdown).toContain('# Evaluation Report: set<&"\'@1.0.0')
+    expect(markdown).toContain('## Scorer aggregates')
+    expect(markdown).toContain('## Aggregates by tag')
+    expect(markdown).toContain('## Failed samples')
+    expect(markdown).toContain('mismatch <&"\'')
+    expect(markdown).toContain('- Target errors: 1')
+    expect(markdown).toContain('- Tokens: 10 input, 5 output')
+  })
+
+  it('truncates long failure reasons in human-readable output', async () => {
+    const fixture = reportFixture()
+    const longReason = 'x'.repeat(300)
+    const report: EvalRunReport = {
+      ...fixture,
+      records: fixture.records.map((record) =>
+        record.recordId === 'record-2' ? { ...record, reason: longReason } : record),
+    }
+    const path = join(await temporaryDirectory(), 'report.md')
+
+    await writeEvalReport(report, { format: 'markdown', path })
+
+    const markdown = await readFile(path, 'utf8')
+    expect(markdown).toContain(`${'x'.repeat(199)}…`)
+    expect(markdown).not.toContain(longReason)
+  })
+
+  it('writes conservative JUnit XML with escaped names, failures, errors, and counts', async () => {
+    const path = join(await temporaryDirectory(), 'report.junit.xml')
+
+    await writeEvalReport(reportFixture(), { format: 'junit', path })
+
+    const xml = await readFile(path, 'utf8')
+    expect(xml).toContain('name="set&lt;&amp;&quot;&apos;@1.0.0" tests="4" failures="1" errors="2" time="2.500"')
+    expect(xml).toContain('name="passing&lt;&amp;&quot;&apos;#r1 · exact&lt;&amp;&quot;&apos;"')
+    expect(xml).toContain('<failure message="mismatch &lt;&amp;&quot;&apos;">')
+    expect(xml).toContain('<error message="judge &lt;&amp;&quot;&apos; failed">')
+    expect(xml).toContain('<testcase')
+  })
+})
@@ -0,0 +1,8 @@
+export default [{
+  name: 'exact',
+  version: '1',
+  score({ output, evalCase }) {
+    const pass = output === evalCase.expected
+    return { score: pass ? 1 : 0, pass }
+  },
+}]
@@ -0,0 +1,18 @@
+{
+  "name": "uppercase",
+  "version": "1.0.0",
+  "cases": [
+    {
+      "id": "hello",
+      "input": "hello",
+      "expected": "HELLO",
+      "tags": ["upper"]
+    },
+    {
+      "id": "world",
+      "input": "world",
+      "expected": "WORLD",
+      "tags": ["upper"]
+    }
+  ]
+}
@@ -0,0 +1,6 @@
+export default {
+  async target() {
+    throw new Error('target failed')
+  },
+  scorers: [{ name: 'exact', score: () => ({ score: 1, pass: true }) }],
+}
@@ -0,0 +1,3 @@
+export default async function target(input) {
+  return { output: String(input).toUpperCase() }
+}
@@ -0,0 +1,13 @@
+export default {
+  async target(input) {
+    if (input === 'world') throw new Error('target failed')
+    return { output: String(input).toUpperCase() }
+  },
+  scorers: [{
+    name: 'exact',
+    score({ output, evalCase }) {
+      const pass = output === evalCase.expected
+      return { score: pass ? 1 : 0, pass }
+    },
+  }],
+}
@@ -0,0 +1,12 @@
+const target = async (input) => ({ output: String(input).toUpperCase() })
+
+const exact = {
+  name: 'exact',
+  version: '1',
+  score({ output, evalCase }) {
+    const pass = output === evalCase.expected
+    return { score: pass ? 1 : 0, pass, reason: pass ? 'matched' : 'mismatch' }
+  },
+}
+
+export default { target, scorers: [exact] }
@@ -29,4 +29,10 @@ describe('subpath export barrels', () => {
    expect(typeof mod.runEvalSet).toBe('function')
    expect(typeof mod.targetFromAgent).toBe('function')
  })
+
+  it('/eval/file exposes Node-only EvalSet and report file helpers', async () => {
+    const mod = await import('../src/eval/file.js')
+    expect(typeof mod.loadEvalSet).toBe('function')
+    expect(typeof mod.writeEvalReport).toBe('function')
+  })
 })