diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37759ac..afb5df9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,8 +72,8 @@ jobs: - name: Smoke-test that the built entry points import and the CLI runs working-directory: packages/core run: | - node -e "Promise.all([import('./dist/index.js'), import('./dist/observability/index.js'), import('./dist/observability/file.js'), import('./dist/acp.js'), import('./dist/process.js'), import('./dist/mcp.js'), import('./dist/ai-sdk.js'), import('./dist/eval/index.js')]).then(() => console.log('entry points import OK')).catch((e) => { console.error(e); process.exit(1); })" - node dist/cli/oma.js help > /dev/null + node -e "Promise.all([import('./dist/index.js'), import('./dist/observability/index.js'), import('./dist/observability/file.js'), import('./dist/acp.js'), import('./dist/process.js'), import('./dist/mcp.js'), import('./dist/ai-sdk.js'), import('./dist/eval/index.js'), import('./dist/eval/file.js')]).then(() => console.log('entry points import OK')).catch((e) => { console.error(e); process.exit(1); })" + node dist/cli/oma.js help | grep -q "oma eval run" echo "CLI help runs OK" - name: Assert the create-oma-app template pins the current core version run: | @@ -99,7 +99,7 @@ jobs: exit 1 fi missing="" - for f in dist/index.js dist/index.d.ts dist/observability/index.js dist/observability/index.d.ts dist/observability/file.js dist/observability/file.d.ts dist/cli/oma.js dist/acp.js dist/acp.d.ts dist/process.js dist/process.d.ts dist/mcp.js dist/mcp.d.ts dist/ai-sdk.js dist/ai-sdk.d.ts dist/eval/index.js dist/eval/index.d.ts; do + for f in dist/index.js dist/index.d.ts dist/observability/index.js dist/observability/index.d.ts dist/observability/file.js dist/observability/file.d.ts dist/cli/oma.js dist/acp.js dist/acp.d.ts dist/process.js dist/process.d.ts dist/mcp.js dist/mcp.d.ts dist/ai-sdk.js dist/ai-sdk.d.ts dist/eval/index.js dist/eval/index.d.ts dist/eval/file.js dist/eval/file.d.ts; do echo "$paths" | grep -qxF "$f" || missing="$missing $f" done if [ -n "$missing" ]; then diff --git a/docs/cli.md b/docs/cli.md index dd06ec3..5961b60 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,6 +1,6 @@ # Command-line interface (`oma`) -The package ships a small binary **`oma`** that exposes the same primitives as the TypeScript API: `runTeam`, `runTasks`, single-run dashboard export, plus a static provider reference. It is meant for **shell scripts and CI** (JSON on stdout, stable exit codes). +The package ships a small binary **`oma`** that exposes the same primitives as the TypeScript API: `runTeam`, `runTasks`, offline EvalSet execution, single-run dashboard export, plus a static provider reference. It is meant for **shell scripts and CI** (JSON on stdout, stable exit codes). It does **not** provide an interactive REPL, working-directory injection into tools, human approval gates, or session persistence. Those stay in application code. @@ -90,6 +90,60 @@ Runs **`OpenMultiAgent.runTasks(team, tasks)`** with a fixed task list (no coord Global flags: [`--pretty`](#output-flags), [`--include-messages`](#output-flags). +### `oma eval run` + +Runs a versioned EvalSet against a user-supplied target and writes one or more +offline reports without invoking a built-in gate: + +```bash +oma eval run --set ./evals/greetings.json --target ./evals/target.mjs \ + [--scorers ./evals/scorers.mjs] \ + [--repeats 3] [--concurrency 2] [--tags smoke,regression] \ + [--report json] [--report markdown] [--report junit] \ + [--out ./eval-results] [--meta prompt_version=v2] [--pretty] +``` + +`--set` and `--target` are required. The EvalSet file is parsed as JSON and +validated by the same `defineEvalSet()` contract used by the TypeScript API. +`--repeats`, `--concurrency`, and `--tags` override the matching runner +options. Repeat `--meta key=value` to attach string metadata to every record. + +The target path is dynamically imported as an ES module. Its default export is +either an `EvalTarget` function or an object containing `{ target, scorers? }`: + +```js +const target = async (input) => ({ output: String(input).toUpperCase() }) + +const exact = { + name: 'exact', + score({ output, evalCase }) { + const pass = output === evalCase.expected + return { score: pass ? 1 : 0, pass } + }, +} + +export default { target, scorers: [exact] } +``` + +When `--scorers` is set, that ES module must default-export a `Scorer[]`. +Explicit scorers are appended to any embedded scorers. Every scorer name must +be unique, and an evaluation with no scorers is a usage error. Dynamic import +executes the supplied modules with the current process permissions; only load +code you trust. The CLI does not sandbox target or scorer modules. + +Repeat `--report` to request any combination of `json`, `markdown`, and +`junit`; JSON is the default. The output root defaults to `./eval-results`. +Every invocation writes into `//`, using `report.json`, +`report.md`, and `report.junit.xml` respectively. JSON is the authoritative +`EvalRunReport` representation. Markdown contains human-readable aggregates +and failure details. JUnit maps `pass: false` to `` and target/scorer +errors to ``. + +A completed evaluation exits 0 even when it contains low or failing scores. +It exits 1 only when every selected case/repeat target invocation failed. File, +module, argument, and contract errors exit 2. Score gates and baseline +comparison are not part of `oma eval run`. + ### `oma provider` Read-only helper for wiring JSON configs and env vars. @@ -256,6 +310,25 @@ Every invocation prints **one JSON document** to stdout, followed by a newline. } ``` +**Successful offline evaluation** + +```json +{ + "command": "eval", + "subcommand": "run", + "evalRunId": "eval_run_...", + "caseCount": 2, + "repeats": 1, + "targetErrors": 0, + "scorers": [ + { "name": "exact", "avg": 1, "passRate": 1, "errorCount": 0 } + ], + "reports": { + "json": "/workspace/eval-results/eval_run_.../report.json" + } +} +``` + **Errors (usage, validation, I/O, runtime)** ```json @@ -290,9 +363,9 @@ separate progress stream; for live telemetry use the TypeScript API with | Code | Meaning | |------|---------| -| **0** | Success: `run`/`task` finished with `success === true`, dashboard export completed, or help / `provider` completed normally. | -| **1** | Run finished but **`success === false`** (agent or task failure as reported by the library). | -| **2** | Usage, validation, readable JSON errors, or file access issues (e.g. missing file). | +| **0** | Success: `run`/`task` succeeded; dashboard export completed; eval completed without every target failing; or help / `provider` completed normally. Low eval scores alone still exit 0. | +| **1** | `run`/`task` reported failure, or every selected eval target invocation failed. | +| **2** | Usage, validation, readable JSON errors, module-load errors, or file access issues (e.g. missing file). | | **3** | Unexpected error, including typical LLM/API failures surfaced as thrown errors. | In scripts: @@ -314,6 +387,7 @@ esac - Long options only: `--goal`, `--team`, `--file`, etc. - Values may be attached with `=`: `--team=./team.json`. +- `oma eval run` accepts repeated `--report` and `--meta` options in either attached or separate-value form. - Boolean-style flags (`--pretty`, `--include-messages`) take no value; if the next token does not start with `--`, it is treated as the value of the previous option (standard `getopt`-style pairing). --- diff --git a/docs/evaluation.md b/docs/evaluation.md index ea12fc8..4384484 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -115,6 +115,67 @@ Agent and team targets convert non-string input with `String(input)` and use it Record metadata merges in this order, with later values winning: case metadata, `runEvalSet()` metadata, then metadata echoed by a convenience target (including its configuration fingerprint). +## Load EvalSets and write reports + +File I/O is isolated in the Node-only `@open-multi-agent/core/eval/file` +subpath. The root package and `@open-multi-agent/core/eval` do not import this +entry point. + +```ts +import { + loadEvalSet, + writeEvalReport, +} from '@open-multi-agent/core/eval/file' + +const setFromJson = await loadEvalSet('./evals/greetings.json') +const fileReport = await runEvalSet(setFromJson, target, { scorers: [exact] }) + +await writeEvalReport(fileReport, { format: 'json', path: './report.json' }) +await writeEvalReport(fileReport, { format: 'markdown', path: './report.md' }) +await writeEvalReport(fileReport, { format: 'junit', path: './report.junit.xml' }) +``` + +`loadEvalSet()` parses JSON, applies the same validation and deep freezing as +`defineEvalSet()`, and includes the resolved file path plus the first schema +issue in validation errors. `writeEvalReport()` creates parent directories as +needed and supports: + +- `json`: the authoritative, pretty-printed `EvalRunReport` representation. +- `markdown`: metadata, scorer and tag aggregates, failed samples, and totals + for human review. Long failure reasons are truncated. +- `junit`: one testcase per record. `pass: false` becomes ``; + `scorer_error` and `target_error` become ``; records without `pass` + and without an error are successful testcases. XML names and messages are + fully escaped. + +## Run evaluations from the CLI + +After building or installing the package, a no-network target can be evaluated +from a shell or CI job: + +```bash +oma eval run --set ./evals/greetings.json --target ./evals/target.mjs \ + --report json --report junit --out ./eval-results \ + --meta prompt_version=v2 +``` + +The target module must default-export an `EvalTarget` function or +`{ target, scorers? }`. An optional `--scorers` module default-exports a +`Scorer[]`; scorer names must be unique across both sources. The CLI dynamically +imports and executes these user modules with the current process permissions, +so they must be trusted. + +Reports are written below `//`; `--out` defaults to +`./eval-results`. `--report` is repeatable and defaults to JSON. `--meta +key=value` is also repeatable and all values are strings. See +[the CLI reference](cli.md#oma-eval-run) for the complete argument and exit-code +contract. + +Low scores and `pass: false` records do not change the exit code in this +command. A normal completed evaluation exits 0, every selected target failing +exits 1, and usage/file/module errors exit 2. Baseline comparison and quality +gates are intentionally separate from this command. + ## Payload privacy EvalSet cases may contain private user data. `storePayloads` therefore defaults to `'none'`, so records contain scores, reasons, metadata, and run references but no input/output snapshots. `'redacted'` serializes each payload field, caps it at 8 KiB, and applies OMA's existing secret redaction. `'full'` keeps the serialized text without redaction but still applies the 8 KiB cap; opt into it only for data you are prepared to retain. A model-based judge necessarily sends the evaluated output to the configured judge model regardless of record payload storage. diff --git a/packages/core/package.json b/packages/core/package.json index 9a0cc51..8f00c86 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -53,6 +53,10 @@ "types": "./dist/eval/index.d.ts", "import": "./dist/eval/index.js" }, + "./eval/file": { + "types": "./dist/eval/file.d.ts", + "import": "./dist/eval/file.js" + }, "./package.json": "./package.json" }, "scripts": { diff --git a/packages/core/src/cli/oma.ts b/packages/core/src/cli/oma.ts index 29ebd83..ba6fc6a 100644 --- a/packages/core/src/cli/oma.ts +++ b/packages/core/src/cli/oma.ts @@ -14,10 +14,14 @@ import { mkdir, stat, writeFile } from 'node:fs/promises' import { readFileSync } from 'node:fs' import { dirname, join, resolve } from 'node:path' -import { fileURLToPath } from 'node:url' +import { fileURLToPath, pathToFileURL } from 'node:url' import { OpenMultiAgent } from '../orchestrator/orchestrator.js' import { renderRunViewer } from '../dashboard/render-run-viewer.js' +import { loadEvalSet, writeEvalReport, type EvalReportFormat } from '../eval/file.js' +import { runEvalSet } from '../eval/runner.js' +import type { Scorer } from '../eval/scorer.js' +import type { EvalTarget } from '../eval/target.js' import { materializeRun } from '../observability/materialize.js' import type { TraceRecord } from '../observability/records.js' import type { StoredRun } from '../observability/store.js' @@ -226,11 +230,217 @@ export function parseArgs(argv: string[]): { return { _, flags, kv } } -function getOpt(kv: Map, flags: Set, key: string): string | undefined { +function getOpt( + kv: ReadonlyMap, + flags: ReadonlySet, + key: string, +): string | undefined { if (flags.has(key)) return '' return kv.get(key) } +function getRepeatedOpts(args: readonly string[], key: string): readonly string[] { + const option = `--${key}` + const values: string[] = [] + for (let index = 0; index < args.length; index += 1) { + const token = args[index]! + if (token === '--') break + if (token === option) { + const value = args[index + 1] + if (value === undefined || value.startsWith('--')) { + throw new OmaValidationError(`${option} requires a value`) + } + values.push(value) + index += 1 + } else if (token.startsWith(`${option}=`)) { + values.push(token.slice(option.length + 1)) + } + } + return values +} + +function positiveIntegerOption(value: string | undefined, option: string): number | undefined { + if (value === undefined) return undefined + if (!/^\d+$/.test(value) || Number(value) < 1) { + throw new OmaValidationError(`${option} must be a positive integer`) + } + return Number(value) +} + +function evalMetadata(values: readonly string[]): Readonly> { + const metadata: Record = {} + for (const entry of values) { + const separator = entry.indexOf('=') + if (separator < 1) throw new OmaValidationError('--meta must use key=value') + metadata[entry.slice(0, separator)] = entry.slice(separator + 1) + } + return Object.freeze(metadata) +} + +function evalReportFormats(values: readonly string[]): readonly EvalReportFormat[] { + const requested = values.length === 0 ? ['json'] : values + const formats: EvalReportFormat[] = [] + for (const value of requested) { + if (value !== 'json' && value !== 'markdown' && value !== 'junit') { + throw new OmaValidationError('--report must be one of json, markdown, or junit') + } + if (!formats.includes(value)) formats.push(value) + } + return formats +} + +function asScorers(value: unknown, label: string): readonly Scorer[] { + if (!Array.isArray(value)) throw new OmaValidationError(`${label} must default-export a Scorer[]`) + for (const scorer of value) { + if (!isObject(scorer) || typeof scorer['name'] !== 'string' || scorer['name'].trim().length === 0 + || typeof scorer['score'] !== 'function') { + throw new OmaValidationError(`${label} contains an invalid Scorer`) + } + } + return value as unknown as readonly Scorer[] +} + +async function importDefault(modulePath: string, label: string): Promise { + const absolutePath = resolve(modulePath) + try { + const imported = await import(pathToFileURL(absolutePath).href) as { readonly default?: unknown } + if (!('default' in imported)) { + throw new Error('module has no default export') + } + return imported.default + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + throw new OmaValidationError(`Unable to load ${label} module ${absolutePath}: ${message}`) + } +} + +async function loadEvalTarget(modulePath: string): Promise<{ + readonly target: EvalTarget + readonly scorers: readonly Scorer[] +}> { + const exported = await importDefault(modulePath, 'target') + if (typeof exported === 'function') { + return { target: exported as EvalTarget, scorers: [] } + } + if (!isObject(exported) || typeof exported['target'] !== 'function') { + throw new OmaValidationError( + 'Target module must default-export an EvalTarget function or { target, scorers? }', + ) + } + const scorers = exported['scorers'] === undefined + ? [] + : asScorers(exported['scorers'], 'target module scorers') + return { target: exported['target'] as EvalTarget, scorers } +} + +function mergeEvalScorers( + embedded: readonly Scorer[], + explicit: readonly Scorer[], +): readonly Scorer[] { + const merged: Scorer[] = [] + const names = new Set() + for (const scorer of [...embedded, ...explicit]) { + if (names.has(scorer.name)) { + throw new OmaValidationError(`Duplicate scorer name: ${scorer.name}`) + } + names.add(scorer.name) + merged.push(scorer) + } + if (merged.length === 0) throw new OmaValidationError('At least one scorer is required') + return merged +} + +const EVAL_REPORT_FILENAMES: Readonly> = { + json: 'report.json', + markdown: 'report.md', + junit: 'report.junit.xml', +} + +export async function runEvalCli(argv: { + readonly _: readonly string[] + readonly flags: ReadonlySet + readonly kv: ReadonlyMap +}): Promise<{ readonly exit: number; readonly summary: Readonly> }> { + const setPath = getOpt(argv.kv, argv.flags, 'set') + const targetPath = getOpt(argv.kv, argv.flags, 'target') + const scorersPath = getOpt(argv.kv, argv.flags, 'scorers') + const tags = getOpt(argv.kv, argv.flags, 'tags') + const out = getOpt(argv.kv, argv.flags, 'out') + if (!setPath || !targetPath) { + throw new OmaValidationError('oma eval run requires --set and --target') + } + if (scorersPath === '') throw new OmaValidationError('--scorers requires a non-empty path') + if (tags === '') throw new OmaValidationError('--tags requires a comma-separated value') + if (out === '') throw new OmaValidationError('--out requires a non-empty path') + + let set + try { + set = await loadEvalSet(setPath) + } catch (error) { + const code = (error as NodeJS.ErrnoException).code + if (code === 'ENOENT' || code === 'EACCES') throw error + throw new OmaValidationError(error instanceof Error ? error.message : String(error)) + } + const targetModule = await loadEvalTarget(targetPath) + const explicitScorers = scorersPath === undefined + ? [] + : asScorers(await importDefault(scorersPath, 'scorers'), 'Scorers module') + const scorers = mergeEvalScorers(targetModule.scorers, explicitScorers) + const repeats = positiveIntegerOption( + getOpt(argv.kv, argv.flags, 'repeats'), + '--repeats', + ) + const concurrency = positiveIntegerOption( + getOpt(argv.kv, argv.flags, 'concurrency'), + '--concurrency', + ) + const filterTags = tags === undefined + ? undefined + : tags.split(',').map((tag) => tag.trim()).filter((tag) => tag.length > 0) + if (tags !== undefined && filterTags?.length === 0) { + throw new OmaValidationError('--tags requires at least one non-empty tag') + } + const metadata = evalMetadata(getRepeatedOpts(argv._, 'meta')) + const formats = evalReportFormats(getRepeatedOpts(argv._, 'report')) + const report = await runEvalSet(set, targetModule.target, { + scorers, + ...(repeats !== undefined ? { repeats } : {}), + ...(concurrency !== undefined ? { concurrency } : {}), + ...(filterTags !== undefined ? { filterTags } : {}), + metadata, + }) + + const outputDirectory = resolve(out ?? 'eval-results', report.evalRunId) + const reports = Object.fromEntries(formats.map((format) => [ + format, + join(outputDirectory, EVAL_REPORT_FILENAMES[format]), + ])) as Partial> + await Promise.all(formats.map((format) => + writeEvalReport(report, { format, path: reports[format]! }))) + const sampleCount = report.caseCount * report.repeats + return { + exit: sampleCount > 0 && report.totals.targetErrors === sampleCount + ? EXIT.RUN_FAILED + : EXIT.SUCCESS, + summary: { + command: 'eval', + subcommand: 'run', + evalRunId: report.evalRunId, + caseCount: report.caseCount, + repeats: report.repeats, + targetErrors: report.totals.targetErrors, + scorers: report.aggregates.map((aggregate) => ({ + name: aggregate.scorer.name, + ...(aggregate.scorer.version !== undefined ? { version: aggregate.scorer.version } : {}), + avg: aggregate.avg, + ...(aggregate.passRate !== undefined ? { passRate: aggregate.passRate } : {}), + errorCount: aggregate.errorCount, + })), + reports, + }, + } +} + function readJson(path: string): unknown { const abs = resolve(path) const raw = readFileSync(abs, 'utf8') @@ -380,6 +590,9 @@ function help(): string { ' oma run --goal --team [--orchestrator ] [--coordinator ]', ' oma task --file [--team ]', ' oma dashboard --trace-store --run-id [--output ]', + ' oma eval run --set --target [--scorers ]', + ' [--repeats ] [--concurrency ] [--tags ]', + ' [--report ]... [--out ] [--meta key=value]...', ' oma provider [list | template ]', '', 'Flags:', @@ -394,7 +607,7 @@ function help(): string { 'tasks.json: { "team": TeamConfig, "tasks": [ ... ], "orchestrator"?: { ... } }.', ' Optional --team overrides the embedded team object.', '', - 'Exit codes: 0 success, 1 run failed, 2 usage/validation, 3 internal', + 'Exit codes: 0 success, 1 run/all eval targets failed, 2 usage/validation, 3 internal', ].join('\n') } @@ -598,6 +811,23 @@ async function main(): Promise { const jsonOpts: CliJsonOptions = { pretty, includeMessages } try { + if (cmd === 'eval') { + if (argv._[1] !== 'run') { + printJson({ + error: { + kind: 'usage', + message: argv._[1] === undefined + ? 'usage: oma eval run --set --target ' + : `unknown eval subcommand: ${argv._[1]}`, + }, + }, pretty) + return EXIT.USAGE + } + const evaluated = await runEvalCli(argv) + printJson(evaluated.summary, pretty) + return evaluated.exit + } + if (cmd === 'dashboard') { const traceStore = getOpt(argv.kv, argv.flags, 'trace-store') const runId = getOpt(argv.kv, argv.flags, 'run-id') diff --git a/packages/core/src/eval/file.ts b/packages/core/src/eval/file.ts new file mode 100644 index 0000000..d3fe911 --- /dev/null +++ b/packages/core/src/eval/file.ts @@ -0,0 +1,211 @@ +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { dirname, resolve } from 'node:path' + +import { ZodError } from 'zod' + +import { defineEvalSet, type EvalSet } from './evalset.js' +import type { EvalRunReport, ScorerAggregate } from './report.js' +import type { EvalRecord } from './record.js' + +export type EvalReportFormat = 'json' | 'markdown' | 'junit' + +const REASON_MAX_CHARS = 200 + +function truncateReason(value: string): string { + if (value.length <= REASON_MAX_CHARS) return value + return `${value.slice(0, REASON_MAX_CHARS - 1)}…` +} + +function firstZodIssue(error: ZodError): string { + const issue = error.issues[0] + if (issue === undefined) return error.message + const path = issue.path.length === 0 ? '' : `${issue.path.join('.')}: ` + return `${path}${issue.message}` +} + +/** Load, validate, defensively copy, and freeze an EvalSet JSON file. */ +export async function loadEvalSet(filePath: string): Promise { + const absolutePath = resolve(filePath) + const raw = await readFile(absolutePath, 'utf8') + let parsed: unknown + try { + parsed = JSON.parse(raw) as unknown + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + throw new Error(`Invalid EvalSet JSON in ${absolutePath}: ${message}`) + } + + try { + return defineEvalSet(parsed as EvalSet) + } catch (error) { + const message = error instanceof ZodError + ? firstZodIssue(error) + : error instanceof Error ? error.message : String(error) + throw new Error(`Invalid EvalSet in ${absolutePath}: ${message}`) + } +} + +function markdownCell(value: unknown): string { + if (value === undefined) return '—' + const text = typeof value === 'string' ? value : JSON.stringify(value) + return text.replaceAll('\\', '\\\\').replaceAll('|', '\\|').replaceAll('\n', '
') +} + +function metric(value: number | undefined): string { + if (value === undefined) return '—' + return Number(value.toFixed(6)).toString() +} + +function passRate(value: number | undefined): string { + return value === undefined ? '—' : `${metric(value * 100)}%` +} + +function aggregateRow(aggregate: ScorerAggregate, tag?: string): string { + return [ + tag, + aggregate.scorer.name, + aggregate.scorer.version, + aggregate.scoredCount, + metric(aggregate.avg), + metric(aggregate.p50), + metric(aggregate.p95), + metric(aggregate.min), + metric(aggregate.max), + passRate(aggregate.passRate), + aggregate.errorCount, + ].map(markdownCell).join(' | ') +} + +function recordFailureReason(record: EvalRecord): string { + if (record.status === 'scored') return truncateReason(record.reason ?? 'Score did not pass.') + return truncateReason(record.error?.message ?? record.reason ?? record.status) +} + +function markdownReport(report: EvalRunReport): string { + const lines = [ + `# Evaluation Report: ${report.evalSet.name}@${report.evalSet.version}`, + '', + `- Eval run: \`${report.evalRunId}\``, + `- Started: ${new Date(report.startedAtUnixMs).toISOString()}`, + `- Duration: ${report.durationMs} ms`, + `- Cases: ${report.caseCount}`, + `- Repeats: ${report.repeats}`, + `- Aborted: ${report.aborted === true ? 'yes' : 'no'}`, + '', + '## Metadata', + '', + '```json', + JSON.stringify(report.metadata, null, 2), + '```', + '', + '## Scorer aggregates', + '', + '| Scorer | Version | Scored | Avg | P50 | P95 | Min | Max | Pass rate | Errors |', + '| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', + ...report.aggregates.map((aggregate) => `| ${aggregateRow(aggregate).split(' | ').slice(1).join(' | ')} |`), + ] + + const byTag = report.aggregates.flatMap((aggregate) => + Object.entries(aggregate.byTag ?? {}).map(([tag, tagged]) => ({ tag, aggregate: tagged }))) + if (byTag.length > 0) { + lines.push( + '', + '## Aggregates by tag', + '', + '| Tag | Scorer | Version | Scored | Avg | P50 | P95 | Min | Max | Pass rate | Errors |', + '| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', + ...byTag.map(({ tag, aggregate }) => `| ${aggregateRow(aggregate, tag)} |`), + ) + } + + const failures = report.records.filter((record) => + record.pass === false || record.status === 'scorer_error' || record.status === 'target_error') + lines.push('', '## Failed samples', '') + if (failures.length === 0) { + lines.push('None.') + } else { + lines.push( + '| Case | Repeat | Scorer | Status | Reason |', + '| --- | ---: | --- | --- | --- |', + ...failures.map((record) => `| ${[ + record.caseId, + record.repeat, + record.scorer.name, + record.status, + recordFailureReason(record), + ].map(markdownCell).join(' | ')} |`), + ) + } + + lines.push( + '', + '## Totals', + '', + `- Target errors: ${report.totals.targetErrors}`, + `- Tokens: ${report.totals.tokens === undefined + ? '—' + : `${report.totals.tokens.input_tokens} input, ${report.totals.tokens.output_tokens} output`}`, + `- Costs: ${report.totals.costs === undefined || report.totals.costs.length === 0 + ? '—' + : report.totals.costs.map((cost) => `${cost.amount} ${cost.currency}`).join(', ')}`, + '', + ) + return lines.join('\n') +} + +function xmlEscape(value: unknown): string { + return String(value) + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", ''') +} + +function seconds(milliseconds: number | undefined): string { + return ((milliseconds ?? 0) / 1_000).toFixed(3) +} + +function junitTestcase(report: EvalRunReport, record: EvalRecord): string { + const name = `${record.caseId ?? '_unknown'}#r${record.repeat ?? 0} · ${record.scorer.name}` + const start = ` \n ${xmlEscape(reason)}\n ` + } + if (record.pass === false) { + const reason = recordFailureReason(record) + return `${start}>\n ${xmlEscape(reason)}\n ` + } + return `${start} />` +} + +function junitReport(report: EvalRunReport): string { + const failures = report.records.filter((record) => + record.status !== 'scorer_error' && record.status !== 'target_error' && record.pass === false).length + const errors = report.records.filter((record) => + record.status === 'scorer_error' || record.status === 'target_error').length + return [ + '', + ``, + ...report.records.map((record) => junitTestcase(report, record)), + '', + '', + ].join('\n') +} + +function serializeReport(report: EvalRunReport, format: EvalReportFormat): string { + if (format === 'json') return JSON.stringify(report, null, 2) + if (format === 'markdown') return markdownReport(report) + return junitReport(report) +} + +/** Write an EvalRunReport in its authoritative JSON, Markdown, or JUnit form. */ +export async function writeEvalReport( + report: EvalRunReport, + options: { readonly format: EvalReportFormat; readonly path: string }, +): Promise { + const outputPath = resolve(options.path) + await mkdir(dirname(outputPath), { recursive: true }) + await writeFile(outputPath, serializeReport(report, options.format), 'utf8') +} diff --git a/packages/core/tests/eval-cli.test.ts b/packages/core/tests/eval-cli.test.ts new file mode 100644 index 0000000..c4d4f16 --- /dev/null +++ b/packages/core/tests/eval-cli.test.ts @@ -0,0 +1,181 @@ +import { readFile, realpath } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { mkdtempSync } from 'node:fs' +import { spawnSync } from 'node:child_process' + +import { describe, expect, it } from 'vitest' + +const cliSource = fileURLToPath(new URL('../src/cli/oma.ts', import.meta.url)) +const tsxBinary = fileURLToPath(new URL('../../../node_modules/.bin/tsx', import.meta.url)) +const fixtures = fileURLToPath(new URL('./fixtures/eval/', import.meta.url)) +const setPath = join(fixtures, 'set.json') + +interface CliRun { + readonly status: number | null + readonly stdout: string + readonly stderr: string +} + +function temporaryDirectory(): string { + return mkdtempSync(join(tmpdir(), 'oma-eval-cli-')) +} + +function runCli(args: readonly string[], cwd = temporaryDirectory()): CliRun { + const result = spawnSync(tsxBinary, [cliSource, ...args], { + cwd, + encoding: 'utf8', + env: { ...process.env, NO_COLOR: '1' }, + }) + return { + status: result.status, + stdout: result.stdout, + stderr: result.stderr, + } +} + +function output(run: CliRun): Record { + return JSON.parse(run.stdout) as Record +} + +describe('oma eval run', () => { + it('loads an object target, merges metadata, and writes every requested report', async () => { + const out = temporaryDirectory() + const run = runCli([ + 'eval', 'run', + '--set', setPath, + '--target', join(fixtures, 'target.mjs'), + '--repeats', '2', + '--concurrency', '1', + '--tags', 'upper', + '--report', 'json', + '--report=markdown', + '--report', 'junit', + '--out', out, + '--meta', 'prompt_version=v2', + '--meta=release=canary', + ]) + + expect(run.status, run.stderr).toBe(0) + const summary = output(run) + expect(summary).toMatchObject({ + command: 'eval', + subcommand: 'run', + caseCount: 2, + repeats: 2, + targetErrors: 0, + scorers: [{ name: 'exact', version: '1', avg: 1, passRate: 1, errorCount: 0 }], + }) + const reports = summary['reports'] as Record + expect(reports['json']).toBe(join(out, String(summary['evalRunId']), 'report.json')) + expect(reports['markdown']).toBe(join(out, String(summary['evalRunId']), 'report.md')) + expect(reports['junit']).toBe(join(out, String(summary['evalRunId']), 'report.junit.xml')) + + const report = JSON.parse(await readFile(reports['json']!, 'utf8')) as Record + expect(report['schemaVersion']).toBe(1) + expect(report['metadata']).toEqual({ prompt_version: 'v2', release: 'canary' }) + expect((report['records'] as unknown[])).toHaveLength(4) + expect(await readFile(reports['markdown']!, 'utf8')).toContain('# Evaluation Report: uppercase@1.0.0') + expect(await readFile(reports['junit']!, 'utf8')).toContain('tests="4" failures="0" errors="0"') + }) + + it('accepts a function target plus a separate scorer module and uses the default output root', async () => { + const cwd = temporaryDirectory() + const run = runCli([ + 'eval', 'run', + '--set', setPath, + '--target', join(fixtures, 'target-function.mjs'), + '--scorers', join(fixtures, 'scorers.mjs'), + ], cwd) + + expect(run.status, run.stderr).toBe(0) + const summary = output(run) + const reports = summary['reports'] as Record + expect(Object.keys(reports)).toEqual(['json']) + expect(reports['json']).toBe(join( + await realpath(cwd), + 'eval-results', + String(summary['evalRunId']), + 'report.json', + )) + expect(JSON.parse(await readFile(reports['json']!, 'utf8'))).toMatchObject({ schemaVersion: 1 }) + }) + + it('returns usage for duplicate scorer names and for no scorers', () => { + const duplicate = runCli([ + 'eval', 'run', + '--set', setPath, + '--target', join(fixtures, 'target.mjs'), + '--scorers', join(fixtures, 'scorers.mjs'), + ]) + expect(duplicate.status).toBe(2) + expect(output(duplicate)).toMatchObject({ + error: { kind: 'validation', message: 'Duplicate scorer name: exact' }, + }) + + const missing = runCli([ + 'eval', 'run', + '--set', setPath, + '--target', join(fixtures, 'target-function.mjs'), + ]) + expect(missing.status).toBe(2) + expect(output(missing)).toMatchObject({ + error: { kind: 'validation', message: 'At least one scorer is required' }, + }) + }) + + it('maps missing set and target-module load failures to usage exit code 2', () => { + const missingSet = runCli([ + 'eval', 'run', + '--set', join(fixtures, 'missing.json'), + '--target', join(fixtures, 'target.mjs'), + ]) + expect(missingSet.status).toBe(2) + expect(output(missingSet)).toMatchObject({ error: { kind: 'io' } }) + + const missingTarget = runCli([ + 'eval', 'run', + '--set', setPath, + '--target', join(fixtures, 'missing-target.mjs'), + ]) + expect(missingTarget.status).toBe(2) + expect(output(missingTarget)).toMatchObject({ error: { kind: 'validation' } }) + }) + + it('returns 0 for partial target failure and 1 only when every target fails', () => { + const partial = runCli([ + 'eval', 'run', + '--set', setPath, + '--target', join(fixtures, 'target-partial-failure.mjs'), + '--out', temporaryDirectory(), + ]) + expect(partial.status, partial.stderr).toBe(0) + expect(output(partial)).toMatchObject({ targetErrors: 1 }) + + const all = runCli([ + 'eval', 'run', + '--set', setPath, + '--target', join(fixtures, 'target-all-failure.mjs'), + '--out', temporaryDirectory(), + ]) + expect(all.status, all.stderr).toBe(1) + expect(output(all)).toMatchObject({ targetErrors: 2 }) + }) +}) + +describe('oma eval help and routing', () => { + it('documents eval run in help', () => { + const run = runCli(['help']) + expect(run.status).toBe(0) + expect(run.stdout).toContain('oma eval run --set --target ') + }) + + it('returns usage for unknown eval subcommands', () => { + const run = runCli(['eval', 'xyz']) + expect(run.status).toBe(2) + expect(output(run)).toEqual({ + error: { kind: 'usage', message: 'unknown eval subcommand: xyz' }, + }) + }) +}) diff --git a/packages/core/tests/eval-file.test.ts b/packages/core/tests/eval-file.test.ts new file mode 100644 index 0000000..4e3109b --- /dev/null +++ b/packages/core/tests/eval-file.test.ts @@ -0,0 +1,233 @@ +import { mkdtemp, readFile, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +import { describe, expect, it } from 'vitest' + +import { loadEvalSet, writeEvalReport } from '../src/eval/file.js' +import type { EvalRunReport } from '../src/eval/report.js' + +async function temporaryDirectory(): Promise { + return mkdtemp(join(tmpdir(), 'oma-eval-file-')) +} + +function reportFixture(): EvalRunReport { + return { + schemaVersion: 1, + evalRunId: 'eval-run-1', + startedAtUnixMs: Date.UTC(2026, 6, 18), + durationMs: 2_500, + evalSet: { name: 'set<&"\'', version: '1.0.0' }, + metadata: { prompt_version: 'v2' }, + caseCount: 4, + repeats: 1, + records: [ + { + schemaVersion: 1, + recordId: 'record-1', + evalRunId: 'eval-run-1', + source: 'offline', + timestampUnixMs: 1, + evalSet: { name: 'set<&"\'', version: '1.0.0' }, + caseId: 'passing<&"\'', + repeat: 1, + scorer: { name: 'exact<&"\'' }, + status: 'scored', + score: 1, + metadata: {}, + usage: { durationMs: 100 }, + }, + { + schemaVersion: 1, + recordId: 'record-2', + evalRunId: 'eval-run-1', + source: 'offline', + timestampUnixMs: 2, + evalSet: { name: 'set<&"\'', version: '1.0.0' }, + caseId: 'failed', + repeat: 1, + scorer: { name: 'exact<&"\'' }, + status: 'scored', + score: 0, + pass: false, + reason: 'mismatch <&"\'', + metadata: {}, + usage: { durationMs: 200 }, + }, + { + schemaVersion: 1, + recordId: 'record-3', + evalRunId: 'eval-run-1', + source: 'offline', + timestampUnixMs: 3, + evalSet: { name: 'set<&"\'', version: '1.0.0' }, + caseId: 'scorer-error', + repeat: 1, + scorer: { name: 'exact<&"\'' }, + status: 'scorer_error', + metadata: {}, + error: { + kind: 'unknown', + name: 'Error', + message: 'judge <&"\' failed', + }, + }, + { + schemaVersion: 1, + recordId: 'record-4', + evalRunId: 'eval-run-1', + source: 'offline', + timestampUnixMs: 4, + evalSet: { name: 'set<&"\'', version: '1.0.0' }, + caseId: 'target-error', + repeat: 1, + scorer: { name: '_target' }, + status: 'target_error', + metadata: {}, + error: { + kind: 'unknown', + name: 'Error', + message: 'target failed', + }, + }, + ], + aggregates: [ + { + scorer: { name: 'exact<&"\'', version: '1' }, + scoredCount: 2, + errorCount: 1, + avg: 0.5, + p50: 0, + p95: 1, + min: 0, + max: 1, + passRate: 0.5, + byTag: { + edge: { + scorer: { name: 'exact<&"\'', version: '1' }, + scoredCount: 1, + errorCount: 1, + avg: 0, + p50: 0, + p95: 0, + min: 0, + max: 0, + passRate: 0, + }, + }, + }, + ], + totals: { + tokens: { input_tokens: 10, output_tokens: 5 }, + costs: [{ amount: 0.01, currency: 'USD' }], + targetErrors: 1, + }, + } +} + +describe('loadEvalSet', () => { + it('loads a valid file through defineEvalSet and deeply freezes the result', async () => { + const directory = await temporaryDirectory() + const path = join(directory, 'set.json') + await writeFile(path, JSON.stringify({ + name: 'fixture', + version: '1', + cases: [{ id: 'a', input: 'hello', tags: ['smoke'] }], + }), 'utf8') + + const set = await loadEvalSet(path) + + expect(set.name).toBe('fixture') + expect(Object.isFrozen(set)).toBe(true) + expect(Object.isFrozen(set.cases)).toBe(true) + expect(Object.isFrozen(set.cases[0])).toBe(true) + }) + + it('reports the path for invalid JSON', async () => { + const directory = await temporaryDirectory() + const path = join(directory, 'broken.json') + await writeFile(path, '{', 'utf8') + + await expect(loadEvalSet(path)).rejects.toThrow(`Invalid EvalSet JSON in ${resolve(path)}`) + }) + + it('reports the path and first validation issue', async () => { + const directory = await temporaryDirectory() + const path = join(directory, 'invalid.json') + await writeFile(path, JSON.stringify({ name: 'fixture', version: '', cases: [] }), 'utf8') + + await expect(loadEvalSet(path)).rejects.toThrow(`Invalid EvalSet in ${resolve(path)}: version:`) + }) + + it('reports duplicate case ids with the file path', async () => { + const directory = await temporaryDirectory() + const path = join(directory, 'duplicate.json') + await writeFile(path, JSON.stringify({ + name: 'fixture', + version: '1', + cases: [{ id: 'a', input: 1 }, { id: 'a', input: 2 }], + }), 'utf8') + + await expect(loadEvalSet(path)).rejects.toThrow( + `Invalid EvalSet in ${resolve(path)}: EvalSet case id "a" must be unique.`, + ) + }) +}) + +describe('writeEvalReport', () => { + it('writes authoritative JSON that round-trips without changes', async () => { + const report = reportFixture() + const path = join(await temporaryDirectory(), 'nested', 'report.json') + + await writeEvalReport(report, { format: 'json', path }) + + const raw = await readFile(path, 'utf8') + expect(raw).toBe(JSON.stringify(report, null, 2)) + expect(JSON.parse(raw)).toEqual(report) + }) + + it('writes Markdown aggregates, tag breakdowns, failure details, and totals', async () => { + const path = join(await temporaryDirectory(), 'report.md') + + await writeEvalReport(reportFixture(), { format: 'markdown', path }) + + const markdown = await readFile(path, 'utf8') + expect(markdown).toContain('# Evaluation Report: set<&"\'@1.0.0') + expect(markdown).toContain('## Scorer aggregates') + expect(markdown).toContain('## Aggregates by tag') + expect(markdown).toContain('## Failed samples') + expect(markdown).toContain('mismatch <&"\'') + expect(markdown).toContain('- Target errors: 1') + expect(markdown).toContain('- Tokens: 10 input, 5 output') + }) + + it('truncates long failure reasons in human-readable output', async () => { + const fixture = reportFixture() + const longReason = 'x'.repeat(300) + const report: EvalRunReport = { + ...fixture, + records: fixture.records.map((record) => + record.recordId === 'record-2' ? { ...record, reason: longReason } : record), + } + const path = join(await temporaryDirectory(), 'report.md') + + await writeEvalReport(report, { format: 'markdown', path }) + + const markdown = await readFile(path, 'utf8') + expect(markdown).toContain(`${'x'.repeat(199)}…`) + expect(markdown).not.toContain(longReason) + }) + + it('writes conservative JUnit XML with escaped names, failures, errors, and counts', async () => { + const path = join(await temporaryDirectory(), 'report.junit.xml') + + await writeEvalReport(reportFixture(), { format: 'junit', path }) + + const xml = await readFile(path, 'utf8') + expect(xml).toContain('name="set<&"'@1.0.0" tests="4" failures="1" errors="2" time="2.500"') + expect(xml).toContain('name="passing<&"'#r1 · exact<&"'"') + expect(xml).toContain('') + expect(xml).toContain('') + expect(xml).toContain(' ({ score: 1, pass: true }) }], +} diff --git a/packages/core/tests/fixtures/eval/target-function.mjs b/packages/core/tests/fixtures/eval/target-function.mjs new file mode 100644 index 0000000..eb0c4ce --- /dev/null +++ b/packages/core/tests/fixtures/eval/target-function.mjs @@ -0,0 +1,3 @@ +export default async function target(input) { + return { output: String(input).toUpperCase() } +} diff --git a/packages/core/tests/fixtures/eval/target-partial-failure.mjs b/packages/core/tests/fixtures/eval/target-partial-failure.mjs new file mode 100644 index 0000000..7cc4345 --- /dev/null +++ b/packages/core/tests/fixtures/eval/target-partial-failure.mjs @@ -0,0 +1,13 @@ +export default { + async target(input) { + if (input === 'world') throw new Error('target failed') + return { output: String(input).toUpperCase() } + }, + scorers: [{ + name: 'exact', + score({ output, evalCase }) { + const pass = output === evalCase.expected + return { score: pass ? 1 : 0, pass } + }, + }], +} diff --git a/packages/core/tests/fixtures/eval/target.mjs b/packages/core/tests/fixtures/eval/target.mjs new file mode 100644 index 0000000..a67d09a --- /dev/null +++ b/packages/core/tests/fixtures/eval/target.mjs @@ -0,0 +1,12 @@ +const target = async (input) => ({ output: String(input).toUpperCase() }) + +const exact = { + name: 'exact', + version: '1', + score({ output, evalCase }) { + const pass = output === evalCase.expected + return { score: pass ? 1 : 0, pass, reason: pass ? 'matched' : 'mismatch' } + }, +} + +export default { target, scorers: [exact] } diff --git a/packages/core/tests/subpath-exports.test.ts b/packages/core/tests/subpath-exports.test.ts index 866ee10..bc58165 100644 --- a/packages/core/tests/subpath-exports.test.ts +++ b/packages/core/tests/subpath-exports.test.ts @@ -29,4 +29,10 @@ describe('subpath export barrels', () => { expect(typeof mod.runEvalSet).toBe('function') expect(typeof mod.targetFromAgent).toBe('function') }) + + it('/eval/file exposes Node-only EvalSet and report file helpers', async () => { + const mod = await import('../src/eval/file.js') + expect(typeof mod.loadEvalSet).toBe('function') + expect(typeof mod.writeEvalReport).toBe('function') + }) })