From feead27b0425dca4675be3635ed7a37ad811e546 Mon Sep 17 00:00:00 2001 From: Jack Chen Date: Sun, 19 Jul 2026 10:59:17 +0800 Subject: [PATCH] feat(core): add reference evaluation scorers (#409) --- .github/workflows/ci.yml | 3 + README.md | 1 + README_zh.md | 1 + docs/cli.md | 5 + docs/evaluation.md | 123 ++++++- package.json | 1 + packages/core/examples/README.md | 2 + .../patterns/eval-offline-regression.ts | 132 +++++++ .../examples/patterns/eval-online-sampling.ts | 59 +++ packages/core/src/eval/index.ts | 15 + packages/core/src/eval/scorer.ts | 8 + packages/core/src/eval/scorers/cost.ts | 77 ++++ packages/core/src/eval/scorers/dependency.ts | 78 ++++ .../core/src/eval/scorers/duplicate-work.ts | 121 +++++++ packages/core/src/eval/scorers/index.ts | 11 + packages/core/src/eval/scorers/no-progress.ts | 112 ++++++ packages/core/src/eval/scorers/relevancy.ts | 31 ++ .../src/eval/scorers/structured-output.ts | 35 ++ packages/core/src/eval/scorers/tool-call.ts | 59 +++ packages/core/tests/eval-scorers.test.ts | 340 ++++++++++++++++++ scripts/eval-example-smoke.mjs | 29 ++ scripts/observability-tarball-smoke.mjs | 26 +- 22 files changed, 1258 insertions(+), 11 deletions(-) create mode 100644 packages/core/examples/patterns/eval-offline-regression.ts create mode 100644 packages/core/examples/patterns/eval-online-sampling.ts create mode 100644 packages/core/src/eval/scorers/cost.ts create mode 100644 packages/core/src/eval/scorers/dependency.ts create mode 100644 packages/core/src/eval/scorers/duplicate-work.ts create mode 100644 packages/core/src/eval/scorers/index.ts create mode 100644 packages/core/src/eval/scorers/no-progress.ts create mode 100644 packages/core/src/eval/scorers/relevancy.ts create mode 100644 packages/core/src/eval/scorers/structured-output.ts create mode 100644 packages/core/src/eval/scorers/tool-call.ts create mode 100644 packages/core/tests/eval-scorers.test.ts create mode 100644 scripts/eval-example-smoke.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3630d9..bf56b36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -181,6 +181,9 @@ jobs: - name: Run public observability examples without network or API keys run: npm run test:observability-examples + - name: Run public evaluation example without network or API keys + run: npm run test:eval-examples + - name: Run tolerant same-host observability regression gate run: npm run bench:observability:ci diff --git a/README.md b/README.md index 40dd11d..b430c5f 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,7 @@ npm test # run the test suite - [Providers](https://github.com/open-multi-agent/open-multi-agent/blob/main/docs/providers.md) — env vars, model examples, local tool-calling, timeouts, troubleshooting. - [Tool configuration](https://github.com/open-multi-agent/open-multi-agent/blob/main/docs/tool-configuration.md) — tool presets, custom tools, the filesystem sandbox, and MCP. - [Observability](https://github.com/open-multi-agent/open-multi-agent/blob/main/docs/observability.md) — stable identity/status, TraceRecord v2, bounded sink/exporter lifecycle, InMemory/File TraceStore, and the offline single-run DAG/Waterfall Viewer. Existing callbacks have a staged [`onTrace` migration guide](docs/observability-migration.md); [`@open-multi-agent/otel`](packages/otel/README.md) uses an application-owned provider. +- [Evaluation](https://github.com/open-multi-agent/open-multi-agent/blob/main/docs/evaluation.md) — versioned EvalSets, reference rule/structure/judge scorers, offline reports and CI gates, EvalStore, and best-effort online sampling. - [Shared memory](https://github.com/open-multi-agent/open-multi-agent/blob/main/docs/shared-memory.md) — the default store and custom `MemoryStore` backends. - [Checkpoint & resume](https://github.com/open-multi-agent/open-multi-agent/blob/main/docs/checkpoint.md) — opt-in snapshots over any `MemoryStore`; restore preserves `runId`, increments `attempt`, and starts a fresh trace. - [Context management](https://github.com/open-multi-agent/open-multi-agent/blob/main/docs/context-management.md) — sliding window, summarization, compaction, and custom compressors. diff --git a/README_zh.md b/README_zh.md index fdda90f..c826ca1 100644 --- a/README_zh.md +++ b/README_zh.md @@ -137,6 +137,7 @@ npm test # 运行测试套件 - [Provider](https://github.com/open-multi-agent/open-multi-agent/blob/main/docs/providers.md) — 环境变量、模型示例、本地模型工具调用、超时、常见问题。 - [工具配置](https://github.com/open-multi-agent/open-multi-agent/blob/main/docs/tool-configuration.md) — 工具预设、自定义工具、文件系统沙箱、MCP。 - [可观测性](https://github.com/open-multi-agent/open-multi-agent/blob/main/docs/observability.md) — 稳定 identity/status、TraceRecord v2、有界 sink/exporter 生命周期、InMemory/File TraceStore，以及离线单次运行 DAG/Waterfall Viewer。旧 callback 可按 [`onTrace` 分阶段迁移指南](docs/observability-migration.md) 无停机迁移；[`@open-multi-agent/otel`](packages/otel/README.md) 使用应用自有 provider。 +- [评测](https://github.com/open-multi-agent/open-multi-agent/blob/main/docs/evaluation.md) — 版本化 EvalSet、规则/结构/judge 参考 scorer、离线报告与 CI gate、EvalStore，以及尽力而为的在线抽样。 - [共享记忆](https://github.com/open-multi-agent/open-multi-agent/blob/main/docs/shared-memory.md) — 默认存储与自定义 `MemoryStore` 后端。 - [Checkpoint & resume](https://github.com/open-multi-agent/open-multi-agent/blob/main/docs/checkpoint.md) — 在任意 `MemoryStore` 上保存可选快照；`restore()` 保留 `runId`、递增 `attempt`，并启动新的 trace。 - [上下文管理](https://github.com/open-multi-agent/open-multi-agent/blob/main/docs/context-management.md) — 滑动窗口、摘要、压缩、自定义压缩器。 diff --git a/docs/cli.md b/docs/cli.md index be7a6cb..aa3a4c6 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -132,6 +132,11 @@ be unique, and an evaluation with no scorers is a usage error. Dynamic import executes the supplied modules with the current process permissions; only load code you trust. The CLI does not sandbox target or scorer modules. +Reference factories such as `toolCallSuccessScorer()`, +`costBudgetScorer()`, and `createAnswerRelevancyScorer()` can be imported from +`@open-multi-agent/core/eval` inside either module. Version custom and judge +scorers so baseline drift warnings remain actionable. + Repeat `--report` to request any combination of `json`, `markdown`, and `junit`; JSON is the default. The output root defaults to `./eval-results`. Every invocation writes into `//`, using `report.json`, diff --git a/docs/evaluation.md b/docs/evaluation.md index 550a793..4f0a54a 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -1,8 +1,23 @@ # Evaluation -The experimental `@open-multi-agent/core/eval` subpath defines quality scorers for offline regression checks and future online sampling. Evaluation runs outside the business execution path: it observes a result but never changes that result. +The `@open-multi-agent/core/eval` subpath measures agent and multi-agent quality +offline or through best-effort online sampling. Evaluation observes completed +results; it never changes the business result. -## Define a scorer +Use runtime verification and evaluation for different jobs: + +| | `runConsensus()` / per-task `verify` | Evaluation | +|---|---|---| +| When | During one business run | Offline in batches or asynchronously after a live run | +| Changes the business result | Yes: may accept, revise, or reject | No | +| Measures | One result | Cases, versions, regressions, and trends | +| Failure meaning | Affects the runtime verdict | Produces `scorer_error`; never becomes score zero | +| Output | `ConsensusResult` | `EvalRecord`, `EvalRunReport`, and `GateVerdict` | + +The two mechanisms compose: verification protects a single run, while an +EvalSet can detect changes in verification pass rate over time. + +## Scorers ```ts import { defineScorer, type ScorerContext } from '@open-multi-agent/core/eval' @@ -27,7 +42,7 @@ const result = await exact.score(context) console.log(result.score) // 1 ``` -Scores must be finite numbers from `0` through `1`. `pass` is optional so a later gate can apply its own threshold. `defineScorer()` freezes the scorer definition and validates both synchronous and asynchronous results. +Scores must be finite numbers from `0` through `1`. `pass` is optional so a later gate can apply its own threshold. `defineScorer()` freezes the scorer definition and validates both synchronous and asynchronous results. A scorer may omit `version`, but OMA warns once per scorer name because a gate then cannot distinguish scoring-logic drift from target drift. Bump the version whenever a rule, prompt, judge model, or judge configuration changes. ## Scorer failures are not zero scores @@ -35,7 +50,44 @@ A scorer that throws, rejects, or exceeds its timeout has not measured quality. If the target itself throws, the sample produces one `target_error` record under the reserved scorer name `_target`; its scorers do not run. The eval subpath also defines the `EvalRecord` shape and schema major version. -## Run an EvalSet offline +### Reference scorers + +Reference scorers are deliberately small examples, not universal quality +standards. Import them from `@open-multi-agent/core/eval`: + +| Factory | Score meaning | Required data and missing-data behavior | +|---|---|---| +| `toolCallSuccessScorer()` | Successful tool spans / all tool spans | Uses trace status when available. A result-only `ToolCallRecord` has no error flag, so completed result calls are treated as successful. No calls returns `1` with `details.tool_calls = 0` and an explicit not-applicable reason. | +| `structuredOutputComplianceScorer(schema?)` | `1` when `AgentRunResult.structured` exists and, when supplied, passes the Zod schema | Intended only for targets whose agent config has `outputSchema`. Missing structured output is a measured failure (`0`), not missing infrastructure. | +| `costBudgetScorer({ maxTokens?, maxCostAmount? })` | Hard step: `1` within every observable ceiling, otherwise `0` | Tokens come from the OMA result; cost comes from `StoredRun.costs`. Unavailable dimensions are named in `reason` and `details.data_complete`; no observable dimension returns `1` with `applicable: false`. Multiple currencies throw and become `scorer_error` instead of being added incorrectly. | +| `dependencyUtilizationScorer()` | Completed dependency-bearing task spans / all dependency-bearing task spans | Requires a trace. This is a conservative dependency-chain completion proxy; it proves linked prerequisites and the dependent task completed, not that the model semantically used prerequisite text. | +| `duplicateWorkScorer({ threshold? })` | `1 - duplicatePairs / comparedPairs` | Requires both a trace and `TeamRunResult`. Trace identifies task IDs; actual outputs come from `agentResults`. Similarity is Jaccard over normalized character trigrams. Fewer than two outputs returns explicit not-applicable `1`. | +| `noProgressScorer({ maxStallTurns? })` | `1` within the allowed consecutive stalls; above it, `maxStallTurns / observedMaximum` | Requires a trace. A stall is a failed task-agent attempt with LLM work, no tool call, and no completed task. This measures agent attempts, not semantic reasoning turns. | +| `createAnswerRelevancyScorer({ judges, ... })` | Mean judge score for direct relevance to input and expected output | Thin `createJudgeScorer()` wrapper with a fixed `{ score, reason }` schema. Treat it as a prompt template to version and validate against your own data. | + +The three structure-aware scorers expose behavior specific to a multi-agent DAG, +but remain privacy-aware: traces do not persist task output bodies. Dependency +utilization and no-progress therefore use honest structural proxies, while +duplicate-work reads outputs from the in-memory `TeamRunResult` and uses the +trace only to select task executions. Use these three with offline +`runEvalSet(..., { traceStore })`; online sampling does not load a trace for a scorer. + +```ts +import { + createAnswerRelevancyScorer, + toolCallSuccessScorer, +} from '@open-multi-agent/core/eval' + +const scorers = [ + toolCallSuccessScorer(), + createAnswerRelevancyScorer({ + version: 'relevancy-prompt-v1', + judges: [{ name: 'judge', model: 'claude-sonnet-4-6', provider: 'anthropic' }], + }), +] +``` + +## Quick start: offline evaluation in five minutes ```ts import { @@ -61,6 +113,7 @@ const target: EvalTarget = async (input) => ({ const exact = defineScorer({ name: 'exact', + version: '1', score({ output, evalCase }) { const pass = output === evalCase.expected return { score: pass ? 1 : 0, pass } @@ -227,7 +280,7 @@ p95 (`0.30 µs` mean) on the implementation host. Absolute microseconds vary by host; CI additionally retains the existing observability same-host regression gate for the unconfigured path. -## Persist evaluation records +## EvalStore Use `InMemoryEvalStore` for short-lived local runs, tests, or an adapter prototype. Pass it to `runEvalSet()` to persist one atomic batch per completed @@ -533,11 +586,34 @@ the gate control the step status: path: eval-results/**/report.junit.xml ``` -## Payload privacy +## Memory evaluation metrics + +`MemoryExtractionSample` and `MemoryRetrievalSample` are experimental input +shapes for future memory scorers; this release adds no memory runtime and no +automatic memory writer. The following metrics can be implemented with existing +rule or judge scorers: + +| Stage | Metric | Definition | +|---|---|---| +| Extraction | Yield | Valid extracted records relative to conversation, token, latency, or monetary cost. Report raw counts beside any ratio. | +| Extraction | Duplicate and conflict rate | Share of records that repeat, contradict, or add no durable information. Rule checks can catch exact duplicates; semantic conflict needs a versioned judge. | +| Extraction | Staleness annotation rate | Share of time-sensitive records carrying enough provenance or expiry information to identify staleness risk. | +| Extraction | Scope leakage | Private content written into team scope. This is a safety gate: any non-zero leakage fails. | +| Extraction | Cost and reasons | Extraction latency and tokens, plus the distribution of skipped and merged reasons. | +| Retrieval | Relevance | Judge score between the query and retrieved records. Version the rubric and judge configuration. | +| Retrieval | Omission | Available positive records that should have been returned but were not. | +| Retrieval | Pollution | Run the same case with and without retrieved memory; a lower primary score after injection is harmful pollution. | +| Retrieval | Added cost | Extra tokens and latency caused by retrieval and prompt injection. | + +Automatic extraction or consolidation should not be enabled by default until it +passes both a versioned offline EvalSet gate and online sampling. Scope leakage +is always a hard safety gate, independent of average quality. + +## Privacy EvalSet cases may contain private user data. `storePayloads` therefore defaults to `'none'`, so records contain scores, reasons, metadata, and run references but no input/output snapshots. `'redacted'` serializes each payload field, caps it at 8 KiB, and applies OMA's existing secret redaction. `'full'` keeps the serialized text without redaction but still applies the 8 KiB cap; opt into it only for data you are prepared to retain. A model-based judge necessarily sends the evaluated output to the configured judge model regardless of record payload storage. -## Why there is no seed +## Reproducibility and the absence of a seed OMA's current provider contract has no cross-provider seed parameter or LLM response recording. Adding a seed to `EvalSet` would therefore promise determinism the framework cannot provide. Use `repeats` to sample nondeterministic behavior and compare the aggregate statistics. `targetFromPlan()` fixes the orchestration plan, but model responses can still vary. @@ -571,6 +647,35 @@ Judge scores are averaged. When the verdict schema returns a boolean `pass`, the `result.details.judges`, `result.details.models`, and `result.details.scores` are parallel arrays: values at the same index describe one judge. This flat representation remains compatible with trace attribute values while preserving model-drift evidence. Bump the scorer `version` whenever judge models, configuration, or prompts change. -## Runtime verification versus evaluation +## FAQ -Use `runConsensus()` or per-task `verify` to accept, revise, or reject a result during a live business run. Use scorers to measure regressions and trends after or alongside a run. Runtime verification can affect the business result; evaluation never does. The two mechanisms can be used together. +### Should a scorer error count as zero? + +No. It means quality was not measured. OMA records `scorer_error`, excludes it +from score denominators, and lets gate health limits decide whether the +evaluation infrastructure is reliable enough. + +### Why did a baseline scorer comparison produce a warning? + +Candidate and baseline scorer versions differ, or one side omitted the scorer. +OMA still applies absolute thresholds and health checks, but skips an invalid +apples-to-oranges regression comparison. Review the scorer change and create a +new accepted baseline intentionally. + +### Can evaluation delay or fail the business response? + +Offline evaluation is a separate call. Online evaluation performs only a +synchronous sampling and bounded-queue decision after the run settles; scoring +and persistence are best-effort and isolated. Call `forceFlush()` when the host +must wait for accepted samples before exit. + +### Does `targetFromPlan()` make an LLM run deterministic? + +It fixes the task graph and avoids another coordinator decomposition. Model +responses can still vary because OMA has no cross-provider seed contract. Use +`repeats` and compare distributions. + +### Where can I see complete examples? + +Run `examples/patterns/eval-offline-regression.ts` for a no-key two-target gate +or `examples/patterns/eval-online-sampling.ts` for `FileEvalStore` lifecycle. diff --git a/package.json b/package.json index 75dd176..0b5220b 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "test:coverage": "npm run test:coverage -w @open-multi-agent/core", "test:e2e": "npm run test:e2e -w @open-multi-agent/core", "bench:observability:ci": "node scripts/observability-benchmark-ci.mjs", + "test:eval-examples": "node scripts/eval-example-smoke.mjs", "test:observability-examples": "node scripts/observability-example-smoke.mjs", "test:observability-pack": "node scripts/observability-tarball-smoke.mjs", "test:scaffold": "npm run test:scaffold --workspaces --if-present", diff --git a/packages/core/examples/README.md b/packages/core/examples/README.md index e5a5a28..0e860a8 100644 --- a/packages/core/examples/README.md +++ b/packages/core/examples/README.md @@ -61,6 +61,8 @@ Reusable shapes for common multi-agent problems. | [`patterns/plan-replay`](patterns/plan-replay.ts) | Pin a coordinator plan with `createPlanArtifact`, then replay it with `runFromPlan`, no coordinator re-run. | | [`patterns/consensus`](patterns/consensus.ts) | Proposer→judge refutation loop via `runConsensus()`: default judge prompt and per-judge `judgePrompt` function. | | [`patterns/cross-provider-reasoning`](patterns/cross-provider-reasoning.ts) | Preserve a reasoning model's thought stream across providers via `preserveReasoningAsText`. | +| [`patterns/eval-offline-regression`](patterns/eval-offline-regression.ts) | No-key EvalSet regression across two model configurations with rule + judge scorers and a gate. | +| [`patterns/eval-online-sampling`](patterns/eval-online-sampling.ts) | Best-effort online sampling into `FileEvalStore` with explicit flush and shutdown. | ## cookbook — use-case recipes diff --git a/packages/core/examples/patterns/eval-offline-regression.ts b/packages/core/examples/patterns/eval-offline-regression.ts new file mode 100644 index 0000000..b7e830d --- /dev/null +++ b/packages/core/examples/patterns/eval-offline-regression.ts @@ -0,0 +1,132 @@ +/** + * Offline evaluation regression — two model configurations, rule + judge scorers, and a gate. + * + * Run: + * npx tsx packages/core/examples/patterns/eval-offline-regression.ts + * + * This fixture uses local mock adapters so it runs without network access or API keys. + * Replace the adapters and fixture model names with production AgentConfig values. + */ +import type { LLMAdapter, LLMMessage } from '../../src/index.js' +import { + createAnswerRelevancyScorer, + defineEvalSet, + defineScorer, + evaluateGate, + runEvalSet, + targetFromAgent, + type EvalRunReport, +} from '../../src/eval/index.js' + +function lastUserText(messages: LLMMessage[]): string { + const user = [...messages].reverse().find((message) => message.role === 'user') + return (user?.content ?? []) + .filter((block): block is { type: 'text'; text: string } => block.type === 'text') + .map((block) => block.text) + .join('\n') +} + +const fixtureTargetAdapter: LLMAdapter = { + name: 'fixture-target', + async chat(messages, options) { + const prompt = lastUserText(messages) + const text = prompt.includes('2 + 2') ? '4' : 'Paris' + return { + id: `fixture-${options.model}`, + content: [{ type: 'text', text }], + model: options.model, + stop_reason: 'end_turn', + usage: { input_tokens: 8, output_tokens: 1 }, + } + }, + async *stream() {}, +} + +const fixtureJudgeAdapter: LLMAdapter = { + name: 'fixture-judge', + async chat(_messages, options) { + return { + id: 'fixture-judge-response', + content: [{ type: 'text', text: '{"score":1,"reason":"directly answers the case"}' }], + model: options.model, + stop_reason: 'end_turn', + usage: { input_tokens: 20, output_tokens: 8 }, + } + }, + async *stream() {}, +} + +const set = defineEvalSet({ + name: 'offline-regression-fixture', + version: '1.0.0', + cases: [ + { id: 'arithmetic', input: 'What is 2 + 2?', expected: '4', tags: ['critical'] }, + { id: 'capital', input: 'What is the capital of France?', expected: 'Paris' }, + ], +}) + +const exact = defineScorer({ + name: 'exact_match', + version: '1', + score({ output, evalCase }) { + const pass = output === evalCase.expected + return { score: pass ? 1 : 0, pass } + }, +}) +const relevancy = createAnswerRelevancyScorer({ + version: 'fixture-prompt-v1', + judges: [{ + name: 'relevancy-judge', + model: 'fixture-judge-v1', + adapter: fixtureJudgeAdapter, + }], +}) + +const targets = [ + { + name: 'baseline-model', + target: targetFromAgent({ + name: 'baseline-agent', + model: 'fixture-baseline-v1', + adapter: fixtureTargetAdapter, + }), + }, + { + name: 'candidate-model', + target: targetFromAgent({ + name: 'candidate-agent', + model: 'fixture-candidate-v2', + systemPrompt: 'Answer directly and concisely.', + adapter: fixtureTargetAdapter, + }), + }, +] as const + +const reports: EvalRunReport[] = [] +for (const entry of targets) { + reports.push(await runEvalSet(set, entry.target, { + scorers: [exact, relevancy], + repeats: 2, + metadata: { target_name: entry.name }, + })) +} + +const [baseline, candidate] = reports +if (baseline === undefined || candidate === undefined) throw new Error('Expected two reports.') +const verdict = evaluateGate(candidate, { + schemaVersion: 1, + thresholds: [ + { scorer: 'exact_match', metric: 'passRate', min: 1 }, + { scorer: 'answer_relevancy', metric: 'avg', min: 0.9 }, + ], + maxScorerErrorRate: 0, + maxTargetErrorRate: 0, + baseline: { maxRegression: 0 }, +}, baseline) + +if (!verdict.pass) throw new Error(`Fixture gate failed: ${JSON.stringify(verdict.failures)}`) +console.log(JSON.stringify({ + targets: targets.map((target) => target.name), + candidate: candidate.aggregates, + verdict, +}, null, 2)) diff --git a/packages/core/examples/patterns/eval-online-sampling.ts b/packages/core/examples/patterns/eval-online-sampling.ts new file mode 100644 index 0000000..57ceac4 --- /dev/null +++ b/packages/core/examples/patterns/eval-online-sampling.ts @@ -0,0 +1,59 @@ +/** + * Online evaluation sampling — asynchronous scoring with a durable local FileEvalStore. + * + * Run: + * npx tsx packages/core/examples/patterns/eval-online-sampling.ts + * + * This fixture uses a local adapter. Set OMA_EVAL_STORE_PATH to choose the NDJSON file. + */ +import { OpenMultiAgent, type LLMAdapter } from '../../src/index.js' +import { costBudgetScorer } from '../../src/eval/index.js' +import { FileEvalStore } from '../../src/eval/file.js' + +const adapter: LLMAdapter = { + name: 'fixture-online', + async chat(_messages, options) { + return { + id: 'fixture-online-response', + content: [{ type: 'text', text: 'Evaluation runs after this response settles.' }], + model: options.model, + stop_reason: 'end_turn', + usage: { input_tokens: 12, output_tokens: 7 }, + } + }, + async *stream() {}, +} + +const path = process.env['OMA_EVAL_STORE_PATH'] ?? './eval-results/online-eval.ndjson' +const store = await FileEvalStore.open(path) +const orchestrator = new OpenMultiAgent({ + evaluation: { + scorers: [costBudgetScorer({ maxTokens: 100 })], + sample: 1, + maxConcurrent: 1, + maxQueueLength: 10, + budget: { maxEvaluationsPerMinute: 10 }, + store, + }, +}) + +try { + const result = await orchestrator.runAgent({ + name: 'online-fixture', + model: 'fixture-model-v1', + adapter, + }, 'Explain when online evaluation runs.') + + const flush = await orchestrator.evaluation.forceFlush({ timeoutMs: 5_000 }) + await store.flush() + console.log(JSON.stringify({ + businessOutput: result.output, + evaluation: flush, + stats: orchestrator.evaluation.getStats(), + store: path, + }, null, 2)) +} finally { + await orchestrator.evaluation.shutdown({ timeoutMs: 5_000 }) + await store.flush() + await store.close() +} diff --git a/packages/core/src/eval/index.ts b/packages/core/src/eval/index.ts index aa0db8c..6609da7 100644 --- a/packages/core/src/eval/index.ts +++ b/packages/core/src/eval/index.ts @@ -2,6 +2,21 @@ export { defineScorer } from './scorer.js' export type { ScoreResult, Scorer, ScorerContext } from './scorer.js' export { createJudgeScorer } from './judge.js' export type { JudgeScorerOptions } from './judge.js' +export { + costBudgetScorer, + createAnswerRelevancyScorer, + dependencyUtilizationScorer, + duplicateWorkScorer, + noProgressScorer, + structuredOutputComplianceScorer, + toolCallSuccessScorer, +} from './scorers/index.js' +export type { + AnswerRelevancyScorerOptions, + CostBudgetScorerOptions, + DuplicateWorkScorerOptions, + NoProgressScorerOptions, +} from './scorers/index.js' export { EVAL_STORE_SCHEMA_MAJOR } from './record.js' export type { EvalRecord } from './record.js' export { EvalStoreError, InMemoryEvalStore } from './store.js' diff --git a/packages/core/src/eval/scorer.ts b/packages/core/src/eval/scorer.ts index 37edec3..864baab 100644 --- a/packages/core/src/eval/scorer.ts +++ b/packages/core/src/eval/scorer.ts @@ -37,6 +37,7 @@ export interface Scorer { } const SCORE_COST_INPUTS = Symbol('oma.eval.score_cost_inputs') +const warnedUnversionedScorers = new Set() /** Internal usage detail attached non-enumerably by framework-backed scorers. */ export interface ScoreCostInput { @@ -139,6 +140,13 @@ export function defineScorer(scorer: Scorer): Scorer { if (typeof scorer.score !== 'function') { throw new TypeError('Scorer.score must be a function.') } + if (scorer.version === undefined && !warnedUnversionedScorers.has(scorer.name)) { + warnedUnversionedScorers.add(scorer.name) + console.warn( + `[open-multi-agent eval] Scorer "${scorer.name}" has no version; ` + + 'baseline comparisons cannot detect scoring drift.', + ) + } const score = scorer.score const frozen: Scorer = { diff --git a/packages/core/src/eval/scorers/cost.ts b/packages/core/src/eval/scorers/cost.ts new file mode 100644 index 0000000..ad3fa03 --- /dev/null +++ b/packages/core/src/eval/scorers/cost.ts @@ -0,0 +1,77 @@ +import type { TokenUsage } from '../../types.js' +import { defineScorer, type Scorer } from '../scorer.js' + +export interface CostBudgetScorerOptions { + readonly maxTokens?: number + readonly maxCostAmount?: number +} + +function positiveLimit(value: number | undefined, name: string): number | undefined { + if (value === undefined) return undefined + if (!Number.isFinite(value) || value <= 0) { + throw new RangeError(`${name} must be a positive finite number.`) + } + return value +} + +function resultTokens(result: Parameters[0]['result']): TokenUsage | undefined { + if (result === undefined) return undefined + return 'totalTokenUsage' in result ? result.totalTokenUsage : result.tokenUsage +} + +/** Apply hard token and/or observed-cost ceilings to a run. */ +export function costBudgetScorer(options: CostBudgetScorerOptions): Scorer { + const maxTokens = positiveLimit(options.maxTokens, 'maxTokens') + const maxCostAmount = positiveLimit(options.maxCostAmount, 'maxCostAmount') + if (maxTokens === undefined && maxCostAmount === undefined) { + throw new TypeError('costBudgetScorer requires maxTokens and/or maxCostAmount.') + } + + return defineScorer({ + name: 'cost_budget', + version: '1', + score(context) { + const usage = resultTokens(context.result) + const totalTokens = usage === undefined + ? undefined + : usage.input_tokens + usage.output_tokens + const currencies = new Set(context.trace?.costs.map((cost) => cost.currency) ?? []) + if (maxCostAmount !== undefined && currencies.size > 1) { + throw new TypeError('costBudgetScorer cannot compare cost amounts across multiple currencies.') + } + const costAmount = context.trace?.costs.length + ? context.trace.costs.reduce((total, cost) => total + cost.amount, 0) + : undefined + const tokenPass = maxTokens === undefined || totalTokens === undefined + ? undefined + : totalTokens <= maxTokens + const costPass = maxCostAmount === undefined || costAmount === undefined + ? undefined + : costAmount <= maxCostAmount + const checks = [tokenPass, costPass].filter((value): value is boolean => value !== undefined) + const pass = checks.every(Boolean) + const missing: string[] = [] + if (maxTokens !== undefined && totalTokens === undefined) missing.push('token usage') + if (maxCostAmount !== undefined && costAmount === undefined) missing.push('cost data') + + return { + score: pass ? 1 : 0, + ...(checks.length > 0 ? { pass } : {}), + reason: missing.length > 0 + ? `${pass ? 'Configured observed budgets passed' : 'A configured observed budget was exceeded'}; ${missing.join(' and ')} unavailable.` + : pass + ? 'All configured budgets passed.' + : 'At least one configured budget was exceeded.', + details: { + applicable: checks.length > 0, + data_complete: missing.length === 0, + ...(totalTokens !== undefined ? { total_tokens: totalTokens } : {}), + ...(maxTokens !== undefined ? { max_tokens: maxTokens } : {}), + ...(costAmount !== undefined ? { cost_amount: costAmount } : {}), + ...(maxCostAmount !== undefined ? { max_cost_amount: maxCostAmount } : {}), + ...(currencies.size === 1 ? { currency: [...currencies][0]! } : {}), + }, + } + }, + }) +} diff --git a/packages/core/src/eval/scorers/dependency.ts b/packages/core/src/eval/scorers/dependency.ts new file mode 100644 index 0000000..da5a025 --- /dev/null +++ b/packages/core/src/eval/scorers/dependency.ts @@ -0,0 +1,78 @@ +import type { MaterializedSpan } from '../../observability/store.js' +import { defineScorer, type Scorer } from '../scorer.js' + +function taskSpans(spans: readonly MaterializedSpan[]): readonly MaterializedSpan[] { + return spans.filter((span) => span.kind === 'task') +} + +/** + * Score dependency-chain completion using task span links and terminal status. + * This deliberately does not claim semantic use of a dependency's contents. + */ +export function dependencyUtilizationScorer(): Scorer { + return defineScorer({ + name: 'dependency_utilization', + version: '1', + score(context) { + if (context.trace === undefined) { + return { + score: 1, + reason: 'No trace was supplied; dependency utilization was not evaluated.', + details: { + applicable: false, + dependency_tasks: 0, + complete_dependency_tasks: 0, + missing_dependency_links: 0, + }, + } + } + + const tasks = taskSpans(context.trace.spans) + const bySpanId = new Map(tasks.map((span) => [span.spanId, span])) + const dependentTasks = tasks + .map((span) => ({ + span, + links: span.links.filter((link) => link.relation === 'depends_on'), + })) + .filter((entry) => entry.links.length > 0) + + if (dependentTasks.length === 0) { + return { + score: 1, + reason: 'The trace contained no task dependency links; the scorer is not applicable.', + details: { + applicable: false, + dependency_tasks: 0, + complete_dependency_tasks: 0, + missing_dependency_links: 0, + }, + } + } + + let missingLinks = 0 + const complete = dependentTasks.filter(({ span, links }) => { + const dependenciesComplete = links.every((link) => { + const dependency = bySpanId.get(link.spanId) + if (dependency === undefined) { + missingLinks++ + return false + } + return dependency.status === 'ok' + }) + return span.status === 'ok' && dependenciesComplete + }).length + const score = complete / dependentTasks.length + + return { + score, + reason: `${complete} of ${dependentTasks.length} dependency-bearing tasks completed with all linked dependencies successful.`, + details: { + applicable: true, + dependency_tasks: dependentTasks.length, + complete_dependency_tasks: complete, + missing_dependency_links: missingLinks, + }, + } + }, + }) +} diff --git a/packages/core/src/eval/scorers/duplicate-work.ts b/packages/core/src/eval/scorers/duplicate-work.ts new file mode 100644 index 0000000..409f650 --- /dev/null +++ b/packages/core/src/eval/scorers/duplicate-work.ts @@ -0,0 +1,121 @@ +import type { MaterializedSpan } from '../../observability/store.js' +import type { TeamRunResult } from '../../types.js' +import { defineScorer, type Scorer } from '../scorer.js' + +export interface DuplicateWorkScorerOptions { + /** Similarity at or above this value counts as duplicate work. Default: 0.8. */ + readonly threshold?: number +} + +function stringAttribute(span: MaterializedSpan, name: string): string | undefined { + const value = span.attributes[name] + return typeof value === 'string' ? value : undefined +} + +function teamResult(result: Parameters[0]['result']): TeamRunResult | undefined { + return result !== undefined && 'agentResults' in result ? result : undefined +} + +function taskOutput(result: TeamRunResult, span: MaterializedSpan): string | undefined { + const taskId = stringAttribute(span, 'oma.task.id') + if (taskId === undefined) return undefined + const agent = stringAttribute(span, 'oma.agent.name') + const exact = agent === undefined ? undefined : result.agentResults.get(`${agent}:${taskId}`) + const matched = exact ?? [...result.agentResults.entries()] + .find(([key]) => key.endsWith(`:${taskId}`))?.[1] + const output = matched?.output.trim() + return output ? output : undefined +} + +function shingles(value: string): Set { + const normalized = Array.from(value.toLowerCase().replace(/\s+/g, ' ').trim()) + if (normalized.length === 0) return new Set() + const width = Math.min(3, normalized.length) + const values = new Set() + for (let index = 0; index <= normalized.length - width; index++) { + values.add(normalized.slice(index, index + width).join('')) + } + return values +} + +function similarity(left: string, right: string): number { + const a = shingles(left) + const b = shingles(right) + if (a.size === 0 && b.size === 0) return 1 + let intersection = 0 + for (const value of a) if (b.has(value)) intersection++ + return intersection / (a.size + b.size - intersection) +} + +/** Score pairwise task-output uniqueness with zero-dependency character shingles. */ +export function duplicateWorkScorer(options: DuplicateWorkScorerOptions = {}): Scorer { + const threshold = options.threshold ?? 0.8 + if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) { + throw new RangeError('DuplicateWorkScorerOptions.threshold must be in the range [0, 1].') + } + + return defineScorer({ + name: 'duplicate_work', + version: '1', + score(context) { + const result = teamResult(context.result) + if (context.trace === undefined || result === undefined) { + return { + score: 1, + reason: 'A StoredRun trace and TeamRunResult are required; duplicate work was not evaluated.', + details: { + applicable: false, + compared_pairs: 0, + duplicate_pairs: 0, + threshold, + max_similarity: 0, + }, + } + } + + const outputs = context.trace.spans + .filter((span) => span.kind === 'task') + .flatMap((span) => { + const output = taskOutput(result, span) + return output === undefined ? [] : [output] + }) + let pairs = 0 + let duplicates = 0 + let maxSimilarity = 0 + for (let left = 0; left < outputs.length; left++) { + for (let right = left + 1; right < outputs.length; right++) { + const value = similarity(outputs[left]!, outputs[right]!) + pairs++ + maxSimilarity = Math.max(maxSimilarity, value) + if (value >= threshold) duplicates++ + } + } + + if (pairs === 0) { + return { + score: 1, + reason: 'Fewer than two non-empty task outputs were available; duplicate work was not evaluated.', + details: { + applicable: false, + compared_pairs: 0, + duplicate_pairs: 0, + threshold, + max_similarity: 0, + }, + } + } + + return { + score: 1 - duplicates / pairs, + reason: `${duplicates} of ${pairs} task-output pairs met the duplicate-work threshold.`, + details: { + applicable: true, + compared_pairs: pairs, + duplicate_pairs: duplicates, + threshold, + max_similarity: maxSimilarity, + }, + } + }, + }) +} diff --git a/packages/core/src/eval/scorers/index.ts b/packages/core/src/eval/scorers/index.ts new file mode 100644 index 0000000..a60d7fe --- /dev/null +++ b/packages/core/src/eval/scorers/index.ts @@ -0,0 +1,11 @@ +export { toolCallSuccessScorer } from './tool-call.js' +export { structuredOutputComplianceScorer } from './structured-output.js' +export { costBudgetScorer } from './cost.js' +export type { CostBudgetScorerOptions } from './cost.js' +export { dependencyUtilizationScorer } from './dependency.js' +export { duplicateWorkScorer } from './duplicate-work.js' +export type { DuplicateWorkScorerOptions } from './duplicate-work.js' +export { noProgressScorer } from './no-progress.js' +export type { NoProgressScorerOptions } from './no-progress.js' +export { createAnswerRelevancyScorer } from './relevancy.js' +export type { AnswerRelevancyScorerOptions } from './relevancy.js' diff --git a/packages/core/src/eval/scorers/no-progress.ts b/packages/core/src/eval/scorers/no-progress.ts new file mode 100644 index 0000000..c1fe003 --- /dev/null +++ b/packages/core/src/eval/scorers/no-progress.ts @@ -0,0 +1,112 @@ +import type { MaterializedSpan } from '../../observability/store.js' +import { defineScorer, type Scorer } from '../scorer.js' + +export interface NoProgressScorerOptions { + /** Consecutive stalled agent attempts allowed before the score is reduced. Default: 2. */ + readonly maxStallTurns?: number +} + +function numberAttribute(span: MaterializedSpan, name: string): number | undefined { + const value = span.attributes[name] + return typeof value === 'number' ? value : undefined +} + +function stringAttribute(span: MaterializedSpan, name: string): string | undefined { + const value = span.attributes[name] + return typeof value === 'string' ? value : undefined +} + +/** Score consecutive failed task-agent attempts that called an LLM but made no tool call. */ +export function noProgressScorer(options: NoProgressScorerOptions = {}): Scorer { + const maxStallTurns = options.maxStallTurns ?? 2 + if (!Number.isInteger(maxStallTurns) || maxStallTurns < 0) { + throw new RangeError('NoProgressScorerOptions.maxStallTurns must be a non-negative integer.') + } + + return defineScorer({ + name: 'no_progress', + version: '1', + score(context) { + if (context.trace === undefined) { + return { + score: 1, + reason: 'No trace was supplied; no-progress attempts were not evaluated.', + details: { + applicable: false, + observed_attempts: 0, + stalled_attempts: 0, + max_consecutive_stalls: 0, + max_stall_turns: maxStallTurns, + }, + } + } + + const spans = context.trace.spans + const tasksById = new Map(spans + .filter((span) => span.kind === 'task') + .flatMap((span) => { + const taskId = stringAttribute(span, 'oma.task.id') + return taskId === undefined ? [] : [[taskId, span] as const] + })) + const agents = spans + .filter((span) => span.kind === 'agent' && stringAttribute(span, 'oma.task.id') !== undefined) + .sort((left, right) => (left.startUnixMs ?? 0) - (right.startUnixMs ?? 0)) + let observed = 0 + let stalled = 0 + let consecutive = 0 + let maxConsecutive = 0 + + for (const agent of agents) { + const llmCalls = spans.filter((span) => span.kind === 'llm' && span.parentSpanId === agent.spanId).length + const turns = numberAttribute(agent, 'oma.agent.turns') ?? llmCalls + if (turns === 0) continue + observed++ + const childTools = spans.filter((span) => span.kind === 'tool' && span.parentSpanId === agent.spanId).length + const toolCalls = numberAttribute(agent, 'oma.agent.tool_calls') ?? childTools + const taskId = stringAttribute(agent, 'oma.task.id')! + const task = tasksById.get(taskId) + const taskCompletedByAttempt = task?.status === 'ok' + && task.endUnixMs !== undefined + && agent.endUnixMs !== undefined + && task.endUnixMs <= agent.endUnixMs + const isStalled = toolCalls === 0 && agent.status !== 'ok' && !taskCompletedByAttempt + if (isStalled) { + stalled++ + consecutive++ + maxConsecutive = Math.max(maxConsecutive, consecutive) + } else { + consecutive = 0 + } + } + + if (observed === 0) { + return { + score: 1, + reason: 'No task-agent attempts with LLM calls were present; the scorer is not applicable.', + details: { + applicable: false, + observed_attempts: 0, + stalled_attempts: 0, + max_consecutive_stalls: 0, + max_stall_turns: maxStallTurns, + }, + } + } + + const score = maxConsecutive <= maxStallTurns + ? 1 + : maxStallTurns === 0 ? 0 : maxStallTurns / maxConsecutive + return { + score, + reason: `Maximum consecutive stalled task-agent attempts: ${maxConsecutive} (allowed: ${maxStallTurns}).`, + details: { + applicable: true, + observed_attempts: observed, + stalled_attempts: stalled, + max_consecutive_stalls: maxConsecutive, + max_stall_turns: maxStallTurns, + }, + } + }, + }) +} diff --git a/packages/core/src/eval/scorers/relevancy.ts b/packages/core/src/eval/scorers/relevancy.ts new file mode 100644 index 0000000..c2ef266 --- /dev/null +++ b/packages/core/src/eval/scorers/relevancy.ts @@ -0,0 +1,31 @@ +import { z } from 'zod' +import type { JudgeScorerOptions } from '../judge.js' +import { createJudgeScorer } from '../judge.js' +import type { Scorer } from '../scorer.js' + +export interface AnswerRelevancyScorerOptions + extends Omit { + readonly name?: string +} + +const ANSWER_RELEVANCY_SCHEMA = z.object({ + score: z.number().min(0).max(1), + reason: z.string(), +}) + +const ANSWER_RELEVANCY_PROMPT = [ + 'Evaluate only answer relevancy.', + 'A fully relevant answer directly addresses the case input, stays on topic, and uses the expected output when one is supplied.', + 'Do not reward style, verbosity, or factual claims that are unrelated to the requested answer.', + 'Return a score from 0 to 1 and a concise reason.', +].join(' ') + +/** Create a versionable answer-relevancy judge using OMA's generic judge scorer. */ +export function createAnswerRelevancyScorer(options: AnswerRelevancyScorerOptions): Scorer { + return createJudgeScorer({ + ...options, + name: options.name ?? 'answer_relevancy', + verdictSchema: ANSWER_RELEVANCY_SCHEMA, + judgePrompt: ANSWER_RELEVANCY_PROMPT, + }) +} diff --git a/packages/core/src/eval/scorers/structured-output.ts b/packages/core/src/eval/scorers/structured-output.ts new file mode 100644 index 0000000..426fda0 --- /dev/null +++ b/packages/core/src/eval/scorers/structured-output.ts @@ -0,0 +1,35 @@ +import type { ZodSchema } from 'zod' +import { defineScorer, type Scorer } from '../scorer.js' + +/** Score whether an Agent result contains structured output and optionally matches a schema. */ +export function structuredOutputComplianceScorer(schema?: ZodSchema): Scorer { + return defineScorer({ + name: 'structured_output_compliance', + version: '1', + score(context) { + const structured = context.result !== undefined && 'structured' in context.result + ? context.result.structured + : undefined + if (structured === undefined) { + return { + score: 0, + pass: false, + reason: 'The target result did not contain structured output.', + details: { structured_present: false, schema_checked: schema !== undefined }, + } + } + + const valid = schema === undefined || schema.safeParse(structured).success + return { + score: valid ? 1 : 0, + pass: valid, + reason: valid + ? schema === undefined + ? 'Structured output was present.' + : 'Structured output matched the supplied schema.' + : 'Structured output did not match the supplied schema.', + details: { structured_present: true, schema_checked: schema !== undefined }, + } + }, + }) +} diff --git a/packages/core/src/eval/scorers/tool-call.ts b/packages/core/src/eval/scorers/tool-call.ts new file mode 100644 index 0000000..7747249 --- /dev/null +++ b/packages/core/src/eval/scorers/tool-call.ts @@ -0,0 +1,59 @@ +import type { MaterializedSpan } from '../../observability/store.js' +import type { AgentRunResult, ConsensusResult, TeamRunResult } from '../../types.js' +import { defineScorer, type Scorer } from '../scorer.js' + +type RunResult = AgentRunResult | TeamRunResult | ConsensusResult + +function resultToolCallCount(result: RunResult | undefined): number | undefined { + if (result === undefined) return undefined + if ('toolCalls' in result) return result.toolCalls.length + if ('agentResults' in result) { + return [...result.agentResults.values()] + .reduce((total, agentResult) => total + agentResult.toolCalls.length, 0) + } + return undefined +} + +function toolSucceeded(span: MaterializedSpan): boolean { + if (span.attributes['oma.tool.is_error'] === true) return false + return span.status === 'ok' +} + +/** Score the share of recorded tool calls that completed successfully. */ +export function toolCallSuccessScorer(): Scorer { + return defineScorer({ + name: 'tool_call_success', + version: '1', + score(context) { + const toolSpans = context.trace?.spans.filter((span) => span.kind === 'tool') ?? [] + const fromTrace = toolSpans.length > 0 + const total = fromTrace ? toolSpans.length : (resultToolCallCount(context.result) ?? 0) + const successful = fromTrace + ? toolSpans.filter(toolSucceeded).length + // ToolCallRecord exposes completed calls but no error flag. Without a trace, + // recorded result calls are therefore treated as successful. + : total + + if (total === 0) { + return { + score: 1, + pass: true, + reason: 'No tool calls were recorded; the scorer is not applicable.', + details: { tool_calls: 0, successful_tool_calls: 0, source: 'none' }, + } + } + + const score = successful / total + return { + score, + pass: successful === total, + reason: `${successful} of ${total} recorded tool calls succeeded.`, + details: { + tool_calls: total, + successful_tool_calls: successful, + source: fromTrace ? 'trace' : 'result', + }, + } + }, + }) +} diff --git a/packages/core/tests/eval-scorers.test.ts b/packages/core/tests/eval-scorers.test.ts new file mode 100644 index 0000000..33b49ae --- /dev/null +++ b/packages/core/tests/eval-scorers.test.ts @@ -0,0 +1,340 @@ +import { describe, expect, it, vi } from 'vitest' +import { z } from 'zod' +import { + costBudgetScorer, + createAnswerRelevancyScorer, + defineScorer, + dependencyUtilizationScorer, + duplicateWorkScorer, + noProgressScorer, + structuredOutputComplianceScorer, + toolCallSuccessScorer, + type ScorerContext, +} from '../src/eval/index.js' +import type { MaterializedSpan, StoredRun } from '../src/observability/store.js' +import type { + AgentRunResult, + LLMAdapter, + LLMMessage, + LLMResponse, + TeamRunResult, +} from '../src/types.js' + +function context(overrides: Partial = {}): ScorerContext { + return { + evalCase: { id: 'case-1', input: 'input', expected: 'expected' }, + output: 'output', + metadata: {}, + signal: new AbortController().signal, + ...overrides, + } +} + +function agentResult( + output = 'output', + overrides: Partial = {}, +): AgentRunResult { + return { + success: true, + output, + messages: [], + tokenUsage: { input_tokens: 4, output_tokens: 2 }, + toolCalls: [], + ...overrides, + } +} + +function teamResult(outputs: Readonly>): TeamRunResult { + return { + success: true, + agentResults: new Map(Object.entries(outputs).map(([key, output]) => [key, agentResult(output)])), + totalTokenUsage: { input_tokens: 12, output_tokens: 6 }, + } +} + +function span( + spanId: string, + kind: MaterializedSpan['kind'], + options: Partial = {}, +): MaterializedSpan { + return { + traceId: '1'.repeat(32), + spanId, + kind, + name: String(kind), + startUnixMs: Number(spanId.replace(/\D/g, '')) || 0, + endUnixMs: (Number(spanId.replace(/\D/g, '')) || 0) + 1, + status: 'ok', + attributes: {}, + links: [], + events: [], + incomplete: false, + ...options, + } +} + +function storedRun(spans: readonly MaterializedSpan[], costs: StoredRun['costs'] = []): StoredRun { + return { + schemaVersion: 2, + runId: 'run-1', + attempts: [{ + attempt: 1, + traceId: '1'.repeat(32), + rootSpanId: '0'.repeat(16), + startedAt: '2026-01-01T00:00:00.000Z', + endedAt: '2026-01-01T00:00:01.000Z', + durationMs: 1_000, + status: 'ok', + incomplete: false, + }], + startedAt: '2026-01-01T00:00:00.000Z', + endedAt: '2026-01-01T00:00:01.000Z', + durationMs: 1_000, + status: 'ok', + agents: [], + taskIds: [], + models: [], + providers: [], + tokens: { input_tokens: 0, output_tokens: 0 }, + costs, + incomplete: false, + spans, + } +} + +function userPrompt(messages: LLMMessage[]): string { + const message = [...messages].reverse().find((candidate) => candidate.role === 'user') + return (message?.content ?? []) + .filter((block): block is { type: 'text'; text: string } => block.type === 'text') + .map((block) => block.text) + .join('\n') +} + +describe('reference rule scorers', () => { + it('scores trace tool outcomes and treats no tool calls as explicitly not applicable', async () => { + const scorer = toolCallSuccessScorer() + await expect(Promise.resolve(scorer.score(context({ result: agentResult() })))).resolves.toMatchObject({ + score: 1, + pass: true, + details: { tool_calls: 0, source: 'none' }, + }) + + const trace = storedRun([ + span('tool-1', 'tool', { attributes: { 'oma.tool.is_error': false } }), + span('tool-2', 'tool', { status: 'error', attributes: { 'oma.tool.is_error': true } }), + ]) + await expect(Promise.resolve(scorer.score(context({ trace })))).resolves.toMatchObject({ + score: 0.5, + pass: false, + details: { tool_calls: 2, successful_tool_calls: 1, source: 'trace' }, + }) + }) + + it('uses completed result tool calls when a trace is unavailable', async () => { + const result = agentResult('done', { + toolCalls: [ + { toolName: 'read', input: {}, output: 'ok', duration: 1 }, + { toolName: 'write', input: {}, output: 'ok', duration: 1 }, + ], + }) + await expect(Promise.resolve(toolCallSuccessScorer().score(context({ result })))).resolves.toMatchObject({ + score: 1, + details: { tool_calls: 2, source: 'result' }, + }) + }) + + it('requires structured output and optionally validates it with Zod', async () => { + const schema = z.object({ answer: z.number() }) + const scorer = structuredOutputComplianceScorer(schema) + await expect(Promise.resolve(scorer.score(context({ result: agentResult() })))).resolves.toMatchObject({ + score: 0, + pass: false, + details: { structured_present: false }, + }) + await expect(Promise.resolve(scorer.score(context({ + result: agentResult('ok', { structured: { answer: 'wrong' } }), + })))).resolves.toMatchObject({ score: 0, pass: false }) + await expect(Promise.resolve(scorer.score(context({ + result: agentResult('ok', { structured: { answer: 42 } }), + })))).resolves.toMatchObject({ score: 1, pass: true }) + }) + + it('applies hard token and cost ceilings and exposes unavailable data', async () => { + const scorer = costBudgetScorer({ maxTokens: 7, maxCostAmount: 0.5 }) + await expect(Promise.resolve(scorer.score(context({ + result: agentResult(), + trace: storedRun([], [{ amount: 0.25, currency: 'USD' }]), + })))).resolves.toMatchObject({ + score: 1, + pass: true, + details: { total_tokens: 6, cost_amount: 0.25, data_complete: true }, + }) + await expect(Promise.resolve(scorer.score(context({ + result: agentResult('over', { tokenUsage: { input_tokens: 8, output_tokens: 2 } }), + trace: storedRun([], [{ amount: 0.75, currency: 'USD' }]), + })))).resolves.toMatchObject({ score: 0, pass: false }) + await expect(Promise.resolve(costBudgetScorer({ maxCostAmount: 1 }).score(context()))) + .resolves.toMatchObject({ + score: 1, + details: { applicable: false, data_complete: false }, + }) + }) + + it('rejects invalid budget options and incomparable cost currencies', async () => { + expect(() => costBudgetScorer({})).toThrow(/requires/i) + const scorer = costBudgetScorer({ maxCostAmount: 10 }) + expect(() => scorer.score(context({ + trace: storedRun([], [ + { amount: 1, currency: 'USD' }, + { amount: 1, currency: 'EUR' }, + ]), + }))).toThrow(/multiple currencies/i) + }) +}) + +describe('reference structure scorers', () => { + it('scores complete dependency chains and makes missing traces explicit', async () => { + const scorer = dependencyUtilizationScorer() + await expect(Promise.resolve(scorer.score(context()))).resolves.toMatchObject({ + score: 1, + details: { applicable: false }, + }) + await expect(Promise.resolve(scorer.score(context({ + trace: storedRun([span('task-standalone', 'task', { + attributes: { 'oma.task.id': 'standalone' }, + })]), + })))).resolves.toMatchObject({ score: 1, details: { applicable: false } }) + const trace = storedRun([ + span('task-1', 'task', { attributes: { 'oma.task.id': 't1' } }), + span('task-2', 'task', { + attributes: { 'oma.task.id': 't2' }, + links: [{ traceId: '1'.repeat(32), spanId: 'task-1', relation: 'depends_on' }], + }), + span('task-3', 'task', { + attributes: { 'oma.task.id': 't3' }, + links: [{ traceId: '1'.repeat(32), spanId: 'missing', relation: 'depends_on' }], + }), + ]) + await expect(Promise.resolve(scorer.score(context({ trace })))).resolves.toMatchObject({ + score: 0.5, + details: { + dependency_tasks: 2, + complete_dependency_tasks: 1, + missing_dependency_links: 1, + }, + }) + }) + + it('penalizes duplicate task outputs from a traced Team run', async () => { + const trace = storedRun([ + span('task-1', 'task', { attributes: { 'oma.task.id': 't1', 'oma.agent.name': 'a' } }), + span('task-2', 'task', { attributes: { 'oma.task.id': 't2', 'oma.agent.name': 'b' } }), + span('task-3', 'task', { attributes: { 'oma.task.id': 't3', 'oma.agent.name': 'c' } }), + ]) + const result = teamResult({ + 'a:t1': 'same task output', + 'b:t2': 'same task output', + 'c:t3': 'completely unrelated material', + }) + const scored = await duplicateWorkScorer().score(context({ trace, result })) + expect(scored.score).toBeCloseTo(2 / 3) + expect(scored.details).toMatchObject({ + applicable: true, + compared_pairs: 3, + duplicate_pairs: 1, + max_similarity: 1, + }) + await expect(Promise.resolve(duplicateWorkScorer().score(context({ result })))) + .resolves.toMatchObject({ + score: 1, + details: { applicable: false }, + }) + }) + + it('scores consecutive failed no-tool task attempts as a conservative stall proxy', async () => { + const trace = storedRun([ + span('task-1', 'task', { status: 'error', attributes: { 'oma.task.id': 't1' } }), + span('agent-1', 'agent', { + parentSpanId: 'task-1', + status: 'error', + attributes: { 'oma.task.id': 't1', 'oma.agent.turns': 1, 'oma.agent.tool_calls': 0 }, + }), + span('task-2', 'task', { status: 'error', attributes: { 'oma.task.id': 't2' } }), + span('agent-2', 'agent', { + parentSpanId: 'task-2', + status: 'error', + attributes: { 'oma.task.id': 't2', 'oma.agent.turns': 1, 'oma.agent.tool_calls': 0 }, + }), + span('task-3', 'task', { attributes: { 'oma.task.id': 't3' } }), + span('agent-3', 'agent', { + parentSpanId: 'task-3', + attributes: { 'oma.task.id': 't3', 'oma.agent.turns': 1, 'oma.agent.tool_calls': 0 }, + }), + ]) + await expect(Promise.resolve(noProgressScorer({ maxStallTurns: 1 }).score(context({ trace })))) + .resolves.toMatchObject({ + score: 0.5, + details: { observed_attempts: 3, stalled_attempts: 2, max_consecutive_stalls: 2 }, + }) + await expect(Promise.resolve(noProgressScorer().score(context()))).resolves.toMatchObject({ + score: 1, + details: { applicable: false }, + }) + }) +}) + +describe('reference judge scorer and version discipline', () => { + it('runs the answer-relevancy template end to end with a mock adapter', async () => { + const prompts: string[] = [] + const adapter: LLMAdapter = { + name: 'mock', + async chat(messages): Promise { + const prompt = userPrompt(messages) + prompts.push(prompt) + const irrelevant = prompt.includes('unrelated answer') + return { + id: 'judge-response', + content: [{ + type: 'text', + text: irrelevant + ? '{"score":0.1,"reason":"unrelated"}' + : '{"score":0.75,"reason":"mostly relevant"}', + }], + model: 'mock-model', + stop_reason: 'end_turn', + usage: { input_tokens: 2, output_tokens: 1 }, + } + }, + async *stream() { + yield { type: 'done' as const, data: {} } + }, + } + const scorer = createAnswerRelevancyScorer({ + version: 'prompt-v1', + judges: [{ name: 'judge', model: 'mock-model', adapter }], + }) + await expect(scorer.score(context({ + evalCase: { id: 'case-1', input: 'Name the capital of France', expected: 'Paris' }, + output: 'Paris is the capital.', + }))).resolves.toMatchObject({ score: 0.75, reason: 'judge: mostly relevant' }) + expect(prompts[0]).toContain('Evaluate only answer relevancy.') + expect(prompts[0]).toContain('Name the capital of France') + expect(prompts[0]).toContain('Paris is the capital.') + await expect(scorer.score(context({ output: 'unrelated answer' }))) + .resolves.toMatchObject({ score: 0.1, reason: 'judge: unrelated' }) + await expect(scorer.score(context({ + evalCase: { id: 'no-expected', input: 'Summarize this.' }, + output: 'A concise summary.', + }))).resolves.toMatchObject({ score: 0.75 }) + expect(prompts[2]).toContain('(not provided)') + }) + + it('warns once per scorer name when version is omitted', () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + defineScorer({ name: 'unversioned-reference-test', score: () => ({ score: 1 }) }) + defineScorer({ name: 'unversioned-reference-test', score: () => ({ score: 1 }) }) + expect(warn).toHaveBeenCalledTimes(1) + expect(warn).toHaveBeenCalledWith(expect.stringContaining('has no version')) + warn.mockRestore() + }) +}) diff --git a/scripts/eval-example-smoke.mjs b/scripts/eval-example-smoke.mjs new file mode 100644 index 0000000..77a9f47 --- /dev/null +++ b/scripts/eval-example-smoke.mjs @@ -0,0 +1,29 @@ +import { spawn } from 'node:child_process' +import { fileURLToPath } from 'node:url' +import { join } from 'node:path' + +const root = fileURLToPath(new URL('..', import.meta.url)) +const tsx = join(root, 'node_modules', 'tsx', 'dist', 'cli.mjs') +const example = join( + root, + 'packages', + 'core', + 'examples', + 'patterns', + 'eval-offline-regression.ts', +) + +await new Promise((resolve, reject) => { + const child = spawn(process.execPath, [tsx, example], { + cwd: root, + stdio: 'inherit', + env: { ...process.env, NO_COLOR: '1' }, + }) + child.once('error', reject) + child.once('exit', (code, signal) => { + if (code === 0) resolve() + else reject(new Error(`eval example failed with code ${code ?? 'null'} signal ${signal ?? 'none'}`)) + }) +}) + +console.log('evaluation example smoke: 1 passed') diff --git a/scripts/observability-tarball-smoke.mjs b/scripts/observability-tarball-smoke.mjs index d6eb377..bfce036 100644 --- a/scripts/observability-tarball-smoke.mjs +++ b/scripts/observability-tarball-smoke.mjs @@ -92,10 +92,23 @@ try { import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' + import * as coreRoot from '@open-multi-agent/core' import { OpenMultiAgent } from '@open-multi-agent/core' + import { defineEvalSet, toolCallSuccessScorer } from '@open-multi-agent/core/eval' + import { FileEvalStore } from '@open-multi-agent/core/eval/file' import { InMemoryTraceStore } from '@open-multi-agent/core/observability' import { FileTraceStore } from '@open-multi-agent/core/observability/file' if (typeof OpenMultiAgent !== 'function') throw new Error('core root import failed') + if (typeof defineEvalSet !== 'function' || typeof toolCallSuccessScorer !== 'function') { + throw new Error('eval import failed') + } + if ('defineEvalSet' in coreRoot) throw new Error('core root unexpectedly re-exported eval') + defineEvalSet({ name: 'pack', version: '1', cases: [{ id: 'case', input: 'x' }] }) + const score = await toolCallSuccessScorer().score({ + evalCase: { id: 'case', input: 'x' }, output: 'x', metadata: {}, + signal: new AbortController().signal, + }) + if (score.score !== 1) throw new Error('reference scorer failed') const records = [{ schemaVersion: 2, recordId: 'start', sequence: 1, timestampUnixMs: 1, runId: 'pack-core-only', attempt: 1, traceId: '1'.repeat(32), spanId: '2'.repeat(16), @@ -112,12 +125,14 @@ try { const directory = await mkdtemp(join(tmpdir(), 'oma-core-only-')) const file = await FileTraceStore.open(join(directory, 'traces.ndjson')) await file.append(records); await file.flush(); await file.close() + const evalFile = await FileEvalStore.open(join(directory, 'evals.ndjson')) + await evalFile.flush(); await evalFile.close() await rm(directory, { recursive: true, force: true }) `) const coreManifest = JSON.parse(await readFile(join(coreOnly, 'node_modules/@open-multi-agent/core/package.json'), 'utf8')) const installedCore = join(coreOnly, 'node_modules/@open-multi-agent/core') await assertExportsInstalled(installedCore, coreManifest) - for (const entry of ['dist/index.js', 'dist/observability/index.js']) { + for (const entry of ['dist/index.js', 'dist/observability/index.js', 'dist/eval/index.js']) { const graph = await staticImportGraph(join(installedCore, entry)) if ([...graph.specifiers].some((specifier) => specifier.startsWith('@opentelemetry/'))) { throw new Error(`${entry} eagerly reaches OpenTelemetry`) @@ -125,7 +140,10 @@ try { if ([...graph.visited].some((file) => file.endsWith('/dist/observability/file-store.js'))) { throw new Error(`${entry} eagerly reaches FileTraceStore`) } - if (entry === 'dist/observability/index.js' + if ([...graph.visited].some((file) => file.endsWith('/dist/eval/file-store.js'))) { + throw new Error(`${entry} eagerly reaches FileEvalStore`) + } + if ((entry === 'dist/observability/index.js' || entry === 'dist/eval/index.js') && [...graph.specifiers].some((specifier) => specifier === 'node:fs' || specifier.startsWith('node:fs/'))) { throw new Error(`${entry} eagerly reaches node:fs`) } @@ -134,6 +152,10 @@ try { if (![...fileGraph.specifiers].some((specifier) => specifier === 'node:fs' || specifier.startsWith('node:fs/'))) { throw new Error('observability/file did not reach its Node-only implementation') } + const evalFileGraph = await staticImportGraph(join(installedCore, 'dist/eval/file.js')) + if (![...evalFileGraph.specifiers].some((specifier) => specifier === 'node:fs' || specifier.startsWith('node:fs/'))) { + throw new Error('eval/file did not reach its Node-only implementation') + } await readdir(join(coreOnly, 'node_modules/@open-multi-agent')).then((names) => { if (names.includes('otel')) throw new Error('core-only consumer unexpectedly installed OTel') })