diff --git a/docs/evaluation-ci.md b/docs/evaluation-ci.md index 8e1fe8a..9d406fd 100644 --- a/docs/evaluation-ci.md +++ b/docs/evaluation-ci.md @@ -50,9 +50,9 @@ const verdict = evaluateGate(report, { schemaVersion: 1, thresholds: [ // A rule scorer plus passRate=1 is a deterministic quality gate. - { scorer: 'exact', metric: 'passRate', min: 1 }, - { scorer: 'relevancy', metric: 'avg', min: 0.8 }, - { scorer: 'relevancy', metric: 'p50', min: 0.85, tag: 'critical' }, + { scorer: 'exact', metric: 'passRate', min: 1, minSamples: 20 }, + { scorer: 'relevancy', metric: 'avg', min: 0.8, minSamples: 10 }, + { scorer: 'relevancy', metric: 'p50', min: 0.85, tag: 'critical', minSamples: 5 }, ], maxScorerErrorRate: 0.1, maxTargetErrorRate: 0, @@ -71,9 +71,9 @@ The equivalent JSON policy is: { "schemaVersion": 1, "thresholds": [ - { "scorer": "exact", "metric": "passRate", "min": 1 }, - { "scorer": "relevancy", "metric": "avg", "min": 0.8 }, - { "scorer": "relevancy", "metric": "p50", "min": 0.85, "tag": "critical" } + { "scorer": "exact", "metric": "passRate", "min": 1, "minSamples": 20 }, + { "scorer": "relevancy", "metric": "avg", "min": 0.8, "minSamples": 10 }, + { "scorer": "relevancy", "metric": "p50", "min": 0.85, "tag": "critical", "minSamples": 5 } ], "maxScorerErrorRate": 0.1, "maxTargetErrorRate": 0, @@ -85,16 +85,25 @@ The equivalent JSON policy is: ``` Thresholds support `avg`, `p50`, `p95`, `min`, and `passRate`, with optional -tag scoping and inclusive `min`/`max` boundaries. A missing scorer, tag, or -`passRate` source is a configuration failure rather than a silent pass. The -health defaults fail when scorer errors exceed 10% of scored plus scorer-error -records, or when any selected target fails. +tag scoping, inclusive `min`/`max` boundaries, and an optional positive integer +`minSamples`. Score-metric guards use the selected aggregate's `scoredCount`; +`passRate` guards use its `passSampleCount`. Tag-scoped thresholds therefore +use the tag aggregate's own counts. A missing scorer, tag, or `passRate` source +is a configuration failure rather than a silent pass. A count below +`minSamples` reports `insufficient_samples` with the observed `actual` count +and configured `limit`. For `passRate`, an older report that omits +`passSampleCount` fails closed with `actual: 0` rather than skipping the guard. +Omitting `minSamples` preserves the previous threshold behavior. The health +defaults fail when scorer errors exceed 10% of scored plus scorer-error records, +or when any selected target fails. Every verdict contains only `pass`, `failures`, and `warnings`. A failure has a stable `kind`, optional scorer/metric/tag coordinates, the observed `actual`, the configured `limit`, and a human-readable `message`. For failures about availability rather than a measured score, `missing_scorer` uses `actual: 0` and `limit: 1`, while `baseline_mismatch` uses `actual: 1` and `limit: 0`. +`insufficient_samples` uses the available sample count as `actual` and the +required count as `limit`. A baseline is an ordinary JSON `EvalRunReport`, not a second file format. The recommended workflow is: diff --git a/docs/evaluation.md b/docs/evaluation.md index 50886d9..c95cc60 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -135,7 +135,7 @@ console.log(report.aggregates[0]?.passRate) // 1 Each case/repeat target runs once, then that sample's scorers run serially. Different samples run in parallel up to `concurrency` (default `2`). Aborting stops new samples from being scheduled, waits for already-started samples, and returns a partial report with `aborted: true`. -Report percentiles use the nearest-rank method. For two sorted scores, p50 is the lower score and p95 is the higher score. `passRate` only includes scored records that explicitly contain `pass`; scorer errors are excluded from every score denominator. `byTag` repeats the same aggregation for each case tag. Target token usage is counted once per sample, even when multiple scorers run, and costs are summed only within the same currency. +Report percentiles use the nearest-rank method. For two sorted scores, p50 is the lower score and p95 is the higher score. `passRate` only includes scored records that explicitly contain `pass`; `passSampleCount` records that same denominator, while scorer errors are excluded from every score denominator. `byTag` repeats the same aggregation for each case tag. Target token usage is counted once per sample, even when multiple scorers run, and costs are summed only within the same currency. ## Sample production runs online diff --git a/packages/core/src/eval/file.ts b/packages/core/src/eval/file.ts index aa17c87..32559d2 100644 --- a/packages/core/src/eval/file.ts +++ b/packages/core/src/eval/file.ts @@ -120,6 +120,7 @@ const scorerAggregateSchema: z.ZodTypeAny = z.lazy(() => z.object({ min: z.number().finite().min(0).max(1), max: z.number().finite().min(0).max(1), passRate: z.number().finite().min(0).max(1).optional(), + passSampleCount: z.number().int().nonnegative().optional(), byTag: z.record(scorerAggregateSchema).optional(), })) diff --git a/packages/core/src/eval/gate.ts b/packages/core/src/eval/gate.ts index f789e5e..0fee65c 100644 --- a/packages/core/src/eval/gate.ts +++ b/packages/core/src/eval/gate.ts @@ -9,6 +9,7 @@ export interface GateThreshold { readonly metric: GateMetric readonly min?: number readonly max?: number + readonly minSamples?: number readonly tag?: string } @@ -36,6 +37,7 @@ export interface GateFailure { | 'target_health' | 'baseline_mismatch' | 'missing_scorer' + | 'insufficient_samples' readonly scorer?: string readonly metric?: string readonly tag?: string @@ -57,6 +59,10 @@ const gateThresholdSchema = z.object({ metric: z.enum(['avg', 'p50', 'p95', 'min', 'passRate']), min: unitInterval.optional(), max: unitInterval.optional(), + minSamples: z.number() + .int('minSamples must be a positive integer') + .positive('minSamples must be a positive integer') + .optional(), tag: z.string().trim().min(1).optional(), }).superRefine((threshold, context) => { if (threshold.min === undefined && threshold.max === undefined) { @@ -130,12 +136,67 @@ function missingThresholdFailure(threshold: GateThreshold): GateFailure { } } +function sampleCountFor( + aggregate: ScorerAggregate, + threshold: GateThreshold, +): number { + return threshold.metric === 'passRate' + ? aggregate.passSampleCount ?? 0 + : aggregate.scoredCount +} + +function insufficientSamplesFailure( + threshold: GateThreshold, + sampleCount: number, + aggregate: ScorerAggregate, + limit: number, +): GateFailure { + const target = thresholdTarget(threshold) + const missingPassSampleCount = threshold.metric === 'passRate' + && aggregate.passSampleCount === undefined + const message = missingPassSampleCount + ? `Gate aggregate for ${target} does not carry passSampleCount, so metric "${threshold.metric}" has 0 samples, below minimum ${threshold.minSamples}.` + : `Gate metric "${threshold.metric}" for ${target} has ${sampleCount} samples, below minimum ${threshold.minSamples}.` + + return { + kind: 'insufficient_samples', + scorer: threshold.scorer, + metric: threshold.metric, + ...(threshold.tag !== undefined ? { tag: threshold.tag } : {}), + actual: sampleCount, + limit, + message, + } +} + +function regressionSampleCountWarning( + side: 'current' | 'baseline', + threshold: GateThreshold, + sampleCount: number, + minSamples: number, +): string { + return `Regression check for gate metric "${threshold.metric}" for ${thresholdTarget(threshold)} was skipped because the ${side} aggregate has ${sampleCount} samples, below minimum ${minSamples}.` +} + function thresholdFailures( report: EvalRunReport, threshold: GateThreshold, ): readonly GateFailure[] { - const actual = aggregateMetric(aggregateFor(report, threshold), threshold.metric) - if (actual === undefined) return [missingThresholdFailure(threshold)] + const aggregate = aggregateFor(report, threshold) + const actual = aggregateMetric(aggregate, threshold.metric) + if (aggregate === undefined || actual === undefined) return [missingThresholdFailure(threshold)] + + if (threshold.minSamples !== undefined) { + const sampleCount = sampleCountFor(aggregate, threshold) + if (sampleCount < threshold.minSamples) { + return [insufficientSamplesFailure( + threshold, + sampleCount, + aggregate, + threshold.minSamples, + )] + } + } const target = thresholdTarget(threshold) const failures: GateFailure[] = [] @@ -256,8 +317,38 @@ export function evaluateGate( continue } - const currentValue = aggregateMetric(aggregateFor(report, threshold), threshold.metric) - const baselineValue = aggregateMetric(aggregateFor(baseline, threshold), threshold.metric) + const currentAggregate = aggregateFor(report, threshold) + const baselineAggregate = aggregateFor(baseline, threshold) + if (threshold.minSamples !== undefined) { + const currentSampleCount = currentAggregate === undefined + ? undefined + : sampleCountFor(currentAggregate, threshold) + if (currentSampleCount !== undefined && currentSampleCount < threshold.minSamples) { + warnings.add(regressionSampleCountWarning( + 'current', + threshold, + currentSampleCount, + threshold.minSamples, + )) + continue + } + + const baselineSampleCount = baselineAggregate === undefined + ? undefined + : sampleCountFor(baselineAggregate, threshold) + if (baselineSampleCount !== undefined && baselineSampleCount < threshold.minSamples) { + warnings.add(regressionSampleCountWarning( + 'baseline', + threshold, + baselineSampleCount, + threshold.minSamples, + )) + continue + } + } + + const currentValue = aggregateMetric(currentAggregate, threshold.metric) + const baselineValue = aggregateMetric(baselineAggregate, threshold.metric) if (currentValue === undefined || baselineValue === undefined) { warnings.add( `Baseline metric "${threshold.metric}" for ${thresholdTarget(threshold)} is unavailable; regression check was skipped.`, diff --git a/packages/core/src/eval/report.ts b/packages/core/src/eval/report.ts index 384d7e0..cbb9c39 100644 --- a/packages/core/src/eval/report.ts +++ b/packages/core/src/eval/report.ts @@ -14,6 +14,8 @@ export interface ScorerAggregate { readonly min: number readonly max: number readonly passRate?: number + /** Number of scored records that define `pass`; present when `passRate` is. */ + readonly passSampleCount?: number readonly byTag?: Readonly> } @@ -69,7 +71,10 @@ function aggregateOne( min: scores[0] ?? 0, max: scores[scores.length - 1] ?? 0, ...(passed.length > 0 - ? { passRate: passed.filter((record) => record.pass === true).length / passed.length } + ? { + passRate: passed.filter((record) => record.pass === true).length / passed.length, + passSampleCount: passed.length, + } : {}), } } diff --git a/packages/core/tests/eval-file.test.ts b/packages/core/tests/eval-file.test.ts index 786e410..0653e69 100644 --- a/packages/core/tests/eval-file.test.ts +++ b/packages/core/tests/eval-file.test.ts @@ -108,6 +108,7 @@ function reportFixture(): EvalRunReport { min: 0, max: 1, passRate: 0.5, + passSampleCount: 2, byTag: { edge: { scorer: { name: 'exact<&"\'', version: '1' }, @@ -119,6 +120,7 @@ function reportFixture(): EvalRunReport { min: 0, max: 0, passRate: 0, + passSampleCount: 1, }, }, }, @@ -200,6 +202,35 @@ describe('loadGatePolicy', () => { expect(Object.isFrozen(policy.thresholds)).toBe(true) }) + it('preserves minSamples in gate policy thresholds', async () => { + const path = join(await temporaryDirectory(), 'gate.json') + await writeFile(path, JSON.stringify({ + schemaVersion: 1, + thresholds: [{ scorer: 'exact', metric: 'passRate', min: 1, minSamples: 5 }], + }), 'utf8') + + const policy = await loadGatePolicy(path) + + expect(policy.thresholds[0]).toEqual({ + scorer: 'exact', + metric: 'passRate', + min: 1, + minSamples: 5, + }) + }) + + it('rejects invalid minSamples values in gate policies', async () => { + for (const minSamples of [0, -1, 1.5]) { + const path = join(await temporaryDirectory(), 'gate.json') + await writeFile(path, JSON.stringify({ + schemaVersion: 1, + thresholds: [{ scorer: 'exact', metric: 'passRate', min: 1, minSamples }], + }), 'utf8') + + await expect(loadGatePolicy(path)).rejects.toThrow('minSamples') + } + }) + it('rejects malformed JSON and reports schema issue paths', async () => { const directory = await temporaryDirectory() const broken = join(directory, 'broken-gate.json') diff --git a/packages/core/tests/eval-gate.test.ts b/packages/core/tests/eval-gate.test.ts index cfa9096..ac6d1a1 100644 --- a/packages/core/tests/eval-gate.test.ts +++ b/packages/core/tests/eval-gate.test.ts @@ -26,13 +26,15 @@ function aggregate(options: { readonly version?: string readonly avg?: number readonly passRate?: number | null + readonly scoredCount?: number + readonly passSampleCount?: number | null readonly byTag?: Readonly> } = {}): ScorerAggregate { const avg = options.avg ?? 0.8 const passRate = options.passRate === undefined ? 0.8 : options.passRate return { scorer: { name: options.name ?? 'exact', version: options.version ?? '1' }, - scoredCount: 10, + scoredCount: options.scoredCount ?? 10, errorCount: 0, avg, p50: avg, @@ -40,6 +42,9 @@ function aggregate(options: { min: avg, max: avg, ...(passRate !== null ? { passRate } : {}), + ...(passRate !== null && options.passSampleCount !== null && options.passSampleCount !== undefined + ? { passSampleCount: options.passSampleCount } + : {}), ...(options.byTag !== undefined ? { byTag: options.byTag } : {}), } } @@ -123,6 +128,123 @@ describe('evaluateGate thresholds', () => { expect(verdict.failures[1]?.message).toContain('tag "missing"') expect(verdict.failures[2]?.message).toContain('no scored records with a pass value') }) + + it('rejects minSamples that are zero, negative, or non-integer', () => { + for (const minSamples of [0, -1, 1.5]) { + expect(() => evaluateGate(report(), policy([ + { scorer: 'exact', metric: 'avg', min: 0.5, minSamples }, + ]))).toThrow('minSamples must be a positive integer') + } + }) + + it('fails below, passes at and above minSamples for score metrics', () => { + const below = evaluateGate( + report({ aggregate: aggregate({ scoredCount: 2 }) }), + policy([{ scorer: 'exact', metric: 'avg', min: 0.5, minSamples: 5 }]), + ) + const at = evaluateGate( + report({ aggregate: aggregate({ scoredCount: 5 }) }), + policy([{ scorer: 'exact', metric: 'avg', min: 0.5, minSamples: 5 }]), + ) + const above = evaluateGate( + report({ aggregate: aggregate({ scoredCount: 10 }) }), + policy([{ scorer: 'exact', metric: 'avg', min: 0.5, minSamples: 5 }]), + ) + + expect(below.failures).toEqual([ + expect.objectContaining({ + kind: 'insufficient_samples', + scorer: 'exact', + metric: 'avg', + actual: 2, + limit: 5, + }), + ]) + expect(at.pass).toBe(true) + expect(above.pass).toBe(true) + }) + + it('respects tag scoping for minSamples', () => { + const tagged = aggregate({ scoredCount: 2, avg: 0.9, passRate: 0.9, passSampleCount: 2 }) + const current = report({ + aggregate: aggregate({ scoredCount: 20, byTag: { critical: tagged } }), + }) + + const verdict = evaluateGate(current, policy([ + { scorer: 'exact', metric: 'avg', min: 0.5, minSamples: 5, tag: 'critical' }, + ])) + + expect(verdict.failures).toEqual([ + expect.objectContaining({ + kind: 'insufficient_samples', + scorer: 'exact', + metric: 'avg', + tag: 'critical', + actual: 2, + limit: 5, + }), + ]) + }) + + it('uses passSampleCount for passRate, not scoredCount', () => { + const current = report({ + aggregate: aggregate({ scoredCount: 10, passRate: 1, passSampleCount: 3 }), + }) + + const verdict = evaluateGate(current, policy([ + { scorer: 'exact', metric: 'passRate', min: 1, minSamples: 5 }, + ])) + + expect(verdict.failures).toEqual([ + expect.objectContaining({ + kind: 'insufficient_samples', + scorer: 'exact', + metric: 'passRate', + actual: 3, + limit: 5, + }), + ]) + }) + + it('fails closed when an old report omits passSampleCount for passRate', () => { + const oldAggregate: ScorerAggregate = { + scorer: { name: 'exact', version: '1' }, + scoredCount: 10, + errorCount: 0, + avg: 0.8, + p50: 0.8, + p95: 0.8, + min: 0.8, + max: 0.8, + passRate: 1, + } + const current = report({ aggregate: oldAggregate }) + + const verdict = evaluateGate(current, policy([ + { scorer: 'exact', metric: 'passRate', min: 1, minSamples: 5 }, + ])) + + expect(verdict.failures).toEqual([ + expect.objectContaining({ + kind: 'insufficient_samples', + scorer: 'exact', + metric: 'passRate', + actual: 0, + limit: 5, + }), + ]) + expect(verdict.failures[0]?.message).toContain('does not carry passSampleCount') + }) + + it('preserves current verdicts when minSamples is omitted', () => { + const verdict = evaluateGate( + report({ aggregate: aggregate({ scoredCount: 2 }) }), + policy([{ scorer: 'exact', metric: 'avg', min: 0.5 }]), + ) + + expect(verdict.pass).toBe(true) + expect(verdict.failures).toEqual([]) + }) }) describe('evaluateGate baseline regression', () => { @@ -203,6 +325,67 @@ describe('evaluateGate baseline regression', () => { expect(verdict.warnings).toHaveLength(1) expect(verdict.warnings[0]).toContain('Scorer version drift') }) + + it('skips regression when either report is below minSamples', () => { + const thresholds = [ + { scorer: 'exact', metric: 'avg' as const, min: 0, minSamples: 5 }, + ] + + const currentBelow = evaluateGate( + report({ aggregate: aggregate({ scoredCount: 2, avg: 0.5 }) }), + policy(thresholds, { baseline: { maxRegression: 0 } }), + report({ aggregate: aggregate({ scoredCount: 5, avg: 1 }) }), + ) + const baselineBelow = evaluateGate( + report({ aggregate: aggregate({ scoredCount: 5, avg: 0.5 }) }), + policy(thresholds, { baseline: { maxRegression: 0 } }), + report({ aggregate: aggregate({ scoredCount: 2, avg: 1 }) }), + ) + const sufficient = evaluateGate( + report({ aggregate: aggregate({ scoredCount: 5, avg: 0.5 }) }), + policy(thresholds, { baseline: { maxRegression: 0 } }), + report({ aggregate: aggregate({ scoredCount: 5, avg: 1 }) }), + ) + + expect(currentBelow.failures).toEqual([ + expect.objectContaining({ kind: 'insufficient_samples', actual: 2, limit: 5 }), + ]) + expect(currentBelow.warnings).toHaveLength(1) + expect(currentBelow.warnings[0]).toContain('current aggregate has 2 samples') + expect(baselineBelow.pass).toBe(true) + expect(baselineBelow.failures).toEqual([]) + expect(baselineBelow.warnings).toHaveLength(1) + expect(baselineBelow.warnings[0]).toContain('baseline aggregate has 2 samples') + expect(sufficient.failures).toEqual([ + expect.objectContaining({ kind: 'regression', actual: 0.5, limit: 0 }), + ]) + }) + + it('treats a baseline without passSampleCount as insufficient for passRate', () => { + const oldBaseline: ScorerAggregate = { + scorer: { name: 'exact', version: '1' }, + scoredCount: 10, + errorCount: 0, + avg: 1, + p50: 1, + p95: 1, + min: 1, + max: 1, + passRate: 1, + } + const verdict = evaluateGate( + report({ aggregate: aggregate({ scoredCount: 10, passRate: 0.5, passSampleCount: 10 }) }), + policy([ + { scorer: 'exact', metric: 'passRate', min: 0, minSamples: 5 }, + ], { baseline: { maxRegression: 0 } }), + report({ aggregate: oldBaseline }), + ) + + expect(verdict.pass).toBe(true) + expect(verdict.failures).toEqual([]) + expect(verdict.warnings).toHaveLength(1) + expect(verdict.warnings[0]).toContain('baseline aggregate has 0 samples') + }) }) describe('evaluateGate health and warnings', () => { diff --git a/packages/core/tests/eval-runner.test.ts b/packages/core/tests/eval-runner.test.ts index 88c21f0..64ee0ff 100644 --- a/packages/core/tests/eval-runner.test.ts +++ b/packages/core/tests/eval-runner.test.ts @@ -250,9 +250,20 @@ describe('runEvalSet', () => { min: 0.2, max: 1, passRate: 0.5, + passSampleCount: 2, + }) + expect(aggregate.byTag?.['single']).toMatchObject({ + p50: 0.2, + p95: 0.2, + scoredCount: 1, + passSampleCount: 1, + }) + expect(aggregate.byTag?.['pair']).toMatchObject({ + p50: 0.2, + p95: 0.8, + scoredCount: 2, + passSampleCount: 2, }) - expect(aggregate.byTag?.['single']).toMatchObject({ p50: 0.2, p95: 0.2, scoredCount: 1 }) - expect(aggregate.byTag?.['pair']).toMatchObject({ p50: 0.2, p95: 0.8, scoredCount: 2 }) }) it('links StoredRun traces and run identity into scorer context and records', async () => {