From 1d40455844c112a6516bd4e426412866d0faadea Mon Sep 17 00:00:00 2001 From: bit <146006641+Zhongghaoming@users.noreply.github.com> Date: Mon, 14 Sep 2026 18:19:56 +0800 Subject: [PATCH] feat(core): enforce minimum eval gate samples (#602) Add an optional positive-integer `minSamples` guard to `GateThreshold` so a threshold only counts when the selected aggregate carries enough evidence. Score metrics compare it against `scoredCount`; `passRate` compares it against the new `ScorerAggregate.passSampleCount`, which records the pass denominator whenever `passRate` is emitted. Tag-scoped thresholds use the tag aggregate's own counts. A count below the minimum reports a new `insufficient_samples` failure carrying the observed count and the configured limit, and a report written before `passSampleCount` existed fails closed at zero rather than skipping the guard. Baseline regression checks honor the same minimum. When either the current or the baseline aggregate is below it, the comparison is skipped with a warning instead of producing a regression number from a sample set the gate already called too small to judge. Omitting `minSamples` preserves the previous behavior, so the report schema version is unchanged and no migration is required. Fixes #573 --- docs/evaluation-ci.md | 29 ++-- docs/evaluation.md | 2 +- packages/core/src/eval/file.ts | 1 + packages/core/src/eval/gate.ts | 99 ++++++++++++- packages/core/src/eval/report.ts | 7 +- packages/core/tests/eval-file.test.ts | 31 ++++ packages/core/tests/eval-gate.test.ts | 185 +++++++++++++++++++++++- packages/core/tests/eval-runner.test.ts | 15 +- 8 files changed, 350 insertions(+), 19 deletions(-) diff --git a/docs/evaluation-ci.md b/docs/evaluation-ci.md index 8e1fe8a..9d406fd 100644 --- a/docs/evaluation-ci.md +++ b/docs/evaluation-ci.md @@ -50,9 +50,9 @@ const verdict = evaluateGate(report, { schemaVersion: 1, thresholds: [ // A rule scorer plus passRate=1 is a deterministic quality gate. - { scorer: 'exact', metric: 'passRate', min: 1 }, - { scorer: 'relevancy', metric: 'avg', min: 0.8 }, - { scorer: 'relevancy', metric: 'p50', min: 0.85, tag: 'critical' }, + { scorer: 'exact', metric: 'passRate', min: 1, minSamples: 20 }, + { scorer: 'relevancy', metric: 'avg', min: 0.8, minSamples: 10 }, + { scorer: 'relevancy', metric: 'p50', min: 0.85, tag: 'critical', minSamples: 5 }, ], maxScorerErrorRate: 0.1, maxTargetErrorRate: 0, @@ -71,9 +71,9 @@ The equivalent JSON policy is: { "schemaVersion": 1, "thresholds": [ - { "scorer": "exact", "metric": "passRate", "min": 1 }, - { "scorer": "relevancy", "metric": "avg", "min": 0.8 }, - { "scorer": "relevancy", "metric": "p50", "min": 0.85, "tag": "critical" } + { "scorer": "exact", "metric": "passRate", "min": 1, "minSamples": 20 }, + { "scorer": "relevancy", "metric": "avg", "min": 0.8, "minSamples": 10 }, + { "scorer": "relevancy", "metric": "p50", "min": 0.85, "tag": "critical", "minSamples": 5 } ], "maxScorerErrorRate": 0.1, "maxTargetErrorRate": 0, @@ -85,16 +85,25 @@ The equivalent JSON policy is: ``` Thresholds support `avg`, `p50`, `p95`, `min`, and `passRate`, with optional -tag scoping and inclusive `min`/`max` boundaries. A missing scorer, tag, or -`passRate` source is a configuration failure rather than a silent pass. The -health defaults fail when scorer errors exceed 10% of scored plus scorer-error -records, or when any selected target fails. +tag scoping, inclusive `min`/`max` boundaries, and an optional positive integer +`minSamples`. Score-metric guards use the selected aggregate's `scoredCount`; +`passRate` guards use its `passSampleCount`. Tag-scoped thresholds therefore +use the tag aggregate's own counts. A missing scorer, tag, or `passRate` source +is a configuration failure rather than a silent pass. A count below +`minSamples` reports `insufficient_samples` with the observed `actual` count +and configured `limit`. For `passRate`, an older report that omits +`passSampleCount` fails closed with `actual: 0` rather than skipping the guard. +Omitting `minSamples` preserves the previous threshold behavior. The health +defaults fail when scorer errors exceed 10% of scored plus scorer-error records, +or when any selected target fails. Every verdict contains only `pass`, `failures`, and `warnings`. A failure has a stable `kind`, optional scorer/metric/tag coordinates, the observed `actual`, the configured `limit`, and a human-readable `message`. For failures about availability rather than a measured score, `missing_scorer` uses `actual: 0` and `limit: 1`, while `baseline_mismatch` uses `actual: 1` and `limit: 0`. +`insufficient_samples` uses the available sample count as `actual` and the +required count as `limit`. A baseline is an ordinary JSON `EvalRunReport`, not a second file format. The recommended workflow is: diff --git a/docs/evaluation.md b/docs/evaluation.md index 50886d9..c95cc60 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -135,7 +135,7 @@ console.log(report.aggregates[0]?.passRate) // 1 Each case/repeat target runs once, then that sample's scorers run serially. Different samples run in parallel up to `concurrency` (default `2`). Aborting stops new samples from being scheduled, waits for already-started samples, and returns a partial report with `aborted: true`. -Report percentiles use the nearest-rank method. For two sorted scores, p50 is the lower score and p95 is the higher score. `passRate` only includes scored records that explicitly contain `pass`; scorer errors are excluded from every score denominator. `byTag` repeats the same aggregation for each case tag. Target token usage is counted once per sample, even when multiple scorers run, and costs are summed only within the same currency. +Report percentiles use the nearest-rank method. For two sorted scores, p50 is the lower score and p95 is the higher score. `passRate` only includes scored records that explicitly contain `pass`; `passSampleCount` records that same denominator, while scorer errors are excluded from every score denominator. `byTag` repeats the same aggregation for each case tag. Target token usage is counted once per sample, even when multiple scorers run, and costs are summed only within the same currency. ## Sample production runs online diff --git a/packages/core/src/eval/file.ts b/packages/core/src/eval/file.ts index aa17c87..32559d2 100644 --- a/packages/core/src/eval/file.ts +++ b/packages/core/src/eval/file.ts @@ -120,6 +120,7 @@ const scorerAggregateSchema: z.ZodTypeAny = z.lazy(() => z.object({ min: z.number().finite().min(0).max(1), max: z.number().finite().min(0).max(1), passRate: z.number().finite().min(0).max(1).optional(), + passSampleCount: z.number().int().nonnegative().optional(), byTag: z.record(scorerAggregateSchema).optional(), })) diff --git a/packages/core/src/eval/gate.ts b/packages/core/src/eval/gate.ts index f789e5e..0fee65c 100644 --- a/packages/core/src/eval/gate.ts +++ b/packages/core/src/eval/gate.ts @@ -9,6 +9,7 @@ export interface GateThreshold { readonly metric: GateMetric readonly min?: number readonly max?: number + readonly minSamples?: number readonly tag?: string } @@ -36,6 +37,7 @@ export interface GateFailure { | 'target_health' | 'baseline_mismatch' | 'missing_scorer' + | 'insufficient_samples' readonly scorer?: string readonly metric?: string readonly tag?: string @@ -57,6 +59,10 @@ const gateThresholdSchema = z.object({ metric: z.enum(['avg', 'p50', 'p95', 'min', 'passRate']), min: unitInterval.optional(), max: unitInterval.optional(), + minSamples: z.number() + .int('minSamples must be a positive integer') + .positive('minSamples must be a positive integer') + .optional(), tag: z.string().trim().min(1).optional(), }).superRefine((threshold, context) => { if (threshold.min === undefined && threshold.max === undefined) { @@ -130,12 +136,67 @@ function missingThresholdFailure(threshold: GateThreshold): GateFailure { } } +function sampleCountFor( + aggregate: ScorerAggregate, + threshold: GateThreshold, +): number { + return threshold.metric === 'passRate' + ? aggregate.passSampleCount ?? 0 + : aggregate.scoredCount +} + +function insufficientSamplesFailure( + threshold: GateThreshold, + sampleCount: number, + aggregate: ScorerAggregate, + limit: number, +): GateFailure { + const target = thresholdTarget(threshold) + const missingPassSampleCount = threshold.metric === 'passRate' + && aggregate.passSampleCount === undefined + const message = missingPassSampleCount + ? `Gate aggregate for ${target} does not carry passSampleCount, so metric "${threshold.metric}" has 0 samples, below minimum ${threshold.minSamples}.` + : `Gate metric "${threshold.metric}" for ${target} has ${sampleCount} samples, below minimum ${threshold.minSamples}.` + + return { + kind: 'insufficient_samples', + scorer: threshold.scorer, + metric: threshold.metric, + ...(threshold.tag !== undefined ? { tag: threshold.tag } : {}), + actual: sampleCount, + limit, + message, + } +} + +function regressionSampleCountWarning( + side: 'current' | 'baseline', + threshold: GateThreshold, + sampleCount: number, + minSamples: number, +): string { + return `Regression check for gate metric "${threshold.metric}" for ${thresholdTarget(threshold)} was skipped because the ${side} aggregate has ${sampleCount} samples, below minimum ${minSamples}.` +} + function thresholdFailures( report: EvalRunReport, threshold: GateThreshold, ): readonly GateFailure[] { - const actual = aggregateMetric(aggregateFor(report, threshold), threshold.metric) - if (actual === undefined) return [missingThresholdFailure(threshold)] + const aggregate = aggregateFor(report, threshold) + const actual = aggregateMetric(aggregate, threshold.metric) + if (aggregate === undefined || actual === undefined) return [missingThresholdFailure(threshold)] + + if (threshold.minSamples !== undefined) { + const sampleCount = sampleCountFor(aggregate, threshold) + if (sampleCount < threshold.minSamples) { + return [insufficientSamplesFailure( + threshold, + sampleCount, + aggregate, + threshold.minSamples, + )] + } + } const target = thresholdTarget(threshold) const failures: GateFailure[] = [] @@ -256,8 +317,38 @@ export function evaluateGate( continue } - const currentValue = aggregateMetric(aggregateFor(report, threshold), threshold.metric) - const baselineValue = aggregateMetric(aggregateFor(baseline, threshold), threshold.metric) + const currentAggregate = aggregateFor(report, threshold) + const baselineAggregate = aggregateFor(baseline, threshold) + if (threshold.minSamples !== undefined) { + const currentSampleCount = currentAggregate === undefined + ? undefined + : sampleCountFor(currentAggregate, threshold) + if (currentSampleCount !== undefined && currentSampleCount < threshold.minSamples) { + warnings.add(regressionSampleCountWarning( + 'current', + threshold, + currentSampleCount, + threshold.minSamples, + )) + continue + } + + const baselineSampleCount = baselineAggregate === undefined + ? undefined + : sampleCountFor(baselineAggregate, threshold) + if (baselineSampleCount !== undefined && baselineSampleCount < threshold.minSamples) { + warnings.add(regressionSampleCountWarning( + 'baseline', + threshold, + baselineSampleCount, + threshold.minSamples, + )) + continue + } + } + + const currentValue = aggregateMetric(currentAggregate, threshold.metric) + const baselineValue = aggregateMetric(baselineAggregate, threshold.metric) if (currentValue === undefined || baselineValue === undefined) { warnings.add( `Baseline metric "${threshold.metric}" for ${thresholdTarget(threshold)} is unavailable; regression check was skipped.`, diff --git a/packages/core/src/eval/report.ts b/packages/core/src/eval/report.ts index 384d7e0..cbb9c39 100644 --- a/packages/core/src/eval/report.ts +++ b/packages/core/src/eval/report.ts @@ -14,6 +14,8 @@ export interface ScorerAggregate { readonly min: number readonly max: number readonly passRate?: number + /** Number of scored records that define `pass`; present when `passRate` is. */ + readonly passSampleCount?: number readonly byTag?: Readonly> } @@ -69,7 +71,10 @@ function aggregateOne( min: scores[0] ?? 0, max: scores[scores.length - 1] ?? 0, ...(passed.length > 0 - ? { passRate: passed.filter((record) => record.pass === true).length / passed.length } + ? { + passRate: passed.filter((record) => record.pass === true).length / passed.length, + passSampleCount: passed.length, + } : {}), } } diff --git a/packages/core/tests/eval-file.test.ts b/packages/core/tests/eval-file.test.ts index 786e410..0653e69 100644 --- a/packages/core/tests/eval-file.test.ts +++ b/packages/core/tests/eval-file.test.ts @@ -108,6 +108,7 @@ function reportFixture(): EvalRunReport { min: 0, max: 1, passRate: 0.5, + passSampleCount: 2, byTag: { edge: { scorer: { name: 'exact<&"\'', version: '1' }, @@ -119,6 +120,7 @@ function reportFixture(): EvalRunReport { min: 0, max: 0, passRate: 0, + passSampleCount: 1, }, }, }, @@ -200,6 +202,35 @@ describe('loadGatePolicy', () => { expect(Object.isFrozen(policy.thresholds)).toBe(true) }) + it('preserves minSamples in gate policy thresholds', async () => { + const path = join(await temporaryDirectory(), 'gate.json') + await writeFile(path, JSON.stringify({ + schemaVersion: 1, + thresholds: [{ scorer: 'exact', metric: 'passRate', min: 1, minSamples: 5 }], + }), 'utf8') + + const policy = await loadGatePolicy(path) + + expect(policy.thresholds[0]).toEqual({ + scorer: 'exact', + metric: 'passRate', + min: 1, + minSamples: 5, + }) + }) + + it('rejects invalid minSamples values in gate policies', async () => { + for (const minSamples of [0, -1, 1.5]) { + const path = join(await temporaryDirectory(), 'gate.json') + await writeFile(path, JSON.stringify({ + schemaVersion: 1, + thresholds: [{ scorer: 'exact', metric: 'passRate', min: 1, minSamples }], + }), 'utf8') + + await expect(loadGatePolicy(path)).rejects.toThrow('minSamples') + } + }) + it('rejects malformed JSON and reports schema issue paths', async () => { const directory = await temporaryDirectory() const broken = join(directory, 'broken-gate.json') diff --git a/packages/core/tests/eval-gate.test.ts b/packages/core/tests/eval-gate.test.ts index cfa9096..ac6d1a1 100644 --- a/packages/core/tests/eval-gate.test.ts +++ b/packages/core/tests/eval-gate.test.ts @@ -26,13 +26,15 @@ function aggregate(options: { readonly version?: string readonly avg?: number readonly passRate?: number | null + readonly scoredCount?: number + readonly passSampleCount?: number | null readonly byTag?: Readonly> } = {}): ScorerAggregate { const avg = options.avg ?? 0.8 const passRate = options.passRate === undefined ? 0.8 : options.passRate return { scorer: { name: options.name ?? 'exact', version: options.version ?? '1' }, - scoredCount: 10, + scoredCount: options.scoredCount ?? 10, errorCount: 0, avg, p50: avg, @@ -40,6 +42,9 @@ function aggregate(options: { min: avg, max: avg, ...(passRate !== null ? { passRate } : {}), + ...(passRate !== null && options.passSampleCount !== null && options.passSampleCount !== undefined + ? { passSampleCount: options.passSampleCount } + : {}), ...(options.byTag !== undefined ? { byTag: options.byTag } : {}), } } @@ -123,6 +128,123 @@ describe('evaluateGate thresholds', () => { expect(verdict.failures[1]?.message).toContain('tag "missing"') expect(verdict.failures[2]?.message).toContain('no scored records with a pass value') }) + + it('rejects minSamples that are zero, negative, or non-integer', () => { + for (const minSamples of [0, -1, 1.5]) { + expect(() => evaluateGate(report(), policy([ + { scorer: 'exact', metric: 'avg', min: 0.5, minSamples }, + ]))).toThrow('minSamples must be a positive integer') + } + }) + + it('fails below, passes at and above minSamples for score metrics', () => { + const below = evaluateGate( + report({ aggregate: aggregate({ scoredCount: 2 }) }), + policy([{ scorer: 'exact', metric: 'avg', min: 0.5, minSamples: 5 }]), + ) + const at = evaluateGate( + report({ aggregate: aggregate({ scoredCount: 5 }) }), + policy([{ scorer: 'exact', metric: 'avg', min: 0.5, minSamples: 5 }]), + ) + const above = evaluateGate( + report({ aggregate: aggregate({ scoredCount: 10 }) }), + policy([{ scorer: 'exact', metric: 'avg', min: 0.5, minSamples: 5 }]), + ) + + expect(below.failures).toEqual([ + expect.objectContaining({ + kind: 'insufficient_samples', + scorer: 'exact', + metric: 'avg', + actual: 2, + limit: 5, + }), + ]) + expect(at.pass).toBe(true) + expect(above.pass).toBe(true) + }) + + it('respects tag scoping for minSamples', () => { + const tagged = aggregate({ scoredCount: 2, avg: 0.9, passRate: 0.9, passSampleCount: 2 }) + const current = report({ + aggregate: aggregate({ scoredCount: 20, byTag: { critical: tagged } }), + }) + + const verdict = evaluateGate(current, policy([ + { scorer: 'exact', metric: 'avg', min: 0.5, minSamples: 5, tag: 'critical' }, + ])) + + expect(verdict.failures).toEqual([ + expect.objectContaining({ + kind: 'insufficient_samples', + scorer: 'exact', + metric: 'avg', + tag: 'critical', + actual: 2, + limit: 5, + }), + ]) + }) + + it('uses passSampleCount for passRate, not scoredCount', () => { + const current = report({ + aggregate: aggregate({ scoredCount: 10, passRate: 1, passSampleCount: 3 }), + }) + + const verdict = evaluateGate(current, policy([ + { scorer: 'exact', metric: 'passRate', min: 1, minSamples: 5 }, + ])) + + expect(verdict.failures).toEqual([ + expect.objectContaining({ + kind: 'insufficient_samples', + scorer: 'exact', + metric: 'passRate', + actual: 3, + limit: 5, + }), + ]) + }) + + it('fails closed when an old report omits passSampleCount for passRate', () => { + const oldAggregate: ScorerAggregate = { + scorer: { name: 'exact', version: '1' }, + scoredCount: 10, + errorCount: 0, + avg: 0.8, + p50: 0.8, + p95: 0.8, + min: 0.8, + max: 0.8, + passRate: 1, + } + const current = report({ aggregate: oldAggregate }) + + const verdict = evaluateGate(current, policy([ + { scorer: 'exact', metric: 'passRate', min: 1, minSamples: 5 }, + ])) + + expect(verdict.failures).toEqual([ + expect.objectContaining({ + kind: 'insufficient_samples', + scorer: 'exact', + metric: 'passRate', + actual: 0, + limit: 5, + }), + ]) + expect(verdict.failures[0]?.message).toContain('does not carry passSampleCount') + }) + + it('preserves current verdicts when minSamples is omitted', () => { + const verdict = evaluateGate( + report({ aggregate: aggregate({ scoredCount: 2 }) }), + policy([{ scorer: 'exact', metric: 'avg', min: 0.5 }]), + ) + + expect(verdict.pass).toBe(true) + expect(verdict.failures).toEqual([]) + }) }) describe('evaluateGate baseline regression', () => { @@ -203,6 +325,67 @@ describe('evaluateGate baseline regression', () => { expect(verdict.warnings).toHaveLength(1) expect(verdict.warnings[0]).toContain('Scorer version drift') }) + + it('skips regression when either report is below minSamples', () => { + const thresholds = [ + { scorer: 'exact', metric: 'avg' as const, min: 0, minSamples: 5 }, + ] + + const currentBelow = evaluateGate( + report({ aggregate: aggregate({ scoredCount: 2, avg: 0.5 }) }), + policy(thresholds, { baseline: { maxRegression: 0 } }), + report({ aggregate: aggregate({ scoredCount: 5, avg: 1 }) }), + ) + const baselineBelow = evaluateGate( + report({ aggregate: aggregate({ scoredCount: 5, avg: 0.5 }) }), + policy(thresholds, { baseline: { maxRegression: 0 } }), + report({ aggregate: aggregate({ scoredCount: 2, avg: 1 }) }), + ) + const sufficient = evaluateGate( + report({ aggregate: aggregate({ scoredCount: 5, avg: 0.5 }) }), + policy(thresholds, { baseline: { maxRegression: 0 } }), + report({ aggregate: aggregate({ scoredCount: 5, avg: 1 }) }), + ) + + expect(currentBelow.failures).toEqual([ + expect.objectContaining({ kind: 'insufficient_samples', actual: 2, limit: 5 }), + ]) + expect(currentBelow.warnings).toHaveLength(1) + expect(currentBelow.warnings[0]).toContain('current aggregate has 2 samples') + expect(baselineBelow.pass).toBe(true) + expect(baselineBelow.failures).toEqual([]) + expect(baselineBelow.warnings).toHaveLength(1) + expect(baselineBelow.warnings[0]).toContain('baseline aggregate has 2 samples') + expect(sufficient.failures).toEqual([ + expect.objectContaining({ kind: 'regression', actual: 0.5, limit: 0 }), + ]) + }) + + it('treats a baseline without passSampleCount as insufficient for passRate', () => { + const oldBaseline: ScorerAggregate = { + scorer: { name: 'exact', version: '1' }, + scoredCount: 10, + errorCount: 0, + avg: 1, + p50: 1, + p95: 1, + min: 1, + max: 1, + passRate: 1, + } + const verdict = evaluateGate( + report({ aggregate: aggregate({ scoredCount: 10, passRate: 0.5, passSampleCount: 10 }) }), + policy([ + { scorer: 'exact', metric: 'passRate', min: 0, minSamples: 5 }, + ], { baseline: { maxRegression: 0 } }), + report({ aggregate: oldBaseline }), + ) + + expect(verdict.pass).toBe(true) + expect(verdict.failures).toEqual([]) + expect(verdict.warnings).toHaveLength(1) + expect(verdict.warnings[0]).toContain('baseline aggregate has 0 samples') + }) }) describe('evaluateGate health and warnings', () => { diff --git a/packages/core/tests/eval-runner.test.ts b/packages/core/tests/eval-runner.test.ts index 88c21f0..64ee0ff 100644 --- a/packages/core/tests/eval-runner.test.ts +++ b/packages/core/tests/eval-runner.test.ts @@ -250,9 +250,20 @@ describe('runEvalSet', () => { min: 0.2, max: 1, passRate: 0.5, + passSampleCount: 2, + }) + expect(aggregate.byTag?.['single']).toMatchObject({ + p50: 0.2, + p95: 0.2, + scoredCount: 1, + passSampleCount: 1, + }) + expect(aggregate.byTag?.['pair']).toMatchObject({ + p50: 0.2, + p95: 0.8, + scoredCount: 2, + passSampleCount: 2, }) - expect(aggregate.byTag?.['single']).toMatchObject({ p50: 0.2, p95: 0.2, scoredCount: 1 }) - expect(aggregate.byTag?.['pair']).toMatchObject({ p50: 0.2, p95: 0.8, scoredCount: 2 }) }) it('links StoredRun traces and run identity into scorer context and records', async () => {