From ca1b2f8608c52ed721d3576ba5eeb1d66717b08c Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Wed, 14 Oct 2026 22:56:25 +0800 Subject: [PATCH] test(perf): measure real sigrok annotation selection workflows Pin two public sigrok recordings and independently verify raw waveform bytes/sample locations through real CLI/MCP selection workflows. Record all eight actual Codex task costs, including a fixed-plan deviation and mixed whole-task token/time outcomes. Existing annotation controls reduce reply bytes; the cohort does not justify a new production cache/default/API change. Harden corpus identity, UART data-bit coverage, I2C interval ownership and preallocation ZIP/metadata admission. Focused tests and original Evidence/transcript replays retain observed facts, outcomes, provenance and resource limitations. Fixes #3131. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- experiments/analysis-efficiency/README.md | 6 + .../analysis-efficiency/agent-trial.py | 1 + experiments/analysis-efficiency/compare.mjs | 12 +- experiments/analysis-efficiency/corpus.json | 30 ++ experiments/analysis-efficiency/measure.mjs | 24 ++ .../analysis-efficiency/review-signal.mjs | 237 ++++++++++++++++ .../analysis-efficiency/signal-entry.mjs | 264 ++++++++++++++++++ .../signal-model-prompt.mjs | 22 ++ .../analysis-efficiency/signal-oracle.py | 211 ++++++++++++++ .../signal-selection-workflows.md | 188 +++++++++++++ .../analysis-efficiency/test_signal_oracle.py | 149 ++++++++++ .../analysisEfficiencyComparison.test.ts | 34 ++- .../signalWorkflowSemantics.test.ts | 46 +++ 13 files changed, 1217 insertions(+), 7 deletions(-) create mode 100644 experiments/analysis-efficiency/review-signal.mjs create mode 100644 experiments/analysis-efficiency/signal-entry.mjs create mode 100644 experiments/analysis-efficiency/signal-model-prompt.mjs create mode 100644 experiments/analysis-efficiency/signal-oracle.py create mode 100644 experiments/analysis-efficiency/signal-selection-workflows.md create mode 100644 experiments/analysis-efficiency/test_signal_oracle.py create mode 100644 tests/evaluation/signalWorkflowSemantics.test.ts diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.md index 9a14b764a..2ca9da3a4 100644 --- a/experiments/analysis-efficiency/README.md +++ b/experiments/analysis-efficiency/README.md @@ -78,6 +78,12 @@ field/filter selection, discovery and output windows. Lower delivered JSON did not establish stable total-token or speed gains; failed plans and negative controls remain visible. +The [sigrok selection checkpoint](signal-selection-workflows.md) records two +real hardware recordings, independently verified bytes, repeated decoder/pages, +and an eight-task Codex cohort with mixed token/time results. Existing annotation +controls reduce replies; these measurements introduce no production cache, +default or API changes. + Agent token consumption and end-to-end task time are the primary objectives. Memory and CPU consumption are also measured and controlled. Correctness, Evidence provenance, caller choices and existing-user compatibility are diff --git a/experiments/analysis-efficiency/agent-trial.py b/experiments/analysis-efficiency/agent-trial.py index db994cea1..365337a06 100644 --- a/experiments/analysis-efficiency/agent-trial.py +++ b/experiments/analysis-efficiency/agent-trial.py @@ -182,6 +182,7 @@ def main(): 'inspect_firmware_regions', 'inspect_managed_artifact', 'inspect_sqlite_database', 'inspect_pcap_capture', 'inspect_packet_capture', 'inspect_packet_fields', + 'decode_signal_recording', 'inspect_signal_decoders', 'inspect_managed_members', 'inspect_managed_native_boundaries', 'inspect_artifact', 'inspect_wasm_artifact', 'inspect_evm_interface', 'get_evidence_bundle', 'export_evidence_bundle', diff --git a/experiments/analysis-efficiency/compare.mjs b/experiments/analysis-efficiency/compare.mjs index 01b9f97af..e1e9f4cda 100644 --- a/experiments/analysis-efficiency/compare.mjs +++ b/experiments/analysis-efficiency/compare.mjs @@ -3,6 +3,7 @@ import { readFile, writeFile } from "node:fs/promises"; import { resolve } from "node:path"; import { parseArgs } from "node:util"; import { PACKET_ARTIFACTS } from "./packet-entry.mjs"; +import { SIGNAL_ARTIFACTS } from "./signal-entry.mjs"; const { values } = parseArgs({ options: { @@ -36,6 +37,8 @@ const sameSettings = [ "native_workflow_source_sha256", "packet_workflow_source_sha256", "packet_oracle_source_sha256", + "signal_workflow_source_sha256", + "signal_oracle_source_sha256", "package_lock_sha256", "node_version", "sdk_client_version", @@ -73,13 +76,14 @@ for (const [side, answer] of [ value: answer.evidence_ids, }); } -// PCAP delivery observations preserve each actual receipt's identity, limitations +// Recording delivery observations preserve each actual receipt's identity, limitations // and view coverage. Summary/page scopes differ from complete source scope; // retain them separately rather than equating page exhaustion with capture // coverage. Repeated equivalent deliveries must also match every observation // field except the receipt and parent IDs, which identify their own invocations. if ( Object.hasOwn(PACKET_ARTIFACTS, left.artifact) || + Object.hasOwn(SIGNAL_ARTIFACTS, left.artifact) || ["wireshark-http.cap", "wireshark-http_redirects.pcapng"].includes( left.artifact, ) @@ -92,7 +96,7 @@ if ( !Array.isArray(answer.delivery_observations) || !answer.delivery_observations.length ) - throw new Error("Missing original PCAP delivery observations"); + throw new Error("Missing original recording delivery observations"); runSpecific.push({ side, path: ["delivery_observations"], @@ -101,7 +105,9 @@ if ( if ( left.delivery === right.delivery && (!Object.hasOwn(PACKET_ARTIFACTS, left.artifact) || - left.packet_selection === right.packet_selection) + left.packet_selection === right.packet_selection) && + (!Object.hasOwn(SIGNAL_ARTIFACTS, left.artifact) || + left.signal_selection === right.signal_selection) ) { answer.delivery_observations = answer.delivery_observations.map( (observation) => diff --git a/experiments/analysis-efficiency/corpus.json b/experiments/analysis-efficiency/corpus.json index e595dde34..983b81799 100644 --- a/experiments/analysis-efficiency/corpus.json +++ b/experiments/analysis-efficiency/corpus.json @@ -343,6 +343,36 @@ "derivation": "Unmodified official saved sample; decompressed gzip where indicated. Local digest pins acquired bytes, not an upstream-published digest.", "compressed_source": null } + }, + { + "id": "sigrok-uart-max3232e.sr", + "file": "uart-max3232e.sr", + "bytes": 12742, + "sha256": "f797d60d6b8e12d2a1549b9a5a499ef389f44cb9e870e8610af26b57c3b2be76", + "provenance": { + "project": "sigrok-dumps", + "version": "0ad13477abc959d37fc9a5acbd23901c371c9c76", + "upstreamFile": "uart/maxim_max3232e/max3232e_hello_world_57600_8n1.sr", + "url": "https://raw.githubusercontent.com/sigrokproject/sigrok-dumps/0ad13477abc959d37fc9a5acbd23901c371c9c76/uart/maxim_max3232e/max3232e_hello_world_57600_8n1.sr", + "git_blob_sha1": "52e47b3014819471a2f348a0d8c7694fe304e052", + "reference": "https://raw.githubusercontent.com/sigrokproject/sigrok-dumps/0ad13477abc959d37fc9a5acbd23901c371c9c76/uart/maxim_max3232e/README", + "license_scope": "Repository README declares dumps public domain unless otherwise noted; selected READMEs state no exception; decoder software licensing is separate." + } + }, + { + "id": "sigrok-i2c-nunchuk.sr", + "file": "i2c-nunchuk.sr", + "bytes": 20373, + "sha256": "0fa6adb8297d7f3fe5ba19c94ee655c9b0f952a8b5e48a17aaa499a2f6d10d70", + "provenance": { + "project": "sigrok-dumps", + "version": "0ad13477abc959d37fc9a5acbd23901c371c9c76", + "upstreamFile": "i2c/wii_nunchuk/wii_nunchuk_init_reg_3xdata.sr", + "url": "https://raw.githubusercontent.com/sigrokproject/sigrok-dumps/0ad13477abc959d37fc9a5acbd23901c371c9c76/i2c/wii_nunchuk/wii_nunchuk_init_reg_3xdata.sr", + "git_blob_sha1": "a7232fce7b54e68e2be44e15946c90a3c756e8e9", + "reference": "https://raw.githubusercontent.com/sigrokproject/sigrok-dumps/0ad13477abc959d37fc9a5acbd23901c371c9c76/i2c/wii_nunchuk/README", + "license_scope": "Repository README declares dumps public domain unless otherwise noted; selected READMEs state no exception; decoder software licensing is separate." + } } ] } diff --git a/experiments/analysis-efficiency/measure.mjs b/experiments/analysis-efficiency/measure.mjs index 07a39e3c1..20f1e7d61 100644 --- a/experiments/analysis-efficiency/measure.mjs +++ b/experiments/analysis-efficiency/measure.mjs @@ -13,6 +13,7 @@ import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { Ajv2020 } from "ajv/dist/2020.js"; import { PACKET_ARTIFACTS, measurePacketEntry } from "./packet-entry.mjs"; +import { SIGNAL_ARTIFACTS, measureSignalEntry } from "./signal-entry.mjs"; const root = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); const { values } = parseArgs({ @@ -31,6 +32,7 @@ const { values } = parseArgs({ schemas: { type: "string", default: "default" }, "sqlite-selection": { type: "string", default: "all" }, "packet-selection": { type: "string", default: "baseline" }, + "signal-selection": { type: "string", default: "baseline" }, }, }); if ( @@ -41,6 +43,13 @@ if ( !isAbsolute(values.path) || !isAbsolute(values.output) || !["complete", "summary"].includes(values.delivery) || + !["baseline", "focused", "discovery", "pages"].includes( + values["signal-selection"], + ) || + (!Object.hasOwn(SIGNAL_ARTIFACTS, values.artifact) && + values["signal-selection"] !== "baseline") || + (Object.hasOwn(SIGNAL_ARTIFACTS, values.artifact) && + values.delivery !== "complete") || !["entry", "entry-and-main", "entry-views"].includes( values["native-question"], ) || @@ -159,6 +168,12 @@ const fingerprints = { packet_oracle_source_sha256: await fileSha( fileURLToPath(new URL("./packet-oracle.py", import.meta.url)), ), + signal_workflow_source_sha256: await fileSha( + fileURLToPath(new URL("./signal-entry.mjs", import.meta.url)), + ), + signal_oracle_source_sha256: await fileSha( + fileURLToPath(new URL("./signal-oracle.py", import.meta.url)), + ), runtime_files_sha256: sha(JSON.stringify(runtimeFiles)), runtime_dispatcher_sha256: await fileSha( resolve(values["runtime-root"], "scripts/rea.mjs"), @@ -739,6 +754,12 @@ const inspectPcapTarget = async () => { }; const workflows = { + ...Object.fromEntries( + Object.keys(SIGNAL_ARTIFACTS).map((artifact) => [ + artifact, + () => measureSignalEntry({ values, call, inputSha, catalogDefinitions }), + ]), + ), ...Object.fromEntries( Object.keys(PACKET_ARTIFACTS).map((artifact) => [ artifact, @@ -1186,6 +1207,9 @@ const report = { packet_selection: Object.hasOwn(PACKET_ARTIFACTS, values.artifact) ? values["packet-selection"] : null, + signal_selection: Object.hasOwn(SIGNAL_ARTIFACTS, values.artifact) + ? values["signal-selection"] + : null, provider_settings_sha256: sha(JSON.stringify(selectedProviders)), provider_search_path_sha256: sha( selectedProviders.PATH ?? process.env.PATH ?? "", diff --git a/experiments/analysis-efficiency/review-signal.mjs b/experiments/analysis-efficiency/review-signal.mjs new file mode 100644 index 000000000..9ae454207 --- /dev/null +++ b/experiments/analysis-efficiency/review-signal.mjs @@ -0,0 +1,237 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { isDeepStrictEqual, parseArgs } from "node:util"; +import { + SIGNAL_ARTIFACTS, + signalRequest, + signalDataRows, + signalFacts, + verifySignalRows, + verifySignalOracleIdentity, +} from "./signal-entry.mjs"; + +const { values } = parseArgs({ + options: { + trial: { type: "string" }, + oracle: { type: "string" }, + "runtime-root": { type: "string" }, + artifact: { type: "string" }, + selection: { type: "string" }, + }, +}); +if ( + !values.trial || + !values.oracle || + !values["runtime-root"] || + !Object.hasOwn(SIGNAL_ARTIFACTS, values.artifact ?? "") || + !["baseline", "focused"].includes(values.selection) +) + throw new Error( + "Use --trial DIR --oracle JSON --runtime-root DIR --artifact ID --selection baseline|focused", + ); +const json = async (name) => + JSON.parse(await readFile(resolve(values.trial, name), "utf8")); +const settings = await json("trial-settings.json"), + summary = await json("summary.json"); +const oracle = JSON.parse(await readFile(values.oracle, "utf8")); +const kind = SIGNAL_ARTIFACTS[values.artifact].kind; +const expectedRows = + kind === "uart" + ? oracle.rows + : oracle.reads.flatMap((read) => read.bytes.slice(1)); +const expected = signalFacts(kind, expectedRows); +const runtime = (file) => + pathToFileURL(resolve(values["runtime-root"], file)).href; +const { parseEvidence } = await import(runtime("dist/domain/evidence.js")); +const { decodeSignalRecordingInputSchema, signalRecordingSchema } = + await import(runtime("dist/domain/signals/signalRecording.js")); +const { inspectSignalDecodersInputSchema, signalDecodersSchema } = await import( + runtime("dist/domain/signals/signalDecoders.js") +); +const calls = (await readFile(resolve(values.trial, "events.jsonl"), "utf8")) + .split("\n") + .filter(Boolean) + .map(JSON.parse) + .filter( + (event) => + event.type === "item.completed" && event.item?.type === "mcp_tool_call", + ) + .map((event) => event.item); +const reasons = [], + workflowReasons = [], + authenticated = new Map(), + conflicting = new Set(); +try { + verifySignalOracleIdentity(values.artifact, oracle); + if (oracle.kind !== kind) + throw new Error("Wrong independent signal question oracle"); + if (calls.length !== 1 || calls[0]?.tool !== "decode_signal_recording") + workflowReasons.push( + "Completed public call sequence differs from the fixed single-call plan", + ); + const planned = decodeSignalRecordingInputSchema.parse( + signalRequest( + values.artifact, + settings.selected_artifact_path, + values.selection, + ), + ); + for (const call of calls) { + try { + if ( + call.server !== "rea" || + call.status !== "completed" || + call.error || + call.result?.isError + ) + throw new Error("Call did not return successful REA observations"); + const text = + call.result?.content?.filter((item) => item.type === "text") ?? []; + if (!call.result?.structuredContent && text.length !== 1) + throw new Error("No unambiguous Evidence envelope"); + const evidence = + call.result.structuredContent ?? JSON.parse(text[0].text); + parseEvidence(evidence); + const capture = call.tool === "decode_signal_recording"; + if (!capture && call.tool !== "inspect_signal_decoders") + throw new Error("Not a signal observation tool"); + const input = ( + capture + ? decodeSignalRecordingInputSchema + : inspectSignalDecodersInputSchema + ).parse(call.arguments); + const report = ( + capture ? signalRecordingSchema : signalDecodersSchema + ).parse(evidence.normalized_result); + if ( + evidence.operation !== call.tool || + !isDeepStrictEqual(evidence.parameters, input) || + !isDeepStrictEqual(report.selection, input) + ) + throw new Error( + "Evidence differs from actual operation or caller choices", + ); + if ( + capture + ? evidence.subject?.local_path !== input.path || + evidence.subject?.digest?.sha256 !== oracle.sha256 || + report.artifact.path !== input.path || + report.artifact.sha256 !== oracle.sha256 + : evidence.subject?.local_path !== report.tool.path || + evidence.subject?.digest?.sha256 !== report.tool.sha256 + ) + throw new Error("Evidence identifies different recording/tool bytes"); + if (capture && !isDeepStrictEqual(input, planned)) + workflowReasons.push("Capture selection differs from the fixed plan"); + const observation = { + operation: call.tool, + input, + report, + rows: capture ? signalDataRows(kind, report.annotations_text) : null, + }; + const id = evidence.evidence_id; + if (conflicting.has(id)) + throw new Error("Previously conflicting Evidence identity"); + if ( + authenticated.has(id) && + !isDeepStrictEqual(authenticated.get(id), observation) + ) { + authenticated.delete(id); + conflicting.add(id); + throw new Error( + "Repeated Evidence identity has inconsistent observations", + ); + } + authenticated.set(id, observation); + } catch (error) { + workflowReasons.push( + `Call ${call.tool} did not provide authenticated observations: ${String(error)}`, + ); + } + } + const answer = JSON.parse( + (await readFile(resolve(values.trial, "answer.txt"), "utf8")).trim(), + ); + if ( + !isDeepStrictEqual( + Object.keys(answer).sort(), + [...Object.keys(expected), "evidence_ids", "limitations"].sort(), + ) + ) + workflowReasons.push("Final answer shape differs from the fixed plan"); + if ( + !isDeepStrictEqual( + Object.fromEntries( + Object.keys(expected).map((key) => [key, answer[key]]), + ), + expected, + ) + ) + throw new Error("Final answer differs from independent sample facts"); + if ( + !Array.isArray(answer.evidence_ids) || + new Set(answer.evidence_ids).size !== answer.evidence_ids.length || + answer.evidence_ids.some((id) => !authenticated.has(id)) + ) + throw new Error( + "Final citations are not authenticated signal observations", + ); + const rows = new Map(); + for (const id of answer.evidence_ids) { + for (const row of authenticated.get(id).rows ?? []) { + const old = rows.get(row.start); + if (old && !isDeepStrictEqual(old, row)) + throw new Error("Cited sample ranges have conflicting decoded data"); + rows.set(row.start, row); + } + } + verifySignalRows( + kind, + [...rows.values()].sort((a, b) => a.start - b.start), + oracle, + oracle.sha256, + ); + const limitations = + typeof answer.limitations === "string" + ? [answer.limitations] + : answer.limitations; + if ( + !Array.isArray(limitations) || + limitations.length === 0 || + limitations.some((value) => typeof value !== "string" || !value.trim()) + ) + workflowReasons.push("Final answer omits investigation limitations"); + if ( + summary.exit_code !== 0 || + summary.timeout || + summary.selected_artifact_bytes_unchanged !== true || + settings.selected_artifact_sha256_before !== oracle.sha256 || + summary.selected_artifact_sha256_after !== oracle.sha256 + ) + workflowReasons.push( + "Trial did not complete with original selected bytes unchanged", + ); +} catch (error) { + reasons.push(String(error)); +} +const report = { + factual_correctness: reasons.length ? "failed" : "passed", + workflow_correctness: + reasons.length || workflowReasons.length ? "failed" : "passed", + reasons, + workflow_reasons: workflowReasons, + completed_calls: calls.length, + wall_seconds: summary.wall_seconds, + usage: summary.usage, + selected_artifact_sha256: oracle.sha256, + scope: + "Two pinned single-device recordings; independently sampled bytes and authenticated cited decoder observations; hardware behavior is not inferred", + limitation_prose: "Presence checked; wording requires transcript review", +}; +await writeFile( + resolve(values.trial, "signal-review.json"), + JSON.stringify(report, null, 2) + "\n", +); +console.log(JSON.stringify(report)); +if (reasons.length || workflowReasons.length) process.exitCode = 1; diff --git a/experiments/analysis-efficiency/signal-entry.mjs b/experiments/analysis-efficiency/signal-entry.mjs new file mode 100644 index 000000000..b44d4d66a --- /dev/null +++ b/experiments/analysis-efficiency/signal-entry.mjs @@ -0,0 +1,264 @@ +import { execFile } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { isDeepStrictEqual, promisify } from "node:util"; + +const corpus = JSON.parse( + await readFile(new URL("./corpus.json", import.meta.url), "utf8"), +); + +export const SIGNAL_ARTIFACTS = { + "sigrok-uart-max3232e.sr": { + kind: "uart", + decoders: "uart:rx=MAX3232E DOUT1:baudrate=57600:format=hex:invert_rx=yes", + annotations: "uart=rx-data", + query: "rx-data", + firstPage: 32, + }, + "sigrok-i2c-nunchuk.sr": { + kind: "i2c", + decoders: "i2c:scl=SCL:sda=SDA", + annotations: "i2c=data-read", + query: "data-read", + firstPage: 9, + }, +}; + +export function signalRequest(artifact, path, selection) { + const descriptor = SIGNAL_ARTIFACTS[artifact]; + if (!descriptor) throw new Error("Unknown fixed signal question"); + return { + path, + decoders: descriptor.decoders, + input_format: null, + annotations: selection === "baseline" ? null : descriptor.annotations, + line_offset: 0, + line_limit: selection === "pages" ? descriptor.firstPage : null, + }; +} + +export function signalDataRows(kind, text) { + const pattern = + kind === "uart" + ? /^(\d+)-(\d+) uart-1: ([0-9A-F]{2})$/ + : /^(\d+)-(\d+) i2c-1: Data read: ([0-9A-F]{2})$/; + const rows = []; + for (const line of text.split("\n")) { + const match = pattern.exec(line); + if (!match) continue; + const start = Number(match[1]), + end = Number(match[2]); + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(end) || + start > end + ) + throw new Error("Unsupported decoded sample interval"); + rows.push({ start, end, byte: Number.parseInt(match[3], 16) }); + } + return rows; +} + +export function signalFacts(kind, rows) { + if (kind === "uart") return { kind, bytes: rows.map((row) => row.byte) }; + if (kind !== "i2c" || rows.length !== 18) + throw new Error( + "The fixed I2C question requires three complete six-byte reads", + ); + return { + kind, + reads: [0, 6, 12].map((offset) => + rows.slice(offset, offset + 6).map((row) => row.byte), + ), + }; +} + +export function verifySignalOracleIdentity(artifact, oracle) { + const pinned = corpus.artifacts.find((entry) => entry.id === artifact); + if ( + !pinned || + oracle.kind !== SIGNAL_ARTIFACTS[artifact]?.kind || + oracle.sha256 !== pinned.sha256 + ) + throw new Error( + "Independent oracle does not identify the pinned signal corpus", + ); +} + +export function verifySignalRows(kind, rows, oracle, digest) { + const expected = + kind === "uart" + ? oracle.rows + : oracle.reads.flatMap((read) => + read.bytes.slice(1).map((byte, index) => ({ + ...byte, + after_previous: read.bytes[index].last_clock, + before_next: read.bytes[index + 2]?.first_clock ?? read.stop, + })), + ); + if ( + oracle.kind !== kind || + oracle.sha256 !== digest || + rows.length !== expected.length + ) + throw new Error( + "Signal observations differ from independent sample identity/coverage", + ); + for (let index = 0; index < rows.length; index++) { + const observed = rows[index], + raw = expected[index]; + if ( + observed.byte !== raw.byte || + observed.start < 0 || + observed.end >= oracle.total_samples + ) + throw new Error("Decoded data differs from independent sample bits"); + if ( + kind === "uart" + ? !Number.isSafeInteger(raw.first_data_sample) || + !Number.isSafeInteger(raw.last_data_sample) || + observed.start < raw.start || + observed.end >= raw.end_exclusive || + observed.start > raw.first_data_sample || + observed.end < raw.last_data_sample + : !Number.isSafeInteger(raw.after_previous) || + !Number.isSafeInteger(raw.before_next) || + observed.start <= raw.after_previous || + observed.end >= raw.before_next || + observed.start > raw.first_clock || + observed.end < raw.last_clock + ) + throw new Error( + "Decoded interval does not identify the independently read byte", + ); + } +} + +export async function measureSignalEntry({ + values, + call, + inputSha, + catalogDefinitions, +}) { + const descriptor = SIGNAL_ARTIFACTS[values.artifact]; + const runtime = (file) => + pathToFileURL(resolve(values["runtime-root"], file)).href; + const { parseEvidence } = await import(runtime("dist/domain/evidence.js")); + const { decodeSignalRecordingInputSchema, signalRecordingSchema } = + await import(runtime("dist/domain/signals/signalRecording.js")); + const { inspectSignalDecodersInputSchema, signalDecodersSchema } = + await import(runtime("dist/domain/signals/signalDecoders.js")); + const observations = []; + const checked = (evidence, operation, input, schema) => { + parseEvidence(evidence); + const report = schema.parse(evidence.normalized_result); + if ( + evidence.operation !== operation || + !isDeepStrictEqual(evidence.parameters, input) || + !isDeepStrictEqual(report.selection, input) + ) + throw new Error( + "Signal Evidence differs from the actual requested operation or choices", + ); + if ( + operation === "decode_signal_recording" + ? evidence.subject?.local_path !== values.path || + evidence.subject?.digest?.sha256 !== inputSha || + report.artifact.path !== values.path || + report.artifact.sha256 !== inputSha + : evidence.subject?.local_path !== report.tool.path || + evidence.subject?.digest?.sha256 !== report.tool.sha256 + ) + throw new Error( + "Signal Evidence identifies different recording/tool bytes", + ); + observations.push({ + evidence_id: evidence.evidence_id, + operation, + selection: report.selection, + output_window: report.output_window, + limitations: evidence.limitations, + }); + return report; + }; + const selection = values["signal-selection"]; + for (const name of [ + "decode_signal_recording", + ...(selection === "discovery" ? ["inspect_signal_decoders"] : []), + ]) + if (!catalogDefinitions.has(name)) + throw new Error(`Missing live signal contract ${name}`); + if (selection === "discovery") { + const input = inspectSignalDecodersInputSchema.parse({ + decoder: descriptor.kind, + query: descriptor.query, + limit: 5, + }); + const report = checked( + await call("inspect_signal_decoders", input), + "inspect_signal_decoders", + input, + signalDecodersSchema, + ); + if (!report.help_lines.some((line) => line.text.includes(descriptor.query))) + throw new Error( + "Installed decoder help does not establish the selected annotation class", + ); + } + const input = decodeSignalRecordingInputSchema.parse( + signalRequest(values.artifact, values.path, selection), + ); + const first = checked( + await call("decode_signal_recording", input), + "decode_signal_recording", + input, + signalRecordingSchema, + ); + let rows = signalDataRows(descriptor.kind, first.annotations_text); + if (selection === "pages") { + const nextInput = decodeSignalRecordingInputSchema.parse({ + ...input, + line_offset: descriptor.firstPage, + line_limit: null, + }); + const next = checked( + await call("decode_signal_recording", nextInput), + "decode_signal_recording", + nextInput, + signalRecordingSchema, + ); + const repeated = checked( + await call("decode_signal_recording", input), + "decode_signal_recording", + input, + signalRecordingSchema, + ); + if ( + first.annotations_text !== repeated.annotations_text || + !isDeepStrictEqual(first.output_window, repeated.output_window) || + first.output_window.total_lines !== next.output_window.total_lines + ) + throw new Error("Repeated/next display pages changed observation scope"); + rows = [...rows, ...signalDataRows(descriptor.kind, next.annotations_text)]; + } + const { stdout } = await promisify(execFile)( + "python3", + [ + fileURLToPath(new URL("./signal-oracle.py", import.meta.url)), + descriptor.kind, + values.path, + ...(descriptor.kind === "uart" ? ["--invert"] : []), + ], + { timeout: 30_000, maxBuffer: 256 * 1024 }, + ); + const oracle = JSON.parse(stdout); + verifySignalOracleIdentity(values.artifact, oracle); + verifySignalRows(descriptor.kind, rows, oracle, inputSha); + return { + artifact_sha256: inputSha, + facts: signalFacts(descriptor.kind, rows), + delivery_observations: observations, + evidence_ids: [...new Set(observations.map((item) => item.evidence_id))], + }; +} diff --git a/experiments/analysis-efficiency/signal-model-prompt.mjs b/experiments/analysis-efficiency/signal-model-prompt.mjs new file mode 100644 index 000000000..64f87fee1 --- /dev/null +++ b/experiments/analysis-efficiency/signal-model-prompt.mjs @@ -0,0 +1,22 @@ +import { parseArgs } from "node:util"; +import { signalRequest, SIGNAL_ARTIFACTS } from "./signal-entry.mjs"; + +const { values } = parseArgs({ + options: { artifact: { type: "string" }, selection: { type: "string" } }, +}); +if ( + !Object.hasOwn(SIGNAL_ARTIFACTS, values.artifact ?? "") || + !["baseline", "focused"].includes(values.selection) +) + throw new Error( + "Use --artifact fixed-signal-ID --selection baseline|focused", + ); +const kind = SIGNAL_ARTIFACTS[values.artifact].kind; +const question = + kind === "uart" + ? "Report every decoded RX data octet on the specified DOUT1 channel, in recording order, as decimal integer bytes. Include all repetitions; do not deduplicate data." + : "Report every I2C READ data payload in recording order, grouped into consecutive six-byte payloads. Exclude address bytes and write data. Do not infer button or hardware behavior."; +console.log(`${question} +Use the configured REA MCP and make exactly one decode_signal_recording call with this selection: +${JSON.stringify(signalRequest(values.artifact, "{{artifact_path}}", values.selection))} +Return one JSON object with exactly these keys: kind (${JSON.stringify(kind)}), ${kind === "uart" ? "bytes (an array of decimal octets)" : "reads (an array of six-octet decimal arrays)"}, evidence_ids (unique actual returned Evidence IDs), limitations (nonempty investigation limitations). Preserve source identity and recorded order. Output lines are not sample or annotation counts; selected decoder observations do not prove complete hardware traffic or original application behavior. Do not run shell commands, browse, acquire hardware samples, invent missing data, or fabricate citations.`); diff --git a/experiments/analysis-efficiency/signal-oracle.py b/experiments/analysis-efficiency/signal-oracle.py new file mode 100644 index 000000000..dbccf01a2 --- /dev/null +++ b/experiments/analysis-efficiency/signal-oracle.py @@ -0,0 +1,211 @@ +"""Independent readers for the two pinned, single-device digital SR recordings.""" + +import argparse +from bisect import bisect_right +import configparser +import hashlib +import io +import json +from pathlib import Path +import re +import struct +import zipfile + + +def read_edges(path, channels): + size = path.stat().st_size + if size > 32 * 1024 * 1024: + raise ValueError('Compressed SR input exceeds the experiment budget') + with path.open('rb') as source: + admitted = source.read(size + 1) + if len(admitted) != size: + raise ValueError('SR input size changed during admission') + marker = admitted.rfind(b'PK\x05\x06', max(0, len(admitted) - 65557)) + if marker < 0 or len(admitted) - marker < 22: + raise ValueError('Missing classic ZIP directory receipt') + header = struct.unpack_from('<4s4H2IH', admitted, marker) + if (marker >= 20 and admitted[marker-20:marker-16] == b'PK\x06\x07'): + raise ValueError('ZIP64 directory is unsupported') + if header[1] or header[2] or header[3] != header[4] or not 1 <= header[4] <= 64 or header[5] > 65536 or marker + 22 + header[7] != len(admitted) or header[6] + header[5] != marker: + raise ValueError('Unsupported or excessive ZIP directory representation') + directory = memoryview(admitted)[header[6]:marker] + position = count = 0 + while position < len(directory): + if count >= 64 or len(directory) - position < 46 or directory[position:position+4] != b'PK\x01\x02': + raise ValueError('Unsupported or excessive actual ZIP directory records') + name_size, extra_size, comment_size = struct.unpack_from('<3H', directory, position + 28) + end = position + 46 + name_size + extra_size + comment_size + if end > len(directory): + raise ValueError('Incomplete actual ZIP directory record') + extra = position + 46 + name_size + extra_end = extra + extra_size + while extra < extra_end: + if extra_end - extra < 4: + raise ValueError('Incomplete ZIP extra field') + tag, length = struct.unpack_from('<2H', directory, extra) + if tag == 1: + raise ValueError('ZIP64 member is unsupported') + extra += 4 + length + if extra > extra_end: + raise ValueError('Incomplete ZIP extra field') + count += 1 + position = end + if count != header[4]: + raise ValueError('Declared and actual ZIP directory counts differ') + before = hashlib.sha256(admitted).hexdigest() + # Admission and ZipFile parse the same bounded snapshot, not a reopened path. + with zipfile.ZipFile(io.BytesIO(admitted), allowZip64=False) as archive: + entries = archive.infolist() + names = [item.filename for item in entries] + if len(names) != len(set(names)) or len(names) > 64: + raise ValueError("Ambiguous or excessive SR members") + if sum(item.file_size for item in entries) > 32 * 1024 * 1024: + raise ValueError("Expanded SR input exceeds the experiment budget") + if archive.getinfo('metadata').file_size > 8192: + raise ValueError("Excessive SR metadata") + # Declared sizes are untrusted. A bounded read also caps each inflater + # step; logic members below likewise use bounded block reads. + with archive.open('metadata') as source: + metadata_bytes = source.read(8193) + if len(metadata_bytes) > 8192: + raise ValueError("Expanded SR metadata exceeds the experiment budget") + metadata = metadata_bytes.decode('utf-8') + config = configparser.ConfigParser(interpolation=None) + config.read_string(metadata) + if set(config.sections()) != {'global', 'device 1'}: + raise ValueError("Only the pinned single-device profile is supported") + device = config['device 1'] + unit = int(device['unitsize']) + probes = int(device['total probes']) + if unit not in [1, 2] or not 1 <= probes <= unit * 8: + raise ValueError("Unsupported digital sample geometry") + rate = re.fullmatch(r'(\d+)\s+(Hz|kHz|MHz)', device['samplerate']) + if rate is None or int(rate[1]) <= 0: + raise ValueError("Unsupported sample-rate representation") + samplerate = int(rate[1]) * {'Hz': 1, 'kHz': 1000, 'MHz': 1000000}[rate[2]] + bits = [] + for channel in channels: + matches = [index - 1 for index in range(1, probes + 1) + if device.get(f'probe{index}') == channel] + if len(matches) != 1: + raise ValueError("Channel identity is missing or ambiguous") + bits.append(matches[0]) + prefix = device['capturefile'] + chunks = [name for name in names if re.fullmatch(re.escape(prefix) + r'-\d+', name)] + if prefix in names and chunks: + raise ValueError("Ambiguous direct and chunked logic data") + if prefix in names: + chunks = [prefix] + else: + chunks.sort(key=lambda name: int(name.rsplit('-', 1)[1])) + if [int(name.rsplit('-', 1)[1]) for name in chunks] != list(range(1, len(chunks) + 1)): + raise ValueError("Missing or ambiguous logic chunk order") + if not chunks or any(archive.getinfo(name).file_size % unit for name in chunks): + raise ValueError("Missing or incomplete digital samples") + edges = [[] for _ in channels] + previous = None + sample = 0 + mask = sum(1 << bit for bit in bits) + for name in chunks: + with archive.open(name) as source: + while block := source.read(65536): + values = block if unit == 1 else (item[0] for item in struct.iter_unpack('> bit) & 1 + if not edges[index] or edges[index][-1][1] != level: + if sum(map(len, edges)) >= 65536: + raise ValueError('Selected transitions exceed the experiment representation budget') + edges[index].append((sample, level)) + previous = selected + sample += 1 + with path.open('rb') as source: + after = source.read(size + 1) + if len(after) != size or hashlib.sha256(after).hexdigest() != before: + raise ValueError("Recording bytes changed during the independent read") + return {'sha256': before, 'metadata': metadata, 'sample_rate_hz': samplerate, + 'total_samples': sample, 'channels': dict(zip(channels, bits))}, edges + + +def uart(path, invert): + metadata, traces = read_edges(path, ['MAX3232E DOUT1']) + edges = [(sample, level ^ invert) for sample, level in traces[0]] + positions = [sample for sample, _ in edges] + rate, baud = metadata['sample_rate_hz'], 57600 + def level(sample): + return edges[bisect_right(positions, sample) - 1][1] + rows = [] + busy_until = 0 + for index, (start, value) in enumerate(edges): + if index == 0 or value != 0 or start < busy_until: + continue + center = lambda halfbits: start + (halfbits * rate) // (2 * baud) + if center(19) >= metadata['total_samples']: + raise ValueError("Incomplete UART frame at the capture boundary") + if level(center(1)) != 0 or level(center(19)) != 1: + raise ValueError(f"UART framing differs from selected polarity at sample {start}") + byte = sum(level(center(3 + 2 * bit)) << bit for bit in range(8)) + # A UART receiver becomes ready after sampling the stop bit. Waiting + # a full nominal ten bits would miss a valid slightly faster sender. + rows.append({'start': start, 'end_exclusive': center(19) + 1, 'byte': byte, + 'first_data_sample': center(3), 'last_data_sample': center(17)}) + busy_until = center(19) + 1 + return metadata | {'kind': 'uart', 'baud_rate': baud, 'invert': bool(invert), + 'complete_frames': len(rows), 'rows': rows} + + +def i2c(path): + metadata, traces = read_edges(path, ['SCL', 'SDA']) + transitions = {} + for channel, edges in enumerate(traces): + for sample, value in edges: + transitions.setdefault(sample, {})[channel] = value + clock = data = None + active = None + bits = [] + transactions = [] + for sample, changes in sorted(transitions.items()): + next_clock, next_data = changes.get(0, clock), changes.get(1, data) + if clock is not None and data is not None: + if clock == next_clock == 1 and next_data != data: + if next_data == 0: + if active is not None: + raise ValueError("The pinned profile does not contain repeated START") + active = {'start': sample, 'bytes': []} + bits = [] + elif active is not None: + # SCL may rise with SDA low to prepare STOP; that final + # rise is not the first bit of a new data byte. + if len(bits) == 1 and bits[0][1] == 0: + bits = [] + if bits: + raise ValueError("Incomplete I2C byte before STOP") + active['stop'] = sample + transactions.append(active) + active = None + elif active is not None and clock == 0 and next_clock == 1: + if next_data != data: + raise ValueError("Ambiguous simultaneous clock/data transition") + bits.append((sample, next_data)) + if len(bits) == 9: + byte = sum(bit << (7-index) for index, (_, bit) in enumerate(bits[:8])) + active['bytes'].append({'byte': byte, 'first_clock': bits[0][0], + 'last_clock': bits[7][0], 'ack': bits[8][1] == 0}) + bits = [] + clock, data = next_clock, next_data + if active is not None or bits: + raise ValueError("Incomplete I2C transaction at capture end") + reads = [transaction for transaction in transactions + if transaction['bytes'] and transaction['bytes'][0]['byte'] & 1] + return metadata | {'kind': 'i2c', 'transactions': transactions, 'reads': reads} + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('kind', choices=['uart', 'i2c']) + parser.add_argument('path', type=Path) + parser.add_argument('--invert', action='store_true') + args = parser.parse_args() + print(json.dumps(uart(args.path, args.invert) if args.kind == 'uart' else i2c(args.path))) diff --git a/experiments/analysis-efficiency/signal-selection-workflows.md b/experiments/analysis-efficiency/signal-selection-workflows.md new file mode 100644 index 000000000..4cc216827 --- /dev/null +++ b/experiments/analysis-efficiency/signal-selection-workflows.md @@ -0,0 +1,188 @@ +# Sigrok selection and repeated-page checkpoint (#3131) + +This experiment preserves production APIs, defaults, caller selections and Evidence +semantics. Existing annotation selectors substantially reduce replies, but this +small real-model cohort does **not** establish a consistent reduction in total +input tokens or elapsed task time. No decoder cache or new production preset is +introduced. Complete necessary output stays available through existing explicit +null limits; original defaults remain 50 display lines. + +## Immutable corpus and independent questions + +The runtime is pinned to `ca7450beffff59325ce3bddc78e181693ce77177`, with Node +24.18.0, SDK 2.3.1 and Linux x86-64. Experiment driver code for model trials is +`23317a58d`; each SDK result separately records workflow/oracle source hashes, +runtime-file and lockfile digests, explicit provider settings and original bytes. +Later integration checks must remain separate from this cohort. + +Two official sigrok-dumps recordings are pinned to upstream commit +`0ad13477abc959d37fc9a5acbd23901c371c9c76` in [corpus.json](corpus.json). +Original Git blob IDs, SHA-256, sizes and immutable reference/download URLs are +recorded there. No sample binaries or raw output are committed. The upstream +repository declares dumps public domain unless otherwise noted; the inspected +sample READMEs state no exception. Decoder software licensing is separate. + +- MAX3232E UART: 12,742 compressed bytes, 10,000,000 logic bytes, 5,000,000 samples + at the recorded nominal 100 MHz. Stored DOUT1 is bit 1; DIN1 is bit 2. The exact + question requests all RX octets in order, including repetitions. Independent + 57,600-baud 8N1 stop-bit sampling with DOUT1 inversion yields 65 bytes: five + copies of `Hello world\r\n`. The source metadata and physical probe numbers + remain distinct from stored bit positions. +- Wii Nunchuk I2C: 20,373 compressed bytes, 20,000,000 samples at nominal 1 MHz. + The source README's physical probes 3/7 are repacked as stored named channels + SCL/SDA at bits 0/1. The question asks every read payload as consecutive + six-byte arrays, excluding address/write bytes and hardware/button inference. + Independent START/STOP, rising-clock, byte and ACK/NACK reading identifies seven + transactions and three read payloads, matching the upstream reference: + `75 7F 77 4F 82 3B`, `75 7F 75 44 82 34`, `75 7F 77 43 83 5D`. + +The independent reader accepts the scoped single-device classic-ZIP profile, +checks directory representation before object allocation, bounds compressed and +expanded input to 32 MiB, and limits selected transitions to 65,536. It preserves +source metadata/digest, chunk ordering and channel identity; incomplete samples, +ambiguous names, partial bytes and unsupported transactions fail explicitly. +Receiver rearming follows the verified UART stop sample, accommodating the +recording's slightly faster sender. I2C STOP preparation is distinct from data. +These checks do not establish other formats, decoders, OSes or physical acquisition. + +An initial DOUT1 decode without inversion exited successfully but produced wrong +bytes. Its cost/output is retained privately as configuration exploration, outside +eligible measurements. An independent sample oracle prevents equal-but-wrong +producer results from passing. Correct inversion was established before the cohort. + +## Actual CLI and SDK evidence + +sigrok-cli 0.7.2 reports libsigrok 0.5.2 and libsigrokdecode 0.5.3. Its executable +SHA-256 is `e1b1eb2f25edc17d24a78acbab16ff7e1dfdee74bb2cccb7ad00497ccc7a0c44`. +Selected installed UART/I2C decoder module bytes are fingerprinted privately. +The banner and executable digest do not fingerprint the complete installation; +modules and libraries remain trusted. Both actual CLI and MCP results match raw +sample bytes; data intervals correlate with the independently examined samples. + +Fourteen actual SDK workflows comprise two interleaved repetitions per recording +and arm (baseline/focused/discovery), plus first/next/repeated pages. All 22 calls +preserve source bytes and necessary decoded facts. Baseline returns all necessary +annotation output. Focused selects `uart=rx-data` or `i2c=data-read`, with explicit +null line limits because the task needs the entire selected payload. Discovery +first queries the installed decoder help for the already specified class; it does +not establish discovery benefits for unknown protocols. + +| Recording | Baseline bytes | Focused bytes | Discovery + focused bytes | First + next + repeat bytes | +| --------- | -------------: | ------------: | ------------------------: | --------------------------: | +| UART | 49,417 | 12,441 | 18,839 | 31,779 | +| I2C | 26,829 | 9,928 | 16,274 | 27,623 | + +Focused reply reductions are 74.8% and 63.0%. These are parsed SDK JSON bytes, +not wire bytes or model tokens. Baseline/focused RPC times (ms, repetitions 1/2) +are UART 2010/582 versus 1459/911, and I2C 2851/676 versus 2330/685; n=2, +with order/cold/warm variation, does not establish a general speedup. Pages retain +input/class/window coverage separately, rather than equating display exhaustion +with a complete protocol decode. Repeated first pages match their original text +and scope; reused content-based Evidence identities are valid. + +## Complete real-model task costs + +Eight actual Codex CLI 0.161.0 tasks requested `gpt-5.6-terra` / `high`, in isolated +profiles using this runtime's REA MCP. Both tools were enabled and preapproved +identically in all arms; shell, browser and extra agents were disabled. The model +alias is a requested identifier, not independently resolved backend identity. +Questions contain no expected payloads. Trial order is baseline 1, focused 1, +focused 2, baseline 2, with UART then I2C in each step. + +| Trial | Total input | Cached input | Uncached input (derived) | Output | Seconds | Completed tool calls | Facts / fixed plan | +| --------------- | ----------: | -----------: | -----------------------: | -----: | ------: | -------------------: | ------------------ | +| uart baseline 1 | 92,399 | 49,408 | 42,991 | 1,295 | 86.411 | 1 | passed / passed | +| uart focused 1 | 130,029 | 98,048 | 31,981 | 780 | 49.623 | 1 | passed / passed | +| uart focused 2 | 98,894 | 78,592 | 20,302 | 878 | 73.861 | 1 | passed / passed | +| uart baseline 2 | 90,395 | 48,384 | 42,011 | 926 | 43.974 | 1 | passed / passed | +| i2c baseline 1 | 172,162 | 121,344 | 50,818 | 1,194 | 36.330 | 2 | passed / failed | +| i2c focused 1 | 74,815 | 49,408 | 25,407 | 937 | 114.515 | 1 | passed / passed | +| i2c focused 2 | 74,684 | 48,384 | 26,300 | 668 | 73.877 | 1 | passed / passed | +| i2c baseline 2 | 86,241 | 49,408 | 36,833 | 778 | 41.084 | 1 | passed / passed | + +All eight answers match independent raw samples and cite authenticated decoder +observations. I2C baseline 1 also queried generic MCP resources before its single +required decode. The predeclared strict single-call grader marks its plan failed; +its actual 172,162 input tokens and 36.330 seconds remain recorded, separately +from the eligible fixed-plan pairs. It is not retried into a better result. + +UART eligible pairs increase total input by 40.7% and 9.4%; elapsed changes are +-42.6% and +68.0%. The one eligible I2C pair reduces total input by 13.4% but +increases elapsed time by 79.8%. Reduced uncached input does not replace total +input as the primary token measure. No reliable whole-task token/time benefit is +established across these two questions; reply projection alone is insufficient. + +## Producer, window and resource scope + +Eight separately bounded direct producer runs preserve source bytes and render +correct stdout: UART baseline 159/131 ms versus focused 163/130 ms; I2C baseline +355/298 ms versus focused 317/290 ms. This isolates the producer process from REA +RPC, version probing and snapshot work, but startup/decoding/annotation formatting +remain combined. Decoding-only versus formatting-only time is unmeasured. + +REA literal-window extraction and JSON encoding were measured separately from +retained producer text (one cold observation and 200 warm iterations per arm). +Warm mean extraction ranges from 0.0040 to 0.1794 ms; JSON encoding from 0.0021 to 0.0121 ms. These exclude file reads, RPC and decoding. + +Every page still invokes decoding. Retained output might help repeated inspection, +but this fixed cohort does not demonstrate a completed-task improvement warranting +a new cache. Any future work should own bounded output explicitly by original +Evidence identity and lifetime, with cancellation/cleanup, rather than use a +CLI-digest-only generic cache of an incompletely fingerprinted installation. + +Heavy work ran serially under one CPU, 3 GiB early stop / 4 GiB hard memory, +no swap and host-availability checks. Verified payload-exit peak bytes were +SDK 587,513,856; model 889,323,520; direct producer 26,288,128. No OOM/OOM-kill or resource abort occurred in that verified phase; task descendants +were cleaned. Terminal whole-scope counters are unavailable and remain unknown. + +## Verification and replay + +Eleven independent-oracle regressions and 12 comparator/sample-interval tests pass. +Twenty-four private replays using actual SDK Evidence cover valid answers, failed +calls before/after recovery, wrong bytes, unknown citations and missing limitations. +These are grading checks with injected state, not additional model/provider trials. +The grader retains authenticated observations by identity, rejects conflicts, +combines cited pages in sample order and keeps failed calls/plan deviations separate +from supported facts. Relevant typecheck/lint and pre-push checks pass. + +Review hardening ties the oracle to the named corpus digest, bounds each I2C +interval against neighboring byte clocks/transaction STOP, and counts actual +classic-ZIP directory records on the same bounded snapshot before allocating +`ZipFile` objects; ZIP64 is rejected. Forged-count and ZIP64 admission regressions +fail on the original implementation. Replaying all eight original model trials +and 24 injected grading cases with these checks preserves their factual/plan +grades and recorded usage/time; these replays are not new model measurements. + +Metadata reads cap inflater output even when ZIP sizes are forged. A streamed +8 MiB malformed metadata member declaring 8 KiB is rejected with traced Python +allocation below 1 MiB; the same regression fails on the preceding implementation. +Compressed snapshots use actual file size plus one growth-detection byte rather +than preallocating the entire admission budget. Both genuine waveform oracles +retain identical bytes, metadata, sample intervals and source digests. + +UART interval checks also require coverage of the first and last independently +sampled data-bit centers within the owning frame. Shortened, shifted and +single-sample intervals are rejected even when the decoded byte is correct. +The added oracle coordinates retain the original byte/frame facts; replaying +saved real Evidence and all eight model transcripts preserves the original +grades and costs. This is stronger retrospective verification, not a new cohort. + +After rebasing onto main `44d98de2367b99808d78db732ec7d72533266233`, +`npm run build:cached` and the 16 focused comparator/sample-interval/MCP-boundary +tests passed. Two additional real focused MCP workflows independently verified +UART/I2C bytes and intervals on that base. These integration checks are separate +from the original cost cohort and do not replace or extend its measurements. + +A subsequent rebase onto main `3b79af28e80785afce9f5998d8f03289402a1847` +passed the 17 focused tests, including the upstream-fixed Hopper doctor case, +and two additional real UART/I2C MCP workflows. Original cohort costs and grades +remain unchanged; these checks verify integration on the updated main only. + +Use `measure.mjs --signal-selection baseline|focused|discovery|pages`, preserving +runtime/source fingerprints and explicit provider JSON. Generate the exact task +with `signal-model-prompt.mjs --artifact ID --selection baseline|focused`, run the +isolated `agent-trial.py --mode direct --capture-tool-context`, and grade with +`review-signal.mjs` and the independently produced oracle. Keep original failures, +resources and raw observations private. For migration, existing optional annotation +selection is sufficient for scoped reply reduction; no default/API change follows +from this mixed whole-task result. diff --git a/experiments/analysis-efficiency/test_signal_oracle.py b/experiments/analysis-efficiency/test_signal_oracle.py new file mode 100644 index 000000000..9decd9a18 --- /dev/null +++ b/experiments/analysis-efficiency/test_signal_oracle.py @@ -0,0 +1,149 @@ +"""Distinct framing, identity and admission failures in the sample-bit oracle.""" + +import importlib.util +from pathlib import Path +import struct +import tempfile +import tracemalloc +import unittest +from unittest.mock import patch +import zipfile + +spec = importlib.util.spec_from_file_location( + 'signal_oracle', Path(__file__).with_name('signal-oracle.py')) +oracle = importlib.util.module_from_spec(spec) +spec.loader.exec_module(oracle) + + +class SignalOracleTests(unittest.TestCase): + def setUp(self): + self.root = tempfile.TemporaryDirectory() + self.addCleanup(self.root.cleanup) + self.path = Path(self.root.name) / 'sample.sr' + + def recording(self, data, unit, channels, chunk='logic-1', extra=0): + metadata = '[global]\nsigrok version=0.3.0\n[device 1]\ncapturefile=logic-1\n' + metadata += f'total probes={len(channels)}\nsamplerate=1 MHz\nunitsize={unit}\n' + metadata += ''.join(f'probe{index+1}={name}\n' for index, name in enumerate(channels)) + with zipfile.ZipFile(self.path, 'w') as archive: + archive.writestr('version', '2') + archive.writestr('metadata', metadata) + archive.writestr(chunk, data) + for index in range(extra): + archive.writestr(f'extra-{index}', '') + + def test_uart_rearms_after_stop_sample_for_a_slightly_faster_sender(self): + levels = [1] * 100 + for byte in [0x48, 0x65]: + for bit in [0, *[(byte >> index) & 1 for index in range(8)], 1]: + levels.extend([bit] * 17) + levels.extend([1] * 100) + data = b''.join(struct.pack('> index) & 1 for index in reversed(range(8))] + [ack] + if incomplete and byte == 0x75: + values = values[:4] + for bit in values: + states.extend([(0, bit), (1, bit)]) + states.extend([(0, 0), (1, 0), (1, 1)]) + return bytes(clock | data << 1 for clock, data in states) + + def test_i2c_stop_setup_is_distinct_from_another_data_byte(self): + self.recording(self.i2c_levels(), 1, ['SCL', 'SDA']) + result = oracle.i2c(self.path) + self.assertEqual(len(result['reads']), 1) + self.assertEqual([row['byte'] for row in result['reads'][0]['bytes']], [0xA5, 0x75]) + self.assertFalse(result['reads'][0]['bytes'][-1]['ack']) + + def test_incomplete_i2c_byte_is_not_invented_at_stop(self): + self.recording(self.i2c_levels(incomplete=True), 1, ['SCL', 'SDA']) + with self.assertRaisesRegex(ValueError, 'Incomplete I2C byte'): + oracle.i2c(self.path) + + def test_channel_names_must_identify_one_stored_bit(self): + self.recording(b'\x00', 1, ['SCL', 'SCL']) + with self.assertRaisesRegex(ValueError, 'ambiguous'): + oracle.read_edges(self.path, ['SCL']) + + def test_missing_chunk_is_not_renumbered(self): + self.recording(b'\x00', 1, ['SCL', 'SDA'], chunk='logic-1-2') + with self.assertRaisesRegex(ValueError, 'chunk order'): + oracle.read_edges(self.path, ['SCL', 'SDA']) + + def test_partial_sample_word_is_rejected(self): + self.recording(b'\x00', 2, ['unused', 'MAX3232E DOUT1']) + with self.assertRaisesRegex(ValueError, 'incomplete digital samples'): + oracle.read_edges(self.path, ['MAX3232E DOUT1']) + + def test_directory_budget_precedes_zip_object_allocation(self): + self.recording(b'\x00', 1, ['SCL', 'SDA'], extra=62) + with patch.object(oracle.zipfile, 'ZipFile', side_effect=AssertionError('Too late')): + with self.assertRaisesRegex(ValueError, 'excessive ZIP directory'): + oracle.read_edges(self.path, ['SCL', 'SDA']) + + def test_declared_expansion_is_rejected_before_decompression(self): + self.recording(b'\x00', 1, ['SCL', 'SDA']) + data = bytearray(self.path.read_bytes()) + directory = data.index(b'PK\x01\x02') + struct.pack_into(' row.value), [left.answer.delivery_observations, right.answer.delivery_observations], ); - right.answer.facts!.matching_packets++; + right.answer.facts!.matching_packets = 1338; assert.equal(compare(left, right).report.facts_match, false); - right.answer.facts!.matching_packets--; + right.answer.facts!.matching_packets = 1337; right.summary.packet_selection = "baseline"; assert.equal(compare(left, right).report.facts_match, false); }); } +for (const artifact of ["sigrok-uart-max3232e.sr", "sigrok-i2c-nunchuk.sr"]) { + test(`${artifact}: compare decoded facts across signal plans and preserve display scope`, () => { + const left = fixture("complete", artifact); + left.summary.signal_selection = "baseline"; + left.answer.facts = { kind: "signal", bytes: [72, 101] }; + const right = structuredClone(left); + right.summary.signal_selection = "focused"; + right.answer.delivery_observations[0]!.coverage = + "selected annotation lines"; + const equivalent = compare(left, right); + assert.equal(equivalent.status, 0); + assert.equal(equivalent.report.facts_match, true); + assert.deepEqual( + equivalent.report.preserved_run_specific_values.map((row) => row.value), + [left.answer.delivery_observations, right.answer.delivery_observations], + ); + right.answer.facts!.bytes![0] = 73; + assert.equal(compare(left, right).report.facts_match, false); + right.answer.facts!.bytes![0] = 72; + right.summary.signal_selection = "baseline"; + assert.equal(compare(left, right).report.facts_match, false); + }); +} + for (const artifact of [ "wireshark-http.cap", "wireshark-http_redirects.pcapng", diff --git a/tests/evaluation/signalWorkflowSemantics.test.ts b/tests/evaluation/signalWorkflowSemantics.test.ts new file mode 100644 index 000000000..7877c494e --- /dev/null +++ b/tests/evaluation/signalWorkflowSemantics.test.ts @@ -0,0 +1,46 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { test } from "vitest"; + +test("signal verification rejects skipped identical UART bytes and reordered I2C samples", () => { + const moduleUrl = new URL( + "../../experiments/analysis-efficiency/signal-entry.mjs", + import.meta.url, + ).href; + const source = ` + import assert from 'node:assert/strict'; + import {verifySignalRows,signalDataRows,verifySignalOracleIdentity} from ${JSON.stringify(moduleUrl)}; + const sha256='a'.repeat(64); + const rows=Array.from({length:8},(_,i)=>({start:i*100+10,end:i*100+80,byte:72})); + const uart={kind:'uart',sha256,total_samples:2000,rows:rows.map((_,i)=>({start:i*100,end_exclusive:i*100+90,first_data_sample:i*100+15,last_data_sample:i*100+75,byte:72}))}; + verifySignalRows('uart',rows,uart,sha256); + verifySignalRows('uart',rows.map((row,i)=>({...row,start:i*100+15,end:i*100+75})),uart,sha256); + assert.throws(()=>verifySignalRows('uart',[{...rows[0],start:0,end:0},...rows.slice(1)],uart,sha256),/interval/); + assert.throws(()=>verifySignalRows('uart',[{...rows[0],end:74},...rows.slice(1)],uart,sha256),/interval/); + assert.throws(()=>verifySignalRows('uart',[{...rows[0],start:16},...rows.slice(1)],uart,sha256),/interval/); + assert.throws(()=>verifySignalRows('uart',rows,{...uart,rows:uart.rows.map(({first_data_sample,...row})=>row)},sha256),/interval/); + assert.throws(()=>verifySignalRows('uart',rows.slice(1).concat({start:810,end:880,byte:72}),uart,sha256),/interval/); + assert.throws(()=>verifySignalRows('uart',rows.slice(1),uart,sha256),/coverage/); + assert.throws(()=>verifySignalRows('uart',rows,uart,'b'.repeat(64)),/identity/); + const data=Array.from({length:18},(_,i)=>({start:i*100+10,end:i*100+90,byte:117})); + const raw=data.map((_,i)=>({byte:117,first_clock:i*100+20,last_clock:i*100+80})); + const i2c={kind:'i2c',sha256,total_samples:2000,reads:[{bytes:[{byte:165,last_clock:0},...raw],stop:1900}]}; + verifySignalRows('i2c',data,i2c,sha256); + assert.throws(()=>verifySignalRows('i2c',data.toReversed(),i2c,sha256),/interval/); + assert.throws(()=>verifySignalRows('i2c',[{...data[0],byte:118},...data.slice(1)],i2c,sha256),/sample bits/); + assert.throws(()=>verifySignalRows('i2c',data.map((row,i)=>({...row,start:i,end:1999})),i2c,sha256),/interval/); + assert.throws(()=>verifySignalRows('i2c',[{...data[0],start:0},...data.slice(1)],i2c,sha256),/interval/); + assert.throws(()=>verifySignalRows('i2c',[...data.slice(0,-1),{...data.at(-1),end:1900}],i2c,sha256),/interval/); + verifySignalOracleIdentity('sigrok-uart-max3232e.sr',{kind:'uart',sha256:'f797d60d6b8e12d2a1549b9a5a499ef389f44cb9e870e8610af26b57c3b2be76'}); + assert.throws(()=>verifySignalOracleIdentity('sigrok-uart-max3232e.sr',uart),/pinned signal corpus/); + assert.throws(()=>verifySignalOracleIdentity('sigrok-i2c-nunchuk.sr',{kind:'uart',sha256:'0fa6adb8297d7f3fe5ba19c94ee655c9b0f952a8b5e48a17aaa499a2f6d10d70'}),/pinned signal corpus/); + assert.throws(()=>signalDataRows('uart','9007199254740993-9007199254740994 uart-1: 48'),/sample interval/); + `; + // Checker semantics only; synthesized rows do not establish provider behavior. + const result = spawnSync(process.execPath, ["--input-type=module", "-"], { + input: source, + encoding: "utf8", + }); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stderr); +});