diff --git a/docs/packet-captures.md b/docs/packet-captures.md index 32cedb11b..8db097386 100644 --- a/docs/packet-captures.md +++ b/docs/packet-captures.md @@ -54,6 +54,22 @@ resource limit; use selected fields, a display filter or an input-packet limit when dissection output is too large. Each call checks the recording's current digest, so pages from different source digests must not be combined. +For a question about a known input prefix, select `packet_limit` for that +prefix, the fields needed for the answer, and a matching output window. For +example, a question about the first twelve PostgreSQL TCP packets within +original frames 1–2,000 can use fields `tcp.srcport`, `tcp.dstport`, `tcp.flags`, +filter `tcp.port == 5432`, `packet_limit: 2000` and `match_limit: 12`. +The matching count then describes that input prefix. A filter containing +`frame.number <= 2000` without an input limit still reads the entire capture; +returning twelve rows does not limit producer output. These controls answer a +bounded question and must not be used to infer whole-capture absence. + +Discover unfamiliar field names with `inspect_packet_fields`, using an exact +protocol and a narrow query. If the installed names are already established, +direct field selection avoids an additional glossary probe. Count discovery +and any retries when comparing investigation cost; a small page alone does +not establish a faster workflow. + Additional selections: - `display_filter`: Wireshark display-filter syntax; omitted means no filter. diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.md index cdb6c3c9b..9a14b764a 100644 --- a/experiments/analysis-efficiency/README.md +++ b/experiments/analysis-efficiency/README.md @@ -72,6 +72,12 @@ checkpoint commits retain their original receipts; no history was rewritten. ## Optimization priorities and gradual migration +The [larger/non-HTTP packet checkpoint](packet-field-workflows.md) records +same-question SDK and actual Codex comparisons using existing input limits, +field/filter selection, discovery and output windows. Lower delivered JSON did +not establish stable total-token or speed gains; failed plans and negative +controls remain visible. + Agent token consumption and end-to-end task time are the primary objectives. Memory and CPU consumption are also measured and controlled. Correctness, Evidence provenance, caller choices and existing-user compatibility are diff --git a/experiments/analysis-efficiency/agent-trial.py b/experiments/analysis-efficiency/agent-trial.py index edb848a28..db994cea1 100644 --- a/experiments/analysis-efficiency/agent-trial.py +++ b/experiments/analysis-efficiency/agent-trial.py @@ -181,6 +181,7 @@ def main(): 'inspect_android_class', 'inspect_android_method', 'inspect_firmware_regions', 'inspect_managed_artifact', 'inspect_sqlite_database', 'inspect_pcap_capture', + 'inspect_packet_capture', 'inspect_packet_fields', 'inspect_managed_members', 'inspect_managed_native_boundaries', 'inspect_artifact', 'inspect_wasm_artifact', 'inspect_evm_interface', 'get_evidence_bundle', 'export_evidence_bundle', diff --git a/experiments/analysis-efficiency/compare.mjs b/experiments/analysis-efficiency/compare.mjs index 4e33e5ead..01b9f97af 100644 --- a/experiments/analysis-efficiency/compare.mjs +++ b/experiments/analysis-efficiency/compare.mjs @@ -2,6 +2,7 @@ import { isDeepStrictEqual } from "node:util"; import { readFile, writeFile } from "node:fs/promises"; import { resolve } from "node:path"; import { parseArgs } from "node:util"; +import { PACKET_ARTIFACTS } from "./packet-entry.mjs"; const { values } = parseArgs({ options: { @@ -33,6 +34,8 @@ const sameSettings = [ "provider_environment_policy_sha256", "measure_source_sha256", "native_workflow_source_sha256", + "packet_workflow_source_sha256", + "packet_oracle_source_sha256", "package_lock_sha256", "node_version", "sdk_client_version", @@ -76,6 +79,7 @@ for (const [side, answer] of [ // coverage. Repeated equivalent deliveries must also match every observation // field except the receipt and parent IDs, which identify their own invocations. if ( + Object.hasOwn(PACKET_ARTIFACTS, left.artifact) || ["wireshark-http.cap", "wireshark-http_redirects.pcapng"].includes( left.artifact, ) @@ -94,7 +98,11 @@ if ( path: ["delivery_observations"], value: answer.delivery_observations, }); - if (left.delivery === right.delivery) { + if ( + left.delivery === right.delivery && + (!Object.hasOwn(PACKET_ARTIFACTS, left.artifact) || + left.packet_selection === right.packet_selection) + ) { answer.delivery_observations = answer.delivery_observations.map( (observation) => Object.fromEntries( diff --git a/experiments/analysis-efficiency/corpus.json b/experiments/analysis-efficiency/corpus.json index 7896c558d..e595dde34 100644 --- a/experiments/analysis-efficiency/corpus.json +++ b/experiments/analysis-efficiency/corpus.json @@ -291,6 +291,58 @@ "upstream_expected_digest": null, "derivation": "Unmodified capture published by official Wireshark wiki. Local digest pins downloaded bytes; no independently published upstream digest is claimed." } + }, + { + "id": "wireshark-pgsql-jdbc.pcap", + "file": "pgsql-jdbc.pcap", + "bytes": 4906316, + "sha256": "6e68b8b792a0f4336092f98856abc7de8c54b55a29112854d28d676f9093c16b", + "provenance": { + "project": "Wireshark public sample captures", + "source_page": "https://wiki.wireshark.org/SampleCaptures", + "url": "https://wiki.wireshark.org/uploads/__moin_import__/attachments/SampleCaptures/pgsql-jdbc.pcap.gz", + "resolved_url": "https://gitlab.com/wireshark/wireshark/-/wikis/uploads/__moin_import__/attachments/SampleCaptures/pgsql-jdbc.pcap.gz", + "upstream_expected_digest": null, + "license": "not stated in inspected upstream catalogue", + "derivation": "Unmodified official saved sample; decompressed gzip where indicated. Local digest pins acquired bytes, not an upstream-published digest.", + "compressed_source": { + "name": "pgsql-jdbc.pcap.gz", + "bytes": 590093, + "sha256": "15b51bb4cc197753c1ac6b7dc3cf5697f22a81e3bb5d2e1d24d15de469c3a06b" + } + } + }, + { + "id": "wireshark-dns.cap", + "file": "dns.cap", + "bytes": 4338, + "sha256": "041eeb6f98bb398f1ee8b09651b5b5a84f6a62639f95bf226f9e7b77355d9f28", + "provenance": { + "project": "Wireshark public sample captures", + "source_page": "https://wiki.wireshark.org/SampleCaptures", + "url": "https://wiki.wireshark.org/uploads/__moin_import__/attachments/SampleCaptures/dns.cap", + "resolved_url": "https://gitlab.com/wireshark/wireshark/-/wikis/uploads/__moin_import__/attachments/SampleCaptures/dns.cap", + "upstream_expected_digest": null, + "license": "not stated in inspected upstream catalogue", + "derivation": "Unmodified official saved sample; decompressed gzip where indicated. Local digest pins acquired bytes, not an upstream-published digest.", + "compressed_source": null + } + }, + { + "id": "wireshark-dhcp.pcap", + "file": "dhcp.pcap", + "bytes": 1400, + "sha256": "2471b5420bdac826eecf8f61a2bbb4a3eb20dbfab7c02ff2be502f349f368214", + "provenance": { + "project": "Wireshark public sample captures", + "source_page": "https://wiki.wireshark.org/SampleCaptures", + "url": "https://wiki.wireshark.org/uploads/__moin_import__/attachments/SampleCaptures/dhcp.pcap", + "resolved_url": "https://gitlab.com/wireshark/wireshark/-/wikis/uploads/__moin_import__/attachments/SampleCaptures/dhcp.pcap", + "upstream_expected_digest": null, + "license": "not stated in inspected upstream catalogue", + "derivation": "Unmodified official saved sample; decompressed gzip where indicated. Local digest pins acquired bytes, not an upstream-published digest.", + "compressed_source": null + } } ] } diff --git a/experiments/analysis-efficiency/measure.mjs b/experiments/analysis-efficiency/measure.mjs index 60dce4f80..07a39e3c1 100644 --- a/experiments/analysis-efficiency/measure.mjs +++ b/experiments/analysis-efficiency/measure.mjs @@ -12,6 +12,7 @@ import { import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { Ajv2020 } from "ajv/dist/2020.js"; +import { PACKET_ARTIFACTS, measurePacketEntry } from "./packet-entry.mjs"; const root = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); const { values } = parseArgs({ @@ -29,6 +30,7 @@ const { values } = parseArgs({ "managed-question": { type: "string", default: "inventory" }, schemas: { type: "string", default: "default" }, "sqlite-selection": { type: "string", default: "all" }, + "packet-selection": { type: "string", default: "baseline" }, }, }); if ( @@ -45,6 +47,15 @@ if ( !["inventory", "selected"].includes(values["javascript-question"]) || !["default", "selected"].includes(values.schemas) || !["all", "columns"].includes(values["sqlite-selection"]) || + !["baseline", "focused", "discovery", "pages"].includes( + values["packet-selection"], + ) || + (!Object.hasOwn(PACKET_ARTIFACTS, values.artifact) && + values["packet-selection"] !== "baseline") || + (Object.hasOwn(PACKET_ARTIFACTS, values.artifact) && + values.delivery !== "complete") || + (values["packet-selection"] === "pages" && + values.artifact !== "wireshark-pgsql-jdbc.pcap") || (values.artifact !== "qgis-world-map.gpkg" && values["sqlite-selection"] !== "all") || (values.artifact === "qgis-world-map.gpkg" && @@ -142,6 +153,12 @@ const fingerprints = { native_workflow_source_sha256: await fileSha( fileURLToPath(new URL("./native-entry.mjs", import.meta.url)), ), + packet_workflow_source_sha256: await fileSha( + fileURLToPath(new URL("./packet-entry.mjs", import.meta.url)), + ), + packet_oracle_source_sha256: await fileSha( + fileURLToPath(new URL("./packet-oracle.py", import.meta.url)), + ), runtime_files_sha256: sha(JSON.stringify(runtimeFiles)), runtime_dispatcher_sha256: await fileSha( resolve(values["runtime-root"], "scripts/rea.mjs"), @@ -722,6 +739,12 @@ const inspectPcapTarget = async () => { }; const workflows = { + ...Object.fromEntries( + Object.keys(PACKET_ARTIFACTS).map((artifact) => [ + artifact, + () => measurePacketEntry({ values, call, inputSha, catalogDefinitions }), + ]), + ), "wireshark-http.cap": inspectPcapTarget, "wireshark-http_redirects.pcapng": inspectPcapTarget, "qgis-world-map.gpkg": async () => { @@ -1160,6 +1183,9 @@ const report = { values.artifact === "qgis-world-map.gpkg" ? values["sqlite-selection"] : null, + packet_selection: Object.hasOwn(PACKET_ARTIFACTS, values.artifact) + ? values["packet-selection"] + : null, provider_settings_sha256: sha(JSON.stringify(selectedProviders)), provider_search_path_sha256: sha( selectedProviders.PATH ?? process.env.PATH ?? "", diff --git a/experiments/analysis-efficiency/packet-entry.mjs b/experiments/analysis-efficiency/packet-entry.mjs new file mode 100644 index 000000000..5c6722a74 --- /dev/null +++ b/experiments/analysis-efficiency/packet-entry.mjs @@ -0,0 +1,316 @@ +import { isDeepStrictEqual } from "node:util"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { fileURLToPath } from "node:url"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +export const verifyPacketPage = (first, next, oracle) => { + if ( + first.artifact.sha256 !== oracle.sha256 || + next.artifact.sha256 !== oracle.sha256 || + first.output_window.matching_packets !== oracle.matching_packets || + next.output_window.matching_packets !== oracle.matching_packets || + !isDeepStrictEqual( + packetFacts("postgres", first).rows, + oracle.rows.slice(0, 12), + ) || + !isDeepStrictEqual( + packetFacts("postgres", next).rows, + oracle.rows.slice(12, 24), + ) + ) + throw new Error( + "Packet pages differ from independently parsed original matches", + ); +}; + +export const PACKET_ARTIFACTS = { + "wireshark-pgsql-jdbc.pcap": { + kind: "postgres", + fields: ["tcp.srcport", "tcp.dstport", "tcp.flags"], + extra: ["tcp.len"], + protocol: "tcp", + query: "tcp.flags", + discovered: "tcp.flags", + filter: "tcp.port == 5432", + }, + "wireshark-dns.cap": { + kind: "dns", + fields: [ + "dns.id", + "dns.flags.response", + "dns.flags.rcode", + "dns.qry.name", + "dns.qry.type", + ], + extra: ["dns.resp.name", "dns.resp.ttl", "udp.srcport", "udp.dstport"], + protocol: "dns", + query: "dns.flags.rcode", + discovered: "dns.flags.rcode", + filter: "dns.flags.response == 1 && dns.flags.rcode != 0", + }, + "wireshark-dhcp.pcap": { + kind: "dhcp", + fields: [ + "dhcp.id", + "dhcp.option.dhcp", + "dhcp.ip.your", + "dhcp.option.dhcp_server_id", + ], + extra: [ + "dhcp.type", + "dhcp.ip.client", + "dhcp.hw.mac_addr", + "udp.srcport", + "udp.dstport", + ], + protocol: "dhcp", + query: "Option 53:", + discovered: "dhcp.option.dhcp", + filter: "dhcp", + }, +}; + +export const packetRequest = (artifact, path, selection) => { + const descriptor = PACKET_ARTIFACTS[artifact]; + if (!descriptor) throw new Error("Unknown pinned packet question"); + const baseline = selection === "baseline"; + return { + path, + fields: baseline + ? [...descriptor.fields, ...descriptor.extra] + : descriptor.fields, + display_filter: + descriptor.kind === "postgres" && baseline + ? `${descriptor.filter} && frame.number <= 2000` + : descriptor.kind === "dns" && baseline + ? "dns" + : descriptor.filter, + ...(descriptor.kind === "postgres" && !baseline + ? { packet_limit: 2000, match_limit: 12 } + : {}), + }; +}; + +const number = (row, field) => { + const value = row[field]; + if ( + !Array.isArray(value) || + value.length !== 1 || + !/^(?:[0-9]+|0x[0-9a-f]+)$/iu.test(value[0]) + ) + throw new Error(`Expected one recorded integer for ${field}`); + const parsed = Number(value[0]); + if (!Number.isSafeInteger(parsed)) + throw new Error(`Unsafe recorded integer for ${field}`); + return parsed; +}; + +export const packetFacts = (kind, report) => { + let packets = report.packets; + if (kind === "postgres") packets = packets.slice(0, 12); + if (kind === "dns") + packets = packets.filter( + (row) => + number(row, "dns.flags.response") === 1 && + number(row, "dns.flags.rcode") !== 0, + ); + return { + kind, + matching_packets: + kind === "dns" ? packets.length : report.output_window.matching_packets, + rows: packets.map((row) => { + const frame = number(row, "frame.number"); + if (kind === "postgres") + return { + frame, + source_port: number(row, "tcp.srcport"), + destination_port: number(row, "tcp.dstport"), + flags: number(row, "tcp.flags"), + }; + if (kind === "dns") { + const names = row["dns.qry.name"]; + const types = row["dns.qry.type"]; + if ( + !Array.isArray(names) || + names.length === 0 || + !Array.isArray(types) || + types.length !== names.length + ) + throw new Error( + "The fixed DNS question requires recorded names and types; missing fields are unknown", + ); + return { + frame, + id: number(row, "dns.id"), + rcode: number(row, "dns.flags.rcode"), + names, + types: types.map((value) => number({ type: [value] }, "type")), + }; + } + const server = row["dhcp.option.dhcp_server_id"]; + const address = row["dhcp.ip.your"]; + if ( + !Array.isArray(address) || + address.length !== 1 || + (server !== undefined && + (!Array.isArray(server) || server.length !== 1)) + ) + throw new Error("Expected recorded DHCP address scalar"); + return { + frame, + id: number(row, "dhcp.id"), + message_type: number(row, "dhcp.option.dhcp"), + your_address: address[0], + ...(server === undefined ? {} : { server_identifier: server[0] }), + }; + }), + }; +}; + +export const measurePacketEntry = async ({ + values, + call, + inputSha, + catalogDefinitions, +}) => { + const descriptor = PACKET_ARTIFACTS[values.artifact]; + const { parseEvidence } = await import( + pathToFileURL(resolve(values["runtime-root"], "dist/domain/evidence.js")) + .href + ); + const { inspectPacketCaptureInputSchema, packetCaptureSchema } = await import( + pathToFileURL( + resolve(values["runtime-root"], "dist/domain/captures/packetCapture.js"), + ).href + ); + const { inspectPacketFieldsInputSchema, packetFieldsCatalogSchema } = + await import( + pathToFileURL( + resolve(values["runtime-root"], "dist/domain/captures/packetFields.js"), + ).href + ); + const observations = []; + const selection = values["packet-selection"]; + const checked = (evidence, operation, input, schema) => { + parseEvidence(evidence); + const report = schema.parse(evidence.normalized_result); + if ( + evidence.operation !== operation || + !isDeepStrictEqual(evidence.parameters, input) || + !isDeepStrictEqual(report.selection, input) + ) + throw new Error( + "Packet Evidence operation or choices differs from the request", + ); + if ( + operation === "inspect_packet_capture" && + (evidence.subject?.local_path !== values.path || + evidence.subject?.digest?.sha256 !== inputSha || + report.artifact.path !== values.path || + report.artifact.sha256 !== inputSha) + ) + throw new Error("Packet Evidence identifies different capture bytes"); + if ( + operation === "inspect_packet_fields" && + (evidence.subject?.local_path !== report.tool.path || + evidence.subject?.digest?.sha256 !== report.tool.sha256) + ) + throw new Error("Field discovery identifies a different executable"); + observations.push({ + evidence_id: evidence.evidence_id, + operation, + selection: report.selection, + output_window: report.output_window, + read_scope: report.read_scope ?? null, + limitations: evidence.limitations, + }); + return report; + }; + if (selection === "discovery") { + const input = inspectPacketFieldsInputSchema.parse({ + kind: "fields", + protocol: descriptor.protocol, + query: descriptor.query, + limit: 1, + }); + const found = checked( + await call("inspect_packet_fields", input), + "inspect_packet_fields", + input, + packetFieldsCatalogSchema, + ); + if ( + !found.records.some((row) => row.abbreviation === descriptor.discovered) + ) + throw new Error( + "Selected installation does not advertise the required protocol field", + ); + } + const input = inspectPacketCaptureInputSchema.parse( + packetRequest(values.artifact, values.path, selection), + ); + for (const name of [ + "inspect_packet_capture", + ...(selection === "discovery" ? ["inspect_packet_fields"] : []), + ]) + if (!catalogDefinitions.has(name)) + throw new Error(`Missing live packet contract ${name}`); + const report = checked( + await call("inspect_packet_capture", input), + "inspect_packet_capture", + input, + packetCaptureSchema, + ); + const facts = packetFacts(descriptor.kind, report); + if (selection === "pages") { + const { stdout } = await promisify(execFile)( + "python3", + [ + fileURLToPath(new URL("./packet-oracle.py", import.meta.url)), + "postgres", + values.path, + "--row-limit", + "24", + ], + { timeout: 30_000, maxBuffer: 128 * 1024 }, + ); + const oracle = JSON.parse(stdout); + const nextInput = inspectPacketCaptureInputSchema.parse({ + ...input, + match_offset: 12, + }); + const next = checked( + await call("inspect_packet_capture", nextInput), + "inspect_packet_capture", + nextInput, + packetCaptureSchema, + ); + const repeated = checked( + await call("inspect_packet_capture", input), + "inspect_packet_capture", + input, + packetCaptureSchema, + ); + verifyPacketPage(report, next, oracle); + if ( + !isDeepStrictEqual(report.packets, repeated.packets) || + !isDeepStrictEqual(report.output_window, repeated.output_window) || + next.artifact.sha256 !== report.artifact.sha256 || + next.output_window.matching_packets !== + report.output_window.matching_packets || + Number(next.packets[0]?.["frame.number"]?.[0]) <= + Number(report.packets.at(-1)?.["frame.number"]?.[0]) + ) + throw new Error( + "Packet page order, digest or repeated observations changed", + ); + } + return { + artifact_sha256: inputSha, + facts, + delivery_observations: observations, + evidence_ids: [...new Set(observations.map((row) => row.evidence_id))], + }; +}; diff --git a/experiments/analysis-efficiency/packet-field-workflows.md b/experiments/analysis-efficiency/packet-field-workflows.md new file mode 100644 index 000000000..8f893c30c --- /dev/null +++ b/experiments/analysis-efficiency/packet-field-workflows.md @@ -0,0 +1,226 @@ +# Larger and non-HTTP packet investigation costs + +This closes the bounded experiment in [#3130](https://github.com/morluto/rea/issues/3130). +Existing field, filter and input/output-window controls substantially reduce +delivered JSON for the selected questions. Actual Codex total input and elapsed +time are less consistent. Requiring field discovery for a known field adds cost. +These results support documenting caller-selected controls, while retaining the +current API, defaults and uncached provider behavior. + +## Runtime and public corpus + +Measurements use immutable main +`ec67aebed02dbd20fa5c7703fcd183ad0b1c343c`, Node 24.18.0, the locked MCP client and +server, Linux, TShark/capinfos 4.0.17 and util-linux prlimit. The SDK captures live +advertised contracts and passes selected definitions to actual public MCP calls. +Runtime-file, dispatcher, lockfile, experiment-source and explicit provider +configuration digests remain with the private trial records. Subsequent main +integration checks do not relabel these measurements. + +Three additional saved recordings come from the +[official Wireshark catalogue](https://wiki.wireshark.org/SampleCaptures). +`corpus.json` pins their acquisition URLs, exact bytes and SHA-256 values. +The PostgreSQL recording is described upstream as a JDBC test session; it is +not a representative production workload. Capture-specific licenses were not +stated in the inspected catalogue. No captures or raw trial records are +redistributed in Git. + +| Recording | Acquired/analyzed size | Recorded frames | Fixed question | +| -------------------- | --------------------------------------------: | --------------: | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `pgsql-jdbc.pcap.gz` | 590,093 compressed / 4,906,316 expanded bytes | 18,472 | Within original frames 1–2,000, report the first twelve TCP packets using port 5432, their source/destination ports and flags, and the matching count. | +| `dns.cap` | 4,338 bytes | 38 | Report nonzero-RCODE DNS responses, transaction IDs and question names/types. | +| `dhcp.pcap` | 1,400 bytes | 4 | Report message types, transaction IDs, offered address and present server identifiers in frame order. | + +The larger application recording and two small non-HTTP controls expand the +question/protocol coverage. Large DNS/DHCP, PCAPNG non-HTTP, USB, Bluetooth, +Windows and macOS provider performance remain unverified. + +`packet-oracle.py` independently reads classic PCAP, Ethernet/IPv4 and selected +TCP/UDP headers. It does not consume TShark output. Its narrow supported scope +fails on truncation, IP fragmentation, DNS compression cycles, duplicate DHCP +options or DHCP overload. TCP header facts follow +[RFC 9293](https://www.rfc-editor.org/rfc/rfc9293.html), DNS names/header fields +[RFC 1035](https://www.rfc-editor.org/rfc/rfc1035.html), and DHCP options +[RFC 2132](https://www.rfc-editor.org/rfc/rfc2132.html). +SQL interpretation, TCP reassembly and network completeness are outside the +oracle. The question and oracle were fixed before measured cohorts. + +The oracle establishes 1,337 selected PostgreSQL TCP packets in the input prefix +and six DNS error responses. DHCP records contain Discover/Offer with transaction +ID 15645 and Request/ACK with ID 15646. An absent server identifier stays omitted; +the recording does not establish a single transaction containing all four steps. + +## Equivalent questions and SDK measurements + +Each arm uses fresh actual MCP sessions; heavy trials run sequentially. +Two interleaved repetitions cover each arm and each recording. Discovery, catalog, +RPC, schema checking, encoding and cleanup costs are retained. JSON bytes describe +serialized SDK replies, not raw transport bytes or model tokens. + +The PostgreSQL baseline selects `tcp.srcport`, `tcp.dstport`, `tcp.flags` and +`tcp.len`, retains the default fifty-row output window and reads to EOF with +`tcp.port == 5432 && frame.number <= 2000`. The focused arm selects the three +required fields, uses `packet_limit: 2000`, filter `tcp.port == 5432` and +`match_limit: 12`. Both establish the same requested prefix facts and count. +Their source read scopes differ explicitly: to EOF versus an input limit. +Neither scope establishes whole-network completeness. + +DNS focuses the nine-field baseline on five requested fields and the error-response +filter. DHCP selects four required fields from the nine-field baseline; all four +records are still read. The discovery arm adds one narrow installed-field probe +before the focused request. It tests the cost of validating an already specified +field, rather than proving a benefit for a previously unknown field or guessing +that an installed registration appears in traffic. + +| Question | Baseline reply bytes | Focused reply bytes | Focused change | Discovery + focused bytes | Discovery change | +| ----------------- | -------------------: | ------------------: | -------------: | ------------------------: | ---------------: | +| PostgreSQL prefix | 23,054 | 13,304 | −42.29% | 21,096 | −8.49% | +| DNS errors | 29,002 | 13,424 | −53.71% | 21,226 | −26.81% | +| DHCP sequence | 13,770 | 12,020 | −12.71% | 19,840 | +44.08% | + +Both repetitions produce the same reply size in each cell and pass the independent +fact comparison. Authentication uses the selected runtime's `parseEvidence` and +named input/output schemas, checking operation, normalized caller choices, +artifact or executable identity, original frame order and omission semantics. + +| Question | Baseline tool-call time, ms | Focused tool-call time, ms | Discovery + focused tool-call time, ms | +| ----------------- | --------------------------- | -------------------------- | -------------------------------------- | +| PostgreSQL prefix | 1,670 / 2,255 | 1,708 / 933 | 3,549 / 2,674 | +| DNS errors | 892 / 786 | 889 / 778 | 2,506 / 2,608 | +| DHCP sequence | 763 / 759 | 755 / 731 | 2,502 / 2,502 | + +Whole SDK workflow times, including connection/catalog, vary from 4.23–8.79 seconds +for these arms. The PostgreSQL focused tool time improves in one repetition and +regresses slightly in the other. Discovery consistently adds tool work. Two +repetitions are descriptive observations, not a statistical speed guarantee. + +A separate first-page, next-page and repeated-first-page sequence costs +922 / 804 / 1,244 ms and returns 39,918 reply bytes in total. Each call repeats +dissection, reports the same 1,337 matching count and capture digest, and preserves +the original frame order. Identical repeated observations retain the same +content-addressed Evidence ID; distinct pages have distinct IDs. Page delivery +does not establish capture or network completeness. This narrow repeat does not +justify introducing a generic cache or weakening provider identity/cleanup rules. + +The two original HTTP controls were also replayed with two SDK pairs each on the +new pinned runtime. Useful summary/prefix facts remain equivalent. For `http.cap`, +complete delivery returns 7,601 bytes and summary plus a two-URI retained page +17,540 bytes (+130.76%); for `http_redirects.pcapng`, 7,535 versus 17,240 bytes +(+128.80%). These remain negative controls. The original model observations in +[the earlier checkpoint](pcap-small-captures.md) keep their original runtime pin. + +## Actual Codex CLI observations + +Six actual Codex CLI 0.161.0 runs use requested alias `gpt-5.6-terra`, reasoning +`high`, a disposable configuration and the pinned REA MCP runtime. Resolved +server-side model identity is not inferred. Both tool names are enabled in every +arm, exact-name discovery guidance is identical, shell/browser/other-agent tools +are disabled and copied authentication is removed after each run. +The shared question and generated plans contain no oracle answers. + +Order is baseline 1, focused 1, discovery 1, focused 2, baseline 2, discovery 2. +The rubric checks authenticated actual tool results, exact header-derived facts, +citations and the planned calls. Answer correctness and adherence to the plan +are reported separately. Limitation wording is reviewed in transcripts; a string +or string array is accepted because the prompt does not require one representation. + +| Trial | Total input tokens | Cached input tokens | Output tokens | Elapsed seconds | Actual MCP calls | Facts / plan | +| ----------- | -----------------: | ------------------: | ------------: | --------------: | ---------------: | ---------------------------------- | +| Baseline 1 | 40,649 | 27,904 | 1,574 | 27.989 | 1 | pass / pass | +| Focused 1 | 36,642 | 19,968 | 943 | 38.542 | 1 | pass / pass | +| Discovery 1 | 52,802 | 33,024 | 1,089 | 44.398 | 2 | pass / pass | +| Focused 2 | 40,912 | 22,016 | 726 | 32.520 | 1 | pass / pass | +| Baseline 2 | 40,181 | 19,968 | 896 | 40.299 | 1 | pass / pass | +| Discovery 2 | 48,754 | 30,976 | 1,780 | 76.081 | 1 | pass / **fail: discovery omitted** | + +The two eligible baseline/focused pairs show total-input changes of −9.86% and ++1.82%, and elapsed changes of +37.70% and −19.30%. Thus reduced SDK reply size +does not establish a stable total-token or speed improvement in these runs. +The first complete discovery comparison adds 29.90% input and 58.63% elapsed time. +The second discovery attempt is retained as a workflow failure and cannot serve +as a successful two-call discovery comparison. Its cost is not discarded or +counted as a passing trial. No model-side recovery or retries occurred in these +six attempts. + +An earlier configuration preflight rejected the packet tool under Codex's +`approval_policy: never`: 36,651 input tokens, 549 output tokens, 21.603 seconds, +one rejected call and no packet observations. The experiment runner now explicitly +preapproves the two authorized passive packet tools, consistently across all arms. +That failed preflight remains separate from the corrected cohort and its cost is +reported here; CLI exit zero alone is never a factual or workflow pass. + +## Resource and compatibility scope + +All engine/model workloads use an owned Linux resource scope with one CPU, +3 GiB early-stop threshold, 4 GiB hard memory ceiling, no swap and a 192-task cap. +Node's heap is additionally limited to 512 MiB. The nineteen non-HTTP SDK trials +have a maximum observed payload-exit memory peak of 860,909,568 bytes; the six model +attempts peak at 940,806,144 bytes. Payload-exit counters and cleanup were verified +with the observer still alive and report no memory-limit/OOM events in that phase. +Final whole-scope counters remain unavailable, so no stronger whole-scope claim +is made. CPU usage and observed process ownership remain in each private receipt. + +A full PostgreSQL probe with four selected fields and no input prefix exceeds the +existing 4 MiB producer-output ceiling even with the default fifty-row window. +It correctly fails instead of delivering partial success. The SDK error retains +large truncated diagnostics (5,790,979 serialized bytes); this separate product +concern is recorded privately for later repair, outside #3130. Paging alone cannot +prevent producer expansion. The successful prefix experiment does not prove that +the full recording can be inspected with those selections. + +Only experiment sources, corpus provenance, a question template and guidance +change. Production schemas, provider limits, CLI/MCP defaults, Evidence identity +and session/resource ownership stay as before. Eight parser regressions retain +malformed/truncated-input and omission checks. Existing clients need no migration. + +## Reproduction + +Acquire the exact corpus outside the repository and verify the pinned hashes. +Use the pinned Node/npm toolchain, locked dependencies, `npm run build:cached` and +an external resource guard before invoking actual engines/models. Supply explicit +provider settings in a private JSON file. An example SDK arm is: + +```sh +node experiments/analysis-efficiency/measure.mjs \ + --runtime-root /absolute/pinned/runtime --revision PINNED_SHA \ + --artifact wireshark-pgsql-jdbc.pcap --path /private/pgsql-jdbc.pcap \ + --providers /private/providers.json --schemas selected \ + --packet-selection focused --output /private/new-trial +``` + +Use `baseline`, `focused`, `discovery`, or the PostgreSQL-only `pages` diagnostic. +Run `packet-oracle.py postgres PATH` independently and compare `answer.json` facts. +Generate each model prompt with `packet-model-prompt.mjs --selection ARM`, then use +the existing `agent-trial.py` with those two enabled tools and context capture. +Run `review-packet.mjs` against the actual trial, oracle, runtime and selected arm. +Its nonzero exit includes a plan failure even when all requested facts are correct. +Retain failed attempts, actual usage, digests, coverage, cleanup and resource +receipts alongside passing observations. No raw results need to be committed. + +## Review verification + +The comparator now recognizes the three non-HTTP questions: observations remain +attached to each run, while different plans compare requested facts. Repeated +identical plans still compare stable observation scope. Capture grading authenticates +Evidence against the actual call and assesses plan differences separately; an +actual 24-row SDK window replay retains a factual pass and a plan failure. This +is a grading replay, not an additional model trial. The stricter header oracle +preserves all original sample facts and rejects truncated IPv4/TCP options and +unsupported Ethernet protocols. The page diagnostic now derives matches 13–24 +independently before accepting the second page; skipped, reversed and changed-byte +pages are rejected. Original cohort measurements retain their recorded source +digests and runtime pin; review checks do not relabel them. + +Capture grading keeps authenticated facts by Evidence ID and evaluates the cited +observations. Four replays using actual first-page and second-page SDK Evidence +verify that later or reordered pages and repeated reads preserve a correct cited +answer, while citing only the wrong page fails factual grading. Extra calls still +fail the fixed workflow plan. Regrading all six original model records preserves +their outcomes and recorded costs; these replays are not new model trials. + +Failed or malformed calls remain workflow failures and never become authenticated +Evidence. Nine failure-injection and answer-shape replays using an actual successful +capture verify that later recovery preserves supported facts, while failed-only +attempts and missing or unobserved citations fail factual grading. Missing +limitations and extra answer keys fail the fixed plan without changing correct +recorded facts. Successful original-model outcomes remain unchanged. diff --git a/experiments/analysis-efficiency/packet-model-prompt.mjs b/experiments/analysis-efficiency/packet-model-prompt.mjs new file mode 100644 index 000000000..f49348fbd --- /dev/null +++ b/experiments/analysis-efficiency/packet-model-prompt.mjs @@ -0,0 +1,40 @@ +import { readFile } from "node:fs/promises"; +import { parseArgs } from "node:util"; +import { packetRequest } from "./packet-entry.mjs"; + +const { values } = parseArgs({ options: { selection: { type: "string" } } }); +if (!["baseline", "focused", "discovery"].includes(values.selection)) + throw new Error("Select baseline, focused or discovery"); +const question = await readFile( + new URL("./questions/packet-postgres.txt", import.meta.url), + "utf8", +); +const plan = [ + ...(values.selection === "discovery" + ? [ + { + tool: "inspect_packet_fields", + arguments: { + kind: "fields", + protocol: "tcp", + query: "tcp.flags", + limit: 1, + }, + }, + ] + : []), + { + tool: "inspect_packet_capture", + arguments: packetRequest( + "wireshark-pgsql-jdbc.pcap", + "{{artifact_path}}", + values.selection, + ), + }, +]; +process.stdout.write( + question + + "\nExecute this fixed public REA MCP plan in order. These are input selections, not expected answers. Preserve omitted defaults. All calls count toward the comparison.\n" + + JSON.stringify(plan, null, 2) + + "\n", +); diff --git a/experiments/analysis-efficiency/packet-oracle.py b/experiments/analysis-efficiency/packet-oracle.py new file mode 100644 index 000000000..db5797b5f --- /dev/null +++ b/experiments/analysis-efficiency/packet-oracle.py @@ -0,0 +1,180 @@ +"""Independent, narrow oracle for the three pinned Ethernet/IPv4 PCAPs. + +Reads header bytes, not TShark output. No PCAPNG, IP fragmentation or TCP +reassembly support is claimed. DNS compression follows RFC 1035; DHCP options +53/54 follow RFC 2132. Unsupported selected traffic fails rather than guessing. +""" +import argparse +import hashlib +import json +import struct +from pathlib import Path + + +def require(condition, reason): + if not condition: + raise ValueError(reason) + + +def packets(data): + require(len(data) >= 24, 'truncated PCAP header') + magic = data[:4] + require(magic in (b'\xd4\xc3\xb2\xa1', b'\xa1\xb2\xc3\xd4'), + 'oracle requires microsecond classic PCAP') + endian = '<' if magic == b'\xd4\xc3\xb2\xa1' else '>' + require(struct.unpack_from(endian + 'HH', data, 4) == (2, 4), + 'unsupported PCAP version') + require(struct.unpack_from(endian + 'I', data, 20)[0] == 1, + 'oracle requires Ethernet') + offset, frame = 24, 0 + while offset < len(data): + require(offset + 16 <= len(data), 'truncated PCAP record header') + captured, original = struct.unpack_from(endian + 'II', data, offset + 8) + offset += 16 + require(captured <= original and offset + captured <= len(data), + 'invalid captured packet length') + frame += 1 + yield frame, data[offset:offset + captured] + offset += captured + + +def transport(packet): + require(len(packet) >= 14, 'truncated Ethernet header') + if packet[12:14] == b'\x08\x06': + return None # ARP cannot contain the selected TCP/UDP messages. + require(packet[12:14] == b'\x08\x00', 'unsupported Ethernet protocol') + ip = packet[14:] + require(len(ip) >= 20 and ip[0] >> 4 == 4, 'invalid IPv4 header') + header = (ip[0] & 15) * 4 + total = struct.unpack_from('!H', ip, 2)[0] + require(20 <= header <= len(ip) and total >= header, 'invalid IPv4 lengths') + require(total <= len(ip), 'truncated IPv4 packet') + require(struct.unpack_from('!H', ip, 6)[0] & 0x3fff == 0, + 'fragmentation is outside oracle scope') + return ip[9], ip[header:min(total, len(ip))] + + +def dns_name(message, offset): + labels, visited, end = [], set(), None + while True: + require(offset < len(message) and offset not in visited, + 'invalid DNS name or compression cycle') + visited.add(offset) + size = message[offset] + if size & 0xc0 == 0xc0: + require(offset + 1 < len(message), 'truncated DNS compression pointer') + if end is None: + end = offset + 2 + offset = ((size & 0x3f) << 8) | message[offset + 1] + continue + require(size & 0xc0 == 0 and offset + 1 + size <= len(message), + 'invalid DNS label') + offset += 1 + if size == 0: + return '.'.join(labels), end if end is not None else offset + labels.append(message[offset:offset + size].decode('ascii')) + require(sum(len(label) + 1 for label in labels) <= 255, + 'DNS name exceeds protocol length') + offset += size + + +def dhcp_options(payload): + require(len(payload) >= 240 and payload[236:240] == b'\x63\x82\x53\x63', + 'invalid DHCP fixed header or cookie') + options, offset = {}, 240 + while offset < len(payload): + kind = payload[offset] + offset += 1 + if kind == 255: + return options + if kind == 0: + continue + require(offset < len(payload), 'missing DHCP option length') + size = payload[offset] + offset += 1 + require(offset + size <= len(payload), 'truncated DHCP option') + require(kind not in options, 'duplicate DHCP options outside oracle scope') + options[kind] = payload[offset:offset + size] + offset += size + raise ValueError('DHCP end option missing') + + +def analyze(data, kind, row_limit=12): + rows, matching, count = [], 0, 0 + for frame, packet in packets(data): + count = frame + decoded = transport(packet) + if decoded is None: + continue + protocol, segment = decoded + if kind == 'postgres' and protocol == 6 and frame <= 2000: + require(len(segment) >= 20, 'truncated TCP header') + tcp_header = (segment[12] >> 4) * 4 + require(20 <= tcp_header <= len(segment), 'invalid or truncated TCP header/options') + source, destination = struct.unpack_from('!HH', segment) + if 5432 not in (source, destination): + continue + matching += 1 + if len(rows) < row_limit: + rows.append({'frame': frame, 'source_port': source, + 'destination_port': destination, + 'flags': struct.unpack_from('!H', segment, 12)[0] & 0xfff}) + elif kind in ('dns', 'dhcp') and protocol == 17: + require(len(segment) >= 8, 'truncated UDP header') + source, destination, length = struct.unpack_from('!HHH', segment) + selected = 53 in (source, destination) if kind == 'dns' else ( + source, destination) in ((67, 68), (68, 67)) + if not selected: + continue + require(8 <= length <= len(segment), 'truncated selected UDP message') + payload = segment[8:length] + if kind == 'dns': + require(len(payload) >= 12, 'truncated DNS header') + identifier, flags, questions = struct.unpack_from('!HHH', payload) + if not flags & 0x8000 or flags & 15 == 0: + continue + matching += 1 + offset, names, types = 12, [], [] + for _ in range(questions): + name, offset = dns_name(payload, offset) + require(offset + 4 <= len(payload), 'truncated DNS question') + types.append(struct.unpack_from('!H', payload, offset)[0]) + names.append(name) + offset += 4 + rows.append({'frame': frame, 'id': identifier, 'rcode': flags & 15, + 'names': names, 'types': types}) + else: + options = dhcp_options(payload) + require(53 in options and len(options[53]) == 1, + 'missing DHCP message type') + server = options.get(54) + require(server is None or len(server) == 4, 'invalid DHCP server ID') + require(52 not in options, 'DHCP overload outside oracle scope') + matching += 1 + row = {'frame': frame, 'id': struct.unpack_from('!I', payload, 4)[0], + 'message_type': options[53][0], + 'your_address': '.'.join(map(str, payload[16:20]))} + if server is not None: + row['server_identifier'] = '.'.join(map(str, server)) + rows.append(row) + return {'kind': kind, 'capture_frames': count, 'matching_packets': matching, + 'question_input_frames': min(2000, count) if kind == 'postgres' else count, + 'rows': rows, 'scope': 'selected recorded headers; no network completeness claim'} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('kind', choices=['postgres', 'dns', 'dhcp']) + parser.add_argument('path', type=Path) + parser.add_argument('--row-limit', type=int, choices=[12, 24], default=12, + help='PostgreSQL rows; 24 independently checks the second page') + args = parser.parse_args() + with args.path.open('rb') as source: + data = source.read(32 * 1024 * 1024 + 1) + require(len(data) <= 32 * 1024 * 1024, 'oracle input budget exceeded') + print(json.dumps({'sha256': hashlib.sha256(data).hexdigest(), + **analyze(data, args.kind, args.row_limit)}, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/experiments/analysis-efficiency/questions/packet-postgres.txt b/experiments/analysis-efficiency/questions/packet-postgres.txt new file mode 100644 index 000000000..82914ba3e --- /dev/null +++ b/experiments/analysis-efficiency/questions/packet-postgres.txt @@ -0,0 +1,3 @@ +Inspect the selected saved PostgreSQL capture {{artifact_path}} with REA. Within original capture frames 1 through 2000, return the first 12 TCP packets with source or destination port 5432 in original frame order. For each return integer frame, source_port, destination_port and flags (the numeric TCP flags word, not prose). Also report the matching packet count within this 2000-frame question scope. + +Do not claim whole-capture or network completeness. Return a single JSON object with exactly kind ("postgres"), matching_packets, rows, evidence_ids and limitations. Use actual successful REA Evidence IDs. Include a concise limitation explaining the recorded input scope; no extra claims or tool calls are needed. diff --git a/experiments/analysis-efficiency/review-packet.mjs b/experiments/analysis-efficiency/review-packet.mjs new file mode 100644 index 000000000..e7e3b0def --- /dev/null +++ b/experiments/analysis-efficiency/review-packet.mjs @@ -0,0 +1,247 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { isDeepStrictEqual, parseArgs } from "node:util"; +import { packetFacts, packetRequest } from "./packet-entry.mjs"; + +const { values } = parseArgs({ + options: { + trial: { type: "string" }, + oracle: { type: "string" }, + "runtime-root": { type: "string" }, + selection: { type: "string" }, + }, +}); +if ( + !values.trial || + !values.oracle || + !values["runtime-root"] || + !["baseline", "focused", "discovery"].includes(values.selection) +) + throw new Error( + "Use --trial DIR --oracle JSON --runtime-root DIR --selection baseline|focused|discovery", + ); + +const root = values["runtime-root"]; +const { parseEvidence } = await import( + pathToFileURL(resolve(root, "dist/domain/evidence.js")).href +); +const { inspectPacketCaptureInputSchema, packetCaptureSchema } = await import( + pathToFileURL(resolve(root, "dist/domain/captures/packetCapture.js")).href +); +const { inspectPacketFieldsInputSchema, packetFieldsCatalogSchema } = + await import( + pathToFileURL(resolve(root, "dist/domain/captures/packetFields.js")).href + ); +const oracle = JSON.parse(await readFile(values.oracle, "utf8")); +const settings = JSON.parse( + await readFile(resolve(values.trial, "trial-settings.json"), "utf8"), +); +const summary = JSON.parse( + await readFile(resolve(values.trial, "summary.json"), "utf8"), +); +const expected = Object.fromEntries( + ["kind", "matching_packets", "rows"].map((key) => [key, oracle[key]]), +); +const calls = (await readFile(resolve(values.trial, "events.jsonl"), "utf8")) + .split("\n") + .filter(Boolean) + .map(JSON.parse) + .filter( + (event) => + event.type === "item.completed" && event.item?.type === "mcp_tool_call", + ) + .map((event) => event.item); +const reasons = []; +const workflowReasons = []; +let answer; +const captures = new Map(); +try { + if (oracle.kind !== "postgres") + throw new Error("This model rubric is the fixed PostgreSQL question only"); + const expectedCalls = + values.selection === "discovery" + ? ["inspect_packet_fields", "inspect_packet_capture"] + : ["inspect_packet_capture"]; + if ( + !isDeepStrictEqual( + calls.map((call) => call.tool), + expectedCalls, + ) || + calls.some( + (call) => + call.server !== "rea" || call.status !== "completed" || call.error, + ) + ) + workflowReasons.push( + "Completed public REA call sequence differs from the fixed plan", + ); + const ids = new Set(); + const conflictingIds = new Set(); + for (const call of calls) { + if (call.server !== "rea" || call.status !== "completed" || call.error) + continue; + try { + const result = call.result; + if (result?.isError) + throw new Error("Tool failure is not observed packet evidence"); + const text = + result?.content?.filter((item) => item.type === "text") ?? []; + if (!result?.structuredContent && text.length !== 1) + throw new Error("No unambiguous captured Evidence envelope"); + const evidence = result.structuredContent ?? JSON.parse(text[0].text); + parseEvidence(evidence); + if (conflictingIds.has(evidence.evidence_id)) + throw new Error("Previously conflicting Evidence identity"); + if (call.tool === "inspect_packet_fields") { + const input = inspectPacketFieldsInputSchema.parse({ + kind: "fields", + protocol: "tcp", + query: "tcp.flags", + limit: 1, + }); + const report = packetFieldsCatalogSchema.parse( + evidence.normalized_result, + ); + if ( + !isDeepStrictEqual( + inspectPacketFieldsInputSchema.parse(call.arguments), + input, + ) || + !isDeepStrictEqual(evidence.parameters, input) || + !isDeepStrictEqual(report.selection, input) || + evidence.operation !== call.tool || + !report.records.some((row) => row.abbreviation === "tcp.flags") || + evidence.subject?.local_path !== report.tool.path || + evidence.subject?.digest?.sha256 !== report.tool.sha256 + ) + throw new Error( + "Discovery Evidence does not establish the requested installed field", + ); + } else if (call.tool === "inspect_packet_capture") { + const planned = inspectPacketCaptureInputSchema.parse( + packetRequest( + "wireshark-pgsql-jdbc.pcap", + settings.selected_artifact_path, + values.selection, + ), + ); + const input = inspectPacketCaptureInputSchema.parse(call.arguments); + if (!isDeepStrictEqual(input, planned)) + workflowReasons.push("Capture selection differs from the fixed plan"); + const report = packetCaptureSchema.parse(evidence.normalized_result); + if ( + !isDeepStrictEqual(evidence.parameters, input) || + !isDeepStrictEqual(report.selection, input) || + evidence.operation !== call.tool || + evidence.subject?.local_path !== input.path || + evidence.subject?.digest?.sha256 !== oracle.sha256 || + report.artifact.path !== input.path || + report.artifact.sha256 !== oracle.sha256 + ) + throw new Error( + "Capture Evidence differs from the fixed input/byte identity", + ); + const observed = packetFacts("postgres", report); + if ( + captures.has(evidence.evidence_id) && + !isDeepStrictEqual(captures.get(evidence.evidence_id), observed) + ) { + captures.delete(evidence.evidence_id); + ids.delete(evidence.evidence_id); + conflictingIds.add(evidence.evidence_id); + throw new Error("Repeated Evidence identity has inconsistent facts"); + } + captures.set(evidence.evidence_id, observed); + } else { + throw new Error("Call is not a packet observation tool"); + } + ids.add(evidence.evidence_id); + } catch (error) { + workflowReasons.push( + `Call ${call.tool} did not provide authenticated packet observations: ${String(error)}`, + ); + } + } + answer = JSON.parse( + (await readFile(resolve(values.trial, "answer.txt"), "utf8")).trim(), + ); + if ( + !isDeepStrictEqual( + Object.keys(answer).sort(), + [ + "kind", + "matching_packets", + "rows", + "evidence_ids", + "limitations", + ].sort(), + ) + ) + workflowReasons.push("Final answer shape differs from the fixed plan"); + if ( + !isDeepStrictEqual( + Object.fromEntries( + Object.keys(expected).map((key) => [key, answer[key]]), + ), + expected, + ) + ) + throw new Error("Final answer is not the exact requested recorded facts"); + if ( + !Array.isArray(answer.evidence_ids) || + answer.evidence_ids.some((id) => !ids.has(id)) || + new Set(answer.evidence_ids).size !== answer.evidence_ids.length + ) + throw new Error("Final citations lack the authenticated packet Evidence"); + const supportingFacts = answer.evidence_ids + .map((id) => captures.get(id)) + .find((observed) => isDeepStrictEqual(observed, expected)); + if (!supportingFacts) + throw new Error( + "Cited observations differ from independent raw header oracle", + ); + const limitations = + typeof answer.limitations === "string" + ? [answer.limitations] + : answer.limitations; + if ( + !Array.isArray(limitations) || + limitations.length === 0 || + limitations.some((item) => typeof item !== "string" || !item.trim()) + ) + workflowReasons.push("Final answer omits investigation limitations"); + if ( + summary.exit_code !== 0 || + summary.timeout || + summary.selected_artifact_bytes_unchanged !== true || + settings.selected_artifact_sha256_before !== oracle.sha256 || + summary.selected_artifact_sha256_after !== oracle.sha256 + ) + workflowReasons.push( + "Trial did not complete with the selected artifact unchanged", + ); +} catch (error) { + reasons.push(String(error)); +} +const review = { + factual_correctness: reasons.length ? "failed" : "passed", + scope: + "Fixed first-2000-frame PostgreSQL question; authenticated tool observations and independent raw header oracle", + workflow_correctness: + reasons.length || workflowReasons.length ? "failed" : "passed", + reasons, + workflow_reasons: workflowReasons, + completed_calls: calls.length, + wall_seconds: summary.wall_seconds, + usage: summary.usage, + selected_artifact_sha256: oracle.sha256, + limitation_prose: + "Presence checked; wording requires transcript review, not heuristic factual certification", +}; +await writeFile( + resolve(values.trial, "packet-review.json"), + JSON.stringify(review, null, 2) + "\n", +); +console.log(JSON.stringify(review)); +if (reasons.length || workflowReasons.length) process.exitCode = 1; diff --git a/experiments/analysis-efficiency/test_packet_oracle.py b/experiments/analysis-efficiency/test_packet_oracle.py new file mode 100644 index 000000000..6727b3b7e --- /dev/null +++ b/experiments/analysis-efficiency/test_packet_oracle.py @@ -0,0 +1,67 @@ +import importlib.util +import struct +import unittest +from pathlib import Path + +spec = importlib.util.spec_from_file_location('packet_oracle', Path(__file__).with_name('packet-oracle.py')) +oracle = importlib.util.module_from_spec(spec) +spec.loader.exec_module(oracle) + + +class PacketOracleTest(unittest.TestCase): + def test_compression_pointer_preserves_question_cursor(self): + message = b'\x03www\x07example\x03com\0' + b'\xc0\0' + b'\0\x01\0\x01' + self.assertEqual(oracle.dns_name(message, 17), ('www.example.com', 19)) + + def test_compression_cycle_and_missing_pointer_byte_fail(self): + for message in (b'\xc0\0', b'\xc0', b'\xc0\x02\xc0\0'): + with self.subTest(message=message), self.assertRaises(ValueError): + oracle.dns_name(message, 0) + + def test_truncated_pcap_record_is_not_exhaustion(self): + header = struct.pack(' { + const left = fixture("complete", artifact); + left.summary.packet_selection = "baseline"; + left.answer.facts = { + kind: "packet", + matching_packets: 1337, + rows: [{ frame: 9 }], + }; + const right = structuredClone(left); + right.summary.packet_selection = "focused"; + right.answer.delivery_observations[0]!.evidence_id = "ev_focused"; + right.answer.delivery_observations[0]!.coverage = + "up-to-input-packet-limit"; + const equivalent = compare(left, right); + assert.equal(equivalent.status, 0); + assert.equal(equivalent.report.facts_match, true); + assert.deepEqual( + equivalent.report.preserved_run_specific_values.map((row) => row.value), + [left.answer.delivery_observations, right.answer.delivery_observations], + ); + right.answer.facts!.matching_packets++; + assert.equal(compare(left, right).report.facts_match, false); + right.answer.facts!.matching_packets--; + right.summary.packet_selection = "baseline"; + assert.equal(compare(left, right).report.facts_match, false); + }); +} + for (const artifact of [ "wireshark-http.cap", "wireshark-http_redirects.pcapng", diff --git a/tests/evaluation/packetWorkflowSemantics.test.ts b/tests/evaluation/packetWorkflowSemantics.test.ts new file mode 100644 index 000000000..aac1f5976 --- /dev/null +++ b/tests/evaluation/packetWorkflowSemantics.test.ts @@ -0,0 +1,32 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { test } from "vitest"; + +test("packet page verification rejects skipped matches with unchanged counts and monotone frames", () => { + const moduleUrl = new URL( + "../../experiments/analysis-efficiency/packet-entry.mjs", + import.meta.url, + ).href; + const source = ` + import assert from 'node:assert/strict'; + import {verifyPacketPage} from ${JSON.stringify(moduleUrl)}; + const rows = Array.from({length:36}, (_,i)=>({frame:i+1, source_port:1000,destination_port:5432,flags:16})); + const sha256 = 'a'.repeat(64); + const report = (selected)=>({artifact:{sha256},output_window:{matching_packets:36},packets:selected.map(row=>({ + 'frame.number':[String(row.frame)],'tcp.srcport':['1000'],'tcp.dstport':['5432'],'tcp.flags':['0x0010'] + }))}); + const oracle={sha256,matching_packets:36,rows:rows.slice(0,24)}; + const first=report(rows.slice(0,12)); + verifyPacketPage(first,report(rows.slice(12,24)),oracle); + assert.throws(()=>verifyPacketPage(first,report(rows.slice(24,36)),oracle),/independently parsed/); + assert.throws(()=>verifyPacketPage(first,report(rows.slice(12,24).reverse()),oracle),/independently parsed/); + assert.throws(()=>verifyPacketPage(first,{...report(rows.slice(12,24)),artifact:{sha256:'b'.repeat(64)}},oracle),/independently parsed/); + `; + // Semantic checker only; synthetic rows do not establish provider behavior. + const result = spawnSync(process.execPath, ["--input-type=module", "-"], { + input: source, + encoding: "utf8", + }); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stderr); +});