mirror of
https://github.com/morluto/rea.git
synced 2026-10-18 20:10:13 -05:00
test(perf): evaluate larger and non-HTTP packet workflows (#3232)
Add reproducible larger/non-HTTP packet cost measurements, an independent header oracle, exact page validation and separate factual/plan grading. Document caller-selected controls and retain negative/mixed model results. Production defaults and APIs remain unchanged. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
15 files changed
+1260
-1
No files matched your search
@@ -54,6 +54,22 @@ resource limit; use selected fields, a display filter or an input-packet limit
|
||||
when dissection output is too large. Each call checks the recording's current
|
||||
digest, so pages from different source digests must not be combined.
|
||||
|
||||
For a question about a known input prefix, select `packet_limit` for that
|
||||
prefix, the fields needed for the answer, and a matching output window. For
|
||||
example, a question about the first twelve PostgreSQL TCP packets within
|
||||
original frames 1–2,000 can use fields `tcp.srcport`, `tcp.dstport`, `tcp.flags`,
|
||||
filter `tcp.port == 5432`, `packet_limit: 2000` and `match_limit: 12`.
|
||||
The matching count then describes that input prefix. A filter containing
|
||||
`frame.number <= 2000` without an input limit still reads the entire capture;
|
||||
returning twelve rows does not limit producer output. These controls answer a
|
||||
bounded question and must not be used to infer whole-capture absence.
|
||||
|
||||
Discover unfamiliar field names with `inspect_packet_fields`, using an exact
|
||||
protocol and a narrow query. If the installed names are already established,
|
||||
direct field selection avoids an additional glossary probe. Count discovery
|
||||
and any retries when comparing investigation cost; a small page alone does
|
||||
not establish a faster workflow.
|
||||
|
||||
Additional selections:
|
||||
|
||||
- `display_filter`: Wireshark display-filter syntax; omitted means no filter.
|
||||
|
||||
@@ -72,6 +72,12 @@ checkpoint commits retain their original receipts; no history was rewritten.
|
||||
|
||||
## Optimization priorities and gradual migration
|
||||
|
||||
The [larger/non-HTTP packet checkpoint](packet-field-workflows.md) records
|
||||
same-question SDK and actual Codex comparisons using existing input limits,
|
||||
field/filter selection, discovery and output windows. Lower delivered JSON did
|
||||
not establish stable total-token or speed gains; failed plans and negative
|
||||
controls remain visible.
|
||||
|
||||
Agent token consumption and end-to-end task time are the primary objectives.
|
||||
Memory and CPU consumption are also measured and controlled. Correctness,
|
||||
Evidence provenance, caller choices and existing-user compatibility are
|
||||
|
||||
@@ -181,6 +181,7 @@ def main():
|
||||
'inspect_android_class', 'inspect_android_method',
|
||||
'inspect_firmware_regions', 'inspect_managed_artifact',
|
||||
'inspect_sqlite_database', 'inspect_pcap_capture',
|
||||
'inspect_packet_capture', 'inspect_packet_fields',
|
||||
'inspect_managed_members', 'inspect_managed_native_boundaries',
|
||||
'inspect_artifact', 'inspect_wasm_artifact',
|
||||
'inspect_evm_interface', 'get_evidence_bundle', 'export_evidence_bundle',
|
||||
|
||||
@@ -2,6 +2,7 @@ import { isDeepStrictEqual } from "node:util";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
import { parseArgs } from "node:util";
|
||||
import { PACKET_ARTIFACTS } from "./packet-entry.mjs";
|
||||
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
@@ -33,6 +34,8 @@ const sameSettings = [
|
||||
"provider_environment_policy_sha256",
|
||||
"measure_source_sha256",
|
||||
"native_workflow_source_sha256",
|
||||
"packet_workflow_source_sha256",
|
||||
"packet_oracle_source_sha256",
|
||||
"package_lock_sha256",
|
||||
"node_version",
|
||||
"sdk_client_version",
|
||||
@@ -76,6 +79,7 @@ for (const [side, answer] of [
|
||||
// coverage. Repeated equivalent deliveries must also match every observation
|
||||
// field except the receipt and parent IDs, which identify their own invocations.
|
||||
if (
|
||||
Object.hasOwn(PACKET_ARTIFACTS, left.artifact) ||
|
||||
["wireshark-http.cap", "wireshark-http_redirects.pcapng"].includes(
|
||||
left.artifact,
|
||||
)
|
||||
@@ -94,7 +98,11 @@ if (
|
||||
path: ["delivery_observations"],
|
||||
value: answer.delivery_observations,
|
||||
});
|
||||
if (left.delivery === right.delivery) {
|
||||
if (
|
||||
left.delivery === right.delivery &&
|
||||
(!Object.hasOwn(PACKET_ARTIFACTS, left.artifact) ||
|
||||
left.packet_selection === right.packet_selection)
|
||||
) {
|
||||
answer.delivery_observations = answer.delivery_observations.map(
|
||||
(observation) =>
|
||||
Object.fromEntries(
|
||||
|
||||
@@ -291,6 +291,58 @@
|
||||
"upstream_expected_digest": null,
|
||||
"derivation": "Unmodified capture published by official Wireshark wiki. Local digest pins downloaded bytes; no independently published upstream digest is claimed."
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "wireshark-pgsql-jdbc.pcap",
|
||||
"file": "pgsql-jdbc.pcap",
|
||||
"bytes": 4906316,
|
||||
"sha256": "6e68b8b792a0f4336092f98856abc7de8c54b55a29112854d28d676f9093c16b",
|
||||
"provenance": {
|
||||
"project": "Wireshark public sample captures",
|
||||
"source_page": "https://wiki.wireshark.org/SampleCaptures",
|
||||
"url": "https://wiki.wireshark.org/uploads/__moin_import__/attachments/SampleCaptures/pgsql-jdbc.pcap.gz",
|
||||
"resolved_url": "https://gitlab.com/wireshark/wireshark/-/wikis/uploads/__moin_import__/attachments/SampleCaptures/pgsql-jdbc.pcap.gz",
|
||||
"upstream_expected_digest": null,
|
||||
"license": "not stated in inspected upstream catalogue",
|
||||
"derivation": "Unmodified official saved sample; decompressed gzip where indicated. Local digest pins acquired bytes, not an upstream-published digest.",
|
||||
"compressed_source": {
|
||||
"name": "pgsql-jdbc.pcap.gz",
|
||||
"bytes": 590093,
|
||||
"sha256": "15b51bb4cc197753c1ac6b7dc3cf5697f22a81e3bb5d2e1d24d15de469c3a06b"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "wireshark-dns.cap",
|
||||
"file": "dns.cap",
|
||||
"bytes": 4338,
|
||||
"sha256": "041eeb6f98bb398f1ee8b09651b5b5a84f6a62639f95bf226f9e7b77355d9f28",
|
||||
"provenance": {
|
||||
"project": "Wireshark public sample captures",
|
||||
"source_page": "https://wiki.wireshark.org/SampleCaptures",
|
||||
"url": "https://wiki.wireshark.org/uploads/__moin_import__/attachments/SampleCaptures/dns.cap",
|
||||
"resolved_url": "https://gitlab.com/wireshark/wireshark/-/wikis/uploads/__moin_import__/attachments/SampleCaptures/dns.cap",
|
||||
"upstream_expected_digest": null,
|
||||
"license": "not stated in inspected upstream catalogue",
|
||||
"derivation": "Unmodified official saved sample; decompressed gzip where indicated. Local digest pins acquired bytes, not an upstream-published digest.",
|
||||
"compressed_source": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "wireshark-dhcp.pcap",
|
||||
"file": "dhcp.pcap",
|
||||
"bytes": 1400,
|
||||
"sha256": "2471b5420bdac826eecf8f61a2bbb4a3eb20dbfab7c02ff2be502f349f368214",
|
||||
"provenance": {
|
||||
"project": "Wireshark public sample captures",
|
||||
"source_page": "https://wiki.wireshark.org/SampleCaptures",
|
||||
"url": "https://wiki.wireshark.org/uploads/__moin_import__/attachments/SampleCaptures/dhcp.pcap",
|
||||
"resolved_url": "https://gitlab.com/wireshark/wireshark/-/wikis/uploads/__moin_import__/attachments/SampleCaptures/dhcp.pcap",
|
||||
"upstream_expected_digest": null,
|
||||
"license": "not stated in inspected upstream catalogue",
|
||||
"derivation": "Unmodified official saved sample; decompressed gzip where indicated. Local digest pins acquired bytes, not an upstream-published digest.",
|
||||
"compressed_source": null
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { Ajv2020 } from "ajv/dist/2020.js";
|
||||
import { PACKET_ARTIFACTS, measurePacketEntry } from "./packet-entry.mjs";
|
||||
|
||||
const root = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const { values } = parseArgs({
|
||||
@@ -29,6 +30,7 @@ const { values } = parseArgs({
|
||||
"managed-question": { type: "string", default: "inventory" },
|
||||
schemas: { type: "string", default: "default" },
|
||||
"sqlite-selection": { type: "string", default: "all" },
|
||||
"packet-selection": { type: "string", default: "baseline" },
|
||||
},
|
||||
});
|
||||
if (
|
||||
@@ -45,6 +47,15 @@ if (
|
||||
!["inventory", "selected"].includes(values["javascript-question"]) ||
|
||||
!["default", "selected"].includes(values.schemas) ||
|
||||
!["all", "columns"].includes(values["sqlite-selection"]) ||
|
||||
!["baseline", "focused", "discovery", "pages"].includes(
|
||||
values["packet-selection"],
|
||||
) ||
|
||||
(!Object.hasOwn(PACKET_ARTIFACTS, values.artifact) &&
|
||||
values["packet-selection"] !== "baseline") ||
|
||||
(Object.hasOwn(PACKET_ARTIFACTS, values.artifact) &&
|
||||
values.delivery !== "complete") ||
|
||||
(values["packet-selection"] === "pages" &&
|
||||
values.artifact !== "wireshark-pgsql-jdbc.pcap") ||
|
||||
(values.artifact !== "qgis-world-map.gpkg" &&
|
||||
values["sqlite-selection"] !== "all") ||
|
||||
(values.artifact === "qgis-world-map.gpkg" &&
|
||||
@@ -142,6 +153,12 @@ const fingerprints = {
|
||||
native_workflow_source_sha256: await fileSha(
|
||||
fileURLToPath(new URL("./native-entry.mjs", import.meta.url)),
|
||||
),
|
||||
packet_workflow_source_sha256: await fileSha(
|
||||
fileURLToPath(new URL("./packet-entry.mjs", import.meta.url)),
|
||||
),
|
||||
packet_oracle_source_sha256: await fileSha(
|
||||
fileURLToPath(new URL("./packet-oracle.py", import.meta.url)),
|
||||
),
|
||||
runtime_files_sha256: sha(JSON.stringify(runtimeFiles)),
|
||||
runtime_dispatcher_sha256: await fileSha(
|
||||
resolve(values["runtime-root"], "scripts/rea.mjs"),
|
||||
@@ -722,6 +739,12 @@ const inspectPcapTarget = async () => {
|
||||
};
|
||||
|
||||
const workflows = {
|
||||
...Object.fromEntries(
|
||||
Object.keys(PACKET_ARTIFACTS).map((artifact) => [
|
||||
artifact,
|
||||
() => measurePacketEntry({ values, call, inputSha, catalogDefinitions }),
|
||||
]),
|
||||
),
|
||||
"wireshark-http.cap": inspectPcapTarget,
|
||||
"wireshark-http_redirects.pcapng": inspectPcapTarget,
|
||||
"qgis-world-map.gpkg": async () => {
|
||||
@@ -1160,6 +1183,9 @@ const report = {
|
||||
values.artifact === "qgis-world-map.gpkg"
|
||||
? values["sqlite-selection"]
|
||||
: null,
|
||||
packet_selection: Object.hasOwn(PACKET_ARTIFACTS, values.artifact)
|
||||
? values["packet-selection"]
|
||||
: null,
|
||||
provider_settings_sha256: sha(JSON.stringify(selectedProviders)),
|
||||
provider_search_path_sha256: sha(
|
||||
selectedProviders.PATH ?? process.env.PATH ?? "",
|
||||
|
||||
@@ -0,0 +1,316 @@
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
export const verifyPacketPage = (first, next, oracle) => {
|
||||
if (
|
||||
first.artifact.sha256 !== oracle.sha256 ||
|
||||
next.artifact.sha256 !== oracle.sha256 ||
|
||||
first.output_window.matching_packets !== oracle.matching_packets ||
|
||||
next.output_window.matching_packets !== oracle.matching_packets ||
|
||||
!isDeepStrictEqual(
|
||||
packetFacts("postgres", first).rows,
|
||||
oracle.rows.slice(0, 12),
|
||||
) ||
|
||||
!isDeepStrictEqual(
|
||||
packetFacts("postgres", next).rows,
|
||||
oracle.rows.slice(12, 24),
|
||||
)
|
||||
)
|
||||
throw new Error(
|
||||
"Packet pages differ from independently parsed original matches",
|
||||
);
|
||||
};
|
||||
|
||||
export const PACKET_ARTIFACTS = {
|
||||
"wireshark-pgsql-jdbc.pcap": {
|
||||
kind: "postgres",
|
||||
fields: ["tcp.srcport", "tcp.dstport", "tcp.flags"],
|
||||
extra: ["tcp.len"],
|
||||
protocol: "tcp",
|
||||
query: "tcp.flags",
|
||||
discovered: "tcp.flags",
|
||||
filter: "tcp.port == 5432",
|
||||
},
|
||||
"wireshark-dns.cap": {
|
||||
kind: "dns",
|
||||
fields: [
|
||||
"dns.id",
|
||||
"dns.flags.response",
|
||||
"dns.flags.rcode",
|
||||
"dns.qry.name",
|
||||
"dns.qry.type",
|
||||
],
|
||||
extra: ["dns.resp.name", "dns.resp.ttl", "udp.srcport", "udp.dstport"],
|
||||
protocol: "dns",
|
||||
query: "dns.flags.rcode",
|
||||
discovered: "dns.flags.rcode",
|
||||
filter: "dns.flags.response == 1 && dns.flags.rcode != 0",
|
||||
},
|
||||
"wireshark-dhcp.pcap": {
|
||||
kind: "dhcp",
|
||||
fields: [
|
||||
"dhcp.id",
|
||||
"dhcp.option.dhcp",
|
||||
"dhcp.ip.your",
|
||||
"dhcp.option.dhcp_server_id",
|
||||
],
|
||||
extra: [
|
||||
"dhcp.type",
|
||||
"dhcp.ip.client",
|
||||
"dhcp.hw.mac_addr",
|
||||
"udp.srcport",
|
||||
"udp.dstport",
|
||||
],
|
||||
protocol: "dhcp",
|
||||
query: "Option 53:",
|
||||
discovered: "dhcp.option.dhcp",
|
||||
filter: "dhcp",
|
||||
},
|
||||
};
|
||||
|
||||
export const packetRequest = (artifact, path, selection) => {
|
||||
const descriptor = PACKET_ARTIFACTS[artifact];
|
||||
if (!descriptor) throw new Error("Unknown pinned packet question");
|
||||
const baseline = selection === "baseline";
|
||||
return {
|
||||
path,
|
||||
fields: baseline
|
||||
? [...descriptor.fields, ...descriptor.extra]
|
||||
: descriptor.fields,
|
||||
display_filter:
|
||||
descriptor.kind === "postgres" && baseline
|
||||
? `${descriptor.filter} && frame.number <= 2000`
|
||||
: descriptor.kind === "dns" && baseline
|
||||
? "dns"
|
||||
: descriptor.filter,
|
||||
...(descriptor.kind === "postgres" && !baseline
|
||||
? { packet_limit: 2000, match_limit: 12 }
|
||||
: {}),
|
||||
};
|
||||
};
|
||||
|
||||
const number = (row, field) => {
|
||||
const value = row[field];
|
||||
if (
|
||||
!Array.isArray(value) ||
|
||||
value.length !== 1 ||
|
||||
!/^(?:[0-9]+|0x[0-9a-f]+)$/iu.test(value[0])
|
||||
)
|
||||
throw new Error(`Expected one recorded integer for ${field}`);
|
||||
const parsed = Number(value[0]);
|
||||
if (!Number.isSafeInteger(parsed))
|
||||
throw new Error(`Unsafe recorded integer for ${field}`);
|
||||
return parsed;
|
||||
};
|
||||
|
||||
export const packetFacts = (kind, report) => {
|
||||
let packets = report.packets;
|
||||
if (kind === "postgres") packets = packets.slice(0, 12);
|
||||
if (kind === "dns")
|
||||
packets = packets.filter(
|
||||
(row) =>
|
||||
number(row, "dns.flags.response") === 1 &&
|
||||
number(row, "dns.flags.rcode") !== 0,
|
||||
);
|
||||
return {
|
||||
kind,
|
||||
matching_packets:
|
||||
kind === "dns" ? packets.length : report.output_window.matching_packets,
|
||||
rows: packets.map((row) => {
|
||||
const frame = number(row, "frame.number");
|
||||
if (kind === "postgres")
|
||||
return {
|
||||
frame,
|
||||
source_port: number(row, "tcp.srcport"),
|
||||
destination_port: number(row, "tcp.dstport"),
|
||||
flags: number(row, "tcp.flags"),
|
||||
};
|
||||
if (kind === "dns") {
|
||||
const names = row["dns.qry.name"];
|
||||
const types = row["dns.qry.type"];
|
||||
if (
|
||||
!Array.isArray(names) ||
|
||||
names.length === 0 ||
|
||||
!Array.isArray(types) ||
|
||||
types.length !== names.length
|
||||
)
|
||||
throw new Error(
|
||||
"The fixed DNS question requires recorded names and types; missing fields are unknown",
|
||||
);
|
||||
return {
|
||||
frame,
|
||||
id: number(row, "dns.id"),
|
||||
rcode: number(row, "dns.flags.rcode"),
|
||||
names,
|
||||
types: types.map((value) => number({ type: [value] }, "type")),
|
||||
};
|
||||
}
|
||||
const server = row["dhcp.option.dhcp_server_id"];
|
||||
const address = row["dhcp.ip.your"];
|
||||
if (
|
||||
!Array.isArray(address) ||
|
||||
address.length !== 1 ||
|
||||
(server !== undefined &&
|
||||
(!Array.isArray(server) || server.length !== 1))
|
||||
)
|
||||
throw new Error("Expected recorded DHCP address scalar");
|
||||
return {
|
||||
frame,
|
||||
id: number(row, "dhcp.id"),
|
||||
message_type: number(row, "dhcp.option.dhcp"),
|
||||
your_address: address[0],
|
||||
...(server === undefined ? {} : { server_identifier: server[0] }),
|
||||
};
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
export const measurePacketEntry = async ({
|
||||
values,
|
||||
call,
|
||||
inputSha,
|
||||
catalogDefinitions,
|
||||
}) => {
|
||||
const descriptor = PACKET_ARTIFACTS[values.artifact];
|
||||
const { parseEvidence } = await import(
|
||||
pathToFileURL(resolve(values["runtime-root"], "dist/domain/evidence.js"))
|
||||
.href
|
||||
);
|
||||
const { inspectPacketCaptureInputSchema, packetCaptureSchema } = await import(
|
||||
pathToFileURL(
|
||||
resolve(values["runtime-root"], "dist/domain/captures/packetCapture.js"),
|
||||
).href
|
||||
);
|
||||
const { inspectPacketFieldsInputSchema, packetFieldsCatalogSchema } =
|
||||
await import(
|
||||
pathToFileURL(
|
||||
resolve(values["runtime-root"], "dist/domain/captures/packetFields.js"),
|
||||
).href
|
||||
);
|
||||
const observations = [];
|
||||
const selection = values["packet-selection"];
|
||||
const checked = (evidence, operation, input, schema) => {
|
||||
parseEvidence(evidence);
|
||||
const report = schema.parse(evidence.normalized_result);
|
||||
if (
|
||||
evidence.operation !== operation ||
|
||||
!isDeepStrictEqual(evidence.parameters, input) ||
|
||||
!isDeepStrictEqual(report.selection, input)
|
||||
)
|
||||
throw new Error(
|
||||
"Packet Evidence operation or choices differs from the request",
|
||||
);
|
||||
if (
|
||||
operation === "inspect_packet_capture" &&
|
||||
(evidence.subject?.local_path !== values.path ||
|
||||
evidence.subject?.digest?.sha256 !== inputSha ||
|
||||
report.artifact.path !== values.path ||
|
||||
report.artifact.sha256 !== inputSha)
|
||||
)
|
||||
throw new Error("Packet Evidence identifies different capture bytes");
|
||||
if (
|
||||
operation === "inspect_packet_fields" &&
|
||||
(evidence.subject?.local_path !== report.tool.path ||
|
||||
evidence.subject?.digest?.sha256 !== report.tool.sha256)
|
||||
)
|
||||
throw new Error("Field discovery identifies a different executable");
|
||||
observations.push({
|
||||
evidence_id: evidence.evidence_id,
|
||||
operation,
|
||||
selection: report.selection,
|
||||
output_window: report.output_window,
|
||||
read_scope: report.read_scope ?? null,
|
||||
limitations: evidence.limitations,
|
||||
});
|
||||
return report;
|
||||
};
|
||||
if (selection === "discovery") {
|
||||
const input = inspectPacketFieldsInputSchema.parse({
|
||||
kind: "fields",
|
||||
protocol: descriptor.protocol,
|
||||
query: descriptor.query,
|
||||
limit: 1,
|
||||
});
|
||||
const found = checked(
|
||||
await call("inspect_packet_fields", input),
|
||||
"inspect_packet_fields",
|
||||
input,
|
||||
packetFieldsCatalogSchema,
|
||||
);
|
||||
if (
|
||||
!found.records.some((row) => row.abbreviation === descriptor.discovered)
|
||||
)
|
||||
throw new Error(
|
||||
"Selected installation does not advertise the required protocol field",
|
||||
);
|
||||
}
|
||||
const input = inspectPacketCaptureInputSchema.parse(
|
||||
packetRequest(values.artifact, values.path, selection),
|
||||
);
|
||||
for (const name of [
|
||||
"inspect_packet_capture",
|
||||
...(selection === "discovery" ? ["inspect_packet_fields"] : []),
|
||||
])
|
||||
if (!catalogDefinitions.has(name))
|
||||
throw new Error(`Missing live packet contract ${name}`);
|
||||
const report = checked(
|
||||
await call("inspect_packet_capture", input),
|
||||
"inspect_packet_capture",
|
||||
input,
|
||||
packetCaptureSchema,
|
||||
);
|
||||
const facts = packetFacts(descriptor.kind, report);
|
||||
if (selection === "pages") {
|
||||
const { stdout } = await promisify(execFile)(
|
||||
"python3",
|
||||
[
|
||||
fileURLToPath(new URL("./packet-oracle.py", import.meta.url)),
|
||||
"postgres",
|
||||
values.path,
|
||||
"--row-limit",
|
||||
"24",
|
||||
],
|
||||
{ timeout: 30_000, maxBuffer: 128 * 1024 },
|
||||
);
|
||||
const oracle = JSON.parse(stdout);
|
||||
const nextInput = inspectPacketCaptureInputSchema.parse({
|
||||
...input,
|
||||
match_offset: 12,
|
||||
});
|
||||
const next = checked(
|
||||
await call("inspect_packet_capture", nextInput),
|
||||
"inspect_packet_capture",
|
||||
nextInput,
|
||||
packetCaptureSchema,
|
||||
);
|
||||
const repeated = checked(
|
||||
await call("inspect_packet_capture", input),
|
||||
"inspect_packet_capture",
|
||||
input,
|
||||
packetCaptureSchema,
|
||||
);
|
||||
verifyPacketPage(report, next, oracle);
|
||||
if (
|
||||
!isDeepStrictEqual(report.packets, repeated.packets) ||
|
||||
!isDeepStrictEqual(report.output_window, repeated.output_window) ||
|
||||
next.artifact.sha256 !== report.artifact.sha256 ||
|
||||
next.output_window.matching_packets !==
|
||||
report.output_window.matching_packets ||
|
||||
Number(next.packets[0]?.["frame.number"]?.[0]) <=
|
||||
Number(report.packets.at(-1)?.["frame.number"]?.[0])
|
||||
)
|
||||
throw new Error(
|
||||
"Packet page order, digest or repeated observations changed",
|
||||
);
|
||||
}
|
||||
return {
|
||||
artifact_sha256: inputSha,
|
||||
facts,
|
||||
delivery_observations: observations,
|
||||
evidence_ids: [...new Set(observations.map((row) => row.evidence_id))],
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,226 @@
|
||||
# Larger and non-HTTP packet investigation costs
|
||||
|
||||
This closes the bounded experiment in [#3130](https://github.com/morluto/rea/issues/3130).
|
||||
Existing field, filter and input/output-window controls substantially reduce
|
||||
delivered JSON for the selected questions. Actual Codex total input and elapsed
|
||||
time are less consistent. Requiring field discovery for a known field adds cost.
|
||||
These results support documenting caller-selected controls, while retaining the
|
||||
current API, defaults and uncached provider behavior.
|
||||
|
||||
## Runtime and public corpus
|
||||
|
||||
Measurements use immutable main
|
||||
`ec67aebed02dbd20fa5c7703fcd183ad0b1c343c`, Node 24.18.0, the locked MCP client and
|
||||
server, Linux, TShark/capinfos 4.0.17 and util-linux prlimit. The SDK captures live
|
||||
advertised contracts and passes selected definitions to actual public MCP calls.
|
||||
Runtime-file, dispatcher, lockfile, experiment-source and explicit provider
|
||||
configuration digests remain with the private trial records. Subsequent main
|
||||
integration checks do not relabel these measurements.
|
||||
|
||||
Three additional saved recordings come from the
|
||||
[official Wireshark catalogue](https://wiki.wireshark.org/SampleCaptures).
|
||||
`corpus.json` pins their acquisition URLs, exact bytes and SHA-256 values.
|
||||
The PostgreSQL recording is described upstream as a JDBC test session; it is
|
||||
not a representative production workload. Capture-specific licenses were not
|
||||
stated in the inspected catalogue. No captures or raw trial records are
|
||||
redistributed in Git.
|
||||
|
||||
| Recording | Acquired/analyzed size | Recorded frames | Fixed question |
|
||||
| -------------------- | --------------------------------------------: | --------------: | ------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `pgsql-jdbc.pcap.gz` | 590,093 compressed / 4,906,316 expanded bytes | 18,472 | Within original frames 1–2,000, report the first twelve TCP packets using port 5432, their source/destination ports and flags, and the matching count. |
|
||||
| `dns.cap` | 4,338 bytes | 38 | Report nonzero-RCODE DNS responses, transaction IDs and question names/types. |
|
||||
| `dhcp.pcap` | 1,400 bytes | 4 | Report message types, transaction IDs, offered address and present server identifiers in frame order. |
|
||||
|
||||
The larger application recording and two small non-HTTP controls expand the
|
||||
question/protocol coverage. Large DNS/DHCP, PCAPNG non-HTTP, USB, Bluetooth,
|
||||
Windows and macOS provider performance remain unverified.
|
||||
|
||||
`packet-oracle.py` independently reads classic PCAP, Ethernet/IPv4 and selected
|
||||
TCP/UDP headers. It does not consume TShark output. Its narrow supported scope
|
||||
fails on truncation, IP fragmentation, DNS compression cycles, duplicate DHCP
|
||||
options or DHCP overload. TCP header facts follow
|
||||
[RFC 9293](https://www.rfc-editor.org/rfc/rfc9293.html), DNS names/header fields
|
||||
[RFC 1035](https://www.rfc-editor.org/rfc/rfc1035.html), and DHCP options
|
||||
[RFC 2132](https://www.rfc-editor.org/rfc/rfc2132.html).
|
||||
SQL interpretation, TCP reassembly and network completeness are outside the
|
||||
oracle. The question and oracle were fixed before measured cohorts.
|
||||
|
||||
The oracle establishes 1,337 selected PostgreSQL TCP packets in the input prefix
|
||||
and six DNS error responses. DHCP records contain Discover/Offer with transaction
|
||||
ID 15645 and Request/ACK with ID 15646. An absent server identifier stays omitted;
|
||||
the recording does not establish a single transaction containing all four steps.
|
||||
|
||||
## Equivalent questions and SDK measurements
|
||||
|
||||
Each arm uses fresh actual MCP sessions; heavy trials run sequentially.
|
||||
Two interleaved repetitions cover each arm and each recording. Discovery, catalog,
|
||||
RPC, schema checking, encoding and cleanup costs are retained. JSON bytes describe
|
||||
serialized SDK replies, not raw transport bytes or model tokens.
|
||||
|
||||
The PostgreSQL baseline selects `tcp.srcport`, `tcp.dstport`, `tcp.flags` and
|
||||
`tcp.len`, retains the default fifty-row output window and reads to EOF with
|
||||
`tcp.port == 5432 && frame.number <= 2000`. The focused arm selects the three
|
||||
required fields, uses `packet_limit: 2000`, filter `tcp.port == 5432` and
|
||||
`match_limit: 12`. Both establish the same requested prefix facts and count.
|
||||
Their source read scopes differ explicitly: to EOF versus an input limit.
|
||||
Neither scope establishes whole-network completeness.
|
||||
|
||||
DNS focuses the nine-field baseline on five requested fields and the error-response
|
||||
filter. DHCP selects four required fields from the nine-field baseline; all four
|
||||
records are still read. The discovery arm adds one narrow installed-field probe
|
||||
before the focused request. It tests the cost of validating an already specified
|
||||
field, rather than proving a benefit for a previously unknown field or guessing
|
||||
that an installed registration appears in traffic.
|
||||
|
||||
| Question | Baseline reply bytes | Focused reply bytes | Focused change | Discovery + focused bytes | Discovery change |
|
||||
| ----------------- | -------------------: | ------------------: | -------------: | ------------------------: | ---------------: |
|
||||
| PostgreSQL prefix | 23,054 | 13,304 | −42.29% | 21,096 | −8.49% |
|
||||
| DNS errors | 29,002 | 13,424 | −53.71% | 21,226 | −26.81% |
|
||||
| DHCP sequence | 13,770 | 12,020 | −12.71% | 19,840 | +44.08% |
|
||||
|
||||
Both repetitions produce the same reply size in each cell and pass the independent
|
||||
fact comparison. Authentication uses the selected runtime's `parseEvidence` and
|
||||
named input/output schemas, checking operation, normalized caller choices,
|
||||
artifact or executable identity, original frame order and omission semantics.
|
||||
|
||||
| Question | Baseline tool-call time, ms | Focused tool-call time, ms | Discovery + focused tool-call time, ms |
|
||||
| ----------------- | --------------------------- | -------------------------- | -------------------------------------- |
|
||||
| PostgreSQL prefix | 1,670 / 2,255 | 1,708 / 933 | 3,549 / 2,674 |
|
||||
| DNS errors | 892 / 786 | 889 / 778 | 2,506 / 2,608 |
|
||||
| DHCP sequence | 763 / 759 | 755 / 731 | 2,502 / 2,502 |
|
||||
|
||||
Whole SDK workflow times, including connection/catalog, vary from 4.23–8.79 seconds
|
||||
for these arms. The PostgreSQL focused tool time improves in one repetition and
|
||||
regresses slightly in the other. Discovery consistently adds tool work. Two
|
||||
repetitions are descriptive observations, not a statistical speed guarantee.
|
||||
|
||||
A separate first-page, next-page and repeated-first-page sequence costs
|
||||
922 / 804 / 1,244 ms and returns 39,918 reply bytes in total. Each call repeats
|
||||
dissection, reports the same 1,337 matching count and capture digest, and preserves
|
||||
the original frame order. Identical repeated observations retain the same
|
||||
content-addressed Evidence ID; distinct pages have distinct IDs. Page delivery
|
||||
does not establish capture or network completeness. This narrow repeat does not
|
||||
justify introducing a generic cache or weakening provider identity/cleanup rules.
|
||||
|
||||
The two original HTTP controls were also replayed with two SDK pairs each on the
|
||||
new pinned runtime. Useful summary/prefix facts remain equivalent. For `http.cap`,
|
||||
complete delivery returns 7,601 bytes and summary plus a two-URI retained page
|
||||
17,540 bytes (+130.76%); for `http_redirects.pcapng`, 7,535 versus 17,240 bytes
|
||||
(+128.80%). These remain negative controls. The original model observations in
|
||||
[the earlier checkpoint](pcap-small-captures.md) keep their original runtime pin.
|
||||
|
||||
## Actual Codex CLI observations
|
||||
|
||||
Six actual Codex CLI 0.161.0 runs use requested alias `gpt-5.6-terra`, reasoning
|
||||
`high`, a disposable configuration and the pinned REA MCP runtime. Resolved
|
||||
server-side model identity is not inferred. Both tool names are enabled in every
|
||||
arm, exact-name discovery guidance is identical, shell/browser/other-agent tools
|
||||
are disabled and copied authentication is removed after each run.
|
||||
The shared question and generated plans contain no oracle answers.
|
||||
|
||||
Order is baseline 1, focused 1, discovery 1, focused 2, baseline 2, discovery 2.
|
||||
The rubric checks authenticated actual tool results, exact header-derived facts,
|
||||
citations and the planned calls. Answer correctness and adherence to the plan
|
||||
are reported separately. Limitation wording is reviewed in transcripts; a string
|
||||
or string array is accepted because the prompt does not require one representation.
|
||||
|
||||
| Trial | Total input tokens | Cached input tokens | Output tokens | Elapsed seconds | Actual MCP calls | Facts / plan |
|
||||
| ----------- | -----------------: | ------------------: | ------------: | --------------: | ---------------: | ---------------------------------- |
|
||||
| Baseline 1 | 40,649 | 27,904 | 1,574 | 27.989 | 1 | pass / pass |
|
||||
| Focused 1 | 36,642 | 19,968 | 943 | 38.542 | 1 | pass / pass |
|
||||
| Discovery 1 | 52,802 | 33,024 | 1,089 | 44.398 | 2 | pass / pass |
|
||||
| Focused 2 | 40,912 | 22,016 | 726 | 32.520 | 1 | pass / pass |
|
||||
| Baseline 2 | 40,181 | 19,968 | 896 | 40.299 | 1 | pass / pass |
|
||||
| Discovery 2 | 48,754 | 30,976 | 1,780 | 76.081 | 1 | pass / **fail: discovery omitted** |
|
||||
|
||||
The two eligible baseline/focused pairs show total-input changes of −9.86% and
|
||||
+1.82%, and elapsed changes of +37.70% and −19.30%. Thus reduced SDK reply size
|
||||
does not establish a stable total-token or speed improvement in these runs.
|
||||
The first complete discovery comparison adds 29.90% input and 58.63% elapsed time.
|
||||
The second discovery attempt is retained as a workflow failure and cannot serve
|
||||
as a successful two-call discovery comparison. Its cost is not discarded or
|
||||
counted as a passing trial. No model-side recovery or retries occurred in these
|
||||
six attempts.
|
||||
|
||||
An earlier configuration preflight rejected the packet tool under Codex's
|
||||
`approval_policy: never`: 36,651 input tokens, 549 output tokens, 21.603 seconds,
|
||||
one rejected call and no packet observations. The experiment runner now explicitly
|
||||
preapproves the two authorized passive packet tools, consistently across all arms.
|
||||
That failed preflight remains separate from the corrected cohort and its cost is
|
||||
reported here; CLI exit zero alone is never a factual or workflow pass.
|
||||
|
||||
## Resource and compatibility scope
|
||||
|
||||
All engine/model workloads use an owned Linux resource scope with one CPU,
|
||||
3 GiB early-stop threshold, 4 GiB hard memory ceiling, no swap and a 192-task cap.
|
||||
Node's heap is additionally limited to 512 MiB. The nineteen non-HTTP SDK trials
|
||||
have a maximum observed payload-exit memory peak of 860,909,568 bytes; the six model
|
||||
attempts peak at 940,806,144 bytes. Payload-exit counters and cleanup were verified
|
||||
with the observer still alive and report no memory-limit/OOM events in that phase.
|
||||
Final whole-scope counters remain unavailable, so no stronger whole-scope claim
|
||||
is made. CPU usage and observed process ownership remain in each private receipt.
|
||||
|
||||
A full PostgreSQL probe with four selected fields and no input prefix exceeds the
|
||||
existing 4 MiB producer-output ceiling even with the default fifty-row window.
|
||||
It correctly fails instead of delivering partial success. The SDK error retains
|
||||
large truncated diagnostics (5,790,979 serialized bytes); this separate product
|
||||
concern is recorded privately for later repair, outside #3130. Paging alone cannot
|
||||
prevent producer expansion. The successful prefix experiment does not prove that
|
||||
the full recording can be inspected with those selections.
|
||||
|
||||
Only experiment sources, corpus provenance, a question template and guidance
|
||||
change. Production schemas, provider limits, CLI/MCP defaults, Evidence identity
|
||||
and session/resource ownership stay as before. Eight parser regressions retain
|
||||
malformed/truncated-input and omission checks. Existing clients need no migration.
|
||||
|
||||
## Reproduction
|
||||
|
||||
Acquire the exact corpus outside the repository and verify the pinned hashes.
|
||||
Use the pinned Node/npm toolchain, locked dependencies, `npm run build:cached` and
|
||||
an external resource guard before invoking actual engines/models. Supply explicit
|
||||
provider settings in a private JSON file. An example SDK arm is:
|
||||
|
||||
```sh
|
||||
node experiments/analysis-efficiency/measure.mjs \
|
||||
--runtime-root /absolute/pinned/runtime --revision PINNED_SHA \
|
||||
--artifact wireshark-pgsql-jdbc.pcap --path /private/pgsql-jdbc.pcap \
|
||||
--providers /private/providers.json --schemas selected \
|
||||
--packet-selection focused --output /private/new-trial
|
||||
```
|
||||
|
||||
Use `baseline`, `focused`, `discovery`, or the PostgreSQL-only `pages` diagnostic.
|
||||
Run `packet-oracle.py postgres PATH` independently and compare `answer.json` facts.
|
||||
Generate each model prompt with `packet-model-prompt.mjs --selection ARM`, then use
|
||||
the existing `agent-trial.py` with those two enabled tools and context capture.
|
||||
Run `review-packet.mjs` against the actual trial, oracle, runtime and selected arm.
|
||||
Its nonzero exit includes a plan failure even when all requested facts are correct.
|
||||
Retain failed attempts, actual usage, digests, coverage, cleanup and resource
|
||||
receipts alongside passing observations. No raw results need to be committed.
|
||||
|
||||
## Review verification
|
||||
|
||||
The comparator now recognizes the three non-HTTP questions: observations remain
|
||||
attached to each run, while different plans compare requested facts. Repeated
|
||||
identical plans still compare stable observation scope. Capture grading authenticates
|
||||
Evidence against the actual call and assesses plan differences separately; an
|
||||
actual 24-row SDK window replay retains a factual pass and a plan failure. This
|
||||
is a grading replay, not an additional model trial. The stricter header oracle
|
||||
preserves all original sample facts and rejects truncated IPv4/TCP options and
|
||||
unsupported Ethernet protocols. The page diagnostic now derives matches 13–24
|
||||
independently before accepting the second page; skipped, reversed and changed-byte
|
||||
pages are rejected. Original cohort measurements retain their recorded source
|
||||
digests and runtime pin; review checks do not relabel them.
|
||||
|
||||
Capture grading keeps authenticated facts by Evidence ID and evaluates the cited
|
||||
observations. Four replays using actual first-page and second-page SDK Evidence
|
||||
verify that later or reordered pages and repeated reads preserve a correct cited
|
||||
answer, while citing only the wrong page fails factual grading. Extra calls still
|
||||
fail the fixed workflow plan. Regrading all six original model records preserves
|
||||
their outcomes and recorded costs; these replays are not new model trials.
|
||||
|
||||
Failed or malformed calls remain workflow failures and never become authenticated
|
||||
Evidence. Nine failure-injection and answer-shape replays using an actual successful
|
||||
capture verify that later recovery preserves supported facts, while failed-only
|
||||
attempts and missing or unobserved citations fail factual grading. Missing
|
||||
limitations and extra answer keys fail the fixed plan without changing correct
|
||||
recorded facts. Successful original-model outcomes remain unchanged.
|
||||
@@ -0,0 +1,40 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { parseArgs } from "node:util";
|
||||
import { packetRequest } from "./packet-entry.mjs";
|
||||
|
||||
const { values } = parseArgs({ options: { selection: { type: "string" } } });
|
||||
if (!["baseline", "focused", "discovery"].includes(values.selection))
|
||||
throw new Error("Select baseline, focused or discovery");
|
||||
const question = await readFile(
|
||||
new URL("./questions/packet-postgres.txt", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
const plan = [
|
||||
...(values.selection === "discovery"
|
||||
? [
|
||||
{
|
||||
tool: "inspect_packet_fields",
|
||||
arguments: {
|
||||
kind: "fields",
|
||||
protocol: "tcp",
|
||||
query: "tcp.flags",
|
||||
limit: 1,
|
||||
},
|
||||
},
|
||||
]
|
||||
: []),
|
||||
{
|
||||
tool: "inspect_packet_capture",
|
||||
arguments: packetRequest(
|
||||
"wireshark-pgsql-jdbc.pcap",
|
||||
"{{artifact_path}}",
|
||||
values.selection,
|
||||
),
|
||||
},
|
||||
];
|
||||
process.stdout.write(
|
||||
question +
|
||||
"\nExecute this fixed public REA MCP plan in order. These are input selections, not expected answers. Preserve omitted defaults. All calls count toward the comparison.\n" +
|
||||
JSON.stringify(plan, null, 2) +
|
||||
"\n",
|
||||
);
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Independent, narrow oracle for the three pinned Ethernet/IPv4 PCAPs.
|
||||
|
||||
Reads header bytes, not TShark output. No PCAPNG, IP fragmentation or TCP
|
||||
reassembly support is claimed. DNS compression follows RFC 1035; DHCP options
|
||||
53/54 follow RFC 2132. Unsupported selected traffic fails rather than guessing.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import struct
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def require(condition, reason):
|
||||
if not condition:
|
||||
raise ValueError(reason)
|
||||
|
||||
|
||||
def packets(data):
|
||||
require(len(data) >= 24, 'truncated PCAP header')
|
||||
magic = data[:4]
|
||||
require(magic in (b'\xd4\xc3\xb2\xa1', b'\xa1\xb2\xc3\xd4'),
|
||||
'oracle requires microsecond classic PCAP')
|
||||
endian = '<' if magic == b'\xd4\xc3\xb2\xa1' else '>'
|
||||
require(struct.unpack_from(endian + 'HH', data, 4) == (2, 4),
|
||||
'unsupported PCAP version')
|
||||
require(struct.unpack_from(endian + 'I', data, 20)[0] == 1,
|
||||
'oracle requires Ethernet')
|
||||
offset, frame = 24, 0
|
||||
while offset < len(data):
|
||||
require(offset + 16 <= len(data), 'truncated PCAP record header')
|
||||
captured, original = struct.unpack_from(endian + 'II', data, offset + 8)
|
||||
offset += 16
|
||||
require(captured <= original and offset + captured <= len(data),
|
||||
'invalid captured packet length')
|
||||
frame += 1
|
||||
yield frame, data[offset:offset + captured]
|
||||
offset += captured
|
||||
|
||||
|
||||
def transport(packet):
|
||||
require(len(packet) >= 14, 'truncated Ethernet header')
|
||||
if packet[12:14] == b'\x08\x06':
|
||||
return None # ARP cannot contain the selected TCP/UDP messages.
|
||||
require(packet[12:14] == b'\x08\x00', 'unsupported Ethernet protocol')
|
||||
ip = packet[14:]
|
||||
require(len(ip) >= 20 and ip[0] >> 4 == 4, 'invalid IPv4 header')
|
||||
header = (ip[0] & 15) * 4
|
||||
total = struct.unpack_from('!H', ip, 2)[0]
|
||||
require(20 <= header <= len(ip) and total >= header, 'invalid IPv4 lengths')
|
||||
require(total <= len(ip), 'truncated IPv4 packet')
|
||||
require(struct.unpack_from('!H', ip, 6)[0] & 0x3fff == 0,
|
||||
'fragmentation is outside oracle scope')
|
||||
return ip[9], ip[header:min(total, len(ip))]
|
||||
|
||||
|
||||
def dns_name(message, offset):
|
||||
labels, visited, end = [], set(), None
|
||||
while True:
|
||||
require(offset < len(message) and offset not in visited,
|
||||
'invalid DNS name or compression cycle')
|
||||
visited.add(offset)
|
||||
size = message[offset]
|
||||
if size & 0xc0 == 0xc0:
|
||||
require(offset + 1 < len(message), 'truncated DNS compression pointer')
|
||||
if end is None:
|
||||
end = offset + 2
|
||||
offset = ((size & 0x3f) << 8) | message[offset + 1]
|
||||
continue
|
||||
require(size & 0xc0 == 0 and offset + 1 + size <= len(message),
|
||||
'invalid DNS label')
|
||||
offset += 1
|
||||
if size == 0:
|
||||
return '.'.join(labels), end if end is not None else offset
|
||||
labels.append(message[offset:offset + size].decode('ascii'))
|
||||
require(sum(len(label) + 1 for label in labels) <= 255,
|
||||
'DNS name exceeds protocol length')
|
||||
offset += size
|
||||
|
||||
|
||||
def dhcp_options(payload):
|
||||
require(len(payload) >= 240 and payload[236:240] == b'\x63\x82\x53\x63',
|
||||
'invalid DHCP fixed header or cookie')
|
||||
options, offset = {}, 240
|
||||
while offset < len(payload):
|
||||
kind = payload[offset]
|
||||
offset += 1
|
||||
if kind == 255:
|
||||
return options
|
||||
if kind == 0:
|
||||
continue
|
||||
require(offset < len(payload), 'missing DHCP option length')
|
||||
size = payload[offset]
|
||||
offset += 1
|
||||
require(offset + size <= len(payload), 'truncated DHCP option')
|
||||
require(kind not in options, 'duplicate DHCP options outside oracle scope')
|
||||
options[kind] = payload[offset:offset + size]
|
||||
offset += size
|
||||
raise ValueError('DHCP end option missing')
|
||||
|
||||
|
||||
def analyze(data, kind, row_limit=12):
|
||||
rows, matching, count = [], 0, 0
|
||||
for frame, packet in packets(data):
|
||||
count = frame
|
||||
decoded = transport(packet)
|
||||
if decoded is None:
|
||||
continue
|
||||
protocol, segment = decoded
|
||||
if kind == 'postgres' and protocol == 6 and frame <= 2000:
|
||||
require(len(segment) >= 20, 'truncated TCP header')
|
||||
tcp_header = (segment[12] >> 4) * 4
|
||||
require(20 <= tcp_header <= len(segment), 'invalid or truncated TCP header/options')
|
||||
source, destination = struct.unpack_from('!HH', segment)
|
||||
if 5432 not in (source, destination):
|
||||
continue
|
||||
matching += 1
|
||||
if len(rows) < row_limit:
|
||||
rows.append({'frame': frame, 'source_port': source,
|
||||
'destination_port': destination,
|
||||
'flags': struct.unpack_from('!H', segment, 12)[0] & 0xfff})
|
||||
elif kind in ('dns', 'dhcp') and protocol == 17:
|
||||
require(len(segment) >= 8, 'truncated UDP header')
|
||||
source, destination, length = struct.unpack_from('!HHH', segment)
|
||||
selected = 53 in (source, destination) if kind == 'dns' else (
|
||||
source, destination) in ((67, 68), (68, 67))
|
||||
if not selected:
|
||||
continue
|
||||
require(8 <= length <= len(segment), 'truncated selected UDP message')
|
||||
payload = segment[8:length]
|
||||
if kind == 'dns':
|
||||
require(len(payload) >= 12, 'truncated DNS header')
|
||||
identifier, flags, questions = struct.unpack_from('!HHH', payload)
|
||||
if not flags & 0x8000 or flags & 15 == 0:
|
||||
continue
|
||||
matching += 1
|
||||
offset, names, types = 12, [], []
|
||||
for _ in range(questions):
|
||||
name, offset = dns_name(payload, offset)
|
||||
require(offset + 4 <= len(payload), 'truncated DNS question')
|
||||
types.append(struct.unpack_from('!H', payload, offset)[0])
|
||||
names.append(name)
|
||||
offset += 4
|
||||
rows.append({'frame': frame, 'id': identifier, 'rcode': flags & 15,
|
||||
'names': names, 'types': types})
|
||||
else:
|
||||
options = dhcp_options(payload)
|
||||
require(53 in options and len(options[53]) == 1,
|
||||
'missing DHCP message type')
|
||||
server = options.get(54)
|
||||
require(server is None or len(server) == 4, 'invalid DHCP server ID')
|
||||
require(52 not in options, 'DHCP overload outside oracle scope')
|
||||
matching += 1
|
||||
row = {'frame': frame, 'id': struct.unpack_from('!I', payload, 4)[0],
|
||||
'message_type': options[53][0],
|
||||
'your_address': '.'.join(map(str, payload[16:20]))}
|
||||
if server is not None:
|
||||
row['server_identifier'] = '.'.join(map(str, server))
|
||||
rows.append(row)
|
||||
return {'kind': kind, 'capture_frames': count, 'matching_packets': matching,
|
||||
'question_input_frames': min(2000, count) if kind == 'postgres' else count,
|
||||
'rows': rows, 'scope': 'selected recorded headers; no network completeness claim'}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('kind', choices=['postgres', 'dns', 'dhcp'])
|
||||
parser.add_argument('path', type=Path)
|
||||
parser.add_argument('--row-limit', type=int, choices=[12, 24], default=12,
|
||||
help='PostgreSQL rows; 24 independently checks the second page')
|
||||
args = parser.parse_args()
|
||||
with args.path.open('rb') as source:
|
||||
data = source.read(32 * 1024 * 1024 + 1)
|
||||
require(len(data) <= 32 * 1024 * 1024, 'oracle input budget exceeded')
|
||||
print(json.dumps({'sha256': hashlib.sha256(data).hexdigest(),
|
||||
**analyze(data, args.kind, args.row_limit)}, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,3 @@
|
||||
Inspect the selected saved PostgreSQL capture {{artifact_path}} with REA. Within original capture frames 1 through 2000, return the first 12 TCP packets with source or destination port 5432 in original frame order. For each return integer frame, source_port, destination_port and flags (the numeric TCP flags word, not prose). Also report the matching packet count within this 2000-frame question scope.
|
||||
|
||||
Do not claim whole-capture or network completeness. Return a single JSON object with exactly kind ("postgres"), matching_packets, rows, evidence_ids and limitations. Use actual successful REA Evidence IDs. Include a concise limitation explaining the recorded input scope; no extra claims or tool calls are needed.
|
||||
@@ -0,0 +1,247 @@
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { isDeepStrictEqual, parseArgs } from "node:util";
|
||||
import { packetFacts, packetRequest } from "./packet-entry.mjs";
|
||||
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
trial: { type: "string" },
|
||||
oracle: { type: "string" },
|
||||
"runtime-root": { type: "string" },
|
||||
selection: { type: "string" },
|
||||
},
|
||||
});
|
||||
if (
|
||||
!values.trial ||
|
||||
!values.oracle ||
|
||||
!values["runtime-root"] ||
|
||||
!["baseline", "focused", "discovery"].includes(values.selection)
|
||||
)
|
||||
throw new Error(
|
||||
"Use --trial DIR --oracle JSON --runtime-root DIR --selection baseline|focused|discovery",
|
||||
);
|
||||
|
||||
const root = values["runtime-root"];
|
||||
const { parseEvidence } = await import(
|
||||
pathToFileURL(resolve(root, "dist/domain/evidence.js")).href
|
||||
);
|
||||
const { inspectPacketCaptureInputSchema, packetCaptureSchema } = await import(
|
||||
pathToFileURL(resolve(root, "dist/domain/captures/packetCapture.js")).href
|
||||
);
|
||||
const { inspectPacketFieldsInputSchema, packetFieldsCatalogSchema } =
|
||||
await import(
|
||||
pathToFileURL(resolve(root, "dist/domain/captures/packetFields.js")).href
|
||||
);
|
||||
const oracle = JSON.parse(await readFile(values.oracle, "utf8"));
|
||||
const settings = JSON.parse(
|
||||
await readFile(resolve(values.trial, "trial-settings.json"), "utf8"),
|
||||
);
|
||||
const summary = JSON.parse(
|
||||
await readFile(resolve(values.trial, "summary.json"), "utf8"),
|
||||
);
|
||||
const expected = Object.fromEntries(
|
||||
["kind", "matching_packets", "rows"].map((key) => [key, oracle[key]]),
|
||||
);
|
||||
const calls = (await readFile(resolve(values.trial, "events.jsonl"), "utf8"))
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.map(JSON.parse)
|
||||
.filter(
|
||||
(event) =>
|
||||
event.type === "item.completed" && event.item?.type === "mcp_tool_call",
|
||||
)
|
||||
.map((event) => event.item);
|
||||
const reasons = [];
|
||||
const workflowReasons = [];
|
||||
let answer;
|
||||
const captures = new Map();
|
||||
try {
|
||||
if (oracle.kind !== "postgres")
|
||||
throw new Error("This model rubric is the fixed PostgreSQL question only");
|
||||
const expectedCalls =
|
||||
values.selection === "discovery"
|
||||
? ["inspect_packet_fields", "inspect_packet_capture"]
|
||||
: ["inspect_packet_capture"];
|
||||
if (
|
||||
!isDeepStrictEqual(
|
||||
calls.map((call) => call.tool),
|
||||
expectedCalls,
|
||||
) ||
|
||||
calls.some(
|
||||
(call) =>
|
||||
call.server !== "rea" || call.status !== "completed" || call.error,
|
||||
)
|
||||
)
|
||||
workflowReasons.push(
|
||||
"Completed public REA call sequence differs from the fixed plan",
|
||||
);
|
||||
const ids = new Set();
|
||||
const conflictingIds = new Set();
|
||||
for (const call of calls) {
|
||||
if (call.server !== "rea" || call.status !== "completed" || call.error)
|
||||
continue;
|
||||
try {
|
||||
const result = call.result;
|
||||
if (result?.isError)
|
||||
throw new Error("Tool failure is not observed packet evidence");
|
||||
const text =
|
||||
result?.content?.filter((item) => item.type === "text") ?? [];
|
||||
if (!result?.structuredContent && text.length !== 1)
|
||||
throw new Error("No unambiguous captured Evidence envelope");
|
||||
const evidence = result.structuredContent ?? JSON.parse(text[0].text);
|
||||
parseEvidence(evidence);
|
||||
if (conflictingIds.has(evidence.evidence_id))
|
||||
throw new Error("Previously conflicting Evidence identity");
|
||||
if (call.tool === "inspect_packet_fields") {
|
||||
const input = inspectPacketFieldsInputSchema.parse({
|
||||
kind: "fields",
|
||||
protocol: "tcp",
|
||||
query: "tcp.flags",
|
||||
limit: 1,
|
||||
});
|
||||
const report = packetFieldsCatalogSchema.parse(
|
||||
evidence.normalized_result,
|
||||
);
|
||||
if (
|
||||
!isDeepStrictEqual(
|
||||
inspectPacketFieldsInputSchema.parse(call.arguments),
|
||||
input,
|
||||
) ||
|
||||
!isDeepStrictEqual(evidence.parameters, input) ||
|
||||
!isDeepStrictEqual(report.selection, input) ||
|
||||
evidence.operation !== call.tool ||
|
||||
!report.records.some((row) => row.abbreviation === "tcp.flags") ||
|
||||
evidence.subject?.local_path !== report.tool.path ||
|
||||
evidence.subject?.digest?.sha256 !== report.tool.sha256
|
||||
)
|
||||
throw new Error(
|
||||
"Discovery Evidence does not establish the requested installed field",
|
||||
);
|
||||
} else if (call.tool === "inspect_packet_capture") {
|
||||
const planned = inspectPacketCaptureInputSchema.parse(
|
||||
packetRequest(
|
||||
"wireshark-pgsql-jdbc.pcap",
|
||||
settings.selected_artifact_path,
|
||||
values.selection,
|
||||
),
|
||||
);
|
||||
const input = inspectPacketCaptureInputSchema.parse(call.arguments);
|
||||
if (!isDeepStrictEqual(input, planned))
|
||||
workflowReasons.push("Capture selection differs from the fixed plan");
|
||||
const report = packetCaptureSchema.parse(evidence.normalized_result);
|
||||
if (
|
||||
!isDeepStrictEqual(evidence.parameters, input) ||
|
||||
!isDeepStrictEqual(report.selection, input) ||
|
||||
evidence.operation !== call.tool ||
|
||||
evidence.subject?.local_path !== input.path ||
|
||||
evidence.subject?.digest?.sha256 !== oracle.sha256 ||
|
||||
report.artifact.path !== input.path ||
|
||||
report.artifact.sha256 !== oracle.sha256
|
||||
)
|
||||
throw new Error(
|
||||
"Capture Evidence differs from the fixed input/byte identity",
|
||||
);
|
||||
const observed = packetFacts("postgres", report);
|
||||
if (
|
||||
captures.has(evidence.evidence_id) &&
|
||||
!isDeepStrictEqual(captures.get(evidence.evidence_id), observed)
|
||||
) {
|
||||
captures.delete(evidence.evidence_id);
|
||||
ids.delete(evidence.evidence_id);
|
||||
conflictingIds.add(evidence.evidence_id);
|
||||
throw new Error("Repeated Evidence identity has inconsistent facts");
|
||||
}
|
||||
captures.set(evidence.evidence_id, observed);
|
||||
} else {
|
||||
throw new Error("Call is not a packet observation tool");
|
||||
}
|
||||
ids.add(evidence.evidence_id);
|
||||
} catch (error) {
|
||||
workflowReasons.push(
|
||||
`Call ${call.tool} did not provide authenticated packet observations: ${String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
answer = JSON.parse(
|
||||
(await readFile(resolve(values.trial, "answer.txt"), "utf8")).trim(),
|
||||
);
|
||||
if (
|
||||
!isDeepStrictEqual(
|
||||
Object.keys(answer).sort(),
|
||||
[
|
||||
"kind",
|
||||
"matching_packets",
|
||||
"rows",
|
||||
"evidence_ids",
|
||||
"limitations",
|
||||
].sort(),
|
||||
)
|
||||
)
|
||||
workflowReasons.push("Final answer shape differs from the fixed plan");
|
||||
if (
|
||||
!isDeepStrictEqual(
|
||||
Object.fromEntries(
|
||||
Object.keys(expected).map((key) => [key, answer[key]]),
|
||||
),
|
||||
expected,
|
||||
)
|
||||
)
|
||||
throw new Error("Final answer is not the exact requested recorded facts");
|
||||
if (
|
||||
!Array.isArray(answer.evidence_ids) ||
|
||||
answer.evidence_ids.some((id) => !ids.has(id)) ||
|
||||
new Set(answer.evidence_ids).size !== answer.evidence_ids.length
|
||||
)
|
||||
throw new Error("Final citations lack the authenticated packet Evidence");
|
||||
const supportingFacts = answer.evidence_ids
|
||||
.map((id) => captures.get(id))
|
||||
.find((observed) => isDeepStrictEqual(observed, expected));
|
||||
if (!supportingFacts)
|
||||
throw new Error(
|
||||
"Cited observations differ from independent raw header oracle",
|
||||
);
|
||||
const limitations =
|
||||
typeof answer.limitations === "string"
|
||||
? [answer.limitations]
|
||||
: answer.limitations;
|
||||
if (
|
||||
!Array.isArray(limitations) ||
|
||||
limitations.length === 0 ||
|
||||
limitations.some((item) => typeof item !== "string" || !item.trim())
|
||||
)
|
||||
workflowReasons.push("Final answer omits investigation limitations");
|
||||
if (
|
||||
summary.exit_code !== 0 ||
|
||||
summary.timeout ||
|
||||
summary.selected_artifact_bytes_unchanged !== true ||
|
||||
settings.selected_artifact_sha256_before !== oracle.sha256 ||
|
||||
summary.selected_artifact_sha256_after !== oracle.sha256
|
||||
)
|
||||
workflowReasons.push(
|
||||
"Trial did not complete with the selected artifact unchanged",
|
||||
);
|
||||
} catch (error) {
|
||||
reasons.push(String(error));
|
||||
}
|
||||
const review = {
|
||||
factual_correctness: reasons.length ? "failed" : "passed",
|
||||
scope:
|
||||
"Fixed first-2000-frame PostgreSQL question; authenticated tool observations and independent raw header oracle",
|
||||
workflow_correctness:
|
||||
reasons.length || workflowReasons.length ? "failed" : "passed",
|
||||
reasons,
|
||||
workflow_reasons: workflowReasons,
|
||||
completed_calls: calls.length,
|
||||
wall_seconds: summary.wall_seconds,
|
||||
usage: summary.usage,
|
||||
selected_artifact_sha256: oracle.sha256,
|
||||
limitation_prose:
|
||||
"Presence checked; wording requires transcript review, not heuristic factual certification",
|
||||
};
|
||||
await writeFile(
|
||||
resolve(values.trial, "packet-review.json"),
|
||||
JSON.stringify(review, null, 2) + "\n",
|
||||
);
|
||||
console.log(JSON.stringify(review));
|
||||
if (reasons.length || workflowReasons.length) process.exitCode = 1;
|
||||
@@ -0,0 +1,67 @@
|
||||
import importlib.util
|
||||
import struct
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
spec = importlib.util.spec_from_file_location('packet_oracle', Path(__file__).with_name('packet-oracle.py'))
|
||||
oracle = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(oracle)
|
||||
|
||||
|
||||
class PacketOracleTest(unittest.TestCase):
|
||||
def test_compression_pointer_preserves_question_cursor(self):
|
||||
message = b'\x03www\x07example\x03com\0' + b'\xc0\0' + b'\0\x01\0\x01'
|
||||
self.assertEqual(oracle.dns_name(message, 17), ('www.example.com', 19))
|
||||
|
||||
def test_compression_cycle_and_missing_pointer_byte_fail(self):
|
||||
for message in (b'\xc0\0', b'\xc0', b'\xc0\x02\xc0\0'):
|
||||
with self.subTest(message=message), self.assertRaises(ValueError):
|
||||
oracle.dns_name(message, 0)
|
||||
|
||||
def test_truncated_pcap_record_is_not_exhaustion(self):
|
||||
header = struct.pack('<IHHIIII', 0xa1b2c3d4, 2, 4, 0, 0, 65535, 1)
|
||||
with self.assertRaises(ValueError):
|
||||
list(oracle.packets(header + b'\0'))
|
||||
record = struct.pack('<IIII', 0, 0, 20, 20)
|
||||
with self.assertRaises(ValueError):
|
||||
list(oracle.packets(header + record + b'\0' * 19))
|
||||
|
||||
def test_missing_dhcp_server_id_stays_omitted(self):
|
||||
payload = b'\0' * 236 + b'\x63\x82\x53\x63\x35\x01\x01\xff'
|
||||
self.assertEqual(oracle.dhcp_options(payload), {53: b'\x01'})
|
||||
|
||||
def test_dhcp_duplicates_truncation_and_missing_end_fail(self):
|
||||
prefix = b'\0' * 236 + b'\x63\x82\x53\x63'
|
||||
for suffix in (b'\x35\x01\x01\x35\x01\x02\xff', b'\x36\x04\0', b'\x35\x01\x01'):
|
||||
with self.subTest(suffix=suffix), self.assertRaises(ValueError):
|
||||
oracle.dhcp_options(prefix + suffix)
|
||||
|
||||
def test_fragmented_selected_traffic_cannot_pass_header_oracle(self):
|
||||
ip = bytearray(20)
|
||||
ip[0], ip[9] = 0x45, 17
|
||||
struct.pack_into('!H', ip, 2, 20)
|
||||
struct.pack_into('!H', ip, 6, 0x2000)
|
||||
with self.assertRaises(ValueError):
|
||||
oracle.transport(b'\0' * 12 + b'\x08\0' + ip)
|
||||
|
||||
def test_declared_ip_length_and_tcp_options_cannot_be_truncated(self):
|
||||
ip = bytearray(40)
|
||||
ip[0], ip[9], ip[32] = 0x45, 6, 0xf0
|
||||
struct.pack_into('!H', ip, 2, 40)
|
||||
struct.pack_into('!HH', ip, 20, 12345, 5432)
|
||||
packet = b'\0' * 12 + b'\x08\0' + ip
|
||||
header = struct.pack('<IHHIIII', 0xa1b2c3d4, 2, 4, 0, 0, 65535, 1)
|
||||
record = struct.pack('<IIII', 0, 0, len(packet), len(packet))
|
||||
with self.assertRaisesRegex(ValueError, 'TCP header'):
|
||||
oracle.analyze(header + record + packet, 'postgres')
|
||||
struct.pack_into('!H', ip, 2, 60)
|
||||
with self.assertRaisesRegex(ValueError, 'truncated IPv4'):
|
||||
oracle.transport(b'\0' * 12 + b'\x08\0' + ip)
|
||||
|
||||
def test_unknown_link_protocol_does_not_prove_tcp_absence(self):
|
||||
with self.assertRaisesRegex(ValueError, 'unsupported Ethernet'):
|
||||
oracle.transport(b'\0' * 12 + b'\x86\xdd' + b'\0' * 40)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -30,11 +30,17 @@ type Fixture = {
|
||||
delivery: Delivery;
|
||||
provider_search_path_sha256: string;
|
||||
calls: unknown[];
|
||||
packet_selection?: string;
|
||||
};
|
||||
answer: {
|
||||
capture: { coverage: string; capture_start: null };
|
||||
http_request_uris_first2: string[];
|
||||
delivery_observations: Observation[];
|
||||
facts?: {
|
||||
kind: string;
|
||||
matching_packets: number;
|
||||
rows: { frame: number }[];
|
||||
};
|
||||
};
|
||||
};
|
||||
const reportSchema = z.object({
|
||||
@@ -123,6 +129,39 @@ function compare(left: Fixture, right: Fixture) {
|
||||
}
|
||||
}
|
||||
|
||||
for (const artifact of [
|
||||
"wireshark-pgsql-jdbc.pcap",
|
||||
"wireshark-dns.cap",
|
||||
"wireshark-dhcp.pcap",
|
||||
]) {
|
||||
test(`${artifact}: compare requested facts across packet plans and preserve observations`, () => {
|
||||
const left = fixture("complete", artifact);
|
||||
left.summary.packet_selection = "baseline";
|
||||
left.answer.facts = {
|
||||
kind: "packet",
|
||||
matching_packets: 1337,
|
||||
rows: [{ frame: 9 }],
|
||||
};
|
||||
const right = structuredClone(left);
|
||||
right.summary.packet_selection = "focused";
|
||||
right.answer.delivery_observations[0]!.evidence_id = "ev_focused";
|
||||
right.answer.delivery_observations[0]!.coverage =
|
||||
"up-to-input-packet-limit";
|
||||
const equivalent = compare(left, right);
|
||||
assert.equal(equivalent.status, 0);
|
||||
assert.equal(equivalent.report.facts_match, true);
|
||||
assert.deepEqual(
|
||||
equivalent.report.preserved_run_specific_values.map((row) => row.value),
|
||||
[left.answer.delivery_observations, right.answer.delivery_observations],
|
||||
);
|
||||
right.answer.facts!.matching_packets++;
|
||||
assert.equal(compare(left, right).report.facts_match, false);
|
||||
right.answer.facts!.matching_packets--;
|
||||
right.summary.packet_selection = "baseline";
|
||||
assert.equal(compare(left, right).report.facts_match, false);
|
||||
});
|
||||
}
|
||||
|
||||
for (const artifact of [
|
||||
"wireshark-http.cap",
|
||||
"wireshark-http_redirects.pcapng",
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { test } from "vitest";
|
||||
|
||||
test("packet page verification rejects skipped matches with unchanged counts and monotone frames", () => {
|
||||
const moduleUrl = new URL(
|
||||
"../../experiments/analysis-efficiency/packet-entry.mjs",
|
||||
import.meta.url,
|
||||
).href;
|
||||
const source = `
|
||||
import assert from 'node:assert/strict';
|
||||
import {verifyPacketPage} from ${JSON.stringify(moduleUrl)};
|
||||
const rows = Array.from({length:36}, (_,i)=>({frame:i+1, source_port:1000,destination_port:5432,flags:16}));
|
||||
const sha256 = 'a'.repeat(64);
|
||||
const report = (selected)=>({artifact:{sha256},output_window:{matching_packets:36},packets:selected.map(row=>({
|
||||
'frame.number':[String(row.frame)],'tcp.srcport':['1000'],'tcp.dstport':['5432'],'tcp.flags':['0x0010']
|
||||
}))});
|
||||
const oracle={sha256,matching_packets:36,rows:rows.slice(0,24)};
|
||||
const first=report(rows.slice(0,12));
|
||||
verifyPacketPage(first,report(rows.slice(12,24)),oracle);
|
||||
assert.throws(()=>verifyPacketPage(first,report(rows.slice(24,36)),oracle),/independently parsed/);
|
||||
assert.throws(()=>verifyPacketPage(first,report(rows.slice(12,24).reverse()),oracle),/independently parsed/);
|
||||
assert.throws(()=>verifyPacketPage(first,{...report(rows.slice(12,24)),artifact:{sha256:'b'.repeat(64)}},oracle),/independently parsed/);
|
||||
`;
|
||||
// Semantic checker only; synthetic rows do not establish provider behavior.
|
||||
const result = spawnSync(process.execPath, ["--input-type=module", "-"], {
|
||||
input: source,
|
||||
encoding: "utf8",
|
||||
});
|
||||
assert.ifError(result.error);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
});
|
||||
Reference in new issue
Block a user