rea/scripts/verify-signal-recording.mjs

#!/usr/bin/env node
import assert from "node:assert/strict";
import { estimateTokenCount } from "tokenx";
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { readFile, writeFile, readdir } from "node:fs/promises";
import { isAbsolute, join } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { Ajv2020 } from "ajv/dist/2020.js";
import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
import { parseEvidence } from "../dist/domain/evidence.js";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";

if (process.platform !== "linux")
  throw new Error(
    "Real signal verification requires Linux with util-linux prlimit.",
  );
if (!isAbsolute(process.env.REA_SIGROK_COMMAND ?? ""))
  throw new Error(
    "Supply an absolute REA_SIGROK_COMMAND for sigrok-cli 0.7.2.",
  );
const entrypoint =
  process.argv[2] ?? fileURLToPath(new URL("./rea.mjs", import.meta.url));
const run = createVerifierRun();
const root = await PrivateRuntimeRoot.create({
  prefix: "rea-signal-verifier-",
});
const path = join(root.path, "recording with spaces.vcd");
const pagedPath = join(root.path, "paged.vcd");
const execute = promisify(execFile);
const environment = {
  ...process.env,
  TMPDIR: root.path,
  TMP: root.path,
  TEMP: root.path,
};
const client = new Client({ name: "signal-recording-verifier", version: "1" });
const transport = new StdioClientTransport({
  command: process.execPath,
  args: [entrypoint, "mcp"],
  env: environment,
  stderr: "pipe",
});
// Independently synthesize 9600-baud 8N1 samples at 1 MHz: bytes 52 45 41.
const lines = [
  "$timescale 1 us $end",
  "$scope module logic $end",
  "$var wire 1 ! RX $end",
  "$upscope $end",
  "$enddefinitions $end",
  "#0",
  "1!",
];
let tick = 1000;
for (const byte of Buffer.from("REA")) {
  for (const bit of [
    0,
    ...Array.from({ length: 8 }, (_, i) => (byte >> i) & 1),
    1,
  ]) {
    lines.push(`#${tick}`, `${bit}!`);
    tick += 104;
  }
  tick += 208;
}
lines.push(`#${tick + 1000}`, "1!");
const source = lines.join("\n") + "\n";
let cases = 0;
const discoveryMeasurements = [];
const inheritedAddressSpace = (await readFile("/proc/self/limits", "utf8"))
  .split("\n")
  .find((line) => line.startsWith("Max address space "))
  ?.trim()
  .match(/^Max address space\s+(\S+)\s+(\S+)\s+bytes$/u);
assert.ok(inheritedAddressSpace);
const expectedAddressSpace = Math.min(
  1073741824,
  ...inheritedAddressSpace
    .slice(1)
    .map((value) => (value === "unlimited" ? Infinity : Number(value))),
);

try {
  await writeFile(path, source);
  await client.connect(transport);
  const advertised = (await client.listTools()).tools.find(
    ({ name }) => name === "decode_signal_recording",
  );
  assert.ok(advertised?.outputSchema);
  const ajv = new Ajv2020({ strict: false, validateFormats: false });
  assert.ok(ajv.validateSchema(advertised.inputSchema));
  assert.ok(ajv.validateSchema(advertised.outputSchema));
  for (const extra of [
    { input_format: "vcd", annotations: "uart=rx-data" },
    {},
  ]) {
    const input = { path, decoders: "uart:rx=RX:baudrate=9600", ...extra };
    const response = await client.callTool({
      name: advertised.name,
      arguments: input,
    });
    assert.notEqual(response.isError, true, JSON.stringify(response));
    assert.ok(
      ajv.validate(advertised.outputSchema, response.structuredContent),
      JSON.stringify(ajv.errors),
    );
    const evidence = parseEvidence(response.structuredContent);
    const report = evidence.normalized_result;
    for (const byte of ["52", "45", "41"])
      assert.match(
        report.annotations_text,
        new RegExp(`\\d+-\\d+ uart-1: ${byte}\\n`),
      );
    assert.equal(
      report.artifact.sha256,
      createHash("sha256").update(source).digest("hex"),
    );
    assert.equal(report.hardware_acquisition, "not-performed");
    assert.equal(report.limits.address_space_bytes, expectedAddressSpace);
    assert.equal(report.tool.configured_path, process.env.REA_SIGROK_COMMAND);
    assert.equal(
      report.resource_limiter.configured_path,
      process.env.REA_SIGROK_PRLIMIT_COMMAND ?? "/usr/bin/prlimit",
    );
    assert.ok(
      report.resource_limiter.arguments.includes(
        `--as=${expectedAddressSpace}:${expectedAddressSpace}`,
      ),
    );
    assert.equal(report.diagnostics, "");
    const flags = Object.entries(input)
      .filter(([key]) => key !== "path")
      .flatMap(([key, value]) => [`--${key}`, value]);
    const cli = JSON.parse(
      (
        await execute(
          process.execPath,
          [
            entrypoint,
            "decode-signal-recording",
            path,
            ...flags,
            "--format",
            "json",
          ],
          { env: environment, timeout: 60000, maxBuffer: 16 * 1024 * 1024 },
        )
      ).stdout,
    );
    assert.equal(parseEvidence(cli).evidence_id, evidence.evidence_id);
    const bundle = await client.callTool({
      name: "get_evidence_bundle",
      arguments: {},
    });
    assert.ok(JSON.stringify(bundle).includes(evidence.evidence_id));
    assert.equal(await readFile(path, "utf8"), source);
    cases++;
  }
  const pagedLines = lines.slice(0, 7);
  let pagedTick = 1000;
  for (const byte of Array.from({ length: 120 }, (_, i) => i)) {
    for (const bit of [
      0,
      ...Array.from({ length: 8 }, (_, i) => (byte >> i) & 1),
      1,
    ]) {
      pagedLines.push(`#${pagedTick}`, `${bit}!`);
      pagedTick += 104;
    }
    pagedTick += 208;
  }
  pagedLines.push(`#${pagedTick + 1000}`, "1!");
  const pagedSource = pagedLines.join("\n") + "\n";
  await writeFile(pagedPath, pagedSource);
  for (const [options, first, count] of [
    [{}, 0, 50],
    [{ line_offset: 50, line_limit: 40 }, 50, 40],
    [{ line_limit: null }, 0, 120],
  ]) {
    const input = {
      path: pagedPath,
      decoders: "uart:rx=RX:baudrate=9600",
      input_format: "vcd",
      annotations: "uart=rx-data",
      ...options,
    };
    const response = await client.callTool({
      name: advertised.name,
      arguments: input,
    });
    assert.notEqual(response.isError, true, JSON.stringify(response));
    assert.ok(
      ajv.validate(advertised.outputSchema, response.structuredContent),
      JSON.stringify(ajv.errors),
    );
    const evidence = parseEvidence(response.structuredContent);
    const report = evidence.normalized_result;
    assert.deepEqual(report.output_window, {
      total_lines: 120,
      returned_lines: count,
      has_more: first + count < 120,
    });
    const observed = report.annotations_text
      .trimEnd()
      .split("\n")
      .map((line) => {
        const match = /^\d+-\d+ uart-1: ([0-9A-F]{2})$/.exec(line);
        assert.ok(match, line);
        return Number.parseInt(match[1], 16);
      });
    assert.deepEqual(
      observed,
      Array.from({ length: count }, (_, i) => first + i),
    );
    const flags = Object.entries(input)
      .filter(([key]) => key !== "path")
      .flatMap(([key, value]) =>
        key === "line_limit" && value === null
          ? ["--all_lines"]
          : [`--${key}`, String(value)],
      );
    const cli = JSON.parse(
      (
        await execute(
          process.execPath,
          [
            entrypoint,
            "decode-signal-recording",
            pagedPath,
            ...flags,
            "--format",
            "json",
          ],
          { env: environment, timeout: 60000, maxBuffer: 16 * 1024 * 1024 },
        )
      ).stdout,
    );
    assert.equal(parseEvidence(cli).evidence_id, evidence.evidence_id);
    assert.equal(await readFile(pagedPath, "utf8"), pagedSource);
    cases++;
  }
  for (const input of [
    { path, decoders: "no_such_decoder" },
    { path: join(root.path, "absent.vcd"), decoders: "uart" },
    { path, decoders: "uart", driver: "demo" },
  ]) {
    const response = await client.callTool({
      name: advertised.name,
      arguments: input,
    });
    assert.equal(response.isError, true);
    await client.ping();
    cases++;
  }
  await writeFile(path, "not a recording");
  const invalid = await client.callTool({
    name: advertised.name,
    arguments: { path, decoders: "uart" },
  });
  assert.equal(invalid.isError, true);
  cases++;
  const discoveryTool = (await client.listTools()).tools.find(
    ({ name }) => name === "inspect_signal_decoders",
  );
  assert.ok(discoveryTool?.outputSchema);
  assert.ok(ajv.validateSchema(discoveryTool.inputSchema));
  assert.ok(ajv.validateSchema(discoveryTool.outputSchema));
  const nativeHelp = (
    await execute(
      process.env.REA_SIGROK_COMMAND,
      ["--dont-scan", "--protocol-decoders=uart", "--show"],
      { env: environment, timeout: 30000, maxBuffer: 4 * 1024 * 1024 },
    )
  ).stdout.split(/\r?\n/);
  if (nativeHelp.at(-1) === "") nativeHelp.pop();
  const nativeCatalogText = (
    await execute(
      process.env.REA_SIGROK_COMMAND,
      ["--dont-scan", "--list-supported"],
      { env: environment, timeout: 30000, maxBuffer: 4 * 1024 * 1024 },
    )
  ).stdout;
  const catalogHeading = "Supported protocol decoders:\n";
  assert.ok(nativeCatalogText.includes(catalogHeading));
  // Independent native-output oracle, without importing the production parser.
  const nativeCatalog = nativeCatalogText
    .split(catalogHeading)[1]
    .trim()
    .split("\n")
    .map((line) => {
      const record = line.trimStart();
      const separator = record.indexOf(" ");
      assert.ok(separator > 0);
      return {
        id: record.slice(0, separator),
        name: record.slice(separator).trimStart(),
      };
    });
  assert.ok(nativeCatalog.length > 0);
  for (const input of [
    {},
    { query: "UaRt" },
    { limit: 1 },
    { offset: 1, limit: 2 },
    { limit: null },
    { offset: nativeCatalog.length, limit: 2 },
    { decoder: "uart" },
    { decoder: "uart", query: "BaUd" },
    { decoder: "uart", offset: 50, limit: null },
    { decoder: "uart", query: "[" },
    { query: "rea_no_such_decoder" },
  ]) {
    const started = performance.now();
    const response = await client.callTool({
      name: discoveryTool.name,
      arguments: input,
    });
    assert.notEqual(response.isError, true, JSON.stringify(response));
    assert.ok(
      ajv.validate(discoveryTool.outputSchema, response.structuredContent),
      JSON.stringify(ajv.errors),
    );
    const evidence = parseEvidence(response.structuredContent);
    const report = evidence.normalized_result;
    discoveryMeasurements.push({
      input,
      elapsed_ms: +(performance.now() - started).toFixed(2),
      response_bytes: Buffer.byteLength(JSON.stringify(response)),
      estimated_tokens_one_tool_request_response: estimateTokenCount(
        JSON.stringify(discoveryTool) +
          JSON.stringify(input) +
          JSON.stringify(response),
      ),
    });
    const nativeRecords = input.decoder
      ? nativeHelp.map((text, i) => ({ line_number: i + 1, text }))
      : nativeCatalog;
    const matched = nativeRecords.filter((record) =>
      ("text" in record ? record.text : `${record.id} ${record.name}`)
        .toLowerCase()
        .includes((input.query ?? "").toLowerCase()),
    );
    const offset = input.offset ?? 0;
    const page = matched.slice(
      offset,
      input.limit === null ? undefined : offset + (input.limit ?? 50),
    );
    assert.deepEqual(input.decoder ? report.help_lines : report.decoders, page);
    assert.deepEqual(input.decoder ? report.decoders : report.help_lines, []);
    assert.deepEqual(report.output_window, {
      unit: input.decoder ? "help-lines" : "decoders",
      examined: nativeRecords.length,
      matching: matched.length,
      returned: page.length,
      has_more: offset + page.length < matched.length,
    });
    const flags = Object.entries(input).flatMap(([key, value]) =>
      value === null ? ["--all"] : [`--${key}`, String(value)],
    );
    const cli = JSON.parse(
      (
        await execute(
          process.execPath,
          [entrypoint, "inspect-signal-decoders", ...flags, "--format", "json"],
          { env: environment, timeout: 60000, maxBuffer: 8 * 1024 * 1024 },
        )
      ).stdout,
    );
    assert.deepEqual(cli.normalized_result, report);
    cases++;
  }
  const unavailable = await client.callTool({
    name: discoveryTool.name,
    arguments: { decoder: "rea_no_such_decoder" },
  });
  assert.equal(unavailable.isError, true);
  cases++;
  await client.close();
  assert.deepEqual((await readdir(root.path)).sort(), [
    "paged.vcd",
    "recording with spaces.vcd",
  ]);
} finally {
  await client.close();
  await transport.close();
  await root.close();
}
console.log(
  JSON.stringify(
    {
      status: "passed",
      cases,
      discovery_measurements: discoveryMeasurements,
      measurement_scope:
        "Warm MCP server, fresh sigrok processes; tokenx estimates include one advertised tool/request/response, not billed model usage. No model-driven selection benchmark, CPU or peak process-family memory measurement; no cache or speedup claim.",
      host: `${process.platform}/${process.arch}`,
      scope:
        "real sigrok UART VCD through compiled CLI/MCP; no physical acquisition",
      ...(await completeVerifierRun(run)),
    },
    null,
    2,
  ),
);