diff --git a/scripts/verify-reconstruction-readiness.mjs b/scripts/verify-reconstruction-readiness.mjs new file mode 100644 index 0000000000..ed8caa4fa9 --- /dev/null +++ b/scripts/verify-reconstruction-readiness.mjs @@ -0,0 +1,261 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { access, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { promisify } from "node:util"; + +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; + +import { CATALOG_IDENTITY } from "../dist/catalogIdentity.js"; +import { RECONSTRUCTION_READINESS_EXAMPLE } from "../dist/contracts/reconstructionReadinessExample.js"; +import { parseEvidence } from "../dist/domain/evidence.js"; +import { createEvidenceBundle } from "../dist/domain/evidenceBundle.js"; +import { reconstructionReadinessReportSchema } from "../dist/domain/reconstructionReadinessSchemas.js"; +import { PACKAGE_METADATA } from "../dist/generatedPackageMetadata.js"; + +const execute = promisify(execFile); +const root = process.cwd(); +const nativePath = resolve(root, "build/conformance/c"); +const fixtureRoot = resolve(root, "tests/conformance/readiness"); +const javascriptPath = join(fixtureRoot, "javascript-cli"); +const electronPath = join(fixtureRoot, "electron"); +const environment = { + ...process.env, + REA_INVESTIGATION_INPUT_ROOTS_JSON: JSON.stringify([fixtureRoot]), +}; +const temporaryRoot = await mkdtemp(join(tmpdir(), "rea-readiness-")); + +await access(nativePath); +const nativeEvidence = await runCli(["inspect-artifact", nativePath, "--json"]); +const javascriptCliEvidence = await runCli([ + "analyze-javascript-application", + javascriptPath, + "--approved", + "--json", +]); +const electronCliEvidence = await runCli([ + "analyze-javascript-application", + electronPath, + "--approved", + "--json", +]); + +const transport = new StdioClientTransport({ + command: process.execPath, + args: ["scripts/rea.mjs", "mcp"], + cwd: root, + env: environment, + stderr: "pipe", +}); +const client = new Client({ + name: "rea-readiness-verifier", + version: "1", + capabilities: { elicitation: {} }, +}); + +try { + await client.connect(transport); + const catalog = await client.listTools(); + const readinessTool = requireTool( + catalog.tools, + "evaluate_reconstruction_readiness", + ); + const analysisTool = requireTool( + catalog.tools, + "analyze_javascript_application", + ); + assertExactAstLimit(analysisTool); + assert.equal( + readinessTool.annotations?.readOnlyHint, + false, + "readiness Evidence retention side effect must stay visible", + ); + + const javascriptMcp = await analyzeMcp(client, javascriptPath); + const electronMcp = await analyzeMcp(client, electronPath); + assert.equal( + javascriptMcp.evidence_id, + javascriptCliEvidence.evidence_id, + "CLI/MCP JavaScript analysis Evidence identity drifted", + ); + assert.equal( + electronMcp.evidence_id, + electronCliEvidence.evidence_id, + "CLI/MCP Electron analysis Evidence identity drifted", + ); + + const input = structuredClone(RECONSTRUCTION_READINESS_EXAMPLE); + input.identity.cli_version = PACKAGE_METADATA.version; + input.identity.server_version = PACKAGE_METADATA.version; + input.identity.catalog_digest = CATALOG_IDENTITY.digests.combined_sha256; + const records = [ + ...input.evidence_bundle.records, + nativeEvidence, + javascriptCliEvidence, + electronCliEvidence, + ].map(parseEvidence); + input.evidence_bundle = createEvidenceBundle(records); + setFixtureDigest(input, "native", nativeEvidence); + setFixtureDigest(input, "javascript-cli", javascriptCliEvidence); + setFixtureDigest(input, "electron", electronCliEvidence); + linkCheck(input, "discover-classify", "artifact-inventory", nativeEvidence); + linkCheck(input, "static-analysis", "native-routed", nativeEvidence); + linkCheck( + input, + "static-analysis", + "javascript-routed", + javascriptCliEvidence, + ); + linkCheck( + input, + "reactive-scenarios", + "electron-correlated", + electronCliEvidence, + ); + + const inputPath = join(temporaryRoot, "input.json"); + await writeFile(inputPath, JSON.stringify(input)); + const cliEvidence = await runCli([ + "evaluate-reconstruction-readiness", + inputPath, + "--json", + ]); + const cliReport = reconstructionReadinessReportSchema.parse( + cliEvidence.normalized_result, + ); + assert.equal(cliReport.status, "pass"); + if (process.env.REA_READINESS_REPORT_PATH !== undefined) + await writeFile( + resolve(root, process.env.REA_READINESS_REPORT_PATH), + `${JSON.stringify(cliReport, null, 2)}\n`, + ); + + const mcpResult = await client.callTool({ + name: "evaluate_reconstruction_readiness", + arguments: input, + }); + assert.notEqual(mcpResult.isError, true); + const projection = requireObject(mcpResult.structuredContent); + const projectedReport = requireObject(projection.result); + assert.equal(projectedReport.report_digest, cliReport.report_digest); + assert.equal(projectedReport.status, "pass"); + const reportUri = projectedReport.report_resource_uri; + assert.equal(typeof reportUri, "string"); + const resource = await client.readResource({ uri: reportUri }); + const resourceContent = resource.contents[0]; + if (resourceContent === undefined || !("text" in resourceContent)) + throw new TypeError("Readiness report resource is missing JSON text"); + const retained = JSON.parse(resourceContent.text); + assert.equal(retained.report.report_digest, cliReport.report_digest); + + const tampered = structuredClone(input); + tampered.replay.expected_source_digest = cliReport.source_digest; + tampered.identity.server_version = "tampered"; + const tamperedPath = join(temporaryRoot, "tampered.json"); + await writeFile(tamperedPath, JSON.stringify(tampered)); + const tamperedEvidence = await runCli([ + "evaluate-reconstruction-readiness", + tamperedPath, + "--json", + ]); + const tamperedReport = reconstructionReadinessReportSchema.parse( + tamperedEvidence.normalized_result, + ); + assert.equal(tamperedReport.status, "fail"); + assert.ok( + tamperedReport.findings.some( + ({ code }) => code === "replay-source-digest-mismatch", + ), + ); + + process.stdout.write( + `${JSON.stringify({ + schema_version: 1, + status: cliReport.status, + report_id: cliReport.report_id, + report_digest: cliReport.report_digest, + stages: cliReport.summary.stages, + findings: cliReport.summary.findings, + native_evidence_id: nativeEvidence.evidence_id, + javascript_evidence_id: javascriptCliEvidence.evidence_id, + electron_evidence_id: electronCliEvidence.evidence_id, + provider: "local-stdio", + })}\n`, + ); +} finally { + await client.close(); + await rm(temporaryRoot, { recursive: true, force: true }); +} + +async function runCli(arguments_) { + const { stdout } = await execute( + process.execPath, + ["scripts/rea.mjs", ...arguments_], + { + cwd: root, + env: environment, + maxBuffer: 64 * 1_024 * 1_024, + }, + ); + return parseEvidence(JSON.parse(stdout)); +} + +async function analyzeMcp(client_, inputPath) { + const result = await client_.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: inputPath, approved: true }, + }); + assert.notEqual(result.isError, true); + const projected = requireObject(result.structuredContent); + assert.equal(typeof projected.evidence_id, "string"); + const evidence = await client_.readResource({ + uri: `rea://evidence/${projected.evidence_id}`, + }); + const content = evidence.contents[0]; + if (content === undefined || !("text" in content)) + throw new TypeError("Analysis Evidence resource is missing JSON text"); + return parseEvidence(JSON.parse(content.text)); +} + +function requireTool(tools, name) { + const tool = tools.find(({ name: toolName }) => toolName === name); + if (tool === undefined) throw new Error(`Missing public MCP tool: ${name}`); + return tool; +} + +function assertExactAstLimit(tool) { + const properties = requireObject(tool.inputSchema.properties); + const limits = requireObject(properties.limits); + const limitProperties = requireObject(limits.properties); + const astLimit = requireObject(limitProperties.max_ast_nodes); + assert.equal(astLimit.default, 2_000_000, "effective AST limit is hidden"); + assert.equal(astLimit.maximum, 20_000_000, "schema AST limit is hidden"); +} + +function requireObject(value) { + if (typeof value !== "object" || value === null || Array.isArray(value)) + throw new TypeError("Expected an object"); + return value; +} + +function setFixtureDigest(input, kind, evidence) { + const fixture = input.fixtures.find( + ({ kind: fixtureKind }) => fixtureKind === kind, + ); + if (fixture === undefined || evidence.subject === null) + throw new TypeError(`Missing ${kind} fixture identity`); + fixture.artifact_sha256 = evidence.subject.digest.sha256; +} + +function linkCheck(input, stageId, checkId, evidence) { + const stage = input.stages.find(({ stage_id: id }) => id === stageId); + const check = stage?.checks.find(({ check_id: id }) => id === checkId); + if (stage === undefined || check === undefined) + throw new TypeError(`Missing readiness check ${stageId}/${checkId}`); + stage.evidence_ids = [ + ...new Set([...stage.evidence_ids, evidence.evidence_id]), + ].sort(); + check.evidence_ids = [evidence.evidence_id]; +} diff --git a/src/application/ReconstructionReadinessService.ts b/src/application/ReconstructionReadinessService.ts new file mode 100644 index 0000000000..e7da145ffd --- /dev/null +++ b/src/application/ReconstructionReadinessService.ts @@ -0,0 +1,67 @@ +import { z } from "zod"; + +import { + AnalysisInputError, + AnalysisProtocolError, + type AnalysisError, +} from "../domain/errors.js"; +import { createEvidence, type Evidence } from "../domain/evidence.js"; +import { jsonObjectSchema, jsonValueSchema } from "../domain/jsonValue.js"; +import { createReconstructionReadinessReport } from "../domain/reconstructionReadiness.js"; +import { + reconstructionReadinessInputSchema, + type ReconstructionReadinessInput, +} from "../domain/reconstructionReadinessSchemas.js"; +import { err, ok, type Result } from "../domain/result.js"; +import { JAVASCRIPT_APPLICATION_WORKFLOW_PROVIDER } from "./InvestigationProviders.js"; + +const OPERATION = "evaluate_reconstruction_readiness" as const; + +/** Parse one portable readiness request for both CLI and MCP adapters. */ +export const resolveReconstructionReadinessRequest = ( + input: unknown, +): Result => { + const parsed = reconstructionReadinessInputSchema.safeParse(input); + return parsed.success + ? ok(parsed.data) + : err(new AnalysisInputError(OPERATION, { cause: parsed.error })); +}; + +/** Evaluate and wrap one deterministic readiness report in Evidence v2. */ +export const evaluateReconstructionReadinessValidated = ( + input: ReconstructionReadinessInput, +): Result => { + try { + const report = createReconstructionReadinessReport(input); + return ok( + createEvidence(undefined, JAVASCRIPT_APPLICATION_WORKFLOW_PROVIDER, { + predicateType: "rea.reconstruction-readiness-report/v1", + operation: OPERATION, + parameters: jsonObjectSchema.parse({ + source_digest: report.source_digest, + expected_source_digest: input.replay.expected_source_digest, + }), + result: jsonValueSchema.parse(report), + rawResult: null, + confidence: "inferred", + authority: "analyst-inference", + environment: null, + limitations: + report.status === "pass" + ? [] + : [ + "Aggregate pass is withheld until every required conformance stage passes.", + ], + evidenceLinks: report.evidence_links, + }), + ); + } catch (cause: unknown) { + if (cause instanceof z.ZodError) + return err(new AnalysisInputError(OPERATION, { cause })); + return err( + new AnalysisProtocolError("Reconstruction readiness evaluation failed", { + cause, + }), + ); + } +}; diff --git a/src/contracts/reconstructionReadinessExample.ts b/src/contracts/reconstructionReadinessExample.ts new file mode 100644 index 0000000000..7780f27a28 --- /dev/null +++ b/src/contracts/reconstructionReadinessExample.ts @@ -0,0 +1,447 @@ +import { createEvidence } from "../domain/evidence.js"; +import { createEvidenceBundle } from "../domain/evidenceBundle.js"; +import { + READINESS_REQUIRED_CHECKS, + READINESS_STAGE_IDS, +} from "../domain/reconstructionReadinessJourneyFindings.js"; +import type { ReconstructionReadinessInput } from "../domain/reconstructionReadinessSchemas.js"; + +const HASH = "a".repeat(64); +const sourceEvidence = (variant: string) => + createEvidence( + undefined, + { + id: "rea-readiness-fixture", + name: "REA readiness fixture", + version: "1", + }, + { + predicateType: "rea.reconstruction-readiness-fixture/v1", + operation: "run_reconstruction_readiness_fixture", + parameters: { variant }, + result: { passed: true, variant }, + confidence: "observed", + authority: "controlled-replay", + }, + ); +const SOURCE = sourceEvidence("authority"); +const CONTRADICTING_SOURCE = sourceEvidence("contradiction"); +const EVIDENCE_ID = SOURCE.evidence_id; +const CASES = [ + "positive", + "negative", + "malformed", + "cancellation", + "teardown", +] as const; +const FIXTURES = [ + "native", + "javascript-cli", + "electron", + "incomplete-reconstruction", + "broken-runtime", +] as const; +const STATUSES = [ + "unowned", + "characterized", + "implemented", + "verified", + "contradicted", + "blocked", + "out-of-scope", + "unknown", +] as const; +const LAYERS = [ + "cli", + "protocol", + "electron", + "persistence", + "process", + "runtime", + "packaging", + "native-abi", + "application", + "other", +] as const; +const EVIDENCE_STATES = [ + "candidate", + "observed", + "reviewed", + "unknown", +] as const; + +/** Complete synthetic input shared by docs, tests, CLI, MCP, and verifier. */ +export const RECONSTRUCTION_READINESS_EXAMPLE: ReconstructionReadinessInput = { + schema_version: 1, + identity: { + cli_version: "2.4.0", + server_version: "2.4.0", + catalog_digest: HASH, + skill_digest: HASH, + versions: [ + { + component: "rea-cli", + version: "2.4.0", + digest: HASH, + state: "current", + remediation: null, + }, + { + component: "rea-server", + version: "2.3.0", + digest: "b".repeat(64), + state: "stale", + remediation: "Restart the connected MCP server.", + }, + { + component: "rea-server", + version: "2.4.0", + digest: HASH, + state: "current", + remediation: null, + }, + ], + providers: [ + { + provider_id: "javascript-application", + version: "1", + digest: HASH, + state: "available", + reason_code: null, + }, + { + provider_id: "broken-runtime-probe", + version: null, + digest: null, + state: "broken-host", + reason_code: "host-runtime-linker-failure", + }, + ], + }, + client: { + name: "synthetic-mcp-client", + version: "1", + negotiated_capabilities: ["elicitation", "progress", "resources"], + }, + capabilities: [ + { + capability_id: "analyze-javascript-application", + available: true, + bounded: true, + side_effect: "reads-target", + authority_scopes: ["investigation-input"], + limits: [{ name: "max-ast-nodes", value: 2_000_000, unit: "count" }], + unavailable_reason: null, + }, + ], + fixtures: FIXTURES.map((kind) => ({ + fixture_id: `fixture.${kind}`, + kind, + artifact_sha256: HASH, + environment_id: "environment.synthetic", + deliberate_faults: [`fault.${kind}`], + })), + workflow_candidates: FIXTURES.map((kind) => ({ + fixture_id: `fixture.${kind}`, + workflow_id: `workflow.${kind}`, + provider_id: kind === "broken-runtime" ? null : `provider.${kind}`, + selected: true, + invoked: true, + compatibility: "compatible", + reason_code: "compatible-artifact-family", + })), + grants: [ + { + grant_id: "grant.denied", + scope: "process-capture", + decision: "denied", + launched_process: false, + evidence_ids: [EVIDENCE_ID], + }, + { + grant_id: "grant.session", + scope: "process-capture", + decision: "granted", + launched_process: true, + evidence_ids: [EVIDENCE_ID], + }, + ], + evidence_bundle: createEvidenceBundle([SOURCE, CONTRADICTING_SOURCE]), + comparisons: [ + { + comparison_id: "comparison.concurrent", + verdict: "equivalent", + schedule_semantics: "partial-order", + concurrent: true, + truncated: false, + unavailable_authority: false, + unstable: false, + deliberate_divergence_ref: null, + divergence_refs: [], + contradiction_ids: [], + evidence_ids: [EVIDENCE_ID], + }, + { + comparison_id: "comparison.divergence", + verdict: "different", + schedule_semantics: "finite-traces", + concurrent: true, + truncated: false, + unavailable_authority: false, + unstable: false, + deliberate_divergence_ref: "/events/primary", + divergence_refs: ["/events/primary"], + contradiction_ids: [], + evidence_ids: [EVIDENCE_ID], + }, + { + comparison_id: "comparison.contradicted", + verdict: "unknown", + schedule_semantics: "finite-traces", + concurrent: false, + truncated: false, + unavailable_authority: false, + unstable: false, + deliberate_divergence_ref: null, + divergence_refs: [], + contradiction_ids: ["contradiction.artifact"], + evidence_ids: [EVIDENCE_ID], + }, + ], + contradictions: [ + { + contradiction_id: "contradiction.artifact", + declared_sha256: HASH, + observed_sha256: "b".repeat(64), + policy: "record-and-continue", + affected_comparison_ids: ["comparison.contradicted"], + evidence_ids: [EVIDENCE_ID, CONTRADICTING_SOURCE.evidence_id], + }, + ], + obligation_ledger: { + schema: "ReconstructionObligationLedger", + schema_version: 1, + ledger_id: `rol_${HASH}`, + closure_digest: HASH, + status: "ready", + coverage: { + status: "complete", + truncated: false, + omitted_count: 0, + max_obligations: 100, + }, + summary: { + total: 1, + required: 1, + verified: 1, + required_open: 0, + by_status: STATUSES.map((key) => ({ + key, + count: key === "verified" ? 1 : 0, + })), + by_application_layer: LAYERS.map((key) => ({ + key, + count: key === "process" ? 1 : 0, + })), + by_evidence_authority: EVIDENCE_STATES.map((key) => ({ + key, + count: key === "observed" ? 1 : 0, + })), + }, + reports: { + missing_owner_obligation_ids: [], + missing_verifier_obligation_ids: [], + contradicted_obligation_ids: [], + residual_unknown_ids: [], + }, + ownership_graph: [ + { + obligation_id: "obl.readiness", + module_path: "src/readiness.ts", + symbol: "runReadiness", + }, + ], + dependency_graph: [], + obligations: [ + { + obligation_id: "obl.readiness", + obligation_version: 1, + title: "Preserve the packaged readiness fixture lifecycle", + origin: "reviewed", + application_layer: "process", + family: "packaged-process-lifecycle", + target: { + artifact_sha256: HASH, + application_node_id: null, + semantic_node_id: null, + location: "/fixture", + }, + authority_references: [ + { + evidence_id: EVIDENCE_ID, + authority: "controlled-replay", + state: "observed", + location: "/fixture", + }, + ], + source_state: "observed", + observed_cases: CASES.map((kind) => ({ + kind, + evidence_id: EVIDENCE_ID, + location: `/original/${kind}`, + })), + required: true, + required_case_kinds: [...CASES], + required_original_authority: "process", + required_fixture_authority: "packaged-process", + required_verifier_authority: "packaged-process", + requires_parser_type: false, + dependency_obligation_ids: [], + residual_unknown_ids: [], + unavailable_authority: [], + required_next_evidence: [], + binding: { + obligation_id: "obl.readiness", + owner: { + module_path: "src/readiness.ts", + symbol: "runReadiness", + owner_sha256: HASH, + }, + parser_type: null, + original_cases: CASES.map((kind) => ({ + kind, + evidence_id: EVIDENCE_ID, + location: `/original/${kind}`, + })), + fixtures: CASES.map((kind) => ({ + fixture_id: `fixture.${kind}`, + case_kind: kind, + authority: "packaged-process", + evidence_ids: [EVIDENCE_ID], + })), + verifier: { + verifier_id: "verifier.readiness", + claim_id: "claim.readiness", + command: "npm run verify:readiness", + authority: "packaged-process", + status: "pass", + result_evidence_id: EVIDENCE_ID, + enumerated_obligation_ids: ["obl.readiness"], + nondeterminism: { + mode: "partial-order", + specification: "Startup precedes teardown.", + }, + }, + }, + status: "verified", + diagnostics: [], + }, + ], + evidence_links: [EVIDENCE_ID], + limitations: [], + }, + operation_outcomes: [ + { + sequence: 0, + operation: "inspect-artifact", + surface: "cli", + call_kind: "valid", + expected_success: true, + cli_exit_code: 0, + mcp_status: null, + error_code: null, + recovered: false, + evidence_ids: [EVIDENCE_ID], + }, + { + sequence: 1, + operation: "inspect-artifact", + surface: "mcp", + call_kind: "valid", + expected_success: true, + cli_exit_code: null, + mcp_status: "success", + error_code: null, + recovered: false, + evidence_ids: [EVIDENCE_ID], + }, + { + sequence: 2, + operation: "capture-process", + surface: "cli", + call_kind: "permission", + expected_success: false, + cli_exit_code: 2, + mcp_status: null, + error_code: "permission-required", + recovered: true, + evidence_ids: [EVIDENCE_ID], + }, + { + sequence: 3, + operation: "capture-process", + surface: "mcp", + call_kind: "permission", + expected_success: false, + cli_exit_code: null, + mcp_status: "non-success", + error_code: "permission-required", + recovered: true, + evidence_ids: [EVIDENCE_ID], + }, + ], + cleanup: [ + { + run_id: "run.cancelled", + cancelled: true, + owned_resources_remaining: 0, + diagnostic_evidence_ids: [EVIDENCE_ID], + }, + ], + closure_history: [ + { + sequence: 0, + ledger_digest: "b".repeat(64), + status: "open", + required_open: 1, + newly_verified_obligation_ids: [], + evidence_ids: [EVIDENCE_ID], + }, + { + sequence: 1, + ledger_digest: HASH, + status: "ready", + required_open: 0, + newly_verified_obligation_ids: ["obl.readiness"], + evidence_ids: [EVIDENCE_ID], + }, + ], + delegation_checks: [ + { + candidate_id: "candidate.reconstructed", + delegates_to_authority: false, + evidence_ids: [EVIDENCE_ID], + }, + ], + replay: { + expected_source_digest: null, + deterministic: true, + tamper_detected: true, + stale_input_detected: true, + evidence_ids: [EVIDENCE_ID], + }, + stages: READINESS_STAGE_IDS.map((stageId) => ({ + stage_id: stageId, + required: true, + status: "pass", + capability_issue: null, + next_action: null, + evidence_ids: [EVIDENCE_ID], + checks: READINESS_REQUIRED_CHECKS[stageId].map((checkId) => ({ + check_id: checkId, + status: "pass", + detail: `Synthetic conformance proved ${checkId}.`, + evidence_ids: [EVIDENCE_ID], + })), + })), +}; diff --git a/src/domain/reconstructionReadiness.ts b/src/domain/reconstructionReadiness.ts new file mode 100644 index 0000000000..6c2566cea5 --- /dev/null +++ b/src/domain/reconstructionReadiness.ts @@ -0,0 +1,193 @@ +import { createHash } from "node:crypto"; + +import canonicalize from "canonicalize"; + +import { parseEvidenceBundle } from "./evidenceBundle.js"; +import { readinessFindingOrder } from "./reconstructionReadinessFindingHelpers.js"; +import { + READINESS_STAGE_IDS, + readinessJourneyFindings, +} from "./reconstructionReadinessJourneyFindings.js"; +import { readinessIntegrityFindings } from "./reconstructionReadinessIntegrityFindings.js"; +import { + reconstructionReadinessInputSchema, + reconstructionReadinessReportSchema, + type ReadinessFinding, + type ReconstructionReadinessInput, + type ReconstructionReadinessReport, +} from "./reconstructionReadinessSchemas.js"; + +/** Evaluate one public-contract journey into a deterministic fail-closed report. */ +export const createReconstructionReadinessReport = ( + inputValue: unknown, +): ReconstructionReadinessReport => { + const input = reconstructionReadinessInputSchema.parse(inputValue); + const sourceDigest = digest(sourceValue(input)); + const evidenceIds = new Set( + parseEvidenceBundle(input.evidence_bundle).records.map( + ({ evidence_id: evidenceId }) => evidenceId, + ), + ); + const findings = [ + ...readinessJourneyFindings(input, evidenceIds), + ...readinessIntegrityFindings(input, evidenceIds, sourceDigest), + ].sort(readinessFindingOrder); + const stages = effectiveStages(input, findings); + const status = aggregateStatus(stages); + const semantic = { + source_digest: sourceDigest, + status, + summary: { + stages: stages.length, + required_stages: stages.filter(({ required }) => required).length, + passed_required_stages: stages.filter( + ({ required, status: stageStatus }) => + required && stageStatus === "pass", + ).length, + findings: findings.length, + failed_findings: findings.filter( + ({ status: findingStatus }) => findingStatus === "fail", + ).length, + unknown_findings: findings.filter( + ({ status: findingStatus }) => findingStatus === "unknown", + ).length, + }, + metrics: readinessMetrics(input), + stages, + findings, + evidence_links: [...evidenceIds].sort(), + snapshot: input, + }; + const reportDigest = digest({ + schema: "rea.reconstruction-readiness-report/v1", + ...semantic, + }); + return reconstructionReadinessReportSchema.parse({ + schema: "ReconstructionReadinessReport", + schema_version: 1, + report_id: `rr_${reportDigest}`, + report_digest: reportDigest, + ...semantic, + }); +}; + +const effectiveStages = ( + input: ReconstructionReadinessInput, + findings: readonly ReadinessFinding[], +): ReconstructionReadinessReport["stages"] => + READINESS_STAGE_IDS.map((stageId) => { + const stage = input.stages.find(({ stage_id: id }) => id === stageId); + const stageFindings = findings.filter(({ stage_id: id }) => id === stageId); + const status = stageFindings.some( + ({ status: findingStatus }) => findingStatus === "fail", + ) + ? "fail" + : stage !== undefined && stage.status !== "pass" + ? stage.status + : stageFindings.length > 0 + ? "unknown" + : (stage?.status ?? "fail"); + return stage === undefined + ? { + stage_id: stageId, + required: true, + status, + capability_issue: null, + next_action: "Supply the missing conformance stage.", + evidence_ids: [], + checks: [], + } + : { ...stage, required: true, status }; + }); + +const aggregateStatus = ( + stages: ReconstructionReadinessReport["stages"], +): ReconstructionReadinessReport["status"] => { + const required = stages.filter(({ required }) => required); + if (required.some(({ status }) => status === "fail")) return "fail"; + if (required.every(({ status }) => status === "pass")) return "pass"; + for (const status of [ + "unknown", + "truncated", + "unsupported", + "skipped", + ] as const) + if (required.some(({ status: stageStatus }) => stageStatus === status)) + return status; + return "unknown"; +}; + +const readinessMetrics = ( + input: ReconstructionReadinessInput, +): ReconstructionReadinessReport["metrics"] => { + const selectedFixtures = input.fixtures.filter((fixture) => + input.workflow_candidates.some( + ({ fixture_id: fixtureId, selected, compatibility }) => + fixtureId === fixture.fixture_id && + selected && + compatibility === "compatible", + ), + ).length; + const prerequisiteFailures = input.operation_outcomes.filter( + ({ call_kind: kind, expected_success: success }) => + !success && ["setup", "permission"].includes(kind), + ); + const localized = input.comparisons.filter( + ({ deliberate_divergence_ref: expected, divergence_refs: observed }) => + expected !== null && observed.includes(expected), + ).length; + const deliberate = input.comparisons.filter( + ({ deliberate_divergence_ref: expected }) => expected !== null, + ).length; + const falseEquivalence = input.comparisons.filter( + (comparison) => + comparison.verdict === "equivalent" && + (comparison.truncated || + comparison.unavailable_authority || + comparison.unstable || + comparison.contradiction_ids.length > 0), + ).length; + return { + first_valid_workflow_selection_rate: ratio( + selectedFixtures, + input.fixtures.length, + ), + malformed_or_over_limit_call_rate: ratio( + input.operation_outcomes.filter(({ call_kind: kind }) => + ["malformed", "over-limit"].includes(kind), + ).length, + input.operation_outcomes.length, + ), + incompatible_provider_invocations: input.workflow_candidates.filter( + ({ invoked, compatibility }) => invoked && compatibility !== "compatible", + ).length, + failed_calls_before_prerequisites: prerequisiteFailures.length, + recovery_rate: ratio( + prerequisiteFailures.filter(({ recovered }) => recovered).length, + prerequisiteFailures.length, + ), + divergence_localization_precision: ratio(localized, deliberate), + false_equivalence_count: falseEquivalence, + unowned_obligation_count: + input.obligation_ledger.reports.missing_owner_obligation_ids.length, + cleanup_leak_count: input.cleanup.filter( + ({ owned_resources_remaining: remaining }) => remaining > 0, + ).length, + nondeterministic_replay_count: input.replay.deterministic ? 0 : 1, + }; +}; + +const sourceValue = (input: ReconstructionReadinessInput): unknown => ({ + ...input, + replay: { ...input.replay, expected_source_digest: null }, +}); + +const ratio = (numerator: number, denominator: number): number => + denominator === 0 ? 1 : numerator / denominator; + +const digest = (value: unknown): string => { + const encoded = canonicalize(value); + if (encoded === undefined) + throw new TypeError("Reconstruction readiness value is not canonical JSON"); + return createHash("sha256").update(encoded).digest("hex"); +}; diff --git a/src/domain/reconstructionReadinessFindingHelpers.ts b/src/domain/reconstructionReadinessFindingHelpers.ts new file mode 100644 index 0000000000..3b79581c6a --- /dev/null +++ b/src/domain/reconstructionReadinessFindingHelpers.ts @@ -0,0 +1,46 @@ +import type { + ReadinessFinding, + ReadinessStageId, +} from "./reconstructionReadinessSchemas.js"; + +export const addMissingEvidenceFindings = ( + findings: ReadinessFinding[], + stageId: ReadinessStageId, + referencedIds: readonly string[], + availableIds: ReadonlySet, +): void => { + for (const evidenceId of new Set(referencedIds)) + if (!availableIds.has(evidenceId)) + findings.push( + readinessFinding( + "referenced-evidence-missing", + stageId, + "unknown", + `Referenced Evidence is absent from the bundle: ${evidenceId}.`, + ), + ); +}; + +export const readinessFinding = ( + ...[code, stageId, status, detail, evidenceIds = []]: readonly [ + code: string, + stageId: ReadinessStageId, + status: ReadinessFinding["status"], + detail: string, + evidenceIds?: readonly string[], + ] +): ReadinessFinding => ({ + code, + stage_id: stageId, + status, + detail, + evidence_ids: [...new Set(evidenceIds)].sort(), +}); + +export const readinessFindingOrder = ( + left: ReadinessFinding, + right: ReadinessFinding, +): number => + `${left.stage_id}:${left.code}:${left.detail}`.localeCompare( + `${right.stage_id}:${right.code}:${right.detail}`, + ); diff --git a/src/domain/reconstructionReadinessIntegrityFindings.ts b/src/domain/reconstructionReadinessIntegrityFindings.ts new file mode 100644 index 0000000000..843b48cb55 --- /dev/null +++ b/src/domain/reconstructionReadinessIntegrityFindings.ts @@ -0,0 +1,385 @@ +import { + addMissingEvidenceFindings, + readinessFinding as finding, +} from "./reconstructionReadinessFindingHelpers.js"; +import type { + ReadinessFinding, + ReconstructionReadinessInput, +} from "./reconstructionReadinessSchemas.js"; + +export const readinessIntegrityFindings = ( + input: ReconstructionReadinessInput, + evidenceIds: ReadonlySet, + sourceDigest: string, +): ReadinessFinding[] => [ + ...comparisonFindings(input, evidenceIds), + ...contradictionFindings(input, evidenceIds), + ...closureFindings(input, evidenceIds), + ...cleanupFindings(input, evidenceIds), + ...replayFindings(input, evidenceIds, sourceDigest), +]; + +const comparisonFindings = ( + input: ReconstructionReadinessInput, + evidenceIds: ReadonlySet, +): ReadinessFinding[] => { + const findings: ReadinessFinding[] = []; + for (const comparison of input.comparisons) { + const unsafeEquivalent = + comparison.verdict === "equivalent" && + (comparison.truncated || + comparison.unavailable_authority || + comparison.unstable || + comparison.contradiction_ids.length > 0); + if (unsafeEquivalent) + findings.push( + finding( + "false-equivalence", + "compare-authority-candidate", + "fail", + `${comparison.comparison_id} claims equivalence across incomplete or contradicted Evidence.`, + comparison.evidence_ids, + ), + ); + if ( + comparison.concurrent && + comparison.schedule_semantics === "total-order" + ) + findings.push( + finding( + "unsafe-concurrency-semantics", + "compare-authority-candidate", + "fail", + `${comparison.comparison_id} uses total-order semantics for concurrent behavior.`, + comparison.evidence_ids, + ), + ); + if ( + comparison.deliberate_divergence_ref !== null && + !comparison.divergence_refs.includes(comparison.deliberate_divergence_ref) + ) + findings.push( + finding( + "divergence-not-localized", + "compare-authority-candidate", + "fail", + `${comparison.comparison_id} did not localize its deliberate primary divergence.`, + comparison.evidence_ids, + ), + ); + addMissingEvidenceFindings( + findings, + "compare-authority-candidate", + comparison.evidence_ids, + evidenceIds, + ); + } + return findings; +}; + +const contradictionFindings = ( + input: ReconstructionReadinessInput, + evidenceIds: ReadonlySet, +): ReadinessFinding[] => { + const findings: ReadinessFinding[] = []; + const comparisons = new Map( + input.comparisons.map((comparison) => [ + comparison.comparison_id, + comparison, + ]), + ); + for (const contradiction of input.contradictions) { + if ( + contradiction.declared_sha256 === contradiction.observed_sha256 || + new Set(contradiction.evidence_ids).size < 2 + ) + findings.push( + finding( + "contradiction-proof-incomplete", + "preserve-contradictions", + "fail", + `${contradiction.contradiction_id} must preserve distinct hashes and Evidence records.`, + contradiction.evidence_ids, + ), + ); + for (const comparisonId of contradiction.affected_comparison_ids) { + const comparison = comparisons.get(comparisonId); + if (comparison === undefined) + findings.push( + finding( + "contradiction-comparison-missing", + "preserve-contradictions", + "unknown", + `Contradiction references missing comparison ${comparisonId}.`, + contradiction.evidence_ids, + ), + ); + else if ( + comparison.verdict === "equivalent" || + !comparison.contradiction_ids.includes(contradiction.contradiction_id) + ) + findings.push( + finding( + "contradiction-equivalence-not-blocked", + "preserve-contradictions", + "fail", + `${comparisonId} does not preserve contradiction ${contradiction.contradiction_id}.`, + contradiction.evidence_ids, + ), + ); + } + addMissingEvidenceFindings( + findings, + "preserve-contradictions", + contradiction.evidence_ids, + evidenceIds, + ); + } + return findings; +}; + +const closureFindings = ( + input: ReconstructionReadinessInput, + evidenceIds: ReadonlySet, +): ReadinessFinding[] => { + const findings: ReadinessFinding[] = []; + if (input.obligation_ledger.summary.required === 0) + findings.push( + finding( + "reconstruction-obligations-empty", + "verify-reconstruction-closure", + "unknown", + "A ready ledger must include at least one required reconstruction obligation.", + ), + ); + if ( + input.obligation_ledger.status !== "ready" || + input.obligation_ledger.summary.required_open > 0 + ) + findings.push( + finding( + "reconstruction-closure-open", + "verify-reconstruction-closure", + "unknown", + `Obligation ledger is ${input.obligation_ledger.status} with ${String(input.obligation_ledger.summary.required_open)} required obligations open.`, + input.obligation_ledger.evidence_links, + ), + ); + addMissingEvidenceFindings( + findings, + "verify-reconstruction-closure", + input.obligation_ledger.evidence_links, + evidenceIds, + ); + evaluateClosureHistory(input, findings, evidenceIds); + for (const check of input.delegation_checks) { + if (check.delegates_to_authority) + findings.push( + finding( + "candidate-delegates-to-authority", + "verify-reconstruction-closure", + "fail", + `${check.candidate_id} delegates behavior back to the authority.`, + check.evidence_ids, + ), + ); + addMissingEvidenceFindings( + findings, + "verify-reconstruction-closure", + check.evidence_ids, + evidenceIds, + ); + } + return findings; +}; + +const evaluateClosureHistory = ( + input: ReconstructionReadinessInput, + findings: ReadinessFinding[], + evidenceIds: ReadonlySet, +): void => { + const history = [...input.closure_history].sort( + (left, right) => left.sequence - right.sequence, + ); + if ( + new Set(history.map(({ sequence }) => sequence)).size !== history.length || + history.at(-1)?.ledger_digest !== input.obligation_ledger.closure_digest + ) + findings.push( + finding( + "closure-history-identity-mismatch", + "verify-reconstruction-closure", + "fail", + "Closure history sequences must be unique and end at the supplied ledger digest.", + ), + ); + if ( + history[0]?.required_open === 0 || + history.at(-1)?.required_open !== 0 || + history.at(-1)?.status !== "ready" + ) + findings.push( + finding( + "incremental-closure-history-missing", + "verify-reconstruction-closure", + "unknown", + "Closure history must begin partial and end ready with zero required obligations open.", + ), + ); + for (const [index, observation] of history.entries()) { + const previous = history[index - 1]; + if ( + previous !== undefined && + (observation.required_open > previous.required_open || + observation.newly_verified_obligation_ids.length !== + previous.required_open - observation.required_open) + ) + findings.push( + finding( + "invalid-obligation-closure-transition", + "verify-reconstruction-closure", + "fail", + `Closure observation ${String(observation.sequence)} does not match its newly verified obligations.`, + observation.evidence_ids, + ), + ); + if ( + observation.newly_verified_obligation_ids.some( + (obligationId) => + !input.obligation_ledger.obligations.some( + ({ obligation_id: candidateId, status }) => + candidateId === obligationId && status === "verified", + ), + ) + ) + findings.push( + finding( + "closure-obligation-proof-missing", + "verify-reconstruction-closure", + "fail", + `Closure observation ${String(observation.sequence)} names an obligation without verified ledger proof.`, + observation.evidence_ids, + ), + ); + addMissingEvidenceFindings( + findings, + "verify-reconstruction-closure", + observation.evidence_ids, + evidenceIds, + ); + } +}; + +const cleanupFindings = ( + input: ReconstructionReadinessInput, + evidenceIds: ReadonlySet, +): ReadinessFinding[] => { + const findings: ReadinessFinding[] = []; + for (const cleanup of input.cleanup) { + if (cleanup.owned_resources_remaining > 0) + findings.push( + finding( + "owned-resource-leak", + "reactive-scenarios", + "fail", + `${cleanup.run_id} retained ${String(cleanup.owned_resources_remaining)} owned resources.`, + cleanup.diagnostic_evidence_ids, + ), + ); + if (cleanup.cancelled && cleanup.diagnostic_evidence_ids.length === 0) + findings.push( + finding( + "cancelled-diagnostic-evidence-missing", + "reactive-scenarios", + "unknown", + `${cleanup.run_id} cancellation retained no diagnostic Evidence.`, + ), + ); + addMissingEvidenceFindings( + findings, + "reactive-scenarios", + cleanup.diagnostic_evidence_ids, + evidenceIds, + ); + } + for (const grant of input.grants) { + if ( + ["denied", "cancelled"].includes(grant.decision) && + grant.launched_process + ) + findings.push( + finding( + "denied-authority-launched-process", + "acquire-authority", + "fail", + `${grant.grant_id} launched a process after ${grant.decision}.`, + grant.evidence_ids, + ), + ); + addMissingEvidenceFindings( + findings, + "acquire-authority", + grant.evidence_ids, + evidenceIds, + ); + } + return findings; +}; + +const replayFindings = ( + input: ReconstructionReadinessInput, + evidenceIds: ReadonlySet, + sourceDigest: string, +): ReadinessFinding[] => { + const findings: ReadinessFinding[] = []; + if (!input.replay.deterministic) + findings.push( + finding( + "replay-nondeterministic", + "export-replay", + "fail", + "Clean replay did not reproduce the same semantic report.", + input.replay.evidence_ids, + ), + ); + if (!input.replay.tamper_detected) + findings.push( + finding( + "tamper-not-detected", + "export-replay", + "fail", + "The conformance replay did not reject a tampered input.", + input.replay.evidence_ids, + ), + ); + if (!input.replay.stale_input_detected) + findings.push( + finding( + "stale-input-not-detected", + "export-replay", + "fail", + "The conformance replay did not reject a stale digest.", + input.replay.evidence_ids, + ), + ); + if ( + input.replay.expected_source_digest !== null && + input.replay.expected_source_digest !== sourceDigest + ) + findings.push( + finding( + "replay-source-digest-mismatch", + "export-replay", + "fail", + `Expected ${input.replay.expected_source_digest}; observed ${sourceDigest}.`, + input.replay.evidence_ids, + ), + ); + addMissingEvidenceFindings( + findings, + "export-replay", + input.replay.evidence_ids, + evidenceIds, + ); + return findings; +}; diff --git a/src/domain/reconstructionReadinessJourneyFindings.ts b/src/domain/reconstructionReadinessJourneyFindings.ts new file mode 100644 index 0000000000..2acde376e3 --- /dev/null +++ b/src/domain/reconstructionReadinessJourneyFindings.ts @@ -0,0 +1,401 @@ +import { + addMissingEvidenceFindings, + readinessFinding as finding, +} from "./reconstructionReadinessFindingHelpers.js"; +import { + readinessStageIdSchema, + type ReadinessFinding, + type ReadinessStageId, + type ReconstructionReadinessInput, +} from "./reconstructionReadinessSchemas.js"; + +export const READINESS_REQUIRED_CHECKS: Readonly< + Record +> = { + "discover-classify": [ + "target-free-start", + "artifact-inventory", + "compatible-workflow-selected", + "limits-visible", + ], + "diagnose-environment": [ + "broken-runtime-distinguished", + "version-skew-detected", + "revision-changed-after-recovery", + ], + "acquire-authority": [ + "missing-scope-reported", + "narrow-grant-resumed", + "expansion-reprompted", + "denial-launched-nothing", + ], + "static-analysis": [ + "native-routed", + "javascript-routed", + "unknowns-preserved", + ], + "reactive-scenarios": [ + "predicate-driven", + "descendant-settled", + "electron-correlated", + "cancel-cleanup-preserved-evidence", + ], + "compare-authority-candidate": [ + "partial-order-accepted", + "divergence-localized", + "incomplete-is-unknown", + ], + "preserve-contradictions": [ + "record-and-continue", + "affected-equivalence-blocked", + "unaffected-comparison-continued", + ], + "verify-reconstruction-closure": [ + "incomplete-detected", + "incremental-verifier-closure", + "delegation-rejected", + ], + "export-replay": [ + "cli-mcp-parity", + "failure-status-parity", + "deterministic-replay", + "tamper-detected", + "stale-input-detected", + ], +}; + +export const READINESS_STAGE_IDS = readinessStageIdSchema.options; + +export const readinessJourneyFindings = ( + input: ReconstructionReadinessInput, + evidenceIds: ReadonlySet, +): ReadinessFinding[] => [ + ...identityFindings(input), + ...fixtureFindings(input), + ...stageFindings(input, evidenceIds), + ...capabilityFindings(input), + ...workflowFindings(input), + ...authorityFindings(input), + ...outcomeFindings(input, evidenceIds), +]; + +const identityFindings = ( + input: ReconstructionReadinessInput, +): ReadinessFinding[] => { + const findings: ReadinessFinding[] = []; + if (!input.identity.versions.some(({ state }) => state === "stale")) + findings.push( + finding( + "version-skew-observation-missing", + "diagnose-environment", + "fail", + "The journey did not preserve a stale version observation.", + ), + ); + if (!input.identity.versions.some(({ state }) => state === "current")) + findings.push( + finding( + "corrected-version-observation-missing", + "diagnose-environment", + "fail", + "The journey did not preserve corrected current version state.", + ), + ); + if (!input.identity.providers.some(({ state }) => state === "broken-host")) + findings.push( + finding( + "broken-host-observation-missing", + "diagnose-environment", + "fail", + "The journey did not distinguish a broken host dependency.", + ), + ); + return findings; +}; + +const fixtureFindings = ( + input: ReconstructionReadinessInput, +): ReadinessFinding[] => { + const requiredKinds = [ + "native", + "javascript-cli", + "electron", + "incomplete-reconstruction", + "broken-runtime", + ] as const; + return requiredKinds.flatMap((kind) => + input.fixtures.some(({ kind: fixtureKind }) => fixtureKind === kind) + ? [] + : [ + finding( + "representative-fixture-missing", + "discover-classify", + "fail", + `The required ${kind} fixture is missing.`, + ), + ], + ); +}; + +const stageFindings = ( + input: ReconstructionReadinessInput, + evidenceIds: ReadonlySet, +): ReadinessFinding[] => { + const findings: ReadinessFinding[] = []; + for (const stageId of READINESS_STAGE_IDS) { + const matching = input.stages.filter(({ stage_id: id }) => id === stageId); + if (matching.length !== 1) { + findings.push( + finding( + "stage-cardinality", + stageId, + "fail", + `Expected exactly one ${stageId} stage; observed ${String(matching.length)}.`, + ), + ); + continue; + } + const stage = matching[0]; + if (stage === undefined) continue; + if (!stage.required) + findings.push( + finding( + "required-stage-disabled", + stageId, + "fail", + "The public conformance journey requires this stage.", + ), + ); + const checks = new Map( + stage.checks.map((check) => [check.check_id, check]), + ); + if (checks.size !== stage.checks.length) + findings.push( + finding( + "duplicate-stage-check", + stageId, + "fail", + "Stage check IDs must be unique.", + ), + ); + for (const checkId of READINESS_REQUIRED_CHECKS[stageId]) { + const check = checks.get(checkId); + if (check === undefined) { + findings.push( + finding( + "missing-stage-check", + stageId, + "fail", + `Required conformance check is missing: ${checkId}.`, + ), + ); + continue; + } + if (check.status !== "pass") + findings.push( + finding( + "stage-check-not-pass", + stageId, + check.status === "fail" ? "fail" : "unknown", + `${checkId} is ${check.status}: ${check.detail}`, + check.evidence_ids, + ), + ); + if (check.evidence_ids.length === 0) + findings.push( + finding( + "stage-check-evidence-missing", + stageId, + "unknown", + `${checkId} has no attributable Evidence.`, + ), + ); + addMissingEvidenceFindings( + findings, + stageId, + [...stage.evidence_ids, ...check.evidence_ids], + evidenceIds, + ); + } + } + return findings; +}; + +const capabilityFindings = ( + input: ReconstructionReadinessInput, +): ReadinessFinding[] => + input.capabilities.flatMap((capability) => { + const findings: ReadinessFinding[] = []; + if (capability.bounded && capability.limits.length === 0) + findings.push( + finding( + "bounded-limit-hidden", + "discover-classify", + "fail", + `${capability.capability_id} is bounded but declares no exact limits.`, + ), + ); + if (!capability.available && capability.unavailable_reason === null) + findings.push( + finding( + "unavailable-reason-missing", + "discover-classify", + "unknown", + `${capability.capability_id} is unavailable without an actionable reason.`, + ), + ); + return findings; + }); + +const workflowFindings = ( + input: ReconstructionReadinessInput, +): ReadinessFinding[] => { + const findings: ReadinessFinding[] = []; + for (const fixture of input.fixtures) { + const selected = input.workflow_candidates.filter( + ({ fixture_id: fixtureId, selected: isSelected }) => + fixtureId === fixture.fixture_id && isSelected, + ); + if (selected.length !== 1 || selected[0]?.compatibility !== "compatible") + findings.push( + finding( + "compatible-workflow-not-selected", + "discover-classify", + "fail", + `${fixture.fixture_id} does not have exactly one compatible selected workflow.`, + ), + ); + } + for (const candidate of input.workflow_candidates) + if (candidate.invoked && candidate.compatibility !== "compatible") + findings.push( + finding( + "incompatible-provider-invoked", + "static-analysis", + "fail", + `${candidate.workflow_id} was invoked despite ${candidate.compatibility} routing.`, + ), + ); + return findings; +}; + +const authorityFindings = ( + input: ReconstructionReadinessInput, +): ReadinessFinding[] => { + const deniedScopes = new Set( + input.grants + .filter(({ decision }) => ["denied", "cancelled"].includes(decision)) + .map(({ scope }) => scope), + ); + const recoveredScopes = new Set( + input.grants + .filter(({ decision }) => decision === "granted") + .map(({ scope }) => scope), + ); + return [...deniedScopes].some((scope) => recoveredScopes.has(scope)) + ? [] + : [ + finding( + "narrow-authority-recovery-missing", + "acquire-authority", + "fail", + "No denied or cancelled scope was later recovered by a narrow grant.", + ), + ]; +}; + +const outcomeFindings = ( + input: ReconstructionReadinessInput, + evidenceIds: ReadonlySet, +): ReadinessFinding[] => { + const findings: ReadinessFinding[] = []; + const parity = new Map>(); + for (const outcome of input.operation_outcomes) { + const expectedCliExit = outcome.expected_success ? 0 : 1; + if ( + outcome.surface === "cli" && + (outcome.cli_exit_code === null || + (expectedCliExit === 0 + ? outcome.cli_exit_code !== 0 + : outcome.cli_exit_code === 0)) + ) + findings.push( + finding( + "cli-status-mismatch", + "export-replay", + "fail", + `${outcome.operation} CLI process status contradicts its structured outcome.`, + outcome.evidence_ids, + ), + ); + if ( + outcome.surface === "mcp" && + outcome.mcp_status !== + (outcome.expected_success ? "success" : "non-success") + ) + findings.push( + finding( + "mcp-status-mismatch", + "export-replay", + "fail", + `${outcome.operation} MCP status contradicts its structured outcome.`, + outcome.evidence_ids, + ), + ); + const key = `${outcome.operation}:${outcome.call_kind}:${String(outcome.expected_success)}`; + parity.set(key, new Set([...(parity.get(key) ?? []), outcome.surface])); + addMissingEvidenceFindings( + findings, + "export-replay", + outcome.evidence_ids, + evidenceIds, + ); + } + for (const [key, surfaces] of parity) + if (surfaces.size !== 2) + findings.push( + finding( + "surface-parity-missing", + "export-replay", + "fail", + `CLI/MCP outcome pair is incomplete: ${key}.`, + ), + ); + for (const [key, outcomes] of groupedOutcomes(input)) { + const cli = outcomes.find(({ surface }) => surface === "cli"); + const mcp = outcomes.find(({ surface }) => surface === "mcp"); + if ( + cli !== undefined && + mcp !== undefined && + (cli.expected_success !== mcp.expected_success || + cli.error_code !== mcp.error_code) + ) + findings.push( + finding( + "surface-semantic-drift", + "export-replay", + "fail", + `CLI/MCP structured outcomes disagree: ${key}.`, + [...cli.evidence_ids, ...mcp.evidence_ids], + ), + ); + } + return findings; +}; + +const groupedOutcomes = ( + input: ReconstructionReadinessInput, +): ReadonlyMap< + string, + readonly ReconstructionReadinessInput["operation_outcomes"][number][] +> => { + const grouped = new Map< + string, + ReconstructionReadinessInput["operation_outcomes"] + >(); + for (const outcome of input.operation_outcomes) { + const key = `${outcome.operation}:${outcome.call_kind}`; + grouped.set(key, [...(grouped.get(key) ?? []), outcome]); + } + return grouped; +}; diff --git a/src/domain/reconstructionReadinessSchemas.ts b/src/domain/reconstructionReadinessSchemas.ts new file mode 100644 index 0000000000..8941afacda --- /dev/null +++ b/src/domain/reconstructionReadinessSchemas.ts @@ -0,0 +1,272 @@ +import { z } from "zod"; + +import { evidenceBundleSchema } from "./evidenceBundle.js"; +import { reconstructionObligationLedgerSchema } from "./reconstructionObligationLedgerSchemas.js"; + +const digestSchema = z.string().regex(/^[a-f0-9]{64}$/u); +const evidenceIdSchema = z.string().regex(/^ev_[a-f0-9]{64}$/u); +const stableIdSchema = z.string().regex(/^[A-Za-z][A-Za-z0-9._:/-]{0,199}$/u); +const boundedTextSchema = z.string().trim().min(1).max(4_096); + +export const readinessStatusSchema = z.enum([ + "pass", + "fail", + "unknown", + "unsupported", + "truncated", + "skipped", +]); + +export const readinessStageIdSchema = z.enum([ + "discover-classify", + "diagnose-environment", + "acquire-authority", + "static-analysis", + "reactive-scenarios", + "compare-authority-candidate", + "preserve-contradictions", + "verify-reconstruction-closure", + "export-replay", +]); + +const versionObservationSchema = z.strictObject({ + component: stableIdSchema, + version: boundedTextSchema.nullable(), + digest: digestSchema.nullable(), + state: z.enum(["current", "stale", "unavailable"]), + remediation: boundedTextSchema.nullable(), +}); + +const providerIdentitySchema = z.strictObject({ + provider_id: stableIdSchema, + version: boundedTextSchema.nullable(), + digest: digestSchema.nullable(), + state: z.enum(["available", "unavailable", "incompatible", "broken-host"]), + reason_code: stableIdSchema.nullable(), +}); + +const limitSchema = z.strictObject({ + name: stableIdSchema, + value: z.number().finite().nonnegative(), + unit: z.enum(["count", "bytes", "milliseconds", "ratio"]), +}); + +const capabilitySchema = z.strictObject({ + capability_id: stableIdSchema, + available: z.boolean(), + bounded: z.boolean(), + side_effect: z.enum([ + "none", + "reads-target", + "launches-process", + "writes-workspace", + "network-observation", + ]), + authority_scopes: z.array(stableIdSchema).max(100), + limits: z.array(limitSchema).max(100), + unavailable_reason: boundedTextSchema.nullable(), +}); + +const fixtureSchema = z.strictObject({ + fixture_id: stableIdSchema, + kind: z.enum([ + "native", + "javascript-cli", + "electron", + "incomplete-reconstruction", + "broken-runtime", + ]), + artifact_sha256: digestSchema, + environment_id: stableIdSchema, + deliberate_faults: z.array(stableIdSchema).min(1).max(100), +}); + +const workflowCandidateSchema = z.strictObject({ + fixture_id: stableIdSchema, + workflow_id: stableIdSchema, + provider_id: stableIdSchema.nullable(), + selected: z.boolean(), + invoked: z.boolean(), + compatibility: z.enum(["compatible", "incompatible", "unavailable"]), + reason_code: stableIdSchema, +}); + +const grantSchema = z.strictObject({ + grant_id: stableIdSchema, + scope: stableIdSchema, + decision: z.enum(["granted", "denied", "cancelled", "restart-required"]), + launched_process: z.boolean(), + evidence_ids: z.array(evidenceIdSchema).max(100), +}); + +const comparisonSchema = z.strictObject({ + comparison_id: stableIdSchema, + verdict: z.enum(["equivalent", "different", "unknown"]), + schedule_semantics: z.enum(["total-order", "partial-order", "finite-traces"]), + concurrent: z.boolean(), + truncated: z.boolean(), + unavailable_authority: z.boolean(), + unstable: z.boolean(), + deliberate_divergence_ref: boundedTextSchema.nullable(), + divergence_refs: z.array(boundedTextSchema).max(100), + contradiction_ids: z.array(stableIdSchema).max(100), + evidence_ids: z.array(evidenceIdSchema).min(1).max(1_000), +}); + +const contradictionSchema = z.strictObject({ + contradiction_id: stableIdSchema, + declared_sha256: digestSchema, + observed_sha256: digestSchema, + policy: z.literal("record-and-continue"), + affected_comparison_ids: z.array(stableIdSchema).min(1).max(100), + evidence_ids: z.array(evidenceIdSchema).min(2).max(100), +}); + +const operationOutcomeSchema = z.strictObject({ + sequence: z.number().int().min(0), + operation: stableIdSchema, + surface: z.enum(["cli", "mcp"]), + call_kind: z.enum([ + "valid", + "malformed", + "over-limit", + "incompatible-provider", + "setup", + "permission", + "operation", + ]), + expected_success: z.boolean(), + cli_exit_code: z.number().int().min(0).nullable(), + mcp_status: z.enum(["success", "non-success"]).nullable(), + error_code: stableIdSchema.nullable(), + recovered: z.boolean(), + evidence_ids: z.array(evidenceIdSchema).max(100), +}); + +const cleanupSchema = z.strictObject({ + run_id: stableIdSchema, + cancelled: z.boolean(), + owned_resources_remaining: z.number().int().min(0), + diagnostic_evidence_ids: z.array(evidenceIdSchema).max(100), +}); + +const closureObservationSchema = z.strictObject({ + sequence: z.number().int().min(0), + ledger_digest: digestSchema, + status: z.enum(["ready", "open", "failed", "unknown"]), + required_open: z.number().int().min(0), + newly_verified_obligation_ids: z.array(stableIdSchema).max(10_000), + evidence_ids: z.array(evidenceIdSchema).min(1).max(1_000), +}); + +const delegationCheckSchema = z.strictObject({ + candidate_id: stableIdSchema, + delegates_to_authority: z.boolean(), + evidence_ids: z.array(evidenceIdSchema).min(1).max(100), +}); + +const stageCheckSchema = z.strictObject({ + check_id: stableIdSchema, + status: readinessStatusSchema, + detail: boundedTextSchema, + evidence_ids: z.array(evidenceIdSchema).max(100), +}); + +const stageSchema = z.strictObject({ + stage_id: readinessStageIdSchema, + required: z.boolean(), + status: readinessStatusSchema, + capability_issue: stableIdSchema.nullable(), + next_action: boundedTextSchema.nullable(), + evidence_ids: z.array(evidenceIdSchema).max(1_000), + checks: z.array(stageCheckSchema).max(100), +}); + +const replaySchema = z.strictObject({ + expected_source_digest: digestSchema.nullable(), + deterministic: z.boolean(), + tamper_detected: z.boolean(), + stale_input_detected: z.boolean(), + evidence_ids: z.array(evidenceIdSchema).max(100), +}); + +export const reconstructionReadinessInputSchema = z.strictObject({ + schema_version: z.literal(1), + identity: z.strictObject({ + cli_version: boundedTextSchema, + server_version: boundedTextSchema, + catalog_digest: digestSchema, + skill_digest: digestSchema.nullable(), + versions: z.array(versionObservationSchema).min(1).max(100), + providers: z.array(providerIdentitySchema).max(100), + }), + client: z.strictObject({ + name: boundedTextSchema, + version: boundedTextSchema, + negotiated_capabilities: z.array(stableIdSchema).max(100), + }), + capabilities: z.array(capabilitySchema).min(1).max(1_000), + fixtures: z.array(fixtureSchema).min(1).max(100), + workflow_candidates: z.array(workflowCandidateSchema).min(1).max(1_000), + grants: z.array(grantSchema).max(1_000), + evidence_bundle: evidenceBundleSchema, + comparisons: z.array(comparisonSchema).max(1_000), + contradictions: z.array(contradictionSchema).max(1_000), + obligation_ledger: reconstructionObligationLedgerSchema, + operation_outcomes: z.array(operationOutcomeSchema).max(10_000), + cleanup: z.array(cleanupSchema).max(1_000), + closure_history: z.array(closureObservationSchema).min(2).max(1_000), + delegation_checks: z.array(delegationCheckSchema).min(1).max(1_000), + replay: replaySchema, + stages: z.array(stageSchema).max(20), +}); + +const findingSchema = z.strictObject({ + code: stableIdSchema, + stage_id: readinessStageIdSchema, + status: z.enum(["fail", "unknown"]), + detail: boundedTextSchema, + evidence_ids: z.array(evidenceIdSchema).max(100), +}); + +export const reconstructionReadinessReportSchema = z.strictObject({ + schema: z.literal("ReconstructionReadinessReport"), + schema_version: z.literal(1), + report_id: z.string().regex(/^rr_[a-f0-9]{64}$/u), + source_digest: digestSchema, + report_digest: digestSchema, + status: readinessStatusSchema, + summary: z.strictObject({ + stages: z.number().int().min(0), + required_stages: z.number().int().min(0), + passed_required_stages: z.number().int().min(0), + findings: z.number().int().min(0), + failed_findings: z.number().int().min(0), + unknown_findings: z.number().int().min(0), + }), + metrics: z.strictObject({ + first_valid_workflow_selection_rate: z.number().min(0).max(1), + malformed_or_over_limit_call_rate: z.number().min(0).max(1), + incompatible_provider_invocations: z.number().int().min(0), + failed_calls_before_prerequisites: z.number().int().min(0), + recovery_rate: z.number().min(0).max(1), + divergence_localization_precision: z.number().min(0).max(1), + false_equivalence_count: z.number().int().min(0), + unowned_obligation_count: z.number().int().min(0), + cleanup_leak_count: z.number().int().min(0), + nondeterministic_replay_count: z.number().int().min(0), + }), + stages: z.array(stageSchema).max(20), + findings: z.array(findingSchema).max(10_000), + evidence_links: z.array(evidenceIdSchema).max(100_000), + snapshot: reconstructionReadinessInputSchema, +}); + +export type ReconstructionReadinessInput = z.infer< + typeof reconstructionReadinessInputSchema +>; +export type ReconstructionReadinessReport = z.infer< + typeof reconstructionReadinessReportSchema +>; +export type ReadinessStageId = z.infer; +export type ReadinessFinding = z.infer; diff --git a/src/server/registerApplicationTools/readiness.ts b/src/server/registerApplicationTools/readiness.ts new file mode 100644 index 0000000000..8d9324d143 --- /dev/null +++ b/src/server/registerApplicationTools/readiness.ts @@ -0,0 +1,79 @@ +import type { McpServer } from "@modelcontextprotocol/server"; + +import { + evaluateReconstructionReadinessValidated, + resolveReconstructionReadinessRequest, +} from "../../application/ReconstructionReadinessService.js"; +import { applicationToolContract } from "../../contracts/applicationToolContracts.js"; +import { reconstructionReadinessInputSchema } from "../../domain/reconstructionReadinessSchemas.js"; +import { reconstructionReadinessReportSchema } from "../../domain/reconstructionReadinessSchemas.js"; +import { logToolExecution } from "../toolLogging.js"; +import { safeParseToolInput } from "../toolInputValidation.js"; +import { toolRegistrationOptions } from "../toolRegistrationOptions.js"; +import { toCallToolResult } from "../toolResult.js"; +import { recordSources } from "./helpers.js"; +import type { ApplicationToolRegistration } from "./types.js"; + +const contract = applicationToolContract("evaluate_reconstruction_readiness"); + +/** Register deterministic public-contract reconstruction conformance. */ +export const registerReconstructionReadinessTool = ( + server: McpServer, + options: ApplicationToolRegistration, +): void => { + server.registerTool( + contract.name, + toolRegistrationOptions(contract), + async (input) => { + const parsed = safeParseToolInput( + reconstructionReadinessInputSchema, + input, + contract.name, + ); + if (!parsed.ok) return toCallToolResult(parsed, contract); + const resolved = resolveReconstructionReadinessRequest(parsed.value); + if (!resolved.ok) return toCallToolResult(resolved, contract); + const result = await logToolExecution(options.logger, contract.name, () => + Promise.resolve( + evaluateReconstructionReadinessValidated(resolved.value), + ), + ); + if (!result.ok) return toCallToolResult(result, contract); + const recorded = recordSources( + options.recordEvidence, + resolved.value.evidence_bundle.records, + ); + if (!recorded.ok) return toCallToolResult(recorded, contract); + const recordedResult = options.recordEvidence?.(result.value); + if (recordedResult !== undefined && !recordedResult.ok) + return toCallToolResult(recordedResult, contract); + const report = reconstructionReadinessReportSchema.parse( + result.value.normalized_result, + ); + const reportUri = `rea://evidence/${result.value.evidence_id}/reconstruction-readiness-report`; + const projection = { + schema: report.schema, + schema_version: report.schema_version, + report_id: report.report_id, + source_digest: report.source_digest, + report_digest: report.report_digest, + status: report.status, + summary: report.summary, + metrics: report.metrics, + report_resource_uri: reportUri, + }; + return toCallToolResult({ ok: true, value: result.value }, contract, { + evidenceResourcesAvailable: recordedResult !== undefined, + evidenceResultProjection: projection, + evidenceTextProjection: projection, + resourceLinks: [ + { + uri: reportUri, + name: report.report_id, + description: "Full deterministic reconstruction readiness report", + }, + ], + }); + }, + ); +}; diff --git a/src/server/registerReconstructionReadinessResource.ts b/src/server/registerReconstructionReadinessResource.ts new file mode 100644 index 0000000000..b965a6f851 --- /dev/null +++ b/src/server/registerReconstructionReadinessResource.ts @@ -0,0 +1,69 @@ +import { + ResourceNotFoundError, + ResourceTemplate, + type McpServer, +} from "@modelcontextprotocol/server"; + +import type { BinarySessionPort } from "../application/BinarySessionPort.js"; +import { reconstructionReadinessReportSchema } from "../domain/reconstructionReadinessSchemas.js"; + +/** Expose retained full readiness reports outside compact tool responses. */ +export const registerReconstructionReadinessResource = ( + server: McpServer, + session: BinarySessionPort, +): void => { + server.registerResource( + "reconstruction-readiness-report", + new ResourceTemplate( + "rea://evidence/{evidenceId}/reconstruction-readiness-report", + { + list: undefined, + complete: { + evidenceId: (prefix) => + session + .exportEvidenceBundle() + .records.filter( + ({ predicate_type: predicateType }) => + predicateType === "rea.reconstruction-readiness-report/v1", + ) + .map(({ evidence_id: evidenceId }) => evidenceId) + .filter((evidenceId) => evidenceId.startsWith(prefix)), + }, + }, + ), + { + title: "Reconstruction readiness report", + description: + "Full deterministic nine-stage conformance report and replay identity.", + mimeType: "application/json", + }, + (uri, variables) => { + const evidenceId = variables.evidenceId; + if (typeof evidenceId !== "string") + throw new ResourceNotFoundError(uri.href); + const evidence = session.evidenceById(evidenceId); + if ( + evidence === undefined || + evidence.predicate_type !== "rea.reconstruction-readiness-report/v1" + ) + throw new ResourceNotFoundError(uri.href); + const report = reconstructionReadinessReportSchema.safeParse( + evidence.normalized_result, + ); + if (!report.success) throw new ResourceNotFoundError(uri.href); + return { + contents: [ + { + uri: uri.href, + mimeType: "application/json", + text: JSON.stringify( + { evidence_id: evidence.evidence_id, report: report.data }, + null, + 2, + ), + }, + ], + }; + }, + ); +}; diff --git a/tests/conformance/readiness/electron/main.js b/tests/conformance/readiness/electron/main.js new file mode 100644 index 0000000000..9478471153 --- /dev/null +++ b/tests/conformance/readiness/electron/main.js @@ -0,0 +1,23 @@ +const { + app, + BrowserWindow, + ipcMain, + shell, + utilityProcess, +} = require("electron"); + +const windows = new Set(); +app.on("open-url", (_event, url) => { + for (const window of windows) window.webContents.send("deep-link", url); +}); +app.whenReady().then(() => { + const window = new BrowserWindow({ + webPreferences: { preload: require.resolve("./preload.js") }, + }); + windows.add(window); + window.loadFile("renderer.html"); + utilityProcess.fork(require.resolve("./utility.js")); + ipcMain.handle("readiness:echo", (_event, value) => ({ value })); + window.webContents.on("render-process-gone", () => window.reload()); + shell.openExternal("https://blocked.invalid").catch(() => undefined); +}); diff --git a/tests/conformance/readiness/electron/package.json b/tests/conformance/readiness/electron/package.json new file mode 100644 index 0000000000..d154971db2 --- /dev/null +++ b/tests/conformance/readiness/electron/package.json @@ -0,0 +1,5 @@ +{ + "name": "rea-readiness-electron", + "version": "1.0.0", + "main": "main.js" +} diff --git a/tests/conformance/readiness/electron/preload.js b/tests/conformance/readiness/electron/preload.js new file mode 100644 index 0000000000..460e37b27c --- /dev/null +++ b/tests/conformance/readiness/electron/preload.js @@ -0,0 +1,6 @@ +const { contextBridge, ipcRenderer } = require("electron"); + +contextBridge.exposeInMainWorld("readiness", { + echo: (value) => ipcRenderer.invoke("readiness:echo", value), + onDeepLink: (listener) => ipcRenderer.on("deep-link", listener), +}); diff --git a/tests/conformance/readiness/electron/renderer.html b/tests/conformance/readiness/electron/renderer.html new file mode 100644 index 0000000000..b78405bbaa --- /dev/null +++ b/tests/conformance/readiness/electron/renderer.html @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/tests/conformance/readiness/electron/renderer.js b/tests/conformance/readiness/electron/renderer.js new file mode 100644 index 0000000000..9c18ac5654 --- /dev/null +++ b/tests/conformance/readiness/electron/renderer.js @@ -0,0 +1,7 @@ +document.querySelector("#run").addEventListener("click", async () => { + document.querySelector("#result").textContent = + await window.readiness.echo("renderer"); +}); +window.readiness.onDeepLink((_event, url) => { + document.querySelector("#result").textContent = url; +}); diff --git a/tests/conformance/readiness/electron/utility.js b/tests/conformance/readiness/electron/utility.js new file mode 100644 index 0000000000..3424604f1d --- /dev/null +++ b/tests/conformance/readiness/electron/utility.js @@ -0,0 +1,5 @@ +const { parentPort } = require("electron"); + +parentPort.on("message", (event) => { + parentPort.postMessage({ echoed: event.data }); +}); diff --git a/tests/conformance/readiness/javascript-cli/index.js b/tests/conformance/readiness/javascript-cli/index.js new file mode 100644 index 0000000000..b0b692f2e7 --- /dev/null +++ b/tests/conformance/readiness/javascript-cli/index.js @@ -0,0 +1,34 @@ +import { spawn } from "node:child_process"; +import { readFile, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import process from "node:process"; +import readline from "node:readline"; + +const configurationPath = + process.env.REA_READINESS_CONFIG ?? "./readiness-config.json"; +const configuration = JSON.parse( + await readFile(configurationPath, "utf8").catch(() => "{}"), +); +const prompt = readline.createInterface({ + input: process.stdin, + output: process.stdout, +}); +process.stdout.write("\u001b[?1049h"); +const answer = await new Promise((resolve) => + prompt.question("readiness> ", resolve), +); +prompt.close(); +const child = spawn(process.execPath, ["-e", "process.stdout.write('child')"]); +const server = createServer((request, response) => { + response.end(JSON.stringify({ path: request.url, answer })); +}); +server.listen(Number(configuration.port ?? 0), "127.0.0.1"); +process.on("SIGTERM", () => { + child.kill("SIGTERM"); + server.close(); + process.stdout.write("\u001b[?1049l"); +}); +await writeFile( + configurationPath, + JSON.stringify({ ...configuration, lastAnswer: answer }), +); diff --git a/tests/conformance/readiness/javascript-cli/package.json b/tests/conformance/readiness/javascript-cli/package.json new file mode 100644 index 0000000000..64b35d1ad5 --- /dev/null +++ b/tests/conformance/readiness/javascript-cli/package.json @@ -0,0 +1,6 @@ +{ + "name": "rea-readiness-javascript-cli", + "version": "1.0.0", + "type": "module", + "bin": "index.js" +} diff --git a/tests/reconstructionReadiness.test.ts b/tests/reconstructionReadiness.test.ts new file mode 100644 index 0000000000..ad2b2c7ef4 --- /dev/null +++ b/tests/reconstructionReadiness.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, it } from "vitest"; + +import { RECONSTRUCTION_READINESS_EXAMPLE } from "../src/contracts/reconstructionReadinessExample.js"; +import { createReconstructionReadinessReport } from "../src/domain/reconstructionReadiness.js"; +import type { ReconstructionReadinessInput } from "../src/domain/reconstructionReadinessSchemas.js"; + +const input = ( + overrides: Partial = {}, +): ReconstructionReadinessInput => ({ + ...structuredClone(RECONSTRUCTION_READINESS_EXAMPLE), + ...overrides, +}); + +describe("reconstruction readiness report", () => { + it("passes the complete public-contract journey deterministically", () => { + const first = createReconstructionReadinessReport(input()); + const second = createReconstructionReadinessReport(input()); + + expect(first.status).toBe("pass"); + expect(first.summary).toMatchObject({ + required_stages: 9, + passed_required_stages: 9, + findings: 0, + }); + expect(first.report_id).toBe(second.report_id); + expect(first.report_digest).toBe(second.report_digest); + expect(first.metrics).toMatchObject({ + first_valid_workflow_selection_rate: 1, + incompatible_provider_invocations: 0, + false_equivalence_count: 0, + unowned_obligation_count: 0, + cleanup_leak_count: 0, + nondeterministic_replay_count: 0, + }); + }); + + it("rejects equivalence over truncated concurrent evidence", () => { + const value = input(); + const comparison = value.comparisons[0]; + if (comparison === undefined) throw new TypeError("Missing comparison"); + value.comparisons[0] = { + ...comparison, + schedule_semantics: "total-order", + truncated: true, + }; + const report = createReconstructionReadinessReport(value); + + expect(report.status).toBe("fail"); + expect(report.findings.map(({ code }) => code)).toEqual( + expect.arrayContaining([ + "false-equivalence", + "unsafe-concurrency-semantics", + ]), + ); + expect(report.metrics.false_equivalence_count).toBe(1); + }); + + it("keeps green stages unknown while one reconstruction obligation is open", () => { + const value = input(); + value.obligation_ledger = { + ...value.obligation_ledger, + status: "open", + summary: { + ...value.obligation_ledger.summary, + verified: 0, + required_open: 1, + }, + }; + const report = createReconstructionReadinessReport(value); + + expect(report.status).toBe("unknown"); + expect(report.stages).toContainEqual( + expect.objectContaining({ + stage_id: "verify-reconstruction-closure", + status: "unknown", + }), + ); + expect(report.findings).toContainEqual( + expect.objectContaining({ code: "reconstruction-closure-open" }), + ); + }); + + it("fails typed operation outcomes that ordinary automation sees as success", () => { + const value = input(); + const cliOutcome = value.operation_outcomes[2]; + const mcpOutcome = value.operation_outcomes[3]; + if (cliOutcome === undefined || mcpOutcome === undefined) + throw new TypeError("Missing operation outcome pair"); + value.operation_outcomes[2] = { + ...cliOutcome, + cli_exit_code: 0, + }; + value.operation_outcomes[3] = { + ...mcpOutcome, + mcp_status: "success", + }; + const report = createReconstructionReadinessReport(value); + + expect(report.status).toBe("fail"); + expect(report.findings.map(({ code }) => code)).toEqual( + expect.arrayContaining(["cli-status-mismatch", "mcp-status-mismatch"]), + ); + }); + + it("detects tampering against the exported source digest", () => { + const baseline = createReconstructionReadinessReport(input()); + const replay = input({ + replay: { + ...RECONSTRUCTION_READINESS_EXAMPLE.replay, + expected_source_digest: baseline.source_digest, + }, + }); + expect(createReconstructionReadinessReport(replay).source_digest).toBe( + baseline.source_digest, + ); + replay.identity.cli_version = "tampered"; + const tampered = createReconstructionReadinessReport(replay); + + expect(tampered.status).toBe("fail"); + expect(tampered.findings).toContainEqual( + expect.objectContaining({ code: "replay-source-digest-mismatch" }), + ); + }); + + it("fails a reconstruction candidate that delegates to the authority", () => { + const value = input(); + const check = value.delegation_checks[0]; + if (check === undefined) throw new TypeError("Missing delegation check"); + value.delegation_checks[0] = { + ...check, + delegates_to_authority: true, + }; + const report = createReconstructionReadinessReport(value); + + expect(report.status).toBe("fail"); + expect(report.findings).toContainEqual( + expect.objectContaining({ code: "candidate-delegates-to-authority" }), + ); + }); + + it("preserves an exact unsupported stage without collapsing it to pass", () => { + const value = input(); + const stage = value.stages.find( + ({ stage_id: stageId }) => stageId === "reactive-scenarios", + ); + const check = stage?.checks[0]; + if (stage === undefined || check === undefined) + throw new TypeError("Missing reactive stage"); + stage.status = "unsupported"; + check.status = "unsupported"; + stage.capability_issue = "electron-runtime-unavailable"; + stage.next_action = "Run on a host with the approved Electron runtime."; + const report = createReconstructionReadinessReport(value); + + expect(report.status).toBe("unsupported"); + expect(report.stages).toContainEqual( + expect.objectContaining({ + stage_id: "reactive-scenarios", + status: "unsupported", + }), + ); + }); +}); diff --git a/tests/reconstructionReadinessCli.test.ts b/tests/reconstructionReadinessCli.test.ts new file mode 100644 index 0000000000..665ad1a221 --- /dev/null +++ b/tests/reconstructionReadinessCli.test.ts @@ -0,0 +1,36 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +import { describe, expect, it } from "vitest"; + +import { RECONSTRUCTION_READINESS_EXAMPLE } from "../src/contracts/reconstructionReadinessExample.js"; + +const execute = promisify(execFile); + +describe("reconstruction readiness CLI parity", () => { + it("emits the same passing report through the public CLI", async () => { + const { stdout } = await execute( + process.execPath, + [ + "scripts/rea.mjs", + "evaluate-reconstruction-readiness", + JSON.stringify(RECONSTRUCTION_READINESS_EXAMPLE), + "--json", + ], + { + cwd: process.cwd(), + env: process.env, + maxBuffer: 32 * 1_024 * 1_024, + }, + ); + expect(JSON.parse(stdout)).toMatchObject({ + operation: "evaluate_reconstruction_readiness", + predicate_type: "rea.reconstruction-readiness-report/v1", + normalized_result: { + schema: "ReconstructionReadinessReport", + status: "pass", + summary: { passed_required_stages: 9 }, + }, + }); + }, 20_000); +}); diff --git a/tests/reconstructionReadinessMcp.test.ts b/tests/reconstructionReadinessMcp.test.ts new file mode 100644 index 0000000000..4f5007711d --- /dev/null +++ b/tests/reconstructionReadinessMcp.test.ts @@ -0,0 +1,65 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { describe, expect, it } from "vitest"; + +import { BinarySession } from "../src/application/BinarySession.js"; +import { RECONSTRUCTION_READINESS_EXAMPLE } from "../src/contracts/reconstructionReadinessExample.js"; +import { createServer } from "../src/server/createServer.js"; +import { observed } from "./fixtures/analysisExecution.js"; + +describe("reconstruction readiness MCP parity", () => { + it("returns a passing compact result and retains the full report resource", async () => { + const session = new BinarySession(() => ({ + execute: () => Promise.resolve(observed(null)), + close: () => Promise.resolve(), + })); + const server = createServer(session, session); + const client = new Client({ name: "readiness-test", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const result = await client.callTool({ + name: "evaluate_reconstruction_readiness", + arguments: RECONSTRUCTION_READINESS_EXAMPLE, + }); + expect(result.isError).not.toBe(true); + expect(result.structuredContent).toMatchObject({ + evidence_id: expect.stringMatching(/^ev_[a-f0-9]{64}$/u), + result: { + schema: "ReconstructionReadinessReport", + status: "pass", + summary: { passed_required_stages: 9 }, + report_resource_uri: expect.stringMatching( + /^rea:\/\/evidence\/ev_[a-f0-9]{64}\/reconstruction-readiness-report$/u, + ), + }, + }); + expect(result.structuredContent).not.toHaveProperty("result.snapshot"); + expect(result.structuredContent).not.toHaveProperty("result.stages"); + const content = result.structuredContent; + if ( + content === undefined || + content === null || + typeof content !== "object" || + !("evidence_id" in content) || + typeof content.evidence_id !== "string" + ) + throw new TypeError("Missing readiness Evidence ID"); + const resource = await client.readResource({ + uri: `rea://evidence/${content.evidence_id}/reconstruction-readiness-report`, + }); + expect(resource.contents[0]).toEqual( + expect.objectContaining({ + text: expect.stringContaining( + '"schema": "ReconstructionReadinessReport"', + ), + }), + ); + } finally { + await client.close(); + await server.close(); + await session.close(); + } + }); +});