diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 2f56858cc..a4de713cb 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -876,6 +876,21 @@ includes all dimension limitations. Equal CFGs establish only equal observed block sizes and successor relationships. Text from different provider identities remains unknown. +The `references` dimension preserves each edge's direction, endpoint identity +and available kind facts. Within one artifact it compares exact raw endpoint +coordinates, including the address space; hexadecimal case and zero padding +do not change identity. Resolving an endpoint to a containing procedure does +not replace the raw coordinate. Unparseable coordinates stay unknown. + +Across artifacts, reference endpoints use stable procedure names. Missing or +address-generated names remain unknown. A name observed at multiple procedure +entries retains its label constraint but cannot establish equal endpoints. +Pairing preserves duplicate counts and considers alternative compatible +pairings; array order never identifies an edge. Compatible records with +unknown endpoint identity or kind facts remain `unknown`. Independent known +facts that contradict every possible pairing still establish `changed`. +Source Evidence retains the original addresses, names and unavailable facts. + ### Comparison input shapes `compare_web_captures` accepts exactly one of two input shapes: diff --git a/src/domain/callPath.ts b/src/domain/callPath.ts index 3b8bd1cfe..0591be032 100644 --- a/src/domain/callPath.ts +++ b/src/domain/callPath.ts @@ -2,11 +2,11 @@ import { relationshipCoverageLimitations } from "./relationshipCoverage.js"; import { callPathInputSchema, callPathResultSchema, - parseCallPathAddress, type CallPathInput, type CallPathResult, type OutputCallPath, } from "./callPathSchemas.js"; +import { parseNativeAddress } from "./native/nativeAddress.js"; import { parseFunctionEvidence, type FunctionSnapshot, @@ -158,9 +158,9 @@ const parseSnapshots = ( const snapshots = new Map(); for (const group of groups) { const snapshot = parseFunctionEvidence(group); - const address = parseCallPathAddress(snapshot.dossier.procedure.address); + const address = parseNativeAddress(snapshot.dossier.procedure.address); for (const callee of snapshot.dossier.callees) - parseCallPathAddress(callee.address); + parseNativeAddress(callee.address); if (snapshots.has(address)) throw new TypeError(`Duplicate function Evidence for ${address}`); snapshots.set(address, snapshot); @@ -197,7 +197,7 @@ const createGraph = ( for (const [address, snapshot] of snapshots) { graph.mergeNode(address); for (const callee of snapshot.dossier.callees) { - const calleeAddress = parseCallPathAddress(callee.address); + const calleeAddress = parseNativeAddress(callee.address); graph.mergeNode(calleeAddress); graph.mergeDirectedEdge(address, calleeAddress); } diff --git a/src/domain/callPathSchemas.ts b/src/domain/callPathSchemas.ts index a7286e366..7c40073f4 100644 --- a/src/domain/callPathSchemas.ts +++ b/src/domain/callPathSchemas.ts @@ -3,43 +3,27 @@ import { z } from "zod"; import { emptyArraySchema } from "./emptyArraySchema.js"; import { evidenceSchema } from "./evidence.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; - -const addressSchema = z - .string() - .regex( - /^(?:(?:[A-Za-z0-9._~\x2d]|%[0-9A-F]{2})+:)?0x(?:0|[1-9a-f][0-9a-f]*)$/u, - ); -const inputAddressSchema = z - .string() - .regex(/^(?:(?:[A-Za-z0-9._~\x2d]|%[0-9a-fA-F]{2})+:)?0[xX][0-9a-fA-F]+$/u) - .transform((address) => { - const separator = address.lastIndexOf(":") + 1; - const space = address - .slice(0, separator) - .replace(/%[0-9a-fA-F]{2}/gu, (escape) => escape.toUpperCase()); - return `${space}0x${BigInt(address.slice(separator)).toString(16)}`; - }); +import { + nativeAddressSchema, + nativeAddressInputSchema, +} from "./native/nativeAddress.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); -/** Normalize a hexadecimal offset while preserving its optional address space. */ -export const parseCallPathAddress = (input: unknown): string => - inputAddressSchema.parse(input); - /** Strict input for explicit call-path reconstruction from complete dossiers. */ export const callPathInputSchema = z.strictObject({ functions: z.array(evidenceSchema).min(1), - start: z.object({ address: inputAddressSchema }).strict(), - goal: z.object({ address: inputAddressSchema }).strict(), + start: z.object({ address: nativeAddressInputSchema }).strict(), + goal: z.object({ address: nativeAddressInputSchema }).strict(), }); const citedNodeSchema = z.object({ - address: addressSchema, + address: nativeAddressSchema, name: z.string().nullable(), evidence_links: z.array(evidenceIdSchema).min(1), }); const citedEdgeSchema = z.object({ - source: addressSchema, - target: addressSchema, + source: nativeAddressSchema, + target: nativeAddressSchema, evidence_links: z.array(evidenceIdSchema).min(1), }); const pathSchema = z @@ -66,8 +50,8 @@ const pathSchema = z }); }); const resultContextShape = { - start: addressSchema, - goal: addressSchema, + start: nativeAddressSchema, + goal: nativeAddressSchema, explored: z.object({ nodes: z.number().int().min(0), edges: z.number().int().min(0), diff --git a/src/domain/functionComparisonDimensions.ts b/src/domain/functionComparisonDimensions.ts index 0f9eb91bd..be6ea7eff 100644 --- a/src/domain/functionComparisonDimensions.ts +++ b/src/domain/functionComparisonDimensions.ts @@ -1,4 +1,5 @@ import { canonicalJson } from "./comparisonSemantics.js"; +import { compareFunctionReferences } from "./functionReferenceComparison.js"; import { diffLines } from "diff"; import type { FunctionSnapshot } from "./functionDossierEvidence.js"; @@ -7,8 +8,6 @@ import { identityProjection, isAutoName, normalizeCfg, - referenceKindProjection, - referenceProjection, sorted, stringAndNameProjection, swiftLiteralSitesConflict, @@ -253,78 +252,25 @@ const swiftLiteralCoverage = ( ? "absent" : "complete"; -const hasUnnamedEndpoint = (snapshot: FunctionSnapshot): boolean => - [ - ...snapshot.dossier.incoming_references, - ...snapshot.dossier.outgoing_references, - ].some( - (edge) => edge.source_procedure === null || edge.target_procedure === null, - ); - const compareReferences = ( left: FunctionSnapshot, right: FunctionSnapshot, context: ComparisonContext, ): FunctionDimension => { - const sameArtifact = - left.subject.digest.sha256 === right.subject.digest.sha256; - const leftProjection = referenceProjection(left, sameArtifact); - const rightProjection = referenceProjection(right, sameArtifact); - if (leftProjection.length === 0 && rightProjection.length === 0) - return compareValues( - "references", - leftProjection, - rightProjection, - context, - ); - if ( - canonicalJson(leftProjection, "Function comparison") !== - canonicalJson(rightProjection, "Function comparison") - ) - return dimensionResult({ - dimension: "references", - status: "changed", - left: leftProjection, - right: rightProjection, - ...context, - leftCount: leftProjection.length, - rightCount: rightProjection.length, - textDelta: null, - limitations: [ - "Reference endpoints differ independently of provider reference-kind metadata.", - ], - }); - const leftKinds = referenceKindProjection(left, sameArtifact); - const rightKinds = referenceKindProjection(right, sameArtifact); - const kindsKnown = leftKinds !== null && rightKinds !== null; - // Differing known edge kinds are a change whatever the endpoint identity. - if ( - kindsKnown && - canonicalJson(leftKinds, "Function comparison") !== - canonicalJson(rightKinds, "Function comparison") - ) - return compareValues("references", leftKinds, rightKinds, context); - // Equal projections with unknown endpoints do not establish equal targets. - if (!sameArtifact && (hasUnnamedEndpoint(left) || hasUnnamedEndpoint(right))) + const compared = compareFunctionReferences(left, right); + if (compared.status === "unknown") return unresolvedDimension({ dimension: "references", links: context.links, - leftCount: leftProjection.length, - rightCount: rightProjection.length, - limitations: [ - "Unnamed reference endpoints have no stable identity across different artifacts, so equal named endpoints do not establish equal references.", - ], + leftCount: compared.leftCount, + rightCount: compared.rightCount, + limitations: compared.limitations, }); - if (kindsKnown) - return compareValues("references", leftKinds, rightKinds, context); - return unresolvedDimension({ + return dimensionResult({ dimension: "references", - links: context.links, - leftCount: leftProjection.length, - rightCount: rightProjection.length, - limitations: [ - "At least one provider did not expose reference kinds, so equal endpoints do not prove equal edge semantics.", - ], + ...compared, + ...context, + textDelta: null, }); }; diff --git a/src/domain/functionComparisonNormalization.ts b/src/domain/functionComparisonNormalization.ts index 85b7d0e44..4b980dc5d 100644 --- a/src/domain/functionComparisonNormalization.ts +++ b/src/domain/functionComparisonNormalization.ts @@ -72,101 +72,6 @@ export const normalizeCfg = ( return graph; }; -type ReferenceEdge = FunctionSnapshot["dossier"]["outgoing_references"][number]; - -/** - * A named endpoint survives relocation. An unnamed endpoint is identified - * only by its address, which names one location only within one artifact; - * across artifacts it stays unknown (null). - */ -const referenceEndpoint = ( - procedure: ReferenceEdge["source_procedure"], - address: string, - sameArtifact: boolean, -): string | { readonly address: string } | null => - procedure !== null ? procedure.name : sameArtifact ? { address } : null; - -export const referenceProjection = ( - snapshot: FunctionSnapshot, - sameArtifact: boolean, -) => - sorted( - [ - ...snapshot.dossier.incoming_references.map((item) => ({ - direction: "in", - item, - })), - ...snapshot.dossier.outgoing_references.map((item) => ({ - direction: "out", - item, - })), - ].map(({ direction, item }) => ({ - direction, - source: referenceEndpoint( - item.source_procedure, - item.source_address, - sameArtifact, - ), - target: referenceEndpoint( - item.target_procedure, - item.target_address, - sameArtifact, - ), - })), - ); - -export const referenceKindProjection = ( - snapshot: FunctionSnapshot, - sameArtifact: boolean, -): readonly unknown[] | null => { - const edges = [ - ...snapshot.dossier.incoming_references.map((edge) => ({ - direction: "in", - edge, - })), - ...snapshot.dossier.outgoing_references.map((edge) => ({ - direction: "out", - edge, - })), - ]; - if (edges.some(({ edge }) => !edge.kind.available)) return null; - return sorted( - edges.map(({ direction, edge }) => { - if (!edge.kind.available) - throw new TypeError( - "Reference-kind availability changed during projection", - ); - return { - direction, - source: referenceEndpoint( - edge.source_procedure, - edge.source_address, - sameArtifact, - ), - target: referenceEndpoint( - edge.target_procedure, - edge.target_address, - sameArtifact, - ), - type: edge.kind.type, - flow: edge.kind.flow, - call: edge.kind.call, - jump: edge.kind.jump, - data: edge.kind.data, - read: edge.kind.read, - write: edge.kind.write, - indirect: edge.kind.indirect, - computed: edge.kind.computed, - conditional: edge.kind.conditional, - terminal: edge.kind.terminal, - primary: edge.kind.primary, - operand_index: edge.kind.operand_index, - external: edge.kind.external, - }; - }), - ); -}; - export const commentProjection = ( snapshot: FunctionSnapshot, ): readonly unknown[] | null => { @@ -287,7 +192,7 @@ const swiftLiteralSites = (snapshot: FunctionSnapshot) => { return { values, complete }; }; -export const sorted = (values: readonly unknown[]): readonly unknown[] => +export const sorted = (values: readonly T[]): readonly T[] => values .map((value) => ({ value, diff --git a/src/domain/functionReferenceComparison.ts b/src/domain/functionReferenceComparison.ts new file mode 100644 index 000000000..e48e376f8 --- /dev/null +++ b/src/domain/functionReferenceComparison.ts @@ -0,0 +1,246 @@ +import { canonicalJson } from "./comparisonSemantics.js"; +import type { FunctionSnapshot } from "./functionDossierEvidence.js"; +import { isAutoName, sorted } from "./functionComparisonNormalization.js"; +import { + referenceFactsCompatible, + type ReferenceCompatibilityRow, +} from "./functionReferenceCompatibility.js"; +import { nativeAddressInputSchema } from "./native/nativeAddress.js"; + +type Reference = FunctionSnapshot["dossier"]["outgoing_references"][number]; +type KnownKind = Omit< + Extract, + "available" | "provenance" +>; +type Endpoint = + | { readonly kind: "address"; readonly address: string } + | { + readonly kind: "name"; + readonly name: string; + readonly ambiguous: boolean; + } + | { + readonly kind: "unknown"; + readonly reason: "unparsed_address" | "missing_name" | "generated_name"; + readonly reported_address: string; + readonly reported_name: string | null; + }; + +interface ReferenceFact { + readonly direction: "in" | "out"; + readonly source: Endpoint; + readonly target: Endpoint; + readonly kind: KnownKind | null; +} + +type ReferenceComparison = { + readonly leftCount: number; + readonly rightCount: number; + readonly limitations: readonly string[]; +} & ( + | { readonly status: "unknown" } + | { + readonly status: "changed" | "unchanged"; + readonly left: readonly ReferenceFact[]; + readonly right: readonly ReferenceFact[]; + } +); + +/** Names observed at different procedure entries do not uniquely identify an overload. */ +const ambiguousProcedureNames = ( + snapshots: readonly FunctionSnapshot[], +): Set => { + const ambiguous = new Set(); + for (const snapshot of snapshots) { + const entries = new Map(); + const observe = (procedure: Reference["source_procedure"]): void => { + if (procedure === null) return; + const address = nativeAddressInputSchema.safeParse(procedure.address); + if (!address.success) { + ambiguous.add(procedure.name); + return; + } + const previous = entries.get(procedure.name); + if (previous !== undefined && previous !== address.data) + ambiguous.add(procedure.name); + entries.set(procedure.name, address.data); + }; + observe(snapshot.dossier.procedure); + for (const edge of snapshot.dossier.incoming_references) { + observe(edge.source_procedure); + observe(edge.target_procedure); + } + for (const edge of snapshot.dossier.outgoing_references) { + observe(edge.source_procedure); + observe(edge.target_procedure); + } + } + return ambiguous; +}; + +const endpoint = ( + procedure: Reference["source_procedure"], + address: string, + sameArtifact: boolean, + ambiguousNames: ReadonlySet, +): Endpoint => { + if (sameArtifact) { + const parsed = nativeAddressInputSchema.safeParse(address); + if (parsed.success) return { kind: "address", address: parsed.data }; + return { + kind: "unknown", + reason: "unparsed_address", + reported_address: address, + reported_name: procedure?.name ?? null, + }; + } + if ( + procedure === null || + procedure.name.length === 0 || + isAutoName(procedure.name) + ) + return { + kind: "unknown", + reason: + procedure !== null && isAutoName(procedure.name) + ? "generated_name" + : "missing_name", + reported_address: address, + reported_name: procedure?.name ?? null, + }; + return { + kind: "name", + name: procedure.name, + ambiguous: ambiguousNames.has(procedure.name), + }; +}; + +const project = ( + snapshot: FunctionSnapshot, + sameArtifact: boolean, + ambiguousNames: ReadonlySet, +): ReferenceFact[] => { + const facts: ReferenceFact[] = []; + const collect = ( + references: readonly Reference[], + direction: "in" | "out", + ): void => { + for (const edge of references) { + let kind: KnownKind | null = null; + if (edge.kind.available) { + const { + available: _available, + provenance: _provenance, + ...known + } = edge.kind; + kind = known; + } + facts.push({ + direction, + source: endpoint( + edge.source_procedure, + edge.source_address, + sameArtifact, + ambiguousNames, + ), + target: endpoint( + edge.target_procedure, + edge.target_address, + sameArtifact, + ambiguousNames, + ), + kind, + }); + } + }; + collect(snapshot.dossier.incoming_references, "in"); + collect(snapshot.dossier.outgoing_references, "out"); + return facts; +}; + +/** Ambiguity blocks equality, but a known label can still contradict another label. */ +const endpointKey = (value: Endpoint): string | null => { + if (value.kind === "unknown") return null; + return canonicalJson( + value.kind === "address" + ? { address: value.address } + : { name: value.name }, + ); +}; + +const constraints = ( + facts: readonly ReferenceFact[], +): ReferenceCompatibilityRow[] => + facts.map((fact) => ({ + direction: fact.direction, + source: endpointKey(fact.source), + target: endpointKey(fact.target), + kind: fact.kind === null ? null : canonicalJson(fact.kind), + })); + +/** Compare parsed endpoint and kind facts without replacing partial knowledge with absence. */ +export const compareFunctionReferences = ( + left: FunctionSnapshot, + right: FunctionSnapshot, +): ReferenceComparison => { + const sameArtifact = + left.subject.digest.sha256 === right.subject.digest.sha256; + const ambiguousNames = sameArtifact + ? new Set() + : ambiguousProcedureNames([left, right]); + const leftFacts = project(left, sameArtifact, ambiguousNames); + const rightFacts = project(right, sameArtifact, ambiguousNames); + let unknownIdentity = false; + let ambiguousIdentity = false; + let unknownKinds = false; + for (const facts of [leftFacts, rightFacts]) { + for (const fact of facts) { + unknownKinds ||= fact.kind === null; + for (const value of [fact.source, fact.target]) { + unknownIdentity ||= value.kind === "unknown"; + ambiguousIdentity ||= value.kind === "name" && value.ambiguous; + } + } + } + const limitations: string[] = []; + if (unknownIdentity) + limitations.push( + sameArtifact + ? "Reference endpoint coordinates could not be parsed, so their identity remains unknown." + : "Unnamed reference endpoints and generated procedure names have no stable identity across different artifacts.", + ); + if (ambiguousIdentity) + limitations.push( + "A reference name does not uniquely identify an observed procedure entry, so matching names do not establish equal endpoints.", + ); + if (unknownKinds) + limitations.push( + "At least one provider did not expose reference kinds, so compatible endpoints do not prove equal edge semantics.", + ); + const context = { + leftCount: leftFacts.length, + rightCount: rightFacts.length, + limitations, + }; + if ( + !referenceFactsCompatible(constraints(leftFacts), constraints(rightFacts)) + ) + return { + ...context, + status: "changed", + left: sorted(leftFacts), + right: sorted(rightFacts), + limitations: [ + "Known reference facts cannot be paired without contradicting an endpoint or reference kind.", + ...limitations, + ], + }; + if (unknownIdentity || ambiguousIdentity || unknownKinds) + return { ...context, status: "unknown" }; + return { + ...context, + status: "unchanged", + left: sorted(leftFacts), + right: sorted(rightFacts), + }; +}; diff --git a/src/domain/functionReferenceCompatibility.test.ts b/src/domain/functionReferenceCompatibility.test.ts new file mode 100644 index 000000000..599df7e8f --- /dev/null +++ b/src/domain/functionReferenceCompatibility.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it } from "vitest"; + +import { + referenceFactsCompatible, + type ReferenceCompatibilityRow, +} from "./functionReferenceCompatibility.js"; + +const row = ( + source: string | null, + target: string | null, + kind: string | null = "call", + direction: "in" | "out" = "out", +): ReferenceCompatibilityRow => ({ direction, source, target, kind }); + +const oracle = ( + left: readonly ReferenceCompatibilityRow[], + right: readonly ReferenceCompatibilityRow[], +): boolean => { + if (left.length !== right.length) return false; + const assigned = new Set(); + const visit = (index: number): boolean => { + if (index === left.length) return true; + const candidate = left[index]; + if (candidate === undefined) return false; + for (let rightIndex = 0; rightIndex < right.length; rightIndex++) { + if (assigned.has(rightIndex)) continue; + const other = right[rightIndex]; + if (other === undefined || candidate.direction !== other.direction) + continue; + if ( + (candidate.source !== null && + other.source !== null && + candidate.source !== other.source) || + (candidate.target !== null && + other.target !== null && + candidate.target !== other.target) || + (candidate.kind !== null && + other.kind !== null && + candidate.kind !== other.kind) + ) + continue; + assigned.add(rightIndex); + if (visit(index + 1)) return true; + assigned.delete(rightIndex); + } + return false; + }; + return visit(0); +}; + +const multisetsThroughTwo = (): ReferenceCompatibilityRow[][] => { + const values: (string | null)[] = [null, "a", "b"]; + const rows = values.flatMap((source) => + values.map((target) => row(source, target)), + ); + const multisets: ReferenceCompatibilityRow[][] = [[]]; + for (let first = 0; first < rows.length; first++) { + const firstRow = rows[first]; + if (firstRow === undefined) continue; + multisets.push([firstRow]); + for (let second = first; second < rows.length; second++) { + const secondRow = rows[second]; + if (secondRow !== undefined) multisets.push([firstRow, secondRow]); + } + } + return multisets; +}; + +describe("referenceFactsCompatible", () => { + it("agrees with an independent permutation oracle for exhaustive two-field multisets", () => { + const multisets = multisetsThroughTwo(); + for (const left of multisets) { + for (const right of multisets) { + expect( + referenceFactsCompatible(left, right), + JSON.stringify({ left, right }), + ).toBe(oracle(left, right)); + } + } + }); + + it("agrees with the oracle for seeded three-field sets with duplicates and directions", () => { + const values: (string | null)[] = [null, "a", "b"]; + const directions = ["in", "out"] as const; + let state = 0x4210; + const next = (limit: number): number => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + return (state >>> 8) % limit; + }; + for (let sample = 0; sample < 700; sample++) { + const length = next(7); + const makeRows = (): ReferenceCompatibilityRow[] => + Array.from({ length }, () => + row( + values[next(values.length)] ?? null, + values[next(values.length)] ?? null, + values[next(values.length)] ?? null, + directions[next(directions.length)] ?? "out", + ), + ); + const left = makeRows(); + const right = makeRows(); + expect( + referenceFactsCompatible(left, right), + JSON.stringify({ left, right }), + ).toBe(oracle(left, right)); + } + }); + + it("does not greedily cancel identical partial rows before a compatible cross-pairing", () => { + const left = [row(null, null, null), row("left", "left-target", "call")]; + const right = [row(null, null, null), row("right", "right-target", "data")]; + + expect(referenceFactsCompatible(left, right)).toBe(true); + expect(oracle(left, right)).toBe(true); + }); + + it("requires direction equality even when all other fields are unknown", () => { + expect( + referenceFactsCompatible( + [row(null, null, null, "in")], + [row(null, null, null, "out")], + ), + ).toBe(false); + }); + + it("finds a perfect match through one wildcard hub among ten thousand rows", () => { + const count = 10_000; + const left = Array.from({ length: count }, (_, index) => + row(`s${index}`, `t${index}`, `k${index}`), + ); + const right = left.slice(0, -1); + right.push(row(null, null, null)); + + expect(referenceFactsCompatible(left, right)).toBe(true); + }); + + it("handles ten thousand rows in one dense compatible bucket", () => { + const count = 10_000; + const left = Array.from({ length: count }, (_, index) => + row(null, `t${index}`, "call"), + ); + const right = Array.from({ length: count }, (_, index) => + row(`s${index}`, null, "call"), + ); + + expect(referenceFactsCompatible(left, right)).toBe(true); + }); +}); diff --git a/src/domain/functionReferenceCompatibility.ts b/src/domain/functionReferenceCompatibility.ts new file mode 100644 index 000000000..dd7354854 --- /dev/null +++ b/src/domain/functionReferenceCompatibility.ts @@ -0,0 +1,228 @@ +/** Only these three facts can be unknown; direction is always observed. */ +export interface ReferenceCompatibilityRow { + readonly direction: "in" | "out"; + readonly source: string | null; + readonly target: string | null; + readonly kind: string | null; +} + +interface ReferenceGroup extends ReferenceCompatibilityRow { + readonly mask: number; + count: number; +} + +interface FlowEdge { + readonly to: number; + readonly reverse: number; + remaining: number; +} + +/** A local residual network: no recursive walk or pairwise adjacency matrix. */ +class ReferenceFlow { + private readonly edges: FlowEdge[][] = []; + + node(): number { + this.edges.push([]); + return this.edges.length - 1; + } + + connect(from: number, to: number, capacity: number): void { + const forward = this.edges[from]; + const reverse = this.edges[to]; + if (forward === undefined || reverse === undefined) + throw new Error("Reference flow endpoint was not allocated."); + forward.push({ to, reverse: reverse.length, remaining: capacity }); + reverse.push({ to: from, reverse: forward.length - 1, remaining: 0 }); + } + + covers(source: number, sink: number, required: number): boolean { + let delivered = 0; + while (delivered < required) { + const levels = Array(this.edges.length).fill(-1); + const queue = [source]; + levels[source] = 0; + for (let index = 0; index < queue.length; index++) { + const node = queue[index]; + if (node === undefined) break; + for (const edge of this.edges[node] ?? []) { + if (edge.remaining <= 0 || levels[edge.to] !== -1) continue; + levels[edge.to] = (levels[node] ?? -1) + 1; + queue.push(edge.to); + } + } + if (levels[sink] === -1) return false; + const next = Array(this.edges.length).fill(0); + let amount: number; + while ( + delivered < required && + (amount = this.augment( + source, + sink, + required - delivered, + levels, + next, + )) > 0 + ) + delivered += amount; + } + return true; + } + + private augment( + source: number, + sink: number, + capacity: number, + levels: number[], + next: number[], + ): number { + const nodes = [source]; + const path: FlowEdge[] = []; + const capacities = [capacity]; + while (nodes.length > 0) { + const node = nodes[nodes.length - 1]; + if (node === undefined) break; + if (node === sink) { + const amount = capacities[capacities.length - 1] ?? 0; + for (const edge of path) { + edge.remaining -= amount; + const reverse = this.edges[edge.to]?.[edge.reverse]; + if (reverse === undefined) + throw new Error("Reference flow reverse edge was not allocated."); + reverse.remaining += amount; + } + return amount; + } + const adjacent = this.edges[node] ?? []; + let edge = adjacent[next[node] ?? 0]; + while ( + edge !== undefined && + (edge.remaining <= 0 || levels[edge.to] !== (levels[node] ?? -1) + 1) + ) { + next[node] = (next[node] ?? 0) + 1; + edge = adjacent[next[node] ?? 0]; + } + if (edge === undefined) { + levels[node] = -1; + nodes.pop(); + path.pop(); + capacities.pop(); + continue; + } + path.push(edge); + nodes.push(edge.to); + capacities.push( + Math.min(capacities[capacities.length - 1] ?? 0, edge.remaining), + ); + } + return 0; + } +} + +const groupRows = ( + rows: readonly ReferenceCompatibilityRow[], +): Map => { + const groups = new Map(); + for (const row of rows) { + const key = JSON.stringify([ + row.direction, + row.source, + row.target, + row.kind, + ]); + const previous = groups.get(key); + if (previous !== undefined) previous.count += 1; + else + groups.set(key, { + ...row, + mask: + (row.source === null ? 0 : 1) | + (row.target === null ? 0 : 2) | + (row.kind === null ? 0 : 4), + count: 1, + }); + } + return groups; +}; + +/** + * Decide whether every row can pair without contradicting a known fact. + * A bucket represents one pair of knowledge masks and their shared facts. + * Each group joins at most eight buckets, so graph storage is linear in the + * number of distinct rows. Integer flow preserves multiplicities exactly. + */ +export const referenceFactsCompatible = ( + left: readonly ReferenceCompatibilityRow[], + right: readonly ReferenceCompatibilityRow[], +): boolean => { + if (left.length !== right.length) return false; + if (left.length === 0) return true; + const leftGroups = groupRows(left); + const rightGroups = groupRows(right); + if ( + leftGroups.size === rightGroups.size && + [...leftGroups].every( + ([key, group]) => rightGroups.get(key)?.count === group.count, + ) + ) + return true; + if ( + [...leftGroups.values(), ...rightGroups.values()].every( + ({ mask }) => mask === 7, + ) + ) + return false; + // Cancelling an identical complete row is safe: any two partial partners of + // that row agree on their shared facts and can instead pair with each other. + // Cancelling identical partial rows is unsafe and must not be done. + let required = left.length; + for (const [key, group] of leftGroups) { + if (group.mask !== 7) continue; + const other = rightGroups.get(key); + if (other === undefined) continue; + const cancelled = Math.min(group.count, other.count); + group.count -= cancelled; + other.count -= cancelled; + required -= cancelled; + if (group.count === 0) leftGroups.delete(key); + if (other.count === 0) rightGroups.delete(key); + } + const leftMasks = new Set([...leftGroups.values()].map(({ mask }) => mask)); + const rightMasks = new Set([...rightGroups.values()].map(({ mask }) => mask)); + const flow = new ReferenceFlow(); + const source = flow.node(); + const sink = flow.node(); + const buckets = new Map(); + const bucket = ( + group: ReferenceGroup, + leftMask: number, + rightMask: number, + ): number => { + const intersection = leftMask & rightMask; + const key = JSON.stringify([ + group.direction, + leftMask, + rightMask, + intersection & 1 ? group.source : null, + intersection & 2 ? group.target : null, + intersection & 4 ? group.kind : null, + ]); + const existing = buckets.get(key); + if (existing !== undefined) return existing; + const node = flow.node(); + buckets.set(key, node); + return node; + }; + for (const group of leftGroups.values()) { + const node = flow.node(); + flow.connect(source, node, group.count); + for (const mask of rightMasks) + flow.connect(node, bucket(group, group.mask, mask), group.count); + } + for (const group of rightGroups.values()) { + const node = flow.node(); + flow.connect(node, sink, group.count); + for (const mask of leftMasks) + flow.connect(bucket(group, mask, group.mask), node, group.count); + } + return flow.covers(source, sink, required); +}; diff --git a/src/domain/native/nativeAddress.ts b/src/domain/native/nativeAddress.ts new file mode 100644 index 000000000..f15d53b0c --- /dev/null +++ b/src/domain/native/nativeAddress.ts @@ -0,0 +1,23 @@ +import { z } from "zod"; + +/** Canonical hexadecimal coordinate, retaining its optional address space. */ +export const nativeAddressSchema = z + .string() + .regex( + /^(?:(?:[A-Za-z0-9._~\x2d]|%[0-9A-F]{2})+:)?0x(?:0|[1-9a-f][0-9a-f]*)$/u, + ); + +/** Accept equivalent hexadecimal spellings without changing address-space identity. */ +export const nativeAddressInputSchema = z + .string() + .regex(/^(?:(?:[A-Za-z0-9._~\x2d]|%[0-9a-fA-F]{2})+:)?0[xX][0-9a-fA-F]+$/u) + .transform((address) => { + const separator = address.lastIndexOf(":") + 1; + const space = address + .slice(0, separator) + .replace(/%[0-9a-fA-F]{2}/gu, (escape) => escape.toUpperCase()); + return `${space}0x${BigInt(address.slice(separator)).toString(16)}`; + }); + +export const parseNativeAddress = (input: unknown): string => + nativeAddressInputSchema.parse(input); diff --git a/tests/boundary/mcp/functionComparisonReferenceFactsMcp.test.ts b/tests/boundary/mcp/functionComparisonReferenceFactsMcp.test.ts new file mode 100644 index 000000000..2759f2758 --- /dev/null +++ b/tests/boundary/mcp/functionComparisonReferenceFactsMcp.test.ts @@ -0,0 +1,511 @@ +import type { Client } from "@modelcontextprotocol/client"; +import { expect, it, onTestFinished } from "vitest"; + +import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { functionComparisonResultSchema } from "../../../src/domain/functionComparisonSchemas.js"; +import { functionDossierSchema } from "../../../src/domain/hopperValues.js"; +import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; +import { reconstructionVerificationResultSchema } from "../../../src/domain/reconstructionVerificationSchemas.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; + +const example = FUNCTION_COMPARISON_EXAMPLE.left; +const dossier = functionDossierSchema.parse(example.normalized_result); +type Reference = (typeof dossier.outgoing_references)[number]; +type Classification = "call" | "data" | "unknown"; + +const createHarness = async () => { + const harness = await createApplicationMcpHarness({ + session: createTestBinarySession(() => { + throw new Error("Function comparison must not launch a provider"); + }), + }); + onTestFinished(harness.close); + return harness; +}; + +const procedure = (address: string, name: string) => ({ + address, + name, + classification: null, + body: dossier.procedure.body, +}); + +const reference = ( + sourceAddress: string, + targetAddress: string, + classification: Classification = "call", + options: { + readonly sourceName?: string; + readonly targetName?: string | null; + readonly sourceEntryAddress?: string; + readonly targetEntryAddress?: string; + } = {}, +): Reference => ({ + source_address: sourceAddress, + target_address: targetAddress, + source_procedure: + options.sourceName === undefined + ? procedure(options.sourceEntryAddress ?? sourceAddress, "main") + : procedure( + options.sourceEntryAddress ?? sourceAddress, + options.sourceName, + ), + target_procedure: + options.targetName === null + ? null + : procedure( + options.targetEntryAddress ?? targetAddress, + options.targetName ?? "helper", + ), + kind: + classification === "unknown" + ? { available: false, reason: "Reference kind was not exposed." } + : { + available: true, + provenance: "controlled-reference-fact", + type: classification === "call" ? "UNCONDITIONAL_CALL" : "READ", + flow: classification === "call", + call: classification === "call", + jump: false, + data: classification === "data", + read: classification === "data", + write: false, + indirect: false, + computed: false, + conditional: false, + terminal: false, + primary: true, + operand_index: 0, + external: false, + }, +}); + +const evidence = ( + id: string, + digest: string, + outgoing: readonly Reference[] = [], + options: { + readonly incoming?: readonly Reference[]; + readonly procedureAddress?: string; + readonly procedureName?: string; + } = {}, +) => + createEvidence( + { + path: `/fixture/function-reference/${id}`, + sha256: digest.repeat(64), + format: "mach-o", + }, + example.provider, + { + operation: "analyze_function", + parameters: { ...example.parameters, observation: id }, + result: jsonValueSchema.parse({ + ...dossier, + procedure: { + ...dossier.procedure, + address: options.procedureAddress ?? dossier.procedure.address, + name: options.procedureName ?? dossier.procedure.name, + }, + incoming_references: [...(options.incoming ?? [])], + outgoing_references: [...outgoing], + }), + confidence: "derived", + authority: "shipped-artifact", + relationshipCoverage: { status: "complete" }, + }, + ); + +const compare = async ( + client: Client, + left: ReturnType, + right: ReturnType, +) => { + const response = await client.callTool({ + name: "compare_functions", + arguments: { left, right }, + }); + expect( + response.isError, + JSON.stringify({ + content: response.content, + structured: response.structuredContent, + }), + ).not.toBe(true); + const resultEvidence = parseEvidence(response.structuredContent); + expect(resultEvidence.operation).toBe("compare_functions"); + expect(resultEvidence.evidence_links).toEqual( + expect.arrayContaining([left.evidence_id, right.evidence_id]), + ); + return { + evidence: resultEvidence, + result: functionComparisonResultSchema.parse( + resultEvidence.normalized_result, + ), + }; +}; + +const expectReferences = ( + comparison: Awaited>, + status: "unchanged" | "changed" | "unknown", +) => + expect + .soft( + comparison.result.dimensions.find( + ({ dimension }) => dimension === "references", + ), + ) + .toMatchObject({ status }); + +const verifyReferences = async ( + client: Client, + comparisonEvidenceId: string, + status: "pass" | "fail" | "unknown", +) => { + const claimId = `references-${comparisonEvidenceId}`; + const response = await client.callTool({ + name: "verify_reconstruction", + arguments: { + specification: { + name: `Reference identity check ${comparisonEvidenceId}`, + claims: [ + { + kind: "structural-function", + claim_id: claimId, + title: "Reference facts agree", + comparison_evidence_id: comparisonEvidenceId, + dimension: "references", + }, + ], + }, + }, + }); + expect(response.isError, JSON.stringify(response.content)).not.toBe(true); + const verification = reconstructionVerificationResultSchema.parse( + parseEvidence(response.structuredContent).normalized_result, + ); + const observed = + status === "pass" ? "unchanged" : status === "fail" ? "changed" : "unknown"; + expect.soft(verification).toMatchObject({ + status, + summary: { + total: 1, + passed: status === "pass" ? 1 : 0, + failed: status === "fail" ? 1 : 0, + unknown: status === "unknown" ? 1 : 0, + }, + claims: { + items: [ + expect.objectContaining({ + claim_id: claimId, + status, + observed_status: observed, + }), + ], + }, + }); +}; + +it("preserves reference identity uncertainty and same-artifact coordinates", async () => { + const { client } = await createHarness(); + const namedLeft = evidence("named-left", "a", [ + reference("0x1000", "0x1010"), + ]); + const namedRight = evidence( + "named-right", + "b", + [reference("0x2000", "0x2020")], + { procedureAddress: "0x2000" }, + ); + expectReferences(await compare(client, namedLeft, namedRight), "unchanged"); + + const named = evidence("known-helper", "c", [reference("0x1000", "0x1010")]); + const unnamedElsewhere = evidence( + "unknown-helper", + "d", + [reference("0x2000", "0x2020", "call", { targetName: null })], + { procedureAddress: "0x2000" }, + ); + const namedToNull = await compare(client, named, unnamedElsewhere); + expectReferences(namedToNull, "unknown"); + await verifyReferences(client, namedToNull.evidence.evidence_id, "unknown"); + const nullToNamed = await compare(client, unnamedElsewhere, named); + expectReferences(nullToNamed, "unknown"); + + const knownSameArtifact = evidence("same-named", "e", [ + reference("0x1000", "0x1010"), + ]); + expectReferences( + await compare( + client, + knownSameArtifact, + evidence("same-unnamed", "e", [ + reference("0x1000", "0x1010", "call", { targetName: null }), + ]), + ), + "unchanged", + ); + expectReferences( + await compare( + client, + knownSameArtifact, + evidence("same-different-raw", "e", [ + reference("0x1000", "0x1020", "call", { targetName: null }), + ]), + ), + "changed", + ); + + const reversed = await compare( + client, + namedLeft, + evidence("reverse", "f", [], { + procedureAddress: "0x2000", + incoming: [ + reference("0x2020", "0x2000", "call", { + sourceName: "helper", + targetName: "main", + }), + ], + }), + ); + expectReferences(reversed, "changed"); + expectReferences(await compare(client, namedRight, namedLeft), "unchanged"); +}); + +it("keeps generated and ambiguous procedure names from proving cross-artifact identity", async () => { + const { client } = await createHarness(); + for (const [index, name] of ["sub_1010", "fcn.1010", "FUN_1010"].entries()) { + const left = evidence(`auto-left-${index}`, "1", [ + reference("0x1000", "0x1010", "call", { targetName: name }), + ]); + const right = evidence( + `auto-right-${index}`, + "2", + [reference("0x2000", "0x2020", "call", { targetName: name })], + { procedureAddress: "0x2000" }, + ); + expectReferences(await compare(client, left, right), "unknown"); + const otherName = ["fcn.2020", "FUN_2020", "sub_2020"][index] ?? "sub_2020"; + expectReferences( + await compare( + client, + left, + evidence( + `auto-different-${index}`, + "3", + [reference("0x2000", "0x2020", "call", { targetName: otherName })], + { procedureAddress: "0x2000" }, + ), + ), + "unknown", + ); + } + + const sameCoordinates = await compare( + client, + evidence("auto-coordinate-left", "4", [ + reference("0x1000", "0x1010", "call", { targetName: "sub_1010" }), + ]), + evidence("auto-coordinate-right", "4", [ + reference("0x1000", "0x1010", "call", { targetName: "FUN_1010" }), + ]), + ); + expectReferences(sameCoordinates, "unchanged"); + + const overloads = await compare( + client, + evidence("duplicate-helper-left", "5", [ + reference("0x1000", "0x1010"), + reference("0x1000", "0x1020"), + ]), + evidence( + "duplicate-helper-right", + "6", + [reference("0x2000", "0x2010"), reference("0x2000", "0x2020")], + { procedureAddress: "0x2000" }, + ), + ); + expectReferences(overloads, "unknown"); + + const generatedFunctions = await compare( + client, + evidence("generated-function-left", "7", [], { + procedureName: "sub_1000", + }), + evidence("generated-function-right", "8", [], { + procedureAddress: "0x2000", + procedureName: "FUN_2000", + }), + ); + expect.soft(generatedFunctions.result.function_match).toMatchObject({ + status: "ambiguous", + }); + expect.soft(generatedFunctions.result.status).toBe("unknown"); +}); + +it("normalizes numeric coordinates but preserves address spaces and source sites", async () => { + const { client } = await createHarness(); + const at = (id: string, address: string) => + evidence(id, "9", [ + { + ...reference("0x1000", "0x1010", "call", { targetName: null }), + target_address: address, + }, + ]); + const equivalentCoordinates = [ + ["0x1010", "0x00001010"], + ["0x1010", "0X001010"], + ["0x10a", "0x10A"], + ] as const; + for (const [index, [left, right]] of equivalentCoordinates.entries()) + expectReferences( + await compare( + client, + at(`coordinate-left-${index}`, left), + at(`coordinate-right-${index}`, right), + ), + "unchanged", + ); + + expectReferences( + await compare( + client, + at("space-left", "left:0x1010"), + at("space-right", "right:0x1010"), + ), + "changed", + ); + + expectReferences( + await compare( + client, + at("escaped-space-left", "seg%2f:0x0010"), + at("escaped-space-right", "seg%2F:0x10"), + ), + "unchanged", + ); + expectReferences( + await compare( + client, + at("space-case-left", "RAM:0x10"), + at("space-case-right", "ram:0x10"), + ), + "changed", + ); + + const unsupported = ( + id: string, + digest: string, + address: string, + classification: Classification = "call", + ) => + evidence(id, digest, [ + reference("0x1000", address, classification, { targetName: null }), + ]); + expectReferences( + await compare( + client, + unsupported("unsupported-same-left", "a", "bad coordinate"), + unsupported("unsupported-same-right", "a", "bad coordinate"), + ), + "unknown", + ); + expectReferences( + await compare( + client, + unsupported("unsupported-different-left", "b", "bad coordinate"), + unsupported("unsupported-different-right", "c", "other bad coordinate"), + ), + "unknown", + ); + expectReferences( + await compare( + client, + unsupported("unsupported-call", "d", "bad coordinate", "call"), + unsupported("unsupported-data", "e", "other bad coordinate", "data"), + ), + "changed", + ); + + const atSite = (id: string, site: string) => + evidence(id, "a", [ + reference(site, "0x1010", "call", { sourceEntryAddress: "0x1000" }), + ]); + expectReferences( + await compare( + client, + atSite("site-left", "0x1004"), + atSite("site-right", "0x1008"), + ), + "changed", + ); +}); + +it("keeps known reference changes decisive beside unrelated unknown kinds", async () => { + const { client } = await createHarness(); + const left = evidence("mixed-left", "b", [ + reference("0x1000", "0x1010", "call"), + reference("0x1000", "0x1020", "unknown", { targetName: "other" }), + ]); + const right = evidence( + "mixed-right", + "c", + [ + reference("0x2000", "0x2020", "unknown", { targetName: "other" }), + reference("0x2000", "0x2010", "data"), + ], + { procedureAddress: "0x2000" }, + ); + const compared = await compare(client, left, right); + expectReferences(compared, "changed"); + await verifyReferences(client, compared.evidence.evidence_id, "fail"); +}); + +it("keeps ambiguous duplicate edge kinds unknown independent of input order", async () => { + const { client } = await createHarness(); + const left = [ + reference("0x1000", "0x1010", "call"), + reference("0x1000", "0x1010", "unknown"), + ]; + const right = [ + reference("0x2000", "0x2010", "unknown"), + reference("0x2000", "0x2010", "data"), + ]; + const first = await compare( + client, + evidence("ambiguous-left", "d", left), + evidence("ambiguous-right", "e", right, { procedureAddress: "0x2000" }), + ); + const reversed = await compare( + client, + evidence("ambiguous-left-reversed", "f", [...left].reverse()), + evidence("ambiguous-right-reversed", "1", [...right].reverse(), { + procedureAddress: "0x2000", + }), + ); + expectReferences(first, "unknown"); + expectReferences(reversed, "unknown"); + expect( + reversed.result.dimensions.find( + ({ dimension }) => dimension === "references", + )?.limitations, + ).toEqual( + first.result.dimensions.find(({ dimension }) => dimension === "references") + ?.limitations, + ); + + const changed = await compare( + client, + evidence("changed-target-left", "2", [reference("0x1000", "0x1010")]), + evidence( + "changed-target-right", + "3", + [reference("0x2000", "0x2020", "call", { targetName: "replacement" })], + { procedureAddress: "0x2000" }, + ), + ); + expectReferences(changed, "changed"); +});