mirror of
https://github.com/morluto/rea.git
synced 2026-10-23 14:22:30 -05:00
fix(javascript): isolate source analysis and retain bounded partial evidence (#1651)
* refactor(javascript): validate transferable file projections * test(javascript): consume decoded module and static facts * fix(javascript): isolate source analysis and retain bounded partial evidence * test(javascript): verify cleanup retention and repeated source paths
This commit is contained in:
1 parent
92cc0edd8e
commit
4999e4be2c
41 files changed
+3748
-176
No files matched your search
@@ -0,0 +1,414 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { createReadStream } from "node:fs";
|
||||
import { open, stat } from "node:fs/promises";
|
||||
import { z } from "zod";
|
||||
import { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js";
|
||||
|
||||
import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js";
|
||||
import type { JavaScriptModuleSemanticIr } from "../../domain/javascript/javascriptModuleSemanticIr.js";
|
||||
import type { JavaScriptSemanticFileProjection } from "../../domain/javascript/javascriptSemanticFileProjection.js";
|
||||
import { digestSchema } from "../../domain/digests.js";
|
||||
import {
|
||||
javaScriptWorkerGraphRecordSchema,
|
||||
javaScriptWorkerModuleHeaderSchema,
|
||||
javaScriptWorkerModuleRecordSchema,
|
||||
javaScriptWorkerProjectionHeaderSchema,
|
||||
} from "./JavaScriptAnalysisWorkerSchemas.js";
|
||||
|
||||
// Budgets bound transfer representation, independently of source length.
|
||||
/** Record bound whose transient decoding expansion is reserved by the consumer. */
|
||||
export const JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES = 8 * 1024 * 1024;
|
||||
const FRAME_BYTES = JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES;
|
||||
const TRANSFER_BYTES = 512 * 1024 * 1024;
|
||||
const BATCH_BYTES = 1024 * 1024;
|
||||
const recordSchema = z.discriminatedUnion("kind", [
|
||||
z.strictObject({
|
||||
kind: z.literal("projection-header"),
|
||||
value: javaScriptWorkerProjectionHeaderSchema,
|
||||
}),
|
||||
z.strictObject({
|
||||
kind: z.literal("module-header"),
|
||||
value: javaScriptWorkerModuleHeaderSchema,
|
||||
}),
|
||||
...javaScriptWorkerGraphRecordSchema.options,
|
||||
...javaScriptWorkerModuleRecordSchema.options,
|
||||
]);
|
||||
|
||||
/** Complete-file coordinates authenticated before adopting any worker facts. */
|
||||
export const javaScriptAnalysisTransferDescriptorSchema = z.strictObject({
|
||||
bytes: z.number().int().nonnegative().max(TRANSFER_BYTES),
|
||||
sha256: digestSchema,
|
||||
records: z.number().int().nonnegative(),
|
||||
module_bytes: z.number().int().nonnegative(),
|
||||
module_records: z.number().int().nonnegative(),
|
||||
graph_bytes: z.number().int().nonnegative(),
|
||||
graph_records: z.number().int().nonnegative(),
|
||||
});
|
||||
type Descriptor = z.output<typeof javaScriptAnalysisTransferDescriptorSchema>;
|
||||
|
||||
/** Conservative retained-object and sealing expansion for validated transfer rows. */
|
||||
export const javaScriptTransferRetentionBytes = (
|
||||
bytes: number,
|
||||
records: number,
|
||||
): number => bytes * 8 + records * 64;
|
||||
|
||||
const encodeTransferRecord = (input: unknown, previousBytes: number) => {
|
||||
const record = recordSchema.parse(input);
|
||||
const line = `${JSON.stringify(record)}\n`;
|
||||
const size = Buffer.byteLength(line);
|
||||
if (size > FRAME_BYTES || previousBytes + size > TRANSFER_BYTES)
|
||||
throw new AnalysisResourceConstraintError(
|
||||
"analyze_javascript_application",
|
||||
"transport",
|
||||
size > FRAME_BYTES
|
||||
? "A complete JavaScript fact exceeds the transfer record byte budget"
|
||||
: "Complete JavaScript file facts exceed the transfer byte budget",
|
||||
{
|
||||
transfer_record_bytes: FRAME_BYTES,
|
||||
transfer_file_bytes: TRANSFER_BYTES,
|
||||
record_bytes: size,
|
||||
transfer_bytes_at_least: previousBytes + size,
|
||||
},
|
||||
);
|
||||
return { line, size, kind: record.kind };
|
||||
};
|
||||
|
||||
function* moduleTransferRecords(
|
||||
module: JavaScriptModuleSemanticIr,
|
||||
): Generator<unknown> {
|
||||
yield {
|
||||
kind: "module-header",
|
||||
value: { coverage: module.coverage, limitations: module.limitations },
|
||||
};
|
||||
for (const value of module.scopes) yield { kind: "scope", value };
|
||||
for (const value of module.bindings) yield { kind: "binding", value };
|
||||
for (const value of module.callables) yield { kind: "callable", value };
|
||||
for (const value of module.moduleLinks) yield { kind: "module-link", value };
|
||||
}
|
||||
|
||||
/** Measure actual validated module rows before allocating an unadmittable graph. */
|
||||
export const measureJavaScriptModuleTransfer = (
|
||||
module: JavaScriptModuleSemanticIr,
|
||||
) => {
|
||||
let bytes = 0;
|
||||
let records = 0;
|
||||
for (const record of moduleTransferRecords(module)) {
|
||||
bytes += encodeTransferRecord(record, bytes).size;
|
||||
records += 1;
|
||||
}
|
||||
return {
|
||||
bytes,
|
||||
records,
|
||||
retentionBytes: javaScriptTransferRetentionBytes(bytes, records),
|
||||
};
|
||||
};
|
||||
|
||||
/** Encode validated facts in bounded records, without serializing a whole graph or IR. */
|
||||
export const writeJavaScriptAnalysisTransfer = async (
|
||||
path: string,
|
||||
module: JavaScriptModuleSemanticIr,
|
||||
projection: JavaScriptSemanticFileProjection,
|
||||
): Promise<Descriptor> => {
|
||||
const output = await open(path, "wx", 0o600);
|
||||
const digest = createHash("sha256");
|
||||
let bytes = 0;
|
||||
let records = 0;
|
||||
let moduleBytes = 0;
|
||||
let moduleRecords = 0;
|
||||
let graphBytes = 0;
|
||||
let graphRecords = 0;
|
||||
let batch: string[] = [];
|
||||
let batchBytes = 0;
|
||||
const flush = async (): Promise<void> => {
|
||||
if (batch.length === 0) return;
|
||||
const buffer = Buffer.from(batch.join(""));
|
||||
// FileHandle.write can be short; writeFile completes the buffer.
|
||||
await output.writeFile(buffer);
|
||||
digest.update(buffer);
|
||||
batch = [];
|
||||
batchBytes = 0;
|
||||
};
|
||||
const append = async (input: unknown): Promise<void> => {
|
||||
const { line, size, kind } = encodeTransferRecord(input, bytes);
|
||||
bytes += size;
|
||||
records += 1;
|
||||
if (
|
||||
["module-header", "scope", "binding", "callable", "module-link"].includes(
|
||||
kind,
|
||||
)
|
||||
) {
|
||||
moduleBytes += size;
|
||||
moduleRecords += 1;
|
||||
} else {
|
||||
graphBytes += size;
|
||||
graphRecords += 1;
|
||||
}
|
||||
batch.push(line);
|
||||
batchBytes += size;
|
||||
if (batchBytes >= BATCH_BYTES) await flush();
|
||||
};
|
||||
try {
|
||||
await append({
|
||||
kind: "projection-header",
|
||||
value: { roots: projection.roots, truncated: projection.truncated },
|
||||
});
|
||||
for (const record of moduleTransferRecords(module)) await append(record);
|
||||
for (const value of projection.evidenceContexts)
|
||||
await append({ kind: "evidence-context", value });
|
||||
for (const value of projection.nodes) await append({ kind: "node", value });
|
||||
for (const value of projection.relations)
|
||||
await append({ kind: "relation", value });
|
||||
for (const value of projection.unknowns)
|
||||
await append({ kind: "unknown", value });
|
||||
for (const value of projection.fingerprints)
|
||||
await append({ kind: "fingerprint", value });
|
||||
await flush();
|
||||
return {
|
||||
bytes,
|
||||
sha256: digest.digest("hex"),
|
||||
records,
|
||||
module_bytes: moduleBytes,
|
||||
module_records: moduleRecords,
|
||||
graph_bytes: graphBytes,
|
||||
graph_records: graphRecords,
|
||||
};
|
||||
} finally {
|
||||
await output.close();
|
||||
}
|
||||
};
|
||||
|
||||
/** Decode producer records into realm-local, source-bound facts; sealing follows in the consumer. */
|
||||
export function readJavaScriptAnalysisTransfer(
|
||||
path: string,
|
||||
descriptor: Descriptor,
|
||||
source: Pick<JavaScriptArtifactFile, "path" | "sha256">,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{
|
||||
readonly module: JavaScriptModuleSemanticIr;
|
||||
readonly projection: JavaScriptSemanticFileProjection;
|
||||
}>;
|
||||
/** Validate the transfer while retaining only module facts when graph admission fails. */
|
||||
export function readJavaScriptAnalysisTransfer(
|
||||
path: string,
|
||||
descriptor: Descriptor,
|
||||
source: Pick<JavaScriptArtifactFile, "path" | "sha256">,
|
||||
signal: AbortSignal | undefined,
|
||||
retainGraph: boolean,
|
||||
): Promise<{
|
||||
readonly module: JavaScriptModuleSemanticIr;
|
||||
readonly projection: JavaScriptSemanticFileProjection | null;
|
||||
}>;
|
||||
export async function readJavaScriptAnalysisTransfer(
|
||||
path: string,
|
||||
descriptor: Descriptor,
|
||||
source: Pick<JavaScriptArtifactFile, "path" | "sha256">,
|
||||
signal?: AbortSignal,
|
||||
retainGraph = true,
|
||||
): Promise<{
|
||||
readonly module: JavaScriptModuleSemanticIr;
|
||||
readonly projection: JavaScriptSemanticFileProjection | null;
|
||||
}> {
|
||||
const expected = javaScriptAnalysisTransferDescriptorSchema.parse(descriptor);
|
||||
if ((await stat(path)).size !== expected.bytes)
|
||||
throw new TypeError(
|
||||
"JavaScript analysis transfer byte count does not match",
|
||||
);
|
||||
const scopes: JavaScriptModuleSemanticIr["scopes"][number][] = [];
|
||||
const bindings: JavaScriptModuleSemanticIr["bindings"][number][] = [];
|
||||
const callables: JavaScriptModuleSemanticIr["callables"][number][] = [];
|
||||
const moduleLinks: JavaScriptModuleSemanticIr["moduleLinks"][number][] = [];
|
||||
const evidenceContexts: JavaScriptSemanticFileProjection["evidenceContexts"][number][] =
|
||||
[];
|
||||
const nodes: JavaScriptSemanticFileProjection["nodes"][number][] = [];
|
||||
const relations: JavaScriptSemanticFileProjection["relations"][number][] = [];
|
||||
const unknowns: JavaScriptSemanticFileProjection["unknowns"][number][] = [];
|
||||
const fingerprints: JavaScriptSemanticFileProjection["fingerprints"][number][] =
|
||||
[];
|
||||
let projectionHeader:
|
||||
| z.output<typeof javaScriptWorkerProjectionHeaderSchema>
|
||||
| undefined;
|
||||
let moduleHeader:
|
||||
| z.output<typeof javaScriptWorkerModuleHeaderSchema>
|
||||
| undefined;
|
||||
const identities = new Map<string, Set<string>>();
|
||||
const retainIdentity = (kind: string, identifier: string): void => {
|
||||
let seen = identities.get(kind);
|
||||
if (seen === undefined) {
|
||||
seen = new Set();
|
||||
identities.set(kind, seen);
|
||||
}
|
||||
if (seen.has(identifier))
|
||||
throw new TypeError(
|
||||
`Duplicate JavaScript transfer ${kind} identity: ${identifier}`,
|
||||
);
|
||||
seen.add(identifier);
|
||||
};
|
||||
let records = 0;
|
||||
let moduleBytes = 0;
|
||||
let moduleRecords = 0;
|
||||
let graphBytes = 0;
|
||||
let graphRecords = 0;
|
||||
const digest = createHash("sha256");
|
||||
const stream = createReadStream(path, {
|
||||
highWaterMark: 64 * 1024,
|
||||
...(signal === undefined ? {} : { signal }),
|
||||
});
|
||||
let bytes = 0;
|
||||
let pending = "";
|
||||
stream.setEncoding("utf8");
|
||||
try {
|
||||
for await (const chunk of stream) {
|
||||
signal?.throwIfAborted();
|
||||
if (typeof chunk !== "string")
|
||||
throw new TypeError("JavaScript transfer stream is not UTF-8 text");
|
||||
bytes += Buffer.byteLength(chunk);
|
||||
if (bytes > expected.bytes)
|
||||
throw new RangeError("JavaScript transfer grew while being read");
|
||||
digest.update(chunk);
|
||||
pending += chunk;
|
||||
let newline: number;
|
||||
while ((newline = pending.indexOf("\n")) >= 0) {
|
||||
const line = pending.slice(0, newline);
|
||||
pending = pending.slice(newline + 1);
|
||||
if (Buffer.byteLength(line) + 1 > FRAME_BYTES)
|
||||
throw new RangeError(
|
||||
"JavaScript transfer record exceeds its byte budget",
|
||||
);
|
||||
const input: unknown = JSON.parse(line);
|
||||
const record = recordSchema.parse(input);
|
||||
if (
|
||||
[
|
||||
"module-header",
|
||||
"scope",
|
||||
"binding",
|
||||
"callable",
|
||||
"module-link",
|
||||
].includes(record.kind)
|
||||
) {
|
||||
moduleBytes += Buffer.byteLength(line) + 1;
|
||||
moduleRecords += 1;
|
||||
} else {
|
||||
graphBytes += Buffer.byteLength(line) + 1;
|
||||
graphRecords += 1;
|
||||
}
|
||||
if (record.kind === "projection-header") {
|
||||
if (records !== 0 || projectionHeader !== undefined)
|
||||
throw new TypeError(
|
||||
"Duplicate or late JavaScript projection header",
|
||||
);
|
||||
projectionHeader = record.value;
|
||||
} else if (record.kind === "module-header") {
|
||||
if (records !== 1 || moduleHeader !== undefined)
|
||||
throw new TypeError("Duplicate or late JavaScript module header");
|
||||
moduleHeader = record.value;
|
||||
} else {
|
||||
if (projectionHeader === undefined || moduleHeader === undefined)
|
||||
throw new TypeError(
|
||||
"JavaScript transfer facts precede their headers",
|
||||
);
|
||||
if (
|
||||
!retainGraph &&
|
||||
[
|
||||
"evidence-context",
|
||||
"node",
|
||||
"relation",
|
||||
"unknown",
|
||||
"fingerprint",
|
||||
].includes(record.kind)
|
||||
) {
|
||||
if (
|
||||
record.kind === "node" &&
|
||||
(record.value.identity.artifact_sha256 !== source.sha256 ||
|
||||
record.value.identity.module_path !== source.path)
|
||||
)
|
||||
throw new TypeError(
|
||||
"JavaScript transfer node belongs to a different artifact source",
|
||||
);
|
||||
records += 1;
|
||||
continue;
|
||||
}
|
||||
switch (record.kind) {
|
||||
case "scope":
|
||||
retainIdentity(record.kind, record.value.scopeId);
|
||||
scopes.push(record.value);
|
||||
break;
|
||||
case "binding":
|
||||
retainIdentity(record.kind, record.value.bindingId);
|
||||
bindings.push(record.value);
|
||||
break;
|
||||
case "callable":
|
||||
retainIdentity(record.kind, record.value.callableId);
|
||||
callables.push(record.value);
|
||||
break;
|
||||
case "module-link":
|
||||
moduleLinks.push(record.value);
|
||||
break;
|
||||
case "evidence-context":
|
||||
retainIdentity(record.kind, record.value.context_id);
|
||||
evidenceContexts.push(record.value);
|
||||
break;
|
||||
case "node": {
|
||||
retainIdentity(record.kind, record.value.node_id);
|
||||
if (
|
||||
record.value.identity.artifact_sha256 !== source.sha256 ||
|
||||
record.value.identity.module_path !== source.path
|
||||
)
|
||||
throw new TypeError(
|
||||
"JavaScript transfer node belongs to a different artifact source",
|
||||
);
|
||||
nodes.push(record.value);
|
||||
break;
|
||||
}
|
||||
case "relation":
|
||||
retainIdentity(record.kind, record.value.relation_id);
|
||||
relations.push(record.value);
|
||||
break;
|
||||
case "unknown":
|
||||
retainIdentity(record.kind, record.value.unknown_id);
|
||||
unknowns.push(record.value);
|
||||
break;
|
||||
case "fingerprint":
|
||||
retainIdentity(record.kind, record.value.fingerprint_id);
|
||||
fingerprints.push(record.value);
|
||||
break;
|
||||
}
|
||||
}
|
||||
records += 1;
|
||||
}
|
||||
if (Buffer.byteLength(pending) + 1 > FRAME_BYTES)
|
||||
throw new RangeError(
|
||||
"Unterminated JavaScript transfer record exceeds its byte budget",
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
stream.destroy();
|
||||
}
|
||||
if (
|
||||
pending !== "" ||
|
||||
records !== expected.records ||
|
||||
bytes !== expected.bytes ||
|
||||
moduleBytes !== expected.module_bytes ||
|
||||
moduleRecords !== expected.module_records ||
|
||||
graphBytes !== expected.graph_bytes ||
|
||||
graphRecords !== expected.graph_records ||
|
||||
digest.digest("hex") !== expected.sha256
|
||||
)
|
||||
throw new TypeError(
|
||||
"JavaScript analysis transfer is incomplete or its digest does not match",
|
||||
);
|
||||
if (projectionHeader === undefined || moduleHeader === undefined)
|
||||
throw new TypeError("JavaScript analysis transfer is missing its headers");
|
||||
return {
|
||||
module: { ...moduleHeader, scopes, bindings, callables, moduleLinks },
|
||||
projection: retainGraph
|
||||
? {
|
||||
...projectionHeader,
|
||||
evidenceContexts,
|
||||
nodes,
|
||||
relations,
|
||||
unknowns,
|
||||
fingerprints,
|
||||
}
|
||||
: null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,726 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { StringDecoder } from "node:string_decoder";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { getHeapStatistics } from "node:v8";
|
||||
import { z } from "zod";
|
||||
import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js";
|
||||
import type { JavaScriptStaticAnalysis } from "../../domain/javascript/javascriptStaticAnalysisTypes.js";
|
||||
import { estimateJavaScriptStaticApplicationProjection } from "../../domain/javascript/javascriptApplicationProjectionResources.js";
|
||||
import type { JavaScriptModuleSemanticIr } from "../../domain/javascript/javascriptModuleSemanticIr.js";
|
||||
import type { JavaScriptSemanticFileProjection } from "../../domain/javascript/javascriptSemanticFileProjection.js";
|
||||
import type {
|
||||
JavaScriptAnalyzedSource,
|
||||
JavaScriptSourceAnalysisFailure,
|
||||
JavaScriptSourceAnalysisLimits,
|
||||
} from "../../domain/javascript/javascriptSourceAnalysis.js";
|
||||
import { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import {
|
||||
AnalysisCancelledError,
|
||||
AnalysisTimeoutError,
|
||||
AnalysisOutputError,
|
||||
AnalysisResourceConstraintError,
|
||||
AnalysisCapabilityUnavailableError,
|
||||
} from "../../domain/analysisErrorCore.js";
|
||||
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
|
||||
import { ProviderCleanupError } from "../../domain/providerCleanupError.js";
|
||||
import { ok, err, type Result } from "../../domain/result.js";
|
||||
import {
|
||||
PrivateRuntimeRoot,
|
||||
PrivateRuntimeRootUnavailableError,
|
||||
} from "../../process/PrivateRuntimeRoot.js";
|
||||
import {
|
||||
ProviderProcessSupervisor,
|
||||
spawnOwnedProviderProcess,
|
||||
} from "../../process/ProviderProcess.js";
|
||||
import { cleanupOwnedProcessGroup } from "../../process/ProcessOwnership.js";
|
||||
import { prepareProcessOwnershipInspection } from "../../process/ProcessOwnershipObservation.js";
|
||||
import {
|
||||
ProviderStartupDeadline,
|
||||
waitForAbortableDelay,
|
||||
} from "../../process/ProviderDeadline.js";
|
||||
import {
|
||||
readJavaScriptAnalysisTransfer,
|
||||
JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES,
|
||||
javaScriptTransferRetentionBytes,
|
||||
} from "./JavaScriptAnalysisTransfer.js";
|
||||
import { javaScriptWorkerStaticAnalysisSchema } from "./JavaScriptAnalysisWorkerSchemas.js";
|
||||
import {
|
||||
JAVASCRIPT_ANALYSIS_WORKER_DEFAULTS as DEFAULTS,
|
||||
javaScriptWorkerMessageSchema,
|
||||
javaScriptWorkerRequestSchema,
|
||||
javaScriptWorkerAdmissionSchema,
|
||||
type JavaScriptWorkerMessage,
|
||||
} from "./JavaScriptAnalysisWorkerProtocol.js";
|
||||
|
||||
const OPERATION = "analyze_javascript_application";
|
||||
const PROVIDER = "javascript-static-analysis";
|
||||
const MIB = 1024 * 1024;
|
||||
const pendingCleanup = new Set<JavaScriptAnalysisWorker>();
|
||||
|
||||
/** One sequential owned worker, retained between source files and closed after analysis. */
|
||||
export class JavaScriptAnalysisWorker {
|
||||
readonly #runId = `rea-javascript-analysis-${randomUUID()}`;
|
||||
#root: PrivateRuntimeRoot | undefined;
|
||||
#supervisor: ProviderProcessSupervisor | undefined;
|
||||
#ready = false;
|
||||
#heapLimitBytes = 0;
|
||||
#frames: JavaScriptWorkerMessage[] = [];
|
||||
#pending = "";
|
||||
#decoder = new StringDecoder("utf8");
|
||||
#protocolFailure: string | undefined;
|
||||
#phase = "startup";
|
||||
#lastFailure: AnalysisError | undefined;
|
||||
#estimatedRetainedBytes = 0;
|
||||
#applicationProjectionFailure: AnalysisResourceConstraintError | undefined;
|
||||
#parentCapacityExhausted = false;
|
||||
#retainedStaticBytes = 0;
|
||||
#sourcesRemaining: number | undefined;
|
||||
readonly #resultBudgetBytes: number;
|
||||
readonly #onStdout = (chunk: Buffer | string): void => {
|
||||
try {
|
||||
this.#pending +=
|
||||
typeof chunk === "string" ? chunk : this.#decoder.write(chunk);
|
||||
let newline: number;
|
||||
while ((newline = this.#pending.indexOf("\n")) >= 0) {
|
||||
const line = this.#pending.slice(0, newline);
|
||||
this.#pending = this.#pending.slice(newline + 1);
|
||||
if (
|
||||
Buffer.byteLength(line) > DEFAULTS.protocolBytes ||
|
||||
this.#frames.length >= 16
|
||||
)
|
||||
throw new RangeError(
|
||||
"JavaScript worker exceeded its control-frame budget",
|
||||
);
|
||||
const raw: unknown = JSON.parse(line);
|
||||
this.#frames.push(javaScriptWorkerMessageSchema.parse(raw));
|
||||
}
|
||||
if (Buffer.byteLength(this.#pending) > DEFAULTS.protocolBytes)
|
||||
throw new RangeError(
|
||||
"JavaScript worker control frame is unterminated or oversized",
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
this.#protocolFailure = message(cause);
|
||||
}
|
||||
};
|
||||
|
||||
constructor(
|
||||
readonly limits: JavaScriptSourceAnalysisLimits = DEFAULTS,
|
||||
readonly launcher: typeof spawnOwnedProviderProcess = spawnOwnedProviderProcess,
|
||||
sourceCount?: number,
|
||||
) {
|
||||
if (
|
||||
!Number.isSafeInteger(limits.heapMb) ||
|
||||
limits.heapMb < 128 ||
|
||||
limits.heapMb > 16_384 ||
|
||||
!Number.isSafeInteger(limits.timeoutMs) ||
|
||||
limits.timeoutMs <= 0 ||
|
||||
limits.timeoutMs > 2_147_483_647
|
||||
)
|
||||
throw new RangeError(
|
||||
"JavaScript worker requires valid heap and execution limits",
|
||||
);
|
||||
if (
|
||||
sourceCount !== undefined &&
|
||||
(!Number.isSafeInteger(sourceCount) || sourceCount < 0)
|
||||
)
|
||||
throw new RangeError(
|
||||
"JavaScript worker source count must be a nonnegative safe integer",
|
||||
);
|
||||
this.#sourcesRemaining = sourceCount;
|
||||
// Reserve host-side control, schema/Evidence work and a maximum record's
|
||||
// transient copies. Admission also accounts for retained earlier files.
|
||||
this.#resultBudgetBytes = Math.max(
|
||||
0,
|
||||
Math.floor(
|
||||
(getHeapStatistics().heap_size_limit -
|
||||
process.memoryUsage().heapUsed -
|
||||
24 * JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES) /
|
||||
2,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/** Analyze inert source; hard worker failure does not terminate this process. */
|
||||
async analyze(
|
||||
file: JavaScriptArtifactFile,
|
||||
nodeBudget: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<
|
||||
Result<JavaScriptAnalyzedSource, JavaScriptSourceAnalysisFailure>
|
||||
> {
|
||||
let javascript: JavaScriptStaticAnalysis | null = null;
|
||||
let module: JavaScriptModuleSemanticIr | null = null;
|
||||
let projection: JavaScriptSemanticFileProjection | null = null;
|
||||
const id = randomUUID();
|
||||
this.#applicationProjectionFailure = undefined;
|
||||
this.#parentCapacityExhausted = false;
|
||||
if (this.#sourcesRemaining !== undefined)
|
||||
this.#sourcesRemaining = Math.max(0, this.#sourcesRemaining - 1);
|
||||
let deadline: ProviderStartupDeadline | undefined;
|
||||
let result: Result<
|
||||
JavaScriptAnalyzedSource,
|
||||
JavaScriptSourceAnalysisFailure
|
||||
>;
|
||||
try {
|
||||
if (signal?.aborted === true) throw new AnalysisCancelledError(OPERATION);
|
||||
if (!file.text.included || file.kind !== "javascript")
|
||||
throw new TypeError(
|
||||
"JavaScript analysis worker requires included JavaScript text",
|
||||
);
|
||||
// Host ownership preparation is outside the parser execution deadline.
|
||||
await prepareProcessOwnershipInspection(signal);
|
||||
for (const owner of pendingCleanup) await owner.close();
|
||||
deadline = new ProviderStartupDeadline(this.limits.timeoutMs, signal);
|
||||
await this.#start(deadline);
|
||||
const root = this.#root;
|
||||
const supervisor = this.#supervisor;
|
||||
if (root === undefined || supervisor === undefined)
|
||||
throw new TypeError("JavaScript worker resources were not acquired");
|
||||
if (this.#frames.length !== 0 || this.#pending !== "")
|
||||
throw new AnalysisOutputError(
|
||||
OPERATION,
|
||||
"JavaScript worker has unsolicited or incomplete control frames",
|
||||
);
|
||||
supervisor.resetOutput();
|
||||
this.#phase = "source-transfer";
|
||||
await writeFile(join(root.path, `${id}.source`), file.text.value, {
|
||||
flag: "wx",
|
||||
mode: 0o600,
|
||||
signal: deadline.signal,
|
||||
});
|
||||
const { text: _text, ...metadata } = file;
|
||||
let offeredResultBudget = 0;
|
||||
const request = javaScriptWorkerRequestSchema.parse({
|
||||
id,
|
||||
node_budget: nodeBudget,
|
||||
text_sha256: createHash("sha256").update(file.text.value).digest("hex"),
|
||||
file: metadata,
|
||||
});
|
||||
const frame = `${JSON.stringify(request)}\n`;
|
||||
if (Buffer.byteLength(frame) > DEFAULTS.protocolBytes)
|
||||
throw new AnalysisResourceConstraintError(
|
||||
OPERATION,
|
||||
"transport",
|
||||
`Source identity for ${file.path} exceeds the worker request byte budget`,
|
||||
{ worker_request_bytes: DEFAULTS.protocolBytes },
|
||||
);
|
||||
const stdin = supervisor.launch.process.stdin;
|
||||
if (stdin === undefined || stdin === null)
|
||||
throw new TypeError("JavaScript worker has no writable request stream");
|
||||
await new Promise<void>((resolve, reject) =>
|
||||
stdin.write(frame, (error) =>
|
||||
error === null || error === undefined ? resolve() : reject(error),
|
||||
),
|
||||
);
|
||||
for (;;) {
|
||||
const response = await this.#next(deadline, file.path);
|
||||
if (response.kind === "ready" || response.id !== id)
|
||||
throw new AnalysisOutputError(
|
||||
OPERATION,
|
||||
"JavaScript worker response names a different source job",
|
||||
);
|
||||
if (response.kind === "phase") {
|
||||
this.#phase = response.phase;
|
||||
continue;
|
||||
}
|
||||
if (response.kind === "static") {
|
||||
if (javascript !== null)
|
||||
throw new AnalysisOutputError(
|
||||
OPERATION,
|
||||
"JavaScript worker emitted a duplicate static checkpoint",
|
||||
);
|
||||
javascript = await this.#readCheckpoint(id, file.path, response);
|
||||
if (this.#parentCapacityExhausted)
|
||||
throw new AnalysisResourceConstraintError(
|
||||
OPERATION,
|
||||
"memory",
|
||||
`The parent's result capacity was exhausted after retaining static facts for ${file.path}; remaining sources were not analyzed`,
|
||||
{
|
||||
...this.#limits(),
|
||||
parent_result_budget_bytes: this.#resultBudgetBytes,
|
||||
parent_result_budget_exhausted: true,
|
||||
},
|
||||
);
|
||||
offeredResultBudget =
|
||||
this.#applicationProjectionFailure !== undefined &&
|
||||
this.#sourcesRemaining !== undefined &&
|
||||
this.#sourcesRemaining > 0
|
||||
? 0
|
||||
: this.#remainingResultBudget();
|
||||
const admission = javaScriptWorkerAdmissionSchema.parse({
|
||||
id,
|
||||
semantic_result_budget_bytes: offeredResultBudget,
|
||||
});
|
||||
await new Promise<void>((resolve, reject) =>
|
||||
stdin.write(`${JSON.stringify(admission)}\n`, (error) =>
|
||||
error === null || error === undefined ? resolve() : reject(error),
|
||||
),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (response.kind === "failure") {
|
||||
if (response.reason === "resource") {
|
||||
result = err({
|
||||
javascript,
|
||||
module,
|
||||
projection,
|
||||
error: new AnalysisResourceConstraintError(
|
||||
OPERATION,
|
||||
response.resource ?? "memory",
|
||||
`${file.path}: ${response.message}`,
|
||||
{ ...this.#limits(), ...response.limits },
|
||||
),
|
||||
});
|
||||
break;
|
||||
}
|
||||
throw new AnalysisOutputError(
|
||||
OPERATION,
|
||||
`${file.path} during ${this.#phase}: ${response.message}`,
|
||||
);
|
||||
}
|
||||
if (javascript === null)
|
||||
throw new AnalysisOutputError(
|
||||
OPERATION,
|
||||
"JavaScript worker completed before a static checkpoint",
|
||||
);
|
||||
if (response.descriptor === null) {
|
||||
result = ok({ javascript, module: null, projection: null });
|
||||
break;
|
||||
}
|
||||
const descriptor = response.descriptor;
|
||||
// JSON bytes alone omit object/array overhead and the later graph/Evidence
|
||||
// sealing/serialization copies. Reserve a conservative expansion budget.
|
||||
const moduleEstimate = javaScriptTransferRetentionBytes(
|
||||
descriptor.module_bytes,
|
||||
descriptor.module_records,
|
||||
);
|
||||
const graphEstimate = javaScriptTransferRetentionBytes(
|
||||
descriptor.graph_bytes,
|
||||
descriptor.graph_records,
|
||||
);
|
||||
const remaining = Math.min(
|
||||
offeredResultBudget,
|
||||
this.#remainingResultBudget(),
|
||||
);
|
||||
if (moduleEstimate > remaining) {
|
||||
result = err({
|
||||
javascript,
|
||||
module: null,
|
||||
projection: null,
|
||||
error: new AnalysisResourceConstraintError(
|
||||
OPERATION,
|
||||
"memory",
|
||||
`Completed module facts for ${file.path} exceed the parent's remaining result budget; static facts were retained`,
|
||||
{
|
||||
...this.#limits(),
|
||||
parent_result_budget_bytes: this.#resultBudgetBytes,
|
||||
module_transfer_bytes: descriptor.module_bytes,
|
||||
},
|
||||
),
|
||||
});
|
||||
break;
|
||||
}
|
||||
const retainGraph = moduleEstimate + graphEstimate <= remaining;
|
||||
this.#phase = "result-admission";
|
||||
const decoded = await readJavaScriptAnalysisTransfer(
|
||||
join(root.path, `${id}.records`),
|
||||
descriptor,
|
||||
file,
|
||||
deadline.signal,
|
||||
retainGraph,
|
||||
);
|
||||
module = decoded.module;
|
||||
projection = decoded.projection;
|
||||
this.#estimatedRetainedBytes +=
|
||||
moduleEstimate + (retainGraph ? graphEstimate : 0);
|
||||
result = retainGraph
|
||||
? ok({ javascript, module, projection: decoded.projection })
|
||||
: err({
|
||||
javascript,
|
||||
module,
|
||||
projection,
|
||||
error: new AnalysisResourceConstraintError(
|
||||
OPERATION,
|
||||
"memory",
|
||||
`Semantic graph for ${file.path} exceeds the parent's remaining result budget; static and module facts were retained`,
|
||||
{
|
||||
...this.#limits(),
|
||||
parent_result_budget_bytes: this.#resultBudgetBytes,
|
||||
graph_transfer_bytes: descriptor.graph_bytes,
|
||||
graph_records: descriptor.graph_records,
|
||||
},
|
||||
),
|
||||
});
|
||||
break;
|
||||
}
|
||||
} catch (cause: unknown) {
|
||||
const error = this.#failure(cause, file.path, deadline);
|
||||
this.#lastFailure = error;
|
||||
// Complete checkpoints remain readable after a child crash or cancellation.
|
||||
const cleanup = await this.#stop(error);
|
||||
if (
|
||||
cleanup === undefined &&
|
||||
javascript === null &&
|
||||
this.#root !== undefined
|
||||
) {
|
||||
try {
|
||||
javascript = await this.#readCheckpoint(id, file.path);
|
||||
} catch (checkpointCause: unknown) {
|
||||
void checkpointCause;
|
||||
}
|
||||
}
|
||||
result = err({ error: cleanup ?? error, javascript, module, projection });
|
||||
} finally {
|
||||
deadline?.dispose();
|
||||
}
|
||||
if (result.ok) this.#lastFailure = undefined;
|
||||
else if (!(result.error.error instanceof ProviderCleanupError))
|
||||
this.#lastFailure = result.error.error;
|
||||
// Child stop must be verified before temporary files are released. On
|
||||
// cleanup uncertainty, close retains this owner for a later retry.
|
||||
if (this.#root !== undefined && !pendingCleanup.has(this)) {
|
||||
try {
|
||||
for (const suffix of ["source", "static.json", "records"])
|
||||
await rm(join(this.#root.path, `${id}.${suffix}`), { force: true });
|
||||
} catch (cause: unknown) {
|
||||
pendingCleanup.add(this);
|
||||
return err({
|
||||
javascript,
|
||||
module,
|
||||
projection,
|
||||
error: this.#cleanupError(message(cause)),
|
||||
});
|
||||
}
|
||||
}
|
||||
if (this.#applicationProjectionFailure !== undefined)
|
||||
return result.ok
|
||||
? ok({
|
||||
...result.value,
|
||||
applicationProjectionFailure: this.#applicationProjectionFailure,
|
||||
})
|
||||
: err({
|
||||
...result.error,
|
||||
applicationProjectionFailure: this.#applicationProjectionFailure,
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Verify process cleanup, then release the private runtime; failed close is retryable. */
|
||||
async close(): Promise<void> {
|
||||
const failure = await this.#stop();
|
||||
if (failure !== undefined) throw failure;
|
||||
if (this.#root !== undefined) {
|
||||
try {
|
||||
await this.#root.close();
|
||||
this.#root = undefined;
|
||||
} catch (cause: unknown) {
|
||||
pendingCleanup.add(this);
|
||||
throw this.#cleanupError(message(cause));
|
||||
}
|
||||
}
|
||||
pendingCleanup.delete(this);
|
||||
}
|
||||
|
||||
async #start(deadline: ProviderStartupDeadline): Promise<void> {
|
||||
if (this.#supervisor !== undefined && this.#ready) return;
|
||||
this.#phase = "startup";
|
||||
this.#heapLimitBytes = 0;
|
||||
this.#root ??= await PrivateRuntimeRoot.create({
|
||||
prefix: "rea-javascript-analysis-",
|
||||
});
|
||||
const environment: NodeJS.ProcessEnv = {
|
||||
...process.env,
|
||||
UV_THREADPOOL_SIZE: "1",
|
||||
};
|
||||
delete environment.NODE_OPTIONS;
|
||||
const spawned = await this.launcher({
|
||||
command: process.execPath,
|
||||
arguments: [
|
||||
`--max-old-space-size=${String(this.limits.heapMb)}`,
|
||||
"--max-semi-space-size=8",
|
||||
"--v8-pool-size=1",
|
||||
"--expose-gc",
|
||||
"--stack_size=2048",
|
||||
fileURLToPath(import.meta.resolve("#javascript-analysis-worker")),
|
||||
],
|
||||
runId: `${this.#runId}-${randomUUID()}`,
|
||||
cwd: this.#root.path,
|
||||
hostEnvironment: environment,
|
||||
stdin: "pipe",
|
||||
signal: deadline.signal,
|
||||
});
|
||||
this.#supervisor = new ProviderProcessSupervisor(
|
||||
{
|
||||
...spawned,
|
||||
ownsProcessLifetime: true,
|
||||
cleanup:
|
||||
spawned.cleanup ??
|
||||
(() => cleanupOwnedProcessGroup(spawned.ownership)),
|
||||
},
|
||||
{ captureStdout: false, maxDiagnosticBytes: 256 * 1024 },
|
||||
);
|
||||
spawned.process.stdout?.on("data", this.#onStdout);
|
||||
const ready = await this.#next(deadline, "worker startup");
|
||||
if (ready.kind !== "ready")
|
||||
throw new AnalysisOutputError(
|
||||
OPERATION,
|
||||
"JavaScript worker emitted facts before its startup handshake",
|
||||
);
|
||||
this.#heapLimitBytes = ready.heap_limit_bytes;
|
||||
this.#ready = true;
|
||||
}
|
||||
|
||||
async #next(
|
||||
deadline: ProviderStartupDeadline,
|
||||
path: string,
|
||||
): Promise<JavaScriptWorkerMessage> {
|
||||
for (;;) {
|
||||
if (deadline.interruption === "cancelled")
|
||||
throw new AnalysisCancelledError(OPERATION);
|
||||
if (deadline.interruption === "timeout")
|
||||
throw new AnalysisTimeoutError(OPERATION, this.limits.timeoutMs);
|
||||
if (this.#protocolFailure !== undefined)
|
||||
throw new AnalysisOutputError(OPERATION, this.#protocolFailure);
|
||||
const frame = this.#frames.shift();
|
||||
if (frame !== undefined) return frame;
|
||||
const snapshot = this.#supervisor?.snapshot();
|
||||
if (
|
||||
snapshot !== undefined &&
|
||||
(snapshot.exitCode !== undefined || snapshot.signal !== undefined)
|
||||
) {
|
||||
// Exit is not complete output: let any final control frame drain first.
|
||||
if (!(await this.#supervisor?.waitForOutputClose(10))) continue;
|
||||
if (this.#frames.length !== 0) continue;
|
||||
const drained = this.#supervisor?.snapshot() ?? snapshot;
|
||||
throw /heap out of memory|heap limit|allocation failed/iu.test(
|
||||
drained.stderr.text,
|
||||
)
|
||||
? new AnalysisResourceConstraintError(
|
||||
OPERATION,
|
||||
"memory",
|
||||
`Owned JavaScript worker exhausted its heap while analyzing ${path} during ${this.#phase}`,
|
||||
this.#limits(),
|
||||
{ capturedOutput: this.#captured() },
|
||||
)
|
||||
: new ProviderAdapterError(PROVIDER, OPERATION, {
|
||||
capturedOutput: this.#captured(),
|
||||
diagnostics: {
|
||||
source_path: path,
|
||||
phase: this.#phase,
|
||||
exit_code: drained.exitCode ?? null,
|
||||
signal: drained.signal ?? null,
|
||||
},
|
||||
});
|
||||
}
|
||||
await waitForAbortableDelay(10, deadline.signal);
|
||||
}
|
||||
}
|
||||
|
||||
async #readCheckpoint(
|
||||
id: string,
|
||||
sourcePath: string,
|
||||
expected?: { readonly bytes: number; readonly sha256: string },
|
||||
): Promise<JavaScriptStaticAnalysis> {
|
||||
if (this.#root === undefined)
|
||||
throw new TypeError("JavaScript checkpoint has no owned runtime");
|
||||
const path = join(this.#root.path, `${id}.static.json`);
|
||||
const bytes = (await stat(path)).size;
|
||||
if (
|
||||
bytes > DEFAULTS.checkpointBytes ||
|
||||
(expected !== undefined && expected.bytes !== bytes)
|
||||
)
|
||||
throw new AnalysisOutputError(
|
||||
OPERATION,
|
||||
"JavaScript static checkpoint byte count exceeds or differs from its declared budget",
|
||||
);
|
||||
if (
|
||||
bytes * 16 + 4 * MIB >
|
||||
getHeapStatistics().heap_size_limit -
|
||||
process.memoryUsage().heapUsed -
|
||||
64 * MIB
|
||||
)
|
||||
throw new AnalysisResourceConstraintError(
|
||||
OPERATION,
|
||||
"memory",
|
||||
"The parent's remaining heap cannot safely decode the completed JavaScript static checkpoint",
|
||||
{ ...this.#limits(), checkpoint_bytes: bytes },
|
||||
);
|
||||
const text = await readFile(path, "utf8");
|
||||
if (
|
||||
expected !== undefined &&
|
||||
createHash("sha256").update(text).digest("hex") !== expected.sha256
|
||||
)
|
||||
throw new AnalysisOutputError(
|
||||
OPERATION,
|
||||
"JavaScript static checkpoint digest does not match",
|
||||
);
|
||||
const value: unknown = JSON.parse(text);
|
||||
const facts = javaScriptWorkerStaticAnalysisSchema.parse(value);
|
||||
const remaining = this.#remainingTotalResultBudget();
|
||||
const retainedBytes = bytes * 6;
|
||||
this.#parentCapacityExhausted = retainedBytes > remaining;
|
||||
this.#estimatedRetainedBytes += retainedBytes;
|
||||
this.#retainedStaticBytes += retainedBytes;
|
||||
const expandedBytes = estimateJavaScriptStaticApplicationProjection(
|
||||
facts,
|
||||
bytes,
|
||||
sourcePath,
|
||||
);
|
||||
if (expandedBytes <= this.#remainingResultBudget())
|
||||
this.#estimatedRetainedBytes += expandedBytes;
|
||||
else
|
||||
this.#applicationProjectionFailure = new AnalysisResourceConstraintError(
|
||||
OPERATION,
|
||||
"memory",
|
||||
`Expanded application relationships for ${sourcePath} exceed the parent's result budget; complete static facts were retained on the file node`,
|
||||
{
|
||||
parent_result_budget_bytes: this.#resultBudgetBytes,
|
||||
application_projection_required_bytes: expandedBytes,
|
||||
},
|
||||
);
|
||||
return facts;
|
||||
}
|
||||
|
||||
#limits() {
|
||||
return {
|
||||
worker_old_space_mb: this.limits.heapMb,
|
||||
...(this.#heapLimitBytes === 0
|
||||
? {}
|
||||
: { worker_heap_limit_bytes: this.#heapLimitBytes }),
|
||||
worker_timeout_ms: this.limits.timeoutMs,
|
||||
};
|
||||
}
|
||||
#remainingResultBudget(): number {
|
||||
// Future sources need their compact observations before current modules
|
||||
// consume capacity with expanded relationships. A final or sole source
|
||||
// can use the unused reserve as well.
|
||||
const staticReserve =
|
||||
this.#sourcesRemaining !== undefined && this.#sourcesRemaining > 0
|
||||
? Math.max(
|
||||
0,
|
||||
Math.floor((this.#resultBudgetBytes * 4) / 5) -
|
||||
this.#retainedStaticBytes,
|
||||
)
|
||||
: 0;
|
||||
return Math.max(0, this.#remainingTotalResultBudget() - staticReserve);
|
||||
}
|
||||
#remainingTotalResultBudget(): number {
|
||||
return Math.max(
|
||||
0,
|
||||
Math.min(
|
||||
this.#resultBudgetBytes - this.#estimatedRetainedBytes,
|
||||
getHeapStatistics().heap_size_limit -
|
||||
process.memoryUsage().heapUsed -
|
||||
24 * JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES,
|
||||
),
|
||||
);
|
||||
}
|
||||
#captured() {
|
||||
const snapshot = this.#supervisor?.snapshot();
|
||||
return {
|
||||
stdout: "",
|
||||
stderr: snapshot?.stderr.text ?? "",
|
||||
truncated: snapshot?.diagnosticTruncated ?? false,
|
||||
stderr_bytes: snapshot?.stderr.observedBytes ?? 0,
|
||||
exit_code: snapshot?.exitCode ?? null,
|
||||
signal: snapshot?.signal ?? null,
|
||||
};
|
||||
}
|
||||
#failure(
|
||||
cause: unknown,
|
||||
path: string,
|
||||
deadline: ProviderStartupDeadline | undefined,
|
||||
): AnalysisError {
|
||||
if (deadline?.interruption === "cancelled")
|
||||
return new AnalysisCancelledError(OPERATION, {
|
||||
capturedOutput: this.#captured(),
|
||||
});
|
||||
if (deadline?.interruption === "timeout")
|
||||
return new AnalysisTimeoutError(OPERATION, this.limits.timeoutMs, {
|
||||
capturedOutput: this.#captured(),
|
||||
});
|
||||
if (cause instanceof AnalysisError) return cause;
|
||||
if (
|
||||
cause instanceof z.ZodError ||
|
||||
cause instanceof TypeError ||
|
||||
cause instanceof SyntaxError
|
||||
)
|
||||
return new AnalysisOutputError(
|
||||
OPERATION,
|
||||
`${path} during ${this.#phase}: ${message(cause)}`,
|
||||
{ cause, capturedOutput: this.#captured() },
|
||||
);
|
||||
return cause instanceof PrivateRuntimeRootUnavailableError
|
||||
? new AnalysisCapabilityUnavailableError(
|
||||
PROVIDER,
|
||||
OPERATION,
|
||||
cause.message,
|
||||
{ cause },
|
||||
)
|
||||
: new ProviderAdapterError(PROVIDER, OPERATION, {
|
||||
cause,
|
||||
capturedOutput: this.#captured(),
|
||||
diagnostics: {
|
||||
source_path: path,
|
||||
phase: this.#phase,
|
||||
message: message(cause),
|
||||
},
|
||||
});
|
||||
}
|
||||
#cleanupError(
|
||||
reason: string,
|
||||
original: AnalysisError | undefined = this.#lastFailure,
|
||||
): ProviderCleanupError {
|
||||
return new ProviderCleanupError(
|
||||
PROVIDER,
|
||||
[this.#runId, ...(this.#root === undefined ? [] : [this.#root.path])],
|
||||
{
|
||||
reason,
|
||||
worker_pid: this.#supervisor?.launch.process.pid ?? null,
|
||||
...(original === undefined
|
||||
? {}
|
||||
: {
|
||||
execution_failure: {
|
||||
tag: original._tag,
|
||||
message: original.message,
|
||||
captured_output: {
|
||||
stdout: original.capturedOutput?.stdout ?? "",
|
||||
stderr: original.capturedOutput?.stderr ?? "",
|
||||
truncated: original.capturedOutput?.truncated ?? false,
|
||||
},
|
||||
},
|
||||
}),
|
||||
},
|
||||
{
|
||||
operation: OPERATION,
|
||||
...(original === undefined ? {} : { cause: original }),
|
||||
},
|
||||
);
|
||||
}
|
||||
async #stop(
|
||||
original: AnalysisError | undefined = this.#lastFailure,
|
||||
): Promise<ProviderCleanupError | undefined> {
|
||||
if (this.#supervisor !== undefined) {
|
||||
const supervisor = this.#supervisor;
|
||||
const stopped = await supervisor.stop();
|
||||
if (stopped.status === "incomplete") {
|
||||
pendingCleanup.add(this);
|
||||
return this.#cleanupError(stopped.reason, original);
|
||||
}
|
||||
supervisor.launch.process.stdout?.off("data", this.#onStdout);
|
||||
supervisor.dispose();
|
||||
this.#supervisor = undefined;
|
||||
}
|
||||
this.#ready = false;
|
||||
this.#frames = [];
|
||||
this.#pending = "";
|
||||
this.#decoder = new StringDecoder("utf8");
|
||||
this.#protocolFailure = undefined;
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
const message = (cause: unknown): string =>
|
||||
cause instanceof Error ? cause.message : String(cause);
|
||||
@@ -0,0 +1,73 @@
|
||||
import { z } from "zod";
|
||||
import { jsonValueSchema } from "../../domain/jsonValue.js";
|
||||
import { digestSchema } from "../../domain/digests.js";
|
||||
import { JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS } from "../../domain/javascript/javascriptAnalysisResourceControls.js";
|
||||
import { javaScriptAnalysisTransferDescriptorSchema } from "./JavaScriptAnalysisTransfer.js";
|
||||
|
||||
/** Independent worker execution budget; source length is not an admission rule. */
|
||||
export const JAVASCRIPT_ANALYSIS_WORKER_DEFAULTS = {
|
||||
...JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS,
|
||||
checkpointBytes: 8 * 1024 * 1024,
|
||||
protocolBytes: 64 * 1024,
|
||||
} as const;
|
||||
|
||||
/** Caller-selected artifact identity, separate from the decoded source-text digest. */
|
||||
export const javaScriptWorkerRequestSchema = z.strictObject({
|
||||
id: z.uuid(),
|
||||
node_budget: z.number().int().min(0).max(20_000),
|
||||
text_sha256: digestSchema,
|
||||
file: z.strictObject({
|
||||
path: z.string(),
|
||||
sha256: digestSchema,
|
||||
container_sha256: digestSchema,
|
||||
bytes: z.number().int().nonnegative(),
|
||||
inventory_artifact_id: z.string(),
|
||||
kind: z.literal("javascript"),
|
||||
unpacked: z.boolean(),
|
||||
}),
|
||||
});
|
||||
|
||||
/** Parent admission after validated static observations have been retained. */
|
||||
export const javaScriptWorkerAdmissionSchema = z.strictObject({
|
||||
id: z.uuid(),
|
||||
semantic_result_budget_bytes: z.number().int().nonnegative(),
|
||||
});
|
||||
|
||||
const identity = { id: z.uuid() };
|
||||
|
||||
/** Control frames never contain AST, full IR, graph rows or source text. */
|
||||
export const javaScriptWorkerMessageSchema = z.discriminatedUnion("kind", [
|
||||
z.strictObject({
|
||||
kind: z.literal("ready"),
|
||||
heap_limit_bytes: z.number().int().positive(),
|
||||
}),
|
||||
z.strictObject({
|
||||
kind: z.literal("phase"),
|
||||
...identity,
|
||||
phase: z.enum(["parse", "static", "semantic", "projection", "transfer"]),
|
||||
}),
|
||||
z.strictObject({
|
||||
kind: z.literal("static"),
|
||||
...identity,
|
||||
bytes: z.number().int().nonnegative(),
|
||||
sha256: digestSchema,
|
||||
}),
|
||||
z.strictObject({
|
||||
kind: z.literal("complete"),
|
||||
...identity,
|
||||
descriptor: javaScriptAnalysisTransferDescriptorSchema.nullable(),
|
||||
}),
|
||||
z.strictObject({
|
||||
kind: z.literal("failure"),
|
||||
...identity,
|
||||
reason: z.enum(["resource", "output", "io"]),
|
||||
resource: z.enum(["memory", "cpu", "file-size", "transport"]).nullable(),
|
||||
limits: z.record(z.string(), jsonValueSchema).nullable(),
|
||||
message: z.string(),
|
||||
}),
|
||||
]);
|
||||
|
||||
/** One decoded control frame from the independently owned worker. */
|
||||
export type JavaScriptWorkerMessage = z.output<
|
||||
typeof javaScriptWorkerMessageSchema
|
||||
>;
|
||||
@@ -0,0 +1,582 @@
|
||||
import { z } from "zod";
|
||||
import { digestSchema } from "../../domain/digests.js";
|
||||
import type { JavaScriptStaticAnalysis } from "../../domain/javascript/javascriptStaticAnalysisTypes.js";
|
||||
import type {
|
||||
JavaScriptSemanticProperty,
|
||||
JavaScriptSemanticValue,
|
||||
} from "../../domain/javascript/javascriptSemanticValueTypes.js";
|
||||
import {
|
||||
javaScriptSemanticNodeSchema,
|
||||
javaScriptSemanticRelationSchema,
|
||||
javaScriptSemanticUnknownSchema,
|
||||
javaScriptSemanticFingerprintSchema,
|
||||
javaScriptSemanticEvidenceContextSchema,
|
||||
} from "../../domain/javascript/javascriptSemanticGraphSchemas.js";
|
||||
|
||||
const range =
|
||||
javaScriptSemanticNodeSchema.shape.identity.shape.source_range.unwrap();
|
||||
const count = z.number().int().nonnegative();
|
||||
const strings = z.array(z.string());
|
||||
const pathContext = z.enum(["module-specifier", "filesystem-expression"]);
|
||||
const findingContext = {
|
||||
module_key: z.string().nullable(),
|
||||
module_runtime: z.string().optional(),
|
||||
location: range,
|
||||
};
|
||||
const withRuntime = <Value extends { module_runtime?: string | undefined }>(
|
||||
value: Value,
|
||||
) => {
|
||||
const { module_runtime, ...rest } = value;
|
||||
return {
|
||||
...rest,
|
||||
...(module_runtime === undefined ? {} : { module_runtime }),
|
||||
};
|
||||
};
|
||||
const staticValue = z.discriminatedUnion("status", [
|
||||
z.strictObject({
|
||||
status: z.literal("literal"),
|
||||
value: z.union([z.string(), z.number(), z.boolean(), z.null()]),
|
||||
expression: z.null(),
|
||||
}),
|
||||
z.strictObject({
|
||||
status: z.literal("dynamic"),
|
||||
value: z.null(),
|
||||
expression: z.string(),
|
||||
}),
|
||||
]);
|
||||
const nullHandler = { handler_kind: z.null(), handler_location: z.null() };
|
||||
const missingHandler = {
|
||||
handler_kind: z.literal("missing"),
|
||||
handler_location: z.null(),
|
||||
};
|
||||
const locatedHandler = {
|
||||
handler_kind: z.enum([
|
||||
"inline-function",
|
||||
"identifier",
|
||||
"member-expression",
|
||||
"dynamic-expression",
|
||||
]),
|
||||
handler_location: range,
|
||||
};
|
||||
const ipcChannel = <Shape extends z.ZodRawShape>(shape: Shape) =>
|
||||
z.union([
|
||||
z.strictObject({
|
||||
...findingContext,
|
||||
...shape,
|
||||
channel: z.string(),
|
||||
channel_expression: z.null(),
|
||||
}),
|
||||
z.strictObject({
|
||||
...findingContext,
|
||||
...shape,
|
||||
channel: z.null(),
|
||||
channel_expression: z.string(),
|
||||
}),
|
||||
]);
|
||||
const ipc = z
|
||||
.union([
|
||||
ipcChannel({
|
||||
side: z.literal("renderer"),
|
||||
operation: z.enum(["send", "send-sync", "post-message", "send-to-host"]),
|
||||
mode: z.literal("send"),
|
||||
...nullHandler,
|
||||
}),
|
||||
ipcChannel({
|
||||
side: z.literal("renderer"),
|
||||
operation: z.literal("invoke"),
|
||||
mode: z.literal("invoke"),
|
||||
...nullHandler,
|
||||
}),
|
||||
ipcChannel({
|
||||
side: z.enum(["renderer", "main"]),
|
||||
operation: z.enum(["on", "once"]),
|
||||
mode: z.literal("listen"),
|
||||
...missingHandler,
|
||||
}),
|
||||
ipcChannel({
|
||||
side: z.enum(["renderer", "main"]),
|
||||
operation: z.enum(["on", "once"]),
|
||||
mode: z.literal("listen"),
|
||||
...locatedHandler,
|
||||
}),
|
||||
ipcChannel({
|
||||
side: z.literal("main"),
|
||||
operation: z.enum(["handle", "handle-once"]),
|
||||
mode: z.literal("handle"),
|
||||
...missingHandler,
|
||||
}),
|
||||
ipcChannel({
|
||||
side: z.literal("main"),
|
||||
operation: z.enum(["handle", "handle-once"]),
|
||||
mode: z.literal("handle"),
|
||||
...locatedHandler,
|
||||
}),
|
||||
])
|
||||
.transform((value) => {
|
||||
const { module_runtime, ...rest } = value;
|
||||
return {
|
||||
...rest,
|
||||
...(module_runtime === undefined ? {} : { module_runtime }),
|
||||
};
|
||||
});
|
||||
const windowShape = {
|
||||
...findingContext,
|
||||
options_status: z.enum(["object-literal", "dynamic", "missing"]),
|
||||
web_preferences_status: z.enum(["object-literal", "dynamic", "missing"]),
|
||||
web_preferences: z.array(
|
||||
z.strictObject({ name: z.string(), value: staticValue }),
|
||||
),
|
||||
};
|
||||
const bridgeShape = {
|
||||
...findingContext,
|
||||
world: z.enum(["main", "isolated"]),
|
||||
world_id: staticValue.nullable(),
|
||||
api_status: z.enum(["object-literal", "dynamic", "missing"]),
|
||||
members: strings,
|
||||
unknown_members: count,
|
||||
};
|
||||
const utilityShape = { ...findingContext, service_name: z.string().nullable() };
|
||||
const electron = z.strictObject({
|
||||
browser_windows: z.array(
|
||||
z.union([
|
||||
z
|
||||
.strictObject({
|
||||
...windowShape,
|
||||
preload_path: z.string(),
|
||||
preload_resolution_context: pathContext,
|
||||
})
|
||||
.transform(withRuntime),
|
||||
z
|
||||
.strictObject({
|
||||
...windowShape,
|
||||
preload_path: z.null(),
|
||||
preload_resolution_context: z.null(),
|
||||
})
|
||||
.transform(withRuntime),
|
||||
]),
|
||||
),
|
||||
context_bridge_apis: z.array(
|
||||
z.union([
|
||||
z
|
||||
.strictObject({
|
||||
...bridgeShape,
|
||||
api_key: z.string(),
|
||||
api_key_expression: z.null(),
|
||||
})
|
||||
.transform(withRuntime),
|
||||
z
|
||||
.strictObject({
|
||||
...bridgeShape,
|
||||
api_key: z.null(),
|
||||
api_key_expression: z.string(),
|
||||
})
|
||||
.transform(withRuntime),
|
||||
z
|
||||
.strictObject({
|
||||
...bridgeShape,
|
||||
api_key: z.null(),
|
||||
api_key_expression: z.null(),
|
||||
})
|
||||
.transform(withRuntime),
|
||||
]),
|
||||
),
|
||||
ipc: z.array(ipc),
|
||||
sender_validations: z.array(
|
||||
z.strictObject({
|
||||
...findingContext,
|
||||
subject: z.enum([
|
||||
"sender-url",
|
||||
"sender-origin",
|
||||
"sender-frame",
|
||||
"sender-id",
|
||||
"frame-id",
|
||||
"process-id",
|
||||
]),
|
||||
mechanism: z.string(),
|
||||
expected: staticValue,
|
||||
enforcement: z.literal("unknown"),
|
||||
}),
|
||||
),
|
||||
utility_processes: z.array(
|
||||
z.union([
|
||||
z
|
||||
.strictObject({
|
||||
...utilityShape,
|
||||
module_path: z.string(),
|
||||
module_resolution_context: pathContext,
|
||||
module_expression: z.null(),
|
||||
})
|
||||
.transform(withRuntime),
|
||||
z
|
||||
.strictObject({
|
||||
...utilityShape,
|
||||
module_path: z.null(),
|
||||
module_resolution_context: z.null(),
|
||||
module_expression: z.string(),
|
||||
})
|
||||
.transform(withRuntime),
|
||||
]),
|
||||
),
|
||||
native_addon_bindings: z.array(
|
||||
z.strictObject({
|
||||
...findingContext,
|
||||
specifier: z.string(),
|
||||
binding_kind: z.enum(["import", "require", "re-export"]),
|
||||
module_kind: z.enum(["import", "require"]),
|
||||
members: strings,
|
||||
namespace_access: z.boolean().optional(),
|
||||
dynamic_member_access: z.boolean().optional(),
|
||||
}),
|
||||
),
|
||||
});
|
||||
const referenceShape = {
|
||||
...findingContext,
|
||||
kind: z.enum([
|
||||
"static-import",
|
||||
"dynamic-import",
|
||||
"require",
|
||||
"worker",
|
||||
"service-worker",
|
||||
]),
|
||||
};
|
||||
|
||||
/** Exact producer facts recoverable before a later semantic worker failure. */
|
||||
export const javaScriptWorkerStaticAnalysisSchema = z
|
||||
.strictObject({
|
||||
parse_status: z.enum(["complete", "partial", "failed"]),
|
||||
parse_error_count: count,
|
||||
visited_ast_nodes: count,
|
||||
references: z.array(
|
||||
z.union([
|
||||
z
|
||||
.strictObject({
|
||||
...referenceShape,
|
||||
specifier: z.string(),
|
||||
expression: z.null(),
|
||||
})
|
||||
.transform(withRuntime),
|
||||
z
|
||||
.strictObject({
|
||||
...referenceShape,
|
||||
specifier: z.null(),
|
||||
expression: z.string(),
|
||||
})
|
||||
.transform(withRuntime),
|
||||
]),
|
||||
),
|
||||
endpoints: z.array(
|
||||
z.strictObject({
|
||||
...findingContext,
|
||||
kind: z.enum(["route", "network"]),
|
||||
value: z.string(),
|
||||
mechanism: z.string(),
|
||||
}),
|
||||
),
|
||||
storage: z.array(
|
||||
z.strictObject({
|
||||
...findingContext,
|
||||
kind: z.enum([
|
||||
"local-storage",
|
||||
"session-storage",
|
||||
"indexed-db",
|
||||
"cache-storage",
|
||||
"sqlite",
|
||||
]),
|
||||
name: z.string().nullable(),
|
||||
mechanism: z.string(),
|
||||
}),
|
||||
),
|
||||
bundler_registrations: z.array(
|
||||
z.strictObject({
|
||||
bundler: z.enum(["webpack", "rspack", "esbuild"]),
|
||||
runtime: z.string(),
|
||||
chunk_keys: strings,
|
||||
unknown_chunk_keys: count,
|
||||
runtime_require_name: z.string().nullable(),
|
||||
runtime_module_cache_status: z.enum(["observed", "not-observed"]),
|
||||
entry_module_keys: strings,
|
||||
unknown_entry_module_keys: count,
|
||||
async_chunk_keys: strings,
|
||||
unknown_async_chunk_keys: count,
|
||||
modules: z.array(
|
||||
z.strictObject({
|
||||
module_key: z.string(),
|
||||
factory_require_name: z.string().nullable(),
|
||||
source_sha256: digestSchema,
|
||||
exports: strings,
|
||||
location: range,
|
||||
structural_fingerprint_sha256: digestSchema,
|
||||
structural_fingerprint_algorithm: z.literal("babel-ast-v1"),
|
||||
}),
|
||||
),
|
||||
location: range,
|
||||
}),
|
||||
),
|
||||
role_paths: z.array(
|
||||
z.strictObject({
|
||||
...findingContext,
|
||||
role: z.enum(["preload", "renderer"]),
|
||||
path: z.string(),
|
||||
resolution_context: pathContext,
|
||||
mechanism: z.string(),
|
||||
}),
|
||||
),
|
||||
source_map_urls: z.array(
|
||||
z.strictObject({ declared_url: z.string(), location: range }),
|
||||
),
|
||||
vendors: strings,
|
||||
electron,
|
||||
limitations: strings,
|
||||
})
|
||||
.transform((analysis) => ({
|
||||
...analysis,
|
||||
endpoints: analysis.endpoints.map(withRuntime),
|
||||
storage: analysis.storage.map(withRuntime),
|
||||
role_paths: analysis.role_paths.map(withRuntime),
|
||||
electron: {
|
||||
...analysis.electron,
|
||||
sender_validations: analysis.electron.sender_validations.map(withRuntime),
|
||||
native_addon_bindings: analysis.electron.native_addon_bindings.map(
|
||||
(value) => {
|
||||
const { namespace_access, dynamic_member_access, ...rest } =
|
||||
withRuntime(value);
|
||||
return {
|
||||
...rest,
|
||||
...(namespace_access === undefined ? {} : { namespace_access }),
|
||||
...(dynamic_member_access === undefined
|
||||
? {}
|
||||
: { dynamic_member_access }),
|
||||
};
|
||||
},
|
||||
),
|
||||
},
|
||||
})) satisfies z.ZodType<JavaScriptStaticAnalysis>;
|
||||
|
||||
const resourceLimit = z.enum([
|
||||
"primitive-candidates",
|
||||
"primitive-bytes",
|
||||
"expression-depth",
|
||||
]);
|
||||
const primitive = z.union([z.string(), z.number(), z.boolean(), z.null()]);
|
||||
const property: z.ZodType<JavaScriptSemanticProperty> = z.lazy(() =>
|
||||
z.strictObject({
|
||||
name: z.string(),
|
||||
value: semanticValue,
|
||||
presence: z.enum(["present", "absent", "unknown-coverage"]),
|
||||
}),
|
||||
);
|
||||
const semanticValue: z.ZodType<JavaScriptSemanticValue> = z.lazy(() =>
|
||||
z
|
||||
.discriminatedUnion("status", [
|
||||
z.strictObject({ status: z.literal("literal"), value: primitive }),
|
||||
z.strictObject({
|
||||
status: z.literal("union"),
|
||||
values: z.array(primitive),
|
||||
}),
|
||||
z.discriminatedUnion("unknownProperties", [
|
||||
z.strictObject({
|
||||
status: z.literal("object"),
|
||||
properties: z.array(property),
|
||||
unknownProperties: z.literal(false),
|
||||
omittedProperties: z.literal(0),
|
||||
}),
|
||||
z.strictObject({
|
||||
status: z.literal("object"),
|
||||
properties: z.array(property),
|
||||
unknownProperties: z.literal(true),
|
||||
omittedProperties: count.nullable(),
|
||||
}),
|
||||
]),
|
||||
z.discriminatedUnion("unknownItems", [
|
||||
z.strictObject({
|
||||
status: z.literal("array"),
|
||||
items: z.array(property),
|
||||
unknownItems: z.literal(false),
|
||||
omittedItems: z.literal(0),
|
||||
}),
|
||||
z.strictObject({
|
||||
status: z.literal("array"),
|
||||
items: z.array(property),
|
||||
unknownItems: z.literal(true),
|
||||
omittedItems: count.nullable(),
|
||||
}),
|
||||
]),
|
||||
z.strictObject({
|
||||
status: z.enum(["unknown", "ambiguous", "cycle"]),
|
||||
reason: z.string(),
|
||||
resourceLimit: resourceLimit.optional(),
|
||||
}),
|
||||
])
|
||||
.transform((value) => {
|
||||
if (
|
||||
value.status === "literal" ||
|
||||
value.status === "union" ||
|
||||
value.status === "object" ||
|
||||
value.status === "array"
|
||||
)
|
||||
return value;
|
||||
const { resourceLimit: limit, ...rest } = value;
|
||||
return {
|
||||
...rest,
|
||||
...(limit === undefined ? {} : { resourceLimit: limit }),
|
||||
};
|
||||
}),
|
||||
);
|
||||
const origin = z.strictObject({ specifier: z.string(), importedPath: strings });
|
||||
const provenance = z.discriminatedUnion("status", [
|
||||
z.strictObject({
|
||||
status: z.literal("module"),
|
||||
origins: z.tuple([origin]),
|
||||
reason: z.null(),
|
||||
}),
|
||||
z.strictObject({
|
||||
status: z.literal("local"),
|
||||
origins: z.tuple([]),
|
||||
reason: z.null(),
|
||||
}),
|
||||
z.strictObject({
|
||||
status: z.literal("ambiguous"),
|
||||
origins: z.array(origin),
|
||||
reason: z.string(),
|
||||
}),
|
||||
z.strictObject({
|
||||
status: z.enum(["unknown", "cycle"]),
|
||||
origins: z.tuple([]),
|
||||
reason: z.string(),
|
||||
}),
|
||||
]);
|
||||
const coverage = z
|
||||
.discriminatedUnion("status", [
|
||||
z.strictObject({
|
||||
status: z.literal("complete"),
|
||||
omittedCount: z.literal(0),
|
||||
resourceLimits: z.array(resourceLimit).optional(),
|
||||
}),
|
||||
z.strictObject({
|
||||
status: z.literal("partial"),
|
||||
omittedCount: z.union([z.literal(0), z.null()]),
|
||||
resourceLimits: z.array(resourceLimit).optional(),
|
||||
}),
|
||||
z.strictObject({
|
||||
status: z.literal("failed"),
|
||||
omittedCount: z.null(),
|
||||
resourceLimits: z.array(resourceLimit).optional(),
|
||||
}),
|
||||
])
|
||||
.transform((value) => {
|
||||
const { resourceLimits, ...rest } = value;
|
||||
return {
|
||||
...rest,
|
||||
...(resourceLimits === undefined ? {} : { resourceLimits }),
|
||||
};
|
||||
});
|
||||
const moduleLink = z.strictObject({
|
||||
kind: z.enum(["import", "require", "export", "re-export", "commonjs-export"]),
|
||||
specifier: z.string().nullable(),
|
||||
importedName: z.string().nullable(),
|
||||
localName: z.string().nullable(),
|
||||
exportedName: z.string().nullable(),
|
||||
callableId: z.string().nullable(),
|
||||
location: range,
|
||||
});
|
||||
const definitionKind = z.enum([
|
||||
"import",
|
||||
"variable",
|
||||
"parameter",
|
||||
"function",
|
||||
"class",
|
||||
"catch",
|
||||
"assignment",
|
||||
]);
|
||||
const scope = z.strictObject({
|
||||
scopeId: z.string(),
|
||||
parentScopeId: z.string().nullable(),
|
||||
kind: z.enum([
|
||||
"program",
|
||||
"function",
|
||||
"block",
|
||||
"static-block",
|
||||
"class",
|
||||
"catch",
|
||||
]),
|
||||
location: range,
|
||||
bindingsComplete: z.boolean(),
|
||||
bindingIds: strings,
|
||||
});
|
||||
const binding = z.strictObject({
|
||||
bindingId: z.string(),
|
||||
scopeId: z.string(),
|
||||
name: z.string(),
|
||||
kind: definitionKind,
|
||||
mutable: z.boolean(),
|
||||
definitions: z.array(
|
||||
z.strictObject({ kind: definitionKind, location: range }),
|
||||
),
|
||||
value: semanticValue,
|
||||
provenance,
|
||||
});
|
||||
const callable = z.strictObject({
|
||||
callableId: z.string(),
|
||||
kind: z.enum(["function", "class", "method"]),
|
||||
name: z.string().nullable(),
|
||||
containerScopeId: z.string(),
|
||||
bodyScopeId: z.string().nullable(),
|
||||
location: range,
|
||||
returnSites: z.array(
|
||||
z.strictObject({
|
||||
returnSiteId: z.string(),
|
||||
location: range,
|
||||
identityReferenceLocation: range.nullable(),
|
||||
value: semanticValue,
|
||||
}),
|
||||
),
|
||||
returnCoverage: z.strictObject({
|
||||
retainedCount: count,
|
||||
status: z.enum(["complete", "partial"]),
|
||||
omittedCount: z.union([z.literal(0), z.null()]),
|
||||
}),
|
||||
});
|
||||
|
||||
/** Validate each transferred graph fact independently of its emitting realm. */
|
||||
export const javaScriptWorkerGraphRecordSchema = z.discriminatedUnion("kind", [
|
||||
z.strictObject({
|
||||
kind: z.literal("evidence-context"),
|
||||
value: javaScriptSemanticEvidenceContextSchema,
|
||||
}),
|
||||
z.strictObject({
|
||||
kind: z.literal("node"),
|
||||
value: javaScriptSemanticNodeSchema,
|
||||
}),
|
||||
z.strictObject({
|
||||
kind: z.literal("relation"),
|
||||
value: javaScriptSemanticRelationSchema,
|
||||
}),
|
||||
z.strictObject({
|
||||
kind: z.literal("unknown"),
|
||||
value: javaScriptSemanticUnknownSchema,
|
||||
}),
|
||||
z.strictObject({
|
||||
kind: z.literal("fingerprint"),
|
||||
value: javaScriptSemanticFingerprintSchema,
|
||||
}),
|
||||
]);
|
||||
|
||||
/** Portable per-file header, before any graph binding or immutable proof. */
|
||||
export const javaScriptWorkerProjectionHeaderSchema = z.strictObject({
|
||||
roots: z.array(javaScriptSemanticNodeSchema.shape.node_id),
|
||||
truncated: z.boolean(),
|
||||
});
|
||||
|
||||
/** Module facts are transferred individually to avoid a whole-IR JSON string. */
|
||||
export const javaScriptWorkerModuleRecordSchema = z.discriminatedUnion("kind", [
|
||||
z.strictObject({ kind: z.literal("scope"), value: scope }),
|
||||
z.strictObject({ kind: z.literal("binding"), value: binding }),
|
||||
z.strictObject({ kind: z.literal("callable"), value: callable }),
|
||||
z.strictObject({ kind: z.literal("module-link"), value: moduleLink }),
|
||||
]);
|
||||
|
||||
/** Coverage and limitations are independent of module collection sizes. */
|
||||
export const javaScriptWorkerModuleHeaderSchema = z.strictObject({
|
||||
coverage,
|
||||
limitations: strings,
|
||||
});
|
||||
@@ -0,0 +1,185 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { createInterface } from "node:readline";
|
||||
import { getHeapStatistics } from "node:v8";
|
||||
import { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js";
|
||||
import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js";
|
||||
import type { JavaScriptStaticAnalysis } from "../../domain/javascript/javascriptStaticAnalysisTypes.js";
|
||||
import type { JavaScriptModuleSemanticIr } from "../../domain/javascript/javascriptModuleSemanticIr.js";
|
||||
import type { JavaScriptSemanticFileProjection } from "../../domain/javascript/javascriptSemanticFileProjection.js";
|
||||
import { writeJavaScriptAnalysisTransfer } from "./JavaScriptAnalysisTransfer.js";
|
||||
import { javaScriptWorkerStaticAnalysisSchema } from "./JavaScriptAnalysisWorkerSchemas.js";
|
||||
import {
|
||||
JAVASCRIPT_ANALYSIS_WORKER_DEFAULTS as DEFAULTS,
|
||||
javaScriptWorkerRequestSchema,
|
||||
javaScriptWorkerAdmissionSchema,
|
||||
javaScriptWorkerMessageSchema,
|
||||
type JavaScriptWorkerMessage,
|
||||
} from "./JavaScriptAnalysisWorkerProtocol.js";
|
||||
|
||||
/** Trusted composition callback; this provider owns framing and temporary I/O. */
|
||||
export interface JavaScriptSourceWorkerAnalyzer {
|
||||
(
|
||||
file: JavaScriptArtifactFile,
|
||||
nodeBudget: number,
|
||||
callbacks: {
|
||||
readonly phase: (
|
||||
phase: "parse" | "static" | "semantic" | "projection",
|
||||
) => Promise<void>;
|
||||
readonly checkpoint: (facts: JavaScriptStaticAnalysis) => Promise<number>;
|
||||
},
|
||||
): Promise<{
|
||||
readonly module: JavaScriptModuleSemanticIr;
|
||||
readonly projection: JavaScriptSemanticFileProjection;
|
||||
} | null>;
|
||||
}
|
||||
|
||||
const send = async (input: JavaScriptWorkerMessage): Promise<void> => {
|
||||
const value = javaScriptWorkerMessageSchema.parse(input);
|
||||
const frame = `${JSON.stringify(value)}\n`;
|
||||
if (Buffer.byteLength(frame) > DEFAULTS.protocolBytes)
|
||||
throw new RangeError(
|
||||
"JavaScript worker control frame exceeds its byte budget",
|
||||
);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
process.stdout.write(frame, (error) =>
|
||||
error === null || error === undefined ? resolve() : reject(error),
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
/** Serve sequential inert-source requests from a parent-owned private runtime root. */
|
||||
export const serveJavaScriptAnalysisWorker = async (
|
||||
analyze: JavaScriptSourceWorkerAnalyzer,
|
||||
): Promise<void> => {
|
||||
const root = process.cwd();
|
||||
const input = createInterface({ input: process.stdin, crlfDelay: Infinity });
|
||||
const requests = input[Symbol.asyncIterator]();
|
||||
await send({
|
||||
kind: "ready",
|
||||
heap_limit_bytes: getHeapStatistics().heap_size_limit,
|
||||
});
|
||||
try {
|
||||
for (;;) {
|
||||
const next = await requests.next();
|
||||
if (next.done) break;
|
||||
const line = next.value;
|
||||
if (Buffer.byteLength(line) > DEFAULTS.protocolBytes)
|
||||
throw new RangeError(
|
||||
"JavaScript worker request exceeds its byte budget",
|
||||
);
|
||||
const raw: unknown = JSON.parse(line);
|
||||
const request = javaScriptWorkerRequestSchema.parse(raw);
|
||||
// Function scope releases source AST/IR/results before accepting another job.
|
||||
await analyzeJob(root, request, analyze, requests);
|
||||
if (
|
||||
process.memoryUsage().heapUsed >
|
||||
getHeapStatistics().heap_size_limit / 2
|
||||
)
|
||||
global.gc?.();
|
||||
}
|
||||
} finally {
|
||||
input.close();
|
||||
}
|
||||
};
|
||||
|
||||
const analyzeJob = async (
|
||||
root: string,
|
||||
request: ReturnType<typeof javaScriptWorkerRequestSchema.parse>,
|
||||
analyze: JavaScriptSourceWorkerAnalyzer,
|
||||
requests: AsyncIterator<string>,
|
||||
): Promise<void> => {
|
||||
try {
|
||||
const source = await readFile(join(root, `${request.id}.source`), "utf8");
|
||||
if (
|
||||
createHash("sha256").update(source).digest("hex") !== request.text_sha256
|
||||
)
|
||||
throw new TypeError(
|
||||
"JavaScript worker source text digest does not match its request",
|
||||
);
|
||||
const result = await analyze(
|
||||
{
|
||||
...request.file,
|
||||
text: { included: true, value: source },
|
||||
},
|
||||
request.node_budget,
|
||||
{
|
||||
phase: async (phase) => send({ kind: "phase", id: request.id, phase }),
|
||||
checkpoint: async (facts) => {
|
||||
const value = javaScriptWorkerStaticAnalysisSchema.parse(facts);
|
||||
const text = JSON.stringify(value);
|
||||
const bytes = Buffer.byteLength(text);
|
||||
if (bytes > DEFAULTS.checkpointBytes)
|
||||
throw new AnalysisResourceConstraintError(
|
||||
"analyze_javascript_application",
|
||||
"transport",
|
||||
"JavaScript static checkpoint exceeds its complete-fact byte budget",
|
||||
{
|
||||
checkpoint_bytes: bytes,
|
||||
checkpoint_limit_bytes: DEFAULTS.checkpointBytes,
|
||||
},
|
||||
);
|
||||
await writeFile(join(root, `${request.id}.static.json`), text, {
|
||||
flag: "wx",
|
||||
mode: 0o600,
|
||||
});
|
||||
await send({
|
||||
kind: "static",
|
||||
id: request.id,
|
||||
bytes,
|
||||
sha256: createHash("sha256").update(text).digest("hex"),
|
||||
});
|
||||
const next = await requests.next();
|
||||
if (
|
||||
next.done ||
|
||||
Buffer.byteLength(next.value) > DEFAULTS.protocolBytes
|
||||
)
|
||||
throw new TypeError(
|
||||
"JavaScript worker admission frame is missing or oversized",
|
||||
);
|
||||
const raw: unknown = JSON.parse(next.value);
|
||||
const admission = javaScriptWorkerAdmissionSchema.parse(raw);
|
||||
if (admission.id !== request.id)
|
||||
throw new TypeError(
|
||||
"JavaScript worker admission names a different source job",
|
||||
);
|
||||
return admission.semantic_result_budget_bytes;
|
||||
},
|
||||
},
|
||||
);
|
||||
await send({ kind: "phase", id: request.id, phase: "transfer" });
|
||||
const descriptor =
|
||||
result === null
|
||||
? null
|
||||
: await writeJavaScriptAnalysisTransfer(
|
||||
join(root, `${request.id}.records`),
|
||||
result.module,
|
||||
result.projection,
|
||||
);
|
||||
await send({ kind: "complete", id: request.id, descriptor });
|
||||
} catch (cause: unknown) {
|
||||
await send({
|
||||
kind: "failure",
|
||||
id: request.id,
|
||||
reason:
|
||||
cause instanceof AnalysisResourceConstraintError ||
|
||||
cause instanceof RangeError
|
||||
? "resource"
|
||||
: cause instanceof TypeError
|
||||
? "output"
|
||||
: "io",
|
||||
resource:
|
||||
cause instanceof AnalysisResourceConstraintError
|
||||
? cause.resource
|
||||
: cause instanceof RangeError
|
||||
? "memory"
|
||||
: null,
|
||||
limits:
|
||||
cause instanceof AnalysisResourceConstraintError
|
||||
? cause.reportedLimits
|
||||
: null,
|
||||
message: cause instanceof Error ? cause.message : String(cause),
|
||||
});
|
||||
}
|
||||
};
|
||||
Reference in new issue
Block a user