diff --git a/docs/binary-diagnostics.md b/docs/binary-diagnostics.md index 0e6a3f196..b49c0ca23 100644 --- a/docs/binary-diagnostics.md +++ b/docs/binary-diagnostics.md @@ -147,6 +147,24 @@ received signal alone does not establish its exact cause. Dynamic tags require a complete DT_NULL inside PT_DYNAMIC; interpreter names and ranges end at the first NUL, with any padding still represented by the original segment range. +## Ghidra process failures + +When output from the failing Ghidra startup or operation reports +`java.lang.OutOfMemoryError`, REA returns `resource_constraint`. Java heap or GC +overhead exhaustion includes heap-specific recovery advice. Metaspace and array +size failures do not recommend increasing the heap; native-thread exhaustion is +reported as a capacity constraint with unknown limits. + +`configured_jvm_max_heap` records the explicit `-Xmx` value selected by REA's +launch builder. It is a configuration fact, not measured memory use or proof of +the effective JVM limit; JVM environment options may affect that limit. +When a Windows script wrapper selects the JVM, this value remains unknown. +Historical stdout and stderr remain available, but classification uses only the +failing operation's `failure_output`. Output draining waits at most one second, +subject to earlier cancellation or the startup deadline. `output_incomplete` +marks a snapshot taken before stream closure; cleanup ownership remains with +the session until release succeeds. + ## Portable PE resources `inspect_pe_resources` / `inspect-pe-resources` inspects an explicit PE32 or diff --git a/src/ghidra/GhidraClient.ts b/src/ghidra/GhidraClient.ts index acbd58ee6..66b09caae 100644 --- a/src/ghidra/GhidraClient.ts +++ b/src/ghidra/GhidraClient.ts @@ -20,6 +20,7 @@ import { ProviderStartupDeadline } from "../process/ProviderDeadline.js"; import { ProviderRunLineage } from "../process/ProviderRunLineage.js"; import { type ProviderProcessDiagnostic, + type ProviderProcessOutputCheckpoint, type ProviderProcessSnapshot, ProviderProcessSupervisor, } from "../process/ProviderProcess.js"; @@ -33,6 +34,7 @@ import type { GhidraInventoryOperation } from "./GhidraInventoryValues.js"; import type { GhidraFunctionOperation } from "./GhidraFunctionValues.js"; import { createGhidraDiagnostics, + ghidraDiagnosticStream, GHIDRA_PROCESS_DIAGNOSTIC_BYTES, } from "./GhidraDiagnostics.js"; import { GhidraLaunchError, type GhidraLaunch } from "./GhidraLauncher.js"; @@ -113,6 +115,10 @@ export class GhidraClient { #nextId = 1; #closing = false; #processSnapshot: ProviderProcessSnapshot | undefined; + #startupOutputCheckpoint: ProviderProcessOutputCheckpoint | undefined; + #activeOutputCheckpoint: ProviderProcessOutputCheckpoint | undefined; + #outputDrainPromise: Promise | undefined; + #outputDrainIncomplete = false; #lastDiagnostics: Readonly> = {}; #startupController: AbortController | undefined; #startPromise: Promise | undefined; @@ -145,7 +151,6 @@ export class GhidraClient { if (!this.#closing) this.#failAll(this.#failure("process", "Ghidra bridge socket closed")); }; - constructor(options: GhidraClientOptions) { this.#options = { ...options, @@ -164,8 +169,45 @@ export class GhidraClient { failure: this.#failure, }); this.#requestQueue = new GhidraRequestQueue( - (method, parameters, requestOptions) => - this.#wire.request(method, parameters, requestOptions), + async (method, parameters, requestOptions) => { + const owner = this.#process; + const checkpoint = owner?.checkpointOutput(); + const token = this.#token; + this.#activeOutputCheckpoint = checkpoint; + try { + const result = await this.#wire.request( + method, + parameters, + requestOptions, + ); + if ( + !result.ok && + (result.error.kind === "process" || + result.error.kind === "protocol") + ) { + const drained = await this.#drainOutputBounded( + requestOptions.signal, + ); + if (requestOptions.signal?.aborted === true) + return err( + this.#failure("cancelled", "Ghidra request was cancelled"), + ); + return err( + this.#refreshProcessFailure( + result.error, + owner, + checkpoint, + token, + !drained, + ), + ); + } + return result; + } finally { + if (this.#activeOutputCheckpoint === checkpoint) + this.#activeOutputCheckpoint = undefined; + } + }, (kind, message, cause) => this.#failure(kind, message, cause), () => this.#stopAfterActiveCancellation(), ); @@ -273,7 +315,7 @@ export class GhidraClient { ): Promise> { const started = await this.start(options.signal); if (!started.ok) return started; - const result = await this.#wire.request("ping", {}, options); + const result = await this.#requestQueue.run("ping", {}, options); if (!result.ok) return result; return this.#parseSessionInfo(result.value); } @@ -463,7 +505,19 @@ export class GhidraClient { async #startupFailure( primary: GhidraSessionError, + deadline?: ProviderStartupDeadline, ): Promise { + await this.#drainOutputBounded(deadline?.signal); + if (deadline?.interruption === "cancelled") + primary = this.#interruptionFailure(deadline, primary.cause); + primary = this.#failure(primary.kind, primary.message, primary.cause, { + ...(primary.timeoutMs === undefined + ? {} + : { timeoutMs: primary.timeoutMs }), + ...(primary.remoteCode === undefined + ? {} + : { remoteCode: primary.remoteCode }), + }); try { await this.#cleanup(); return err(primary); @@ -499,14 +553,17 @@ export class GhidraClient { return this.#closePromise; } - #stopAfterActiveCancellation(): Promise { - return this.#requestClose(true).catch((cause: unknown) => { - this.#logger.error( - { error: cause instanceof Error ? cause.message : String(cause) }, - "Ghidra cleanup after active request cancellation failed", - ); - throw cause; - }); + #stopAfterActiveCancellation(): Promise> { + return this.#requestClose(true).then( + () => ok(null), + (cause: unknown) => { + this.#logger.error( + { error: cause instanceof Error ? cause.message : String(cause) }, + "Ghidra cleanup after active request cancellation failed", + ); + return err(this.#cleanupError(cause)); + }, + ); } /** Observed failures that prevent a retained session from being reused. */ @@ -546,11 +603,18 @@ export class GhidraClient { } async #start(signal: AbortSignal): Promise { + if (this.#socket === undefined && this.#runtimeRoot === undefined) { + this.#lastDiagnostics = {}; + this.#processSnapshot = undefined; + this.#startupOutputCheckpoint = undefined; + this.#activeOutputCheckpoint = undefined; + this.#outputDrainPromise = undefined; + this.#outputDrainIncomplete = false; + } if (signal.aborted) return err(this.#failure("cancelled", "Ghidra startup was cancelled")); if (this.#socket !== undefined || this.#runtimeRoot !== undefined) return err(this.#failure("protocol", "Ghidra client is already started")); - this.#processSnapshot = undefined; const deadline = new ProviderStartupDeadline( this.#options.startupTimeoutMs, signal, @@ -596,7 +660,7 @@ export class GhidraClient { "Ghidra private endpoint allocation failed", cause, ); - return this.#startupFailure(failure); + return this.#startupFailure(failure, deadline); } try { const snapshot = await createGhidraTargetSnapshot( @@ -614,7 +678,7 @@ export class GhidraClient { "Ghidra target snapshot failed admission", cause, ); - return this.#startupFailure(failure); + return this.#startupFailure(failure, deadline); } if (deadline.signal.aborted) return this.#startupInterrupted(deadline); this.#token = randomBytes(32).toString("hex"); @@ -650,21 +714,24 @@ export class GhidraClient { maxDiagnosticBytes: GHIDRA_PROCESS_DIAGNOSTIC_BYTES, onDiagnostic: (event) => this.#onProcessDiagnostic(event), }); + this.#startupOutputCheckpoint = this.#process.checkpointOutput(); } const failure = deadline.signal.aborted ? this.#interruptionFailure(deadline, launched.error) : launched.error instanceof GhidraSessionError ? launched.error : this.#failure("start", launched.error.message, launched.error); - return this.#startupFailure(failure); + return this.#startupFailure(failure, deadline); } this.#launch = launched.value; this.#process = new ProviderProcessSupervisor(launched.value, { maxDiagnosticBytes: GHIDRA_PROCESS_DIAGNOSTIC_BYTES, onDiagnostic: (event) => this.#onProcessDiagnostic(event), }); + this.#startupOutputCheckpoint = this.#process.checkpointOutput(); const completed = await this.#connectAndHandshake(endpoint, deadline); - if (!completed.ok) return this.#startupFailure(completed.error); + if (!completed.ok) return this.#startupFailure(completed.error, deadline); + this.#startupOutputCheckpoint = undefined; await this.#lineage.observe(this.#launch); this.#requestQueue.reopen(); return completed; @@ -777,7 +844,7 @@ export class GhidraClient { async #cleanup(forceStop = false): Promise { this.#closing = true; - let forcedStopFailure: GhidraSessionError | undefined; + let forcedStopFailure: { readonly cause: unknown } | undefined; try { this.#requestQueue.failQueued( this.#failure("process", "Ghidra session closed"), @@ -843,12 +910,7 @@ export class GhidraClient { try { await this.#removeRuntimeRoots(); } catch (cause: unknown) { - if (forceStop) - forcedStopFailure ??= this.#failure( - "process", - "Ghidra runtime cleanup after cancellation failed", - cause, - ); + if (forceStop) forcedStopFailure = { cause }; else throw cause; } this.#snapshotPath = undefined; @@ -856,7 +918,7 @@ export class GhidraClient { this.#token = undefined; this.#runId = undefined; this.#responseBuffer.reset(); - if (forcedStopFailure !== undefined) throw forcedStopFailure; + if (forcedStopFailure !== undefined) throw forcedStopFailure.cause; } finally { this.#closing = false; } @@ -955,7 +1017,7 @@ export class GhidraClient { deadline: ProviderStartupDeadline, ): Promise { const failure = this.#interruptionFailure(deadline); - return this.#startupFailure(failure); + return this.#startupFailure(failure, deadline); } #interruptionFailure( @@ -970,7 +1032,29 @@ export class GhidraClient { } #diagnostics(): Readonly> { - const snapshot = this.#process?.snapshot() ?? this.#processSnapshot; + const currentSnapshot = this.#process?.snapshot() ?? this.#processSnapshot; + const snapshot: ProviderProcessSnapshot | undefined = + this.#outputDrainIncomplete && currentSnapshot !== undefined + ? { + ...currentSnapshot, + diagnosticTruncated: true, + outputIncomplete: true, + } + : currentSnapshot; + const checkpoint = + this.#activeOutputCheckpoint ?? this.#startupOutputCheckpoint; + const scoped = + checkpoint === undefined + ? undefined + : this.#process?.snapshotOutputSince(checkpoint); + const failureOutput: JsonValue | undefined = + scoped === undefined + ? undefined + : { + stdout: ghidraDiagnosticStream(scoped.snapshot.stdout, this.#token), + stderr: ghidraDiagnosticStream(scoped.snapshot.stderr, this.#token), + complete: scoped.complete && !this.#outputDrainIncomplete, + }; return createGhidraDiagnostics({ targetPath: this.#options.targetPath, targetSha256: this.#options.targetSha256, @@ -994,6 +1078,69 @@ export class GhidraClient { ...(snapshot === undefined ? {} : { snapshot }), ...(this.#token === undefined ? {} : { token: this.#token }), previous: this.#lastDiagnostics, + ...(failureOutput === undefined ? {} : { failure_output: failureOutput }), }); } + + #refreshProcessFailure( + failure: GhidraSessionError, + owner: ProviderProcessSupervisor | undefined, + checkpoint: ProviderProcessOutputCheckpoint | undefined, + token: string | undefined, + drainIncomplete: boolean, + ): GhidraSessionError { + if (owner === undefined || checkpoint === undefined) return failure; + const snapshot = owner.snapshot(); + const scoped = owner.snapshotOutputSince(checkpoint); + const incomplete = drainIncomplete || !scoped.complete; + return new GhidraSessionError( + failure.kind, + failure.message, + { + ...failure.diagnostics, + exit_code: snapshot.exitCode ?? null, + exit_signal: snapshot.signal ?? null, + diagnostic_truncated: + snapshot.diagnosticTruncated === true || incomplete, + output_incomplete: + snapshot.outputIncomplete === true || drainIncomplete, + stdout: ghidraDiagnosticStream(snapshot.stdout, token), + stderr: ghidraDiagnosticStream(snapshot.stderr, token), + failure_output: { + stdout: ghidraDiagnosticStream(scoped.snapshot.stdout, token), + stderr: ghidraDiagnosticStream(scoped.snapshot.stderr, token), + complete: scoped.complete && !drainIncomplete, + }, + }, + { + ...(failure.cause === undefined ? {} : { cause: failure.cause }), + ...(failure.timeoutMs === undefined + ? {} + : { timeoutMs: failure.timeoutMs }), + ...(failure.remoteCode === undefined + ? {} + : { remoteCode: failure.remoteCode }), + ...(failure.cleanupFailure === undefined + ? {} + : { cleanupFailure: failure.cleanupFailure }), + }, + ); + } + + #drainOutputBounded(signal?: AbortSignal): Promise { + if (this.#outputDrainPromise !== undefined) return this.#outputDrainPromise; + const process = this.#process; + if (process === undefined) return Promise.resolve(true); + const pending = process.waitForOutputClose(1_000, signal).then((closed) => { + if (this.#process !== process) return; + if (!closed) this.#outputDrainIncomplete = true; + const snapshot = process.snapshot(); + this.#processSnapshot = this.#outputDrainIncomplete + ? { ...snapshot, diagnosticTruncated: true, outputIncomplete: true } + : snapshot; + return closed; + }); + this.#outputDrainPromise = pending.then((closed) => closed ?? false); + return this.#outputDrainPromise; + } } diff --git a/src/ghidra/GhidraDiagnostics.test.ts b/src/ghidra/GhidraDiagnostics.test.ts new file mode 100644 index 000000000..be17253eb --- /dev/null +++ b/src/ghidra/GhidraDiagnostics.test.ts @@ -0,0 +1,23 @@ +import { expect, it } from "vitest"; +import { createGhidraDiagnostics } from "./GhidraDiagnostics.js"; + +it("does not reuse historical operation-scoped failure output without a current owner", () => { + const diagnostics = createGhidraDiagnostics({ + targetPath: "/tmp/target", + targetSha256: "a".repeat(64), + transport: "unix-socket", + providerVersion: "11.4", + profileDigest: "b".repeat(64), + previous: { + stdout: { text: "historical OOME" }, + failure_output: { + stdout: { text: "historical OOME" }, + stderr: { text: "" }, + complete: true, + }, + }, + }); + + expect(diagnostics.stdout).toEqual({ text: "historical OOME" }); + expect(diagnostics).not.toHaveProperty("failure_output"); +}); diff --git a/src/ghidra/GhidraDiagnostics.ts b/src/ghidra/GhidraDiagnostics.ts index 8ec3de36d..3dddd737c 100644 --- a/src/ghidra/GhidraDiagnostics.ts +++ b/src/ghidra/GhidraDiagnostics.ts @@ -23,6 +23,7 @@ export interface GhidraDiagnosticsOptions { readonly snapshot?: ProviderProcessSnapshot; readonly token?: string; readonly previous: Readonly>; + readonly failure_output?: JsonValue; } /** Preserve actionable runtime coordinates after cleanup and redact bearer data. */ @@ -33,8 +34,11 @@ export const createGhidraDiagnostics = ( options.runtimeRoot === undefined && options.launch === undefined && Object.keys(options.previous).length > 0 - ) - return options.previous; + ) { + const history: Record = { ...options.previous }; + delete history.failure_output; + return history; + } return { target_path: options.targetPath, target_sha256: options.targetSha256, @@ -69,19 +73,32 @@ export const createGhidraDiagnostics = ( })), }), }), + ...(options.launch?.configuredMaxHeap === undefined + ? {} + : { configured_jvm_max_heap: options.launch.configuredMaxHeap }), ...(options.snapshot === undefined ? {} : { exit_code: options.snapshot.exitCode ?? null, exit_signal: options.snapshot.signal ?? null, diagnostic_truncated: options.snapshot.diagnosticTruncated === true, - stdout: boundedStream(options.snapshot.stdout, options.token), - stderr: boundedStream(options.snapshot.stderr, options.token), + output_incomplete: options.snapshot.outputIncomplete === true, + stdout: ghidraDiagnosticStream( + options.snapshot.stdout, + options.token, + ), + stderr: ghidraDiagnosticStream( + options.snapshot.stderr, + options.token, + ), }), + ...(options.failure_output === undefined + ? {} + : { failure_output: options.failure_output }), }; }; -const boundedStream = ( +export const ghidraDiagnosticStream = ( stream: ProviderProcessSnapshot["stdout"], token: string | undefined, ): JsonValue => ({ @@ -89,4 +106,5 @@ const boundedStream = ( // Preserve the historical total-byte counter; text is now bounded separately. bytes: stream.observedBytes, retained_bytes: stream.bytes, + ...(stream.invalidUtf8 === true ? { invalid_utf8: true } : {}), }); diff --git a/src/ghidra/GhidraJavaLaunch.ts b/src/ghidra/GhidraJavaLaunch.ts index 598a152c2..94639d015 100644 --- a/src/ghidra/GhidraJavaLaunch.ts +++ b/src/ghidra/GhidraJavaLaunch.ts @@ -83,6 +83,8 @@ export const ghidraJavaLaunch = async (options: { readonly command: string; readonly arguments: readonly string[]; readonly environment: NodeJS.ProcessEnv; + /** REA-configured -Xmx value, not effective or observed JVM memory. */ + readonly configuredMaxHeap?: string; }> => { const supportRoot = dirname(options.analyzeHeadlessPath); const installRoot = dirname(supportRoot); @@ -146,6 +148,8 @@ export const ghidraJavaLaunch = async (options: { if ((environment[key] ?? "") === "") environment[key] = line.slice(separator + 1); } + const configuredMaxHeap = + environment.GHIDRA_HEADLESS_MAXMEM || environment.GHIDRA_MAXMEM || "2G"; // The merged environment becomes the analysis JVM's environment, and the // JVM applies these options variables automatically. An injected // -Dos.arch/-Dos.name there selects an unvalidated native directory without @@ -172,10 +176,11 @@ export const ghidraJavaLaunch = async (options: { return { command, environment, + configuredMaxHeap, arguments: [ ...vmArguments, ...(options.platform === "darwin" ? ["-Xdock:name=Ghidra-Headless"] : []), - `-Xmx${environment.GHIDRA_HEADLESS_MAXMEM || environment.GHIDRA_MAXMEM || "2G"}`, + `-Xmx${configuredMaxHeap}`, "-XX:ParallelGCThreads=2", "-XX:CICompilerCount=2", "-Djava.awt.headless=true", diff --git a/src/ghidra/GhidraLauncher.ts b/src/ghidra/GhidraLauncher.ts index c64ba1a7e..fea00e270 100644 --- a/src/ghidra/GhidraLauncher.ts +++ b/src/ghidra/GhidraLauncher.ts @@ -65,6 +65,8 @@ export type GhidraLaunch = ProviderProcessLaunch & { readonly projectRoot: string; readonly ghidraLogPath: string; readonly scriptLogPath: string; + /** Value supplied to the JVM as -Xmx, not a measurement of effective memory. */ + readonly configuredMaxHeap?: string; readonly launchSupport?: readonly GhidraLaunchSupportDiagnostic[]; }; @@ -195,6 +197,7 @@ export class GhidraHeadlessLauncher implements GhidraLauncher { const paths = ghidraRuntimePaths(session.runtimeRoot); const platform = this.options.platform ?? process.platform; let started: SpawnedOwnedProviderProcess | undefined; + let configuredMaxHeap: string | undefined; const launchSupport: GhidraLaunchSupportDiagnostic[] = []; try { const psp = this.options.pspExtension; @@ -341,6 +344,11 @@ export class GhidraHeadlessLauncher implements GhidraLauncher { }, ...(options.signal === undefined ? {} : { signal: options.signal }), }); + // Windows starts the script wrapper rather than this inspected JVM command. + configuredMaxHeap = + platform === "win32" + ? undefined + : inspectedJavaLaunch.configuredMaxHeap; const command = platform === "win32" ? { ...scriptCommand, environment } @@ -363,6 +371,7 @@ export class GhidraHeadlessLauncher implements GhidraLauncher { paths, platform, launchSupport, + configuredMaxHeap, ); await writeGhidraRuntimeFile( paths.ownershipPath, @@ -416,6 +425,7 @@ export class GhidraHeadlessLauncher implements GhidraLauncher { paths, platform, launchSupport, + configuredMaxHeap, ); const primaryMessage = isAborted(options.signal) ? primary.message @@ -446,11 +456,13 @@ const ownedGhidraLaunch = ( paths: ReturnType, platform: NodeJS.Platform, launchSupport: readonly GhidraLaunchSupportDiagnostic[], + configuredMaxHeap?: string, ): GhidraLaunch => ({ ...ownedGhidraProcess(spawned, platform), projectRoot: paths.projectRoot, ghidraLogPath: paths.ghidraLogPath, scriptLogPath: paths.scriptLogPath, + ...(configuredMaxHeap === undefined ? {} : { configuredMaxHeap }), ...(launchSupport.length === 0 ? {} : { launchSupport: [...launchSupport] }), }); diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index 73c7524b7..ab707d20f 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -17,7 +17,7 @@ import { nativeAotPeDigest, nativeAotPeFixture, } from "../../tests/fixtures/nativeaotPe.js"; -import { jsonValueSchema } from "../domain/jsonValue.js"; +import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; import { parseConfig } from "../config/parseConfig.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; @@ -2743,3 +2743,203 @@ describe("Ghidra queries after the owned provider is lost", () => { ); }); }); + +const output = (text: string) => ({ + text, + bytes: Buffer.byteLength(text), + retained_bytes: Buffer.byteLength(text), +}); +const failureOutput = (stdout = "", stderr = "") => ({ + stdout: output(stdout), + stderr: output(stderr), + complete: true, +}); +const projectFailure = async ( + kind: "cancelled" | "process" | "protocol" | "remote" | "timeout", + diagnostics: Readonly>, +) => { + const ghidra = provider(installationHost(), () => ({ + start: async () => + err(new GhidraSessionError("start", "startup is not exercised")), + callTool: async () => + err( + new GhidraSessionError( + kind, + "Ghidra request failed", + { failure_kind: kind, ...diagnostics }, + kind === "remote" ? { remoteCode: "internal" } : {}, + ), + ), + close: async () => ok(null), + })); + const target = executableTarget("elf", "x86_64"); + const profile = await ghidra.resolveAnalysisProfile(target); + if (!profile.ok || profile.value.profile === null) + throw new Error("Expected Ghidra analysis profile"); + const result = await ghidra + .createClient(target, profile.value.profile) + .execute("list_documents", {}); + if (result.ok) throw new Error("Expected a Ghidra failure"); + return projectAnalysisError(result.error); +}; + +const issueOutput = [ + "INFO ANALYZING all memory and code: /hugo (HeadlessAnalyzer)", + "ERROR Abort due to Headless analyzer error: Java heap space (HeadlessAnalyzer) java.lang.OutOfMemoryError:", + "\tat ghidra.app.plugin.core.analysis.GolangSymbolAnalyzer$PropagateRttiBackgroundCommand.applyTo", +].join("\n"); + +describe("Ghidra OOME error projection", () => { + it("recognizes the headless analyzer's real detail-before-class output", async () => { + const projected = await projectFailure("process", { + stdout: output(issueOutput), + stderr: output(""), + failure_output: failureOutput(issueOutput), + configured_jvm_max_heap: "4G", + }); + expect(projected).toMatchObject({ + code: "resource_constraint", + details: { + resource: "memory", + reported_limits: { configured_jvm_max_heap: "4G" }, + }, + }); + expect(projected.message).toContain("Java heap space"); + expect(projected.remediation.action).toContain("GHIDRA_HEADLESS_MAXMEM"); + expect(projected.remediation.action).toContain( + "REA_GHIDRA_DISABLED_ANALYZERS", + ); + }); + + it("retains an unknown heap for GC overhead when the launch did not report one", async () => { + const projected = await projectFailure("timeout", { + stdout: output(""), + stderr: output(""), + failure_output: failureOutput( + "", + 'Exception in thread "main" java.lang.OutOfMemoryError: GC overhead limit exceeded', + ), + }); + expect(projected).toMatchObject({ + code: "resource_constraint", + details: { + resource: "memory", + reported_limits: { configured_jvm_max_heap: null }, + }, + }); + expect(projected.remediation.action).toContain("GHIDRA_HEADLESS_MAXMEM"); + }); + + it.each([ + ["Metaspace", "java.lang.OutOfMemoryError: Metaspace"], + [ + "array VM limit", + "java.lang.OutOfMemoryError: Requested array size exceeds VM limit", + ], + ])("does not recommend a larger Java heap for %s", async (_name, text) => { + const projected = await projectFailure("process", { + stdout: output(""), + stderr: output(""), + failure_output: failureOutput("", text), + configured_jvm_max_heap: "4G", + }); + expect(projected.code).toBe("resource_constraint"); + expect(projected.message).toContain(text); + expect(projected.remediation.action).not.toContain( + "GHIDRA_HEADLESS_MAXMEM", + ); + }); + + it("reports native-thread exhaustion as capacity without inventing a limit", async () => { + const projected = await projectFailure("process", { + stdout: output(""), + stderr: output(""), + failure_output: failureOutput( + "", + "java.lang.OutOfMemoryError: unable to create native thread: possibly out of memory or process/resource limits reached", + ), + configured_jvm_max_heap: "4G", + }); + expect(projected).toMatchObject({ + code: "resource_constraint", + details: { resource: "capacity", reported_limits: null }, + }); + expect(projected.remediation.action).not.toContain( + "GHIDRA_HEADLESS_MAXMEM", + ); + }); + + it("keeps a bare OOME class at unknown subtype without heap advice", async () => { + const projected = await projectFailure("process", { + failure_output: failureOutput("", "java.lang.OutOfMemoryError"), + }); + expect(projected).toMatchObject({ + code: "resource_constraint", + details: { resource: "memory", reported_limits: null }, + }); + expect(projected.remediation.action).not.toContain( + "GHIDRA_HEADLESS_MAXMEM", + ); + }); + + it("does not reclassify a later failure from OOME retained after an earlier success", async () => { + let calls = 0; + const ghidra = provider(installationHost(), () => ({ + start: async () => + err(new GhidraSessionError("start", "startup is not exercised")), + callTool: async () => { + calls += 1; + if (calls === 1) return ok(["fixture"]); + return err( + new GhidraSessionError( + "protocol", + "Ghidra returned an invalid later response", + { + stdout: output(issueOutput), + stderr: output(""), + // The OOME remains visible in full process history above, but + // the current operation produced no such error. + failure_output: failureOutput(), + }, + ), + ); + }, + close: async () => ok(null), + })); + const target = executableTarget("elf", "x86_64"); + const profile = await ghidra.resolveAnalysisProfile(target); + if (!profile.ok || profile.value.profile === null) + throw new Error("Expected Ghidra analysis profile"); + const client = ghidra.createClient(target, profile.value.profile); + const earlier = await client.execute("list_documents", {}); + expect(earlier.ok).toBe(true); + const later = await client.execute("list_procedures", {}); + expect(later.ok).toBe(false); + if (later.ok) return; + const projected = projectAnalysisError(later.error); + expect(projected.code).toBe("execution_failure"); + expect(projected.details?.diagnostics).toMatchObject({ + stdout: { text: issueOutput }, + }); + }); + + it("keeps cancellation primary even when the current operation reports an OOME", async () => { + const projected = await projectFailure("cancelled", { + stdout: output(""), + stderr: output(""), + failure_output: failureOutput(issueOutput), + configured_jvm_max_heap: "4G", + }); + expect(projected.code).toBe("cancelled"); + }); + + it("does not reinterpret a bridge-reported OOME as a local process failure", async () => { + const projected = await projectFailure("remote", { + stdout: output(""), + stderr: output(""), + failure_output: failureOutput(issueOutput), + configured_jvm_max_heap: "4G", + }); + expect(projected.code).not.toBe("resource_constraint"); + }); +}); diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index a57e23386..2f6ce6d13 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -1092,7 +1092,7 @@ const projectPrimarySessionError = ( (failure.kind === "remote" && failure.remoteCode === "decompile_cancelled") ) return new AnalysisCancelledError(operation, context); - if (failure.kind === "timeout" || failure.kind === "analysis_timeout") + if (failure.kind === "analysis_timeout") return new AnalysisTimeoutError( operation, failure.timeoutMs ?? startupTimeoutMs, @@ -1129,6 +1129,18 @@ const projectPrimarySessionError = ( failure.message, context, ); + const memoryFailure = projectObservedJvmMemoryFailure( + operation, + failure, + context, + ); + if (memoryFailure !== undefined) return memoryFailure; + if (failure.kind === "timeout") + return new AnalysisTimeoutError( + operation, + failure.timeoutMs ?? startupTimeoutMs, + context, + ); const exited = exitedProviderMessage(operation, failure, health); if (exited !== undefined) return new ProviderAdapterError("ghidra", operation, { @@ -1138,6 +1150,90 @@ const projectPrimarySessionError = ( return new ProviderAdapterError("ghidra", operation, context); }; +/** Project only OOME evidence captured during the operation that failed. */ +const projectObservedJvmMemoryFailure = ( + operation: AnalysisOperation, + failure: GhidraSessionError, + context: { + readonly cause: GhidraSessionError; + readonly diagnostics: Readonly>; + }, +): AnalysisResourceConstraintError | undefined => { + if ( + failure.kind !== "start" && + failure.kind !== "process" && + failure.kind !== "protocol" && + failure.kind !== "timeout" + ) + return undefined; + const scoped = failure.diagnostics.failure_output; + if ( + scoped === null || + typeof scoped !== "object" || + Array.isArray(scoped) || + typeof scoped.stdout !== "object" || + scoped.stdout === null || + typeof scoped.stderr !== "object" || + scoped.stderr === null + ) + return undefined; + const streamText = (value: JsonValue): string => + typeof value === "object" && + value !== null && + !Array.isArray(value) && + typeof value.text === "string" + ? value.text + : ""; + const output = `${streamText(scoped.stdout)}\n${streamText(scoped.stderr)}`; + if (!/java\.lang\.OutOfMemoryError(?:\s*:|\b)/u.test(output)) + return undefined; + const threadCapacity = /unable to create native thread/iu.test(output); + const heapExhaustion = /Java heap space|GC overhead limit exceeded/iu.test( + output, + ); + const detail = [ + "Java heap space", + "GC overhead limit exceeded", + "Metaspace", + "Requested array size exceeds VM limit", + "unable to create native thread", + ].find((candidate) => output.includes(candidate)); + if (threadCapacity) + return new AnalysisResourceConstraintError( + operation, + "capacity", + "Ghidra's process output reports an OutOfMemoryError while creating a native thread. This is observed capacity evidence; it does not establish a process limit or the precise cause.", + null, + { + ...context, + remediationAction: + "Reduce concurrent analysis processes or review the host's available thread and process capacity, then retry.", + }, + ); + const configuredHeap = failure.diagnostics.configured_jvm_max_heap; + const reportedLimits = heapExhaustion + ? { + configured_jvm_max_heap: + typeof configuredHeap === "string" ? configuredHeap : null, + } + : null; + return new AnalysisResourceConstraintError( + operation, + "memory", + `Ghidra's operation-scoped process output reports java.lang.OutOfMemoryError${detail === undefined ? "" : `: ${detail}`}. This is reported evidence and does not prove the exact allocation failure.`, + reportedLimits, + { + ...context, + ...(heapExhaustion + ? { + remediationAction: + "If the host has available memory, consider increasing GHIDRA_HEADLESS_MAXMEM or disabling expensive analyzers with REA_GHIDRA_DISABLED_ANALYZERS, then retry.", + } + : {}), + }, + ); +}; + /** A provider already observed as running can be restarted by reopening. */ const exitedProviderMessage = ( operation: AnalysisOperation, diff --git a/src/ghidra/GhidraRequestQueue.test.ts b/src/ghidra/GhidraRequestQueue.test.ts index e5dd0f012..c529295e7 100644 --- a/src/ghidra/GhidraRequestQueue.test.ts +++ b/src/ghidra/GhidraRequestQueue.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { err, ok } from "../domain/result.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { GhidraRequestQueue } from "./GhidraRequestQueue.js"; import { bindGhidraSessionFailure } from "./GhidraSessionError.js"; @@ -20,7 +21,7 @@ describe("Ghidra request queue failure settlement", () => { return ok({ method }); }, failure, - () => Promise.resolve(), + () => Promise.resolve(ok(null)), ); const first = queue.run("first", {}, {}); const second = queue.run("second", {}, {}); @@ -46,16 +47,24 @@ describe("Ghidra request queue failure settlement", () => { const queue = new GhidraRequestQueue( () => Promise.resolve(err(original)), failure, - () => Promise.resolve(), + () => Promise.resolve(ok(null)), ); const result = await queue.run("first", {}, {}); expect(result).toEqual(err(original)); if (!result.ok) expect(result.error).toBe(original); }); +}); +describe("Ghidra request queue cancellation", () => { it("preserves cancellation cleanup failures and rejects queued work", async () => { const controller = new AbortController(); const cause = new Error("Runtime removal failed"); + const cleanupFailure = new ProviderCleanupError( + "ghidra", + ["/tmp/runtime"], + { reason: cause.message }, + { cause }, + ); let release: (() => void) | undefined; const completed = new Promise((resolve) => { release = resolve; @@ -66,9 +75,7 @@ describe("Ghidra request queue failure settlement", () => { return err(failure("cancelled", "Request cancelled")); }, failure, - async () => { - throw cause; - }, + () => Promise.resolve(err(cleanupFailure)), ); const active = queue.run("active", {}, { signal: controller.signal }); const queued = queue.run("queued", {}, {}); @@ -78,15 +85,88 @@ describe("Ghidra request queue failure settlement", () => { expect(result).toMatchObject({ ok: false, error: { - kind: "process", - diagnostics: { failure_cause: { message: cause.message } }, + kind: "cancelled", + message: "Request cancelled", }, }); - if (result.ok) throw new Error("Expected cleanup failure"); - expect(result.error.cause).toBe(cause); + if (result.ok) throw new Error("Expected cancellation"); + expect(result.error.cleanupFailure).toBe(cleanupFailure); await expect(queued).resolves.toMatchObject({ ok: false, error: { kind: "process" }, }); }); + + it.each([ + { + cause: new Error("Unexpected cleanup rejection"), + diagnostic: { message: "Unexpected cleanup rejection" }, + }, + { cause: undefined, diagnostic: { type: "undefined" } }, + ])( + "keeps cancellation primary when cleanup rejects with $diagnostic", + async ({ cause, diagnostic }) => { + const controller = new AbortController(); + const cancellationCause = new Error("Caller cancelled"); + const original = failure( + "cancelled", + "Request cancelled", + cancellationCause, + ); + let release: (() => void) | undefined; + const completed = new Promise((resolve) => { + release = resolve; + }); + const queue = new GhidraRequestQueue( + async () => { + await completed; + return err(original); + }, + failure, + async () => { + throw cause; + }, + ); + const active = queue.run("active", {}, { signal: controller.signal }); + controller.abort(); + release?.(); + + const result = await active; + expect(result).toMatchObject({ + ok: false, + error: { + kind: "cancelled", + message: "Request cancelled", + diagnostics: { + unexpected_cleanup_rejection: diagnostic, + }, + }, + }); + if (result.ok) throw new Error("Expected cancellation"); + expect(result.error.cause).toBe(cancellationCause); + expect(result.error.cleanupFailure).toBeUndefined(); + }, + ); + + it("preserves the cancellation result when cleanup succeeds", async () => { + const controller = new AbortController(); + const original = failure("cancelled", "Request cancelled"); + let release: (() => void) | undefined; + const completed = new Promise((resolve) => { + release = resolve; + }); + const queue = new GhidraRequestQueue( + async () => { + await completed; + return err(original); + }, + failure, + () => Promise.resolve(ok(null)), + ); + const active = queue.run("active", {}, { signal: controller.signal }); + controller.abort(); + release?.(); + + await expect(active).resolves.toEqual(err(original)); + }); }); diff --git a/src/ghidra/GhidraRequestQueue.ts b/src/ghidra/GhidraRequestQueue.ts index c114341ce..e3072dab2 100644 --- a/src/ghidra/GhidraRequestQueue.ts +++ b/src/ghidra/GhidraRequestQueue.ts @@ -1,9 +1,10 @@ +import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { err, type Result } from "../domain/result.js"; import type { GhidraRequestOptions } from "./GhidraClientTypes.js"; -import type { +import { GhidraSessionError, - GhidraSessionFailureKind, + type GhidraSessionFailureKind, } from "./GhidraSessionError.js"; /** Result settled by one queued Ghidra Program request. */ @@ -22,7 +23,9 @@ export type GhidraQueueFailureFactory = ( ) => GhidraSessionError; /** Stop the provider session when an admitted operation is cancelled. */ -export type GhidraActiveCancellationHandler = () => Promise; +export type GhidraActiveCancellationHandler = () => Promise< + Result +>; interface QueuedRequest { readonly method: string; @@ -108,7 +111,7 @@ export class GhidraRequestQueue { } this.#active = true; let executionSettled = false; - let activeCancellation: Promise | undefined; + let activeCancellation: Promise> | undefined; const onAbort = entry.signal === undefined ? undefined @@ -131,18 +134,15 @@ export class GhidraRequestQueue { ) .then(async (result) => { if (activeCancellation !== undefined) { + const primary = result.ok + ? this.failure("cancelled", "Ghidra request was cancelled") + : result.error; try { - await activeCancellation; + const cleanup = await activeCancellation; + if (!cleanup.ok) + result = err(this.#withCleanupFailure(primary, cleanup.error)); } catch (cause: unknown) { - result = err( - this.failure( - "process", - cause instanceof Error - ? `Ghidra cleanup after cancellation failed: ${cause.message}` - : "Ghidra cleanup after cancellation failed", - cause, - ), - ); + result = err(this.#withUnexpectedCleanupRejection(primary, cause)); } } executionSettled = true; @@ -176,6 +176,52 @@ export class GhidraRequestQueue { entry.resolve(result); } + #withCleanupFailure( + primary: GhidraSessionError, + cleanupFailure: AnalysisError, + ): GhidraSessionError { + return new GhidraSessionError( + primary.kind, + primary.message, + primary.diagnostics, + { + ...(primary.cause === undefined ? {} : { cause: primary.cause }), + ...(primary.timeoutMs === undefined + ? {} + : { timeoutMs: primary.timeoutMs }), + ...(primary.remoteCode === undefined + ? {} + : { remoteCode: primary.remoteCode }), + cleanupFailure, + }, + ); + } + + #withUnexpectedCleanupRejection( + primary: GhidraSessionError, + cause: unknown, + ): GhidraSessionError { + const described = this.failure(primary.kind, primary.message, cause); + const rejection = described.diagnostics.failure_cause; + return new GhidraSessionError( + primary.kind, + primary.message, + { + ...primary.diagnostics, + unexpected_cleanup_rejection: rejection ?? { type: typeof cause }, + }, + { + ...(primary.cause === undefined ? {} : { cause: primary.cause }), + ...(primary.timeoutMs === undefined + ? {} + : { timeoutMs: primary.timeoutMs }), + ...(primary.remoteCode === undefined + ? {} + : { remoteCode: primary.remoteCode }), + }, + ); + } + #release(entry: QueuedRequest): void { if (entry.signal !== undefined && entry.onAbort !== undefined) entry.signal.removeEventListener("abort", entry.onAbort); diff --git a/src/process/ProviderProcess.ts b/src/process/ProviderProcess.ts index 17eda48d7..6f42dab6f 100644 --- a/src/process/ProviderProcess.ts +++ b/src/process/ProviderProcess.ts @@ -126,6 +126,28 @@ export interface ProviderProcessSnapshot { readonly outputIncomplete?: true; } +declare const processOutputCheckpointBrand: unique symbol; +/** Opaque cursor into one supervisor's raw, byte-counted output history. */ +export interface ProviderProcessOutputCheckpoint { + readonly [processOutputCheckpointBrand]: true; +} + +/** Output produced after a checkpoint; incomplete when bytes were not retained or drained. */ +export interface ProviderProcessOutputSince { + readonly snapshot: ProviderProcessSnapshot; + readonly complete: boolean; +} + +const outputCheckpoints = new WeakMap< + ProviderProcessOutputCheckpoint, + { + readonly owner: ProviderProcessSupervisor; + readonly epoch: number; + readonly stdout: number; + readonly stderr: number; + } +>(); + /** Resource-safe events emitted without imposing provider protocol semantics. */ export type ProviderProcessDiagnostic = | { @@ -313,6 +335,7 @@ export class ProviderProcessSupervisor { #stopPromise: Promise | undefined; #disposed = false; #maxDiagnosticBytes: number | undefined; + #outputEpoch = 0; constructor( readonly launch: ProviderProcessLaunch, @@ -358,8 +381,75 @@ export class ProviderProcessSupervisor { }; } + /** Mark the current raw-byte position without copying or retaining output. */ + checkpointOutput(): ProviderProcessOutputCheckpoint { + const checkpoint = Object.freeze({}) as ProviderProcessOutputCheckpoint; + outputCheckpoints.set(checkpoint, { + owner: this, + epoch: this.#outputEpoch, + stdout: this.#stdoutObservedBytes, + stderr: this.#stderrObservedBytes, + }); + return checkpoint; + } + + /** Snapshot only bytes produced after a valid checkpoint on this supervisor. */ + snapshotOutputSince( + checkpoint: ProviderProcessOutputCheckpoint, + ): ProviderProcessOutputSince { + const position = + typeof checkpoint === "object" && checkpoint !== null + ? outputCheckpoints.get(checkpoint) + : undefined; + if ( + position === undefined || + position.owner !== this || + position.epoch !== this.#outputEpoch + ) + return { + snapshot: { + stdout: { text: "", bytes: 0, observedBytes: 0 }, + stderr: { text: "", bytes: 0, observedBytes: 0 }, + exitCode: this.#exitObservation?.code, + signal: this.#exitObservation?.signal, + outputIncomplete: true, + diagnosticTruncated: true, + }, + complete: false, + }; + const stdout = this.#stdout.snapshotRange( + position.stdout, + this.#stdoutObservedBytes, + ); + const stderr = this.#stderr.snapshotRange( + position.stderr, + this.#stderrObservedBytes, + ); + const incomplete = + stdout.complete === false || + stderr.complete === false || + this.launch.process.outputIncomplete === true; + return { + snapshot: { + ...(incomplete || this.#diagnosticTruncated + ? { diagnosticTruncated: true } + : {}), + stdout: stdout.stream, + stderr: stderr.stream, + exitCode: this.#exitObservation?.code, + signal: this.#exitObservation?.signal, + ...(this.launch.process.memoryLimitReached === true + ? { memoryLimitReached: true } + : {}), + ...(incomplete ? { outputIncomplete: true } : {}), + }, + complete: !incomplete && !this.#diagnosticTruncated, + }; + } + /** Begin a new observation on a retained process without retaining earlier output. */ resetOutput(): void { + this.#outputEpoch += 1; this.#stdout.reset(); this.#stderr.reset(); this.#stdoutObservedBytes = 0; @@ -377,11 +467,15 @@ export class ProviderProcessSupervisor { } /** Wait until output streams close; WindowsOwnedProcess closes only after its whole Job Object settles. */ - async waitForOutputClose(timeoutMs: number): Promise { + async waitForOutputClose( + timeoutMs: number, + signal?: AbortSignal, + ): Promise { return waitForProcessEvent( this.#outputClosed, this.#outputClose.promise, timeoutMs, + signal, ); } @@ -627,7 +721,39 @@ class ProcessOutputCapture { } snapshot(observedBytes: number): ProviderProcessSnapshot["stdout"] { - const buffer = Buffer.concat(this.#chunks, this.#bytes); + return this.#snapshotBuffer( + Buffer.concat(this.#chunks, this.#bytes), + observedBytes, + ); + } + + snapshotRange( + start: number, + end: number, + ): { stream: ProviderProcessSnapshot["stdout"]; complete: boolean } { + const availableEnd = Math.min(end, this.#bytes); + const availableStart = Math.min(start, availableEnd); + const chunks: Buffer[] = []; + let offset = 0; + for (const chunk of this.#chunks) { + const chunkEnd = offset + chunk.byteLength; + const from = Math.max(availableStart, offset); + const to = Math.min(availableEnd, chunkEnd); + if (to > from) chunks.push(chunk.subarray(from - offset, to - offset)); + offset = chunkEnd; + if (offset >= availableEnd) break; + } + const buffer = Buffer.concat(chunks, availableEnd - availableStart); + return { + stream: this.#snapshotBuffer(buffer, Math.max(0, end - start)), + complete: start <= this.#bytes && end <= this.#bytes, + }; + } + + #snapshotBuffer( + buffer: Buffer, + observedBytes: number, + ): ProviderProcessSnapshot["stdout"] { const text = buffer.toString("utf8"); let invalidUtf8 = false; try { @@ -637,7 +763,7 @@ class ProcessOutputCapture { } return { text, - bytes: this.#bytes, + bytes: buffer.byteLength, observedBytes, ...(invalidUtf8 ? { invalidUtf8: true } : {}), }; @@ -680,17 +806,25 @@ const waitForProcessEvent = async ( observed: boolean, event: Promise, timeoutMs: number, + signal?: AbortSignal, ): Promise => { if (observed) return true; + if (signal?.aborted === true) return false; let timer: NodeJS.Timeout | undefined; + let onAbort: (() => void) | undefined; try { return await Promise.race([ event.then(() => true), new Promise((resolve) => { timer = setTimeout(() => resolve(false), Math.max(0, timeoutMs)); + if (signal !== undefined) { + onAbort = () => resolve(false); + signal.addEventListener("abort", onAbort, { once: true }); + } }), ]); } finally { if (timer !== undefined) clearTimeout(timer); + if (onAbort !== undefined) signal?.removeEventListener("abort", onAbort); } }; diff --git a/src/process/ProviderProcessOutput.test.ts b/src/process/ProviderProcessOutput.test.ts index 993386e79..c11b93ce9 100644 --- a/src/process/ProviderProcessOutput.test.ts +++ b/src/process/ProviderProcessOutput.test.ts @@ -2,6 +2,7 @@ import { EventEmitter } from "node:events"; import { PassThrough } from "node:stream"; import { expect, it, vi } from "vitest"; import { ProviderProcessSupervisor } from "./ProviderProcess.js"; +import { redactCapturedTransportCredential } from "./ProviderDiagnosticRedaction.js"; it("exit does not establish complete output when a producer closes its streams later", async () => { const producer = Object.assign(new EventEmitter(), { @@ -66,6 +67,101 @@ it("reads an observed exit without materializing stream diagnostics", () => { } }); +it("scopes output checkpoints to raw bytes on one supervisor and epoch", () => { + const first = Object.assign(new EventEmitter(), { + stdout: new PassThrough(), + stderr: new PassThrough(), + exitCode: null, + signalCode: null, + kill: () => false, + }); + const second = Object.assign(new EventEmitter(), { + stdout: new PassThrough(), + stderr: new PassThrough(), + exitCode: null, + signalCode: null, + kill: () => false, + }); + const owner = new ProviderProcessSupervisor( + { process: first, ownsProcessLifetime: false }, + { maxDiagnosticBytes: 128 }, + ); + const other = new ProviderProcessSupervisor({ + process: second, + ownsProcessLifetime: false, + }); + try { + first.stdout.write(Buffer.from("prior: 🧭\n", "utf8")); + const checkpoint = owner.checkpointOutput(); + first.stdout.write(Buffer.from("current: é\n", "utf8")); + first.stderr.write(Buffer.from([0xff, 0x61])); + const scoped = owner.snapshotOutputSince(checkpoint); + expect(scoped.complete).toBe(true); + expect(scoped.snapshot.stdout.text).toBe("current: é\n"); + expect(scoped.snapshot.stdout.bytes).toBe( + Buffer.byteLength("current: é\n"), + ); + expect(scoped.snapshot.stderr).toEqual({ + text: "\ufffda", + bytes: 2, + observedBytes: 2, + invalidUtf8: true, + }); + expect(other.snapshotOutputSince(checkpoint).complete).toBe(false); + owner.resetOutput(); + expect(owner.snapshotOutputSince(checkpoint).complete).toBe(false); + } finally { + owner.dispose(); + other.dispose(); + first.stdout.destroy(); + first.stderr.destroy(); + second.stdout.destroy(); + second.stderr.destroy(); + } +}); + +it.each(["stdout", "stderr"] as const)( + "redacts a budget-cut credential in scoped %s after prior output", + (stream) => { + const producer = Object.assign(new EventEmitter(), { + stdout: new PassThrough(), + stderr: new PassThrough(), + exitCode: null, + signalCode: null, + kill: () => false, + }); + const supervisor = new ProviderProcessSupervisor( + { process: producer, ownsProcessLifetime: false }, + { maxDiagnosticBytes: 40 }, + ); + try { + producer.stdout.write("p".repeat(32)); + const checkpoint = supervisor.checkpointOutput(); + const token = "private-transport-token"; + producer[stream].write(token); + const scoped = supervisor.snapshotOutputSince(checkpoint); + expect(scoped.complete).toBe(false); + expect(scoped.snapshot.outputIncomplete).toBe(true); + expect(scoped.snapshot[stream]).toMatchObject({ + bytes: 8, + observedBytes: Buffer.byteLength(token), + text: token.slice(0, 8), + }); + expect( + redactCapturedTransportCredential( + scoped.snapshot[stream], + token, + "[REDACTED]", + ), + ).toBe("[REDACTED]"); + } finally { + supervisor.dispose(); + producer.stdout.destroy(); + producer.stderr.destroy(); + } + }, +); + it("observes an exit that completed before supervision", () => { const producer = Object.assign(new EventEmitter(), { stdout: new PassThrough(), diff --git a/tests/boundary/mcp/ghidraFailureDiagnostics.test.ts b/tests/boundary/mcp/ghidraFailureDiagnostics.test.ts index d6d5eeb51..24b728397 100644 --- a/tests/boundary/mcp/ghidraFailureDiagnostics.test.ts +++ b/tests/boundary/mcp/ghidraFailureDiagnostics.test.ts @@ -3,6 +3,7 @@ import { expect, it } from "vitest"; import { logCliCommand } from "../../../src/cliLogging.js"; import { analysisErrorProjectionSchema } from "../../../src/contracts/errorSchemas.js"; +import { ok } from "../../../src/domain/result.js"; import { GhidraRequestQueue } from "../../../src/ghidra/GhidraRequestQueue.js"; import { bindGhidraSessionFailure } from "../../../src/ghidra/GhidraSessionError.js"; import { silentLogger } from "../../../src/logger.js"; @@ -28,7 +29,7 @@ it("preserves queue failure diagnostics through the provider, CLI adapter and SD throw cause; }, failure, - () => Promise.resolve(), + () => Promise.resolve(ok(null)), ); const harness = await connectGhidraMcp( "ghidra-failure-diagnostics", diff --git a/tests/boundary/providers/ghidra/ghidraClient.test.ts b/tests/boundary/providers/ghidra/ghidraClient.test.ts index 4bd24f5f7..452c122f7 100644 --- a/tests/boundary/providers/ghidra/ghidraClient.test.ts +++ b/tests/boundary/providers/ghidra/ghidraClient.test.ts @@ -1,8 +1,10 @@ import { spawn, type ChildProcess } from "node:child_process"; import { createHash } from "node:crypto"; +import { EventEmitter } from "node:events"; import { readFileSync } from "node:fs"; import { access, chmod, mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { PassThrough } from "node:stream"; import { fileURLToPath } from "node:url"; import { afterEach, describe, expect, it, onTestFinished, vi } from "vitest"; @@ -20,6 +22,7 @@ import type { GhidraLaunchSession, GhidraLauncher, } from "../../../../src/ghidra/GhidraLauncher.js"; +import type { ProviderProcessHandle } from "../../../../src/process/ProviderProcess.js"; import type { GhidraTransportKind } from "../../../../src/ghidra/GhidraTransport.js"; import { GHIDRA_MUTATING_OPERATIONS, @@ -59,6 +62,7 @@ type FixtureMode = | "exit_tools" | "silent" | "exit" + | "old_oom_then_malformed" | "diagnostic_output"; class FixtureLauncher implements GhidraLauncher { @@ -117,6 +121,58 @@ class FixtureLauncher implements GhidraLauncher { } } +const preExitedProcessLauncher = (options: { + readonly output: string; + readonly closeOutput: boolean; + readonly outputDelayMs?: number; +}) => { + const stdout = new PassThrough(); + const stderr = new PassThrough(); + const emitter = new EventEmitter(); + const process_: ProviderProcessHandle = { + on: emitter.on.bind(emitter), + once: emitter.once.bind(emitter), + off: emitter.off.bind(emitter), + pid: undefined, + stdout, + stderr, + exitCode: 73, + signalCode: null, + kill: () => false, + }; + const finishOutput = () => { + if (!stdout.writableEnded) stdout.end(); + if (!stderr.writableEnded) stderr.end(); + }; + let endedStreams = 0; + const maybeClose = () => { + endedStreams += 1; + if (endedStreams === 2) emitter.emit("close", 73, null); + }; + stdout.once("end", maybeClose); + stderr.once("end", maybeClose); + const launcher: GhidraLauncher = { + launch: async (session) => { + const projectRoot = join(session.runtimeRoot, "project"); + await mkdir(projectRoot, { recursive: true }); + setTimeout(() => { + if (options.closeOutput) { + if (!stdout.writableEnded) stdout.end(options.output); + if (!stderr.writableEnded) stderr.end(); + } else if (!stdout.writableEnded) stdout.write(options.output); + }, options.outputDelayMs ?? 15); + return ok({ + process: process_, + ownsProcessLifetime: true as const, + projectRoot, + ghidraLogPath: join(session.runtimeRoot, "ghidra.log"), + scriptLogPath: join(session.runtimeRoot, "script.log"), + }); + }, + }; + return { launcher, stdout, stderr, finishOutput }; +}; + const clients: GhidraClient[] = []; const clientFor = ( launcher: GhidraLauncher, @@ -604,24 +660,60 @@ describe("GhidraClient established requests", () => { }); describe("GhidraClient runtime removal failures", () => { - it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( - "reports a runtime root it could not remove and removes it on a later close", - async () => { - const launcher = new FixtureLauncher(); + it + .skipIf(process.platform === "win32" || process.getuid?.() === 0) + .each(["close", "active cancellation"] as const)( + "retains a runtime root after %s cleanup failure and removes it on retry", + async (stage) => { + const launcher = new FixtureLauncher( + stage === "active cancellation" ? "hang_tools" : "success", + ); const client = clientFor(launcher); await expect(client.start()).resolves.toMatchObject({ ok: true }); const runtimeRoot = launcher.runtimeRoots[0] ?? ""; // Without write permission its entries cannot be removed. await chmod(runtimeRoot, 0o500); try { - await expect(client.close()).resolves.toMatchObject({ - ok: false, - error: { - cleanupIncomplete: true, - cleanupResources: [runtimeRoot], - diagnostics: { leftover_paths: [runtimeRoot] }, - }, - }); + if (stage === "close") { + await expect(client.close()).resolves.toMatchObject({ + ok: false, + error: { + cleanupIncomplete: true, + cleanupResources: [runtimeRoot], + diagnostics: { leftover_paths: [runtimeRoot] }, + }, + }); + } else { + const controller = new AbortController(); + const active = client.callTool( + "procedure_pseudo_code", + { document: null, procedure: "fixture_main" }, + { signal: controller.signal }, + ); + await vi.waitFor(() => + expect(launcher.requests).toContain("procedure_pseudo_code"), + ); + const queued = client.callTool("procedure_info", { + document: null, + procedure: "fixture_main", + }); + controller.abort(); + await expect(active).resolves.toMatchObject({ + ok: false, + error: { + kind: "cancelled", + cleanupFailure: { + cleanupIncomplete: true, + cleanupResources: [runtimeRoot], + diagnostics: { leftover_paths: [runtimeRoot] }, + }, + }, + }); + await expect(queued).resolves.toMatchObject({ + ok: false, + error: { kind: "process" }, + }); + } await expect(access(runtimeRoot)).resolves.toBeUndefined(); expect(client.operationHealthSnapshot()).toMatchObject({ state: "unknown", @@ -871,6 +963,102 @@ describe("GhidraClient rejected requests", () => { }); }); +describe("GhidraClient process output drain before startup failure", () => { + const oomeOutput = [ + "ERROR Abort due to Headless analyzer error: Java heap space (HeadlessAnalyzer) java.lang.OutOfMemoryError:", + "\tat ghidra.app.plugin.core.analysis.GolangSymbolAnalyzer.applyTo", + ].join("\n"); + + it("captures deferred stdio from a child that exited before supervision", async () => { + const { launcher, finishOutput } = preExitedProcessLauncher({ + output: oomeOutput, + closeOutput: true, + }); + onTestFinished(finishOutput); + const client = clientFor(launcher, { startupTimeoutMs: 5_000 }); + const result = await client.start(); + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error.kind).toBe("process"); + expect(result.error.diagnostics).toMatchObject({ + exit_code: 73, + stdout: { text: oomeOutput }, + failure_output: { complete: true, stdout: { text: oomeOutput } }, + }); + expect(result.error.diagnostics).not.toHaveProperty( + "output_incomplete", + true, + ); + }); + + it("bounds an output stream that never closes and marks its snapshot incomplete", async () => { + const { launcher, finishOutput } = preExitedProcessLauncher({ + output: oomeOutput, + closeOutput: false, + }); + onTestFinished(finishOutput); + const client = clientFor(launcher, { startupTimeoutMs: 5_000 }); + const result = await client.start(); + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error.diagnostics).toMatchObject({ + exit_code: 73, + output_incomplete: true, + stdout: { text: oomeOutput }, + }); + }); + + it("keeps caller cancellation primary when cancellation wins during output drain", async () => { + const { launcher, finishOutput } = preExitedProcessLauncher({ + output: oomeOutput, + closeOutput: true, + outputDelayMs: 150, + }); + onTestFinished(finishOutput); + const controller = new AbortController(); + let abortScheduled = false; + const client = clientFor(launcher, { + startupTimeoutMs: 5_000, + onDiagnostic: (event) => { + if (event.type !== "launcher-exit" || abortScheduled) return; + abortScheduled = true; + setTimeout(() => controller.abort(), 25); + }, + }); + const result = await client.start(controller.signal); + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error.kind).toBe("cancelled"); + }); +}); + +describe("GhidraClient operation-scoped failure output", () => { + it("keeps earlier successful-operation OOME in history but out of later failure evidence", async () => { + const client = clientFor(new FixtureLauncher("old_oom_then_malformed")); + const started = await client.start(); + if (!started.ok) throw started.error; + + const earlier = await client.callTool("list_documents", {}); + expect(earlier.ok).toBe(true); + const later = await client.callTool("list_procedures", {}); + expect(later.ok).toBe(false); + if (later.ok) return; + + expect(later.error.kind).toBe("protocol"); + expect(later.error.diagnostics.stderr).toMatchObject({ + text: expect.stringContaining("java.lang.OutOfMemoryError:"), + }); + expect(later.error.diagnostics.failure_output).toMatchObject({ + complete: false, + stdout: { text: "" }, + stderr: { text: "Current operation diagnostic: 雪\n" }, + }); + }); +}); + describe("GhidraClient shutdown diagnostics", () => { it("logs an unexpected shutdown rejection and still removes its process and runtime", async () => { const launcher = new FixtureLauncher(); diff --git a/tests/boundary/providers/ghidra/ghidraJavaLaunch.test.ts b/tests/boundary/providers/ghidra/ghidraJavaLaunch.test.ts index 09ad518ab..4e6c8f99b 100644 --- a/tests/boundary/providers/ghidra/ghidraJavaLaunch.test.ts +++ b/tests/boundary/providers/ghidra/ghidraJavaLaunch.test.ts @@ -58,7 +58,9 @@ if (mode === "env-failure" || (probe === "-vmargs" && mode === "vm-failure")) { ? "-Dos.arch=x86_64\\n" : mode === "os-name-override" ? "-Dos.name=Windows\\n" - : "-Dfile.encoding=UTF8\\n"; + : mode === "heap-override" + ? "-Xmx128M\\n" + : "-Dfile.encoding=UTF8\\n"; process.stdout.write(vmArguments); } `, @@ -111,6 +113,29 @@ if (mode === "env-failure" || (probe === "-vmargs" && mode === "vm-failure")) { describe.skipIf(process.platform === "win32")( "Ghidra LaunchSupport JVM compatibility", () => { + it("reports the selected heap after LaunchSupport's earlier heap argument", async () => { + const { session, launcher, analysisArguments } = + await fixture("heap-override"); + const launched = await launcher.launch(session); + if (!launched.ok) throw launched.error; + expect( + analysisArguments[0]?.filter((argument) => argument.startsWith("-Xmx")), + ).toEqual(["-Xmx128M", "-Xmx1G"]); + expect(launched.value.configuredMaxHeap).toBe("1G"); + expect( + createGhidraDiagnostics({ + targetPath: session.targetPath, + targetSha256: session.targetSha256, + transport: session.transport, + providerVersion: session.providerVersion, + profileDigest: session.profileDigest, + launch: launched.value, + token: session.token, + previous: {}, + }), + ).toMatchObject({ configured_jvm_max_heap: "1G" }); + }); + it.each(["Eclipse OpenJ9 VM", "unrecognized VM", ""])( "keeps HotSpot-only flags off a VM identified as %s", async (javaVmName) => { diff --git a/tests/fixtures/fakeGhidra.mjs b/tests/fixtures/fakeGhidra.mjs index a7f29ba12..fc7b78700 100644 --- a/tests/fixtures/fakeGhidra.mjs +++ b/tests/fixtures/fakeGhidra.mjs @@ -47,6 +47,7 @@ else { const state = { buffer: { value: "" }, pings: { value: 0 }, + emittedOome: false, mode, token, runId, @@ -108,6 +109,28 @@ const handleRequest = (socket, server, request, state) => { handlePing(socket, request, state); return; } + if ( + state.mode === "old_oom_then_malformed" && + request.method === "list_documents" && + !state.emittedOome + ) { + state.emittedOome = true; + process.stderr.write( + "ERROR Abort due to Headless analyzer error: Java heap space (HeadlessAnalyzer) java.lang.OutOfMemoryError:\n", + () => setTimeout(() => handleRequest(socket, server, request, state), 10), + ); + return; + } + if ( + state.mode === "old_oom_then_malformed" && + request.method === "list_procedures" && + state.emittedOome + ) { + process.stderr.write("Current operation diagnostic: 雪\n", () => + setTimeout(() => socket.write("{invalid\n"), 10), + ); + return; + } if (state.mode === "remote_error" && request.method === "list_procedures") { socket.write( `${JSON.stringify({