diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index 63f16b9780..fc6237dcd1 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -83,6 +83,15 @@ explicit path or endpoint and do not need it. absent metadata does not prove a binary is not Go, and unparsed text records remain explicit. See the [Go binary guide](https://github.com/morluto/rea/blob/main/docs/go-binaries.md). + +- Explicit local SQLite database snapshot: `inspect_sqlite_database`. Inspect + with Node.js 24.x >=24.15 or 26+ providing native SQLite limits. Read schema + first (compact inventory by default), or select an exact ordinary table + and `columns` when rows are needed. Defaults are selected-table schema and + 20 rows; request `schema_scope: "database"` for the complete stored schema. Preserve both database and existing sibling WAL identity; row samples + report truncation and have unspecified order. Use a quiescent snapshot rather + than assuming a live database/WAL pair was copied atomically. See the + [SQLite guide](https://github.com/morluto/rea/blob/main/docs/sqlite-databases.md). - Explicit Linux ELF file for offline layout, symbols, relocations or static mitigation evidence: `inspect_binary_layout`. This target-free operation uses caller-supplied pwntools without opening a disassembler database. Preserve its diff --git a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md index e2f3230dbc..9dbd2b092a 100644 --- a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -11,6 +11,21 @@ unparsed module-text line. A missing record is an unknown, not proof of a non-Go binary; observed metadata does not authenticate its claims. Function, source and type recovery remain a separate native-analysis question. +## SQLite snapshots + +Use `inspect_sqlite_database` with an explicit local database snapshot for its +schema and optional rows from one exact ordinary table. It requires Node.js +24.x >=24.15 or 26+ with the native `DatabaseSync.limits` API. It also consumes the +existing sibling `-wal`; retain both original digests when citing its result. +Default output is a compact inventory; a selected table expands only its own +schema and samples 20 rows. Select `columns` to avoid irrelevant or large cells, +and request `schema_scope: "database"` only for complete schema details. +Choose `row_limit` when a finite sample is useful and keep truncation and +unspecified row order explicit. Virtual tables, shadow tables and views are +schema facts, not row-read targets. REA inspects private copies and leaves +the source files untouched; obtain a coherent quiescent database/WAL snapshot +before inspection. See [SQLite snapshots](https://github.com/morluto/rea/blob/main/docs/sqlite-databases.md). + ## Recorded Linux crashes Use `inspect_recorded_crash` with an explicit Linux x86-64 ELF core path to diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e882d07265..86f1ba2e49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -209,6 +209,11 @@ jobs: if-no-files-found: error retention-days: 7 + sqlite-runtime: + needs: [changes, build] + if: needs.changes.outputs.code == 'true' + uses: ./.github/workflows/sqlite-runtime.yml + test: name: Test (coverage + timing) needs: [changes, test-shard] @@ -659,6 +664,7 @@ jobs: static, build, test-shard, + sqlite-runtime, test, conformance-fixtures, windows-native-build, diff --git a/.github/workflows/sqlite-runtime.yml b/.github/workflows/sqlite-runtime.yml new file mode 100644 index 0000000000..6e5bbcd108 --- /dev/null +++ b/.github/workflows/sqlite-runtime.yml @@ -0,0 +1,37 @@ +name: SQLite runtime compatibility + +on: + workflow_call: + +permissions: + contents: read + +env: + HUSKY: "0" + +jobs: + sqlite: + name: SQLite runtime compatibility (Node ${{ matrix.node }}) + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # Exercise unavailable-runtime diagnostics, the first supported release, + # and new patches on both supported lines without hand-editing versions. + # The repository-pinned Node 24 version also runs the complete test suite. + node: ["22.x", "24.15.0", "24.x", "26.x"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + check-latest: true + cache: npm + - run: npm ci + - uses: actions/download-artifact@v4 + with: + name: rea-test-runtime + path: . + - name: Verify SQLite discovery, inspection and cleanup + run: npm run test:sqlite:run diff --git a/README.md b/README.md index 83b8f03bc7..a656171284 100644 --- a/README.md +++ b/README.md @@ -157,6 +157,7 @@ Additional tools and host support depend on the target: | Go binaries | Embedded compiler version, modules, replacements and build settings | Built-in static ELF/PE/thin Mach-O inspection; no Go installation required | | WASM artifact | Validation, sections, exact import/export WAT forms and decoded WAT | Caller-supplied WABT 1.0.42; `inspect_wasm_artifact` | | EVM bytecode | Dispatch selectors, byte offsets, inferred arguments and mutability | Local raw/hex carrier on Linux x64; [offline EVM guide](docs/evm-bytecode.md) | +| SQLite databases | Tables, columns, indexes, views, triggers and selected ordinary-table rows | Local database/WAL snapshot; Node.js 24.x >=24.15 or 26+ with native SQLite limits | | Recorded Linux crashes | Raw notes, every recorded thread's registers/signals and optional mapping candidates | Caller-supplied pwntools on Linux x64; optional GDB/pwndbg; [recorded crashes](docs/recorded-crashes.md) | | JavaScript / Electron | Modules, imports, source maps, routes, IPC and native add-on relationships | Node.js and npm; [application analysis](https://rea.tools/guides/javascript/) | | Websites | Page structure, scripts, network observations and requested screenshots | A Chrome-family browser; [browser analysis](https://rea.tools/guides/browser/) | diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index c906c4176c..07ca2e72df 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -43,6 +43,7 @@ export default defineConfig({ { text: "Android applications", link: "/android-analysis" }, { text: "Apple applications", link: "/apple-application-analysis" }, { text: "Firmware", link: "/firmware-analysis" }, + { text: "SQLite databases", link: "/sqlite-databases" }, ], }, { diff --git a/docs/cli.md b/docs/cli.md index 3e565c8d79..2d7567d061 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -62,6 +62,14 @@ unpack .NET single-file hosts or decode IL2CPP metadata. NativeAOT method bodies require native analysis. See the [managed-code guide](managed-code-analysis.md) for supported deployment forms, selection semantics, and coverage limits. +## SQLite snapshots + +For a local database snapshot, use `rea inspect-sqlite-database PATH --json`. +Add `--table NAME --row-limit 20` to read a bounded sample from an ordinary +table. This command does not require a native analysis provider. See +[SQLite database snapshots](sqlite-databases.md) for database/WAL identity, +source-file handling and row-value semantics. + ## Native analysis Inspect embedded Go build metadata without installing Go or configuring a diff --git a/docs/sqlite-databases.md b/docs/sqlite-databases.md new file mode 100644 index 0000000000..3b8f40b01f --- /dev/null +++ b/docs/sqlite-databases.md @@ -0,0 +1,144 @@ +# SQLite database snapshots + +`inspect_sqlite_database` / `inspect-sqlite-database` inspects one explicitly +selected local SQLite database snapshot without an active native analysis +target. Use it to understand application storage, configuration or retained +database evidence. No separate SQLite installation is required. + +This operation requires Node.js 24.x >=24.15 or 26+ with the native +`DatabaseSync.limits` API. The worker verifies that capability before querying +the copied database. Other REA features retain their published Node.js 22.x, +24.x or 26+ requirements. + +`rea capabilities` reports `inspect_sqlite_database` availability and the +runtime requirement without opening a database. Unsupported runtimes show +`available: false`, `availability_code: "unsupported_version"`, and an upgrade +note while the inventory command still succeeds. + +```sh +rea capabilities --json +rea inspect-sqlite-database ./application.sqlite --json +rea inspect-sqlite-database ./application.sqlite --table settings --column key --column value --row-limit 20 --json +``` + +```json +{ + "name": "inspect_sqlite_database", + "arguments": { + "path": "/artifacts/application.sqlite", + "table": "settings", + "columns": ["key", "value"], + "row_limit": 20 + } +} +``` + +MCP paths must be absolute filesystem paths on the current host. CLI paths +resolve against the operator's working directory. Select a quiescent snapshot: +stop the writer or obtain a consistent database/WAL pair using the application's +backup workflow before inspection. A file copy of a live pair does not establish +an atomic SQLite snapshot, even if no change is detected during inspection. + +## Returned evidence + +Without a table selection, the default `schema_scope` is `inventory`: compact +table metadata and index, view and trigger names, without SQL declarations or +column details. Selecting a table defaults to `selected_table`, which expands +that table and its indexes and triggers. Explicit `schema_scope: "database"` +(or CLI `--schema-scope database`) expands all stored declarations and supported +columns. `schema.completeness` refers to the reported scope; omitted column +details are marked `not_requested`. SQL declarations retain the original text. Virtual and +shadow tables are identified as such; their supported metadata coverage is +reported. If a virtual table cannot be resolved, possible backing tables have +`kind: "unknown"` and an explicit limitation; their rows are refused. Unrelated +ordinary tables remain selectable. Schema inventory does not execute views, +triggers or arbitrary SQL. + +Rows are optional. `table` selects one exact ordinary-table name. `columns` +(CLI repeated `--column`) selects exact visible column names in the requested +order; omission selects all visible columns, including generated columns. +Unselected cells are not evaluated or projected. Column selection requires a +table and at least one name; empty-string identifiers are valid. `row_limit` +requires `table`, defaults to 20 and accepts positive safe integers. Samples +remain subject to the byte and worker resource limits below, regardless of row count. The result reports +the selected limit, returned count and whether more rows were observed. Row +order is unspecified; a truncated sample does not prove anything about omitted +records. View, virtual-table and shadow-table row reads are unsupported. + +SQLite integers are decimal strings so every signed 64-bit value survives JSON +serialization. Real values retain their numbers; `"Infinity"`, `"-Infinity"` +and `"-0"` explicitly preserve infinities and negative zero through JSON. +Text returns the decoded string together with its bytes as hex and the database +encoding, preserving malformed text that the SQLite binding may replace for +display. Blobs are hex, and SQL NULL is a distinct tagged value. + +Database and existing sibling `-wal` files have separate original SHA-256 and +byte counts. The Evidence subject identifies the original database file; +the WAL identity is also necessary to interpret pending committed data. The +SQLite engine version is reported with the observed result. Neither digest is +a claim that the database is healthy, complete or trusted. + +## Source files and failures + +REA copies the selected database and existing sibling WAL to an owned private +temporary directory before SQLite opens them. Source files are not opened by +SQLite, so its locks or shared-memory sidecars cannot modify the original +snapshot. Temporary copies and the owned worker are cleaned up on success, +failure and cancellation. Extension loading and schema trust are disabled. +Empty rollback journals and fully zeroed journal headers left by successful +`TRUNCATE` and `PERSIST` commits are accepted after stable identity checks. +Journal headers are checked both before and after acquisition; unchanged file +metadata alone does not establish that a journal remains inactive. +Other retained rollback journals are refused because recovery is outside this +profile. Journal files are never modified or copied into the worker snapshot. + +A provider admits at most two inspections through acquisition, worker execution +and cleanup. Additional requests receive a typed capacity error before reading +files or creating workers; retry after an active inspection finishes. This +bounds simultaneous input buffering and private snapshot/worker counts. + +The database and WAL together are limited to 256 MiB. SQLite values/records, +SQL statements and the complete worker JSON reply each have a 16 MiB limit. +The owned worker has a 256 MiB JavaScript heap limit and a 30-second inspection +deadline. SQLite native allocations are separate from the JavaScript heap; +overall native heap and total resident memory are not capped. Row limits do not +override these byte and resource limits. Resource-limit failures do not return +an incomplete schema as successful complete evidence. + +MCP also applies its configured response budget to the complete Evidence +envelope. If delivery exceeds that budget, it reports a transport constraint +with guidance for exporting retained Evidence, requesting a matching larger +client/server response budget or using complete CLI JSON output. + +A cancellation received after the worker has completed does not erase the +completed observation. If subsequent private-root cleanup fails, the error +retains the completed Evidence and the resource owner for cleanup retry. MCP +records that Evidence before delivering the error, which carries its retained +evidence ID for later retrieval or export. + +Changed input identities, special files, malformed databases, unsupported +formats and resource failures are reported as errors with the selected path and +meaningful reason. Encrypted databases, database repair, arbitrary SQL, +cross-database queries and live capture are outside this inspection's coverage. + +## Compatibility verification + +Run `npm run test:sqlite` after changing the SQLite adapter or its shared +process, filesystem, CLI or MCP boundaries. It prepares the compiled runtime +and runs SQLite service, snapshot, inspection, cleanup and public CLI/MCP +tests. Directory-based selection includes new SQLite service and boundary +tests automatically. `npm run test:sqlite:run` runs the same cases when the +compiled runtime and generated test artifacts are already current. + +The reusable SQLite CI workflow consumes the shared build artifact and tests +the latest Node 22.x unavailable-runtime diagnostics, the minimum supported +24.15.0 release, and the latest 24.x and 26.x releases. The repository-pinned +runtime also runs the full suite. CI's required gate waits for every selected +SQLite matrix job; a failed, cancelled, skipped or missing selected lane blocks +the gate. Workflow changes themselves select the build and compatibility lane. + +PR checks run against GitHub's merge with the current base branch. A later +change to `main` can introduce failures outside SQLite; inspect the failing +job and recheck the updated merge before treating an earlier green run as +current verification. Compatibility checks detect regressions, but cannot +guarantee that an arbitrary future upstream change remains compatible. diff --git a/package.json b/package.json index faefead3b1..8a58184c8d 100644 --- a/package.json +++ b/package.json @@ -244,6 +244,8 @@ "verify:native-calls": "npm run build:cached && node scripts/verify-native-calls.mjs", "verify:native-ui:permissions": "npm run build:cached && node scripts/verify-native-ui.mjs --allow-permission-denial", "test:focused": "node scripts/run-development-tests.mjs focused", + "test:sqlite": "npm run test:prepare && npm run test:sqlite:run", + "test:sqlite:run": "vitest run src/application/sqlite/ tests/boundary/sqlite/ tests/acceptance/sqliteDatabaseCli.test.ts tests/boundary/mcp/serverLifecycleMcp.test.ts", "build:windows-native": "node scripts/build-windows-native.mjs", "verify:windows-native:artifact": "node scripts/verify-windows-native-artifact.mjs", "prepublishOnly": "npm run verify:windows-native:artifact", diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs index bea640f55c..8b99efce51 100644 --- a/scripts/ci/plan.mjs +++ b/scripts/ci/plan.mjs @@ -24,6 +24,7 @@ const NATIVE_TEST_OWNERS = ["apple", "package"]; const GUARDED_WORKFLOWS = new Set([ ".github/workflows/release.yml", ".github/workflows/real-ghidra-windows.yml", + ".github/workflows/sqlite-runtime.yml", ]); // Sources compiled by build-conformance-fixtures.mjs. The readiness journey // analyzes the built C fixture as well as its own applications. diff --git a/scripts/ci/required.mjs b/scripts/ci/required.mjs index fb5115b8ee..4f65867372 100644 --- a/scripts/ci/required.mjs +++ b/scripts/ci/required.mjs @@ -2,7 +2,7 @@ const jobs = JSON.parse(process.env.CI_JOBS); const outputs = jobs.changes.outputs; const required = ["changes", "static", "test"]; const lanes = { - code: ["build", "test-shard"], + code: ["build", "test-shard", "sqlite-runtime"], runtime: ["windows-native-build", "package-e2e"], package: ["windows-curated"], apple: ["apple-artifacts"], diff --git a/scripts/lib/catalog-builders.mjs b/scripts/lib/catalog-builders.mjs index 2ecafd9ae9..5cf27d4861 100644 --- a/scripts/lib/catalog-builders.mjs +++ b/scripts/lib/catalog-builders.mjs @@ -29,6 +29,7 @@ export const toolFamilyCatalog = (sources) => { ...sources.evmContracts.EVM_TOOL_CONTRACTS, ...sources.goContracts.GO_TOOL_CONTRACTS, ...sources.wasmContracts.WASM_TOOL_CONTRACTS, + ...sources.sqliteContracts.SQLITE_TOOL_CONTRACTS, ], }, { @@ -207,6 +208,10 @@ export const providerCatalog = (sources) => { identity: sources.goProvider.GO_BINARY_PROVIDER_IDENTITY, contracts: sources.goContracts.GO_TOOL_CONTRACTS, }, + { + identity: sources.sqliteDomain.SQLITE_PROVIDER_IDENTITY, + contracts: sources.sqliteContracts.SQLITE_TOOL_CONTRACTS, + }, { identity: sources.artifactProviders.ARTIFACT_GRAPH_PROVIDER, contracts: sources.artifactContracts.ARTIFACT_TOOL_CONTRACTS, diff --git a/scripts/lib/catalog-core.mjs b/scripts/lib/catalog-core.mjs index 6287f257b7..14c8bf3f89 100644 --- a/scripts/lib/catalog-core.mjs +++ b/scripts/lib/catalog-core.mjs @@ -50,6 +50,8 @@ const SOURCE_PATHS = { wasmContracts: "dist/contracts/wasm/wasmToolContracts.js", wasmProvider: "dist/wasm/WabtRelease.js", evmContracts: "dist/contracts/evm/evmToolContracts.js", + sqliteContracts: "dist/contracts/sqlite/sqliteToolContracts.js", + sqliteDomain: "dist/domain/sqlite/sqliteDatabase.js", evmProvider: "dist/evm/EvmoleRelease.js", goContracts: "dist/contracts/go/goToolContracts.js", goProvider: "dist/go/GoBinaryProvider.js", diff --git a/scripts/lib/module-boundaries.mjs b/scripts/lib/module-boundaries.mjs index a7a6bbf91b..d4acb4f00c 100644 --- a/scripts/lib/module-boundaries.mjs +++ b/scripts/lib/module-boundaries.mjs @@ -5,6 +5,7 @@ import { fileURLToPath, pathToFileURL } from "node:url"; const PURE_LAYERS = new Set(["domain", "contracts"]); const MIGRATED_PROVIDER_ROOTS = new Set([ + "sqlite", "evm", "go", "wasm", diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index ac825bc049..73ffaecf97 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { sqliteDatabaseCapability } from "./sqlite/SqliteDatabaseCapability.js"; import type { OptionalProviderLoadFailures, OptionalProviderLoadFailure, @@ -341,6 +342,15 @@ const workflowAvailabilityFor = ({ }: AvailabilityContext): Availability | null => { if (name === "inspect_go_binary") return { reason: "available", remediation: null }; + if (name === "inspect_sqlite_database") { + const capability = sqliteDatabaseCapability(process.versions.node); + return capability.available + ? { reason: "available", remediation: null } + : { + reason: "provider_unavailable", + remediation: capability.reason, + }; + } if (name === "trace_web_module_imports") return { reason: "available", diff --git a/src/application/DirectAnalysisStatus.ts b/src/application/DirectAnalysisStatus.ts index c19bd0c167..d67dcc9205 100644 --- a/src/application/DirectAnalysisStatus.ts +++ b/src/application/DirectAnalysisStatus.ts @@ -4,6 +4,9 @@ import { createServerIdentity } from "../serverIdentity.js"; import { silentLogger } from "../logger.js"; import type { Logger } from "pino"; import type { DirectAnalysisDependencies } from "./DirectAnalysisDependencies.js"; +import { providerCapabilitySet } from "../contracts/toolOutputSchemaPrimitives.js"; +import { providerCapabilitySets } from "./binary/BinarySessionStatus.js"; +import { sqliteDatabaseCapability } from "./sqlite/SqliteDatabaseCapability.js"; /** Read target-free provider status and operation availability. */ export const runSessionStatus = async ( @@ -17,8 +20,15 @@ export const runSessionStatus = async ( if (!config.ok) return { error: projectAnalysisError(config.error) }; const session = await dependencies.createBinarySession(config.value, logger); try { + const status = jsonObjectSchema.parse(session.status()); return { - ...jsonObjectSchema.parse(session.status()), + ...status, + capabilities: [ + ...providerCapabilitySet.array().parse(status.capabilities), + ...providerCapabilitySets([ + sqliteDatabaseCapability(process.versions.node), + ]), + ], server_identity: createServerIdentity({ startedAt: new Date().toISOString(), }), diff --git a/src/application/binary/BinarySessionStatus.ts b/src/application/binary/BinarySessionStatus.ts index 4794bed493..15e0cfd32c 100644 --- a/src/application/binary/BinarySessionStatus.ts +++ b/src/application/binary/BinarySessionStatus.ts @@ -198,15 +198,19 @@ type CapabilityAvailabilityCode = Exclude< CapabilityDescriptor["availabilityCode"], undefined >; -type CapabilityProjection = Omit & - ( +// Wire capability projection also covers target-free operations outside binary routing. +type CapabilityProjection = Omit< + CapabilityDescriptor, + "operation" | "available" | "reason" +> & { readonly operation: string } & ( | { readonly available: true; readonly reason: null } | { readonly available: false; readonly reason: string } ); type ProviderCapabilityGroup = ProviderCapabilitySet["groups"][number]; /** Group inline capability facts without merging distinct providers or states. */ -const providerCapabilitySets = ( +/** Project provider descriptors once for binary and target-free status consumers. */ +export const providerCapabilitySets = ( descriptors: readonly CapabilityProjection[], ): ProviderCapabilitySet[] => { const providers = new Map< diff --git a/src/application/sqlite/SqliteDatabaseCapability.test.ts b/src/application/sqlite/SqliteDatabaseCapability.test.ts new file mode 100644 index 0000000000..7ae3a6f94a --- /dev/null +++ b/src/application/sqlite/SqliteDatabaseCapability.test.ts @@ -0,0 +1,31 @@ +import { expect, it } from "vitest"; +import { expandProviderCapabilities } from "../../contracts/toolOutputSchemaPrimitives.js"; +import { providerCapabilitySets } from "../binary/BinarySessionStatus.js"; +import { sqliteDatabaseCapability } from "./SqliteDatabaseCapability.js"; + +it.each([ + ["22.23.1", false], + ["24.11.0", false], + ["24.14.9", false], + ["24.15.0", true], + ["25.1.0", false], + ["26.0.0", true], + ["24.15.0-rc.1", false], + ["invalid", false], +])( + "reports SQLite inspection availability on Node %s as %s", + (version, available) => { + const [capability] = expandProviderCapabilities( + providerCapabilitySets([sqliteDatabaseCapability(version)]), + ); + expect(capability).toMatchObject({ + operation: "inspect_sqlite_database", + available, + reason: available + ? null + : expect.stringContaining("Node.js 24.x >=24.15 or 26+"), + availability_code: available ? null : "unsupported_version", + limitations: [expect.stringContaining("DatabaseSync.limits")], + }); + }, +); diff --git a/src/application/sqlite/SqliteDatabaseCapability.ts b/src/application/sqlite/SqliteDatabaseCapability.ts new file mode 100644 index 0000000000..f3c0308676 --- /dev/null +++ b/src/application/sqlite/SqliteDatabaseCapability.ts @@ -0,0 +1,33 @@ +import { satisfies } from "semver"; +import { toolContract } from "../../contracts/toolContracts.js"; +import { SQLITE_PROVIDER_IDENTITY } from "../../domain/sqlite/sqliteDatabase.js"; + +const SQLITE_RUNTIME_REQUIREMENT = + "SQLite inspection requires Node.js 24.x >=24.15 or 26+ with the native DatabaseSync.limits API. Other REA features retain their published runtime requirements. The inspection worker verifies the API before querying the snapshot."; + +/** Discover inspection readiness without opening a database or starting a worker. */ +export const sqliteDatabaseCapability = (nodeVersion: string) => { + const contract = toolContract("inspect_sqlite_database"); + const facts = { + operation: contract.name, + provider: SQLITE_PROVIDER_IDENTITY, + effects: { + mutatesArtifact: contract.effects.mutatesTarget, + launchesProcess: contract.effects.launchesProcess, + mayShowUi: contract.effects.changesUiState, + mayAccessNetwork: contract.effects.accessesNetwork, + mayWriteFilesystem: contract.effects.writesFilesystem, + changesPermissions: false, + requiresRoot: false, + }, + limitations: [SQLITE_RUNTIME_REQUIREMENT], + }; + return satisfies(nodeVersion, "^24.15.0 || >=26.0.0") + ? { ...facts, available: true as const, reason: null } + : { + ...facts, + available: false as const, + reason: SQLITE_RUNTIME_REQUIREMENT, + availabilityCode: "unsupported_version" as const, + }; +}; diff --git a/src/application/sqlite/SqliteDatabasePort.ts b/src/application/sqlite/SqliteDatabasePort.ts new file mode 100644 index 0000000000..ed5322b678 --- /dev/null +++ b/src/application/sqlite/SqliteDatabasePort.ts @@ -0,0 +1,17 @@ +import type { + AnalysisExecution, + ExecutionOptions, +} from "../AnalysisProvider.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import type { InspectSqliteDatabaseInput } from "../../domain/sqlite/sqliteDatabase.js"; +import type { Result } from "../../domain/result.js"; + +/** Offline database inspection boundary shared by CLI and MCP. */ +export interface SqliteDatabasePort { + /** Await and retry provider-owned snapshot cleanup. */ + close?(): Promise; + inspect( + input: InspectSqliteDatabaseInput, + options?: ExecutionOptions, + ): Promise>; +} diff --git a/src/application/sqlite/SqliteDatabaseService.test.ts b/src/application/sqlite/SqliteDatabaseService.test.ts new file mode 100644 index 0000000000..543cd0b027 --- /dev/null +++ b/src/application/sqlite/SqliteDatabaseService.test.ts @@ -0,0 +1,147 @@ +import { expect, it } from "vitest"; +import { SqliteDatabaseService } from "./SqliteDatabaseService.js"; +import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCore.js"; +import { err } from "../../domain/result.js"; +import { ok } from "../../domain/result.js"; +import { createAnalysisExecution } from "../AnalysisProvider.js"; + +const fixture = () => ({ + artifact: { path: "/selected.db", sha256: "a".repeat(64), bytes: 4096 }, + wal: null, + engine: { name: "SQLite", version: "3.50.2" }, + database_encoding: "UTF-8", + schema: { + scope: "inventory", + completeness: "complete", + tables: [], + indexes: [], + views: [], + triggers: [], + }, + rows: null, + limitations: [], +}); + +const execution = (value: unknown) => + createAnalysisExecution( + value, + { id: "sqlite", name: "SQLite", version: "3.50.2" }, + { + subject: { path: "/selected.db", sha256: "a".repeat(64), format: "file" }, + locations: [{ kind: "artifact-path", path: "/selected.db" }], + }, + ); + +it.each([ + { path: "relative.db" }, + { path: "/selected.db", approval: true }, + { path: "/selected.db", row_limit: 1 }, + { path: "/selected.db", table: 1 }, + { path: "/selected.db", table: "data", row_limit: 0 }, + { + path: "/selected.db", + table: "data", + row_limit: Number.MAX_SAFE_INTEGER + 1, + }, +])( + "rejects invalid SQLite selection before provider effects: %j", + async (input) => { + const service = new SqliteDatabaseService({ + inspect: () => { + throw new Error("Invalid selection must not read a database"); + }, + }); + expect(await service.inspect(input)).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + }, +); + +it("preserves producer failures and rejects pre-cancelled requests before effects", async () => { + const failure = new AnalysisCapabilityUnavailableError( + "rea-sqlite", + "inspect_sqlite_database", + "Selected worker could not start", + ); + const service = new SqliteDatabaseService({ + inspect: () => Promise.resolve(err(failure)), + }); + expect(await service.inspect({ path: "/selected.db" })).toEqual(err(failure)); + const signal = AbortSignal.abort(); + const untouched = new SqliteDatabaseService({ + inspect: () => { + throw new Error("Cancelled selection must not read a database"); + }, + }); + expect( + await untouched.inspect({ path: "/selected.db" }, { signal }), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); +}); + +it.each(["path", "digest", "subject", "wal", "table"])( + "rejects producer results with changed caller binding: %s", + async (problem) => { + const value = fixture(); + if (problem === "path") value.artifact.path = "/other.db"; + if (problem === "digest") value.artifact.sha256 = "b".repeat(64); + const observed = execution({ + ...value, + ...(problem === "wal" + ? { wal: { path: "/other.db-wal", sha256: "b".repeat(64), bytes: 32 } } + : {}), + }); + const service = new SqliteDatabaseService({ + inspect: () => + Promise.resolve( + ok({ + ...observed, + ...(problem === "subject" ? { subject: null } : {}), + }), + ), + }); + expect( + await service.inspect({ + path: "/selected.db", + ...(problem === "table" ? { table: "selected" } : {}), + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); + }, +); + +it("preserves completed observed evidence when cancellation arrives after provider completion", async () => { + const value = fixture(); + const service = new SqliteDatabaseService({ + inspect: () => Promise.resolve(ok(execution(value))), + }); + expect(await service.inspect({ path: "/selected.db" })).toMatchObject({ + ok: true, + value: { + confidence: "observed", + subject: { + local_path: "/selected.db", + digest: { sha256: "a".repeat(64) }, + }, + normalized_result: value, + }, + }); + const controller = new AbortController(); + const cancelled = new SqliteDatabaseService({ + inspect: () => { + controller.abort(); + return Promise.resolve(ok(execution(value))); + }, + }); + expect( + await cancelled.inspect( + { path: "/selected.db" }, + { signal: controller.signal }, + ), + ).toMatchObject({ + ok: true, + value: { normalized_result: value }, + }); +}); diff --git a/src/application/sqlite/SqliteDatabaseService.ts b/src/application/sqlite/SqliteDatabaseService.ts new file mode 100644 index 0000000000..7f8aaeb73e --- /dev/null +++ b/src/application/sqlite/SqliteDatabaseService.ts @@ -0,0 +1,121 @@ +import { isAbsolute } from "node:path"; +import type { ExecutionOptions } from "../AnalysisProvider.js"; +import type { SqliteDatabasePort } from "./SqliteDatabasePort.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisInputError, + AnalysisOutputError, +} from "../../domain/analysisErrorCore.js"; +import { + createEvidence, + type Evidence, + type EvidenceSubjectTarget, +} from "../../domain/evidence.js"; +import { + inspectSqliteDatabaseInputSchema, + sqliteDatabaseSchema, + type InspectSqliteDatabaseInput, + type SqliteDatabase, + SQLITE_ROW_LIMIT_DEFAULT, + sqliteSchemaScope, + sqliteInspectionParameters, +} from "../../domain/sqlite/sqliteDatabase.js"; +import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { jsonValueSchema } from "../../domain/jsonValue.js"; + +const OPERATION = "inspect_sqlite_database"; + +const matchesSelectedDatabase = ( + report: SqliteDatabase, + input: InspectSqliteDatabaseInput, + subject: EvidenceSubjectTarget, +): boolean => + report.artifact.path === input.path && + subject.path === input.path && + subject.sha256 === report.artifact.sha256 && + subject.format === "file" && + (report.wal === null || report.wal.path === `${input.path}-wal`) && + report.schema.scope === sqliteSchemaScope(input) && + (report.schema.scope !== "selected_table" || + (report.schema.tables.length === 1 && + report.schema.tables[0]?.name === input.table && + report.schema.indexes.every((index) => index.table === input.table) && + report.schema.triggers.every( + (trigger) => trigger.table === input.table, + ) && + report.schema.views.length === 0)) && + (input.table === undefined + ? report.rows === null + : report.rows?.table === input.table && + report.rows.row_limit === (input.row_limit ?? SQLITE_ROW_LIMIT_DEFAULT) && + (input.columns === undefined || + (report.rows.columns.length === input.columns.length && + report.rows.columns.every( + (name, index) => name === input.columns?.[index], + )))); + +/** Shared snapshot inspection and caller-selection validation for CLI and MCP. */ +export class SqliteDatabaseService { + constructor(readonly provider: SqliteDatabasePort) {} + + /** Await provider shutdown, retaining failed cleanup owners for retry. */ + close(): Promise { + return this.provider.close?.() ?? Promise.resolve(); + } + + /** Inspect an explicit database snapshot and preserve its original database/WAL identity. */ + async inspect( + rawInput: unknown, + options?: ExecutionOptions, + ): Promise> { + if (options?.signal?.aborted) + return err(new AnalysisCancelledError(OPERATION)); + const input = inspectSqliteDatabaseInputSchema.safeParse(rawInput); + if (!input.success) + return err( + analysisInputErrorFromIssues(OPERATION, input.error.issues, rawInput, { + cause: input.error, + }), + ); + if (!isAbsolute(input.data.path)) + return err( + new AnalysisInputError(OPERATION, undefined, [ + { + path: ["path"], + reason: "invalid_format", + message: "Expected an absolute filesystem path on this host.", + }, + ]), + ); + const inspected = await this.provider.inspect(input.data, options); + if (!inspected.ok) return inspected; + const execution = inspected.value; + const report = sqliteDatabaseSchema.safeParse(execution.result); + if ( + !report.success || + execution.subject === null || + !matchesSelectedDatabase(report.data, input.data, execution.subject) + ) + return err( + new AnalysisOutputError( + OPERATION, + "SQLite adapter returned malformed data or changed the selected database identity/table/row limit.", + ), + ); + return ok( + createEvidence(execution.subject, execution.provider, { + operation: OPERATION, + parameters: sqliteInspectionParameters(input.data), + result: jsonValueSchema.parse(report.data), + ...(execution.rawResult === null + ? {} + : { rawResult: execution.rawResult }), + confidence: "observed", + locations: execution.locations, + limitations: execution.limitations, + }), + ); + } +} diff --git a/src/artifacts/readStableArtifact.ts b/src/artifacts/readStableArtifact.ts index 560d00c959..e35a20783f 100644 --- a/src/artifacts/readStableArtifact.ts +++ b/src/artifacts/readStableArtifact.ts @@ -2,6 +2,8 @@ import { createHash } from "node:crypto"; import { constants, type BigIntStats } from "node:fs"; import { lstat, open, type FileHandle } from "node:fs/promises"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; +import type { ArtifactResourceScope } from "./ArtifactResourceScope.js"; +import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; /** Filesystem boundary for stable reads, including deterministic replacement probes. */ export interface StableArtifactFileSystem { @@ -42,6 +44,7 @@ export const readStableArtifact = async ( maximumBytes: number, signal?: AbortSignal, fileSystem: StableArtifactFileSystem = FILE_SYSTEM, + resourceScope?: ArtifactResourceScope, ): Promise<{ readonly bytes: Buffer; readonly sha256: string }> => { signal?.throwIfAborted(); const before = await fileSystem.lstat(path); @@ -61,6 +64,9 @@ export const readStableArtifact = async ( constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, ), ); + const owner = + resourceScope === undefined ? undefined : new OwnedFileHandle(file); + let failure: unknown; try { const initial = await file.stat({ bigint: true }); if ( @@ -116,7 +122,22 @@ export const readStableArtifact = async ( ); const bytes = Buffer.concat(chunks, count); return { bytes, sha256: createHash("sha256").update(bytes).digest("hex") }; + } catch (cause: unknown) { + failure = cause; + throw cause; } finally { - await file.close(); + if (owner === undefined || resourceScope === undefined) await file.close(); + else { + const cleanup = await resourceScope.release({ + kind: "file-handle", + resource: path, + handle: owner, + }); + if (cleanup.kind === "failed") + throw ArtifactReaderFailure.withCleanup( + failure ?? cleanup.cause, + ArtifactReaderFailure.cleanupObservation(cleanup.cause, path), + ); + } } }; diff --git a/src/cli.ts b/src/cli.ts index 36c05f6c98..cfb83b71ed 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -3,6 +3,7 @@ import { snapshotEnvironment } from "./process/snapshotEnvironment.js"; import { createDirectAnalysis } from "./composition/directAnalysis.js"; import { registerEvmCommands } from "./cli/evmCommands.js"; import { registerGoCommands } from "./cli/goCommands.js"; +import { registerSqliteCommands } from "./cli/sqliteCommands.js"; import { registerAnalysisViewCommands } from "./cli/analysisViewCommands.js"; import { registerBinaryDiagnosticsCommands } from "./cli/binaryDiagnosticsCommands.js"; import { registerFridaCommands } from "./cli/fridaCommands.js"; @@ -95,6 +96,7 @@ export const createCli = ( registerWasmCommands(cli, logger, environment); registerEvmCommands(cli, logger, environment); registerGoCommands(cli, logger); + registerSqliteCommands(cli, logger, environment); registerArtifactCommands(cli, logger, analysis.runProviderAnalysis); registerManagedCommands(cli, logger, analysis.runProviderAnalysis); registerAndroidCommands(cli, logger, environment); diff --git a/src/cli/sqliteCommands.ts b/src/cli/sqliteCommands.ts new file mode 100644 index 0000000000..49ded4801d --- /dev/null +++ b/src/cli/sqliteCommands.ts @@ -0,0 +1,111 @@ +import { z } from "incur"; +import { resolve } from "node:path"; +import { createSqliteDatabaseService } from "../composition/sqlite.js"; +import { CLI_COMMANDS } from "../cliCommandNames.js"; +import { logCliCommand } from "../cliLogging.js"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; +import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import type { Logger } from "pino"; +import type { CliInstance } from "./types.js"; +import { withCommandCancellation } from "./commandCancellation.js"; + +/** Resolve an operator-selected SQLite snapshot through the shared inspection workflow. */ +export const registerSqliteCommands = ( + cli: CliInstance, + logger: Logger, + environment: Readonly, +): void => { + const service = createSqliteDatabaseService(environment); + cli.command(CLI_COMMANDS.inspectSqliteDatabase, { + description: + "Inspect SQLite schema and optional ordinary-table rows from a local snapshot", + args: z.object({ + path: z.string().describe("Local SQLite database snapshot"), + }), + options: z.object({ + table: z + .string() + .optional() + .describe("Exact ordinary table name to read"), + column: z + .array(z.string()) + .optional() + .describe( + "Repeat to select exact row columns in the requested order; requires table", + ), + schemaScope: z + .enum(["inventory", "selected_table", "database"]) + .optional() + .describe( + "Compact inventory by default, or selected-table details when table is supplied; database requests full schema", + ), + rowLimit: z + .number() + .int() + .min(1) + .optional() + .describe( + "Positive safe integer row count; default 20; requires table", + ), + }), + run: ({ args, options }) => + withCommandCancellation((signal) => + logCliCommand(logger, CLI_COMMANDS.inspectSqliteDatabase, async () => { + const inspection = await service + .inspect( + { + path: resolve(args.path), + ...(options.table === undefined + ? {} + : { table: options.table }), + ...(options.rowLimit === undefined + ? {} + : { row_limit: options.rowLimit }), + ...(options.column === undefined + ? {} + : { columns: options.column }), + ...(options.schemaScope === undefined + ? {} + : { schema_scope: options.schemaScope }), + }, + { signal }, + ) + .then( + (result) => ({ result }), + (cause: unknown) => ({ cause }), + ); + try { + await service.close(); + } catch (cause: unknown) { + const primary = + "cause" in inspection + ? inspection.cause + : inspection.result.ok + ? undefined + : inspection.result.error; + if ( + primary instanceof AnalysisError && + cause instanceof AnalysisError + ) + throw analysisErrorWithCleanupFailure( + primary, + cause, + "inspect_sqlite_database", + ); + if ("cause" in inspection) + throw new AggregateError( + [inspection.cause, cause], + "SQLite inspection and provider cleanup both failed", + { cause: inspection.cause }, + ); + throw cause; + } + if ("cause" in inspection) throw inspection.cause; + return inspection.result.ok + ? inspection.result.value + : projectAnalysisError(inspection.result.error); + }), + ), + }); +}; diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index 8fa0505f1b..154c49396d 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -1,6 +1,7 @@ /** Canonical names used by every one-shot CLI command registration. */ export const CLI_COMMANDS = Object.freeze({ inspectWasmArtifact: "inspect-wasm-artifact", + inspectSqliteDatabase: "inspect-sqlite-database", inspectEvmInterface: "inspect-evm-interface", inspectWebNetworkCapture: "inspect-web-network-capture", observeWebExecution: "observe-web-execution", diff --git a/src/composition/sqlite.ts b/src/composition/sqlite.ts new file mode 100644 index 0000000000..786684a2a0 --- /dev/null +++ b/src/composition/sqlite.ts @@ -0,0 +1,8 @@ +import { SqliteDatabaseService } from "../application/sqlite/SqliteDatabaseService.js"; +import { SqliteDatabaseProvider } from "../sqlite/SqliteDatabaseProvider.js"; + +/** Compose local snapshot inspection without opening a native analysis provider. */ +export const createSqliteDatabaseService = ( + environment: Readonly = process.env, +): SqliteDatabaseService => + new SqliteDatabaseService(new SqliteDatabaseProvider(environment)); diff --git a/src/contracts/sqlite/sqliteToolContracts.ts b/src/contracts/sqlite/sqliteToolContracts.ts new file mode 100644 index 0000000000..6ee837c0c5 --- /dev/null +++ b/src/contracts/sqlite/sqliteToolContracts.ts @@ -0,0 +1,42 @@ +import { + inspectSqliteDatabaseInputSchema, + sqliteDatabaseSchema, +} from "../../domain/sqlite/sqliteDatabase.js"; +import type { ToolContract } from "../toolContractTypes.js"; +import { toolContractMetadata } from "../toolEffects.js"; +import { evidenceResultOf } from "../toolOutputSchemaPrimitives.js"; + +/** Snapshot inspection is independent of any active native-analysis target. */ +export const SQLITE_TOOL_CONTRACTS = [ + { + name: "inspect_sqlite_database", + ...toolContractMetadata("inspect_sqlite_database"), + kind: "artifact-provider", + description: + "Inspect an explicit local SQLite database snapshot and its existing sibling WAL without modifying source files or opening a native analysis provider. Requires Node.js 24.x >=24.15 or 26+ with native DatabaseSync.limits support, checked by the worker. Returns observed Evidence with database/WAL SHA-256 and a compact schema inventory. Selecting an exact ordinary table expands only its schema and samples rows (default 20, subject to byte and worker resource limits). Select exact columns in the requested order to avoid unrelated or large values; schema_scope=database explicitly expands the whole schema; returned rows retain 64-bit integers, text bytes and blobs with explicit truncation. No arbitrary SQL, view or virtual-table execution, extension loading, database repair, encryption support or live-snapshot atomicity claim. At most two inspections per provider are admitted; occupied capacity is retryable. An isolated temporary copy and owned worker enforce input, value/record, SQL-statement, JSON-reply, JavaScript-heap and deadline limits; overall native heap is not capped. Supply a quiescent snapshot for coherent database/WAL evidence.", + inputSchema: inspectSqliteDatabaseInputSchema, + outputSchema: evidenceResultOf(sqliteDatabaseSchema), + examples: [ + { + title: "Inspect a compact inventory of a local SQLite snapshot", + input: { path: "/artifacts/application.sqlite" }, + }, + { + title: "Read selected rows from an ordinary table", + input: { + path: "/artifacts/application.sqlite", + table: "settings", + columns: ["key", "value"], + row_limit: 20, + }, + }, + { + title: "Expand the complete stored schema", + input: { + path: "/artifacts/application.sqlite", + schema_scope: "database", + }, + }, + ], + }, +] as const satisfies readonly ToolContract[]; diff --git a/src/contracts/toolContracts.ts b/src/contracts/toolContracts.ts index 8725f30408..216e2b7928 100644 --- a/src/contracts/toolContracts.ts +++ b/src/contracts/toolContracts.ts @@ -1,6 +1,7 @@ import { WASM_TOOL_CONTRACTS } from "./wasm/wasmToolContracts.js"; import { EVM_TOOL_CONTRACTS } from "./evm/evmToolContracts.js"; import { GO_TOOL_CONTRACTS } from "./go/goToolContracts.js"; +import { SQLITE_TOOL_CONTRACTS } from "./sqlite/sqliteToolContracts.js"; import { ANALYSIS_VIEW_TOOL_CONTRACTS } from "./analysisViewToolContracts.js"; import { BINARY_DIAGNOSTICS_TOOL_CONTRACTS } from "./native/binaryDiagnosticsToolContracts.js"; import { NATIVE_TOOL_CONTRACTS } from "./native/nativeToolContracts.js"; @@ -37,6 +38,7 @@ export const TOOL_CONTRACTS = [ ...EVM_TOOL_CONTRACTS, ...GO_TOOL_CONTRACTS, ...WASM_TOOL_CONTRACTS, + ...SQLITE_TOOL_CONTRACTS, ...MANAGED_TOOL_CONTRACTS, ...ANDROID_TOOL_CONTRACTS, ...APKTOOL_TOOL_CONTRACTS, diff --git a/src/contracts/toolEffects.test.ts b/src/contracts/toolEffects.test.ts index 151c7d5f49..836d190b19 100644 --- a/src/contracts/toolEffects.test.ts +++ b/src/contracts/toolEffects.test.ts @@ -19,6 +19,8 @@ const EXECUTION_WORDING_EXCEPTIONS: Readonly> = { resolve_native_call_targets: "states that it does not execute code", inspect_pe_resources: "states that it does not execute the image", inspect_go_binary: "reads embedded metadata without executing the target", + inspect_sqlite_database: + "uses a trusted inspector worker without executing application code or caller SQL", inspect_binary_layout: "parses the binary offline with operator-configured pwntools", inspect_recorded_crash: "inspects a core file without launching its target", diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index 11a268c0e9..74b8a96651 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -54,6 +54,11 @@ const sessionEvidence = effects({ mutatesSession: true }); /** Explicit effect audit for every public tool. */ export const TOOL_EFFECTS: Readonly> = { inspect_go_binary: evidence, + inspect_sqlite_database: effects({ + mutatesSession: true, + writesFilesystem: true, + launchesProcess: true, + }), inspect_wasm_artifact: effects({ mutatesSession: true, writesFilesystem: true, diff --git a/src/domain/analysisErrorCore.ts b/src/domain/analysisErrorCore.ts index 8e1138f19f..9e08f29a36 100644 --- a/src/domain/analysisErrorCore.ts +++ b/src/domain/analysisErrorCore.ts @@ -21,13 +21,22 @@ export class AnalysisUnsupportedTargetError extends AnalysisError { } /** An analysis or transport resource failure, distinct from malformed input or unsupported coverage. */ +export const ANALYSIS_RESOURCE_KINDS = [ + "memory", + "cpu", + "file-size", + "transport", + "capacity", +] as const; + +/** A measured resource constraint or bounded admission refusal. */ export class AnalysisResourceConstraintError extends AnalysisError { readonly _tag = "AnalysisResourceConstraintError"; readonly remediationAction: string | undefined; constructor( readonly operation: string, - readonly resource: "memory" | "cpu" | "file-size" | "transport", + readonly resource: (typeof ANALYSIS_RESOURCE_KINDS)[number], readonly reason: string, readonly reportedLimits: Readonly> | null, options?: AnalysisErrorOptions & { readonly remediationAction?: string }, diff --git a/src/domain/analysisErrorPresentation.ts b/src/domain/analysisErrorPresentation.ts index fd5db4823a..215ed40fc4 100644 --- a/src/domain/analysisErrorPresentation.ts +++ b/src/domain/analysisErrorPresentation.ts @@ -45,13 +45,15 @@ export const analysisErrorRemediationAction = ( if (error instanceof AnalysisResourceConstraintError) return ( error.remediationAction ?? - (error.resource === "transport" - ? "Discover retained records with get_evidence_bundle and detail: summary, then inspect compatible records through inspect_analysis_view using an exact Evidence reference. Export complete retained session evidence through export_evidence_bundle to a caller-selected path, or use complete CLI JSON output. The connection remains usable." - : error.resource === "cpu" - ? "Review the reported worker CPU limits and observed signal. Retry with sufficient CPU time or a smaller artifact; REA retains tighter inherited limits." - : error.resource === "file-size" - ? "Review the reported worker file-size limits and write failure. Retry with a sufficient file-size allowance for the evidence reply; REA retains tighter inherited limits." - : "Review the reported worker memory limits and available host memory. Retry with sufficient memory or a smaller artifact; REA retains tighter inherited limits.") + (error.resource === "capacity" + ? "Wait for an active analysis to complete, then retry within the reported admission limits." + : error.resource === "transport" + ? "Discover retained records with get_evidence_bundle and detail: summary, then inspect compatible records through inspect_analysis_view using an exact Evidence reference. Export complete retained session evidence through export_evidence_bundle to a caller-selected path, or use complete CLI JSON output. The connection remains usable." + : error.resource === "cpu" + ? "Review the reported worker CPU limits and observed signal. Retry with sufficient CPU time or a smaller artifact; REA retains tighter inherited limits." + : error.resource === "file-size" + ? "Review the reported worker file-size limits and write failure. Retry with a sufficient file-size allowance for the evidence reply; REA retains tighter inherited limits." + : "Review the reported worker memory limits and available host memory. Retry with sufficient memory or a smaller artifact; REA retains tighter inherited limits.") ); if (error instanceof HopperTimeoutError) return error.operation === undefined diff --git a/src/domain/analysisErrorProjection.test.ts b/src/domain/analysisErrorProjection.test.ts index 12597fc945..b166385a08 100644 --- a/src/domain/analysisErrorProjection.test.ts +++ b/src/domain/analysisErrorProjection.test.ts @@ -116,6 +116,40 @@ it("distinguishes unsupported target formats from unavailable provider capabilit expect(analysisErrorProjectionSchema.safeParse(projected).success).toBe(true); }); +it.each([false, true])( + "marks occupied capacity retryable only while cleanup is confirmed: %s", + (cleanupIncomplete) => { + const projected = projectAnalysisError( + new AnalysisResourceConstraintError( + "inspect_sqlite_database", + "capacity", + "Inspection slots are occupied", + { max_concurrent_inspections: 2 }, + cleanupIncomplete + ? { + cleanup: { + reason: "Snapshot removal denied", + resources: ["/private/snapshot"], + }, + } + : undefined, + ), + ); + expect(projected.retryable).toBe(!cleanupIncomplete); + expect(projected.code).toBe( + cleanupIncomplete ? "cleanup_incomplete" : "resource_constraint", + ); + expect(projected.details).toMatchObject({ + resource: "capacity", + reported_limits: { max_concurrent_inspections: 2 }, + }); + expect(projected.remediation.action).toContain("Wait"); + expect(analysisErrorProjectionSchema.safeParse(projected).success).toBe( + true, + ); + }, +); + it.each([ ["memory", null], ["memory", { address_space_bytes: 67108864 }], diff --git a/src/domain/analysisErrorProjection.ts b/src/domain/analysisErrorProjection.ts index 6c0699eb87..cd0c00987f 100644 --- a/src/domain/analysisErrorProjection.ts +++ b/src/domain/analysisErrorProjection.ts @@ -78,7 +78,11 @@ export const projectAnalysisError = ( code, category: analysisErrorCategory(error), message: analysisErrorUserMessage(error), - retryable: RETRYABLE_CODES.has(code), + retryable: + RETRYABLE_CODES.has(code) || + (code === "resource_constraint" && + error instanceof AnalysisResourceConstraintError && + error.resource === "capacity"), remediation: { action: analysisErrorRemediationAction(error), }, diff --git a/src/domain/sqlite/sqliteDatabase.ts b/src/domain/sqlite/sqliteDatabase.ts new file mode 100644 index 0000000000..60f3c6d3fa --- /dev/null +++ b/src/domain/sqlite/sqliteDatabase.ts @@ -0,0 +1,235 @@ +import { z } from "zod"; +import { localPathStringSchema } from "../localPath.js"; +import type { JsonValue } from "../jsonValue.js"; + +/** Format identity is shared by discovery, catalog, and observed SQLite results. */ +export const SQLITE_PROVIDER_IDENTITY = { + id: "sqlite", + name: "REA SQLite database inspector", + version: null, +} as const; + +/** Default ordinary-table sample size when the caller selects a table. */ +export const SQLITE_ROW_LIMIT_DEFAULT = 20; + +const nonnegative = z.number().int().nonnegative(); +const hex = z.string().regex(/^(?:[0-9a-f]{2})*$/); +const encoding = z.enum(["UTF-8", "UTF-16le", "UTF-16be"]); +const artifactSchema = z.strictObject({ + path: z.string().min(1), + sha256: z.string().regex(/^[0-9a-f]{64}$/), + bytes: nonnegative, +}); + +/** Inspect one frozen database and its sibling WAL, with optional ordinary-table rows. */ +export const inspectSqliteDatabaseInputSchema = z + .strictObject({ + path: localPathStringSchema.describe( + "Absolute path to an offline SQLite database snapshot", + ), + table: z + .string() + .optional() + .describe( + "Exact ordinary table name; views and virtual/shadow tables are not executed", + ), + columns: z + .array(z.string()) + .min(1) + .optional() + .describe( + "Exact row columns in caller-selected order; requires table. Omit to sample all visible columns.", + ), + schema_scope: z + .enum(["inventory", "selected_table", "database"]) + .optional() + .describe( + "Default: compact inventory without a table, or detailed selected-table schema with a table. Database explicitly expands all stored declarations and supported columns.", + ), + row_limit: z + .number() + .int() + .min(1) + .optional() + .describe( + "Selected-table sample size, default 20; subject to byte and worker resource limits", + ), + }) + .superRefine((input, context) => { + if (input.row_limit !== undefined && input.table === undefined) + context.addIssue({ + code: "custom", + path: ["row_limit"], + message: "row_limit requires an explicit table selection", + }); + if (input.columns !== undefined && input.table === undefined) + context.addIssue({ + code: "custom", + path: ["columns"], + message: "columns requires an explicit table selection", + }); + if (input.schema_scope === "selected_table" && input.table === undefined) + context.addIssue({ + code: "custom", + path: ["schema_scope"], + message: "selected_table schema scope requires table", + }); + }) + .meta({ + dependentRequired: { row_limit: ["table"], columns: ["table"] }, + if: { + properties: { schema_scope: { const: "selected_table" } }, + required: ["schema_scope"], + }, + then: { required: ["table"] }, + }); + +/** JSON-safe SQLite cells retain integers, binary content and special REAL values. */ +export const sqliteValueSchema = z.discriminatedUnion("type", [ + z.strictObject({ type: z.literal("null") }), + z.strictObject({ + type: z.literal("integer"), + value: z.string().regex(/^-?(?:0|[1-9][0-9]*)$/), + }), + z.strictObject({ + type: z.literal("real"), + value: z.union([z.number(), z.enum(["Infinity", "-Infinity", "-0"])]), + }), + z.strictObject({ + type: z.literal("text"), + value: z.string(), + bytes_hex: hex, + encoding, + }), + z.strictObject({ type: z.literal("blob"), hex }), +]); + +/** Frozen schema facts and explicitly bounded selected records, without arbitrary SQL. */ +export const sqliteDatabaseSchema = z + .strictObject({ + artifact: artifactSchema, + wal: artifactSchema.nullable(), + engine: z.strictObject({ + name: z.literal("SQLite"), + version: z.string().min(1), + }), + database_encoding: encoding, + schema: z.strictObject({ + scope: z.enum(["inventory", "selected_table", "database"]), + completeness: z + .literal("complete") + .describe( + "Complete within the indicated schema scope; inventory omits declarations and column details", + ), + tables: z.array( + z.strictObject({ + name: z.string(), + sql: z.string().nullable().optional(), + root_page: nonnegative, + kind: z + .enum(["table", "virtual", "shadow", "unknown"]) + .describe( + "Unknown means a possible shadow table of an unresolved virtual table; its rows cannot be inspected", + ), + without_rowid: z.boolean(), + strict: z.boolean(), + column_count: nonnegative.nullable(), + columns_completeness: z.enum([ + "complete", + "unsupported", + "not_requested", + ]), + columns: z.array( + z.strictObject({ + cid: nonnegative, + name: z.string(), + declared_type: z.string(), + not_null: z.boolean(), + default_sql: z.string().nullable(), + primary_key_position: nonnegative, + hidden: z.union([ + z.literal(0), + z.literal(1), + z.literal(2), + z.literal(3), + ]), + }), + ), + }), + ), + indexes: z.array( + z.strictObject({ + name: z.string(), + table: z.string(), + sql: z.string().nullable().optional(), + root_page: nonnegative, + }), + ), + views: z.array( + z.strictObject({ + name: z.string(), + sql: z.string().nullable().optional(), + }), + ), + triggers: z.array( + z.strictObject({ + name: z.string(), + table: z.string(), + sql: z.string().nullable().optional(), + }), + ), + }), + rows: z + .strictObject({ + table: z.string(), + columns: z.array(z.string()), + values: z.array(z.array(sqliteValueSchema)), + row_limit: z.number().int().min(1), + returned_rows: nonnegative, + truncated: z.boolean(), + order: z.literal("unspecified"), + }) + .nullable(), + limitations: z.array(z.string()), + }) + .superRefine((report, context) => { + const rows = report.rows; + if ( + rows !== null && + (rows.returned_rows !== rows.values.length || + rows.returned_rows > rows.row_limit || + rows.values.some((row) => row.length !== rows.columns.length)) + ) + context.addIssue({ + code: "custom", + path: ["rows"], + message: + "Selected row counts and column dimensions must match the retained values", + }); + }); + +export type InspectSqliteDatabaseInput = z.infer< + typeof inspectSqliteDatabaseInputSchema +>; +export type SqliteDatabase = z.infer; +export type SqliteValue = z.infer; + +/** Preserve explicit caller selections while omitting absent optional values from JSON. */ +export const sqliteInspectionParameters = ( + input: InspectSqliteDatabaseInput, +): Readonly> => ({ + path: input.path, + ...(input.table === undefined ? {} : { table: input.table }), + ...(input.columns === undefined ? {} : { columns: input.columns }), + ...(input.schema_scope === undefined + ? {} + : { schema_scope: input.schema_scope }), + ...(input.row_limit === undefined ? {} : { row_limit: input.row_limit }), +}); + +/** One contract owner interprets omitted scope for adapters and selection checks. */ +export const sqliteSchemaScope = ( + input: InspectSqliteDatabaseInput, +): SqliteDatabase["schema"]["scope"] => + input.schema_scope ?? + (input.table === undefined ? "inventory" : "selected_table"); diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts index 0cf42176e8..5c81c5ce0f 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { ANALYSIS_RESOURCE_KINDS } from "../../domain/analysisErrorCore.js"; import { jsonValueSchema } from "../../domain/jsonValue.js"; import { digestSchema } from "../../domain/digests.js"; import { @@ -65,7 +66,7 @@ export const javaScriptWorkerMessageSchema = z.discriminatedUnion("kind", [ kind: z.literal("failure"), ...identity, reason: z.enum(["resource", "output", "io"]), - resource: z.enum(["memory", "cpu", "file-size", "transport"]).nullable(), + resource: z.enum(ANALYSIS_RESOURCE_KINDS).nullable(), limits: z.record(z.string(), jsonValueSchema).nullable(), message: z.string(), }), diff --git a/src/main/transport.ts b/src/main/transport.ts index f9a5cd0ea8..eb7b157d75 100644 --- a/src/main/transport.ts +++ b/src/main/transport.ts @@ -56,6 +56,17 @@ export const startMcpTransport = async ( ); } const androidProviders: AndroidAnalysisPort[] = []; + const servers = new Set>(); + const closeServers = async (): Promise => { + const results = await Promise.allSettled( + [...servers].map((server) => server.close()), + ); + const failures = results.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + if (failures.length > 0) + throw new AggregateError(failures, "REA server cleanup failed"); + }; const closeAndroid = async (): Promise => { const results = await Promise.allSettled( androidProviders.map((provider) => provider.close()), @@ -77,7 +88,7 @@ export const startMcpTransport = async ( serverContext.environment, ); androidProviders.push(android); - return (dependencies.createServer ?? createServer)( + const server = (dependencies.createServer ?? createServer)( { kind: "session", session }, { logger: serverContext.logger, @@ -87,6 +98,14 @@ export const startMcpTransport = async ( androidAnalysis: android, }, ); + // The SDK can discard a server even when its provider cleanup failed. + servers.add(server); + const closeServer = server.close.bind(server); + server.close = async () => { + await closeServer(); + servers.delete(server); + }; + return server; }, { onerror: () => { @@ -96,10 +115,12 @@ export const startMcpTransport = async ( }, ); } catch (cause: unknown) { - const [binaryCleanup, androidCleanup] = await Promise.allSettled([ - session.close(), - closeAndroid(), - ]); + const [binaryCleanup, androidCleanup, serverCleanup] = + await Promise.allSettled([ + session.close(), + closeAndroid(), + closeServers(), + ]); serverLogger.error( { error: cause instanceof Error ? cause.message : String(cause), @@ -115,7 +136,7 @@ export const startMcpTransport = async ( ); dependencies.writeStderr(`${cleanupError.message}\n`); } - for (const cleanup of [binaryCleanup, androidCleanup]) { + for (const cleanup of [binaryCleanup, androidCleanup, serverCleanup]) { if (cleanup.status === "rejected") { const reason = cleanup.reason instanceof Error @@ -130,5 +151,23 @@ export const startMcpTransport = async ( } return { ok: false }; } - return { ok: true, handle, closeAndroid, closeOptionalProviders }; + return { + ok: true, + handle: { + close: async () => { + const results = await Promise.allSettled([ + handle.close(), + closeServers(), + ]); + const failures = results.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + if (failures.length === 1) throw failures[0]; + if (failures.length > 1) + throw new AggregateError(failures, "REA transport cleanup failed"); + }, + }, + closeAndroid, + closeOptionalProviders, + }; }; diff --git a/src/server/contractToolHandler.ts b/src/server/contractToolHandler.ts index ea3e8935b3..0ad6fcd160 100644 --- a/src/server/contractToolHandler.ts +++ b/src/server/contractToolHandler.ts @@ -61,8 +61,19 @@ export const createEvidenceToolHandler = const result = await logToolExecution(logger, name, () => operation(input, { signal: context.mcpReq.signal }), ); - if (!result.ok) + if (!result.ok) { + const partial = result.error.partialObservation; + if (partial !== undefined && "evidence_id" in partial) { + const recorded = recordEvidence?.(partial); + if (recorded !== undefined) + return server.delivery.toRecordedPartialErrorToolResult( + result.error, + partial.evidence_id, + recorded, + ); + } return server.delivery.toCallToolResult(result, contract); + } const recorded = recordEvidence?.(result.value); return server.delivery.toEvidenceToolResult( result.value, diff --git a/src/server/createServer.ts b/src/server/createServer.ts index 844cc5d72f..19d169bd10 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -7,6 +7,9 @@ import { registerPeResourcesTool } from "./registerPeResourcesTool.js"; import type { EvmInterfaceService } from "../application/evm/EvmInterfaceService.js"; import { evmInterfaceModule } from "../composition/evm.js"; import { registerEvmTools } from "./registerEvmTools.js"; +import type { SqliteDatabaseService } from "../application/sqlite/SqliteDatabaseService.js"; +import { createSqliteDatabaseService } from "../composition/sqlite.js"; +import { registerSqliteTools } from "./registerSqliteTools.js"; import { registerRecordedCrashTools } from "./registerRecordedCrashTools.js"; import { createRecordedCrashService } from "../composition/binaryDiagnostics.js"; import { createGoBinaryService } from "../composition/go.js"; @@ -118,6 +121,7 @@ export interface CreateServerOptions { readonly delivery?: ToolResultDelivery; readonly wasmArtifact?: WasmArtifactService; readonly evmInterface?: EvmInterfaceService; + readonly sqliteDatabase?: SqliteDatabaseService; readonly logger?: Logger; readonly binaryLayout?: BinaryLayoutService; readonly goBinary?: GoBinaryService; @@ -243,6 +247,8 @@ export const createServer = ( const delivery = selectToolResultDelivery(environment, options.delivery); const evmInterface = options.evmInterface ?? evmInterfaceModule.create(environment); + const sqliteDatabase = + options.sqliteDatabase ?? createSqliteDatabaseService(environment); const binaryLayout = options.binaryLayout ?? createBinaryLayoutService(environment); const recordedCrash = @@ -262,6 +268,7 @@ export const createServer = ( environment, delivery, evmInterface, + sqliteDatabase, binaryLayout, recordedCrash, webSourceLocation, @@ -302,6 +309,7 @@ export const createServer = ( adbDevice.close(), flutter.close(), evmInterface.close(), + sqliteDatabase.close(), binaryLayout.close(), recordedCrash.close(), webSourceLocation.close(), @@ -520,6 +528,13 @@ const registerConfiguredAnalysisTools = ( recordEvidence, withAdmittedAnalysis, ); + registerSqliteTools( + server, + options.sqliteDatabase ?? createSqliteDatabaseService(environment), + toolLogger, + recordEvidence, + withAdmittedAnalysis, + ); registerRecordedCrashTools( server, options.recordedCrash ?? createRecordedCrashService(environment), diff --git a/src/server/registerSqliteTools.ts b/src/server/registerSqliteTools.ts new file mode 100644 index 0000000000..8d2e07d320 --- /dev/null +++ b/src/server/registerSqliteTools.ts @@ -0,0 +1,31 @@ +import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; +import type { SqliteDatabaseService } from "../application/sqlite/SqliteDatabaseService.js"; +import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; +import { toolContract } from "../contracts/toolContracts.js"; +import type { Logger } from "pino"; +import type { WithAdmittedAnalysis } from "./analysisAdmission.js"; +import { createEvidenceToolHandler } from "./contractToolHandler.js"; +import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; + +/** Bind snapshot inspection to its named contract and additive session Evidence. */ +export const registerSqliteTools = ( + server: EvidenceMcpServer, + service: SqliteDatabaseService, + logger: Logger, + recordEvidence?: EvidenceWriter["recordEvidence"], + withAdmittedAnalysis?: WithAdmittedAnalysis, +): void => { + const contract = toolContract("inspect_sqlite_database"); + const handler = createEvidenceToolHandler( + server, + (input, options) => service.inspect(input, options), + logger, + recordEvidence, + withAdmittedAnalysis, + ); + server.registerTool( + contract.name, + toolRegistrationOptions(contract), + handler(contract), + ); +}; diff --git a/src/sqlite/SqliteDatabaseFailures.ts b/src/sqlite/SqliteDatabaseFailures.ts new file mode 100644 index 0000000000..06bf4e71ed --- /dev/null +++ b/src/sqlite/SqliteDatabaseFailures.ts @@ -0,0 +1,165 @@ +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { + AnalysisError, + type AnalysisCapturedOutput, +} from "../domain/analysisErrorBase.js"; +import { + AnalysisAccessDeniedError, + AnalysisArtifactChangedError, + AnalysisCancelledError, + AnalysisInputError, + AnalysisOutputError, + AnalysisResourceConstraintError, + AnalysisTimeoutError, +} from "../domain/analysisErrorCore.js"; +import { ProviderAdapterError } from "../domain/providerAdapterError.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { OwnedCommandFailure } from "../process/OwnedCommand.js"; +import type { ProviderProcessSnapshot } from "../process/ProviderProcess.js"; +import { SQLITE_PROVIDER_IDENTITY } from "../domain/sqlite/sqliteDatabase.js"; +import { SQLITE_DATABASE_LIMITS } from "./SqliteDatabaseLimits.js"; + +/** Preserve diagnostics independently of worker exit and private-root cleanup. */ +export const capturedSqliteOutput = ( + snapshot: ProviderProcessSnapshot, +): AnalysisCapturedOutput => ({ + stdout: snapshot.stdout.text, + stderr: snapshot.stderr.text, + truncated: snapshot.diagnosticTruncated === true, +}); + +/** Preserve artifact, caller, process, resource and output failure meanings. */ +export const sqliteDatabaseFailure = ( + cause: unknown, + path: string, + phase: string, + signal?: AbortSignal, +): AnalysisError => { + const operation = "inspect_sqlite_database"; + if (cause instanceof AnalysisError) return cause; + if (cause instanceof ArtifactReaderFailure) { + if (cause.cleanup !== undefined) + return new ProviderCleanupError( + SQLITE_PROVIDER_IDENTITY.id, + cause.cleanup.resources, + { + reason: cause.cleanup.reason, + previous_error: + cause.cause === undefined + ? null + : projectAnalysisError( + signal?.aborted + ? new AnalysisCancelledError(operation, { + cause: cause.cause, + }) + : sqliteDatabaseFailure(cause.cause, path, phase), + ), + }, + { operation, cause }, + ); + if (cause.reason === "io" && cause.cause !== undefined) + return sqliteDatabaseFailure(cause.cause, path, phase, signal); + if (cause.reason === "integrity") + return new AnalysisArtifactChangedError(operation, path, cause.message, { + cause, + }); + if (cause.reason === "cancelled") + return new AnalysisCancelledError(operation, { cause }); + return new AnalysisInputError(operation, { cause }, [ + { + path: ["path"], + reason: cause.reason === "limit" ? "out_of_range" : "invalid_format", + message: cause.message, + }, + ]); + } + if (cause instanceof Error && cause.name === "AbortError") + return new AnalysisCancelledError(operation, { cause }); + if (cause instanceof OwnedCommandFailure) { + const projected = ownedCommandFailure(cause); + if (projected !== undefined) return projected; + } + if (phase === "artifact-read" && cause instanceof Error && "code" in cause) { + if (cause.code === "EACCES" || cause.code === "EPERM") + return new AnalysisAccessDeniedError(operation, path, cause.code, { + cause, + }); + if (cause.code === "ENOENT" || cause.code === "ENOTDIR") + return new AnalysisInputError(operation, { cause }, [ + { + path: ["path"], + reason: "invalid_value", + message: `Selected database file set could not be read (${String(cause.code)}): ${path}`, + }, + ]); + } + return new ProviderAdapterError(SQLITE_PROVIDER_IDENTITY.id, operation, { + cause, + diagnostics: { + phase, + path, + reason: cause instanceof Error ? cause.message : String(cause), + ...(cause instanceof OwnedCommandFailure && cause.snapshot !== null + ? { + failure_kind: cause.reason, + exit_code: cause.snapshot.exitCode ?? null, + signal: cause.snapshot.signal ?? null, + captured_output: { ...capturedSqliteOutput(cause.snapshot) }, + } + : {}), + }, + }); +}; + +const ownedCommandFailure = ( + cause: OwnedCommandFailure, +): AnalysisError | undefined => { + const operation = "inspect_sqlite_database"; + const capturedOutput = + cause.snapshot === null ? undefined : capturedSqliteOutput(cause.snapshot); + const options = { + cause, + ...(capturedOutput === undefined ? {} : { capturedOutput }), + }; + if (cause.cleanupFailure !== null) + return new ProviderCleanupError( + SQLITE_PROVIDER_IDENTITY.id, + cause.resources, + { + reason: cause.cleanupFailure, + previous_error: { + failure_kind: cause.reason, + message: cause.message, + }, + ...(capturedOutput === undefined + ? {} + : { captured_output: { ...capturedOutput } }), + }, + { operation, cause }, + ); + if (cause.reason === "cancelled") + return new AnalysisCancelledError(operation, options); + if (cause.reason === "timeout") + return new AnalysisTimeoutError( + operation, + SQLITE_DATABASE_LIMITS.timeoutMs, + options, + ); + if (cause.reason === "output-limit") + return new AnalysisOutputError(operation, cause.message, options); + if ( + cause.snapshot?.stderr.text.includes("JavaScript heap out of memory") === + true + ) + return new AnalysisResourceConstraintError( + operation, + "memory", + "Owned SQLite worker reported JavaScript heap exhaustion.", + { + javascript_heap_bytes: 256 * 1024 * 1024, + }, + options, + ); + return undefined; +}; diff --git a/src/sqlite/SqliteDatabaseInspection.ts b/src/sqlite/SqliteDatabaseInspection.ts new file mode 100644 index 0000000000..f5f92bdf49 --- /dev/null +++ b/src/sqlite/SqliteDatabaseInspection.ts @@ -0,0 +1,514 @@ +import { DatabaseSync } from "node:sqlite"; +import { z } from "zod"; +import { jsonParts } from "../domain/jsonSerialization.js"; +import { + SQLITE_ROW_LIMIT_DEFAULT, + sqliteSchemaScope, + type InspectSqliteDatabaseInput, + type SqliteDatabase, + type SqliteValue, +} from "../domain/sqlite/sqliteDatabase.js"; +import { + SQLITE_DATABASE_LIMITS, + SqliteInspectionFailure, +} from "./SqliteDatabaseLimits.js"; + +const natural = z + .bigint() + .nonnegative() + .max(BigInt(Number.MAX_SAFE_INTEGER)) + .transform(Number); +const schemaRow = z.object({ + type: z.enum(["table", "index", "view", "trigger"]), + name: z.instanceof(Uint8Array), + tbl_name: z.instanceof(Uint8Array), + rootpage: natural, + sql: z.instanceof(Uint8Array).nullable().optional(), +}); +const tableRow = z.object({ + schema: z.string(), + name: z.string(), + type: z.enum(["table", "view", "virtual", "shadow"]), + ncol: natural, + wr: z.bigint(), + strict: z.bigint(), +}); +const columnRow = z.object({ + cid: natural, + name: z.string(), + type: z.string(), + notnull: z.bigint(), + dflt_value: z.string().nullable(), + pk: natural, + hidden: z + .bigint() + .transform(Number) + .pipe(z.union([z.literal(0), z.literal(1), z.literal(2), z.literal(3)])), +}); +const arrayRow = z.array( + z.union([ + z.null(), + z.bigint(), + z.custom( + (value) => typeof value === "number" && !Number.isNaN(value), + ), + z.string(), + ]), +); +const encodingSchema = z.enum(["UTF-8", "UTF-16le", "UTF-16be"]); +const cellByteLengths = z.array(z.bigint().nonnegative()); +type DatabaseEncoding = z.output; +type Inspection = Pick< + SqliteDatabase, + "engine" | "database_encoding" | "schema" | "rows" | "limitations" +>; +type SelectedTableFacts = Pick< + SqliteDatabase["schema"]["tables"][number], + "name" | "kind" | "columns" +>; + +class OutputBudget { + #bytes = 1024; + retain(value: unknown): void { + // Count bounded encoded parts, including escapes, without allocating a + // document-sized string just to decide whether it fits the reply. + this.#bytes += 1; + for (const part of jsonParts(value)) { + this.#bytes += Buffer.byteLength(part); + if (this.#bytes > SQLITE_DATABASE_LIMITS.outputBytes) + throw new SqliteInspectionFailure( + "output-limit", + "Complete SQLite evidence exceeds the 16 MiB JSON reply budget; no partial success was returned.", + ); + } + } + beforeDecode(row: z.output): void { + let minimumBytes = this.#bytes; + for (const value of row) + if (typeof value === "string") minimumBytes += Buffer.byteLength(value); + if (minimumBytes > SQLITE_DATABASE_LIMITS.outputBytes) + throw new SqliteInspectionFailure( + "output-limit", + "Selected cells exceed the 16 MiB reply budget before text/hex expansion.", + ); + } + beforeProjection(lengths: readonly bigint[]): void { + const encodedBytes = lengths.reduce( + (total, length) => total + length * 2n, + BigInt(lengths.length * 64), + ); + if ( + encodedBytes + BigInt(this.#bytes) > + BigInt(SQLITE_DATABASE_LIMITS.outputBytes) + ) + throw new SqliteInspectionFailure( + "output-limit", + "Selected row byte lengths exceed the 16 MiB reply budget before hex projection.", + ); + } +} +const quoteIdentifier = (name: string): string => + `"${name.replaceAll('"', '""')}"`; +// SQLite folds only ASCII letters when matching virtual/shadow table names. +const sqliteIdentifierKey = (name: string): string => + name.replace(/[A-Z]/g, (letter) => letter.toLowerCase()); +const textEncoding = (encoding: DatabaseEncoding): string => + encoding === "UTF-8" + ? "utf-8" + : encoding === "UTF-16le" + ? "utf-16le" + : "utf-16be"; + +const decodeSchemaText = ( + value: Uint8Array, + encoding: DatabaseEncoding, + label: string, +): string => { + try { + return new TextDecoder(textEncoding(encoding), { + fatal: true, + ignoreBOM: true, + }).decode(value); + } catch (cause: unknown) { + throw new SqliteInspectionFailure( + "format", + `SQLite schema ${label} contains malformed ${encoding} text: ${cause instanceof Error ? cause.message : String(cause)}`, + ); + } +}; + +const cell = ( + value: z.output[number], + encoding: DatabaseEncoding, +): SqliteValue => { + if (value === null) return { type: "null" }; + if (typeof value === "bigint") + return { type: "integer", value: String(value) }; + if (typeof value === "number") + return { + type: "real", + value: + value === Infinity + ? "Infinity" + : value === -Infinity + ? "-Infinity" + : Object.is(value, -0) + ? "-0" + : value, + }; + const hex = value.slice(1).toLowerCase(); + if (!/^(?:[0-9a-f]{2})*$/.test(hex)) + throw new SqliteInspectionFailure( + "format", + "SQLite returned an invalid encoded cell.", + ); + if (value[0] === "b") return { type: "blob", hex }; + if (value[0] !== "t") + throw new SqliteInspectionFailure( + "format", + "SQLite returned an unknown encoded cell type.", + ); + return { + type: "text", + value: new TextDecoder(textEncoding(encoding), { ignoreBOM: true }).decode( + Buffer.from(hex, "hex"), + ), + bytes_hex: hex, + encoding, + }; +}; + +const readColumns = ( + db: DatabaseSync, + name: string, + budget: OutputBudget, +): SqliteDatabase["schema"]["tables"][number]["columns"] => { + const columns: SqliteDatabase["schema"]["tables"][number]["columns"] = []; + // Direct PRAGMA syntax cannot be shadowed by a stored table or view. + for (const raw of db + .prepare(`PRAGMA main.table_xinfo(${quoteIdentifier(name)})`) + .iterate()) { + const value = columnRow.parse(raw); + const column = { + cid: value.cid, + name: value.name, + declared_type: value.type, + not_null: value.notnull !== 0n, + default_sql: value.dflt_value, + primary_key_position: value.pk, + hidden: value.hidden, + }; + budget.retain(column); + columns.push(column); + } + return columns; +}; + +const readSchema = ( + db: DatabaseSync, + input: InspectSqliteDatabaseInput, + context: { + readonly budget: OutputBudget; + readonly limitations: string[]; + readonly encoding: DatabaseEncoding; + }, +): { + schema: SqliteDatabase["schema"]; + selectedTable: SelectedTableFacts | undefined; +} => { + const { budget, limitations, encoding } = context; + const scope = sqliteSchemaScope(input); + const selectedName = scope === "selected_table" ? input.table : undefined; + if (scope === "selected_table" && selectedName === undefined) + throw new SqliteInspectionFailure( + "selection", + "Selected-table schema scope requires an exact table name.", + ); + let selectedTable: SelectedTableFacts | undefined; + const metadata = new Map>(); + // Names bound the minimum eventual schema output while metadata is collected. + // The full schema objects below account for them in the actual reply budget. + const metadataBudget = new OutputBudget(); + for (const raw of db.prepare("PRAGMA main.table_list").iterate()) { + const value = tableRow.parse(raw); + if (value.schema === "main") { + metadataBudget.retain(value.name); + metadata.set(value.name, value); + } + } + const unresolvedVirtualTables = [...metadata.values()] + .filter((value) => value.type === "virtual" && value.ncol === 0) + .map((value) => ({ + name: value.name, + prefix: sqliteIdentifierKey(`${value.name}_`), + })); + const schema: SqliteDatabase["schema"] = { + scope, + completeness: "complete", + tables: [], + indexes: [], + views: [], + triggers: [], + }; + // Raw bytes preserve text without hex expansion; avoid sorter records that + // duplicate large names or definitions beyond SQLite's native record limit. + const declaration = scope === "inventory" ? "" : ", CAST(sql AS BLOB) AS sql"; + const selection = + scope === "selected_table" ? " WHERE tbl_name = ? COLLATE BINARY" : ""; + const statement = db.prepare( + `SELECT type, CAST(name AS BLOB) AS name, CAST(tbl_name AS BLOB) AS tbl_name, rootpage${declaration} FROM main.sqlite_schema${selection}`, + ); + for (const raw of statement.iterate( + ...(selectedName === undefined ? [] : [selectedName]), + )) { + const parsed = schemaRow.parse(raw); + const value = { + ...parsed, + name: decodeSchemaText(parsed.name, encoding, "name"), + tbl_name: decodeSchemaText(parsed.tbl_name, encoding, "table"), + sql: + parsed.sql === undefined + ? undefined + : parsed.sql === null + ? null + : decodeSchemaText(parsed.sql, encoding, "sql"), + }; + if (value.type === "table") { + const listed = metadata.get(value.name); + if (listed === undefined || listed.type === "view") + throw new SqliteInspectionFailure( + "format", + `SQLite schema/table inventory mismatch: ${value.name}`, + ); + const identifier = sqliteIdentifierKey(value.name); + const possibleShadowParent = + listed.type === "table" + ? unresolvedVirtualTables.find((parent) => + identifier.startsWith(parent.prefix), + ) + : undefined; + const table: SqliteDatabase["schema"]["tables"][number] = { + name: value.name, + ...(scope === "inventory" ? {} : { sql: value.sql }), + root_page: value.rootpage, + kind: possibleShadowParent === undefined ? listed.type : "unknown", + without_rowid: listed.wr !== 0n, + strict: listed.strict !== 0n, + column_count: + listed.type === "virtual" && listed.ncol === 0 ? null : listed.ncol, + columns_completeness: "complete", + columns: [], + }; + if (possibleShadowParent !== undefined) + limitations.push( + `SQLite could not resolve virtual table ${possibleShadowParent.name}. Table ${value.name} may be its shadow table; its kind is unknown and row inspection is unavailable.`, + ); + budget.retain(table); + if (scope === "inventory") table.columns_completeness = "not_requested"; + if (listed.type === "virtual" && scope !== "inventory") { + table.columns_completeness = "unsupported"; + limitations.push( + `Virtual table column expansion is not performed: ${value.name}. Its complete stored CREATE statement is retained.`, + ); + } else if ( + listed.type !== "virtual" && + (scope !== "inventory" || input.table === value.name) + ) { + const columns = readColumns( + db, + value.name, + scope === "inventory" ? new OutputBudget() : budget, + ); + if (input.table === value.name) + selectedTable = { name: table.name, kind: table.kind, columns }; + if (scope !== "inventory") table.columns = columns; + } + if (input.table === value.name && selectedTable === undefined) + selectedTable = { + name: table.name, + kind: table.kind, + columns: table.columns, + }; + schema.tables.push(table); + } else if (value.type === "index") { + const index = { + name: value.name, + table: value.tbl_name, + ...(scope === "inventory" ? {} : { sql: value.sql }), + root_page: value.rootpage, + }; + budget.retain(index); + schema.indexes.push(index); + } else if (value.type === "view") { + const view = { + name: value.name, + ...(scope === "inventory" ? {} : { sql: value.sql }), + }; + budget.retain(view); + schema.views.push(view); + } else { + const trigger = { + name: value.name, + table: value.tbl_name, + ...(scope === "inventory" ? {} : { sql: value.sql }), + }; + budget.retain(trigger); + schema.triggers.push(trigger); + } + } + return { schema, selectedTable }; +}; + +const readRows = ( + db: DatabaseSync, + input: InspectSqliteDatabaseInput, + table: SelectedTableFacts | undefined, + context: { + readonly encoding: DatabaseEncoding; + readonly budget: OutputBudget; + }, +): SqliteDatabase["rows"] => { + const { encoding, budget } = context; + if (input.table === undefined) return null; + if (table === undefined) + throw new SqliteInspectionFailure( + "selection", + `Selected ordinary table does not exist with that exact name: ${input.table}`, + ); + if (table.kind === "unknown") + throw new SqliteInspectionFailure( + "selection", + `SQLite cannot determine whether selected table ${input.table} is a shadow table of an unresolved virtual table. Row inspection requires a confirmed ordinary table.`, + ); + if (table.kind !== "table") + throw new SqliteInspectionFailure( + "selection", + `Rows from ${table.kind} table ${input.table} are outside the ordinary-table inspection profile.`, + ); + const limit = input.row_limit ?? SQLITE_ROW_LIMIT_DEFAULT; + const visibleColumns = table.columns + .filter((column) => column.hidden !== 1) + .map((column) => column.name); + const columns = input.columns ?? visibleColumns; + const available = new Set(visibleColumns); + for (const name of columns) + if (!available.has(name)) + throw new SqliteInspectionFailure( + "selection", + `Selected column does not exist with that exact name in table ${input.table}: ${name}`, + ); + const projection = columns + .map((name) => { + const column = quoteIdentifier(name); + return `CASE typeof(${column}) WHEN 'text' THEN 't' || hex(CAST(${column} AS BLOB)) WHEN 'blob' THEN 'b' || hex(${column}) ELSE ${column} END`; + }) + .join(", "); + const source = `main.${quoteIdentifier(input.table)} NOT INDEXED`; + const lengths = db.prepare( + `SELECT ${columns + .map((name) => { + const column = quoteIdentifier(name); + return `CASE typeof(${column}) WHEN 'text' THEN length(CAST(${column} AS BLOB)) WHEN 'blob' THEN length(${column}) ELSE 0 END`; + }) + .join(", ")} FROM ${source} LIMIT ?`, + ); + lengths.setReturnArrays(true); + for (const raw of lengths.iterate(limit)) + budget.beforeProjection(cellByteLengths.parse(raw)); + const statement = db.prepare(`SELECT ${projection} FROM ${source} LIMIT ?`); + statement.setReturnArrays(true); + const rows: NonNullable = { + table: input.table, + columns, + values: [], + row_limit: limit, + returned_rows: 0, + truncated: false, + order: "unspecified", + }; + for (const raw of statement.iterate(limit)) { + const rawRow = arrayRow.parse(raw); + // Hex expansion already belongs to the retained-output budget, before decoding/copying cells. + budget.beforeDecode(rawRow); + const values = rawRow.map((value) => cell(value, encoding)); + budget.retain(values); + rows.values.push(values); + } + rows.returned_rows = rows.values.length; + rows.truncated = + db.prepare(`SELECT 1 FROM ${source} LIMIT 1 OFFSET ?`).get(limit) !== + undefined; + return rows; +}; + +/** Query only a provider-owned snapshot; no caller SQL, extensions, views or virtual rows. */ +export const inspectSqliteDatabaseSnapshot = ( + snapshotPath: string, + input: InspectSqliteDatabaseInput, +): Inspection => { + const db = new DatabaseSync(snapshotPath, { + readOnly: true, + allowExtension: false, + readBigInts: true, + enableDoubleQuotedStringLiterals: false, + timeout: 0, + limits: { + length: SQLITE_DATABASE_LIMITS.outputBytes, + sqlLength: SQLITE_DATABASE_LIMITS.outputBytes, + }, + }); + try { + const limits: unknown = db.limits; + const supported = z + .object({ + length: z + .number() + .int() + .positive() + .max(SQLITE_DATABASE_LIMITS.outputBytes), + sqlLength: z + .number() + .int() + .positive() + .max(SQLITE_DATABASE_LIMITS.outputBytes), + }) + .safeParse(limits); + if (!supported.success) + throw new SqliteInspectionFailure( + "unavailable", + "SQLite inspection requires the native DatabaseSync.limits API to bound untrusted values and SQL. Use Node.js 24.15 or newer in the supported 24.x line, or a compatible Node.js 26+ runtime.", + ); + db.exec( + "PRAGMA trusted_schema=OFF; PRAGMA query_only=ON; PRAGMA mmap_size=0; PRAGMA cell_size_check=ON; BEGIN;", + ); + const version = z + .object({ version: z.string() }) + .parse(db.prepare("SELECT sqlite_version() AS version").get()).version; + const encoding = z + .object({ encoding: encodingSchema }) + .parse(db.prepare("PRAGMA encoding").get()).encoding; + const limitations = [ + "Inspects an offline stable file set. Filesystem identity checks detect observed acquisition changes; they do not establish an atomic snapshot of an actively changing database. Supply an application-created backup or quiescent database/WAL pair.", + "The source database and WAL are never opened by SQLite or modified. A private copy may acquire its own WAL-index sidecar and is removed after inspection.", + "SQLite reports current schema and selected committed rows from its valid database/WAL state. Deleted records, uncommitted or invalid WAL tails, encrypted databases, forensic recovery, and full integrity verification are outside this profile.", + "Stored CREATE statements preserve index/view/trigger definitions. Views, triggers, and virtual/shadow table rows are not executed; virtual-table column expansion is unsupported.", + "Selected rows have unspecified order and are a bounded sample. Text display may replace invalid sequences; bytes_hex and encoding preserve SQLite's text byte representation, not original on-disk page offsets.", + "Resource boundaries: at most two concurrent inspections per provider; combined input 256 MiB per inspection, complete worker JSON 16 MiB, native SQLite value/record and SQL statement lengths 16 MiB, owned-worker JavaScript heap 256 MiB and supervised deadline 30 seconds. These are distinct from total resident memory; an overall native SQLite heap cap is not claimed. Failures return no partial success.", + ]; + const budget = new OutputBudget(); + const { schema, selectedTable } = readSchema(db, input, { + budget, + limitations, + encoding, + }); + return { + engine: { name: "SQLite", version }, + database_encoding: encoding, + schema, + rows: readRows(db, input, selectedTable, { encoding, budget }), + limitations, + }; + } finally { + db.close(); + } +}; diff --git a/src/sqlite/SqliteDatabaseLimits.ts b/src/sqlite/SqliteDatabaseLimits.ts new file mode 100644 index 0000000000..99df7ed9d9 --- /dev/null +++ b/src/sqlite/SqliteDatabaseLimits.ts @@ -0,0 +1,23 @@ +/** Bound frozen input, retained JSON and worker allocation before parsing untrusted files. */ +export const SQLITE_DATABASE_LIMITS = { + concurrentInspections: 2, + inputBytes: 256 * 1024 * 1024, + outputBytes: 16 * 1024 * 1024, + timeoutMs: 30_000, + diagnosticBytes: 1024 * 1024, +} as const; +/** Internal rejection retains the failed boundary without returning partial success. */ +export class SqliteInspectionFailure extends Error { + constructor( + readonly reason: + | "format" + | "selection" + | "unsupported" + | "unavailable" + | "output-limit" + | "memory", + message: string, + ) { + super(message); + } +} diff --git a/src/sqlite/SqliteDatabaseProvider.ts b/src/sqlite/SqliteDatabaseProvider.ts new file mode 100644 index 0000000000..c44185616f --- /dev/null +++ b/src/sqlite/SqliteDatabaseProvider.ts @@ -0,0 +1,410 @@ +import { + createAnalysisExecution, + type AnalysisExecution, + type ExecutionOptions, +} from "../application/AnalysisProvider.js"; +import type { SqliteDatabasePort } from "../application/sqlite/SqliteDatabasePort.js"; +import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import type { + AnalysisCapturedOutput, + AnalysisError, +} from "../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisInputError, + AnalysisOutputError, + AnalysisResourceConstraintError, +} from "../domain/analysisErrorCore.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { createEvidence } from "../domain/evidence.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { err, ok, type Result } from "../domain/result.js"; +import { + inspectSqliteDatabaseInputSchema, + sqliteDatabaseSchema, + type InspectSqliteDatabaseInput, + SQLITE_PROVIDER_IDENTITY, + sqliteInspectionParameters, +} from "../domain/sqlite/sqliteDatabase.js"; +import { isAbsolute } from "node:path"; +import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js"; +import { OwnedCommandFailure } from "../process/OwnedCommand.js"; +import type { ProviderProcessSupervisor } from "../process/ProviderProcess.js"; +import { sqliteDatabaseFailure } from "./SqliteDatabaseFailures.js"; +import { captureSqliteDatabaseSnapshot } from "./SqliteDatabaseSnapshot.js"; +import { SQLITE_DATABASE_LIMITS } from "./SqliteDatabaseLimits.js"; + +import { + executeSqliteDatabaseWorker, + type SqliteWorkerLauncher, +} from "./SqliteDatabaseWorkerExecution.js"; + +const operation = "inspect_sqlite_database"; +type SqliteOutcome = Result; +type SqliteRuntimeRoot = Pick; +interface PendingSnapshotCleanup { + readonly input: InspectSqliteDatabaseInput; + readonly result: SqliteOutcome; + readonly output: AnalysisCapturedOutput | undefined; + process: ProviderProcessSupervisor | undefined; +} +/** Inspect a provider-owned copy through a cancellable SQLite child, without source writes. */ +export class SqliteDatabaseProvider implements SqliteDatabasePort { + readonly #active = new Set>(); + readonly #artifactResources = new ArtifactResourceScope(); + readonly #pendingCleanup = new Map< + SqliteRuntimeRoot, + PendingSnapshotCleanup + >(); + #closed = false; + #closePromise: Promise | undefined; + #retryPromise: Promise> | undefined; + constructor( + readonly environment: Readonly = process.env, + readonly launcher?: SqliteWorkerLauncher, + readonly createRuntime: () => Promise< + Pick + > = () => PrivateRuntimeRoot.create({ prefix: "rea-sqlite-database-" }), + ) {} + + /** Return exact original DB/WAL identities and current schema/selected committed rows. */ + async inspect( + input: InspectSqliteDatabaseInput, + options?: ExecutionOptions, + ): Promise { + if (this.#closed || options?.signal?.aborted) + return err(new AnalysisCancelledError(operation)); + if (this.#active.size >= SQLITE_DATABASE_LIMITS.concurrentInspections) + return err( + new AnalysisResourceConstraintError( + operation, + "capacity", + "SQLite inspection capacity is occupied. Retry after an active inspection completes.", + { + active_inspections: this.#active.size, + max_concurrent_inspections: + SQLITE_DATABASE_LIMITS.concurrentInspections, + input_bytes_per_inspection: SQLITE_DATABASE_LIMITS.inputBytes, + }, + { + remediationAction: + "Retry after an active SQLite inspection completes.", + }, + ), + ); + const inspection = this.#artifactResources + .run(() => this.#inspect(input, options)) + .catch((cause: unknown) => + err( + selectedFailure( + sqliteDatabaseFailure( + cause, + input.path, + "artifact-read", + options?.signal, + ), + options?.signal, + ), + ), + ); + this.#active.add(inspection); + try { + return await inspection; + } finally { + this.#active.delete(inspection); + } + } + + /** Await inspections and retry retained descriptors, workers, and snapshot roots. */ + close(): Promise { + this.#closed = true; + this.#closePromise ??= Promise.allSettled(this.#active) + .then(async () => { + let descriptorFailure: AnalysisError | undefined; + try { + await this.#artifactResources.close(); + } catch (cause: unknown) { + const cleanup = ArtifactReaderFailure.cleanupObservation( + cause, + "SQLite artifact descriptors", + ); + descriptorFailure = new ProviderCleanupError( + SQLITE_PROVIDER_IDENTITY.id, + cleanup.resources, + { reason: cleanup.reason }, + { operation, cause }, + ); + } + const cleaned = await this.#retryCleanup(); + if (descriptorFailure !== undefined && !cleaned.ok) + throw new ProviderCleanupError( + SQLITE_PROVIDER_IDENTITY.id, + [ + ...descriptorFailure.cleanupResources, + ...cleaned.error.cleanupResources, + ], + { + failures: [ + projectAnalysisError(descriptorFailure), + projectAnalysisError(cleaned.error), + ], + }, + { operation }, + ); + if (descriptorFailure !== undefined) throw descriptorFailure; + if (!cleaned.ok) throw cleaned.error; + }) + .catch((cause: unknown) => { + this.#closePromise = undefined; + throw cause; + }); + return this.#closePromise; + } + + #retryCleanup(): Promise> { + this.#retryPromise ??= this.#cleanupPending().finally(() => { + this.#retryPromise = undefined; + }); + return this.#retryPromise; + } + + async #cleanupPending(): Promise> { + const failures: AnalysisError[] = []; + for (const [root, pending] of [...this.#pendingCleanup]) { + if (pending.process !== undefined) { + const stopped = await pending.process.stop(); + if (stopped.status === "incomplete") { + failures.push( + new ProviderCleanupError( + SQLITE_PROVIDER_IDENTITY.id, + [ + pending.process.launch.ownership?.runId ?? + "owned-sqlite-worker", + root.path, + ], + { + reason: stopped.reason, + previous_error: pending.result.ok + ? null + : projectAnalysisError(pending.result.error), + ...(pending.output === undefined + ? {} + : { captured_output: { ...pending.output } }), + }, + { operation }, + ), + ); + continue; + } + pending.process = undefined; + } + const cleaned = await cleanupSqliteRoot( + root, + pending.result, + pending.input, + pending.output, + ); + if (cleaned.ok) this.#pendingCleanup.delete(root); + else failures.push(cleaned.error); + } + if (failures.length === 1 && failures[0] !== undefined) + return err(failures[0]); + return failures.length === 0 + ? ok(null) + : err( + new ProviderCleanupError( + SQLITE_PROVIDER_IDENTITY.id, + failures.flatMap((failure) => failure.cleanupResources), + { failures: failures.map(projectAnalysisError) }, + { operation }, + ), + ); + } + + async #inspect( + input: InspectSqliteDatabaseInput, + options?: ExecutionOptions, + ): Promise { + let root: Pick | undefined; + let processOwner: ProviderProcessSupervisor | undefined; + let retainedOutput: AnalysisCapturedOutput | undefined; + let phase = "configuration"; + let result: Result; + try { + if (options?.signal?.aborted) throw new AnalysisCancelledError(operation); + validateSqliteInput(input); + const retired = await this.#retryCleanup(); + if (!retired.ok) return retired; + if (this.#closed || options?.signal?.aborted) + throw new AnalysisCancelledError(operation); + root = await this.createRuntime(); + phase = "artifact-read"; + const snapshot = await captureSqliteDatabaseSnapshot( + input.path, + root.path, + this.#artifactResources, + options?.signal, + ); + phase = "worker"; + const inspection = await executeSqliteDatabaseWorker({ + input, + root: root.path, + snapshotPath: snapshot.snapshotPath, + environment: this.environment, + resourceScope: this.#artifactResources, + ...(this.launcher === undefined ? {} : { launcher: this.launcher }), + ...(options?.signal === undefined ? {} : { signal: options.signal }), + captured: (output) => { + retainedOutput = output; + }, + }); + const validated = sqliteDatabaseSchema.safeParse({ + artifact: snapshot.artifact, + wal: snapshot.wal, + ...inspection, + }); + if (!validated.success) + throw new AnalysisOutputError( + operation, + "SQLite worker report dimensions or identity fields are invalid", + retainedOutput === undefined + ? undefined + : { capturedOutput: retainedOutput }, + ); + result = ok(createSqliteObservation(validated.data, input.path)); + } catch (cause: unknown) { + if (cause instanceof OwnedCommandFailure) + processOwner = cause.cleanupOwner; + const failure = sqliteDatabaseFailure( + cause, + input.path, + phase, + options?.signal, + ); + retainedOutput ??= failure.capturedOutput; + result = err(selectedFailure(failure, options?.signal, retainedOutput)); + } + if (root !== undefined) { + if (processOwner !== undefined) { + this.#pendingCleanup.set(root, { + input, + result, + output: retainedOutput, + process: processOwner, + }); + return result; + } + const cleaned = await cleanupSqliteRoot( + root, + result, + input, + retainedOutput, + ); + if (!cleaned.ok) { + this.#pendingCleanup.set(root, { + input, + result, + output: retainedOutput, + process: undefined, + }); + return cleaned; + } + } + // Completion was acknowledged before cleanup. A later abort does not erase it. + return result; + } +} + +const selectedFailure = ( + failure: AnalysisError, + signal?: AbortSignal, + output?: AnalysisCapturedOutput, +): AnalysisError => + signal?.aborted && + failure._tag !== "AnalysisCancelledError" && + !failure.cleanupIncomplete + ? new AnalysisCancelledError( + operation, + output === undefined ? undefined : { capturedOutput: output }, + ) + : failure; +const createSqliteObservation = ( + report: ReturnType, + path: string, +): AnalysisExecution => { + const locations: AnalysisExecution["locations"][number][] = [ + { kind: "artifact-path", path }, + ]; + if (report.wal !== null) + locations.push({ kind: "artifact-path", path: report.wal.path }); + return createAnalysisExecution( + report, + { ...SQLITE_PROVIDER_IDENTITY, version: `SQLite ${report.engine.version}` }, + { + rawResult: null, + subject: { path, format: "file", sha256: report.artifact.sha256 }, + locations, + limitations: report.limitations, + }, + ); +}; +const cleanupSqliteRoot = async ( + root: Pick, + result: Result, + input: InspectSqliteDatabaseInput, + output?: AnalysisCapturedOutput, +): Promise> => { + try { + await root.close(); + return ok(null); + } catch (cause: unknown) { + return err( + new ProviderCleanupError( + SQLITE_PROVIDER_IDENTITY.id, + [root.path], + { + reason: cause instanceof Error ? cause.message : String(cause), + previous_error: result.ok ? null : projectAnalysisError(result.error), + ...(output === undefined ? {} : { captured_output: { ...output } }), + }, + { + operation, + cause, + ...(result.ok + ? { + partialObservation: createEvidence( + result.value.subject ?? undefined, + result.value.provider, + { + operation, + parameters: sqliteInspectionParameters(input), + result: result.value.result, + confidence: "observed", + locations: result.value.locations, + limitations: result.value.limitations, + }, + ), + } + : result.error.partialObservation === undefined + ? {} + : { partialObservation: result.error.partialObservation }), + }, + ), + ); + } +}; + +const validateSqliteInput = (input: InspectSqliteDatabaseInput): void => { + const parsed = inspectSqliteDatabaseInputSchema.safeParse(input); + if (!parsed.success || !isAbsolute(input.path)) + throw new AnalysisInputError(operation, undefined, [ + { + path: ["path"], + reason: "invalid_format", + message: parsed.success + ? "path must be an absolute local filesystem path" + : parsed.error.message, + }, + ]); +}; diff --git a/src/sqlite/SqliteDatabaseSnapshot.ts b/src/sqlite/SqliteDatabaseSnapshot.ts new file mode 100644 index 0000000000..4acf896113 --- /dev/null +++ b/src/sqlite/SqliteDatabaseSnapshot.ts @@ -0,0 +1,270 @@ +import { lstat, open, writeFile } from "node:fs/promises"; +import { constants, type BigIntStats } from "node:fs"; +import { join } from "node:path"; +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { + readStableArtifact, + type StableArtifactFileSystem, +} from "../artifacts/readStableArtifact.js"; +import { SQLITE_DATABASE_LIMITS } from "./SqliteDatabaseLimits.js"; + +const fileSystem: StableArtifactFileSystem = { + lstat: (path) => lstat(path, { bigint: true }), + open, +}; +const optionalStat = async ( + path: string, + selectedFileSystem: StableArtifactFileSystem, +): Promise => { + try { + return await selectedFileSystem.lstat(path); + } catch (cause: unknown) { + if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") + return null; + throw cause; + } +}; +const sameState = ( + before: BigIntStats | null, + after: BigIntStats | null, +): boolean => + before === null || after === null + ? before === after + : before.dev === after.dev && + before.ino === after.ino && + before.mode === after.mode && + before.size === after.size && + before.mtimeNs === after.mtimeNs && + before.ctimeNs === after.ctimeNs; + +const duringAcquisition = async ( + path: string, + read: () => Promise, +): Promise => { + try { + return await read(); + } catch (cause: unknown) { + if ( + cause instanceof Error && + "code" in cause && + ["ENOENT", "ENOTDIR", "ELOOP"].includes(String(cause.code)) + ) + throw new ArtifactReaderFailure( + "integrity", + `SQLite snapshot file set changed during acquisition: ${path}`, + { cause }, + ); + throw cause; + } +}; + +const readObservedArtifact = ( + path: string, + observed: BigIntStats, + maximumBytes: number, + selectedFileSystem: StableArtifactFileSystem, + resourceScope: ArtifactResourceScope, + signal?: AbortSignal, +) => + duringAcquisition(path, () => + readStableArtifact( + path, + maximumBytes, + signal, + { + open: (selectedPath, flags) => + selectedFileSystem.open(selectedPath, flags), + lstat: async (selectedPath) => { + const current = await selectedFileSystem.lstat(selectedPath); + if (!sameState(observed, current)) + throw new ArtifactReaderFailure( + "integrity", + `SQLite snapshot file set changed during acquisition: ${selectedPath}`, + ); + return current; + }, + }, + resourceScope, + ), + ); + +const verifyInactiveJournal = async ( + path: string, + before: BigIntStats, + selectedFileSystem: StableArtifactFileSystem, + resourceScope: ArtifactResourceScope, + phase: "initial" | "after-copy", + signal?: AbortSignal, +): Promise => { + if (!before.isFile() || before.isSymbolicLink()) + throw new ArtifactReaderFailure( + "path", + `Expected a regular journal without a symlink: ${path}`, + ); + const file = await duringAcquisition(path, () => + selectedFileSystem.open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ), + ); + const owner = new OwnedFileHandle(file); + let failure: unknown; + try { + if (!sameState(before, await file.stat({ bigint: true }))) + throw new ArtifactReaderFailure( + "integrity", + `SQLite journal changed before open: ${path}`, + ); + // TRUNCATE commits leave an empty file; PERSIST commits zero the complete + // 28-byte fixed header. A new active journal may zero only its magic bytes. + const header = Buffer.alloc(28); + let bytesRead = 0; + if (before.size > 0n) { + while (bytesRead < header.length) { + signal?.throwIfAborted(); + const read = await file.read( + header, + bytesRead, + header.length - bytesRead, + bytesRead, + ); + if (read.bytesRead === 0) break; + bytesRead += read.bytesRead; + } + } + if (!sameState(before, await file.stat({ bigint: true }))) + throw new ArtifactReaderFailure( + "integrity", + `SQLite journal changed during inspection: ${path}`, + ); + if ( + before.size !== 0n && + (bytesRead !== header.length || header.some((byte) => byte !== 0)) + ) + throw new ArtifactReaderFailure( + phase === "initial" ? "format" : "integrity", + phase === "initial" + ? `Rollback journal may require recovery: ${path}. Supply a clean offline snapshot; only empty journals or a fully zeroed header are supported.` + : `Rollback journal became active or malformed during database acquisition: ${path}`, + ); + } catch (cause: unknown) { + failure = cause; + throw cause; + } finally { + const cleanup = await resourceScope.release({ + kind: "file-handle", + resource: path, + handle: owner, + }); + if (cleanup.kind === "failed") + throw ArtifactReaderFailure.withCleanup( + failure ?? cleanup.cause, + ArtifactReaderFailure.cleanupObservation(cleanup.cause, path), + ); + } +}; + +/** Freeze a stable offline file set without ever opening the source through SQLite. */ +export const captureSqliteDatabaseSnapshot = async ( + path: string, + root: string, + resourceScope: ArtifactResourceScope, + signal?: AbortSignal, + selectedFileSystem: StableArtifactFileSystem = fileSystem, +) => { + signal?.throwIfAborted(); + const paths = [path, `${path}-wal`, `${path}-journal`]; + const before = await Promise.all( + paths.map((path) => optionalStat(path, selectedFileSystem)), + ); + const database = before[0]; + const wal = before[1]; + if (database === undefined || wal === undefined || before[2] === undefined) + throw new Error("Incomplete database file-set observation"); + if (database === null) + throw new ArtifactReaderFailure( + "path", + `Selected SQLite database does not exist: ${path}`, + ); + if (before[2] !== null) + await verifyInactiveJournal( + `${path}-journal`, + before[2], + selectedFileSystem, + resourceScope, + "initial", + signal, + ); + const selectedSize = database.size + (wal?.size ?? 0n); + if (selectedSize > BigInt(SQLITE_DATABASE_LIMITS.inputBytes)) + throw new ArtifactReaderFailure( + "limit", + `Selected SQLite database and WAL exceed the combined ${String(SQLITE_DATABASE_LIMITS.inputBytes)}-byte snapshot budget: ${path}`, + ); + const snapshotPath = join(root, "database.snapshot"); + const captured = await readObservedArtifact( + path, + database, + SQLITE_DATABASE_LIMITS.inputBytes, + selectedFileSystem, + resourceScope, + signal, + ); + await writeFile(snapshotPath, captured.bytes, { + flag: "wx", + mode: 0o600, + ...(signal === undefined ? {} : { signal }), + }); + const artifact = { + path, + sha256: captured.sha256, + bytes: captured.bytes.length, + }; + let walArtifact: typeof artifact | null = null; + if (wal !== null) { + const capturedWal = await readObservedArtifact( + `${path}-wal`, + wal, + SQLITE_DATABASE_LIMITS.inputBytes - artifact.bytes, + selectedFileSystem, + resourceScope, + signal, + ); + await writeFile(`${snapshotPath}-wal`, capturedWal.bytes, { + flag: "wx", + mode: 0o600, + ...(signal === undefined ? {} : { signal }), + }); + walArtifact = { + path: `${path}-wal`, + sha256: capturedWal.sha256, + bytes: capturedWal.bytes.length, + }; + } + // Identity alone does not establish an inactive journal when timestamps are + // coarse. Re-read the header after copying, using the same validity owner. + if (before[2] !== null) + await verifyInactiveJournal( + `${path}-journal`, + before[2], + selectedFileSystem, + resourceScope, + "after-copy", + signal, + ); + const after = await Promise.all( + paths.map((path) => + duringAcquisition(path, () => optionalStat(path, selectedFileSystem)), + ), + ); + for (const [index, expected] of before.entries()) + if (!sameState(expected, after[index] ?? null)) + throw new ArtifactReaderFailure( + "integrity", + `SQLite snapshot file set changed during acquisition: ${paths[index] ?? path}`, + ); + signal?.throwIfAborted(); + return { snapshotPath, artifact, wal: walArtifact }; +}; diff --git a/src/sqlite/SqliteDatabaseWorker.ts b/src/sqlite/SqliteDatabaseWorker.ts new file mode 100644 index 0000000000..ebd2d5bac1 --- /dev/null +++ b/src/sqlite/SqliteDatabaseWorker.ts @@ -0,0 +1,57 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { z } from "zod"; +import { inspectSqliteDatabaseInputSchema } from "../domain/sqlite/sqliteDatabase.js"; +import { inspectSqliteDatabaseSnapshot } from "./SqliteDatabaseInspection.js"; +import { + SQLITE_DATABASE_LIMITS, + SqliteInspectionFailure, +} from "./SqliteDatabaseLimits.js"; + +const requestSchema = z.strictObject({ + snapshot_path: z.string(), + reply_path: z.string(), + input: inspectSqliteDatabaseInputSchema, +}); +const main = async (): Promise => { + const requestPath = process.argv[2]; + if (requestPath === undefined) + throw new Error("Owned SQLite worker request path missing."); + const request = requestSchema.parse( + JSON.parse(await readFile(requestPath, "utf8")), + ); + let reply: unknown; + try { + reply = { + ok: true, + inspection: inspectSqliteDatabaseSnapshot( + request.snapshot_path, + request.input, + ), + }; + } catch (cause: unknown) { + const code = + cause instanceof Error && "errcode" in cause ? cause.errcode : undefined; + reply = { + ok: false, + reason: + cause instanceof SqliteInspectionFailure + ? cause.reason + : code === 7 + ? "memory" + : code === 18 + ? "output-limit" + : "format", + message: cause instanceof Error ? cause.message : String(cause), + }; + } + let encoded = JSON.stringify(reply); + if (Buffer.byteLength(encoded) > SQLITE_DATABASE_LIMITS.outputBytes) + encoded = JSON.stringify({ + ok: false, + reason: "output-limit", + message: + "Complete SQLite evidence exceeds the 16 MiB JSON reply budget; no partial success was returned.", + }); + await writeFile(request.reply_path, encoded, { flag: "wx", mode: 0o600 }); +}; +await main(); diff --git a/src/sqlite/SqliteDatabaseWorkerExecution.ts b/src/sqlite/SqliteDatabaseWorkerExecution.ts new file mode 100644 index 0000000000..9474a6d27c --- /dev/null +++ b/src/sqlite/SqliteDatabaseWorkerExecution.ts @@ -0,0 +1,169 @@ +import { randomUUID } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { z } from "zod"; +import type { AnalysisCapturedOutput } from "../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisInputError, + AnalysisCapabilityUnavailableError, + AnalysisOutputError, + AnalysisResourceConstraintError, +} from "../domain/analysisErrorCore.js"; +import { + sqliteDatabaseSchema, + type InspectSqliteDatabaseInput, +} from "../domain/sqlite/sqliteDatabase.js"; +import { readStableArtifact } from "../artifacts/readStableArtifact.js"; +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { runOwnedCommand } from "../process/OwnedCommand.js"; +import { capturedSqliteOutput } from "./SqliteDatabaseFailures.js"; +import { SQLITE_DATABASE_LIMITS } from "./SqliteDatabaseLimits.js"; + +const operation = "inspect_sqlite_database"; +const { + artifact: _artifactSchema, + wal: _walSchema, + ...inspectionShape +} = sqliteDatabaseSchema.shape; +const inspectionSchema = z.strictObject(inspectionShape); +const replySchema = z.discriminatedUnion("ok", [ + z.strictObject({ ok: z.literal(true), inspection: inspectionSchema }), + z.strictObject({ + ok: z.literal(false), + reason: z.enum([ + "format", + "selection", + "unsupported", + "unavailable", + "output-limit", + "memory", + ]), + message: z.string(), + }), +]); +/** Replaceable launcher retains the shared owned-process supervision boundary. */ +export type SqliteWorkerLauncher = NonNullable< + Parameters[2] +>["launcher"]; + +const rejectedReply = ( + reply: z.output & { ok: false }, + capturedOutput: AnalysisCapturedOutput, +) => { + if (reply.reason === "unavailable") + return new AnalysisCapabilityUnavailableError( + "sqlite", + operation, + reply.message, + { capturedOutput, userMessage: reply.message }, + ); + if (reply.reason === "memory") + return new AnalysisResourceConstraintError( + operation, + "memory", + reply.message, + { + javascript_heap_bytes: 256 * 1024 * 1024, + }, + { capturedOutput }, + ); + if (reply.reason === "output-limit") + return new AnalysisResourceConstraintError( + operation, + "transport", + reply.message, + { + sqlite_value_or_record_bytes: SQLITE_DATABASE_LIMITS.outputBytes, + worker_reply_bytes: SQLITE_DATABASE_LIMITS.outputBytes, + }, + { capturedOutput }, + ); + return new AnalysisInputError(operation, { capturedOutput }, [ + { + path: [reply.reason === "selection" ? "table" : "path"], + reason: "invalid_format", + message: reply.message, + }, + ]); +}; + +/** Execute only the private snapshot and validate the bounded worker protocol. */ +export const executeSqliteDatabaseWorker = async (request: { + readonly input: InspectSqliteDatabaseInput; + readonly root: string; + readonly snapshotPath: string; + readonly environment: Readonly; + readonly resourceScope: ArtifactResourceScope; + readonly launcher?: SqliteWorkerLauncher; + readonly signal?: AbortSignal; + readonly captured: (output: AnalysisCapturedOutput) => void; +}): Promise> => { + const requestPath = join(request.root, "request.json"); + const replyPath = join(request.root, "reply.json"); + await writeFile( + requestPath, + JSON.stringify({ + snapshot_path: request.snapshotPath, + reply_path: replyPath, + input: request.input, + }), + { flag: "wx", mode: 0o600 }, + ); + const { NODE_OPTIONS: _ambientNodeOptions, ...environment } = + request.environment; + const execution = await runOwnedCommand( + { + command: process.execPath, + arguments: [ + "--max-old-space-size=256", + fileURLToPath(new URL("./SqliteDatabaseWorker.js", import.meta.url)), + requestPath, + ], + cwd: request.root, + expectedCommand: null, + runId: `rea-sqlite-database-${randomUUID()}`, + hostEnvironment: environment, + }, + { + timeoutMs: SQLITE_DATABASE_LIMITS.timeoutMs, + diagnosticBytes: SQLITE_DATABASE_LIMITS.diagnosticBytes, + }, + { + ...(request.signal === undefined ? {} : { signal: request.signal }), + ...(request.launcher === undefined ? {} : { launcher: request.launcher }), + }, + ); + const capturedOutput = capturedSqliteOutput(execution); + request.captured(capturedOutput); + let reply: z.output; + try { + const file = await readStableArtifact( + replyPath, + SQLITE_DATABASE_LIMITS.outputBytes, + request.signal, + undefined, + request.resourceScope, + ); + reply = replySchema.parse(JSON.parse(file.bytes.toString("utf8"))); + } catch (cause: unknown) { + const options = { + cause, + capturedOutput, + ...(cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined + ? { cleanup: cause.cleanup } + : {}), + }; + if (request.signal?.aborted) + throw new AnalysisCancelledError(operation, options); + throw new AnalysisOutputError( + operation, + `Owned SQLite worker reply is missing or malformed: ${cause instanceof Error ? cause.message : String(cause)}`, + options, + ); + } + if (!reply.ok) throw rejectedReply(reply, capturedOutput); + return reply.inspection; +}; diff --git a/tests/acceptance/sqliteDatabaseCli.test.ts b/tests/acceptance/sqliteDatabaseCli.test.ts new file mode 100644 index 0000000000..d23c032a6e --- /dev/null +++ b/tests/acceptance/sqliteDatabaseCli.test.ts @@ -0,0 +1,597 @@ +import { createHash } from "node:crypto"; +import { readFile, stat, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { expect, onTestFinished } from "vitest"; +import { z } from "zod"; +import { + expandProviderCapabilities, + expandToolAvailability, + providerCapabilitySet, + toolAvailabilityInventory, +} from "../../src/contracts/toolOutputSchemaPrimitives.js"; + +import { SQLITE_NATIVE_LIMITS_AVAILABLE } from "../fixtures/sqlite/database.js"; +import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; +import { parseMcpToolError } from "../fixtures/mcpToolError.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +const digest = (bytes: Buffer): string => + createHash("sha256").update(bytes).digest("hex"); + +cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "bounds default context and projects exact columns without evaluating unrelated cells", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-selective-"); + const path = join(root, "selected.db"); + const database = new DatabaseSync(path); + try { + database.exec(` + CREATE TABLE selected(id INTEGER, "" TEXT, payload BLOB GENERATED ALWAYS AS (zeroblob(72*1024*1024)) VIRTUAL); + CREATE TABLE unrelated(value TEXT); + CREATE VIEW unrelated_view AS SELECT value FROM unrelated; + CREATE INDEX selected_id ON selected(id); + CREATE TRIGGER selected_trigger AFTER INSERT ON selected BEGIN SELECT 1; END; + WITH RECURSIVE numbers(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM numbers WHERE n<21) + INSERT INTO selected(id, "") SELECT n, 'retained' FROM numbers; + `); + } finally { + database.close(); + } + const before = await readFile(path); + const { call } = await connectLocalToolsMcp(); + const inventory = await call("inspect_sqlite_database", { path }); + expect(inventory.isError).not.toBe(true); + const schema = z + .object({ + normalized_result: z.object({ + schema: z.object({ + scope: z.literal("inventory"), + tables: z.array( + z.object({ + name: z.string(), + columns: z.array(z.unknown()), + columns_completeness: z.literal("not_requested"), + }), + ), + }), + }), + }) + .parse(inventory.structuredContent).normalized_result.schema; + expect(schema.tables.map((table) => table.name)).toEqual([ + "selected", + "unrelated", + ]); + expect(schema.tables.every((table) => table.columns.length === 0)).toBe( + true, + ); + expect(JSON.stringify(inventory.structuredContent)).not.toContain( + "zeroblob", + ); + const selected = await call("inspect_sqlite_database", { + path, + table: "selected", + columns: ["", "id", "id"], + }); + expect(selected.isError).not.toBe(true); + expect(selected.structuredContent).toMatchObject({ + normalized_result: { + schema: { + scope: "selected_table", + tables: [{ name: "selected" }], + views: [], + indexes: [{ name: "selected_id" }], + triggers: [{ name: "selected_trigger" }], + }, + rows: { + columns: ["", "id", "id"], + row_limit: 20, + returned_rows: 20, + truncated: true, + values: expect.arrayContaining([ + [ + { + type: "text", + value: "retained", + bytes_hex: "72657461696e6564", + encoding: "UTF-8", + }, + { type: "integer", value: "1" }, + { type: "integer", value: "1" }, + ], + ]), + }, + }, + }); + const cliResult = await cli.run({ + arguments: [ + "inspect-sqlite-database", + path, + "--table", + "selected", + "--column", + "id", + "--column", + "", + "--row-limit", + "1", + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(cliResult.exitCode, cliResult.stdout + cliResult.stderr).toBe(0); + expect(cliResult.json).toMatchObject({ + normalized_result: { rows: { columns: ["id", ""], returned_rows: 1 } }, + }); + for (const input of [ + { path, columns: ["id"] }, + { path, schema_scope: "selected_table" }, + { path, table: "selected", columns: [] }, + { path, table: "selected", columns: ["ID"] }, + ]) + expect((await call("inspect_sqlite_database", input)).isError).toBe(true); + expect(await readFile(path)).toEqual(before); + }, +); + +cliTest( + "discovers SQLite runtime availability before inspecting a database through CLI and MCP", + async ({ cli }) => { + const result = await cli.run({ + arguments: ["capabilities", "--json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode, result.stdout + result.stderr).toBe(0); + const status = z + .object({ + open: z.literal(false), + capabilities: z.array(providerCapabilitySet), + }) + .parse(result.json); + expect(expandProviderCapabilities(status.capabilities)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + operation: "inspect_sqlite_database", + available: SQLITE_NATIVE_LIMITS_AVAILABLE, + reason: SQLITE_NATIVE_LIMITS_AVAILABLE + ? null + : expect.stringContaining("Node.js 24.x >=24.15 or 26+"), + availability_code: SQLITE_NATIVE_LIMITS_AVAILABLE + ? null + : "unsupported_version", + limitations: [expect.stringContaining("DatabaseSync.limits")], + effects: { + mutates_artifact: false, + launches_process: true, + may_show_ui: false, + may_access_network: false, + may_write_filesystem: true, + changes_permissions: false, + requires_root: false, + }, + }), + ]), + ); + const { call } = await connectLocalToolsMcp(); + const response = await call("binary_session", {}); + expect(response.isError).not.toBe(true); + const sessionStatus = z + .object({ + result: z.object({ tool_availability: toolAvailabilityInventory }), + }) + .parse(response.structuredContent); + expect( + expandToolAvailability(sessionStatus.result.tool_availability), + ).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: "inspect_sqlite_database", + available: SQLITE_NATIVE_LIMITS_AVAILABLE, + remediation: SQLITE_NATIVE_LIMITS_AVAILABLE + ? null + : expect.stringContaining("Node.js 24.x >=24.15 or 26+"), + }), + ]), + ); + }, +); + +cliTest.skipIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "reports the required native SQLite limits capability through CLI and MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-unavailable-"); + const path = join(root, "selected.db"); + const database = new DatabaseSync(path); + try { + database.exec("CREATE TABLE items(value INTEGER)"); + } finally { + database.close(); + } + const result = await cli.run({ + arguments: ["inspect-sqlite-database", path, "--json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + code: "capability_unavailable", + message: expect.stringContaining("DatabaseSync.limits"), + }); + const { call } = await connectLocalToolsMcp(); + const response = await call("inspect_sqlite_database", { path }); + expect(response.isError).toBe(true); + expect(parseMcpToolError(response).error).toMatchObject({ + code: "capability_unavailable", + message: expect.stringContaining("DatabaseSync.limits"), + }); + }, +); + +cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "accepts small samples above 1000 rows within the real resource budgets", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-row-budget-"); + const path = join(root, "rows.db"); + const database = new DatabaseSync(path); + try { + database.exec(`CREATE TABLE items(value INTEGER); + WITH RECURSIVE numbers(value) AS (SELECT 1 UNION ALL SELECT value + 1 FROM numbers WHERE value < 1002) + INSERT INTO items SELECT value FROM numbers;`); + } finally { + database.close(); + } + const { call } = await connectLocalToolsMcp(); + for (const limit of [1001, Number.MAX_SAFE_INTEGER]) { + const result = await cli.run({ + arguments: [ + "inspect-sqlite-database", + path, + "--table", + "items", + "--row-limit", + String(limit), + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode, result.stdout + result.stderr).toBe(0); + expect(result.json).toMatchObject({ + normalized_result: { + rows: { + row_limit: limit, + returned_rows: Math.min(limit, 1002), + truncated: limit < 1002, + }, + }, + }); + const response = await call("inspect_sqlite_database", { + path, + table: "items", + row_limit: limit, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(response.structuredContent).toEqual(result.json); + } + }, +); + +const sourceState = async (path: string) => { + const bytes = await readFile(path); + const metadata = await stat(path, { bigint: true }); + return { + bytes, + device: metadata.dev, + inode: metadata.ino, + size: metadata.size, + modified: metadata.mtimeNs, + changed: metadata.ctimeNs, + }; +}; + +const createLosslessSnapshot = (path: string, journalMode: string): void => { + const database = new DatabaseSync(path); + try { + database.exec(`PRAGMA journal_mode=${journalMode}`); + database.exec(` + CREATE TABLE "odd "" table" ( + id INTEGER PRIMARY KEY, + payload BLOB, + amount REAL, + missing TEXT, + text_value TEXT, + negative_zero, + generated TEXT GENERATED ALWAYS AS (hex(payload)) VIRTUAL + ); + CREATE UNIQUE INDEX "quoted "" index" ON "odd "" table"(payload, id); + CREATE VIEW readable AS SELECT id FROM "odd "" table"; + CREATE TRIGGER observed AFTER INSERT ON "odd "" table" BEGIN SELECT 1; END; + CREATE TABLE keyed(k TEXT PRIMARY KEY) WITHOUT ROWID, STRICT; + `); + const insert = database.prepare( + 'INSERT INTO "odd "" table"(id,payload,amount,missing,text_value,negative_zero) VALUES(?,?,?,NULL,CAST(X\'80ff\' AS TEXT),?)', + ); + insert.setReadBigInts(true); + expect( + insert.run( + 9007199254740993n, + Buffer.from("deadbeef", "hex"), + Infinity, + -0, + ).lastInsertRowid, + ).toBe(9007199254740993n); + insert.run(9007199254740994n, Buffer.from("deadbeef", "hex"), Infinity, -0); + } finally { + database.close(); + } +}; + +cliTest + .runIf(SQLITE_NATIVE_LIMITS_AVAILABLE) + .for(["DELETE", "PERSIST", "TRUNCATE"])( + "inspects a real %s SQLite schema and lossless selected records identically through CLI and MCP", + async (journalMode, { cli }) => { + const root = await createTestTempDirectory("rea-sqlite-public-"); + const path = join(root, "selected.db"); + createLosslessSnapshot(path, journalMode); + const original = await readFile(path); + const result = await cli.run({ + arguments: [ + "inspect-sqlite-database", + "selected.db", + "--schema-scope", + "database", + "--table", + 'odd " table', + "--row-limit", + "1", + "--json", + ], + cwd: root, + environment: { + REA_LOG_LEVEL: "silent", + REA_ANALYSIS_PROVIDER: "ghidra", + REA_GHIDRA_HOME: "/unconfigured/ghidra", + }, + }); + expect(result.exitCode, result.stdout + result.stderr).toBe(0); + expect(result.json).toMatchObject({ + operation: "inspect_sqlite_database", + confidence: "observed", + subject: { local_path: path, digest: { sha256: digest(original) } }, + normalized_result: { + artifact: { path, sha256: digest(original), bytes: original.length }, + wal: null, + schema: { + completeness: "complete", + tables: expect.arrayContaining([ + expect.objectContaining({ + name: 'odd " table', + columns: expect.arrayContaining([ + expect.objectContaining({ + name: "id", + primary_key_position: 1, + }), + expect.objectContaining({ name: "generated", hidden: 2 }), + ]), + }), + expect.objectContaining({ + name: "keyed", + without_rowid: true, + strict: true, + }), + ]), + indexes: expect.arrayContaining([ + expect.objectContaining({ + name: 'quoted " index', + table: 'odd " table', + }), + ]), + views: [ + { name: "readable", sql: expect.stringContaining("SELECT id") }, + ], + triggers: [ + { + name: "observed", + table: 'odd " table', + sql: expect.stringContaining("AFTER INSERT"), + }, + ], + }, + rows: { + table: 'odd " table', + columns: [ + "id", + "payload", + "amount", + "missing", + "text_value", + "negative_zero", + "generated", + ], + returned_rows: 1, + row_limit: 1, + truncated: true, + values: [ + [ + { + type: "integer", + value: expect.stringMatching(/^900719925474099[34]$/), + }, + { type: "blob", hex: "deadbeef" }, + { type: "real", value: "Infinity" }, + { type: "null" }, + { type: "text", value: "\ufffd\ufffd", bytes_hex: "80ff" }, + { type: "real", value: "-0" }, + { + type: "text", + value: "DEADBEEF", + bytes_hex: "4445414442454546", + }, + ], + ], + }, + }, + }); + const { call } = await connectLocalToolsMcp(); + const response = await call("inspect_sqlite_database", { + path, + table: 'odd " table', + schema_scope: "database", + row_limit: 1, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(response.structuredContent).toEqual(result.json); + expect(await readFile(path)).toEqual(original); + }, +); + +cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "includes uncheckpointed committed WAL data without modifying the source database or sidecars", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-wal-public-"); + const path = join(root, "wal.db"); + const database = new DatabaseSync(path); + onTestFinished(() => database.close()); + database.exec( + "PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0; CREATE TABLE committed(value TEXT); INSERT INTO committed VALUES('only in WAL');", + ); + const files = [path, `${path}-wal`, `${path}-shm`]; + const before = await Promise.all(files.map(sourceState)); + const db = before[0]; + const wal = before[1]; + expect(db).toBeDefined(); + expect(wal).toBeDefined(); + const result = await cli.run({ + arguments: [ + "inspect-sqlite-database", + path, + "--table", + "committed", + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode, result.stdout + result.stderr).toBe(0); + expect(result.json).toMatchObject({ + normalized_result: { + artifact: { sha256: db === undefined ? "missing" : digest(db.bytes) }, + wal: { + path: `${path}-wal`, + sha256: wal === undefined ? "missing" : digest(wal.bytes), + }, + rows: { + values: [ + [ + { + type: "text", + value: "only in WAL", + bytes_hex: Buffer.from("only in WAL").toString("hex"), + }, + ], + ], + returned_rows: 1, + truncated: false, + }, + }, + }); + const { call } = await connectLocalToolsMcp(); + const response = await call("inspect_sqlite_database", { + path, + table: "committed", + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(response.structuredContent).toEqual(result.json); + const after = await Promise.all(files.map(sourceState)); + expect(after).toEqual(before); + }, +); + +cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "returns actionable public errors for corrupt input, unknown or ambiguous tables and invalid record choices", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-errors-public-"); + const path = join(root, "invalid.db"); + await writeFile(path, "not a SQLite database"); + const { call } = await connectLocalToolsMcp(); + const corrupt = await cli.run({ + arguments: ["inspect-sqlite-database", path, "--json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(corrupt.exitCode).toBe(1); + expect(corrupt.json).toMatchObject({ code: "invalid_request" }); + expect((await call("inspect_sqlite_database", { path })).isError).toBe( + true, + ); + const valid = join(root, "valid.db"); + const database = new DatabaseSync(valid); + try { + database.exec( + "CREATE TABLE present(value TEXT); CREATE VIRTUAL TABLE search USING fts5(body);", + ); + } finally { + database.close(); + } + const bytes = await readFile(valid); + const declaration = bytes.indexOf(Buffer.from("USING fts5")); + if (declaration < 0) throw new Error("Virtual declaration missing"); + bytes.write("zzzz", declaration + Buffer.byteLength("USING "), "utf8"); + await writeFile(valid, bytes); + const schema = await call("inspect_sqlite_database", { path: valid }); + expect(schema.isError).not.toBe(true); + expect(schema.structuredContent).toMatchObject({ + normalized_result: { + schema: { + tables: expect.arrayContaining([ + expect.objectContaining({ name: "search_data", kind: "unknown" }), + ]), + }, + }, + }); + const ambiguous = await cli.run({ + arguments: [ + "inspect-sqlite-database", + valid, + "--table", + "search_data", + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(ambiguous.exitCode).toBe(1); + expect(ambiguous.stdout).toContain("shadow table"); + expect( + ( + await call("inspect_sqlite_database", { + path: valid, + table: "search_data", + }) + ).isError, + ).toBe(true); + const missing = await cli.run({ + arguments: [ + "inspect-sqlite-database", + valid, + "--table", + "absent", + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(missing.exitCode).toBe(1); + expect(missing.stdout).toContain("absent"); + expect( + (await call("inspect_sqlite_database", { path: valid, table: "absent" })) + .isError, + ).toBe(true); + for (const input of [ + { path: valid, row_limit: 1 }, + { path: valid, table: "present", row_limit: 0 }, + { path: valid, query: "DELETE FROM present" }, + ]) { + expect( + (await call("inspect_sqlite_database", input)).isError, + JSON.stringify(input), + ).toBe(true); + } + }, +); diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index f1b16a52c6..9176dec349 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -442,6 +442,10 @@ it("rejects skipped selected work while allowing unselected lanes to skip", asyn }); it.each([ + [ + ".github/workflows/sqlite-runtime.yml", + { workflows: true, code: true, package: false, website: false }, + ], [ ".github/workflows/real-termux.yml", { workflows: true, code: false, package: false, website: false }, diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts new file mode 100644 index 0000000000..4cff44e968 --- /dev/null +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -0,0 +1,104 @@ +import { readFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import { expect, it } from "vitest"; +import { parse } from "yaml"; +import { z } from "zod"; +import { executeWorkflowFixture } from "../../support/workflowGit.js"; + +const workflow = async (filename = "ci.yml") => + z + .object({ + jobs: z.record( + z.string(), + z + .object({ + needs: z.union([z.string(), z.array(z.string())]).optional(), + }) + .passthrough(), + ), + }) + .parse( + parse( + await readFile( + new URL(`../../../.github/workflows/${filename}`, import.meta.url), + "utf8", + ), + ), + ); + +it("binds SQLite compatibility jobs to the build's prepared runtime artifact and root layout", async () => { + const steps = z.array( + z.object({ + uses: z.string().optional(), + with: z + .object({ name: z.string().optional(), path: z.string().optional() }) + .optional(), + }), + ); + const uploads = steps.parse((await workflow()).jobs.build?.steps).filter( + (step) => + step.uses?.startsWith("actions/upload-artifact@") && + step.with?.path + ?.split(/\r?\n/u) + .map((path) => path.trim()) + .includes("dist/"), + ); + expect(uploads).toHaveLength(1); + const name = z.string().min(1).parse(uploads[0]?.with?.name); + const downloads = steps + .parse((await workflow("sqlite-runtime.yml")).jobs.sqlite?.steps) + .filter((step) => step.uses?.startsWith("actions/download-artifact@")); + expect(downloads).toHaveLength(1); + expect(downloads[0]?.with).toEqual({ name, path: "." }); +}); + +const requiredNeeds = async () => { + const { jobs } = await workflow(); + return { jobs, needs: z.array(z.string()).parse(jobs.required?.needs) }; +}; + +it("makes the required gate wait for every CI lane, including future additions", async () => { + const { jobs, needs } = await requiredNeeds(); + expect(new Set(needs)).toEqual( + new Set(Object.keys(jobs).filter((name) => name !== "required")), + ); +}); + +it.each([ + ["true", "success", true], + ["true", "failure", false], + ["true", "cancelled", false], + ["true", "skipped", false], + ["true", "missing", false], + ["false", "skipped", true], +] as const)( + "enforces the SQLite required gate for code=%s and result=%s", + async (code, result, passes) => { + const workflow = await requiredNeeds(); + // GitHub exposes only direct dependencies in the required job's needs context. + const needs: Record< + string, + { result: string; outputs?: Record } + > = Object.fromEntries( + workflow.needs.map((name) => [name, { result: "success" }]), + ); + needs.changes = { result: "success", outputs: { code } }; + if (result === "missing") delete needs["sqlite-runtime"]; + else if ("sqlite-runtime" in needs) needs["sqlite-runtime"] = { result }; + const execution = executeWorkflowFixture( + process.execPath, + [ + fileURLToPath( + new URL("../../../scripts/ci/required.mjs", import.meta.url), + ), + ], + { env: { ...process.env, CI_JOBS: JSON.stringify(needs) } }, + ); + if (passes) await expect(execution).resolves.toMatchObject({ stderr: "" }); + else + await expect(execution).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("sqlite-runtime"), + }); + }, +); diff --git a/tests/boundary/mcp/serverLifecycleMcp.test.ts b/tests/boundary/mcp/serverLifecycleMcp.test.ts index 6a140befe7..c841e08ede 100644 --- a/tests/boundary/mcp/serverLifecycleMcp.test.ts +++ b/tests/boundary/mcp/serverLifecycleMcp.test.ts @@ -8,9 +8,332 @@ import { HopperRemoteError } from "../../../src/domain/hopperErrors.js"; import { err } from "../../../src/domain/result.js"; import { observed as ok } from "../../fixtures/analysisExecution.js"; import { createServer } from "../../../src/server/createServer.js"; +import { SqliteDatabaseService } from "../../../src/application/sqlite/SqliteDatabaseService.js"; +import { PassThrough } from "node:stream"; +import { once } from "node:events"; +import { access } from "node:fs/promises"; +import { + serveStdio, + StdioServerTransport, +} from "@modelcontextprotocol/server/stdio"; +import pino from "pino"; +import { createManagedBinarySession } from "../../../src/composition/binary.js"; +import { startMcpTransport } from "../../../src/main/transport.js"; +import { createToolResultDelivery } from "../../../src/server/toolResult.js"; +import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; +import { SqliteDatabaseProvider } from "../../../src/sqlite/SqliteDatabaseProvider.js"; +import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; +import { fileURLToPath } from "node:url"; +import { + createSqliteDatabaseFixture, + SQLITE_NATIVE_LIMITS_AVAILABLE, +} from "../../fixtures/sqlite/database.js"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; +import { z } from "zod"; +import { createDeferred } from "../../fixtures/binarySession.js"; const resources: Array<{ close(): Promise }> = []; +it.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "drains an admitted SQLite inspection before changing the session binding", + async () => { + const workspace = await createTestWorkspace("rea-sqlite-mcp-admission-"); + const database = createSqliteDatabaseFixture(workspace.root); + database.close(); + const acquired = createDeferred(); + const release = createDeferred(); + const provider = new SqliteDatabaseProvider( + process.env, + (spawn) => + spawnOwnedProviderProcess({ + ...spawn, + arguments: spawn.arguments.map((argument, index) => + index === 1 + ? fileURLToPath( + new URL( + "../../../dist/sqlite/SqliteDatabaseWorker.js", + import.meta.url, + ), + ) + : argument, + ), + }), + async () => { + const root = await PrivateRuntimeRoot.create({ + parent: workspace.root, + }); + acquired.resolve(); + await release.promise; + return root; + }, + ); + const session = createManagedBinarySession(); + const server = createServer( + { kind: "session", session }, + { sqliteDatabase: new SqliteDatabaseService(provider) }, + ); + const client = new Client({ name: "sqlite-admission", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const inspection = client.callTool({ + name: "inspect_sqlite_database", + arguments: { path: database.path, table: "records", columns: ["id"] }, + }); + await acquired.promise; + let closed = false; + const closing = session.close().then((result) => { + closed = true; + return result; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(closed).toBe(false); + release.resolve(); + const result = await inspection; + expect(result.isError).not.toBe(true); + const evidenceId = z + .object({ evidence_id: z.string() }) + .parse(result.structuredContent).evidence_id; + expect(result.structuredContent).toMatchObject({ + normalized_result: { rows: { columns: ["id"], returned_rows: 3 } }, + }); + expect(await closing).toMatchObject({ ok: true }); + // Closing a binding clears its ledger after the admitted recording succeeds. + expect(session.evidenceById(evidenceId)).toBeUndefined(); + expect(closed).toBe(true); + } finally { + release.resolve(); + await client.close(); + await server.close(); + await session.close(); + await removeTestWorkspace(workspace.root); + } + }, +); + +it.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "records completed SQLite evidence before delivering a cleanup failure", + async () => { + const workspace = await createTestWorkspace("rea-sqlite-mcp-retention-"); + const database = createSqliteDatabaseFixture(workspace.root); + database.close(); + let removalAllowed = false; + const provider = new SqliteDatabaseProvider( + process.env, + (spawn) => + spawnOwnedProviderProcess({ + ...spawn, + arguments: spawn.arguments.map((argument, index) => + index === 1 + ? fileURLToPath( + new URL( + "../../../dist/sqlite/SqliteDatabaseWorker.js", + import.meta.url, + ), + ) + : argument, + ), + }), + async () => { + const root = await PrivateRuntimeRoot.create({ + parent: workspace.root, + }); + return { + path: root.path, + close: async () => { + if (!removalAllowed) + throw new Error( + "Snapshot cleanup denied after completed observation", + ); + await root.close(); + }, + }; + }, + ); + const session = createManagedBinarySession(); + const server = createServer( + { kind: "session", session }, + { sqliteDatabase: new SqliteDatabaseService(provider) }, + ); + const client = new Client({ + name: "sqlite-retained-observation", + version: "1", + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const result = await client.callTool({ + name: "inspect_sqlite_database", + arguments: { path: database.path, table: "records", columns: ["id"] }, + }); + expect(result.isError).toBe(true); + const error = z + .object({ + error: z.object({ + details: z.object({ + partial_observation: z.object({ + kind: z.literal("retained-evidence"), + evidence_id: z.string(), + }), + }), + }), + }) + .parse(parseMcpToolError(result)).error; + const retained = session.evidenceById( + error.details.partial_observation.evidence_id, + ); + expect(retained).toMatchObject({ + operation: "inspect_sqlite_database", + parameters: { columns: ["id"] }, + normalized_result: { rows: { columns: ["id"], returned_rows: 3 } }, + }); + expect(JSON.stringify(parseMcpToolError(result))).not.toContain( + "normalized_result", + ); + const bundle = await client.callTool({ + name: "get_evidence_bundle", + arguments: {}, + }); + expect(bundle.isError).not.toBe(true); + expect(JSON.stringify(bundle.structuredContent)).toContain( + error.details.partial_observation.evidence_id, + ); + } finally { + removalAllowed = true; + await client.close(); + await server.close(); + await session.close(); + await removeTestWorkspace(workspace.root); + } + }, +); + +it.each(["handle", "stdin"] as const)( + "retains failed SQLite cleanup through production stdio shutdown: %s", + async (shutdown) => { + const root = await PrivateRuntimeRoot.create({ + prefix: "rea-sqlite-stdio-cleanup-", + }); + let removalAllowed = false; + let attempts = 0; + const sqliteDatabase = new SqliteDatabaseService({ + inspect: async () => { + throw new Error("No inspection requested"); + }, + close: async () => { + attempts += 1; + if (!removalAllowed) throw new Error("SQLite snapshot removal denied"); + await root.close(); + }, + }); + const stdin = new PassThrough(); + const stdout = new PassThrough(); + const logger = pino({ enabled: false }); + const session = createManagedBinarySession(); + const started = await startMcpTransport( + { + env: {}, + serve: (factory, options) => + serveStdio(factory, { + ...options, + transport: new StdioServerTransport(stdin, stdout), + }), + createServer: (source, options) => + createServer(source, { ...options, sqliteDatabase }), + writeStderr: () => undefined, + setExitCode: () => undefined, + registerShutdown: () => () => undefined, + }, + session, + { + environment: {}, + delivery: createToolResultDelivery(undefined), + logger, + serverLogger: logger, + loadOptionalProviders: async () => ({}), + }, + ); + if (!started.ok) throw new Error("Transport startup failed"); + try { + const response = once(stdout, "data"); + stdin.write( + `${JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "sqlite-cleanup-test", version: "1.0.0" }, + }, + })}\n`, + ); + const [reply] = await response; + expect(JSON.parse(String(reply))).toMatchObject({ + id: 1, + result: { serverInfo: { name: "rea" } }, + }); + if (shutdown === "stdin") { + stdin.end(); + await new Promise((resolve) => setImmediate(resolve)); + } + await expect(started.handle.close()).rejects.toBeInstanceOf( + AggregateError, + ); + expect(attempts).toBeGreaterThan(0); + await expect(access(root.path)).resolves.toBeUndefined(); + removalAllowed = true; + await started.handle.close(); + await expect(access(root.path)).rejects.toMatchObject({ code: "ENOENT" }); + const completedAttempts = attempts; + await started.handle.close(); + expect(attempts).toBe(completedAttempts); + } finally { + removalAllowed = true; + await started.handle.close(); + await started.closeAndroid(); + await session.close(); + await root.close(); + stdin.destroy(); + stdout.destroy(); + } + }, +); + +it("awaits SQLite cleanup and retries its retained owner after failed server shutdown", async () => { + let removalAllowed = false; + let attempts = 0; + const sqliteDatabase = new SqliteDatabaseService({ + inspect: async () => { + throw new Error("No inspection requested"); + }, + close: async () => { + attempts += 1; + if (!removalAllowed) throw new Error("SQLite snapshot removal denied"); + }, + }); + const server = createServer( + { kind: "fixed", analysis: { execute: async () => ok(null) } }, + { sqliteDatabase }, + ); + resources.push(server); + try { + await expect(server.close()).rejects.toBeInstanceOf(AggregateError); + expect(attempts).toBe(1); + } finally { + removalAllowed = true; + } + await server.close(); + expect(attempts).toBe(2); +}); + afterEach(async () => { await Promise.all( resources.splice(0).map(async (resource) => resource.close()), diff --git a/tests/boundary/sqlite/sqliteDatabaseCleanup.test.ts b/tests/boundary/sqlite/sqliteDatabaseCleanup.test.ts new file mode 100644 index 0000000000..de8e142980 --- /dev/null +++ b/tests/boundary/sqlite/sqliteDatabaseCleanup.test.ts @@ -0,0 +1,275 @@ +import { access } from "node:fs/promises"; +import { expect, it, onTestFinished } from "vitest"; +import { SqliteDatabaseProvider } from "../../../src/sqlite/SqliteDatabaseProvider.js"; +import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; +import { OwnedCommandFailure } from "../../../src/process/OwnedCommand.js"; +import { ProviderProcessSupervisor } from "../../../src/process/ProviderProcess.js"; +import { + spawnProviderProcessFixture, + stopProviderProcessFixture, + waitForProviderProcessReady, +} from "../../fixtures/providerProcess.js"; +import { createDeferred } from "../../fixtures/binarySession.js"; +import { createSqliteDatabaseFixture } from "../../fixtures/sqlite/database.js"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; + +const createUnavailableProvider = ( + createRuntime: SqliteDatabaseProvider["createRuntime"], +) => + new SqliteDatabaseProvider( + {}, + () => { + throw new Error("SQLite worker deliberately unavailable"); + }, + createRuntime, + ); + +const cleanupFixture = async () => { + const workspace = await createTestWorkspace("rea-sqlite-cleanup-"); + const database = createSqliteDatabaseFixture(workspace.root); + database.close(); + const roots: PrivateRuntimeRoot[] = []; + let removalAllowed = false; + let attempts = 0; + const provider = createUnavailableProvider(async () => { + const root = await PrivateRuntimeRoot.create({ parent: workspace.root }); + roots.push(root); + return { + path: root.path, + close: async () => { + attempts += 1; + if (!removalAllowed) throw new Error("snapshot removal denied"); + await root.close(); + }, + }; + }); + onTestFinished(async () => { + await Promise.all(roots.map((root) => root.close())); + await removeTestWorkspace(workspace.root); + }); + return { + provider, + roots, + path: database.path, + allowRemoval: () => { + removalAllowed = true; + }, + attempts: () => attempts, + }; +}; + +it("retains failed snapshot owners and blocks new acquisition until retry succeeds", async () => { + const fixture = await cleanupFixture(); + const first = await fixture.provider.inspect({ path: fixture.path }); + expect(first).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + const root = fixture.roots[0]; + if (root === undefined) throw new Error("Missing owned root"); + await expect(access(root.path)).resolves.toBeUndefined(); + const second = await fixture.provider.inspect({ path: fixture.path }); + expect(second).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + expect(fixture.roots).toHaveLength(1); + expect(fixture.attempts()).toBe(2); + fixture.allowRemoval(); + await fixture.provider.inspect({ path: fixture.path }); + expect(fixture.roots).toHaveLength(2); + await expect(access(root.path)).rejects.toMatchObject({ code: "ENOENT" }); + await fixture.provider.close(); +}); + +it("awaits shared shutdown cleanup and retains failures for a later close", async () => { + const fixture = await cleanupFixture(); + await fixture.provider.inspect({ path: fixture.path }); + const closed = await Promise.allSettled([ + fixture.provider.close(), + fixture.provider.close(), + ]); + expect(closed.map((result) => result.status)).toEqual([ + "rejected", + "rejected", + ]); + expect(fixture.attempts()).toBe(2); + expect(await fixture.provider.inspect({ path: fixture.path })).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + fixture.allowRemoval(); + await fixture.provider.close(); + await fixture.provider.close(); + expect(fixture.attempts()).toBe(3); + for (const root of fixture.roots) + await expect(access(root.path)).rejects.toMatchObject({ code: "ENOENT" }); +}); + +it("waits for concurrent acquisitions and releases only owners whose cleanup succeeds", async () => { + const workspace = await createTestWorkspace("rea-sqlite-concurrent-cleanup-"); + const database = createSqliteDatabaseFixture(workspace.root); + database.close(); + const acquired = createDeferred(); + const release = createDeferred(); + const roots: PrivateRuntimeRoot[] = []; + const allowed = new Set(); + const provider = createUnavailableProvider(async () => { + const root = await PrivateRuntimeRoot.create({ parent: workspace.root }); + roots.push(root); + if (roots.length === 2) acquired.resolve(); + await release.promise; + return { + path: root.path, + close: async () => { + if (!allowed.has(root)) throw new Error("snapshot removal denied"); + await root.close(); + }, + }; + }); + onTestFinished(async () => { + release.resolve(); + roots.forEach((root) => allowed.add(root)); + await provider.close(); + await removeTestWorkspace(workspace.root); + }); + const inspections = [ + provider.inspect({ path: database.path }), + provider.inspect({ path: database.path }), + ]; + await acquired.promise; + // Two admitted snapshots occupy capacity even before the worker launches. + expect(await provider.inspect({ path: database.path })).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError", resource: "capacity" }, + }); + expect(roots).toHaveLength(2); + expect( + await provider.inspect( + { path: database.path }, + { signal: AbortSignal.abort() }, + ), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + let settled = false; + const closing = provider.close().finally(() => { + settled = true; + }); + const rejected = expect(closing).rejects.toMatchObject({ + cleanupIncomplete: true, + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).toBe(false); + release.resolve(); + await Promise.all(inspections); + await rejected; + const [first, second] = roots; + if (first === undefined || second === undefined) + throw new Error("Missing owned roots"); + allowed.add(first); + await expect(provider.close()).rejects.toMatchObject({ + cleanupResources: [second.path], + }); + await expect(access(first.path)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(access(second.path)).resolves.toBeUndefined(); + allowed.add(second); + await provider.close(); + await expect(access(second.path)).rejects.toMatchObject({ code: "ENOENT" }); +}); + +it("retains the snapshot until a failed worker cleanup owner can be stopped", async () => { + const workspace = await createTestWorkspace("rea-sqlite-worker-cleanup-"); + const database = createSqliteDatabaseFixture(workspace.root); + database.close(); + const child = spawnProviderProcessFixture("graceful"); + let workerRemovalAllowed = false; + let rootRemovalAllowed = false; + let workerAttempts = 0; + let rootAttempts = 0; + const roots: PrivateRuntimeRoot[] = []; + const supervisor = new ProviderProcessSupervisor({ + process: child, + ownsProcessLifetime: true, + cleanup: async () => { + workerAttempts += 1; + if (!workerRemovalAllowed) + return { cleaned: false, reason: "SQLite worker cleanup denied" }; + await stopProviderProcessFixture(child); + return { cleaned: true, signaled: true }; + }, + }); + const provider = new SqliteDatabaseProvider( + {}, + async (spawn) => { + const stopped = await supervisor.stop(); + if (stopped.status !== "incomplete") + throw new Error("Expected retained worker"); + throw new OwnedCommandFailure( + "process", + "SQLite worker cleanup failed", + supervisor.snapshot(), + stopped.reason, + undefined, + [spawn.runId, spawn.cwd ?? ""], + supervisor, + ); + }, + async () => { + const root = await PrivateRuntimeRoot.create({ parent: workspace.root }); + roots.push(root); + return { + path: root.path, + close: async () => { + rootAttempts += 1; + if (!rootRemovalAllowed) + throw new Error("SQLite snapshot removal denied"); + await root.close(); + }, + }; + }, + ); + onTestFinished(async () => { + workerRemovalAllowed = true; + rootRemovalAllowed = true; + await supervisor.stop(); + await provider.close(); + await Promise.all(roots.map((root) => root.close())); + await removeTestWorkspace(workspace.root); + }); + await waitForProviderProcessReady(child); + expect(await provider.inspect({ path: database.path })).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + // The snapshot cannot be removed while its worker may still use it. + expect(rootAttempts).toBe(0); + const root = roots[0]; + if (root === undefined) throw new Error("Missing retained snapshot"); + await expect(access(root.path)).resolves.toBeUndefined(); + expect(await provider.inspect({ path: database.path })).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + expect(roots).toHaveLength(1); + expect(workerAttempts).toBe(2); + await expect(provider.close()).rejects.toMatchObject({ + cleanupIncomplete: true, + }); + expect(rootAttempts).toBe(0); + workerRemovalAllowed = true; + await expect(provider.close()).rejects.toMatchObject({ + cleanupResources: [root.path], + }); + expect(child.signalCode).toBe("SIGKILL"); + expect(workerAttempts).toBe(4); + expect(rootAttempts).toBe(1); + rootRemovalAllowed = true; + await provider.close(); + expect(workerAttempts).toBe(4); + await expect(access(root.path)).rejects.toMatchObject({ code: "ENOENT" }); +}); diff --git a/tests/boundary/sqlite/sqliteDatabaseDescriptorCleanup.test.ts b/tests/boundary/sqlite/sqliteDatabaseDescriptorCleanup.test.ts new file mode 100644 index 0000000000..f656c35f48 --- /dev/null +++ b/tests/boundary/sqlite/sqliteDatabaseDescriptorCleanup.test.ts @@ -0,0 +1,378 @@ +import { Cli } from "incur"; +import { access, writeFile } from "node:fs/promises"; +import type { FileHandle } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, it, onTestFinished, vi } from "vitest"; + +import { SqliteDatabaseService } from "../../../src/application/sqlite/SqliteDatabaseService.js"; +import { registerSqliteCommands } from "../../../src/cli/sqliteCommands.js"; +import { createSqliteDatabaseService } from "../../../src/composition/sqlite.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { silentLogger } from "../../../src/logger.js"; +import { SqliteDatabaseProvider } from "../../../src/sqlite/SqliteDatabaseProvider.js"; +import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; +import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; +import { createSqliteDatabaseFixture } from "../../fixtures/sqlite/database.js"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; + +const descriptorCleanup = vi.hoisted(() => ({ + path: "", + denied: false, + invalidate: false, + attempts: 0, + handles: [] as FileHandle[], + readFailure: undefined as Error | undefined, + onOpen: undefined as (() => void) | undefined, +})); + +vi.mock("../../../src/composition/sqlite.js", () => ({ + createSqliteDatabaseService: vi.fn(), +})); + +vi.mock("node:fs/promises", async (importOriginal) => { + const filesystem = await importOriginal(); + return { + ...filesystem, + open: async (...args: Parameters) => { + const handle = await filesystem.open(...args); + if (args[0] === descriptorCleanup.path) { + descriptorCleanup.handles.push(handle); + const close = handle.close.bind(handle); + handle.close = async () => { + // FileHandle.close also asks attached streams to close the invalidated handle. + if (handle.fd < 0) { + await close(); + return; + } + descriptorCleanup.attempts += 1; + if (descriptorCleanup.denied) { + if (descriptorCleanup.invalidate) await close(); + throw new Error("injected SQLite descriptor close failure"); + } + await close(); + }; + if (descriptorCleanup.readFailure !== undefined) + vi.spyOn(handle, "stat").mockRejectedValue( + descriptorCleanup.readFailure, + ); + descriptorCleanup.onOpen?.(); + } + return handle; + }, + }; +}); + +const fixture = async ( + member: "database" | "WAL" | "journal" | "reply", + invalidate = false, +) => { + const workspace = await createTestWorkspace("rea-sqlite-descriptor-cleanup-"); + const database = createSqliteDatabaseFixture(workspace.root, true); + const path = + member === "database" + ? database.path + : member === "WAL" + ? database.walPath + : member === "journal" + ? `${database.path}-journal` + : ""; + if (member === "journal") await writeFile(path, Buffer.alloc(28)); + const roots: PrivateRuntimeRoot[] = []; + const provider = new SqliteDatabaseProvider( + process.env, + (spawn) => { + if (member !== "reply") + throw new Error("SQLite worker deliberately unavailable"); + return spawnOwnedProviderProcess({ + ...spawn, + arguments: spawn.arguments.map((argument, index) => + index === 1 + ? fileURLToPath( + new URL( + "../../../dist/sqlite/SqliteDatabaseWorker.js", + import.meta.url, + ), + ) + : argument, + ), + }); + }, + async () => { + const root = await PrivateRuntimeRoot.create({ parent: workspace.root }); + if (member === "reply") + descriptorCleanup.path = join(root.path, "reply.json"); + roots.push(root); + return root; + }, + ); + descriptorCleanup.path = path; + descriptorCleanup.denied = true; + descriptorCleanup.invalidate = invalidate; + descriptorCleanup.attempts = 0; + descriptorCleanup.handles = []; + descriptorCleanup.readFailure = undefined; + descriptorCleanup.onOpen = undefined; + onTestFinished(async () => { + descriptorCleanup.denied = false; + try { + await provider.close(); + } catch (cause) { + if (!invalidate) throw cause; + } + for (const handle of descriptorCleanup.handles) await handle.close(); + descriptorCleanup.path = ""; + database.close(); + await removeTestWorkspace(workspace.root); + }); + return { provider, roots, path, database }; +}; + +const serveInspection = async ( + provider: SqliteDatabaseProvider, + path: string, +) => { + const previousExitCode = process.exitCode; + onTestFinished(() => { + process.exitCode = previousExitCode; + }); + vi.mocked(createSqliteDatabaseService).mockReturnValue( + new SqliteDatabaseService(provider), + ); + const cli = Cli.create("rea-sqlite-cleanup-test", { sync: false }); + registerSqliteCommands(cli, silentLogger, {}); + let stdout = ""; + await cli.serve(["inspect-sqlite-database", path, "--json"], { + env: {}, + stdout: (text) => { + stdout += text; + }, + exit: () => {}, + }); + return { output: JSON.parse(stdout) as unknown, exitCode: process.exitCode }; +}; + +it.each(["database", "WAL", "journal"] as const)( + "retains a failed %s descriptor and retries it at shutdown", + async (member) => { + const selected = await fixture(member); + const first = await selected.provider.inspect({ + path: selected.database.path, + }); + expect(first).toMatchObject({ + ok: false, + error: { + cleanupIncomplete: true, + cleanupResources: [expect.stringContaining(selected.path)], + }, + }); + const handle = descriptorCleanup.handles[0]; + if (handle === undefined) + throw new Error("Expected owned source descriptor"); + expect(handle.fd).toBeGreaterThanOrEqual(0); + const blocked = await selected.provider.inspect({ + path: selected.database.path, + }); + expect(blocked).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + expect(selected.roots).toHaveLength(1); + await expect(selected.provider.close()).rejects.toMatchObject({ + cleanupIncomplete: true, + }); + expect(descriptorCleanup.attempts).toBe(3); + descriptorCleanup.denied = false; + await selected.provider.close(); + expect(descriptorCleanup.attempts).toBe(4); + expect(handle.fd).toBe(-1); + for (const root of selected.roots) + await expect(access(root.path)).rejects.toMatchObject({ code: "ENOENT" }); + }, +); + +it.each(["EACCES", "EPERM"])( + "retains the original %s inspection diagnostics through CLI shutdown", + async (code) => { + const selected = await fixture("database"); + descriptorCleanup.readFailure = Object.assign( + new Error("injected source read denial"), + { code }, + ); + const result = await serveInspection( + selected.provider, + selected.database.path, + ); + expect(result).toMatchObject({ + exitCode: 1, + output: { + code: "cleanup_incomplete", + details: { + resources: [expect.stringContaining(selected.path)], + diagnostics: { + primary_error: { + details: { + diagnostics: { + previous_error: { + code: "access_denied", + details: { path: selected.path, system_code: code }, + }, + }, + }, + }, + cleanup_error: { code: "cleanup_incomplete" }, + }, + }, + }, + }); + expect(descriptorCleanup.attempts).toBe(2); + }, +); + +it("retains cancellation diagnostics and the interrupt exit code through CLI shutdown", async () => { + const selected = await fixture("database"); + descriptorCleanup.onOpen = () => { + const interrupt = process.listeners("SIGINT").at(-1); + if (interrupt === undefined) + throw new Error("Missing CLI interrupt handler"); + interrupt("SIGINT"); + }; + const result = await serveInspection( + selected.provider, + selected.database.path, + ); + expect(result).toMatchObject({ + exitCode: 130, + output: { + code: "cleanup_incomplete", + details: { + resources: [expect.stringContaining(selected.path)], + diagnostics: { + primary_error: { + details: { + diagnostics: { previous_error: { code: "cancelled" } }, + }, + }, + cleanup_error: { code: "cleanup_incomplete" }, + }, + }, + }, + }); + expect(descriptorCleanup.attempts).toBeGreaterThanOrEqual(2); +}); + +it("retains reply-read cancellation and retries the failed reply descriptor close", async () => { + const selected = await fixture("reply"); + const controller = new AbortController(); + descriptorCleanup.onOpen = () => + controller.abort(new Error("requested cancellation during reply read")); + const result = await selected.provider.inspect( + { path: selected.database.path }, + { signal: controller.signal }, + ); + if (result.ok) throw new Error("Expected cancelled reply read"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "cleanup_incomplete", + details: { + execution_failure: "cancelled", + resources: [expect.stringContaining(descriptorCleanup.path)], + captured_output: { + stdout: "", + stderr: expect.any(String), + truncated: false, + }, + }, + }); + const handle = descriptorCleanup.handles[0]; + if (handle === undefined) throw new Error("Expected owned reply descriptor"); + expect(handle.fd).toBeGreaterThanOrEqual(0); + await expect(selected.provider.close()).rejects.toMatchObject({ + cleanupIncomplete: true, + }); + descriptorCleanup.denied = false; + await selected.provider.close(); + expect(handle.fd).toBe(-1); + for (const root of selected.roots) + await expect(access(root.path)).rejects.toMatchObject({ code: "ENOENT" }); +}); + +it("retries source cleanup before admitting another snapshot", async () => { + const selected = await fixture("database"); + await selected.provider.inspect({ path: selected.database.path }); + const first = descriptorCleanup.handles[0]; + if (first === undefined) throw new Error("Expected source descriptor"); + descriptorCleanup.denied = false; + const resumed = await selected.provider.inspect({ + path: selected.database.path, + }); + expect(selected.roots).toHaveLength(2); + expect(first.fd).toBe(-1); + if (resumed.ok) throw new Error("The worker is deliberately unavailable"); + expect(resumed.error.cleanupIncomplete).toBe(false); + await selected.provider.close(); +}); + +it("keeps an invalidated descriptor close failure unconfirmed across retries", async () => { + const selected = await fixture("journal", true); + const first = await selected.provider.inspect({ + path: selected.database.path, + }); + expect(first).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + expect(descriptorCleanup.handles[0]?.fd).toBe(-1); + descriptorCleanup.denied = false; + await expect(selected.provider.close()).rejects.toMatchObject({ + cleanupIncomplete: true, + }); + await expect(selected.provider.close()).rejects.toMatchObject({ + cleanupIncomplete: true, + }); + expect(descriptorCleanup.attempts).toBe(1); +}); + +it.each(["EACCES", "EPERM"])( + "preserves %s read diagnostics alongside a failed descriptor close", + async (code) => { + const selected = await fixture("database"); + descriptorCleanup.readFailure = Object.assign( + new Error("injected source read denial"), + { code }, + ); + const result = await selected.provider.inspect({ + path: selected.database.path, + }); + expect(result).toMatchObject({ + ok: false, + error: { + cleanupIncomplete: true, + diagnostics: { previous_error: { code: "access_denied" } }, + }, + }); + }, +); + +it.each(["database", "WAL", "journal"] as const)( + "preserves caller cancellation alongside a failed %s descriptor close", + async (member) => { + const selected = await fixture(member); + const controller = new AbortController(); + descriptorCleanup.onOpen = () => + controller.abort(new Error("requested caller cancellation")); + const result = await selected.provider.inspect( + { path: selected.database.path }, + { signal: controller.signal }, + ); + expect(result).toMatchObject({ + ok: false, + error: { + cleanupIncomplete: true, + diagnostics: { previous_error: { code: "cancelled" } }, + }, + }); + }, +); diff --git a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts new file mode 100644 index 0000000000..ba94792a57 --- /dev/null +++ b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts @@ -0,0 +1,328 @@ +import { DatabaseSync } from "node:sqlite"; +import { readFile, writeFile } from "node:fs/promises"; +import { expect, it as nativeTest, onTestFinished } from "vitest"; +import { SQLITE_NATIVE_LIMITS_AVAILABLE } from "../../fixtures/sqlite/database.js"; +import { inspectSqliteDatabaseSnapshot } from "../../../src/sqlite/SqliteDatabaseInspection.js"; +import { SqliteInspectionFailure } from "../../../src/sqlite/SqliteDatabaseLimits.js"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; + +const it = nativeTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE); + +const createDatabase = async (sql: string): Promise => { + const workspace = await createTestWorkspace("rea-sqlite-expansion-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const path = workspace.path("private.db"); + const producer = new DatabaseSync(path); + try { + producer.exec(sql); + } finally { + producer.close(); + } + return path; +}; + +it("preserves negative-zero REAL values through JSON separately from positive and integer zero", async () => { + const path = await createDatabase(` + CREATE TABLE numbers(value); + INSERT INTO numbers VALUES (-0.0), (0.0), (0), (5e-324), (-5e-324); + `); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + table: "numbers", + }); + expect(JSON.parse(JSON.stringify(result.rows))).toMatchObject({ + values: [ + [{ type: "real", value: "-0" }], + [{ type: "real", value: 0 }], + [{ type: "integer", value: "0" }], + [{ type: "real", value: Number.MIN_VALUE }], + [{ type: "real", value: -Number.MIN_VALUE }], + ], + }); +}); + +it.each(["table", "view"] as const)( + "inspects columns when a %s shadows the table-valued PRAGMA name", + async (kind) => { + const path = await createDatabase(` + CREATE TABLE "selected "" table" ("0" TEXT, "" INTEGER); + INSERT INTO "selected "" table" VALUES ('retained', 7); + ${kind === "table" ? "CREATE TABLE pragma_table_xinfo(value TEXT)" : "CREATE VIEW pragma_table_xinfo AS SELECT 1"}; + `); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + table: 'selected " table', + }); + expect(result.schema.tables).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: 'selected " table', + columns: [ + expect.objectContaining({ name: "0", declared_type: "TEXT" }), + expect.objectContaining({ name: "", declared_type: "INTEGER" }), + ], + }), + ]), + ); + expect(result.rows).toMatchObject({ + columns: ["0", ""], + values: [ + [ + { type: "text", value: "retained" }, + { type: "integer", value: "7" }, + ], + ], + }); + }, +); + +it.each([6, 9])( + "preserves a %i MiB schema statement without expanding its native record", + async (mebibytes) => { + const trigger = `CREATE TRIGGER large_definition AFTER INSERT ON records BEGIN /*${"x".repeat(mebibytes * 1024 * 1024)}*/ SELECT 1; END`; + const path = await createDatabase(` + CREATE TABLE records(value TEXT); + ${trigger}; + `); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + schema_scope: "database", + }); + expect(result.schema.triggers).toEqual([ + { name: "large_definition", table: "records", sql: trigger }, + ]); + expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThan( + 16 * 1024 * 1024, + ); + }, +); + +it("inspects a large schema identifier without expanding a native sort record", async () => { + const name = "x".repeat(4 * 1024 * 1024); + const sql = `CREATE TABLE "${name}" (value TEXT)`; + const path = await createDatabase(sql); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + schema_scope: "database", + }); + expect(result.schema.tables).toMatchObject([ + { name, sql, columns: [{ name: "value", declared_type: "TEXT" }] }, + ]); + expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThan( + 16 * 1024 * 1024, + ); +}); + +it("accounts for JSON escape expansion of a stored declaration in the reply limit", async () => { + const path = await createDatabase(` + CREATE TABLE records(value TEXT); + CREATE TRIGGER expanded AFTER INSERT ON records BEGIN /*${"\u0001".repeat(3 * 1024 * 1024)}*/ SELECT 1; END; + `); + expect(() => + inspectSqliteDatabaseSnapshot(path, { path, schema_scope: "database" }), + ).toThrowError(expect.objectContaining({ reason: "output-limit" })); + const inventory = inspectSqliteDatabaseSnapshot(path, { path }); + expect(inventory.schema.triggers).toEqual([ + { name: "expanded", table: "records" }, + ]); +}); + +it.each(["UTF-8", "UTF-16le", "UTF-16be"])( + "preserves exact %s schema text when decoding native bytes", + async (encoding) => { + const sql = 'CREATE TABLE "\uFEFFé𝄞" ("列" TEXT DEFAULT \'\uFEFFvalue\')'; + const path = await createDatabase(`PRAGMA encoding='${encoding}'; ${sql}`); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + schema_scope: "database", + }); + expect(result.schema.tables).toMatchObject([ + { + name: "\uFEFFé𝄞", + sql, + columns: [{ name: "列", default_sql: "'\uFEFFvalue'" }], + }, + ]); + }, +); + +it("counts schema names once when enforcing the complete reply budget", async () => { + const names = Array.from( + { length: 48 }, + (_, index) => `${index}_${"x".repeat(128 * 1024)}`, + ); + const path = await createDatabase( + names.map((name) => `CREATE TABLE "${name}" (value TEXT);`).join("\n"), + ); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + schema_scope: "database", + }); + expect(result.schema.tables).toHaveLength(names.length); + expect(result.schema.tables.map((table) => table.name)).toEqual( + expect.arrayContaining(names), + ); + expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThan( + 16 * 1024 * 1024, + ); +}); + +it("rejects malformed schema bytes instead of returning replacement text", async () => { + const path = await createDatabase( + "CREATE TABLE records(value TEXT /*schema-marker*/)", + ); + const bytes = await readFile(path); + const marker = bytes.indexOf(Buffer.from("schema-marker")); + if (marker < 0) throw new Error("Schema marker missing"); + bytes[marker] = 0xff; + await writeFile(path, bytes); + expect(() => + inspectSqliteDatabaseSnapshot(path, { path, schema_scope: "database" }), + ).toThrow(/schema sql contains malformed UTF-8 text/); +}); + +it("refuses possible shadow tables when their virtual module is unavailable", async () => { + const path = await createDatabase(` + CREATE VIRTUAL TABLE "SeArCh_É" USING fts5(body); + INSERT INTO "SeArCh_É" VALUES ('retained'); + CREATE TABLE "search_É_manual" (value TEXT); + CREATE TABLE "search_é_manual" (value TEXT); + INSERT INTO "search_é_manual" VALUES ('distinct Unicode name'); + CREATE TABLE "SeArCh_Élite" (value TEXT); + `); + const bytes = await readFile(path); + const declaration = bytes.indexOf(Buffer.from("USING fts5")); + if (declaration < 0) throw new Error("Virtual declaration missing"); + bytes.write("zzzz", declaration + Buffer.byteLength("USING "), "utf8"); + await writeFile(path, bytes); + + for (const table of ["SeArCh_É_data", "search_É_manual"]) + expect(() => inspectSqliteDatabaseSnapshot(path, { path, table })).toThrow( + /cannot determine.*shadow table/i, + ); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + table: "search_é_manual", + schema_scope: "database", + }); + expect(result.schema.tables).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: "SeArCh_É", kind: "virtual" }), + expect.objectContaining({ name: "SeArCh_É_data", kind: "unknown" }), + expect.objectContaining({ name: "search_É_manual", kind: "unknown" }), + expect.objectContaining({ name: "search_é_manual", kind: "table" }), + expect.objectContaining({ name: "SeArCh_Élite", kind: "table" }), + ]), + ); + expect(result.rows).toMatchObject({ + values: [[{ type: "text", value: "distinct Unicode name" }]], + }); + expect(result.limitations.join("\n")).toMatch(/shadow table/i); +}); + +it("rejects an oversized generated cell through SQLite's native value limit", async () => { + const path = await createDatabase(` + CREATE TABLE records ( + id INTEGER, + payload BLOB GENERATED ALWAYS AS (zeroblob(72 * 1024 * 1024)) VIRTUAL + ); + INSERT INTO records(id) VALUES (1); + `); + try { + inspectSqliteDatabaseSnapshot(path, { + path, + table: "records", + row_limit: 1, + }); + throw new Error("An oversized generated cell must not be projected"); + } catch (cause: unknown) { + expect(cause).toMatchObject({ + code: "ERR_SQLITE_ERROR", + errcode: 18, + }); + } +}); + +it("reports additional rows without expanding an unselected generated cell", async () => { + const path = await createDatabase(` + CREATE TABLE records ( + id INTEGER, + payload BLOB GENERATED ALWAYS AS ( + CASE WHEN id = 2 THEN zeroblob(72 * 1024 * 1024) ELSE X'01' END + ) VIRTUAL + ); + INSERT INTO records(id) VALUES (1), (2); + `); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + table: "records", + row_limit: 1, + }); + expect(result.rows).toMatchObject({ + columns: ["id", "payload"], + values: [ + [ + { type: "integer", value: "1" }, + { type: "blob", hex: "01" }, + ], + ], + returned_rows: 1, + truncated: true, + }); +}); + +it("rejects the combined representation of individually small generated cells", async () => { + const columns = Array.from( + { length: 6 }, + (_, index) => + `payload_${String(index)} BLOB GENERATED ALWAYS AS (zeroblob(2 * 1024 * 1024)) VIRTUAL`, + ); + const path = await createDatabase(` + CREATE TABLE records (id INTEGER, ${columns.join(", ")}); + INSERT INTO records(id) VALUES (1); + `); + try { + inspectSqliteDatabaseSnapshot(path, { + path, + table: "records", + row_limit: 1, + }); + throw new Error( + "Aggregate row expansion must be rejected before projection", + ); + } catch (cause: unknown) { + expect(cause).toBeInstanceOf(SqliteInspectionFailure); + expect(cause).toMatchObject({ reason: "output-limit" }); + } +}); + +it("samples above 1000 rows and safe integer limits within the byte budget", async () => { + const path = await createDatabase(`CREATE TABLE items(value INTEGER); + WITH RECURSIVE numbers(value) AS (SELECT 1 UNION ALL SELECT value + 1 FROM numbers WHERE value < 1002) + INSERT INTO items SELECT value FROM numbers;`); + for (const row_limit of [1001, Number.MAX_SAFE_INTEGER]) { + const result = inspectSqliteDatabaseSnapshot(path, { + path, + table: "items", + row_limit, + }); + expect(result.rows).toMatchObject({ + row_limit, + returned_rows: Math.min(row_limit, 1002), + truncated: row_limit < 1002, + }); + } +}); + +nativeTest.skipIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "rejects a runtime without native limits before querying the snapshot", + async () => { + const path = await createDatabase("CREATE TABLE items(value INTEGER)"); + expect(() => + inspectSqliteDatabaseSnapshot(path, { path, schema_scope: "database" }), + ).toThrow("DatabaseSync.limits"); + }, +); diff --git a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts new file mode 100644 index 0000000000..24dadc1287 --- /dev/null +++ b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts @@ -0,0 +1,595 @@ +import { createHash } from "node:crypto"; +import { execFile } from "node:child_process"; +import { access, lstat, readFile, symlink, writeFile } from "node:fs/promises"; +import { DatabaseSync } from "node:sqlite"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { expect, it as nativeTest, onTestFinished } from "vitest"; +import { SqliteDatabaseProvider } from "../../../src/sqlite/SqliteDatabaseProvider.js"; +import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; +import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; +import { waitForProviderProcessReady } from "../../fixtures/providerProcess.js"; +import { + createSqliteDatabaseFixture, + SQLITE_NATIVE_LIMITS_AVAILABLE, +} from "../../fixtures/sqlite/database.js"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; + +const it = nativeTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE); + +const workerUrl = new URL( + "../../../dist/sqlite/SqliteDatabaseWorker.js", + import.meta.url, +); +const createProvider = () => + new SqliteDatabaseProvider(process.env, (spawn) => + spawnOwnedProviderProcess({ + ...spawn, + arguments: spawn.arguments.map((argument, index) => + index === 1 ? fileURLToPath(workerUrl) : argument, + ), + }), + ); + +const createFixture = async (wal = false) => { + const workspace = await createTestWorkspace("rea-sqlite-boundary-"); + const database = createSqliteDatabaseFixture(workspace.root, wal); + if (!wal) database.close(); + onTestFinished(async () => { + if (wal) database.close(); + await removeTestWorkspace(workspace.root); + }); + return { workspace, database }; +}; + +const expectPrivateRootRemoved = (path: string) => + expect(access(path)).rejects.toMatchObject({ code: "ENOENT" }); + +it.each(["late-cancellation", "cleanup-failure"])( + "preserves completed native-worker observations across %s", + async (scenario) => { + const { workspace, database } = await createFixture(); + const original = await readFile(database.path); + const controller = new AbortController(); + let cleanupAllowed = false; + let root: PrivateRuntimeRoot | undefined; + const provider = new SqliteDatabaseProvider( + process.env, + (spawn) => + spawnOwnedProviderProcess({ + ...spawn, + arguments: spawn.arguments.map((argument, index) => + index === 1 ? fileURLToPath(workerUrl) : argument, + ), + }), + async () => { + root = await PrivateRuntimeRoot.create({ parent: workspace.root }); + const acquired = root; + return { + path: acquired.path, + close: async () => { + if (scenario === "late-cancellation") controller.abort(); + else if (!cleanupAllowed) + throw new Error("Completed snapshot cleanup denied"); + await acquired.close(); + }, + }; + }, + ); + onTestFinished(async () => { + cleanupAllowed = true; + await provider.close(); + }); + const result = await provider.inspect( + { path: database.path, table: "records", columns: ["id"], row_limit: 1 }, + { signal: controller.signal }, + ); + if (scenario === "late-cancellation") { + expect(controller.signal.aborted).toBe(true); + expect(result).toMatchObject({ + ok: true, + value: { result: { rows: { columns: ["id"], returned_rows: 1 } } }, + }); + } else { + expect(result).toMatchObject({ + ok: false, + error: { + cleanupIncomplete: true, + partialObservation: { + operation: "inspect_sqlite_database", + confidence: "observed", + subject: { + local_path: database.path, + digest: { sha256: digestForTest(original) }, + }, + parameters: { + path: database.path, + table: "records", + columns: ["id"], + row_limit: 1, + }, + normalized_result: { rows: { columns: ["id"], returned_rows: 1 } }, + }, + }, + }); + if (root === undefined) throw new Error("Missing retained root"); + await expect(access(root.path)).resolves.toBeUndefined(); + cleanupAllowed = true; + await provider.close(); + await expectPrivateRootRemoved(root.path); + } + expect(await readFile(database.path)).toEqual(original); + }, +); + +const digestForTest = (bytes: Buffer): string => + createHash("sha256").update(bytes).digest("hex"); + +it("inspects committed WAL schema and rows while preserving every source artifact", async () => { + const { database } = await createFixture(true); + const paths = [database.path, database.walPath, `${database.path}-shm`]; + const original = await Promise.all( + paths.map(async (path) => ({ + path, + bytes: await readFile(path), + stat: await lstat(path), + })), + ); + const result = await createProvider().inspect({ + path: database.path, + table: "records", + schema_scope: "database", + row_limit: 1, + }); + if (!result.ok) throw result.error; + const main = original[0]; + const wal = original[1]; + if (main === undefined || wal === undefined) + throw new Error("Missing fixture identity"); + expect(result.value.result).toMatchObject({ + artifact: { + path: database.path, + bytes: main.bytes.length, + sha256: createHash("sha256").update(main.bytes).digest("hex"), + }, + wal: { + path: database.walPath, + bytes: wal.bytes.length, + sha256: createHash("sha256").update(wal.bytes).digest("hex"), + }, + schema: { + completeness: "complete", + tables: expect.arrayContaining([ + { + name: "records", + sql: expect.stringContaining("CREATE TABLE"), + root_page: expect.any(Number), + kind: "table", + without_rowid: false, + strict: false, + column_count: 7, + columns_completeness: "complete", + columns: expect.arrayContaining([ + expect.objectContaining({ name: "generated_value", hidden: 2 }), + ]), + }, + ]), + indexes: [ + expect.objectContaining({ name: "records_text", table: "records" }), + ], + views: [ + { + name: "record_names", + sql: expect.stringContaining("SELECT text_value"), + }, + ], + triggers: [ + { + name: "records_insert", + table: "records", + sql: expect.stringContaining("AFTER INSERT"), + }, + ], + }, + rows: { + table: "records", + returned_rows: 1, + truncated: true, + row_limit: 1, + order: "unspecified", + }, + }); + for (const artifact of original) { + expect(await readFile(artifact.path)).toEqual(artifact.bytes); + const after = await lstat(artifact.path); + expect({ + ino: after.ino, + size: after.size, + mtimeMs: after.mtimeMs, + ctimeMs: after.ctimeMs, + }).toEqual({ + ino: artifact.stat.ino, + size: artifact.stat.size, + mtimeMs: artifact.stat.mtimeMs, + ctimeMs: artifact.stat.ctimeMs, + }); + } +}); + +it("preserves SQLite integer, binary, nonfinite, null, and text byte values", async () => { + const { database } = await createFixture(); + const result = await createProvider().inspect({ + path: database.path, + table: "records", + row_limit: 10, + }); + if (!result.ok) throw result.error; + expect(result.value.result).toMatchObject({ + wal: null, + rows: { + columns: [ + "id", + "integer_value", + "blob_value", + "real_value", + "nullable_value", + "text_value", + "generated_value", + ], + returned_rows: 3, + truncated: false, + values: [ + [ + { type: "integer", value: "1" }, + { type: "integer", value: "9223372036854775807" }, + { type: "blob", hex: "001fefff" }, + { type: "real", value: "Infinity" }, + { type: "null" }, + { + type: "text", + value: "first é\0value", + bytes_hex: Buffer.from("first é\0value").toString("hex"), + }, + { + type: "text", + value: "integer", + bytes_hex: Buffer.from("integer").toString("hex"), + }, + ], + [ + { type: "integer", value: "2" }, + { type: "integer", value: "-9223372036854775808" }, + { type: "blob", hex: "" }, + { type: "real", value: "-Infinity" }, + { + type: "text", + value: "present", + bytes_hex: Buffer.from("present").toString("hex"), + }, + { + type: "text", + value: "second", + bytes_hex: Buffer.from("second").toString("hex"), + }, + { + type: "text", + value: "integer", + bytes_hex: Buffer.from("integer").toString("hex"), + }, + ], + [ + { type: "integer", value: "3" }, + { type: "integer", value: "0" }, + { type: "blob", hex: "ff" }, + { type: "real", value: 1.25 }, + { type: "null" }, + { type: "text", bytes_hex: "ff00c0af" }, + { + type: "text", + value: "integer", + bytes_hex: Buffer.from("integer").toString("hex"), + }, + ], + ], + }, + }); + expect(() => JSON.stringify(result.value.result)).not.toThrow(); +}); + +it("selects an exact quoted table and refuses to execute table input as SQL", async () => { + const { database } = await createFixture(); + const provider = createProvider(); + const selected = await provider.inspect({ + path: database.path, + table: database.quotedTable, + row_limit: 1, + }); + if (!selected.ok) throw selected.error; + expect(selected.value.result).toMatchObject({ + rows: { + table: database.quotedTable, + columns: ["__proto__"], + values: [ + [ + { + type: "text", + value: "exact selected table", + bytes_hex: Buffer.from("exact selected table").toString("hex"), + }, + ], + ], + returned_rows: 1, + truncated: false, + }, + }); + const unknown = await provider.inspect({ + path: database.path, + table: 'records"; DROP TABLE records;--', + row_limit: 1, + }); + expect(unknown).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + const after = await provider.inspect({ + path: database.path, + table: "records", + row_limit: 10, + }); + if (!after.ok) throw after.error; + expect(after.value.result).toMatchObject({ rows: { returned_rows: 3 } }); +}); + +it("does not execute views when selecting rows", async () => { + const { database } = await createFixture(); + const result = await createProvider().inspect({ + path: database.path, + table: "record_names", + row_limit: 1, + }); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); +}); + +it("rejects malformed databases without altering the selected file", async () => { + const workspace = await createTestWorkspace("rea-sqlite-invalid-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const path = await workspace.write( + "not-a-database.db", + "selected non-SQLite bytes", + ); + const bytes = await readFile(path); + const result = await createProvider().inspect({ path }); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + expect(await readFile(path)).toEqual(bytes); +}); + +nativeTest.runIf(process.platform !== "win32")( + "rejects a symlink database", + async () => { + const { database, workspace } = await createFixture(); + const alias = workspace.path("alias.db"); + await symlink(database.path, alias); + const provider = createProvider(); + expect(await provider.inspect({ path: alias })).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + }, +); + +nativeTest( + "rejects an actual open rollback transaction without changing its journal", + async () => { + const { database } = await createFixture(); + const writer = new DatabaseSync(database.path); + writer.exec( + "BEGIN IMMEDIATE; INSERT INTO records(id, text_value) VALUES (4, 'uncommitted')", + ); + try { + const main = await readFile(database.path); + const journalPath = `${database.path}-journal`; + const journal = await readFile(journalPath); + expect(journal.length).toBeGreaterThan(0); + expect( + await createProvider().inspect({ path: database.path }), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + expect(await readFile(database.path)).toEqual(main); + expect(await readFile(journalPath)).toEqual(journal); + } finally { + writer.exec("ROLLBACK"); + writer.close(); + } + }, +); + +nativeTest.runIf(process.platform !== "win32")( + "rejects FIFO input without waiting for a writer", + async () => { + const workspace = await createTestWorkspace("rea-sqlite-fifo-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const path = workspace.path("database.fifo"); + await promisify(execFile)("mkfifo", [path]); + const result = await createProvider().inspect({ path }); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + }, +); + +nativeTest( + "honors a cancelled caller before opening the database", + async () => { + const { database } = await createFixture(); + const controller = new AbortController(); + controller.abort(); + expect( + await createProvider().inspect( + { path: database.path }, + { signal: controller.signal }, + ), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisCancelledError" } }); + }, +); + +nativeTest( + "removes an acquired private root when cancelled before snapshot creation", + async () => { + const { database } = await createFixture(); + const controller = new AbortController(); + let ownedRoot = ""; + const provider = new SqliteDatabaseProvider( + {}, + () => { + throw new Error("Cancelled request must not launch SQLite"); + }, + async () => { + const root = await PrivateRuntimeRoot.create({ + prefix: "rea-sqlite-cancel-root-", + }); + ownedRoot = root.path; + controller.abort(); + return root; + }, + ); + expect( + await provider.inspect( + { path: database.path }, + { signal: controller.signal }, + ), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisCancelledError" } }); + await expectPrivateRootRemoved(ownedRoot); + }, +); + +it.each(["UTF-8", "UTF-16le", "UTF-16be"] as const)( + "preserves %s text, a leading BOM and empty table selection", + async (encoding) => { + const workspace = await createTestWorkspace("rea-sqlite-encoding-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const path = workspace.path("encoding.db"); + const writer = new DatabaseSync(path); + writer.exec( + `PRAGMA encoding='${encoding}'; CREATE TABLE "" ("1" TEXT, "0" TEXT)`, + ); + const value = "\uFEFFé𝄞\0selected"; + writer + .prepare('INSERT INTO "" VALUES (?, ?)') + .run(value, "second numeric column"); + writer.close(); + const bytes = Buffer.from(value, encoding === "UTF-8" ? "utf8" : "utf16le"); + if (encoding === "UTF-16be") bytes.swap16(); + const result = await createProvider().inspect({ + path, + table: "", + }); + if (!result.ok) throw result.error; + expect(result.value.result).toMatchObject({ + database_encoding: encoding, + rows: { + table: "", + columns: ["1", "0"], + returned_rows: 1, + values: [ + [ + { type: "text", value, bytes_hex: bytes.toString("hex"), encoding }, + { type: "text", value: "second numeric column" }, + ], + ], + }, + }); + }, +); + +it("retains an unavailable virtual module definition without executing it", async () => { + const workspace = await createTestWorkspace("rea-sqlite-virtual-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const path = workspace.path("virtual.db"); + const writer = new DatabaseSync(path); + writer.exec( + "CREATE VIRTUAL TABLE optional_search USING fts5(body); INSERT INTO optional_search VALUES ('selected source')", + ); + writer.close(); + const bytes = await readFile(path); + const declaration = bytes.indexOf(Buffer.from("USING fts5")); + if (declaration < 0) + throw new Error("SQLite virtual module declaration not found"); + bytes.write("zzzz", declaration + Buffer.byteLength("USING "), "utf8"); + await writeFile(path, bytes); + const provider = createProvider(); + const result = await provider.inspect({ path, schema_scope: "database" }); + if (!result.ok) throw result.error; + expect(result.value.result).toMatchObject({ + rows: null, + schema: { + tables: expect.arrayContaining([ + expect.objectContaining({ + name: "optional_search", + kind: "virtual", + sql: expect.stringContaining("USING zzzz"), + columns: [], + columns_completeness: "unsupported", + }), + ]), + }, + }); + expect( + await provider.inspect({ path, table: "optional_search" }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisInputError" } }); + expect(await readFile(path)).toEqual(bytes); +}); + +nativeTest( + "cancels an acquired SQLite worker and removes its private snapshot", + async () => { + const { database } = await createFixture(); + const controller = new AbortController(); + let ownedRoot = ""; + let pid: number | undefined; + const provider = new SqliteDatabaseProvider({}, async (spawn) => { + ownedRoot = spawn.cwd ?? ""; + const requestPath = spawn.arguments[2]; + if (requestPath === undefined) + throw new Error("SQLite worker arguments missing"); + const launched = await spawnOwnedProviderProcess({ + ...spawn, + arguments: [ + "--input-type=module", + "-e", + `await import(${JSON.stringify(workerUrl.href)}); process.stdout.write('ready\\n'); setInterval(() => {}, 1000);`, + "sqlite-worker-test", + requestPath, + ], + }); + pid = launched.process.pid; + await waitForProviderProcessReady(launched.process); + await access(`${ownedRoot}/reply.json`); + setImmediate(() => controller.abort()); + return launched; + }); + const result = await provider.inspect( + { path: database.path, table: "records" }, + { signal: controller.signal }, + ); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + if (pid === undefined) throw new Error("SQLite worker not acquired"); + const acquiredPid = pid; + expect(() => process.kill(acquiredPid, 0)).toThrow(); + await expectPrivateRootRemoved(ownedRoot); + }, +); diff --git a/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts b/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts new file mode 100644 index 0000000000..4fce636ceb --- /dev/null +++ b/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts @@ -0,0 +1,264 @@ +import { + lstat, + open, + readFile, + rename, + symlink, + unlink, + utimes, + writeFile, +} from "node:fs/promises"; +import { DatabaseSync } from "node:sqlite"; +import { expect, it, onTestFinished, vi } from "vitest"; +import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js"; +import type { StableArtifactFileSystem } from "../../../src/artifacts/readStableArtifact.js"; +import { captureSqliteDatabaseSnapshot } from "../../../src/sqlite/SqliteDatabaseSnapshot.js"; +import { inspectSqliteDatabaseSnapshot } from "../../../src/sqlite/SqliteDatabaseInspection.js"; +import { sqliteDatabaseFailure } from "../../../src/sqlite/SqliteDatabaseFailures.js"; +import { + createSqliteDatabaseFixture, + SQLITE_NATIVE_LIMITS_AVAILABLE, +} from "../../fixtures/sqlite/database.js"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; + +const fixture = async (wal = false) => { + const workspace = await createTestWorkspace("rea-sqlite-snapshot-race-"); + const database = createSqliteDatabaseFixture(workspace.root, wal); + if (!wal) database.close(); + const resourceScope = new ArtifactResourceScope(); + onTestFinished(async () => { + await resourceScope.close(); + if (wal) database.close(); + await removeTestWorkspace(workspace.root); + }); + const root = await workspace.mkdir("private"); + const capture = (fileSystem?: StableArtifactFileSystem) => + captureSqliteDatabaseSnapshot( + database.path, + root, + resourceScope, + undefined, + fileSystem, + ); + return { workspace, database, root, capture }; +}; + +it.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE).each(["PERSIST", "TRUNCATE"])( + "inspects a committed %s journal without modifying the source file set", + async (mode) => { + const { database, capture } = await fixture(); + const writer = new DatabaseSync(database.path); + try { + writer.exec( + `PRAGMA journal_mode=${mode}; INSERT INTO records(id, text_value) VALUES (4, 'committed');`, + ); + } finally { + writer.close(); + } + const journalPath = `${database.path}-journal`; + const before = await readFile(journalPath); + const beforeStat = await lstat(journalPath, { bigint: true }); + const snapshot = await capture(); + const result = inspectSqliteDatabaseSnapshot(snapshot.snapshotPath, { + path: database.path, + table: "records", + }); + expect(result.rows).toMatchObject({ returned_rows: 4, truncated: false }); + expect(await readFile(journalPath)).toEqual(before); + const afterStat = await lstat(journalPath, { bigint: true }); + expect([ + afterStat.ino, + afterStat.size, + afterStat.mtimeNs, + afterStat.ctimeNs, + ]).toEqual([ + beforeStat.ino, + beforeStat.size, + beforeStat.mtimeNs, + beforeStat.ctimeNs, + ]); + }, +); + +it.runIf(process.platform !== "win32").each([ + ["database", "remove"], + ["WAL", "remove"], + ["database", "replace with symlink"], + ["WAL", "replace with symlink"], +])( + "reports an observed %s change during acquisition as changed evidence: %s", + async (member, change) => { + const { database, workspace, capture } = await fixture(true); + const replacement = await workspace.write("replacement", ""); + const selectedPath = + member === "database" ? database.path : database.walPath; + let observations = 0; + const fileSystem: StableArtifactFileSystem = { + lstat: async (path) => { + if (path === selectedPath && ++observations === 2) { + await unlink(path); + if (change === "replace with symlink") + await symlink(replacement, path); + } + return lstat(path, { bigint: true }); + }, + open, + }; + try { + await capture(fileSystem); + throw new Error("A disappearing source must be rejected"); + } catch (cause) { + expect( + sqliteDatabaseFailure(cause, database.path, "artifact-read"), + ).toMatchObject({ _tag: "AnalysisArtifactChangedError" }); + expect(cause).toMatchObject({ + message: expect.stringContaining(selectedPath), + }); + } + }, +); + +it("refuses an active journal even before SQLite writes its magic header", async () => { + const { database, capture } = await fixture(); + const writer = new DatabaseSync(database.path); + try { + writer.exec( + "PRAGMA journal_mode=PERSIST; BEGIN IMMEDIATE; INSERT INTO records(id, text_value) VALUES (4, 'uncommitted');", + ); + const journalPath = `${database.path}-journal`; + const journal = await readFile(journalPath); + expect(journal.subarray(0, 8).every((byte) => byte === 0)).toBe(true); + expect(journal.subarray(8, 28).some((byte) => byte !== 0)).toBe(true); + const original = await readFile(database.path); + await expect(capture()).rejects.toMatchObject({ + reason: "format", + message: expect.stringContaining("recovery"), + }); + expect(await readFile(database.path)).toEqual(original); + expect(await readFile(journalPath)).toEqual(journal); + } finally { + writer.exec("ROLLBACK"); + writer.close(); + } +}); + +it("refuses an incomplete zeroed journal header", async () => { + const { database, capture } = await fixture(); + await writeFile(`${database.path}-journal`, Buffer.alloc(27)); + await expect(capture()).rejects.toMatchObject({ reason: "format" }); +}); + +it.runIf(process.platform !== "win32")( + "refuses a journal symlink even when its target is empty", + async () => { + const { database, workspace, capture } = await fixture(); + const empty = await workspace.write("empty-journal", ""); + await symlink(empty, `${database.path}-journal`); + await expect(capture()).rejects.toMatchObject({ reason: "path" }); + }, +); + +it.each(["activate", "replace", "remove"])( + "rejects a cold journal changed during database acquisition: %s", + async (change) => { + const { database, capture } = await fixture(); + const journalPath = `${database.path}-journal`; + await writeFile(journalPath, Buffer.alloc(28)); + const fileSystem: StableArtifactFileSystem = { + lstat: (path) => lstat(path, { bigint: true }), + open: async (path, flags) => { + if (path === database.path) { + if (change === "activate") + await writeFile(journalPath, Buffer.alloc(28, 1)); + else if (change === "remove") await unlink(journalPath); + else { + await rename(journalPath, `${journalPath}.previous`); + await writeFile(journalPath, Buffer.alloc(28)); + } + } + return open(path, flags); + }, + }; + await expect(capture(fileSystem)).rejects.toMatchObject({ + reason: "integrity", + }); + }, +); + +it("refuses an activated journal even when filesystem metadata cannot distinguish the write", async () => { + const { database, capture } = await fixture(); + const journalPath = `${database.path}-journal`; + await writeFile(journalPath, Buffer.alloc(28)); + const observed = await lstat(journalPath, { bigint: true }); + const fileSystem: StableArtifactFileSystem = { + lstat: (path) => + path === journalPath + ? Promise.resolve(observed) + : lstat(path, { bigint: true }), + open: async (path, flags) => { + if (path === database.path) + await writeFile(journalPath, Buffer.alloc(28, 1)); + const handle = await open(path, flags); + if (path === journalPath) + vi.spyOn(handle, "stat").mockResolvedValue(observed); + return handle; + }, + }; + await expect(capture(fileSystem)).rejects.toMatchObject({ + reason: "integrity", + }); +}); + +it("rejects a database changed after its copy but before WAL acquisition", async () => { + const { database, root, capture } = await fixture(true); + const initial = await readFile(database.path); + const initialStat = await lstat(database.path); + let changed = false; + const fileSystem: StableArtifactFileSystem = { + lstat: (path) => lstat(path, { bigint: true }), + open: async (path, flags) => { + if (path === database.walPath) { + const bytes = Buffer.from(initial); + const offset = bytes.length - 1; + bytes[offset] = (bytes[offset] ?? 0) ^ 1; + await writeFile(database.path, bytes); + await utimes(database.path, initialStat.atime, initialStat.mtime); + changed = true; + } + return open(path, flags); + }, + }; + await expect(capture(fileSystem)).rejects.toMatchObject({ + reason: "integrity", + }); + expect(changed).toBe(true); + expect(await readFile(`${root}/database.snapshot`)).toEqual(initial); +}); + +it.each(["-wal", "-journal"] as const)( + "rejects a previously absent %s created during database acquisition", + async (suffix) => { + const { database, capture } = await fixture(); + let created = false; + const fileSystem: StableArtifactFileSystem = { + lstat: (path) => lstat(path, { bigint: true }), + open: async (path, flags) => { + if (path === database.path) { + await writeFile( + `${database.path}${suffix}`, + "newly selected sidecar", + ); + created = true; + } + return open(path, flags); + }, + }; + await expect(capture(fileSystem)).rejects.toMatchObject({ + reason: "integrity", + }); + expect(created).toBe(true); + }, +); diff --git a/tests/fixtures/android/jadx-mcp.mjs b/tests/fixtures/android/jadx-mcp.mjs index 212cf4f5b3..7319a5501a 100644 --- a/tests/fixtures/android/jadx-mcp.mjs +++ b/tests/fixtures/android/jadx-mcp.mjs @@ -159,3 +159,6 @@ for (const name of ["get_xrefs_to_class", "get_xrefs_to_method"]) }), ); await server.connect(new StdioServerTransport()); +// A refused cleanup must leave a live process for verified fixture teardown. +// Otherwise stdin EOF races the ownership scan with the server's natural exit. +if (mode === "cleanup-failure") setInterval(() => {}, 1000); diff --git a/tests/fixtures/sqlite/database.ts b/tests/fixtures/sqlite/database.ts new file mode 100644 index 0000000000..ae174c44d7 --- /dev/null +++ b/tests/fixtures/sqlite/database.ts @@ -0,0 +1,74 @@ +import { DatabaseSync } from "node:sqlite"; +import { join } from "node:path"; + +/** Engine-dependent cases require the same native limit capability as the worker. */ +export const SQLITE_NATIVE_LIMITS_AVAILABLE = (() => { + const database = new DatabaseSync(":memory:"); + try { + return "limits" in database; + } finally { + database.close(); + } +})(); + +/** Exact selected SQLite fixture with committed rows and optional live WAL. */ +export interface SqliteDatabaseFixture { + readonly path: string; + readonly walPath: string; + readonly quotedTable: string; + close(): void; +} + +/** Produce schema and lossless SQLite values through the actual SQLite engine. */ +export const createSqliteDatabaseFixture = ( + root: string, + wal = false, +): SqliteDatabaseFixture => { + const path = join(root, "selected.db"); + const db = new DatabaseSync(path); + if (wal) db.exec("PRAGMA journal_mode = WAL; PRAGMA wal_autocheckpoint = 0"); + db.exec(` + CREATE TABLE records ( + id INTEGER PRIMARY KEY, + integer_value INTEGER, + blob_value BLOB, + real_value REAL, + nullable_value TEXT, + text_value TEXT, + generated_value TEXT GENERATED ALWAYS AS (typeof(integer_value)) VIRTUAL + ); + CREATE INDEX records_text ON records(text_value); + CREATE VIEW record_names AS SELECT text_value FROM records; + CREATE TRIGGER records_insert AFTER INSERT ON records BEGIN SELECT 1; END; + CREATE TABLE "strange""'; DROP TABLE records;--" ("__proto__" TEXT); + INSERT INTO "strange""'; DROP TABLE records;--" VALUES ('exact selected table'); + `); + const insert = db.prepare( + "INSERT INTO records(id, integer_value, blob_value, real_value, nullable_value, text_value) VALUES (?, ?, ?, ?, ?, ?)", + ); + insert.run( + 1, + 9223372036854775807n, + Buffer.from([0, 31, 239, 255]), + Infinity, + null, + "first é\0value", + ); + insert.run( + 2, + -9223372036854775808n, + Buffer.alloc(0), + -Infinity, + "present", + "second", + ); + db.exec( + "INSERT INTO records(id, integer_value, blob_value, real_value, nullable_value, text_value) VALUES (3, 0, X'FF', 1.25, NULL, CAST(X'FF00C0AF' AS TEXT))", + ); + return { + path, + walPath: `${path}-wal`, + quotedTable: "strange\"'; DROP TABLE records;--", + close: () => db.close(), + }; +};