perf(ci): upload only current portable build cache entries

Stage only the current portable Turbo task archives, metadata and available manifests for same-run artifact consumers. Share task identity/hash validation with cross-run keys and fail publication on incomplete preparation, while retaining existing cache authority and consumer build/prepack verification.

Validation: 252 focused tests, full check:ci, docs:check, check:fast, actionlint, real Turbo output equivalence and invalidation, restored CLI/MCP scenarios and full Linux package verification passed. Fault injection detected historical-entry inclusion. Exact-head hosted CI and available review passed; measurement limits are recorded in the PR.

Refs #4744.

.github/actions/portable-build-cache/action.yml

@@ -24,19 +24,7 @@ runs:
         BUILD_CACHE_PREFIX: rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.npmrc', '.nvmrc', 'package-lock.json') }}
       run: |
         npm run --silent test:prepare -- --dry=json |
-          node --input-type=module -e '
-            import { createHash } from "node:crypto";
-            import { readFileSync } from "node:fs";
-            const { tasks } = JSON.parse(readFileSync(0, "utf8"));
-            const hashes = tasks.map(({ taskId, hash }) => {
-              if (typeof hash !== "string")
-                throw new Error(`Build task ${taskId} must have an eager Turbo hash`);
-              return `${taskId}:${hash}`;
-            }).sort();
-            const hash = createHash("sha256").update(hashes.join("\n")).digest("hex");
-            console.log(`key=${process.env.BUILD_CACHE_PREFIX}-${hash}`);
-            console.log(`prefix=${process.env.BUILD_CACHE_PREFIX}-`);
-          ' >> "$GITHUB_OUTPUT"
+          node scripts/ci/portable-build-cache.mjs key >> "$GITHUB_OUTPUT"
     - name: Restore portable build outputs
       if: inputs.build_cache_artifact == '' && inputs.save != 'true'
       uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0

.github/workflows/ci.yml

@@ -357,10 +357,15 @@ jobs:
       # test:prepare caches only portable JS, skill, and catalog outputs.
       # Native package jobs still run their build and prepack checks; Turbo
       # restores matching outputs or rebuilds when their inputs differ.
+      - name: Stage current portable build task cache
+        shell: bash
+        run: |
+          npm run --silent test:prepare -- --dry=json |
+            node scripts/ci/portable-build-cache.mjs snapshot
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: rea-build-cache
-          path: .turbo/cache/
+          path: .cache/ci-build-cache/
           include-hidden-files: true
           if-no-files-found: error
           retention-days: 7

docs/testing.md

@@ -503,7 +503,7 @@ costs. Its queue, execution, and failures are independent of the required gate
 and publication. CPU time and peak process-family memory are not measured by this API.
 
 The build job saves portable Turbo outputs across runs. Eight Linux real-system
-lanes restore this cache without waiting for the build job. The shared action
+lanes reuse its current-run artifact; standalone dispatches restore the cross-run cache. The shared action
 derives its key from `test:prepare --dry=json` task hashes, runner OS and
 architecture, `.npmrc`, `.nvmrc`, and the lockfile; test-only changes can reuse compiled outputs. Every
 build and verification command still runs, so Turbo checks task inputs and
@@ -513,7 +513,10 @@ that generator's entry. Native outputs are not stored in this cache.
 
 Native package jobs, the Windows curated lane, and the Apple artifact lane reuse
 the same workflow's Turbo cache for portable compiled JavaScript, skills, and
-catalogs. The Windows lane relies on the Linux static job for typechecking. They retain the normal build and prepack
+catalogs. The producer stages only the completed `test:prepare` task hashes into
+`.cache/ci-build-cache/`, preserving their archives, metadata, and available manifests.
+Missing required entries fail publication; older cross-run entries stay in the
+original cache. The Windows lane relies on the Linux static job for typechecking. They retain the normal build and prepack
 commands, which restore matching outputs or rebuild on a cache miss. Native
 artifacts, dependency installs, package archives, and runtime verification remain
 host-local. Keep platform-dependent outputs out of this shared cache.

scripts/ci/portable-build-cache.mjs

@@ -0,0 +1,86 @@
+import { createHash } from "node:crypto";
+import { readFileSync } from "node:fs";
+import { copyFile, lstat, mkdir, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { z } from "zod";
+
+const output = ".cache/ci-build-cache";
+
+function readTasks() {
+  const { tasks } = z
+    .object({
+      tasks: z
+        .array(
+          z.object({
+            taskId: z.string().min(1),
+            // The pinned Turbo version uses eager 64-bit hexadecimal hashes.
+            hash: z.string().regex(/^[0-9a-f]{16}$/u),
+          }),
+        )
+        .min(1),
+    })
+    .parse(JSON.parse(readFileSync(0, "utf8")));
+  const identities = new Set();
+  for (const { taskId } of tasks) {
+    if (identities.has(taskId))
+      throw new Error(`Duplicate portable build task: ${taskId}`);
+    identities.add(taskId);
+  }
+  return tasks;
+}
+
+async function snapshot(tasks) {
+  await mkdir(output, { recursive: true });
+  let files = 0;
+  let bytes = 0;
+  try {
+    for (const hash of new Set(tasks.map((task) => task.hash))) {
+      for (const suffix of [".tar.zst", "-meta.json", "-manifest.json"]) {
+        const name = `${hash}${suffix}`;
+        const source = join(".turbo/cache", name);
+        const entry = await lstat(source).catch((error) => {
+          // Manifests are not present in every supported local cache entry.
+          if (suffix === "-manifest.json" && error.code === "ENOENT")
+            return undefined;
+          throw error;
+        });
+        if (!entry) continue;
+        if (!entry.isFile())
+          throw new Error(
+            `Portable build cache entry must be a regular file: ${source}`,
+          );
+        await copyFile(source, join(output, name));
+        files++;
+        bytes += entry.size;
+      }
+    }
+  } catch (error) {
+    await rm(output, { recursive: true, force: true });
+    throw error;
+  }
+  console.log(
+    JSON.stringify({ tasks: tasks.length, files, bytes, path: output }),
+  );
+}
+
+const [mode, ...extra] = process.argv.slice(2);
+if (extra.length || !["key", "snapshot"].includes(mode))
+  throw new Error(
+    "Usage: portable-build-cache.mjs key|snapshot < turbo-dry-plan.json",
+  );
+// Remove any earlier snapshot before parsing: a failed second invocation must
+// never leave an old successful artifact eligible for upload.
+if (mode === "snapshot") await rm(output, { recursive: true, force: true });
+const tasks = readTasks();
+if (mode === "key") {
+  const prefix = z
+    .string()
+    .min(1)
+    .regex(/^[^\r\n]+$/u)
+    .parse(process.env.BUILD_CACHE_PREFIX);
+  const hashes = tasks.map(({ taskId, hash }) => `${taskId}:${hash}`).sort();
+  const digest = createHash("sha256").update(hashes.join("\n")).digest("hex");
+  console.log(`key=${prefix}-${digest}\nprefix=${prefix}-`);
+} else {
+  await snapshot(tasks);
+}

tests/boundary/filesystem/ciPortableBuildCache.test.ts

@@ -0,0 +1,176 @@
+import { spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import {
+  mkdir,
+  readFile,
+  readdir,
+  rm,
+  symlink,
+  writeFile,
+} from "node:fs/promises";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { expect, it } from "vitest";
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+
+const script = fileURLToPath(
+  new URL("../../../scripts/ci/portable-build-cache.mjs", import.meta.url),
+);
+const first = { taskId: "compile", hash: "0123456789abcdef" };
+const second = { taskId: "artifacts:skills", hash: "fedcba9876543210" };
+const plan = { tasks: [first, second] };
+const prefix = "rea-portable-build-v2-Linux-X64-toolchain";
+const output = ".cache/ci-build-cache";
+
+function run(root: string, mode: "key" | "snapshot", input: unknown = plan) {
+  const result = spawnSync(process.execPath, [script, mode], {
+    cwd: root,
+    input: typeof input === "string" ? input : JSON.stringify(input),
+    encoding: "utf8",
+    env: { ...process.env, BUILD_CACHE_PREFIX: prefix },
+  });
+  expect(result.error).toBeUndefined();
+  return result;
+}
+
+async function fixture() {
+  const root = await createTestTempDirectory("rea-ci-portable-cache-");
+  await mkdir(join(root, ".turbo/cache"), { recursive: true });
+  return root;
+}
+
+async function entry(root: string, hash: string, manifest = true) {
+  for (const suffix of [
+    ".tar.zst",
+    "-meta.json",
+    ...(manifest ? ["-manifest.json"] : []),
+  ])
+    await writeFile(
+      join(root, ".turbo/cache", `${hash}${suffix}`),
+      `${hash}${suffix}\n`,
+    );
+}
+
+async function inventory(root: string, path: string) {
+  const files = (await readdir(join(root, path))).sort();
+  return Promise.all(
+    files.map(async (name) => [
+      name,
+      await readFile(join(root, path, name), "hex"),
+    ]),
+  );
+}
+
+it("publishes only current task companions without changing the source cache or retaining old snapshots", async () => {
+  const root = await fixture();
+  await entry(root, first.hash);
+  await entry(root, second.hash, false);
+  await entry(root, "aaaaaaaaaaaaaaaa");
+  const before = await inventory(root, ".turbo/cache");
+  const result = run(root, "snapshot");
+  expect(result.status, result.stderr).toBe(0);
+  const current = before.filter(
+    ([name]) => name?.startsWith(first.hash) || name?.startsWith(second.hash),
+  );
+  expect(await inventory(root, output)).toEqual(current);
+  expect(JSON.parse(result.stdout)).toMatchObject({
+    tasks: 2,
+    files: 5,
+    path: output,
+  });
+  expect(run(root, "snapshot", { tasks: [second] }).status).toBe(0);
+  expect(await inventory(root, output)).toEqual(
+    before.filter(([name]) => name?.startsWith(second.hash)),
+  );
+  expect(await inventory(root, ".turbo/cache")).toEqual(before);
+});
+
+it("retains existing cross-run keys independent of plan order and invalidates changed task hashes", async () => {
+  const root = await fixture();
+  const digest = createHash("sha256")
+    .update(
+      plan.tasks
+        .map(({ taskId, hash }) => `${taskId}:${hash}`)
+        .sort()
+        .join("\n"),
+    )
+    .digest("hex");
+  const expected = `key=${prefix}-${digest}\nprefix=${prefix}-\n`;
+  const result = run(root, "key");
+  expect(result.status, result.stderr).toBe(0);
+  expect(result.stdout).toBe(expected);
+  expect(run(root, "key", { tasks: [second, first] }).stdout).toBe(expected);
+  const changed = run(root, "key", {
+    tasks: [first, { ...second, hash: "aaaaaaaaaaaaaaaa" }],
+  });
+  expect(changed.status, changed.stderr).toBe(0);
+  expect(changed.stdout).not.toBe(expected);
+});
+
+it("rejects malformed or ambiguous plans in both commands and removes stale publication output", async () => {
+  const root = await fixture();
+  for (const invalid of [
+    "not JSON",
+    {},
+    { tasks: [] },
+    { tasks: [{ taskId: "compile" }] },
+    { tasks: [{ ...first, hash: null }] },
+    { tasks: [{ ...first, hash: "../outside" }] },
+    { tasks: [first, { ...first, hash: second.hash }] },
+  ]) {
+    await mkdir(join(root, output), { recursive: true });
+    await writeFile(join(root, output, "old-success"), "stale");
+    for (const mode of ["key", "snapshot"] as const) {
+      const result = run(root, mode, invalid);
+      expect(result.status, result.stderr).not.toBe(0);
+      expect(result.stdout).toBe("");
+    }
+    await expect(readdir(join(root, output))).rejects.toMatchObject({
+      code: "ENOENT",
+    });
+  }
+});
+
+it("fails publication on a cold cache, missing companions or non-file entries, including after a partial copy", async () => {
+  const root = await fixture();
+  expect(run(root, "snapshot").status).not.toBe(0);
+  await expect(readdir(join(root, output))).rejects.toMatchObject({
+    code: "ENOENT",
+  });
+  await entry(root, first.hash);
+  for (const suffix of [".tar.zst", "-meta.json", "-manifest.json"]) {
+    await entry(root, second.hash);
+    const path = join(root, ".turbo/cache", `${second.hash}${suffix}`);
+    await rm(path);
+    if (suffix === "-manifest.json") await mkdir(path);
+    const result = run(root, "snapshot");
+    expect(result.status, result.stderr).not.toBe(0);
+    await expect(readdir(join(root, output))).rejects.toMatchObject({
+      code: "ENOENT",
+    });
+    await rm(path, { recursive: true, force: true });
+  }
+});
+
+it.skipIf(process.platform === "win32")(
+  "rejects symlinked archives and optional manifests without copying their targets",
+  async () => {
+    const root = await fixture();
+    const outside = join(root, "outside");
+    await writeFile(outside, "must not publish");
+    for (const suffix of [".tar.zst", "-manifest.json"]) {
+      await entry(root, first.hash);
+      const path = join(root, ".turbo/cache", `${first.hash}${suffix}`);
+      await rm(path);
+      await symlink(outside, path);
+      const result = run(root, "snapshot", { tasks: [first] });
+      expect(result.status, result.stderr).not.toBe(0);
+      expect(result.stderr).toContain("must be a regular file");
+      await expect(readdir(join(root, output))).rejects.toMatchObject({
+        code: "ENOENT",
+      });
+      expect(await readFile(outside, "utf8")).toBe("must not publish");
+      await rm(path);
+    }
+  },
+);

tests/boundary/filesystem/ciRequiredLanes.test.ts

@@ -175,6 +175,41 @@ it("holds native and provider fan-out behind static validation while portable fe
 
 it("binds provider build reuse to the current run while keeping standalone dispatches independent", async () => {
   const { jobs } = await workflow();
+  const buildSteps = z
+    .array(
+      z.object({
+        run: z.string().optional(),
+        uses: z.string().optional(),
+        shell: z.string().optional(),
+        if: z.string().optional(),
+        with: z.record(z.string(), z.unknown()).optional(),
+        "continue-on-error": z.boolean().optional(),
+      }),
+    )
+    .parse(jobs.build?.steps);
+  const prepare = buildSteps.findIndex(
+    (step) => step.run === "npm run test:prepare",
+  );
+  const stage = buildSteps.findIndex((step) =>
+    step.run?.includes("portable-build-cache.mjs snapshot"),
+  );
+  const upload = buildSteps.findIndex(
+    (step) => step.with?.name === "rea-build-cache",
+  );
+  expect(prepare).toBeGreaterThanOrEqual(0);
+  expect(stage).toBeGreaterThan(prepare);
+  expect(upload).toBeGreaterThan(stage);
+  expect(buildSteps[stage]?.run).toContain(
+    "npm run --silent test:prepare -- --dry=json |",
+  );
+  expect(buildSteps[stage]?.if).toBeUndefined();
+  expect(buildSteps[stage]?.shell).toBe("bash");
+  expect(buildSteps[stage]?.["continue-on-error"]).not.toBe(true);
+  expect(buildSteps[upload]?.if).toBeUndefined();
+  expect(buildSteps[upload]?.with).toMatchObject({
+    path: ".cache/ci-build-cache/",
+    "if-no-files-found": "error",
+  });
   const consumers = [
     "browser",
     "binary_layout",
@@ -252,6 +287,7 @@ it("binds provider build reuse to the current run while keeping standalone dispa
             if: z.string().optional(),
             with: z.record(z.string(), z.unknown()).optional(),
             "continue-on-error": z.boolean().optional(),
+            run: z.string().optional(),
           }),
         ),
       }),
@@ -276,6 +312,9 @@ it("binds provider build reuse to the current run while keeping standalone dispa
   });
   expect(download?.if).toBe("inputs.build_cache_artifact != ''");
   expect(download?.["continue-on-error"]).not.toBe(true);
+  expect(action.runs.steps.find((step) => step.id === "plan")?.run).toContain(
+    'node scripts/ci/portable-build-cache.mjs key >> "$GITHUB_OUTPUT"',
+  );
   for (const step of action.runs.steps.filter(
     (step) => step.id === "plan" || step.uses?.startsWith("actions/cache"),
   ))