mirror of
https://github.com/morluto/rea.git
synced 2026-10-19 04:19:54 -05:00
fix(ci): handle cache eviction and composed cleanup assertions (#4824)
* fix(ci): handle eviction when staging portable build cache
* docs(readme): fix French heading spacing
(cherry picked from commit 0ccd0e6999)
* test(ci): verify observed cache coverage during eviction
* test(electron): assert composed launch cleanup diagnostics
(cherry picked from commit e1466dc480e715b427cddfc3569b2baf03574405)
---------
Co-authored-by: ShuaixinHuang <1836656514@qq.com>
This commit is contained in:
1 parent
7016951394
commit
3ddb2e4cf7
7 files changed
+508
-67
No files matched your search
@@ -360,8 +360,8 @@ jobs:
|
||||
- name: Stage current portable build task cache
|
||||
shell: bash
|
||||
run: |
|
||||
npm run --silent test:prepare -- --dry=json |
|
||||
node scripts/ci/portable-build-cache.mjs snapshot
|
||||
# A new Turbo dry run can evict the entries this build just used.
|
||||
node scripts/ci/portable-build-cache.mjs snapshot < .cache/ci-build-plan.json
|
||||
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: rea-build-cache
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
[English](../../README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · **Français** · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) · [فارسی](README_fa.md) · [Bahasa Indonesia](README_id.md) · [Italiano](README_it.md)
|
||||
· [Ελληνικά](README_el.md) · [മലയാളം](README_ml.md)
|
||||
|
||||
# REA : rétro-ingénierie de tout
|
||||
|
||||
### Un seul MCP pour la rétro-ingénierie des binaires, des applications et du comportement à l'exécution.
|
||||
|
||||
+8
-3
@@ -534,9 +534,14 @@ that generator's entry. Native outputs are not stored in this cache.
|
||||
|
||||
Native package jobs, the Windows curated lane, and the Apple artifact lane reuse
|
||||
the same workflow's Turbo cache for portable compiled JavaScript, skills, and
|
||||
catalogs. The producer stages only the completed `test:prepare` task hashes into
|
||||
`.cache/ci-build-cache/`, preserving their archives, metadata, and available manifests.
|
||||
Missing required entries fail publication; older cross-run entries stay in the
|
||||
catalogs. The key parser retains the validated `test:prepare` task identities in
|
||||
`.cache/ci-build-plan.json`; staging reads this plan without starting another
|
||||
Turbo run that could evict cache entries. The producer stages matching archives,
|
||||
metadata, and available manifests into `.cache/ci-build-cache/`.
|
||||
`portable-build-cache.json` records availability per task. Evicted or partially
|
||||
missing entries are omitted as whole entries, and consumers rebuild cache misses,
|
||||
including when every entry is unavailable. Malformed plans, non-file entries,
|
||||
and I/O errors other than disappearance still fail publication. Older cross-run entries stay in the
|
||||
original cache. The Windows lane relies on the Linux static job for typechecking. They retain the normal build and prepack
|
||||
commands, which restore matching outputs or rebuild on a cache miss. Native
|
||||
artifacts, dependency installs, package archives, and runtime verification remain
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { copyFile, lstat, mkdir, rm } from "node:fs/promises";
|
||||
import { copyFile, lstat, mkdir, rm, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { z } from "zod";
|
||||
|
||||
const output = ".cache/ci-build-cache";
|
||||
const plan = ".cache/ci-build-plan.json";
|
||||
|
||||
function readTasks() {
|
||||
const { tasks } = z
|
||||
@@ -29,37 +30,93 @@ function readTasks() {
|
||||
return tasks;
|
||||
}
|
||||
|
||||
async function copyEntry(hash) {
|
||||
const entries = [];
|
||||
for (const suffix of [".tar.zst", "-meta.json", "-manifest.json"]) {
|
||||
const name = `${hash}${suffix}`;
|
||||
const source = join(".turbo/cache", name);
|
||||
const entry = await lstat(source).catch((error) => {
|
||||
// Eviction can remove a cache hit even during the successful build.
|
||||
// Manifests are also optional in supported local cache entries.
|
||||
if (error.code === "ENOENT") return undefined;
|
||||
throw error;
|
||||
});
|
||||
if (!entry) continue;
|
||||
if (!entry.isFile())
|
||||
throw new Error(
|
||||
`Portable build cache entry must be a regular file: ${source}`,
|
||||
);
|
||||
entries.push({ suffix, name, source, size: entry.size });
|
||||
}
|
||||
// Publish whole reusable entries only. Consumers rebuild missing tasks.
|
||||
if (
|
||||
![".tar.zst", "-meta.json"].every((suffix) =>
|
||||
entries.some((entry) => entry.suffix === suffix),
|
||||
)
|
||||
)
|
||||
return undefined;
|
||||
const copied = [];
|
||||
let missing = false;
|
||||
for (const entry of entries) {
|
||||
try {
|
||||
await copyFile(entry.source, join(output, entry.name));
|
||||
copied.push(entry);
|
||||
} catch (error) {
|
||||
if (error.code !== "ENOENT") throw error;
|
||||
if (entry.suffix === "-manifest.json") continue;
|
||||
missing = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (missing) {
|
||||
for (const entry of copied) await rm(join(output, entry.name));
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
files: copied.length,
|
||||
bytes: copied.reduce((sum, entry) => sum + entry.size, 0),
|
||||
};
|
||||
}
|
||||
|
||||
async function snapshot(tasks) {
|
||||
await mkdir(output, { recursive: true });
|
||||
let files = 0;
|
||||
let bytes = 0;
|
||||
const available = new Set();
|
||||
try {
|
||||
for (const hash of new Set(tasks.map((task) => task.hash))) {
|
||||
for (const suffix of [".tar.zst", "-meta.json", "-manifest.json"]) {
|
||||
const name = `${hash}${suffix}`;
|
||||
const source = join(".turbo/cache", name);
|
||||
const entry = await lstat(source).catch((error) => {
|
||||
// Manifests are not present in every supported local cache entry.
|
||||
if (suffix === "-manifest.json" && error.code === "ENOENT")
|
||||
return undefined;
|
||||
throw error;
|
||||
});
|
||||
if (!entry) continue;
|
||||
if (!entry.isFile())
|
||||
throw new Error(
|
||||
`Portable build cache entry must be a regular file: ${source}`,
|
||||
);
|
||||
await copyFile(source, join(output, name));
|
||||
files++;
|
||||
bytes += entry.size;
|
||||
}
|
||||
const entry = await copyEntry(hash);
|
||||
if (!entry) continue;
|
||||
available.add(hash);
|
||||
files += entry.files;
|
||||
bytes += entry.bytes;
|
||||
}
|
||||
// A fully evicted cache still has an explicit artifact and coverage record;
|
||||
// required build outputs and downstream verification remain separate gates.
|
||||
const manifest = `${JSON.stringify({
|
||||
version: 1,
|
||||
tasks: tasks.map((task) => ({
|
||||
...task,
|
||||
available: available.has(task.hash),
|
||||
})),
|
||||
})}\n`;
|
||||
await writeFile(join(output, "portable-build-cache.json"), manifest);
|
||||
files++;
|
||||
bytes += Buffer.byteLength(manifest);
|
||||
} catch (error) {
|
||||
await rm(output, { recursive: true, force: true });
|
||||
throw error;
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify({ tasks: tasks.length, files, bytes, path: output }),
|
||||
JSON.stringify({
|
||||
tasks: tasks.length,
|
||||
files,
|
||||
bytes,
|
||||
path: output,
|
||||
missing_tasks: tasks
|
||||
.filter((task) => !available.has(task.hash))
|
||||
.map((task) => task.taskId),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -68,9 +125,10 @@ if (extra.length || !["key", "snapshot"].includes(mode))
|
||||
throw new Error(
|
||||
"Usage: portable-build-cache.mjs key|snapshot < turbo-dry-plan.json",
|
||||
);
|
||||
// Remove any earlier snapshot before parsing: a failed second invocation must
|
||||
// never leave an old successful artifact eligible for upload.
|
||||
// Invalidate this mode's previous result before parsing its input. Failed
|
||||
// validation must not leave stale plans or snapshots for downstream reuse.
|
||||
if (mode === "snapshot") await rm(output, { recursive: true, force: true });
|
||||
else await rm(plan, { force: true });
|
||||
const tasks = readTasks();
|
||||
if (mode === "key") {
|
||||
const prefix = z
|
||||
@@ -80,6 +138,10 @@ if (mode === "key") {
|
||||
.parse(process.env.BUILD_CACHE_PREFIX);
|
||||
const hashes = tasks.map(({ taskId, hash }) => `${taskId}:${hash}`).sort();
|
||||
const digest = createHash("sha256").update(hashes.join("\n")).digest("hex");
|
||||
// Retain only validated identities for the later snapshot. Running Turbo
|
||||
// again after the build can evict cache entries before they are copied.
|
||||
await mkdir(".cache", { recursive: true });
|
||||
await writeFile(plan, `${JSON.stringify({ tasks })}\n`);
|
||||
console.log(`key=${prefix}-${digest}\nprefix=${prefix}-`);
|
||||
} else {
|
||||
await snapshot(tasks);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import {
|
||||
cp,
|
||||
mkdir,
|
||||
readFile,
|
||||
readdir,
|
||||
@@ -9,8 +10,10 @@ import {
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { expect, it } from "vitest";
|
||||
import { parse } from "yaml";
|
||||
import { z } from "zod";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
|
||||
const script = fileURLToPath(
|
||||
@@ -21,13 +24,19 @@ const second = { taskId: "artifacts:skills", hash: "fedcba9876543210" };
|
||||
const plan = { tasks: [first, second] };
|
||||
const prefix = "rea-portable-build-v2-Linux-X64-toolchain";
|
||||
const output = ".cache/ci-build-cache";
|
||||
const manifestName = "portable-build-cache.json";
|
||||
|
||||
function run(root: string, mode: "key" | "snapshot", input: unknown = plan) {
|
||||
function run(
|
||||
root: string,
|
||||
mode: "key" | "snapshot",
|
||||
input: unknown = plan,
|
||||
environment: NodeJS.ProcessEnv = {},
|
||||
) {
|
||||
const result = spawnSync(process.execPath, [script, mode], {
|
||||
cwd: root,
|
||||
input: typeof input === "string" ? input : JSON.stringify(input),
|
||||
encoding: "utf8",
|
||||
env: { ...process.env, BUILD_CACHE_PREFIX: prefix },
|
||||
env: { ...process.env, BUILD_CACHE_PREFIX: prefix, ...environment },
|
||||
});
|
||||
expect(result.error).toBeUndefined();
|
||||
return result;
|
||||
@@ -61,6 +70,12 @@ async function inventory(root: string, path: string) {
|
||||
);
|
||||
}
|
||||
|
||||
const payload = async (root: string) =>
|
||||
(await inventory(root, output)).filter(([name]) => name !== manifestName);
|
||||
|
||||
const coverage = async (root: string) =>
|
||||
JSON.parse(await readFile(join(root, output, manifestName), "utf8"));
|
||||
|
||||
it("publishes only current task companions without changing the source cache or retaining old snapshots", async () => {
|
||||
const root = await fixture();
|
||||
await entry(root, first.hash);
|
||||
@@ -72,14 +87,19 @@ it("publishes only current task companions without changing the source cache or
|
||||
const current = before.filter(
|
||||
([name]) => name?.startsWith(first.hash) || name?.startsWith(second.hash),
|
||||
);
|
||||
expect(await inventory(root, output)).toEqual(current);
|
||||
expect(await payload(root)).toEqual(current);
|
||||
expect(JSON.parse(result.stdout)).toMatchObject({
|
||||
tasks: 2,
|
||||
files: 5,
|
||||
files: 6,
|
||||
path: output,
|
||||
missing_tasks: [],
|
||||
});
|
||||
expect(await coverage(root)).toEqual({
|
||||
version: 1,
|
||||
tasks: plan.tasks.map((task) => ({ ...task, available: true })),
|
||||
});
|
||||
expect(run(root, "snapshot", { tasks: [second] }).status).toBe(0);
|
||||
expect(await inventory(root, output)).toEqual(
|
||||
expect(await payload(root)).toEqual(
|
||||
before.filter(([name]) => name?.startsWith(second.hash)),
|
||||
);
|
||||
expect(await inventory(root, ".turbo/cache")).toEqual(before);
|
||||
@@ -96,9 +116,15 @@ it("retains existing cross-run keys independent of plan order and invalidates ch
|
||||
)
|
||||
.digest("hex");
|
||||
const expected = `key=${prefix}-${digest}\nprefix=${prefix}-\n`;
|
||||
const result = run(root, "key");
|
||||
const result = run(root, "key", {
|
||||
tasks: plan.tasks.map((task) => ({ ...task, command: "not retained" })),
|
||||
environmentVariables: { passthrough: ["not retained"] },
|
||||
});
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
expect(result.stdout).toBe(expected);
|
||||
expect(
|
||||
JSON.parse(await readFile(join(root, ".cache/ci-build-plan.json"), "utf8")),
|
||||
).toEqual(plan);
|
||||
expect(run(root, "key", { tasks: [second, first] }).stdout).toBe(expected);
|
||||
const changed = run(root, "key", {
|
||||
tasks: [first, { ...second, hash: "aaaaaaaaaaaaaaaa" }],
|
||||
@@ -120,6 +146,10 @@ it("rejects malformed or ambiguous plans in both commands and removes stale publ
|
||||
]) {
|
||||
await mkdir(join(root, output), { recursive: true });
|
||||
await writeFile(join(root, output, "old-success"), "stale");
|
||||
await writeFile(
|
||||
join(root, ".cache/ci-build-plan.json"),
|
||||
JSON.stringify(plan),
|
||||
);
|
||||
for (const mode of ["key", "snapshot"] as const) {
|
||||
const result = run(root, mode, invalid);
|
||||
expect(result.status, result.stderr).not.toBe(0);
|
||||
@@ -128,21 +158,58 @@ it("rejects malformed or ambiguous plans in both commands and removes stale publ
|
||||
await expect(readdir(join(root, output))).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
await expect(
|
||||
readFile(join(root, ".cache/ci-build-plan.json")),
|
||||
).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("fails publication on a cold cache, missing companions or non-file entries, including after a partial copy", async () => {
|
||||
it("records cold caches and omits whole entries with missing required companions", async () => {
|
||||
const root = await fixture();
|
||||
expect(run(root, "snapshot").status).not.toBe(0);
|
||||
await expect(readdir(join(root, output))).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
const cold = run(root, "snapshot");
|
||||
expect(cold.status, cold.stderr).toBe(0);
|
||||
expect(JSON.parse(cold.stdout)).toMatchObject({
|
||||
files: 1,
|
||||
missing_tasks: plan.tasks.map((task) => task.taskId),
|
||||
});
|
||||
expect(await payload(root)).toEqual([]);
|
||||
expect(await coverage(root)).toEqual({
|
||||
version: 1,
|
||||
tasks: plan.tasks.map((task) => ({ ...task, available: false })),
|
||||
});
|
||||
await entry(root, first.hash);
|
||||
for (const suffix of [".tar.zst", "-meta.json"]) {
|
||||
await entry(root, second.hash);
|
||||
await rm(join(root, ".turbo/cache", `${second.hash}${suffix}`));
|
||||
const result = run(root, "snapshot");
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
expect(JSON.parse(result.stdout)).toMatchObject({
|
||||
files: 4,
|
||||
missing_tasks: [second.taskId],
|
||||
});
|
||||
expect(
|
||||
(await payload(root)).every(([name]) => name?.startsWith(first.hash)),
|
||||
).toBe(true);
|
||||
expect(await coverage(root)).toEqual({
|
||||
version: 1,
|
||||
tasks: [
|
||||
{ ...first, available: true },
|
||||
{ ...second, available: false },
|
||||
],
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects non-file cache companions and cleans up already copied entries", async () => {
|
||||
const root = await fixture();
|
||||
await entry(root, first.hash);
|
||||
for (const suffix of [".tar.zst", "-meta.json", "-manifest.json"]) {
|
||||
await entry(root, second.hash);
|
||||
const path = join(root, ".turbo/cache", `${second.hash}${suffix}`);
|
||||
await rm(path);
|
||||
if (suffix === "-manifest.json") await mkdir(path);
|
||||
await mkdir(path);
|
||||
const result = run(root, "snapshot");
|
||||
expect(result.status, result.stderr).not.toBe(0);
|
||||
await expect(readdir(join(root, output))).rejects.toMatchObject({
|
||||
@@ -152,6 +219,59 @@ it("fails publication on a cold cache, missing companions or non-file entries, i
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ suffix: "-meta.json", code: "ENOENT" },
|
||||
{ suffix: "-manifest.json", code: "ENOENT" },
|
||||
{ suffix: "-meta.json", code: "EACCES" },
|
||||
])(
|
||||
"handles $code during $suffix copying without publishing partial entries",
|
||||
async ({ suffix, code }) => {
|
||||
const root = await fixture();
|
||||
await entry(root, first.hash);
|
||||
await entry(root, second.hash);
|
||||
const preload = join(root, "evict-during-copy.mjs");
|
||||
await writeFile(
|
||||
preload,
|
||||
`
|
||||
import fs from "node:fs/promises";
|
||||
import { syncBuiltinESMExports } from "node:module";
|
||||
const copy = fs.copyFile;
|
||||
fs.copyFile = async (source, ...args) => {
|
||||
if (source.endsWith(${JSON.stringify(second.hash + suffix)})) {
|
||||
if (${JSON.stringify(code)} === "ENOENT") await fs.unlink(source);
|
||||
else throw Object.assign(new Error("copy denied"), { code: "EACCES" });
|
||||
}
|
||||
return copy(source, ...args);
|
||||
};
|
||||
syncBuiltinESMExports();
|
||||
`,
|
||||
);
|
||||
const result = run(root, "snapshot", plan, {
|
||||
NODE_OPTIONS: `${process.env.NODE_OPTIONS ?? ""} --import=${pathToFileURL(preload).href}`,
|
||||
});
|
||||
if (code === "EACCES") {
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("EACCES");
|
||||
await expect(readdir(join(root, output))).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
} else {
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
const missing = suffix === "-meta.json";
|
||||
expect(JSON.parse(result.stdout).missing_tasks).toEqual(
|
||||
missing ? [second.taskId] : [],
|
||||
);
|
||||
expect(
|
||||
(await payload(root)).filter(([name]) => name?.startsWith(second.hash)),
|
||||
).toHaveLength(missing ? 0 : 2);
|
||||
expect((await coverage(root)).tasks).toEqual([
|
||||
{ ...first, available: true },
|
||||
{ ...second, available: !missing },
|
||||
]);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"rejects symlinked archives and optional manifests without copying their targets",
|
||||
async () => {
|
||||
@@ -174,3 +294,214 @@ it.skipIf(process.platform === "win32")(
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
async function workflowFixture(historyKiB: number) {
|
||||
const repository = fileURLToPath(new URL("../../../", import.meta.url));
|
||||
const { packageManager } = z
|
||||
.object({ packageManager: z.string() })
|
||||
.parse(
|
||||
JSON.parse(await readFile(join(repository, "package.json"), "utf8")),
|
||||
);
|
||||
const steps = z.array(
|
||||
z.object({
|
||||
id: z.string().optional(),
|
||||
name: z.string().optional(),
|
||||
run: z.string().optional(),
|
||||
}),
|
||||
);
|
||||
const action = z
|
||||
.object({ runs: z.object({ steps }) })
|
||||
.parse(
|
||||
parse(
|
||||
await readFile(
|
||||
join(repository, ".github/actions/portable-build-cache/action.yml"),
|
||||
"utf8",
|
||||
),
|
||||
),
|
||||
);
|
||||
const workflow = z
|
||||
.object({ jobs: z.object({ build: z.object({ steps }) }) })
|
||||
.parse(
|
||||
parse(
|
||||
await readFile(join(repository, ".github/workflows/ci.yml"), "utf8"),
|
||||
),
|
||||
);
|
||||
const planCommand = action.runs.steps.find((step) => step.id === "plan")?.run;
|
||||
const stageCommand = workflow.jobs.build.steps.find(
|
||||
(step) => step.name === "Stage current portable build task cache",
|
||||
)?.run;
|
||||
expect(planCommand).toBeTypeOf("string");
|
||||
expect(stageCommand).toBeTypeOf("string");
|
||||
const root = await fixture();
|
||||
await symlink(
|
||||
join(repository, "node_modules"),
|
||||
join(root, "node_modules"),
|
||||
"junction",
|
||||
);
|
||||
await mkdir(join(root, "scripts/ci"), { recursive: true });
|
||||
await writeFile(
|
||||
join(root, "scripts/ci/portable-build-cache.mjs"),
|
||||
await readFile(script),
|
||||
);
|
||||
await writeFile(join(root, "input.bin"), randomBytes(192 * 1024));
|
||||
await writeFile(
|
||||
join(root, "package.json"),
|
||||
JSON.stringify({
|
||||
name: "portable-cache-workflow-fixture",
|
||||
version: "1.0.0",
|
||||
private: true,
|
||||
packageManager,
|
||||
scripts: {
|
||||
"test:prepare": "turbo run build",
|
||||
"seed:prepare": "turbo run seed",
|
||||
"metadata:check": "node -e \"console.log('validated')\"",
|
||||
build: "node produce.mjs build",
|
||||
seed: "node produce.mjs seed",
|
||||
},
|
||||
}),
|
||||
);
|
||||
await writeFile(
|
||||
join(root, "package-lock.json"),
|
||||
JSON.stringify({
|
||||
name: "portable-cache-workflow-fixture",
|
||||
version: "1.0.0",
|
||||
lockfileVersion: 3,
|
||||
packages: {
|
||||
"": { name: "portable-cache-workflow-fixture", version: "1.0.0" },
|
||||
},
|
||||
}),
|
||||
);
|
||||
await writeFile(
|
||||
join(root, "produce.mjs"),
|
||||
`
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
const name = process.argv[2];
|
||||
await mkdir("out", { recursive: true });
|
||||
const bytes = await readFile("input.bin");
|
||||
await writeFile("out/" + name + ".bin", bytes.subarray(0, (name === "seed" ? ${historyKiB} : 64) * 1024));
|
||||
`,
|
||||
);
|
||||
await writeFile(
|
||||
join(root, "turbo.json"),
|
||||
JSON.stringify({
|
||||
cacheDir: ".turbo/cache",
|
||||
cacheMaxSize: "0.125MB",
|
||||
tasks: {
|
||||
"metadata:check": { outputs: [] },
|
||||
build: { dependsOn: ["metadata:check"], outputs: ["out/build.bin"] },
|
||||
seed: { dependsOn: ["metadata:check"], outputs: ["out/seed.bin"] },
|
||||
},
|
||||
}),
|
||||
);
|
||||
await writeFile(
|
||||
join(root, ".gitignore"),
|
||||
"node_modules/\n.turbo/\n.cache/\nout/\n",
|
||||
);
|
||||
const execute = (command: string | undefined, expectedStatus = 0) => {
|
||||
if (command === undefined) throw new Error("Missing workflow command");
|
||||
const result = spawnSync(
|
||||
"bash",
|
||||
["--noprofile", "--norc", "-e", "-o", "pipefail", "-c", command],
|
||||
{
|
||||
cwd: root,
|
||||
encoding: "utf8",
|
||||
timeout: 30_000,
|
||||
env: {
|
||||
...process.env,
|
||||
BUILD_CACHE_PREFIX: prefix,
|
||||
GITHUB_OUTPUT: join(root, ".turbo/action-output"),
|
||||
TURBO_CACHE_DIR: ".turbo/cache",
|
||||
TURBO_CACHE_MAX_SIZE: "0.125MB",
|
||||
TURBO_CACHE_MAX_AGE: "0",
|
||||
TURBO_TELEMETRY_DISABLED: "1",
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status, result.stdout + result.stderr).toBe(expectedStatus);
|
||||
return expectedStatus === 0 ? result.stdout : result.stdout + result.stderr;
|
||||
};
|
||||
return { root, planCommand, stageCommand, execute };
|
||||
}
|
||||
|
||||
it.skipIf(process.platform === "win32").each([
|
||||
{ historyKiB: 96, evictAll: false },
|
||||
{ historyKiB: 192, evictAll: false },
|
||||
{ historyKiB: 96, evictAll: true },
|
||||
])(
|
||||
"stages and restores current build entries through the real workflow (history=$historyKiB KiB, evictAll=$evictAll)",
|
||||
async ({ historyKiB, evictAll }) => {
|
||||
const { root, planCommand, stageCommand, execute } =
|
||||
await workflowFixture(historyKiB);
|
||||
execute(planCommand);
|
||||
// Exercise eviction at build startup, after building, and complete cache
|
||||
// loss. All are cache misses; successful build artifacts remain required.
|
||||
execute("npm run --silent seed:prepare");
|
||||
const prepared = execute("npm run --silent test:prepare");
|
||||
expect(prepared).toContain("2 successful, 2 total");
|
||||
if (historyKiB === 96)
|
||||
expect(prepared).toContain("Cached: 1 cached, 2 total");
|
||||
const artifact = await readFile(join(root, "out/build.bin"));
|
||||
if (evictAll)
|
||||
execute(
|
||||
"TURBO_CACHE_MAX_SIZE=1B npm run --silent test:prepare -- --dry=json > /dev/null",
|
||||
);
|
||||
const staged = JSON.parse(execute(stageCommand));
|
||||
const index = z
|
||||
.object({
|
||||
version: z.literal(1),
|
||||
tasks: z.array(
|
||||
z.object({
|
||||
taskId: z.string(),
|
||||
hash: z.string(),
|
||||
available: z.boolean(),
|
||||
}),
|
||||
),
|
||||
})
|
||||
.parse(await coverage(root));
|
||||
const identities = JSON.parse(
|
||||
await readFile(join(root, ".cache/ci-build-plan.json"), "utf8"),
|
||||
);
|
||||
expect(
|
||||
index.tasks.map(({ available: _available, ...task }) => task),
|
||||
).toEqual(identities.tasks);
|
||||
const hits = index.tasks.filter((task) => task.available).length;
|
||||
expect(staged).toMatchObject({
|
||||
tasks: 2,
|
||||
files: 1 + hits * 3,
|
||||
missing_tasks: index.tasks
|
||||
.filter((task) => !task.available)
|
||||
.map((task) => task.taskId),
|
||||
});
|
||||
if (evictAll) expect(hits).toBe(0);
|
||||
const remaining = await readdir(join(root, ".turbo/cache"));
|
||||
for (const task of index.tasks)
|
||||
expect(task.available).toBe(
|
||||
[".tar.zst", "-meta.json"].every((suffix) =>
|
||||
remaining.includes(`${task.hash}${suffix}`),
|
||||
),
|
||||
);
|
||||
await rm(join(root, ".turbo/cache"), { recursive: true });
|
||||
await cp(join(root, output), join(root, ".turbo/cache"), {
|
||||
recursive: true,
|
||||
});
|
||||
await rm(join(root, "out"), { recursive: true });
|
||||
const restored = execute("npm run --silent test:prepare");
|
||||
expect(restored).toContain(`Cached: ${hits} cached, 2 total`);
|
||||
expect(await readFile(join(root, "out/build.bin"))).toEqual(artifact);
|
||||
await expect(readFile(join(root, "out/seed.bin"))).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
execute(planCommand);
|
||||
execute("npm run --silent test:prepare");
|
||||
expect(JSON.parse(execute(stageCommand))).toMatchObject({
|
||||
tasks: 2,
|
||||
files: 7,
|
||||
missing_tasks: [],
|
||||
});
|
||||
if (evictAll) {
|
||||
await writeFile(join(root, "produce.mjs"), "process.exitCode = 47;\n");
|
||||
expect(execute("npm run --silent test:prepare", 47)).toContain("(47)");
|
||||
}
|
||||
},
|
||||
);
|
||||
@@ -200,7 +200,7 @@ it("binds provider build reuse to the current run while keeping standalone dispa
|
||||
expect(stage).toBeGreaterThan(prepare);
|
||||
expect(upload).toBeGreaterThan(stage);
|
||||
expect(buildSteps[stage]?.run).toContain(
|
||||
"npm run --silent test:prepare -- --dry=json |",
|
||||
"node scripts/ci/portable-build-cache.mjs snapshot < .cache/ci-build-plan.json",
|
||||
);
|
||||
expect(buildSteps[stage]?.if).toBeUndefined();
|
||||
expect(buildSteps[stage]?.shell).toBe("bash");
|
||||
|
||||
@@ -2,6 +2,7 @@ import { chmod, mkdir, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { expect, vi } from "vitest";
|
||||
import type { AnalysisErrorProjection } from "../../../src/domain/analysisErrorProjection.js";
|
||||
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import {
|
||||
@@ -12,6 +13,7 @@ import {
|
||||
const script = `
|
||||
import { PlaywrightElectronActiveProvider } from "./dist/browser/PlaywrightElectronActiveProvider.js";
|
||||
import { electronActiveObservationInputSchema } from "./dist/domain/javascript/electronActiveObservation.js";
|
||||
import { projectAnalysisError } from "./dist/domain/analysisErrorProjection.js";
|
||||
|
||||
const [scenario, executable, root] = JSON.parse(process.argv[1]);
|
||||
let observedRejections = 0;
|
||||
@@ -114,11 +116,8 @@ console.log(JSON.stringify({
|
||||
during,
|
||||
after: process.listenerCount("unhandledRejection"),
|
||||
observedRejections,
|
||||
reasons: results.map((result) => result.ok ? null : result.error.reason ?? null),
|
||||
errors: results.map((result) => result.ok ? null : projectAnalysisError(result.error)),
|
||||
ok: results.map((result) => result.ok),
|
||||
causeReasons: results.map((result) => result.ok ? null : result.error.cause?.reason ?? null),
|
||||
cleanupResources: results.map((result) => result.ok ? null : result.error.cleanupResources ?? null),
|
||||
causes: results.map((result) => result.ok ? null : String(result.error.cause?.message).split("\\n")[0]),
|
||||
}));
|
||||
`;
|
||||
|
||||
@@ -131,11 +130,8 @@ type Outcome = {
|
||||
readonly during: number;
|
||||
readonly after: number;
|
||||
readonly ok: readonly boolean[];
|
||||
readonly causes: readonly (string | null)[];
|
||||
readonly errors: readonly (AnalysisErrorProjection | null)[];
|
||||
readonly observedRejections: number;
|
||||
readonly reasons: readonly (string | null)[];
|
||||
readonly causeReasons: readonly (string | null)[];
|
||||
readonly cleanupResources: readonly (readonly string[] | null)[];
|
||||
};
|
||||
|
||||
const run = async (
|
||||
@@ -176,7 +172,17 @@ processTest(
|
||||
during: outcome(result).before + 1,
|
||||
after: outcome(result).before,
|
||||
ok: [false],
|
||||
causes: ["electron.launch: Process failed to launch!"],
|
||||
errors: [
|
||||
{
|
||||
details: {
|
||||
diagnostics: {
|
||||
message: expect.stringContaining(
|
||||
"electron.launch: Process failed to launch!",
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -198,7 +204,17 @@ processTest.skipIf(process.platform === "win32")(
|
||||
expect(result.exitCode, result.stderr).toBe(0);
|
||||
expect(outcome(result)).toMatchObject({
|
||||
ok: [false],
|
||||
causes: ["electron.launch: Process failed to launch!"],
|
||||
errors: [
|
||||
{
|
||||
details: {
|
||||
diagnostics: {
|
||||
message: expect.stringContaining(
|
||||
"electron.launch: Process failed to launch!",
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(outcome(result).after).toBe(outcome(result).before);
|
||||
},
|
||||
@@ -228,7 +244,17 @@ processTest(
|
||||
expect(outcome(result)).toMatchObject({
|
||||
observedRejections: 1,
|
||||
ok: [false],
|
||||
causes: ["electron.launch: Process failed to launch!"],
|
||||
errors: [
|
||||
{
|
||||
details: {
|
||||
diagnostics: {
|
||||
message: expect.stringContaining(
|
||||
"electron.launch: Process failed to launch!",
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -303,21 +329,37 @@ processTest(
|
||||
expect(observed.after).toBe(observed.before);
|
||||
expect(observed.observedRejections).toBe(3);
|
||||
expect(observed.ok).toEqual([false, false, false, false, false]);
|
||||
expect(observed.causes[0]).toBe("electron.launch: alpha failed");
|
||||
expect(observed.reasons).toEqual([
|
||||
null,
|
||||
"cleanup_failed",
|
||||
"cleanup_failed",
|
||||
"cleanup_failed",
|
||||
"cleanup_failed",
|
||||
]);
|
||||
expect(observed.causeReasons).toEqual([
|
||||
null,
|
||||
...Array.from({ length: 4 }, () => "process_ownership_unavailable"),
|
||||
]);
|
||||
expect(observed.cleanupResources).toEqual([
|
||||
[],
|
||||
...Array.from({ length: 4 }, () => ["owned_process_group"]),
|
||||
expect(observed.errors).toMatchObject([
|
||||
{
|
||||
details: {
|
||||
diagnostics: {
|
||||
message: expect.stringContaining("electron.launch: alpha failed"),
|
||||
},
|
||||
},
|
||||
},
|
||||
...Array.from({ length: 4 }, () => ({
|
||||
code: "cleanup_incomplete",
|
||||
details: {
|
||||
cleanup: "incomplete",
|
||||
resources: ["owned_process_group"],
|
||||
diagnostics: {
|
||||
primary_error: {
|
||||
code: "unreadable_output",
|
||||
details: {
|
||||
reason: "process_ownership_unavailable",
|
||||
},
|
||||
},
|
||||
cleanup_error: {
|
||||
code: "cleanup_incomplete",
|
||||
details: {
|
||||
reason: "cleanup_failed",
|
||||
cleanup: "incomplete",
|
||||
resources: ["owned_process_group"],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})),
|
||||
]);
|
||||
},
|
||||
);
|
||||
Reference in new issue
Block a user