ps5upload/bench/resume-test.mjs

#!/usr/bin/env node
/*
 * Interrupt + resume hardware test (manual).
 *
 * 1. Start a 1 GiB single-file upload with a fixed tx_id.
 * 2. Kill the engine mid-transfer (simulates a crash / hard network drop).
 * 3. Restart the engine, re-issue the SAME tx_id → payload resumes from its
 *    journaled last_acked_shard (BeginTxAck), engine sends only the remainder.
 * 4. Verify: run-2 sent < full size (proves resume, not re-send) AND the
 *    final file on the PS5 byte-matches the source (download-back sha256).
 *
 * Usage: node bench/resume-test.mjs <ps5-ip>
 */
import { spawn, execFileSync } from "node:child_process";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import crypto from "node:crypto";
import { fileURLToPath } from "node:url";

const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const ENGINE_BIN = path.join(repoRoot, "engine", "target", "release", "ps5upload-engine");
const ENGINE = "http://127.0.0.1:19113";
const HOST = process.argv[2];
if (!HOST) { console.error("usage: resume-test.mjs <ps5-ip>"); process.exit(2); }
const ADDR = `${HOST}:9113`;
const SRC = path.join(repoRoot, "bench", "fixtures", "huge-file", "huge-file.bin");
const TXID = "aabbccddeeff00112233445566778899";
const DEST = "/data/ps5upload/tests/resume/huge.bin";
const DESTDIR = "/data/ps5upload/tests/resume";

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

let engineProc = null;
function startEngine() {
  engineProc = spawn(ENGINE_BIN, [], { cwd: path.join(repoRoot, "engine"), stdio: "ignore", detached: false });
}
function killEngine() { if (engineProc) { try { engineProc.kill("SIGKILL"); } catch {} engineProc = null; } }
async function waitEngine(up = true, timeoutMs = 15000) {
  const t0 = Date.now();
  for (;;) {
    let ok = false;
    try { const r = await fetch(`${ENGINE}/api/jobs`); ok = r.ok; } catch {}
    if (ok === up) return true;
    if (Date.now() - t0 > timeoutMs) throw new Error(`engine ${up ? "up" : "down"} wait timeout`);
    await sleep(200);
  }
}
async function jpost(p, body) {
  const r = await fetch(`${ENGINE}${p}`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
  return await r.json().catch(() => ({}));
}
async function jget(p) { const r = await fetch(`${ENGINE}${p}`); return await r.json().catch(() => ({})); }
async function pollJob(id, timeoutMs = 180000) {
  const t0 = Date.now();
  for (;;) {
    const j = await jget(`/api/jobs/${id}`).catch(() => ({}));
    if (j.status === "done") return { ok: true, j };
    if (j.status === "failed" || j.status === "error") return { ok: false, error: j.error || JSON.stringify(j) };
    if (Date.now() - t0 > timeoutMs) return { ok: false, error: "timeout" };
    await sleep(300);
  }
}

async function main() {
  console.log(`\n==== INTERRUPT + RESUME → ${ADDR} ====`);
  const srcBuf = await fs.readFile(SRC);
  const srcSha = crypto.createHash("sha256").update(srcBuf).digest("hex");
  const fullSize = srcBuf.length;
  console.log(`  source: ${(fullSize / 1048576).toFixed(0)} MiB  sha=${srcSha.slice(0, 16)}…`);

  // Clean dest, ensure engine up
  killEngine(); await waitEngine(false).catch(() => {});
  startEngine(); await waitEngine(true);
  await jpost("/api/ps5/fs/delete", { path: DESTDIR, addr: ADDR });
  await sleep(500);

  // ---- Run 1: start, then kill mid-transfer ----
  console.log("  run-1: starting upload, will kill engine mid-transfer...");
  const r1 = await jpost("/api/transfer/file", { src: SRC, dest: DEST, addr: ADDR, tx_id: TXID });
  if (!r1.job_id) { console.log("  ✗ run-1 start failed", JSON.stringify(r1)); process.exit(1); }
  // Wait for partial progress (some bytes on the wire), then hard-kill.
  let partialSeen = false;
  for (let i = 0; i < 40; i++) {
    await sleep(250);
    const j = await jget(`/api/jobs/${r1.job_id}`).catch(() => ({}));
    const sent = j.bytes_sent ?? 0;
    if (sent > 0 && sent < fullSize) { partialSeen = true; console.log(`  run-1: ${(sent / 1048576).toFixed(0)} MiB sent — killing engine now`); break; }
    if (j.status === "done") { console.log("  run-1 finished before we could interrupt (too fast); resume still valid as no-op"); break; }
  }
  killEngine();
  await waitEngine(false).catch(() => {});
  console.log("  engine killed (simulated crash). Payload should have journaled progress.");
  await sleep(1500); // let the payload notice the dropped connection

  // ---- Run 2: restart engine, resume same tx_id ----
  startEngine(); await waitEngine(true);
  console.log("  run-2: re-issuing SAME tx_id (resume)...");
  const r2 = await jpost("/api/transfer/file", { src: SRC, dest: DEST, addr: ADDR, tx_id: TXID });
  if (!r2.job_id) { console.log("  ✗ run-2 start failed", JSON.stringify(r2)); process.exit(1); }
  // Diagnostic: log run-2 progress + payload active-tx every 5s so a stall is visible.
  const diag = setInterval(async () => {
    const j = await jget(`/api/jobs/${r2.job_id}`).catch(() => ({}));
    const st = await jget(`/api/ps5/status?addr=${encodeURIComponent(ADDR)}`).catch(() => ({}));
    console.log(`    [diag] job=${j.status} sent=${((j.bytes_sent||0)/1048576).toFixed(0)}MiB | payload active_tx=${st.active_transactions} last_seq=${st.last_tx_seq}`);
  }, 5000);
  const res2 = await pollJob(r2.job_id, 150000);
  clearInterval(diag);
  if (!res2.ok) { console.log(`  ✗ run-2 FAILED: ${res2.error}`); process.exit(1); }
  const sent2 = res2.j.bytes_sent ?? -1;
  const skipped2 = res2.j.skipped_bytes ?? 0;
  console.log(`  run-2 done: bytes_sent=${(sent2 / 1048576).toFixed(0)} MiB skipped=${(skipped2 / 1048576).toFixed(0)} MiB`);

  // ---- Verify final file integrity via download-back ----
  const dlDir = await fs.mkdtemp(path.join(os.tmpdir(), "resume-dl-"));
  const dl = await jpost("/api/transfer/download", { src_path: DEST, dest_dir: dlDir, kind: "file", addr: ADDR });
  if (!dl.job_id) { console.log("  ✗ download start failed", JSON.stringify(dl)); process.exit(1); }
  const dlres = await pollJob(dl.job_id);
  if (!dlres.ok) { console.log(`  ✗ download FAILED: ${dlres.error}`); process.exit(1); }
  const back = await fs.readFile(path.join(dlDir, "huge.bin"));
  const backSha = crypto.createHash("sha256").update(back).digest("hex");
  await fs.rm(dlDir, { recursive: true, force: true });

  // ---- Verdict ----
  let okAll = true;
  if (back.length !== fullSize) { console.log(`  ✗ size mismatch: ${back.length} vs ${fullSize}`); okAll = false; }
  if (backSha !== srcSha) { console.log(`  ✗ SHA MISMATCH — resumed file is corrupt`); okAll = false; }
  else console.log(`  ✓ final file byte-identical to source (sha match)`);
  if (partialSeen && sent2 >= fullSize) console.log(`  ⚠ run-2 re-sent the whole file (${(sent2/1048576).toFixed(0)} MiB) — resume did not skip acked shards`);
  else if (partialSeen) console.log(`  ✓ resume skipped already-acked data (run-2 sent only ${(sent2 / 1048576).toFixed(0)}/${(fullSize/1048576).toFixed(0)} MiB)`);

  await jpost("/api/ps5/fs/delete", { path: DESTDIR, addr: ADDR });
  killEngine();
  console.log(okAll ? "\n  RESUME TEST: PASS" : "\n  RESUME TEST: FAIL");
  process.exit(okAll ? 0 : 1);
}
main().catch((e) => { console.error("FATAL", e); killEngine(); process.exit(3); });