fix(import): stop flagging models that mention robots as a MakerWorld CAPTCHA, and let a manual retry try again (#18)

* fix(import): stop flagging models that mention robots as a MakerWorld CAPTCHA, and let a manual retry try again

isCaptchaChallenge looked for "captcha" or "robot" in every top-level string of a reply,
design titles and summaries included, so a model named "Robot ..." failed as a CAPTCHA on
every retry and started the 2h cooloff for every other import. Only a captcha key or the
reply's own error, message or msg text counts now.

The cooloff also failed a retry straight away after the user solved the puzzle in the
browser, which is what the error message tells them to do. Starting or retrying a queue job
by hand, a bulk start or retry, and POST /import/jobs/:id/retry now lift it. Automatic
collection syncs still respect it.

* feat(extension): offer "Add to the queue" when MakerWorld blocks an import

A CAPTCHA or the daily download limit won't clear by clicking Import again, so the error
panel lets an admin park the link in the queue with the same collection and profile scope.

* fix(import): let a queue job cut off by a restart be started or retried

A restart marks a running job ERROR with its links still PENDING or RUNNING. The queue page
only offered Start for PAUSED jobs and Retry for jobs with FAILED links, and the API agreed,
so such a job could never run again. The same held for a job whose runner crashed.

Start now takes an ERROR job with links waiting, and a retry reruns links an interrupted
run never reached even when none of them failed.

backend/src/routes/admin.ts

@@ -180,7 +180,7 @@ router.get(
 router.post(
   "/import-queue/jobs/:id/start",
   asyncHandler(async (req, res) => {
-    await startLinksJob(await getQueueJob(req.params.id));
+    await startLinksJob(await getQueueJob(req.params.id), { manual: true });
     res.status(202).json({ ok: true });
   }),
 );
@@ -188,7 +188,7 @@ router.post(
 router.post(
   "/import-queue/jobs/:id/retry",
   asyncHandler(async (req, res) => {
-    await startLinksJob(await getQueueJob(req.params.id), { retryFailed: true });
+    await startLinksJob(await getQueueJob(req.params.id), { retryFailed: true, manual: true });
     res.status(202).json({ ok: true });
   }),
 );

backend/src/routes/imports.ts

@@ -563,7 +563,7 @@ router.post(
   asyncHandler(async (req, res) => {
     const job = await getJob(req.params.id, req.userId!);
     if (!job) throw new HttpError(404, "Import job not found");
-    await startLinksJob(job, { retryFailed: true });
+    await startLinksJob(job, { retryFailed: true, manual: true });
     res.status(202).json({ job_id: job.id });
   }),
 );

backend/src/services/importQueueService.ts

@@ -16,6 +16,7 @@ import {
   type QueuedLinkOptions,
 } from "./importJobRunner";
 import { getUserMakerworldCookie } from "./makerworldCookieService";
+import { clearCaptchaCooloff } from "./makerworldCaptcha";
 
 /** The shared body a LINKS job reruns with: its stored payload plus the owner's saved MakerWorld
  *  cookie. The cookie is never stored on the job. */
@@ -35,14 +36,19 @@ async function linksJobBody(job: ImportJob): Promise<LinksImportJobBody> {
 
 /** Starts a LINKS job's PENDING links, or with `retryFailed` puts its FAILED links back first.
  *  Resolves once the runner is launched; `run` settles when it stops (finished or paused), which is
- *  what the bulk run waits on. */
+ *  what the bulk run waits on. `manual` is a person clicking start or retry: it lifts the MakerWorld
+ *  CAPTCHA cooloff, which would otherwise fail every link without trying. */
 export async function startLinksJob(
   job: ImportJob,
-  opts: { retryFailed?: boolean } = {},
+  opts: { retryFailed?: boolean; manual?: boolean } = {},
 ): Promise<{ run: Promise<void> }> {
   if (job.type !== "LINKS") throw new HttpError(400, "Only a link-list import can be started or retried");
   if (job.status === "RUNNING" && isRunnerLive(job.id)) throw new HttpError(409, "This import is already running");
-  if (!opts.retryFailed && job.status !== "PAUSED") throw new HttpError(400, "Only a paused import can be started");
+  // An ERROR job was cut off (a restart, or its runner crashed) with links still waiting, and has
+  // no failed links for a retry to pick up, so it starts like a paused one.
+  if (!opts.retryFailed && job.status !== "PAUSED" && job.status !== "ERROR") {
+    throw new HttpError(400, "Only a paused or interrupted import can be started");
+  }
   if (isRunnerLive(job.id)) {
     // A paused job's runner finishes the link it's on before it stops, which can take a while.
     if (job.status === "PAUSED") {
@@ -56,7 +62,11 @@ export async function startLinksJob(
 
   if (opts.retryFailed) {
     const failed = await resetFailedItems(job.id);
-    if (!failed) throw new HttpError(400, "This import has no failed links to retry");
+    // Links an interrupted run never reached are rerun too, so a retry alone can finish it.
+    const waiting = await prisma.importJobItem.count({
+      where: { jobId: job.id, status: { in: ["PENDING", "RUNNING"] } },
+    });
+    if (!failed && !waiting) throw new HttpError(400, "This import has no failed links to retry");
   }
   // No runner is alive, so a RUNNING item was cut off (a restart, or a pause mid-link).
   await prisma.importJobItem.updateMany({ where: { jobId: job.id, status: "RUNNING" }, data: { status: "PENDING" } });
@@ -74,6 +84,7 @@ export async function startLinksJob(
     },
   });
   if (!count) throw new HttpError(409, "This import changed while it was being started, reload and try again");
+  if (opts.manual) clearCaptchaCooloff();
   const body = await linksJobBody(job);
   return { run: runLinksImportJob(job.id, job.userId, body) };
 }
@@ -287,7 +298,10 @@ async function drainBulkQueue(): Promise<void> {
   }
 }
 
+/** Bulk runs are started by hand, so they lift the CAPTCHA cooloff once (see startLinksJob). Not
+ *  per job: after a CAPTCHA, the jobs behind it should fail fast, not each probe MakerWorld again. */
 function enqueueBulk(entries: BulkEntry[]): number {
+  if (entries.length) clearCaptchaCooloff();
   const known = new Set([bulkCurrent, ...bulkQueue.map((e) => e.jobId)]);
   const fresh = entries.filter((e) => !known.has(e.jobId));
   bulkQueue.push(...fresh);

backend/src/services/makerworldCaptcha.ts

@@ -6,13 +6,15 @@ function isRecord(value: unknown): value is Record<string, unknown> {
 // body naming a captchaId, under either HTTP 418 or 200. It can't be solved without a real
 // browser, so it's detected by shape and reported instead of surfacing as "no file found".
 // The cloud API and collection listing share one cooldown so one can't re-trip the other.
+// Only a captcha key or the reply's own error text counts: a design whose title or summary says
+// "robot" would otherwise fail as a CAPTCHA on every retry, and start the cooloff for everyone.
 export function isCaptchaChallenge(data: unknown): boolean {
   if (!isRecord(data)) return false;
-  const haystack = Object.entries(data)
-    .map(([key, value]) => `${key} ${typeof value === "string" ? value : ""}`)
-    .join(" ")
-    .toLowerCase();
-  return haystack.includes("captchaid") || haystack.includes("captcha") || haystack.includes("robot");
+  if (Object.keys(data).some((key) => key.toLowerCase().includes("captcha"))) return true;
+  return ["error", "message", "msg"].some((key) => {
+    const value = data[key];
+    return typeof value === "string" && /captcha|robot/i.test(value);
+  });
 }
 
 export const MAKERWORLD_CAPTCHA_MESSAGE =
@@ -34,6 +36,12 @@ export function noteCaptchaChallenge(): void {
   captchaBlockedUntil = Date.now() + CAPTCHA_COOLOFF_MS;
 }
 
+/** For a run someone starts by hand: they may have just solved the puzzle in the browser, which
+ *  the cooloff can't know. A block that's still there costs one request before it trips again. */
+export function clearCaptchaCooloff(): void {
+  captchaBlockedUntil = 0;
+}
+
 export class MakerworldCaptchaError extends Error {
   constructor() {
     super(MAKERWORLD_CAPTCHA_MESSAGE);

backend/tests/importQueue.test.ts

@@ -30,6 +30,7 @@ vi.mock("../src/services/notificationService", async (importOriginal) => {
 
 import { createApp } from "../src/app";
 import { prisma } from "../src/db";
+import { clearCaptchaCooloff, makerworldCaptchaCooloffActive } from "../src/services/makerworldCaptcha";
 
 const app = createApp();
 const stamp = Date.now();
@@ -80,6 +81,8 @@ function design(designId: string) {
 const designCalls = new Map<string, number>();
 const gates = new Map<string, { held: Promise<void>; release: () => void }>();
 const gone = new Set<string>();
+// Design ids MakerWorld answers with its CAPTCHA challenge.
+const challenged = new Set<string>();
 let inFlight = 0;
 let maxInFlight = 0;
 
@@ -99,6 +102,12 @@ function mockFetch() {
         inFlight--;
       }
       if (gone.has(id)) return new Response("gone", { status: 404 });
+      if (challenged.has(id)) {
+        return new Response(JSON.stringify({ captchaId: "c1", error: "confirm you are not a robot" }), {
+          status: 418,
+          headers: { "content-type": "application/json" },
+        });
+      }
       return json(design(id));
     }
     const profile = url.match(/^https:\/\/api\.bambulab\.com\/v1\/iot-service\/api\/user\/profile\/(\d+)/);
@@ -192,6 +201,8 @@ describe("the import queue", () => {
     mockFetch();
     designCalls.clear();
     gone.clear();
+    challenged.clear();
+    clearCaptchaCooloff();
     inFlight = 0;
     maxInFlight = 0;
     notifyDelay.ms = 0;
@@ -454,6 +465,60 @@ describe("the import queue", () => {
       expect(finished.imported).toBe(1);
     });
 
+    it("starts a job a restart cut off, which has links waiting but none failed", async () => {
+      const res = await queue(member, { url: modelUrl(base + 86) });
+      await queue(member, { url: modelUrl(base + 87) });
+      const jobId = res.body.job_id;
+      // What the boot recovery in server.ts leaves: the link that was importing still RUNNING.
+      const [first] = await items(jobId);
+      await prisma.importJobItem.update({ where: { id: first.id }, data: { status: "RUNNING" } });
+      await prisma.importJob.update({
+        where: { id: jobId },
+        data: { status: "ERROR", errorMessage: "Interrupted by server restart" },
+      });
+
+      const start = await request(app).post(`/api/admin/import-queue/jobs/${jobId}/start`).set(auth(admin));
+      expect(start.status).toBe(202);
+      const finished = await waitStatus(jobId, "DONE");
+      expect(finished.imported).toBe(2);
+      expect(finished.errorMessage).toBeNull();
+      expect((await items(jobId)).map((item) => item.status)).toEqual(["DONE", "DONE"]);
+    });
+
+    it("lets a retry finish a job a restart cut off, too, though none of its links failed", async () => {
+      const res = await queue(member, { url: modelUrl(base + 88) });
+      await queue(member, { url: modelUrl(base + 89) });
+      const jobId = res.body.job_id;
+      const [first] = await items(jobId);
+      await prisma.importJobItem.update({ where: { id: first.id }, data: { status: "RUNNING" } });
+      await prisma.importJob.update({ where: { id: jobId }, data: { status: "ERROR" } });
+
+      const retry = await request(app).post(`/api/admin/import-queue/jobs/${jobId}/retry`).set(auth(admin));
+      expect(retry.status).toBe(202);
+      const finished = await waitFor(
+        () => job(jobId),
+        (j) => j.status === "DONE" && j.imported === 2,
+      );
+      expect(finished.failedCount).toBe(0);
+    });
+
+    it("still won't start or retry a finished job with nothing left", async () => {
+      const res = await queue(member, { url: modelUrl(base + 79) });
+      const jobId = res.body.job_id;
+      await request(app).post(`/api/admin/import-queue/jobs/${jobId}/start`).set(auth(admin));
+      await waitStatus(jobId, "DONE");
+      await waitFor(
+        async () => (await request(app).get("/api/admin/import-queue").set(auth(admin))).body,
+        (body) => !body.jobs.some((j: { runner_live: boolean }) => j.runner_live),
+      );
+      expect((await request(app).post(`/api/admin/import-queue/jobs/${jobId}/start`).set(auth(admin))).status).toBe(
+        400,
+      );
+      expect((await request(app).post(`/api/admin/import-queue/jobs/${jobId}/retry`).set(auth(admin))).status).toBe(
+        400,
+      );
+    });
+
     it("retries just the failed links of a job", async () => {
       gone.add(String(base + 81));
       await queue(member, { url: modelUrl(base + 80) });
@@ -473,6 +538,41 @@ describe("the import queue", () => {
       expect(designCalls.get(String(base + 80))).toBe(1);
     });
 
+    it("retries a CAPTCHA'd link for real once someone solved the puzzle, instead of failing it on the cooloff", async () => {
+      challenged.add(String(base + 83));
+      const res = await queue(member, { url: modelUrl(base + 83) });
+      await request(app).post(`/api/admin/import-queue/jobs/${res.body.job_id}/start`).set(auth(admin));
+      const first = await waitStatus(res.body.job_id, "DONE");
+      expect(first.failedCount).toBe(1);
+      expect(makerworldCaptchaCooloffActive()).toBe(true);
+
+      // Solved in the browser: the cooloff is still on, but the retry has to ask MakerWorld again.
+      challenged.clear();
+      const callsBefore = designCalls.get(String(base + 83)) ?? 0;
+      await request(app).post(`/api/admin/import-queue/jobs/${res.body.job_id}/retry`).set(auth(admin));
+      const second = await waitFor(
+        () => job(res.body.job_id),
+        (j) => j.status === "DONE" && j.failedCount === 0,
+      );
+      expect(second.imported).toBe(1);
+      expect(designCalls.get(String(base + 83))).toBeGreaterThan(callsBefore);
+    });
+
+    it("imports a model whose title talks about robots instead of calling it a CAPTCHA", async () => {
+      const res = await queue(member, { url: modelUrl(base + 84) });
+      const realFetch = global.fetch;
+      global.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+        const response = await realFetch(input, init);
+        if (!String(input).includes(`/design-service/design/${base + 84}`)) return response;
+        return json({ ...(await response.json()), title: "Robot vacuum dock", summary: "<p>Not a robot.</p>" });
+      }) as typeof fetch;
+      await request(app).post(`/api/admin/import-queue/jobs/${res.body.job_id}/start`).set(auth(admin));
+      const finished = await waitStatus(res.body.job_id, "DONE");
+      expect(finished.failedCount).toBe(0);
+      expect(finished.imported).toBe(1);
+      expect(makerworldCaptchaCooloffActive()).toBe(false);
+    });
+
     it("retries a finished job right away, while its runner is still writing the notification", async () => {
       gone.add(String(base + 85));
       const res = await queue(member, { url: modelUrl(base + 85) });

backend/tests/makerworldCaptcha.test.ts

@@ -0,0 +1,39 @@
+import { afterEach, describe, expect, it } from "vitest";
+import {
+  clearCaptchaCooloff,
+  isCaptchaChallenge,
+  makerworldCaptchaCooloffActive,
+  noteCaptchaChallenge,
+} from "../src/services/makerworldCaptcha";
+
+describe("isCaptchaChallenge", () => {
+  it("spots the real challenge shapes", () => {
+    expect(isCaptchaChallenge({ captchaId: "abc", error: "We need you to confirm you are not a robot" })).toBe(true);
+    expect(isCaptchaChallenge({ captcha_id: "abc" })).toBe(true);
+    expect(isCaptchaChallenge({ code: 1, message: "Please complete the captcha" })).toBe(true);
+    expect(isCaptchaChallenge({ msg: "confirm you are not a robot" })).toBe(true);
+  });
+
+  it("doesn't take a design that talks about robots or captchas for a challenge", () => {
+    expect(isCaptchaChallenge({ id: 1, title: "Robot vacuum dock", summary: "<p>For my robot</p>" })).toBe(false);
+    expect(isCaptchaChallenge({ id: 2, title: "CAPTCHA keychain", tags: ["captcha"] })).toBe(false);
+    expect(isCaptchaChallenge({ message: "success", url: "https://cdn.example.com/robot-arm.3mf" })).toBe(false);
+  });
+
+  it("ignores anything that isn't an object", () => {
+    expect(isCaptchaChallenge(null)).toBe(false);
+    expect(isCaptchaChallenge("captcha")).toBe(false);
+    expect(isCaptchaChallenge(["captchaId"])).toBe(false);
+  });
+});
+
+describe("the CAPTCHA cooloff", () => {
+  afterEach(() => clearCaptchaCooloff());
+
+  it("can be lifted by hand once a person has solved the puzzle", () => {
+    noteCaptchaChallenge();
+    expect(makerworldCaptchaCooloffActive()).toBe(true);
+    clearCaptchaCooloff();
+    expect(makerworldCaptchaCooloffActive()).toBe(false);
+  });
+});

extension/src/content/panels/single.ts

@@ -298,10 +298,24 @@ async function runDirectImport(opts?: { entries?: string[] }): Promise<void> {
       renderPanel(successHtml(link));
     }
   } catch (err) {
-    renderPanel(errorHtml(err));
+    renderBlockedError(err, "url", collectionId);
   }
 }
 
+/** A CAPTCHA or the daily limit won't clear by clicking Import again, so an admin can park the
+ *  link in the queue with the same choices instead, and start it once MakerWorld lets go. The
+ *  backend's errors arrive as plain text, hence the message check. */
+function renderBlockedError(err: unknown, scope: MakerworldProfileScope, collectionId: string | null): void {
+  const blocked =
+    isMakerworldBlockingError(err) || (err instanceof Error && /captcha|download limit/i.test(err.message));
+  if (!blocked || !ctx().canQueue) {
+    renderPanel(errorHtml(err));
+    return;
+  }
+  renderPanel(`${errorHtml(err)}${queueButtonHtml()}`);
+  onPanelAction("queue", () => void runQueueImport(scope, collectionId));
+}
+
 /** Null lets the backend resolve it; a CAPTCHA or the download limit would stop it too. */
 function nullUnlessBlocking(err: unknown): null {
   if (isMakerworldBlockingError(err)) throw err;
@@ -354,7 +368,7 @@ async function runProfilesImport(scope: MakerworldProfileScope, collectionId: st
   }
 
   if (!printId) {
-    renderPanel(errorHtml(lastError ?? new Error("Import failed")));
+    renderBlockedError(lastError ?? new Error("Import failed"), scope, collectionId);
     return;
   }
   const parts = [`${added} print profile${added === 1 ? "" : "s"} imported`];

frontend/src/pages/ImportQueuePage/index.tsx

@@ -324,7 +324,12 @@ function JobRow({
   const isLinks = job.type === "LINKS";
   const status = jobStatus(job);
   const idle = job.status !== "RUNNING" && !job.runner_live;
-  const canStart = isLinks && job.status === "PAUSED" && !job.runner_live && job.pending_count + job.running_count > 0;
+  // An ERROR job was cut off with links still waiting (a restart, say), and starts like a paused one.
+  const canStart =
+    isLinks &&
+    (job.status === "PAUSED" || job.status === "ERROR") &&
+    !job.runner_live &&
+    job.pending_count + job.running_count > 0;
   const canPause = isLinks && job.status === "RUNNING";
   const canRetry = isLinks && idle && job.failed_items_count > 0;
   const progress = job.total > 0 ? Math.min(100, (job.processed / job.total) * 100) : 0;