fix: retry file downloads blocked by Cloudflare or rate limits

Retry downloads once with a delay when blocked by a 429 or Cloudflare
challenge, then report the failure so the caller can handle it. Limit
Thingiverse download concurrency to 2 to reduce triggering rate limits.

backend/src/config.ts

@@ -115,6 +115,8 @@ export const IMPORT_HTML_MAX_KB = envInt("IMPORT_HTML_MAX_KB", 4096);
 export const IMPORT_HTML_MAX_BYTES = Math.max(64, IMPORT_HTML_MAX_KB) * 1024;
 // Gap between items in a batch import, to avoid anti-abuse bursts.
 export const IMPORT_COLLECTION_DELAY_MS = envInt("IMPORT_COLLECTION_DELAY_MS", 1000);
+// Wait before retrying a file download Cloudflare challenged or rate-limited.
+export const IMPORT_DOWNLOAD_RETRY_DELAY_MS = envInt("IMPORT_DOWNLOAD_RETRY_DELAY_MS", 2000);
 // MakerWorld's CAPTCHA trips on bursts, so in collection imports this delay precedes every
 // MakerWorld request (listing pages, each resolution step, each image), not just each model.
 export const IMPORT_MAKERWORLD_CALL_DELAY_MS = envInt("IMPORT_MAKERWORLD_CALL_DELAY_MS", 5000);

backend/src/services/flaresolverr.ts

@@ -39,12 +39,17 @@ export async function testFlaresolverr(
 // Cloudflare still said no after FlareSolverr, or with none set up: nothing to do with the link.
 export const CLOUDFLARE_BLOCKED_CODE = "CLOUDFLARE_BLOCKED";
 
+/** Cloudflare challenged this request. Narrower than looksLikeCloudflareBlock: a site behind
+ * Cloudflare sends `server: cloudflare` on its own 403s and 404s too. */
+export function isCloudflareChallenge(headers: Headers): boolean {
+  return headers.has("cf-mitigated");
+}
+
 /** A Cloudflare edge block rather than an application-level 403. */
 export function looksLikeCloudflareBlock(headers: Headers): boolean {
   const server = (headers.get("server") || "").toLowerCase();
   if (server.includes("cloudflare")) return true;
-  if (headers.has("cf-mitigated")) return true;
-  return false;
+  return isCloudflareChallenge(headers);
 }
 
 function parseCookieHeader(raw: string | null | undefined): Record<string, string> {

backend/src/services/importService.ts

@@ -5,6 +5,7 @@ import path from "node:path";
 import crypto from "node:crypto";
 import {
   IMPORT_ALLOWED_EXTS,
+  IMPORT_DOWNLOAD_RETRY_DELAY_MS,
   IMPORT_HTML_MAX_BYTES,
   IMPORT_MAX_BYTES,
   IMPORT_PREVIEW_IMAGE_DELAY_MS,
@@ -21,10 +22,11 @@ import {
 } from "../utils/fileUtils";
 import { validateRemoteUrl } from "../utils/urlUtils";
 import { readCapped } from "../utils/readCapped";
-import { maybeSleep, sleep } from "../utils/concurrency";
+import { mapWithConcurrency, maybeSleep, sleep } from "../utils/concurrency";
 import {
   CLOUDFLARE_BLOCKED_CODE,
   fetchViaFlaresolverr,
+  isCloudflareChallenge,
   looksLikeCloudflareBlock,
   shouldProxyHost,
 } from "./flaresolverr";
@@ -737,12 +739,26 @@ export const MULTI_FILE_PLATE_EXTS = new Set([...IMPORT_ALLOWED_EXTS].filter((ex
 
 export type PlainDownloadResult = { input: NewPlateInput } | { rateLimited: true } | null;
 
-/** Best-effort: null on failure so one bad file doesn't fail the whole import, except a 429,
- * which is reported so the caller can say why. */
+// Thingiverse's CDN challenges request bursts; parallel downloads trip it more often.
+const THINGIVERSE_DOWNLOAD_CONCURRENCY = 2;
+
+/** A 429, or a 403 Cloudflare challenge (which FlareSolverr can't solve for a binary download). */
+function isDownloadBlocked(res: Response): boolean {
+  return res.status === 429 || (res.status === 403 && isCloudflareChallenge(res.headers));
+}
+
+/** Best-effort: null on failure so one bad file doesn't fail the whole import, except a block,
+ * which is retried once and then reported so the caller can say why. */
 export async function downloadPlainFileToTemp(url: string, suggestedName: string): Promise<PlainDownloadResult> {
   try {
-    const res = await rawFetch(url, { "User-Agent": IMPORT_USER_AGENT, Accept: "*/*" });
-    if (res.status === 429) {
+    const headers = { "User-Agent": IMPORT_USER_AGENT, Accept: "*/*" };
+    let res = await rawFetch(url, headers);
+    if (isDownloadBlocked(res)) {
+      await res.body?.cancel().catch(() => undefined);
+      await maybeSleep(IMPORT_DOWNLOAD_RETRY_DELAY_MS);
+      res = await rawFetch(url, headers);
+    }
+    if (isDownloadBlocked(res)) {
       await res.body?.cancel().catch(() => undefined);
       return { rateLimited: true };
     }
@@ -818,8 +834,10 @@ async function importThingiverseThing(
   const modelFiles = plateFiles.filter((f: ThingiversePlateFile) =>
     MULTI_FILE_PLATE_EXTS.has(path.extname(f.name).toLowerCase()),
   );
-  const downloadResults = await Promise.all(
-    modelFiles.map((f: ThingiversePlateFile) => downloadPlainFileToTemp(f.url, f.name)),
+  const downloadResults = await mapWithConcurrency(
+    modelFiles,
+    THINGIVERSE_DOWNLOAD_CONCURRENCY,
+    (f: ThingiversePlateFile) => downloadPlainFileToTemp(f.url, f.name),
   );
   const downloaded = downloadResults
     .filter((result): result is { input: NewPlateInput } => result !== null && "input" in result)
@@ -904,6 +922,12 @@ async function importPrintablesModel(
     .filter((result): result is { input: NewPlateInput } => result !== null && "input" in result)
     .map((result) => result.input);
   if (!downloaded.length) {
+    if (downloadResults.some((result) => result !== null && "rateLimited" in result)) {
+      throw new HttpError(
+        429,
+        "Printables blocked a file download (rate limit or Cloudflare challenge). This usually clears after a while -- wait, then retry the same import.",
+      );
+    }
     throw new HttpError(400, "None of this model's files could be downloaded.");
   }
 

backend/src/services/thingiverseApi.ts

@@ -5,6 +5,7 @@ import type { ProviderCollectionCard } from "./providerCollectionCard";
 import {
   extractJsonFromBrowserBody,
   fetchViaFlaresolverr,
+  isCloudflareChallenge,
   looksLikeCloudflareBlock,
   shouldProxyHost,
 } from "./flaresolverr";
@@ -135,7 +136,7 @@ async function fetchThingiverseApi(
     }
   }
   // An unsolved challenge is a 403 too; it says nothing about the token.
-  if (!viaBrowser && (res.status === 429 || res.headers.get("cf-mitigated") === "challenge")) {
+  if (!viaBrowser && (res.status === 429 || isCloudflareChallenge(res.headers))) {
     throw new ThingiverseRateLimitError((await getFlaresolverrUrl()) ? "failed" : "missing");
   }
   if (res.status === 429) throw new ThingiverseRateLimitError();

backend/tests/providerImports.test.ts

@@ -302,6 +302,37 @@ describe("importing from Printables", () => {
       filename: "hook.3mf",
     });
   });
+
+  it("reports a Cloudflare-challenged file download as a block, not a generic failure", async () => {
+    const modelId = String((stamp + 1) % 1_000_000_000);
+    mockFetch({
+      "https://api.printables.com/graphql/": (init) => {
+        const body = JSON.parse(String(init?.body ?? "{}")) as { query?: string };
+        if (body.query?.includes("getDownloadLink")) {
+          return json({
+            data: {
+              getDownloadLink: {
+                ok: true,
+                errors: null,
+                output: { files: [{ id: "78", link: "https://files.printables.com/media/prints/blocked.3mf" }] },
+              },
+            },
+          });
+        }
+        return json({
+          data: { print: { id: modelId, name: "Blocked Hook", stls: [{ id: "78", name: "blocked.3mf" }] } },
+        });
+      },
+      "https://files.printables.com/media/prints/blocked.3mf": cloudflareChallenge,
+    });
+
+    const res = await request(app)
+      .post("/api/import")
+      .set(auth())
+      .send({ url: `https://www.printables.com/model/${modelId}-blocked-hook` });
+    expect(res.status).toBe(429);
+    expect(res.body.detail).toMatch(/blocked a file download/i);
+  });
 });
 
 describe("importing from MakerWorld", () => {

backend/tests/thingiverseImport.test.ts

@@ -1,7 +1,7 @@
 import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
 import request from "supertest";
 import { createApp } from "../src/app";
-import { importPrintFromUrl } from "../src/services/importService";
+import { downloadPlainFileToTemp, importPrintFromUrl } from "../src/services/importService";
 import { setThingiverseAccessToken } from "../src/services/settingsService";
 import { prisma } from "../src/db";
 
@@ -65,9 +65,17 @@ function pdfResponse(): Response {
   return new Response("%PDF-1.4 fake", { status: 200, headers: { "content-type": "application/pdf" } });
 }
 
-function mockThingiverseFetch(overrides?: { detailStatus?: number; detailBody?: unknown }) {
+function challengeResponse(): Response {
+  return new Response("<html>Just a moment...</html>", {
+    status: 403,
+    headers: { "content-type": "text/html", "cf-mitigated": "challenge", server: "cloudflare" },
+  });
+}
+
+function mockThingiverseFetch(overrides?: { detailStatus?: number; detailBody?: unknown; cdnChallenge?: boolean }) {
   return vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async (input) => {
     const url = String(input);
+    if (overrides?.cdnChallenge && url.startsWith("https://cdn.thingiverse.com/assets/")) return challengeResponse();
     if (url.startsWith(`https://api.thingiverse.com/things/${THING_ID}/categories`)) {
       return jsonResponse(200, THING_CATEGORIES);
     }
@@ -118,6 +126,29 @@ describe("importPrintFromUrl -- Thingiverse", () => {
     await expect(importPrintFromUrl(userId, THING_URL, { url: THING_URL, tags: [] })).rejects.toThrow(/app token/i);
   });
 
+  it("reports a Cloudflare-challenged file download as a block, not a generic failure", async () => {
+    await setThingiverseAccessToken(ACCESS_TOKEN);
+    global.fetch = mockThingiverseFetch({ cdnChallenge: true });
+
+    await expect(importPrintFromUrl(userId, THING_URL, { url: THING_URL, tags: [] })).rejects.toThrow(
+      /blocked a file download/i,
+    );
+  });
+
+  it("retries a challenged file download once", async () => {
+    const url = "https://cdn.thingiverse.com/assets/test/body.stl";
+    const fetchMock = vi
+      .fn<(input: RequestInfo | URL) => Promise<Response>>()
+      .mockResolvedValueOnce(challengeResponse())
+      .mockResolvedValueOnce(stlResponse("body"));
+    global.fetch = fetchMock as unknown as typeof fetch;
+
+    const result = await downloadPlainFileToTemp(url, "body.stl");
+
+    expect(result && "input" in result ? result.input.filename : null).toBe("body.stl");
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+  });
+
   it("imports a Thing's model files as plates, matches category, and attaches metadata + images", async () => {
     await setThingiverseAccessToken(ACCESS_TOKEN);
     // Several category ids, only one matching: any overlap should match.

backend/vitest.config.ts

@@ -7,6 +7,7 @@ export default defineConfig({
     setupFiles: ["tests/setup.ts"],
     testTimeout: 20000,
     hookTimeout: 20000,
+    env: { IMPORT_DOWNLOAD_RETRY_DELAY_MS: "0" },
     // Files share one database and mutate the same global rows, so parallel files race.
     fileParallelism: false,
   },