feat: enable back flaresolverr

backend/src/services/importJobRunner.ts

@@ -185,6 +185,7 @@ async function runThingiverseThingsImportJob(
     let alreadyInLibrary = 0;
     let processed = 0;
     let unavailable = 0;
+    let rateLimited = 0;
     let authFailed = 0;
     const failed: string[] = [];
     const successPrintIds: string[] = [];
@@ -206,6 +207,7 @@ async function runThingiverseThingsImportJob(
         failed.push(thingId);
         const reason = classifyImportFailure(err);
         if (reason === "unavailable") unavailable++;
+        else if (reason === "rateLimited") rateLimited++;
         else if (reason === "auth") authFailed++;
       } finally {
         processed++;
@@ -234,8 +236,17 @@ async function runThingiverseThingsImportJob(
 
     const bodyParts: string[] = [];
     if (alreadyInLibrary) bodyParts.push(`${alreadyInLibrary} already in your library`);
-    const otherFailed = failed.length - unavailable - authFailed;
+    const otherFailed = failed.length - unavailable - rateLimited - authFailed;
     if (unavailable) bodyParts.push(`${unavailable} unavailable (private, deleted, or hidden)`);
+    // Same idea as the MakerWorld CAPTCHA case above: once Thingiverse's Cloudflare bot-management
+    // trips, every remaining item fails the same way for a while, so call out the actionable
+    // cause instead of a vague "N failed" -- especially since this is usually the bulk of a
+    // large batch's failures once it trips.
+    if (rateLimited) {
+      bodyParts.push(
+        `${rateLimited} blocked by Thingiverse's rate-limit protection (too many requests at once) — wait a while, then retry`,
+      );
+    }
     if (authFailed) bodyParts.push(`${authFailed} failed because the configured Access Token was rejected`);
     if (otherFailed) bodyParts.push(`${otherFailed} failed`);
     const label = sourceLabel(collectionTitle);

backend/src/services/importService.ts

@@ -37,6 +37,7 @@ import {
   parseThingiverseThingUrl,
   resolveThingiverseThing,
   ThingiverseAuthError,
+  ThingiverseRateLimitError,
   type ThingiversePlateFile,
 } from "./thingiverseApi";
 import { getThingiverseAccessToken } from "./settingsService";
@@ -336,7 +337,6 @@ const PREVIEW_IMAGE_MAX_COUNT = 20;
 
 async function fetchImageBytes(url: string): Promise<Buffer | null> {
   try {
-  console.error("DEBUG fetchImageBytes url=", url);
     const res = await rawFetch(url, { "User-Agent": IMPORT_USER_AGENT, Accept: "image/*" });
     if (!res.ok) {
       await res.body?.cancel().catch(() => undefined);
@@ -463,12 +463,21 @@ async function findExistingImportedPrint(
 
 const THINGIVERSE_PLATE_EXTS = new Set([...IMPORT_ALLOWED_EXTS].filter((ext) => ext !== ".zip"));
 
+type PlainDownloadResult = { input: NewPlateInput } | { rateLimited: true } | null;
+
 /** Downloads one plain, unauthenticated file URL (a Thingiverse CDN asset -- see
  * thingiverseApi.ts's zip_data.files) to a temp file. Best-effort: returns null instead of
- * throwing, so one unreachable file among several doesn't fail the whole Thing import. */
-async function downloadPlainFileToTemp(url: string, suggestedName: string): Promise<NewPlateInput | null> {
+ * throwing, so one unreachable file among several doesn't fail the whole Thing import -- except
+ * a 429 (Cloudflare rate-limit challenge, same as api.thingiverse.com can return -- see
+ * ThingiverseRateLimitError), which is reported back distinctly since the caller needs to know
+ * *why* every file failed to report that honestly instead of a generic "couldn't be downloaded". */
+async function downloadPlainFileToTemp(url: string, suggestedName: string): Promise<PlainDownloadResult> {
   try {
     const res = await rawFetch(url, { "User-Agent": IMPORT_USER_AGENT, Accept: "*/*" });
+    if (res.status === 429) {
+      await res.body?.cancel().catch(() => undefined);
+      return { rateLimited: true };
+    }
     if (!res.ok) {
       await res.body?.cancel().catch(() => undefined);
       return null;
@@ -479,7 +488,7 @@ async function downloadPlainFileToTemp(url: string, suggestedName: string): Prom
       `printstash-thingiverse-${crypto.randomBytes(8).toString("hex")}${path.extname(filename)}`,
     );
     await streamToFileCapped(res, tempFilePath, IMPORT_MAX_BYTES);
-    return { filename, mime: guessMimeFromPath(filename), tempFilePath };
+    return { input: { filename, mime: guessMimeFromPath(filename), tempFilePath } };
   } catch {
     return null;
   }
@@ -508,6 +517,7 @@ async function importThingiverseThing(
   try {
     resolved = await resolveThingiverseThing(source.externalId, accessToken);
   } catch (err) {
+    if (err instanceof ThingiverseRateLimitError) throw new HttpError(429, err.message);
     if (err instanceof ThingiverseAuthError) throw new HttpError(401, err.message);
     throw err;
   }
@@ -531,10 +541,18 @@ async function importThingiverseThing(
   };
 
   const modelFiles = plateFiles.filter((f: ThingiversePlateFile) => THINGIVERSE_PLATE_EXTS.has(path.extname(f.name).toLowerCase()));
-  const downloaded = (
-    await Promise.all(modelFiles.map((f: ThingiversePlateFile) => downloadPlainFileToTemp(f.url, f.name)))
-  ).filter((input): input is NewPlateInput => input !== null);
+  const downloadResults = await Promise.all(modelFiles.map((f: ThingiversePlateFile) => downloadPlainFileToTemp(f.url, f.name)));
+  const downloaded = downloadResults
+    .filter((result): result is { input: NewPlateInput } => result !== null && "input" in result)
+    .map((result) => result.input);
   if (!downloaded.length) {
+    const wasRateLimited = downloadResults.some((result) => result !== null && "rateLimited" in result);
+    if (wasRateLimited) {
+      throw new HttpError(
+        429,
+        "Thingiverse blocked a file download with a rate-limit challenge (Cloudflare). This usually clears after a while -- wait, then retry the same import.",
+      );
+    }
     throw new HttpError(400, "None of this Thing's files could be downloaded.");
   }
 

backend/src/services/thingiverseApi.ts

@@ -1,15 +1,28 @@
 import { IMPORT_BROWSER_USER_AGENT, IMPORT_TIMEOUT_SECONDS } from "../config";
 import type { ImportedAuthorInfo, ImportedPageMetadata } from "./importResolvers";
+import {
+  extractJsonFromBrowserBody,
+  fetchViaFlaresolverr,
+  isFlaresolverrEnabled,
+  looksLikeCloudflareBlock,
+  shouldProxyHost,
+} from "./flaresolverr";
 
-// The official, documented Thingiverse Developer API -- confirmed live to be a clean JSON API
-// with none of www.thingiverse.com's Cloudflare bot-management (that domain actively challenges
-// automated requests, including its own legacy `download:{id}` links and the internal
-// `/api/v2/*` endpoints the website's own frontend uses). api.thingiverse.com sits on a
-// separate host, requires its own Access Token (from an app registered at
+// The official, documented Thingiverse Developer API -- much less aggressively gated than
+// www.thingiverse.com (that domain actively challenges automated requests, including its own
+// legacy `download:{id}` links and the internal `/api/v2/*` endpoints the website's own frontend
+// uses), and it requires its own Access Token (from an app registered at
 // thingiverse.com/apps/create -- NOT a browser session cookie, confirmed by a clean
-// INVALID_ACCESS_TOKEN response when a real logged-in session token was tried against it), and
-// answers with well-formed JSON/plain HTTP errors instead of a challenge page.
+// INVALID_ACCESS_TOKEN response when a real logged-in session token was tried against it).
+// BUT it is still sitting behind Cloudflare (`server: cloudflare`, `cf-mitigated: challenge`
+// response headers) and confirmed live to start returning a Cloudflare managed-challenge page
+// (HTTP 429, HTML body) after only a handful of rapid requests -- sticky for a while once
+// tripped, every subsequent request fails the same way. classifyImportFailure/ThingiverseRateLimitError
+// exist to surface that distinctly instead of misreporting it as "not found", and
+// fetchThingiverseApiJson below falls back to FlareSolverr (when configured -- see
+// flaresolverr.ts) to push through a challenge rather than just failing the whole batch.
 const THINGIVERSE_API_BASE = "https://api.thingiverse.com";
+const THINGIVERSE_API_HOSTNAME = new URL(THINGIVERSE_API_BASE).hostname;
 const API_TIMEOUT_MS = IMPORT_TIMEOUT_SECONDS * 1000;
 const THINGIVERSE_PROVIDER = "thingiverse";
 
@@ -27,6 +40,20 @@ export class ThingiverseAuthError extends Error {
   }
 }
 
+/** api.thingiverse.com tripped its Cloudflare bot-management (a "managed challenge" HTML page,
+ * HTTP 429) rather than answering the request -- distinct from a genuine 404 (Thing doesn't
+ * exist/isn't accessible) or a rejected token, and worth its own bucket since the fix (wait,
+ * then retry) is completely different. See classifyImportFailure in importJobRunner.ts. */
+export class ThingiverseRateLimitError extends Error {
+  constructor() {
+    super(
+      "Thingiverse blocked this request with a rate-limit challenge (Cloudflare). This usually " +
+        "clears after a while -- wait, then retry the same import.",
+    );
+    this.name = "ThingiverseRateLimitError";
+  }
+}
+
 export function parseThingiverseThingUrl(url: string): { thingId: string } | null {
   let parsed: URL;
   try {
@@ -58,30 +85,66 @@ export function parseThingiverseLikesUrl(url: string): { username: string } | nu
   return m ? { username: m[1] } : null;
 }
 
-async function fetchThingiverseApiJson(path: string, accessToken: string): Promise<unknown> {
-  const url = `${THINGIVERSE_API_BASE}${path}${path.includes("?") ? "&" : "?"}access_token=${encodeURIComponent(accessToken)}`;
+async function rawApiFetch(url: string): Promise<Response> {
   const controller = new AbortController();
   const timeout = setTimeout(() => controller.abort(), API_TIMEOUT_MS);
   try {
-    const res = await fetch(url, { headers: { "User-Agent": IMPORT_BROWSER_USER_AGENT, Accept: "application/json" }, signal: controller.signal });
-    const text = await res.text();
-    let data: unknown = null;
-    if (text.trim()) {
-      try {
-        data = JSON.parse(text);
-      } catch {
-        data = null;
-      }
-    }
-    if (res.status === 401 || res.status === 403) throw new ThingiverseAuthError();
-    if (res.status === 404) return null;
-    if (!res.ok) return null;
-    return data;
+    return await fetch(url, { headers: { "User-Agent": IMPORT_BROWSER_USER_AGENT, Accept: "application/json" }, signal: controller.signal });
   } finally {
     clearTimeout(timeout);
   }
 }
 
+/** Same detect-a-Cloudflare-block-then-proxy-through-FlareSolverr pattern as importService.ts's
+ * fetchWithGuard (used for Printables/MakerWorld page scraping), adapted for a JSON API instead
+ * of an HTML page: once api.thingiverse.com has been seen returning its managed-challenge page
+ * for this run, every subsequent call goes straight through FlareSolverr's real headless browser
+ * for a while (shouldProxyHost's TTL) instead of wasting a direct request that would just get
+ * challenged again. A browser-rendered JSON response comes back as Chrome's own JSON-viewer DOM,
+ * not raw text -- extractJsonFromBrowserBody unwraps that. Only ever used for the JSON API calls
+ * (GET, no body) -- file downloads from Thingiverse's CDN are a separate, unguarded path (see
+ * downloadPlainFileToTemp in importService.ts): FlareSolverr renders pages, it can't relay
+ * arbitrary binary bytes back, so it isn't a fit there. */
+async function fetchThingiverseApiJson(path: string, accessToken: string): Promise<unknown> {
+  const url = `${THINGIVERSE_API_BASE}${path}${path.includes("?") ? "&" : "?"}access_token=${encodeURIComponent(accessToken)}`;
+
+  let res: Response;
+  let viaBrowser = false;
+  if (isFlaresolverrEnabled() && shouldProxyHost(THINGIVERSE_API_HOSTNAME)) {
+    const solved = await fetchViaFlaresolverr(url);
+    if (solved) {
+      viaBrowser = true;
+      res = new Response(solved.body, { status: solved.status });
+    } else {
+      res = await rawApiFetch(url);
+    }
+  } else {
+    res = await rawApiFetch(url);
+    if ((res.status === 429 || res.status === 403) && isFlaresolverrEnabled() && looksLikeCloudflareBlock(res.headers)) {
+      const solved = await fetchViaFlaresolverr(url);
+      if (solved) {
+        viaBrowser = true;
+        res = new Response(solved.body, { status: solved.status });
+      }
+    }
+  }
+
+  const text = await res.text();
+  let data: unknown = null;
+  if (text.trim()) {
+    try {
+      data = viaBrowser ? extractJsonFromBrowserBody(text) : JSON.parse(text);
+    } catch {
+      data = null;
+    }
+  }
+  if (res.status === 429) throw new ThingiverseRateLimitError();
+  if (res.status === 401 || res.status === 403) throw new ThingiverseAuthError();
+  if (res.status === 404) return null;
+  if (!res.ok) return null;
+  return data;
+}
+
 export type ThingiversePlateFile = { name: string; url: string };
 export type ThingiverseGalleryImage = { name: string; url: string };
 

backend/tests/thingiverseFlaresolverr.test.ts

@@ -0,0 +1,70 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+const ACCESS_TOKEN = "test-access-token";
+const FLARESOLVERR_URL = "http://fake-flaresolverr.test/v1";
+
+function cloudflareChallengeResponse(): Response {
+  return new Response("<!doctype html><html><body>Just a moment...</body></html>", {
+    status: 429,
+    headers: { server: "cloudflare", "cf-mitigated": "challenge", "content-type": "text/html" },
+  });
+}
+
+/** Mimics FlareSolverr's real reply shape: its headless Chrome renders a JSON API response as
+ * its own JSON-viewer DOM (a <pre> tag), not raw JSON text -- see extractJsonFromBrowserBody. */
+function flaresolverrEnvelope(thingJson: unknown): Response {
+  const rendered = `<html><head></head><body><pre>${JSON.stringify(thingJson)}</pre></body></html>`;
+  return new Response(JSON.stringify({ status: "ok", solution: { status: 200, response: rendered } }), {
+    status: 200,
+    headers: { "content-type": "application/json" },
+  });
+}
+
+describe("fetchThingiverseApiJson FlareSolverr fallback", () => {
+  const originalFetch = global.fetch;
+  const originalEnv = process.env.FLARESOLVERR_URL;
+
+  afterEach(() => {
+    global.fetch = originalFetch;
+    vi.restoreAllMocks();
+    if (originalEnv === undefined) delete process.env.FLARESOLVERR_URL;
+    else process.env.FLARESOLVERR_URL = originalEnv;
+    vi.resetModules();
+  });
+
+  it("solves a Cloudflare challenge via FlareSolverr, then reuses the proxy for later calls to the same host", async () => {
+    // config.ts reads FLARESOLVERR_URL from process.env once, at first import -- must be set
+    // before (re-)importing thingiverseApi.ts's dependency chain, hence the dynamic import below
+    // instead of a static one (which ES import hoisting would run before this assignment).
+    process.env.FLARESOLVERR_URL = FLARESOLVERR_URL;
+    vi.resetModules();
+    const { resolveThingiverseThing } = await import("../src/services/thingiverseApi");
+
+    let directCalls = 0;
+    let flaresolverrCalls = 0;
+    global.fetch = vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async (input) => {
+      const url = String(input);
+      if (url.startsWith(FLARESOLVERR_URL)) {
+        flaresolverrCalls++;
+        return flaresolverrEnvelope({ name: "Rate Limited Thing", zip_data: { files: [], images: [] } });
+      }
+      if (url.startsWith("https://api.thingiverse.com/")) {
+        directCalls++;
+        return cloudflareChallengeResponse();
+      }
+      throw new Error(`Unexpected fetch: ${url}`);
+    }) as unknown as typeof fetch;
+
+    const first = await resolveThingiverseThing("999", ACCESS_TOKEN);
+    expect(first?.meta.title).toBe("Rate Limited Thing");
+    expect(directCalls).toBe(1);
+    expect(flaresolverrCalls).toBe(1);
+
+    // The host was just seen returning a Cloudflare challenge -- shouldProxyHost's TTL should
+    // route this next call straight through FlareSolverr without wasting a direct request first.
+    const second = await resolveThingiverseThing("1000", ACCESS_TOKEN);
+    expect(second?.meta.title).toBe("Rate Limited Thing");
+    expect(directCalls).toBe(1);
+    expect(flaresolverrCalls).toBe(2);
+  });
+});