feat: add flaresolverr

.env.example

@@ -25,3 +25,7 @@ IMPORT_MOUNT_PATH_HOST=/path/to/imports
 IMPORT_MOUNT_EXTS=stl,3mf,step,stp,obj,svg,png,jpg,jpeg,webp,bmp,lbrn,lbrn2,zip
 IMPORT_MOUNT_INCLUDE_HIDDEN=false
 IMPORT_MOUNT_ON_STARTUP=true
+# Used by the backend to get past Cloudflare's bot challenge on sites like MakerWorld during
+# link import. Defaults to the flaresolverr container both compose files already start -- only
+# set this if you're running FlareSolverr elsewhere.
+# FLARESOLVERR_URL=http://flaresolverr:8191/v1

backend/.env.example

@@ -15,3 +15,4 @@ IMPORT_MOUNT_EXTS=stl,3mf,step,stp,obj,svg,png,jpg,jpeg,webp,bmp,lbrn,lbrn2,zip
 IMPORT_MOUNT_INCLUDE_HIDDEN=false
 IMPORT_MOUNT_ON_STARTUP=true
 IMPORT_MOUNT_COPY=true
+FLARESOLVERR_URL=

backend/src/config.ts

@@ -39,6 +39,10 @@ export const IMPORT_USER_AGENT = "PrintStash/1.0";
 export const IMPORT_BROWSER_USER_AGENT =
   "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36";
 
+export const FLARESOLVERR_URL = (process.env.FLARESOLVERR_URL || "").trim();
+export const FLARESOLVERR_TIMEOUT_MS = envInt("FLARESOLVERR_TIMEOUT_MS", 60000);
+export const FLARESOLVERR_SESSION_TTL_MS = envInt("FLARESOLVERR_SESSION_TTL_MS", 15 * 60 * 1000);
+
 export const MOUNT_IMPORT_PATH = (process.env.IMPORT_MOUNT_PATH || "").trim();
 export const MOUNT_IMPORT_EXTS_RAW = (process.env.IMPORT_MOUNT_EXTS || "").trim();
 export const MOUNT_IMPORT_INCLUDE_HIDDEN = envBool("IMPORT_MOUNT_INCLUDE_HIDDEN", false);

backend/src/services/flaresolverr.ts

@@ -0,0 +1,119 @@
+import { FLARESOLVERR_SESSION_TTL_MS, FLARESOLVERR_TIMEOUT_MS, FLARESOLVERR_URL } from "../config";
+
+export function isFlaresolverrEnabled(): boolean {
+  return Boolean(FLARESOLVERR_URL);
+}
+
+/** True when a response looks like a Cloudflare edge block/challenge rather than an
+ * application-level 403 (wrong credentials, missing auth, etc). Only these are worth
+ * paying FlareSolverr's solve cost for. */
+export function looksLikeCloudflareBlock(headers: Headers): boolean {
+  const server = (headers.get("server") || "").toLowerCase();
+  if (server.includes("cloudflare")) return true;
+  if (headers.has("cf-mitigated")) return true;
+  return false;
+}
+
+function parseCookieHeader(raw: string | null | undefined): Record<string, string> {
+  const jar: Record<string, string> = {};
+  for (const part of (raw || "").split(";")) {
+    const trimmed = part.trim();
+    if (!trimmed) continue;
+    const eq = trimmed.indexOf("=");
+    if (eq === -1) continue;
+    jar[trimmed.slice(0, eq)] = trimmed.slice(eq + 1);
+  }
+  return jar;
+}
+
+// Cloudflare's clearance here is bound to the exact browser fingerprint that solved the
+// challenge, so replaying its cookies via our own fetch() does not work (verified against
+// makerworld.com). Once a host is seen blocking us, every request to it is proxied through
+// FlareSolverr's real headless browser instead, for a TTL, rather than re-probing with a
+// wasted direct request each time.
+const proxyHosts = new Map<string, number>();
+
+export function shouldProxyHost(hostname: string): boolean {
+  const expiresAt = proxyHosts.get(hostname.toLowerCase());
+  return typeof expiresAt === "number" && Date.now() > 0 && Date.now() < expiresAt;
+}
+
+function markHostNeedsProxy(hostname: string): void {
+  proxyHosts.set(hostname.toLowerCase(), Date.now() + FLARESOLVERR_SESSION_TTL_MS);
+}
+
+export type FlaresolverrResult = {
+  status: number;
+  body: string;
+};
+
+/** Loads `url` inside FlareSolverr's real headless browser (passing Cloudflare's JS/managed
+ * challenge along the way) and returns the page body it rendered. GET only: FlareSolverr's
+ * POST command submits a browser form, not a raw JSON body, so it isn't used here. */
+export async function fetchViaFlaresolverr(url: string, cookieHeader?: string | null): Promise<FlaresolverrResult | null> {
+  if (!FLARESOLVERR_URL) return null;
+  let hostname: string;
+  try {
+    hostname = new URL(url).hostname.toLowerCase();
+  } catch {
+    return null;
+  }
+
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), FLARESOLVERR_TIMEOUT_MS);
+  try {
+    const seedCookies = parseCookieHeader(cookieHeader);
+    const cookieList = Object.entries(seedCookies).map(([name, value]) => ({ name, value, domain: hostname }));
+    const body: Record<string, unknown> = {
+      cmd: "request.get",
+      url,
+      maxTimeout: FLARESOLVERR_TIMEOUT_MS,
+    };
+    if (cookieList.length) body.cookies = cookieList;
+
+    const res = await fetch(FLARESOLVERR_URL, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify(body),
+      signal: controller.signal,
+    });
+    if (!res.ok) return null;
+    const data = (await res.json()) as {
+      status?: string;
+      solution?: { status?: number; response?: string };
+    };
+    if (data.status !== "ok" || !data.solution) return null;
+
+    markHostNeedsProxy(hostname);
+    return { status: data.solution.status ?? 200, body: data.solution.response || "" };
+  } catch {
+    return null;
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+/** FlareSolverr's captured body is what the browser rendered, so a JSON API response comes
+ * back as Chrome's own JSON-viewer DOM (`<html>...<pre>{...}</pre>...</html>`) rather than
+ * raw JSON text. Unwraps that, HTML-entity-decodes it, and parses it. */
+export function extractJsonFromBrowserBody(body: string): unknown | null {
+  const trimmed = body.trim();
+  try {
+    return JSON.parse(trimmed);
+  } catch {
+    // fall through to unwrap
+  }
+  const match = trimmed.match(/<pre[^>]*>([\s\S]*?)<\/pre>/i);
+  if (!match) return null;
+  const unescaped = match[1]
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&amp;/g, "&");
+  try {
+    return JSON.parse(unescaped);
+  } catch {
+    return null;
+  }
+}

backend/src/services/importResolvers.ts

@@ -10,6 +10,13 @@ import {
   IMPORT_USER_AGENT,
 } from "../config";
 import { HttpError, isJsonContentType } from "../utils/fileUtils";
+import {
+  extractJsonFromBrowserBody,
+  fetchViaFlaresolverr,
+  isFlaresolverrEnabled,
+  looksLikeCloudflareBlock,
+  shouldProxyHost,
+} from "./flaresolverr";
 
 export type ImportCookies = {
   makerworld_cookie?: string | null;
@@ -134,7 +141,7 @@ function thingiverseApiHeaders(referer?: string | null, cookie?: string | null,
 // -- Small bounded fetch helpers (used only for resolver-side HTML/JSON probes; the actual
 // model-file download streams straight to disk in importService.ts, not through here). ------
 
-async function fetchCappedBuffer(
+async function rawFetchBuffer(
   url: string,
   headers: Record<string, string>,
   init?: { method?: string; body?: string },
@@ -159,6 +166,56 @@ async function fetchCappedBuffer(
   }
 }
 
+/** Loads `url` through FlareSolverr's real browser (GET only - its POST command submits a
+ * browser form, not a raw JSON body, so callers with a POST init fall back to a direct
+ * request instead). Unwraps the JSON it renders back into the same {status, headers, buffer,
+ * url} shape rawFetchBuffer returns, so callers don't need to know which path served them. */
+async function proxiedBuffer(
+  url: string,
+  cookieHeader?: string | null,
+): Promise<{ status: number; headers: Headers; buffer: Buffer; url: string } | null> {
+  const solved = await fetchViaFlaresolverr(url, cookieHeader);
+  if (!solved) return null;
+  const json = extractJsonFromBrowserBody(solved.body);
+  const bodyText = json !== null ? JSON.stringify(json) : solved.body;
+  const contentType = json !== null ? "application/json" : "text/html; charset=utf-8";
+  return {
+    status: solved.status,
+    headers: new Headers({ "content-type": contentType }),
+    buffer: Buffer.from(bodyText, "utf-8"),
+    url,
+  };
+}
+
+async function fetchCappedBuffer(
+  url: string,
+  headers: Record<string, string>,
+  init?: { method?: string; body?: string },
+): Promise<{ status: number; headers: Headers; buffer: Buffer; url: string } | null> {
+  const isGet = !init?.method || init.method.toUpperCase() === "GET";
+  if (!isGet || !isFlaresolverrEnabled()) {
+    return rawFetchBuffer(url, headers, init);
+  }
+
+  let hostname = "";
+  try {
+    hostname = new URL(url).hostname;
+  } catch {
+    return rawFetchBuffer(url, headers, init);
+  }
+
+  if (shouldProxyHost(hostname)) {
+    return (await proxiedBuffer(url, headers.Cookie)) || rawFetchBuffer(url, headers, init);
+  }
+
+  const result = await rawFetchBuffer(url, headers, init);
+  if (result && result.status === 403 && looksLikeCloudflareBlock(result.headers)) {
+    const proxied = await proxiedBuffer(url, headers.Cookie);
+    if (proxied) return proxied;
+  }
+  return result;
+}
+
 function parseJsonErrorMessage(raw: Buffer): string | null {
   try {
     const data = JSON.parse(raw.toString("utf-8"));

backend/src/services/importService.ts

@@ -11,6 +11,7 @@ import {
 } from "../config";
 import { HttpError, buildImportFilename, isHtmlContentType, mimeFromContentType } from "../utils/fileUtils";
 import { validateRemoteUrl } from "../utils/urlUtils";
+import { fetchViaFlaresolverr, isFlaresolverrEnabled, looksLikeCloudflareBlock, shouldProxyHost } from "./flaresolverr";
 import {
   findDownloadUrl,
   isPrintablesPageHost,
@@ -66,11 +67,48 @@ async function readCapped(response: Response, maxBytes: number): Promise<{ buffe
   return { buffer: Buffer.concat(chunks), truncated };
 }
 
-async function fetchWithGuard(url: string, headers: Record<string, string>): Promise<Response> {
+async function rawFetch(url: string, headers: Record<string, string>): Promise<Response> {
   const controller = new AbortController();
   const timeout = setTimeout(() => controller.abort(), IMPORT_TIMEOUT_SECONDS * 1000);
   try {
-    const res = await fetch(url, { headers, redirect: "follow", signal: controller.signal });
+    return await fetch(url, { headers, redirect: "follow", signal: controller.signal });
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+/** Loads `url` through FlareSolverr's real browser and wraps the rendered body as an HTML
+ * Response, so callers (openImportResponse's page-scraping path) can treat it exactly like
+ * an ordinary fetch result. */
+async function proxiedResponse(url: string, cookieHeader?: string | null): Promise<Response | null> {
+  const solved = await fetchViaFlaresolverr(url, cookieHeader);
+  if (!solved) return null;
+  return new Response(solved.body, {
+    status: solved.status,
+    headers: { "content-type": "text/html; charset=utf-8" },
+  });
+}
+
+async function fetchWithGuard(url: string, headers: Record<string, string>): Promise<Response> {
+  try {
+    let hostname = "";
+    try {
+      hostname = new URL(url).hostname;
+    } catch {
+      // validateRemoteUrl already rejects unparsable URLs upstream
+    }
+
+    let res: Response;
+    if (isFlaresolverrEnabled() && shouldProxyHost(hostname)) {
+      res = (await proxiedResponse(url, headers.Cookie)) || (await rawFetch(url, headers));
+    } else {
+      res = await rawFetch(url, headers);
+      if (res.status === 403 && isFlaresolverrEnabled() && looksLikeCloudflareBlock(res.headers)) {
+        const proxied = await proxiedResponse(url, headers.Cookie);
+        if (proxied) res = proxied;
+      }
+    }
+
     if (!res.ok) {
       throw new HttpError(res.status || 400, `Failed to fetch URL: ${res.statusText || res.status}`);
     }
@@ -78,8 +116,6 @@ async function fetchWithGuard(url: string, headers: Record<string, string>): Pro
   } catch (err) {
     if (err instanceof HttpError) throw err;
     throw new HttpError(400, "Failed to reach the provided URL");
-  } finally {
-    clearTimeout(timeout);
   }
 }
 

docker-compose.deploy.yml

@@ -16,6 +16,15 @@ services:
       timeout: 5s
       retries: 10
 
+  flaresolverr:
+    image: ghcr.io/flaresolverr/flaresolverr:latest
+    restart: unless-stopped
+    environment:
+      - LOG_LEVEL=${FLARESOLVERR_LOG_LEVEL:-info}
+      - TZ=${TZ:-UTC}
+    networks:
+      - app-net
+
   backend:
     # Pin to a specific build instead of :latest, e.g.:
     #   BACKEND_IMAGE=ghcr.io/tautvydasderzinskas/printstash-backend:sha-a8fef7d
@@ -36,9 +45,12 @@ services:
       - IMPORT_MOUNT_INCLUDE_HIDDEN=${IMPORT_MOUNT_INCLUDE_HIDDEN:-false}
       - IMPORT_MOUNT_ON_STARTUP=${IMPORT_MOUNT_ON_STARTUP:-false}
       - IMPORT_MOUNT_COPY=${IMPORT_MOUNT_COPY:-true}
+      - FLARESOLVERR_URL=${FLARESOLVERR_URL:-http://flaresolverr:8191/v1}
     depends_on:
       db:
         condition: service_healthy
+      flaresolverr:
+        condition: service_started
     volumes:
       - printstash_storage:/app/storage
       - ${IMPORT_MOUNT_PATH_HOST:-./imports}:/imports:ro

docker-compose.yml

@@ -16,6 +16,15 @@ services:
       timeout: 5s
       retries: 10
 
+  flaresolverr:
+    image: ghcr.io/flaresolverr/flaresolverr:latest
+    restart: unless-stopped
+    environment:
+      - LOG_LEVEL=${FLARESOLVERR_LOG_LEVEL:-info}
+      - TZ=${TZ:-UTC}
+    networks:
+      - app-net
+
   backend:
     build:
       context: ./backend
@@ -31,9 +40,12 @@ services:
       - AUTH_PASSWORD=${AUTH_PASSWORD:-super-secret}
       - AUTH_SECRET=${AUTH_SECRET:-changeme-secret}
       - AUTH_TOKEN_TTL=${AUTH_TOKEN_TTL:-43200}
+      - FLARESOLVERR_URL=${FLARESOLVERR_URL:-http://flaresolverr:8191/v1}
     depends_on:
       db:
         condition: service_healthy
+      flaresolverr:
+        condition: service_started
     volumes:
       - ./data/storage:/app/storage
     networks: