diff --git a/.env.example b/.env.example index 33838db..74ea6f6 100644 --- a/.env.example +++ b/.env.example @@ -25,3 +25,7 @@ IMPORT_MOUNT_PATH_HOST=/path/to/imports IMPORT_MOUNT_EXTS=stl,3mf,step,stp,obj,svg,png,jpg,jpeg,webp,bmp,lbrn,lbrn2,zip IMPORT_MOUNT_INCLUDE_HIDDEN=false IMPORT_MOUNT_ON_STARTUP=true +# Used by the backend to get past Cloudflare's bot challenge on sites like MakerWorld during +# link import. Defaults to the flaresolverr container both compose files already start -- only +# set this if you're running FlareSolverr elsewhere. +# FLARESOLVERR_URL=http://flaresolverr:8191/v1 diff --git a/backend/.env.example b/backend/.env.example index ddcf084..20e5c91 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -15,3 +15,4 @@ IMPORT_MOUNT_EXTS=stl,3mf,step,stp,obj,svg,png,jpg,jpeg,webp,bmp,lbrn,lbrn2,zip IMPORT_MOUNT_INCLUDE_HIDDEN=false IMPORT_MOUNT_ON_STARTUP=true IMPORT_MOUNT_COPY=true +FLARESOLVERR_URL= diff --git a/backend/src/config.ts b/backend/src/config.ts index 8b9af26..37fc1fa 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -39,6 +39,10 @@ export const IMPORT_USER_AGENT = "PrintStash/1.0"; export const IMPORT_BROWSER_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"; +export const FLARESOLVERR_URL = (process.env.FLARESOLVERR_URL || "").trim(); +export const FLARESOLVERR_TIMEOUT_MS = envInt("FLARESOLVERR_TIMEOUT_MS", 60000); +export const FLARESOLVERR_SESSION_TTL_MS = envInt("FLARESOLVERR_SESSION_TTL_MS", 15 * 60 * 1000); + export const MOUNT_IMPORT_PATH = (process.env.IMPORT_MOUNT_PATH || "").trim(); export const MOUNT_IMPORT_EXTS_RAW = (process.env.IMPORT_MOUNT_EXTS || "").trim(); export const MOUNT_IMPORT_INCLUDE_HIDDEN = envBool("IMPORT_MOUNT_INCLUDE_HIDDEN", false); diff --git a/backend/src/services/flaresolverr.ts b/backend/src/services/flaresolverr.ts new file mode 100644 index 0000000..6dd64ce --- /dev/null +++ b/backend/src/services/flaresolverr.ts @@ -0,0 +1,119 @@ +import { FLARESOLVERR_SESSION_TTL_MS, FLARESOLVERR_TIMEOUT_MS, FLARESOLVERR_URL } from "../config"; + +export function isFlaresolverrEnabled(): boolean { + return Boolean(FLARESOLVERR_URL); +} + +/** True when a response looks like a Cloudflare edge block/challenge rather than an + * application-level 403 (wrong credentials, missing auth, etc). Only these are worth + * paying FlareSolverr's solve cost for. */ +export function looksLikeCloudflareBlock(headers: Headers): boolean { + const server = (headers.get("server") || "").toLowerCase(); + if (server.includes("cloudflare")) return true; + if (headers.has("cf-mitigated")) return true; + return false; +} + +function parseCookieHeader(raw: string | null | undefined): Record { + const jar: Record = {}; + for (const part of (raw || "").split(";")) { + const trimmed = part.trim(); + if (!trimmed) continue; + const eq = trimmed.indexOf("="); + if (eq === -1) continue; + jar[trimmed.slice(0, eq)] = trimmed.slice(eq + 1); + } + return jar; +} + +// Cloudflare's clearance here is bound to the exact browser fingerprint that solved the +// challenge, so replaying its cookies via our own fetch() does not work (verified against +// makerworld.com). Once a host is seen blocking us, every request to it is proxied through +// FlareSolverr's real headless browser instead, for a TTL, rather than re-probing with a +// wasted direct request each time. +const proxyHosts = new Map(); + +export function shouldProxyHost(hostname: string): boolean { + const expiresAt = proxyHosts.get(hostname.toLowerCase()); + return typeof expiresAt === "number" && Date.now() > 0 && Date.now() < expiresAt; +} + +function markHostNeedsProxy(hostname: string): void { + proxyHosts.set(hostname.toLowerCase(), Date.now() + FLARESOLVERR_SESSION_TTL_MS); +} + +export type FlaresolverrResult = { + status: number; + body: string; +}; + +/** Loads `url` inside FlareSolverr's real headless browser (passing Cloudflare's JS/managed + * challenge along the way) and returns the page body it rendered. GET only: FlareSolverr's + * POST command submits a browser form, not a raw JSON body, so it isn't used here. */ +export async function fetchViaFlaresolverr(url: string, cookieHeader?: string | null): Promise { + if (!FLARESOLVERR_URL) return null; + let hostname: string; + try { + hostname = new URL(url).hostname.toLowerCase(); + } catch { + return null; + } + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), FLARESOLVERR_TIMEOUT_MS); + try { + const seedCookies = parseCookieHeader(cookieHeader); + const cookieList = Object.entries(seedCookies).map(([name, value]) => ({ name, value, domain: hostname })); + const body: Record = { + cmd: "request.get", + url, + maxTimeout: FLARESOLVERR_TIMEOUT_MS, + }; + if (cookieList.length) body.cookies = cookieList; + + const res = await fetch(FLARESOLVERR_URL, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + signal: controller.signal, + }); + if (!res.ok) return null; + const data = (await res.json()) as { + status?: string; + solution?: { status?: number; response?: string }; + }; + if (data.status !== "ok" || !data.solution) return null; + + markHostNeedsProxy(hostname); + return { status: data.solution.status ?? 200, body: data.solution.response || "" }; + } catch { + return null; + } finally { + clearTimeout(timeout); + } +} + +/** FlareSolverr's captured body is what the browser rendered, so a JSON API response comes + * back as Chrome's own JSON-viewer DOM (`...
{...}
...`) rather than + * raw JSON text. Unwraps that, HTML-entity-decodes it, and parses it. */ +export function extractJsonFromBrowserBody(body: string): unknown | null { + const trimmed = body.trim(); + try { + return JSON.parse(trimmed); + } catch { + // fall through to unwrap + } + const match = trimmed.match(/]*>([\s\S]*?)<\/pre>/i); + if (!match) return null; + const unescaped = match[1] + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&/g, "&"); + try { + return JSON.parse(unescaped); + } catch { + return null; + } +} diff --git a/backend/src/services/importResolvers.ts b/backend/src/services/importResolvers.ts index d749d27..9617c3a 100644 --- a/backend/src/services/importResolvers.ts +++ b/backend/src/services/importResolvers.ts @@ -10,6 +10,13 @@ import { IMPORT_USER_AGENT, } from "../config"; import { HttpError, isJsonContentType } from "../utils/fileUtils"; +import { + extractJsonFromBrowserBody, + fetchViaFlaresolverr, + isFlaresolverrEnabled, + looksLikeCloudflareBlock, + shouldProxyHost, +} from "./flaresolverr"; export type ImportCookies = { makerworld_cookie?: string | null; @@ -134,7 +141,7 @@ function thingiverseApiHeaders(referer?: string | null, cookie?: string | null, // -- Small bounded fetch helpers (used only for resolver-side HTML/JSON probes; the actual // model-file download streams straight to disk in importService.ts, not through here). ------ -async function fetchCappedBuffer( +async function rawFetchBuffer( url: string, headers: Record, init?: { method?: string; body?: string }, @@ -159,6 +166,56 @@ async function fetchCappedBuffer( } } +/** Loads `url` through FlareSolverr's real browser (GET only - its POST command submits a + * browser form, not a raw JSON body, so callers with a POST init fall back to a direct + * request instead). Unwraps the JSON it renders back into the same {status, headers, buffer, + * url} shape rawFetchBuffer returns, so callers don't need to know which path served them. */ +async function proxiedBuffer( + url: string, + cookieHeader?: string | null, +): Promise<{ status: number; headers: Headers; buffer: Buffer; url: string } | null> { + const solved = await fetchViaFlaresolverr(url, cookieHeader); + if (!solved) return null; + const json = extractJsonFromBrowserBody(solved.body); + const bodyText = json !== null ? JSON.stringify(json) : solved.body; + const contentType = json !== null ? "application/json" : "text/html; charset=utf-8"; + return { + status: solved.status, + headers: new Headers({ "content-type": contentType }), + buffer: Buffer.from(bodyText, "utf-8"), + url, + }; +} + +async function fetchCappedBuffer( + url: string, + headers: Record, + init?: { method?: string; body?: string }, +): Promise<{ status: number; headers: Headers; buffer: Buffer; url: string } | null> { + const isGet = !init?.method || init.method.toUpperCase() === "GET"; + if (!isGet || !isFlaresolverrEnabled()) { + return rawFetchBuffer(url, headers, init); + } + + let hostname = ""; + try { + hostname = new URL(url).hostname; + } catch { + return rawFetchBuffer(url, headers, init); + } + + if (shouldProxyHost(hostname)) { + return (await proxiedBuffer(url, headers.Cookie)) || rawFetchBuffer(url, headers, init); + } + + const result = await rawFetchBuffer(url, headers, init); + if (result && result.status === 403 && looksLikeCloudflareBlock(result.headers)) { + const proxied = await proxiedBuffer(url, headers.Cookie); + if (proxied) return proxied; + } + return result; +} + function parseJsonErrorMessage(raw: Buffer): string | null { try { const data = JSON.parse(raw.toString("utf-8")); diff --git a/backend/src/services/importService.ts b/backend/src/services/importService.ts index 4eec278..b31e4d9 100644 --- a/backend/src/services/importService.ts +++ b/backend/src/services/importService.ts @@ -11,6 +11,7 @@ import { } from "../config"; import { HttpError, buildImportFilename, isHtmlContentType, mimeFromContentType } from "../utils/fileUtils"; import { validateRemoteUrl } from "../utils/urlUtils"; +import { fetchViaFlaresolverr, isFlaresolverrEnabled, looksLikeCloudflareBlock, shouldProxyHost } from "./flaresolverr"; import { findDownloadUrl, isPrintablesPageHost, @@ -66,11 +67,48 @@ async function readCapped(response: Response, maxBytes: number): Promise<{ buffe return { buffer: Buffer.concat(chunks), truncated }; } -async function fetchWithGuard(url: string, headers: Record): Promise { +async function rawFetch(url: string, headers: Record): Promise { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), IMPORT_TIMEOUT_SECONDS * 1000); try { - const res = await fetch(url, { headers, redirect: "follow", signal: controller.signal }); + return await fetch(url, { headers, redirect: "follow", signal: controller.signal }); + } finally { + clearTimeout(timeout); + } +} + +/** Loads `url` through FlareSolverr's real browser and wraps the rendered body as an HTML + * Response, so callers (openImportResponse's page-scraping path) can treat it exactly like + * an ordinary fetch result. */ +async function proxiedResponse(url: string, cookieHeader?: string | null): Promise { + const solved = await fetchViaFlaresolverr(url, cookieHeader); + if (!solved) return null; + return new Response(solved.body, { + status: solved.status, + headers: { "content-type": "text/html; charset=utf-8" }, + }); +} + +async function fetchWithGuard(url: string, headers: Record): Promise { + try { + let hostname = ""; + try { + hostname = new URL(url).hostname; + } catch { + // validateRemoteUrl already rejects unparsable URLs upstream + } + + let res: Response; + if (isFlaresolverrEnabled() && shouldProxyHost(hostname)) { + res = (await proxiedResponse(url, headers.Cookie)) || (await rawFetch(url, headers)); + } else { + res = await rawFetch(url, headers); + if (res.status === 403 && isFlaresolverrEnabled() && looksLikeCloudflareBlock(res.headers)) { + const proxied = await proxiedResponse(url, headers.Cookie); + if (proxied) res = proxied; + } + } + if (!res.ok) { throw new HttpError(res.status || 400, `Failed to fetch URL: ${res.statusText || res.status}`); } @@ -78,8 +116,6 @@ async function fetchWithGuard(url: string, headers: Record): Pro } catch (err) { if (err instanceof HttpError) throw err; throw new HttpError(400, "Failed to reach the provided URL"); - } finally { - clearTimeout(timeout); } } diff --git a/docker-compose.deploy.yml b/docker-compose.deploy.yml index 4dbc6f4..00a7988 100644 --- a/docker-compose.deploy.yml +++ b/docker-compose.deploy.yml @@ -16,6 +16,15 @@ services: timeout: 5s retries: 10 + flaresolverr: + image: ghcr.io/flaresolverr/flaresolverr:latest + restart: unless-stopped + environment: + - LOG_LEVEL=${FLARESOLVERR_LOG_LEVEL:-info} + - TZ=${TZ:-UTC} + networks: + - app-net + backend: # Pin to a specific build instead of :latest, e.g.: # BACKEND_IMAGE=ghcr.io/tautvydasderzinskas/printstash-backend:sha-a8fef7d @@ -36,9 +45,12 @@ services: - IMPORT_MOUNT_INCLUDE_HIDDEN=${IMPORT_MOUNT_INCLUDE_HIDDEN:-false} - IMPORT_MOUNT_ON_STARTUP=${IMPORT_MOUNT_ON_STARTUP:-false} - IMPORT_MOUNT_COPY=${IMPORT_MOUNT_COPY:-true} + - FLARESOLVERR_URL=${FLARESOLVERR_URL:-http://flaresolverr:8191/v1} depends_on: db: condition: service_healthy + flaresolverr: + condition: service_started volumes: - printstash_storage:/app/storage - ${IMPORT_MOUNT_PATH_HOST:-./imports}:/imports:ro diff --git a/docker-compose.yml b/docker-compose.yml index 7d950db..1b58cb9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,15 @@ services: timeout: 5s retries: 10 + flaresolverr: + image: ghcr.io/flaresolverr/flaresolverr:latest + restart: unless-stopped + environment: + - LOG_LEVEL=${FLARESOLVERR_LOG_LEVEL:-info} + - TZ=${TZ:-UTC} + networks: + - app-net + backend: build: context: ./backend @@ -31,9 +40,12 @@ services: - AUTH_PASSWORD=${AUTH_PASSWORD:-super-secret} - AUTH_SECRET=${AUTH_SECRET:-changeme-secret} - AUTH_TOKEN_TTL=${AUTH_TOKEN_TTL:-43200} + - FLARESOLVERR_URL=${FLARESOLVERR_URL:-http://flaresolverr:8191/v1} depends_on: db: condition: service_healthy + flaresolverr: + condition: service_started volumes: - ./data/storage:/app/storage networks: