From 8e6e5b2e0ee63b8d3c09bbbd3a086ad5543d2974 Mon Sep 17 00:00:00 2001 From: Tautvydas Derzinskas Date: Sun, 18 Oct 2026 04:38:16 +0200 Subject: [PATCH] fix: download from Thingiverse with a Chrome TLS fingerprint Thingiverse's Cloudflare now challenges every request whose TLS handshake isn't a browser's, whatever the headers or cookies, so file downloads, preview images and API calls all failed. Route all thingiverse.com requests through impit, which impersonates Chrome. --- backend/package-lock.json | 148 ++++++++++++++++++ backend/package.json | 1 + .../src/services/descriptionImageService.ts | 10 +- backend/src/services/importService.ts | 2 + backend/src/services/thingiverseApi.ts | 14 +- backend/src/utils/browserFetch.ts | 49 ++++++ backend/tests/setup.ts | 14 ++ 7 files changed, 222 insertions(+), 16 deletions(-) create mode 100644 backend/src/utils/browserFetch.ts diff --git a/backend/package-lock.json b/backend/package-lock.json index 8df625f..93406a2 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -15,6 +15,7 @@ "cors": "^2.8.6", "dotenv": "^18.0.8", "express": "^4.22.3", + "impit": "0.14.6", "jsonwebtoken": "^9.0.3", "jszip": "^3.10.2", "meshoptimizer": "^1.3.0", @@ -3148,6 +3149,153 @@ "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", "license": "MIT" }, + "node_modules/impit": { + "version": "0.14.6", + "resolved": "https://registry.npmjs.org/impit/-/impit-0.14.6.tgz", + "integrity": "sha512-1i5DxW0eRIUCtX9x+ZCSzD+FlRiEdrDOXazg0DYNMKUgnVwRWEwRhbA23Lpo2WOC674wZpwTQUubBi5z7m04Ng==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + }, + "optionalDependencies": { + "impit-darwin-arm64": "0.14.6", + "impit-darwin-x64": "0.14.6", + "impit-linux-arm64-gnu": "0.14.6", + "impit-linux-arm64-musl": "0.14.6", + "impit-linux-x64-gnu": "0.14.6", + "impit-linux-x64-musl": "0.14.6", + "impit-win32-arm64-msvc": "0.14.6", + "impit-win32-x64-msvc": "0.14.6" + } + }, + "node_modules/impit-darwin-arm64": { + "version": "0.14.6", + "resolved": "https://registry.npmjs.org/impit-darwin-arm64/-/impit-darwin-arm64-0.14.6.tgz", + "integrity": "sha512-QhEENTXA727SlgIIIfPg4Ij2FEgwX/3rKCBpR84JxZZfQVVNO/KB2cVvt9aupokY1NZIo6oBFYJXF5PKyRVKbQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-darwin-x64": { + "version": "0.14.6", + "resolved": "https://registry.npmjs.org/impit-darwin-x64/-/impit-darwin-x64-0.14.6.tgz", + "integrity": "sha512-BYcBdG7+smXHrhwLGqtWWw1jTsQSWwJO7ZyfkVPMUGRcbPFJAXXKRUjdFWcp2AdMWyI5JufNF3IZzAJ6B++8mw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-linux-arm64-gnu": { + "version": "0.14.6", + "resolved": "https://registry.npmjs.org/impit-linux-arm64-gnu/-/impit-linux-arm64-gnu-0.14.6.tgz", + "integrity": "sha512-yoUxNCBDsQ72H7uTtaLp7hUQJxr1/OtIZ7KnaerFeJwPpuK2KCeAIXa/9ajoa0GZ6BErm9ZwkggOENhAXu0xxw==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-linux-arm64-musl": { + "version": "0.14.6", + "resolved": "https://registry.npmjs.org/impit-linux-arm64-musl/-/impit-linux-arm64-musl-0.14.6.tgz", + "integrity": "sha512-2UL2ya3uRCdcZwkTL4Oe0RVyhvsONJHhbtD+0VZZkDKyCxyyjtZ6Rc1QQNKh2q/xKa3vNnzj/9qcu0mj/zCk8w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-linux-x64-gnu": { + "version": "0.14.6", + "resolved": "https://registry.npmjs.org/impit-linux-x64-gnu/-/impit-linux-x64-gnu-0.14.6.tgz", + "integrity": "sha512-6TJxs/iUQbxQuznb0gz5MQ15WIT4/IxHdwNG9II8kEKUEzVMLCun8IXiT5AGdYzwlmPYSIuxDjtY2ZALhC1FlA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-linux-x64-musl": { + "version": "0.14.6", + "resolved": "https://registry.npmjs.org/impit-linux-x64-musl/-/impit-linux-x64-musl-0.14.6.tgz", + "integrity": "sha512-AoneCmxaKKCx2wJZGLtFB6LrkTZUEViZ9XEsocpyMsm5IvV3mhj64AGIZI7zHlW7AFXq4MW0DPZkHiL/KxTB1A==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-win32-arm64-msvc": { + "version": "0.14.6", + "resolved": "https://registry.npmjs.org/impit-win32-arm64-msvc/-/impit-win32-arm64-msvc-0.14.6.tgz", + "integrity": "sha512-DRsyV3vNPeUTaRjtFiRqHb8FVFmLDR3NB47cvb+AKKuo+KUbRJdJR1zbDQMD2SOKhYRHf1PTxzamRIkl3yWqWQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-win32-x64-msvc": { + "version": "0.14.6", + "resolved": "https://registry.npmjs.org/impit-win32-x64-msvc/-/impit-win32-x64-msvc-0.14.6.tgz", + "integrity": "sha512-0j7OHcgX8ui9ObSelx5v7OEEIfe00EgHcljq9RFR7oFzkD2ABS/6+fSyrXgu2LcC277vJ9oSfKjEacgGoCHOpQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", diff --git a/backend/package.json b/backend/package.json index f8e5d83..ee59409 100644 --- a/backend/package.json +++ b/backend/package.json @@ -24,6 +24,7 @@ "cors": "^2.8.6", "dotenv": "^18.0.8", "express": "^4.22.3", + "impit": "0.14.6", "jsonwebtoken": "^9.0.3", "jszip": "^3.10.2", "meshoptimizer": "^1.3.0", diff --git a/backend/src/services/descriptionImageService.ts b/backend/src/services/descriptionImageService.ts index 4d83837..e1b25fe 100644 --- a/backend/src/services/descriptionImageService.ts +++ b/backend/src/services/descriptionImageService.ts @@ -5,6 +5,7 @@ import { DESCRIPTION_IMAGES, IMPORT_BROWSER_USER_AGENT, IMPORT_TIMEOUT_SECONDS } import { prisma } from "../db"; import { mapWithConcurrency } from "../utils/concurrency"; import { validateRemoteUrl } from "../utils/urlUtils"; +import { browserFetch, needsBrowserFetch } from "../utils/browserFetch"; import { readCapped } from "../utils/readCapped"; const IMAGE_MAX_BYTES = 16 * 1024 * 1024; @@ -47,11 +48,10 @@ async function fetchImage(url: string, deadline: number): Promise let current = url; for (let hop = 0; hop <= MAX_REDIRECTS; hop++) { await validateRemoteUrl(current); - const res = await fetch(current, { - headers: { "User-Agent": IMPORT_BROWSER_USER_AGENT, Accept: "image/*" }, - redirect: "manual", - signal: controller.signal, - }); + const headers = { "User-Agent": IMPORT_BROWSER_USER_AGENT, Accept: "image/*" }; + const res = needsBrowserFetch(current) + ? await browserFetch(current, { headers, redirect: "manual", signal: controller.signal }) + : await fetch(current, { headers, redirect: "manual", signal: controller.signal }); const location = res.headers.get("location"); if (res.status >= 300 && res.status < 400 && location) { await res.body?.cancel().catch(() => undefined); diff --git a/backend/src/services/importService.ts b/backend/src/services/importService.ts index 6ef224c..ddf5d6b 100644 --- a/backend/src/services/importService.ts +++ b/backend/src/services/importService.ts @@ -22,6 +22,7 @@ import { } from "../utils/fileUtils"; import { validateRemoteUrl } from "../utils/urlUtils"; import { readCapped } from "../utils/readCapped"; +import { browserFetch, needsBrowserFetch } from "../utils/browserFetch"; import { mapWithConcurrency, maybeSleep, sleep } from "../utils/concurrency"; import { CLOUDFLARE_BLOCKED_CODE, @@ -118,6 +119,7 @@ function parseCharset(contentType: string | null): string { } async function rawFetch(url: string, headers: Record): Promise { + if (needsBrowserFetch(url)) return browserFetch(url, { headers, timeoutMs: IMPORT_TIMEOUT_SECONDS * 1000 }); const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), IMPORT_TIMEOUT_SECONDS * 1000); try { diff --git a/backend/src/services/thingiverseApi.ts b/backend/src/services/thingiverseApi.ts index 67de0e7..7f8a4a4 100644 --- a/backend/src/services/thingiverseApi.ts +++ b/backend/src/services/thingiverseApi.ts @@ -1,4 +1,5 @@ -import { IMPORT_BROWSER_USER_AGENT, IMPORT_TIMEOUT_SECONDS } from "../config"; +import { IMPORT_TIMEOUT_SECONDS } from "../config"; +import { browserFetch } from "../utils/browserFetch"; import type { ImportedAuthorInfo, ImportedPageMetadata } from "./importResolvers"; import { cleanThingiverseMarkdown } from "./descriptionMarkdown"; import type { ProviderCollectionCard } from "./providerCollectionCard"; @@ -80,16 +81,7 @@ export function parseThingiverseLikesUrl(url: string): { username: string } | nu } async function rawApiFetch(url: string): Promise { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), API_TIMEOUT_MS); - try { - return await fetch(url, { - headers: { "User-Agent": IMPORT_BROWSER_USER_AGENT, Accept: "application/json" }, - signal: controller.signal, - }); - } finally { - clearTimeout(timeout); - } + return browserFetch(url, { headers: { Accept: "application/json" }, timeoutMs: API_TIMEOUT_MS }); } /** Once a challenge is seen, calls go straight through FlareSolverr for a while. JSON API calls diff --git a/backend/src/utils/browserFetch.ts b/backend/src/utils/browserFetch.ts new file mode 100644 index 0000000..b7c6800 --- /dev/null +++ b/backend/src/utils/browserFetch.ts @@ -0,0 +1,49 @@ +import { Impit } from "impit"; + +// Thingiverse's Cloudflare challenges clients by their TLS handshake, whatever the headers or +// cookies, and Node's fetch is on its list. impit sends Chrome's handshake and headers. +const following = new Impit({ browser: "chrome", timeout: null }); +const manual = new Impit({ browser: "chrome", timeout: null, followRedirects: false }); + +const NULL_BODY_STATUSES = new Set([204, 205, 304]); + +/** Hosts that challenge Node's own fetch, so every request to them goes through browserFetch. */ +export function needsBrowserFetch(url: string): boolean { + try { + const host = new URL(url).hostname.toLowerCase(); + return host === "thingiverse.com" || host.endsWith(".thingiverse.com"); + } catch { + return false; + } +} + +/** fetch with Chrome's TLS fingerprint. `timeoutMs` covers waiting for the response, not reading + * the body. The caller's User-Agent is dropped: one that doesn't match the handshake is itself a + * bot signal. */ +export async function browserFetch( + url: string, + { + headers = {}, + timeoutMs, + signal, + redirect = "follow", + }: { headers?: Record; timeoutMs?: number; signal?: AbortSignal; redirect?: "follow" | "manual" }, +): Promise { + const sent = Object.fromEntries(Object.entries(headers).filter(([name]) => name.toLowerCase() !== "user-agent")); + const controller = new AbortController(); + const onAbort = () => controller.abort(); + signal?.addEventListener("abort", onAbort); + const timeout = timeoutMs ? setTimeout(onAbort, timeoutMs) : undefined; + try { + const client = redirect === "follow" ? following : manual; + const res = await client.fetch(url, { headers: sent, signal: controller.signal }); + // A standard Response, so callers and helpers typed for fetch's take it as is. + const body = NULL_BODY_STATUSES.has(res.status) ? null : res.body; + const out = new Response(body, { status: res.status, statusText: res.statusText, headers: res.headers }); + Object.defineProperty(out, "url", { value: res.url || url }); + return out; + } finally { + // The caller's signal stays wired: like fetch's, it also cuts off reading the body. + clearTimeout(timeout); + } +} diff --git a/backend/tests/setup.ts b/backend/tests/setup.ts index 544bb74..458514b 100644 --- a/backend/tests/setup.ts +++ b/backend/tests/setup.ts @@ -1,5 +1,6 @@ import type { Server } from "node:http"; import { Test } from "supertest"; +import { vi } from "vitest"; // supertest's request(app) listens on port 0 on every interface, then connects to 127.0.0.1 on the // port it got. On macOS a wildcard bind succeeds even when another program already holds that port @@ -45,3 +46,16 @@ proto.end = function (fn) { }); return this; }; + +// Tests mock global.fetch; send browserFetch through it too, read at call time so each test's mock applies. +vi.mock("../src/utils/browserFetch", async (importOriginal) => ({ + ...(await importOriginal()), + browserFetch: ( + url: string, + { + headers, + signal, + redirect, + }: { headers?: Record; signal?: AbortSignal; redirect?: RequestRedirect }, + ) => fetch(url, { headers, signal, redirect }), +}));