Merge pull request #62 from TautvydasDerzinskas/fix/thingiverse-chrome-tls

fix: download from Thingiverse with a Chrome TLS fingerprint

backend/package-lock.json

@@ -15,6 +15,7 @@
         "cors": "^2.8.6",
         "dotenv": "^18.0.8",
         "express": "^4.22.3",
+        "impit": "0.14.6",
         "jsonwebtoken": "^9.0.3",
         "jszip": "^3.10.2",
         "meshoptimizer": "^1.3.0",
@@ -3148,6 +3149,153 @@
       "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==",
       "license": "MIT"
     },
+    "node_modules/impit": {
+      "version": "0.14.6",
+      "resolved": "https://registry.npmjs.org/impit/-/impit-0.14.6.tgz",
+      "integrity": "sha512-1i5DxW0eRIUCtX9x+ZCSzD+FlRiEdrDOXazg0DYNMKUgnVwRWEwRhbA23Lpo2WOC674wZpwTQUubBi5z7m04Ng==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">= 20"
+      },
+      "optionalDependencies": {
+        "impit-darwin-arm64": "0.14.6",
+        "impit-darwin-x64": "0.14.6",
+        "impit-linux-arm64-gnu": "0.14.6",
+        "impit-linux-arm64-musl": "0.14.6",
+        "impit-linux-x64-gnu": "0.14.6",
+        "impit-linux-x64-musl": "0.14.6",
+        "impit-win32-arm64-msvc": "0.14.6",
+        "impit-win32-x64-msvc": "0.14.6"
+      }
+    },
+    "node_modules/impit-darwin-arm64": {
+      "version": "0.14.6",
+      "resolved": "https://registry.npmjs.org/impit-darwin-arm64/-/impit-darwin-arm64-0.14.6.tgz",
+      "integrity": "sha512-QhEENTXA727SlgIIIfPg4Ij2FEgwX/3rKCBpR84JxZZfQVVNO/KB2cVvt9aupokY1NZIo6oBFYJXF5PKyRVKbQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/impit-darwin-x64": {
+      "version": "0.14.6",
+      "resolved": "https://registry.npmjs.org/impit-darwin-x64/-/impit-darwin-x64-0.14.6.tgz",
+      "integrity": "sha512-BYcBdG7+smXHrhwLGqtWWw1jTsQSWwJO7ZyfkVPMUGRcbPFJAXXKRUjdFWcp2AdMWyI5JufNF3IZzAJ6B++8mw==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/impit-linux-arm64-gnu": {
+      "version": "0.14.6",
+      "resolved": "https://registry.npmjs.org/impit-linux-arm64-gnu/-/impit-linux-arm64-gnu-0.14.6.tgz",
+      "integrity": "sha512-yoUxNCBDsQ72H7uTtaLp7hUQJxr1/OtIZ7KnaerFeJwPpuK2KCeAIXa/9ajoa0GZ6BErm9ZwkggOENhAXu0xxw==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/impit-linux-arm64-musl": {
+      "version": "0.14.6",
+      "resolved": "https://registry.npmjs.org/impit-linux-arm64-musl/-/impit-linux-arm64-musl-0.14.6.tgz",
+      "integrity": "sha512-2UL2ya3uRCdcZwkTL4Oe0RVyhvsONJHhbtD+0VZZkDKyCxyyjtZ6Rc1QQNKh2q/xKa3vNnzj/9qcu0mj/zCk8w==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/impit-linux-x64-gnu": {
+      "version": "0.14.6",
+      "resolved": "https://registry.npmjs.org/impit-linux-x64-gnu/-/impit-linux-x64-gnu-0.14.6.tgz",
+      "integrity": "sha512-6TJxs/iUQbxQuznb0gz5MQ15WIT4/IxHdwNG9II8kEKUEzVMLCun8IXiT5AGdYzwlmPYSIuxDjtY2ZALhC1FlA==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/impit-linux-x64-musl": {
+      "version": "0.14.6",
+      "resolved": "https://registry.npmjs.org/impit-linux-x64-musl/-/impit-linux-x64-musl-0.14.6.tgz",
+      "integrity": "sha512-AoneCmxaKKCx2wJZGLtFB6LrkTZUEViZ9XEsocpyMsm5IvV3mhj64AGIZI7zHlW7AFXq4MW0DPZkHiL/KxTB1A==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/impit-win32-arm64-msvc": {
+      "version": "0.14.6",
+      "resolved": "https://registry.npmjs.org/impit-win32-arm64-msvc/-/impit-win32-arm64-msvc-0.14.6.tgz",
+      "integrity": "sha512-DRsyV3vNPeUTaRjtFiRqHb8FVFmLDR3NB47cvb+AKKuo+KUbRJdJR1zbDQMD2SOKhYRHf1PTxzamRIkl3yWqWQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/impit-win32-x64-msvc": {
+      "version": "0.14.6",
+      "resolved": "https://registry.npmjs.org/impit-win32-x64-msvc/-/impit-win32-x64-msvc-0.14.6.tgz",
+      "integrity": "sha512-0j7OHcgX8ui9ObSelx5v7OEEIfe00EgHcljq9RFR7oFzkD2ABS/6+fSyrXgu2LcC277vJ9oSfKjEacgGoCHOpQ==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
+    },
     "node_modules/inherits": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",

backend/package.json

@@ -24,6 +24,7 @@
     "cors": "^2.8.6",
     "dotenv": "^18.0.8",
     "express": "^4.22.3",
+    "impit": "0.14.6",
     "jsonwebtoken": "^9.0.3",
     "jszip": "^3.10.2",
     "meshoptimizer": "^1.3.0",

backend/src/services/descriptionImageService.ts

@@ -5,6 +5,7 @@ import { DESCRIPTION_IMAGES, IMPORT_BROWSER_USER_AGENT, IMPORT_TIMEOUT_SECONDS }
 import { prisma } from "../db";
 import { mapWithConcurrency } from "../utils/concurrency";
 import { validateRemoteUrl } from "../utils/urlUtils";
+import { browserFetch, needsBrowserFetch } from "../utils/browserFetch";
 import { readCapped } from "../utils/readCapped";
 
 const IMAGE_MAX_BYTES = 16 * 1024 * 1024;
@@ -47,11 +48,10 @@ async function fetchImage(url: string, deadline: number): Promise<Buffer | null>
     let current = url;
     for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
       await validateRemoteUrl(current);
-      const res = await fetch(current, {
-        headers: { "User-Agent": IMPORT_BROWSER_USER_AGENT, Accept: "image/*" },
-        redirect: "manual",
-        signal: controller.signal,
-      });
+      const headers = { "User-Agent": IMPORT_BROWSER_USER_AGENT, Accept: "image/*" };
+      const res = needsBrowserFetch(current)
+        ? await browserFetch(current, { headers, redirect: "manual", signal: controller.signal })
+        : await fetch(current, { headers, redirect: "manual", signal: controller.signal });
       const location = res.headers.get("location");
       if (res.status >= 300 && res.status < 400 && location) {
         await res.body?.cancel().catch(() => undefined);

backend/src/services/importService.ts

@@ -22,6 +22,7 @@ import {
 } from "../utils/fileUtils";
 import { validateRemoteUrl } from "../utils/urlUtils";
 import { readCapped } from "../utils/readCapped";
+import { browserFetch, needsBrowserFetch } from "../utils/browserFetch";
 import { mapWithConcurrency, maybeSleep, sleep } from "../utils/concurrency";
 import {
   CLOUDFLARE_BLOCKED_CODE,
@@ -118,6 +119,7 @@ function parseCharset(contentType: string | null): string {
 }
 
 async function rawFetch(url: string, headers: Record<string, string>): Promise<Response> {
+  if (needsBrowserFetch(url)) return browserFetch(url, { headers, timeoutMs: IMPORT_TIMEOUT_SECONDS * 1000 });
   const controller = new AbortController();
   const timeout = setTimeout(() => controller.abort(), IMPORT_TIMEOUT_SECONDS * 1000);
   try {

backend/src/services/thingiverseApi.ts

@@ -1,4 +1,5 @@
-import { IMPORT_BROWSER_USER_AGENT, IMPORT_TIMEOUT_SECONDS } from "../config";
+import { IMPORT_TIMEOUT_SECONDS } from "../config";
+import { browserFetch } from "../utils/browserFetch";
 import type { ImportedAuthorInfo, ImportedPageMetadata } from "./importResolvers";
 import { cleanThingiverseMarkdown } from "./descriptionMarkdown";
 import type { ProviderCollectionCard } from "./providerCollectionCard";
@@ -80,16 +81,7 @@ export function parseThingiverseLikesUrl(url: string): { username: string } | nu
 }
 
 async function rawApiFetch(url: string): Promise<Response> {
-  const controller = new AbortController();
-  const timeout = setTimeout(() => controller.abort(), API_TIMEOUT_MS);
-  try {
-    return await fetch(url, {
-      headers: { "User-Agent": IMPORT_BROWSER_USER_AGENT, Accept: "application/json" },
-      signal: controller.signal,
-    });
-  } finally {
-    clearTimeout(timeout);
-  }
+  return browserFetch(url, { headers: { Accept: "application/json" }, timeoutMs: API_TIMEOUT_MS });
 }
 
 /** Once a challenge is seen, calls go straight through FlareSolverr for a while. JSON API calls

backend/src/utils/browserFetch.ts

@@ -0,0 +1,49 @@
+import { Impit } from "impit";
+
+// Thingiverse's Cloudflare challenges clients by their TLS handshake, whatever the headers or
+// cookies, and Node's fetch is on its list. impit sends Chrome's handshake and headers.
+const following = new Impit({ browser: "chrome", timeout: null });
+const manual = new Impit({ browser: "chrome", timeout: null, followRedirects: false });
+
+const NULL_BODY_STATUSES = new Set([204, 205, 304]);
+
+/** Hosts that challenge Node's own fetch, so every request to them goes through browserFetch. */
+export function needsBrowserFetch(url: string): boolean {
+  try {
+    const host = new URL(url).hostname.toLowerCase();
+    return host === "thingiverse.com" || host.endsWith(".thingiverse.com");
+  } catch {
+    return false;
+  }
+}
+
+/** fetch with Chrome's TLS fingerprint. `timeoutMs` covers waiting for the response, not reading
+ * the body. The caller's User-Agent is dropped: one that doesn't match the handshake is itself a
+ * bot signal. */
+export async function browserFetch(
+  url: string,
+  {
+    headers = {},
+    timeoutMs,
+    signal,
+    redirect = "follow",
+  }: { headers?: Record<string, string>; timeoutMs?: number; signal?: AbortSignal; redirect?: "follow" | "manual" },
+): Promise<Response> {
+  const sent = Object.fromEntries(Object.entries(headers).filter(([name]) => name.toLowerCase() !== "user-agent"));
+  const controller = new AbortController();
+  const onAbort = () => controller.abort();
+  signal?.addEventListener("abort", onAbort);
+  const timeout = timeoutMs ? setTimeout(onAbort, timeoutMs) : undefined;
+  try {
+    const client = redirect === "follow" ? following : manual;
+    const res = await client.fetch(url, { headers: sent, signal: controller.signal });
+    // A standard Response, so callers and helpers typed for fetch's take it as is.
+    const body = NULL_BODY_STATUSES.has(res.status) ? null : res.body;
+    const out = new Response(body, { status: res.status, statusText: res.statusText, headers: res.headers });
+    Object.defineProperty(out, "url", { value: res.url || url });
+    return out;
+  } finally {
+    // The caller's signal stays wired: like fetch's, it also cuts off reading the body.
+    clearTimeout(timeout);
+  }
+}

backend/tests/setup.ts

@@ -1,5 +1,6 @@
 import type { Server } from "node:http";
 import { Test } from "supertest";
+import { vi } from "vitest";
 
 // supertest's request(app) listens on port 0 on every interface, then connects to 127.0.0.1 on the
 // port it got. On macOS a wildcard bind succeeds even when another program already holds that port
@@ -45,3 +46,16 @@ proto.end = function (fn) {
   });
   return this;
 };
+
+// Tests mock global.fetch; send browserFetch through it too, read at call time so each test's mock applies.
+vi.mock("../src/utils/browserFetch", async (importOriginal) => ({
+  ...(await importOriginal<typeof import("../src/utils/browserFetch")>()),
+  browserFetch: (
+    url: string,
+    {
+      headers,
+      signal,
+      redirect,
+    }: { headers?: Record<string, string>; signal?: AbortSignal; redirect?: RequestRedirect },
+  ) => fetch(url, { headers, signal, redirect }),
+}));