feat: pause guided MakerWorld imports on a CAPTCHA, and fix localized and long collection imports

- A MakerWorld CAPTCHA was reported as "No downloadable model file
  found". The extension's real-click download looked for a "Download"
  button that no longer exists in any language: the downloads sit in the
  menu behind the arrow next to "Open in Bambu Studio". It now opens that
  menu and clicks "Download 3MF", else "Download STL/CAD" then "Download
  All" (a zip), matched in all 12 MakerWorld UI languages. A 418/CAPTCHA
  reply is detected by both the extension and the backend, which now
  answers 429 with a clear message instead of trying more endpoints.
- A CAPTCHA now pauses the guided import on the same model instead of
  ending it. A docked panel leaves MakerWorld's puzzle reachable, and
  Resume retries that model. A "Wait between models" choice (5 s to
  5 min) paces long unattended runs; the countdown runs on the next
  model's page, since the extension background is killed after 30 s idle.
- The collection scan stopped early (e.g. 40 of 56) on localized pages:
  the end marker was English-only, so a 4 s no-new-cards timeout ended
  the scan on any slow page load. It now stops at the collection's own
  model count from the page data, knows the end marker in every
  language, and keeps the timeout only as a long fallback.
- The guided import hung every few models with nginx logging 499s:
  Chromium kills an extension service worker whose fetch() takes over
  30 s to answer. POST /import?async=1 now returns a task id to poll at
  GET /import/tasks/:id; an older instance ignores the flag and answers
  as before.
- The overlay's "Import next" button was always visible instead of only
  after a step looks stuck.
This commit is contained in:
Tautvydas Derzinskas committed 2026-10-05 19:59:01 +02:00
1 parent 62480d7828
commit 627a442d53
17 files changed
+780 -135

No files matched your search

+25 -14
View File
@@ -41,6 +41,7 @@ import { getUserMakerworldCookie } from "../services/makerworldCookieService";
import { listZipEntries } from "../services/zipService";
import { createJob, createJobItems, getActiveJob, getJob, listJobItems } from "../services/importJobService";
import { queueLink, startLinksJob } from "../services/importQueueService";
import { getImportTask, startImportTask } from "../services/importTaskService";
import {
runCollectionImportJob,
runLinksImportJob,
@@ -82,26 +83,36 @@ async function withStoredMakerworldCookie<T extends { makerworld_cookie?: string
return stored ? { ...body, makerworld_cookie: stored } : body;
}
async function importAndDescribe(userId: string, url: string, body: z.infer<typeof importRequestSchema>) {
const result = await importPrintFromUrl(userId, url, body);
const { print, plates, author, previewImages } = result;
const importOutcome = result.alreadyImported ? "already_imported" : result.profileAdded ? "profile_added" : "created";
void createLog({ userId, action: "model_imported", targetId: print.id, details: { name: print.name, url } });
return { ...toPrintOut(print, plates, [], null, author, previewImages), import_outcome: importOutcome };
}
// `?async=1` answers 202 with a task id to poll at /import/tasks/:id (see importTaskService.ts).
router.post(
"/import",
requireCaptcha("import"),
asyncHandler(async (req, res) => {
const body = await withStoredMakerworldCookie(req.userId!, parseBody(importRequestSchema, req.body));
const url = await normalizeImportUrl(body.url);
const result = await importPrintFromUrl(req.userId!, url, body);
const { print, plates, author, previewImages } = result;
const importOutcome = result.alreadyImported
? "already_imported"
: result.profileAdded
? "profile_added"
: "created";
res.json({ ...toPrintOut(print, plates, [], null, author, previewImages), import_outcome: importOutcome });
void createLog({
userId: req.userId!,
action: "model_imported",
targetId: print.id,
details: { name: print.name, url },
});
if (req.query.async === "1") {
const taskId = startImportTask(req.userId!, () => importAndDescribe(req.userId!, url, body));
res.status(202).json({ task_id: taskId });
return;
}
res.json(await importAndDescribe(req.userId!, url, body));
}),
);
router.get(
"/import/tasks/:id",
asyncHandler(async (req, res) => {
const task = getImportTask(req.params.id, req.userId!);
if (!task) throw new HttpError(404, "Import task not found -- it may have expired or the server restarted");
res.json(task);
}),
);
+11 -2
View File
@@ -10,6 +10,7 @@ import {
IMPORT_USER_AGENT,
} from "../config";
import { isJsonContentType } from "../utils/fileUtils";
import { isCaptchaChallenge, MakerworldCaptchaError } from "./makerworldCaptcha";
import {
extractJsonFromBrowserBody,
fetchViaFlaresolverr,
@@ -591,6 +592,14 @@ function makerworldModelIdFromUrl(url: string): string | null {
}
}
/** A CAPTCHA reply (HTTP 418) is thrown, not treated as "no URL": the fallbacks after it would only
* hit the same block and end in a misleading "No downloadable model file found". */
function makerworldDownloadUrlOrCaptcha(data: unknown, apiUrl: string): string | null {
const url = data ? extractDownloadUrlFromResponse(data, apiUrl) : null;
if (!url && isCaptchaChallenge(data)) throw new MakerworldCaptchaError();
return url;
}
async function fetchMakerworldInstanceDownloadUrl(
instanceId: string,
pageUrl: string,
@@ -598,7 +607,7 @@ async function fetchMakerworldInstanceDownloadUrl(
): Promise<string | null> {
const apiUrl = `https://makerworld.com/api/v1/design-service/instance/${instanceId}/f3mf?type=download&fileType=3mfstl`;
const data = await fetchJsonFromUrl(apiUrl, pageUrl, makerworldApiHeaders(pageUrl, null, cookie));
return data ? extractDownloadUrlFromResponse(data, apiUrl) : null;
return makerworldDownloadUrlOrCaptcha(data, apiUrl);
}
async function fetchMakerworldModelDownloadUrl(
@@ -609,7 +618,7 @@ async function fetchMakerworldModelDownloadUrl(
): Promise<string | null> {
const apiUrl = `https://makerworld.com/api/v1/models/${modelId}/download`;
const data = await fetchJsonFromUrl(apiUrl, pageUrl, makerworldApiHeaders(pageUrl, nonce, cookie));
return data ? extractDownloadUrlFromResponse(data, apiUrl) : null;
return makerworldDownloadUrlOrCaptcha(data, apiUrl);
}
/** `requestedInstanceId` is passed separately because `pageUrl` never carries the hash. Every
+7 -1
View File
@@ -377,7 +377,13 @@ export async function openImportResponse(
} else {
if (!makerworldCookie) makerworldCookie = resolveMakerworldCookie(body);
const requestedInstanceId = parseMakerworldModelUrl(validatedUrl)?.requestedInstanceId ?? null;
const resolved = await resolveMakerworldDownloadUrl(html, finalUrl, makerworldCookie, requestedInstanceId);
let resolved: Awaited<ReturnType<typeof resolveMakerworldDownloadUrl>>;
try {
resolved = await resolveMakerworldDownloadUrl(html, finalUrl, makerworldCookie, requestedInstanceId);
} catch (err) {
if (err instanceof MakerworldCaptchaError) throw new HttpError(429, err.message);
throw err;
}
if (resolved) {
downloadUrl = resolved.downloadUrl;
resolvedMeta.makerworldProfile = resolved.profile;
+55
View File
@@ -0,0 +1,55 @@
// A single-URL import run in the background so the caller can poll instead of holding one request
// open. The extension needs this: Chromium kills an extension service worker whose fetch() takes
// over 30s to answer, which cut long MakerWorld imports off mid-request (nginx logged a 499).
// In memory on purpose: a task lives for one import, and a lost one just reads as "not found".
import crypto from "node:crypto";
import { HttpError } from "../utils/fileUtils";
export type ImportTaskState =
| { status: "running" }
| { status: "done"; result: unknown }
| { status: "error"; detail: string; http_status: number };
type ImportTask = { userId: string; state: ImportTaskState; finishedAt: number | null };
// Long enough for a client to come back after a stall; a client polls every couple of seconds.
const FINISHED_TASK_TTL_MS = 10 * 60 * 1000;
const tasks = new Map<string, ImportTask>();
function pruneFinished(): void {
const cutoff = Date.now() - FINISHED_TASK_TTL_MS;
for (const [id, task] of tasks) {
if (task.finishedAt !== null && task.finishedAt < cutoff) tasks.delete(id);
}
}
export function startImportTask(userId: string, run: () => Promise<unknown>): string {
pruneFinished();
const id = crypto.randomUUID();
const task: ImportTask = { userId, state: { status: "running" }, finishedAt: null };
tasks.set(id, task);
run().then(
(result) => {
task.state = { status: "done", result };
task.finishedAt = Date.now();
},
(err: unknown) => {
if (!(err instanceof HttpError)) console.error(`Import task ${id} failed:`, err);
task.state =
err instanceof HttpError
? { status: "error", detail: err.message, http_status: err.status }
: { status: "error", detail: "Internal server error", http_status: 500 };
task.finishedAt = Date.now();
},
);
return id;
}
/** Null for an unknown, expired or someone else's task. */
export function getImportTask(id: string, userId: string): ImportTaskState | null {
pruneFinished();
const task = tasks.get(id);
return task && task.userId === userId ? task.state : null;
}
+3 -2
View File
@@ -17,8 +17,9 @@ export function isCaptchaChallenge(data: unknown): boolean {
export const MAKERWORLD_CAPTCHA_MESSAGE =
"MakerWorld is challenging this account with a CAPTCHA before it will hand over a download link. " +
"This can't be solved automatically. Open the model on makerworld.com and click Download there " +
"once -- that usually clears it -- then retry the import.";
"This can't be solved automatically. Open the model on makerworld.com, click the arrow next to " +
'"Open in Bambu Studio", choose "Download 3MF" and solve the puzzle -- that usually clears it -- ' +
"then retry the import.";
// The block is IP-scoped and clears on its own after 1-4 hours; retrying into it extends it.
// The cooloff only needs to stop a batch import hammering a known block, so 2h is enough.
+67
View File
@@ -192,6 +192,73 @@ describe("MakerWorld print profiles", () => {
expect((await post(`${pageUrl}#profileId-200`, "200")).body.import_outcome).toBe("already_imported");
});
it("reports MakerWorld's CAPTCHA instead of 'no downloadable file', and stops asking", async () => {
const fetchMock = vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async (input) => {
const url = String(input);
if (url.startsWith("https://makerworld.com/en/models/")) {
return new Response("<html><head><title>Captcha Test</title></head><body></body></html>", {
headers: { "content-type": "text/html; charset=utf-8" },
});
}
if (url === `https://makerworld.com/api/v1/models/${designId}/download`) {
return new Response(JSON.stringify({ captchaId: "abc", message: "we need to confirm you are not a robot" }), {
status: 418,
headers: { "content-type": "application/json" },
});
}
throw new Error(`Unexpected fetch to ${url}`);
});
global.fetch = fetchMock as unknown as typeof fetch;
const res = await request(app).post("/api/import").set(auth()).send({ url: pageUrl });
expect(res.status).toBe(429);
expect(res.body.detail).toMatch(/CAPTCHA/);
// None of the generic API fallbacks ran after the CAPTCHA.
expect(fetchMock.mock.calls.map(([input]) => String(input)).filter((u) => u.includes("/api/v1/"))).toEqual([
`https://makerworld.com/api/v1/models/${designId}/download`,
]);
});
it("runs POST /import?async=1 in the background and serves the result at /import/tasks/:id", async () => {
mockMakerworldFetch();
const start = await request(app)
.post("/api/import?async=1")
.set(auth())
.send({ url: pageUrl, resolved_download_url: "https://makerworld.com/files/profile-100.stl" });
expect(start.status).toBe(202);
const taskId = start.body.task_id as string;
let task = (await request(app).get(`/api/import/tasks/${taskId}`).set(auth())).body;
for (let i = 0; task.status === "running" && i < 100; i++) {
await new Promise((resolve) => setTimeout(resolve, 50));
task = (await request(app).get(`/api/import/tasks/${taskId}`).set(auth())).body;
}
expect(task.status).toBe("done");
expect(task.result.import_outcome).toBe("created");
expect(task.result.id).toEqual(expect.any(String));
});
it("hides an import task from other users", async () => {
mockMakerworldFetch();
const start = await request(app)
.post("/api/import?async=1")
.set(auth())
.send({ url: pageUrl, resolved_download_url: "https://makerworld.com/files/profile-100.stl" });
const other = await request(app)
.post("/api/register")
.send({ displayName: "Other", email: `imports-other-${Date.now()}@example.com`, password: "password123" });
const res = await request(app)
.get(`/api/import/tasks/${start.body.task_id}`)
.set({ Authorization: `Bearer ${other.body.token}` });
expect(res.status).toBe(404);
// Let the import finish before afterEach restores fetch and cleans up.
for (let i = 0; i < 100; i++) {
const task = await request(app).get(`/api/import/tasks/${start.body.task_id}`).set(auth());
if (task.body.status !== "running") break;
await new Promise((resolve) => setTimeout(resolve, 50));
}
});
it("treats a bare model URL as already imported without asking MakerWorld", async () => {
mockMakerworldFetch();
await importProfile(pageUrl, "100");
+18 -1
View File
@@ -13,6 +13,19 @@ export async function pollJobToCompletion(jobId: string): Promise<ImportJob> {
}
}
type ImportTaskState =
{ status: "running" } | { status: "done"; result: Print } | { status: "error"; detail: string; http_status: number };
/** Each poll is an extension API call (chrome.storage in apiCall), which keeps the worker alive. */
async function pollImportTask(taskId: string): Promise<Print> {
for (;;) {
const task = await apiCall<ImportTaskState>("GET", `/import/tasks/${taskId}`);
if (task.status === "done") return task.result;
if (task.status === "error") throw new Error(task.detail);
await new Promise((resolve) => setTimeout(resolve, JOB_POLL_INTERVAL_MS));
}
}
/** One message covering import and collection filing, so both complete even if the tab navigates
* away: the background outlives the content script. */
export async function importSingle({
@@ -39,7 +52,11 @@ export async function importSingle({
if (job.status === "ERROR") throw new Error(job.error_message || "Import failed");
print = job.result_print_id ? { id: job.result_print_id } : null;
} else {
print = await apiCall<Print>("POST", "/import", { url, ...extra });
// Asynchronous so the request never outlives the worker: Chromium kills an extension service
// worker whose fetch() takes over 30s to answer. An older instance ignores `async` and
// answers with the print itself.
const started = await apiCall<Print | { task_id: string }>("POST", "/import?async=1", { url, ...extra });
print = "task_id" in started ? await pollImportTask(started.task_id) : started;
}
if (collectionId && print?.id) {
+15 -1
View File
@@ -6,7 +6,15 @@ import { apiCall } from "./api";
import { getState, saveConfig, setDisabled, setSendToQueue } from "./config";
import { armDownloadCapture, awaitDownloadCapture } from "./downloadCapture";
import { importSingle, queueImport } from "./importJobs";
import { abortJob, advanceJob, dropJobIfForTab, forceAdvanceJob, getJobForTab, startJob } from "./makerworldJob";
import {
abortJob,
advanceJob,
dropJobIfForTab,
forceAdvanceJob,
getJobForTab,
resumeJob,
startJob,
} from "./makerworldJob";
import { getRecentImports } from "./recentImports";
import { openSetup } from "./setup";
import { setTabIconState } from "./tabIcon";
@@ -30,6 +38,12 @@ listen<BackgroundMessages>({
START_MAKERWORLD_COLLECTION_JOB: (payload, sender) => startJob(senderTabId(sender), payload),
ABORT_MAKERWORLD_COLLECTION_JOB: (_payload, sender) => abortJob(senderTabId(sender)),
FORCE_ADVANCE_MAKERWORLD_JOB: (_payload, sender) => forceAdvanceJob(senderTabId(sender)),
RESUME_MAKERWORLD_JOB: (_payload, sender) => resumeJob(senderTabId(sender)),
// Not awaited: the step runs for as long as the import takes.
MAKERWORLD_JOB_STEP_DUE: (_payload, sender) => {
void advanceJob(senderTabId(sender));
return null;
},
ARM_DOWNLOAD_CAPTURE: () => armDownloadCapture(),
AWAIT_DOWNLOAD_CAPTURE: () => awaitDownloadCapture(),
GET_MAKERWORLD_JOB: (_payload, sender) => getJobForTab(sender.tab?.id),
+60 -24
View File
@@ -2,7 +2,9 @@
// there, one at a time. Real page loads pace the requests the way no config value can, which
// keeps MakerWorld's account-wide CAPTCHA from tripping.
//
// State lives in chrome.storage because an MV3 background can be suspended between steps.
// State lives in chrome.storage because an MV3 background can be suspended between steps. For the
// same reason the wait between models is counted down by the next model's page (see overlays.ts),
// not slept here: the background is killed after 30s idle.
import type { ImportStatus } from "../shared/api";
import type { MakerworldJob, MakerworldJobError } from "../shared/messages";
@@ -12,8 +14,6 @@ import { importSingle } from "./importJobs";
const JOB_STORAGE_KEY = "makerworldCollectionJob";
const JOB_ERROR_STORAGE_KEY = "makerworldJobError";
// Not configurable: a shorter delay defeats the point.
const STEP_DELAY_MS = 5000;
// tabs.sendMessage has no timeout; this is well above the content script's own worst case.
const DOWNLOAD_RESOLVE_TIMEOUT_MS = 30000;
@@ -27,16 +27,24 @@ async function setJob(job: MakerworldJob | null): Promise<void> {
else await chrome.storage.local.remove(JOB_STORAGE_KEY);
}
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T | null> {
return Promise.race([promise, new Promise<null>((resolve) => setTimeout(() => resolve(null), ms))]);
}
/** Both the extension's and the backend's CAPTCHA messages say so; API errors carry only text. */
function isCaptchaMessage(message: string): boolean {
return /captcha/i.test(message);
}
/** `urls` are the not-yet-imported designs. tabs.onUpdated drives every step after the first. */
export async function startJob(
tabId: number,
{ urls, collectionId, originalUrl }: { urls: string[]; collectionId: string | null; originalUrl: string },
{
urls,
collectionId,
originalUrl,
stepDelayMs,
}: { urls: string[]; collectionId: string | null; originalUrl: string; stepDelayMs: number },
): Promise<null> {
if (!urls.length) return null;
await setJob({
@@ -48,6 +56,9 @@ export async function startJob(
imported: 0,
total: urls.length,
awaitingLoad: true,
stepDelayMs,
startAt: 0,
paused: false,
});
await chrome.tabs.update(tabId, { url: urls[0] });
return null;
@@ -62,6 +73,15 @@ export async function abortJob(tabId: number): Promise<null> {
return null;
}
/** Retries the model the run paused on, once the user has solved MakerWorld's CAPTCHA. */
export async function resumeJob(tabId: number): Promise<null> {
const job = await getJob();
if (!job || job.tabId !== tabId || !job.paused) return null;
await setJob({ ...job, paused: false, awaitingLoad: true, startAt: 0 });
await chrome.tabs.update(tabId, { url: job.urls[job.index] });
return null;
}
/** A just-stopped job's error is read-and-clear, so it's shown exactly once. */
export async function getJobForTab(
tabId: number | undefined,
@@ -80,6 +100,7 @@ async function isStillAtStep(tabId: number, stepIndex: number): Promise<boolean>
return Boolean(current && current.tabId === tabId && current.index === stepIndex);
}
/** The next model's page counts down `stepDelayMs` before its step starts. */
async function moveForward(tabId: number, job: MakerworldJob): Promise<void> {
job.index += 1;
if (job.index >= job.urls.length) {
@@ -87,45 +108,60 @@ async function moveForward(tabId: number, job: MakerworldJob): Promise<void> {
await chrome.tabs.update(tabId, { url: job.originalUrl });
return;
}
await setJob(job);
await sleep(STEP_DELAY_MS);
// The user may have hit Abort during the delay.
const stillActive = await getJob();
if (!stillActive || stillActive.tabId !== tabId) return;
job.awaitingLoad = true;
await setJob(job);
await setJob({ ...job, awaitingLoad: true, startAt: Date.now() + job.stepDelayMs });
await chrome.tabs.update(tabId, { url: job.urls[job.index] });
}
/** Any failure stops the run: it almost always means MakerWorld rejected the request, and more
* requests would make it worse. */
let claimQueue: Promise<unknown> = Promise.resolve();
/** Takes the current step if it's due. Serialized: the page load and the page's countdown can both
* ask at once, and the check-then-set on storage would otherwise let both through. */
function claimStep(tabId: number): Promise<MakerworldJob | null> {
const claimed = claimQueue.then(async () => {
const job = await getJob();
if (!job || job.tabId !== tabId || !job.awaitingLoad || job.paused || Date.now() < job.startAt) return null;
job.awaitingLoad = false;
await setJob(job);
return job;
});
claimQueue = claimed.catch(() => undefined);
return claimed;
}
/** A CAPTCHA pauses the run on this model for the user to solve; any other failure stops it, since
* it almost always means MakerWorld rejected the request, and more requests would make it worse. */
export async function advanceJob(tabId: number): Promise<void> {
const job = await getJob();
if (!job || job.tabId !== tabId || !job.awaitingLoad) return;
const job = await claimStep(tabId);
if (!job) return;
const stepIndex = job.index;
job.awaitingLoad = false;
await setJob(job);
const currentUrl = job.urls[stepIndex];
// The content script resolves the download itself with the page's session. Best-effort: null
// lets the backend resolve it.
let resolved = null;
let resolveError: string | null = null;
try {
const reply = await withTimeout(sendToTab(tabId, "RESOLVE_MAKERWORLD_DOWNLOAD_URL"), DOWNLOAD_RESOLVE_TIMEOUT_MS);
// The content script only replies with an error for a CAPTCHA; the backend would hit it too.
if (reply && reply.ok) resolved = reply.data;
else if (reply) resolveError = reply.error;
} catch {}
try {
if (resolveError) throw new Error(resolveError);
await importSingle({ url: currentUrl, collectionId: job.collectionId, resolved });
} catch (err) {
// A manual "Import next" may have moved past this step while the request was in flight.
if (!(await isStillAtStep(tabId, stepIndex))) return;
const message = err instanceof Error ? err.message : String(err);
if (isCaptchaMessage(message)) {
// Stays on this page: the CAPTCHA has to be solved here, in MakerWorld's own UI.
await setJob({ ...job, paused: true });
await sendToTab(tabId, "MAKERWORLD_JOB_UPDATED").catch(() => undefined);
return;
}
await setJob(null);
const error: MakerworldJobError = {
message: err instanceof Error ? err.message : String(err),
imported: job.imported,
total: job.total,
};
const error: MakerworldJobError = { message, imported: job.imported, total: job.total };
await chrome.storage.local.set({ [JOB_ERROR_STORAGE_KEY]: error });
await chrome.tabs.update(tabId, { url: job.originalUrl });
return;
+16 -4
View File
@@ -6,9 +6,9 @@ import { listen, send, type ContentMessages } from "../shared/messages";
import { CONFIG_CHANGE_KEYS } from "../shared/storage";
import { classifyUrl } from "../shared/urls";
import { setContext, type LibraryState } from "./context";
import { resolveMakerworldDownloadUrl } from "./makerworld/downloadResolver";
import { MakerworldCaptchaError, resolveMakerworldDownloadUrl } from "./makerworld/downloadResolver";
import { offerNormalizedDownload, unmountNormalizedDownload } from "./makerworld/normalizedDownload";
import { mountJobOverlay } from "./overlays";
import { mountJobOverlay, mountPausedJobOverlay } from "./overlays";
import { loadPanel } from "./panels";
import { errorHtml } from "./panels/results";
import { api } from "./runtime";
@@ -43,7 +43,9 @@ async function init(): Promise<void> {
const jobRes = await send("GET_MAKERWORLD_JOB");
if (isStale()) return;
if (jobRes?.ok && jobRes.data.job) {
mountJobOverlay(mountHost(), jobRes.data.job);
const job = jobRes.data.job;
if (job.paused) mountPausedJobOverlay(mountHost(), job);
else mountJobOverlay(mountHost(), job);
reportTabIconState(true);
return;
}
@@ -153,7 +155,17 @@ if (!window.thingportGrabInjected) {
});
listen<ContentMessages>({
RESOLVE_MAKERWORLD_DOWNLOAD_URL: () => resolveMakerworldDownloadUrl(location.href).catch(() => null),
// A CAPTCHA comes back as an error reply so the guided import can say so.
RESOLVE_MAKERWORLD_DOWNLOAD_URL: () =>
resolveMakerworldDownloadUrl(location.href).catch((err) => {
if (err instanceof MakerworldCaptchaError) throw err;
return null;
}),
MAKERWORLD_JOB_UPDATED: () => {
unmount();
void init();
return null;
},
});
installNavigationWatcher();
@@ -1,10 +1,13 @@
// Resolves the MakerWorld file URL in the page instead of the backend, whose resolution calls are
// what trip MakerWorld's account-wide CAPTCHA. Same-origin fetches carry the real session and look
// like the page's own traffic. Returns null on any failure so the backend can resolve it instead.
// like the page's own traffic. Returns null on any failure so the backend can resolve it instead,
// except a CAPTCHA, which is thrown: the backend would only hit the same wall.
import type { ResolvedDownload } from "../../shared/messages";
import { send } from "../../shared/messages";
import { parseMakerworldModelUrl } from "../../shared/urls";
import { sleep } from "../runtime";
import { DOWNLOAD_3MF, DOWNLOAD_ALL, DOWNLOAD_OTHER, DOWNLOAD_STL, OPEN_IN_APP } from "./labels";
import {
loadMakerworldDesignForPage,
makerworldDesignForImport,
@@ -39,6 +42,27 @@ function extractDownloadUrl(data: unknown): string | null {
return null;
}
export const MAKERWORLD_CAPTCHA_MESSAGE =
'MakerWorld is asking for a CAPTCHA ("confirm you are not a robot"). Open any model on MakerWorld, ' +
'click the arrow next to "Open in Bambu Studio", choose "Download 3MF" and solve the puzzle, then ' +
"retry the import.";
/** MakerWorld's anti-bot layer answers with HTTP 418 (sometimes 200) and a body naming a captcha.
* Thrown rather than returned as null so it isn't reported as "no file found". */
export class MakerworldCaptchaError extends Error {
constructor() {
super(MAKERWORLD_CAPTCHA_MESSAGE);
this.name = "MakerworldCaptchaError";
}
}
/** Only asked of a body without a download URL, whose filename could otherwise say "robot". */
function looksLikeCaptcha(data: unknown): boolean {
if (!data || typeof data !== "object") return false;
const text = JSON.stringify(data).toLowerCase();
return text.includes("captcha") || text.includes("robot");
}
async function fetchMakerworldApiJson(url: string, nonce: string | null): Promise<unknown> {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), MAKERWORLD_API_FETCH_TIMEOUT_MS);
@@ -46,35 +70,91 @@ async function fetchMakerworldApiJson(url: string, nonce: string | null): Promis
const headers: Record<string, string> = { Accept: "application/json", ...MAKERWORLD_CLIENT_HEADERS };
if (nonce) headers["X-Nonce"] = nonce;
const res = await fetch(url, { headers, signal: controller.signal });
if (!res.ok) return null;
return await res.json();
} catch {
if (res.status === 418) throw new MakerworldCaptchaError();
const data: unknown = await res.json();
if (extractDownloadUrl(data)) return data;
if (looksLikeCaptcha(data)) throw new MakerworldCaptchaError();
return res.ok ? data : null;
} catch (err) {
if (err instanceof MakerworldCaptchaError) throw err;
return null;
} finally {
clearTimeout(timeout);
}
}
/** Matched by text (class names are build hashes), visible elements only. */
function findDownloadButton(): HTMLElement | null {
for (const el of document.querySelectorAll<HTMLElement>("button, a")) {
if (!/^download$/i.test((el.textContent || "").trim())) continue;
if (el.offsetParent === null) continue; // hidden (display:none or detached)
return el;
const MENU_WAIT_MS = 2000;
// The raw-files dialog loads its file list before showing "Download all".
const DIALOG_WAIT_MS = 4000;
/** Polls until `find` returns something or `ms` passes; menus and dialogs render asynchronously. */
async function waitFor<T>(find: () => T | null, ms: number): Promise<T | null> {
for (const end = Date.now() + ms; ; await sleep(150)) {
const found = find();
if (found || Date.now() >= end) return found;
}
}
/** The visible element holding exactly one of `labels` as its text. */
function findByLabel(labels: Set<string>): HTMLElement | null {
const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT);
for (let node = walker.nextNode(); node; node = walker.nextNode()) {
const el = node.parentElement;
if (el && el.offsetParent !== null && labels.has((node.textContent || "").trim())) return el;
}
return null;
}
/** Captures and cancels the download the real button starts (see background/downloadCapture.ts).
* Null means try the fallback. */
async function captureViaRealClick(): Promise<string | null> {
const button = findDownloadButton();
if (!button) return null;
/** The arrow beside "Open in Bambu Studio" (or Suite/Handy) that opens the download menu. */
function findMenuArrow(): HTMLElement | null {
const label = findByLabel(OPEN_IN_APP);
if (!label) return null;
let el = label.parentElement;
for (let depth = 0; el && depth < 4; depth++, el = el.parentElement) {
const arrow = [...el.children].find((child) => !child.contains(label) && child.querySelector("svg"));
if (arrow) return arrow as HTMLElement;
}
return null;
}
async function armCapture(): Promise<boolean> {
const armed = await send("ARM_DOWNLOAD_CAPTURE");
if (!armed || !armed.ok) return null;
button.click();
return Boolean(armed && armed.ok);
}
/** Escape, sent where MakerWorld's dialog listens: it traps focus inside itself. */
function closeFilesDialog(): void {
(document.activeElement ?? document.body).dispatchEvent(
new KeyboardEvent("keydown", { key: "Escape", bubbles: true }),
);
}
/** Clicks through MakerWorld's own download UI the way a person would: "Download 3MF" (the
* selected profile) from the menu under "Open in Bambu Studio", else the model's raw files (a
* zip, via the dialog's "Download all"), else any other download item. The browser download that
* starts is captured and cancelled (see background/downloadCapture.ts). Null means try the API
* fallback -- also what a CAPTCHA looks like here, since MakerWorld shows it instead of
* downloading. */
async function captureViaRealClick(): Promise<{ url: string; isProfile: boolean } | null> {
const arrow = findMenuArrow();
arrow?.click();
const fileItem = () => findByLabel(DOWNLOAD_3MF) ?? findByLabel(DOWNLOAD_STL);
const item = (arrow ? await waitFor(fileItem, MENU_WAIT_MS) : fileItem()) ?? findByLabel(DOWNLOAD_OTHER);
if (!item) return null;
const isProfile = DOWNLOAD_3MF.has((item.textContent || "").trim());
if (!(await armCapture())) return null;
item.click();
if (!isProfile) {
// Raw-file items open a dialog listing the files instead of downloading.
const downloadAll = await waitFor(() => findByLabel(DOWNLOAD_ALL), DIALOG_WAIT_MS);
if (downloadAll) {
if (!(await armCapture())) return null;
downloadAll.click();
}
}
const res = await send("AWAIT_DOWNLOAD_CAPTURE");
return res && res.ok ? res.data : null;
if (!isProfile) closeFilesDialog();
return res && res.ok && res.data ? { url: res.data, isProfile } : null;
}
/** The printer the profile was made for, which MakerWorld's own download request names. */
@@ -116,17 +196,23 @@ async function resolveFromPageApi(pageUrl: string): Promise<ResolvedDownload | n
export async function resolveMakerworldDownloadUrl(pageUrl: string): Promise<ResolvedDownload | null> {
const viaClick = await captureViaRealClick().catch(() => null);
if (viaClick) {
// The button downloads the page's selected profile, which follows the URL hash.
// "Download 3MF" gives the page's selected profile, which follows the URL hash; the raw-files
// zip belongs to no profile.
const page = readMakerworldDesignForPage(pageUrl);
if (!page)
return { downloadUrl: viaClick, instanceId: parseMakerworldModelUrl(pageUrl)?.requestedInstanceId ?? null };
return {
downloadUrl: viaClick,
instanceId: pickMakerworldInstanceId(page.design, page.requestedInstanceId),
design: makerworldDesignForImport(page.design),
};
const requested = parseMakerworldModelUrl(pageUrl)?.requestedInstanceId ?? null;
const instanceId = !viaClick.isProfile
? null
: page
? pickMakerworldInstanceId(page.design, page.requestedInstanceId)
: requested;
return page
? { downloadUrl: viaClick.url, instanceId, design: makerworldDesignForImport(page.design) }
: { downloadUrl: viaClick.url, instanceId };
}
return resolveFromPageApi(pageUrl).catch(() => null);
return resolveFromPageApi(pageUrl).catch((err) => {
if (err instanceof MakerworldCaptchaError) throw err;
return null;
});
}
/** The Download button only gives the selected profile, so other profiles resolve via the API. */
+160
View File
@@ -0,0 +1,160 @@
// MakerWorld UI labels in every language it ships (de, en, es, fr, it, ja, ko, pt, ru, sv, zh,
// zh-tw; its other locales show English). Matched on text because class names are build hashes.
// Each set is one key of the site's own translations, readable from any page's
// __NEXT_DATA__ props.pageProps._nextI18Next.initialI18nStore.
/** common.common.noMore: shown under a fully loaded list. */
export const NO_MORE_DATA = new Set([
"No more data",
"Keine weiteren Daten",
"No hay más datos",
"Aucune donnée disponible",
"Nessun ulteriore dato",
"これ以上データはありません",
"더 이상 데이터 없음",
"Sem mais dados",
"Больше данных нет",
"Inga fler uppgifter",
"没有更多了~",
"沒有更多資料",
]);
/** The model page's main button, whose arrow opens the download menu:
* common.models.detail.open_in_studio / open_in_bambu_suite / open_in_bambu_handy. */
export const OPEN_IN_APP = new Set([
"Open in Bambu Studio",
"Im Bambu Studio öffnen",
"Abrir en Bambu Studio",
"Ouvrir dans Bambu Studio",
"Apri in Bambu Studio",
"Bambu Studioで開く",
"Bambu Studio에서 열기",
"Aberto no Bambu Studio",
"Открыть в Bambu Studio",
"Öppet i Bambu Studio",
"在 Bambu Studio 中打开",
"在 Bambu Studio 中開啟",
"Open in Bambu Suite",
"In der Bambu Suite öffnen",
"Abrir en Bambu Suite",
"Ouvert dans la suite Bambu",
"Apri in Bambu Suite",
"Bambu Suite で開く",
"Bambu Suite에서 열기",
"Aberto na Suíte Bambu",
"Открыть в Bambu Suite",
"Öppna i Bambu Suite",
"在 Bambu Suite 中打开",
"在 Bambu Suite 中開啟",
"Open in Bambu Handy",
"In Bambu Handy öffnen",
"Abrir en Bambu Handy",
"Ouvrir dans Bambu Handy",
"Apri in Bambu Handy",
"Bambu Handyで開く",
"Bambu Handy에서 열기",
"Aberto em Bambu Handy",
"Открыть в Bambu Handy",
"Öppna i Bambu Handy",
"在Bambu Handy中打开",
"在 Bambu Handy 中開啟",
]);
/** common.models.detail.download_3mf: downloads the selected print profile directly. */
export const DOWNLOAD_3MF = new Set([
"Download 3MF",
"3MF herunterladen",
"Descargar 3MF",
"Télécharger en 3MF",
"Scaricare 3MF",
"3MFをダウンロード",
"3MF 다운로드",
"Baixar 3MF",
"Скачать 3MF",
"Ladda ner 3MF",
"下载 3MF",
"下載 3MF",
]);
/** common.models.detail.download_stl: opens the raw-files dialog (see DOWNLOAD_ALL). */
export const DOWNLOAD_STL = new Set([
"Download STL/CAD Files",
"STL/CAD herunterladen",
"Descargar archivos STL/CAD",
"Télécharger en STL",
"Download STL",
"STL/CAD ファイルのダウンロード",
"STL/CAD 파일 다운로드",
"Baixar arquivos STL/CAD",
"Скачать STL/CAD-файлы",
"Ladda ner STL",
"下载 STL/CAD 文件",
"下載 STL/CAD 檔案",
]);
/** Other model types' download items, and the plain button: download_model_files,
* 2d_models_download_svg, download_lac, common.common.download. */
export const DOWNLOAD_OTHER = new Set([
"Download Model Files",
"Modelldateien herunterladen",
"Descargar archivos de modelo",
"Télécharger les Fichiers Modèles",
"Download file del modello",
"モデルファイルをダウンロード",
"모델 파일 다운로드",
"Baixar arquivos de modelo",
"Скачать файлы модели",
"Ladda ner modellfiler",
"下载模型文件",
"下載模型檔案",
"Download SVG Files",
"SVG-Dateien herunterladen",
"Descargar archivos SVG",
"Télécharger les fichiers SVG",
"Scarica file SVG",
"SVG ファイルのダウンロード",
"SVG 파일 다운로드",
"Baixar arquivos SVG",
"Скачать SVG-файлы",
"Ladda ner SVG-filer",
"下载 SVG",
"下載 SVG 檔案",
"Download LAC",
"LAC herunterladen",
"Descargar LAC",
"Télécharger LAC",
"Scarica LAC",
"LACをダウンロード",
"LAC 다운로드",
"Baixar LAC",
"Скачать LAC",
"Ladda ner LAC",
"下载 LAC",
"下載 LAC",
"Download",
"Herunterladen",
"Descargar",
"Télécharger",
"Scarica",
"ダウンロード",
"다운로드",
"Скачать",
"Ladda ned",
"下載",
]);
/** common.models.download_all: the raw-files dialog's button, which downloads a zip of them. */
export const DOWNLOAD_ALL = new Set([
"Download All",
"Alles herunterladen",
"Descargar todo",
"Télécharger Tout",
"Download tutto",
"すべてダウンロード",
"모두 다운로드",
"Baixar tudo",
"Скачать всё",
"Ladda ner alla",
"下载所有",
"全部下載",
]);
+34 -11
View File
@@ -52,13 +52,8 @@ export function readMakerworldDesignForPage(pageUrl: string): MakerworldPage | n
const PAGE_FETCH_TIMEOUT_MS = 10000;
/** Fetches the model's own page once when the tab's data is stale (the usual case after clicking
* through MakerWorld). */
export async function loadMakerworldDesignForPage(pageUrl: string): Promise<MakerworldPage | null> {
const current = readMakerworldDesignForPage(pageUrl);
if (current) return current;
const expected = parseMakerworldModelUrl(pageUrl);
if (!expected) return null;
/** A page's own __NEXT_DATA__, fetched fresh. */
async function fetchNextData(pageUrl: string): Promise<unknown> {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), PAGE_FETCH_TIMEOUT_MS);
try {
@@ -66,10 +61,7 @@ export async function loadMakerworldDesignForPage(pageUrl: string): Promise<Make
if (!res.ok) return null;
const html = await res.text();
const el = new DOMParser().parseFromString(html, "text/html").getElementById("__NEXT_DATA__");
const found = el?.textContent ? designFromNextData(JSON.parse(el.textContent), expected.designId) : null;
if (!found) return null;
fetchedDesigns.set(expected.designId, found);
return { ...found, requestedInstanceId: expected.requestedInstanceId };
return el?.textContent ? JSON.parse(el.textContent) : null;
} catch {
return null;
} finally {
@@ -77,6 +69,37 @@ export async function loadMakerworldDesignForPage(pageUrl: string): Promise<Make
}
}
/** Fetches the model's own page once when the tab's data is stale (the usual case after clicking
* through MakerWorld). */
export async function loadMakerworldDesignForPage(pageUrl: string): Promise<MakerworldPage | null> {
const current = readMakerworldDesignForPage(pageUrl);
if (current) return current;
const expected = parseMakerworldModelUrl(pageUrl);
if (!expected) return null;
const found = designFromNextData(await fetchNextData(pageUrl), expected.designId);
if (!found) return null;
fetchedDesigns.set(expected.designId, found);
return { ...found, requestedInstanceId: expected.requestedInstanceId };
}
function collectionTotalFromNextData(nextData: unknown, collectionId: string): number | null {
const favoriteId = getPath(nextData, "props", "pageProps", "favorite", "id");
if (favoriteId == null || String(favoriteId) !== collectionId) return null;
const total = getPath(nextData, "props", "pageProps", "favoriteDesigns", "total");
return typeof total === "number" && total >= 0 ? total : null;
}
/** How many models the collection page will list (hidden ones excluded), from the page's own data;
* the page itself only renders 20 at a time. Null if it can't be read. */
export async function loadMakerworldCollectionTotal(pageUrl: string): Promise<number | null> {
const collectionId = pageUrl.match(/\/collections\/(\d+)/)?.[1];
if (!collectionId) return null;
return (
collectionTotalFromNextData(readNextData(), collectionId) ??
collectionTotalFromNextData(await fetchNextData(pageUrl), collectionId)
);
}
/** The requested profile (if the design has it), then the default, then the first. */
export function pickMakerworldInstanceId(design: MakerworldDesign, requestedInstanceId: string | null): string | null {
const instances = Array.isArray(design.instances) ? design.instances.filter((inst) => inst && inst.id) : [];
+78 -20
View File
@@ -19,40 +19,98 @@ function overlayHtml(title: string, countHtml: string, extra = ""): string {
`;
}
export function mountJobOverlay(root: ShadowRoot, job: MakerworldJob): void {
const overlay = document.createElement("div");
overlay.className = "tg-overlay";
const percent = job.total ? Math.round((job.imported / job.total) * 100) : 0;
overlay.innerHTML = overlayHtml(
"Importing from MakerWorld…",
`<div class="tg-overlay__count">${job.imported} / ${job.total} models imported (${percent}%)</div>
<div class="tg-progress"><div class="tg-progress__bar" style="width:${percent}%"></div></div>`,
`<button class="tg-btn tg-btn--danger" type="button" data-action="abort">Abort</button>
<button class="tg-btn tg-btn--secondary" type="button" data-action="next" hidden>Import next</button>`,
);
fillIcons(overlay);
root.appendChild(overlay);
function formatWait(ms: number): string {
const seconds = Math.max(0, Math.ceil(ms / 1000));
return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`;
}
function progressHtml(job: MakerworldJob): string {
const percent = job.total ? Math.round((job.imported / job.total) * 100) : 0;
return `<div class="tg-overlay__count">${job.imported} / ${job.total} models imported (${percent}%)</div>
<div class="tg-progress"><div class="tg-progress__bar" style="width:${percent}%"></div></div>`;
}
function wireAbort(overlay: HTMLElement): void {
const abortBtn = overlay.querySelector<HTMLButtonElement>("[data-action=abort]")!;
abortBtn.addEventListener("click", () => {
abortBtn.disabled = true;
abortBtn.textContent = "Aborting…";
void send("ABORT_MAKERWORLD_COLLECTION_JOB");
});
}
/** Counts down the wait between models, then asks the background to start this one. */
export function mountJobOverlay(root: ShadowRoot, job: MakerworldJob): void {
const overlay = document.createElement("div");
overlay.className = "tg-overlay";
overlay.innerHTML = overlayHtml(
"Importing from MakerWorld…",
progressHtml(job),
`<div class="tg-overlay__count" data-role="wait" hidden></div>
<button class="tg-btn tg-btn--danger" type="button" data-action="abort">Abort</button>
<button class="tg-btn tg-btn--secondary" type="button" data-action="next" hidden>Import next</button>`,
);
fillIcons(overlay);
root.appendChild(overlay);
wireAbort(overlay);
// Normally the page navigates on first, tearing this timer down.
const nextBtn = overlay.querySelector<HTMLButtonElement>("[data-action=next]")!;
setTimeout(() => {
nextBtn.hidden = false;
}, JOB_STUCK_REVEAL_MS);
nextBtn.addEventListener("click", () => {
nextBtn.disabled = true;
nextBtn.textContent = "Checking…";
void send("FORCE_ADVANCE_MAKERWORLD_JOB");
});
// Normally the page navigates on first, tearing this timer down.
const startStep = () => {
if (job.awaitingLoad) void send("MAKERWORLD_JOB_STEP_DUE");
setTimeout(() => {
nextBtn.hidden = false;
}, JOB_STUCK_REVEAL_MS);
};
const waitEl = overlay.querySelector<HTMLElement>("[data-role=wait]")!;
if (!job.awaitingLoad || Date.now() >= job.startAt) {
startStep();
return;
}
waitEl.hidden = false;
const tick = () => {
const remaining = job.startAt - Date.now();
waitEl.textContent = `Next model in ${formatWait(remaining)}`;
if (remaining > 0) return;
clearInterval(timer);
waitEl.hidden = true;
startStep();
};
const timer = setInterval(tick, 1000);
tick();
}
export type ScanOverlay = { update(count: number): void; remove(): void };
/** Docked rather than full-page: the user has to reach MakerWorld's own CAPTCHA underneath. */
export function mountPausedJobOverlay(root: ShadowRoot, job: MakerworldJob): void {
const overlay = document.createElement("div");
overlay.className = "tg-overlay tg-overlay--docked";
overlay.innerHTML = overlayHtml(
"Paused: MakerWorld wants a CAPTCHA",
progressHtml(job),
`<div class="tg-overlay__text">Solve the puzzle on this page. If none is showing, click the arrow
next to "Open in Bambu Studio" and choose "Download 3MF" to bring it up. Then resume -- the
run continues from this model.</div>
<button class="tg-btn" type="button" data-action="resume">Resume</button>
<button class="tg-btn tg-btn--danger" type="button" data-action="abort">Abort</button>`,
);
fillIcons(overlay);
root.appendChild(overlay);
wireAbort(overlay);
const resumeBtn = overlay.querySelector<HTMLButtonElement>("[data-action=resume]")!;
resumeBtn.addEventListener("click", () => {
resumeBtn.disabled = true;
resumeBtn.textContent = "Resuming…";
void send("RESUME_MAKERWORLD_JOB");
});
}
export type ScanOverlay = { update(count: number, total?: number | null): void; remove(): void };
export function mountScanOverlay(root: ShadowRoot): ScanOverlay {
const overlay = document.createElement("div");
@@ -65,8 +123,8 @@ export function mountScanOverlay(root: ShadowRoot): ScanOverlay {
root.appendChild(overlay);
const countEl = overlay.querySelector<HTMLElement>('[data-role="count"]')!;
return {
update: (count) => {
countEl.textContent = `${count} found so far`;
update: (count, total) => {
countEl.textContent = total ? `${count} of ${total} found` : `${count} found so far`;
},
remove: () => overlay.remove(),
};
@@ -1,58 +1,78 @@
// MakerWorld collection import. Avoids MakerWorld's collection-listing API (a burst of those calls
// trips an account-wide CAPTCHA) by scrolling the page and scraping model ids from the DOM.
// trips an account-wide CAPTCHA) by scrolling the page and scraping model ids from the DOM, until
// it has as many as the page's own data says the collection holds.
// Every not-yet-imported model is queued.
import type { Collection, ImportStatus } from "../../shared/api";
import { request } from "../../shared/messages";
import { makerworldModelUrl } from "../../shared/urls";
import { ctx } from "../context";
import { NO_MORE_DATA } from "../makerworld/labels";
import { loadMakerworldCollectionTotal } from "../makerworld/pageData";
import { mountScanOverlay } from "../overlays";
import { api, escapeHtml, sleep } from "../runtime";
import { getShadowRoot, onPanelAction, renderPanel } from "../shell";
import { getShadowRoot, onPanelAction, panelQuery, renderPanel } from "../shell";
import { errorHtml, statusHtml, successHtml } from "./results";
// The page may still be hydrating; checking too early sees zero cards and looks like the end.
const SCAN_START_DELAY_MS = 1500;
const SCAN_ROUND_DELAY_MS = 700;
const SCAN_MAX_ROUNDS = 300; // generous: 153 models at ~20/page is ~8 loads
const SCAN_STAGNANT_LIMIT = 6;
const SCAN_MAX_ROUNDS = 600; // generous: ~1,000 models at 20/page is ~50 loads
// ~14s without new cards. Only a fallback: the scan normally stops at the page's own model count
// or the end marker. At ~4s, a slow page load ended it early (e.g. at 40 of 56).
const SCAN_STAGNANT_LIMIT = 20;
/** Matched on text: MakerWorld's class names are build hashes. */
function hasNoMoreDataMarker(): boolean {
const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT);
for (let node = walker.nextNode(); node; node = walker.nextNode()) {
if (node.textContent && node.textContent.trim() === "No more data") return true;
if (node.textContent && NO_MORE_DATA.has(node.textContent.trim())) return true;
}
return false;
}
function extractModelIds(): string[] {
const ids = new Set<string>();
function collectModelIds(into: Set<string>): void {
for (const a of document.querySelectorAll('a[href*="/models/"]')) {
const match = (a.getAttribute("href") || "").match(/\/models\/(\d+)/);
if (match) ids.add(match[1]);
if (match) into.add(match[1]);
}
return [...ids];
}
/** Scrolls the last card into view until the end marker shows or the count stops growing. */
async function scrollToEnd(onProgress: (count: number) => void): Promise<void> {
let lastCount = -1;
function scrollToLastCard(): void {
const links = document.querySelectorAll('a[href*="/models/"]');
const lastLink = links[links.length - 1];
if (lastLink) lastLink.scrollIntoView({ block: "end" });
else window.scrollTo(0, document.body.scrollHeight);
}
/** Scrolls the last card into view until `total` models are seen, the end marker shows, or the
* count stops growing. Ids are collected every round, in case the page drops cards that scrolled
* out of view. */
async function scrollToEnd(
total: number | null,
onProgress: (count: number, total: number | null) => void,
): Promise<string[]> {
const ids = new Set<string>();
let stagnantRounds = 0;
await sleep(SCAN_START_DELAY_MS);
for (let round = 0; round < SCAN_MAX_ROUNDS; round++) {
if (hasNoMoreDataMarker()) return;
const links = document.querySelectorAll('a[href*="/models/"]');
const lastLink = links[links.length - 1];
if (lastLink) lastLink.scrollIntoView({ block: "end" });
else window.scrollTo(0, document.body.scrollHeight);
collectModelIds(ids);
if ((total !== null && ids.size >= total) || hasNoMoreDataMarker()) break;
// Scrolling to where the page already is fires no scroll event, so a stalled infinite-scroll
// loader is nudged by stepping back up first.
if (stagnantRounds > 0 && stagnantRounds % 3 === 0) {
window.scrollBy(0, -window.innerHeight);
await sleep(100);
}
scrollToLastCard();
await sleep(SCAN_ROUND_DELAY_MS);
const count = extractModelIds().length;
onProgress(count);
stagnantRounds = count === lastCount ? stagnantRounds + 1 : 0;
if (stagnantRounds >= SCAN_STAGNANT_LIMIT) return;
lastCount = count;
const before = ids.size;
collectModelIds(ids);
onProgress(ids.size, total);
stagnantRounds = ids.size === before ? stagnantRounds + 1 : 0;
if (stagnantRounds >= SCAN_STAGNANT_LIMIT) break;
}
return [...ids];
}
/** Case-insensitive match on title, created if missing. Null on failure; the import proceeds. */
@@ -70,6 +90,36 @@ async function findOrCreateCollection(name: string | null): Promise<string | nul
}
}
// The first is the default. Longer waits let a big run go unattended under MakerWorld's rate limit.
const STEP_DELAY_CHOICES = [
{ ms: 5_000, label: "5 seconds" },
{ ms: 30_000, label: "30 seconds" },
{ ms: 60_000, label: "1 minute" },
{ ms: 2 * 60_000, label: "2 minutes" },
{ ms: 5 * 60_000, label: "5 minutes" },
];
const STEP_DELAY_STORAGE_KEY = "makerworldStepDelayMs";
async function rememberedStepDelay(): Promise<number> {
try {
const stored = (await chrome.storage.local.get(STEP_DELAY_STORAGE_KEY))[STEP_DELAY_STORAGE_KEY];
if (STEP_DELAY_CHOICES.some((choice) => choice.ms === stored)) return stored as number;
} catch {}
return STEP_DELAY_CHOICES[0].ms;
}
async function rememberStepDelay(ms: number): Promise<void> {
try {
await chrome.storage.local.set({ [STEP_DELAY_STORAGE_KEY]: ms });
} catch {}
}
function stepDelayOptions(selectedMs: number): string {
return STEP_DELAY_CHOICES.map(
({ ms, label }) => `<option value="${ms}"${ms === selectedMs ? " selected" : ""}>${label}</option>`,
).join("");
}
type ModelStatus = { url: string; already_imported: boolean; print_id: string | null };
export async function loadMakerworldGuidedCollection(): Promise<void> {
@@ -78,10 +128,10 @@ export async function loadMakerworldGuidedCollection(): Promise<void> {
const root = getShadowRoot();
if (!root) return;
const scan = mountScanOverlay(root);
await scrollToEnd((count) => scan.update(count));
const total = await loadMakerworldCollectionTotal(location.href);
const ids = await scrollToEnd(total, (count, expected) => scan.update(count, expected));
scan.remove();
const ids = extractModelIds();
if (!ids.length) {
renderPanel(
errorHtml(new Error("Couldn't find any models on this page -- MakerWorld may have changed its page layout.")),
@@ -110,6 +160,8 @@ export async function loadMakerworldGuidedCollection(): Promise<void> {
renderPanel(`
<div class="tg-title">${escapeHtml(collectionTitle || "Import collection")}</div>
<div class="tg-hint">${parts.join(", ") || "No models found."}</div>
<label class="tg-label" for="tg-step-delay">Wait between models</label>
<select id="tg-step-delay" class="tg-select">${stepDelayOptions(await rememberedStepDelay())}</select>
<button class="tg-btn" type="button" data-action="start">Start import</button>
<div class="tg-hint tg-hint--spaced">
${collectionTitle ? `Models go into a Thingport collection named "${escapeHtml(collectionTitle)}" (created if it doesn't exist yet). ` : ""}New
@@ -117,13 +169,18 @@ export async function loadMakerworldGuidedCollection(): Promise<void> {
limiting -- this can take a while for a large collection.
</div>
`);
onPanelAction("start", () => void startGuidedImport(toImport, alreadyImported, collectionTitle));
onPanelAction("start", () => {
const stepDelayMs = Number(panelQuery<HTMLSelectElement>("#tg-step-delay")?.value) || STEP_DELAY_CHOICES[0].ms;
void rememberStepDelay(stepDelayMs);
void startGuidedImport(toImport, alreadyImported, collectionTitle, stepDelayMs);
});
}
async function startGuidedImport(
toImport: ModelStatus[],
alreadyImported: ModelStatus[],
collectionTitle: string | null,
stepDelayMs: number,
): Promise<void> {
const { url: originalUrl, instanceUrl } = ctx();
renderPanel(statusHtml("Preparing your collection…"));
@@ -153,5 +210,10 @@ async function startGuidedImport(
}
renderPanel(statusHtml("Starting guided import…"));
await request("START_MAKERWORLD_COLLECTION_JOB", { urls: toImport.map((e) => e.url), collectionId, originalUrl });
await request("START_MAKERWORLD_COLLECTION_JOB", {
urls: toImport.map((e) => e.url),
collectionId,
originalUrl,
stepDelayMs,
});
}
+19 -1
View File
@@ -39,9 +39,27 @@
font-weight: 700;
}
&__count {
&__count,
&__text {
color: t.c("text-muted");
}
&__text {
font-size: 13px;
line-height: 1.4;
}
// .tg-btn's `display` outranks the browser's `[hidden] { display: none }`, which kept "Import next"
// showing from the start instead of after JOB_STUCK_REVEAL_MS.
[hidden] {
display: none;
}
// Leaves the page usable underneath (see mountPausedJobOverlay).
&--docked {
inset: auto 16px 16px auto;
background: none;
}
}
.tg-progress {
+11 -1
View File
@@ -52,6 +52,12 @@ export type MakerworldJob = {
imported: number;
total: number;
awaitingLoad: boolean;
/** Wait before each model after the first, so a long run stays under MakerWorld's limits. */
stepDelayMs: number;
/** When the current model's step may start; its page counts down to it. */
startAt: number;
/** MakerWorld wants a CAPTCHA: the run waits on this model until the user resumes it. */
paused: boolean;
};
export type MakerworldJobError = { message: string; imported: number; total: number };
@@ -70,10 +76,12 @@ export type BackgroundMessages = {
IMPORT_SINGLE: { payload: ImportSinglePayload; result: Print | null };
QUEUE_IMPORT: { payload: QueueImportPayload; result: QueueImportResult };
START_MAKERWORLD_COLLECTION_JOB: {
payload: { urls: string[]; collectionId: string | null; originalUrl: string };
payload: { urls: string[]; collectionId: string | null; originalUrl: string; stepDelayMs: number };
result: null;
};
ABORT_MAKERWORLD_COLLECTION_JOB: { payload: void; result: null };
RESUME_MAKERWORLD_JOB: { payload: void; result: null };
MAKERWORLD_JOB_STEP_DUE: { payload: void; result: null };
FORCE_ADVANCE_MAKERWORLD_JOB: { payload: void; result: null };
ARM_DOWNLOAD_CAPTURE: { payload: void; result: null };
AWAIT_DOWNLOAD_CAPTURE: { payload: void; result: string | null };
@@ -82,6 +90,8 @@ export type BackgroundMessages = {
export type ContentMessages = {
RESOLVE_MAKERWORLD_DOWNLOAD_URL: { payload: void; result: ResolvedDownload | null };
/** The guided import paused on this page; it re-renders. */
MAKERWORLD_JOB_UPDATED: { payload: void; result: null };
};
type AnyMessages = Record<string, { payload: unknown; result: unknown }>;