feat(sync): pause syncs while a provider isn't connected, and hand back models blocked by it

This commit is contained in:
Tautvydas Derzinskas committed 2026-10-12 21:15:38 +02:00
1 parent 2d433dff17
commit 6993a4c423
20 files changed
+517 -44

No files matched your search

@@ -0,0 +1 @@
ALTER TABLE "User" ADD COLUMN "makerworldRejectedAt" TIMESTAMP(3);
+2
View File
@@ -43,6 +43,8 @@ model User {
apiTokenLastUsedAt DateTime?
// Per-user, unlike the instance-wide Thingiverse token.
makerworldCookie String?
// When MakerWorld last turned the cookie down. Scheduled MakerWorld syncs wait until it changes.
makerworldRejectedAt DateTime?
// Who the user is on providers without a per-user login, so their collections can be listed for
// syncing. Printables keeps its numeric id (the API's key) with the handle to show.
printablesUserId String?
+8 -4
View File
@@ -35,7 +35,7 @@ import {
setStorageTemplate,
validateStorageTemplate,
} from "../services/printService";
import { getUserMakerworldCookie, setUserMakerworldCookie } from "../services/makerworldCookieService";
import { getMakerworldConnection, setUserMakerworldCookie } from "../services/makerworldCookieService";
import { type MakerworldCookieCheck, verifyMakerworldCookie } from "../services/makerworldCloudApi";
import { verifyThingiverseAccessToken } from "../services/thingiverseApi";
import { SLICER_IDS, getUserSlicer, setUserSlicer } from "../services/slicerPreferenceService";
@@ -363,11 +363,13 @@ router.post(
}),
);
// Per-user, unlike the instance-wide Thingiverse token. GET only reports whether one is set.
// Per-user, unlike the instance-wide Thingiverse token. GET only reports whether one is set, and
// whether MakerWorld has since turned it down.
router.get(
"/settings/makerworld",
asyncHandler(async (req, res) => {
res.json({ configured: Boolean(await getUserMakerworldCookie(req.userId!)) });
const connection = await getMakerworldConnection(req.userId!);
res.json({ configured: connection !== "disconnected", expired: connection === "expired" });
}),
);
@@ -390,6 +392,7 @@ router.patch(
asyncHandler(async (req, res) => {
const body = parseBody(makerworldSettingsSchema, req.body);
const trimmed = (body.cookie ?? "").trim();
let verified = false;
if (body.verify && trimmed) {
const check = await verifyMakerworldCookie(trimmed);
if (check.result === "invalid") {
@@ -400,8 +403,9 @@ router.patch(
}
// 503, not 422: nothing is known to be wrong with the cookie.
if (check.result === "unverifiable") throw new HttpError(503, MAKERWORLD_UNVERIFIABLE_MESSAGES[check.reason]);
verified = true;
}
const configured = await setUserMakerworldCookie(req.userId!, body.cookie);
const configured = await setUserMakerworldCookie(req.userId!, body.cookie, { verified });
res.json({ configured });
}),
);
+1 -1
View File
@@ -167,7 +167,7 @@ async function lookUpAuthor(candidate: Candidate, thingiverseToken: string | nul
return (await resolvePrintablesModel(candidate.externalId))?.meta.author ?? null;
}
case "makerworld": {
const cookie = (await getUserMakerworldCookie(candidate.userId)) ?? process.env.MAKERWORLD_COOKIE ?? null;
const cookie = await getUserMakerworldCookie(candidate.userId);
const bearer = extractMakerworldBearerToken(cookie);
return bearer
? fetchMakerworldDesignAuthor(candidate.externalId, bearer, IMPORT_MAKERWORLD_CALL_DELAY_MS)
+126 -5
View File
@@ -26,9 +26,14 @@ import {
import { getActiveJob } from "./importJobService";
import { runLinksImportJob, SYNC_JOB_INTERRUPTED } from "./importJobRunner";
import { startLinksJob } from "./importQueueService";
import { extractMakerworldBearerToken } from "./makerworldCloudApi";
import { extractMakerworldBearerToken, MakerworldAuthError } from "./makerworldCloudApi";
import { checkMakerworldCollection, fetchMakerworldCollectionEntries } from "./makerworldCollections";
import { getUserMakerworldCookie } from "./makerworldCookieService";
import {
getMakerworldConnection,
getUserMakerworldCookie,
MAKERWORLD_REJECTED_ERROR,
markMakerworldCookieRejected,
} from "./makerworldCookieService";
import { checkPrintablesCollection, fetchPrintablesCollectionEntries } from "./printablesApi";
import {
checkThingiverseCollection,
@@ -96,6 +101,7 @@ export type SyncOutcome =
| "busy" // an import of the user's is running; tried again next time
| "resumed" // a job a restart cut off was picked up again
| "error" // the provider couldn't be read
| "disconnected" // the provider isn't connected, or turned the login down; not checked till it is
| "gone" // the provider's collection wasn't found, not yet often enough to unsync
| "unsynced"
| "unchanged"
@@ -128,10 +134,95 @@ export function syncCollection(syncId: string, { reserved = false } = {}): Promi
return run;
}
type ConnectionProblem = { error: string; reason: string; fix: string };
const CONNECTION_PROBLEMS = {
makerworldDisconnected: {
error: "MakerWorld isn't connected",
reason: "MakerWorld isn't connected.",
fix: "you connect it again",
},
makerworldExpired: {
error: MAKERWORLD_REJECTED_ERROR,
reason: "MakerWorld turned your sign-in down; it may have expired.",
fix: "you connect MakerWorld again",
},
thingiverseDisconnected: {
error: "Thingiverse isn't connected on this Thingport",
reason: "Thingiverse isn't connected on this Thingport.",
fix: "an admin connects it again",
},
} satisfies Record<string, ConnectionProblem>;
/** `"A"`, `"A" and "B"`, or the first few and how many more. */
function listNames(names: string[]): string {
const quoted = names.map((name) => `"${name}"`);
if (quoted.length === 1) return quoted[0];
if (quoted.length <= 3) return `${quoted.slice(0, -1).join(", ")} and ${quoted[quoted.length - 1]}`;
return `${quoted.slice(0, 2).join(", ")} and ${quoted.length - 2} more`;
}
/** One notification for every collection the problem holds. */
async function notifyHeld(
userId: string,
provider: string,
problem: ConnectionProblem,
syncs: SyncWithCollection[],
): Promise<void> {
if (!syncs.length) return;
const one = syncs.length === 1 ? syncs[0] : null;
await createNotification(userId, {
type: "sync",
title: one
? `Couldn't sync "${one.collection.name}"`
: `Couldn't sync ${syncs.length} ${providerName(provider)} collections`,
body: `${problem.reason} ${listNames(syncs.map((sync) => sync.collection.name))} won't sync until ${problem.fix} in Configuration → Providers. ${one ? "It stays synced, and its models stay." : "They stay synced, and their models stay."}`,
externalUrl: one?.sourceUrl ?? null,
internalPath: one ? `/models/collections/${one.collectionId}` : "/models/collections",
});
}
/** Why the provider can't be reached for this sync right now, or null when it can. Checked before
* any provider call, so a sync with nothing to sign in with waits instead of failing every round. */
async function connectionProblem(sync: Pick<CollectionSync, "userId" | "provider">): Promise<ConnectionProblem | null> {
if (sync.provider === "makerworld") {
const connection = await getMakerworldConnection(sync.userId);
if (connection === "disconnected") return CONNECTION_PROBLEMS.makerworldDisconnected;
if (connection === "expired") return CONNECTION_PROBLEMS.makerworldExpired;
}
if (sync.provider === "thingiverse" && !(await getThingiverseAccessToken())) {
return CONNECTION_PROBLEMS.thingiverseDisconnected;
}
return null;
}
/** Holds every one of the user's syncs with this provider at once, with one notification for the
* ones not already held; later rounds find the error saved and stay quiet. Last check times are
* left alone, so they're due again as soon as the provider is connected. */
async function holdProvider(userId: string, provider: string, problem: ConnectionProblem): Promise<void> {
const fresh = await prisma.collectionSync.findMany({
where: { userId, provider, OR: [{ lastError: null }, { lastError: { not: problem.error } }] },
include: { collection: true },
orderBy: { createdAt: "asc" },
});
if (!fresh.length) return;
await prisma.collectionSync.updateMany({
where: { id: { in: fresh.map((sync) => sync.id) } },
data: { lastError: problem.error },
});
console.warn(`[sync] ${fresh.length} ${provider} collection(s) of user ${userId} held: ${problem.error}`);
await notifyHeld(userId, provider, problem, fresh);
}
async function runSync(syncId: string): Promise<SyncOutcome> {
const sync = await prisma.collectionSync.findUnique({ where: { id: syncId }, include: { collection: true } });
if (!sync) return "missing";
const { userId } = sync;
const problem = await connectionProblem(sync);
if (problem) {
await holdProvider(userId, sync.provider, problem);
return "disconnected";
}
let openJob = await prisma.importJob.findFirst({
where: { collectionSyncId: sync.id, status: { in: ["RUNNING", "PAUSED"] } },
@@ -157,6 +248,11 @@ async function runSync(syncId: string): Promise<SyncOutcome> {
try {
listing = await fetchListing(sync);
} catch (err) {
if (err instanceof MakerworldAuthError) {
await markMakerworldCookieRejected(userId);
await holdProvider(userId, sync.provider, CONNECTION_PROBLEMS.makerworldExpired);
return "disconnected";
}
const message = err instanceof Error && err.message.trim() ? err.message.trim() : "Sync failed";
console.warn(`[sync] "${sync.collection.name}" (${sync.provider} ${sync.externalId}): ${message}`);
await prisma.collectionSync
@@ -336,6 +432,12 @@ export async function runManualSync(syncId: string, { reserved = false } = {}):
/** The answer to "Sync now" for the outcomes that send no notification of their own. */
async function notifyManualOutcome(before: SyncWithCollection, outcome: SyncOutcome): Promise<void> {
const after = await prisma.collectionSync.findUnique({ where: { id: before.id } });
// holdProvider only notifies the first time; a click after that still gets an answer.
if (outcome === "disconnected" && after?.lastError === before.lastError) {
const problem = Object.values(CONNECTION_PROBLEMS).find((p) => p.error === after?.lastError);
if (problem) await notifyHeld(before.userId, before.provider, problem, [before]);
return;
}
const name = before.collection.name;
const provider = providerName(before.provider);
const message: Partial<Record<SyncOutcome, { title: string; body: string }>> = {
@@ -372,10 +474,29 @@ export function runSyncRound(): Promise<void> {
currentRound = (async () => {
const syncs = await prisma.collectionSync.findMany({
orderBy: { lastCheckedAt: { sort: "asc", nulls: "first" } },
select: { id: true, lastCheckedAt: true, intervalHours: true },
select: { id: true, userId: true, provider: true, lastCheckedAt: true, intervalHours: true },
});
const due = syncs.filter((sync) => isSyncDue(sync));
for (const [index, { id }] of due.entries()) {
const due: string[] = [];
// Each user's connection to each provider, looked up once a round.
const problems = new Map<string, ConnectionProblem | null>();
for (const sync of syncs.filter((s) => isSyncDue(s))) {
const key = `${sync.userId} ${sync.provider}`;
if (problems.has(key)) {
if (!problems.get(key)) due.push(sync.id);
continue;
}
const problem = await connectionProblem(sync);
problems.set(key, problem);
if (!problem) {
due.push(sync.id);
continue;
}
// Held here rather than in the lane: there's no provider call to pace.
await holdProvider(sync.userId, sync.provider, problem).catch((err) =>
console.error(`[sync] holding ${key} failed:`, err),
);
}
for (const [index, id] of due.entries()) {
if (index > 0) await sleep(IMPORT_COLLECTION_DELAY_MS);
try {
const outcome = await syncCollection(id);
@@ -97,6 +97,13 @@ export function modelUrlFor(provider: string, externalId: string): string {
return `https://www.thingiverse.com/thing:${externalId}`;
}
/** The model id in a link modelUrlFor made. */
export function modelIdFromUrl(provider: string, url: string): string | null {
const pattern =
provider === "makerworld" ? /\/models\/(\d+)/ : provider === "printables" ? /\/model\/(\d+)/ : /thing:(\d+)/;
return url.match(pattern)?.[1] ?? null;
}
/** What Thingport Grab reads on a provider's collection page. */
export type SyncLinkOut = CollectionSyncOut & { collection_id: string; collection_name: string };
+3
View File
@@ -36,6 +36,9 @@ export async function testFlaresolverr(
}
}
// Cloudflare still said no after FlareSolverr, or with none set up: nothing to do with the link.
export const CLOUDFLARE_BLOCKED_CODE = "CLOUDFLARE_BLOCKED";
/** A Cloudflare edge block rather than an application-level 403. */
export function looksLikeCloudflareBlock(headers: Headers): boolean {
const server = (headers.get("server") || "").toLowerCase();
+104 -8
View File
@@ -5,7 +5,7 @@ import { IMPORT_COLLECTION_DELAY_MS, IMPORT_MAKERWORLD_CALL_DELAY_MS } from "../
import { countJobItems, getJob, getJobStatus, listJobItems, updateJob, updateJobItem } from "./importJobService";
import { createNotification } from "./notificationService";
import { addPrintsToCollection, findOrCreateCollectionByName } from "./collectionService";
import { likesCollectionName, providerName, syncedCollectionFor } from "./collectionSyncService";
import { likesCollectionName, modelIdFromUrl, providerName, syncedCollectionFor } from "./collectionSyncService";
import { resolveMakerworldCookie } from "./importResolvers";
import { decodeHtmlEntities } from "../utils/htmlEntities";
import {
@@ -13,6 +13,8 @@ import {
parseMakerworldModelUrl,
selectMakerworldProfiles,
type MakerworldProfileScope,
MAKERWORLD_AUTH_CODE,
MakerworldAuthError,
MAKERWORLD_DOWNLOAD_LIMIT_CODE,
MAKERWORLD_DOWNLOAD_LIMIT_MESSAGE,
} from "./makerworldCloudApi";
@@ -29,6 +31,8 @@ import { fetchThingiverseCollectionTitle } from "./thingiverseApi";
import { getThingiverseAccessToken } from "./settingsService";
import { fetchPrintablesCollectionTitle } from "./printablesApi";
import { createLog } from "./auditLog";
import { MAKERWORLD_REJECTED_ERROR, markMakerworldCookieRejected } from "./makerworldCookieService";
import { CLOUDFLARE_BLOCKED_CODE } from "./flaresolverr";
import { HttpError } from "../utils/fileUtils";
import { prisma } from "../db";
import type { ImportJob } from "@prisma/client";
@@ -48,13 +52,19 @@ type PrintablesCollectionImportJobBody = ImportRequestBody & {
type MakerworldProfilesImportJobBody = ImportRequestBody & { scope: Exclude<MakerworldProfileScope, "url"> };
// Why a design failed, so a batch reads as one clear cause instead of "N failed". Once a CAPTCHA
// ("rateLimited"), the daily download limit or an auth failure hits, every remaining item fails
// the same way.
type ImportFailureReason = "unavailable" | "rateLimited" | "downloadLimit" | "auth" | "other";
type StopReason = "rateLimited" | "downloadLimit" | "auth";
// ("rateLimited"), the daily download limit, an auth failure or a Cloudflare block ("blocked")
// hits, every remaining item fails the same way.
type ImportFailureReason = "unavailable" | "rateLimited" | "downloadLimit" | "auth" | "blocked" | "other";
type StopReason = "rateLimited" | "downloadLimit" | "auth" | "blocked";
const BLOCKED_RETRY_TEXT =
"Cloudflare blocked the request and FlareSolverr didn't get past it — check FlareSolverr in Administration → Connections, then retry";
function classifyImportFailure(err: unknown): ImportFailureReason {
if (err instanceof MakerworldAuthError) return "auth";
if (err instanceof HttpError) {
if (err.code === MAKERWORLD_AUTH_CODE) return "auth";
if (err.code === CLOUDFLARE_BLOCKED_CODE) return "blocked";
if (err.status === 403 || err.status === 404) return "unavailable";
if (err.status === 429) return err.code === MAKERWORLD_DOWNLOAD_LIMIT_CODE ? "downloadLimit" : "rateLimited";
if (err.status === 401) return "auth";
@@ -63,7 +73,9 @@ function classifyImportFailure(err: unknown): ImportFailureReason {
}
function asStopReason(reason: ImportFailureReason): StopReason | null {
return reason === "rateLimited" || reason === "downloadLimit" || reason === "auth" ? reason : null;
return reason === "rateLimited" || reason === "downloadLimit" || reason === "auth" || reason === "blocked"
? reason
: null;
}
async function markJobFailed(jobId: string, err: unknown): Promise<void> {
@@ -172,6 +184,7 @@ export async function runMakerworldProfilesImportJob(
details: { provider: "makerworld", sourceLabel: title, imported, alreadyInLibrary, failed },
});
if (stopReason === "auth") await noteMakerworldRejected(userId);
const bodyParts: string[] = [];
if (alreadyInLibrary) bodyParts.push(`${alreadyInLibrary} already on the model`);
if (stopReason === "rateLimited") {
@@ -301,6 +314,7 @@ export async function runCollectionImportJob(
},
});
if (authFailed) await noteMakerworldRejected(userId);
const label = collectionTitle ? `"${collectionTitle}"` : "a MakerWorld collection";
const bodyParts: string[] = [];
if (alreadyInLibrary) bodyParts.push(`${alreadyInLibrary} already in your library`);
@@ -667,6 +681,7 @@ const LINKS_MAX_ATTEMPTS = 3;
const LINKS_RETRY_DELAY_MS = 2000;
function isTransientImportFailure(err: unknown): boolean {
if (err instanceof MakerworldAuthError) return false;
return !(err instanceof HttpError) || err.transient === true;
}
@@ -742,6 +757,10 @@ async function runLinks(jobId: string, userId: string, body: LinksImportJobBody)
let profilesProcessed = 0;
let printId: string | null = null;
let stopReason: StopReason | null = null;
// The link the stop reason came from, so only a MakerWorld one marks its cookie rejected.
let stopUrl: string | null = null;
// A sync's links handed back to it by deferSyncRest.
let deferred: SyncDeferral | null = null;
let processed = (await countJobItems(jobId, "DONE")) + (await countJobItems(jobId, "FAILED"));
// The models this run brought in new, for a sync's notification.
const newPrintIds: string[] = [];
@@ -768,6 +787,7 @@ async function runLinks(jobId: string, userId: string, body: LinksImportJobBody)
rateLimited: "blocked by a MakerWorld CAPTCHA challenge — retry once it clears (usually 1-4 hours)",
downloadLimit: "MakerWorld's daily download limit was reached — retry tomorrow",
auth: "the MakerWorld session expired — update the cookie in Settings, then retry",
blocked: BLOCKED_RETRY_TEXT,
}[stopReason];
settled = true;
break;
@@ -797,6 +817,7 @@ async function runLinks(jobId: string, userId: string, body: LinksImportJobBody)
rateLimited: " (MakerWorld CAPTCHA challenge — retry once it clears)",
downloadLimit: " (MakerWorld's daily download limit — retry tomorrow)",
auth: " (the MakerWorld session expired — update the cookie in Settings, then retry)",
blocked: ` (${BLOCKED_RETRY_TEXT})`,
}[result.stopReason];
}
settled = true;
@@ -819,6 +840,18 @@ async function runLinks(jobId: string, userId: string, body: LinksImportJobBody)
}
if (itemError) console.error(`[import] job ${jobId}: ${item.url} failed: ${itemError}`);
else console.log(`[import] job ${jobId}: ${item.url} imported`);
if (stopReason && !stopUrl) stopUrl = item.url;
// The connection failed, not the models: the rest go back to unseen, for the next sync to find
// again once it works. A link that got some of its profiles in is done.
if (job.collectionSyncId && (stopReason === "auth" || stopReason === "blocked")) {
const nothingIn = itemError !== null && !partlyIn;
if (!nothingIn) {
await updateJobItem(item.id, { status: "FAILED", attempts, errorMessage: itemError }).catch(() => undefined);
processed++;
}
deferred = { count: await deferSyncRest(job, nothingIn ? item.id : null, stopReason), reason: stopReason };
break;
}
// A sync can't ask anyone to retry, so a rate limit pauses it instead of failing the rest. A
// link that got nothing in waits with them; one that got some of its profiles in is done.
if (job.collectionSyncId && stopReason) {
@@ -864,7 +897,10 @@ async function runLinks(jobId: string, userId: string, body: LinksImportJobBody)
details: { provider: "links", imported, alreadyInLibrary, failed: failedCount },
});
if (job.collectionSyncId && (await notifySyncJobDone(job, newPrintIds, alreadyInLibrary, failedCount))) return;
if (stopReason === "auth" && stopUrl && parseMakerworldModelUrl(stopUrl)) await noteMakerworldRejected(userId);
if (job.collectionSyncId && (await notifySyncJobDone(job, newPrintIds, alreadyInLibrary, failedCount, deferred))) {
return;
}
const bodyParts: string[] = [];
if (profilesProcessed) bodyParts.push(`${profilesImported} of ${profilesProcessed} print profiles imported`);
@@ -881,6 +917,8 @@ async function runLinks(jobId: string, userId: string, body: LinksImportJobBody)
bodyParts.push(
`the rest failed because your MakerWorld session expired — update the cookie in Settings, then retry`,
);
} else if (stopReason === "blocked") {
bodyParts.push(`the rest failed: ${BLOCKED_RETRY_TEXT}`);
} else if (failedCount) {
bodyParts.push(`${failedCount} failed — retry them from the import's progress bar`);
}
@@ -926,6 +964,7 @@ async function modelNames(printIds: string[]): Promise<string> {
function syncStopText(reason: StopReason, provider: string): string {
if (reason === "downloadLimit") return "MakerWorld's daily download limit was reached";
if (reason === "blocked") return `Cloudflare blocked ${providerName(provider)} and FlareSolverr didn't get past it`;
if (reason === "auth") {
return provider === "makerworld"
? "your MakerWorld session expired, so connect MakerWorld again in Configuration > Providers"
@@ -971,12 +1010,59 @@ async function pauseSyncJob(job: ImportJob, reason: StopReason, newPrintIds: str
});
}
/** Scheduled MakerWorld syncs then wait for a new cookie instead of meeting the same refusal. */
async function noteMakerworldRejected(userId: string): Promise<void> {
await markMakerworldCookieRejected(userId).catch((err) => console.error("[import] couldn't mark cookie:", err));
}
type SyncDeferral = { count: number; reason: "auth" | "blocked" };
/** Takes the sync job's links that haven't run (and `currentItemId`'s, which got nothing in) out of
* the job, and out of the sync's seen ids, so its next check queues them again. How many. */
async function deferSyncRest(job: ImportJob, currentItemId: string | null, reason: SyncDeferral["reason"]) {
// One transaction: links gone from the job but still seen by the sync would never come in.
const handedBack = await prisma.$transaction(async (tx) => {
const items = await tx.importJobItem.findMany({
where: { jobId: job.id, OR: [{ status: "PENDING" }, ...(currentItemId ? [{ id: currentItemId }] : [])] },
select: { id: true, url: true },
});
await tx.importJobItem.deleteMany({ where: { id: { in: items.map((item) => item.id) } } });
await tx.importJob.update({
where: { id: job.id },
data: { total: await tx.importJobItem.count({ where: { jobId: job.id } }) },
});
const sync = job.collectionSyncId
? await tx.collectionSync.findUnique({ where: { id: job.collectionSyncId } })
: null;
if (sync) {
const unseen = new Set(items.flatMap((item) => modelIdFromUrl(sync.provider, item.url) ?? []));
await tx.collectionSync.update({
where: { id: sync.id },
data: {
knownIds: sync.knownIds.filter((id) => !unseen.has(id)),
// For MakerWorld, already held, so holdProvider leaves it out of the notification about the others.
lastError:
reason === "blocked"
? `Cloudflare blocked ${providerName(sync.provider)} and FlareSolverr didn't get past it`
: sync.provider === "makerworld"
? MAKERWORLD_REJECTED_ERROR
: `${providerName(sync.provider)} turned Thingport's sign-in down`,
},
});
}
return items;
});
console.warn(`[sync] job ${job.id}: ${handedBack.length} links handed back to the sync (${reason})`);
return handedBack.length;
}
/** Says which new models came in. False when the sync is gone, for the plain notification. */
async function notifySyncJobDone(
job: ImportJob,
newPrintIds: string[],
alreadyInLibrary: number,
failedCount: number,
deferred: SyncDeferral | null,
): Promise<boolean> {
const sync = await syncOfJob(job);
if (!sync) return false;
@@ -990,6 +1076,16 @@ async function notifySyncJobDone(
`${filed} model${filed === 1 ? " that was" : "s that were"} already in your library ${filed === 1 ? "was" : "were"} added to it too.`,
);
}
if (deferred) {
const more = `${deferred.count} more model${deferred.count === 1 ? "" : "s"}`;
parts.push(
deferred.reason === "auth" && sync.provider === "makerworld"
? `${more} will come in on the next sync once you connect MakerWorld again in Configuration → Providers: MakerWorld turned your sign-in down, and it may have expired.`
: deferred.reason === "auth"
? `${more} will come in on a later sync: ${providerName(sync.provider)} turned Thingport's sign-in down.`
: `${more} will come in on a later sync: Cloudflare blocked ${providerName(sync.provider)} and FlareSolverr didn't get past it. An admin can check FlareSolverr in Administration → Connections.`,
);
}
if (failedCount) {
parts.push(
`${failedCount} couldn't be imported (private, deleted, or failed); retry ${failedCount === 1 ? "it" : "them"} from the import queue.`,
@@ -1000,7 +1096,7 @@ async function notifySyncJobDone(
type: "sync",
title: count
? `${count} new model${count === 1 ? "" : "s"} synced into "${name}"`
: `Couldn't sync new models into "${name}"`,
: `Couldn't sync new models into "${name}"${deferred ? " yet" : ""}`,
body: parts.join(" ") || null,
externalUrl: sync.sourceUrl,
internalPath: `/models/collections/${sync.collectionId}`,
+1 -1
View File
@@ -46,7 +46,7 @@ function firstNonEmptyLine(raw: string): string | null {
}
export function resolveMakerworldCookie(body: ImportCookies): string | null {
const raw = (body.makerworld_cookie || process.env.MAKERWORLD_COOKIE || "").trim();
const raw = (body.makerworld_cookie || "").trim();
if (!raw) return null;
return firstNonEmptyLine(raw);
}
+18 -3
View File
@@ -22,7 +22,12 @@ import {
import { validateRemoteUrl } from "../utils/urlUtils";
import { readCapped } from "../utils/readCapped";
import { maybeSleep, sleep } from "../utils/concurrency";
import { fetchViaFlaresolverr, looksLikeCloudflareBlock, shouldProxyHost } from "./flaresolverr";
import {
CLOUDFLARE_BLOCKED_CODE,
fetchViaFlaresolverr,
looksLikeCloudflareBlock,
shouldProxyHost,
} from "./flaresolverr";
import {
emptyImportedPageMetadata,
extractNextDataJson,
@@ -38,6 +43,7 @@ import {
} from "./importResolvers";
import {
extractMakerworldBearerToken,
MAKERWORLD_AUTH_CODE,
MakerworldAuthError,
MakerworldCaptchaError,
makerworldCaptchaCooloffActive,
@@ -146,6 +152,13 @@ async function fetchWithGuard(url: string, headers: Record<string, string>): Pro
}
}
if (res.status === 403 && looksLikeCloudflareBlock(res.headers)) {
throw new HttpError(
403,
"Cloudflare blocked the request and FlareSolverr didn't get past it",
CLOUDFLARE_BLOCKED_CODE,
);
}
if (!res.ok) {
throw new HttpError(res.status || 400, `Failed to fetch URL: ${res.statusText || res.status}`);
}
@@ -174,7 +187,9 @@ async function fetchMakerworldPageDesign(
Accept: "*/*",
...makerworldHtmlHeaders(url, cookie),
});
} catch {
} catch (err) {
// Kept apart from "couldn't read the model", so a sync hands the model back for later.
if (err instanceof HttpError && err.code === CLOUDFLARE_BLOCKED_CODE) throw err;
return null;
}
if (!isHtmlContentType(res.headers.get("content-type") || "")) return null;
@@ -273,7 +288,7 @@ async function tryMakerworldCloudApi(url: string, body: ImportRequestBody): Prom
throw new HttpError(429, err.message, MAKERWORLD_DOWNLOAD_LIMIT_CODE);
}
// 400, not 401: the frontend treats any 401 as an expired Thingport session and logs out.
if (err instanceof MakerworldAuthError) throw new HttpError(400, err.message);
if (err instanceof MakerworldAuthError) throw new HttpError(400, err.message, MAKERWORLD_AUTH_CODE);
throw err;
}
}
@@ -55,6 +55,8 @@ export const MAKERWORLD_DOWNLOAD_LIMIT_MESSAGE =
// The HttpError code, so batch runners can tell it from a CAPTCHA (both are 429).
export const MAKERWORLD_DOWNLOAD_LIMIT_CODE = "MAKERWORLD_DOWNLOAD_LIMIT";
// On the 400 a MakerworldAuthError becomes, so a queued import can tell it from a bad link.
export const MAKERWORLD_AUTH_CODE = "MAKERWORLD_AUTH";
export class MakerworldDownloadLimitError extends Error {
constructor() {
@@ -26,6 +26,7 @@ import {
export {
MAKERWORLD_CAPTCHA_MESSAGE,
makerworldCaptchaCooloffActive,
MAKERWORLD_AUTH_CODE,
MakerworldAuthError,
MakerworldCaptchaError,
MAKERWORLD_DOWNLOAD_LIMIT_CODE,
@@ -84,8 +84,9 @@ async function fetchCollectionJson(
redirect: "follow",
signal: controller.signal,
});
// Checked separately, unlike the other callers: without FlareSolverr a 403 falls through to MakerworldAuthError.
if (res.status === 403 && looksLikeCloudflareBlock(res.headers) && (await isFlaresolverrEnabled())) {
// Cloudflare's 403 isn't MakerWorld turning the login down, which would pause the user's syncs.
if (res.status === 403 && looksLikeCloudflareBlock(res.headers)) {
if (!(await isFlaresolverrEnabled())) throw new MakerworldCollectionFetchError("blocked by Cloudflare");
const solved = await fetchViaFlaresolverr(url, bearerToken ? `token=${bearerToken}` : null);
if (!solved) return fail("blocked by Cloudflare");
const data = extractJsonFromBrowserBody(solved.body);
@@ -7,8 +7,39 @@ export async function getUserMakerworldCookie(userId: string): Promise<string |
return user?.makerworldCookie ?? null;
}
export async function setUserMakerworldCookie(userId: string, cookie: string | null): Promise<boolean> {
const trimmed = (cookie ?? "").trim();
await prisma.user.update({ where: { id: userId }, data: { makerworldCookie: trimmed || null } });
/** `verified`: MakerWorld just accepted it, so an earlier rejection no longer stands even if it's
* the same cookie. A different cookie clears it too. */
export async function setUserMakerworldCookie(
userId: string,
cookie: string | null,
{ verified = false } = {},
): Promise<boolean> {
const trimmed = (cookie ?? "").trim() || null;
const current = await getUserMakerworldCookie(userId);
await prisma.user.update({
where: { id: userId },
data: {
makerworldCookie: trimmed,
...(verified || trimmed !== current ? { makerworldRejectedAt: null } : {}),
},
});
return Boolean(trimmed);
}
// A synced collection's error while its MakerWorld syncs wait for a new cookie.
export const MAKERWORLD_REJECTED_ERROR = "Your MakerWorld sign-in has expired";
export async function markMakerworldCookieRejected(userId: string): Promise<void> {
await prisma.user.update({ where: { id: userId }, data: { makerworldRejectedAt: new Date() } });
}
export type MakerworldConnection = "connected" | "disconnected" | "expired";
export async function getMakerworldConnection(userId: string): Promise<MakerworldConnection> {
const user = await prisma.user.findUnique({
where: { id: userId },
select: { makerworldCookie: true, makerworldRejectedAt: true },
});
if (!user?.makerworldCookie) return "disconnected";
return user.makerworldRejectedAt ? "expired" : "connected";
}
+156 -1
View File
@@ -63,6 +63,10 @@ function design(designId: string) {
const collections = new Map<string, string[] | "gone">();
/** Designs whose download hits MakerWorld's daily limit. */
const limited = new Set<string>();
/** Designs MakerWorld answers with 401, as with an expired sign-in. */
const signInRejected = new Set<string>();
/** Designs Cloudflare stands in front of, with no FlareSolverr to get past it. */
const cloudflared = new Set<string>();
function mockFetch() {
global.fetch = vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async (input) => {
@@ -82,7 +86,18 @@ function mockFetch() {
return json({ id: Number(head[1]), title: "Phoenix flock" });
}
const model = url.match(/^https:\/\/api\.bambulab\.com\/v1\/design-service\/design\/(\d+)/);
if (model) return json(design(model[1]));
if (model) {
if (signInRejected.has(model[1])) return json({ error: "unauthorized" }, 401);
if (cloudflared.has(model[1])) return json({ error: "unavailable" }, 500);
return json(design(model[1]));
}
const page = url.match(/^https:\/\/makerworld\.com\/en\/models\/(\d+)/);
if (page && cloudflared.has(page[1])) {
return new Response("Just a moment...", {
status: 403,
headers: { server: "cloudflare", "cf-mitigated": "challenge" },
});
}
const profile = url.match(
/^https:\/\/api\.bambulab\.com\/v1\/iot-service\/api\/user\/profile\/(\d+)\?model_id=US(\d+)/,
);
@@ -138,6 +153,8 @@ describe("synced collections", () => {
global.fetch = originalFetch;
mockFetch();
limited.clear();
signInRejected.clear();
cloudflared.clear();
// A finished job mustn't hold the one-running-import lock for the next test.
await prisma.importJob.updateMany({ where: { userId, status: "RUNNING" }, data: { status: "DONE" } });
});
@@ -705,6 +722,144 @@ describe("synced collections", () => {
expect(after.lastError).toMatch(/HTTP 502/);
});
it("waits without unsyncing while MakerWorld isn't connected, telling the user once for all", async () => {
const [one, two] = [newId(), newId()];
collections.set(one, ["1"]);
collections.set(two, ["1"]);
const first = await enable({ url: collectionUrl(one), title: `Flock ${one}`, known_ids: [] });
const second = await enable({ url: collectionUrl(two), title: `Flock ${two}`, known_ids: [] });
const sync = await syncOf(first.body.collection_id);
await request(app).patch("/api/settings/makerworld").set(auth()).send({ cookie: null });
try {
const calls = vi.mocked(global.fetch).mock.calls.length;
const before = await prisma.notification.count({ where: { userId } });
expect(await syncCollection(sync.id)).toBe("disconnected");
expect(await syncCollection((await syncOf(second.body.collection_id)).id)).toBe("disconnected");
expect(vi.mocked(global.fetch).mock.calls.length).toBe(calls);
for (const enabled of [first, second]) {
const held = await syncOf(enabled.body.collection_id);
expect(held.lastError).toBe("MakerWorld isn't connected");
expect(held.lastCheckedAt).toBeNull();
}
// Every MakerWorld sync of the user's, in one notification.
expect(await prisma.notification.count({ where: { userId } })).toBe(before + 1);
const note = await latestNotification();
expect(note.title).toMatch(/^Couldn't sync \d+ MakerWorld collections$/);
expect(note.body).toMatch(/^MakerWorld isn't connected\. .+ won't sync until you connect it again/);
// Sync now still answers, for its own collection.
expect(await runManualSync(sync.id)).toBe("disconnected");
expect(await prisma.notification.count({ where: { userId } })).toBe(before + 2);
expect((await latestNotification()).title).toBe(`Couldn't sync "Flock ${one}"`);
} finally {
await request(app).patch("/api/settings/makerworld").set(auth()).send({ cookie: "token=test-bearer" });
}
expect(await syncCollection(sync.id)).not.toBe("disconnected");
expect((await syncOf(first.body.collection_id)).lastError).toBeNull();
});
it("holds MakerWorld syncs once the sign-in is turned down, until the cookie changes", async () => {
const cid = newId();
collections.set(cid, ["1"]);
const enabled = await enable({ url: collectionUrl(cid), title: `Flock ${cid}`, known_ids: [] });
const sync = await syncOf(enabled.body.collection_id);
global.fetch = vi.fn<() => Promise<Response>>(async () =>
json({ error: "unauthorized" }, 401),
) as unknown as typeof fetch;
try {
expect(await syncCollection(sync.id)).toBe("disconnected");
expect(await prisma.collectionSync.findUnique({ where: { id: sync.id } })).not.toBeNull();
expect((await syncOf(enabled.body.collection_id)).lastError).toBe("Your MakerWorld sign-in has expired");
const settings = await request(app).get("/api/settings/makerworld").set(auth());
expect(settings.body).toEqual({ configured: true, expired: true });
const calls = vi.mocked(global.fetch).mock.calls.length;
expect(await syncCollection(sync.id)).toBe("disconnected");
expect(vi.mocked(global.fetch).mock.calls.length).toBe(calls);
// The extension sending the same cookie again doesn't count as fixing it.
await request(app).patch("/api/settings/makerworld").set(auth()).send({ cookie: "token=test-bearer" });
expect(await syncCollection(sync.id)).toBe("disconnected");
} finally {
mockFetch();
await request(app).patch("/api/settings/makerworld").set(auth()).send({ cookie: "token=fresh-bearer" });
}
expect(await syncCollection(sync.id)).not.toBe("disconnected");
const settings = await request(app).get("/api/settings/makerworld").set(auth());
expect(settings.body).toEqual({ configured: true, expired: false });
await prisma.user.update({ where: { id: userId }, data: { makerworldCookie: "token=test-bearer" } });
});
it("hands models back to the sync when the sign-in is turned down mid-import, for after reconnecting", async () => {
const cid = newId();
const [first, second, third] = [newId(), newId(), newId()];
collections.set(cid, []);
const enabled = await enable({ url: collectionUrl(cid), title: `Flock ${cid}`, known_ids: [] });
const collectionId = enabled.body.collection_id;
const sync = await syncOf(collectionId);
collections.set(cid, [first, second, third]);
signInRejected.add(second);
expect(await syncCollection(sync.id)).toBe("queued");
expect(await prisma.print.findFirst({ where: { userId, sourceExternalId: first } })).not.toBeNull();
const after = await syncOf(collectionId);
expect(after.knownIds).toContain(first);
expect(after.knownIds).not.toContain(second);
expect(after.knownIds).not.toContain(third);
expect(after.lastError).toBe("Your MakerWorld sign-in has expired");
// Nothing left in the queue to retry by hand.
const job = await prisma.importJob.findFirstOrThrow({ where: { collectionSyncId: sync.id } });
expect(job.status).toBe("DONE");
expect(await prisma.importJobItem.count({ where: { jobId: job.id, status: { not: "DONE" } } })).toBe(0);
const note = await latestNotification();
expect(note.title).toBe(`1 new model synced into "Flock ${cid}"`);
expect(note.body).toContain("2 more models will come in on the next sync once you connect MakerWorld again");
expect((await request(app).get("/api/settings/makerworld").set(auth())).body.expired).toBe(true);
expect(await syncCollection(sync.id)).toBe("disconnected");
signInRejected.clear();
await request(app).patch("/api/settings/makerworld").set(auth()).send({ cookie: "token=fresh-bearer" });
try {
expect(await syncCollection(sync.id)).toBe("queued");
for (const id of [second, third]) {
expect(await prisma.print.findFirst({ where: { userId, sourceExternalId: id } })).not.toBeNull();
}
expect((await syncOf(collectionId)).knownIds).toEqual(expect.arrayContaining([first, second, third]));
} finally {
await prisma.user.update({ where: { id: userId }, data: { makerworldCookie: "token=test-bearer" } });
}
});
it("hands models back to the sync when Cloudflare blocks them, without holding MakerWorld", async () => {
const cid = newId();
const [first, second] = [newId(), newId()];
collections.set(cid, []);
const enabled = await enable({ url: collectionUrl(cid), title: `Flock ${cid}`, known_ids: [] });
const collectionId = enabled.body.collection_id;
// The model's own profile, whose page fallback is what Cloudflare stands in front of.
const sync = await prisma.collectionSync.update({ where: { collectionId }, data: { profileScope: "url" } });
collections.set(cid, [first, second]);
cloudflared.add(first);
expect(await syncCollection(sync.id)).toBe("queued");
const after = await syncOf(collectionId);
expect(after.knownIds).not.toContain(first);
expect(after.knownIds).not.toContain(second);
expect(after.lastError).toBe("Cloudflare blocked MakerWorld and FlareSolverr didn't get past it");
const job = await prisma.importJob.findFirstOrThrow({ where: { collectionSyncId: sync.id } });
expect(await prisma.importJobItem.count({ where: { jobId: job.id } })).toBe(0);
expect((await latestNotification()).body).toContain(
"2 more models will come in on a later sync: Cloudflare blocked",
);
expect((await request(app).get("/api/settings/makerworld").set(auth())).body.expired).toBe(false);
cloudflared.clear();
expect(await syncCollection(sync.id)).toBe("queued");
expect((await syncOf(collectionId)).knownIds).toEqual(expect.arrayContaining([first, second]));
expect((await syncOf(collectionId)).lastError).toBeNull();
});
it("is removed with its collection, and can be turned off without touching the collection", async () => {
const one = await enable({ url: collectionUrl(newId()), title: `Gone ${stamp}`, known_ids: [] });
await request(app).delete(`/api/collection/${one.body.collection_id}`).set(auth());
+2 -1
View File
@@ -342,7 +342,8 @@ export const settingsApi = {
return res.json();
},
getMakerworld: async (): Promise<{ configured: boolean }> => {
// `expired`: MakerWorld turned the saved cookie down; scheduled syncs wait for a new one.
getMakerworld: async (): Promise<{ configured: boolean; expired: boolean }> => {
const res = await fetch(`${apiBase()}/settings/makerworld`, { headers: authHeaders() });
assertOk(res, "Failed to load MakerWorld settings");
return res.json();
@@ -50,6 +50,7 @@ export default function ProvidersPanel({ isAdmin, cookie, onUpdateMakerWorld, on
thangs: false,
});
const [menu, setMenu] = React.useState<{ id: ProviderId; top: number; left: number } | null>(null);
const [makerworldExpired, setMakerworldExpired] = React.useState(false);
const [connecting, setConnecting] = React.useState<ProviderId | null>(null);
const [error, setError] = React.useState<string | null>(null);
const [notice, setNotice] = React.useState<string | null>(null);
@@ -58,7 +59,11 @@ export default function ProvidersPanel({ isAdmin, cookie, onUpdateMakerWorld, on
let active = true;
settingsApi
.getMakerworld()
.then((res) => active && setConnected((current) => ({ ...current, makerworld: res.configured })))
.then((res) => {
if (!active) return;
setConnected((current) => ({ ...current, makerworld: res.configured }));
setMakerworldExpired(res.expired);
})
.catch(() => undefined);
settingsApi
.getThingiverse()
@@ -84,6 +89,7 @@ export default function ProvidersPanel({ isAdmin, cookie, onUpdateMakerWorld, on
const result = await settingsApi.updateMakerworld(value);
onUpdateMakerWorld({ cookie: value ?? "" });
setConnected((current) => ({ ...current, makerworld: result.configured }));
setMakerworldExpired(false);
} else if (id === "thingiverse") {
const result = await settingsApi.updateThingiverse(value);
setConnected((current) => ({ ...current, thingiverse: result.configured }));
@@ -103,7 +109,7 @@ export default function ProvidersPanel({ isAdmin, cookie, onUpdateMakerWorld, on
if (locked) {
setConnecting(null);
setNotice(locked);
} else if (connected[id] === false) {
} else if (connected[id] === false || (id === "makerworld" && makerworldExpired)) {
startConnect(id);
}
};
@@ -137,6 +143,8 @@ export default function ProvidersPanel({ isAdmin, cookie, onUpdateMakerWorld, on
const menuId = menu?.id;
const menuConnected = menuId ? Boolean(connected[menuId]) : false;
// Still connected, so it can be disconnected, but offered a fresh connect first.
const menuExpired = menuId === "makerworld" && menuConnected && makerworldExpired;
const menuLocked = menuId ? lockedReason(menuId) : null;
return (
@@ -146,14 +154,17 @@ export default function ProvidersPanel({ isAdmin, cookie, onUpdateMakerWorld, on
{PROVIDERS.map(({ id, name }) => {
const isConnected = Boolean(connected[id]);
const locked = lockedReason(id);
const status = connected[id] === null ? null : isConnected;
const expired = id === "makerworld" && isConnected && makerworldExpired;
const status = connected[id] === null ? null : isConnected && !expired;
const tooltip = isComingSoon(id)
? t("providers.comingSoon")
: id === "thingiverse" && locked && !isConnected
? locked
: isConnected
? t("providers.connected")
: t("providers.notConnected");
: expired
? t("providers.makerworld.expired")
: id === "thingiverse" && locked && !isConnected
? locked
: isConnected
? t("providers.connected")
: t("providers.notConnected");
return (
<AppTile
key={id}
@@ -163,7 +174,7 @@ export default function ProvidersPanel({ isAdmin, cookie, onUpdateMakerWorld, on
badge={status === null || isComingSoon(id) ? undefined : status}
highlighted={connecting === id || menuId === id}
tooltip={status === null ? undefined : tooltip}
ariaLabel={`${name}: ${isConnected ? t("providers.connected") : t("providers.notConnected")}`}
ariaLabel={`${name}: ${expired ? t("providers.makerworld.expired") : isConnected ? t("providers.connected") : t("providers.notConnected")}`}
onContextMenu={(event) => openMenu(id, event)}
onActivate={() => activate(id)}
/>
@@ -185,12 +196,27 @@ export default function ProvidersPanel({ isAdmin, cookie, onUpdateMakerWorld, on
status={
isComingSoon(menuId)
? t("providers.comingSoon")
: menuConnected
? t("providers.connected")
: t("providers.notConnected")
: menuExpired
? t("providers.makerworld.expiredStatus")
: menuConnected
? t("providers.connected")
: t("providers.notConnected")
}
/>
)}
{menuId && menuExpired && (
<MenuItem
onClick={() => {
setMenu(null);
startConnect(menuId);
}}
>
<ListItemIcon>
<LinkIcon fontSize="small" />
</ListItemIcon>
<ListItemText>{t("providers.connect")}</ListItemText>
</MenuItem>
)}
{menuId && (
<MenuItem
disabled={Boolean(menuLocked)}
+6 -3
View File
@@ -47,6 +47,7 @@ export function UserMenu({ user, onOpenConfiguration, onOpenProfile, onLogout }:
const avatarUrl = useGravatarUrl(user?.email, 128);
const [anchorEl, setAnchorEl] = React.useState<HTMLElement | null>(null);
const [makerworldConfigured, setMakerworldConfigured] = React.useState(false);
const [makerworldExpired, setMakerworldExpired] = React.useState(false);
const [thingiverseConfigured, setThingiverseConfigured] = React.useState(false);
const menuOpen = Boolean(anchorEl);
@@ -57,7 +58,9 @@ export function UserMenu({ user, onOpenConfiguration, onOpenProfile, onLogout }:
settingsApi
.getMakerworld()
.then((res) => {
if (active) setMakerworldConfigured(res.configured);
if (!active) return;
setMakerworldConfigured(res.configured);
setMakerworldExpired(res.expired);
})
.catch(() => undefined);
settingsApi
@@ -77,8 +80,8 @@ export function UserMenu({ user, onOpenConfiguration, onOpenProfile, onLogout }:
{
key: "makerworld",
...IMPORT_PROVIDER_INFO.makerworld,
connected: makerworldConfigured,
disabledReason: t("userMenu.makerworldDisabledReason"),
connected: makerworldConfigured && !makerworldExpired,
disabledReason: makerworldExpired ? t("providers.makerworld.expired") : t("userMenu.makerworldDisabledReason"),
},
{
key: "thingiverse",
+2
View File
@@ -1030,6 +1030,8 @@
"disconnectMessageShared": "The {{name}} token is removed for everyone on this Thingport, and imports from {{name}} stop working until it's connected again.",
"failed": "Failed to save -- try again.",
"makerworld": {
"expiredStatus": "Sign-in expired",
"expired": "Sign-in expired. Connect MakerWorld again with a fresh cookie in Configuration → Providers.",
"placeholder": "example: mw_session=...; mw_token=...;",
"steps": [
"Sign in at makerworld.com in your browser.",
+2
View File
@@ -1057,6 +1057,8 @@
"disconnectMessageShared": "{{name}} raktas bus pašalintas visiems šio Thingport naudotojams, o importavimas iš {{name}} neveiks, kol jis vėl bus prijungtas.",
"failed": "Nepavyko išsaugoti -- bandykite dar kartą.",
"makerworld": {
"expiredStatus": "Prisijungimas nebegalioja",
"expired": "Prisijungimo galiojimas baigėsi. Vėl prijunkite MakerWorld su nauju slapuku skiltyje Konfigūracija → Šaltiniai.",
"placeholder": "pavyzdys: mw_session=...; mw_token=...;",
"steps": [
"Naršyklėje prisijunkite prie makerworld.com.",