mirror of
https://github.com/TautvydasDerzinskas/Thingport.git
synced 2026-10-19 05:10:01 -05:00
feat(sync): pause syncs while a provider isn't connected, and hand back models blocked by it
This commit is contained in:
1 parent
2d433dff17
commit
6993a4c423
20 files changed
+517
-44
No files matched your search
@@ -0,0 +1 @@
|
||||
ALTER TABLE "User" ADD COLUMN "makerworldRejectedAt" TIMESTAMP(3);
|
||||
@@ -43,6 +43,8 @@ model User {
|
||||
apiTokenLastUsedAt DateTime?
|
||||
// Per-user, unlike the instance-wide Thingiverse token.
|
||||
makerworldCookie String?
|
||||
// When MakerWorld last turned the cookie down. Scheduled MakerWorld syncs wait until it changes.
|
||||
makerworldRejectedAt DateTime?
|
||||
// Who the user is on providers without a per-user login, so their collections can be listed for
|
||||
// syncing. Printables keeps its numeric id (the API's key) with the handle to show.
|
||||
printablesUserId String?
|
||||
|
||||
@@ -35,7 +35,7 @@ import {
|
||||
setStorageTemplate,
|
||||
validateStorageTemplate,
|
||||
} from "../services/printService";
|
||||
import { getUserMakerworldCookie, setUserMakerworldCookie } from "../services/makerworldCookieService";
|
||||
import { getMakerworldConnection, setUserMakerworldCookie } from "../services/makerworldCookieService";
|
||||
import { type MakerworldCookieCheck, verifyMakerworldCookie } from "../services/makerworldCloudApi";
|
||||
import { verifyThingiverseAccessToken } from "../services/thingiverseApi";
|
||||
import { SLICER_IDS, getUserSlicer, setUserSlicer } from "../services/slicerPreferenceService";
|
||||
@@ -363,11 +363,13 @@ router.post(
|
||||
}),
|
||||
);
|
||||
|
||||
// Per-user, unlike the instance-wide Thingiverse token. GET only reports whether one is set.
|
||||
// Per-user, unlike the instance-wide Thingiverse token. GET only reports whether one is set, and
|
||||
// whether MakerWorld has since turned it down.
|
||||
router.get(
|
||||
"/settings/makerworld",
|
||||
asyncHandler(async (req, res) => {
|
||||
res.json({ configured: Boolean(await getUserMakerworldCookie(req.userId!)) });
|
||||
const connection = await getMakerworldConnection(req.userId!);
|
||||
res.json({ configured: connection !== "disconnected", expired: connection === "expired" });
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -390,6 +392,7 @@ router.patch(
|
||||
asyncHandler(async (req, res) => {
|
||||
const body = parseBody(makerworldSettingsSchema, req.body);
|
||||
const trimmed = (body.cookie ?? "").trim();
|
||||
let verified = false;
|
||||
if (body.verify && trimmed) {
|
||||
const check = await verifyMakerworldCookie(trimmed);
|
||||
if (check.result === "invalid") {
|
||||
@@ -400,8 +403,9 @@ router.patch(
|
||||
}
|
||||
// 503, not 422: nothing is known to be wrong with the cookie.
|
||||
if (check.result === "unverifiable") throw new HttpError(503, MAKERWORLD_UNVERIFIABLE_MESSAGES[check.reason]);
|
||||
verified = true;
|
||||
}
|
||||
const configured = await setUserMakerworldCookie(req.userId!, body.cookie);
|
||||
const configured = await setUserMakerworldCookie(req.userId!, body.cookie, { verified });
|
||||
res.json({ configured });
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -167,7 +167,7 @@ async function lookUpAuthor(candidate: Candidate, thingiverseToken: string | nul
|
||||
return (await resolvePrintablesModel(candidate.externalId))?.meta.author ?? null;
|
||||
}
|
||||
case "makerworld": {
|
||||
const cookie = (await getUserMakerworldCookie(candidate.userId)) ?? process.env.MAKERWORLD_COOKIE ?? null;
|
||||
const cookie = await getUserMakerworldCookie(candidate.userId);
|
||||
const bearer = extractMakerworldBearerToken(cookie);
|
||||
return bearer
|
||||
? fetchMakerworldDesignAuthor(candidate.externalId, bearer, IMPORT_MAKERWORLD_CALL_DELAY_MS)
|
||||
|
||||
@@ -26,9 +26,14 @@ import {
|
||||
import { getActiveJob } from "./importJobService";
|
||||
import { runLinksImportJob, SYNC_JOB_INTERRUPTED } from "./importJobRunner";
|
||||
import { startLinksJob } from "./importQueueService";
|
||||
import { extractMakerworldBearerToken } from "./makerworldCloudApi";
|
||||
import { extractMakerworldBearerToken, MakerworldAuthError } from "./makerworldCloudApi";
|
||||
import { checkMakerworldCollection, fetchMakerworldCollectionEntries } from "./makerworldCollections";
|
||||
import { getUserMakerworldCookie } from "./makerworldCookieService";
|
||||
import {
|
||||
getMakerworldConnection,
|
||||
getUserMakerworldCookie,
|
||||
MAKERWORLD_REJECTED_ERROR,
|
||||
markMakerworldCookieRejected,
|
||||
} from "./makerworldCookieService";
|
||||
import { checkPrintablesCollection, fetchPrintablesCollectionEntries } from "./printablesApi";
|
||||
import {
|
||||
checkThingiverseCollection,
|
||||
@@ -96,6 +101,7 @@ export type SyncOutcome =
|
||||
| "busy" // an import of the user's is running; tried again next time
|
||||
| "resumed" // a job a restart cut off was picked up again
|
||||
| "error" // the provider couldn't be read
|
||||
| "disconnected" // the provider isn't connected, or turned the login down; not checked till it is
|
||||
| "gone" // the provider's collection wasn't found, not yet often enough to unsync
|
||||
| "unsynced"
|
||||
| "unchanged"
|
||||
@@ -128,10 +134,95 @@ export function syncCollection(syncId: string, { reserved = false } = {}): Promi
|
||||
return run;
|
||||
}
|
||||
|
||||
type ConnectionProblem = { error: string; reason: string; fix: string };
|
||||
|
||||
const CONNECTION_PROBLEMS = {
|
||||
makerworldDisconnected: {
|
||||
error: "MakerWorld isn't connected",
|
||||
reason: "MakerWorld isn't connected.",
|
||||
fix: "you connect it again",
|
||||
},
|
||||
makerworldExpired: {
|
||||
error: MAKERWORLD_REJECTED_ERROR,
|
||||
reason: "MakerWorld turned your sign-in down; it may have expired.",
|
||||
fix: "you connect MakerWorld again",
|
||||
},
|
||||
thingiverseDisconnected: {
|
||||
error: "Thingiverse isn't connected on this Thingport",
|
||||
reason: "Thingiverse isn't connected on this Thingport.",
|
||||
fix: "an admin connects it again",
|
||||
},
|
||||
} satisfies Record<string, ConnectionProblem>;
|
||||
|
||||
/** `"A"`, `"A" and "B"`, or the first few and how many more. */
|
||||
function listNames(names: string[]): string {
|
||||
const quoted = names.map((name) => `"${name}"`);
|
||||
if (quoted.length === 1) return quoted[0];
|
||||
if (quoted.length <= 3) return `${quoted.slice(0, -1).join(", ")} and ${quoted[quoted.length - 1]}`;
|
||||
return `${quoted.slice(0, 2).join(", ")} and ${quoted.length - 2} more`;
|
||||
}
|
||||
|
||||
/** One notification for every collection the problem holds. */
|
||||
async function notifyHeld(
|
||||
userId: string,
|
||||
provider: string,
|
||||
problem: ConnectionProblem,
|
||||
syncs: SyncWithCollection[],
|
||||
): Promise<void> {
|
||||
if (!syncs.length) return;
|
||||
const one = syncs.length === 1 ? syncs[0] : null;
|
||||
await createNotification(userId, {
|
||||
type: "sync",
|
||||
title: one
|
||||
? `Couldn't sync "${one.collection.name}"`
|
||||
: `Couldn't sync ${syncs.length} ${providerName(provider)} collections`,
|
||||
body: `${problem.reason} ${listNames(syncs.map((sync) => sync.collection.name))} won't sync until ${problem.fix} in Configuration → Providers. ${one ? "It stays synced, and its models stay." : "They stay synced, and their models stay."}`,
|
||||
externalUrl: one?.sourceUrl ?? null,
|
||||
internalPath: one ? `/models/collections/${one.collectionId}` : "/models/collections",
|
||||
});
|
||||
}
|
||||
|
||||
/** Why the provider can't be reached for this sync right now, or null when it can. Checked before
|
||||
* any provider call, so a sync with nothing to sign in with waits instead of failing every round. */
|
||||
async function connectionProblem(sync: Pick<CollectionSync, "userId" | "provider">): Promise<ConnectionProblem | null> {
|
||||
if (sync.provider === "makerworld") {
|
||||
const connection = await getMakerworldConnection(sync.userId);
|
||||
if (connection === "disconnected") return CONNECTION_PROBLEMS.makerworldDisconnected;
|
||||
if (connection === "expired") return CONNECTION_PROBLEMS.makerworldExpired;
|
||||
}
|
||||
if (sync.provider === "thingiverse" && !(await getThingiverseAccessToken())) {
|
||||
return CONNECTION_PROBLEMS.thingiverseDisconnected;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Holds every one of the user's syncs with this provider at once, with one notification for the
|
||||
* ones not already held; later rounds find the error saved and stay quiet. Last check times are
|
||||
* left alone, so they're due again as soon as the provider is connected. */
|
||||
async function holdProvider(userId: string, provider: string, problem: ConnectionProblem): Promise<void> {
|
||||
const fresh = await prisma.collectionSync.findMany({
|
||||
where: { userId, provider, OR: [{ lastError: null }, { lastError: { not: problem.error } }] },
|
||||
include: { collection: true },
|
||||
orderBy: { createdAt: "asc" },
|
||||
});
|
||||
if (!fresh.length) return;
|
||||
await prisma.collectionSync.updateMany({
|
||||
where: { id: { in: fresh.map((sync) => sync.id) } },
|
||||
data: { lastError: problem.error },
|
||||
});
|
||||
console.warn(`[sync] ${fresh.length} ${provider} collection(s) of user ${userId} held: ${problem.error}`);
|
||||
await notifyHeld(userId, provider, problem, fresh);
|
||||
}
|
||||
|
||||
async function runSync(syncId: string): Promise<SyncOutcome> {
|
||||
const sync = await prisma.collectionSync.findUnique({ where: { id: syncId }, include: { collection: true } });
|
||||
if (!sync) return "missing";
|
||||
const { userId } = sync;
|
||||
const problem = await connectionProblem(sync);
|
||||
if (problem) {
|
||||
await holdProvider(userId, sync.provider, problem);
|
||||
return "disconnected";
|
||||
}
|
||||
|
||||
let openJob = await prisma.importJob.findFirst({
|
||||
where: { collectionSyncId: sync.id, status: { in: ["RUNNING", "PAUSED"] } },
|
||||
@@ -157,6 +248,11 @@ async function runSync(syncId: string): Promise<SyncOutcome> {
|
||||
try {
|
||||
listing = await fetchListing(sync);
|
||||
} catch (err) {
|
||||
if (err instanceof MakerworldAuthError) {
|
||||
await markMakerworldCookieRejected(userId);
|
||||
await holdProvider(userId, sync.provider, CONNECTION_PROBLEMS.makerworldExpired);
|
||||
return "disconnected";
|
||||
}
|
||||
const message = err instanceof Error && err.message.trim() ? err.message.trim() : "Sync failed";
|
||||
console.warn(`[sync] "${sync.collection.name}" (${sync.provider} ${sync.externalId}): ${message}`);
|
||||
await prisma.collectionSync
|
||||
@@ -336,6 +432,12 @@ export async function runManualSync(syncId: string, { reserved = false } = {}):
|
||||
/** The answer to "Sync now" for the outcomes that send no notification of their own. */
|
||||
async function notifyManualOutcome(before: SyncWithCollection, outcome: SyncOutcome): Promise<void> {
|
||||
const after = await prisma.collectionSync.findUnique({ where: { id: before.id } });
|
||||
// holdProvider only notifies the first time; a click after that still gets an answer.
|
||||
if (outcome === "disconnected" && after?.lastError === before.lastError) {
|
||||
const problem = Object.values(CONNECTION_PROBLEMS).find((p) => p.error === after?.lastError);
|
||||
if (problem) await notifyHeld(before.userId, before.provider, problem, [before]);
|
||||
return;
|
||||
}
|
||||
const name = before.collection.name;
|
||||
const provider = providerName(before.provider);
|
||||
const message: Partial<Record<SyncOutcome, { title: string; body: string }>> = {
|
||||
@@ -372,10 +474,29 @@ export function runSyncRound(): Promise<void> {
|
||||
currentRound = (async () => {
|
||||
const syncs = await prisma.collectionSync.findMany({
|
||||
orderBy: { lastCheckedAt: { sort: "asc", nulls: "first" } },
|
||||
select: { id: true, lastCheckedAt: true, intervalHours: true },
|
||||
select: { id: true, userId: true, provider: true, lastCheckedAt: true, intervalHours: true },
|
||||
});
|
||||
const due = syncs.filter((sync) => isSyncDue(sync));
|
||||
for (const [index, { id }] of due.entries()) {
|
||||
const due: string[] = [];
|
||||
// Each user's connection to each provider, looked up once a round.
|
||||
const problems = new Map<string, ConnectionProblem | null>();
|
||||
for (const sync of syncs.filter((s) => isSyncDue(s))) {
|
||||
const key = `${sync.userId} ${sync.provider}`;
|
||||
if (problems.has(key)) {
|
||||
if (!problems.get(key)) due.push(sync.id);
|
||||
continue;
|
||||
}
|
||||
const problem = await connectionProblem(sync);
|
||||
problems.set(key, problem);
|
||||
if (!problem) {
|
||||
due.push(sync.id);
|
||||
continue;
|
||||
}
|
||||
// Held here rather than in the lane: there's no provider call to pace.
|
||||
await holdProvider(sync.userId, sync.provider, problem).catch((err) =>
|
||||
console.error(`[sync] holding ${key} failed:`, err),
|
||||
);
|
||||
}
|
||||
for (const [index, id] of due.entries()) {
|
||||
if (index > 0) await sleep(IMPORT_COLLECTION_DELAY_MS);
|
||||
try {
|
||||
const outcome = await syncCollection(id);
|
||||
|
||||
@@ -97,6 +97,13 @@ export function modelUrlFor(provider: string, externalId: string): string {
|
||||
return `https://www.thingiverse.com/thing:${externalId}`;
|
||||
}
|
||||
|
||||
/** The model id in a link modelUrlFor made. */
|
||||
export function modelIdFromUrl(provider: string, url: string): string | null {
|
||||
const pattern =
|
||||
provider === "makerworld" ? /\/models\/(\d+)/ : provider === "printables" ? /\/model\/(\d+)/ : /thing:(\d+)/;
|
||||
return url.match(pattern)?.[1] ?? null;
|
||||
}
|
||||
|
||||
/** What Thingport Grab reads on a provider's collection page. */
|
||||
export type SyncLinkOut = CollectionSyncOut & { collection_id: string; collection_name: string };
|
||||
|
||||
|
||||
@@ -36,6 +36,9 @@ export async function testFlaresolverr(
|
||||
}
|
||||
}
|
||||
|
||||
// Cloudflare still said no after FlareSolverr, or with none set up: nothing to do with the link.
|
||||
export const CLOUDFLARE_BLOCKED_CODE = "CLOUDFLARE_BLOCKED";
|
||||
|
||||
/** A Cloudflare edge block rather than an application-level 403. */
|
||||
export function looksLikeCloudflareBlock(headers: Headers): boolean {
|
||||
const server = (headers.get("server") || "").toLowerCase();
|
||||
|
||||
@@ -5,7 +5,7 @@ import { IMPORT_COLLECTION_DELAY_MS, IMPORT_MAKERWORLD_CALL_DELAY_MS } from "../
|
||||
import { countJobItems, getJob, getJobStatus, listJobItems, updateJob, updateJobItem } from "./importJobService";
|
||||
import { createNotification } from "./notificationService";
|
||||
import { addPrintsToCollection, findOrCreateCollectionByName } from "./collectionService";
|
||||
import { likesCollectionName, providerName, syncedCollectionFor } from "./collectionSyncService";
|
||||
import { likesCollectionName, modelIdFromUrl, providerName, syncedCollectionFor } from "./collectionSyncService";
|
||||
import { resolveMakerworldCookie } from "./importResolvers";
|
||||
import { decodeHtmlEntities } from "../utils/htmlEntities";
|
||||
import {
|
||||
@@ -13,6 +13,8 @@ import {
|
||||
parseMakerworldModelUrl,
|
||||
selectMakerworldProfiles,
|
||||
type MakerworldProfileScope,
|
||||
MAKERWORLD_AUTH_CODE,
|
||||
MakerworldAuthError,
|
||||
MAKERWORLD_DOWNLOAD_LIMIT_CODE,
|
||||
MAKERWORLD_DOWNLOAD_LIMIT_MESSAGE,
|
||||
} from "./makerworldCloudApi";
|
||||
@@ -29,6 +31,8 @@ import { fetchThingiverseCollectionTitle } from "./thingiverseApi";
|
||||
import { getThingiverseAccessToken } from "./settingsService";
|
||||
import { fetchPrintablesCollectionTitle } from "./printablesApi";
|
||||
import { createLog } from "./auditLog";
|
||||
import { MAKERWORLD_REJECTED_ERROR, markMakerworldCookieRejected } from "./makerworldCookieService";
|
||||
import { CLOUDFLARE_BLOCKED_CODE } from "./flaresolverr";
|
||||
import { HttpError } from "../utils/fileUtils";
|
||||
import { prisma } from "../db";
|
||||
import type { ImportJob } from "@prisma/client";
|
||||
@@ -48,13 +52,19 @@ type PrintablesCollectionImportJobBody = ImportRequestBody & {
|
||||
type MakerworldProfilesImportJobBody = ImportRequestBody & { scope: Exclude<MakerworldProfileScope, "url"> };
|
||||
|
||||
// Why a design failed, so a batch reads as one clear cause instead of "N failed". Once a CAPTCHA
|
||||
// ("rateLimited"), the daily download limit or an auth failure hits, every remaining item fails
|
||||
// the same way.
|
||||
type ImportFailureReason = "unavailable" | "rateLimited" | "downloadLimit" | "auth" | "other";
|
||||
type StopReason = "rateLimited" | "downloadLimit" | "auth";
|
||||
// ("rateLimited"), the daily download limit, an auth failure or a Cloudflare block ("blocked")
|
||||
// hits, every remaining item fails the same way.
|
||||
type ImportFailureReason = "unavailable" | "rateLimited" | "downloadLimit" | "auth" | "blocked" | "other";
|
||||
type StopReason = "rateLimited" | "downloadLimit" | "auth" | "blocked";
|
||||
|
||||
const BLOCKED_RETRY_TEXT =
|
||||
"Cloudflare blocked the request and FlareSolverr didn't get past it — check FlareSolverr in Administration → Connections, then retry";
|
||||
|
||||
function classifyImportFailure(err: unknown): ImportFailureReason {
|
||||
if (err instanceof MakerworldAuthError) return "auth";
|
||||
if (err instanceof HttpError) {
|
||||
if (err.code === MAKERWORLD_AUTH_CODE) return "auth";
|
||||
if (err.code === CLOUDFLARE_BLOCKED_CODE) return "blocked";
|
||||
if (err.status === 403 || err.status === 404) return "unavailable";
|
||||
if (err.status === 429) return err.code === MAKERWORLD_DOWNLOAD_LIMIT_CODE ? "downloadLimit" : "rateLimited";
|
||||
if (err.status === 401) return "auth";
|
||||
@@ -63,7 +73,9 @@ function classifyImportFailure(err: unknown): ImportFailureReason {
|
||||
}
|
||||
|
||||
function asStopReason(reason: ImportFailureReason): StopReason | null {
|
||||
return reason === "rateLimited" || reason === "downloadLimit" || reason === "auth" ? reason : null;
|
||||
return reason === "rateLimited" || reason === "downloadLimit" || reason === "auth" || reason === "blocked"
|
||||
? reason
|
||||
: null;
|
||||
}
|
||||
|
||||
async function markJobFailed(jobId: string, err: unknown): Promise<void> {
|
||||
@@ -172,6 +184,7 @@ export async function runMakerworldProfilesImportJob(
|
||||
details: { provider: "makerworld", sourceLabel: title, imported, alreadyInLibrary, failed },
|
||||
});
|
||||
|
||||
if (stopReason === "auth") await noteMakerworldRejected(userId);
|
||||
const bodyParts: string[] = [];
|
||||
if (alreadyInLibrary) bodyParts.push(`${alreadyInLibrary} already on the model`);
|
||||
if (stopReason === "rateLimited") {
|
||||
@@ -301,6 +314,7 @@ export async function runCollectionImportJob(
|
||||
},
|
||||
});
|
||||
|
||||
if (authFailed) await noteMakerworldRejected(userId);
|
||||
const label = collectionTitle ? `"${collectionTitle}"` : "a MakerWorld collection";
|
||||
const bodyParts: string[] = [];
|
||||
if (alreadyInLibrary) bodyParts.push(`${alreadyInLibrary} already in your library`);
|
||||
@@ -667,6 +681,7 @@ const LINKS_MAX_ATTEMPTS = 3;
|
||||
const LINKS_RETRY_DELAY_MS = 2000;
|
||||
|
||||
function isTransientImportFailure(err: unknown): boolean {
|
||||
if (err instanceof MakerworldAuthError) return false;
|
||||
return !(err instanceof HttpError) || err.transient === true;
|
||||
}
|
||||
|
||||
@@ -742,6 +757,10 @@ async function runLinks(jobId: string, userId: string, body: LinksImportJobBody)
|
||||
let profilesProcessed = 0;
|
||||
let printId: string | null = null;
|
||||
let stopReason: StopReason | null = null;
|
||||
// The link the stop reason came from, so only a MakerWorld one marks its cookie rejected.
|
||||
let stopUrl: string | null = null;
|
||||
// A sync's links handed back to it by deferSyncRest.
|
||||
let deferred: SyncDeferral | null = null;
|
||||
let processed = (await countJobItems(jobId, "DONE")) + (await countJobItems(jobId, "FAILED"));
|
||||
// The models this run brought in new, for a sync's notification.
|
||||
const newPrintIds: string[] = [];
|
||||
@@ -768,6 +787,7 @@ async function runLinks(jobId: string, userId: string, body: LinksImportJobBody)
|
||||
rateLimited: "blocked by a MakerWorld CAPTCHA challenge — retry once it clears (usually 1-4 hours)",
|
||||
downloadLimit: "MakerWorld's daily download limit was reached — retry tomorrow",
|
||||
auth: "the MakerWorld session expired — update the cookie in Settings, then retry",
|
||||
blocked: BLOCKED_RETRY_TEXT,
|
||||
}[stopReason];
|
||||
settled = true;
|
||||
break;
|
||||
@@ -797,6 +817,7 @@ async function runLinks(jobId: string, userId: string, body: LinksImportJobBody)
|
||||
rateLimited: " (MakerWorld CAPTCHA challenge — retry once it clears)",
|
||||
downloadLimit: " (MakerWorld's daily download limit — retry tomorrow)",
|
||||
auth: " (the MakerWorld session expired — update the cookie in Settings, then retry)",
|
||||
blocked: ` (${BLOCKED_RETRY_TEXT})`,
|
||||
}[result.stopReason];
|
||||
}
|
||||
settled = true;
|
||||
@@ -819,6 +840,18 @@ async function runLinks(jobId: string, userId: string, body: LinksImportJobBody)
|
||||
}
|
||||
if (itemError) console.error(`[import] job ${jobId}: ${item.url} failed: ${itemError}`);
|
||||
else console.log(`[import] job ${jobId}: ${item.url} imported`);
|
||||
if (stopReason && !stopUrl) stopUrl = item.url;
|
||||
// The connection failed, not the models: the rest go back to unseen, for the next sync to find
|
||||
// again once it works. A link that got some of its profiles in is done.
|
||||
if (job.collectionSyncId && (stopReason === "auth" || stopReason === "blocked")) {
|
||||
const nothingIn = itemError !== null && !partlyIn;
|
||||
if (!nothingIn) {
|
||||
await updateJobItem(item.id, { status: "FAILED", attempts, errorMessage: itemError }).catch(() => undefined);
|
||||
processed++;
|
||||
}
|
||||
deferred = { count: await deferSyncRest(job, nothingIn ? item.id : null, stopReason), reason: stopReason };
|
||||
break;
|
||||
}
|
||||
// A sync can't ask anyone to retry, so a rate limit pauses it instead of failing the rest. A
|
||||
// link that got nothing in waits with them; one that got some of its profiles in is done.
|
||||
if (job.collectionSyncId && stopReason) {
|
||||
@@ -864,7 +897,10 @@ async function runLinks(jobId: string, userId: string, body: LinksImportJobBody)
|
||||
details: { provider: "links", imported, alreadyInLibrary, failed: failedCount },
|
||||
});
|
||||
|
||||
if (job.collectionSyncId && (await notifySyncJobDone(job, newPrintIds, alreadyInLibrary, failedCount))) return;
|
||||
if (stopReason === "auth" && stopUrl && parseMakerworldModelUrl(stopUrl)) await noteMakerworldRejected(userId);
|
||||
if (job.collectionSyncId && (await notifySyncJobDone(job, newPrintIds, alreadyInLibrary, failedCount, deferred))) {
|
||||
return;
|
||||
}
|
||||
|
||||
const bodyParts: string[] = [];
|
||||
if (profilesProcessed) bodyParts.push(`${profilesImported} of ${profilesProcessed} print profiles imported`);
|
||||
@@ -881,6 +917,8 @@ async function runLinks(jobId: string, userId: string, body: LinksImportJobBody)
|
||||
bodyParts.push(
|
||||
`the rest failed because your MakerWorld session expired — update the cookie in Settings, then retry`,
|
||||
);
|
||||
} else if (stopReason === "blocked") {
|
||||
bodyParts.push(`the rest failed: ${BLOCKED_RETRY_TEXT}`);
|
||||
} else if (failedCount) {
|
||||
bodyParts.push(`${failedCount} failed — retry them from the import's progress bar`);
|
||||
}
|
||||
@@ -926,6 +964,7 @@ async function modelNames(printIds: string[]): Promise<string> {
|
||||
|
||||
function syncStopText(reason: StopReason, provider: string): string {
|
||||
if (reason === "downloadLimit") return "MakerWorld's daily download limit was reached";
|
||||
if (reason === "blocked") return `Cloudflare blocked ${providerName(provider)} and FlareSolverr didn't get past it`;
|
||||
if (reason === "auth") {
|
||||
return provider === "makerworld"
|
||||
? "your MakerWorld session expired, so connect MakerWorld again in Configuration > Providers"
|
||||
@@ -971,12 +1010,59 @@ async function pauseSyncJob(job: ImportJob, reason: StopReason, newPrintIds: str
|
||||
});
|
||||
}
|
||||
|
||||
/** Scheduled MakerWorld syncs then wait for a new cookie instead of meeting the same refusal. */
|
||||
async function noteMakerworldRejected(userId: string): Promise<void> {
|
||||
await markMakerworldCookieRejected(userId).catch((err) => console.error("[import] couldn't mark cookie:", err));
|
||||
}
|
||||
|
||||
type SyncDeferral = { count: number; reason: "auth" | "blocked" };
|
||||
|
||||
/** Takes the sync job's links that haven't run (and `currentItemId`'s, which got nothing in) out of
|
||||
* the job, and out of the sync's seen ids, so its next check queues them again. How many. */
|
||||
async function deferSyncRest(job: ImportJob, currentItemId: string | null, reason: SyncDeferral["reason"]) {
|
||||
// One transaction: links gone from the job but still seen by the sync would never come in.
|
||||
const handedBack = await prisma.$transaction(async (tx) => {
|
||||
const items = await tx.importJobItem.findMany({
|
||||
where: { jobId: job.id, OR: [{ status: "PENDING" }, ...(currentItemId ? [{ id: currentItemId }] : [])] },
|
||||
select: { id: true, url: true },
|
||||
});
|
||||
await tx.importJobItem.deleteMany({ where: { id: { in: items.map((item) => item.id) } } });
|
||||
await tx.importJob.update({
|
||||
where: { id: job.id },
|
||||
data: { total: await tx.importJobItem.count({ where: { jobId: job.id } }) },
|
||||
});
|
||||
const sync = job.collectionSyncId
|
||||
? await tx.collectionSync.findUnique({ where: { id: job.collectionSyncId } })
|
||||
: null;
|
||||
if (sync) {
|
||||
const unseen = new Set(items.flatMap((item) => modelIdFromUrl(sync.provider, item.url) ?? []));
|
||||
await tx.collectionSync.update({
|
||||
where: { id: sync.id },
|
||||
data: {
|
||||
knownIds: sync.knownIds.filter((id) => !unseen.has(id)),
|
||||
// For MakerWorld, already held, so holdProvider leaves it out of the notification about the others.
|
||||
lastError:
|
||||
reason === "blocked"
|
||||
? `Cloudflare blocked ${providerName(sync.provider)} and FlareSolverr didn't get past it`
|
||||
: sync.provider === "makerworld"
|
||||
? MAKERWORLD_REJECTED_ERROR
|
||||
: `${providerName(sync.provider)} turned Thingport's sign-in down`,
|
||||
},
|
||||
});
|
||||
}
|
||||
return items;
|
||||
});
|
||||
console.warn(`[sync] job ${job.id}: ${handedBack.length} links handed back to the sync (${reason})`);
|
||||
return handedBack.length;
|
||||
}
|
||||
|
||||
/** Says which new models came in. False when the sync is gone, for the plain notification. */
|
||||
async function notifySyncJobDone(
|
||||
job: ImportJob,
|
||||
newPrintIds: string[],
|
||||
alreadyInLibrary: number,
|
||||
failedCount: number,
|
||||
deferred: SyncDeferral | null,
|
||||
): Promise<boolean> {
|
||||
const sync = await syncOfJob(job);
|
||||
if (!sync) return false;
|
||||
@@ -990,6 +1076,16 @@ async function notifySyncJobDone(
|
||||
`${filed} model${filed === 1 ? " that was" : "s that were"} already in your library ${filed === 1 ? "was" : "were"} added to it too.`,
|
||||
);
|
||||
}
|
||||
if (deferred) {
|
||||
const more = `${deferred.count} more model${deferred.count === 1 ? "" : "s"}`;
|
||||
parts.push(
|
||||
deferred.reason === "auth" && sync.provider === "makerworld"
|
||||
? `${more} will come in on the next sync once you connect MakerWorld again in Configuration → Providers: MakerWorld turned your sign-in down, and it may have expired.`
|
||||
: deferred.reason === "auth"
|
||||
? `${more} will come in on a later sync: ${providerName(sync.provider)} turned Thingport's sign-in down.`
|
||||
: `${more} will come in on a later sync: Cloudflare blocked ${providerName(sync.provider)} and FlareSolverr didn't get past it. An admin can check FlareSolverr in Administration → Connections.`,
|
||||
);
|
||||
}
|
||||
if (failedCount) {
|
||||
parts.push(
|
||||
`${failedCount} couldn't be imported (private, deleted, or failed); retry ${failedCount === 1 ? "it" : "them"} from the import queue.`,
|
||||
@@ -1000,7 +1096,7 @@ async function notifySyncJobDone(
|
||||
type: "sync",
|
||||
title: count
|
||||
? `${count} new model${count === 1 ? "" : "s"} synced into "${name}"`
|
||||
: `Couldn't sync new models into "${name}"`,
|
||||
: `Couldn't sync new models into "${name}"${deferred ? " yet" : ""}`,
|
||||
body: parts.join(" ") || null,
|
||||
externalUrl: sync.sourceUrl,
|
||||
internalPath: `/models/collections/${sync.collectionId}`,
|
||||
|
||||
@@ -46,7 +46,7 @@ function firstNonEmptyLine(raw: string): string | null {
|
||||
}
|
||||
|
||||
export function resolveMakerworldCookie(body: ImportCookies): string | null {
|
||||
const raw = (body.makerworld_cookie || process.env.MAKERWORLD_COOKIE || "").trim();
|
||||
const raw = (body.makerworld_cookie || "").trim();
|
||||
if (!raw) return null;
|
||||
return firstNonEmptyLine(raw);
|
||||
}
|
||||
|
||||
@@ -22,7 +22,12 @@ import {
|
||||
import { validateRemoteUrl } from "../utils/urlUtils";
|
||||
import { readCapped } from "../utils/readCapped";
|
||||
import { maybeSleep, sleep } from "../utils/concurrency";
|
||||
import { fetchViaFlaresolverr, looksLikeCloudflareBlock, shouldProxyHost } from "./flaresolverr";
|
||||
import {
|
||||
CLOUDFLARE_BLOCKED_CODE,
|
||||
fetchViaFlaresolverr,
|
||||
looksLikeCloudflareBlock,
|
||||
shouldProxyHost,
|
||||
} from "./flaresolverr";
|
||||
import {
|
||||
emptyImportedPageMetadata,
|
||||
extractNextDataJson,
|
||||
@@ -38,6 +43,7 @@ import {
|
||||
} from "./importResolvers";
|
||||
import {
|
||||
extractMakerworldBearerToken,
|
||||
MAKERWORLD_AUTH_CODE,
|
||||
MakerworldAuthError,
|
||||
MakerworldCaptchaError,
|
||||
makerworldCaptchaCooloffActive,
|
||||
@@ -146,6 +152,13 @@ async function fetchWithGuard(url: string, headers: Record<string, string>): Pro
|
||||
}
|
||||
}
|
||||
|
||||
if (res.status === 403 && looksLikeCloudflareBlock(res.headers)) {
|
||||
throw new HttpError(
|
||||
403,
|
||||
"Cloudflare blocked the request and FlareSolverr didn't get past it",
|
||||
CLOUDFLARE_BLOCKED_CODE,
|
||||
);
|
||||
}
|
||||
if (!res.ok) {
|
||||
throw new HttpError(res.status || 400, `Failed to fetch URL: ${res.statusText || res.status}`);
|
||||
}
|
||||
@@ -174,7 +187,9 @@ async function fetchMakerworldPageDesign(
|
||||
Accept: "*/*",
|
||||
...makerworldHtmlHeaders(url, cookie),
|
||||
});
|
||||
} catch {
|
||||
} catch (err) {
|
||||
// Kept apart from "couldn't read the model", so a sync hands the model back for later.
|
||||
if (err instanceof HttpError && err.code === CLOUDFLARE_BLOCKED_CODE) throw err;
|
||||
return null;
|
||||
}
|
||||
if (!isHtmlContentType(res.headers.get("content-type") || "")) return null;
|
||||
@@ -273,7 +288,7 @@ async function tryMakerworldCloudApi(url: string, body: ImportRequestBody): Prom
|
||||
throw new HttpError(429, err.message, MAKERWORLD_DOWNLOAD_LIMIT_CODE);
|
||||
}
|
||||
// 400, not 401: the frontend treats any 401 as an expired Thingport session and logs out.
|
||||
if (err instanceof MakerworldAuthError) throw new HttpError(400, err.message);
|
||||
if (err instanceof MakerworldAuthError) throw new HttpError(400, err.message, MAKERWORLD_AUTH_CODE);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,6 +55,8 @@ export const MAKERWORLD_DOWNLOAD_LIMIT_MESSAGE =
|
||||
|
||||
// The HttpError code, so batch runners can tell it from a CAPTCHA (both are 429).
|
||||
export const MAKERWORLD_DOWNLOAD_LIMIT_CODE = "MAKERWORLD_DOWNLOAD_LIMIT";
|
||||
// On the 400 a MakerworldAuthError becomes, so a queued import can tell it from a bad link.
|
||||
export const MAKERWORLD_AUTH_CODE = "MAKERWORLD_AUTH";
|
||||
|
||||
export class MakerworldDownloadLimitError extends Error {
|
||||
constructor() {
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
export {
|
||||
MAKERWORLD_CAPTCHA_MESSAGE,
|
||||
makerworldCaptchaCooloffActive,
|
||||
MAKERWORLD_AUTH_CODE,
|
||||
MakerworldAuthError,
|
||||
MakerworldCaptchaError,
|
||||
MAKERWORLD_DOWNLOAD_LIMIT_CODE,
|
||||
|
||||
@@ -84,8 +84,9 @@ async function fetchCollectionJson(
|
||||
redirect: "follow",
|
||||
signal: controller.signal,
|
||||
});
|
||||
// Checked separately, unlike the other callers: without FlareSolverr a 403 falls through to MakerworldAuthError.
|
||||
if (res.status === 403 && looksLikeCloudflareBlock(res.headers) && (await isFlaresolverrEnabled())) {
|
||||
// Cloudflare's 403 isn't MakerWorld turning the login down, which would pause the user's syncs.
|
||||
if (res.status === 403 && looksLikeCloudflareBlock(res.headers)) {
|
||||
if (!(await isFlaresolverrEnabled())) throw new MakerworldCollectionFetchError("blocked by Cloudflare");
|
||||
const solved = await fetchViaFlaresolverr(url, bearerToken ? `token=${bearerToken}` : null);
|
||||
if (!solved) return fail("blocked by Cloudflare");
|
||||
const data = extractJsonFromBrowserBody(solved.body);
|
||||
|
||||
@@ -7,8 +7,39 @@ export async function getUserMakerworldCookie(userId: string): Promise<string |
|
||||
return user?.makerworldCookie ?? null;
|
||||
}
|
||||
|
||||
export async function setUserMakerworldCookie(userId: string, cookie: string | null): Promise<boolean> {
|
||||
const trimmed = (cookie ?? "").trim();
|
||||
await prisma.user.update({ where: { id: userId }, data: { makerworldCookie: trimmed || null } });
|
||||
/** `verified`: MakerWorld just accepted it, so an earlier rejection no longer stands even if it's
|
||||
* the same cookie. A different cookie clears it too. */
|
||||
export async function setUserMakerworldCookie(
|
||||
userId: string,
|
||||
cookie: string | null,
|
||||
{ verified = false } = {},
|
||||
): Promise<boolean> {
|
||||
const trimmed = (cookie ?? "").trim() || null;
|
||||
const current = await getUserMakerworldCookie(userId);
|
||||
await prisma.user.update({
|
||||
where: { id: userId },
|
||||
data: {
|
||||
makerworldCookie: trimmed,
|
||||
...(verified || trimmed !== current ? { makerworldRejectedAt: null } : {}),
|
||||
},
|
||||
});
|
||||
return Boolean(trimmed);
|
||||
}
|
||||
|
||||
// A synced collection's error while its MakerWorld syncs wait for a new cookie.
|
||||
export const MAKERWORLD_REJECTED_ERROR = "Your MakerWorld sign-in has expired";
|
||||
|
||||
export async function markMakerworldCookieRejected(userId: string): Promise<void> {
|
||||
await prisma.user.update({ where: { id: userId }, data: { makerworldRejectedAt: new Date() } });
|
||||
}
|
||||
|
||||
export type MakerworldConnection = "connected" | "disconnected" | "expired";
|
||||
|
||||
export async function getMakerworldConnection(userId: string): Promise<MakerworldConnection> {
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: userId },
|
||||
select: { makerworldCookie: true, makerworldRejectedAt: true },
|
||||
});
|
||||
if (!user?.makerworldCookie) return "disconnected";
|
||||
return user.makerworldRejectedAt ? "expired" : "connected";
|
||||
}
|
||||
@@ -63,6 +63,10 @@ function design(designId: string) {
|
||||
const collections = new Map<string, string[] | "gone">();
|
||||
/** Designs whose download hits MakerWorld's daily limit. */
|
||||
const limited = new Set<string>();
|
||||
/** Designs MakerWorld answers with 401, as with an expired sign-in. */
|
||||
const signInRejected = new Set<string>();
|
||||
/** Designs Cloudflare stands in front of, with no FlareSolverr to get past it. */
|
||||
const cloudflared = new Set<string>();
|
||||
|
||||
function mockFetch() {
|
||||
global.fetch = vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async (input) => {
|
||||
@@ -82,7 +86,18 @@ function mockFetch() {
|
||||
return json({ id: Number(head[1]), title: "Phoenix flock" });
|
||||
}
|
||||
const model = url.match(/^https:\/\/api\.bambulab\.com\/v1\/design-service\/design\/(\d+)/);
|
||||
if (model) return json(design(model[1]));
|
||||
if (model) {
|
||||
if (signInRejected.has(model[1])) return json({ error: "unauthorized" }, 401);
|
||||
if (cloudflared.has(model[1])) return json({ error: "unavailable" }, 500);
|
||||
return json(design(model[1]));
|
||||
}
|
||||
const page = url.match(/^https:\/\/makerworld\.com\/en\/models\/(\d+)/);
|
||||
if (page && cloudflared.has(page[1])) {
|
||||
return new Response("Just a moment...", {
|
||||
status: 403,
|
||||
headers: { server: "cloudflare", "cf-mitigated": "challenge" },
|
||||
});
|
||||
}
|
||||
const profile = url.match(
|
||||
/^https:\/\/api\.bambulab\.com\/v1\/iot-service\/api\/user\/profile\/(\d+)\?model_id=US(\d+)/,
|
||||
);
|
||||
@@ -138,6 +153,8 @@ describe("synced collections", () => {
|
||||
global.fetch = originalFetch;
|
||||
mockFetch();
|
||||
limited.clear();
|
||||
signInRejected.clear();
|
||||
cloudflared.clear();
|
||||
// A finished job mustn't hold the one-running-import lock for the next test.
|
||||
await prisma.importJob.updateMany({ where: { userId, status: "RUNNING" }, data: { status: "DONE" } });
|
||||
});
|
||||
@@ -705,6 +722,144 @@ describe("synced collections", () => {
|
||||
expect(after.lastError).toMatch(/HTTP 502/);
|
||||
});
|
||||
|
||||
it("waits without unsyncing while MakerWorld isn't connected, telling the user once for all", async () => {
|
||||
const [one, two] = [newId(), newId()];
|
||||
collections.set(one, ["1"]);
|
||||
collections.set(two, ["1"]);
|
||||
const first = await enable({ url: collectionUrl(one), title: `Flock ${one}`, known_ids: [] });
|
||||
const second = await enable({ url: collectionUrl(two), title: `Flock ${two}`, known_ids: [] });
|
||||
const sync = await syncOf(first.body.collection_id);
|
||||
await request(app).patch("/api/settings/makerworld").set(auth()).send({ cookie: null });
|
||||
try {
|
||||
const calls = vi.mocked(global.fetch).mock.calls.length;
|
||||
const before = await prisma.notification.count({ where: { userId } });
|
||||
|
||||
expect(await syncCollection(sync.id)).toBe("disconnected");
|
||||
expect(await syncCollection((await syncOf(second.body.collection_id)).id)).toBe("disconnected");
|
||||
expect(vi.mocked(global.fetch).mock.calls.length).toBe(calls);
|
||||
for (const enabled of [first, second]) {
|
||||
const held = await syncOf(enabled.body.collection_id);
|
||||
expect(held.lastError).toBe("MakerWorld isn't connected");
|
||||
expect(held.lastCheckedAt).toBeNull();
|
||||
}
|
||||
// Every MakerWorld sync of the user's, in one notification.
|
||||
expect(await prisma.notification.count({ where: { userId } })).toBe(before + 1);
|
||||
const note = await latestNotification();
|
||||
expect(note.title).toMatch(/^Couldn't sync \d+ MakerWorld collections$/);
|
||||
expect(note.body).toMatch(/^MakerWorld isn't connected\. .+ won't sync until you connect it again/);
|
||||
|
||||
// Sync now still answers, for its own collection.
|
||||
expect(await runManualSync(sync.id)).toBe("disconnected");
|
||||
expect(await prisma.notification.count({ where: { userId } })).toBe(before + 2);
|
||||
expect((await latestNotification()).title).toBe(`Couldn't sync "Flock ${one}"`);
|
||||
} finally {
|
||||
await request(app).patch("/api/settings/makerworld").set(auth()).send({ cookie: "token=test-bearer" });
|
||||
}
|
||||
expect(await syncCollection(sync.id)).not.toBe("disconnected");
|
||||
expect((await syncOf(first.body.collection_id)).lastError).toBeNull();
|
||||
});
|
||||
|
||||
it("holds MakerWorld syncs once the sign-in is turned down, until the cookie changes", async () => {
|
||||
const cid = newId();
|
||||
collections.set(cid, ["1"]);
|
||||
const enabled = await enable({ url: collectionUrl(cid), title: `Flock ${cid}`, known_ids: [] });
|
||||
const sync = await syncOf(enabled.body.collection_id);
|
||||
global.fetch = vi.fn<() => Promise<Response>>(async () =>
|
||||
json({ error: "unauthorized" }, 401),
|
||||
) as unknown as typeof fetch;
|
||||
try {
|
||||
expect(await syncCollection(sync.id)).toBe("disconnected");
|
||||
expect(await prisma.collectionSync.findUnique({ where: { id: sync.id } })).not.toBeNull();
|
||||
expect((await syncOf(enabled.body.collection_id)).lastError).toBe("Your MakerWorld sign-in has expired");
|
||||
const settings = await request(app).get("/api/settings/makerworld").set(auth());
|
||||
expect(settings.body).toEqual({ configured: true, expired: true });
|
||||
|
||||
const calls = vi.mocked(global.fetch).mock.calls.length;
|
||||
expect(await syncCollection(sync.id)).toBe("disconnected");
|
||||
expect(vi.mocked(global.fetch).mock.calls.length).toBe(calls);
|
||||
|
||||
// The extension sending the same cookie again doesn't count as fixing it.
|
||||
await request(app).patch("/api/settings/makerworld").set(auth()).send({ cookie: "token=test-bearer" });
|
||||
expect(await syncCollection(sync.id)).toBe("disconnected");
|
||||
} finally {
|
||||
mockFetch();
|
||||
await request(app).patch("/api/settings/makerworld").set(auth()).send({ cookie: "token=fresh-bearer" });
|
||||
}
|
||||
expect(await syncCollection(sync.id)).not.toBe("disconnected");
|
||||
const settings = await request(app).get("/api/settings/makerworld").set(auth());
|
||||
expect(settings.body).toEqual({ configured: true, expired: false });
|
||||
await prisma.user.update({ where: { id: userId }, data: { makerworldCookie: "token=test-bearer" } });
|
||||
});
|
||||
|
||||
it("hands models back to the sync when the sign-in is turned down mid-import, for after reconnecting", async () => {
|
||||
const cid = newId();
|
||||
const [first, second, third] = [newId(), newId(), newId()];
|
||||
collections.set(cid, []);
|
||||
const enabled = await enable({ url: collectionUrl(cid), title: `Flock ${cid}`, known_ids: [] });
|
||||
const collectionId = enabled.body.collection_id;
|
||||
const sync = await syncOf(collectionId);
|
||||
collections.set(cid, [first, second, third]);
|
||||
signInRejected.add(second);
|
||||
|
||||
expect(await syncCollection(sync.id)).toBe("queued");
|
||||
expect(await prisma.print.findFirst({ where: { userId, sourceExternalId: first } })).not.toBeNull();
|
||||
const after = await syncOf(collectionId);
|
||||
expect(after.knownIds).toContain(first);
|
||||
expect(after.knownIds).not.toContain(second);
|
||||
expect(after.knownIds).not.toContain(third);
|
||||
expect(after.lastError).toBe("Your MakerWorld sign-in has expired");
|
||||
// Nothing left in the queue to retry by hand.
|
||||
const job = await prisma.importJob.findFirstOrThrow({ where: { collectionSyncId: sync.id } });
|
||||
expect(job.status).toBe("DONE");
|
||||
expect(await prisma.importJobItem.count({ where: { jobId: job.id, status: { not: "DONE" } } })).toBe(0);
|
||||
const note = await latestNotification();
|
||||
expect(note.title).toBe(`1 new model synced into "Flock ${cid}"`);
|
||||
expect(note.body).toContain("2 more models will come in on the next sync once you connect MakerWorld again");
|
||||
expect((await request(app).get("/api/settings/makerworld").set(auth())).body.expired).toBe(true);
|
||||
expect(await syncCollection(sync.id)).toBe("disconnected");
|
||||
|
||||
signInRejected.clear();
|
||||
await request(app).patch("/api/settings/makerworld").set(auth()).send({ cookie: "token=fresh-bearer" });
|
||||
try {
|
||||
expect(await syncCollection(sync.id)).toBe("queued");
|
||||
for (const id of [second, third]) {
|
||||
expect(await prisma.print.findFirst({ where: { userId, sourceExternalId: id } })).not.toBeNull();
|
||||
}
|
||||
expect((await syncOf(collectionId)).knownIds).toEqual(expect.arrayContaining([first, second, third]));
|
||||
} finally {
|
||||
await prisma.user.update({ where: { id: userId }, data: { makerworldCookie: "token=test-bearer" } });
|
||||
}
|
||||
});
|
||||
|
||||
it("hands models back to the sync when Cloudflare blocks them, without holding MakerWorld", async () => {
|
||||
const cid = newId();
|
||||
const [first, second] = [newId(), newId()];
|
||||
collections.set(cid, []);
|
||||
const enabled = await enable({ url: collectionUrl(cid), title: `Flock ${cid}`, known_ids: [] });
|
||||
const collectionId = enabled.body.collection_id;
|
||||
// The model's own profile, whose page fallback is what Cloudflare stands in front of.
|
||||
const sync = await prisma.collectionSync.update({ where: { collectionId }, data: { profileScope: "url" } });
|
||||
collections.set(cid, [first, second]);
|
||||
cloudflared.add(first);
|
||||
|
||||
expect(await syncCollection(sync.id)).toBe("queued");
|
||||
const after = await syncOf(collectionId);
|
||||
expect(after.knownIds).not.toContain(first);
|
||||
expect(after.knownIds).not.toContain(second);
|
||||
expect(after.lastError).toBe("Cloudflare blocked MakerWorld and FlareSolverr didn't get past it");
|
||||
const job = await prisma.importJob.findFirstOrThrow({ where: { collectionSyncId: sync.id } });
|
||||
expect(await prisma.importJobItem.count({ where: { jobId: job.id } })).toBe(0);
|
||||
expect((await latestNotification()).body).toContain(
|
||||
"2 more models will come in on a later sync: Cloudflare blocked",
|
||||
);
|
||||
expect((await request(app).get("/api/settings/makerworld").set(auth())).body.expired).toBe(false);
|
||||
|
||||
cloudflared.clear();
|
||||
expect(await syncCollection(sync.id)).toBe("queued");
|
||||
expect((await syncOf(collectionId)).knownIds).toEqual(expect.arrayContaining([first, second]));
|
||||
expect((await syncOf(collectionId)).lastError).toBeNull();
|
||||
});
|
||||
|
||||
it("is removed with its collection, and can be turned off without touching the collection", async () => {
|
||||
const one = await enable({ url: collectionUrl(newId()), title: `Gone ${stamp}`, known_ids: [] });
|
||||
await request(app).delete(`/api/collection/${one.body.collection_id}`).set(auth());
|
||||
|
||||
@@ -342,7 +342,8 @@ export const settingsApi = {
|
||||
return res.json();
|
||||
},
|
||||
|
||||
getMakerworld: async (): Promise<{ configured: boolean }> => {
|
||||
// `expired`: MakerWorld turned the saved cookie down; scheduled syncs wait for a new one.
|
||||
getMakerworld: async (): Promise<{ configured: boolean; expired: boolean }> => {
|
||||
const res = await fetch(`${apiBase()}/settings/makerworld`, { headers: authHeaders() });
|
||||
assertOk(res, "Failed to load MakerWorld settings");
|
||||
return res.json();
|
||||
|
||||
@@ -50,6 +50,7 @@ export default function ProvidersPanel({ isAdmin, cookie, onUpdateMakerWorld, on
|
||||
thangs: false,
|
||||
});
|
||||
const [menu, setMenu] = React.useState<{ id: ProviderId; top: number; left: number } | null>(null);
|
||||
const [makerworldExpired, setMakerworldExpired] = React.useState(false);
|
||||
const [connecting, setConnecting] = React.useState<ProviderId | null>(null);
|
||||
const [error, setError] = React.useState<string | null>(null);
|
||||
const [notice, setNotice] = React.useState<string | null>(null);
|
||||
@@ -58,7 +59,11 @@ export default function ProvidersPanel({ isAdmin, cookie, onUpdateMakerWorld, on
|
||||
let active = true;
|
||||
settingsApi
|
||||
.getMakerworld()
|
||||
.then((res) => active && setConnected((current) => ({ ...current, makerworld: res.configured })))
|
||||
.then((res) => {
|
||||
if (!active) return;
|
||||
setConnected((current) => ({ ...current, makerworld: res.configured }));
|
||||
setMakerworldExpired(res.expired);
|
||||
})
|
||||
.catch(() => undefined);
|
||||
settingsApi
|
||||
.getThingiverse()
|
||||
@@ -84,6 +89,7 @@ export default function ProvidersPanel({ isAdmin, cookie, onUpdateMakerWorld, on
|
||||
const result = await settingsApi.updateMakerworld(value);
|
||||
onUpdateMakerWorld({ cookie: value ?? "" });
|
||||
setConnected((current) => ({ ...current, makerworld: result.configured }));
|
||||
setMakerworldExpired(false);
|
||||
} else if (id === "thingiverse") {
|
||||
const result = await settingsApi.updateThingiverse(value);
|
||||
setConnected((current) => ({ ...current, thingiverse: result.configured }));
|
||||
@@ -103,7 +109,7 @@ export default function ProvidersPanel({ isAdmin, cookie, onUpdateMakerWorld, on
|
||||
if (locked) {
|
||||
setConnecting(null);
|
||||
setNotice(locked);
|
||||
} else if (connected[id] === false) {
|
||||
} else if (connected[id] === false || (id === "makerworld" && makerworldExpired)) {
|
||||
startConnect(id);
|
||||
}
|
||||
};
|
||||
@@ -137,6 +143,8 @@ export default function ProvidersPanel({ isAdmin, cookie, onUpdateMakerWorld, on
|
||||
|
||||
const menuId = menu?.id;
|
||||
const menuConnected = menuId ? Boolean(connected[menuId]) : false;
|
||||
// Still connected, so it can be disconnected, but offered a fresh connect first.
|
||||
const menuExpired = menuId === "makerworld" && menuConnected && makerworldExpired;
|
||||
const menuLocked = menuId ? lockedReason(menuId) : null;
|
||||
|
||||
return (
|
||||
@@ -146,14 +154,17 @@ export default function ProvidersPanel({ isAdmin, cookie, onUpdateMakerWorld, on
|
||||
{PROVIDERS.map(({ id, name }) => {
|
||||
const isConnected = Boolean(connected[id]);
|
||||
const locked = lockedReason(id);
|
||||
const status = connected[id] === null ? null : isConnected;
|
||||
const expired = id === "makerworld" && isConnected && makerworldExpired;
|
||||
const status = connected[id] === null ? null : isConnected && !expired;
|
||||
const tooltip = isComingSoon(id)
|
||||
? t("providers.comingSoon")
|
||||
: id === "thingiverse" && locked && !isConnected
|
||||
? locked
|
||||
: isConnected
|
||||
? t("providers.connected")
|
||||
: t("providers.notConnected");
|
||||
: expired
|
||||
? t("providers.makerworld.expired")
|
||||
: id === "thingiverse" && locked && !isConnected
|
||||
? locked
|
||||
: isConnected
|
||||
? t("providers.connected")
|
||||
: t("providers.notConnected");
|
||||
return (
|
||||
<AppTile
|
||||
key={id}
|
||||
@@ -163,7 +174,7 @@ export default function ProvidersPanel({ isAdmin, cookie, onUpdateMakerWorld, on
|
||||
badge={status === null || isComingSoon(id) ? undefined : status}
|
||||
highlighted={connecting === id || menuId === id}
|
||||
tooltip={status === null ? undefined : tooltip}
|
||||
ariaLabel={`${name}: ${isConnected ? t("providers.connected") : t("providers.notConnected")}`}
|
||||
ariaLabel={`${name}: ${expired ? t("providers.makerworld.expired") : isConnected ? t("providers.connected") : t("providers.notConnected")}`}
|
||||
onContextMenu={(event) => openMenu(id, event)}
|
||||
onActivate={() => activate(id)}
|
||||
/>
|
||||
@@ -185,12 +196,27 @@ export default function ProvidersPanel({ isAdmin, cookie, onUpdateMakerWorld, on
|
||||
status={
|
||||
isComingSoon(menuId)
|
||||
? t("providers.comingSoon")
|
||||
: menuConnected
|
||||
? t("providers.connected")
|
||||
: t("providers.notConnected")
|
||||
: menuExpired
|
||||
? t("providers.makerworld.expiredStatus")
|
||||
: menuConnected
|
||||
? t("providers.connected")
|
||||
: t("providers.notConnected")
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{menuId && menuExpired && (
|
||||
<MenuItem
|
||||
onClick={() => {
|
||||
setMenu(null);
|
||||
startConnect(menuId);
|
||||
}}
|
||||
>
|
||||
<ListItemIcon>
|
||||
<LinkIcon fontSize="small" />
|
||||
</ListItemIcon>
|
||||
<ListItemText>{t("providers.connect")}</ListItemText>
|
||||
</MenuItem>
|
||||
)}
|
||||
{menuId && (
|
||||
<MenuItem
|
||||
disabled={Boolean(menuLocked)}
|
||||
|
||||
@@ -47,6 +47,7 @@ export function UserMenu({ user, onOpenConfiguration, onOpenProfile, onLogout }:
|
||||
const avatarUrl = useGravatarUrl(user?.email, 128);
|
||||
const [anchorEl, setAnchorEl] = React.useState<HTMLElement | null>(null);
|
||||
const [makerworldConfigured, setMakerworldConfigured] = React.useState(false);
|
||||
const [makerworldExpired, setMakerworldExpired] = React.useState(false);
|
||||
const [thingiverseConfigured, setThingiverseConfigured] = React.useState(false);
|
||||
|
||||
const menuOpen = Boolean(anchorEl);
|
||||
@@ -57,7 +58,9 @@ export function UserMenu({ user, onOpenConfiguration, onOpenProfile, onLogout }:
|
||||
settingsApi
|
||||
.getMakerworld()
|
||||
.then((res) => {
|
||||
if (active) setMakerworldConfigured(res.configured);
|
||||
if (!active) return;
|
||||
setMakerworldConfigured(res.configured);
|
||||
setMakerworldExpired(res.expired);
|
||||
})
|
||||
.catch(() => undefined);
|
||||
settingsApi
|
||||
@@ -77,8 +80,8 @@ export function UserMenu({ user, onOpenConfiguration, onOpenProfile, onLogout }:
|
||||
{
|
||||
key: "makerworld",
|
||||
...IMPORT_PROVIDER_INFO.makerworld,
|
||||
connected: makerworldConfigured,
|
||||
disabledReason: t("userMenu.makerworldDisabledReason"),
|
||||
connected: makerworldConfigured && !makerworldExpired,
|
||||
disabledReason: makerworldExpired ? t("providers.makerworld.expired") : t("userMenu.makerworldDisabledReason"),
|
||||
},
|
||||
{
|
||||
key: "thingiverse",
|
||||
|
||||
@@ -1030,6 +1030,8 @@
|
||||
"disconnectMessageShared": "The {{name}} token is removed for everyone on this Thingport, and imports from {{name}} stop working until it's connected again.",
|
||||
"failed": "Failed to save -- try again.",
|
||||
"makerworld": {
|
||||
"expiredStatus": "Sign-in expired",
|
||||
"expired": "Sign-in expired. Connect MakerWorld again with a fresh cookie in Configuration → Providers.",
|
||||
"placeholder": "example: mw_session=...; mw_token=...;",
|
||||
"steps": [
|
||||
"Sign in at makerworld.com in your browser.",
|
||||
|
||||
@@ -1057,6 +1057,8 @@
|
||||
"disconnectMessageShared": "{{name}} raktas bus pašalintas visiems šio Thingport naudotojams, o importavimas iš {{name}} neveiks, kol jis vėl bus prijungtas.",
|
||||
"failed": "Nepavyko išsaugoti -- bandykite dar kartą.",
|
||||
"makerworld": {
|
||||
"expiredStatus": "Prisijungimas nebegalioja",
|
||||
"expired": "Prisijungimo galiojimas baigėsi. Vėl prijunkite MakerWorld su nauju slapuku skiltyje Konfigūracija → Šaltiniai.",
|
||||
"placeholder": "pavyzdys: mw_session=...; mw_token=...;",
|
||||
"steps": [
|
||||
"Naršyklėje prisijunkite prie makerworld.com.",
|
||||
|
||||
Reference in new issue
Block a user