diff --git a/backend/src/services/providerCollectionsService.ts b/backend/src/services/providerCollectionsService.ts index 10281b5..ce60a49 100644 --- a/backend/src/services/providerCollectionsService.ts +++ b/backend/src/services/providerCollectionsService.ts @@ -30,6 +30,8 @@ import { fetchThingiverseLikesSummary, listThingiverseUserCollections, resolveThingiverseUser, + ThingiverseAuthError, + ThingiverseRateLimitError, } from "./thingiverseApi"; export const BROWSABLE_PROVIDERS = ["makerworld", "printables", "thingiverse"] as const; @@ -93,7 +95,8 @@ export async function setProviderAccount( }); if (!user) await deletePrintablesListings(userId); } else if (provider === "thingiverse") { - const user = value ? await resolveThingiverseUser(value, await requireThingiverseToken()) : null; + const token = value ? await requireThingiverseToken() : null; + const user = token ? await providerCall(() => resolveThingiverseUser(value!, token)) : null; if (value && !user) throw new HttpError(404, `Thingiverse has no user "${value}"`); await prisma.user.update({ where: { id: userId }, data: { thingiverseUsername: user?.username ?? null } }); } else { @@ -127,6 +130,8 @@ async function providerCall(call: () => Promise): Promise { if (err instanceof MakerworldAuthError) { throw new HttpError(400, "MakerWorld didn't accept your cookie. Update it in Configuration > Providers."); } + if (err instanceof ThingiverseRateLimitError) throw new HttpError(429, err.message); + if (err instanceof ThingiverseAuthError) throw new HttpError(400, err.message); throw err; } } @@ -185,10 +190,12 @@ export async function listProviderCollections( if (!user?.thingiverseUsername) throw new HttpError(409, "Tell Thingport your Thingiverse username first"); const username = user.thingiverseUsername; // Thingiverse shows anyone's likes, so they sync like a collection; a failed count costs only them. - const [likes, collections] = await Promise.all([ - thingiverseLikesCard(username, token).catch(() => null), - listThingiverseUserCollections(username, token), - ]); + const [likes, collections] = await providerCall(() => + Promise.all([ + thingiverseLikesCard(username, token).catch(() => null), + listThingiverseUserCollections(username, token), + ]), + ); cards = [...(likes ? [likes] : []), ...collections]; } return withSyncs(userId, provider, cards); @@ -200,7 +207,8 @@ export async function lookupProviderCollection(userId: string, url: string): Pro if (!source) throw new HttpError(400, "That isn't a MakerWorld, Printables or Thingiverse collection link"); let card: ProviderCollectionCard | null; if (source.likesOf && source.provider === "thingiverse") { - card = await thingiverseLikesCard(source.likesOf, await requireThingiverseToken()); + const token = await requireThingiverseToken(); + card = await providerCall(() => thingiverseLikesCard(source.likesOf!, token)); } else if (source.likesOf) { // Printables shows likes to their owner only, so only the user's own, as Grab read them. const likes = await pushedLikes(userId); @@ -220,7 +228,8 @@ export async function lookupProviderCollection(userId: string, url: string): Pro (await pushedPrintablesCards(userId)).collections?.find((pushed) => pushed.id === source.externalId) ?? null; } else { - card = await fetchThingiverseCollectionCard(source.externalId, await requireThingiverseToken()); + const token = await requireThingiverseToken(); + card = await providerCall(() => fetchThingiverseCollectionCard(source.externalId, token)); } if (!card) throw new HttpError(404, "That collection can't be found. It may be private or deleted."); return (await withSyncs(userId, source.provider, [card]))[0]; diff --git a/backend/src/services/thingiverseApi.ts b/backend/src/services/thingiverseApi.ts index 1edaa3e..dec17ca 100644 --- a/backend/src/services/thingiverseApi.ts +++ b/backend/src/services/thingiverseApi.ts @@ -8,6 +8,7 @@ import { looksLikeCloudflareBlock, shouldProxyHost, } from "./flaresolverr"; +import { getFlaresolverrUrl } from "./settingsService"; // The official API needs an app Access Token (not a session cookie). It's still behind Cloudflare // and returns a sticky 429 challenge after a few rapid requests, hence ThingiverseRateLimitError @@ -31,12 +32,19 @@ export class ThingiverseAuthError extends Error { } } -/** Cloudflare's 429 challenge, distinct from a 404 or rejected token: the fix is to wait. */ +const FLARESOLVERR_HINTS = { + missing: " Setting up FlareSolverr in Admin Settings > Connections gets past it.", + failed: + " FlareSolverr couldn't get past it either: it may be down or unreachable. Check it in Admin Settings > Connections.", +}; + +/** Cloudflare's 429 challenge, distinct from a 404 or rejected token: the fix is to wait, or a working FlareSolverr. */ export class ThingiverseRateLimitError extends Error { - constructor() { + constructor(flaresolverr?: keyof typeof FLARESOLVERR_HINTS) { super( "Thingiverse blocked this request with a rate-limit challenge (Cloudflare). This usually " + - "clears after a while -- wait, then retry the same import.", + "clears after a while -- wait, then try again." + + (flaresolverr ? FLARESOLVERR_HINTS[flaresolverr] : ""), ); this.name = "ThingiverseRateLimitError"; } @@ -126,6 +134,10 @@ async function fetchThingiverseApi( data = null; } } + // An unsolved challenge is a 403 too; it says nothing about the token. + if (!viaBrowser && (res.status === 429 || res.headers.get("cf-mitigated") === "challenge")) { + throw new ThingiverseRateLimitError((await getFlaresolverrUrl()) ? "failed" : "missing"); + } if (res.status === 429) throw new ThingiverseRateLimitError(); if (res.status === 401 || res.status === 403) throw new ThingiverseAuthError(); return { ok: res.ok, status: res.status, data }; diff --git a/backend/tests/providerCollections.test.ts b/backend/tests/providerCollections.test.ts index 907ee41..9342c04 100644 --- a/backend/tests/providerCollections.test.ts +++ b/backend/tests/providerCollections.test.ts @@ -96,6 +96,13 @@ beforeAll(async () => { // Thingiverse's API as it answers for a user with two likes and one collection. const thingiverse = url.match(/^https:\/\/api\.thingiverse\.com(\/[^?]*)/); if (thingiverse) { + if (url.includes("access_token=rejected-token")) return json({}, 401); + if (url.includes("access_token=challenged-token")) { + return new Response("Just a moment...", { + status: 403, + headers: { "cf-mitigated": "challenge" }, + }); + } const path = thingiverse[1]; if (path === `/users/${THINGIVERSE_USER}`) return json({ name: THINGIVERSE_USER, like_count: 2 }); if (path === `/users/${THINGIVERSE_USER}/likes`) { @@ -229,6 +236,22 @@ describe("Thingiverse likes in the Browse tab", () => { thumbnails: ["https://cdn.thingiverse.com/hook.jpg", "https://cdn.thingiverse.com/clip.jpg"], }); }); + + it("says the instance's token was rejected, rather than failing", async () => { + await setThingiverseAccessToken("rejected-token"); + const list = await request(app).get("/api/provider-collections").query({ provider: "thingiverse" }).set(auth()); + expect(list.status).toBe(400); + expect(list.body.detail).toMatch(/Access Token .* was rejected/); + await setThingiverseAccessToken("test-token"); + }); + + it("tells a Cloudflare challenge from a rejected token", async () => { + await setThingiverseAccessToken("challenged-token"); + const list = await request(app).get("/api/provider-collections").query({ provider: "thingiverse" }).set(auth()); + expect(list.status).toBe(429); + expect(list.body.detail).toMatch(/Cloudflare.*Setting up FlareSolverr/); + await setThingiverseAccessToken("test-token"); + }); }); describe("turning sync on from the app", () => { diff --git a/frontend/src/api/admin.ts b/frontend/src/api/admin.ts index f66e643..9cefb22 100644 --- a/frontend/src/api/admin.ts +++ b/frontend/src/api/admin.ts @@ -35,7 +35,9 @@ export type LogAction = | "collection_item_added" | "collection_item_removed" | "collection_sync_enabled" - | "collection_sync_disabled"; + | "collection_sync_disabled" + | "ai_categorization_run" + | "ai_description_run"; export type LogEntry = { id: string; diff --git a/frontend/src/components/Layout/AppLayout.tsx b/frontend/src/components/Layout/AppLayout.tsx index e8efa8e..17f10d6 100644 --- a/frontend/src/components/Layout/AppLayout.tsx +++ b/frontend/src/components/Layout/AppLayout.tsx @@ -199,8 +199,9 @@ function AppLayoutShell({ {children} - {/* Both can run at once, so they stack here rather than each pinning itself to the bottom. */} - theme.zIndex.snackbar }}> + {/* Both can run at once, so they stack here rather than each pinning itself to the bottom. Under + dialogs and their backdrop, like the top bar. */} + theme.zIndex.appBar }}> diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index ecbcf51..2828f04 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -927,10 +927,14 @@ "collection_sync_enabled": "Collection sync turned on", "collection_sync_disabled": "Collection sync turned off", "authors_linked": "Linked missing authors", - "descriptions_refetched": "Refetched model descriptions" + "descriptions_refetched": "Refetched model descriptions", + "ai_categorization_run": "AI categorization run", + "ai_description_run": "AI description improvement run" }, "authorsLinked": "{{count}} model(s) linked", - "descriptionsRefetched": "{{email}} — {{updated}} of {{total}} description(s) changed" + "descriptionsRefetched": "{{email}} — {{updated}} of {{total}} description(s) changed", + "aiRunSummary": "{{done}} of {{total}} checked — {{suggested}} suggested, {{applied}} applied, {{failed}} failed", + "aiRunStopped": "(stopped)" }, "rendering": { "heading": "Rendering", diff --git a/frontend/src/i18n/locales/lt.json b/frontend/src/i18n/locales/lt.json index 69b9be5..b3162d3 100644 --- a/frontend/src/i18n/locales/lt.json +++ b/frontend/src/i18n/locales/lt.json @@ -935,10 +935,14 @@ "collection_sync_enabled": "Įjungtas kolekcijos sinchronizavimas", "collection_sync_disabled": "Išjungtas kolekcijos sinchronizavimas", "authors_linked": "Susieti trūkstami autoriai", - "descriptions_refetched": "Iš naujo gauti modelių aprašymai" + "descriptions_refetched": "Iš naujo gauti modelių aprašymai", + "ai_categorization_run": "AI kategorizavimo paleidimas", + "ai_description_run": "AI aprašymų tobulinimo paleidimas" }, "authorsLinked": "Susieta modelių: {{count}}", - "descriptionsRefetched": "{{email}} — pakeista aprašymų: {{updated}} iš {{total}}" + "descriptionsRefetched": "{{email}} — pakeista aprašymų: {{updated}} iš {{total}}", + "aiRunSummary": "Patikrinta {{done}} iš {{total}} — pasiūlyta {{suggested}}, pritaikyta {{applied}}, nepavyko {{failed}}", + "aiRunStopped": "(sustabdyta)" }, "rendering": { "heading": "Atvaizdavimas", diff --git a/frontend/src/pages/AiPage/AiDescriptionsPanel.tsx b/frontend/src/pages/AiPage/AiDescriptionsPanel.tsx index 55b4653..8e37a32 100644 --- a/frontend/src/pages/AiPage/AiDescriptionsPanel.tsx +++ b/frontend/src/pages/AiPage/AiDescriptionsPanel.tsx @@ -50,7 +50,7 @@ export default function AiDescriptionsPanel({ onUnauthorized, onRunStarted }: Pr const { t } = useTranslation(["app", "common"]); const showToast = useToast(); const confirm = useConfirm(); - const { running, startRun } = useAiDescriptionRun(); + const { run, running, startRun } = useAiDescriptionRun(); const [suggestions, setSuggestions] = useState<{ total: number; items: AiDescriptionSuggestion[] } | null>(null); const [applied, setApplied] = useState<{ total: number; items: AiDescriptionApplied[] } | null>(null); const [selected, setSelected] = useState([]); @@ -72,14 +72,16 @@ export default function AiDescriptionsPanel({ onUnauthorized, onRunStarted }: Pr ]); setSuggestions(pending); setApplied(done); - setSelected([]); + // Only what's still listed stays ticked, so a run adding suggestions doesn't undo a selection. + setSelected((ids) => ids.filter((id) => pending.items.some((item) => item.id === id))); }, []); - // Again whenever a run stops, since a run adds suggestions. + // On opening, as a run adds suggestions or applies rewrites, and again once it stops. + const progress = running ? `${run?.suggested ?? 0}/${run?.applied ?? 0}` : "idle"; useEffect(() => { - if (running) return; void load().catch(fail); - }, [running, load, fail]); + // oxlint-disable-next-line react/exhaustive-effect-dependencies + }, [progress, load, fail]); const act = async (work: () => Promise, done: string) => { setBusy(true); diff --git a/frontend/src/pages/LogsPage/index.tsx b/frontend/src/pages/LogsPage/index.tsx index d6043c2..2bb8e74 100644 --- a/frontend/src/pages/LogsPage/index.tsx +++ b/frontend/src/pages/LogsPage/index.tsx @@ -50,6 +50,8 @@ const ACTION_COLORS: Record = { collection_item_removed: "default", collection_sync_enabled: "info", collection_sync_disabled: "default", + ai_categorization_run: "info", + ai_description_run: "info", }; function isoDateOnly(d: Date): string { @@ -137,6 +139,18 @@ export default function LogsPage({ onUnauthorized }: Props) { const failed = typeof d.failed === "number" ? d.failed : 0; return t("adminSettings.logs.importSummary", { label, imported, failed }); } + case "ai_categorization_run": + case "ai_description_run": { + const count = (key: string) => (typeof d[key] === "number" ? d[key] : 0); + const summary = t("adminSettings.logs.aiRunSummary", { + done: count("done"), + total: count("total"), + suggested: count("suggested"), + applied: count("applied"), + failed: count("failed"), + }); + return d.cancelled === true ? `${summary} ${t("adminSettings.logs.aiRunStopped")}` : summary; + } default: return ""; } diff --git a/frontend/src/pages/ModelsPage/AiCategorizationPanel.tsx b/frontend/src/pages/ModelsPage/AiCategorizationPanel.tsx index b547a83..f26cefd 100644 --- a/frontend/src/pages/ModelsPage/AiCategorizationPanel.tsx +++ b/frontend/src/pages/ModelsPage/AiCategorizationPanel.tsx @@ -64,7 +64,7 @@ export default function AiCategorizationPanel({ const [selected, setSelected] = useState([]); const [busy, setBusy] = useState(false); const [suggestionsLoading, setSuggestionsLoading] = useState(true); - const { running, startRun } = useAiRun(); + const { run, running, startRun } = useAiRun(); const loadSuggestions = async (offset = 0, append = false) => { const result = await aiCategorizationApi.suggestions(50, offset); @@ -73,9 +73,9 @@ export default function AiCategorizationPanel({ if (!append) setSelected([]); }; - // Reloaded whenever a run stops, since a run adds suggestions. + // On opening, as a run adds suggestions, and again once it stops. + const progress = running ? (run?.suggested ?? 0) : -1; useEffect(() => { - if (running) return; let active = true; setSuggestionsLoading(true); void aiCategorizationApi @@ -84,7 +84,8 @@ export default function AiCategorizationPanel({ if (!active) return; setItems(result.items); setTotal(result.total); - setSelected([]); + // Only what's still listed stays ticked, so a run adding suggestions doesn't undo a selection. + setSelected((ids) => ids.filter((id) => result.items.some((item) => item.id === id))); }) .catch((err: unknown) => { if (err instanceof UnauthorizedError) onUnauthorized?.(); @@ -96,11 +97,11 @@ export default function AiCategorizationPanel({ return () => { active = false; }; - }, [running, onUnauthorized, showToast]); + // oxlint-disable-next-line react/exhaustive-effect-dependencies + }, [progress, onUnauthorized, showToast]); // Counts on opening, and again after a run, which moves models between the groups. useEffect(() => { - if (running) return; let active = true; void aiCategorizationApi .runState() @@ -116,6 +117,7 @@ export default function AiCategorizationPanel({ return () => { active = false; }; + // oxlint-disable-next-line react/exhaustive-effect-dependencies }, [running, onUnauthorized, showToast]); const start = async () => {