mirror of
https://github.com/TautvydasDerzinskas/Thingport.git
synced 2026-10-21 22:22:24 -05:00
feat: link import queue with an admin page and send-to-queue in the extension, plus update from source (#6)
* feat: link queue with retries and update-from-source
Pasting several model links now imports them as one background queue
job (LINKS), one at a time, with an in-place retry for transient
failures and a retry button for whatever still fails. Failed links are
kept per item, so a retry reruns only those.
An imported model can also be refreshed from its source ("Update from
source" in the model menu, "Update model in Thingport" in the
extension). Only empty fields are filled, so an edit always wins and
tags are only added; missing files come in (for MakerWorld, the print
profiles the model doesn't hold yet) and an empty image slot is
filled.
* fix: show the import queue and log its execution
Feedback from testing a mixed batch: the provider chips only showed one
of the pasted links' providers (now every provider in the batch lights
up), the queue had no view while it ran, and the pod's logs said nothing
about what was importing.
The progress bar is now clickable and opens the queue as a list: one row
per link with its status (waiting/importing/imported/failed) and why it
failed. Items carry a RUNNING state so the current link is visible, and
a finished batch with failures shows up in warning colors instead of the
quiet bar. The runner logs each link and the job summary with an
[import] prefix, in the console style the rest of the backend uses.
* feat: admin import queue page with start, pause and retry
Links can now wait in a PAUSED LINKS job until someone starts them. The
new Administration > Import queue page lists every user's batch imports
and starts, pauses, retries and deletes link-list jobs as their owner,
with the owner's saved MakerWorld cookie. Start all and Retry all failed
run jobs one at a time across users, so a bulk run never bursts the
providers from one IP.
POST /import/queue parks one link in the user's paused queue with its
own collection, profile scope and title, keeping a single "N links
waiting" notification up to date. Pausing is cooperative: the runner
finishes the link it is on, then stops, and a job cut off by a restart
can be paused and started again to finish.
* feat(extension): send to queue toggle
A "Send to queue" switch under "Extension enabled" makes the single
model panel add the link to the instance's paused import queue instead
of importing it, keeping the panel's collection and print profile
choice. Choosing files out of a zip still imports directly, since it
needs the download right away.
This commit is contained in:
1 parent
b5ba52d8bf
commit
96da36ae79
49 files changed
+4099
-108
No files matched your search
@@ -0,0 +1,24 @@
|
||||
-- A background batch import of a list of links (see importJobRunner.ts's runLinksImportJob),
|
||||
-- with per-item state so failed links can be retried.
|
||||
ALTER TYPE "ImportJobType" ADD VALUE 'LINKS';
|
||||
|
||||
CREATE TYPE "ImportJobItemStatus" AS ENUM ('PENDING', 'DONE', 'FAILED');
|
||||
|
||||
ALTER TABLE "ImportJob" ADD COLUMN "payload" JSONB;
|
||||
|
||||
CREATE TABLE "ImportJobItem" (
|
||||
"id" TEXT NOT NULL,
|
||||
"jobId" TEXT NOT NULL,
|
||||
"url" TEXT NOT NULL,
|
||||
"status" "ImportJobItemStatus" NOT NULL DEFAULT 'PENDING',
|
||||
"attempts" INTEGER NOT NULL DEFAULT 0,
|
||||
"errorMessage" TEXT,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||
|
||||
CONSTRAINT "ImportJobItem_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE INDEX "ImportJobItem_jobId_status_idx" ON "ImportJobItem"("jobId", "status");
|
||||
|
||||
ALTER TABLE "ImportJobItem" ADD CONSTRAINT "ImportJobItem_jobId_fkey" FOREIGN KEY ("jobId") REFERENCES "ImportJob"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
@@ -0,0 +1,2 @@
|
||||
-- The queue's progress UI shows which link is importing right now, not just the pending ones.
|
||||
ALTER TYPE "ImportJobItemStatus" ADD VALUE 'RUNNING';
|
||||
@@ -0,0 +1,6 @@
|
||||
-- Links sent from the extension wait in a PAUSED job until started from the admin import queue.
|
||||
ALTER TYPE "ImportJobStatus" ADD VALUE 'PAUSED';
|
||||
|
||||
ALTER TABLE "ImportJob" ADD COLUMN "notificationId" TEXT;
|
||||
|
||||
ALTER TABLE "ImportJobItem" ADD COLUMN "payload" JSONB;
|
||||
@@ -203,12 +203,16 @@ enum ImportJobType {
|
||||
COLLECTION
|
||||
ZIP
|
||||
PROFILES
|
||||
LINKS
|
||||
}
|
||||
|
||||
enum ImportJobStatus {
|
||||
RUNNING
|
||||
DONE
|
||||
ERROR
|
||||
// A LINKS job waiting for someone to start it: the extension's "send to queue" collects links
|
||||
// here, and pausing a running job parks its remaining links here too.
|
||||
PAUSED
|
||||
}
|
||||
|
||||
/// A background batch import. At most one RUNNING job per user (enforced in routes/imports.ts).
|
||||
@@ -230,12 +234,42 @@ model ImportJob {
|
||||
resultCollectionId String?
|
||||
// Set when the job created exactly one Print, so the frontend can open it directly.
|
||||
resultPrintId String?
|
||||
// The LINKS job's shared request body (never the MakerWorld cookie), so a retry can rerun it.
|
||||
payload Json?
|
||||
// The "N links waiting" notification of a PAUSED queue, updated as links are added to it.
|
||||
notificationId String?
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
items ImportJobItem[]
|
||||
|
||||
@@index([userId, status])
|
||||
}
|
||||
|
||||
enum ImportJobItemStatus {
|
||||
PENDING
|
||||
RUNNING
|
||||
DONE
|
||||
FAILED
|
||||
}
|
||||
|
||||
/// One queued URL of a LINKS job, kept so failed links can be retried after a LINKS run.
|
||||
model ImportJobItem {
|
||||
id String @id @default(cuid())
|
||||
jobId String
|
||||
job ImportJob @relation(fields: [jobId], references: [id], onDelete: Cascade)
|
||||
url String
|
||||
status ImportJobItemStatus @default(PENDING)
|
||||
attempts Int @default(0)
|
||||
errorMessage String?
|
||||
// Per-link options a queued link carries on its own (collection_id, scope, title), over the
|
||||
// job's shared payload.
|
||||
payload Json?
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
@@index([jobId, status])
|
||||
}
|
||||
|
||||
model Notification {
|
||||
id String @id @default(cuid())
|
||||
userId String
|
||||
|
||||
+59
-2
@@ -4,6 +4,7 @@ import type {
|
||||
Collection,
|
||||
Category,
|
||||
ImportJob,
|
||||
ImportJobItem,
|
||||
Notification,
|
||||
Plate,
|
||||
PreviewImage,
|
||||
@@ -16,6 +17,8 @@ import { previewImageExists, previewImagePath } from "./services/previewImageSer
|
||||
import { preparedFilename } from "./services/preparedPrint";
|
||||
import { modelPreviewGlbExists, modelPreviewGlbPath } from "./services/modelPreviewCache";
|
||||
import { buildImportSourceUrl } from "./services/importService";
|
||||
import { parseQueuedLinkOptions } from "./services/importJobRunner";
|
||||
import type { QueueJobRow } from "./services/importQueueService";
|
||||
|
||||
export type UserOut = {
|
||||
id: string;
|
||||
@@ -373,8 +376,8 @@ export function toCategoryOut(category: Category): CategoryOut {
|
||||
|
||||
export type ImportJobOut = {
|
||||
id: string;
|
||||
type: "COLLECTION" | "ZIP" | "PROFILES";
|
||||
status: "RUNNING" | "DONE" | "ERROR";
|
||||
type: "COLLECTION" | "ZIP" | "PROFILES" | "LINKS";
|
||||
status: "RUNNING" | "DONE" | "ERROR" | "PAUSED";
|
||||
source_url: string;
|
||||
source_label: string | null;
|
||||
provider: string | null;
|
||||
@@ -388,6 +391,33 @@ export type ImportJobOut = {
|
||||
result_print_id: string | null;
|
||||
};
|
||||
|
||||
/** The queue's per-link state, so the progress bar can show what's importing and what failed. */
|
||||
export type ImportJobItemOut = {
|
||||
id: string;
|
||||
url: string;
|
||||
status: "PENDING" | "RUNNING" | "DONE" | "FAILED";
|
||||
attempts: number;
|
||||
error_message: string | null;
|
||||
/** Page title, collection and profile scope of a link sent from the extension's queue. */
|
||||
title: string | null;
|
||||
collection_id: string | null;
|
||||
scope: "url" | "designer" | "all" | null;
|
||||
};
|
||||
|
||||
export function toImportJobItemOut(item: ImportJobItem): ImportJobItemOut {
|
||||
const options = parseQueuedLinkOptions(item.payload);
|
||||
return {
|
||||
id: item.id,
|
||||
url: item.url,
|
||||
status: item.status,
|
||||
attempts: item.attempts,
|
||||
error_message: item.errorMessage,
|
||||
title: options.title ?? null,
|
||||
collection_id: options.collection_id ?? null,
|
||||
scope: options.scope ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export function toImportJobOut(job: ImportJob): ImportJobOut {
|
||||
return {
|
||||
id: job.id,
|
||||
@@ -407,6 +437,33 @@ export function toImportJobOut(job: ImportJob): ImportJobOut {
|
||||
};
|
||||
}
|
||||
|
||||
/** A row of the admin import queue: any user's job, with its links counted by status. */
|
||||
export type AdminImportJobOut = ImportJobOut & {
|
||||
owner: { id: string; display_name: string; email: string };
|
||||
pending_count: number;
|
||||
running_count: number;
|
||||
done_count: number;
|
||||
failed_items_count: number;
|
||||
/** False on a RUNNING job means its runner is gone (a restart): pause it, then start it again. */
|
||||
runner_live: boolean;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
};
|
||||
|
||||
export function toAdminImportJobOut(row: QueueJobRow): AdminImportJobOut {
|
||||
return {
|
||||
...toImportJobOut(row),
|
||||
owner: { id: row.user.id, display_name: row.user.displayName, email: row.user.email },
|
||||
pending_count: row.counts.pending,
|
||||
running_count: row.counts.running,
|
||||
done_count: row.counts.done,
|
||||
failed_items_count: row.counts.failed,
|
||||
runner_live: row.runnerLive,
|
||||
created_at: row.createdAt.toISOString(),
|
||||
updated_at: row.updatedAt.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
export type NotificationOut = {
|
||||
id: string;
|
||||
title: string;
|
||||
|
||||
@@ -8,6 +8,21 @@ import { parseBody } from "../utils/validate";
|
||||
import { deleteAllPrintsForUser, getStorageUsage, listLogs, listUsersWithPrintCounts } from "../services/adminService";
|
||||
import { createLog } from "../services/auditLog";
|
||||
import { INVITATION_TTL_DAYS, inviteUser } from "../services/invitationService";
|
||||
import { listJobItems } from "../services/importJobService";
|
||||
import {
|
||||
bulkRunState,
|
||||
clearFinishedJobs,
|
||||
deleteQueueJob,
|
||||
getQueueJob,
|
||||
listQueueJobs,
|
||||
pauseAll,
|
||||
pauseLinksJob,
|
||||
removeQueueItem,
|
||||
retryAllFailed,
|
||||
startAllPaused,
|
||||
startLinksJob,
|
||||
} from "../services/importQueueService";
|
||||
import { toAdminImportJobOut, toImportJobItemOut } from "../dto";
|
||||
import { authorLinkingSummary, currentAuthorLinkingRun, startAuthorLinking } from "../services/authorLinkingService";
|
||||
|
||||
// Mounted at /api/admin (app.ts), so these guards only see admin routes.
|
||||
@@ -117,4 +132,90 @@ router.post(
|
||||
}),
|
||||
);
|
||||
|
||||
// ---- Import queue: every user's batch imports. Start/retry run as the job's owner, with the
|
||||
// owner's saved MakerWorld cookie. ----
|
||||
|
||||
router.get(
|
||||
"/import-queue",
|
||||
asyncHandler(async (_req, res) => {
|
||||
const jobs = await listQueueJobs();
|
||||
res.json({ jobs: jobs.map(toAdminImportJobOut), bulk: bulkRunState() });
|
||||
}),
|
||||
);
|
||||
|
||||
router.get(
|
||||
"/import-queue/jobs/:id/items",
|
||||
asyncHandler(async (req, res) => {
|
||||
const job = await getQueueJob(req.params.id);
|
||||
res.json((await listJobItems(job.id)).map(toImportJobItemOut));
|
||||
}),
|
||||
);
|
||||
|
||||
router.post(
|
||||
"/import-queue/jobs/:id/start",
|
||||
asyncHandler(async (req, res) => {
|
||||
await startLinksJob(await getQueueJob(req.params.id));
|
||||
res.status(202).json({ ok: true });
|
||||
}),
|
||||
);
|
||||
|
||||
router.post(
|
||||
"/import-queue/jobs/:id/retry",
|
||||
asyncHandler(async (req, res) => {
|
||||
await startLinksJob(await getQueueJob(req.params.id), { retryFailed: true });
|
||||
res.status(202).json({ ok: true });
|
||||
}),
|
||||
);
|
||||
|
||||
router.post(
|
||||
"/import-queue/jobs/:id/pause",
|
||||
asyncHandler(async (req, res) => {
|
||||
await pauseLinksJob(await getQueueJob(req.params.id));
|
||||
res.json({ ok: true });
|
||||
}),
|
||||
);
|
||||
|
||||
router.delete(
|
||||
"/import-queue/jobs/:id",
|
||||
asyncHandler(async (req, res) => {
|
||||
await deleteQueueJob(req.params.id);
|
||||
res.json({ ok: true });
|
||||
}),
|
||||
);
|
||||
|
||||
router.delete(
|
||||
"/import-queue/items/:id",
|
||||
asyncHandler(async (req, res) => {
|
||||
res.json(await removeQueueItem(req.params.id));
|
||||
}),
|
||||
);
|
||||
|
||||
router.post(
|
||||
"/import-queue/start-all",
|
||||
asyncHandler(async (_req, res) => {
|
||||
res.status(202).json({ queued: await startAllPaused(), bulk: bulkRunState() });
|
||||
}),
|
||||
);
|
||||
|
||||
router.post(
|
||||
"/import-queue/retry-failed",
|
||||
asyncHandler(async (_req, res) => {
|
||||
res.status(202).json({ queued: await retryAllFailed(), bulk: bulkRunState() });
|
||||
}),
|
||||
);
|
||||
|
||||
router.post(
|
||||
"/import-queue/pause-all",
|
||||
asyncHandler(async (_req, res) => {
|
||||
res.json({ paused: await pauseAll(), bulk: bulkRunState() });
|
||||
}),
|
||||
);
|
||||
|
||||
router.post(
|
||||
"/import-queue/clear-finished",
|
||||
asyncHandler(async (_req, res) => {
|
||||
res.json({ deleted: await clearFinishedJobs() });
|
||||
}),
|
||||
);
|
||||
|
||||
export default router;
|
||||
@@ -39,9 +39,11 @@ import { fetchPrintablesCollectionEntries, parsePrintablesCollectionUrl } from "
|
||||
import { getThingiverseAccessToken } from "../services/settingsService";
|
||||
import { getUserMakerworldCookie } from "../services/makerworldCookieService";
|
||||
import { listZipEntries } from "../services/zipService";
|
||||
import { createJob, getActiveJob, getJob } from "../services/importJobService";
|
||||
import { createJob, createJobItems, getActiveJob, getJob, listJobItems } from "../services/importJobService";
|
||||
import { queueLink, startLinksJob } from "../services/importQueueService";
|
||||
import {
|
||||
runCollectionImportJob,
|
||||
runLinksImportJob,
|
||||
runMakerworldProfilesImportJob,
|
||||
runPrintablesCollectionImportJob,
|
||||
runThingiverseCollectionImportJob,
|
||||
@@ -49,7 +51,8 @@ import {
|
||||
runZipImportJob,
|
||||
} from "../services/importJobRunner";
|
||||
import { createLog } from "../services/auditLog";
|
||||
import { toImportJobOut, toPrintOut } from "../dto";
|
||||
import { toImportJobItemOut, toImportJobOut, toPrintOut } from "../dto";
|
||||
import type { Prisma } from "@prisma/client";
|
||||
|
||||
const router = Router();
|
||||
router.use(requireAuth);
|
||||
@@ -427,6 +430,98 @@ router.get(
|
||||
}),
|
||||
);
|
||||
|
||||
const linksImportRequestSchema = importRequestSchema.extend({
|
||||
// The queue carries its own urls; this one is only there so the shared body shape still fits.
|
||||
url: z.string().optional(),
|
||||
urls: z.array(z.string()).min(1).max(500),
|
||||
scope: z.enum(["url", "designer", "all"]).default("url"),
|
||||
});
|
||||
|
||||
/** The shared request body a LINKS job reruns on a retry -- never the cookie or the captcha. */
|
||||
function linksJobPayload(body: z.infer<typeof linksImportRequestSchema>): Prisma.InputJsonValue {
|
||||
return {
|
||||
notes: body.notes ?? null,
|
||||
tags: body.tags,
|
||||
category_id: body.category_id ?? null,
|
||||
scope: body.scope,
|
||||
};
|
||||
}
|
||||
|
||||
router.post(
|
||||
"/import/links",
|
||||
requireCaptcha("import"),
|
||||
asyncHandler(async (req, res) => {
|
||||
const body = await withStoredMakerworldCookie(req.userId!, parseBody(linksImportRequestSchema, req.body));
|
||||
await assertNoActiveJob(req.userId!);
|
||||
const urls: string[] = [];
|
||||
for (const raw of body.urls) {
|
||||
const url = await normalizeImportUrl(raw.trim());
|
||||
if (url && !urls.includes(url)) urls.push(url);
|
||||
}
|
||||
if (!urls.length) throw new HttpError(400, "No links to import");
|
||||
const job = await createJob(req.userId!, "LINKS", {
|
||||
sourceUrl: urls[0],
|
||||
total: urls.length,
|
||||
payload: linksJobPayload(body),
|
||||
});
|
||||
await createJobItems(job.id, urls);
|
||||
void runLinksImportJob(job.id, req.userId!, { ...body, url: urls[0] });
|
||||
res.status(202).json({ job_id: job.id });
|
||||
}),
|
||||
);
|
||||
|
||||
/** Reruns just the links that failed, so a CAPTCHA cooloff or a cookie fix is enough to finish a
|
||||
* batch. */
|
||||
router.post(
|
||||
"/import/jobs/:id/retry",
|
||||
requireCaptcha("import"),
|
||||
asyncHandler(async (req, res) => {
|
||||
const job = await getJob(req.params.id, req.userId!);
|
||||
if (!job) throw new HttpError(404, "Import job not found");
|
||||
await startLinksJob(job, { retryFailed: true });
|
||||
res.status(202).json({ job_id: job.id });
|
||||
}),
|
||||
);
|
||||
|
||||
const queueLinkRequestSchema = z.object({
|
||||
url: z.string().min(1),
|
||||
collection_id: z.string().nullable().optional(),
|
||||
scope: z.enum(["url", "designer", "all"]).optional(),
|
||||
title: z.string().max(500).nullable().optional(),
|
||||
});
|
||||
|
||||
/** The extension's "send to queue": the link waits, paused, in the user's queue until it's started
|
||||
* from the admin import queue. Nothing is fetched now, so this is quick and never trips a CAPTCHA. */
|
||||
router.post(
|
||||
"/import/queue",
|
||||
requireCaptcha("import"),
|
||||
asyncHandler(async (req, res) => {
|
||||
const body = parseBody(queueLinkRequestSchema, req.body);
|
||||
const url = await normalizeImportUrl(body.url.trim());
|
||||
if (!url) throw new HttpError(400, "No link to queue");
|
||||
const result = await queueLink(req.userId!, url, {
|
||||
collection_id: body.collection_id ?? null,
|
||||
scope: body.scope,
|
||||
title: body.title?.trim() || null,
|
||||
});
|
||||
res.status(result.duplicate ? 200 : 201).json({
|
||||
job_id: result.job.id,
|
||||
item_id: result.itemId,
|
||||
duplicate: result.duplicate,
|
||||
waiting: result.waiting,
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
router.get(
|
||||
"/import/jobs/:id/items",
|
||||
asyncHandler(async (req, res) => {
|
||||
const job = await getJob(req.params.id, req.userId!);
|
||||
if (!job) throw new HttpError(404, "Import job not found");
|
||||
res.json((await listJobItems(job.id)).map(toImportJobItemOut));
|
||||
}),
|
||||
);
|
||||
|
||||
router.get(
|
||||
"/import/jobs/:id",
|
||||
asyncHandler(async (req, res) => {
|
||||
|
||||
@@ -20,6 +20,7 @@ import { RENDERABLE_MODEL_EXTS } from "../config";
|
||||
import { estimateDownloadSize, resolvePrintsForDownload, sendPrintsZip } from "../services/downloadZip";
|
||||
import { systemCollectionKeyForId } from "../services/collectionService";
|
||||
import { createLog } from "../services/auditLog";
|
||||
import { reimportPrint } from "../services/reimportService";
|
||||
import { isNormalizable3mf, normalize3mfStatus, normalized3mfFor } from "../services/normalized3mfCache";
|
||||
import type { Prisma } from "@prisma/client";
|
||||
|
||||
@@ -586,4 +587,28 @@ router.delete(
|
||||
}),
|
||||
);
|
||||
|
||||
const reimportRequestSchema = z.object({
|
||||
metadata: z.boolean().default(true),
|
||||
files: z.boolean().default(true),
|
||||
images: z.boolean().default(true),
|
||||
});
|
||||
|
||||
/** Refreshes a model from its source: fills empty metadata, brings in files still missing (for
|
||||
* MakerWorld, the print profiles that aren't on it yet) and fills an empty image slot. Nothing a
|
||||
* user edited by hand is overwritten. */
|
||||
router.post(
|
||||
"/print/:id/reimport",
|
||||
asyncHandler(async (req, res) => {
|
||||
const body = parseBody(reimportRequestSchema, req.body ?? {});
|
||||
const result = await reimportPrint(req.userId!, req.params.id, body);
|
||||
res.json(result);
|
||||
void createLog({
|
||||
userId: req.userId!,
|
||||
action: "model_reimported",
|
||||
targetId: req.params.id,
|
||||
details: { source_url: result.source_url, files_added: result.files_added, tags_added: result.tags_added },
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
export default router;
|
||||
@@ -12,6 +12,7 @@ export type LogAction =
|
||||
| "model_imported"
|
||||
| "import_completed"
|
||||
| "model_edited"
|
||||
| "model_reimported"
|
||||
| "model_deleted"
|
||||
| "collection_created"
|
||||
| "collection_edited"
|
||||
|
||||
@@ -2,7 +2,7 @@ import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { mapWithConcurrency, sleep } from "../utils/concurrency";
|
||||
import { IMPORT_COLLECTION_DELAY_MS, IMPORT_MAKERWORLD_CALL_DELAY_MS } from "../config";
|
||||
import { updateJob } from "./importJobService";
|
||||
import { countJobItems, getJob, getJobStatus, listJobItems, updateJob, updateJobItem } from "./importJobService";
|
||||
import { createNotification } from "./notificationService";
|
||||
import { addPrintsToCollection, findOrCreateCollectionByName } from "./collectionService";
|
||||
import { resolveMakerworldCookie } from "./importResolvers";
|
||||
@@ -27,6 +27,7 @@ import { getThingiverseAccessToken } from "./settingsService";
|
||||
import { fetchPrintablesCollectionTitle } from "./printablesApi";
|
||||
import { createLog } from "./auditLog";
|
||||
import { HttpError } from "../utils/fileUtils";
|
||||
import { prisma } from "../db";
|
||||
|
||||
// Sequential on purpose: parallel bursts of api.bambulab.com calls trip MakerWorld's CAPTCHA.
|
||||
const COLLECTION_IMPORT_CONCURRENCY = 1;
|
||||
@@ -57,6 +58,56 @@ async function markJobFailed(jobId: string, err: unknown): Promise<void> {
|
||||
await updateJob(jobId, { status: "ERROR", errorMessage: message }).catch(() => undefined);
|
||||
}
|
||||
|
||||
type ProfileImportTotals = {
|
||||
processed: number;
|
||||
imported: number;
|
||||
alreadyInLibrary: number;
|
||||
failed: number;
|
||||
printId: string | null;
|
||||
stopReason: "rateLimited" | "auth" | null;
|
||||
};
|
||||
|
||||
/** Imports each profile URL onto one print, in order. Once a CAPTCHA ("rateLimited") or auth
|
||||
* failure hits, every remaining profile fails the same way without another MakerWorld call. */
|
||||
async function importProfileUrlsSequentially(
|
||||
userId: string,
|
||||
profileUrls: string[],
|
||||
body: ImportRequestBody,
|
||||
onProgress?: (totals: ProfileImportTotals) => void,
|
||||
): Promise<ProfileImportTotals> {
|
||||
const totals: ProfileImportTotals = {
|
||||
processed: 0,
|
||||
imported: 0,
|
||||
alreadyInLibrary: 0,
|
||||
failed: 0,
|
||||
printId: null,
|
||||
stopReason: null,
|
||||
};
|
||||
for (const profileUrl of profileUrls) {
|
||||
if (totals.stopReason) {
|
||||
totals.failed++;
|
||||
} else {
|
||||
try {
|
||||
const result = await importPrintFromUrl(userId, profileUrl, {
|
||||
...body,
|
||||
url: profileUrl,
|
||||
makerworldPaceMs: IMPORT_MAKERWORLD_CALL_DELAY_MS,
|
||||
});
|
||||
totals.printId = result.print.id;
|
||||
if (result.alreadyImported) totals.alreadyInLibrary++;
|
||||
else totals.imported++;
|
||||
} catch (err) {
|
||||
totals.failed++;
|
||||
const reason = classifyImportFailure(err);
|
||||
if (reason === "rateLimited" || reason === "auth") totals.stopReason = reason;
|
||||
}
|
||||
}
|
||||
totals.processed++;
|
||||
onProgress?.(totals);
|
||||
}
|
||||
return totals;
|
||||
}
|
||||
|
||||
/** Imports several print profiles of one MakerWorld model as one model with a file per profile. */
|
||||
export async function runMakerworldProfilesImportJob(
|
||||
jobId: string,
|
||||
@@ -78,40 +129,25 @@ export async function runMakerworldProfilesImportJob(
|
||||
typeof design.title === "string" && design.title.trim() ? decodeHtmlEntities(design.title.trim()) : null;
|
||||
await updateJob(jobId, { total: profileIds.length, sourceLabel: title });
|
||||
|
||||
let processed = 0;
|
||||
let imported = 0;
|
||||
let alreadyInLibrary = 0;
|
||||
let failed = 0;
|
||||
let stopReason: "rateLimited" | "auth" | null = null;
|
||||
let printId: string | null = null;
|
||||
for (const profileId of profileIds) {
|
||||
const profileUrl = `https://makerworld.com/en/models/${parsed.designId}#profileId-${profileId}`;
|
||||
if (stopReason) {
|
||||
failed++;
|
||||
} else {
|
||||
try {
|
||||
const result = await importPrintFromUrl(userId, profileUrl, {
|
||||
...body,
|
||||
url: profileUrl,
|
||||
makerworldPaceMs: IMPORT_MAKERWORLD_CALL_DELAY_MS,
|
||||
});
|
||||
printId = result.print.id;
|
||||
if (result.alreadyImported) alreadyInLibrary++;
|
||||
else imported++;
|
||||
} catch (err) {
|
||||
failed++;
|
||||
const reason = classifyImportFailure(err);
|
||||
if (reason === "rateLimited" || reason === "auth") stopReason = reason;
|
||||
}
|
||||
}
|
||||
processed++;
|
||||
await updateJob(jobId, { processed, imported, alreadyInLibrary, failedCount: failed }).catch(() => undefined);
|
||||
}
|
||||
const totals = await importProfileUrlsSequentially(
|
||||
userId,
|
||||
profileIds.map((profileId) => `https://makerworld.com/en/models/${parsed.designId}#profileId-${profileId}`),
|
||||
body,
|
||||
(t) =>
|
||||
void updateJob(jobId, {
|
||||
processed: t.processed,
|
||||
imported: t.imported,
|
||||
alreadyInLibrary: t.alreadyInLibrary,
|
||||
failedCount: t.failed,
|
||||
}).catch(() => undefined),
|
||||
);
|
||||
const { imported, alreadyInLibrary, failed, stopReason } = totals;
|
||||
const printId = totals.printId;
|
||||
|
||||
await updateJob(jobId, {
|
||||
status: "DONE",
|
||||
resultPrintId: printId,
|
||||
processed,
|
||||
processed: totals.processed,
|
||||
imported,
|
||||
alreadyInLibrary,
|
||||
failedCount: failed,
|
||||
@@ -549,3 +585,240 @@ export async function runZipImportJob(jobId: string, userId: string, body: ZipIm
|
||||
if (tempPath) await fs.rm(tempPath, { force: true }).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
export type LinksImportJobBody = ImportRequestBody & { scope?: MakerworldProfileScope };
|
||||
|
||||
/** What a link sent from the extension's "send to queue" carries on its own item, over the job's
|
||||
* shared body: the panel's collection and profile scope, and the page title for the queue page. */
|
||||
export type QueuedLinkOptions = {
|
||||
collection_id?: string | null;
|
||||
scope?: MakerworldProfileScope;
|
||||
title?: string | null;
|
||||
};
|
||||
|
||||
export function parseQueuedLinkOptions(payload: unknown): QueuedLinkOptions {
|
||||
if (!payload || typeof payload !== "object" || Array.isArray(payload)) return {};
|
||||
const p = payload as Record<string, unknown>;
|
||||
return {
|
||||
collection_id: typeof p.collection_id === "string" && p.collection_id ? p.collection_id : null,
|
||||
scope: p.scope === "designer" || p.scope === "all" || p.scope === "url" ? p.scope : undefined,
|
||||
title: typeof p.title === "string" && p.title ? p.title : null,
|
||||
};
|
||||
}
|
||||
|
||||
// Runners alive in this process, by job. A PAUSED job's runner may still be finishing its current
|
||||
// link, and a DONE one its notification; starting the job again before then would run it twice.
|
||||
const liveRunners = new Map<string, Promise<void>>();
|
||||
|
||||
export function isRunnerLive(jobId: string): boolean {
|
||||
return liveRunners.has(jobId);
|
||||
}
|
||||
|
||||
/** Settles when the job's runner stops, or right away when none is alive. */
|
||||
export function runnerSettled(jobId: string): Promise<void> {
|
||||
return liveRunners.get(jobId) ?? Promise.resolve();
|
||||
}
|
||||
|
||||
// Three tries for a network-ish failure; a provider's own 4xx answer won't change on a retry.
|
||||
const LINKS_MAX_ATTEMPTS = 3;
|
||||
const LINKS_RETRY_DELAY_MS = 2000;
|
||||
|
||||
function isTransientImportFailure(err: unknown): boolean {
|
||||
return !(err instanceof HttpError) || err.transient === true;
|
||||
}
|
||||
|
||||
function failureMessage(err: unknown): string {
|
||||
return err instanceof Error && err.message.trim() ? err.message.trim() : "Import failed";
|
||||
}
|
||||
|
||||
/** One queued link. A MakerWorld model with a multi-profile scope imports each wanted profile onto
|
||||
* one print, like the PROFILES job; anything else is a plain single import. */
|
||||
async function importOneLink(userId: string, url: string, body: LinksImportJobBody): Promise<ProfileImportTotals> {
|
||||
const parsed = parseMakerworldModelUrl(url);
|
||||
if (parsed && body.scope && body.scope !== "url") {
|
||||
const design = await fetchMakerworldDesignForImport(
|
||||
parsed.designId,
|
||||
resolveMakerworldCookie(body),
|
||||
IMPORT_MAKERWORLD_CALL_DELAY_MS,
|
||||
);
|
||||
if (!design) throw new HttpError(400, "Couldn't read this model's print profiles from MakerWorld");
|
||||
const profileIds = selectMakerworldProfiles(design, body.scope, parsed.requestedInstanceId);
|
||||
if (!profileIds.length) throw new HttpError(400, "This model has no print profiles to import");
|
||||
return importProfileUrlsSequentially(
|
||||
userId,
|
||||
profileIds.map((profileId) => `https://makerworld.com/en/models/${parsed.designId}#profileId-${profileId}`),
|
||||
{ ...body, url },
|
||||
);
|
||||
}
|
||||
const result = await importPrintFromUrl(userId, url, { ...body, url });
|
||||
return {
|
||||
processed: 1,
|
||||
imported: result.alreadyImported ? 0 : 1,
|
||||
alreadyInLibrary: result.alreadyImported ? 1 : 0,
|
||||
failed: 0,
|
||||
printId: result.print.id,
|
||||
stopReason: null,
|
||||
};
|
||||
}
|
||||
|
||||
/** Imports a queued list of links, one at a time (parallel bursts trip MakerWorld's CAPTCHA).
|
||||
* A transient failure is retried in place; what still fails stays FAILED on its item so
|
||||
* POST /import/jobs/:id/retry can rerun just those. Also rerun by that route after a CAPTCHA
|
||||
* cooloff or a cookie update. */
|
||||
export function runLinksImportJob(jobId: string, userId: string, body: LinksImportJobBody): Promise<void> {
|
||||
if (liveRunners.has(jobId)) {
|
||||
console.warn(`[import] job ${jobId}: a runner is already working on it, not starting another`);
|
||||
return Promise.resolve();
|
||||
}
|
||||
const run = runLinks(jobId, userId, body).finally(() => {
|
||||
if (liveRunners.get(jobId) === run) liveRunners.delete(jobId);
|
||||
});
|
||||
liveRunners.set(jobId, run);
|
||||
return run;
|
||||
}
|
||||
|
||||
async function runLinks(jobId: string, userId: string, body: LinksImportJobBody): Promise<void> {
|
||||
try {
|
||||
const job = await getJob(jobId, userId);
|
||||
if (!job) throw new HttpError(404, "Import job not found");
|
||||
const items = (await listJobItems(jobId)).filter((item) => item.status === "PENDING");
|
||||
const total = await countJobItems(jobId, { in: ["PENDING", "RUNNING", "DONE", "FAILED"] });
|
||||
console.log(`[import] job ${jobId}: running ${items.length} of ${total} links`);
|
||||
|
||||
let imported = 0;
|
||||
let alreadyInLibrary = 0;
|
||||
let printId: string | null = null;
|
||||
let stopReason: "rateLimited" | "auth" | null = null;
|
||||
let processed = (await countJobItems(jobId, "DONE")) + (await countJobItems(jobId, "FAILED"));
|
||||
|
||||
for (const item of items) {
|
||||
// Pausing only flips the status; the link that's importing finishes and the rest wait.
|
||||
if ((await getJobStatus(jobId)) === "PAUSED") {
|
||||
console.log(`[import] job ${jobId}: paused, ${processed} of ${total} links processed`);
|
||||
return;
|
||||
}
|
||||
const options = parseQueuedLinkOptions(item.payload);
|
||||
const itemBody: LinksImportJobBody = options.scope ? { ...body, scope: options.scope } : body;
|
||||
await updateJobItem(item.id, { status: "RUNNING" }).catch(() => undefined);
|
||||
let attempts = item.attempts;
|
||||
let itemError: string | null = null;
|
||||
let settled = false;
|
||||
while (!settled) {
|
||||
attempts++;
|
||||
if (stopReason) {
|
||||
itemError =
|
||||
stopReason === "rateLimited"
|
||||
? "blocked by a MakerWorld CAPTCHA challenge — retry once it clears (usually 1-4 hours)"
|
||||
: "the MakerWorld session expired — update the cookie in Settings, then retry";
|
||||
settled = true;
|
||||
break;
|
||||
}
|
||||
try {
|
||||
const result = await importOneLink(userId, item.url, itemBody);
|
||||
imported += result.imported;
|
||||
alreadyInLibrary += result.alreadyInLibrary;
|
||||
if (result.printId) printId = result.printId;
|
||||
if (result.printId && options.collection_id) {
|
||||
await fileIntoCollection(userId, options.collection_id, result.printId);
|
||||
}
|
||||
if (result.failed) {
|
||||
// Some of the model's profiles got in; the rest are a manual retry's job, since
|
||||
// rerunning right away would only re-meet the same CAPTCHA.
|
||||
itemError = `${result.failed} of ${result.processed} print profiles failed`;
|
||||
if (result.stopReason) {
|
||||
stopReason = result.stopReason;
|
||||
itemError +=
|
||||
result.stopReason === "rateLimited"
|
||||
? " (MakerWorld CAPTCHA challenge — retry once it clears)"
|
||||
: " (the MakerWorld session expired — update the cookie in Settings, then retry)";
|
||||
}
|
||||
settled = true;
|
||||
} else {
|
||||
itemError = null;
|
||||
settled = true;
|
||||
}
|
||||
} catch (err) {
|
||||
itemError = failureMessage(err);
|
||||
const reason = classifyImportFailure(err);
|
||||
if (reason === "rateLimited" || reason === "auth") stopReason = reason;
|
||||
if (!isTransientImportFailure(err) || attempts >= LINKS_MAX_ATTEMPTS) {
|
||||
settled = true;
|
||||
} else {
|
||||
console.warn(
|
||||
`[import] job ${jobId}: ${item.url} failed (${itemError}), retrying (attempt ${attempts + 1} of ${LINKS_MAX_ATTEMPTS})`,
|
||||
);
|
||||
await sleep(LINKS_RETRY_DELAY_MS * attempts);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (itemError) console.error(`[import] job ${jobId}: ${item.url} failed: ${itemError}`);
|
||||
else console.log(`[import] job ${jobId}: ${item.url} imported`);
|
||||
await updateJobItem(item.id, {
|
||||
status: itemError ? "FAILED" : "DONE",
|
||||
attempts,
|
||||
errorMessage: itemError,
|
||||
}).catch(() => undefined);
|
||||
processed++;
|
||||
await updateJob(jobId, {
|
||||
processed,
|
||||
imported: job.imported + imported,
|
||||
alreadyInLibrary: job.alreadyInLibrary + alreadyInLibrary,
|
||||
failedCount: await countJobItems(jobId, "FAILED"),
|
||||
}).catch(() => undefined);
|
||||
}
|
||||
|
||||
const totalImported = job.imported + imported;
|
||||
const failedCount = await countJobItems(jobId, "FAILED");
|
||||
console.log(
|
||||
`[import] job ${jobId}: done, ${totalImported} imported, ${job.alreadyInLibrary + alreadyInLibrary} already in library, ${failedCount} failed`,
|
||||
);
|
||||
await updateJob(jobId, {
|
||||
status: "DONE",
|
||||
processed,
|
||||
imported: totalImported,
|
||||
alreadyInLibrary: job.alreadyInLibrary + alreadyInLibrary,
|
||||
failedCount,
|
||||
resultPrintId: totalImported === 1 ? (printId ?? job.resultPrintId) : null,
|
||||
});
|
||||
void createLog({
|
||||
userId,
|
||||
action: "import_completed",
|
||||
targetId: printId ?? job.resultPrintId,
|
||||
details: { provider: "links", imported, alreadyInLibrary, failed: failedCount },
|
||||
});
|
||||
|
||||
const bodyParts: string[] = [];
|
||||
if (alreadyInLibrary) bodyParts.push(`${alreadyInLibrary} already in your library`);
|
||||
if (stopReason === "rateLimited") {
|
||||
bodyParts.push(
|
||||
`the rest blocked by a MakerWorld CAPTCHA challenge — this usually clears in 1-4 hours, then retry the failed links`,
|
||||
);
|
||||
} else if (stopReason === "auth") {
|
||||
bodyParts.push(
|
||||
`the rest failed because your MakerWorld session expired — update the cookie in Settings, then retry`,
|
||||
);
|
||||
} else if (failedCount) {
|
||||
bodyParts.push(`${failedCount} failed — retry them from the import's progress bar`);
|
||||
}
|
||||
await createNotification(userId, {
|
||||
title: `Imported ${totalImported} of ${total} links`,
|
||||
body: bodyParts.length ? `${bodyParts.join(", ")}.` : null,
|
||||
externalUrl: job.sourceUrl,
|
||||
internalPath:
|
||||
totalImported === 1 && (printId ?? job.resultPrintId) ? `/models/${printId ?? job.resultPrintId}` : null,
|
||||
});
|
||||
} catch (err) {
|
||||
await markJobFailed(jobId, err);
|
||||
}
|
||||
}
|
||||
|
||||
/** The collection picked in the extension's panel. Checked against the owner at run time, since
|
||||
* it may have been deleted while the link waited in the queue; a miss doesn't fail the link. */
|
||||
async function fileIntoCollection(userId: string, collectionId: string, printId: string): Promise<void> {
|
||||
try {
|
||||
const collection = await prisma.collection.findFirst({ where: { id: collectionId, userId } });
|
||||
if (collection) await addPrintsToCollection(collection.id, [printId]);
|
||||
} catch (err) {
|
||||
console.warn(`[import] couldn't add ${printId} to collection ${collectionId}:`, err);
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,16 @@
|
||||
import { prisma } from "../db";
|
||||
import type { ImportJob, ImportJobType, Prisma } from "@prisma/client";
|
||||
import type { ImportJob, ImportJobItem, ImportJobType, Prisma } from "@prisma/client";
|
||||
|
||||
export async function getActiveJob(userId: string): Promise<ImportJob | null> {
|
||||
return prisma.importJob.findFirst({ where: { userId, status: "RUNNING" } });
|
||||
}
|
||||
|
||||
/** Read fresh on every link, so a runner sees a pause made while it works. */
|
||||
export async function getJobStatus(id: string): Promise<ImportJob["status"] | null> {
|
||||
const job = await prisma.importJob.findUnique({ where: { id }, select: { status: true } });
|
||||
return job?.status ?? null;
|
||||
}
|
||||
|
||||
export async function getJob(id: string, userId: string): Promise<ImportJob | null> {
|
||||
return prisma.importJob.findFirst({ where: { id, userId } });
|
||||
}
|
||||
@@ -12,7 +18,13 @@ export async function getJob(id: string, userId: string): Promise<ImportJob | nu
|
||||
export async function createJob(
|
||||
userId: string,
|
||||
type: ImportJobType,
|
||||
data: { sourceUrl: string; sourceLabel?: string | null; provider?: string | null; total?: number },
|
||||
data: {
|
||||
sourceUrl: string;
|
||||
sourceLabel?: string | null;
|
||||
provider?: string | null;
|
||||
total?: number;
|
||||
payload?: Prisma.InputJsonValue;
|
||||
},
|
||||
): Promise<ImportJob> {
|
||||
return prisma.importJob.create({
|
||||
data: {
|
||||
@@ -22,6 +34,7 @@ export async function createJob(
|
||||
sourceLabel: data.sourceLabel ?? null,
|
||||
provider: data.provider ?? null,
|
||||
total: data.total ?? 0,
|
||||
payload: data.payload ?? undefined,
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -29,3 +42,28 @@ export async function createJob(
|
||||
export async function updateJob(id: string, data: Prisma.ImportJobUpdateInput): Promise<void> {
|
||||
await prisma.importJob.update({ where: { id }, data });
|
||||
}
|
||||
|
||||
export async function createJobItems(jobId: string, urls: string[]): Promise<ImportJobItem[]> {
|
||||
return prisma.$transaction(urls.map((url) => prisma.importJobItem.create({ data: { jobId, url } })));
|
||||
}
|
||||
|
||||
export async function listJobItems(jobId: string): Promise<ImportJobItem[]> {
|
||||
return prisma.importJobItem.findMany({ where: { jobId }, orderBy: { createdAt: "asc" } });
|
||||
}
|
||||
|
||||
export async function updateJobItem(id: string, data: Prisma.ImportJobItemUpdateInput): Promise<void> {
|
||||
await prisma.importJobItem.update({ where: { id }, data });
|
||||
}
|
||||
|
||||
/** How a retry reruns a LINKS job: only the links that failed last time go back to PENDING. */
|
||||
export async function resetFailedItems(jobId: string): Promise<number> {
|
||||
const { count } = await prisma.importJobItem.updateMany({
|
||||
where: { jobId, status: "FAILED" },
|
||||
data: { status: "PENDING", errorMessage: null },
|
||||
});
|
||||
return count;
|
||||
}
|
||||
|
||||
export async function countJobItems(jobId: string, status: Prisma.ImportJobItemWhereInput["status"]): Promise<number> {
|
||||
return prisma.importJobItem.count({ where: { jobId, status } });
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
// The import queue behind the admin "Import queue" page and the extension's "send to queue". Links
|
||||
// sent from the extension collect in one PAUSED LINKS job per user until someone starts it; the
|
||||
// admin page starts, pauses, retries and cleans up LINKS jobs of every user.
|
||||
|
||||
import type { ImportJob, Prisma } from "@prisma/client";
|
||||
import { prisma } from "../db";
|
||||
import { HttpError } from "../utils/fileUtils";
|
||||
import { getActiveJob, resetFailedItems } from "./importJobService";
|
||||
import {
|
||||
isRunnerLive,
|
||||
runLinksImportJob,
|
||||
runnerSettled,
|
||||
type LinksImportJobBody,
|
||||
type QueuedLinkOptions,
|
||||
} from "./importJobRunner";
|
||||
import { getUserMakerworldCookie } from "./makerworldCookieService";
|
||||
|
||||
/** The shared body a LINKS job reruns with: its stored payload plus the owner's saved MakerWorld
|
||||
* cookie. The cookie is never stored on the job. */
|
||||
async function linksJobBody(job: ImportJob): Promise<LinksImportJobBody> {
|
||||
const payload = (job.payload ?? {}) as Record<string, unknown>;
|
||||
const cookie = await getUserMakerworldCookie(job.userId);
|
||||
return {
|
||||
url: job.sourceUrl,
|
||||
notes: (payload.notes as string | null) ?? null,
|
||||
tags: Array.isArray(payload.tags) ? (payload.tags as string[]) : [],
|
||||
category_id: (payload.category_id as string | null) ?? null,
|
||||
scope: payload.scope === "designer" || payload.scope === "all" ? payload.scope : "url",
|
||||
makerworld_cookie: cookie ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/** Starts a LINKS job's PENDING links, or with `retryFailed` puts its FAILED links back first.
|
||||
* Resolves once the runner is launched; `run` settles when it stops (finished or paused), which is
|
||||
* what the bulk run waits on. */
|
||||
export async function startLinksJob(
|
||||
job: ImportJob,
|
||||
opts: { retryFailed?: boolean } = {},
|
||||
): Promise<{ run: Promise<void> }> {
|
||||
if (job.type !== "LINKS") throw new HttpError(400, "Only a link-list import can be started or retried");
|
||||
if (job.status === "RUNNING" && isRunnerLive(job.id)) throw new HttpError(409, "This import is already running");
|
||||
if (!opts.retryFailed && job.status !== "PAUSED") throw new HttpError(400, "Only a paused import can be started");
|
||||
if (isRunnerLive(job.id)) {
|
||||
// A paused job's runner finishes the link it's on before it stops, which can take a while.
|
||||
if (job.status === "PAUSED") {
|
||||
throw new HttpError(409, "This import is still finishing its current link, try again shortly");
|
||||
}
|
||||
// A finished one is only writing its notification.
|
||||
await runnerSettled(job.id);
|
||||
}
|
||||
const active = await getActiveJob(job.userId);
|
||||
if (active && active.id !== job.id) throw new HttpError(409, "An import is already in progress");
|
||||
|
||||
if (opts.retryFailed) {
|
||||
const failed = await resetFailedItems(job.id);
|
||||
if (!failed) throw new HttpError(400, "This import has no failed links to retry");
|
||||
}
|
||||
// No runner is alive, so a RUNNING item was cut off (a restart, or a pause mid-link).
|
||||
await prisma.importJobItem.updateMany({ where: { jobId: job.id, status: "RUNNING" }, data: { status: "PENDING" } });
|
||||
const pending = await prisma.importJobItem.count({ where: { jobId: job.id, status: "PENDING" } });
|
||||
if (!pending) throw new HttpError(400, "This import has no links waiting to import");
|
||||
|
||||
// Conditional on the status read above, so two starts at once launch one runner.
|
||||
const { count } = await prisma.importJob.updateMany({
|
||||
where: { id: job.id, status: job.status },
|
||||
data: {
|
||||
status: "RUNNING",
|
||||
errorMessage: null,
|
||||
processed: await prisma.importJobItem.count({ where: { jobId: job.id, status: { in: ["DONE", "FAILED"] } } }),
|
||||
failedCount: await prisma.importJobItem.count({ where: { jobId: job.id, status: "FAILED" } }),
|
||||
},
|
||||
});
|
||||
if (!count) throw new HttpError(409, "This import changed while it was being started, reload and try again");
|
||||
const body = await linksJobBody(job);
|
||||
return { run: runLinksImportJob(job.id, job.userId, body) };
|
||||
}
|
||||
|
||||
/** Pausing only flips the status: the runner finishes the link it's on, then stops. */
|
||||
export async function pauseLinksJob(job: ImportJob): Promise<void> {
|
||||
if (job.type !== "LINKS") throw new HttpError(400, "Only a link-list import can be paused");
|
||||
const { count } = await prisma.importJob.updateMany({
|
||||
where: { id: job.id, status: "RUNNING" },
|
||||
data: { status: "PAUSED" },
|
||||
});
|
||||
if (!count) throw new HttpError(400, "Only a running import can be paused");
|
||||
}
|
||||
|
||||
// ---- Sending links to the queue (the extension) ----
|
||||
|
||||
// Two quick sends from the extension would otherwise each create a PAUSED job.
|
||||
const queueLocks = new Map<string, Promise<unknown>>();
|
||||
|
||||
function withUserLock<T>(userId: string, fn: () => Promise<T>): Promise<T> {
|
||||
const previous = queueLocks.get(userId) ?? Promise.resolve();
|
||||
const next = previous.catch(() => undefined).then(fn);
|
||||
queueLocks.set(userId, next);
|
||||
void next.finally(() => {
|
||||
if (queueLocks.get(userId) === next) queueLocks.delete(userId);
|
||||
});
|
||||
return next;
|
||||
}
|
||||
|
||||
function waitingTitle(count: number): string {
|
||||
return count === 1 ? "1 link waiting in the import queue" : `${count} links waiting in the import queue`;
|
||||
}
|
||||
|
||||
/** One notification per PAUSED queue, kept current: each new link refreshes its count and marks it
|
||||
* unread again instead of stacking a notification per link. */
|
||||
async function refreshQueueNotification(jobId: string): Promise<void> {
|
||||
const job = await prisma.importJob.findUnique({ where: { id: jobId }, include: { user: true } });
|
||||
if (!job || job.status !== "PAUSED") return;
|
||||
const waiting = await prisma.importJobItem.count({ where: { jobId, status: { in: ["PENDING", "FAILED"] } } });
|
||||
const isAdmin = job.user.role === "ADMIN";
|
||||
const data = {
|
||||
title: waitingTitle(waiting),
|
||||
body: isAdmin
|
||||
? "Paused until you start it from Administration > Import queue."
|
||||
: "Paused until an admin starts it from the import queue.",
|
||||
internalPath: isAdmin ? "/admin-queue" : null,
|
||||
readAt: null,
|
||||
createdAt: new Date(),
|
||||
};
|
||||
const existing = job.notificationId
|
||||
? await prisma.notification.findFirst({ where: { id: job.notificationId, userId: job.userId } })
|
||||
: null;
|
||||
if (existing) {
|
||||
await prisma.notification.update({ where: { id: existing.id }, data });
|
||||
return;
|
||||
}
|
||||
const created = await prisma.notification.create({ data: { ...data, userId: job.userId } });
|
||||
await prisma.importJob.update({ where: { id: jobId }, data: { notificationId: created.id } });
|
||||
}
|
||||
|
||||
export type QueueLinkResult = { job: ImportJob; itemId: string; duplicate: boolean; waiting: number };
|
||||
|
||||
/** Adds one link to the user's PAUSED queue, creating it on the first link. A link already waiting
|
||||
* there isn't added twice. */
|
||||
export async function queueLink(userId: string, url: string, options: QueuedLinkOptions): Promise<QueueLinkResult> {
|
||||
if (options.collection_id) {
|
||||
const collection = await prisma.collection.findFirst({ where: { id: options.collection_id, userId } });
|
||||
if (!collection) throw new HttpError(404, "Collection not found");
|
||||
}
|
||||
return withUserLock(userId, async () => {
|
||||
let job = await prisma.importJob.findFirst({
|
||||
where: { userId, type: "LINKS", status: "PAUSED" },
|
||||
orderBy: { createdAt: "asc" },
|
||||
});
|
||||
if (!job) {
|
||||
job = await prisma.importJob.create({
|
||||
data: {
|
||||
userId,
|
||||
type: "LINKS",
|
||||
status: "PAUSED",
|
||||
sourceUrl: url,
|
||||
sourceLabel: "Sent from Thingport Grab",
|
||||
payload: { notes: null, tags: [], category_id: null, scope: "url" },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const existing = await prisma.importJobItem.findFirst({
|
||||
where: { jobId: job.id, url, status: { in: ["PENDING", "FAILED"] } },
|
||||
});
|
||||
let itemId: string;
|
||||
if (existing) {
|
||||
itemId = existing.id;
|
||||
} else {
|
||||
const payload: Prisma.InputJsonValue = {
|
||||
collection_id: options.collection_id ?? null,
|
||||
scope: options.scope ?? "url",
|
||||
title: options.title ?? null,
|
||||
};
|
||||
const item = await prisma.importJobItem.create({ data: { jobId: job.id, url, payload } });
|
||||
itemId = item.id;
|
||||
job = await prisma.importJob.update({
|
||||
where: { id: job.id },
|
||||
data: { total: await prisma.importJobItem.count({ where: { jobId: job.id } }) },
|
||||
});
|
||||
}
|
||||
await refreshQueueNotification(job.id);
|
||||
const waiting = await prisma.importJobItem.count({ where: { jobId: job.id, status: "PENDING" } });
|
||||
return { job, itemId, duplicate: Boolean(existing), waiting };
|
||||
});
|
||||
}
|
||||
|
||||
// ---- Admin queue page ----
|
||||
|
||||
export type QueueJobRow = ImportJob & {
|
||||
user: { id: string; displayName: string; email: string };
|
||||
counts: { pending: number; running: number; done: number; failed: number };
|
||||
runnerLive: boolean;
|
||||
};
|
||||
|
||||
const LIST_LIMIT = 200;
|
||||
|
||||
/** Every user's batch imports, newest first. Only LINKS jobs can be started or retried, but the
|
||||
* others are listed too so the page shows everything that ran. */
|
||||
export async function listQueueJobs(): Promise<QueueJobRow[]> {
|
||||
const jobs = await prisma.importJob.findMany({
|
||||
orderBy: { createdAt: "desc" },
|
||||
take: LIST_LIMIT,
|
||||
include: { user: { select: { id: true, displayName: true, email: true } } },
|
||||
});
|
||||
const grouped = jobs.length
|
||||
? await prisma.importJobItem.groupBy({
|
||||
by: ["jobId", "status"],
|
||||
where: { jobId: { in: jobs.map((j) => j.id) } },
|
||||
_count: { status: true },
|
||||
})
|
||||
: [];
|
||||
return jobs.map((job) => {
|
||||
const counts = { pending: 0, running: 0, done: 0, failed: 0 };
|
||||
for (const g of grouped) {
|
||||
if (g.jobId !== job.id) continue;
|
||||
const key = g.status.toLowerCase() as keyof typeof counts;
|
||||
counts[key] = g._count.status;
|
||||
}
|
||||
return Object.assign(job, { counts, runnerLive: isRunnerLive(job.id) });
|
||||
});
|
||||
}
|
||||
|
||||
export async function getQueueJob(jobId: string): Promise<ImportJob> {
|
||||
const job = await prisma.importJob.findUnique({ where: { id: jobId } });
|
||||
if (!job) throw new HttpError(404, "Import job not found");
|
||||
return job;
|
||||
}
|
||||
|
||||
function assertIdle(job: ImportJob): void {
|
||||
if (job.status === "RUNNING" || isRunnerLive(job.id)) {
|
||||
throw new HttpError(409, "Pause this import first, and wait for its current link to finish");
|
||||
}
|
||||
}
|
||||
|
||||
/** A deleted queue's "N links waiting" notification would point at nothing. A started queue's
|
||||
* stays: it's the history the completion notification follows. */
|
||||
async function deleteJobAndWaitingNotification(job: ImportJob): Promise<void> {
|
||||
bulkQueue = bulkQueue.filter((entry) => entry.jobId !== job.id);
|
||||
await prisma.importJob.delete({ where: { id: job.id } });
|
||||
if (job.status === "PAUSED" && job.notificationId) {
|
||||
await prisma.notification.deleteMany({ where: { id: job.notificationId, userId: job.userId } });
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteQueueJob(jobId: string): Promise<void> {
|
||||
const job = await getQueueJob(jobId);
|
||||
assertIdle(job);
|
||||
await deleteJobAndWaitingNotification(job);
|
||||
}
|
||||
|
||||
/** Drops one link from an idle job. A job left without links is deleted with it. */
|
||||
export async function removeQueueItem(itemId: string): Promise<{ jobDeleted: boolean }> {
|
||||
const item = await prisma.importJobItem.findUnique({ where: { id: itemId }, include: { job: true } });
|
||||
if (!item) throw new HttpError(404, "Queue item not found");
|
||||
assertIdle(item.job);
|
||||
await prisma.importJobItem.delete({ where: { id: itemId } });
|
||||
const left = await prisma.importJobItem.count({ where: { jobId: item.jobId } });
|
||||
if (!left) {
|
||||
await deleteJobAndWaitingNotification(item.job);
|
||||
return { jobDeleted: true };
|
||||
}
|
||||
await prisma.importJob.update({
|
||||
where: { id: item.jobId },
|
||||
data: {
|
||||
total: left,
|
||||
processed: await prisma.importJobItem.count({ where: { jobId: item.jobId, status: { in: ["DONE", "FAILED"] } } }),
|
||||
failedCount: await prisma.importJobItem.count({ where: { jobId: item.jobId, status: "FAILED" } }),
|
||||
},
|
||||
});
|
||||
await refreshQueueNotification(item.jobId);
|
||||
return { jobDeleted: false };
|
||||
}
|
||||
|
||||
/** Finished jobs with nothing left to retry. Jobs with failed links stay, so a retry is still
|
||||
* possible. */
|
||||
export async function clearFinishedJobs(): Promise<number> {
|
||||
const { count } = await prisma.importJob.deleteMany({
|
||||
where: { status: "DONE", failedCount: 0, NOT: { id: { in: bulkQueue.map((e) => e.jobId) } } },
|
||||
});
|
||||
return count;
|
||||
}
|
||||
|
||||
// ---- Bulk runs: "Start all" and "Retry all failed" ----
|
||||
|
||||
// One job at a time, across all users: every job of a bulk run hits the same providers from the
|
||||
// same IP, and parallel bursts are what trip MakerWorld's CAPTCHA.
|
||||
type BulkEntry = { jobId: string; retryFailed: boolean };
|
||||
let bulkQueue: BulkEntry[] = [];
|
||||
let bulkCurrent: string | null = null;
|
||||
|
||||
export function bulkRunState(): { running: boolean; currentJobId: string | null; remaining: number } {
|
||||
return {
|
||||
running: bulkCurrent !== null || bulkQueue.length > 0,
|
||||
currentJobId: bulkCurrent,
|
||||
remaining: bulkQueue.length,
|
||||
};
|
||||
}
|
||||
|
||||
async function drainBulkQueue(): Promise<void> {
|
||||
if (bulkCurrent !== null) return;
|
||||
try {
|
||||
for (let entry = bulkQueue.shift(); entry; entry = bulkQueue.shift()) {
|
||||
bulkCurrent = entry.jobId;
|
||||
try {
|
||||
const job = await prisma.importJob.findUnique({ where: { id: entry.jobId } });
|
||||
if (!job) continue;
|
||||
const { run } = await startLinksJob(job, { retryFailed: entry.retryFailed });
|
||||
await run;
|
||||
} catch (err) {
|
||||
// A job that can't start (deleted, already running, nothing left) is skipped, not fatal.
|
||||
console.warn(`[import] bulk run skipped job ${entry.jobId}:`, err instanceof Error ? err.message : err);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
bulkCurrent = null;
|
||||
}
|
||||
}
|
||||
|
||||
function enqueueBulk(entries: BulkEntry[]): number {
|
||||
const known = new Set([bulkCurrent, ...bulkQueue.map((e) => e.jobId)]);
|
||||
const fresh = entries.filter((e) => !known.has(e.jobId));
|
||||
bulkQueue.push(...fresh);
|
||||
void drainBulkQueue();
|
||||
return fresh.length;
|
||||
}
|
||||
|
||||
/** Every PAUSED link-list job, oldest first. */
|
||||
export async function startAllPaused(): Promise<number> {
|
||||
const jobs = await prisma.importJob.findMany({
|
||||
where: { type: "LINKS", status: "PAUSED" },
|
||||
orderBy: { createdAt: "asc" },
|
||||
select: { id: true },
|
||||
});
|
||||
return enqueueBulk(jobs.map((j) => ({ jobId: j.id, retryFailed: false })));
|
||||
}
|
||||
|
||||
/** Every finished link-list job that has failed links, oldest first. */
|
||||
export async function retryAllFailed(): Promise<number> {
|
||||
const jobs = await prisma.importJob.findMany({
|
||||
where: { type: "LINKS", status: { in: ["DONE", "ERROR"] }, items: { some: { status: "FAILED" } } },
|
||||
orderBy: { createdAt: "asc" },
|
||||
select: { id: true },
|
||||
});
|
||||
return enqueueBulk(jobs.map((j) => ({ jobId: j.id, retryFailed: true })));
|
||||
}
|
||||
|
||||
/** Stops a bulk run: what hasn't started is dropped, and every running link-list job is paused. */
|
||||
export async function pauseAll(): Promise<number> {
|
||||
bulkQueue = [];
|
||||
const { count } = await prisma.importJob.updateMany({
|
||||
where: { type: "LINKS", status: "RUNNING" },
|
||||
data: { status: "PAUSED" },
|
||||
});
|
||||
return count;
|
||||
}
|
||||
@@ -169,7 +169,7 @@ async function fetchWithGuard(url: string, headers: Record<string, string>): Pro
|
||||
return res;
|
||||
} catch (err) {
|
||||
if (err instanceof HttpError) throw err;
|
||||
throw new HttpError(400, "Failed to reach the provided URL");
|
||||
throw new HttpError(400, "Failed to reach the provided URL", undefined, true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -541,7 +541,7 @@ const CATEGORY_SITE_CAT_IDS_FIELD = {
|
||||
|
||||
/** Auto-categorizes an import when a category's `*CatIds` for this site overlap the model's own
|
||||
* category ids. Best-effort. */
|
||||
async function resolveCategoryIdByCategory(
|
||||
export async function resolveCategoryIdByCategory(
|
||||
userId: string,
|
||||
categorySite: ImportedPageMetadata["categorySite"],
|
||||
siteCategoryIds: number[],
|
||||
@@ -625,13 +625,13 @@ async function findExistingImportedPrint(
|
||||
|
||||
type ExistingImportedPrint = NonNullable<Awaited<ReturnType<typeof findExistingImportedPrint>>>;
|
||||
|
||||
async function sha256OfFile(filePath: string): Promise<string> {
|
||||
export async function sha256OfFile(filePath: string): Promise<string> {
|
||||
const hash = crypto.createHash("sha256");
|
||||
for await (const chunk of fsSync.createReadStream(filePath)) hash.update(chunk as Buffer);
|
||||
return hash.digest("hex");
|
||||
}
|
||||
|
||||
async function plateContentSha256(plate: Plate): Promise<string | null> {
|
||||
export async function plateContentSha256(plate: Plate): Promise<string | null> {
|
||||
if (plate.contentSha256) return plate.contentSha256;
|
||||
const filePath = resolvePlateFilePath(plate);
|
||||
if (!filePath || !fsSync.existsSync(filePath)) return null;
|
||||
@@ -718,13 +718,13 @@ export async function findImportedExternalIds(
|
||||
return new Set(prints.map((p) => p.sourceExternalId).filter((id): id is string => id !== null));
|
||||
}
|
||||
|
||||
const MULTI_FILE_PLATE_EXTS = new Set([...IMPORT_ALLOWED_EXTS].filter((ext) => ext !== ".zip"));
|
||||
export const MULTI_FILE_PLATE_EXTS = new Set([...IMPORT_ALLOWED_EXTS].filter((ext) => ext !== ".zip"));
|
||||
|
||||
type PlainDownloadResult = { input: NewPlateInput } | { rateLimited: true } | null;
|
||||
export type PlainDownloadResult = { input: NewPlateInput } | { rateLimited: true } | null;
|
||||
|
||||
/** Best-effort: null on failure so one bad file doesn't fail the whole import, except a 429,
|
||||
* which is reported so the caller can say why. */
|
||||
async function downloadPlainFileToTemp(url: string, suggestedName: string): Promise<PlainDownloadResult> {
|
||||
export async function downloadPlainFileToTemp(url: string, suggestedName: string): Promise<PlainDownloadResult> {
|
||||
try {
|
||||
const res = await rawFetch(url, { "User-Agent": IMPORT_USER_AGENT, Accept: "*/*" });
|
||||
if (res.status === 429) {
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
type ImportedPageMetadata,
|
||||
} from "./importResolvers";
|
||||
import { maybeSleep, sleep } from "../utils/concurrency";
|
||||
import { HttpError } from "../utils/fileUtils";
|
||||
import {
|
||||
isCaptchaChallenge,
|
||||
makerworldCaptchaCooloffActive,
|
||||
@@ -78,8 +79,11 @@ async function fetchCloudJson(url: string, bearerToken: string): Promise<{ statu
|
||||
}
|
||||
}
|
||||
return { status: res.status, data };
|
||||
} catch {
|
||||
return null;
|
||||
} catch (err) {
|
||||
if (err instanceof HttpError) throw err;
|
||||
// A network failure, unlike a non-200 answer, is worth surfacing: falling back to page
|
||||
// scraping would only fail there too, with a less honest error.
|
||||
throw new HttpError(502, "Failed to reach the provider", undefined, true);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,333 @@
|
||||
import fs from "node:fs/promises";
|
||||
import fsSync from "node:fs";
|
||||
import { prisma } from "../db";
|
||||
import type { Plate, Print, Prisma } from "@prisma/client";
|
||||
import { HttpError } from "../utils/fileUtils";
|
||||
import { IMPORT_MAKERWORLD_CALL_DELAY_MS } from "../config";
|
||||
import {
|
||||
attachImportedPreviewImages,
|
||||
buildImportSourceUrl,
|
||||
downloadPlainFileToTemp,
|
||||
fetchMakerworldDesignForImport,
|
||||
importPrintFromUrl,
|
||||
MULTI_FILE_PLATE_EXTS,
|
||||
plateContentSha256,
|
||||
resolveCategoryIdByCategory,
|
||||
sha256OfFile,
|
||||
} from "./importService";
|
||||
import { parseMakerworldModelUrl, selectMakerworldProfiles } from "./makerworldCloudApi";
|
||||
import {
|
||||
emptyImportedPageMetadata,
|
||||
makerworldMetaFromDesign,
|
||||
resolveMakerworldCookie,
|
||||
type ImportedPageMetadata,
|
||||
} from "./importResolvers";
|
||||
import { resolveThingiverseThing, ThingiverseAuthError, ThingiverseRateLimitError } from "./thingiverseApi";
|
||||
import { parsePrintablesModelUrl, resolvePrintablesDownloadLinks, resolvePrintablesModel } from "./printablesApi";
|
||||
import { getThingiverseAccessToken } from "./settingsService";
|
||||
import { getUserMakerworldCookie } from "./makerworldCookieService";
|
||||
import { upsertAuthorFromImport } from "./authorService";
|
||||
import { addPlatesToPrint } from "./printCreation";
|
||||
|
||||
export type ReimportOptions = {
|
||||
metadata: boolean;
|
||||
files: boolean;
|
||||
images: boolean;
|
||||
};
|
||||
|
||||
/** What the refresh changed, for the caller's toast. Fields that were already filled are left
|
||||
* alone, so nothing a user edited by hand is overwritten. */
|
||||
export type ReimportResult = {
|
||||
source_url: string;
|
||||
title_filled: boolean;
|
||||
notes_filled: boolean;
|
||||
tags_added: string[];
|
||||
creator_filled: boolean;
|
||||
author_linked: boolean;
|
||||
category_filled: boolean;
|
||||
files_added: number;
|
||||
files_already_present: number;
|
||||
images_added: number;
|
||||
};
|
||||
|
||||
type SourceSnapshot = {
|
||||
meta: ImportedPageMetadata;
|
||||
/** Model files the source currently offers, each with a working download. */
|
||||
fileDownloads: { name: string; url: string }[];
|
||||
/** MakerWorld only: the profiles this design offers. */
|
||||
makerworld?: { designId: string; profileIds: string[] };
|
||||
};
|
||||
|
||||
/** Reads what the source offers right now. Files are only listed, never downloaded here. */
|
||||
async function fetchSourceSnapshot(userId: string, print: Print, sourceUrl: string): Promise<SourceSnapshot> {
|
||||
const provider = print.sourceProvider!;
|
||||
const externalId = print.sourceExternalId!;
|
||||
|
||||
if (provider === "makerworld") {
|
||||
const parsed = parseMakerworldModelUrl(sourceUrl);
|
||||
if (!parsed) throw new HttpError(400, "Not a MakerWorld model link");
|
||||
const cookie = await getUserMakerworldCookie(userId);
|
||||
const design = await fetchMakerworldDesignForImport(
|
||||
parsed.designId,
|
||||
resolveMakerworldCookie({ makerworld_cookie: cookie }),
|
||||
IMPORT_MAKERWORLD_CALL_DELAY_MS,
|
||||
);
|
||||
if (!design) throw new HttpError(400, "Couldn't read this model's print profiles from MakerWorld");
|
||||
return {
|
||||
meta: makerworldMetaFromDesign(design),
|
||||
fileDownloads: [],
|
||||
makerworld: {
|
||||
designId: parsed.designId,
|
||||
profileIds: selectMakerworldProfiles(design, "all", parsed.requestedInstanceId),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (provider === "thingiverse") {
|
||||
const accessToken = await getThingiverseAccessToken();
|
||||
if (!accessToken) {
|
||||
throw new HttpError(
|
||||
503,
|
||||
"Thingiverse import isn't configured for this instance yet -- ask an admin to add an Access Token in Admin Settings.",
|
||||
);
|
||||
}
|
||||
let resolved;
|
||||
try {
|
||||
resolved = await resolveThingiverseThing(externalId, accessToken);
|
||||
} catch (err) {
|
||||
if (err instanceof ThingiverseRateLimitError) throw new HttpError(429, err.message);
|
||||
if (err instanceof ThingiverseAuthError) throw new HttpError(400, err.message);
|
||||
throw err;
|
||||
}
|
||||
if (!resolved) {
|
||||
throw new HttpError(
|
||||
404,
|
||||
"This Thingiverse Thing could not be found, or isn't accessible with the configured Access Token.",
|
||||
);
|
||||
}
|
||||
return {
|
||||
meta: {
|
||||
...emptyImportedPageMetadata(),
|
||||
...resolved.meta,
|
||||
galleryImages: resolved.galleryImages.map((image) => ({ url: image.url, filename: image.name })),
|
||||
},
|
||||
fileDownloads: modelFilesOf(resolved.plateFiles),
|
||||
};
|
||||
}
|
||||
|
||||
if (provider === "printables") {
|
||||
const parsed = parsePrintablesModelUrl(sourceUrl);
|
||||
if (!parsed) throw new HttpError(400, "Not a Printables model link");
|
||||
const resolved = await resolvePrintablesModel(externalId);
|
||||
if (!resolved) throw new HttpError(404, "This Printables model could not be found, or isn't public.");
|
||||
const modelFiles = modelFilesOf(resolved.plateFiles);
|
||||
const links = modelFiles.length
|
||||
? await resolvePrintablesDownloadLinks(
|
||||
externalId,
|
||||
resolved.plateFiles.map((f) => f.id),
|
||||
)
|
||||
: new Map<string, string>();
|
||||
return {
|
||||
meta: {
|
||||
...emptyImportedPageMetadata(),
|
||||
...resolved.meta,
|
||||
galleryImages: resolved.galleryImages.map((image) => ({ url: image.url, filename: image.name })),
|
||||
},
|
||||
fileDownloads: resolved.plateFiles
|
||||
.filter((f) => MULTI_FILE_PLATE_EXTS.has(extensionOf(f.name)))
|
||||
.map((f) => ({ name: f.name, url: links.get(f.id) ?? "" }))
|
||||
.filter((f) => Boolean(f.url)),
|
||||
};
|
||||
}
|
||||
|
||||
throw new HttpError(400, "This model has no source that can be re-imported from");
|
||||
}
|
||||
|
||||
function extensionOf(name: string): string {
|
||||
const dot = name.lastIndexOf(".");
|
||||
return dot >= 0 ? name.slice(dot).toLowerCase() : "";
|
||||
}
|
||||
|
||||
function modelFilesOf(files: { name: string; url?: string }[]): { name: string; url: string }[] {
|
||||
return files
|
||||
.filter((f) => MULTI_FILE_PLATE_EXTS.has(extensionOf(f.name)))
|
||||
.map((f) => ({ name: f.name, url: f.url ?? "" }))
|
||||
.filter((f) => Boolean(f.url));
|
||||
}
|
||||
|
||||
/** Fills only what's empty: an edited title, note or creator always wins over the source. Tags are
|
||||
* additive, never removed. */
|
||||
async function fillEmptyMetadata(
|
||||
userId: string,
|
||||
print: Print,
|
||||
meta: ImportedPageMetadata,
|
||||
): Promise<{
|
||||
title: boolean;
|
||||
notes: boolean;
|
||||
tagsAdded: string[];
|
||||
creator: boolean;
|
||||
author: boolean;
|
||||
category: boolean;
|
||||
}> {
|
||||
const updates: Prisma.PrintUpdateInput = {};
|
||||
let title = false;
|
||||
let notes = false;
|
||||
let creator = false;
|
||||
let author = false;
|
||||
let category = false;
|
||||
let tagsAdded: string[] = [];
|
||||
|
||||
if (!print.title && meta.title) {
|
||||
updates.title = meta.title;
|
||||
title = true;
|
||||
}
|
||||
if (!print.notes && meta.description) {
|
||||
updates.notes = meta.description;
|
||||
notes = true;
|
||||
}
|
||||
const haveTags = new Set(print.tags.map((tag) => tag.toLowerCase()));
|
||||
tagsAdded = meta.tags.filter((tag) => !haveTags.has(tag.toLowerCase()));
|
||||
if (tagsAdded.length) updates.tags = [...print.tags, ...tagsAdded];
|
||||
if (!print.creator && meta.creator) {
|
||||
updates.creator = meta.creator;
|
||||
creator = true;
|
||||
}
|
||||
if (!print.authorId && meta.author) {
|
||||
const linked = await upsertAuthorFromImport(meta.author);
|
||||
if (linked) {
|
||||
updates.author = { connect: { id: linked.id } };
|
||||
author = true;
|
||||
}
|
||||
}
|
||||
if (!print.categoryId) {
|
||||
const categoryId = await resolveCategoryIdByCategory(userId, meta.categorySite, meta.siteCategoryIds);
|
||||
if (categoryId) {
|
||||
updates.category = { connect: { id: categoryId } };
|
||||
category = true;
|
||||
}
|
||||
}
|
||||
if (Object.keys(updates).length) await prisma.print.update({ where: { id: print.id }, data: updates });
|
||||
return { title, notes, tagsAdded, creator, author, category };
|
||||
}
|
||||
|
||||
/** A source file that isn't byte-identical to any plate comes in as a new plate; anything the
|
||||
* library already holds is skipped. */
|
||||
async function addMissingPlainFiles(
|
||||
userId: string,
|
||||
print: Print,
|
||||
plates: Plate[],
|
||||
downloads: { name: string; url: string }[],
|
||||
): Promise<{ added: number; alreadyPresent: number }> {
|
||||
const existingShas = new Set(
|
||||
(await Promise.all(plates.map(plateContentSha256))).filter((sha): sha is string => sha !== null),
|
||||
);
|
||||
let added = 0;
|
||||
let alreadyPresent = 0;
|
||||
for (const file of downloads) {
|
||||
const downloaded = await downloadPlainFileToTemp(file.url, file.name);
|
||||
if (!downloaded) {
|
||||
alreadyPresent++;
|
||||
continue;
|
||||
}
|
||||
if ("rateLimited" in downloaded) {
|
||||
throw new HttpError(
|
||||
429,
|
||||
"The source blocked a file download with a rate-limit challenge. Wait a while, then re-import again.",
|
||||
);
|
||||
}
|
||||
const input = downloaded.input;
|
||||
try {
|
||||
const sha = input.tempFilePath ? await sha256OfFile(input.tempFilePath) : null;
|
||||
if (sha && existingShas.has(sha)) {
|
||||
alreadyPresent++;
|
||||
continue;
|
||||
}
|
||||
await addPlatesToPrint(userId, print.id, [input]);
|
||||
if (sha) existingShas.add(sha);
|
||||
added++;
|
||||
} finally {
|
||||
if (input.tempFilePath && fsSync.existsSync(input.tempFilePath)) {
|
||||
await fs.rm(input.tempFilePath, { force: true }).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
return { added, alreadyPresent };
|
||||
}
|
||||
|
||||
/** Refreshes one imported model from its source: fills empty metadata, brings in files the library
|
||||
* doesn't hold yet (MakerWorld: the print profiles still missing), and fills an empty image slot.
|
||||
* The source is whatever `sourceProvider`/`sourceExternalId` recorded at import time. */
|
||||
export async function reimportPrint(userId: string, printId: string, opts: ReimportOptions): Promise<ReimportResult> {
|
||||
const print = await prisma.print.findFirst({
|
||||
where: { id: printId, userId },
|
||||
include: { plates: true, previewImages: true },
|
||||
});
|
||||
if (!print) throw new HttpError(404, "Model not found");
|
||||
const sourceUrl = buildImportSourceUrl(print.sourceProvider, print.sourceExternalId);
|
||||
if (!sourceUrl) throw new HttpError(400, "This model has no recorded source to re-import from");
|
||||
|
||||
const snapshot = await fetchSourceSnapshot(userId, print, sourceUrl);
|
||||
|
||||
let metadata = {
|
||||
title: false,
|
||||
notes: false,
|
||||
tagsAdded: [] as string[],
|
||||
creator: false,
|
||||
author: false,
|
||||
category: false,
|
||||
};
|
||||
if (opts.metadata) metadata = await fillEmptyMetadata(userId, print, snapshot.meta);
|
||||
|
||||
let imagesAdded = 0;
|
||||
if (opts.images && print.previewImages.length === 0) {
|
||||
await attachImportedPreviewImages(
|
||||
print.id,
|
||||
print.plates[0]?.id,
|
||||
snapshot.meta.previewImageUrl,
|
||||
snapshot.meta.galleryImages.map((image) => ({ url: image.url, filename: image.filename })),
|
||||
);
|
||||
imagesAdded = await prisma.previewImage.count({ where: { printId: print.id } });
|
||||
}
|
||||
|
||||
let filesAdded = 0;
|
||||
let filesAlreadyPresent = 0;
|
||||
if (opts.files) {
|
||||
if (snapshot.makerworld) {
|
||||
// Each profile is its own 3MF with its own settings, so a missing one is a new file on the
|
||||
// model -- the same thing importing its link does.
|
||||
const cookie = await getUserMakerworldCookie(userId);
|
||||
const have = new Set(print.plates.map((plate) => plate.sourceInstanceId).filter(Boolean));
|
||||
for (const profileId of snapshot.makerworld.profileIds) {
|
||||
if (have.has(profileId)) {
|
||||
filesAlreadyPresent++;
|
||||
continue;
|
||||
}
|
||||
const profileUrl = `https://makerworld.com/en/models/${snapshot.makerworld.designId}#profileId-${profileId}`;
|
||||
const result = await importPrintFromUrl(userId, profileUrl, {
|
||||
url: profileUrl,
|
||||
makerworld_cookie: cookie,
|
||||
makerworldPaceMs: IMPORT_MAKERWORLD_CALL_DELAY_MS,
|
||||
});
|
||||
if (result.profileAdded) filesAdded++;
|
||||
else filesAlreadyPresent++;
|
||||
}
|
||||
} else {
|
||||
const plain = await addMissingPlainFiles(userId, print, print.plates, snapshot.fileDownloads);
|
||||
filesAdded = plain.added;
|
||||
filesAlreadyPresent = plain.alreadyPresent;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
source_url: sourceUrl,
|
||||
title_filled: metadata.title,
|
||||
notes_filled: metadata.notes,
|
||||
tags_added: metadata.tagsAdded,
|
||||
creator_filled: metadata.creator,
|
||||
author_linked: metadata.author,
|
||||
category_filled: metadata.category,
|
||||
files_added: filesAdded,
|
||||
files_already_present: filesAlreadyPresent,
|
||||
images_added: imagesAdded,
|
||||
};
|
||||
}
|
||||
@@ -39,10 +39,13 @@ export class HttpError extends Error {
|
||||
status: number;
|
||||
// For the rare case the frontend must branch on why a request failed, e.g. EMAIL_NOT_VERIFIED.
|
||||
code?: string;
|
||||
constructor(status: number, message: string, code?: string) {
|
||||
// Set when the failure is worth trying again (the network, not the provider, said no).
|
||||
transient?: boolean;
|
||||
constructor(status: number, message: string, code?: string, transient?: boolean) {
|
||||
super(message);
|
||||
this.status = status;
|
||||
this.code = code;
|
||||
this.transient = transient;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,650 @@
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import request from "supertest";
|
||||
|
||||
// No pacing or DNS here. config.ts reads the pace on load, so it's hoisted.
|
||||
vi.hoisted(() => {
|
||||
process.env.IMPORT_MAKERWORLD_CALL_DELAY_MS = "0";
|
||||
});
|
||||
vi.mock("node:dns/promises", () => ({
|
||||
default: {
|
||||
resolve4: async () => ["93.184.216.34"],
|
||||
resolve6: async () => {
|
||||
throw new Error("no AAAA");
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
// Holds the runner's final notification, so a test can act while the job already reads DONE but
|
||||
// its runner hasn't returned yet.
|
||||
const notifyDelay = vi.hoisted(() => ({ ms: 0 }));
|
||||
vi.mock("../src/services/notificationService", async (importOriginal) => {
|
||||
const original = await importOriginal<typeof import("../src/services/notificationService")>();
|
||||
return {
|
||||
...original,
|
||||
createNotification: async (...args: Parameters<typeof original.createNotification>) => {
|
||||
if (notifyDelay.ms) await new Promise((resolve) => setTimeout(resolve, notifyDelay.ms));
|
||||
return original.createNotification(...args);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
import { createApp } from "../src/app";
|
||||
import { prisma } from "../src/db";
|
||||
|
||||
const app = createApp();
|
||||
const stamp = Date.now();
|
||||
const base = (stamp % 1_000_000_000) + 1000;
|
||||
const originalFetch = global.fetch;
|
||||
|
||||
type Account = { id: string; token: string };
|
||||
let admin: Account;
|
||||
let member: Account;
|
||||
let other: Account;
|
||||
const auth = (account: Account) => ({ Authorization: `Bearer ${account.token}` });
|
||||
|
||||
async function register(label: string, role: "ADMIN" | "MEMBER"): Promise<Account> {
|
||||
const email = `queue-${label}-${stamp}@example.com`;
|
||||
const res = await request(app)
|
||||
.post("/api/register")
|
||||
.send({ displayName: `Queue ${label}`, email, password: "password123" });
|
||||
if (res.status !== 200) throw new Error(`register ${label}: ${res.status} ${JSON.stringify(res.body)}`);
|
||||
await prisma.user.update({ where: { id: res.body.user.id }, data: { role } });
|
||||
// Tokens embed the role at issue time.
|
||||
const login = await request(app).post("/api/login").send({ email, password: "password123" });
|
||||
return { id: res.body.user.id, token: login.body.token };
|
||||
}
|
||||
|
||||
function json(body: unknown): Response {
|
||||
return new Response(JSON.stringify(body), { headers: { "content-type": "application/json" } });
|
||||
}
|
||||
|
||||
const DESIGNER = { uid: 1692606088, name: "Deus Cat" };
|
||||
function design(designId: string) {
|
||||
return {
|
||||
id: Number(designId),
|
||||
modelId: `US${designId}`,
|
||||
title: `Phoenix ${designId}`,
|
||||
summary: "<p>A phoenix.</p>",
|
||||
tags: ["phoenix"],
|
||||
designCreator: DESIGNER,
|
||||
defaultInstanceId: 2,
|
||||
instances: [
|
||||
{ id: 1, profileId: 101, title: "Single-colour version", instanceCreator: DESIGNER },
|
||||
{ id: 2, profileId: 102, title: "multi-colored version", instanceCreator: DESIGNER },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
// Per design id: how often it was fetched, an optional gate that holds the fetch until released,
|
||||
// and a 404 switch. inFlight/maxInFlight catch two jobs importing at once.
|
||||
const designCalls = new Map<string, number>();
|
||||
const gates = new Map<string, { held: Promise<void>; release: () => void }>();
|
||||
const gone = new Set<string>();
|
||||
let inFlight = 0;
|
||||
let maxInFlight = 0;
|
||||
|
||||
function mockFetch() {
|
||||
global.fetch = vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async (input) => {
|
||||
const url = String(input);
|
||||
const model = url.match(/^https:\/\/api\.bambulab\.com\/v1\/design-service\/design\/(\d+)/);
|
||||
if (model) {
|
||||
const id = model[1];
|
||||
designCalls.set(id, (designCalls.get(id) ?? 0) + 1);
|
||||
inFlight++;
|
||||
maxInFlight = Math.max(maxInFlight, inFlight);
|
||||
try {
|
||||
await gates.get(id)?.held;
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
} finally {
|
||||
inFlight--;
|
||||
}
|
||||
if (gone.has(id)) return new Response("gone", { status: 404 });
|
||||
return json(design(id));
|
||||
}
|
||||
const profile = url.match(/^https:\/\/api\.bambulab\.com\/v1\/iot-service\/api\/user\/profile\/(\d+)/);
|
||||
if (profile)
|
||||
return json({
|
||||
message: "success",
|
||||
url: `https://s3.example.com/phoenix-${profile[1]}.stl`,
|
||||
filename: `phoenix-${profile[1]}.stl`,
|
||||
});
|
||||
const file = url.match(/^https:\/\/s3\.example\.com\/phoenix-(\d+)\.stl$/);
|
||||
if (file)
|
||||
return new Response(`solid phoenix-${file[1]}\nendsolid\n`, {
|
||||
headers: { "content-type": "application/octet-stream" },
|
||||
});
|
||||
return new Response("not found", { status: 404 });
|
||||
}) as unknown as typeof fetch;
|
||||
}
|
||||
|
||||
function gate(id: string): () => void {
|
||||
let release!: () => void;
|
||||
const held = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
gates.set(id, { held, release });
|
||||
return () => {
|
||||
gates.delete(id);
|
||||
release();
|
||||
};
|
||||
}
|
||||
|
||||
const modelUrl = (id: number | string) => `https://makerworld.com/en/models/${id}-phoenix`;
|
||||
|
||||
function queue(account: Account, body: Record<string, unknown>) {
|
||||
return request(app).post("/api/import/queue").set(auth(account)).send(body);
|
||||
}
|
||||
|
||||
async function job(jobId: string) {
|
||||
return prisma.importJob.findUniqueOrThrow({ where: { id: jobId } });
|
||||
}
|
||||
|
||||
async function items(jobId: string) {
|
||||
return prisma.importJobItem.findMany({ where: { jobId }, orderBy: { createdAt: "asc" } });
|
||||
}
|
||||
|
||||
async function waitFor<T>(read: () => Promise<T>, done: (value: T) => boolean, timeoutMs = 15000): Promise<T> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
for (;;) {
|
||||
const value = await read();
|
||||
if (done(value) || Date.now() > deadline) return value;
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
}
|
||||
|
||||
const waitStatus = (jobId: string, status: string) =>
|
||||
waitFor(
|
||||
() => job(jobId),
|
||||
(j) => j.status === status,
|
||||
);
|
||||
|
||||
/** Each test starts from an empty queue: the admin page acts on every user's jobs, so another
|
||||
* test's leftovers would be started by "Start all" too. */
|
||||
async function clearQueue() {
|
||||
await request(app).post("/api/admin/import-queue/pause-all").set(auth(admin));
|
||||
await waitFor(
|
||||
async () => (await request(app).get("/api/admin/import-queue").set(auth(admin))).body,
|
||||
(body) => !body.bulk.running && !body.jobs.some((j: { runner_live: boolean }) => j.runner_live),
|
||||
);
|
||||
await prisma.importJob.deleteMany({ where: { type: "LINKS" } });
|
||||
// Going around the API leaves the queues' notifications behind.
|
||||
await prisma.notification.deleteMany({ where: { userId: { in: [admin.id, member.id, other.id] } } });
|
||||
}
|
||||
|
||||
describe("the import queue", () => {
|
||||
beforeAll(async () => {
|
||||
admin = await register("admin", "ADMIN");
|
||||
member = await register("member", "MEMBER");
|
||||
other = await register("other", "MEMBER");
|
||||
// Start/retry take the owner's saved cookie, never one from the request.
|
||||
for (const account of [admin, member, other]) {
|
||||
await request(app).patch("/api/settings/makerworld").set(auth(account)).send({ cookie: "token=test-bearer" });
|
||||
}
|
||||
mockFetch();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
// A failed assertion can leave a fetch held, which would keep its runner alive forever.
|
||||
for (const { release } of gates.values()) release();
|
||||
gates.clear();
|
||||
await clearQueue();
|
||||
global.fetch = originalFetch;
|
||||
mockFetch();
|
||||
designCalls.clear();
|
||||
gone.clear();
|
||||
inFlight = 0;
|
||||
maxInFlight = 0;
|
||||
notifyDelay.ms = 0;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await prisma.user.deleteMany({ where: { id: { in: [admin.id, member.id, other.id] } } });
|
||||
});
|
||||
|
||||
describe("sending links from the extension", () => {
|
||||
it("parks links in one paused job without fetching anything", async () => {
|
||||
const first = await queue(member, { url: modelUrl(base), title: "Phoenix A" });
|
||||
expect(first.status).toBe(201);
|
||||
expect(first.body).toMatchObject({ duplicate: false, waiting: 1 });
|
||||
const second = await queue(member, { url: modelUrl(base + 1) });
|
||||
expect(second.body.job_id).toBe(first.body.job_id);
|
||||
expect(second.body.waiting).toBe(2);
|
||||
|
||||
const parked = await job(first.body.job_id);
|
||||
expect(parked.status).toBe("PAUSED");
|
||||
expect(parked.total).toBe(2);
|
||||
expect((await items(parked.id)).map((i) => i.status)).toEqual(["PENDING", "PENDING"]);
|
||||
expect(designCalls.size).toBe(0);
|
||||
|
||||
// Wait a beat: nothing starts on its own.
|
||||
await new Promise((resolve) => setTimeout(resolve, 300));
|
||||
expect((await job(parked.id)).status).toBe("PAUSED");
|
||||
});
|
||||
|
||||
it("keeps one notification per queue, updated with the count and unread again", async () => {
|
||||
const first = await queue(member, { url: modelUrl(base + 2) });
|
||||
const parked = await job(first.body.job_id);
|
||||
expect(parked.notificationId).toBeTruthy();
|
||||
await prisma.notification.update({ where: { id: parked.notificationId! }, data: { readAt: new Date() } });
|
||||
|
||||
await queue(member, { url: modelUrl(base + 3) });
|
||||
const notifications = await prisma.notification.findMany({
|
||||
where: { userId: member.id, title: { contains: "waiting in the import queue" } },
|
||||
});
|
||||
expect(notifications).toHaveLength(1);
|
||||
expect(notifications[0].title).toBe("2 links waiting in the import queue");
|
||||
expect(notifications[0].readAt).toBeNull();
|
||||
// A member can't open the admin page, so the notification doesn't link there.
|
||||
expect(notifications[0].internalPath).toBeNull();
|
||||
await prisma.notification.deleteMany({ where: { userId: member.id } });
|
||||
});
|
||||
|
||||
it("links an admin's notification to the queue page", async () => {
|
||||
const res = await queue(admin, { url: modelUrl(base + 4) });
|
||||
const notification = await prisma.notification.findUnique({
|
||||
where: { id: (await job(res.body.job_id)).notificationId! },
|
||||
});
|
||||
expect(notification!.internalPath).toBe("/admin-queue");
|
||||
});
|
||||
|
||||
it("doesn't add a link that's already waiting", async () => {
|
||||
await queue(member, { url: modelUrl(base + 5) });
|
||||
const again = await queue(member, { url: modelUrl(base + 5) });
|
||||
expect(again.status).toBe(200);
|
||||
expect(again.body.duplicate).toBe(true);
|
||||
expect(await items(again.body.job_id)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("collects quick back-to-back sends into one job", async () => {
|
||||
const sends = await Promise.all(
|
||||
Array.from({ length: 6 }, (_, i) => queue(member, { url: modelUrl(base + 10 + i) })),
|
||||
);
|
||||
expect(sends.every((r) => r.status === 201)).toBe(true);
|
||||
expect(new Set(sends.map((r) => r.body.job_id)).size).toBe(1);
|
||||
const paused = await prisma.importJob.findMany({ where: { userId: member.id, status: "PAUSED" } });
|
||||
expect(paused).toHaveLength(1);
|
||||
expect(await items(paused[0].id)).toHaveLength(6);
|
||||
});
|
||||
|
||||
it("keeps each user's queue separate", async () => {
|
||||
const a = await queue(member, { url: modelUrl(base + 20) });
|
||||
const b = await queue(other, { url: modelUrl(base + 20) });
|
||||
expect(a.body.job_id).not.toBe(b.body.job_id);
|
||||
});
|
||||
|
||||
it("rejects bad input", async () => {
|
||||
expect((await queue(member, { url: "" })).status).toBe(400);
|
||||
expect((await queue(member, {})).status).toBe(400);
|
||||
expect((await queue(member, { url: modelUrl(base), scope: "everything" })).status).toBe(400);
|
||||
expect((await queue(member, { url: "not a url" })).status).toBe(400);
|
||||
expect(
|
||||
(
|
||||
await request(app)
|
||||
.post("/api/import/queue")
|
||||
.send({ url: modelUrl(base) })
|
||||
).status,
|
||||
).toBe(401);
|
||||
});
|
||||
|
||||
it("refuses someone else's collection", async () => {
|
||||
const theirs = await request(app)
|
||||
.post("/api/collections")
|
||||
.set(auth(other))
|
||||
.send({ name: `Theirs ${stamp}` });
|
||||
const res = await queue(member, { url: modelUrl(base + 21), collection_id: theirs.body.id });
|
||||
expect(res.status).toBe(404);
|
||||
expect(await prisma.importJob.count({ where: { userId: member.id } })).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the admin queue page", () => {
|
||||
it("is admin-only", async () => {
|
||||
expect((await request(app).get("/api/admin/import-queue").set(auth(member))).status).toBe(403);
|
||||
expect((await request(app).post("/api/admin/import-queue/start-all").set(auth(member))).status).toBe(403);
|
||||
});
|
||||
|
||||
it("lists every user's jobs with their owner and link counts", async () => {
|
||||
await queue(member, { url: modelUrl(base + 30), title: "Phoenix Thirty" });
|
||||
await queue(other, { url: modelUrl(base + 31) });
|
||||
const res = await request(app).get("/api/admin/import-queue").set(auth(admin));
|
||||
expect(res.status).toBe(200);
|
||||
const mine = res.body.jobs.find((j: { owner: { id: string } }) => j.owner.id === member.id);
|
||||
expect(mine).toMatchObject({ status: "PAUSED", pending_count: 1, runner_live: false, total: 1 });
|
||||
expect(res.body.jobs.some((j: { owner: { id: string } }) => j.owner.id === other.id)).toBe(true);
|
||||
|
||||
const listed = await request(app).get(`/api/admin/import-queue/jobs/${mine.id}/items`).set(auth(admin));
|
||||
expect(listed.body[0]).toMatchObject({ title: "Phoenix Thirty", scope: "url", status: "PENDING" });
|
||||
});
|
||||
|
||||
it("starts a paused job as its owner, with each link's collection and profiles", async () => {
|
||||
const collection = await request(app)
|
||||
.post("/api/collections")
|
||||
.set(auth(member))
|
||||
.send({ name: `Queued ${stamp}` });
|
||||
const res = await queue(member, {
|
||||
url: modelUrl(base + 40),
|
||||
collection_id: collection.body.id,
|
||||
scope: "all",
|
||||
});
|
||||
await queue(member, { url: modelUrl(base + 41) });
|
||||
|
||||
const start = await request(app).post(`/api/admin/import-queue/jobs/${res.body.job_id}/start`).set(auth(admin));
|
||||
expect(start.status).toBe(202);
|
||||
const finished = await waitStatus(res.body.job_id, "DONE");
|
||||
expect(finished.status).toBe("DONE");
|
||||
expect(finished.failedCount).toBe(0);
|
||||
|
||||
// The member owns the models, not the admin who pressed start.
|
||||
const withProfiles = await prisma.print.findFirst({
|
||||
where: { userId: member.id, sourceExternalId: String(base + 40) },
|
||||
include: { plates: true, collectionItems: true },
|
||||
});
|
||||
expect(withProfiles!.plates).toHaveLength(2);
|
||||
expect(withProfiles!.collectionItems.map((c) => c.collectionId)).toEqual([collection.body.id]);
|
||||
const plain = await prisma.print.findFirst({
|
||||
where: { userId: member.id, sourceExternalId: String(base + 41) },
|
||||
include: { plates: true, collectionItems: true },
|
||||
});
|
||||
expect(plain!.plates).toHaveLength(1);
|
||||
expect(plain!.collectionItems).toHaveLength(0);
|
||||
expect(await prisma.print.count({ where: { userId: admin.id } })).toBe(0);
|
||||
});
|
||||
|
||||
it("still imports a link whose collection was deleted while it waited", async () => {
|
||||
const collection = await request(app)
|
||||
.post("/api/collections")
|
||||
.set(auth(member))
|
||||
.send({ name: `Doomed ${stamp}` });
|
||||
const res = await queue(member, { url: modelUrl(base + 42), collection_id: collection.body.id });
|
||||
await prisma.collection.delete({ where: { id: collection.body.id } });
|
||||
|
||||
await request(app).post(`/api/admin/import-queue/jobs/${res.body.job_id}/start`).set(auth(admin));
|
||||
const finished = await waitStatus(res.body.job_id, "DONE");
|
||||
expect(finished.failedCount).toBe(0);
|
||||
expect(finished.imported).toBe(1);
|
||||
});
|
||||
|
||||
it("sends new links to a fresh queue once the old one started", async () => {
|
||||
const first = await queue(member, { url: modelUrl(base + 43) });
|
||||
await request(app).post(`/api/admin/import-queue/jobs/${first.body.job_id}/start`).set(auth(admin));
|
||||
const second = await queue(member, { url: modelUrl(base + 44) });
|
||||
expect(second.body.job_id).not.toBe(first.body.job_id);
|
||||
expect((await job(second.body.job_id)).status).toBe("PAUSED");
|
||||
await waitStatus(first.body.job_id, "DONE");
|
||||
});
|
||||
|
||||
it("refuses to start a job that isn't paused, or while the owner has another import running", async () => {
|
||||
const release = gate(String(base + 50));
|
||||
const a = await queue(member, { url: modelUrl(base + 50) });
|
||||
await request(app).post(`/api/admin/import-queue/jobs/${a.body.job_id}/start`).set(auth(admin));
|
||||
await waitFor(
|
||||
async () => designCalls.get(String(base + 50)) ?? 0,
|
||||
(n) => n > 0,
|
||||
);
|
||||
|
||||
const again = await request(app).post(`/api/admin/import-queue/jobs/${a.body.job_id}/start`).set(auth(admin));
|
||||
expect(again.status).toBe(409);
|
||||
|
||||
// A second queue of the same owner can't run next to the first.
|
||||
const b = await queue(member, { url: modelUrl(base + 51) });
|
||||
const blocked = await request(app).post(`/api/admin/import-queue/jobs/${b.body.job_id}/start`).set(auth(admin));
|
||||
expect(blocked.status).toBe(409);
|
||||
expect((await job(b.body.job_id)).status).toBe("PAUSED");
|
||||
|
||||
release();
|
||||
await waitStatus(a.body.job_id, "DONE");
|
||||
const done = await request(app).post(`/api/admin/import-queue/jobs/${a.body.job_id}/start`).set(auth(admin));
|
||||
expect(done.status).toBe(400);
|
||||
});
|
||||
|
||||
it("pauses after the current link and resumes where it stopped", async () => {
|
||||
const ids = [base + 60, base + 61, base + 62];
|
||||
const release = gate(String(ids[0]));
|
||||
let res!: request.Response;
|
||||
for (const id of ids) res = await queue(member, { url: modelUrl(id) });
|
||||
const jobId = res.body.job_id;
|
||||
|
||||
await request(app).post(`/api/admin/import-queue/jobs/${jobId}/start`).set(auth(admin));
|
||||
await waitFor(
|
||||
async () => designCalls.get(String(ids[0])) ?? 0,
|
||||
(n) => n > 0,
|
||||
);
|
||||
const pause = await request(app).post(`/api/admin/import-queue/jobs/${jobId}/pause`).set(auth(admin));
|
||||
expect(pause.status).toBe(200);
|
||||
|
||||
// Its runner is still on the first link, so starting now would run the job twice.
|
||||
const early = await request(app).post(`/api/admin/import-queue/jobs/${jobId}/start`).set(auth(admin));
|
||||
expect(early.status).toBe(409);
|
||||
|
||||
release();
|
||||
await waitFor(
|
||||
async () => (await request(app).get("/api/admin/import-queue").set(auth(admin))).body,
|
||||
(body) => !body.jobs.find((j: { id: string }) => j.id === jobId).runner_live,
|
||||
);
|
||||
expect((await job(jobId)).status).toBe("PAUSED");
|
||||
expect((await items(jobId)).map((i) => i.status)).toEqual(["DONE", "PENDING", "PENDING"]);
|
||||
expect(designCalls.get(String(ids[1]))).toBeUndefined();
|
||||
|
||||
const resume = await request(app).post(`/api/admin/import-queue/jobs/${jobId}/start`).set(auth(admin));
|
||||
expect(resume.status).toBe(202);
|
||||
const finished = await waitStatus(jobId, "DONE");
|
||||
expect(finished.imported).toBe(3);
|
||||
expect(finished.processed).toBe(3);
|
||||
// Nothing was downloaded twice.
|
||||
for (const id of ids) expect(designCalls.get(String(id))).toBe(1);
|
||||
});
|
||||
|
||||
it("recovers a job left RUNNING by a restart: pause, then start", async () => {
|
||||
const res = await queue(member, { url: modelUrl(base + 70) });
|
||||
const jobId = res.body.job_id;
|
||||
// What a crash mid-link leaves behind: no runner, the job and its link still RUNNING.
|
||||
await prisma.importJob.update({ where: { id: jobId }, data: { status: "RUNNING" } });
|
||||
await prisma.importJobItem.updateMany({ where: { jobId }, data: { status: "RUNNING" } });
|
||||
|
||||
const listed = await request(app).get("/api/admin/import-queue").set(auth(admin));
|
||||
expect(listed.body.jobs.find((j: { id: string }) => j.id === jobId).runner_live).toBe(false);
|
||||
|
||||
expect((await request(app).post(`/api/admin/import-queue/jobs/${jobId}/start`).set(auth(admin))).status).toBe(
|
||||
400,
|
||||
);
|
||||
expect((await request(app).post(`/api/admin/import-queue/jobs/${jobId}/pause`).set(auth(admin))).status).toBe(
|
||||
200,
|
||||
);
|
||||
expect((await request(app).post(`/api/admin/import-queue/jobs/${jobId}/start`).set(auth(admin))).status).toBe(
|
||||
202,
|
||||
);
|
||||
const finished = await waitStatus(jobId, "DONE");
|
||||
expect(finished.imported).toBe(1);
|
||||
});
|
||||
|
||||
it("retries just the failed links of a job", async () => {
|
||||
gone.add(String(base + 81));
|
||||
await queue(member, { url: modelUrl(base + 80) });
|
||||
const res = await queue(member, { url: modelUrl(base + 81) });
|
||||
await request(app).post(`/api/admin/import-queue/jobs/${res.body.job_id}/start`).set(auth(admin));
|
||||
const first = await waitStatus(res.body.job_id, "DONE");
|
||||
expect(first.failedCount).toBe(1);
|
||||
|
||||
gone.clear();
|
||||
const retry = await request(app).post(`/api/admin/import-queue/jobs/${res.body.job_id}/retry`).set(auth(admin));
|
||||
expect(retry.status).toBe(202);
|
||||
const second = await waitFor(
|
||||
() => job(res.body.job_id),
|
||||
(j) => j.status === "DONE" && j.failedCount === 0,
|
||||
);
|
||||
expect(second.imported).toBe(2);
|
||||
expect(designCalls.get(String(base + 80))).toBe(1);
|
||||
});
|
||||
|
||||
it("retries a finished job right away, while its runner is still writing the notification", async () => {
|
||||
gone.add(String(base + 85));
|
||||
const res = await queue(member, { url: modelUrl(base + 85) });
|
||||
const jobId = res.body.job_id;
|
||||
notifyDelay.ms = 500;
|
||||
await request(app).post(`/api/admin/import-queue/jobs/${jobId}/start`).set(auth(admin));
|
||||
// Poll the DB directly, so the retry lands as soon as the job reads DONE.
|
||||
await waitFor(
|
||||
() => job(jobId),
|
||||
(j) => j.status === "DONE",
|
||||
);
|
||||
expect(
|
||||
(await request(app).get("/api/admin/import-queue").set(auth(admin))).body.jobs.find(
|
||||
(j: { id: string }) => j.id === jobId,
|
||||
).runner_live,
|
||||
).toBe(true);
|
||||
notifyDelay.ms = 0;
|
||||
gone.clear();
|
||||
const retry = await request(app).post(`/api/admin/import-queue/jobs/${jobId}/retry`).set(auth(admin));
|
||||
expect(retry.status).toBe(202);
|
||||
const finished = await waitFor(
|
||||
() => job(jobId),
|
||||
(j) => j.status === "DONE" && j.failedCount === 0,
|
||||
);
|
||||
expect(finished.imported).toBe(1);
|
||||
});
|
||||
|
||||
it("start all runs every paused queue, one job at a time across users", async () => {
|
||||
const a = await queue(member, { url: modelUrl(base + 90) });
|
||||
await queue(member, { url: modelUrl(base + 91) });
|
||||
const b = await queue(other, { url: modelUrl(base + 92) });
|
||||
const c = await queue(admin, { url: modelUrl(base + 93) });
|
||||
|
||||
const res = await request(app).post("/api/admin/import-queue/start-all").set(auth(admin));
|
||||
expect(res.status).toBe(202);
|
||||
expect(res.body.queued).toBe(3);
|
||||
// Pressing it again while it runs doesn't queue the same jobs twice.
|
||||
expect((await request(app).post("/api/admin/import-queue/start-all").set(auth(admin))).body.queued).toBe(0);
|
||||
|
||||
for (const id of [a.body.job_id, b.body.job_id, c.body.job_id]) await waitStatus(id, "DONE");
|
||||
expect(maxInFlight).toBe(1);
|
||||
expect(await prisma.print.count({ where: { userId: other.id, sourceExternalId: String(base + 92) } })).toBe(1);
|
||||
const listed = await waitFor(
|
||||
async () => (await request(app).get("/api/admin/import-queue").set(auth(admin))).body,
|
||||
(body) => !body.bulk.running,
|
||||
);
|
||||
expect(listed.bulk).toMatchObject({ running: false, remaining: 0 });
|
||||
});
|
||||
|
||||
it("retry all failed reruns the failed links of every finished job", async () => {
|
||||
gone.add(String(base + 100));
|
||||
gone.add(String(base + 101));
|
||||
const a = await queue(member, { url: modelUrl(base + 100) });
|
||||
const b = await queue(other, { url: modelUrl(base + 101) });
|
||||
await request(app).post("/api/admin/import-queue/start-all").set(auth(admin));
|
||||
for (const id of [a.body.job_id, b.body.job_id]) await waitStatus(id, "DONE");
|
||||
await waitFor(
|
||||
async () => (await request(app).get("/api/admin/import-queue").set(auth(admin))).body,
|
||||
(body) => !body.bulk.running,
|
||||
);
|
||||
|
||||
gone.clear();
|
||||
const res = await request(app).post("/api/admin/import-queue/retry-failed").set(auth(admin));
|
||||
expect(res.body.queued).toBe(2);
|
||||
for (const id of [a.body.job_id, b.body.job_id]) {
|
||||
const finished = await waitFor(
|
||||
() => job(id),
|
||||
(j) => j.status === "DONE" && j.failedCount === 0,
|
||||
);
|
||||
expect(finished.imported).toBe(1);
|
||||
}
|
||||
});
|
||||
|
||||
it("pause all stops a bulk run before its next job", async () => {
|
||||
const release = gate(String(base + 110));
|
||||
const a = await queue(member, { url: modelUrl(base + 110) });
|
||||
const b = await queue(other, { url: modelUrl(base + 111) });
|
||||
await request(app).post("/api/admin/import-queue/start-all").set(auth(admin));
|
||||
await waitFor(
|
||||
async () => designCalls.get(String(base + 110)) ?? 0,
|
||||
(n) => n > 0,
|
||||
);
|
||||
|
||||
const res = await request(app).post("/api/admin/import-queue/pause-all").set(auth(admin));
|
||||
expect(res.body.paused).toBe(1);
|
||||
release();
|
||||
await waitFor(
|
||||
async () => (await request(app).get("/api/admin/import-queue").set(auth(admin))).body,
|
||||
(body) => !body.bulk.running && !body.jobs.some((j: { runner_live: boolean }) => j.runner_live),
|
||||
);
|
||||
// The link that was importing finishes; the job stays paused and the next one never starts.
|
||||
expect((await job(a.body.job_id)).status).toBe("DONE");
|
||||
expect((await job(b.body.job_id)).status).toBe("PAUSED");
|
||||
expect(designCalls.get(String(base + 111))).toBeUndefined();
|
||||
});
|
||||
|
||||
it("removes a waiting link, and the job with its last one", async () => {
|
||||
await queue(member, { url: modelUrl(base + 120) });
|
||||
const res = await queue(member, { url: modelUrl(base + 121) });
|
||||
const jobId = res.body.job_id;
|
||||
const [first, second] = await items(jobId);
|
||||
|
||||
const removed = await request(app).delete(`/api/admin/import-queue/items/${first.id}`).set(auth(admin));
|
||||
expect(removed.body).toEqual({ jobDeleted: false });
|
||||
expect((await job(jobId)).total).toBe(1);
|
||||
const notification = await prisma.notification.findUnique({ where: { id: (await job(jobId)).notificationId! } });
|
||||
expect(notification!.title).toBe("1 link waiting in the import queue");
|
||||
|
||||
const last = await request(app).delete(`/api/admin/import-queue/items/${second.id}`).set(auth(admin));
|
||||
expect(last.body).toEqual({ jobDeleted: true });
|
||||
expect(await prisma.importJob.findUnique({ where: { id: jobId } })).toBeNull();
|
||||
expect(await prisma.notification.findUnique({ where: { id: notification!.id } })).toBeNull();
|
||||
expect((await request(app).delete(`/api/admin/import-queue/items/${second.id}`).set(auth(admin))).status).toBe(
|
||||
404,
|
||||
);
|
||||
});
|
||||
|
||||
it("won't remove links or delete a job while it runs", async () => {
|
||||
const release = gate(String(base + 130));
|
||||
await queue(member, { url: modelUrl(base + 130) });
|
||||
const res = await queue(member, { url: modelUrl(base + 131) });
|
||||
const jobId = res.body.job_id;
|
||||
await request(app).post(`/api/admin/import-queue/jobs/${jobId}/start`).set(auth(admin));
|
||||
await waitFor(
|
||||
async () => designCalls.get(String(base + 130)) ?? 0,
|
||||
(n) => n > 0,
|
||||
);
|
||||
|
||||
const [, waiting] = await items(jobId);
|
||||
expect((await request(app).delete(`/api/admin/import-queue/items/${waiting.id}`).set(auth(admin))).status).toBe(
|
||||
409,
|
||||
);
|
||||
expect((await request(app).delete(`/api/admin/import-queue/jobs/${jobId}`).set(auth(admin))).status).toBe(409);
|
||||
release();
|
||||
await waitStatus(jobId, "DONE");
|
||||
expect((await request(app).delete(`/api/admin/import-queue/jobs/${jobId}`).set(auth(admin))).status).toBe(200);
|
||||
expect(await prisma.importJobItem.count({ where: { jobId } })).toBe(0);
|
||||
});
|
||||
|
||||
it("clears finished jobs but keeps ones with failed links to retry", async () => {
|
||||
gone.add(String(base + 141));
|
||||
const clean = await queue(member, { url: modelUrl(base + 140) });
|
||||
const failing = await queue(other, { url: modelUrl(base + 141) });
|
||||
const waiting = await queue(admin, { url: modelUrl(base + 142) });
|
||||
await request(app).post(`/api/admin/import-queue/jobs/${clean.body.job_id}/start`).set(auth(admin));
|
||||
await request(app).post(`/api/admin/import-queue/jobs/${failing.body.job_id}/start`).set(auth(admin));
|
||||
await waitStatus(clean.body.job_id, "DONE");
|
||||
await waitStatus(failing.body.job_id, "DONE");
|
||||
|
||||
const res = await request(app).post("/api/admin/import-queue/clear-finished").set(auth(admin));
|
||||
expect(res.body.deleted).toBeGreaterThanOrEqual(1);
|
||||
expect(await prisma.importJob.findUnique({ where: { id: clean.body.job_id } })).toBeNull();
|
||||
expect(await prisma.importJob.findUnique({ where: { id: failing.body.job_id } })).not.toBeNull();
|
||||
expect(await prisma.importJob.findUnique({ where: { id: waiting.body.job_id } })).not.toBeNull();
|
||||
});
|
||||
|
||||
it("only starts, pauses or retries link-list jobs", async () => {
|
||||
const zip = await prisma.importJob.create({
|
||||
data: { userId: member.id, type: "ZIP", status: "DONE", sourceUrl: "https://example.com/a.zip" },
|
||||
});
|
||||
try {
|
||||
for (const action of ["start", "retry", "pause"]) {
|
||||
const res = await request(app).post(`/api/admin/import-queue/jobs/${zip.id}/${action}`).set(auth(admin));
|
||||
expect(res.status).toBe(400);
|
||||
}
|
||||
expect((await request(app).post("/api/admin/import-queue/jobs/nope/start").set(auth(admin))).status).toBe(404);
|
||||
} finally {
|
||||
await prisma.importJob.delete({ where: { id: zip.id } });
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,276 @@
|
||||
import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import request from "supertest";
|
||||
|
||||
// No pacing or DNS here. config.ts reads the pace on load, so it's hoisted.
|
||||
vi.hoisted(() => {
|
||||
process.env.IMPORT_MAKERWORLD_CALL_DELAY_MS = "0";
|
||||
});
|
||||
vi.mock("node:dns/promises", () => ({
|
||||
default: {
|
||||
resolve4: async () => ["93.184.216.34"],
|
||||
resolve6: async () => {
|
||||
throw new Error("no AAAA");
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
import { createApp } from "../src/app";
|
||||
import { prisma } from "../src/db";
|
||||
|
||||
const app = createApp();
|
||||
const stamp = Date.now();
|
||||
let token: string;
|
||||
const auth = () => ({ Authorization: `Bearer ${token}` });
|
||||
const originalFetch = global.fetch;
|
||||
|
||||
function json(body: unknown): Response {
|
||||
return new Response(JSON.stringify(body), { headers: { "content-type": "application/json" } });
|
||||
}
|
||||
|
||||
const DESIGNER = { uid: 1692606088, name: "Deus Cat" };
|
||||
function design(designId: string) {
|
||||
return {
|
||||
id: Number(designId),
|
||||
modelId: `US${designId}`,
|
||||
title: `Phoenix ${designId}`,
|
||||
summary: "<p>A phoenix.</p>",
|
||||
tags: ["phoenix"],
|
||||
coverUrl: "https://makerworld.bblmw.com/phoenix/cover.jpg",
|
||||
designCreator: DESIGNER,
|
||||
defaultInstanceId: 2,
|
||||
instances: [
|
||||
{ id: 1, profileId: 101, title: "Single-colour version", instanceCreator: DESIGNER },
|
||||
{ id: 2, profileId: 102, title: "multi-colored version", instanceCreator: DESIGNER },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
/** "ok" imports; "fail-once" drops the first design fetch (a network blip); "gone" 404s forever. */
|
||||
type Behavior = "ok" | "fail-once" | "gone";
|
||||
const behavior = new Map<string, Behavior>();
|
||||
const designCalls = new Map<string, number>();
|
||||
|
||||
function mockFetch() {
|
||||
global.fetch = vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async (input) => {
|
||||
const url = String(input);
|
||||
const model = url.match(/^https:\/\/api\.bambulab\.com\/v1\/design-service\/design\/(\d+)/);
|
||||
if (model) {
|
||||
const id = model[1];
|
||||
const calls = (designCalls.get(id) ?? 0) + 1;
|
||||
designCalls.set(id, calls);
|
||||
const b = behavior.get(id) ?? "ok";
|
||||
if (b === "gone") return new Response("gone", { status: 404 });
|
||||
if (b === "fail-once" && calls === 1) throw new TypeError("network down");
|
||||
return json(design(id));
|
||||
}
|
||||
const profile = url.match(/^https:\/\/api\.bambulab\.com\/v1\/iot-service\/api\/user\/profile\/(\d+)/);
|
||||
if (profile)
|
||||
return json({
|
||||
message: "success",
|
||||
url: `https://s3.example.com/phoenix-${profile[1]}.stl`,
|
||||
filename: `phoenix-${profile[1]}.stl`,
|
||||
});
|
||||
const file = url.match(/^https:\/\/s3\.example\.com\/phoenix-(\d+)\.stl$/);
|
||||
if (file)
|
||||
return new Response(`solid phoenix-${file[1]}\nendsolid\n`, {
|
||||
headers: { "content-type": "application/octet-stream" },
|
||||
});
|
||||
if (url === "https://makerworld.bblmw.com/phoenix/cover.jpg") {
|
||||
return new Response(
|
||||
Buffer.from(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
|
||||
"base64",
|
||||
),
|
||||
{ headers: { "content-type": "image/png" } },
|
||||
);
|
||||
}
|
||||
return new Response("not found", { status: 404 }); // e.g. the author profile, behind Cloudflare
|
||||
}) as unknown as typeof fetch;
|
||||
}
|
||||
|
||||
async function waitJob(jobId: string) {
|
||||
let job = (await request(app).get(`/api/import/jobs/${jobId}`).set(auth())).body;
|
||||
for (let i = 0; i < 150 && job.status === "RUNNING"; i++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
job = (await request(app).get(`/api/import/jobs/${jobId}`).set(auth())).body;
|
||||
}
|
||||
return job;
|
||||
}
|
||||
|
||||
function startLinks(urls: string[], extra: Record<string, unknown> = {}) {
|
||||
return request(app)
|
||||
.post("/api/import/links")
|
||||
.set(auth())
|
||||
.send({ urls, makerworld_cookie: "token=test-bearer", ...extra });
|
||||
}
|
||||
|
||||
describe("the links import queue", () => {
|
||||
beforeAll(async () => {
|
||||
const res = await request(app)
|
||||
.post("/api/register")
|
||||
.send({
|
||||
displayName: "Links Import Test",
|
||||
email: `links-${stamp}@example.com`,
|
||||
password: "password123",
|
||||
});
|
||||
token = res.body.token;
|
||||
// The retry route takes the cookie from the user's saved settings, never from a stored payload.
|
||||
await request(app).patch("/api/settings/makerworld").set(auth()).send({ cookie: "token=test-bearer" });
|
||||
mockFetch();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
mockFetch();
|
||||
behavior.clear();
|
||||
designCalls.clear();
|
||||
});
|
||||
|
||||
it("imports every pasted link as one job, with an item per link", async () => {
|
||||
const ids = [String(stamp % 1_000_000_000), String((stamp % 1_000_000_000) + 1)];
|
||||
const start = await startLinks(ids.map((id) => `https://makerworld.com/en/models/${id}-phoenix`));
|
||||
expect(start.status).toBe(202);
|
||||
|
||||
const job = await waitJob(start.body.job_id);
|
||||
expect(job.status).toBe("DONE");
|
||||
expect(job.total).toBe(2);
|
||||
expect(job.imported).toBe(2);
|
||||
expect(job.failed_count).toBe(0);
|
||||
|
||||
const items = (await request(app).get(`/api/import/jobs/${job.id}`).set(auth())).body;
|
||||
expect(items.result_print_id).toBeNull();
|
||||
for (const id of ids) {
|
||||
const print = await prisma.print.findFirst({
|
||||
where: {
|
||||
userId: (await prisma.user.findFirst({ where: { email: `links-${stamp}@example.com` } }))!.id,
|
||||
sourceExternalId: id,
|
||||
},
|
||||
});
|
||||
expect(print).not.toBeNull();
|
||||
}
|
||||
const dbItems = await prisma.importJobItem.findMany({ where: { jobId: job.id } });
|
||||
expect(dbItems.map((i) => i.status)).toEqual(["DONE", "DONE"]);
|
||||
});
|
||||
|
||||
it("collapses repeated links into one queue item", async () => {
|
||||
const id = String((stamp % 1_000_000_000) + 2);
|
||||
const start = await startLinks([
|
||||
`https://makerworld.com/en/models/${id}-phoenix`,
|
||||
`https://makerworld.com/en/models/${id}-phoenix`,
|
||||
]);
|
||||
const job = await waitJob(start.body.job_id);
|
||||
expect(job.total).toBe(1);
|
||||
expect(job.imported).toBe(1);
|
||||
});
|
||||
|
||||
it("keeps importing when one link fails, then a retry reruns just the failed one", async () => {
|
||||
const goodId = String((stamp % 1_000_000_000) + 3);
|
||||
const badId = String((stamp % 1_000_000_000) + 4);
|
||||
behavior.set(badId, "gone");
|
||||
|
||||
const start = await startLinks([
|
||||
`https://makerworld.com/en/models/${goodId}-phoenix`,
|
||||
`https://makerworld.com/en/models/${badId}-phoenix`,
|
||||
]);
|
||||
const job = await waitJob(start.body.job_id);
|
||||
expect(job.status).toBe("DONE");
|
||||
expect(job.imported).toBe(1);
|
||||
expect(job.failed_count).toBe(1);
|
||||
|
||||
let items = await prisma.importJobItem.findMany({ where: { jobId: job.id }, orderBy: { createdAt: "asc" } });
|
||||
expect(items.map((i) => i.status)).toEqual(["DONE", "FAILED"]);
|
||||
expect(items[1].errorMessage).toBeTruthy();
|
||||
expect(items[1].attempts).toBe(1);
|
||||
|
||||
// The model reappears: retrying the job brings in only the failed link.
|
||||
behavior.delete(badId);
|
||||
designCalls.clear();
|
||||
const retry = await request(app).post(`/api/import/jobs/${job.id}/retry`).set(auth()).send({});
|
||||
expect(retry.status).toBe(202);
|
||||
const rerun = await waitJob(job.id);
|
||||
expect(rerun.status).toBe("DONE");
|
||||
expect(rerun.imported).toBe(2);
|
||||
expect(rerun.failed_count).toBe(0);
|
||||
|
||||
items = await prisma.importJobItem.findMany({ where: { jobId: job.id }, orderBy: { createdAt: "asc" } });
|
||||
expect(items.map((i) => i.status)).toEqual(["DONE", "DONE"]);
|
||||
expect(designCalls.get(goodId)).toBeUndefined(); // the good link wasn't downloaded again
|
||||
});
|
||||
|
||||
it("retries a transient network failure on its own", async () => {
|
||||
const id = String((stamp % 1_000_000_000) + 5);
|
||||
behavior.set(id, "fail-once");
|
||||
|
||||
const start = await startLinks([`https://makerworld.com/en/models/${id}-phoenix`]);
|
||||
const job = await waitJob(start.body.job_id);
|
||||
expect(job.status).toBe("DONE");
|
||||
expect(job.imported).toBe(1);
|
||||
expect(job.failed_count).toBe(0);
|
||||
expect(designCalls.get(id)).toBe(2);
|
||||
|
||||
const items = await prisma.importJobItem.findMany({ where: { jobId: job.id } });
|
||||
expect(items[0].status).toBe("DONE");
|
||||
expect(items[0].attempts).toBe(2);
|
||||
});
|
||||
|
||||
it("does not retry a provider's own 4xx answer", async () => {
|
||||
const id = String((stamp % 1_000_000_000) + 6);
|
||||
behavior.set(id, "gone");
|
||||
|
||||
const start = await startLinks([`https://makerworld.com/en/models/${id}-phoenix`]);
|
||||
const job = await waitJob(start.body.job_id);
|
||||
expect(job.failed_count).toBe(1);
|
||||
expect(designCalls.get(id)).toBe(1);
|
||||
|
||||
const items = await prisma.importJobItem.findMany({ where: { jobId: job.id } });
|
||||
expect(items[0].status).toBe("FAILED");
|
||||
});
|
||||
|
||||
it("imports every wanted profile of a MakerWorld link onto one model", async () => {
|
||||
const id = String((stamp % 1_000_000_000) + 7);
|
||||
const start = await startLinks([`https://makerworld.com/en/models/${id}-phoenix`], { scope: "all" });
|
||||
const job = await waitJob(start.body.job_id);
|
||||
expect(job.status).toBe("DONE");
|
||||
expect(job.imported).toBe(2);
|
||||
|
||||
const items = await prisma.importJobItem.findMany({ where: { jobId: job.id } });
|
||||
expect(items[0].status).toBe("DONE");
|
||||
|
||||
const user = await prisma.user.findFirst({ where: { email: `links-${stamp}@example.com` } });
|
||||
const print = await prisma.print.findFirst({
|
||||
where: { userId: user!.id, sourceExternalId: id },
|
||||
include: { plates: true },
|
||||
});
|
||||
expect(print!.plates).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("lists each link with its status and failure reason", async () => {
|
||||
const goodId = String((stamp % 1_000_000_000) + 9);
|
||||
const badId = String((stamp % 1_000_000_000) + 10);
|
||||
behavior.set(badId, "gone");
|
||||
|
||||
const start = await startLinks([
|
||||
`https://makerworld.com/en/models/${goodId}-phoenix`,
|
||||
`https://makerworld.com/en/models/${badId}-phoenix`,
|
||||
]);
|
||||
const job = await waitJob(start.body.job_id);
|
||||
|
||||
const res = await request(app).get(`/api/import/jobs/${job.id}/items`).set(auth());
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toHaveLength(2);
|
||||
expect(res.body[0]).toMatchObject({ status: "DONE", attempts: 1, error_message: null });
|
||||
expect(res.body[1].status).toBe("FAILED");
|
||||
expect(res.body[1].error_message).toBeTruthy();
|
||||
expect(res.body[1].url).toContain(badId);
|
||||
});
|
||||
|
||||
it("refuses a retry with nothing failed", async () => {
|
||||
const id = String((stamp % 1_000_000_000) + 8);
|
||||
const start = await startLinks([`https://makerworld.com/en/models/${id}-phoenix`]);
|
||||
const job = await waitJob(start.body.job_id);
|
||||
expect(job.failed_count).toBe(0);
|
||||
|
||||
const retry = await request(app).post(`/api/import/jobs/${job.id}/retry`).set(auth()).send({});
|
||||
expect(retry.status).toBe(400);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,287 @@
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import request from "supertest";
|
||||
|
||||
// No pacing or DNS here. config.ts reads the pace on load, so it's hoisted.
|
||||
vi.hoisted(() => {
|
||||
process.env.IMPORT_MAKERWORLD_CALL_DELAY_MS = "0";
|
||||
});
|
||||
vi.mock("node:dns/promises", () => ({
|
||||
default: {
|
||||
resolve4: async () => ["93.184.216.34"],
|
||||
resolve6: async () => {
|
||||
throw new Error("no AAAA");
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
import { createApp } from "../src/app";
|
||||
import { prisma } from "../src/db";
|
||||
import { getThingiverseAccessToken, setThingiverseAccessToken } from "../src/services/settingsService";
|
||||
|
||||
const app = createApp();
|
||||
const stamp = Date.now();
|
||||
let token: string;
|
||||
const auth = () => ({ Authorization: `Bearer ${token}` });
|
||||
const originalFetch = global.fetch;
|
||||
|
||||
const ONE_PIXEL_PNG_BASE64 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=";
|
||||
let previousThingiverseToken: string | null = null;
|
||||
|
||||
function json(body: unknown): Response {
|
||||
return new Response(JSON.stringify(body), { headers: { "content-type": "application/json" } });
|
||||
}
|
||||
|
||||
function pngResponse(): Response {
|
||||
return new Response(Buffer.from(ONE_PIXEL_PNG_BASE64, "base64"), {
|
||||
headers: { "content-type": "image/png" },
|
||||
});
|
||||
}
|
||||
|
||||
const DESIGNER = { uid: 1692606088, name: "Deus Cat" };
|
||||
/** Mutable so a re-import can see an updated page: what the source offers changes over time. */
|
||||
const currentDesign = new Map<string, Record<string, unknown>>();
|
||||
const currentThingFiles = new Map<string, string>(); // name -> file contents
|
||||
let coverAvailable = true;
|
||||
|
||||
function setDesign(designId: string, overrides: Partial<Record<string, unknown>> = {}) {
|
||||
currentDesign.set(designId, {
|
||||
id: Number(designId),
|
||||
modelId: `US${designId}`,
|
||||
title: `Phoenix ${designId}`,
|
||||
summary: "<p>A phoenix.</p>",
|
||||
tags: ["phoenix"],
|
||||
coverUrl: "https://makerworld.bblmw.com/cover.jpg",
|
||||
designCreator: DESIGNER,
|
||||
defaultInstanceId: 2,
|
||||
instances: [
|
||||
{ id: 1, profileId: 101, title: "Single-colour version", instanceCreator: DESIGNER },
|
||||
{ id: 2, profileId: 102, title: "multi-colored version", instanceCreator: DESIGNER },
|
||||
],
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
const THING_ID = "8880001";
|
||||
|
||||
function mockFetch() {
|
||||
global.fetch = vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async (input) => {
|
||||
const url = String(input);
|
||||
const model = url.match(/^https:\/\/api\.bambulab\.com\/v1\/design-service\/design\/(\d+)/);
|
||||
if (model) return json(currentDesign.get(model[1]) ?? {});
|
||||
const profile = url.match(/^https:\/\/api\.bambulab\.com\/v1\/iot-service\/api\/user\/profile\/(\d+)/);
|
||||
if (profile)
|
||||
return json({
|
||||
message: "success",
|
||||
url: `https://s3.example.com/phoenix-${profile[1]}.stl`,
|
||||
filename: `phoenix-${profile[1]}.stl`,
|
||||
});
|
||||
const file = url.match(/^https:\/\/s3\.example\.com\/phoenix-(\d+)\.stl$/);
|
||||
if (file)
|
||||
return new Response(`solid phoenix-${file[1]}\nendsolid\n`, {
|
||||
headers: { "content-type": "application/octet-stream" },
|
||||
});
|
||||
if (url === "https://makerworld.bblmw.com/cover.jpg") {
|
||||
return coverAvailable ? pngResponse() : new Response("gone", { status: 404 });
|
||||
}
|
||||
if (url.startsWith(`https://api.thingiverse.com/things/${THING_ID}`)) {
|
||||
return json({
|
||||
id: Number(THING_ID),
|
||||
name: "Test Widget",
|
||||
description: "A widget.",
|
||||
tags: [{ name: "Widget" }],
|
||||
thumbnail: "https://cdn.thingiverse.com/assets/test/thumb.jpg",
|
||||
default_image: { url: "https://cdn.thingiverse.com/assets/test/preview.jpg" },
|
||||
categories_url: `https://api.thingiverse.com/things/${THING_ID}/categories`,
|
||||
creator: { id: 42, name: "TestCreator" },
|
||||
zip_data: {
|
||||
files: [...currentThingFiles.keys()].map((name) => ({
|
||||
name,
|
||||
url: `https://cdn.thingiverse.com/assets/test/${name}`,
|
||||
})),
|
||||
images: [{ name: "render.png", url: "https://cdn.thingiverse.com/renders/test/render.png" }],
|
||||
},
|
||||
});
|
||||
}
|
||||
if (url.startsWith(`https://api.thingiverse.com/things/${THING_ID}/categories`)) {
|
||||
return json([{ id: 129, name: "3D Printing Tests" }]);
|
||||
}
|
||||
const thingFile = url.match(/^https:\/\/cdn\.thingiverse\.com\/assets\/test\/(.+)$/);
|
||||
if (thingFile && currentThingFiles.has(thingFile[1])) {
|
||||
return new Response(currentThingFiles.get(thingFile[1]), {
|
||||
headers: { "content-type": "application/sla" },
|
||||
});
|
||||
}
|
||||
if (url.endsWith(".png") || url.endsWith(".jpg")) return pngResponse();
|
||||
return new Response("not found", { status: 404 });
|
||||
}) as unknown as typeof fetch;
|
||||
}
|
||||
|
||||
async function importMakerworldModel(designId: string, scope?: "url" | "designer" | "all") {
|
||||
const payload = {
|
||||
url: `https://makerworld.com/en/models/${designId}-phoenix`,
|
||||
makerworld_cookie: "token=test-bearer",
|
||||
...(scope && scope !== "url" ? { scope } : {}),
|
||||
};
|
||||
const res =
|
||||
scope && scope !== "url"
|
||||
? await request(app).post("/api/import/makerworld-profiles").set(auth()).send(payload)
|
||||
: await request(app).post("/api/import").set(auth()).send(payload);
|
||||
if (res.status === 202) {
|
||||
let job = (await request(app).get(`/api/import/jobs/${res.body.job_id}`).set(auth())).body;
|
||||
for (let i = 0; i < 100 && job.status === "RUNNING"; i++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
job = (await request(app).get(`/api/import/jobs/${res.body.job_id}`).set(auth())).body;
|
||||
}
|
||||
return (await request(app).get(`/api/print/${job.result_print_id}`).set(auth())).body;
|
||||
}
|
||||
return res.body;
|
||||
}
|
||||
|
||||
async function reimport(printId: string, opts: Record<string, boolean> = {}) {
|
||||
return request(app).post(`/api/print/${printId}/reimport`).set(auth()).send(opts);
|
||||
}
|
||||
|
||||
describe("re-importing a model from its source", () => {
|
||||
beforeAll(async () => {
|
||||
const res = await request(app)
|
||||
.post("/api/register")
|
||||
.send({
|
||||
displayName: "Reimport Test",
|
||||
email: `reimport-${stamp}@example.com`,
|
||||
password: "password123",
|
||||
});
|
||||
token = res.body.token;
|
||||
await request(app).patch("/api/settings/makerworld").set(auth()).send({ cookie: "token=test-bearer" });
|
||||
previousThingiverseToken = await getThingiverseAccessToken();
|
||||
await setThingiverseAccessToken("test-access-token");
|
||||
mockFetch();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
// Files share one database; leave the token as this file found it.
|
||||
await setThingiverseAccessToken(previousThingiverseToken);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
mockFetch();
|
||||
});
|
||||
|
||||
it("fills only the empty metadata and merges tags, never overwriting an edit", async () => {
|
||||
const designId = String(stamp % 1_000_000_000);
|
||||
setDesign(designId);
|
||||
const print = await importMakerworldModel(designId);
|
||||
|
||||
// The user edits by hand, and the source later changes its own title, notes and tags.
|
||||
await prisma.print.update({
|
||||
where: { id: print.id },
|
||||
data: { title: "My own title", notes: null, tags: ["mine"] },
|
||||
});
|
||||
setDesign(designId, {
|
||||
title: "Phoenix Deluxe",
|
||||
summary: "<p>An updated description.</p>",
|
||||
tags: ["phoenix", "articulated"],
|
||||
});
|
||||
|
||||
const res = await reimport(print.id, { metadata: true, files: false, images: false });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.title_filled).toBe(false);
|
||||
expect(res.body.notes_filled).toBe(true);
|
||||
expect(res.body.tags_added).toEqual(["phoenix", "articulated"]);
|
||||
|
||||
const updated = await prisma.print.findUniqueOrThrow({ where: { id: print.id } });
|
||||
expect(updated.title).toBe("My own title"); // the edit wins
|
||||
expect(updated.notes).toBe("An updated description."); // the empty one is filled
|
||||
expect(updated.tags.toSorted()).toEqual(["articulated", "mine", "phoenix"]);
|
||||
expect(updated.creator).toBe("Deus Cat");
|
||||
expect(updated.authorId).not.toBeNull();
|
||||
});
|
||||
|
||||
it("brings in the MakerWorld print profiles the model doesn't hold yet", async () => {
|
||||
const designId = String((stamp % 1_000_000_000) + 1);
|
||||
setDesign(designId);
|
||||
const print = await importMakerworldModel(designId);
|
||||
expect(print.plates).toHaveLength(1);
|
||||
|
||||
const res = await reimport(print.id, { metadata: false, files: true, images: false });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.files_added).toBe(1);
|
||||
expect(res.body.files_already_present).toBe(1);
|
||||
|
||||
const plates = await prisma.plate.findMany({ where: { printId: print.id } });
|
||||
expect(plates).toHaveLength(2);
|
||||
// A second run has nothing left to add.
|
||||
const again = await reimport(print.id, { metadata: false, files: true, images: false });
|
||||
expect(again.body.files_added).toBe(0);
|
||||
expect(again.body.files_already_present).toBe(2);
|
||||
});
|
||||
|
||||
it("fills the image slots only while they are empty", async () => {
|
||||
const designId = String((stamp % 1_000_000_000) + 2);
|
||||
coverAvailable = false;
|
||||
setDesign(designId);
|
||||
const print = await importMakerworldModel(designId);
|
||||
expect(await prisma.previewImage.count({ where: { printId: print.id } })).toBe(0);
|
||||
|
||||
coverAvailable = true;
|
||||
const res = await reimport(print.id, { metadata: false, files: false, images: true });
|
||||
expect(res.body.images_added).toBeGreaterThan(0);
|
||||
|
||||
const count = await prisma.previewImage.count({ where: { printId: print.id } });
|
||||
const again = await reimport(print.id, { metadata: false, files: false, images: true });
|
||||
expect(again.body.images_added).toBe(0);
|
||||
expect(await prisma.previewImage.count({ where: { printId: print.id } })).toBe(count);
|
||||
});
|
||||
|
||||
it("adds a source file the library doesn't hold, without duplicating one it does", async () => {
|
||||
currentThingFiles.set("body.stl", "solid body\nendsolid\n");
|
||||
currentThingFiles.set("wheels.stl", "solid wheels\nendsolid\n");
|
||||
const res = await request(app)
|
||||
.post("/api/import")
|
||||
.set(auth())
|
||||
.send({ url: `https://www.thingiverse.com/thing:${THING_ID}` });
|
||||
expect(res.status).toBe(200);
|
||||
const print = res.body;
|
||||
expect(print.plates).toHaveLength(2);
|
||||
|
||||
// Nothing new at the source: everything is recognized by content.
|
||||
const same = await reimport(print.id, { metadata: false, files: true, images: false });
|
||||
expect(same.body.files_added).toBe(0);
|
||||
expect(same.body.files_already_present).toBe(2);
|
||||
|
||||
// The author replaces one file and adds another; the changed one comes in, the rest is skipped.
|
||||
currentThingFiles.set("wheels.stl", "solid wheels v2\nendsolid\n");
|
||||
currentThingFiles.set("spare.stl", "solid spare\nendsolid\n");
|
||||
const changed = await reimport(print.id, { metadata: false, files: true, images: false });
|
||||
expect(changed.body.files_added).toBe(2);
|
||||
expect(changed.body.files_already_present).toBe(1);
|
||||
expect(await prisma.plate.count({ where: { printId: print.id } })).toBe(4);
|
||||
});
|
||||
|
||||
it("does nothing when every part is turned off", async () => {
|
||||
const designId = String((stamp % 1_000_000_000) + 3);
|
||||
setDesign(designId, { title: "Untouched", tags: ["never"] });
|
||||
const print = await importMakerworldModel(designId);
|
||||
await prisma.print.update({ where: { id: print.id }, data: { title: "Kept", tags: ["kept"] } });
|
||||
|
||||
const res = await reimport(print.id, { metadata: false, files: false, images: false });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.tags_added).toEqual([]);
|
||||
expect(res.body.files_added).toBe(0);
|
||||
expect(res.body.images_added).toBe(0);
|
||||
|
||||
const updated = await prisma.print.findUniqueOrThrow({ where: { id: print.id } });
|
||||
expect(updated.title).toBe("Kept");
|
||||
expect(updated.tags).toEqual(["kept"]);
|
||||
});
|
||||
|
||||
it("refuses a model with no recorded source", async () => {
|
||||
const user = await prisma.user.findFirstOrThrow({ where: { email: `reimport-${stamp}@example.com` } });
|
||||
const print = await prisma.print.create({
|
||||
data: { userId: user.id, name: `no-source-${stamp}`, nameNormalized: `no-source-${stamp}` },
|
||||
});
|
||||
const res = await reimport(print.id);
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
});
|
||||
@@ -17,6 +17,7 @@ export async function getState(): Promise<ExtensionState> {
|
||||
return {
|
||||
configured: isConfigured(config),
|
||||
disabled: Boolean(config.disabled),
|
||||
sendToQueue: Boolean(config.sendToQueue),
|
||||
instanceUrl: config.instanceUrl || "",
|
||||
email: config.email || "",
|
||||
};
|
||||
@@ -43,3 +44,8 @@ export async function setDisabled(disabled: boolean): Promise<null> {
|
||||
await chrome.storage.local.set({ disabled: Boolean(disabled) });
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function setSendToQueue(sendToQueue: boolean): Promise<null> {
|
||||
await chrome.storage.local.set({ sendToQueue: Boolean(sendToQueue) });
|
||||
return null;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { ImportJob, Print } from "../shared/api";
|
||||
import type { ImportSinglePayload } from "../shared/messages";
|
||||
import type { ImportJob, Print, QueueImportResult } from "../shared/api";
|
||||
import type { ImportSinglePayload, QueueImportPayload } from "../shared/messages";
|
||||
import { apiCall } from "./api";
|
||||
import { recordRecentImport } from "./recentImports";
|
||||
|
||||
@@ -51,3 +51,14 @@ export async function importSingle({
|
||||
}
|
||||
return print;
|
||||
}
|
||||
|
||||
/** "Send to queue": nothing is fetched now; the link waits, paused, until it's started from the
|
||||
* instance's Administration > Import queue, which then imports it with the panel's choices. */
|
||||
export async function queueImport({ url, collectionId, scope, title }: QueueImportPayload): Promise<QueueImportResult> {
|
||||
return apiCall<QueueImportResult>("POST", "/import/queue", {
|
||||
url,
|
||||
collection_id: collectionId ?? null,
|
||||
scope: scope ?? "url",
|
||||
title: title ?? null,
|
||||
});
|
||||
}
|
||||
@@ -3,9 +3,9 @@
|
||||
|
||||
import { listen, type BackgroundMessages } from "../shared/messages";
|
||||
import { apiCall } from "./api";
|
||||
import { getState, saveConfig, setDisabled } from "./config";
|
||||
import { getState, saveConfig, setDisabled, setSendToQueue } from "./config";
|
||||
import { armDownloadCapture, awaitDownloadCapture } from "./downloadCapture";
|
||||
import { importSingle } from "./importJobs";
|
||||
import { importSingle, queueImport } from "./importJobs";
|
||||
import { abortJob, advanceJob, dropJobIfForTab, forceAdvanceJob, getJobForTab, startJob } from "./makerworldJob";
|
||||
import { getRecentImports } from "./recentImports";
|
||||
import { openSetup } from "./setup";
|
||||
@@ -20,11 +20,13 @@ listen<BackgroundMessages>({
|
||||
GET_STATE: () => getState(),
|
||||
SAVE_CONFIG: (payload) => saveConfig(payload),
|
||||
SET_DISABLED: ({ disabled }) => setDisabled(disabled),
|
||||
SET_SEND_TO_QUEUE: ({ sendToQueue }) => setSendToQueue(sendToQueue),
|
||||
GET_RECENT_IMPORTS: () => getRecentImports(),
|
||||
OPEN_SETUP: (_payload, sender) => openSetup(sender.tab?.id),
|
||||
SET_TAB_ICON_STATE: ({ active }, sender) => setTabIconState(senderTabId(sender), active),
|
||||
API_CALL: ({ method, path, body }) => apiCall(method, path, body),
|
||||
IMPORT_SINGLE: (payload) => importSingle(payload),
|
||||
QUEUE_IMPORT: (payload) => queueImport(payload),
|
||||
START_MAKERWORLD_COLLECTION_JOB: (payload, sender) => startJob(senderTabId(sender), payload),
|
||||
ABORT_MAKERWORLD_COLLECTION_JOB: (_payload, sender) => abortJob(senderTabId(sender)),
|
||||
FORCE_ADVANCE_MAKERWORLD_JOB: (_payload, sender) => forceAdvanceJob(senderTabId(sender)),
|
||||
|
||||
@@ -9,6 +9,8 @@ export type ImportContext = {
|
||||
instanceUrl: string;
|
||||
classification: Classification;
|
||||
library: LibraryState | null;
|
||||
/** Imports go to the instance's paused import queue (the popup's "Send to queue"). */
|
||||
sendToQueue: boolean;
|
||||
title?: string | null;
|
||||
};
|
||||
|
||||
|
||||
@@ -97,7 +97,13 @@ async function init(): Promise<void> {
|
||||
}
|
||||
if (isStale()) return;
|
||||
|
||||
setContext({ url, instanceUrl: stateRes.data.instanceUrl, classification, library });
|
||||
setContext({
|
||||
url,
|
||||
instanceUrl: stateRes.data.instanceUrl,
|
||||
classification,
|
||||
library,
|
||||
sendToQueue: stateRes.data.sendToQueue,
|
||||
});
|
||||
const root = mountHost();
|
||||
const togglePanel = mountPanel(root, () => void loadPanel());
|
||||
mountFab(root, {
|
||||
|
||||
@@ -4,11 +4,16 @@ export function statusHtml(text: string): string {
|
||||
return `<div class="tg-status">${escapeHtml(text)}</div>`;
|
||||
}
|
||||
|
||||
export function successHtml(link: string, note?: string | null, title = "Imported!"): string {
|
||||
export function successHtml(
|
||||
link: string,
|
||||
note?: string | null,
|
||||
title = "Imported!",
|
||||
linkLabel = "Open in Thingport",
|
||||
): string {
|
||||
return `
|
||||
<div class="tg-title">${escapeHtml(title)}</div>
|
||||
${note ? `<div class="tg-hint">${escapeHtml(note)}</div>` : ""}
|
||||
<a class="tg-btn" href="${escapeHtml(link)}" target="_blank" rel="noopener noreferrer">Open in Thingport</a>
|
||||
<a class="tg-btn" href="${escapeHtml(link)}" target="_blank" rel="noopener noreferrer">${escapeHtml(linkLabel)}</a>
|
||||
`;
|
||||
}
|
||||
|
||||
|
||||
@@ -56,15 +56,27 @@ function profileUrl(url: string, instanceId: string): string {
|
||||
}
|
||||
|
||||
function importHeading(): string {
|
||||
const { title } = ctx();
|
||||
const { title, sendToQueue } = ctx();
|
||||
if (sendToQueue) return title ? `Queue "${escapeHtml(title)}"` : "Queue this model";
|
||||
return title ? `Import "${escapeHtml(title)}"` : "Import this model";
|
||||
}
|
||||
|
||||
function importLabel(): string {
|
||||
return ctx().sendToQueue ? "Add to queue" : "Import";
|
||||
}
|
||||
|
||||
function queueHintHtml(): string {
|
||||
return ctx().sendToQueue
|
||||
? `<div class="tg-hint">Send to queue is on: this waits, paused, in Thingport's import queue.</div>`
|
||||
: "";
|
||||
}
|
||||
|
||||
export async function loadSingleItem(): Promise<void> {
|
||||
// A MakerWorld profile is always one 3MF, so inspecting would only cost a download resolution.
|
||||
if (ctx().library) {
|
||||
renderPanel(addProfileHtml(await profilesPickerHtml()));
|
||||
onPanelAction("import", () => void runDirectImport());
|
||||
onPanelAction("reimport", () => void runReimport());
|
||||
return;
|
||||
}
|
||||
renderPanel(statusHtml("Checking link…"));
|
||||
@@ -89,9 +101,10 @@ export async function loadSingleItem(): Promise<void> {
|
||||
if (zipFilename === null) {
|
||||
renderPanel(`
|
||||
<div class="tg-title">${importHeading()}</div>
|
||||
${queueHintHtml()}
|
||||
${await profilesPickerHtml()}
|
||||
${await collectionPickerHtml()}
|
||||
<button class="tg-btn" type="button" data-action="import">Import</button>
|
||||
<button class="tg-btn" type="button" data-action="import">${importLabel()}</button>
|
||||
`);
|
||||
onPanelAction("import", () => void runDirectImport());
|
||||
return;
|
||||
@@ -100,8 +113,9 @@ export async function loadSingleItem(): Promise<void> {
|
||||
renderPanel(`
|
||||
<div class="tg-title">${importHeading()}</div>
|
||||
<div class="tg-hint">${escapeHtml(zipFilename)} contains multiple files.</div>
|
||||
${queueHintHtml()}
|
||||
${await collectionPickerHtml()}
|
||||
<button class="tg-btn" type="button" data-action="import-as-zip">Import as one model</button>
|
||||
<button class="tg-btn" type="button" data-action="import-as-zip">${ctx().sendToQueue ? "Queue as one model" : "Import as one model"}</button>
|
||||
<button class="tg-btn tg-btn--secondary" type="button" data-action="choose-files">Choose files…</button>
|
||||
`);
|
||||
onPanelAction("import-as-zip", () => void runDirectImport());
|
||||
@@ -121,12 +135,29 @@ function addProfileHtml(profilesPicker: string): string {
|
||||
return `
|
||||
<div class="tg-title">In your library</div>
|
||||
<div class="tg-hint">${hint}</div>
|
||||
${queueHintHtml()}
|
||||
${profilesPicker}
|
||||
<button class="tg-btn" type="button" data-action="import">Add profile</button>
|
||||
<button class="tg-btn" type="button" data-action="import">${ctx().sendToQueue ? "Queue profile" : "Add profile"}</button>
|
||||
<button class="tg-btn tg-btn--secondary" type="button" data-action="reimport">Update model in Thingport</button>
|
||||
<a class="tg-btn tg-btn--secondary" href="${escapeHtml(modelLink)}" target="_blank" rel="noopener noreferrer">Open model in Thingport</a>
|
||||
`;
|
||||
}
|
||||
|
||||
/** Refreshes the library model from this page's source: fills empty metadata and images, and adds
|
||||
* whatever files (e.g. print profiles) it doesn't hold yet. Nothing edited by hand is touched. */
|
||||
async function runReimport(): Promise<void> {
|
||||
const { library, instanceUrl } = ctx();
|
||||
if (!library?.printId) return;
|
||||
renderPanel(statusHtml("Updating model in Thingport…"));
|
||||
try {
|
||||
await api("POST", `/print/${library.printId}/reimport`, { metadata: true, files: true, images: true });
|
||||
const link = `${instanceUrl}/models/${library.printId}`;
|
||||
renderPanel(successHtml(link, "Filled in what the model in your library was missing.", "Model updated"));
|
||||
} catch (err) {
|
||||
renderPanel(errorHtml(err));
|
||||
}
|
||||
}
|
||||
|
||||
async function loadZipEntries(): Promise<void> {
|
||||
renderPanel(statusHtml("Loading files…"));
|
||||
let result: ZipEntriesResult;
|
||||
@@ -162,6 +193,11 @@ async function runDirectImport(opts?: { entries?: string[] }): Promise<void> {
|
||||
// Captured up front: SPA navigation clears the context mid-import.
|
||||
const collectionId = selectedCollectionId();
|
||||
const scope = selectedProfileScope();
|
||||
// Picking files out of a zip needs the download now, so that one always imports directly.
|
||||
if (ctx().sendToQueue && !opts?.entries) {
|
||||
await runQueueImport(scope, collectionId);
|
||||
return;
|
||||
}
|
||||
if (scope !== "url") {
|
||||
await runProfilesImport(scope, collectionId);
|
||||
return;
|
||||
@@ -249,3 +285,26 @@ async function runProfilesImport(scope: MakerworldProfileScope, collectionId: st
|
||||
successHtml(`${instanceUrl}/models/${printId}`, `${parts.join(", ")}.`, failed ? "Partly imported" : undefined),
|
||||
);
|
||||
}
|
||||
|
||||
/** Sends the link to the paused import queue with the panel's collection and profile choices. No
|
||||
* download is resolved here: the instance does that once the queue is started. */
|
||||
async function runQueueImport(scope: MakerworldProfileScope, collectionId: string | null): Promise<void> {
|
||||
const { url, instanceUrl, title } = ctx();
|
||||
renderPanel(statusHtml("Adding to the queue…"));
|
||||
try {
|
||||
const result = await request("QUEUE_IMPORT", { url, collectionId, scope, title: title ?? null });
|
||||
const waiting = `${result.waiting} link${result.waiting === 1 ? "" : "s"} waiting`;
|
||||
renderPanel(
|
||||
successHtml(
|
||||
`${instanceUrl}/admin-queue`,
|
||||
result.duplicate
|
||||
? `This link was already waiting in the queue (${waiting}).`
|
||||
: `Paused until it's started from Administration > Import queue (${waiting}).`,
|
||||
result.duplicate ? "Already in the queue" : "Added to the queue",
|
||||
"Open import queue",
|
||||
),
|
||||
);
|
||||
} catch (err) {
|
||||
renderPanel(errorHtml(err));
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ export const els = {
|
||||
editInstanceBtn: byId<HTMLButtonElement>("edit-instance-btn"),
|
||||
configuredView: byId("configured-view"),
|
||||
enabledSwitch: byId<HTMLInputElement>("enabled-switch"),
|
||||
queueSwitch: byId<HTMLInputElement>("queue-switch"),
|
||||
recentSection: byId("recent-section"),
|
||||
recentGrid: byId("recent-grid"),
|
||||
configuredError: byId("configured-error"),
|
||||
|
||||
@@ -24,6 +24,7 @@ function showConfiguredView(state: ExtensionState): void {
|
||||
els.instanceUrlText.textContent = displayInstanceUrl(state.instanceUrl);
|
||||
els.instanceUrlText.title = state.instanceUrl;
|
||||
els.enabledSwitch.checked = !state.disabled;
|
||||
els.queueSwitch.checked = state.sendToQueue;
|
||||
clearError(els.configuredError);
|
||||
void loadRecentImports();
|
||||
}
|
||||
@@ -68,6 +69,17 @@ els.enabledSwitch.addEventListener("change", async () => {
|
||||
}
|
||||
});
|
||||
|
||||
els.queueSwitch.addEventListener("change", async () => {
|
||||
clearError(els.configuredError);
|
||||
els.queueSwitch.disabled = true;
|
||||
const res = await send("SET_SEND_TO_QUEUE", { sendToQueue: els.queueSwitch.checked });
|
||||
els.queueSwitch.disabled = false;
|
||||
if (!res?.ok) {
|
||||
showError(res?.error ?? "Couldn't save", els.configuredError);
|
||||
els.queueSwitch.checked = !els.queueSwitch.checked;
|
||||
}
|
||||
});
|
||||
|
||||
els.setupForm.addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
clearError();
|
||||
|
||||
@@ -43,6 +43,13 @@
|
||||
<label for="enabled-switch">Extension enabled</label>
|
||||
<input type="checkbox" role="switch" class="switch" id="enabled-switch" />
|
||||
</div>
|
||||
<div class="toggle-row">
|
||||
<label for="queue-switch">Send to queue</label>
|
||||
<input type="checkbox" role="switch" class="switch" id="queue-switch" aria-describedby="queue-switch-hint" />
|
||||
</div>
|
||||
<p class="note" id="queue-switch-hint">
|
||||
Imports wait, paused, in Thingport's import queue until they're started from Administration.
|
||||
</p>
|
||||
<div class="section" id="recent-section" hidden>
|
||||
<h2 class="section__title">Recent imports</h2>
|
||||
<div class="recent-grid" id="recent-grid"></div>
|
||||
|
||||
@@ -35,4 +35,7 @@ export type ImportJob = {
|
||||
error_message?: string | null;
|
||||
};
|
||||
|
||||
/** POST /import/queue: the link waits, paused, in the user's queue on the instance. */
|
||||
export type QueueImportResult = { job_id: string; item_id: string; duplicate: boolean; waiting: number };
|
||||
|
||||
export type LoginResult = { token: string; expires_in: number };
|
||||
@@ -1,9 +1,15 @@
|
||||
// Typed runtime messages between the content script/popup and the background. Replies use a
|
||||
// `{ ok, data } | { ok: false, error }` envelope: `send` returns it, `request` throws on error.
|
||||
|
||||
import type { Print } from "./api";
|
||||
import type { Print, QueueImportResult } from "./api";
|
||||
|
||||
export type ExtensionState = { configured: boolean; disabled: boolean; instanceUrl: string; email: string };
|
||||
export type ExtensionState = {
|
||||
configured: boolean;
|
||||
disabled: boolean;
|
||||
sendToQueue: boolean;
|
||||
instanceUrl: string;
|
||||
email: string;
|
||||
};
|
||||
|
||||
export type RecentImport = {
|
||||
printId: string;
|
||||
@@ -30,6 +36,13 @@ export type ImportSinglePayload = {
|
||||
title?: string | null;
|
||||
};
|
||||
|
||||
export type QueueImportPayload = {
|
||||
url: string;
|
||||
collectionId?: string | null;
|
||||
scope?: "url" | "designer" | "all";
|
||||
title?: string | null;
|
||||
};
|
||||
|
||||
export type MakerworldJob = {
|
||||
tabId: number;
|
||||
originalUrl: string;
|
||||
@@ -49,11 +62,13 @@ export type BackgroundMessages = {
|
||||
GET_STATE: { payload: void; result: ExtensionState };
|
||||
SAVE_CONFIG: { payload: { instanceUrl: string; email: string; password: string }; result: null };
|
||||
SET_DISABLED: { payload: { disabled: boolean }; result: null };
|
||||
SET_SEND_TO_QUEUE: { payload: { sendToQueue: boolean }; result: null };
|
||||
GET_RECENT_IMPORTS: { payload: void; result: RecentImport[] };
|
||||
OPEN_SETUP: { payload: void; result: "popup" | "tab" };
|
||||
SET_TAB_ICON_STATE: { payload: { active: boolean }; result: null };
|
||||
API_CALL: { payload: ApiCallPayload; result: unknown };
|
||||
IMPORT_SINGLE: { payload: ImportSinglePayload; result: Print | null };
|
||||
QUEUE_IMPORT: { payload: QueueImportPayload; result: QueueImportResult };
|
||||
START_MAKERWORLD_COLLECTION_JOB: {
|
||||
payload: { urls: string[]; collectionId: string | null; originalUrl: string };
|
||||
result: null;
|
||||
|
||||
@@ -5,6 +5,8 @@ export const STORAGE_KEYS = {
|
||||
email: "email",
|
||||
password: "password",
|
||||
disabled: "disabled",
|
||||
// Imports go to the instance's paused import queue instead of running right away.
|
||||
sendToQueue: "sendToQueue",
|
||||
token: "token",
|
||||
tokenExpiresAt: "tokenExpiresAt",
|
||||
// Lets the cookie sync skip a redundant PATCH.
|
||||
@@ -16,6 +18,7 @@ export type StoredConfig = {
|
||||
email?: string;
|
||||
password?: string;
|
||||
disabled?: boolean;
|
||||
sendToQueue?: boolean;
|
||||
token?: string;
|
||||
tokenExpiresAt?: number;
|
||||
lastSyncedMakerworldCookie?: string;
|
||||
@@ -26,6 +29,7 @@ export const CONFIG_CHANGE_KEYS: readonly string[] = [
|
||||
STORAGE_KEYS.email,
|
||||
STORAGE_KEYS.password,
|
||||
STORAGE_KEYS.disabled,
|
||||
STORAGE_KEYS.sendToQueue,
|
||||
];
|
||||
|
||||
export function normalizeInstanceUrl(raw: string | undefined | null): string {
|
||||
|
||||
@@ -29,6 +29,7 @@ import LogsPage from "./pages/LogsPage";
|
||||
import TriggersPage from "./pages/TriggersPage";
|
||||
import ConnectionsPage from "./pages/ConnectionsPage";
|
||||
import CaptchaPage from "./pages/CaptchaPage";
|
||||
import ImportQueuePage from "./pages/ImportQueuePage";
|
||||
import RenderingPage from "./pages/RenderingPage";
|
||||
import { healthApi, type HealthInfo } from "./api/health";
|
||||
import { authApi, type AuthUser } from "./api/auth";
|
||||
@@ -247,6 +248,10 @@ function AppShell({
|
||||
path="/admin-connections"
|
||||
element={isAdmin ? <ConnectionsPage onUnauthorized={onUnauthorized} /> : <Navigate to="/" replace />}
|
||||
/>
|
||||
<Route
|
||||
path="/admin-queue"
|
||||
element={isAdmin ? <ImportQueuePage onUnauthorized={onUnauthorized} /> : <Navigate to="/" replace />}
|
||||
/>
|
||||
<Route
|
||||
path="/admin-captcha"
|
||||
element={isAdmin ? <CaptchaPage onUnauthorized={onUnauthorized} /> : <Navigate to="/" replace />}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { authHeaders } from "../utils/auth";
|
||||
import { apiBase, assertOk, readErrorMessage, UnauthorizedError } from "./client";
|
||||
import type { ImportJob, ImportJobItem } from "./imports";
|
||||
|
||||
export type AdminUser = {
|
||||
id: string;
|
||||
@@ -23,6 +24,7 @@ export type LogAction =
|
||||
| "model_imported"
|
||||
| "import_completed"
|
||||
| "model_edited"
|
||||
| "model_reimported"
|
||||
| "model_deleted"
|
||||
| "collection_created"
|
||||
| "collection_edited"
|
||||
@@ -62,6 +64,31 @@ export type AuthorLinkingRun = {
|
||||
|
||||
export type AuthorLinkingStatus = { linkable: number; lookup: number; run: AuthorLinkingRun | null };
|
||||
|
||||
/** Any user's batch import, as the admin import queue lists it. */
|
||||
export type AdminImportJob = ImportJob & {
|
||||
owner: { id: string; display_name: string; email: string };
|
||||
pending_count: number;
|
||||
running_count: number;
|
||||
done_count: number;
|
||||
failed_items_count: number;
|
||||
/** False on a RUNNING job means its runner is gone (a restart): pause it, then start it again. */
|
||||
runner_live: boolean;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
};
|
||||
|
||||
/** "Start all" and "Retry all failed" run jobs one after another, across users. */
|
||||
export type ImportQueueBulk = { running: boolean; currentJobId: string | null; remaining: number };
|
||||
|
||||
export type ImportQueue = { jobs: AdminImportJob[]; bulk: ImportQueueBulk };
|
||||
|
||||
async function queueAction<T>(method: "POST" | "DELETE", path: string, fallback: string): Promise<T> {
|
||||
const res = await fetch(`${apiBase()}/admin/import-queue${path}`, { method, headers: authHeaders() });
|
||||
if (res.status === 401) throw new UnauthorizedError();
|
||||
if (!res.ok) throw new Error(await readErrorMessage(res, fallback));
|
||||
return res.json();
|
||||
}
|
||||
|
||||
export const adminApi = {
|
||||
getStorageUsage: async (): Promise<StorageUsage> => {
|
||||
const res = await fetch(`${apiBase()}/admin/storage`, { headers: authHeaders() });
|
||||
@@ -108,6 +135,29 @@ export const adminApi = {
|
||||
return res.json();
|
||||
},
|
||||
|
||||
getImportQueue: async (): Promise<ImportQueue> => {
|
||||
const res = await fetch(`${apiBase()}/admin/import-queue`, { headers: authHeaders() });
|
||||
assertOk(res, "Failed to load the import queue");
|
||||
return res.json();
|
||||
},
|
||||
|
||||
getImportQueueItems: async (jobId: string): Promise<ImportJobItem[]> => {
|
||||
const res = await fetch(`${apiBase()}/admin/import-queue/jobs/${jobId}/items`, { headers: authHeaders() });
|
||||
assertOk(res, "Failed to load the import's links");
|
||||
return res.json();
|
||||
},
|
||||
|
||||
startQueueJob: (jobId: string) => queueAction<{ ok: true }>("POST", `/jobs/${jobId}/start`, "Failed to start"),
|
||||
retryQueueJob: (jobId: string) => queueAction<{ ok: true }>("POST", `/jobs/${jobId}/retry`, "Failed to retry"),
|
||||
pauseQueueJob: (jobId: string) => queueAction<{ ok: true }>("POST", `/jobs/${jobId}/pause`, "Failed to pause"),
|
||||
deleteQueueJob: (jobId: string) => queueAction<{ ok: true }>("DELETE", `/jobs/${jobId}`, "Failed to delete"),
|
||||
removeQueueItem: (itemId: string) =>
|
||||
queueAction<{ jobDeleted: boolean }>("DELETE", `/items/${itemId}`, "Failed to remove the link"),
|
||||
startAllQueued: () => queueAction<{ queued: number }>("POST", "/start-all", "Failed to start the queue"),
|
||||
retryAllFailed: () => queueAction<{ queued: number }>("POST", "/retry-failed", "Failed to retry"),
|
||||
pauseAllQueued: () => queueAction<{ paused: number }>("POST", "/pause-all", "Failed to pause the queue"),
|
||||
clearFinishedQueued: () => queueAction<{ deleted: number }>("POST", "/clear-finished", "Failed to clear"),
|
||||
|
||||
listLogs: async (filter: { userId?: string; from?: string; to?: string }): Promise<LogEntry[]> => {
|
||||
const params = new URLSearchParams();
|
||||
if (filter.userId) params.set("user_id", filter.userId);
|
||||
|
||||
@@ -27,10 +27,11 @@ export type ImportCollectionEntriesResult = {
|
||||
entries: ImportCollectionEntry[];
|
||||
};
|
||||
|
||||
export type ImportJobType = "COLLECTION" | "ZIP" | "PROFILES";
|
||||
export type ImportJobType = "COLLECTION" | "ZIP" | "PROFILES" | "LINKS";
|
||||
|
||||
export type MakerworldProfileScope = "url" | "designer" | "all";
|
||||
export type ImportJobStatus = "RUNNING" | "DONE" | "ERROR";
|
||||
// PAUSED: a link queue waiting to be started from the admin import queue.
|
||||
export type ImportJobStatus = "RUNNING" | "DONE" | "ERROR" | "PAUSED";
|
||||
|
||||
/** Polled by ImportJobContext until status leaves RUNNING. */
|
||||
export type ImportJob = {
|
||||
@@ -51,6 +52,19 @@ export type ImportJob = {
|
||||
result_print_id: string | null;
|
||||
};
|
||||
|
||||
/** One queued link of a LINKS job. Polled alongside the job to show the queue's detail. */
|
||||
export type ImportJobItem = {
|
||||
id: string;
|
||||
url: string;
|
||||
status: "PENDING" | "RUNNING" | "DONE" | "FAILED";
|
||||
attempts: number;
|
||||
error_message: string | null;
|
||||
/** Set on links sent from the extension's "send to queue". */
|
||||
title: string | null;
|
||||
collection_id: string | null;
|
||||
scope: MakerworldProfileScope | null;
|
||||
};
|
||||
|
||||
type ImportLinkPayload = {
|
||||
url: string;
|
||||
title?: string;
|
||||
@@ -167,6 +181,32 @@ export const importsApi = {
|
||||
return res.json();
|
||||
},
|
||||
|
||||
/** Returns immediately; ImportJobContext polls the job. A pasted list of links, imported one at
|
||||
* a time; what fails can be rerun with retryImportJob. */
|
||||
fromLinks: async (
|
||||
payload: Omit<ImportLinkPayload, "url"> & { urls: string[]; scope?: MakerworldProfileScope },
|
||||
): Promise<{ job_id: string }> => {
|
||||
const res = await fetch(`${apiBase()}/import/links`, {
|
||||
method: "POST",
|
||||
headers: authHeaders({ "Content-Type": "application/json" }),
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
if (res.status === 401) throw new UnauthorizedError();
|
||||
if (!res.ok) throw new Error(await readErrorMessage(res, "Import failed"));
|
||||
return res.json();
|
||||
},
|
||||
|
||||
/** Reruns just a link-list job's failed links. */
|
||||
retryImportJob: async (jobId: string): Promise<{ job_id: string }> => {
|
||||
const res = await fetch(`${apiBase()}/import/jobs/${jobId}/retry`, {
|
||||
method: "POST",
|
||||
headers: authHeaders({ "Content-Type": "application/json" }),
|
||||
});
|
||||
if (res.status === 401) throw new UnauthorizedError();
|
||||
if (!res.ok) throw new Error(await readErrorMessage(res, "Retry failed"));
|
||||
return res.json();
|
||||
},
|
||||
|
||||
/** Returns immediately; ImportJobContext polls the job. */
|
||||
fromCollection: async (payload: ImportLinkPayload & { design_ids: string[] }): Promise<{ job_id: string }> => {
|
||||
const res = await fetch(`${apiBase()}/import/collection`, {
|
||||
@@ -300,4 +340,12 @@ export const importsApi = {
|
||||
if (!res.ok) throw new Error(await readErrorMessage(res, "Failed to load import progress"));
|
||||
return res.json();
|
||||
},
|
||||
|
||||
/** Per-link state of a queue job, for the progress bar's detail view. */
|
||||
getImportJobItems: async (jobId: string): Promise<ImportJobItem[]> => {
|
||||
const res = await fetch(`${apiBase()}/import/jobs/${jobId}/items`, { headers: authHeaders() });
|
||||
if (res.status === 401) throw new UnauthorizedError();
|
||||
if (!res.ok) throw new Error(await readErrorMessage(res, "Failed to load the import queue"));
|
||||
return res.json();
|
||||
},
|
||||
};
|
||||
@@ -92,6 +92,19 @@ export type UploadPrintsResult = {
|
||||
prints: Print[];
|
||||
};
|
||||
|
||||
export type ReimportResult = {
|
||||
source_url: string;
|
||||
title_filled: boolean;
|
||||
notes_filled: boolean;
|
||||
tags_added: string[];
|
||||
creator_filled: boolean;
|
||||
author_linked: boolean;
|
||||
category_filled: boolean;
|
||||
files_added: number;
|
||||
files_already_present: number;
|
||||
images_added: number;
|
||||
};
|
||||
|
||||
export type PrintSortMode = "newest" | "popular" | "downloads";
|
||||
|
||||
/** Filters combine with AND. */
|
||||
@@ -356,6 +369,23 @@ export const printsApi = {
|
||||
return res.json();
|
||||
},
|
||||
|
||||
/** Refreshes a model from its source; only empty fields are filled, never an edit. */
|
||||
reimport: async (
|
||||
id: string,
|
||||
payload: { metadata: boolean; files: boolean; images: boolean },
|
||||
): Promise<ReimportResult> => {
|
||||
const res = await fetch(`${apiBase()}/print/${id}/reimport`, {
|
||||
method: "POST",
|
||||
headers: authHeaders({ "Content-Type": "application/json" }),
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
if (res.status === 401) throw new UnauthorizedError();
|
||||
if (!res.ok) {
|
||||
throw new Error(await readErrorMessage(res, "Update from source failed"));
|
||||
}
|
||||
return res.json();
|
||||
},
|
||||
|
||||
delete: async (id: string) => {
|
||||
const res = await fetch(`${apiBase()}/print/${id}`, { method: "DELETE", headers: authHeaders() });
|
||||
assertOk(res, "Delete print failed");
|
||||
|
||||
@@ -32,6 +32,9 @@ import {
|
||||
IMPORT_LINK_EXAMPLES,
|
||||
isMakerworldCollectionUrl,
|
||||
isMakerworldModelUrl,
|
||||
isPrintablesCollectionUrl,
|
||||
isThingiverseCollectionUrl,
|
||||
isThingiverseLikesUrl,
|
||||
type ImportProviderKey,
|
||||
} from "../../utils/importLinkDetection";
|
||||
import type { MakerworldProfileScope } from "../../api/imports";
|
||||
@@ -60,9 +63,35 @@ export default function AddMenu({ categoryId, makerworldCookie, onUploaded, onUn
|
||||
const [captcha, setCaptcha] = React.useState<CaptchaAnswer | null>(null);
|
||||
const needsCaptcha = Boolean(captchaSettings?.import);
|
||||
|
||||
const detectedProvider = detectImportProvider(linkValue);
|
||||
const isBlockedCollection = isMakerworldCollectionUrl(linkValue);
|
||||
const isMakerworldModel = isMakerworldModelUrl(linkValue);
|
||||
// A pasted block of links becomes one queue job; a single link keeps the existing flows
|
||||
// (collection pickers, zip inspection, immediate import).
|
||||
const links = React.useMemo(
|
||||
() =>
|
||||
linkValue
|
||||
.split(/\s+/)
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean),
|
||||
[linkValue],
|
||||
);
|
||||
const isBatch = links.length > 1;
|
||||
const detectedProviders = React.useMemo(() => {
|
||||
const found = new Set<ImportProviderKey>();
|
||||
for (const link of links) {
|
||||
const key = detectImportProvider(link);
|
||||
if (key) found.add(key);
|
||||
}
|
||||
return found;
|
||||
}, [links]);
|
||||
const isBlockedCollection = isBatch
|
||||
? links.some(
|
||||
(link) =>
|
||||
isMakerworldCollectionUrl(link) ||
|
||||
isThingiverseLikesUrl(link) ||
|
||||
isThingiverseCollectionUrl(link) ||
|
||||
isPrintablesCollectionUrl(link),
|
||||
)
|
||||
: isMakerworldCollectionUrl(linkValue);
|
||||
const isMakerworldModel = links.some((link) => isMakerworldModelUrl(link));
|
||||
|
||||
const closeMenu = () => setAnchorEl(null);
|
||||
|
||||
@@ -90,9 +119,14 @@ export default function AddMenu({ categoryId, makerworldCookie, onUploaded, onUn
|
||||
};
|
||||
|
||||
const submitImport = async () => {
|
||||
if (!linkValue.trim() || isBlockedCollection) return;
|
||||
if (!links.length || isBlockedCollection) return;
|
||||
if (needsCaptcha && !captcha?.captcha_answer.trim()) return;
|
||||
// Each captcha works once; reopening shows a fresh one.
|
||||
if (isBatch) {
|
||||
await upload.submitImportMany(links, needsCaptcha ? captcha : null, isMakerworldModel ? profileScope : "url");
|
||||
setImportOpen(false);
|
||||
return;
|
||||
}
|
||||
await upload.submitImport(linkValue, needsCaptcha ? captcha : null, isMakerworldModel ? profileScope : "url");
|
||||
setImportOpen(false);
|
||||
};
|
||||
@@ -141,7 +175,7 @@ export default function AddMenu({ categoryId, makerworldCookie, onUploaded, onUn
|
||||
<Stack direction="row" spacing={1} sx={{ mb: 1.5 }}>
|
||||
{IMPORT_PROVIDERS.map((key) => {
|
||||
const info = IMPORT_PROVIDER_INFO[key];
|
||||
const active = detectedProvider === key;
|
||||
const active = detectedProviders.has(key);
|
||||
return (
|
||||
<Chip
|
||||
key={key}
|
||||
@@ -175,25 +209,31 @@ export default function AddMenu({ categoryId, makerworldCookie, onUploaded, onUn
|
||||
|
||||
{isBlockedCollection && (
|
||||
<Alert severity="warning" sx={{ mb: 1.5 }}>
|
||||
{t("addMenu.makerworldCollectionBlocked")}
|
||||
{t(isBatch ? "addMenu.batchCollectionsBlocked" : "addMenu.makerworldCollectionBlocked")}
|
||||
</Alert>
|
||||
)}
|
||||
|
||||
<TextField
|
||||
fullWidth
|
||||
type="url"
|
||||
multiline
|
||||
minRows={2}
|
||||
margin="dense"
|
||||
value={linkValue}
|
||||
onChange={(e) => setLinkValue(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === "Enter") {
|
||||
if (e.key === "Enter" && !e.shiftKey) {
|
||||
e.preventDefault();
|
||||
submitImport();
|
||||
}
|
||||
}}
|
||||
placeholder={t("uploadBar.linkPlaceholder") ?? undefined}
|
||||
placeholder={t("addMenu.linksPlaceholder") ?? undefined}
|
||||
disabled={upload.importing}
|
||||
/>
|
||||
{isBatch && (
|
||||
<Typography variant="caption" color="text.secondary" component="div" sx={{ mt: 0.5 }}>
|
||||
{t("addMenu.linksDetected", { count: links.length })}
|
||||
</Typography>
|
||||
)}
|
||||
{isMakerworldModel && (
|
||||
<TextField
|
||||
select
|
||||
@@ -227,7 +267,7 @@ export default function AddMenu({ categoryId, makerworldCookie, onUploaded, onUn
|
||||
onClick={submitImport}
|
||||
disabled={
|
||||
upload.importing ||
|
||||
!linkValue.trim() ||
|
||||
!links.length ||
|
||||
isBlockedCollection ||
|
||||
(needsCaptcha && !captcha?.captcha_answer.trim())
|
||||
}
|
||||
|
||||
@@ -103,6 +103,9 @@ function useRouteChrome() {
|
||||
} else if (path.startsWith("/admin-users")) {
|
||||
title = t("adminSettings.users.heading");
|
||||
onBack = () => navigate("/admin");
|
||||
} else if (path.startsWith("/admin-queue")) {
|
||||
title = t("adminSettings.importQueue.heading");
|
||||
onBack = () => navigate("/admin");
|
||||
} else if (path.startsWith("/admin-logs")) {
|
||||
title = t("adminSettings.logs.heading");
|
||||
onBack = () => navigate("/admin");
|
||||
|
||||
@@ -10,6 +10,7 @@ type StartThingiverseLikesImportPayload = Parameters<typeof importsApi.fromThing
|
||||
type StartThingiverseCollectionImportPayload = Parameters<typeof importsApi.fromThingiverseCollection>[0];
|
||||
type StartPrintablesCollectionImportPayload = Parameters<typeof importsApi.fromPrintablesCollection>[0];
|
||||
type StartMakerworldProfilesImportPayload = Parameters<typeof importsApi.fromMakerworldProfiles>[0];
|
||||
type StartLinksImportPayload = Parameters<typeof importsApi.fromLinks>[0];
|
||||
|
||||
type ImportJobContextValue = {
|
||||
/** Non-null while a batch import runs; drives the progress bar and disables importing. */
|
||||
@@ -21,6 +22,10 @@ type ImportJobContextValue = {
|
||||
startThingiverseCollectionImport: (payload: StartThingiverseCollectionImportPayload) => Promise<void>;
|
||||
startPrintablesCollectionImport: (payload: StartPrintablesCollectionImportPayload) => Promise<void>;
|
||||
startMakerworldProfilesImport: (payload: StartMakerworldProfilesImportPayload) => Promise<void>;
|
||||
startLinksImport: (payload: StartLinksImportPayload) => Promise<void>;
|
||||
/** Reruns the links that failed on the finished job shown in the progress bar. */
|
||||
retryJob: () => Promise<void>;
|
||||
dismissJob: () => void;
|
||||
};
|
||||
|
||||
const ImportJobContext = createContext<ImportJobContextValue | null>(null);
|
||||
@@ -58,8 +63,18 @@ export function ImportJobProvider({
|
||||
return;
|
||||
}
|
||||
stopPolling();
|
||||
setActiveJob(null);
|
||||
// Paused from the admin import queue: nothing finished, so don't navigate anywhere.
|
||||
if (job.status === "PAUSED") {
|
||||
setActiveJob(null);
|
||||
return;
|
||||
}
|
||||
onJobCompletedRef.current?.();
|
||||
// A link queue with failures stays on screen so they can be retried or dismissed.
|
||||
if (job.type === "LINKS" && (job.failed_count > 0 || job.status === "ERROR")) {
|
||||
setActiveJob(job);
|
||||
return;
|
||||
}
|
||||
setActiveJob(null);
|
||||
// One resulting print opens its details page; more go to the models grid; none stays put.
|
||||
if (job.status === "DONE") {
|
||||
if (job.result_print_id) {
|
||||
@@ -164,6 +179,29 @@ export function ImportJobProvider({
|
||||
[startPolling],
|
||||
);
|
||||
|
||||
const startLinksImport = useCallback(
|
||||
async (payload: StartLinksImportPayload) => {
|
||||
const { job_id } = await importsApi.fromLinks(payload);
|
||||
const job = await importsApi.getImportJob(job_id);
|
||||
setActiveJob(job);
|
||||
startPolling(job_id);
|
||||
},
|
||||
[startPolling],
|
||||
);
|
||||
|
||||
const retryJob = useCallback(async () => {
|
||||
if (!activeJob) return;
|
||||
const { job_id } = await importsApi.retryImportJob(activeJob.id);
|
||||
const job = await importsApi.getImportJob(job_id);
|
||||
setActiveJob(job);
|
||||
startPolling(job_id);
|
||||
}, [activeJob, startPolling]);
|
||||
|
||||
const dismissJob = useCallback(() => {
|
||||
stopPolling();
|
||||
setActiveJob(null);
|
||||
}, [stopPolling]);
|
||||
|
||||
const value = useMemo(
|
||||
() => ({
|
||||
activeJob,
|
||||
@@ -174,6 +212,9 @@ export function ImportJobProvider({
|
||||
startThingiverseCollectionImport,
|
||||
startPrintablesCollectionImport,
|
||||
startMakerworldProfilesImport,
|
||||
startLinksImport,
|
||||
retryJob,
|
||||
dismissJob,
|
||||
}),
|
||||
[
|
||||
activeJob,
|
||||
@@ -183,6 +224,9 @@ export function ImportJobProvider({
|
||||
startThingiverseCollectionImport,
|
||||
startPrintablesCollectionImport,
|
||||
startMakerworldProfilesImport,
|
||||
startLinksImport,
|
||||
retryJob,
|
||||
dismissJob,
|
||||
],
|
||||
);
|
||||
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import Dialog from "@mui/material/Dialog";
|
||||
import DialogTitle from "@mui/material/DialogTitle";
|
||||
import DialogContent from "@mui/material/DialogContent";
|
||||
import DialogActions from "@mui/material/DialogActions";
|
||||
import Button from "@mui/material/Button";
|
||||
import Stack from "@mui/material/Stack";
|
||||
import Box from "@mui/material/Box";
|
||||
import Typography from "@mui/material/Typography";
|
||||
import Chip from "@mui/material/Chip";
|
||||
import CircularProgress from "@mui/material/CircularProgress";
|
||||
import CheckCircleIcon from "@mui/icons-material/CheckCircle";
|
||||
import ErrorIcon from "@mui/icons-material/Error";
|
||||
import ScheduleIcon from "@mui/icons-material/Schedule";
|
||||
import { importsApi, type ImportJobItem } from "../../api/imports";
|
||||
|
||||
type Props = {
|
||||
jobId: string;
|
||||
open: boolean;
|
||||
onClose: () => void;
|
||||
};
|
||||
|
||||
const STATUS_COLOR: Record<ImportJobItem["status"], "default" | "success" | "warning" | "error"> = {
|
||||
PENDING: "default",
|
||||
RUNNING: "warning",
|
||||
DONE: "success",
|
||||
FAILED: "error",
|
||||
};
|
||||
|
||||
const STATUS_LABEL: Record<ImportJobItem["status"], string> = {
|
||||
PENDING: "importProgress.itemPending",
|
||||
RUNNING: "importProgress.itemRunning",
|
||||
DONE: "importProgress.itemDone",
|
||||
FAILED: "importProgress.itemFailed",
|
||||
};
|
||||
|
||||
function StatusIcon({ status }: { status: ImportJobItem["status"] }) {
|
||||
if (status === "RUNNING") return <CircularProgress size={16} />;
|
||||
if (status === "DONE") return <CheckCircleIcon fontSize="small" color="success" />;
|
||||
if (status === "FAILED") return <ErrorIcon fontSize="small" color="error" />;
|
||||
return <ScheduleIcon fontSize="small" color="disabled" />;
|
||||
}
|
||||
|
||||
/** The queue behind the progress bar: one row per pasted link, with why it failed. Opened by
|
||||
* clicking the bar, so a batch's failures are visible without waiting on the notification. */
|
||||
export default function ImportJobItemsDialog({ jobId, open, onClose }: Props) {
|
||||
const { t } = useTranslation("app");
|
||||
const [items, setItems] = useState<ImportJobItem[] | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) return;
|
||||
let cancelled = false;
|
||||
const load = async () => {
|
||||
try {
|
||||
const loaded = await importsApi.getImportJobItems(jobId);
|
||||
if (!cancelled) setItems(loaded);
|
||||
} catch {
|
||||
if (!cancelled) setItems([]);
|
||||
}
|
||||
};
|
||||
void load();
|
||||
const timer = window.setInterval(load, 1000);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
window.clearInterval(timer);
|
||||
};
|
||||
}, [open, jobId]);
|
||||
|
||||
return (
|
||||
<Dialog open={open} onClose={onClose} fullWidth maxWidth="sm">
|
||||
<DialogTitle>{t("importProgress.queueTitle")}</DialogTitle>
|
||||
<DialogContent dividers>
|
||||
<Stack spacing={1}>
|
||||
{(items ?? []).map((item) => (
|
||||
<Stack key={item.id} direction="row" spacing={1.5} alignItems="flex-start">
|
||||
<Box sx={{ pt: 0.25 }}>
|
||||
<StatusIcon status={item.status} />
|
||||
</Box>
|
||||
<Box sx={{ minWidth: 0, flex: 1 }}>
|
||||
<Typography variant="body2" sx={{ wordBreak: "break-all" }}>
|
||||
{item.url}
|
||||
</Typography>
|
||||
{item.error_message && (
|
||||
<Typography variant="caption" color="error" component="div" sx={{ wordBreak: "break-word" }}>
|
||||
{item.error_message}
|
||||
</Typography>
|
||||
)}
|
||||
</Box>
|
||||
<Chip
|
||||
size="small"
|
||||
color={STATUS_COLOR[item.status]}
|
||||
variant={item.status === "PENDING" ? "outlined" : "filled"}
|
||||
label={t(STATUS_LABEL[item.status])}
|
||||
/>
|
||||
</Stack>
|
||||
))}
|
||||
{items && !items.length && (
|
||||
<Typography variant="body2" color="text.secondary">
|
||||
{t("importProgress.queueEmpty")}
|
||||
</Typography>
|
||||
)}
|
||||
</Stack>
|
||||
</DialogContent>
|
||||
<DialogActions>
|
||||
<Button onClick={onClose}>{t("importProgress.queueClose")}</Button>
|
||||
</DialogActions>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
@@ -1,49 +1,106 @@
|
||||
import { useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import Box from "@mui/material/Box";
|
||||
import Stack from "@mui/material/Stack";
|
||||
import Typography from "@mui/material/Typography";
|
||||
import LinearProgress from "@mui/material/LinearProgress";
|
||||
import Button from "@mui/material/Button";
|
||||
import IconButton from "@mui/material/IconButton";
|
||||
import Tooltip from "@mui/material/Tooltip";
|
||||
import CloseIcon from "@mui/icons-material/Close";
|
||||
import ReplayIcon from "@mui/icons-material/Replay";
|
||||
import ListIcon from "@mui/icons-material/List";
|
||||
import { useImportJob } from "./ImportJobContext";
|
||||
import ImportJobItemsDialog from "./ImportJobItemsDialog";
|
||||
|
||||
/** Shown while a batch import runs; hovering shows the full breakdown. */
|
||||
/** Shown while a batch import runs; clicking shows the per-link queue. A link queue that finished
|
||||
* with failures stays up, in warning colors, with a retry button until it's dismissed. */
|
||||
export default function ImportProgressBar() {
|
||||
const { t } = useTranslation("app");
|
||||
const { activeJob } = useImportJob();
|
||||
const { activeJob, retryJob, dismissJob } = useImportJob();
|
||||
const [detailsOpen, setDetailsOpen] = useState(false);
|
||||
if (!activeJob) return null;
|
||||
|
||||
const { total, processed, imported, already_in_library: alreadyInLibrary, failed_count: failedCount } = activeJob;
|
||||
const progressValue = total > 0 ? Math.min(100, (processed / total) * 100) : 0;
|
||||
const label = total > 0 ? t("importProgress.label", { processed, total }) : t("importProgress.starting");
|
||||
const tooltip = t("importProgress.tooltip", { imported, alreadyInLibrary, failedCount });
|
||||
const running = activeJob.status === "RUNNING";
|
||||
const hasFailures = !running && (failedCount > 0 || activeJob.status === "ERROR");
|
||||
const label = running
|
||||
? total > 0
|
||||
? t("importProgress.label", { processed, total })
|
||||
: t("importProgress.starting")
|
||||
: hasFailures
|
||||
? t("importProgress.doneWithFailures", { imported, alreadyInLibrary, failedCount })
|
||||
: t("importProgress.done", { imported, alreadyInLibrary });
|
||||
const tooltip = hasFailures
|
||||
? t("importProgress.tooltipFailed", { failedCount })
|
||||
: t("importProgress.tooltip", { imported, alreadyInLibrary, failedCount });
|
||||
|
||||
return (
|
||||
<Tooltip title={tooltip} placement="top">
|
||||
<Box
|
||||
sx={{
|
||||
position: "fixed",
|
||||
left: 0,
|
||||
right: 0,
|
||||
bottom: 0,
|
||||
zIndex: (theme) => theme.zIndex.snackbar,
|
||||
bgcolor: "background.paper",
|
||||
borderTop: "1px solid",
|
||||
borderColor: "divider",
|
||||
px: 2,
|
||||
py: 0.75,
|
||||
}}
|
||||
>
|
||||
<Stack spacing={0.5} sx={{ maxWidth: 480, mx: "auto" }}>
|
||||
<Typography variant="caption" color="text.secondary" textAlign="center">
|
||||
{label}
|
||||
</Typography>
|
||||
<LinearProgress
|
||||
variant={total > 0 ? "determinate" : "indeterminate"}
|
||||
value={progressValue}
|
||||
sx={{ borderRadius: 1, height: 6 }}
|
||||
/>
|
||||
</Stack>
|
||||
</Box>
|
||||
</Tooltip>
|
||||
<>
|
||||
<Tooltip title={tooltip} placement="top">
|
||||
<Box
|
||||
onClick={() => setDetailsOpen(true)}
|
||||
sx={{
|
||||
position: "fixed",
|
||||
left: 0,
|
||||
right: 0,
|
||||
bottom: 0,
|
||||
zIndex: (theme) => theme.zIndex.snackbar,
|
||||
bgcolor: hasFailures ? "warning.main" : "background.paper",
|
||||
color: hasFailures ? "warning.contrastText" : "inherit",
|
||||
borderTop: "1px solid",
|
||||
borderColor: hasFailures ? "warning.dark" : "divider",
|
||||
px: 2,
|
||||
py: 0.75,
|
||||
cursor: "pointer",
|
||||
}}
|
||||
>
|
||||
<Stack spacing={0.5} sx={{ maxWidth: 480, mx: "auto" }}>
|
||||
<Stack direction="row" spacing={0.75} justifyContent="center" alignItems="center">
|
||||
<ListIcon fontSize="small" sx={{ opacity: 0.7 }} />
|
||||
<Typography variant="caption" textAlign="center">
|
||||
{label}
|
||||
</Typography>
|
||||
</Stack>
|
||||
{running ? (
|
||||
<LinearProgress
|
||||
variant={total > 0 ? "determinate" : "indeterminate"}
|
||||
value={progressValue}
|
||||
sx={{ borderRadius: 1, height: 6 }}
|
||||
/>
|
||||
) : (
|
||||
<Stack direction="row" spacing={1} justifyContent="center" alignItems="center">
|
||||
<Button
|
||||
size="small"
|
||||
startIcon={<ReplayIcon fontSize="small" />}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
void retryJob();
|
||||
}}
|
||||
sx={hasFailures ? { color: "warning.contrastText" } : undefined}
|
||||
>
|
||||
{t("importProgress.retryFailed")}
|
||||
</Button>
|
||||
<IconButton
|
||||
size="small"
|
||||
aria-label={t("importProgress.dismiss")}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
dismissJob();
|
||||
}}
|
||||
sx={hasFailures ? { color: "warning.contrastText" } : undefined}
|
||||
>
|
||||
<CloseIcon fontSize="small" />
|
||||
</IconButton>
|
||||
</Stack>
|
||||
)}
|
||||
</Stack>
|
||||
</Box>
|
||||
</Tooltip>
|
||||
{activeJob.type === "LINKS" && (
|
||||
<ImportJobItemsDialog jobId={activeJob.id} open={detailsOpen} onClose={() => setDetailsOpen(false)} />
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -59,6 +59,7 @@ export function useUploadImport({ onUploaded, categoryId, makerworldCookie, onUn
|
||||
startThingiverseCollectionImport,
|
||||
startPrintablesCollectionImport,
|
||||
startMakerworldProfilesImport,
|
||||
startLinksImport,
|
||||
} = useImportJob();
|
||||
const isBusy = uploading || importing || zipPrompt.isOpen || collectionPrompt.isOpen || importModePrompt.isOpen;
|
||||
|
||||
@@ -409,6 +410,34 @@ export function useUploadImport({ onUploaded, categoryId, makerworldCookie, onUn
|
||||
}
|
||||
};
|
||||
|
||||
/** Several pasted links run as one queue job, with per-link retries for whatever fails. */
|
||||
const submitImportMany = async (
|
||||
urls: string[],
|
||||
captcha?: CaptchaAnswer | null,
|
||||
profileScope: MakerworldProfileScope = "url",
|
||||
) => {
|
||||
if (!urls.length) return;
|
||||
setImporting(true);
|
||||
try {
|
||||
await startLinksImport({
|
||||
urls,
|
||||
scope: profileScope,
|
||||
category_id: categoryId || undefined,
|
||||
makerworld_cookie: (makerworldCookie || "").trim() || undefined,
|
||||
...captcha,
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof UnauthorizedError) {
|
||||
onUnauthorized?.();
|
||||
return;
|
||||
}
|
||||
console.error("Import failed for", urls, err);
|
||||
alert(err instanceof Error ? err.message : t("uploadBar.importFailed"));
|
||||
} finally {
|
||||
setImporting(false);
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
fileInput,
|
||||
folderInput,
|
||||
@@ -418,6 +447,7 @@ export function useUploadImport({ onUploaded, categoryId, makerworldCookie, onUn
|
||||
triggerUpload,
|
||||
triggerFolderUpload,
|
||||
submitImport,
|
||||
submitImportMany,
|
||||
modals: (
|
||||
<>
|
||||
{zipPrompt.modal}
|
||||
|
||||
@@ -217,7 +217,10 @@
|
||||
"profilesDesigner": "All designer print profiles",
|
||||
"profilesAll": "Designer and community print profiles",
|
||||
"profilesHelpUrl": "The profile the link points to, or the model's default one. Other profiles can be added later.",
|
||||
"profilesHelpMany": "Each profile becomes its own file on the model. They're imported one at a time in the background, which can take a few minutes."
|
||||
"profilesHelpMany": "Each profile becomes its own file on the model. They're imported one at a time in the background, which can take a few minutes.",
|
||||
"linksPlaceholder": "Paste model links, one per line. Several links are imported as a queue.",
|
||||
"linksDetected": "{{count}} links will be imported one at a time as a background job. Failed links can be retried when it finishes.",
|
||||
"batchCollectionsBlocked": "Collection links (likes, collections) can only be imported one at a time. Paste model links here instead, or import a collection on its own."
|
||||
},
|
||||
"userMenu": {
|
||||
"theme": "Theme",
|
||||
@@ -256,7 +259,19 @@
|
||||
"importProgress": {
|
||||
"starting": "Starting import…",
|
||||
"label": "Importing… {{processed}} of {{total}}",
|
||||
"tooltip": "{{imported}} imported, {{alreadyInLibrary}} already in your library, {{failedCount}} failed"
|
||||
"tooltip": "{{imported}} imported, {{alreadyInLibrary}} already in your library, {{failedCount}} failed. Click to see the queue.",
|
||||
"done": "Imported {{imported}}, {{alreadyInLibrary}} already in your library. Click to see the queue.",
|
||||
"doneWithFailures": "Imported {{imported}}, {{alreadyInLibrary}} already in your library, {{failedCount}} failed. Click to see which and why.",
|
||||
"tooltipFailed": "{{failedCount}} links failed. Click to see which ones and why.",
|
||||
"retryFailed": "Retry failed",
|
||||
"dismiss": "Dismiss",
|
||||
"queueTitle": "Import queue",
|
||||
"queueEmpty": "No links in this queue.",
|
||||
"queueClose": "Close",
|
||||
"itemPending": "waiting",
|
||||
"itemRunning": "importing",
|
||||
"itemDone": "imported",
|
||||
"itemFailed": "failed"
|
||||
},
|
||||
"dashboard": {
|
||||
"loadError": "Failed to load your dashboard.",
|
||||
@@ -615,6 +630,77 @@
|
||||
"loadFailed": "Failed to load rendering settings.",
|
||||
"saveFailed": "Failed to save rendering settings."
|
||||
}
|
||||
},
|
||||
"importQueue": {
|
||||
"heading": "Import queue",
|
||||
"loadFailed": "Unable to load the import queue.",
|
||||
"empty": "Nothing in the queue. Links sent from Thingport Grab with \"Send to queue\" on wait here until you start them.",
|
||||
"emptyFilter": "No imports match this filter.",
|
||||
"startAll": "Start all ({{count}})",
|
||||
"retryAll": "Retry all failed ({{count}})",
|
||||
"pauseAll": "Pause all",
|
||||
"clearFinished": "Clear finished",
|
||||
"startedAll": "Started {{count}} import(s). They run one at a time.",
|
||||
"retriedAll": "Retrying {{count}} import(s), one at a time.",
|
||||
"cleared": "Cleared {{count}} finished import(s).",
|
||||
"bulkRunning": "Running imports one at a time, {{count}} more waiting.",
|
||||
"filter": {
|
||||
"all": "All",
|
||||
"paused": "Paused",
|
||||
"running": "Running",
|
||||
"failed": "With failures",
|
||||
"done": "Finished"
|
||||
},
|
||||
"columnSource": "Import",
|
||||
"columnUser": "User",
|
||||
"columnStatus": "Status",
|
||||
"columnProgress": "Progress",
|
||||
"columnCreated": "Created",
|
||||
"columnActions": "Actions",
|
||||
"progress": "{{processed}} of {{total}}, {{imported}} imported, {{failed}} failed",
|
||||
"type": {
|
||||
"LINKS": "Links",
|
||||
"COLLECTION": "Collection",
|
||||
"ZIP": "ZIP",
|
||||
"PROFILES": "Print profiles"
|
||||
},
|
||||
"status": {
|
||||
"paused": "Paused",
|
||||
"pausing": "Pausing",
|
||||
"running": "Running",
|
||||
"stalled": "Stalled",
|
||||
"error": "Error",
|
||||
"done": "Done",
|
||||
"doneWithFailures": "Done, with failures"
|
||||
},
|
||||
"statusHint": {
|
||||
"paused": "Waiting to be started.",
|
||||
"pausing": "Finishing the link it's on, then it stops.",
|
||||
"running": "Importing now.",
|
||||
"stalled": "Cut off by a restart. Pause it, then start it again to finish.",
|
||||
"error": "The import failed.",
|
||||
"done": "Every link imported.",
|
||||
"doneWithFailures": "Finished, but some links failed. Retry them once the cause is fixed."
|
||||
},
|
||||
"start": "Start",
|
||||
"pause": "Pause after the current link",
|
||||
"retry": "Retry {{count}} failed",
|
||||
"delete": "Delete",
|
||||
"deleteTitle": "Delete this import?",
|
||||
"deleteMessage": "Its {{count}} link(s) are removed from the queue. Models already imported stay in the library.",
|
||||
"showLinks": "Show links",
|
||||
"hideLinks": "Hide links",
|
||||
"removeLink": "Remove from queue",
|
||||
"itemStatus": {
|
||||
"PENDING": "waiting",
|
||||
"RUNNING": "importing",
|
||||
"DONE": "imported",
|
||||
"FAILED": "failed"
|
||||
},
|
||||
"scope": {
|
||||
"designer": "All designer print profiles",
|
||||
"all": "Designer and community print profiles"
|
||||
}
|
||||
}
|
||||
},
|
||||
"download": {
|
||||
@@ -1304,6 +1390,17 @@
|
||||
"confirmRemoveFromCollection": "Remove \"{{name}}\" from this collection?",
|
||||
"removeFromCollectionFailed": "Failed to remove from collection. Please try again.",
|
||||
"openInProvider": "Open in {{provider}}",
|
||||
"reimport": "Update from source",
|
||||
"reimportSource": "its source",
|
||||
"reimportTitle": "Update from source",
|
||||
"reimportHint": "Pulls what {{provider}} offers right now. Only empty fields are filled: an edit you made always wins, and tags are only added, never removed.",
|
||||
"reimportMetadata": "Fill empty metadata (title, description, tags, author, category)",
|
||||
"reimportFiles": "Add missing files (e.g. print profiles not on the model yet)",
|
||||
"reimportImages": "Fill empty preview images",
|
||||
"reimportSubmit": "Update",
|
||||
"reimportDone": "Updated: {{filesAdded}} files added, {{imagesAdded}} images added, {{tagsAdded}} tags added.",
|
||||
"reimportNothing": "Nothing new to add: the model already has everything the source offers.",
|
||||
"reimportFailed": "Couldn't update from source. Please try again.",
|
||||
"openInSlicer": "Open in {{slicer}}",
|
||||
"openNormalizedInSlicer": "Open normalized in {{slicer}}",
|
||||
"normalizedInfo": "MakerWorld files are Bambu Studio projects, which {{slicer}} often opens without their colors or print settings. This opens a converted copy that keeps painted colors, filament colors, plates and the designer's settings (walls, layer height, infill, supports). Heads up: multi-part objects may be merged into one, per-object overrides and layer color changes are dropped, and you pick your own printer profile. The stored file isn't changed.",
|
||||
|
||||
@@ -217,7 +217,10 @@
|
||||
"profilesDesigner": "Visi dizainerio spausdinimo profiliai",
|
||||
"profilesAll": "Dizainerio ir bendruomenės spausdinimo profiliai",
|
||||
"profilesHelpUrl": "Profilis, į kurį rodo nuoroda, arba numatytasis modelio profilis. Kitus profilius galima pridėti vėliau.",
|
||||
"profilesHelpMany": "Kiekvienas profilis tampa atskiru modelio failu. Jie importuojami po vieną fone, todėl tai gali užtrukti kelias minutes."
|
||||
"profilesHelpMany": "Kiekvienas profilis tampa atskiru modelio failu. Jie importuojami po vieną fone, todėl tai gali užtrukti kelias minutes.",
|
||||
"linksPlaceholder": "Įklijuokite modelių nuorodas, po vieną eilutėje. Kelios nuorodos importuojamos kaip eilė.",
|
||||
"linksDetected": "{{count}} nuorodos bus importuojamos po vieną kaip fono užduotis. Nepavykusias nuorodas galėsite pakartoti jai pasibaigus.",
|
||||
"batchCollectionsBlocked": "Kolekcijų nuorodas (patikimai, kolekcijas) galima importuoti tik po vieną. Įklijuokite modelių nuorodas arba importuokite kolekciją atskirai."
|
||||
},
|
||||
"userMenu": {
|
||||
"theme": "Tema",
|
||||
@@ -256,7 +259,19 @@
|
||||
"importProgress": {
|
||||
"starting": "Pradedamas importas…",
|
||||
"label": "Importuojama… {{processed}} iš {{total}}",
|
||||
"tooltip": "{{imported}} importuota, {{alreadyInLibrary}} jau bibliotekoje, {{failedCount}} nepavyko"
|
||||
"tooltip": "{{imported}} importuota, {{alreadyInLibrary}} jau bibliotekoje, {{failedCount}} nepavyko. Spustelėkite, kad pamatytumėte eilę.",
|
||||
"done": "Importuota {{imported}}, {{alreadyInLibrary}} jau bibliotekoje. Spustelėkite, kad pamatytumėte eilę.",
|
||||
"doneWithFailures": "Importuota {{imported}}, {{alreadyInLibrary}} jau bibliotekoje, {{failedCount}} nepavyko. Spustelėkite, kad pamatytumėte, kurios ir kodėl.",
|
||||
"tooltipFailed": "{{failedCount}} nuorodos nepavyko. Spustelėkite, kad pamatytumėte, kurios ir kodėl.",
|
||||
"retryFailed": "Pakartoti nepavykusius",
|
||||
"dismiss": "Uždaryti",
|
||||
"queueTitle": "Importo eilė",
|
||||
"queueEmpty": "Šioje eilėje nėra nuorodų.",
|
||||
"queueClose": "Uždaryti",
|
||||
"itemPending": "laukiama",
|
||||
"itemRunning": "importuojama",
|
||||
"itemDone": "importuota",
|
||||
"itemFailed": "nepavyko"
|
||||
},
|
||||
"dashboard": {
|
||||
"loadError": "Nepavyko įkelti valdymo skydo.",
|
||||
@@ -617,6 +632,77 @@
|
||||
"loadFailed": "Nepavyko įkelti atvaizdavimo nustatymų.",
|
||||
"saveFailed": "Nepavyko išsaugoti atvaizdavimo nustatymų."
|
||||
}
|
||||
},
|
||||
"importQueue": {
|
||||
"heading": "Importavimo eilė",
|
||||
"loadFailed": "Nepavyko įkelti importavimo eilės.",
|
||||
"empty": "Eilė tuščia. Nuorodos, išsiųstos iš Thingport Grab su įjungtu „Siųsti į eilę“, laukia čia, kol jas paleisite.",
|
||||
"emptyFilter": "Nėra importų pagal šį filtrą.",
|
||||
"startAll": "Paleisti visus ({{count}})",
|
||||
"retryAll": "Kartoti visus nepavykusius ({{count}})",
|
||||
"pauseAll": "Pristabdyti visus",
|
||||
"clearFinished": "Išvalyti baigtus",
|
||||
"startedAll": "Paleista importų: {{count}}. Jie vykdomi po vieną.",
|
||||
"retriedAll": "Kartojama importų: {{count}}, po vieną.",
|
||||
"cleared": "Išvalyta baigtų importų: {{count}}.",
|
||||
"bulkRunning": "Importai vykdomi po vieną, dar laukia: {{count}}.",
|
||||
"filter": {
|
||||
"all": "Visi",
|
||||
"paused": "Pristabdyti",
|
||||
"running": "Vykdomi",
|
||||
"failed": "Su klaidomis",
|
||||
"done": "Baigti"
|
||||
},
|
||||
"columnSource": "Importas",
|
||||
"columnUser": "Naudotojas",
|
||||
"columnStatus": "Būsena",
|
||||
"columnProgress": "Eiga",
|
||||
"columnCreated": "Sukurta",
|
||||
"columnActions": "Veiksmai",
|
||||
"progress": "{{processed}} iš {{total}}, importuota {{imported}}, nepavyko {{failed}}",
|
||||
"type": {
|
||||
"LINKS": "Nuorodos",
|
||||
"COLLECTION": "Kolekcija",
|
||||
"ZIP": "ZIP",
|
||||
"PROFILES": "Spausdinimo profiliai"
|
||||
},
|
||||
"status": {
|
||||
"paused": "Pristabdytas",
|
||||
"pausing": "Stabdomas",
|
||||
"running": "Vykdomas",
|
||||
"stalled": "Įstrigęs",
|
||||
"error": "Klaida",
|
||||
"done": "Baigtas",
|
||||
"doneWithFailures": "Baigtas su klaidomis"
|
||||
},
|
||||
"statusHint": {
|
||||
"paused": "Laukia paleidimo.",
|
||||
"pausing": "Baigia dabartinę nuorodą ir sustoja.",
|
||||
"running": "Dabar importuojama.",
|
||||
"stalled": "Nutrauktas paleidus serverį iš naujo. Pristabdykite ir paleiskite vėl, kad užbaigtumėte.",
|
||||
"error": "Importas nepavyko.",
|
||||
"done": "Visos nuorodos importuotos.",
|
||||
"doneWithFailures": "Baigta, bet kai kurios nuorodos nepavyko. Pakartokite jas, kai priežastis bus pašalinta."
|
||||
},
|
||||
"start": "Paleisti",
|
||||
"pause": "Pristabdyti po dabartinės nuorodos",
|
||||
"retry": "Kartoti nepavykusias ({{count}})",
|
||||
"delete": "Ištrinti",
|
||||
"deleteTitle": "Ištrinti šį importą?",
|
||||
"deleteMessage": "Jo nuorodos ({{count}}) pašalinamos iš eilės. Jau importuoti modeliai lieka bibliotekoje.",
|
||||
"showLinks": "Rodyti nuorodas",
|
||||
"hideLinks": "Slėpti nuorodas",
|
||||
"removeLink": "Pašalinti iš eilės",
|
||||
"itemStatus": {
|
||||
"PENDING": "laukia",
|
||||
"RUNNING": "importuojama",
|
||||
"DONE": "importuota",
|
||||
"FAILED": "nepavyko"
|
||||
},
|
||||
"scope": {
|
||||
"designer": "Visi dizainerio spausdinimo profiliai",
|
||||
"all": "Dizainerio ir bendruomenės spausdinimo profiliai"
|
||||
}
|
||||
}
|
||||
},
|
||||
"download": {
|
||||
@@ -1307,6 +1393,17 @@
|
||||
"confirmRemoveFromCollection": "Pašalinti „{{name}}“ iš šios kolekcijos?",
|
||||
"removeFromCollectionFailed": "Nepavyko pašalinti iš kolekcijos. Bandykite dar kartą.",
|
||||
"openInProvider": "Atverti „{{provider}}“",
|
||||
"reimport": "Atnaujinti iš šaltinio",
|
||||
"reimportSource": "jo šaltinio",
|
||||
"reimportTitle": "Atnaujinti iš šaltinio",
|
||||
"reimportHint": "Paima tai, ką {{provider}} siūlo dabar. Užpildomi tik tušti laukai: jūsų pakeitimai visada išlieka, o žymos tik pridedamos, niekada nepašalinamos.",
|
||||
"reimportMetadata": "Užpildyti tuščius metaduomenis (pavadinimą, aprašymą, žymas, autorių, kategoriją)",
|
||||
"reimportFiles": "Pridėti trūkstamus failus (pvz., spausdinimo profilius, kurių modelis dar neturi)",
|
||||
"reimportImages": "Užpildyti tuščias peržiūros nuotraukas",
|
||||
"reimportSubmit": "Atnaujinti",
|
||||
"reimportDone": "Atnaujinta: pridėta {{filesAdded}} failų, {{imagesAdded}} nuotraukų, {{tagsAdded}} žymų.",
|
||||
"reimportNothing": "Nėra ko pridėti: modelis jau turi viską, ką siūlo šaltinis.",
|
||||
"reimportFailed": "Nepavyko atnaujinti iš šaltinio. Bandykite dar kartą.",
|
||||
"openInSlicer": "Atverti su {{slicer}}",
|
||||
"openNormalizedInSlicer": "Atverti normalizuotą su {{slicer}}",
|
||||
"normalizedInfo": "MakerWorld failai yra Bambu Studio projektai, kuriuos {{slicer}} dažnai atveria be spalvų ar spausdinimo nustatymų. Ši parinktis atveria konvertuotą kopiją, kurioje išlieka nudažytos spalvos, gijų spalvos, plokštės ir autoriaus nustatymai (sienelės, sluoksnio aukštis, užpildas, atramos). Atkreipkite dėmesį: kelių dalių objektai gali būti sujungti į vieną, atskirų objektų nustatymai ir spalvų keitimai tarp sluoksnių pašalinami, o spausdintuvo profilį pasirenkate patys. Saugomas failas nekeičiamas.",
|
||||
|
||||
@@ -20,6 +20,7 @@ import CableIcon from "@mui/icons-material/Cable";
|
||||
import PublicIcon from "@mui/icons-material/Public";
|
||||
import VerifiedUserIcon from "@mui/icons-material/VerifiedUser";
|
||||
import ViewInArIcon from "@mui/icons-material/ViewInAr";
|
||||
import QueueIcon from "@mui/icons-material/Queue";
|
||||
import UpdateCheckSection from "./UpdateCheckSection";
|
||||
import { adminApi, type StorageUsage } from "../../api/admin";
|
||||
import { UnauthorizedError } from "../../api/client";
|
||||
@@ -36,6 +37,7 @@ const SECTIONS: Section[] = [
|
||||
{ path: "/admin-settings", icon: <SettingsIcon fontSize="small" />, labelKey: "common:settings" },
|
||||
{ path: "/admin-rendering", icon: <ViewInArIcon fontSize="small" />, labelKey: "adminSettings.rendering.heading" },
|
||||
{ path: "/admin-users", icon: <PeopleIcon fontSize="small" />, labelKey: "adminSettings.users.heading" },
|
||||
{ path: "/admin-queue", icon: <QueueIcon fontSize="small" />, labelKey: "adminSettings.importQueue.heading" },
|
||||
{ path: "/admin-logs", icon: <HistoryIcon fontSize="small" />, labelKey: "adminSettings.logs.heading" },
|
||||
{ path: "/admin-triggers", icon: <BoltIcon fontSize="small" />, labelKey: "adminSettings.triggers.heading" },
|
||||
{ path: "/admin-connections", icon: <CableIcon fontSize="small" />, labelKey: "adminSettings.connections.heading" },
|
||||
|
||||
@@ -0,0 +1,541 @@
|
||||
import React from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import Stack from "@mui/material/Stack";
|
||||
import Paper from "@mui/material/Paper";
|
||||
import Box from "@mui/material/Box";
|
||||
import Button from "@mui/material/Button";
|
||||
import IconButton from "@mui/material/IconButton";
|
||||
import Tooltip from "@mui/material/Tooltip";
|
||||
import Chip from "@mui/material/Chip";
|
||||
import Collapse from "@mui/material/Collapse";
|
||||
import CircularProgress from "@mui/material/CircularProgress";
|
||||
import LinearProgress from "@mui/material/LinearProgress";
|
||||
import Alert from "@mui/material/Alert";
|
||||
import Link from "@mui/material/Link";
|
||||
import ToggleButton from "@mui/material/ToggleButton";
|
||||
import ToggleButtonGroup from "@mui/material/ToggleButtonGroup";
|
||||
import Typography from "@mui/material/Typography";
|
||||
import Table from "@mui/material/Table";
|
||||
import TableBody from "@mui/material/TableBody";
|
||||
import TableCell from "@mui/material/TableCell";
|
||||
import TableContainer from "@mui/material/TableContainer";
|
||||
import TableHead from "@mui/material/TableHead";
|
||||
import TableRow from "@mui/material/TableRow";
|
||||
import PlayArrowIcon from "@mui/icons-material/PlayArrow";
|
||||
import PauseIcon from "@mui/icons-material/Pause";
|
||||
import ReplayIcon from "@mui/icons-material/Replay";
|
||||
import DeleteOutlineIcon from "@mui/icons-material/DeleteOutline";
|
||||
import DeleteSweepIcon from "@mui/icons-material/DeleteSweep";
|
||||
import CloseIcon from "@mui/icons-material/Close";
|
||||
import KeyboardArrowDownIcon from "@mui/icons-material/KeyboardArrowDown";
|
||||
import KeyboardArrowRightIcon from "@mui/icons-material/KeyboardArrowRight";
|
||||
import { UnauthorizedError } from "../../api/client";
|
||||
import { adminApi, type AdminImportJob, type ImportQueue } from "../../api/admin";
|
||||
import type { ImportJobItem } from "../../api/imports";
|
||||
import { useConfirm } from "../../components/ConfirmProvider";
|
||||
import { useToast } from "../../components/ToastProvider";
|
||||
|
||||
type Props = {
|
||||
onUnauthorized?: () => void;
|
||||
};
|
||||
|
||||
type Filter = "all" | "paused" | "running" | "failed" | "done";
|
||||
|
||||
// Fast while something moves, so progress reads live; slow otherwise, to pick up links the
|
||||
// extension sends.
|
||||
const POLL_ACTIVE_MS = 1500;
|
||||
const POLL_IDLE_MS = 8000;
|
||||
|
||||
type StatusColor = "default" | "success" | "warning" | "error" | "info";
|
||||
|
||||
/** A RUNNING job without a live runner was cut off by a restart. */
|
||||
function isStalled(job: AdminImportJob): boolean {
|
||||
return job.status === "RUNNING" && !job.runner_live;
|
||||
}
|
||||
|
||||
function jobStatus(job: AdminImportJob): { key: string; color: StatusColor } {
|
||||
if (isStalled(job)) return { key: "stalled", color: "error" };
|
||||
if (job.status === "PAUSED") return { key: job.runner_live ? "pausing" : "paused", color: "info" };
|
||||
if (job.status === "RUNNING") return { key: "running", color: "warning" };
|
||||
if (job.status === "ERROR") return { key: "error", color: "error" };
|
||||
return job.failed_count > 0 ? { key: "doneWithFailures", color: "warning" } : { key: "done", color: "success" };
|
||||
}
|
||||
|
||||
function matchesFilter(job: AdminImportJob, filter: Filter): boolean {
|
||||
switch (filter) {
|
||||
case "paused":
|
||||
return job.status === "PAUSED";
|
||||
case "running":
|
||||
return job.status === "RUNNING";
|
||||
case "failed":
|
||||
return job.status === "ERROR" || job.failed_items_count > 0 || job.failed_count > 0;
|
||||
case "done":
|
||||
return job.status === "DONE";
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
const ITEM_COLOR: Record<ImportJobItem["status"], StatusColor> = {
|
||||
PENDING: "default",
|
||||
RUNNING: "warning",
|
||||
DONE: "success",
|
||||
FAILED: "error",
|
||||
};
|
||||
|
||||
/** Every user's batch imports. Links sent from the extension's "send to queue" wait here, paused,
|
||||
* until started; "Start all" and "Retry all failed" run the jobs one after another. */
|
||||
export default function ImportQueuePage({ onUnauthorized }: Props) {
|
||||
const { t, i18n } = useTranslation(["app", "common"]);
|
||||
const confirm = useConfirm();
|
||||
const showToast = useToast();
|
||||
const [queue, setQueue] = React.useState<ImportQueue | null>(null);
|
||||
const [error, setError] = React.useState<string | null>(null);
|
||||
const [filter, setFilter] = React.useState<Filter>("all");
|
||||
const [expanded, setExpanded] = React.useState<string | null>(null);
|
||||
const [busy, setBusy] = React.useState<string | null>(null);
|
||||
// The expanded job's links, reloaded with the queue on every poll.
|
||||
const [items, setItems] = React.useState<{ jobId: string; items: ImportJobItem[] } | null>(null);
|
||||
const expandedRef = React.useRef<string | null>(null);
|
||||
|
||||
const load = React.useCallback(async () => {
|
||||
try {
|
||||
const jobId = expandedRef.current;
|
||||
const [loaded, loadedItems] = await Promise.all([
|
||||
adminApi.getImportQueue(),
|
||||
jobId ? adminApi.getImportQueueItems(jobId).catch(() => null) : Promise.resolve(null),
|
||||
]);
|
||||
setQueue(loaded);
|
||||
if (jobId && expandedRef.current === jobId) setItems(loadedItems ? { jobId, items: loadedItems } : null);
|
||||
setError(null);
|
||||
} catch (err) {
|
||||
if (err instanceof UnauthorizedError) onUnauthorized?.();
|
||||
else setError(t("adminSettings.importQueue.loadFailed"));
|
||||
}
|
||||
}, [onUnauthorized, t]);
|
||||
|
||||
// Opening a job loads its links right away instead of on the next poll.
|
||||
React.useEffect(() => {
|
||||
expandedRef.current = expanded;
|
||||
if (expanded) void load();
|
||||
}, [expanded, load]);
|
||||
|
||||
const active = Boolean(
|
||||
queue && (queue.bulk.running || queue.jobs.some((job) => job.status === "RUNNING" || job.runner_live)),
|
||||
);
|
||||
|
||||
React.useEffect(() => {
|
||||
void load();
|
||||
const timer = window.setInterval(() => void load(), active ? POLL_ACTIVE_MS : POLL_IDLE_MS);
|
||||
return () => window.clearInterval(timer);
|
||||
}, [load, active]);
|
||||
|
||||
/** Runs one action with its button disabled, then reloads; a refusal shows the server's reason. */
|
||||
const run = async (key: string, action: () => Promise<unknown>, success?: (result: unknown) => string | null) => {
|
||||
setBusy(key);
|
||||
try {
|
||||
const result = await action();
|
||||
const message = success?.(result);
|
||||
if (message) showToast({ message, severity: "success" });
|
||||
} catch (err) {
|
||||
if (err instanceof UnauthorizedError) onUnauthorized?.();
|
||||
else showToast({ message: err instanceof Error ? err.message : String(err), severity: "error" });
|
||||
} finally {
|
||||
setBusy(null);
|
||||
await load();
|
||||
}
|
||||
};
|
||||
|
||||
const jobs = queue?.jobs ?? [];
|
||||
const visible = jobs.filter((job) => matchesFilter(job, filter));
|
||||
const pausedCount = jobs.filter((job) => job.status === "PAUSED" && job.type === "LINKS").length;
|
||||
const failedCount = jobs.filter(
|
||||
(job) => job.type === "LINKS" && job.status !== "RUNNING" && job.status !== "PAUSED" && job.failed_items_count > 0,
|
||||
).length;
|
||||
const runningCount = jobs.filter((job) => job.status === "RUNNING" && job.type === "LINKS").length;
|
||||
const finishedCount = jobs.filter((job) => job.status === "DONE" && job.failed_count === 0).length;
|
||||
|
||||
const deleteJob = async (job: AdminImportJob) => {
|
||||
const ok = await confirm({
|
||||
title: t("adminSettings.importQueue.deleteTitle"),
|
||||
message: t("adminSettings.importQueue.deleteMessage", { count: job.total }),
|
||||
confirmLabel: t("adminSettings.importQueue.delete"),
|
||||
destructive: true,
|
||||
});
|
||||
if (ok) await run(`delete:${job.id}`, () => adminApi.deleteQueueJob(job.id));
|
||||
};
|
||||
|
||||
return (
|
||||
<Stack spacing={2}>
|
||||
<Paper variant="outlined" sx={{ p: 2 }}>
|
||||
<Stack spacing={1.5}>
|
||||
<Stack direction="row" spacing={1} flexWrap="wrap" useFlexGap>
|
||||
<Button
|
||||
variant="contained"
|
||||
size="small"
|
||||
startIcon={<PlayArrowIcon />}
|
||||
disabled={!pausedCount || busy !== null}
|
||||
onClick={() =>
|
||||
void run("startAll", adminApi.startAllQueued, (r) =>
|
||||
t("adminSettings.importQueue.startedAll", { count: (r as { queued: number }).queued }),
|
||||
)
|
||||
}
|
||||
>
|
||||
{t("adminSettings.importQueue.startAll", { count: pausedCount })}
|
||||
</Button>
|
||||
<Button
|
||||
variant="outlined"
|
||||
size="small"
|
||||
startIcon={<ReplayIcon />}
|
||||
disabled={!failedCount || busy !== null}
|
||||
onClick={() =>
|
||||
void run("retryAll", adminApi.retryAllFailed, (r) =>
|
||||
t("adminSettings.importQueue.retriedAll", { count: (r as { queued: number }).queued }),
|
||||
)
|
||||
}
|
||||
>
|
||||
{t("adminSettings.importQueue.retryAll", { count: failedCount })}
|
||||
</Button>
|
||||
<Button
|
||||
variant="outlined"
|
||||
size="small"
|
||||
startIcon={<PauseIcon />}
|
||||
disabled={(!runningCount && !queue?.bulk.running) || busy !== null}
|
||||
onClick={() => void run("pauseAll", adminApi.pauseAllQueued)}
|
||||
>
|
||||
{t("adminSettings.importQueue.pauseAll")}
|
||||
</Button>
|
||||
<Button
|
||||
variant="text"
|
||||
size="small"
|
||||
startIcon={<DeleteSweepIcon />}
|
||||
disabled={!finishedCount || busy !== null}
|
||||
onClick={() =>
|
||||
void run("clear", adminApi.clearFinishedQueued, (r) =>
|
||||
t("adminSettings.importQueue.cleared", { count: (r as { deleted: number }).deleted }),
|
||||
)
|
||||
}
|
||||
>
|
||||
{t("adminSettings.importQueue.clearFinished")}
|
||||
</Button>
|
||||
</Stack>
|
||||
{queue?.bulk.running && (
|
||||
<Typography variant="body2" color="text.secondary">
|
||||
{t("adminSettings.importQueue.bulkRunning", { count: queue.bulk.remaining })}
|
||||
</Typography>
|
||||
)}
|
||||
<ToggleButtonGroup
|
||||
size="small"
|
||||
exclusive
|
||||
value={filter}
|
||||
onChange={(_e, value: Filter | null) => value && setFilter(value)}
|
||||
sx={{ flexWrap: "wrap" }}
|
||||
>
|
||||
{(["all", "paused", "running", "failed", "done"] as Filter[]).map((f) => (
|
||||
<ToggleButton key={f} value={f} sx={{ px: 1.5, py: 0.25, textTransform: "none" }}>
|
||||
{t(`adminSettings.importQueue.filter.${f}`)}
|
||||
</ToggleButton>
|
||||
))}
|
||||
</ToggleButtonGroup>
|
||||
</Stack>
|
||||
</Paper>
|
||||
|
||||
{error && <Alert severity="error">{error}</Alert>}
|
||||
|
||||
{!queue ? (
|
||||
<Stack alignItems="center" sx={{ py: 2 }}>
|
||||
<CircularProgress size={20} />
|
||||
</Stack>
|
||||
) : visible.length === 0 ? (
|
||||
<Paper variant="outlined" sx={{ p: 3 }}>
|
||||
<Typography variant="body2" color="text.secondary">
|
||||
{t(jobs.length ? "adminSettings.importQueue.emptyFilter" : "adminSettings.importQueue.empty")}
|
||||
</Typography>
|
||||
</Paper>
|
||||
) : (
|
||||
<Paper variant="outlined">
|
||||
<TableContainer>
|
||||
<Table size="small">
|
||||
<TableHead>
|
||||
<TableRow>
|
||||
<TableCell padding="checkbox" />
|
||||
<TableCell>{t("adminSettings.importQueue.columnSource")}</TableCell>
|
||||
<TableCell>{t("adminSettings.importQueue.columnUser")}</TableCell>
|
||||
<TableCell>{t("adminSettings.importQueue.columnStatus")}</TableCell>
|
||||
<TableCell sx={{ minWidth: 140 }}>{t("adminSettings.importQueue.columnProgress")}</TableCell>
|
||||
<TableCell>{t("adminSettings.importQueue.columnCreated")}</TableCell>
|
||||
<TableCell align="right">{t("adminSettings.importQueue.columnActions")}</TableCell>
|
||||
</TableRow>
|
||||
</TableHead>
|
||||
<TableBody>
|
||||
{visible.map((job) => (
|
||||
<JobRow
|
||||
key={job.id}
|
||||
job={job}
|
||||
expanded={expanded === job.id}
|
||||
onToggle={() => setExpanded((id) => (id === job.id ? null : job.id))}
|
||||
busy={busy}
|
||||
items={items?.jobId === job.id ? items.items : null}
|
||||
locale={i18n.language}
|
||||
onStart={() => void run(`start:${job.id}`, () => adminApi.startQueueJob(job.id))}
|
||||
onPause={() => void run(`pause:${job.id}`, () => adminApi.pauseQueueJob(job.id))}
|
||||
onRetry={() => void run(`retry:${job.id}`, () => adminApi.retryQueueJob(job.id))}
|
||||
onDelete={() => void deleteJob(job)}
|
||||
onRemoveItem={(item) => void run(`item:${item.id}`, () => adminApi.removeQueueItem(item.id))}
|
||||
/>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</TableContainer>
|
||||
</Paper>
|
||||
)}
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
|
||||
type JobRowProps = {
|
||||
job: AdminImportJob;
|
||||
expanded: boolean;
|
||||
onToggle: () => void;
|
||||
busy: string | null;
|
||||
items: ImportJobItem[] | null;
|
||||
locale: string;
|
||||
onStart: () => void;
|
||||
onPause: () => void;
|
||||
onRetry: () => void;
|
||||
onDelete: () => void;
|
||||
onRemoveItem: (item: ImportJobItem) => void;
|
||||
};
|
||||
|
||||
function JobRow({
|
||||
job,
|
||||
expanded,
|
||||
onToggle,
|
||||
busy,
|
||||
items,
|
||||
locale,
|
||||
onStart,
|
||||
onPause,
|
||||
onRetry,
|
||||
onDelete,
|
||||
onRemoveItem,
|
||||
}: JobRowProps) {
|
||||
const { t } = useTranslation("app");
|
||||
const isLinks = job.type === "LINKS";
|
||||
const status = jobStatus(job);
|
||||
const idle = job.status !== "RUNNING" && !job.runner_live;
|
||||
const canStart = isLinks && job.status === "PAUSED" && !job.runner_live && job.pending_count + job.running_count > 0;
|
||||
const canPause = isLinks && job.status === "RUNNING";
|
||||
const canRetry = isLinks && idle && job.failed_items_count > 0;
|
||||
const progress = job.total > 0 ? Math.min(100, (job.processed / job.total) * 100) : 0;
|
||||
const disabled = busy !== null;
|
||||
|
||||
return (
|
||||
<>
|
||||
<TableRow hover sx={{ "& > td": { borderBottom: expanded ? "none" : undefined } }}>
|
||||
<TableCell padding="checkbox">
|
||||
{isLinks && (
|
||||
<IconButton
|
||||
size="small"
|
||||
onClick={onToggle}
|
||||
aria-expanded={expanded}
|
||||
aria-label={t(expanded ? "adminSettings.importQueue.hideLinks" : "adminSettings.importQueue.showLinks")}
|
||||
>
|
||||
{expanded ? <KeyboardArrowDownIcon fontSize="small" /> : <KeyboardArrowRightIcon fontSize="small" />}
|
||||
</IconButton>
|
||||
)}
|
||||
</TableCell>
|
||||
<TableCell sx={{ maxWidth: 320 }}>
|
||||
<Typography variant="body2" noWrap title={job.source_label ?? job.source_url}>
|
||||
{job.source_label ?? job.source_url}
|
||||
</Typography>
|
||||
<Typography variant="caption" color="text.secondary">
|
||||
{t(`adminSettings.importQueue.type.${job.type}`)}
|
||||
{job.provider ? ` · ${job.provider}` : ""}
|
||||
</Typography>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<Typography variant="body2">{job.owner.display_name}</Typography>
|
||||
<Typography variant="caption" color="text.secondary">
|
||||
{job.owner.email}
|
||||
</Typography>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<Tooltip title={job.error_message ?? t(`adminSettings.importQueue.statusHint.${status.key}`)}>
|
||||
<Chip size="small" color={status.color} label={t(`adminSettings.importQueue.status.${status.key}`)} />
|
||||
</Tooltip>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<Typography variant="caption" component="div">
|
||||
{t("adminSettings.importQueue.progress", {
|
||||
processed: job.processed,
|
||||
total: job.total,
|
||||
imported: job.imported,
|
||||
failed: job.failed_count,
|
||||
})}
|
||||
</Typography>
|
||||
<LinearProgress
|
||||
variant="determinate"
|
||||
value={progress}
|
||||
color={job.failed_count > 0 ? "warning" : "primary"}
|
||||
sx={{ borderRadius: 1, height: 4, mt: 0.5 }}
|
||||
/>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<Typography variant="body2" noWrap>
|
||||
{new Date(job.created_at).toLocaleString(locale)}
|
||||
</Typography>
|
||||
</TableCell>
|
||||
<TableCell align="right" sx={{ whiteSpace: "nowrap" }}>
|
||||
{canStart && (
|
||||
<Tooltip title={t("adminSettings.importQueue.start")}>
|
||||
<span>
|
||||
<IconButton
|
||||
size="small"
|
||||
color="primary"
|
||||
onClick={onStart}
|
||||
disabled={disabled}
|
||||
aria-label={t("adminSettings.importQueue.start")}
|
||||
>
|
||||
<PlayArrowIcon fontSize="small" />
|
||||
</IconButton>
|
||||
</span>
|
||||
</Tooltip>
|
||||
)}
|
||||
{canPause && (
|
||||
<Tooltip title={t("adminSettings.importQueue.pause")}>
|
||||
<span>
|
||||
<IconButton
|
||||
size="small"
|
||||
onClick={onPause}
|
||||
disabled={disabled}
|
||||
aria-label={t("adminSettings.importQueue.pause")}
|
||||
>
|
||||
<PauseIcon fontSize="small" />
|
||||
</IconButton>
|
||||
</span>
|
||||
</Tooltip>
|
||||
)}
|
||||
{canRetry && (
|
||||
<Tooltip title={t("adminSettings.importQueue.retry", { count: job.failed_items_count })}>
|
||||
<span>
|
||||
<IconButton
|
||||
size="small"
|
||||
onClick={onRetry}
|
||||
disabled={disabled}
|
||||
aria-label={t("adminSettings.importQueue.retry", { count: job.failed_items_count })}
|
||||
>
|
||||
<ReplayIcon fontSize="small" />
|
||||
</IconButton>
|
||||
</span>
|
||||
</Tooltip>
|
||||
)}
|
||||
{idle && (
|
||||
<Tooltip title={t("adminSettings.importQueue.delete")}>
|
||||
<span>
|
||||
<IconButton
|
||||
size="small"
|
||||
onClick={onDelete}
|
||||
disabled={disabled}
|
||||
aria-label={t("adminSettings.importQueue.delete")}
|
||||
>
|
||||
<DeleteOutlineIcon fontSize="small" />
|
||||
</IconButton>
|
||||
</span>
|
||||
</Tooltip>
|
||||
)}
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
{isLinks && (
|
||||
<TableRow>
|
||||
<TableCell colSpan={7} sx={{ py: 0, borderBottom: expanded ? undefined : "none" }}>
|
||||
<Collapse in={expanded} timeout="auto" unmountOnExit>
|
||||
<JobItems items={items} canRemove={idle} disabled={disabled} onRemove={onRemoveItem} />
|
||||
</Collapse>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function JobItems({
|
||||
items,
|
||||
canRemove,
|
||||
disabled,
|
||||
onRemove,
|
||||
}: {
|
||||
items: ImportJobItem[] | null;
|
||||
canRemove: boolean;
|
||||
disabled: boolean;
|
||||
onRemove: (item: ImportJobItem) => void;
|
||||
}) {
|
||||
const { t } = useTranslation("app");
|
||||
if (!items) {
|
||||
return (
|
||||
<Box sx={{ py: 1.5 }}>
|
||||
<CircularProgress size={16} />
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<Stack spacing={1} sx={{ py: 1.5, pl: { xs: 0, sm: 5 } }}>
|
||||
{items.map((item) => (
|
||||
<Stack key={item.id} direction="row" spacing={1.5} alignItems="flex-start">
|
||||
<Chip
|
||||
size="small"
|
||||
color={ITEM_COLOR[item.status]}
|
||||
variant={item.status === "PENDING" ? "outlined" : "filled"}
|
||||
label={t(`adminSettings.importQueue.itemStatus.${item.status}`)}
|
||||
sx={{ minWidth: 84 }}
|
||||
/>
|
||||
<Box sx={{ minWidth: 0, flex: 1 }}>
|
||||
{item.title && (
|
||||
<Typography variant="body2" sx={{ wordBreak: "break-word" }}>
|
||||
{item.title}
|
||||
</Typography>
|
||||
)}
|
||||
<Link
|
||||
href={item.url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
variant={item.title ? "caption" : "body2"}
|
||||
sx={{ wordBreak: "break-all" }}
|
||||
>
|
||||
{item.url}
|
||||
</Link>
|
||||
{item.scope && item.scope !== "url" && (
|
||||
<Typography variant="caption" color="text.secondary" component="div">
|
||||
{t(`adminSettings.importQueue.scope.${item.scope}`)}
|
||||
</Typography>
|
||||
)}
|
||||
{item.error_message && (
|
||||
<Typography variant="caption" color="error" component="div" sx={{ wordBreak: "break-word" }}>
|
||||
{item.error_message}
|
||||
</Typography>
|
||||
)}
|
||||
</Box>
|
||||
{canRemove && item.status !== "DONE" && (
|
||||
<Tooltip title={t("adminSettings.importQueue.removeLink")}>
|
||||
<span>
|
||||
<IconButton
|
||||
size="small"
|
||||
onClick={() => onRemove(item)}
|
||||
disabled={disabled}
|
||||
aria-label={t("adminSettings.importQueue.removeLink")}
|
||||
>
|
||||
<CloseIcon fontSize="small" />
|
||||
</IconButton>
|
||||
</span>
|
||||
</Tooltip>
|
||||
)}
|
||||
</Stack>
|
||||
))}
|
||||
{!items.length && (
|
||||
<Typography variant="body2" color="text.secondary">
|
||||
{t("importProgress.queueEmpty")}
|
||||
</Typography>
|
||||
)}
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
@@ -38,6 +38,7 @@ const ACTION_COLORS: Record<LogAction, ActionColor> = {
|
||||
model_imported: "info",
|
||||
import_completed: "info",
|
||||
model_edited: "warning",
|
||||
model_reimported: "info",
|
||||
model_deleted: "error",
|
||||
collection_created: "success",
|
||||
collection_edited: "warning",
|
||||
@@ -114,6 +115,8 @@ export default function LogsPage({ onUnauthorized }: Props) {
|
||||
return t("adminSettings.logs.authorsLinked", { count: typeof d.linked === "number" ? d.linked : 0 });
|
||||
case "model_edited":
|
||||
return typeof d.field === "string" ? t("adminSettings.logs.editedField", { field: d.field }) : "";
|
||||
case "model_reimported":
|
||||
return typeof d.source_url === "string" ? d.source_url : "";
|
||||
case "import_completed": {
|
||||
const provider = typeof d.provider === "string" ? d.provider : "";
|
||||
const label = typeof d.sourceLabel === "string" && d.sourceLabel ? d.sourceLabel : provider;
|
||||
|
||||
@@ -16,6 +16,7 @@ import DeleteIcon from "@mui/icons-material/Delete";
|
||||
import PlaylistAddIcon from "@mui/icons-material/PlaylistAdd";
|
||||
import PlaylistRemoveIcon from "@mui/icons-material/PlaylistRemove";
|
||||
import OpenInNewIcon from "@mui/icons-material/OpenInNew";
|
||||
import SyncIcon from "@mui/icons-material/Sync";
|
||||
import LaunchIcon from "@mui/icons-material/Launch";
|
||||
import type { SxProps, Theme } from "@mui/material/styles";
|
||||
import { UnauthorizedError } from "../../api/client";
|
||||
@@ -32,6 +33,7 @@ import { useNormalizedOpen } from "./useNormalizedOpen";
|
||||
import DownloadPickerDialog from "./DownloadPickerDialog";
|
||||
import AddToCollectionModal from "./AddToCollectionModal";
|
||||
import EditModelModal from "./EditModelModal";
|
||||
import ReimportDialog from "./ReimportDialog";
|
||||
|
||||
type Props = {
|
||||
print: Print;
|
||||
@@ -69,6 +71,7 @@ export default function ModelActionsMenu({
|
||||
const [deleting, setDeleting] = useState(false);
|
||||
const [removingFromCollection, setRemovingFromCollection] = useState(false);
|
||||
const [addToCollectionOpen, setAddToCollectionOpen] = useState(false);
|
||||
const [reimportOpen, setReimportOpen] = useState(false);
|
||||
// Driven by ?edit=<id> so links and the back button open/close it.
|
||||
const editOpen = searchParams.get("edit") === print.id;
|
||||
const {
|
||||
@@ -267,6 +270,19 @@ export default function ModelActionsMenu({
|
||||
<ListItemText>{normalizedMenuLabel(slicerOption.label)}</ListItemText>
|
||||
</MenuItem>
|
||||
)}
|
||||
{providerInfo && print.source_url && (
|
||||
<MenuItem
|
||||
onClick={() => {
|
||||
closeMenu();
|
||||
setReimportOpen(true);
|
||||
}}
|
||||
>
|
||||
<ListItemIcon>
|
||||
<SyncIcon fontSize="small" />
|
||||
</ListItemIcon>
|
||||
<ListItemText>{t("models:detail.reimport")}</ListItemText>
|
||||
</MenuItem>
|
||||
)}
|
||||
{providerInfo && print.source_url && (
|
||||
<MenuItem component="a" href={print.source_url} target="_blank" rel="noopener noreferrer" onClick={closeMenu}>
|
||||
<ListItemIcon>
|
||||
@@ -325,6 +341,16 @@ export default function ModelActionsMenu({
|
||||
viewer={viewer}
|
||||
/>
|
||||
)}
|
||||
|
||||
{reimportOpen && (
|
||||
<ReimportDialog
|
||||
open
|
||||
print={print}
|
||||
onClose={() => setReimportOpen(false)}
|
||||
onUpdated={(updated) => onUpdated?.(updated)}
|
||||
onUnauthorized={onUnauthorized}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
import { useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import Dialog from "@mui/material/Dialog";
|
||||
import DialogTitle from "@mui/material/DialogTitle";
|
||||
import DialogContent from "@mui/material/DialogContent";
|
||||
import DialogActions from "@mui/material/DialogActions";
|
||||
import Button from "@mui/material/Button";
|
||||
import FormControlLabel from "@mui/material/FormControlLabel";
|
||||
import Checkbox from "@mui/material/Checkbox";
|
||||
import CircularProgress from "@mui/material/CircularProgress";
|
||||
import Stack from "@mui/material/Stack";
|
||||
import Typography from "@mui/material/Typography";
|
||||
import { UnauthorizedError } from "../../api/client";
|
||||
import { printsApi, type Print, type ReimportResult } from "../../api/prints";
|
||||
import { importProviderInfo } from "../../constants/importProviders";
|
||||
import { useToast } from "../../components/ToastProvider";
|
||||
|
||||
type Props = {
|
||||
open: boolean;
|
||||
print: Print;
|
||||
onClose: () => void;
|
||||
onUpdated: (print: Print) => void;
|
||||
onUnauthorized?: () => void;
|
||||
};
|
||||
|
||||
/** "Update from source": pulls what the source offers now. Only empty fields are filled, so a
|
||||
* title, note or creator the user typed always wins, and tags are only ever added. */
|
||||
export default function ReimportDialog({ open, print, onClose, onUpdated, onUnauthorized }: Props) {
|
||||
const { t } = useTranslation(["models", "common"]);
|
||||
const showToast = useToast();
|
||||
const [metadata, setMetadata] = useState(true);
|
||||
const [files, setFiles] = useState(true);
|
||||
const [images, setImages] = useState(true);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const providerLabel = importProviderInfo(print.source_provider)?.label ?? t("models:detail.reimportSource");
|
||||
|
||||
const summaryOf = (result: ReimportResult): string => {
|
||||
const filesAdded = result.files_added;
|
||||
const imagesAdded = result.images_added;
|
||||
const tagsAdded = result.tags_added.length;
|
||||
if (
|
||||
!filesAdded &&
|
||||
!imagesAdded &&
|
||||
!tagsAdded &&
|
||||
!result.title_filled &&
|
||||
!result.notes_filled &&
|
||||
!result.creator_filled &&
|
||||
!result.author_linked &&
|
||||
!result.category_filled
|
||||
) {
|
||||
return t("models:detail.reimportNothing");
|
||||
}
|
||||
return t("models:detail.reimportDone", { filesAdded, imagesAdded, tagsAdded });
|
||||
};
|
||||
|
||||
const submit = async () => {
|
||||
setBusy(true);
|
||||
try {
|
||||
const result = await printsApi.reimport(print.id, { metadata, files, images });
|
||||
const updated = await printsApi.get(print.id);
|
||||
onUpdated(updated);
|
||||
showToast({ message: summaryOf(result) });
|
||||
onClose();
|
||||
} catch (err) {
|
||||
if (err instanceof UnauthorizedError) {
|
||||
onUnauthorized?.();
|
||||
return;
|
||||
}
|
||||
showToast({ message: err instanceof Error ? err.message : t("models:detail.reimportFailed") });
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Dialog open={open} onClose={onClose} fullWidth maxWidth="xs">
|
||||
<DialogTitle>{t("models:detail.reimportTitle")}</DialogTitle>
|
||||
<DialogContent>
|
||||
<Typography variant="body2" color="text.secondary" sx={{ mb: 1.5 }}>
|
||||
{t("models:detail.reimportHint", { provider: providerLabel })}
|
||||
</Typography>
|
||||
<Stack>
|
||||
<FormControlLabel
|
||||
control={<Checkbox checked={metadata} onChange={(e) => setMetadata(e.target.checked)} disabled={busy} />}
|
||||
label={t("models:detail.reimportMetadata")}
|
||||
/>
|
||||
<FormControlLabel
|
||||
control={<Checkbox checked={files} onChange={(e) => setFiles(e.target.checked)} disabled={busy} />}
|
||||
label={t("models:detail.reimportFiles")}
|
||||
/>
|
||||
<FormControlLabel
|
||||
control={<Checkbox checked={images} onChange={(e) => setImages(e.target.checked)} disabled={busy} />}
|
||||
label={t("models:detail.reimportImages")}
|
||||
/>
|
||||
</Stack>
|
||||
</DialogContent>
|
||||
<DialogActions>
|
||||
<Button onClick={onClose} disabled={busy}>
|
||||
{t("common:cancel")}
|
||||
</Button>
|
||||
<Button
|
||||
variant="contained"
|
||||
onClick={() => void submit()}
|
||||
disabled={busy || (!metadata && !files && !images)}
|
||||
startIcon={busy ? <CircularProgress size={14} /> : undefined}
|
||||
>
|
||||
{t("models:detail.reimportSubmit")}
|
||||
</Button>
|
||||
</DialogActions>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user