mirror of
https://github.com/TautvydasDerzinskas/Thingport.git
synced 2026-10-18 20:59:55 -05:00
fix: count queued links as models and let imported images replace 3D renders
- A links import with a profile scope counted print profiles against the
number of links ("Imported 11 of 3 links"). Each link now counts as one
model, and the profile totals go in the notification body.
- Opening a model while its import was still downloading images rendered a
3D thumbnail that then stuck as the card image and first gallery image.
Renders are now flagged, imported images replace them, and a late render
no longer overwrites an imported cover. "Update from source" treats a
model with only renders as having no images.
This commit is contained in:
1 parent
96da36ae79
commit
dd4d2592ea
10 files changed
+164
-27
No files matched your search
@@ -0,0 +1,4 @@
|
||||
-- Marks our own 3D renders, so an import's images replace one that got in first.
|
||||
ALTER TABLE "Plate" ADD COLUMN "thumbGenerated" BOOLEAN NOT NULL DEFAULT false;
|
||||
|
||||
ALTER TABLE "PreviewImage" ADD COLUMN "generated" BOOLEAN NOT NULL DEFAULT false;
|
||||
@@ -336,6 +336,8 @@ model Plate {
|
||||
sourceInstanceId String?
|
||||
// Computed lazily; plate files are never rewritten in place.
|
||||
contentSha256 String?
|
||||
// The thumb is our own 3D render, which an imported image replaces.
|
||||
thumbGenerated Boolean @default(false)
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
@@ -351,6 +353,8 @@ model PreviewImage {
|
||||
printId String
|
||||
print Print @relation(fields: [printId], references: [id], onDelete: Cascade)
|
||||
position Int
|
||||
// Our own 3D render, which an imported image replaces.
|
||||
generated Boolean @default(false)
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
@@unique([printId, position])
|
||||
|
||||
@@ -15,7 +15,13 @@ import {
|
||||
resolvePlateFilePath,
|
||||
type NewPlateInput,
|
||||
} from "../services/printCreation";
|
||||
import { availablePlateFilename, plateThumbPath, relocatePrint, saveThumbFromBytes } from "../services/printService";
|
||||
import {
|
||||
availablePlateFilename,
|
||||
plateThumbPath,
|
||||
plateThumbReplaceable,
|
||||
relocatePrint,
|
||||
saveThumbFromBytes,
|
||||
} from "../services/printService";
|
||||
import { addGeneratedPreviewImageIfNone } from "../services/previewImageService";
|
||||
import { printOutById } from "../services/printLoader";
|
||||
import { generateModelPreviewGlb, modelPreviewGlbPath, modelPreviewState } from "../services/modelPreviewCache";
|
||||
@@ -193,10 +199,14 @@ router.post(
|
||||
where: { id: req.params.plateId, print: { userId: req.userId } },
|
||||
});
|
||||
if (!plate) throw new HttpError(404, "Not found");
|
||||
const ok = await saveThumbFromBytes(plate.id, file.buffer);
|
||||
if (!ok) throw new HttpError(400, "Invalid thumbnail image");
|
||||
// Only when no better preview (e.g. an imported cover) exists.
|
||||
await addGeneratedPreviewImageIfNone(plate.printId, file.buffer);
|
||||
// A render that finishes after an import set the cover as the thumb mustn't replace it.
|
||||
if (await plateThumbReplaceable(plate.id)) {
|
||||
const ok = await saveThumbFromBytes(plate.id, file.buffer);
|
||||
if (!ok) throw new HttpError(400, "Invalid thumbnail image");
|
||||
await prisma.plate.update({ where: { id: plate.id }, data: { thumbGenerated: true } });
|
||||
// Only when no better preview (e.g. an imported cover) exists.
|
||||
await addGeneratedPreviewImageIfNone(plate.printId, file.buffer);
|
||||
}
|
||||
res.json({ print: await printOutById(req.userId!, plate.printId) });
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -632,8 +632,13 @@ function failureMessage(err: unknown): string {
|
||||
}
|
||||
|
||||
/** One queued link. A MakerWorld model with a multi-profile scope imports each wanted profile onto
|
||||
* one print, like the PROFILES job; anything else is a plain single import. */
|
||||
async function importOneLink(userId: string, url: string, body: LinksImportJobBody): Promise<ProfileImportTotals> {
|
||||
* one print, like the PROFILES job; anything else is a plain single import. `profiles` marks the
|
||||
* former, whose totals count print profiles rather than models. */
|
||||
async function importOneLink(
|
||||
userId: string,
|
||||
url: string,
|
||||
body: LinksImportJobBody,
|
||||
): Promise<ProfileImportTotals & { profiles: boolean }> {
|
||||
const parsed = parseMakerworldModelUrl(url);
|
||||
if (parsed && body.scope && body.scope !== "url") {
|
||||
const design = await fetchMakerworldDesignForImport(
|
||||
@@ -644,11 +649,12 @@ async function importOneLink(userId: string, url: string, body: LinksImportJobBo
|
||||
if (!design) throw new HttpError(400, "Couldn't read this model's print profiles from MakerWorld");
|
||||
const profileIds = selectMakerworldProfiles(design, body.scope, parsed.requestedInstanceId);
|
||||
if (!profileIds.length) throw new HttpError(400, "This model has no print profiles to import");
|
||||
return importProfileUrlsSequentially(
|
||||
const totals = await importProfileUrlsSequentially(
|
||||
userId,
|
||||
profileIds.map((profileId) => `https://makerworld.com/en/models/${parsed.designId}#profileId-${profileId}`),
|
||||
{ ...body, url },
|
||||
);
|
||||
return { ...totals, profiles: true };
|
||||
}
|
||||
const result = await importPrintFromUrl(userId, url, { ...body, url });
|
||||
return {
|
||||
@@ -658,6 +664,7 @@ async function importOneLink(userId: string, url: string, body: LinksImportJobBo
|
||||
failed: 0,
|
||||
printId: result.print.id,
|
||||
stopReason: null,
|
||||
profiles: false,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -687,6 +694,9 @@ async function runLinks(jobId: string, userId: string, body: LinksImportJobBody)
|
||||
|
||||
let imported = 0;
|
||||
let alreadyInLibrary = 0;
|
||||
// This run's print profiles, from links with a profile scope; the counts above are models.
|
||||
let profilesImported = 0;
|
||||
let profilesProcessed = 0;
|
||||
let printId: string | null = null;
|
||||
let stopReason: "rateLimited" | "auth" | null = null;
|
||||
let processed = (await countJobItems(jobId, "DONE")) + (await countJobItems(jobId, "FAILED"));
|
||||
@@ -715,8 +725,13 @@ async function runLinks(jobId: string, userId: string, body: LinksImportJobBody)
|
||||
}
|
||||
try {
|
||||
const result = await importOneLink(userId, item.url, itemBody);
|
||||
imported += result.imported;
|
||||
alreadyInLibrary += result.alreadyInLibrary;
|
||||
// A link is one model however many of its profiles came in.
|
||||
if (result.imported) imported++;
|
||||
else if (result.alreadyInLibrary) alreadyInLibrary++;
|
||||
if (result.profiles) {
|
||||
profilesImported += result.imported;
|
||||
profilesProcessed += result.processed;
|
||||
}
|
||||
if (result.printId) printId = result.printId;
|
||||
if (result.printId && options.collection_id) {
|
||||
await fileIntoCollection(userId, options.collection_id, result.printId);
|
||||
@@ -788,6 +803,7 @@ async function runLinks(jobId: string, userId: string, body: LinksImportJobBody)
|
||||
});
|
||||
|
||||
const bodyParts: string[] = [];
|
||||
if (profilesProcessed) bodyParts.push(`${profilesImported} of ${profilesProcessed} print profiles imported`);
|
||||
if (alreadyInLibrary) bodyParts.push(`${alreadyInLibrary} already in your library`);
|
||||
if (stopReason === "rateLimited") {
|
||||
bodyParts.push(
|
||||
@@ -801,7 +817,7 @@ async function runLinks(jobId: string, userId: string, body: LinksImportJobBody)
|
||||
bodyParts.push(`${failedCount} failed — retry them from the import's progress bar`);
|
||||
}
|
||||
await createNotification(userId, {
|
||||
title: `Imported ${totalImported} of ${total} links`,
|
||||
title: `Imported ${totalImported} of ${total} models`,
|
||||
body: bodyParts.length ? `${bodyParts.join(", ")}.` : null,
|
||||
externalUrl: job.sourceUrl,
|
||||
internalPath:
|
||||
|
||||
@@ -67,8 +67,8 @@ import {
|
||||
type NewPlateInput,
|
||||
type PrintMetaInput,
|
||||
} from "./printCreation";
|
||||
import { plateThumbExists, saveThumbFromBytes } from "./printService";
|
||||
import { addPreviewImage } from "./previewImageService";
|
||||
import { plateThumbReplaceable, saveThumbFromBytes } from "./printService";
|
||||
import { addPreviewImage, removeGeneratedPreviewImages } from "./previewImageService";
|
||||
import { prisma } from "../db";
|
||||
import { Prisma } from "@prisma/client";
|
||||
import type { Author, Plate, PreviewImage, Print } from "@prisma/client";
|
||||
@@ -496,7 +496,8 @@ async function fetchImageBytes(url: string): Promise<Buffer | null> {
|
||||
}
|
||||
|
||||
/** Best-effort: stores the page's cover (at position 0) and gallery as preview images, and seeds
|
||||
* the plate thumbnail from the cover when nothing better exists. Never throws. */
|
||||
* the plate thumbnail from the cover when nothing better exists. A 3D render the app made while
|
||||
* the import ran (the model is browsable from creation) gives way to them. Never throws. */
|
||||
export async function attachImportedPreviewImages(
|
||||
printId: string | undefined,
|
||||
plateId: string | undefined,
|
||||
@@ -517,18 +518,18 @@ export async function attachImportedPreviewImages(
|
||||
orderedUrls.push(image.url);
|
||||
}
|
||||
|
||||
let platesThumbSeeded = false;
|
||||
let firstAdded = false;
|
||||
const urls = orderedUrls.slice(0, PREVIEW_IMAGE_MAX_COUNT);
|
||||
const delayMs = paceMs ?? IMPORT_PREVIEW_IMAGE_DELAY_MS;
|
||||
for (let i = 0; i < urls.length; i++) {
|
||||
await sleep(delayMs);
|
||||
const buf = await fetchImageBytes(urls[i]);
|
||||
if (buf) {
|
||||
await addPreviewImage(printId, buf);
|
||||
if (!platesThumbSeeded && plateId && !plateThumbExists(plateId)) {
|
||||
await saveThumbFromBytes(plateId, buf);
|
||||
platesThumbSeeded = true;
|
||||
}
|
||||
if (!buf) continue;
|
||||
const added = await addPreviewImage(printId, buf);
|
||||
if (added && !firstAdded) {
|
||||
firstAdded = true;
|
||||
await removeGeneratedPreviewImages(printId);
|
||||
if (plateId && (await plateThumbReplaceable(plateId))) await saveThumbFromBytes(plateId, buf);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,10 +33,14 @@ async function saveBuffer(id: string, input: Buffer): Promise<boolean> {
|
||||
}
|
||||
|
||||
/** Null if the buffer isn't a decodable image; never throws. */
|
||||
export async function addPreviewImage(printId: string, buffer: Buffer): Promise<PreviewImage | null> {
|
||||
export async function addPreviewImage(
|
||||
printId: string,
|
||||
buffer: Buffer,
|
||||
generated = false,
|
||||
): Promise<PreviewImage | null> {
|
||||
const last = await prisma.previewImage.findFirst({ where: { printId }, orderBy: { position: "desc" } });
|
||||
const position = last ? last.position + 1 : 0;
|
||||
const row = await prisma.previewImage.create({ data: { printId, position } });
|
||||
const row = await prisma.previewImage.create({ data: { printId, position, generated } });
|
||||
const ok = await saveBuffer(row.id, buffer);
|
||||
if (!ok) {
|
||||
await prisma.previewImage.delete({ where: { id: row.id } }).catch(() => undefined);
|
||||
@@ -49,7 +53,23 @@ export async function addPreviewImage(printId: string, buffer: Buffer): Promise<
|
||||
export async function addGeneratedPreviewImageIfNone(printId: string, buffer: Buffer): Promise<void> {
|
||||
const count = await prisma.previewImage.count({ where: { printId } });
|
||||
if (count > 0) return;
|
||||
await addPreviewImage(printId, buffer);
|
||||
await addPreviewImage(printId, buffer, true);
|
||||
}
|
||||
|
||||
/** Drops our own renders, so the images that replace them start at position 0. */
|
||||
export async function removeGeneratedPreviewImages(printId: string): Promise<void> {
|
||||
const generated = await prisma.previewImage.findMany({ where: { printId, generated: true } });
|
||||
if (!generated.length) return;
|
||||
await prisma.previewImage.deleteMany({ where: { id: { in: generated.map((img) => img.id) } } });
|
||||
const remaining = await prisma.previewImage.findMany({ where: { printId }, orderBy: { position: "asc" } });
|
||||
// Two-phase renumber, as in plates.ts, to avoid (printId, position) collisions.
|
||||
await prisma.$transaction(
|
||||
remaining.map((img, idx) => prisma.previewImage.update({ where: { id: img.id }, data: { position: -(idx + 1) } })),
|
||||
);
|
||||
await prisma.$transaction(
|
||||
remaining.map((img, idx) => prisma.previewImage.update({ where: { id: img.id }, data: { position: idx } })),
|
||||
);
|
||||
for (const img of generated) await fs.rm(previewImagePath(img.id), { force: true }).catch(() => undefined);
|
||||
}
|
||||
|
||||
export async function deleteAllPreviewImages(printId: string): Promise<void> {
|
||||
|
||||
@@ -416,6 +416,8 @@ async function saveThumbBuffer(plateId: string, input: Buffer): Promise<boolean>
|
||||
.jpeg({ quality: 88, mozjpeg: true })
|
||||
.toFile(tmp);
|
||||
await fs.rename(tmp, dest);
|
||||
// Any thumb but our own render; the generated-thumbnail route marks those after saving.
|
||||
await prisma.plate.updateMany({ where: { id: plateId, thumbGenerated: true }, data: { thumbGenerated: false } });
|
||||
return true;
|
||||
} catch {
|
||||
await fs.rm(tmp, { force: true });
|
||||
@@ -479,3 +481,10 @@ export function plateThumbPath(plateId: string): string {
|
||||
export function plateThumbExists(plateId: string): boolean {
|
||||
return fsSync.existsSync(plateThumbPath(plateId));
|
||||
}
|
||||
|
||||
/** True when the plate has no thumb or only our own 3D render, which an imported image beats. */
|
||||
export async function plateThumbReplaceable(plateId: string): Promise<boolean> {
|
||||
if (!plateThumbExists(plateId)) return true;
|
||||
const plate = await prisma.plate.findUnique({ where: { id: plateId }, select: { thumbGenerated: true } });
|
||||
return plate?.thumbGenerated ?? false;
|
||||
}
|
||||
@@ -279,7 +279,8 @@ export async function reimportPrint(userId: string, printId: string, opts: Reimp
|
||||
if (opts.metadata) metadata = await fillEmptyMetadata(userId, print, snapshot.meta);
|
||||
|
||||
let imagesAdded = 0;
|
||||
if (opts.images && print.previewImages.length === 0) {
|
||||
// Only our own 3D renders count as an empty slot; the import replaces them.
|
||||
if (opts.images && print.previewImages.every((image) => image.generated)) {
|
||||
await attachImportedPreviewImages(
|
||||
print.id,
|
||||
print.plates[0]?.id,
|
||||
|
||||
@@ -231,12 +231,19 @@ describe("the links import queue", () => {
|
||||
const start = await startLinks([`https://makerworld.com/en/models/${id}-phoenix`], { scope: "all" });
|
||||
const job = await waitJob(start.body.job_id);
|
||||
expect(job.status).toBe("DONE");
|
||||
expect(job.imported).toBe(2);
|
||||
// One link is one model, however many profiles it brought in.
|
||||
expect(job.imported).toBe(1);
|
||||
|
||||
const items = await prisma.importJobItem.findMany({ where: { jobId: job.id } });
|
||||
expect(items[0].status).toBe("DONE");
|
||||
|
||||
const user = await prisma.user.findFirst({ where: { email: `links-${stamp}@example.com` } });
|
||||
const notification = await prisma.notification.findFirst({
|
||||
where: { userId: user!.id },
|
||||
orderBy: { createdAt: "desc" },
|
||||
});
|
||||
expect(notification!.title).toBe("Imported 1 of 1 models");
|
||||
expect(notification!.body).toBe("2 of 2 print profiles imported.");
|
||||
const print = await prisma.print.findFirst({
|
||||
where: { userId: user!.id, sourceExternalId: id },
|
||||
include: { plates: true },
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
import { beforeAll, describe, expect, it } from "vitest";
|
||||
import { afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import request from "supertest";
|
||||
import sharp from "sharp";
|
||||
import { createApp } from "../src/app";
|
||||
import { prisma } from "../src/db";
|
||||
import { attachImportedPreviewImages } from "../src/services/importService";
|
||||
import { plateThumbPath } from "../src/services/printService";
|
||||
import { previewImagePath } from "../src/services/previewImageService";
|
||||
|
||||
const app = createApp();
|
||||
let token: string;
|
||||
@@ -16,6 +20,11 @@ function png(r: number) {
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
// The stored JPEGs are re-encoded, so an image is recognised by its red channel.
|
||||
async function redOf(filePath: string) {
|
||||
return Math.round((await sharp(filePath).stats()).channels[0].mean);
|
||||
}
|
||||
|
||||
async function uploadStl() {
|
||||
const res = await request(app).post("/api/upload").set(auth()).attach("files", Buffer.from(STL), "part.stl");
|
||||
expect(res.status).toBe(200);
|
||||
@@ -44,3 +53,59 @@ describe("preview images", () => {
|
||||
expect(res.body.print.thumb_url).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("a 3D render made while an import runs", () => {
|
||||
const originalFetch = global.fetch;
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
});
|
||||
|
||||
it("gives way to the imported cover, as thumbnail and first preview image", async () => {
|
||||
const print = await uploadStl();
|
||||
const plate = (await prisma.plate.findFirst({ where: { printId: print.id } }))!;
|
||||
|
||||
// The model is browsable before its images download, so the app renders and saves one.
|
||||
const render = await request(app)
|
||||
.post(`/api/plate/${plate.id}/thumbnail-generated`)
|
||||
.set(auth())
|
||||
.attach("file", await png(10), "render.png");
|
||||
expect(render.status).toBe(200);
|
||||
expect(render.body.print.preview_images).toHaveLength(1);
|
||||
|
||||
const images: Record<string, Buffer> = {
|
||||
"https://img.example.com/cover.png": await png(200),
|
||||
"https://img.example.com/gallery.png": await png(120),
|
||||
};
|
||||
global.fetch = (async (input: RequestInfo | URL) => {
|
||||
const body = images[String(input)];
|
||||
return body
|
||||
? new Response(body, { headers: { "content-type": "image/png" } })
|
||||
: new Response("not found", { status: 404 });
|
||||
}) as typeof fetch;
|
||||
await attachImportedPreviewImages(
|
||||
print.id,
|
||||
plate.id,
|
||||
"https://img.example.com/cover.png",
|
||||
[{ url: "https://img.example.com/gallery.png", filename: "gallery.png" }],
|
||||
0,
|
||||
);
|
||||
|
||||
const previews = await prisma.previewImage.findMany({ where: { printId: print.id }, orderBy: { position: "asc" } });
|
||||
expect(previews.map((img) => [img.position, img.generated])).toEqual([
|
||||
[0, false],
|
||||
[1, false],
|
||||
]);
|
||||
expect(await redOf(previewImagePath(previews[0].id))).toBeGreaterThan(180);
|
||||
expect(await redOf(plateThumbPath(plate.id))).toBeGreaterThan(180);
|
||||
expect((await prisma.plate.findUnique({ where: { id: plate.id } }))!.thumbGenerated).toBe(false);
|
||||
|
||||
// A render that finishes after the cover landed doesn't replace it.
|
||||
const late = await request(app)
|
||||
.post(`/api/plate/${plate.id}/thumbnail-generated`)
|
||||
.set(auth())
|
||||
.attach("file", await png(10), "render.png");
|
||||
expect(late.status).toBe(200);
|
||||
expect(late.body.print.preview_images).toHaveLength(2);
|
||||
expect(await redOf(plateThumbPath(plate.id))).toBeGreaterThan(180);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user