starred/Thingport
1
0 Fork 0
mirror of https://github.com/TautvydasDerzinskas/Thingport.git synced 2026-10-18 20:59:55 -05:00
Code Issues Packages Projects Releases 8 Wiki Activity

feat: support Markdown with embedded images in model descriptions

Browse Source
This commit is contained in:
Tautvydas Derzinskas committed 2026-10-07 00:31:34 +02:00
1 parent b4ead39283
commit 97f0eea4c2
34 files changed
+2634 -113
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/package-lock.json
Generated
+35
View File
@@ -25,6 +25,8 @@
"sharp": "^0.35.4",
"svg-captcha": "^1.4.0",
"three": "^0.160.1",
"turndown": "^7.2.4",
"turndown-plugin-gfm": "^1.0.2",
"yauzl": "^3.2.0",
"zod": "^3.23.8"
},
@@ -38,6 +40,7 @@
"@types/nodemailer": "^8.0.1",
"@types/supertest": "^7.2.1",
"@types/three": "^0.160.0",
"@types/turndown": "^5.0.6",
"@types/yauzl": "^2.10.3",
"oxlint": "^1.81.0",
"supertest": "^7.2.2",
@@ -1008,6 +1011,12 @@
"dev": true,
"license": "MIT"
},
"node_modules/@mixmark-io/domino": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz",
"integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==",
"license": "BSD-2-Clause"
},
"node_modules/@noble/hashes": {
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz",
@@ -1959,6 +1968,13 @@
"dev": true,
"license": "MIT"
},
"node_modules/@types/turndown": {
"version": "5.0.6",
"resolved": "https://registry.npmjs.org/@types/turndown/-/turndown-5.0.6.tgz",
"integrity": "sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/webxr": {
"version": "0.5.24",
"resolved": "https://registry.npmjs.org/@types/webxr/-/webxr-0.5.24.tgz",
@@ -4520,6 +4536,25 @@
"fsevents": "~2.3.3"
}
},
"node_modules/turndown": {
"version": "7.2.4",
"resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.4.tgz",
"integrity": "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==",
"license": "MIT",
"dependencies": {
"@mixmark-io/domino": "^2.2.0"
},
"engines": {
"node": ">=18",
"npm": ">=9"
}
},
"node_modules/turndown-plugin-gfm": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/turndown-plugin-gfm/-/turndown-plugin-gfm-1.0.2.tgz",
"integrity": "sha512-vwz9tfvF7XN/jE0dGoBei3FXWuvll78ohzCZQuOb+ZjWrs3a0XhQVomJEb2Qh4VHTPNRO4GPZh0V7VRbiWwkRg==",
"license": "MIT"
},
"node_modules/type-is": {
"version": "1.6.18",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
backend/package.json
+3
View File
@@ -34,6 +34,8 @@
"sharp": "^0.35.4",
"svg-captcha": "^1.4.0",
"three": "^0.160.1",
"turndown": "^7.2.4",
"turndown-plugin-gfm": "^1.0.2",
"yauzl": "^3.2.0",
"zod": "^3.23.8"
},
@@ -47,6 +49,7 @@
"@types/nodemailer": "^8.0.1",
"@types/supertest": "^7.2.1",
"@types/three": "^0.160.0",
"@types/turndown": "^5.0.6",
"@types/yauzl": "^2.10.3",
"oxlint": "^1.81.0",
"supertest": "^7.2.2",
backend/prisma/migrations/20261006180000_add_description_images/migration.sql
+17
View File
@@ -0,0 +1,17 @@
-- CreateTable
CREATE TABLE "DescriptionImage" (
"id" TEXT NOT NULL,
"printId" TEXT NOT NULL,
"sourceUrl" TEXT,
"mime" TEXT NOT NULL,
"size" INTEGER NOT NULL,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "DescriptionImage_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE INDEX "DescriptionImage_printId_idx" ON "DescriptionImage"("printId");
-- AddForeignKey
ALTER TABLE "DescriptionImage" ADD CONSTRAINT "DescriptionImage_printId_fkey" FOREIGN KEY ("printId") REFERENCES "Print"("id") ON DELETE CASCADE ON UPDATE CASCADE;
backend/prisma/schema.prisma
+47 -31
View File
@@ -96,42 +96,43 @@ model Category {
}
model Print {
id String @id @default(cuid())
userId String
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
name String
nameNormalized String
title String?
notes String?
creator String?
tags String[] @default([])
viewCount Int @default(0)
printCount Int @default(0)
id String @id @default(cuid())
userId String
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
name String
nameNormalized String
title String?
notes String?
creator String?
tags String[] @default([])
viewCount Int @default(0)
printCount Int @default(0)
// Back the Favourites and Browsing History pseudo-collections.
favoritedAt DateTime?
lastViewedAt DateTime?
favoritedAt DateTime?
lastViewedAt DateTime?
// Dedupes re-imports of the same design. Null for uploads and unidentifiable sources.
sourceProvider String?
sourceExternalId String?
sourceProvider String?
sourceExternalId String?
// Gaps (see services/sourceGaps.ts) the source had nothing for at the last fill.
unfillableGaps String[] @default([])
categoryId String?
category Category? @relation(fields: [categoryId], references: [id], onDelete: SetNull)
authorId String?
author Author? @relation(fields: [authorId], references: [id], onDelete: SetNull)
sourcePath String?
plates Plate[]
previewImages PreviewImage[]
files PrintFile[] @relation("PrintFiles")
preparedFileId String? @unique
preparedFile PrintFile? @relation("PreparedFile", fields: [preparedFileId], references: [id], onDelete: SetNull)
preparedMetadata Json?
collectionItems CollectionItem[]
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
unfillableGaps String[] @default([])
categoryId String?
category Category? @relation(fields: [categoryId], references: [id], onDelete: SetNull)
authorId String?
author Author? @relation(fields: [authorId], references: [id], onDelete: SetNull)
sourcePath String?
plates Plate[]
previewImages PreviewImage[]
descriptionImages DescriptionImage[]
files PrintFile[] @relation("PrintFiles")
preparedFileId String? @unique
preparedFile PrintFile? @relation("PreparedFile", fields: [preparedFileId], references: [id], onDelete: SetNull)
preparedMetadata Json?
collectionItems CollectionItem[]
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
// Generated column (name A, tags B, notes C, creator D) created with its GIN index in a
// hand-written migration, since Prisma can't express a generated tsvector.
searchVector Unsupported("tsvector")?
searchVector Unsupported("tsvector")?
@@unique([userId, categoryId, nameNormalized])
@@unique([userId, sourceProvider, sourceExternalId])
@@ -368,6 +369,21 @@ enum PrintFileRole {
PREPARED
}
// Images embedded in a print's Markdown notes, stored locally so the description survives the
// source changing or going away. Referenced from the notes as /description-image/<id>.
model DescriptionImage {
id String @id @default(cuid())
printId String
print Print @relation(fields: [printId], references: [id], onDelete: Cascade)
// Where an imported image came from, so a refresh reuses it instead of downloading again.
sourceUrl String?
mime String
size Int
createdAt DateTime @default(now())
@@index([printId])
}
model PrintFile {
id String @id @default(cuid())
printId String
backend/src/config.ts
+1
View File
@@ -37,6 +37,7 @@ export const CONSUME_DIR = path.resolve(process.env.CONSUME_DIR || "/app/consume
export const THUMBS = path.join(STORAGE, "thumbs");
export const BUNDLES = path.join(STORAGE, "bundles");
export const PREVIEWS = path.join(STORAGE, "previews");
export const DESCRIPTION_IMAGES = path.join(STORAGE, "description-images");
export const MODEL_PREVIEWS = path.join(STORAGE, "model-previews");
// Slicer-compatible copies of MakerWorld 3MFs, rebuilt when the source plate changes.
export const NORMALIZED_3MFS = path.join(STORAGE, "normalized-3mf");
backend/src/db.ts
+2 -2
View File
@@ -1,9 +1,9 @@
import fs from "node:fs";
import path from "node:path";
import { PrismaClient } from "@prisma/client";
import { STORAGE, THUMBS, BUNDLES, PREVIEWS, MODEL_PREVIEWS, NORMALIZED_3MFS } from "./config";
import { STORAGE, THUMBS, BUNDLES, PREVIEWS, DESCRIPTION_IMAGES, MODEL_PREVIEWS, NORMALIZED_3MFS } from "./config";
for (const dir of [STORAGE, THUMBS, BUNDLES, PREVIEWS, MODEL_PREVIEWS, NORMALIZED_3MFS]) {
for (const dir of [STORAGE, THUMBS, BUNDLES, PREVIEWS, DESCRIPTION_IMAGES, MODEL_PREVIEWS, NORMALIZED_3MFS]) {
fs.mkdirSync(dir, { recursive: true });
}
// The cache's name before it became NORMALIZED_3MFS; nothing reads it any more.
backend/src/routes/prints.ts
+44 -1
View File
@@ -12,6 +12,12 @@ import { modelUpload } from "../uploadMiddleware";
import { createPrint, deletePlateFiles, resolvePlateFilePath, type NewPlateInput } from "../services/printCreation";
import { plateThumbPath, relocatePrint, relocatePrintsForToken, uniqueModelName } from "../services/printService";
import { previewImagePath, deleteAllPreviewImages } from "../services/previewImageService";
import {
deleteAllDescriptionImages,
descriptionImagePath,
localizeDescriptionImages,
pruneDescriptionImages,
} from "../services/descriptionImageService";
import { deleteAuthorIfOrphaned, getLinkedAuthorIds } from "../services/authorService";
import { toPrintOut } from "../dto";
import { loadFullPrint, printOutById } from "../services/printLoader";
@@ -20,7 +26,7 @@ import { RENDERABLE_MODEL_EXTS } from "../config";
import { estimateDownloadSize, resolvePrintsForDownload, sendPrintsZip } from "../services/downloadZip";
import { systemCollectionKeyForId } from "../services/collectionService";
import { createLog } from "../services/auditLog";
import { fillSourceGaps } from "../services/fillGapsService";
import { fillSourceGaps, refreshDescriptionFromSource } from "../services/fillGapsService";
import { isNormalizable3mf, normalize3mfStatus, normalized3mfFor } from "../services/normalized3mfCache";
import type { Prisma } from "@prisma/client";
@@ -435,6 +441,22 @@ router.get(
}),
);
router.get(
"/description-image/:id",
asyncHandler(async (req, res) => {
const image = await prisma.descriptionImage.findFirst({
where: { id: req.params.id, print: { userId: req.userId } },
});
if (!image) throw new HttpError(404, "Not found");
const filePath = descriptionImagePath(image.id);
if (!fs.existsSync(filePath)) throw new HttpError(404, "Not found");
res.setHeader("Content-Type", image.mime);
res.setHeader("X-Content-Type-Options", "nosniff");
res.setHeader("Cache-Control", "private, max-age=31536000, immutable");
res.sendFile(path.resolve(filePath));
}),
);
router.get(
"/preview-image/:id/file.jpg",
asyncHandler(async (req, res) => {
@@ -502,6 +524,11 @@ router.post(
const updated = await prisma.print.update({ where: { id: print.id }, data });
const plates = await prisma.plate.findMany({ where: { printId: print.id }, orderBy: { position: "asc" } });
await relocatePrint(updated, plates);
if (body.notes !== undefined) {
await pruneDescriptionImages(print.id);
// Images pasted in by URL are fetched in the background: slow hosts mustn't hold up the save.
void localizeDescriptionImages(print.id);
}
res.json({ print: await printOutById(req.userId!, print.id) });
void createLog({
userId: req.userId!,
@@ -572,6 +599,7 @@ router.delete(
const full = await loadFullPrint(req.userId!, req.params.id);
await deleteAllPrintFiles(req.params.id);
await deleteAllPreviewImages(req.params.id);
await deleteAllDescriptionImages(req.params.id);
await prisma.print.delete({ where: { id: req.params.id } });
for (const plate of full.plates) {
await deletePlateFiles(plate);
@@ -600,6 +628,21 @@ async function fillGaps(req: Request, res: Response): Promise<void> {
}
router.post("/print/:id/fill-gaps", asyncHandler(fillGaps));
/** Replaces the description with the source's current one, formatting and images included. */
router.post(
"/print/:id/refresh-description",
asyncHandler(async (req, res) => {
const sourceUrl = await refreshDescriptionFromSource(req.userId!, req.params.id);
res.json({ print: await printOutById(req.userId!, req.params.id) });
void createLog({
userId: req.userId!,
action: "model_edited",
targetId: req.params.id,
details: { field: "description_refreshed", source_url: sourceUrl },
});
}),
);
// Extension versions up to 1.3.0 call it by its old name.
router.post("/print/:id/reimport", asyncHandler(fillGaps));
backend/src/services/adminService.ts
+2
View File
@@ -3,6 +3,7 @@ import type { Prisma } from "@prisma/client";
import { prisma } from "../db";
import { deleteAllPrintFiles } from "./printFileService";
import { deleteAllPreviewImages } from "./previewImageService";
import { deleteAllDescriptionImages } from "./descriptionImageService";
import { deletePlateFiles } from "./printCreation";
import { plateThumbPath } from "./printService";
import { loadFullPrint } from "./printLoader";
@@ -89,6 +90,7 @@ export async function deleteAllPrintsForUser(userId: string): Promise<number> {
const full = await loadFullPrint(userId, id);
await deleteAllPrintFiles(id);
await deleteAllPreviewImages(id);
await deleteAllDescriptionImages(id);
await prisma.print.delete({ where: { id } });
for (const plate of full.plates) {
await deletePlateFiles(plate);
backend/src/services/descriptionImageService.ts
+150
View File
@@ -0,0 +1,150 @@
import fs from "node:fs/promises";
import path from "node:path";
import sharp from "sharp";
import { DESCRIPTION_IMAGES, IMPORT_TIMEOUT_SECONDS, IMPORT_USER_AGENT } from "../config";
import { prisma } from "../db";
import { mapWithConcurrency } from "../utils/concurrency";
import { validateRemoteUrl } from "../utils/urlUtils";
import { readCapped } from "../utils/readCapped";
const IMAGE_MAX_BYTES = 16 * 1024 * 1024;
// Step-by-step assembly guides run long; this still bounds a hostile description.
const IMAGE_MAX_COUNT = 80;
const DOWNLOAD_CONCURRENCY = 4;
const MAX_REDIRECTS = 5;
// For all of one description's downloads together, so slow hosts can't stretch an import out.
const DOWNLOAD_BUDGET_MS = 2 * 60 * 1000;
// Kept byte-for-byte rather than re-encoded, so animated GIFs stay animated. SVG is left out:
// served from our own origin it could run script.
const ALLOWED_FORMATS: Record<string, string> = {
jpeg: "image/jpeg",
png: "image/png",
gif: "image/gif",
webp: "image/webp",
};
const REMOTE_IMAGE_RE = /!\[([^\]]*)\]\((https?:\/\/[^\s)]+)\)/g;
const LOCAL_REF_RE = /\/description-image\/([a-z0-9]+)/g;
export function descriptionImagePath(id: string): string {
return path.join(DESCRIPTION_IMAGES, id);
}
/** Relative to the API base; the frontend adds the base and auth token when rendering. */
function descriptionImageRef(id: string): string {
return `/description-image/${id}`;
}
/** Redirects are followed by hand so every hop is checked before it's requested; the
* source is third-party content, so a redirect to an internal address must never be fetched. */
async function fetchImage(url: string, deadline: number): Promise<Buffer | null> {
const timeLeft = Math.min(IMPORT_TIMEOUT_SECONDS * 1000, deadline - Date.now());
if (timeLeft <= 0) return null;
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), timeLeft);
try {
let current = url;
for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
await validateRemoteUrl(current);
const res = await fetch(current, {
headers: { "User-Agent": IMPORT_USER_AGENT, Accept: "image/*" },
redirect: "manual",
signal: controller.signal,
});
const location = res.headers.get("location");
if (res.status >= 300 && res.status < 400 && location) {
await res.body?.cancel().catch(() => undefined);
current = new URL(location, current).toString();
continue;
}
if (!res.ok) {
await res.body?.cancel().catch(() => undefined);
return null;
}
const { buffer, truncated } = await readCapped(res, IMAGE_MAX_BYTES);
return truncated ? null : buffer;
}
return null;
} catch {
return null;
} finally {
clearTimeout(timeout);
}
}
async function imageMime(buffer: Buffer): Promise<string | null> {
try {
const { format } = await sharp(buffer, { animated: true }).metadata();
return (format && ALLOWED_FORMATS[format]) || null;
} catch {
return null;
}
}
/** Null when the bytes aren't an allowed image. */
async function saveDescriptionImage(printId: string, buffer: Buffer, sourceUrl: string | null) {
const mime = await imageMime(buffer);
if (!mime) return null;
const row = await prisma.descriptionImage.create({ data: { printId, sourceUrl, mime, size: buffer.length } });
try {
await fs.writeFile(descriptionImagePath(row.id), buffer);
} catch {
await prisma.descriptionImage.delete({ where: { id: row.id } }).catch(() => undefined);
return null;
}
return row;
}
/** Downloads the remote images in a print's notes and points the notes at the local copies.
* An image that can't be fetched in time keeps its remote URL. Best-effort; never throws. */
export async function localizeDescriptionImages(printId: string): Promise<void> {
try {
const print = await prisma.print.findUnique({ where: { id: printId }, select: { notes: true } });
const notes = print?.notes ?? "";
const urls = [...new Set([...notes.matchAll(REMOTE_IMAGE_RE)].map((match) => match[2]))].slice(0, IMAGE_MAX_COUNT);
if (urls.length) {
const existing = await prisma.descriptionImage.findMany({
where: { printId, sourceUrl: { in: urls } },
select: { id: true, sourceUrl: true },
});
const localByUrl = new Map(existing.map((image) => [image.sourceUrl!, image.id]));
const missing = urls.filter((url) => !localByUrl.has(url));
const deadline = Date.now() + DOWNLOAD_BUDGET_MS;
await mapWithConcurrency(missing, DOWNLOAD_CONCURRENCY, async (url) => {
const buffer = await fetchImage(url, deadline);
const saved = buffer ? await saveDescriptionImage(printId, buffer, url) : null;
if (saved) localByUrl.set(url, saved.id);
});
const rewritten = notes.replace(REMOTE_IMAGE_RE, (match, alt: string, url: string) => {
const id = localByUrl.get(url);
return id ? `![${alt}](${descriptionImageRef(id)})` : match;
});
// Only onto the notes it started from: an edit saved meanwhile wins, and its images are
// left for that edit's own run.
if (rewritten !== notes) {
await prisma.print.updateMany({ where: { id: printId, notes }, data: { notes: rewritten } });
}
}
await pruneDescriptionImages(printId);
} catch (err) {
console.error("Failed to store description images", printId, err);
}
}
/** Drops the stored images the print's notes no longer reference, e.g. after an edit. */
export async function pruneDescriptionImages(printId: string): Promise<void> {
const print = await prisma.print.findUnique({ where: { id: printId }, select: { notes: true } });
const referenced = new Set([...(print?.notes ?? "").matchAll(LOCAL_REF_RE)].map((match) => match[1]));
const images = await prisma.descriptionImage.findMany({ where: { printId }, select: { id: true } });
const unused = images.filter((image) => !referenced.has(image.id));
if (!unused.length) return;
await prisma.descriptionImage.deleteMany({ where: { id: { in: unused.map((image) => image.id) } } });
for (const image of unused) await fs.rm(descriptionImagePath(image.id), { force: true }).catch(() => undefined);
}
export async function deleteAllDescriptionImages(printId: string): Promise<void> {
const rows = await prisma.descriptionImage.findMany({ where: { printId }, select: { id: true } });
await prisma.descriptionImage.deleteMany({ where: { printId } });
for (const row of rows) await fs.rm(descriptionImagePath(row.id), { force: true }).catch(() => undefined);
}
backend/src/services/descriptionMarkdown.ts
+166
View File
@@ -0,0 +1,166 @@
import TurndownService from "turndown";
import { gfm } from "turndown-plugin-gfm";
import { decodeHtmlEntities } from "../utils/htmlEntities";
// Descriptions are stored as Markdown: it stays readable as plain text (search, the edit
// field), and the sources' HTML converts to it without needing sanitizing on display.
function createTurndown(): TurndownService {
const service = new TurndownService({
headingStyle: "atx",
bulletListMarker: "-",
codeBlockStyle: "fenced",
emDelimiter: "*",
});
service.use(gfm);
// MakerWorld's "Boost Me" banner, and anything that isn't content.
service.remove(["boostme", "script", "style", "noscript", "button", "form"] as unknown as TurndownService.Filter);
service.addRule("image", {
filter: "img",
replacement: (_content, node) => {
const src = absoluteHttpUrl(attrs(node).getAttribute("src"));
if (!src) return "";
const alt = (attrs(node).getAttribute("alt") || "").replace(/[[\]\n]/g, " ").trim();
return `![${alt}](${src})`;
},
});
// "- item" rather than Turndown's "- item", which is awkward to edit by hand.
service.addRule("listItem", {
filter: "li",
replacement: (content, node, options) => {
const item = node as unknown as { parentNode: ListNode | null; nextSibling: unknown };
const list = item.parentNode;
let prefix = `${options.bulletListMarker} `;
if (list?.nodeName === "OL") {
const start = Number(list.getAttribute("start")) || 1;
prefix = `${start + Array.prototype.indexOf.call(list.children, node)}. `;
}
const body = content
.replace(/^\n+/, "")
.replace(/\n+$/, "\n")
.replace(/\n/gm, `\n${" ".repeat(prefix.length)}`);
return prefix + body + (item.nextSibling && !body.endsWith("\n") ? "\n" : "");
},
});
// Overrides the GFM plugin's tables, which keep any table without a header row as raw HTML.
service.addRule("table", {
filter: "table",
replacement: (_content, node) => `\n\n${tableToMarkdown(service, node as unknown as DomNode)}\n\n`,
});
// Embedded videos can't be kept; a link to them can.
service.addRule("iframe", {
filter: "iframe",
replacement: (_content, node) => {
const src = absoluteHttpUrl(attrs(node).getAttribute("src"));
return src ? `\n\n[${src}](${src})\n\n` : "";
},
});
return service;
}
let turndown: TurndownService | null = null;
// The backend builds without the DOM lib; these are all the rules read from a node.
type ListNode = { nodeName: string; children: ArrayLike<unknown>; getAttribute(name: string): string | null };
type DomNode = {
nodeName: string;
children: ArrayLike<DomNode>;
innerHTML: string;
querySelector(selector: string): DomNode | null;
};
// A cell holding any of these is page layout (text beside a picture, say), not tabular data.
const LAYOUT_CELL_CONTENT = "ul, ol, img, figure, h1, h2, h3, h4, h5, h6, table, blockquote, pre";
function tableRows(table: DomNode): DomNode[][] {
const rows: DomNode[] = [];
for (const child of Array.from(table.children)) {
if (child.nodeName === "TR") rows.push(child);
else if (["THEAD", "TBODY", "TFOOT"].includes(child.nodeName)) {
rows.push(...Array.from(child.children).filter((row) => row.nodeName === "TR"));
}
}
return rows
.map((row) => Array.from(row.children).filter((cell) => cell.nodeName === "TD" || cell.nodeName === "TH"))
.filter((cells) => cells.length);
}
/** A Markdown table when the cells hold plain text; Markdown tables can't hold lists, images or
* more than one line, so a layout table is unrolled into its cells' content instead. */
function tableToMarkdown(service: TurndownService, table: DomNode): string {
const rows = tableRows(table);
if (!rows.length) return "";
if (table.querySelector(LAYOUT_CELL_CONTENT)) {
return rows
.flat()
.map((cell) => service.turndown(cell.innerHTML).trim())
.filter(Boolean)
.join("\n\n");
}
const width = Math.max(...rows.map((cells) => cells.length));
const lines = rows.map((cells) => {
const texts = cells.map((cell) =>
service
.turndown(cell.innerHTML)
.replace(/\s*\n+\s*/g, " ")
.replace(/\|/g, "\\|")
.trim(),
);
while (texts.length < width) texts.push("");
return `| ${texts.join(" | ")} |`;
});
// GFM tables need a header row; without a <th> row, the first row is the closest thing.
lines.splice(1, 0, `|${" --- |".repeat(width)}`);
return lines.join("\n");
}
function attrs(node: unknown): { getAttribute(name: string): string | null } {
return node as { getAttribute(name: string): string | null };
}
function absoluteHttpUrl(value: string | null): string | null {
const raw = (value || "").trim();
if (!raw) return null;
const url = raw.startsWith("//") ? `https:${raw}` : raw;
return /^https?:\/\//i.test(url) ? url.replace(/[()\s]/g, encodeURIComponent) : null;
}
function tidy(markdown: string): string | null {
const text = markdown
.replace(/ /g, " ")
.split("\n")
.map((line) => line.replace(/[ \t]+$/, ""))
.join("\n")
.replace(/\n{3,}/g, "\n\n")
.trim();
return text || null;
}
export function htmlToMarkdown(html: string): string | null {
turndown ??= createTurndown();
return tidy(turndown.turndown(html));
}
/** Thingiverse descriptions are Markdown already, but older ones mix in a little HTML. HTML
* parsing would collapse the Markdown's line breaks, so the common tags are mapped by hand. */
export function cleanThingiverseMarkdown(text: string): string | null {
if (!/<\/?[a-z][^>]*>/i.test(text)) return tidy(text);
const converted = decodeHtmlEntities(
text
.replace(/<br\s*\/?>/gi, "\n")
.replace(/<img\b[^>]*?\bsrc=["']([^"']+)["'][^>]*>/gi, (_match, src: string) => {
const url = absoluteHttpUrl(src);
return url ? `![](${url})` : "";
})
.replace(/<a\b[^>]*?\bhref=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, (_match, href: string, label: string) => {
const url = absoluteHttpUrl(href);
return url ? `[${label.replace(/<[^>]+>/g, "").trim() || url}](${url})` : label;
})
.replace(/<\/?(strong|b)>/gi, "**")
.replace(/<\/?(em|i)>/gi, "*")
.replace(/<\/(p|div|h[1-6]|li)>/gi, "\n\n")
.replace(/<[^>]+>/g, ""),
);
return tidy(converted);
}
backend/src/services/fillGapsService.ts
+15
View File
@@ -21,6 +21,7 @@ import { getThingiverseAccessToken } from "./settingsService";
import { getUserMakerworldCookie } from "./makerworldCookieService";
import { upsertAuthorFromImport } from "./authorService";
import { emptyGaps, openSourceGaps, type SourceGap } from "./sourceGaps";
import { localizeDescriptionImages } from "./descriptionImageService";
export type FillGapsResult = {
source_url: string;
@@ -141,9 +142,23 @@ export async function fillSourceGaps(userId: string, printId: string): Promise<F
meta.galleryImages.map((image) => ({ url: image.url, filename: image.filename })),
);
}
if (wanted.has("description") && meta.description) await localizeDescriptionImages(print.id);
const after = await prisma.print.findUniqueOrThrow({ where: { id: print.id }, include: { previewImages: true } });
const remaining = emptyGaps(after, after.previewImages);
await prisma.print.update({ where: { id: print.id }, data: { unfillableGaps: remaining } });
return { source_url: sourceUrl, filled: gaps.filter((gap) => !remaining.includes(gap)), remaining };
}
/** Overwrites the description with the source's, unlike fillSourceGaps. Returns the source URL. */
export async function refreshDescriptionFromSource(userId: string, printId: string): Promise<string> {
const print = await prisma.print.findFirst({ where: { id: printId, userId } });
if (!print) throw new HttpError(404, "Model not found");
const sourceUrl = buildImportSourceUrl(print.sourceProvider, print.sourceExternalId);
if (!sourceUrl) throw new HttpError(400, "This model has no recorded source to refresh its description from");
const meta = await fetchSourceMeta(userId, print, sourceUrl);
if (!meta.description) throw new HttpError(404, "The source has no description for this model");
await prisma.print.update({ where: { id: print.id }, data: { notes: meta.description } });
await localizeDescriptionImages(print.id);
return sourceUrl;
}
backend/src/services/importJobRunner.ts
+1 -1
View File
@@ -6,7 +6,7 @@ import { countJobItems, getJob, getJobStatus, listJobItems, updateJob, updateJob
import { createNotification } from "./notificationService";
import { addPrintsToCollection, findOrCreateCollectionByName } from "./collectionService";
import { resolveMakerworldCookie } from "./importResolvers";
import { decodeHtmlEntities } from "./importResolvers";
import { decodeHtmlEntities } from "../utils/htmlEntities";
import {
extractMakerworldBearerToken,
parseMakerworldModelUrl,
backend/src/services/importResolvers.ts
+3 -24
View File
@@ -10,6 +10,8 @@ import {
IMPORT_USER_AGENT,
} from "../config";
import { isJsonContentType } from "../utils/fileUtils";
import { htmlToMarkdown } from "./descriptionMarkdown";
import { decodeHtmlEntities } from "../utils/htmlEntities";
import {
isCaptchaChallenge,
isDownloadLimitReply,
@@ -425,7 +427,7 @@ export function makerworldMetaFromDesign(design: unknown): ImportedPageMetadata
.map((tag) => tag.trim())
: [];
meta.description =
typeof design.summary === "string" && design.summary.trim() ? htmlToPlainText(design.summary) : null;
typeof design.summary === "string" && design.summary.trim() ? htmlToMarkdown(design.summary) : null;
const designCreator = isRecord(design.designCreator) ? design.designCreator : null;
meta.creator = designCreator ? pickDesignString(designCreator, ["nickName", "name", "handle"]) : null;
meta.author = makerworldAuthorFromDesignCreator(designCreator);
@@ -462,29 +464,6 @@ export function makerworldAuthorFromDesignCreator(creator: unknown): ImportedAut
};
}
export function decodeHtmlEntities(value: string): string {
return value
.replace(/&nbsp;/g, " ")
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&amp;/g, "&");
}
export function htmlToPlainText(html: string): string | null {
const withBreaks = html.replace(/<\s*(br|\/p|\/li|\/div|\/h[1-6])\s*\/?>/gi, "\n").replace(/<[^>]+>/g, "");
const text = decodeHtmlEntities(withBreaks)
.replace(/\u00A0/g, " ")
.replace(/[ \t]+/g, " ")
.split("\n")
.map((line) => line.trim())
.join("\n")
.replace(/\n{3,}/g, "\n\n")
.trim();
return text || null;
}
function genericTitleFromHtml(html: string): string | null {
const ogMatch =
html.match(/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']*)["']/i) ||
backend/src/services/importService.ts
+5 -23
View File
@@ -20,6 +20,7 @@ import {
sanitizeFilename,
} from "../utils/fileUtils";
import { validateRemoteUrl } from "../utils/urlUtils";
import { readCapped } from "../utils/readCapped";
import { maybeSleep, sleep } from "../utils/concurrency";
import { fetchViaFlaresolverr, isFlaresolverrEnabled, looksLikeCloudflareBlock, shouldProxyHost } from "./flaresolverr";
import {
@@ -72,6 +73,7 @@ import {
} from "./printCreation";
import { plateThumbReplaceable, saveThumbFromBytes } from "./printService";
import { addPreviewImage, removeGeneratedPreviewImages } from "./previewImageService";
import { localizeDescriptionImages } from "./descriptionImageService";
import { prisma } from "../db";
import { Prisma } from "@prisma/client";
import type { Author, Plate, PreviewImage, Print } from "@prisma/client";
@@ -105,29 +107,6 @@ function parseCharset(contentType: string | null): string {
return match ? match[1].trim().replace(/["']/g, "") : "utf-8";
}
async function readCapped(response: Response, maxBytes: number): Promise<{ buffer: Buffer; truncated: boolean }> {
if (!response.body) return { buffer: Buffer.alloc(0), truncated: false };
const reader = response.body.getReader();
const chunks: Buffer[] = [];
let total = 0;
let truncated = false;
for (;;) {
const { done, value } = await reader.read();
if (done) break;
const chunk = Buffer.from(value);
total += chunk.length;
if (total > maxBytes) {
const allowed = chunk.length - (total - maxBytes);
if (allowed > 0) chunks.push(chunk.subarray(0, allowed));
truncated = true;
await reader.cancel().catch(() => undefined);
break;
}
chunks.push(chunk);
}
return { buffer: Buffer.concat(chunks), truncated };
}
async function rawFetch(url: string, headers: Record<string, string>): Promise<Response> {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), IMPORT_TIMEOUT_SECONDS * 1000);
@@ -852,6 +831,7 @@ async function importThingiverseThing(
}
const gallery = galleryImages.map((img) => ({ url: img.url, filename: img.name }));
await attachImportedPreviewImages(result.print.id, result.plates[0]?.id, meta.previewImageUrl ?? null, gallery);
await localizeDescriptionImages(result.print.id);
const previewImages = await prisma.previewImage.findMany({
where: { printId: result.print.id },
orderBy: { position: "asc" },
@@ -938,6 +918,7 @@ async function importPrintablesModel(
}
const gallery = galleryImages.map((img) => ({ url: img.url, filename: img.name }));
await attachImportedPreviewImages(result.print.id, result.plates[0]?.id, meta.previewImageUrl ?? null, gallery);
await localizeDescriptionImages(result.print.id);
const previewImages = await prisma.previewImage.findMany({
where: { printId: result.print.id },
orderBy: { position: "asc" },
@@ -1017,6 +998,7 @@ export async function importPrintFromUrl(
meta.galleryImages,
body.makerworldPaceMs,
);
await localizeDescriptionImages(result.print.id);
const previewImages = await prisma.previewImage.findMany({
where: { printId: result.print.id },
orderBy: { position: "asc" },
backend/src/services/printablesApi.ts
+4 -3
View File
@@ -1,6 +1,7 @@
import { IMPORT_BROWSER_USER_AGENT, IMPORT_HTML_MAX_BYTES, IMPORT_TIMEOUT_SECONDS } from "../config";
import { HttpError } from "../utils/fileUtils";
import { htmlToPlainText, type ImportedAuthorInfo, type ImportedPageMetadata } from "./importResolvers";
import { type ImportedAuthorInfo, type ImportedPageMetadata } from "./importResolvers";
import { htmlToMarkdown } from "./descriptionMarkdown";
// Unlike Cloudflare-gated www.printables.com, the GraphQL API answers unauthenticated requests.
const PRINTABLES_GRAPHQL_URL = "https://api.printables.com/graphql/";
@@ -143,8 +144,8 @@ export async function resolvePrintablesModel(modelId: string): Promise<Printable
const meta: Partial<ImportedPageMetadata> = {};
if (typeof model.name === "string" && model.name.trim()) meta.title = model.name.trim();
if (typeof model.description === "string" && model.description.trim()) {
const plainText = htmlToPlainText(model.description);
if (plainText) meta.description = plainText;
const markdown = htmlToMarkdown(model.description);
if (markdown) meta.description = markdown;
}
if (Array.isArray(model.tags)) {
const tags = model.tags
backend/src/services/thingiverseApi.ts
+11 -1
View File
@@ -1,5 +1,6 @@
import { IMPORT_BROWSER_USER_AGENT, IMPORT_TIMEOUT_SECONDS } from "../config";
import type { ImportedAuthorInfo, ImportedPageMetadata } from "./importResolvers";
import { cleanThingiverseMarkdown } from "./descriptionMarkdown";
import {
extractJsonFromBrowserBody,
fetchViaFlaresolverr,
@@ -172,6 +173,14 @@ function extractCreatorAuthor(creator: Record<string, unknown>): {
return { creator: name, author };
}
/** The Thing's description, with its separate instructions section (when it has one) appended. */
function thingiverseDescription(detail: Record<string, unknown>): string | null {
const description = typeof detail.description === "string" ? cleanThingiverseMarkdown(detail.description) : null;
const instructions = typeof detail.instructions === "string" ? cleanThingiverseMarkdown(detail.instructions) : null;
if (!instructions) return description;
return [description, `## Instructions\n\n${instructions}`].filter(Boolean).join("\n\n");
}
/** Null for a Thing that doesn't exist or isn't accessible; throws ThingiverseAuthError for a
* rejected token. */
export async function resolveThingiverseThing(
@@ -183,7 +192,8 @@ export async function resolveThingiverseThing(
const meta: Partial<ImportedPageMetadata> = {};
if (typeof detail.name === "string" && detail.name.trim()) meta.title = detail.name.trim();
if (typeof detail.description === "string" && detail.description.trim()) meta.description = detail.description.trim();
const description = thingiverseDescription(detail);
if (description) meta.description = description;
if (Array.isArray(detail.tags)) {
const tags = detail.tags
.map((t) => (isRecord(t) && typeof t.name === "string" ? t.name.trim() : null))
backend/src/services/zipService.ts
+2
View File
@@ -9,6 +9,7 @@ import { IMPORT_MAX_BYTES } from "../config";
import { listZipEntries as listRawZipEntries, readZipEntry } from "../utils/zipReader";
import { validateParentCategory } from "./categoryService";
import { attachImportedPreviewImages } from "./importService";
import { localizeDescriptionImages } from "./descriptionImageService";
import { createPrint, type PrintMetaInput } from "./printCreation";
import type { Print, Plate, PreviewImage } from "@prisma/client";
@@ -177,6 +178,7 @@ export async function extractZipEntriesToPrints(
]);
tempPath = null;
await attachImportedPreviewImages(print.id, plates[0]?.id, options.previewImageUrl, options.galleryImages ?? []);
await localizeDescriptionImages(print.id);
const previewImages = await prisma.previewImage.findMany({
where: { printId: print.id },
orderBy: { position: "asc" },
backend/src/types/turndown-plugin-gfm.d.ts
Vendored
+4
View File
@@ -0,0 +1,4 @@
declare module "turndown-plugin-gfm" {
import type TurndownService from "turndown";
export const gfm: TurndownService.Plugin;
}
backend/src/utils/htmlEntities.ts
+9
View File
@@ -0,0 +1,9 @@
export function decodeHtmlEntities(value: string): string {
return value
.replace(/&nbsp;/g, " ")
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&amp;/g, "&");
}
backend/src/utils/readCapped.ts
+26
View File
@@ -0,0 +1,26 @@
/** Reads at most `maxBytes` of a response body, whatever its Content-Length claims. */
export async function readCapped(
response: Response,
maxBytes: number,
): Promise<{ buffer: Buffer; truncated: boolean }> {
if (!response.body) return { buffer: Buffer.alloc(0), truncated: false };
const reader = response.body.getReader();
const chunks: Buffer[] = [];
let total = 0;
let truncated = false;
for (;;) {
const { done, value } = await reader.read();
if (done) break;
const chunk = Buffer.from(value);
total += chunk.length;
if (total > maxBytes) {
const allowed = chunk.length - (total - maxBytes);
if (allowed > 0) chunks.push(chunk.subarray(0, allowed));
truncated = true;
await reader.cancel().catch(() => undefined);
break;
}
chunks.push(chunk);
}
return { buffer: Buffer.concat(chunks), truncated };
}
backend/tests/descriptionImages.test.ts
+240
View File
@@ -0,0 +1,240 @@
import fs from "node:fs";
import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import request from "supertest";
import { createApp } from "../src/app";
import { importPrintFromUrl } from "../src/services/importService";
import { descriptionImagePath, localizeDescriptionImages } from "../src/services/descriptionImageService";
import { prisma } from "../src/db";
// Images embedded in an imported description are stored locally and served from our own API.
const MODEL_ID = "1786599";
const MODEL_URL = `https://www.printables.com/model/${MODEL_ID}-lamp-kit`;
const STEP_IMAGE = "https://media.printables.com/media/prints/step-1.gif";
const MISSING_IMAGE = "https://media.printables.com/media/prints/gone.jpg";
// 1x1 GIF, so the test also covers keeping a GIF as a GIF.
const ONE_PIXEL_GIF_BASE64 = "R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7";
const ONE_PIXEL_PNG_BASE64 =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=";
function modelResponse(description: string) {
return {
data: {
print: {
id: MODEL_ID,
name: "Lamp kit",
description,
user: { id: "991", handle: "lamps", publicUsername: "lamps", avatarFilePath: null },
image: { filePath: "media/prints/cover.jpg" },
images: [{ filePath: "media/prints/cover.jpg" }],
tags: [],
category: null,
stls: [{ id: "5", name: "lamp.3mf" }],
},
},
};
}
function json(body: unknown): Response {
return new Response(JSON.stringify(body), { status: 200, headers: { "content-type": "application/json" } });
}
function mockFetch(description: string) {
return vi.fn<(input: RequestInfo | URL, init?: RequestInit) => Promise<Response>>(async (input, init) => {
const url = String(input);
if (url === "https://api.printables.com/graphql/") {
const body = JSON.parse(String(init?.body ?? "{}"));
if (typeof body.query === "string" && body.query.includes("getDownloadLink")) {
return json({
data: {
getDownloadLink: {
ok: true,
errors: null,
output: { files: [{ id: "5", link: "https://files.printables.com/media/prints/lamp.3mf" }] },
},
},
});
}
return json(modelResponse(description));
}
if (url === "https://files.printables.com/media/prints/lamp.3mf") return new Response("fake-3mf", { status: 200 });
if (url === "https://media.printables.com/media/prints/cover.jpg") {
return new Response(Buffer.from(ONE_PIXEL_PNG_BASE64, "base64"), { status: 200 });
}
if (url === STEP_IMAGE) return new Response(Buffer.from(ONE_PIXEL_GIF_BASE64, "base64"), { status: 200 });
if (url === MISSING_IMAGE) return new Response("nope", { status: 404 });
throw new Error(`Unexpected fetch to ${url}`);
}) as unknown as typeof fetch;
}
describe("description images", () => {
const app = createApp();
let token: string;
let userId: string;
let printId: string;
const originalFetch = global.fetch;
beforeAll(async () => {
const res = await request(app)
.post("/api/register")
.send({
displayName: "Description Images",
email: `desc-images-${Date.now()}@example.com`,
password: "password123",
});
if (res.status !== 200) throw new Error(`Failed to register: ${res.status} ${JSON.stringify(res.body)}`);
token = res.body.token;
userId = res.body.user.id;
});
afterEach(() => {
global.fetch = originalFetch;
vi.restoreAllMocks();
});
it("stores an imported description's images and points the Markdown at them", async () => {
global.fetch = mockFetch(
`<h3>STEP 1:</h3><figure><img src="${STEP_IMAGE}"></figure><p>Then:</p><img src="${MISSING_IMAGE}">`,
);
const result = await importPrintFromUrl(userId, MODEL_URL, { url: MODEL_URL, tags: [] });
printId = result.print.id;
const print = await prisma.print.findUniqueOrThrow({ where: { id: printId } });
const images = await prisma.descriptionImage.findMany({ where: { printId } });
expect(images).toHaveLength(1);
expect(images[0]).toMatchObject({ sourceUrl: STEP_IMAGE, mime: "image/gif" });
expect(print.notes).toContain("### STEP 1:");
expect(print.notes).toContain(`![](/description-image/${images[0].id})`);
// One that couldn't be fetched keeps its remote URL rather than disappearing.
expect(print.notes).toContain(`![](${MISSING_IMAGE})`);
const served = await request(app)
.get(`/api/description-image/${images[0].id}`)
.set("Authorization", `Bearer ${token}`);
expect(served.status).toBe(200);
expect(served.headers["content-type"]).toBe("image/gif");
});
it("doesn't serve another user's description image", async () => {
const image = await prisma.descriptionImage.findFirstOrThrow({ where: { printId } });
const other = await request(app)
.post("/api/register")
.send({ displayName: "Other", email: `desc-images-other-${Date.now()}@example.com`, password: "password123" });
const res = await request(app)
.get(`/api/description-image/${image.id}`)
.set("Authorization", `Bearer ${other.body.token}`);
expect(res.status).toBe(404);
});
it("drops the stored images an edit removes from the description", async () => {
const image = await prisma.descriptionImage.findFirstOrThrow({ where: { printId } });
const res = await request(app)
.post(`/api/print/${printId}/meta`)
.set("Authorization", `Bearer ${token}`)
.send({ notes: "Just text now." });
expect(res.status).toBe(200);
expect(await prisma.descriptionImage.count({ where: { printId } })).toBe(0);
expect(fs.existsSync(descriptionImagePath(image.id))).toBe(false);
});
it("refreshes the description from the source, images included", async () => {
global.fetch = mockFetch(`<p>Updated <strong>steps</strong></p><img src="${STEP_IMAGE}">`);
const res = await request(app)
.post(`/api/print/${printId}/refresh-description`)
.set("Authorization", `Bearer ${token}`);
expect(res.status).toBe(200);
const image = await prisma.descriptionImage.findFirstOrThrow({ where: { printId } });
expect(res.body.print.notes).toBe(`Updated **steps**\n\n![](/description-image/${image.id})`);
});
it("drops all stored images when the description is cleared", async () => {
const image = await prisma.descriptionImage.findFirstOrThrow({ where: { printId } });
const res = await request(app)
.post(`/api/print/${printId}/meta`)
.set("Authorization", `Bearer ${token}`)
.send({ notes: "" });
expect(res.status).toBe(200);
expect(await prisma.descriptionImage.count({ where: { printId } })).toBe(0);
expect(fs.existsSync(descriptionImagePath(image.id))).toBe(false);
// Back again for the next test.
global.fetch = mockFetch(`<img src="${STEP_IMAGE}">`);
await request(app).post(`/api/print/${printId}/refresh-description`).set("Authorization", `Bearer ${token}`);
});
it("removes the stored images with the model", async () => {
const image = await prisma.descriptionImage.findFirstOrThrow({ where: { printId } });
const res = await request(app).delete(`/api/print/${printId}`).set("Authorization", `Bearer ${token}`);
expect(res.status).toBe(200);
expect(fs.existsSync(descriptionImagePath(image.id))).toBe(false);
});
describe("downloading untrusted image URLs", () => {
async function printWithImage(url: string): Promise<string> {
const name = `guarded-${Date.now()}-${Math.random()}`;
const print = await prisma.print.create({
data: { userId, name, nameNormalized: name, notes: `![](${url})` },
});
return print.id;
}
it("follows a redirect to another public host", async () => {
const fetchMock = vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async (input) => {
if (String(input) === MISSING_IMAGE) {
return new Response(null, { status: 302, headers: { location: STEP_IMAGE } });
}
if (String(input) === STEP_IMAGE) {
return new Response(Buffer.from(ONE_PIXEL_GIF_BASE64, "base64"), { status: 200 });
}
throw new Error(`Unexpected fetch to ${String(input)}`);
});
global.fetch = fetchMock as unknown as typeof fetch;
const id = await printWithImage(MISSING_IMAGE);
await localizeDescriptionImages(id);
expect(await prisma.descriptionImage.count({ where: { printId: id } })).toBe(1);
});
it("never requests a redirect target on an internal address", async () => {
const fetchMock = vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async (input) => {
if (String(input) === STEP_IMAGE) {
return new Response(null, { status: 302, headers: { location: "http://127.0.0.1:8000/secret.png" } });
}
throw new Error(`Unexpected fetch to ${String(input)}`);
});
global.fetch = fetchMock as unknown as typeof fetch;
const id = await printWithImage(STEP_IMAGE);
await localizeDescriptionImages(id);
expect(fetchMock.mock.calls.map(([input]) => String(input))).toEqual([STEP_IMAGE]);
expect(await prisma.descriptionImage.count({ where: { printId: id } })).toBe(0);
const print = await prisma.print.findUniqueOrThrow({ where: { id } });
expect(print.notes).toBe(`![](${STEP_IMAGE})`);
});
it("leaves an edit saved during the download alone", async () => {
const id = await printWithImage(STEP_IMAGE);
global.fetch = vi.fn<() => Promise<Response>>(async () => {
await prisma.print.update({ where: { id }, data: { notes: "Edited meanwhile." } });
return new Response(Buffer.from(ONE_PIXEL_GIF_BASE64, "base64"), { status: 200 });
}) as unknown as typeof fetch;
await localizeDescriptionImages(id);
const print = await prisma.print.findUniqueOrThrow({ where: { id } });
expect(print.notes).toBe("Edited meanwhile.");
// The download the edit made unnecessary isn't kept.
expect(await prisma.descriptionImage.count({ where: { printId: id } })).toBe(0);
});
it("stops reading an oversized image that doesn't declare its length", async () => {
// A valid GIF header followed by 17 MB, with no Content-Length.
const oversized = Buffer.concat([Buffer.from(ONE_PIXEL_GIF_BASE64, "base64"), Buffer.alloc(17 * 1024 * 1024)]);
global.fetch = vi.fn<() => Promise<Response>>(
async () => new Response(oversized, { status: 200 }),
) as unknown as typeof fetch;
const id = await printWithImage(STEP_IMAGE);
await localizeDescriptionImages(id);
expect(await prisma.descriptionImage.count({ where: { printId: id } })).toBe(0);
});
});
});
backend/tests/descriptionMarkdown.test.ts
+80
View File
@@ -0,0 +1,80 @@
import { describe, expect, it } from "vitest";
import { cleanThingiverseMarkdown, htmlToMarkdown } from "../src/services/descriptionMarkdown";
describe("htmlToMarkdown", () => {
it("keeps headings, emphasis, lists and step images from a MakerWorld summary", () => {
const html =
"<h2>Excavator Lamp</h2><boostme><boosttitle>Boost Me</boosttitle><boostcontent>A boost will really support my hobby!</boostcontent></boostme>" +
"<ul><li><strong>No supports needed.</strong></li></ul>" +
'<h3>STEP 1:</h3><figure class="image image_resized" style="width: 81.93%"><img src="https://makerworld.bblmw.com/a.jpg"></figure>' +
'<p>&nbsp;</p><figure class="image"><img src="https://makerworld.bblmw.com/b.gif"></figure>';
expect(htmlToMarkdown(html)).toBe(
[
"## Excavator Lamp",
"",
"- **No supports needed.**",
"",
"### STEP 1:",
"",
"![](https://makerworld.bblmw.com/a.jpg)",
"",
"![](https://makerworld.bblmw.com/b.gif)",
].join("\n"),
);
});
it("keeps links and drops images that aren't absolute http(s) URLs", () => {
const html =
'<p>See <a href="https://example.com/guide">the guide</a>.</p><img src="data:image/png;base64,AAAA"><img src="//media.printables.com/x.jpg" alt="Side view">';
expect(htmlToMarkdown(html)).toBe(
"See [the guide](https://example.com/guide).\n\n![Side view](https://media.printables.com/x.jpg)",
);
});
it("writes compact list markers, numbering ordered lists from their start", () => {
expect(htmlToMarkdown('<ol start="3"><li>Print</li><li>Glue<ul><li>Let it dry</li></ul></li></ol>')).toBe(
"3. Print\n4. Glue\n - Let it dry",
);
});
it("turns a data table without a header row into a Markdown table", () => {
const html =
"<table><colgroup><col></colgroup><tbody>" +
"<tr><td><p><strong>Plate #</strong></p></td><td><p><strong>Part</strong></p></td></tr>" +
"<tr><td><p>1</p></td><td><p>Main Hexa</p><p>5 magnets | strong</p></td></tr>" +
"<tr><td>2</td></tr>" +
"</tbody></table>";
expect(htmlToMarkdown(html)).toBe(
["| **Plate #** | **Part** |", "| --- | --- |", "| 1 | Main Hexa 5 magnets \\| strong |", "| 2 | |"].join("\n"),
);
});
it("unrolls a layout table into its cells' content", () => {
const html =
'<table><tbody><tr><td><p><i>Added covers:</i></p><ul><li><a href="https://makerworld.com/m/1">Lisbon</a></li></ul></td>' +
'<td><img src="https://makerworld.bblmw.com/c.png"></td></tr></tbody></table>';
expect(htmlToMarkdown(html)).toBe(
"*Added covers:*\n\n- [Lisbon](https://makerworld.com/m/1)\n\n![](https://makerworld.bblmw.com/c.png)",
);
});
it("is null for an empty description", () => {
expect(htmlToMarkdown("<p>&nbsp;</p>")).toBeNull();
});
});
describe("cleanThingiverseMarkdown", () => {
it("leaves plain Markdown alone", () => {
expect(cleanThingiverseMarkdown("# Title\n\nA **flexible** dragon.\nSecond line.")).toBe(
"# Title\n\nA **flexible** dragon.\nSecond line.",
);
});
it("maps inline HTML onto Markdown without losing line breaks", () => {
expect(
cleanThingiverseMarkdown(
'Line one<br>Line <b>two</b>\n<img src="https://cdn.thingiverse.com/a.jpg">\n<a href="https://x.com">x</a> &amp; <span>y</span>',
),
).toBe("Line one\nLine **two**\n![](https://cdn.thingiverse.com/a.jpg)\n[x](https://x.com) & y");
});
});
backend/tests/providerImports.test.ts
+2 -2
View File
@@ -291,7 +291,7 @@ describe("importing from Printables", () => {
expect(res.status).toBe(200);
await expectFullyImported(res.body.id, {
title: "Sturdy Wall Hook",
descriptionIncludes: "Holds up to 5 kg",
descriptionIncludes: "Holds up to **5 kg**",
tags: ["Hook", "Wall"],
author: {
id: "printables:31337",
@@ -353,7 +353,7 @@ describe("importing from MakerWorld", () => {
const expected = (overrides: Partial<Expected> = {}): Expected => ({
title: "Classic Benchy",
descriptionIncludes: "classic calibration boat",
descriptionIncludes: "The **classic** calibration boat.",
tags: ["Benchy", "Calibration"],
author: { id: `makerworld:${creator.uid}`, name: creator.name, avatarUrl: creator.avatar },
previewImages: 2,
backend/tests/thingiverseImport.test.ts
+5 -1
View File
@@ -19,6 +19,7 @@ const THING_DETAIL = {
id: Number(THING_ID),
name: "Test Articulated Widget",
description: "A **test** widget for import verification.",
instructions: "1. Print both parts.<br>2. Snap the wheels on.",
tags: [{ name: "Widget" }, { name: "Test Fixture" }],
thumbnail: "https://cdn.thingiverse.com/assets/test/thumb.jpg",
default_image: { url: "https://cdn.thingiverse.com/assets/test/preview.jpg" },
@@ -139,7 +140,10 @@ describe("importPrintFromUrl -- Thingiverse", () => {
expect(result.print.creator).toBe("TestCreator");
expect(result.print.sourceProvider).toBe("thingiverse");
expect(result.print.sourceExternalId).toBe(THING_ID);
expect(result.print.notes).toContain("test");
// The separate instructions section is kept, under its own heading, with its HTML mapped onto Markdown.
expect(result.print.notes).toBe(
"A **test** widget for import verification.\n\n## Instructions\n\n1. Print both parts.\n2. Snap the wheels on.",
);
// Canonical tag casing: only the first character stays uppercase.
expect(result.print.tags.toSorted()).toEqual(["Test fixture", "Widget"]);
expect(result.print.categoryId).toBe(category.id);
frontend/package-lock.json
Generated
+1522 -1
View File
File diff suppressed because it is too large. Load diff
frontend/package.json
+3
View File
@@ -29,7 +29,10 @@
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-i18next": "^15.0.0",
"react-markdown": "^10.1.0",
"react-router-dom": "^6.30.6",
"remark-breaks": "^4.0.0",
"remark-gfm": "^4.0.1",
"three": "^0.160.0"
},
"devDependencies": {
frontend/src/api/prints.ts
+13
View File
@@ -378,6 +378,19 @@ export const printsApi = {
return res.json();
},
/** Overwrites the description with the source's current one, formatting and images included. */
refreshDescription: async (id: string): Promise<Print> => {
const res = await fetch(`${apiBase()}/print/${id}/refresh-description`, {
method: "POST",
headers: authHeaders(),
});
if (res.status === 401) throw new UnauthorizedError();
if (!res.ok) {
throw new Error(await readErrorMessage(res, "Refreshing the description failed"));
}
return (await res.json()).print;
},
delete: async (id: string) => {
const res = await fetch(`${apiBase()}/print/${id}`, { method: "DELETE", headers: authHeaders() });
assertOk(res, "Delete print failed");
frontend/src/components/MarkdownDescription.tsx
+76
View File
@@ -0,0 +1,76 @@
import Markdown, { defaultUrlTransform, type Components } from "react-markdown";
import remarkGfm from "remark-gfm";
import remarkBreaks from "remark-breaks";
import Box from "@mui/material/Box";
import Link from "@mui/material/Link";
import { printsApi } from "../api/prints";
import { dividerBorderColor } from "../theme";
// Stored images are referenced relative to the API, without the auth token.
function urlTransform(url: string, key: string): string {
if (key === "src" && url.startsWith("/description-image/")) return printsApi.fileUrl(url);
return defaultUrlTransform(url);
}
const components: Components = {
a: ({ href, children }) => (
<Link href={href} target="_blank" rel="noopener noreferrer">
{children}
</Link>
),
// No referrer, so a source CDN's hotlink protection doesn't block images that stayed remote.
img: ({ src, alt }) => <img src={src} alt={alt ?? ""} loading="lazy" referrerPolicy="no-referrer" />,
};
/** Renders a model description. Raw HTML in it is not rendered, so it needs no sanitizing. */
export default function MarkdownDescription({ markdown }: { markdown: string }) {
return (
<Box
sx={{
typography: "body2",
color: "text.primary",
overflowWrap: "anywhere",
"& > :first-child": { mt: 0 },
"& > :last-child": { mb: 0 },
"& p, & ul, & ol, & blockquote, & pre, & table": { my: 1.25 },
"& h1, & h2, & h3, & h4, & h5, & h6": {
mt: 2.5,
mb: 1,
lineHeight: 1.3,
fontWeight: 700,
color: (theme) => theme.thingport.headingText,
},
"& h1": { fontSize: "1.35rem" },
"& h2": { fontSize: "1.2rem" },
"& h3": { fontSize: "1.05rem" },
"& h4, & h5, & h6": { fontSize: "0.95rem" },
"& ul, & ol": { pl: 3 },
"& li + li": { mt: 0.5 },
"& img": { display: "block", maxWidth: "100%", height: "auto", borderRadius: "8px", my: 1 },
"& blockquote": {
mx: 0,
pl: 1.5,
borderLeft: "3px solid",
borderColor: dividerBorderColor,
color: "text.secondary",
},
"& code": {
fontFamily: "monospace",
fontSize: "0.85em",
px: 0.5,
borderRadius: "4px",
bgcolor: "action.hover",
},
"& pre": { p: 1.5, borderRadius: "8px", bgcolor: "action.hover", overflowX: "auto" },
"& pre code": { p: 0, bgcolor: "transparent" },
"& hr": { border: 0, borderTop: "1px solid", borderColor: dividerBorderColor, my: 2 },
"& table": { display: "block", overflowX: "auto", borderCollapse: "collapse" },
"& th, & td": { border: "1px solid", borderColor: dividerBorderColor, px: 1, py: 0.5, textAlign: "left" },
}}
>
<Markdown remarkPlugins={[remarkGfm, remarkBreaks]} components={components} urlTransform={urlTransform}>
{markdown}
</Markdown>
</Box>
);
}
frontend/src/i18n/locales/en.json
+10 -1
View File
@@ -1400,6 +1400,11 @@
"fillGapsRemaining": "The source has nothing for: {{remaining}}.",
"fillGapsNothing": "The source has nothing to fetch for these.",
"fillGapsFailed": "Couldn't fetch the missing details. Please try again.",
"refreshDescription": "Refresh description from {{provider}}",
"confirmRefreshDescription": "Replace this model's description with the current one from {{provider}}? Any changes you made to the description will be lost.",
"refreshDescriptionConfirm": "Refresh",
"refreshDescriptionDone": "Description refreshed",
"refreshDescriptionFailed": "Couldn't refresh the description. Please try again.",
"gap": {
"title": "title",
"description": "description",
@@ -1454,7 +1459,11 @@
"confirmDiscard": "Discard all unsaved changes to this model?",
"discardConfirmLabel": "Discard",
"updateSuccess": "Model updated",
"updateFailed": "Failed to update the model. Please try again."
"updateFailed": "Failed to update the model. Please try again.",
"descriptionWrite": "Write",
"descriptionPreview": "Preview",
"descriptionPlaceholder": "Describe the model, print settings, assembly steps...",
"descriptionMarkdownHint": "Markdown supported: **bold**, *italic*, # headings, - lists, [links](https://…) and ![images](https://…)."
},
"author": {
"pageTitle": "Author",
frontend/src/i18n/locales/lt.json
+10 -1
View File
@@ -1403,6 +1403,11 @@
"fillGapsRemaining": "Šaltinyje nėra: {{remaining}}.",
"fillGapsNothing": "Šaltinyje šios informacijos nėra.",
"fillGapsFailed": "Nepavyko gauti trūkstamos informacijos. Bandykite dar kartą.",
"refreshDescription": "Atnaujinti aprašymą iš {{provider}}",
"confirmRefreshDescription": "Pakeisti šio modelio aprašymą dabartiniu aprašymu iš {{provider}}? Jūsų atlikti aprašymo pakeitimai bus prarasti.",
"refreshDescriptionConfirm": "Atnaujinti",
"refreshDescriptionDone": "Aprašymas atnaujintas",
"refreshDescriptionFailed": "Nepavyko atnaujinti aprašymo. Bandykite dar kartą.",
"gap": {
"title": "pavadinimas",
"description": "aprašymas",
@@ -1457,7 +1462,11 @@
"confirmDiscard": "Atmesti visus neišsaugotus šio modelio pakeitimus?",
"discardConfirmLabel": "Atmesti",
"updateSuccess": "Modelis atnaujintas",
"updateFailed": "Nepavyko atnaujinti modelio. Bandykite dar kartą."
"updateFailed": "Nepavyko atnaujinti modelio. Bandykite dar kartą.",
"descriptionWrite": "Rašyti",
"descriptionPreview": "Peržiūra",
"descriptionPlaceholder": "Aprašykite modelį, spausdinimo nustatymus, surinkimo žingsnius...",
"descriptionMarkdownHint": "Palaikomas Markdown: **paryškinta**, *kursyvas*, # antraštės, - sąrašai, [nuorodos](https://…) ir ![paveikslėliai](https://…)."
},
"author": {
"pageTitle": "Autorius",
frontend/src/pages/ModelDetailPage/DescriptionEditor.tsx
+71
View File
@@ -0,0 +1,71 @@
import { useState } from "react";
import { useTranslation } from "react-i18next";
import Box from "@mui/material/Box";
import Tabs from "@mui/material/Tabs";
import Tab from "@mui/material/Tab";
import TextField from "@mui/material/TextField";
import Typography from "@mui/material/Typography";
import MarkdownDescription from "../../components/MarkdownDescription";
import { dividerBorderColor } from "../../theme";
type Props = {
value: string;
onChange: (value: string) => void;
disabled?: boolean;
};
/** The description field: Markdown source, with a tab to preview it as the detail page shows it. */
export default function DescriptionEditor({ value, onChange, disabled }: Props) {
const { t } = useTranslation("models");
const [tab, setTab] = useState<"write" | "preview">("write");
return (
<Box>
<Box sx={{ display: "flex", alignItems: "center", justifyContent: "space-between", gap: 1, flexWrap: "wrap" }}>
<Typography variant="body2" color="text.secondary">
{t("detail.description")}
</Typography>
<Tabs value={tab} onChange={(_e, next) => setTab(next)} sx={{ minHeight: 32 }}>
<Tab value="write" label={t("edit.descriptionWrite")} sx={{ minHeight: 32, py: 0.5 }} />
<Tab value="preview" label={t("edit.descriptionPreview")} sx={{ minHeight: 32, py: 0.5 }} />
</Tabs>
</Box>
{tab === "write" ? (
<TextField
value={value}
onChange={(e) => onChange(e.target.value)}
disabled={disabled}
fullWidth
multiline
minRows={6}
maxRows={20}
placeholder={t("edit.descriptionPlaceholder")}
helperText={t("edit.descriptionMarkdownHint")}
inputProps={{ "aria-label": t("detail.description") }}
sx={{ mt: 1, "& textarea": { fontFamily: "monospace", fontSize: "0.85rem" } }}
/>
) : (
<Box
sx={{
mt: 1,
p: 1.5,
minHeight: 140,
maxHeight: 480,
overflowY: "auto",
border: "1px solid",
borderColor: dividerBorderColor,
borderRadius: "4px",
}}
>
{value.trim() ? (
<MarkdownDescription markdown={value} />
) : (
<Typography variant="body2" color="text.disabled">
{t("detail.noDescription")}
</Typography>
)}
</Box>
)}
</Box>
);
}
frontend/src/pages/ModelDetailPage/EditModelModal.tsx
+4 -7
View File
@@ -35,6 +35,7 @@ import { UnauthorizedError } from "../../api/client";
import { useConfirm } from "../../components/ConfirmProvider";
import { useToast } from "../../components/ToastProvider";
import TagInput from "../../components/TagInput";
import DescriptionEditor from "./DescriptionEditor";
import { translateCategoryDisplay } from "../../utils/translateCategoryDisplay";
import { buildCategoryTree, flattenCategoryTree } from "../../utils/categoryTree";
@@ -487,17 +488,13 @@ export default function EditModelModal({ print, onClose, onUnauthorized, onUpdat
</Stack>
</Box>
<TextField
label={t("models:detail.description")}
<DescriptionEditor
value={notes}
onChange={(e) => {
setNotes(e.target.value);
onChange={(next) => {
setNotes(next);
markDirty();
}}
disabled={saving}
fullWidth
multiline
minRows={3}
/>
<Box>
frontend/src/pages/ModelDetailPage/ModelActionsMenu.tsx
+43 -8
View File
@@ -17,6 +17,7 @@ import PlaylistAddIcon from "@mui/icons-material/PlaylistAdd";
import PlaylistRemoveIcon from "@mui/icons-material/PlaylistRemove";
import OpenInNewIcon from "@mui/icons-material/OpenInNew";
import AutoFixHighIcon from "@mui/icons-material/AutoFixHigh";
import NotesIcon from "@mui/icons-material/Notes";
import LaunchIcon from "@mui/icons-material/Launch";
import type { SxProps, Theme } from "@mui/material/styles";
import { UnauthorizedError } from "../../api/client";
@@ -24,6 +25,7 @@ import { type Print, printsApi } from "../../api/prints";
import type { AuthUser } from "../../api/auth";
import { collectionsApi } from "../../api/collections";
import { useConfirm } from "../../components/ConfirmProvider";
import { useToast } from "../../components/ToastProvider";
import { importProviderInfo } from "../../constants/importProviders";
import { useDownloadPrint } from "./useDownloadPrint";
import { useOpenInSlicer } from "./useOpenInSlicer";
@@ -72,6 +74,8 @@ export default function ModelActionsMenu({
const [removingFromCollection, setRemovingFromCollection] = useState(false);
const [addToCollectionOpen, setAddToCollectionOpen] = useState(false);
const [fillGapsOpen, setFillGapsOpen] = useState(false);
const [refreshingDescription, setRefreshingDescription] = useState(false);
const showToast = useToast();
const sourceGaps = print.source_gaps ?? [];
// Driven by ?edit=<id> so links and the back button open/close it.
const editOpen = searchParams.get("edit") === print.id;
@@ -128,6 +132,28 @@ export default function ModelActionsMenu({
}
};
const handleRefreshDescription = async (providerLabel: string) => {
closeMenu();
const confirmed = await confirmDialog({
message: t("models:detail.confirmRefreshDescription", { provider: providerLabel }),
confirmLabel: t("models:detail.refreshDescriptionConfirm"),
});
if (!confirmed) return;
setRefreshingDescription(true);
try {
onUpdated?.(await printsApi.refreshDescription(print.id));
showToast({ message: t("models:detail.refreshDescriptionDone") });
} catch (err) {
if (err instanceof UnauthorizedError) {
onUnauthorized?.();
return;
}
showToast({ message: err instanceof Error ? err.message : t("models:detail.refreshDescriptionFailed") });
} finally {
setRefreshingDescription(false);
}
};
const handleDelete = async () => {
closeMenu();
const confirmed = await confirmDialog({
@@ -271,6 +297,23 @@ export default function ModelActionsMenu({
<ListItemText>{normalizedMenuLabel(slicerOption.label)}</ListItemText>
</MenuItem>
)}
{providerInfo && print.source_url && (
<MenuItem component="a" href={print.source_url} target="_blank" rel="noopener noreferrer" onClick={closeMenu}>
<ListItemIcon>
<OpenInNewIcon fontSize="small" />
</ListItemIcon>
<ListItemText>{t("models:detail.openInProvider", { provider: providerInfo.label })}</ListItemText>
</MenuItem>
)}
{providerInfo && print.source_url && <Divider />}
{providerInfo && print.source_url && (
<MenuItem disabled={refreshingDescription} onClick={() => void handleRefreshDescription(providerInfo.label)}>
<ListItemIcon>
<NotesIcon fontSize="small" />
</ListItemIcon>
<ListItemText>{t("models:detail.refreshDescription", { provider: providerInfo.label })}</ListItemText>
</MenuItem>
)}
{providerInfo && print.source_url && sourceGaps.length > 0 && (
<MenuItem
onClick={() => {
@@ -284,14 +327,6 @@ export default function ModelActionsMenu({
<ListItemText>{t("models:detail.fillGaps", { provider: providerInfo.label })}</ListItemText>
</MenuItem>
)}
{providerInfo && print.source_url && (
<MenuItem component="a" href={print.source_url} target="_blank" rel="noopener noreferrer" onClick={closeMenu}>
<ListItemIcon>
<OpenInNewIcon fontSize="small" />
</ListItemIcon>
<ListItemText>{t("models:detail.openInProvider", { provider: providerInfo.label })}</ListItemText>
</MenuItem>
)}
</Menu>
{slicerOption && normalizedTargets.length > 1 && (
frontend/src/pages/ModelDetailPage/index.tsx
+8 -6
View File
@@ -28,6 +28,7 @@ import Model3DPreviewModal from "./Model3DPreviewModal";
import ModelActionsMenu from "./ModelActionsMenu";
import FavoriteButton from "./FavoriteButton";
import ModelSidePanel from "./ModelSidePanel";
import MarkdownDescription from "../../components/MarkdownDescription";
type Props = {
theme: ResolvedTheme;
@@ -297,12 +298,13 @@ export default function ModelDetailPage({ theme, onSelectCategory, onUnauthorize
>
{t("models:detail.description")}
</Typography>
<Typography
variant="body2"
sx={{ whiteSpace: "pre-wrap", color: print.notes ? "text.primary" : "text.disabled" }}
>
{print.notes || t("models:detail.noDescription")}
</Typography>
{print.notes?.trim() ? (
<MarkdownDescription markdown={print.notes} />
) : (
<Typography variant="body2" color="text.disabled">
{t("models:detail.noDescription")}
</Typography>
)}
<Typography
variant="subtitle1"