fix: import issues

This commit is contained in:
Tautvydas Derzinskas committed 2026-09-07 16:27:02 +02:00
1 parent d8c2ef20ff
commit 0d1efdba02
6 files changed
+240 -27

No files matched your search

+6 -4
View File
@@ -68,14 +68,16 @@ router.post(
asyncHandler(async (req, res) => {
const body = parseBody(zipExtractRequestSchema, req.body);
const url = await normalizeImportUrl(body.url);
const { tempPath, filename } = await downloadImportToTemp(url, body);
const { tempPath, filename, meta } = await downloadImportToTemp(url, body);
try {
if (path.extname(filename).toLowerCase() !== ".zip") throw new HttpError(415, "Imported file is not a zip");
const { prints, failed } = await extractZipEntriesToPrints(tempPath, body.entries, {
title: body.title,
notes: body.notes,
tags: body.tags,
title: body.title ?? meta.title,
notes: body.notes ?? meta.description,
tags: body.tags && body.tags.length ? body.tags : meta.tags,
folderId: body.folder_id,
creator: meta.creator,
previewImageUrl: meta.previewImageUrl,
});
res.json({ prints: prints.map((p) => toPrintOut(p, p.plates, [], null)), failed });
} finally {
+106
View File
@@ -443,6 +443,112 @@ function makerworldDesignIdFromNextData(data: unknown): string | null {
return designId ? String(designId) : null;
}
function makerworldTitleFromNextData(data: unknown): string | null {
const title = getPath(data, "props", "pageProps", "design", "title");
return typeof title === "string" && title.trim() ? title.trim() : null;
}
function makerworldTagsFromNextData(data: unknown): string[] {
const tags = getPath(data, "props", "pageProps", "design", "tags");
if (!Array.isArray(tags)) return [];
return tags.filter((tag): tag is string => typeof tag === "string" && tag.trim().length > 0).map((tag) => tag.trim());
}
function makerworldCreatorFromNextData(data: unknown): string | null {
const creator = getPath(data, "props", "pageProps", "design", "designCreator") as Record<string, unknown> | undefined;
if (!creator) return null;
for (const key of ["nickName", "name", "handle"]) {
const value = creator[key];
if (typeof value === "string" && value.trim()) return value.trim();
}
return null;
}
function makerworldCoverUrlFromNextData(data: unknown): string | null {
for (const key of ["coverUrl", "coverPortrait", "coverLandscape"]) {
const value = getPath(data, "props", "pageProps", "design", key);
if (typeof value === "string" && value.trim()) return value.trim();
}
return null;
}
function decodeHtmlEntities(value: string): string {
return value
.replace(/&nbsp;/g, " ")
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&amp;/g, "&");
}
/** MakerWorld's design summary is a small HTML fragment (paragraphs, links, embedded figures).
* Converts it to plain text for Print.notes: turns block-level closing tags into line breaks,
* strips every remaining tag, decodes entities, and collapses the resulting whitespace. */
function htmlToPlainText(html: string): string | null {
const withBreaks = html
.replace(/<\s*(br|\/p|\/li|\/div|\/h[1-6])\s*\/?>/gi, "\n")
.replace(/<[^>]+>/g, "");
const text = decodeHtmlEntities(withBreaks)
.replace(/\u00A0/g, " ")
.replace(/[ \t]+/g, " ")
.split("\n")
.map((line) => line.trim())
.join("\n")
.replace(/\n{3,}/g, "\n\n")
.trim();
return text || null;
}
function makerworldDescriptionFromNextData(data: unknown): string | null {
const summary = getPath(data, "props", "pageProps", "design", "summary");
if (typeof summary !== "string" || !summary.trim()) return null;
return htmlToPlainText(summary);
}
function genericTitleFromHtml(html: string): string | null {
const ogMatch =
html.match(/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']*)["']/i) ||
html.match(/<meta[^>]+content=["']([^"']*)["'][^>]+property=["']og:title["']/i);
if (ogMatch && ogMatch[1].trim()) return decodeHtmlEntities(ogMatch[1].trim());
const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
if (titleMatch && titleMatch[1].trim()) return decodeHtmlEntities(titleMatch[1].trim());
return null;
}
export type ImportedPageMetadata = {
title: string | null;
tags: string[];
description: string | null;
creator: string | null;
previewImageUrl: string | null;
};
export function emptyImportedPageMetadata(): ImportedPageMetadata {
return { title: null, tags: [], description: null, creator: null, previewImageUrl: null };
}
/** Best-effort metadata for a landing page, used to fill in the Print when the caller didn't
* supply a field explicitly. MakerWorld exposes title/tags/summary/creator/cover image directly
* on the design object in its NEXT_DATA blob; everywhere else only title is filled in, via the
* page's og:title/<title> (still far more useful than the internal filename of whatever the
* page links to). */
export function extractPageMetadata(html: string, pageHost: string): ImportedPageMetadata {
const meta = emptyImportedPageMetadata();
if (pageHost.endsWith("makerworld.com")) {
const nextData = extractNextDataJson(html);
if (nextData) {
meta.title = makerworldTitleFromNextData(nextData);
meta.tags = makerworldTagsFromNextData(nextData);
meta.description = makerworldDescriptionFromNextData(nextData);
meta.creator = makerworldCreatorFromNextData(nextData);
meta.previewImageUrl = makerworldCoverUrlFromNextData(nextData);
}
}
if (!meta.title) meta.title = genericTitleFromHtml(html);
return meta;
}
function makerworldInstanceIdFromNextData(data: unknown): string | null {
const design = getPath(data, "props", "pageProps", "design") as Record<string, unknown> | undefined;
if (!design) return null;
+66 -17
View File
@@ -13,6 +13,8 @@ import { HttpError, buildImportFilename, isHtmlContentType, mimeFromContentType
import { validateRemoteUrl } from "../utils/urlUtils";
import { fetchViaFlaresolverr, isFlaresolverrEnabled, looksLikeCloudflareBlock, shouldProxyHost } from "./flaresolverr";
import {
emptyImportedPageMetadata,
extractPageMetadata,
findDownloadUrl,
isPrintablesPageHost,
isThingiversePageHost,
@@ -25,8 +27,10 @@ import {
resolveThingiverseDownloadUrl,
thingiverseHtmlHeaders,
type ImportCookies,
type ImportedPageMetadata,
} from "./importResolvers";
import { createPrint, type PrintMetaInput } from "./printCreation";
import { plateThumbExists, saveThumbFromBytes } from "./printService";
import type { Plate, Print } from "@prisma/client";
export type ImportRequestBody = ImportCookies & {
@@ -119,18 +123,22 @@ async function fetchWithGuard(url: string, headers: Record<string, string>): Pro
}
}
export type OpenImportResult = { response: Response; finalUrl: string };
export type OpenImportResult = { response: Response; finalUrl: string; meta: ImportedPageMetadata };
/**
* Fetches `url`, following HTML "landing pages" recursively (MakerWorld/Printables/Thingiverse
* page scraping, or generic <a href>/JSON link sniffing) until it lands on the actual model file
* response. Mirrors MakersVault's open_import_response.
* response. Mirrors MakersVault's open_import_response. `inheritedMeta` carries the landing
* page's real metadata (MakerWorld's design title/tags/summary/creator/cover image) down through
* the recursive follow so the final file response can still report it even though the file host
* itself has none of its own.
*/
export async function openImportResponse(
url: string,
body: ImportRequestBody,
referer?: string | null,
depth = 0,
inheritedMeta: ImportedPageMetadata = emptyImportedPageMetadata(),
): Promise<OpenImportResult> {
if (depth > 3) throw new HttpError(400, "Too many redirects while resolving download link");
const validatedUrl = await validateRemoteUrl(url);
@@ -171,6 +179,14 @@ export async function openImportResponse(
} catch {
pageHost = "";
}
const extracted = extractPageMetadata(html, pageHost);
const resolvedMeta: ImportedPageMetadata = {
title: extracted.title ?? inheritedMeta.title,
tags: extracted.tags.length ? extracted.tags : inheritedMeta.tags,
description: extracted.description ?? inheritedMeta.description,
creator: extracted.creator ?? inheritedMeta.creator,
previewImageUrl: extracted.previewImageUrl ?? inheritedMeta.previewImageUrl,
};
if (pageHost.endsWith("makerworld.com")) {
if (!makerworldCookie) makerworldCookie = resolveMakerworldCookie(body);
downloadUrl = await resolveMakerworldDownloadUrl(html, finalUrl, makerworldCookie);
@@ -194,10 +210,10 @@ export async function openImportResponse(
}
throw new HttpError(400, "No downloadable model file found. Use a direct download link.");
}
return openImportResponse(downloadUrl, body, finalUrl, depth + 1);
return openImportResponse(downloadUrl, body, finalUrl, depth + 1, resolvedMeta);
}
return { response: res, finalUrl };
return { response: res, finalUrl, meta: inheritedMeta };
}
async function streamToFileCapped(response: Response, destPath: string, maxBytes: number): Promise<number> {
@@ -230,8 +246,8 @@ async function streamToFileCapped(response: Response, destPath: string, maxBytes
export async function downloadImportToTemp(
url: string,
body: ImportRequestBody,
): Promise<{ tempPath: string; filename: string; mime: string }> {
const { response, finalUrl } = await openImportResponse(url, body);
): Promise<{ tempPath: string; filename: string; mime: string; meta: ImportedPageMetadata }> {
const { response, finalUrl, meta } = await openImportResponse(url, body);
const contentLength = response.headers.get("content-length");
if (contentLength && Number(contentLength) > IMPORT_MAX_BYTES) {
await response.body?.cancel().catch(() => undefined);
@@ -247,21 +263,51 @@ export async function downloadImportToTemp(
await fs.rm(tempPath, { force: true }).catch(() => undefined);
throw err;
}
return { tempPath, filename, mime };
return { tempPath, filename, mime, meta };
}
/** Inspects a link without downloading the file body: returns the filename/mime/is_zip it
/** Inspects a link without downloading the file body: returns the filename/mime/is_zip/title it
* would resolve to. */
export async function inspectImportLink(
url: string,
body: ImportRequestBody,
): Promise<{ filename: string; mime: string; is_zip: boolean }> {
const { response, finalUrl } = await openImportResponse(url, body);
): Promise<{ filename: string; mime: string; is_zip: boolean; title: string | null }> {
const { response, finalUrl, meta } = await openImportResponse(url, body);
await response.body?.cancel().catch(() => undefined);
const filename = buildImportFilename(finalUrl, response.headers, body.filename);
const mime = mimeFromContentType(response.headers.get("content-type"), filename);
const isZip = path.extname(filename).toLowerCase() === ".zip";
return { filename, mime, is_zip: isZip };
return { filename, mime, is_zip: isZip, title: meta.title };
}
const COVER_IMAGE_MAX_BYTES = 16 * 1024 * 1024;
/** Best-effort: downloads a resolved page's cover/preview image and uses it as the plate's
* thumbnail, but only when nothing better (e.g. an embedded .3mf thumbnail extracted from the
* file itself during createPrint) was already set. Never throws -- a broken cover image
* shouldn't fail the import. */
export async function applyCoverThumbnailIfMissing(
plateId: string | undefined,
imageUrl: string | null | undefined,
): Promise<void> {
if (!plateId || !imageUrl || plateThumbExists(plateId)) return;
try {
const res = await rawFetch(imageUrl, { "User-Agent": IMPORT_USER_AGENT, Accept: "image/*" });
if (!res.ok) {
await res.body?.cancel().catch(() => undefined);
return;
}
const contentLength = res.headers.get("content-length");
if (contentLength && Number(contentLength) > COVER_IMAGE_MAX_BYTES) {
await res.body?.cancel().catch(() => undefined);
return;
}
const buf = Buffer.from(await res.arrayBuffer());
if (buf.length > COVER_IMAGE_MAX_BYTES) return;
await saveThumbFromBytes(plateId, buf);
} catch {
// best-effort only
}
}
/** Downloads a URL and creates a single-plate Print from it (POST /import). */
@@ -269,15 +315,18 @@ export async function importPrintFromUrl(
url: string,
body: ImportRequestBody,
): Promise<{ print: Print; plates: Plate[] }> {
const { tempPath, filename, mime } = await downloadImportToTemp(url, body);
const meta: PrintMetaInput = {
title: body.title ?? null,
notes: body.notes ?? null,
tags: body.tags ?? [],
const { tempPath, filename, mime, meta } = await downloadImportToTemp(url, body);
const printMeta: PrintMetaInput = {
title: body.title ?? meta.title ?? null,
notes: body.notes ?? meta.description ?? null,
tags: body.tags && body.tags.length ? body.tags : meta.tags,
folderId: body.folder_id ?? null,
creator: meta.creator ?? null,
};
try {
return await createPrint(meta, path.parse(filename).name, [{ filename, mime, tempFilePath: tempPath }]);
const result = await createPrint(printMeta, path.parse(filename).name, [{ filename, mime, tempFilePath: tempPath }]);
await applyCoverThumbnailIfMissing(result.plates[0]?.id, meta.previewImageUrl);
return result;
} finally {
if (fsSync.existsSync(tempPath)) await fs.rm(tempPath, { force: true }).catch(() => undefined);
}
+12 -1
View File
@@ -7,6 +7,7 @@ import { HttpError, sanitizeFilename, guessMimeFromPath } from "../utils/fileUti
import { IMPORT_MAX_BYTES } from "../config";
import { listZipEntries as listRawZipEntries, readZipEntry } from "../utils/zipReader";
import { validateParentFolder } from "./folderService";
import { applyCoverThumbnailIfMissing } from "./importService";
import { createPrint, type PrintMetaInput } from "./printCreation";
import type { Print, Plate } from "@prisma/client";
@@ -89,6 +90,8 @@ export type ZipExtractOptions = {
notes?: string | null;
tags?: string[];
folderId?: string | null;
creator?: string | null;
previewImageUrl?: string | null;
};
/**
@@ -144,17 +147,25 @@ export async function extractZipEntriesToPrints(
tempPath = path.join(os.tmpdir(), `printstash-zip-${crypto.randomBytes(8).toString("hex")}`);
await fs.writeFile(tempPath, buffer);
// A resolved page title (e.g. MakerWorld's design name) is the same for every entry in
// the zip, so when more than one entry is being extracted into its own Print, keep them
// distinguishable by tagging the entry's own filename onto it instead of overwriting it
// outright the way a single-entry extraction does.
const resolvedTitle =
options.title && ordered.length > 1 ? `${options.title} (${path.parse(filename).name})` : options.title;
const meta: PrintMetaInput = {
title: options.title ?? null,
title: resolvedTitle ?? null,
notes: options.notes ?? null,
tags: options.tags ?? [],
folderId: targetFolderId,
creator: options.creator ?? null,
};
const mime = guessMimeFromPath(filename);
const { print, plates } = await createPrint(meta, path.parse(filename).name, [
{ filename, mime, tempFilePath: tempPath },
]);
tempPath = null;
await applyCoverThumbnailIfMissing(plates[0]?.id, options.previewImageUrl);
prints.push({ ...print, plates });
} catch {
if (tempPath) await fs.rm(tempPath, { force: true }).catch(() => undefined);
+6 -2
View File
@@ -5,8 +5,12 @@ server {
root /usr/share/nginx/html;
index index.html;
# Proxy all /api/* requests to the backend container
location /api/ {
# Proxy all /api/* requests to the backend container.
# ^~ forces this prefix match to win over the regex location below -- without it, any
# backend route ending in one of those extensions (e.g. /api/plate/:id/thumb.jpg, or a
# plate file download named *.png/*.svg) gets shadowed by the static-asset block and
# 404s straight out of nginx instead of ever reaching the backend.
location ^~ /api/ {
proxy_pass http://backend:8000;
proxy_http_version 1.1;
proxy_set_header Host $host;
@@ -1,5 +1,6 @@
import { useEffect, useRef, useState } from "react";
import Box from "@mui/material/Box";
import Stack from "@mui/material/Stack";
import Typography from "@mui/material/Typography";
import Button from "@mui/material/Button";
import { useTranslation } from "react-i18next";
@@ -16,12 +17,34 @@ type ModelSnapshotProps = {
mode?: "automatic" | "on-demand";
};
// Every ModelSnapshot on the page shares one job queue (see queueSnapshotJob), so a single model
// that hangs mid-load (bad network response, pathological geometry) would otherwise wedge every
// other card's preview behind it forever. This bounds each job so the queue always keeps moving.
const SNAPSHOT_TIMEOUT_MS = 20000;
function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T> {
return new Promise<T>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error("Snapshot generation timed out")), ms);
promise.then(
value => {
clearTimeout(timer);
resolve(value);
},
err => {
clearTimeout(timer);
reject(err);
}
);
});
}
export function ModelSnapshot({ url, ext, plateId, mode = "automatic" }: ModelSnapshotProps) {
const { t } = useTranslation(["library"]);
const { t } = useTranslation(["library", "common"]);
const containerRef = useRef<HTMLDivElement | null>(null);
const [snapshot, setSnapshot] = useState<string | null>(null);
const [state, setState] = useState<SnapshotState>("idle");
const [requested, setRequested] = useState(mode === "automatic");
const [retryToken, setRetryToken] = useState(0);
useEffect(() => {
if (mode === "automatic") setRequested(true);
@@ -44,7 +67,7 @@ export function ModelSnapshot({ url, ext, plateId, mode = "automatic" }: ModelSn
const height = Math.max(120, Math.floor(rect.height || 180));
const image = await queueSnapshotJob(async () => {
if (disposed) return null;
return generateModelSnapshot(url, ext, width, height, "light");
return withTimeout(generateModelSnapshot(url, ext, width, height, "light"), SNAPSHOT_TIMEOUT_MS);
});
if (disposed || !image) return;
snapshotCache.set(cacheKey, image);
@@ -83,7 +106,7 @@ export function ModelSnapshot({ url, ext, plateId, mode = "automatic" }: ModelSn
disposed = true;
observer?.disconnect();
};
}, [url, ext, plateId, mode, requested]);
}, [url, ext, plateId, mode, requested, retryToken]);
return (
<Box
@@ -110,6 +133,24 @@ export function ModelSnapshot({ url, ext, plateId, mode = "automatic" }: ModelSn
<Typography variant="caption" color="text.secondary">
{t("library:modelViewer.generatingPreview")}
</Typography>
) : state === "error" ? (
<Stack alignItems="center" spacing={0.5}>
<Typography variant="caption" color="error.main">
{t("library:modelViewer.previewFailed")}
</Typography>
<Button
type="button"
size="small"
variant="outlined"
onClick={event => {
event.stopPropagation();
setState("idle");
setRetryToken(v => v + 1);
}}
>
{t("common:retry")}
</Button>
</Stack>
) : mode === "on-demand" && !requested ? (
<Button
type="button"