mirror of
https://github.com/TautvydasDerzinskas/Thingport.git
synced 2026-10-18 20:59:55 -05:00
fix: import issues
This commit is contained in:
1 parent
d8c2ef20ff
commit
0d1efdba02
6 files changed
+240
-27
No files matched your search
@@ -68,14 +68,16 @@ router.post(
|
||||
asyncHandler(async (req, res) => {
|
||||
const body = parseBody(zipExtractRequestSchema, req.body);
|
||||
const url = await normalizeImportUrl(body.url);
|
||||
const { tempPath, filename } = await downloadImportToTemp(url, body);
|
||||
const { tempPath, filename, meta } = await downloadImportToTemp(url, body);
|
||||
try {
|
||||
if (path.extname(filename).toLowerCase() !== ".zip") throw new HttpError(415, "Imported file is not a zip");
|
||||
const { prints, failed } = await extractZipEntriesToPrints(tempPath, body.entries, {
|
||||
title: body.title,
|
||||
notes: body.notes,
|
||||
tags: body.tags,
|
||||
title: body.title ?? meta.title,
|
||||
notes: body.notes ?? meta.description,
|
||||
tags: body.tags && body.tags.length ? body.tags : meta.tags,
|
||||
folderId: body.folder_id,
|
||||
creator: meta.creator,
|
||||
previewImageUrl: meta.previewImageUrl,
|
||||
});
|
||||
res.json({ prints: prints.map((p) => toPrintOut(p, p.plates, [], null)), failed });
|
||||
} finally {
|
||||
|
||||
@@ -443,6 +443,112 @@ function makerworldDesignIdFromNextData(data: unknown): string | null {
|
||||
return designId ? String(designId) : null;
|
||||
}
|
||||
|
||||
function makerworldTitleFromNextData(data: unknown): string | null {
|
||||
const title = getPath(data, "props", "pageProps", "design", "title");
|
||||
return typeof title === "string" && title.trim() ? title.trim() : null;
|
||||
}
|
||||
|
||||
function makerworldTagsFromNextData(data: unknown): string[] {
|
||||
const tags = getPath(data, "props", "pageProps", "design", "tags");
|
||||
if (!Array.isArray(tags)) return [];
|
||||
return tags.filter((tag): tag is string => typeof tag === "string" && tag.trim().length > 0).map((tag) => tag.trim());
|
||||
}
|
||||
|
||||
function makerworldCreatorFromNextData(data: unknown): string | null {
|
||||
const creator = getPath(data, "props", "pageProps", "design", "designCreator") as Record<string, unknown> | undefined;
|
||||
if (!creator) return null;
|
||||
for (const key of ["nickName", "name", "handle"]) {
|
||||
const value = creator[key];
|
||||
if (typeof value === "string" && value.trim()) return value.trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function makerworldCoverUrlFromNextData(data: unknown): string | null {
|
||||
for (const key of ["coverUrl", "coverPortrait", "coverLandscape"]) {
|
||||
const value = getPath(data, "props", "pageProps", "design", key);
|
||||
if (typeof value === "string" && value.trim()) return value.trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function decodeHtmlEntities(value: string): string {
|
||||
return value
|
||||
.replace(/ /g, " ")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, "&");
|
||||
}
|
||||
|
||||
/** MakerWorld's design summary is a small HTML fragment (paragraphs, links, embedded figures).
|
||||
* Converts it to plain text for Print.notes: turns block-level closing tags into line breaks,
|
||||
* strips every remaining tag, decodes entities, and collapses the resulting whitespace. */
|
||||
function htmlToPlainText(html: string): string | null {
|
||||
const withBreaks = html
|
||||
.replace(/<\s*(br|\/p|\/li|\/div|\/h[1-6])\s*\/?>/gi, "\n")
|
||||
.replace(/<[^>]+>/g, "");
|
||||
const text = decodeHtmlEntities(withBreaks)
|
||||
.replace(/\u00A0/g, " ")
|
||||
.replace(/[ \t]+/g, " ")
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.join("\n")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim();
|
||||
return text || null;
|
||||
}
|
||||
|
||||
function makerworldDescriptionFromNextData(data: unknown): string | null {
|
||||
const summary = getPath(data, "props", "pageProps", "design", "summary");
|
||||
if (typeof summary !== "string" || !summary.trim()) return null;
|
||||
return htmlToPlainText(summary);
|
||||
}
|
||||
|
||||
function genericTitleFromHtml(html: string): string | null {
|
||||
const ogMatch =
|
||||
html.match(/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']*)["']/i) ||
|
||||
html.match(/<meta[^>]+content=["']([^"']*)["'][^>]+property=["']og:title["']/i);
|
||||
if (ogMatch && ogMatch[1].trim()) return decodeHtmlEntities(ogMatch[1].trim());
|
||||
const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
|
||||
if (titleMatch && titleMatch[1].trim()) return decodeHtmlEntities(titleMatch[1].trim());
|
||||
return null;
|
||||
}
|
||||
|
||||
export type ImportedPageMetadata = {
|
||||
title: string | null;
|
||||
tags: string[];
|
||||
description: string | null;
|
||||
creator: string | null;
|
||||
previewImageUrl: string | null;
|
||||
};
|
||||
|
||||
export function emptyImportedPageMetadata(): ImportedPageMetadata {
|
||||
return { title: null, tags: [], description: null, creator: null, previewImageUrl: null };
|
||||
}
|
||||
|
||||
/** Best-effort metadata for a landing page, used to fill in the Print when the caller didn't
|
||||
* supply a field explicitly. MakerWorld exposes title/tags/summary/creator/cover image directly
|
||||
* on the design object in its NEXT_DATA blob; everywhere else only title is filled in, via the
|
||||
* page's og:title/<title> (still far more useful than the internal filename of whatever the
|
||||
* page links to). */
|
||||
export function extractPageMetadata(html: string, pageHost: string): ImportedPageMetadata {
|
||||
const meta = emptyImportedPageMetadata();
|
||||
if (pageHost.endsWith("makerworld.com")) {
|
||||
const nextData = extractNextDataJson(html);
|
||||
if (nextData) {
|
||||
meta.title = makerworldTitleFromNextData(nextData);
|
||||
meta.tags = makerworldTagsFromNextData(nextData);
|
||||
meta.description = makerworldDescriptionFromNextData(nextData);
|
||||
meta.creator = makerworldCreatorFromNextData(nextData);
|
||||
meta.previewImageUrl = makerworldCoverUrlFromNextData(nextData);
|
||||
}
|
||||
}
|
||||
if (!meta.title) meta.title = genericTitleFromHtml(html);
|
||||
return meta;
|
||||
}
|
||||
|
||||
function makerworldInstanceIdFromNextData(data: unknown): string | null {
|
||||
const design = getPath(data, "props", "pageProps", "design") as Record<string, unknown> | undefined;
|
||||
if (!design) return null;
|
||||
|
||||
@@ -13,6 +13,8 @@ import { HttpError, buildImportFilename, isHtmlContentType, mimeFromContentType
|
||||
import { validateRemoteUrl } from "../utils/urlUtils";
|
||||
import { fetchViaFlaresolverr, isFlaresolverrEnabled, looksLikeCloudflareBlock, shouldProxyHost } from "./flaresolverr";
|
||||
import {
|
||||
emptyImportedPageMetadata,
|
||||
extractPageMetadata,
|
||||
findDownloadUrl,
|
||||
isPrintablesPageHost,
|
||||
isThingiversePageHost,
|
||||
@@ -25,8 +27,10 @@ import {
|
||||
resolveThingiverseDownloadUrl,
|
||||
thingiverseHtmlHeaders,
|
||||
type ImportCookies,
|
||||
type ImportedPageMetadata,
|
||||
} from "./importResolvers";
|
||||
import { createPrint, type PrintMetaInput } from "./printCreation";
|
||||
import { plateThumbExists, saveThumbFromBytes } from "./printService";
|
||||
import type { Plate, Print } from "@prisma/client";
|
||||
|
||||
export type ImportRequestBody = ImportCookies & {
|
||||
@@ -119,18 +123,22 @@ async function fetchWithGuard(url: string, headers: Record<string, string>): Pro
|
||||
}
|
||||
}
|
||||
|
||||
export type OpenImportResult = { response: Response; finalUrl: string };
|
||||
export type OpenImportResult = { response: Response; finalUrl: string; meta: ImportedPageMetadata };
|
||||
|
||||
/**
|
||||
* Fetches `url`, following HTML "landing pages" recursively (MakerWorld/Printables/Thingiverse
|
||||
* page scraping, or generic <a href>/JSON link sniffing) until it lands on the actual model file
|
||||
* response. Mirrors MakersVault's open_import_response.
|
||||
* response. Mirrors MakersVault's open_import_response. `inheritedMeta` carries the landing
|
||||
* page's real metadata (MakerWorld's design title/tags/summary/creator/cover image) down through
|
||||
* the recursive follow so the final file response can still report it even though the file host
|
||||
* itself has none of its own.
|
||||
*/
|
||||
export async function openImportResponse(
|
||||
url: string,
|
||||
body: ImportRequestBody,
|
||||
referer?: string | null,
|
||||
depth = 0,
|
||||
inheritedMeta: ImportedPageMetadata = emptyImportedPageMetadata(),
|
||||
): Promise<OpenImportResult> {
|
||||
if (depth > 3) throw new HttpError(400, "Too many redirects while resolving download link");
|
||||
const validatedUrl = await validateRemoteUrl(url);
|
||||
@@ -171,6 +179,14 @@ export async function openImportResponse(
|
||||
} catch {
|
||||
pageHost = "";
|
||||
}
|
||||
const extracted = extractPageMetadata(html, pageHost);
|
||||
const resolvedMeta: ImportedPageMetadata = {
|
||||
title: extracted.title ?? inheritedMeta.title,
|
||||
tags: extracted.tags.length ? extracted.tags : inheritedMeta.tags,
|
||||
description: extracted.description ?? inheritedMeta.description,
|
||||
creator: extracted.creator ?? inheritedMeta.creator,
|
||||
previewImageUrl: extracted.previewImageUrl ?? inheritedMeta.previewImageUrl,
|
||||
};
|
||||
if (pageHost.endsWith("makerworld.com")) {
|
||||
if (!makerworldCookie) makerworldCookie = resolveMakerworldCookie(body);
|
||||
downloadUrl = await resolveMakerworldDownloadUrl(html, finalUrl, makerworldCookie);
|
||||
@@ -194,10 +210,10 @@ export async function openImportResponse(
|
||||
}
|
||||
throw new HttpError(400, "No downloadable model file found. Use a direct download link.");
|
||||
}
|
||||
return openImportResponse(downloadUrl, body, finalUrl, depth + 1);
|
||||
return openImportResponse(downloadUrl, body, finalUrl, depth + 1, resolvedMeta);
|
||||
}
|
||||
|
||||
return { response: res, finalUrl };
|
||||
return { response: res, finalUrl, meta: inheritedMeta };
|
||||
}
|
||||
|
||||
async function streamToFileCapped(response: Response, destPath: string, maxBytes: number): Promise<number> {
|
||||
@@ -230,8 +246,8 @@ async function streamToFileCapped(response: Response, destPath: string, maxBytes
|
||||
export async function downloadImportToTemp(
|
||||
url: string,
|
||||
body: ImportRequestBody,
|
||||
): Promise<{ tempPath: string; filename: string; mime: string }> {
|
||||
const { response, finalUrl } = await openImportResponse(url, body);
|
||||
): Promise<{ tempPath: string; filename: string; mime: string; meta: ImportedPageMetadata }> {
|
||||
const { response, finalUrl, meta } = await openImportResponse(url, body);
|
||||
const contentLength = response.headers.get("content-length");
|
||||
if (contentLength && Number(contentLength) > IMPORT_MAX_BYTES) {
|
||||
await response.body?.cancel().catch(() => undefined);
|
||||
@@ -247,21 +263,51 @@ export async function downloadImportToTemp(
|
||||
await fs.rm(tempPath, { force: true }).catch(() => undefined);
|
||||
throw err;
|
||||
}
|
||||
return { tempPath, filename, mime };
|
||||
return { tempPath, filename, mime, meta };
|
||||
}
|
||||
|
||||
/** Inspects a link without downloading the file body: returns the filename/mime/is_zip it
|
||||
/** Inspects a link without downloading the file body: returns the filename/mime/is_zip/title it
|
||||
* would resolve to. */
|
||||
export async function inspectImportLink(
|
||||
url: string,
|
||||
body: ImportRequestBody,
|
||||
): Promise<{ filename: string; mime: string; is_zip: boolean }> {
|
||||
const { response, finalUrl } = await openImportResponse(url, body);
|
||||
): Promise<{ filename: string; mime: string; is_zip: boolean; title: string | null }> {
|
||||
const { response, finalUrl, meta } = await openImportResponse(url, body);
|
||||
await response.body?.cancel().catch(() => undefined);
|
||||
const filename = buildImportFilename(finalUrl, response.headers, body.filename);
|
||||
const mime = mimeFromContentType(response.headers.get("content-type"), filename);
|
||||
const isZip = path.extname(filename).toLowerCase() === ".zip";
|
||||
return { filename, mime, is_zip: isZip };
|
||||
return { filename, mime, is_zip: isZip, title: meta.title };
|
||||
}
|
||||
|
||||
const COVER_IMAGE_MAX_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
/** Best-effort: downloads a resolved page's cover/preview image and uses it as the plate's
|
||||
* thumbnail, but only when nothing better (e.g. an embedded .3mf thumbnail extracted from the
|
||||
* file itself during createPrint) was already set. Never throws -- a broken cover image
|
||||
* shouldn't fail the import. */
|
||||
export async function applyCoverThumbnailIfMissing(
|
||||
plateId: string | undefined,
|
||||
imageUrl: string | null | undefined,
|
||||
): Promise<void> {
|
||||
if (!plateId || !imageUrl || plateThumbExists(plateId)) return;
|
||||
try {
|
||||
const res = await rawFetch(imageUrl, { "User-Agent": IMPORT_USER_AGENT, Accept: "image/*" });
|
||||
if (!res.ok) {
|
||||
await res.body?.cancel().catch(() => undefined);
|
||||
return;
|
||||
}
|
||||
const contentLength = res.headers.get("content-length");
|
||||
if (contentLength && Number(contentLength) > COVER_IMAGE_MAX_BYTES) {
|
||||
await res.body?.cancel().catch(() => undefined);
|
||||
return;
|
||||
}
|
||||
const buf = Buffer.from(await res.arrayBuffer());
|
||||
if (buf.length > COVER_IMAGE_MAX_BYTES) return;
|
||||
await saveThumbFromBytes(plateId, buf);
|
||||
} catch {
|
||||
// best-effort only
|
||||
}
|
||||
}
|
||||
|
||||
/** Downloads a URL and creates a single-plate Print from it (POST /import). */
|
||||
@@ -269,15 +315,18 @@ export async function importPrintFromUrl(
|
||||
url: string,
|
||||
body: ImportRequestBody,
|
||||
): Promise<{ print: Print; plates: Plate[] }> {
|
||||
const { tempPath, filename, mime } = await downloadImportToTemp(url, body);
|
||||
const meta: PrintMetaInput = {
|
||||
title: body.title ?? null,
|
||||
notes: body.notes ?? null,
|
||||
tags: body.tags ?? [],
|
||||
const { tempPath, filename, mime, meta } = await downloadImportToTemp(url, body);
|
||||
const printMeta: PrintMetaInput = {
|
||||
title: body.title ?? meta.title ?? null,
|
||||
notes: body.notes ?? meta.description ?? null,
|
||||
tags: body.tags && body.tags.length ? body.tags : meta.tags,
|
||||
folderId: body.folder_id ?? null,
|
||||
creator: meta.creator ?? null,
|
||||
};
|
||||
try {
|
||||
return await createPrint(meta, path.parse(filename).name, [{ filename, mime, tempFilePath: tempPath }]);
|
||||
const result = await createPrint(printMeta, path.parse(filename).name, [{ filename, mime, tempFilePath: tempPath }]);
|
||||
await applyCoverThumbnailIfMissing(result.plates[0]?.id, meta.previewImageUrl);
|
||||
return result;
|
||||
} finally {
|
||||
if (fsSync.existsSync(tempPath)) await fs.rm(tempPath, { force: true }).catch(() => undefined);
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import { HttpError, sanitizeFilename, guessMimeFromPath } from "../utils/fileUti
|
||||
import { IMPORT_MAX_BYTES } from "../config";
|
||||
import { listZipEntries as listRawZipEntries, readZipEntry } from "../utils/zipReader";
|
||||
import { validateParentFolder } from "./folderService";
|
||||
import { applyCoverThumbnailIfMissing } from "./importService";
|
||||
import { createPrint, type PrintMetaInput } from "./printCreation";
|
||||
import type { Print, Plate } from "@prisma/client";
|
||||
|
||||
@@ -89,6 +90,8 @@ export type ZipExtractOptions = {
|
||||
notes?: string | null;
|
||||
tags?: string[];
|
||||
folderId?: string | null;
|
||||
creator?: string | null;
|
||||
previewImageUrl?: string | null;
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -144,17 +147,25 @@ export async function extractZipEntriesToPrints(
|
||||
tempPath = path.join(os.tmpdir(), `printstash-zip-${crypto.randomBytes(8).toString("hex")}`);
|
||||
await fs.writeFile(tempPath, buffer);
|
||||
|
||||
// A resolved page title (e.g. MakerWorld's design name) is the same for every entry in
|
||||
// the zip, so when more than one entry is being extracted into its own Print, keep them
|
||||
// distinguishable by tagging the entry's own filename onto it instead of overwriting it
|
||||
// outright the way a single-entry extraction does.
|
||||
const resolvedTitle =
|
||||
options.title && ordered.length > 1 ? `${options.title} (${path.parse(filename).name})` : options.title;
|
||||
const meta: PrintMetaInput = {
|
||||
title: options.title ?? null,
|
||||
title: resolvedTitle ?? null,
|
||||
notes: options.notes ?? null,
|
||||
tags: options.tags ?? [],
|
||||
folderId: targetFolderId,
|
||||
creator: options.creator ?? null,
|
||||
};
|
||||
const mime = guessMimeFromPath(filename);
|
||||
const { print, plates } = await createPrint(meta, path.parse(filename).name, [
|
||||
{ filename, mime, tempFilePath: tempPath },
|
||||
]);
|
||||
tempPath = null;
|
||||
await applyCoverThumbnailIfMissing(plates[0]?.id, options.previewImageUrl);
|
||||
prints.push({ ...print, plates });
|
||||
} catch {
|
||||
if (tempPath) await fs.rm(tempPath, { force: true }).catch(() => undefined);
|
||||
|
||||
+6
-2
@@ -5,8 +5,12 @@ server {
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
|
||||
# Proxy all /api/* requests to the backend container
|
||||
location /api/ {
|
||||
# Proxy all /api/* requests to the backend container.
|
||||
# ^~ forces this prefix match to win over the regex location below -- without it, any
|
||||
# backend route ending in one of those extensions (e.g. /api/plate/:id/thumb.jpg, or a
|
||||
# plate file download named *.png/*.svg) gets shadowed by the static-asset block and
|
||||
# 404s straight out of nginx instead of ever reaching the backend.
|
||||
location ^~ /api/ {
|
||||
proxy_pass http://backend:8000;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import Box from "@mui/material/Box";
|
||||
import Stack from "@mui/material/Stack";
|
||||
import Typography from "@mui/material/Typography";
|
||||
import Button from "@mui/material/Button";
|
||||
import { useTranslation } from "react-i18next";
|
||||
@@ -16,12 +17,34 @@ type ModelSnapshotProps = {
|
||||
mode?: "automatic" | "on-demand";
|
||||
};
|
||||
|
||||
// Every ModelSnapshot on the page shares one job queue (see queueSnapshotJob), so a single model
|
||||
// that hangs mid-load (bad network response, pathological geometry) would otherwise wedge every
|
||||
// other card's preview behind it forever. This bounds each job so the queue always keeps moving.
|
||||
const SNAPSHOT_TIMEOUT_MS = 20000;
|
||||
|
||||
function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error("Snapshot generation timed out")), ms);
|
||||
promise.then(
|
||||
value => {
|
||||
clearTimeout(timer);
|
||||
resolve(value);
|
||||
},
|
||||
err => {
|
||||
clearTimeout(timer);
|
||||
reject(err);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export function ModelSnapshot({ url, ext, plateId, mode = "automatic" }: ModelSnapshotProps) {
|
||||
const { t } = useTranslation(["library"]);
|
||||
const { t } = useTranslation(["library", "common"]);
|
||||
const containerRef = useRef<HTMLDivElement | null>(null);
|
||||
const [snapshot, setSnapshot] = useState<string | null>(null);
|
||||
const [state, setState] = useState<SnapshotState>("idle");
|
||||
const [requested, setRequested] = useState(mode === "automatic");
|
||||
const [retryToken, setRetryToken] = useState(0);
|
||||
|
||||
useEffect(() => {
|
||||
if (mode === "automatic") setRequested(true);
|
||||
@@ -44,7 +67,7 @@ export function ModelSnapshot({ url, ext, plateId, mode = "automatic" }: ModelSn
|
||||
const height = Math.max(120, Math.floor(rect.height || 180));
|
||||
const image = await queueSnapshotJob(async () => {
|
||||
if (disposed) return null;
|
||||
return generateModelSnapshot(url, ext, width, height, "light");
|
||||
return withTimeout(generateModelSnapshot(url, ext, width, height, "light"), SNAPSHOT_TIMEOUT_MS);
|
||||
});
|
||||
if (disposed || !image) return;
|
||||
snapshotCache.set(cacheKey, image);
|
||||
@@ -83,7 +106,7 @@ export function ModelSnapshot({ url, ext, plateId, mode = "automatic" }: ModelSn
|
||||
disposed = true;
|
||||
observer?.disconnect();
|
||||
};
|
||||
}, [url, ext, plateId, mode, requested]);
|
||||
}, [url, ext, plateId, mode, requested, retryToken]);
|
||||
|
||||
return (
|
||||
<Box
|
||||
@@ -110,6 +133,24 @@ export function ModelSnapshot({ url, ext, plateId, mode = "automatic" }: ModelSn
|
||||
<Typography variant="caption" color="text.secondary">
|
||||
{t("library:modelViewer.generatingPreview")}
|
||||
</Typography>
|
||||
) : state === "error" ? (
|
||||
<Stack alignItems="center" spacing={0.5}>
|
||||
<Typography variant="caption" color="error.main">
|
||||
{t("library:modelViewer.previewFailed")}
|
||||
</Typography>
|
||||
<Button
|
||||
type="button"
|
||||
size="small"
|
||||
variant="outlined"
|
||||
onClick={event => {
|
||||
event.stopPropagation();
|
||||
setState("idle");
|
||||
setRetryToken(v => v + 1);
|
||||
}}
|
||||
>
|
||||
{t("common:retry")}
|
||||
</Button>
|
||||
</Stack>
|
||||
) : mode === "on-demand" && !requested ? (
|
||||
<Button
|
||||
type="button"
|
||||
|
||||
Reference in new issue
Block a user