mirror of
https://github.com/TautvydasDerzinskas/Thingport.git
synced 2026-10-19 05:10:01 -05:00
fix: include MakerWorld gallery images when importing from the model page
Imports that read the MakerWorld model page (every extension import that resolved its download in the browser, and web imports without a MakerWorld login) only kept the cover image, and lost the site's category ids. Build the metadata from the design object the same way for the page and the API.
This commit is contained in:
Tautvydas Derzinskas committed
1 parent
484efff1c9
commit
e87eb39ab6
3 files changed
+85
-103
No files matched your search
@@ -375,27 +375,80 @@ function makerworldDesignIdFromNextData(data: unknown): string | null {
|
||||
return designId ? String(designId) : null;
|
||||
}
|
||||
|
||||
function makerworldTitleFromNextData(data: unknown): string | null {
|
||||
const title = getPath(data, "props", "pageProps", "design", "title");
|
||||
return typeof title === "string" && title.trim() ? title.trim() : null;
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return !!value && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function makerworldTagsFromNextData(data: unknown): string[] {
|
||||
const tags = getPath(data, "props", "pageProps", "design", "tags");
|
||||
if (!Array.isArray(tags)) return [];
|
||||
return tags.filter((tag): tag is string => typeof tag === "string" && tag.trim().length > 0).map((tag) => tag.trim());
|
||||
}
|
||||
|
||||
function makerworldCreatorFromNextData(data: unknown): string | null {
|
||||
const creator = getPath(data, "props", "pageProps", "design", "designCreator") as Record<string, unknown> | undefined;
|
||||
if (!creator) return null;
|
||||
for (const key of ["nickName", "name", "handle"]) {
|
||||
const value = creator[key];
|
||||
function pickDesignString(source: Record<string, unknown>, keys: string[]): string | null {
|
||||
for (const key of keys) {
|
||||
const value = source[key];
|
||||
if (typeof value === "string" && value.trim()) return value.trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The model page's photo gallery -- design.designExtension.design_pictures -- distinct from
|
||||
* coverUrl/coverPortrait/coverLandscape, which are just crops of the same single cover image
|
||||
* for different UI contexts. Confirmed live: a model can have several of these (renders and/or
|
||||
* isRealLifePhoto: 1 real-world photos of the print). */
|
||||
function makerworldGalleryImages(design: Record<string, unknown>): { url: string; filename: string }[] {
|
||||
const extension = design.designExtension;
|
||||
if (!isRecord(extension)) return [];
|
||||
const pictures = extension.design_pictures;
|
||||
if (!Array.isArray(pictures)) return [];
|
||||
const images: { url: string; filename: string }[] = [];
|
||||
for (const picture of pictures) {
|
||||
if (!isRecord(picture)) continue;
|
||||
const url = typeof picture.url === "string" ? picture.url.trim() : "";
|
||||
if (!url) continue;
|
||||
const filename = typeof picture.name === "string" && picture.name.trim() ? picture.name.trim() : null;
|
||||
images.push({ url, filename: filename ?? url.split("/").pop() ?? "preview.jpg" });
|
||||
}
|
||||
return images;
|
||||
}
|
||||
|
||||
/** design.categories -- a flat array of `{id, name, ...}` objects, most-specific first (e.g.
|
||||
* "Cosplay Weapons" then its parent "Props & Cosplays"). Used to auto-land the import into a
|
||||
* Category whose makerworldCatIds overlaps one of these (see importService.ts's
|
||||
* resolveCategoryIdByCategory); every id is kept, not just the first, so a category configured for
|
||||
* either the specific or the parent category still matches. */
|
||||
function makerworldCategoryIds(design: Record<string, unknown>): number[] {
|
||||
const categories = design.categories;
|
||||
if (!Array.isArray(categories)) return [];
|
||||
const ids: number[] = [];
|
||||
for (const category of categories) {
|
||||
if (!isRecord(category)) continue;
|
||||
const id = category.id;
|
||||
if (typeof id === "number" && Number.isInteger(id)) ids.push(id);
|
||||
else if (typeof id === "string" && /^\d+$/.test(id)) ids.push(Number(id));
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
/** Everything a MakerWorld design object says about the model -- the same shape whichever way
|
||||
* it was got: the api.bambulab.com design API, the model page's __NEXT_DATA__ (fetched by the
|
||||
* backend), or that same page data sent by the Thingport Grab extension. The author is only the
|
||||
* design's short creator summary (see makerworldAuthorFromDesignCreator); the download-specific
|
||||
* fields (filename, makerworldProfile) are left for the caller. */
|
||||
export function makerworldMetaFromDesign(design: unknown): ImportedPageMetadata {
|
||||
const meta = emptyImportedPageMetadata();
|
||||
if (!isRecord(design)) return meta;
|
||||
const title = pickDesignString(design, ["title"]);
|
||||
meta.title = title ? decodeHtmlEntities(title) : null;
|
||||
meta.tags = Array.isArray(design.tags)
|
||||
? design.tags.filter((tag): tag is string => typeof tag === "string" && tag.trim().length > 0).map((tag) => tag.trim())
|
||||
: [];
|
||||
meta.description = typeof design.summary === "string" && design.summary.trim() ? htmlToPlainText(design.summary) : null;
|
||||
const designCreator = isRecord(design.designCreator) ? design.designCreator : null;
|
||||
meta.creator = designCreator ? pickDesignString(designCreator, ["nickName", "name", "handle"]) : null;
|
||||
meta.author = makerworldAuthorFromDesignCreator(designCreator);
|
||||
meta.previewImageUrl = pickDesignString(design, ["coverUrl", "coverPortrait", "coverLandscape"]);
|
||||
meta.galleryImages = makerworldGalleryImages(design);
|
||||
meta.siteCategoryIds = makerworldCategoryIds(design);
|
||||
meta.categorySite = meta.siteCategoryIds.length ? "makerworld" : null;
|
||||
return meta;
|
||||
}
|
||||
|
||||
/** An author record from MakerWorld's own summary of a design's creator (design.designCreator --
|
||||
* the same shape in the model page's __NEXT_DATA__ and in the api.bambulab.com design API). It
|
||||
* has what an author needs to show up linked, with an avatar -- id, name, handle, avatar -- so an
|
||||
@@ -427,14 +480,6 @@ export function makerworldAuthorFromDesignCreator(creator: unknown): ImportedAut
|
||||
};
|
||||
}
|
||||
|
||||
function makerworldCoverUrlFromNextData(data: unknown): string | null {
|
||||
for (const key of ["coverUrl", "coverPortrait", "coverLandscape"]) {
|
||||
const value = getPath(data, "props", "pageProps", "design", key);
|
||||
if (typeof value === "string" && value.trim()) return value.trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function decodeHtmlEntities(value: string): string {
|
||||
return value
|
||||
.replace(/ /g, " ")
|
||||
@@ -463,12 +508,6 @@ export function htmlToPlainText(html: string): string | null {
|
||||
return text || null;
|
||||
}
|
||||
|
||||
function makerworldDescriptionFromNextData(data: unknown): string | null {
|
||||
const summary = getPath(data, "props", "pageProps", "design", "summary");
|
||||
if (typeof summary !== "string" || !summary.trim()) return null;
|
||||
return htmlToPlainText(summary);
|
||||
}
|
||||
|
||||
function genericTitleFromHtml(html: string): string | null {
|
||||
const ogMatch =
|
||||
html.match(/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']*)["']/i) ||
|
||||
@@ -540,24 +579,18 @@ export function emptyImportedPageMetadata(): ImportedPageMetadata {
|
||||
}
|
||||
|
||||
/** Best-effort metadata for a landing page, used to fill in the Print when the caller didn't
|
||||
* supply a field explicitly. MakerWorld exposes title/tags/summary/creator/cover image directly
|
||||
* on the design object in its NEXT_DATA blob; everywhere else only title is filled in, via the
|
||||
* supply a field explicitly. MakerWorld exposes title/tags/summary/creator/cover image/gallery
|
||||
* directly on the design object in its NEXT_DATA blob (see makerworldMetaFromDesign); everywhere else only title is filled in, via the
|
||||
* page's og:title/<title> (still far more useful than the internal filename of whatever the
|
||||
* page links to). */
|
||||
export function extractPageMetadata(html: string, pageHost: string): ImportedPageMetadata {
|
||||
const meta = emptyImportedPageMetadata();
|
||||
if (pageHost.endsWith("makerworld.com")) {
|
||||
const nextData = extractNextDataJson(html);
|
||||
if (nextData) {
|
||||
meta.title = makerworldTitleFromNextData(nextData);
|
||||
meta.tags = makerworldTagsFromNextData(nextData);
|
||||
meta.description = makerworldDescriptionFromNextData(nextData);
|
||||
meta.creator = makerworldCreatorFromNextData(nextData);
|
||||
meta.author = makerworldAuthorFromDesignCreator(getPath(nextData, "props", "pageProps", "design", "designCreator"));
|
||||
meta.previewImageUrl = makerworldCoverUrlFromNextData(nextData);
|
||||
}
|
||||
const fromDesign = makerworldMetaFromDesign(getPath(extractNextDataJson(html), "props", "pageProps", "design"));
|
||||
if (!fromDesign.title) fromDesign.title = genericTitleFromHtml(html);
|
||||
return fromDesign;
|
||||
}
|
||||
if (!meta.title) meta.title = genericTitleFromHtml(html);
|
||||
meta.title = genericTitleFromHtml(html);
|
||||
return meta;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
import { IMPORT_BROWSER_USER_AGENT, IMPORT_TIMEOUT_SECONDS } from "../config";
|
||||
import { extractJsonFromBrowserBody, fetchViaFlaresolverr, isFlaresolverrEnabled, looksLikeCloudflareBlock } from "./flaresolverr";
|
||||
import {
|
||||
decodeHtmlEntities,
|
||||
htmlToPlainText,
|
||||
makerworldAuthorFromDesignCreator,
|
||||
makerworldMetaFromDesign,
|
||||
type ImportedAuthorInfo,
|
||||
type ImportedPageMetadata,
|
||||
} from "./importResolvers";
|
||||
@@ -343,46 +342,6 @@ export async function fetchMakerworldDesign(
|
||||
return result.status === 200 && isRecord(result.data) ? result.data : null;
|
||||
}
|
||||
|
||||
export type MakerworldGalleryImage = { url: string; filename: string };
|
||||
|
||||
/** The model page's photo gallery -- design.designExtension.design_pictures -- distinct from
|
||||
* coverUrl/coverPortrait/coverLandscape, which are just crops of the same single cover image
|
||||
* for different UI contexts. Confirmed live: a model can have several of these (renders and/or
|
||||
* isRealLifePhoto: 1 real-world photos of the print). */
|
||||
function extractGalleryImages(design: Record<string, unknown>): MakerworldGalleryImage[] {
|
||||
const extension = design.designExtension;
|
||||
if (!isRecord(extension)) return [];
|
||||
const pictures = extension.design_pictures;
|
||||
if (!Array.isArray(pictures)) return [];
|
||||
const images: MakerworldGalleryImage[] = [];
|
||||
for (const picture of pictures) {
|
||||
if (!isRecord(picture)) continue;
|
||||
const url = typeof picture.url === "string" ? picture.url.trim() : "";
|
||||
if (!url) continue;
|
||||
const filename = typeof picture.name === "string" && picture.name.trim() ? picture.name.trim() : null;
|
||||
images.push({ url, filename: filename ?? url.split("/").pop() ?? "preview.jpg" });
|
||||
}
|
||||
return images;
|
||||
}
|
||||
|
||||
/** design.categories -- a flat array of `{id, name, ...}` objects, most-specific first (e.g.
|
||||
* "Cosplay Weapons" then its parent "Props & Cosplays"). Used to auto-land the import into a
|
||||
* Category whose makerworldCatIds overlaps one of these (see importService.ts's
|
||||
* resolveCategoryIdByCategory); every id is kept, not just the first, so a category configured for
|
||||
* either the specific or the parent category still matches. */
|
||||
function extractCategoryIds(design: Record<string, unknown>): number[] {
|
||||
const categories = design.categories;
|
||||
if (!Array.isArray(categories)) return [];
|
||||
const ids: number[] = [];
|
||||
for (const category of categories) {
|
||||
if (!isRecord(category)) continue;
|
||||
const id = category.id;
|
||||
if (typeof id === "number" && Number.isInteger(id)) ids.push(id);
|
||||
else if (typeof id === "string" && /^\d+$/.test(id)) ids.push(Number(id));
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a MakerWorld design to a real, directly-downloadable (signed S3) URL entirely
|
||||
* through api.bambulab.com -- no Cloudflare, no cookie-gated web session, no HTML scraping.
|
||||
@@ -451,31 +410,16 @@ export async function resolveMakerworldViaCloudApi(
|
||||
const body = downloadResult.data;
|
||||
if (body.message !== "success" || typeof body.url !== "string" || !body.url.trim()) return null;
|
||||
|
||||
const tags = Array.isArray(design.tags)
|
||||
? design.tags.filter((tag): tag is string => typeof tag === "string" && tag.trim().length > 0).map((tag) => tag.trim())
|
||||
: [];
|
||||
const summary = typeof design.summary === "string" ? design.summary : null;
|
||||
const designCreator = isRecord(design.designCreator) ? design.designCreator : null;
|
||||
const creator = designCreator ? pickString(designCreator, ["nickName", "name", "handle"]) : null;
|
||||
const previewImageUrl = pickString(design, ["coverUrl", "coverPortrait", "coverLandscape"]);
|
||||
const title = pickString(design, ["title"]);
|
||||
const galleryImages = extractGalleryImages(design);
|
||||
const author = await completeMakerworldAuthor(makerworldAuthorFromDesignCreator(designCreator), paceMs);
|
||||
const siteCategoryIds = extractCategoryIds(design);
|
||||
const meta = makerworldMetaFromDesign(design);
|
||||
const author = await completeMakerworldAuthor(meta.author, paceMs);
|
||||
|
||||
return {
|
||||
downloadUrl: body.url,
|
||||
meta: {
|
||||
title: title ? decodeHtmlEntities(title) : null,
|
||||
tags,
|
||||
description: summary ? htmlToPlainText(summary) : null,
|
||||
creator: author?.name ?? creator,
|
||||
previewImageUrl,
|
||||
...meta,
|
||||
creator: author?.name ?? meta.creator,
|
||||
filename: pickString(body, ["filename"]),
|
||||
galleryImages,
|
||||
author,
|
||||
siteCategoryIds,
|
||||
categorySite: siteCategoryIds.length ? "makerworld" : null,
|
||||
makerworldProfile: { instanceId: selected.id != null ? String(selected.id) : null },
|
||||
},
|
||||
};
|
||||
|
||||
@@ -298,6 +298,9 @@ describe("importing from MakerWorld", () => {
|
||||
tags: ["benchy", "calibration"],
|
||||
coverUrl: "https://makerworld.bblmw.com/makerworld/model/benchy/cover.jpg",
|
||||
designCreator: creator,
|
||||
designExtension: {
|
||||
design_pictures: [{ name: "side.jpg", url: "https://makerworld.bblmw.com/makerworld/model/benchy/side.jpg" }],
|
||||
},
|
||||
defaultInstanceId: Number(instanceId),
|
||||
instances: [{ id: Number(instanceId), title: "0.2mm layer" }],
|
||||
},
|
||||
@@ -312,7 +315,7 @@ describe("importing from MakerWorld", () => {
|
||||
descriptionIncludes: "classic calibration boat",
|
||||
tags: ["Benchy", "Calibration"],
|
||||
author: { id: `makerworld:${creator.uid}`, name: creator.name, avatarUrl: creator.avatar },
|
||||
previewImages: 1,
|
||||
previewImages: 2,
|
||||
filename: "benchy.3mf",
|
||||
...overrides,
|
||||
});
|
||||
@@ -330,6 +333,7 @@ describe("importing from MakerWorld", () => {
|
||||
json({ name: "benchy.3mf", url: "https://makerworld.bblmw.com/makerworld/model/benchy/benchy.3mf" }),
|
||||
"https://makerworld.bblmw.com/makerworld/model/benchy/benchy.3mf": threeMf,
|
||||
"https://makerworld.bblmw.com/makerworld/model/benchy/cover.jpg": png,
|
||||
"https://makerworld.bblmw.com/makerworld/model/benchy/side.jpg": png,
|
||||
});
|
||||
|
||||
const res = await request(app).post("/api/import").set(auth()).send({ url: `https://makerworld.com/en/models/${designId}-classic-benchy` });
|
||||
@@ -346,6 +350,7 @@ describe("importing from MakerWorld", () => {
|
||||
"https://makerworld.com/api/v1/design-user-service/user/profile/": cloudflareChallenge,
|
||||
"https://makerworld.bblmw.com/makerworld/model/benchy/benchy.3mf": threeMf,
|
||||
"https://makerworld.bblmw.com/makerworld/model/benchy/cover.jpg": png,
|
||||
"https://makerworld.bblmw.com/makerworld/model/benchy/side.jpg": png,
|
||||
});
|
||||
|
||||
const res = await request(app)
|
||||
@@ -404,7 +409,7 @@ describe("importing from MakerWorld", () => {
|
||||
.set(auth())
|
||||
.send({ url: `https://makerworld.com/en/models/${designId}-classic-benchy`, makerworld_cookie: "token=test-bearer-token" });
|
||||
expect(res.status).toBe(200);
|
||||
await expectFullyImported(res.body.id, expected({ previewImages: 2 }));
|
||||
await expectFullyImported(res.body.id, expected());
|
||||
}
|
||||
|
||||
it("from the web app with a MakerWorld login: uses MakerWorld's API, with the full author profile when reachable", async () => {
|
||||
|
||||
Reference in new issue
Block a user