mirror of
https://github.com/TautvydasDerzinskas/Thingport.git
synced 2026-10-21 22:22:24 -05:00
feat: collection import pt2
This commit is contained in:
1 parent
031cdf3990
commit
f04ca0c8b8
2 files changed
+23
-11
No files matched your search
@@ -8,6 +8,8 @@ import { normalizeImportUrl } from "../utils/urlUtils";
|
||||
import { parseBody } from "../utils/validate";
|
||||
import { asyncHandler } from "../utils/asyncHandler";
|
||||
import { downloadImportToTemp, importPrintFromUrl, inspectImportLink, type ImportRequestBody } from "../services/importService";
|
||||
import { resolveMakerworldCookie } from "../services/importResolvers";
|
||||
import { extractMakerworldBearerToken } from "../services/makerworldCloudApi";
|
||||
import {
|
||||
fetchMakerworldCollectionEntries,
|
||||
fetchMakerworldCollectionTitle,
|
||||
@@ -98,10 +100,11 @@ router.post(
|
||||
const url = await normalizeImportUrl(body.url);
|
||||
const parsed = parseMakerworldCollectionUrl(url);
|
||||
if (!parsed) throw new HttpError(400, "Not a MakerWorld collection URL");
|
||||
const bearerToken = extractMakerworldBearerToken(resolveMakerworldCookie(body));
|
||||
|
||||
const [title, listing] = await Promise.all([
|
||||
fetchMakerworldCollectionTitle(parsed.collectionId),
|
||||
fetchMakerworldCollectionEntries(parsed.collectionId),
|
||||
fetchMakerworldCollectionTitle(parsed.collectionId, bearerToken),
|
||||
fetchMakerworldCollectionEntries(parsed.collectionId, bearerToken),
|
||||
]);
|
||||
if (!listing.entries.length) throw new HttpError(400, "Could not load this collection's models");
|
||||
|
||||
|
||||
@@ -4,8 +4,11 @@ import { fetchViaFlaresolverr, isFlaresolverrEnabled, looksLikeCloudflareBlock }
|
||||
// MakerWorld's own /api/v1/design-service/favorites/* endpoints back the collection page's
|
||||
// "load more" pagination. Same host as the design/download endpoints we already call directly
|
||||
// (see makerworldCloudApi.ts) and, like those, not behind Cloudflare's bot-management challenge
|
||||
// even unauthenticated -- confirmed against live traffic. No bearer token needed: collection
|
||||
// listings are public the same way the collection page itself is.
|
||||
// even unauthenticated. Public collections work fine without a token, but a private one (e.g.
|
||||
// a user's own default/unshared collection) answers with a clean application-level 403
|
||||
// ("The client does not have access rights to the content.") unless the same bearer token
|
||||
// used for design/download resolution is sent -- confirmed against a live private collection.
|
||||
// So the token is sent whenever we have one, same as everywhere else MakerWorld is called.
|
||||
const COLLECTION_API_BASE = "https://makerworld.com/api/v1/design-service/favorites";
|
||||
const COLLECTION_PAGE_SIZE = 20;
|
||||
const COLLECTION_MAX_ENTRIES = 300;
|
||||
@@ -20,22 +23,24 @@ function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return !!value && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function collectionHeaders(): Record<string, string> {
|
||||
return {
|
||||
function collectionHeaders(bearerToken: string | null): Record<string, string> {
|
||||
const headers: Record<string, string> = {
|
||||
"User-Agent": IMPORT_BROWSER_USER_AGENT,
|
||||
Accept: "application/json",
|
||||
Referer: "https://makerworld.com/",
|
||||
};
|
||||
if (bearerToken) headers.Authorization = `Bearer ${bearerToken}`;
|
||||
return headers;
|
||||
}
|
||||
|
||||
/** Same defensive fallback used elsewhere for MakerWorld's /api/v1/* paths: they haven't been
|
||||
* seen behind Cloudflare's challenge in practice, but if that ever changes, retry once through
|
||||
* FlareSolverr rather than failing outright. */
|
||||
async function fetchCollectionJson(url: string): Promise<unknown | null> {
|
||||
async function fetchCollectionJson(url: string, bearerToken: string | null): Promise<unknown | null> {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), IMPORT_TIMEOUT_SECONDS * 1000);
|
||||
try {
|
||||
const res = await fetch(url, { headers: collectionHeaders(), redirect: "follow", signal: controller.signal });
|
||||
const res = await fetch(url, { headers: collectionHeaders(bearerToken), redirect: "follow", signal: controller.signal });
|
||||
if (res.status === 403 && isFlaresolverrEnabled() && looksLikeCloudflareBlock(res.headers)) {
|
||||
const solved = await fetchViaFlaresolverr(url, null);
|
||||
if (!solved) return null;
|
||||
@@ -72,8 +77,11 @@ export function parseMakerworldCollectionUrl(url: string): { collectionId: strin
|
||||
return match ? { collectionId: match[1] } : null;
|
||||
}
|
||||
|
||||
export async function fetchMakerworldCollectionTitle(collectionId: string): Promise<string | null> {
|
||||
const data = await fetchCollectionJson(`${COLLECTION_API_BASE}/${collectionId}`);
|
||||
export async function fetchMakerworldCollectionTitle(
|
||||
collectionId: string,
|
||||
bearerToken: string | null = null,
|
||||
): Promise<string | null> {
|
||||
const data = await fetchCollectionJson(`${COLLECTION_API_BASE}/${collectionId}`, bearerToken);
|
||||
if (!isRecord(data)) return null;
|
||||
return typeof data.title === "string" && data.title.trim() ? data.title.trim() : null;
|
||||
}
|
||||
@@ -84,6 +92,7 @@ export async function fetchMakerworldCollectionTitle(collectionId: string): Prom
|
||||
* turn this into an unbounded loop against a live upstream. */
|
||||
export async function fetchMakerworldCollectionEntries(
|
||||
collectionId: string,
|
||||
bearerToken: string | null = null,
|
||||
maxItems: number = COLLECTION_MAX_ENTRIES,
|
||||
): Promise<{ total: number; entries: MakerworldCollectionEntry[]; truncated: boolean }> {
|
||||
const entries: MakerworldCollectionEntry[] = [];
|
||||
@@ -92,7 +101,7 @@ export async function fetchMakerworldCollectionEntries(
|
||||
|
||||
for (;;) {
|
||||
const url = `${COLLECTION_API_BASE}/${collectionId}/designs?seed=0&collectionId=${collectionId}&limit=${COLLECTION_PAGE_SIZE}&offset=${offset}`;
|
||||
const data = await fetchCollectionJson(url);
|
||||
const data = await fetchCollectionJson(url, bearerToken);
|
||||
if (!isRecord(data) || !Array.isArray(data.hits) || !data.hits.length) break;
|
||||
if (typeof data.total === "number") total = data.total;
|
||||
|
||||
|
||||
Reference in new issue
Block a user