mirror of
https://github.com/TautvydasDerzinskas/Thingport.git
synced 2026-10-20 21:59:53 -05:00
fix: big collection import printables
This commit is contained in:
1 parent
27948c01ee
commit
8cb97bd315
6 files changed
+209
-140
No files matched your search
Generated
+13
@@ -12,6 +12,7 @@
|
||||
"bcryptjs": "^3.0.3",
|
||||
"cheerio": "^1.0.0",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^17.4.2",
|
||||
"express": "^4.21.2",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"mime-types": "^2.1.35",
|
||||
@@ -2536,6 +2537,18 @@
|
||||
"url": "https://github.com/fb55/domutils?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/dotenv": {
|
||||
"version": "17.4.2",
|
||||
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz",
|
||||
"integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://dotenvx.com"
|
||||
}
|
||||
},
|
||||
"node_modules/dunder-proto": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
"bcryptjs": "^3.0.3",
|
||||
"cheerio": "^1.0.0",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^17.4.2",
|
||||
"express": "^4.21.2",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"mime-types": "^2.1.35",
|
||||
|
||||
@@ -1,4 +1,20 @@
|
||||
import path from "node:path";
|
||||
import dotenv from "dotenv";
|
||||
|
||||
// Every value below reads straight from process.env at module-evaluation time (a one-shot
|
||||
// `export const`, not a function called later) -- so .env has to be loaded before any of them,
|
||||
// not just before the app "starts" in some general sense. There was no explicit .env loading
|
||||
// anywhere in this codebase; local dev only worked at all because requiring @prisma/client has
|
||||
// the side effect of loading .env too, and most existing config reads happened to be evaluated
|
||||
// after something had already pulled in db.ts/@prisma/client first -- an accident of import
|
||||
// order, not a guarantee. A newly added config.ts export (this file is always the first module
|
||||
// in the graph to read env vars) can land ahead of that side effect and silently read an empty
|
||||
// string forever, which is exactly what happened to FLARESOLVERR_URL. Loading .env explicitly,
|
||||
// right here, removes the dependency on that accident entirely. (Docker Compose deployments are
|
||||
// unaffected either way -- environment: entries are already real process env vars before the
|
||||
// Node process even starts, so there's nothing for dotenv to add there; this only matters for
|
||||
// `tsx watch src/server.ts`-style local dev reading a .env file.)
|
||||
dotenv.config();
|
||||
|
||||
function envInt(name: string, fallback: number): number {
|
||||
const raw = process.env[name];
|
||||
|
||||
@@ -228,7 +228,7 @@ router.post(
|
||||
const parsed = parsePrintablesCollectionUrl(url);
|
||||
if (!parsed) throw new HttpError(400, "Not a Printables Collection URL");
|
||||
|
||||
const listing = await fetchPrintablesCollectionEntries(parsed.collectionId, url);
|
||||
const listing = await fetchPrintablesCollectionEntries(parsed.collectionId);
|
||||
if (!listing.entries.length) throw new HttpError(400, "Could not load this collection's models");
|
||||
|
||||
const alreadyImported = await findImportedExternalIds(
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { IMPORT_BROWSER_USER_AGENT, IMPORT_HTML_MAX_BYTES, IMPORT_TIMEOUT_SECONDS } from "../config";
|
||||
import { HttpError } from "../utils/fileUtils";
|
||||
import { extractNextDataJson, htmlToPlainText, type ImportedAuthorInfo, type ImportedPageMetadata } from "./importResolvers";
|
||||
import { fetchViaFlaresolverr, isFlaresolverrEnabled } from "./flaresolverr";
|
||||
import { htmlToPlainText, type ImportedAuthorInfo, type ImportedPageMetadata } from "./importResolvers";
|
||||
|
||||
// The public api.printables.com GraphQL endpoint -- unlike www.printables.com (Cloudflare-gated,
|
||||
// confirmed via a plain fetch returning its "Just a moment..." challenge page), api.printables.com
|
||||
@@ -254,17 +253,6 @@ export function parsePrintablesCollectionUrl(url: string): { collectionId: strin
|
||||
|
||||
export type PrintablesCollectionEntry = { modelId: string; title: string; cover: string | null };
|
||||
|
||||
const COLLECTION_QUERY = `
|
||||
query ($id: ID!) {
|
||||
collection(id: $id) {
|
||||
id
|
||||
name
|
||||
printsCount
|
||||
thumbnails11 { id slug image { filePath } }
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
/** Printables has no human-readable name on a thumbnail-only listing (ThumbnailPrintType has no
|
||||
* `name` field, only `slug`) -- turns "prusa-core-one-nozzle-wiper-remix" into "Prusa core one
|
||||
* nozzle wiper remix" so the collection picker shows something readable instead of a raw slug. */
|
||||
@@ -274,54 +262,11 @@ function titleFromSlug(slug: string): string {
|
||||
return words[0].charAt(0).toUpperCase() + words[0].slice(1) + (words.length > 1 ? " " + words.slice(1).join(" ") : "");
|
||||
}
|
||||
|
||||
function entryFromThumbnail(item: unknown): PrintablesCollectionEntry | null {
|
||||
if (!isRecord(item) || item.id == null || typeof item.slug !== "string" || !item.slug.trim()) return null;
|
||||
const cover = isRecord(item.image) ? mediaUrl(item.image.filePath) : null;
|
||||
return { modelId: String(item.id), title: titleFromSlug(item.slug.trim()), cover };
|
||||
}
|
||||
|
||||
const COLLECTION_SCRAPE_MAX_ENTRIES = 300;
|
||||
|
||||
/** Best-effort full listing for a collection bigger than the public API's hard 11-item preview
|
||||
* cap (see fetchPrintablesCollectionEntries) -- renders the actual collection page through
|
||||
* FlareSolverr (the same Cloudflare-bypass path already used elsewhere in this codebase) and
|
||||
* walks its embedded __NEXT_DATA__ payload for print-shaped objects ({id, slug, image} --
|
||||
* matching the exact PrintType/ThumbnailPrintType shape confirmed against the live GraphQL
|
||||
* schema), rather than relying on any specific, unconfirmed path into that JSON -- Next.js's
|
||||
* SSR payload shape isn't part of any public contract and could shift at any time.
|
||||
* Returns null (never throws) on anything short of a clean, useful result: FlareSolverr not
|
||||
* configured, the fetch failing, no __NEXT_DATA__ found, or nothing print-shaped inside it --
|
||||
* every case the caller falls back to the always-available 11-item preview for instead. */
|
||||
async function scrapeFullPrintablesCollection(pageUrl: string): Promise<PrintablesCollectionEntry[] | null> {
|
||||
if (!isFlaresolverrEnabled()) return null;
|
||||
const solved = await fetchViaFlaresolverr(pageUrl);
|
||||
if (!solved || !solved.body) return null;
|
||||
const nextData = extractNextDataJson(solved.body);
|
||||
if (!nextData) return null;
|
||||
|
||||
const found = new Map<string, PrintablesCollectionEntry>();
|
||||
const stack: unknown[] = [nextData];
|
||||
let visited = 0;
|
||||
while (stack.length && visited < 50000 && found.size < COLLECTION_SCRAPE_MAX_ENTRIES) {
|
||||
const node = stack.pop();
|
||||
visited++;
|
||||
if (Array.isArray(node)) {
|
||||
stack.push(...node);
|
||||
continue;
|
||||
}
|
||||
if (!isRecord(node)) continue;
|
||||
const entry = entryFromThumbnail(node);
|
||||
if (entry && !found.has(entry.modelId)) found.set(entry.modelId, entry);
|
||||
stack.push(...Object.values(node));
|
||||
}
|
||||
return found.size ? Array.from(found.values()) : null;
|
||||
}
|
||||
|
||||
const COLLECTION_TITLE_QUERY = `query ($id: ID!) { collection(id: $id) { id name } }`;
|
||||
|
||||
/** Cheap, title-only fetch -- used by the job runner once the batch import is done (see
|
||||
* importJobRunner.ts's runPrintablesCollectionImportJob), so the PrintStash Collection it files
|
||||
* results into doesn't require re-running the (potentially FlareSolverr-backed) full listing. */
|
||||
* results into doesn't require re-running the full listing. */
|
||||
export async function fetchPrintablesCollectionTitle(collectionId: string): Promise<string | null> {
|
||||
const data = (await fetchPrintablesGraphql(COLLECTION_TITLE_QUERY, { id: collectionId })) as
|
||||
| { data?: { collection?: Record<string, unknown> } }
|
||||
@@ -330,41 +275,78 @@ export async function fetchPrintablesCollectionTitle(collectionId: string): Prom
|
||||
return typeof name === "string" && name.trim() ? name.trim() : null;
|
||||
}
|
||||
|
||||
/** Resolves a Printables Collection's title and model list. The public GraphQL API only ever
|
||||
* exposes up to 11 models per collection (`thumbnails11` -- confirmed live: it takes no
|
||||
* limit/offset/cursor argument, and there is no separate paginated "list every model in this
|
||||
* collection" operation anywhere in Printables' own site traffic either) -- for anything bigger,
|
||||
* this falls back to best-effort page-scraping via FlareSolverr (see
|
||||
* scrapeFullPrintablesCollection). `truncated` reflects whichever source ultimately won: false
|
||||
* only when every model in the collection was actually returned. */
|
||||
// The actual query the site itself sends its own collection page's "load more" scroll trigger --
|
||||
// a real, properly cursor-paginated listing, unlike `collection(id).thumbnails11` (a hard-capped,
|
||||
// unparameterized 11-item preview field used elsewhere on the site, e.g. the "add to collection"
|
||||
// picker). Confirmed live end to end against an 89-model collection: 3 pages of `limit: 30`,
|
||||
// terminated by an empty-string `cursor` on the last page (NOT null -- a naive `cursor == null`
|
||||
// check treats an empty string as "keep going" and the API happily restarts from page 1, an easy
|
||||
// infinite-loop trap). No auth needed, same as every other Printables query here.
|
||||
const COLLECTION_MODELS_QUERY = `
|
||||
query CollectionModels($collectionId: ID!, $limit: Int, $cursor: String, $ordering: CollectionPrintsOrderingEnum) {
|
||||
moreCollectionModels(limit: $limit, cursor: $cursor, collectionId: $collectionId, ordering: $ordering) {
|
||||
items {
|
||||
id
|
||||
model: print { id name slug image { filePath } }
|
||||
}
|
||||
cursor
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const COLLECTION_PAGE_SIZE = 30;
|
||||
const COLLECTION_MAX_PAGES = 20;
|
||||
const COLLECTION_MAX_ENTRIES = 600;
|
||||
|
||||
/** Pages through every model in a Collection via moreCollectionModels, deduplicating and capping
|
||||
* at COLLECTION_MAX_ENTRIES/COLLECTION_MAX_PAGES purely as a safety net against a pathological
|
||||
* collection or an unexpected non-terminating cursor -- not expected to ever actually bite given
|
||||
* the confirmed termination behavior above. An item referencing a deleted/hidden print (no
|
||||
* `model`, only `unavailableModel`) is skipped, same tolerance as everywhere else in this file for
|
||||
* one bad entry not derailing the whole listing. */
|
||||
async function fetchAllPrintablesCollectionModels(collectionId: string): Promise<PrintablesCollectionEntry[]> {
|
||||
const found = new Map<string, PrintablesCollectionEntry>();
|
||||
let cursor: string | null = null;
|
||||
for (let page = 0; page < COLLECTION_MAX_PAGES && found.size < COLLECTION_MAX_ENTRIES; page++) {
|
||||
const data = (await fetchPrintablesGraphql(COLLECTION_MODELS_QUERY, {
|
||||
collectionId,
|
||||
limit: COLLECTION_PAGE_SIZE,
|
||||
cursor,
|
||||
ordering: "added_to_collection",
|
||||
})) as { data?: { moreCollectionModels?: { items?: unknown[]; cursor?: string | null } } } | null;
|
||||
const node = data?.data?.moreCollectionModels;
|
||||
const items = Array.isArray(node?.items) ? node.items : [];
|
||||
if (!items.length) break;
|
||||
|
||||
for (const item of items) {
|
||||
if (found.size >= COLLECTION_MAX_ENTRIES) break;
|
||||
if (!isRecord(item) || !isRecord(item.model)) continue;
|
||||
const model = item.model;
|
||||
if (model.id == null || typeof model.slug !== "string" || !model.slug.trim()) continue;
|
||||
const modelId = String(model.id);
|
||||
if (found.has(modelId)) continue;
|
||||
const cover = isRecord(model.image) ? mediaUrl(model.image.filePath) : null;
|
||||
found.set(modelId, { modelId, title: titleFromSlug(model.slug.trim()), cover });
|
||||
}
|
||||
|
||||
const nextCursor = node?.cursor;
|
||||
if (!nextCursor) break; // empty string or null/undefined both mean "no more pages"
|
||||
cursor = nextCursor;
|
||||
}
|
||||
return Array.from(found.values());
|
||||
}
|
||||
|
||||
/** Resolves a Printables Collection's title and full model list via real cursor pagination (see
|
||||
* fetchAllPrintablesCollectionModels) -- no FlareSolverr, no page-scraping, no arbitrary preview
|
||||
* cap: this reaches every model in the collection in the overwhelming majority of cases.
|
||||
* `truncated` only trips if COLLECTION_MAX_ENTRIES/COLLECTION_MAX_PAGES's safety net was actually
|
||||
* needed (an exceptionally large collection), not as a matter of course. */
|
||||
export async function fetchPrintablesCollectionEntries(
|
||||
collectionId: string,
|
||||
pageUrl: string,
|
||||
): Promise<{ title: string | null; entries: PrintablesCollectionEntry[]; total: number; truncated: boolean }> {
|
||||
const data = (await fetchPrintablesGraphql(COLLECTION_QUERY, { id: collectionId })) as
|
||||
| { data?: { collection?: Record<string, unknown> } }
|
||||
| null;
|
||||
const collection = data?.data?.collection;
|
||||
if (!isRecord(collection)) return { title: null, entries: [], total: 0, truncated: false };
|
||||
|
||||
const title = typeof collection.name === "string" && collection.name.trim() ? collection.name.trim() : null;
|
||||
const printsCount = typeof collection.printsCount === "number" ? collection.printsCount : null;
|
||||
const preview = Array.isArray(collection.thumbnails11)
|
||||
? collection.thumbnails11.map(entryFromThumbnail).filter((e): e is PrintablesCollectionEntry => e !== null)
|
||||
: [];
|
||||
// The real collection size, independent of how many entries actually came back below --
|
||||
// callers (see routes/imports.ts) must report this as `total`, not entries.length, or the
|
||||
// "Showing 11 of X" truncation notice silently reads "Showing 11 of 11" and hides that there
|
||||
// were ever more than 11 models to begin with.
|
||||
const total = printsCount ?? preview.length;
|
||||
|
||||
if (printsCount === null || preview.length >= printsCount) {
|
||||
return { title, entries: preview, total, truncated: false };
|
||||
}
|
||||
|
||||
const scraped = await scrapeFullPrintablesCollection(pageUrl);
|
||||
if (scraped && scraped.length > preview.length) {
|
||||
return { title, entries: scraped, total, truncated: scraped.length < printsCount };
|
||||
}
|
||||
return { title, entries: preview, total, truncated: true };
|
||||
const [title, entries] = await Promise.all([
|
||||
fetchPrintablesCollectionTitle(collectionId),
|
||||
fetchAllPrintablesCollectionModels(collectionId),
|
||||
]);
|
||||
return { title, entries, total: entries.length, truncated: entries.length >= COLLECTION_MAX_ENTRIES };
|
||||
}
|
||||
@@ -4,14 +4,12 @@ import { createApp } from "../src/app";
|
||||
import { fetchPrintablesCollectionEntries, parsePrintablesCollectionUrl } from "../src/services/printablesApi";
|
||||
import { prisma } from "../src/db";
|
||||
|
||||
// Exercises the Printables Collection listing path against the real shape confirmed live
|
||||
// against api.printables.com's public GraphQL endpoint (see printablesApi.ts): `collection(id)`
|
||||
// exposes name/printsCount/thumbnails11, but thumbnails11 is a hard-capped preview (confirmed:
|
||||
// it takes no limit/offset/cursor argument) -- collections at or under 11 models resolve fully
|
||||
// from that alone; bigger ones would need the FlareSolverr scrape fallback, which is not
|
||||
// exercised here since FLARESOLVERR_URL is unset in the test environment (isFlaresolverrEnabled
|
||||
//() false), matching how a real instance without FlareSolverr configured behaves: falls back to
|
||||
// the capped preview with `truncated: true` rather than failing.
|
||||
// Exercises the Printables Collection listing path against the real shape confirmed live against
|
||||
// api.printables.com's public GraphQL endpoint (see printablesApi.ts): `moreCollectionModels`
|
||||
// is the actual query the site's own collection page sends as it lazy-loads more models (found by
|
||||
// inspecting the real site's network traffic), cursor-paginated with `limit: 30` pages, no auth
|
||||
// needed. Confirmed live end to end against a real 89-model collection (30+30+29 across 3 pages,
|
||||
// terminated by an empty-string `cursor` -- not null -- on the last page).
|
||||
|
||||
const COLLECTION_ID = "2348006";
|
||||
const COLLECTION_TITLE = "Printing for printing's sake";
|
||||
@@ -20,21 +18,40 @@ function jsonResponse(status: number, body: unknown): Response {
|
||||
return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } });
|
||||
}
|
||||
|
||||
function thumbnail(id: number) {
|
||||
return { id, slug: `collected-model-${id}`, image: { filePath: `media/prints/${id}/title.jpg` } };
|
||||
function modelItem(id: number) {
|
||||
return { id: String(id), model: { id: String(id), name: `Model ${id}`, slug: `collected-model-${id}`, image: { filePath: `media/prints/${id}/title.jpg` } } };
|
||||
}
|
||||
|
||||
function collectionGraphqlResponse(printsCount: number, thumbnailCount: number) {
|
||||
return jsonResponse(200, {
|
||||
data: {
|
||||
collection: {
|
||||
id: COLLECTION_ID,
|
||||
name: COLLECTION_TITLE,
|
||||
printsCount,
|
||||
thumbnails11: Array.from({ length: thumbnailCount }, (_, i) => thumbnail(i + 1)),
|
||||
},
|
||||
},
|
||||
});
|
||||
/** An item referencing a print that's since been deleted/hidden -- has `unavailableModel`
|
||||
* instead of `model`, and should be silently skipped rather than crash the listing. */
|
||||
function unavailableItem(id: number) {
|
||||
return { id: String(id), unavailableModel: { id: String(id), name: "Deleted print" } };
|
||||
}
|
||||
|
||||
function collectionModelsPage(items: unknown[], cursor: string | null) {
|
||||
return jsonResponse(200, { data: { moreCollectionModels: { items, cursor } } });
|
||||
}
|
||||
|
||||
function collectionTitleResponse(name: string | null) {
|
||||
return jsonResponse(200, { data: { collection: name ? { id: COLLECTION_ID, name } : null } });
|
||||
}
|
||||
|
||||
/** Routes a mocked fetch to either the title query or the right page of moreCollectionModels,
|
||||
* based on the outgoing request body -- fetchPrintablesCollectionEntries fires both in parallel
|
||||
* (see printablesApi.ts), and the models query itself gets called once per page with a different
|
||||
* `cursor` variable each time. `pages` is keyed by the cursor each page expects to be called
|
||||
* with (null for the first page). */
|
||||
function mockCollectionFetch(opts: { title?: string | null; pages: Record<string, { items: unknown[]; nextCursor: string | null }> }) {
|
||||
return vi.fn<(input: RequestInfo | URL, init?: RequestInit) => Promise<Response>>(async (_input, init) => {
|
||||
const body = JSON.parse(String(init?.body ?? "{}"));
|
||||
if (typeof body.query === "string" && body.query.includes("moreCollectionModels")) {
|
||||
const cursorKey = body.variables?.cursor ?? "null";
|
||||
const page = opts.pages[cursorKey];
|
||||
if (!page) throw new Error(`Unexpected cursor requested: ${cursorKey}`);
|
||||
return collectionModelsPage(page.items, page.nextCursor);
|
||||
}
|
||||
return collectionTitleResponse(opts.title ?? null);
|
||||
}) as unknown as typeof fetch;
|
||||
}
|
||||
|
||||
describe("parsePrintablesCollectionUrl", () => {
|
||||
@@ -56,49 +73,83 @@ describe("parsePrintablesCollectionUrl", () => {
|
||||
|
||||
describe("fetchPrintablesCollectionEntries", () => {
|
||||
const originalFetch = global.fetch;
|
||||
const pageUrl = `https://www.printables.com/@joshuaargh/collections/${COLLECTION_ID}`;
|
||||
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("resolves fully from the API alone when the collection is at or under the 11-item preview cap", async () => {
|
||||
global.fetch = vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async () => collectionGraphqlResponse(5, 5)) as unknown as typeof fetch;
|
||||
it("resolves fully from a single page for a small collection", async () => {
|
||||
global.fetch = mockCollectionFetch({
|
||||
title: COLLECTION_TITLE,
|
||||
pages: { null: { items: [modelItem(1), modelItem(2), modelItem(3)], nextCursor: "" } },
|
||||
});
|
||||
|
||||
const listing = await fetchPrintablesCollectionEntries(COLLECTION_ID, pageUrl);
|
||||
const listing = await fetchPrintablesCollectionEntries(COLLECTION_ID);
|
||||
|
||||
expect(listing.title).toBe(COLLECTION_TITLE);
|
||||
expect(listing.truncated).toBe(false);
|
||||
expect(listing.total).toBe(5);
|
||||
expect(listing.total).toBe(3);
|
||||
expect(listing.entries).toEqual([
|
||||
{ modelId: "1", title: "Collected model 1", cover: "https://media.printables.com/media/prints/1/title.jpg" },
|
||||
{ modelId: "2", title: "Collected model 2", cover: "https://media.printables.com/media/prints/2/title.jpg" },
|
||||
{ modelId: "3", title: "Collected model 3", cover: "https://media.printables.com/media/prints/3/title.jpg" },
|
||||
{ modelId: "4", title: "Collected model 4", cover: "https://media.printables.com/media/prints/4/title.jpg" },
|
||||
{ modelId: "5", title: "Collected model 5", cover: "https://media.printables.com/media/prints/5/title.jpg" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("falls back to the 11-item preview, flagged truncated, when the collection is bigger and no scrape path is available", async () => {
|
||||
global.fetch = vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async () => collectionGraphqlResponse(34, 11)) as unknown as typeof fetch;
|
||||
it("pages through a collection bigger than one page, terminating on an empty-string cursor", async () => {
|
||||
global.fetch = mockCollectionFetch({
|
||||
title: COLLECTION_TITLE,
|
||||
pages: {
|
||||
null: { items: [modelItem(1), modelItem(2), modelItem(3)], nextCursor: "page2" },
|
||||
page2: { items: [modelItem(4), modelItem(5)], nextCursor: "" },
|
||||
},
|
||||
});
|
||||
|
||||
const listing = await fetchPrintablesCollectionEntries(COLLECTION_ID, pageUrl);
|
||||
const listing = await fetchPrintablesCollectionEntries(COLLECTION_ID);
|
||||
|
||||
expect(listing.title).toBe(COLLECTION_TITLE);
|
||||
expect(listing.truncated).toBe(true);
|
||||
// Regression guard: `total` must be the real collection size (34), not entries.length (11)
|
||||
// -- otherwise the "Showing 11 of X" truncation notice reads "Showing 11 of 11" and hides
|
||||
// that there were ever more than 11 models to begin with (exactly what a real 17-model
|
||||
// collection surfaced with no FlareSolverr configured).
|
||||
expect(listing.total).toBe(34);
|
||||
expect(listing.entries).toHaveLength(11);
|
||||
// Regression guard for the exact bug found against a real 89-model collection: `total` must
|
||||
// reflect every model actually paginated through, not just the first page.
|
||||
expect(listing.total).toBe(5);
|
||||
expect(listing.truncated).toBe(false);
|
||||
expect(listing.entries.map((e) => e.modelId)).toEqual(["1", "2", "3", "4", "5"]);
|
||||
});
|
||||
|
||||
it("skips items referencing a deleted/unavailable print instead of failing the whole listing", async () => {
|
||||
global.fetch = mockCollectionFetch({
|
||||
title: COLLECTION_TITLE,
|
||||
pages: { null: { items: [modelItem(1), unavailableItem(2), modelItem(3)], nextCursor: "" } },
|
||||
});
|
||||
|
||||
const listing = await fetchPrintablesCollectionEntries(COLLECTION_ID);
|
||||
|
||||
expect(listing.entries.map((e) => e.modelId)).toEqual(["1", "3"]);
|
||||
});
|
||||
|
||||
it("stops after a bounded number of pages instead of looping forever against a non-terminating cursor", async () => {
|
||||
let calls = 0;
|
||||
global.fetch = vi.fn<(input: RequestInfo | URL, init?: RequestInit) => Promise<Response>>(async (_input, init) => {
|
||||
const body = JSON.parse(String(init?.body ?? "{}"));
|
||||
if (typeof body.query === "string" && body.query.includes("moreCollectionModels")) {
|
||||
calls++;
|
||||
// Always claims there's another page, no matter what cursor was sent -- simulates a
|
||||
// pathological/buggy upstream response, exactly what the empty-string-vs-null handling
|
||||
// is there to guard against.
|
||||
return collectionModelsPage([modelItem(calls)], "still-more");
|
||||
}
|
||||
return collectionTitleResponse(COLLECTION_TITLE);
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
const listing = await fetchPrintablesCollectionEntries(COLLECTION_ID);
|
||||
|
||||
expect(calls).toBeLessThanOrEqual(20);
|
||||
expect(listing.entries.length).toBe(calls);
|
||||
});
|
||||
|
||||
it("returns an empty listing for a collection that doesn't exist", async () => {
|
||||
global.fetch = vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async () => jsonResponse(200, { data: { collection: null } })) as unknown as typeof fetch;
|
||||
global.fetch = mockCollectionFetch({ title: null, pages: { null: { items: [], nextCursor: "" } } });
|
||||
|
||||
const listing = await fetchPrintablesCollectionEntries("999999999999", pageUrl);
|
||||
const listing = await fetchPrintablesCollectionEntries("999999999999");
|
||||
|
||||
expect(listing.title).toBeNull();
|
||||
expect(listing.entries).toEqual([]);
|
||||
@@ -131,7 +182,7 @@ describe("POST /import/printables-collection/entries", () => {
|
||||
});
|
||||
|
||||
it("lists a collection's models as importable entries, with the real collection name as title", async () => {
|
||||
global.fetch = vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async () => collectionGraphqlResponse(1, 1)) as unknown as typeof fetch;
|
||||
global.fetch = mockCollectionFetch({ title: COLLECTION_TITLE, pages: { null: { items: [modelItem(1)], nextCursor: "" } } });
|
||||
|
||||
const res = await request(app)
|
||||
.post("/api/import/printables-collection/entries")
|
||||
@@ -147,8 +198,14 @@ describe("POST /import/printables-collection/entries", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("reports the real collection size as `total`, not the truncated entry count, when over the 11-item cap", async () => {
|
||||
global.fetch = vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async () => collectionGraphqlResponse(17, 11)) as unknown as typeof fetch;
|
||||
it("aggregates every page's models across the full collection, end to end through the route", async () => {
|
||||
global.fetch = mockCollectionFetch({
|
||||
title: COLLECTION_TITLE,
|
||||
pages: {
|
||||
null: { items: [modelItem(1), modelItem(2)], nextCursor: "page2" },
|
||||
page2: { items: [modelItem(3)], nextCursor: "" },
|
||||
},
|
||||
});
|
||||
|
||||
const res = await request(app)
|
||||
.post("/api/import/printables-collection/entries")
|
||||
@@ -156,9 +213,9 @@ describe("POST /import/printables-collection/entries", () => {
|
||||
.send({ url: `https://www.printables.com/@joshuaargh/collections/${COLLECTION_ID}`, tags: [] });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.truncated).toBe(true);
|
||||
expect(res.body.total).toBe(17);
|
||||
expect(res.body.entries).toHaveLength(11);
|
||||
expect(res.body.total).toBe(3);
|
||||
expect(res.body.truncated).toBe(false);
|
||||
expect(res.body.entries.map((e: { design_id: string }) => e.design_id)).toEqual(["1", "2", "3"]);
|
||||
});
|
||||
|
||||
it("flags an entry already in the user's library instead of letting it be re-selected", async () => {
|
||||
@@ -171,7 +228,7 @@ describe("POST /import/printables-collection/entries", () => {
|
||||
sourceExternalId: "1",
|
||||
},
|
||||
});
|
||||
global.fetch = vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async () => collectionGraphqlResponse(1, 1)) as unknown as typeof fetch;
|
||||
global.fetch = mockCollectionFetch({ title: COLLECTION_TITLE, pages: { null: { items: [modelItem(1)], nextCursor: "" } } });
|
||||
|
||||
const res = await request(app)
|
||||
.post("/api/import/printables-collection/entries")
|
||||
|
||||
Reference in new issue
Block a user