feat: import MakerWorld models that have only raw files

Some MakerWorld listings have no print profiles, only the STL/CAD files
behind "Download STL/CAD files". The backend now downloads those as the
page's own zip and unpacks every model file onto the model, and a zip
Grab passes on is unpacked the same way instead of being stored whole.
Every-profile and queued imports take the raw files instead of failing.

backend/src/services/importJobRunner.ts

@@ -212,6 +212,21 @@ async function importProfileUrlsSequentially(
   return totals;
 }
 
+/** A link per wanted print profile. A model with none has only its raw files, which the model's own link imports. */
+function makerworldProfileUrls(
+  designId: string,
+  design: Record<string, unknown>,
+  scope: MakerworldProfileScope,
+  requestedInstanceId: string | null,
+): { urls: string[]; rawFiles: boolean } {
+  const profileIds = selectMakerworldProfiles(design, scope, requestedInstanceId);
+  if (!profileIds.length) return { urls: [`https://makerworld.com/en/models/${designId}`], rawFiles: true };
+  return {
+    urls: profileIds.map((profileId) => `https://makerworld.com/en/models/${designId}#profileId-${profileId}`),
+    rawFiles: false,
+  };
+}
+
 /** Imports several print profiles of one MakerWorld model as one model with a file per profile. */
 export async function runMakerworldProfilesImportJob(
   jobId: string,
@@ -227,16 +242,20 @@ export async function runMakerworldProfilesImportJob(
       IMPORT_MAKERWORLD_CALL_DELAY_MS,
     );
     if (!design) throw new HttpError(400, "Couldn't read this model's print profiles from MakerWorld");
-    const profileIds = selectMakerworldProfiles(design, body.scope, parsed.requestedInstanceId);
-    if (!profileIds.length) throw new HttpError(400, "This model has no print profiles to import");
+    const { urls: profileUrls, rawFiles } = makerworldProfileUrls(
+      parsed.designId,
+      design,
+      body.scope,
+      parsed.requestedInstanceId,
+    );
     const title =
       typeof design.title === "string" && design.title.trim() ? decodeHtmlEntities(design.title.trim()) : null;
-    await updateJob(jobId, { total: profileIds.length, sourceLabel: title });
-    log.info("makerworld profiles job started", { job: jobId, url: body.url, profiles: profileIds.length });
+    await updateJob(jobId, { total: profileUrls.length, sourceLabel: title });
+    log.info("makerworld profiles job started", { job: jobId, url: body.url, profiles: profileUrls.length });
 
     const totals = await importProfileUrlsSequentially(
       userId,
-      profileIds.map((profileId) => `https://makerworld.com/en/models/${parsed.designId}#profileId-${profileId}`),
+      profileUrls,
       body,
       (t) =>
         void updateJob(jobId, {
@@ -292,7 +311,10 @@ export async function runMakerworldProfilesImportJob(
     const label = title ? `"${title}"` : "a MakerWorld model";
     await createNotification(userId, {
       type: "import",
-      title: `Imported ${imported} of ${profileIds.length} print profiles from MakerWorld`,
+      // A model with no print profiles comes in as its raw files, all at once.
+      title: rawFiles
+        ? `Imported ${imported} of 1 model from MakerWorld`
+        : `Imported ${imported} of ${profileUrls.length} print profiles from MakerWorld`,
       body: bodyParts.length ? `Of ${label} — ${bodyParts.join(", ")}.` : `Of ${label}.`,
       externalUrl: body.url,
       internalPath: printId ? `/models/${printId}` : null,
@@ -1222,11 +1244,9 @@ async function importOneLink(
       IMPORT_MAKERWORLD_CALL_DELAY_MS,
     );
     if (!design) throw new HttpError(400, "Couldn't read this model's print profiles from MakerWorld");
-    const profileIds = selectMakerworldProfiles(design, body.scope, parsed.requestedInstanceId);
-    if (!profileIds.length) throw new HttpError(400, "This model has no print profiles to import");
     const totals = await importProfileUrlsSequentially(
       userId,
-      profileIds.map((profileId) => `https://makerworld.com/en/models/${parsed.designId}#profileId-${profileId}`),
+      makerworldProfileUrls(parsed.designId, design, body.scope, parsed.requestedInstanceId).urls,
       { ...body, url },
     );
     return { ...totals, profiles: true };

backend/src/services/importService.ts

@@ -1336,19 +1336,26 @@ async function importPrintablesModel(
 const MYMINIFACTORY_NOT_CONFIGURED =
   "MyMiniFactory import isn't configured for this instance yet -- ask an admin to add an API key in Administration > Providers.";
 
-/** The model files in a downloaded zip, each as its own plate; the zip itself is left for the caller. */
+/** The model files in a downloaded zip, each as its own plate; the zip itself is left for the caller. Plates have no
+ *  folders, so files named alike in different folders are named after their folder too: "2x2 - Top.stl". */
 async function modelFilesFromZip(zipPath: string): Promise<NewPlateInput[]> {
   const inputs: NewPlateInput[] = [];
   try {
-    for (const entry of await listZipEntries(zipPath)) {
-      if (entry.isDirectory) continue;
-      const filename = sanitizeFilename(path.basename(entry.name.replace(/\\/g, "/")));
-      if (!MULTI_FILE_PLATE_EXTS.has(path.extname(filename).toLowerCase())) continue;
+    const entries = (await listZipEntries(zipPath))
+      .filter((entry) => !entry.isDirectory)
+      .map((entry) => ({ entry, parts: entry.name.replace(/\\/g, "/").split("/").filter(Boolean) }))
+      .filter(({ parts }) => parts.length && MULTI_FILE_PLATE_EXTS.has(path.extname(parts.at(-1)!).toLowerCase()));
+    const sameName = new Map<string, number>();
+    for (const { parts } of entries) sameName.set(parts.at(-1)!, (sameName.get(parts.at(-1)!) ?? 0) + 1);
+    for (const { entry, parts } of entries) {
+      const name = parts.at(-1)!;
+      const folder = parts.at(-2);
+      const filename = sanitizeFilename(sameName.get(name)! > 1 && folder ? `${folder} - ${name}` : name);
       const buffer = await readZipEntry(zipPath, entry.name, IMPORT_MAX_BYTES);
       if (!buffer) continue;
       const tempFilePath = path.join(
         os.tmpdir(),
-        `thingport-mmf-${crypto.randomBytes(8).toString("hex")}${path.extname(filename)}`,
+        `thingport-zip-${crypto.randomBytes(8).toString("hex")}${path.extname(filename)}`,
       );
       await fs.writeFile(tempFilePath, buffer);
       inputs.push({ filename, mime: guessMimeFromPath(filename), tempFilePath });
@@ -2311,6 +2318,16 @@ async function importNewOrExisting(userId: string, url: string, body: ImportRequ
   }
 
   const { tempPath, filename, mime, meta } = await downloadImportToTemp(url, body);
+  let plates: NewPlateInput[] = [];
+  try {
+    plates = await makerworldZipPlates(source, tempPath, filename);
+  } catch (err) {
+    await fs.rm(tempPath, { force: true }).catch(() => undefined);
+    throw err;
+  }
+  if (!plates.length) {
+    plates = [{ filename, mime, tempFilePath: tempPath, sourceInstanceId: meta.makerworldProfile?.instanceId ?? null }];
+  }
   const author = await upsertAuthorFromImport(meta.author);
   const categoryId =
     body.category_id ?? (await resolveCategoryIdByCategory(userId, meta.categorySite, meta.siteCategoryIds));
@@ -2329,9 +2346,7 @@ async function importNewOrExisting(userId: string, url: string, body: ImportRequ
   try {
     let result: { print: Print; plates: Plate[] };
     try {
-      result = await createPrint(userId, printMeta, path.parse(filename).name, [
-        { filename, mime, tempFilePath: tempPath, sourceInstanceId: meta.makerworldProfile?.instanceId ?? null },
-      ]);
+      result = await createPrint(userId, printMeta, path.parse(filename).name, plates);
     } catch (err) {
       // Race guard: a concurrent import of the same source model won.
       if (source && isUniqueConstraintError(err)) {
@@ -2355,6 +2370,21 @@ async function importNewOrExisting(userId: string, url: string, body: ImportRequ
     });
     return { ...result, author, previewImages, alreadyImported: false };
   } finally {
-    if (fsSync.existsSync(tempPath)) await fs.rm(tempPath, { force: true }).catch(() => undefined);
+    for (const file of [tempPath, ...plates.map((plate) => plate.tempFilePath)]) {
+      if (file && fsSync.existsSync(file)) await fs.rm(file, { force: true }).catch(() => undefined);
+    }
   }
 }
+
+/** A MakerWorld zip -- the raw files of a model with no print profiles -- as its model files, one plate each, the zip
+ *  itself removed. Empty for anything else, or a zip with no model file in it, which is kept as it came. */
+async function makerworldZipPlates(
+  source: { provider: string } | null,
+  tempPath: string,
+  filename: string,
+): Promise<NewPlateInput[]> {
+  if (source?.provider !== "makerworld" || path.extname(filename).toLowerCase() !== ".zip") return [];
+  const plates = await modelFilesFromZip(tempPath);
+  if (plates.length) await fs.rm(tempPath, { force: true }).catch(() => undefined);
+  return plates;
+}

backend/src/services/makerworldCloudApi.ts

@@ -318,6 +318,46 @@ export async function fetchMakerworldDesign(
   return result.status === 200 && isRecord(result.data) ? result.data : null;
 }
 
+/** A download request's JSON body, or null; a CAPTCHA, the daily limit and a rejected login are thrown. */
+function downloadReplyBody(result: { status: number; data: unknown } | null): Record<string, unknown> | null {
+  if (!result) return null;
+  if (isCaptchaChallenge(result.data)) {
+    noteCaptchaChallenge();
+    throw new MakerworldCaptchaError();
+  }
+  if (isDownloadLimitReply(result.data)) throw new MakerworldDownloadLimitError();
+  if (result.status === 401 || result.status === 403) throw new MakerworldAuthError();
+  return result.status === 200 && isRecord(result.data) ? result.data : null;
+}
+
+/** A design with no print profiles, only the raw files the page's "Download STL/CAD files" offers: all of them, as
+ *  one zip, which the import unpacks. It belongs to no profile. */
+async function resolveMakerworldRawFiles(
+  designId: string,
+  design: Record<string, unknown>,
+  bearerToken: string,
+  paceMs?: number,
+): Promise<MakerworldCloudResolution | null> {
+  await maybeSleep(paceMs);
+  const body = downloadReplyBody(
+    await fetchCloudJson(`${DESIGN_API_BASE}/design/${designId}/model?modelType=all&type=download`, bearerToken),
+  );
+  const url = body && pickString(body, ["url"]);
+  if (!body || !url) return null;
+  const meta = makerworldMetaFromDesign(design);
+  const author = await completeMakerworldAuthor(meta.author, paceMs);
+  return {
+    downloadUrl: url,
+    meta: {
+      ...meta,
+      creator: author?.name ?? meta.creator,
+      filename: pickString(body, ["name"]),
+      author,
+      makerworldProfile: { instanceId: null },
+    },
+  };
+}
+
 /**
  * Resolves a design to a signed download URL via api.bambulab.com. Null means fall back to page
  * scraping. `paceMs` is awaited before every request so collection imports stay evenly spaced.
@@ -340,11 +380,12 @@ export async function resolveMakerworldViaCloudApi(
   if (designResult.status === 401 || designResult.status === 403) throw new MakerworldAuthError();
   if (designResult.status !== 200 || !isRecord(designResult.data)) return null;
   const design = designResult.data;
+  const instances = Array.isArray(design.instances) ? design.instances.filter(isRecord) : [];
+  if (!instances.length) return resolveMakerworldRawFiles(designId, design, bearerToken, paceMs);
 
   const modelId = typeof design.modelId === "string" && design.modelId.trim() ? design.modelId.trim() : null;
   if (!modelId) return null;
 
-  const instances = Array.isArray(design.instances) ? design.instances.filter(isRecord) : [];
   const defaultInstanceId = design.defaultInstanceId != null ? String(design.defaultInstanceId) : null;
 
   let selected: Record<string, unknown> | null = null;
@@ -363,20 +404,10 @@ export async function resolveMakerworldViaCloudApi(
   if (!profileId) return null;
 
   await maybeSleep(paceMs);
-  const downloadResult = await fetchCloudJson(
-    `${PROFILE_DOWNLOAD_BASE}/${profileId}?model_id=${encodeURIComponent(modelId)}`,
-    bearerToken,
+  const body = downloadReplyBody(
+    await fetchCloudJson(`${PROFILE_DOWNLOAD_BASE}/${profileId}?model_id=${encodeURIComponent(modelId)}`, bearerToken),
   );
-  if (!downloadResult) return null;
-  if (isCaptchaChallenge(downloadResult.data)) {
-    noteCaptchaChallenge();
-    throw new MakerworldCaptchaError();
-  }
-  if (isDownloadLimitReply(downloadResult.data)) throw new MakerworldDownloadLimitError();
-  if (downloadResult.status === 401 || downloadResult.status === 403) throw new MakerworldAuthError();
-  if (downloadResult.status !== 200 || !isRecord(downloadResult.data)) return null;
-  const body = downloadResult.data;
-  if (body.message !== "success" || typeof body.url !== "string" || !body.url.trim()) return null;
+  if (!body || body.message !== "success" || typeof body.url !== "string" || !body.url.trim()) return null;
 
   const meta = makerworldMetaFromDesign(design);
   const author = await completeMakerworldAuthor(meta.author, paceMs);

backend/tests/makerworldCloudApi.test.ts

@@ -48,6 +48,28 @@ describe("resolveMakerworldViaCloudApi", () => {
     expect(makerworldCaptchaCooloffActive()).toBe(false);
   });
 
+  it("downloads the raw files of a model with no print profiles, as one zip", async () => {
+    const fetched: string[] = [];
+    global.fetch = vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async (input) => {
+      const url = String(input);
+      fetched.push(url);
+      if (url.endsWith("/design-service/design/777")) {
+        return jsonResponse(200, { id: 777, title: "Paint holder", modelId: "", instances: [], defaultInstanceId: 0 });
+      }
+      if (url.includes("/design-service/design/777/model?modelType=all&type=download")) {
+        return jsonResponse(200, { name: "", url: "https://makerworld.bblmw.com/model/777/all.zip?key=k" });
+      }
+      return jsonResponse(404, {});
+    }) as unknown as typeof fetch;
+
+    const result = await resolveMakerworldViaCloudApi("777", null, "test-token");
+
+    expect(result?.downloadUrl).toBe("https://makerworld.bblmw.com/model/777/all.zip?key=k");
+    expect(result?.meta.title).toBe("Paint holder");
+    expect(result?.meta.makerworldProfile?.instanceId).toBeNull();
+    expect(fetched.some((url) => url.includes("/iot-service/"))).toBe(false);
+  });
+
   it("throws MakerworldDownloadLimitError on the daily download limit reply", async () => {
     global.fetch = vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async (input) => {
       const url = String(input);

backend/tests/makerworldProfilesImport.test.ts

@@ -3,6 +3,7 @@ import fs from "node:fs";
 import path from "node:path";
 import request from "supertest";
 import sharp from "sharp";
+import JSZip from "jszip";
 
 // No pacing or DNS here. config.ts reads the pace on load, so it's hoisted.
 vi.hoisted(() => {
@@ -284,3 +285,105 @@ describe("adding a profile the extension already resolved", () => {
     expect(fetched.filter((u) => u.startsWith("https://makerworld.com/en/models/")).length).toBe(pageFetchesAfterFirst);
   });
 });
+
+describe("a MakerWorld model with no print profiles", () => {
+  afterEach(() => {
+    global.fetch = originalFetch;
+  });
+
+  /** Its raw files, as MakerWorld zips them: in the designer's folders, two of them holding a file named alike. */
+  async function rawFilesZip(): Promise<Buffer> {
+    const zip = new JSZip();
+    zip.file("Holder_Top.stl", "solid top\nendsolid\n");
+    zip.file("2x2 Version/Holder_Bottom.stl", "solid bottom 2x2\nendsolid\n");
+    zip.file("3x2 Version/Holder_Bottom.stl", "solid bottom 3x2\nendsolid\n");
+    zip.file("readme.txt", "not a model");
+    return zip.generateAsync({ type: "nodebuffer" });
+  }
+
+  function mockRawFiles(designId: string) {
+    const zipUrl = `https://makerworld.bblmw.com/makerworld/model/${designId}/mszipfile/all.zip`;
+    global.fetch = vi.fn<(input: RequestInfo | URL) => Promise<Response>>(async (input) => {
+      const url = String(input);
+      if (url === `https://api.bambulab.com/v1/design-service/design/${designId}`) {
+        return json({
+          ...design(designId),
+          title: "Paint Bottle Holder",
+          modelId: "",
+          defaultInstanceId: 0,
+          instances: [],
+        });
+      }
+      if (url.startsWith(`https://api.bambulab.com/v1/design-service/design/${designId}/model?`)) {
+        return json({ name: "", url: zipUrl, highCompression: "" });
+      }
+      if (url === zipUrl) {
+        return new Response(await rawFilesZip(), {
+          headers: {
+            "content-type": "application/zip",
+            "content-disposition": "attachment;filename=Paint+Bottle+Holder.zip",
+          },
+        });
+      }
+      return new Response("not found", { status: 404 });
+    }) as unknown as typeof fetch;
+    return zipUrl;
+  }
+
+  const EXPECTED_FILES = ["Holder_Top.stl", "2x2 Version - Holder_Bottom.stl", "3x2 Version - Holder_Bottom.stl"];
+
+  async function filenamesOf(printId: string): Promise<string[]> {
+    const plates = await prisma.plate.findMany({ where: { printId }, orderBy: { position: "asc" } });
+    expect(plates.every((p) => p.sourceInstanceId === null)).toBe(true);
+    return plates.map((p) => p.filename);
+  }
+
+  it("imports its raw files, unpacked, one file each", async () => {
+    const designId = String((stamp % 1_000_000_000) + 100);
+    mockRawFiles(designId);
+    const res = await request(app)
+      .post("/api/import")
+      .set(auth())
+      .send({ url: `https://makerworld.com/en/models/${designId}-paint-holder`, makerworld_cookie: "token=t" });
+    expect(res.status).toBe(200);
+    expect(res.body.name).toBe("Paint Bottle Holder");
+    expect(await filenamesOf(res.body.id)).toEqual(EXPECTED_FILES);
+  });
+
+  it("unpacks the zip Grab resolved from the page's own download button too", async () => {
+    const designId = String((stamp % 1_000_000_000) + 101);
+    const zipUrl = mockRawFiles(designId);
+    const res = await request(app)
+      .post("/api/import")
+      .set({ ...auth(), "X-Thingport-Client": "grab" })
+      .send({
+        url: `https://makerworld.com/en/models/${designId}-paint-holder`,
+        resolved_download_url: zipUrl,
+        resolved_instance_id: null,
+        makerworld_design: { ...design(designId), title: "Paint Bottle Holder", instances: [] },
+      });
+    expect(res.status).toBe(200);
+    expect(await filenamesOf(res.body.id)).toEqual(EXPECTED_FILES);
+  });
+
+  it("imports its raw files from an every-profile import instead of failing", async () => {
+    const designId = String((stamp % 1_000_000_000) + 102);
+    mockRawFiles(designId);
+    const start = await request(app)
+      .post("/api/import/makerworld-profiles")
+      .set(auth())
+      .send({
+        url: `https://makerworld.com/en/models/${designId}-paint-holder`,
+        scope: "all",
+        makerworld_cookie: "token=t",
+      });
+    expect(start.status).toBe(202);
+    let job = (await request(app).get(`/api/import/jobs/${start.body.job_id}`).set(auth())).body;
+    for (let i = 0; i < 100 && job.status === "RUNNING"; i++) {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      job = (await request(app).get(`/api/import/jobs/${start.body.job_id}`).set(auth())).body;
+    }
+    expect(job).toMatchObject({ status: "DONE", total: 1, imported: 1, failed_count: 0 });
+    expect(await filenamesOf(job.result_print_id)).toEqual(EXPECTED_FILES);
+  });
+});