From feb67ab5003d2450d14948eca65dadab0a4abb1e Mon Sep 17 00:00:00 2001
From: Donkie <daniel.cf.hultgren@gmail.com>
Date: Sun, 13 Sep 2026 12:19:50 +0200
Subject: [PATCH] feat(external): page through SpoolmanDB search results

The filament pickers only ever showed the first 8 SpoolmanDB matches, so entries further down a broad search could not be reached. They now load 20 at a time and fetch the next page when the result list is scrolled to the bottom.

GET /external/filament/search takes an optional offset and returns the total match count in x-total-count, like the other list endpoints. Requests without offset behave as before. The lowercased search text is built once per catalog load instead of on every search, since paging means one scan per page.

Fixes #1136
---
 .../src/lib/api/externalSearch.svelte.ts      | 94 +++++++++++++++++++
 client_v2/src/lib/api/spoolSource.ts          | 12 ++-
 .../src/lib/components/AddSpoolModal.svelte   | 29 +++---
 .../library/ChangeFilamentModal.svelte        | 27 +++---
 spoolman/api/v1/externaldb.py                 | 13 ++-
 spoolman/externaldb.py                        | 33 +++----
 tests/test_externaldb_search.py               | 52 ++++++++++
 7 files changed, 207 insertions(+), 53 deletions(-)
 create mode 100644 client_v2/src/lib/api/externalSearch.svelte.ts
 create mode 100644 tests/test_externaldb_search.py

diff --git a/client_v2/src/lib/api/externalSearch.svelte.ts b/client_v2/src/lib/api/externalSearch.svelte.ts
new file mode 100644
index 00000000..cf99676c
--- /dev/null
+++ b/client_v2/src/lib/api/externalSearch.svelte.ts
@@ -0,0 +1,94 @@
+import { spoolSource } from './spoolSource';
+import type { ExternalFilament } from './external';
+
+/** Rows per request. Enough to overflow a picker's result list, so there is something to scroll. */
+const PAGE = 20;
+/** Distance from the bottom of the result list at which the next page is fetched. */
+const SCROLL_MARGIN = 120;
+
+/**
+ * The external-catalog half of a filament picker's search (#1136): `search` loads the
+ * first page of matches, and scrolling the result list towards its end appends the next.
+ * Shared by the add-spools and change-filament dialogs.
+ */
+export class ExternalSearch {
+	items = $state<ExternalFilament[]>([]);
+	error = $state(false);
+	loadingMore = $state(false);
+	#total = 0;
+	#query = '';
+	#loading = false;
+	// Bumped by every new search, so a page that lands after the query changed is dropped.
+	#req = 0;
+	#ctrl: AbortController | undefined;
+	// The result list, remembered from its first scroll event.
+	#list: HTMLElement | undefined;
+
+	async search(query: string): Promise<void> {
+		const req = this.#restart();
+		this.#query = query.trim();
+		this.#loading = true;
+		this.error = false;
+		try {
+			const page = await spoolSource.searchExternalFilaments(this.#query, PAGE, 0, this.#ctrl!.signal);
+			if (req !== this.#req) return;
+			this.items = page.items;
+			this.#total = page.total;
+			// A new query starts at the top: left scrolled down, the list would fetch
+			// its second page straight away.
+			this.#list?.scrollTo(0, 0);
+		} catch {
+			if (req !== this.#req) return;
+			this.items = [];
+			this.#total = 0;
+			this.error = true;
+		} finally {
+			if (req === this.#req) this.#loading = false;
+		}
+	}
+
+	async more(): Promise<void> {
+		if (this.#loading || this.loadingMore || this.error || this.items.length >= this.#total) return;
+		const req = this.#req;
+		this.loadingMore = true;
+		try {
+			const page = await spoolSource.searchExternalFilaments(
+				this.#query,
+				PAGE,
+				this.items.length,
+				this.#ctrl!.signal
+			);
+			if (req !== this.#req) return;
+			// A catalog sync between two pages shifts the offsets and can hand back a row
+			// that is already listed; a repeated id would break the keyed list.
+			const fresh = page.items.filter((f) => !this.items.some((e) => e.id === f.id));
+			this.items = [...this.items, ...fresh];
+			this.#total = page.total;
+		} catch (e) {
+			if (req === this.#req) console.error('Failed to load more external filaments', e);
+		} finally {
+			if (req === this.#req) this.loadingMore = false;
+		}
+	}
+
+	/** Forget the results, dropping any request still in flight. */
+	clear(): void {
+		this.#restart();
+		this.items = [];
+		this.#total = 0;
+	}
+
+	/** Scroll handler for the result list. An arrow function so it can be passed as-is. */
+	onscroll = (e: Event) => {
+		const el = (this.#list = e.currentTarget as HTMLElement);
+		if (el.scrollTop + el.clientHeight >= el.scrollHeight - SCROLL_MARGIN) void this.more();
+	};
+
+	#restart(): number {
+		this.#ctrl?.abort();
+		this.#ctrl = new AbortController();
+		this.#loading = false;
+		this.loadingMore = false;
+		return ++this.#req;
+	}
+}
diff --git a/client_v2/src/lib/api/spoolSource.ts b/client_v2/src/lib/api/spoolSource.ts
index 946f6b07..02e146b3 100644
--- a/client_v2/src/lib/api/spoolSource.ts
+++ b/client_v2/src/lib/api/spoolSource.ts
@@ -234,12 +234,18 @@ class HttpSpoolSource {
 		return filaments.map((m) => m.entity).slice(0, limit);
 	}
 
-	async searchExternalFilaments(query: string, limit = 8): Promise<ExternalFilament[]> {
+	async searchExternalFilaments(
+		query: string,
+		limit: number,
+		offset = 0,
+		signal?: AbortSignal
+	): Promise<Page<ExternalFilament>> {
 		// Filtering happens server-side (/external/filament/search) so the whole catalog
 		// — thousands of entries — never has to be downloaded to the client.
 		const q = query.trim();
-		if (!q) return [];
-		return getJson<ExternalFilament[]>('/external/filament/search', { query: q, limit });
+		if (!q) return { items: [], total: 0 };
+		const { items, total } = await getList('/external/filament/search', { query: q, limit, offset }, signal);
+		return { items: items as ExternalFilament[], total };
 	}
 
 	/**
diff --git a/client_v2/src/lib/components/AddSpoolModal.svelte b/client_v2/src/lib/components/AddSpoolModal.svelte
index 71f9bfcc..6487bd9d 100644
--- a/client_v2/src/lib/components/AddSpoolModal.svelte
+++ b/client_v2/src/lib/components/AddSpoolModal.svelte
@@ -17,6 +17,7 @@
 	import { fields } from '$lib/stores/fields.svelte';
 	import type { EntityType } from '$lib/api/fields';
 	import { externalColors, externalDirection, type ExternalFilament } from '$lib/api/external';
+	import { ExternalSearch } from '$lib/api/externalSearch.svelte';
 	import { roundGrams, weightAuto } from '$lib/utils/format';
 	import { numErr } from '$lib/utils/validate';
 	import {
@@ -49,9 +50,8 @@
 	let query = $state('');
 	let searchInput = $state<HTMLInputElement | undefined>();
 	let localResults = $state<Filament[]>([]);
-	let externalResults = $state<ExternalFilament[]>([]);
+	const extSearch = new ExternalSearch();
 	let searching = $state(false);
-	let extError = $state(false);
 	let chosen = $state<Choice | null>(null);
 	let creating = $state(false);
 	let submitting = $state(false);
@@ -123,17 +123,11 @@
 	}
 	async function runSearch() {
 		searching = true;
-		extError = false;
-		const [local, external] = await Promise.allSettled([
+		const [local] = await Promise.allSettled([
 			spoolSource.searchFilaments(query.trim()),
-			spoolSource.searchExternalFilaments(query.trim())
+			extSearch.search(query)
 		]);
 		localResults = local.status === 'fulfilled' ? local.value : [];
-		if (external.status === 'fulfilled') externalResults = external.value;
-		else {
-			externalResults = [];
-			extError = true;
-		}
 		searching = false;
 	}
 
@@ -339,7 +333,7 @@
 		step = 1;
 		query = '';
 		localResults = [];
-		externalResults = [];
+		extSearch.clear();
 		chosen = null;
 		creating = false;
 		cloneSource = null;
@@ -536,7 +530,7 @@
 						oninput={(e) => onSearch(e.currentTarget.value)}
 						placeholder={m['add.searchPlaceholder']({ name: serverInfo.externalDbName })}
 					/>
-					<div class="results">
+					<div class="results" onscroll={extSearch.onscroll}>
 						<div class="res-hdr">{m['add.yourCatalog']()}</div>
 						{#if searching && localResults.length === 0}
 							<div class="res-note">{m['add.searching']()}</div>
@@ -562,16 +556,16 @@
 						{/if}
 
 						<div class="res-hdr"><span class="hdr-note">{serverInfo.externalDbName}</span></div>
-						{#if extError}
+						{#if extSearch.error}
 							<div class="res-note">{m['add.dbUnavailable']({ name: serverInfo.externalDbName })}</div>
-						{:else if searching && externalResults.length === 0}
+						{:else if searching && extSearch.items.length === 0}
 							<div class="res-note">{m['add.searching']()}</div>
-						{:else if externalResults.length === 0}
+						{:else if extSearch.items.length === 0}
 							<div class="res-note">
 								{query.trim() ? m['add.typeToSearchMatches']() : m['add.typeToSearchAll']()}
 							</div>
 						{:else}
-							{#each externalResults as ext (ext.id)}
+							{#each extSearch.items as ext (ext.id)}
 								<button class="res-item" onclick={() => choose({ source: 'external', ext })}>
 									<Swatch
 										colors={externalColors(ext)}
@@ -593,6 +587,9 @@
 									<span class="tag external">{serverInfo.externalDbName}</span>
 								</button>
 							{/each}
+							{#if extSearch.loadingMore}
+								<div class="res-note">{m['add.searching']()}</div>
+							{/if}
 						{/if}
 					</div>
 
diff --git a/client_v2/src/lib/components/library/ChangeFilamentModal.svelte b/client_v2/src/lib/components/library/ChangeFilamentModal.svelte
index 23bbcbe0..0330ff05 100644
--- a/client_v2/src/lib/components/library/ChangeFilamentModal.svelte
+++ b/client_v2/src/lib/components/library/ChangeFilamentModal.svelte
@@ -20,6 +20,7 @@
 	import { spoolSource } from '$lib/api/spoolSource';
 	import { fields } from '$lib/stores/fields.svelte';
 	import { externalColors, externalDirection, type ExternalFilament } from '$lib/api/external';
+	import { ExternalSearch } from '$lib/api/externalSearch.svelte';
 	import { toasts } from '$lib/stores/toasts.svelte';
 	import { weightAuto } from '$lib/utils/format';
 	import { parseDecimal } from '$lib/utils/numeric';
@@ -53,9 +54,8 @@
 	let query = $state('');
 	let searchInput = $state<HTMLInputElement | undefined>();
 	let localResults = $state<Filament[]>([]);
-	let externalResults = $state<ExternalFilament[]>([]);
+	const extSearch = new ExternalSearch();
 	let searching = $state(false);
-	let extError = $state(false);
 	let chosen = $state<Choice | null>(null);
 	let busy = $state(false);
 
@@ -187,17 +187,11 @@
 	}
 	async function runSearch() {
 		searching = true;
-		extError = false;
-		const [local, external] = await Promise.allSettled([
+		const [local] = await Promise.allSettled([
 			spoolSource.searchFilaments(query.trim()),
-			spoolSource.searchExternalFilaments(query.trim())
+			extSearch.search(query)
 		]);
 		localResults = local.status === 'fulfilled' ? local.value : [];
-		if (external.status === 'fulfilled') externalResults = external.value;
-		else {
-			externalResults = [];
-			extError = true;
-		}
 		searching = false;
 	}
 
@@ -434,7 +428,7 @@
 						placeholder={m['add.searchPlaceholder']({ name: serverInfo.externalDbName })}
 						aria-label={m['add.searchPlaceholder']({ name: serverInfo.externalDbName })}
 					/>
-					<div class="results">
+					<div class="results" onscroll={extSearch.onscroll}>
 						<div class="res-hdr">{m['add.yourCatalog']()}</div>
 						{#if searching && localResults.length === 0}
 							<div class="res-note">{m['add.searching']()}</div>
@@ -465,16 +459,16 @@
 						{/if}
 
 						<div class="res-hdr"><span class="hdr-note">{serverInfo.externalDbName}</span></div>
-						{#if extError}
+						{#if extSearch.error}
 							<div class="res-note">{m['add.dbUnavailable']({ name: serverInfo.externalDbName })}</div>
-						{:else if searching && externalResults.length === 0}
+						{:else if searching && extSearch.items.length === 0}
 							<div class="res-note">{m['add.searching']()}</div>
-						{:else if externalResults.length === 0}
+						{:else if extSearch.items.length === 0}
 							<div class="res-note">
 								{query.trim() ? m['add.typeToSearchMatches']() : m['add.typeToSearchAll']()}
 							</div>
 						{:else}
-							{#each externalResults as ext (ext.id)}
+							{#each extSearch.items as ext (ext.id)}
 								<button
 									class="res-item"
 									class:sel={chosen?.source === 'external' && chosen.ext.id === ext.id}
@@ -496,6 +490,9 @@
 									<span class="tag external">{serverInfo.externalDbName}</span>
 								</button>
 							{/each}
+							{#if extSearch.loadingMore}
+								<div class="res-note">{m['add.searching']()}</div>
+							{/if}
 						{/if}
 					</div>
 					<button class="create-new" onclick={startCreate}>
diff --git a/spoolman/api/v1/externaldb.py b/spoolman/api/v1/externaldb.py
index 058e74eb..74feebf1 100644
--- a/spoolman/api/v1/externaldb.py
+++ b/spoolman/api/v1/externaldb.py
@@ -3,7 +3,7 @@
 import logging
 from typing import Annotated
 
-from fastapi import APIRouter, Query
+from fastapi import APIRouter, Query, Response
 from fastapi.responses import FileResponse
 
 from spoolman.externaldb import (
@@ -40,6 +40,7 @@ async def filaments() -> FileResponse:
     response_model_exclude_none=True,
 )
 async def search_external_filaments(
+    response: Response,
     query: Annotated[
         str,
         Query(
@@ -51,13 +52,19 @@ async def search_external_filaments(
         int,
         Query(ge=1, le=100, description="Maximum number of results to return."),
     ] = 20,
+    offset: Annotated[
+        int,
+        Query(ge=0, description="Number of matches to skip, for paging through the results."),
+    ] = 0,
 ) -> list[ExternalFilament]:
     """Search the external filament catalog.
 
     Filters server-side so clients don't have to download the entire catalog just to
-    search it.
+    search it. The total number of matches is returned in the x-total-count header.
     """
-    return search_filaments(query, limit)
+    items, total = search_filaments(query, limit, offset)
+    response.headers["x-total-count"] = str(total)
+    return items
 
 
 @router.get(
diff --git a/spoolman/externaldb.py b/spoolman/externaldb.py
index c38a2121..61579862 100644
--- a/spoolman/externaldb.py
+++ b/spoolman/externaldb.py
@@ -183,11 +183,16 @@ def get_materials_file() -> Path:
 # because mtime alone can miss a rewrite: filesystems with 1-second mtime granularity
 # report the same stamp for two writes within the same second, which would leave us
 # serving the previous catalog until the next sync.
-_filaments_cache: tuple[tuple[float, int], list[ExternalFilament]] | None = None
+_filaments_cache: tuple[tuple[float, int], list[tuple[str, ExternalFilament]]] | None = None
 
 
-def _load_filaments() -> list[ExternalFilament]:
-    """Load and parse the cached filament catalog, memoized by the file's mtime and size."""
+def _load_filaments() -> list[tuple[str, ExternalFilament]]:
+    """Load and parse the cached filament catalog, memoized by the file's mtime and size.
+
+    Each filament is paired with its lowercased search text, built once here instead of on
+    every search: on a large catalog (the community one has ~50k entries) that halves the
+    cost of a scan, and paging means a scan per page.
+    """
     global _filaments_cache  # noqa: PLW0603
     path = get_filaments_file()
     if not path.exists():
@@ -195,30 +200,26 @@ def _load_filaments() -> list[ExternalFilament]:
     stat = path.stat()
     key = (stat.st_mtime, stat.st_size)
     if _filaments_cache is None or _filaments_cache[0] != key:
-        _filaments_cache = (key, _parse_filaments_from_bytes(path.read_bytes()).root)
+        filaments = _parse_filaments_from_bytes(path.read_bytes()).root
+        _filaments_cache = (key, [(f"{f.manufacturer} {f.name} {f.material}".lower(), f) for f in filaments])
     return _filaments_cache[1]
 
 
-def search_filaments(query: str, limit: int) -> list[ExternalFilament]:
+def search_filaments(query: str, limit: int, offset: int = 0) -> tuple[list[ExternalFilament], int]:
     """Search the external filament catalog server-side.
 
     Keeps the same semantics as the client-side search it replaces: the query is split
     into whitespace-separated words and every word must appear (case-insensitively) as a
     substring of the filament's "manufacturer name material" text. Results preserve
-    catalog order and are capped at `limit`, so the entire catalog never has to be sent
-    to the client.
+    catalog order, so `offset` and `limit` page through them consistently.
+
+    Returns the requested page of matches and the total number of matches.
     """
     words = query.lower().split()
     if not words:
-        return []
-    results: list[ExternalFilament] = []
-    for filament in _load_filaments():
-        haystack = f"{filament.manufacturer} {filament.name} {filament.material}".lower()
-        if all(word in haystack for word in words):
-            results.append(filament)
-            if len(results) >= limit:
-                break
-    return results
+        return [], 0
+    matches = [filament for haystack, filament in _load_filaments() if all(word in haystack for word in words)]
+    return matches[offset : offset + limit], len(matches)
 
 
 async def _sync() -> None:
diff --git a/tests/test_externaldb_search.py b/tests/test_externaldb_search.py
new file mode 100644
index 00000000..bd4b6744
--- /dev/null
+++ b/tests/test_externaldb_search.py
@@ -0,0 +1,52 @@
+"""Tests for searching and paging through the external filament catalog."""
+
+import json
+from pathlib import Path
+
+import pytest
+
+from spoolman import externaldb
+
+
+def filament(i: int, manufacturer: str, name: str, material: str = "PLA") -> dict:
+    return {
+        "id": f"filament_{i}",
+        "manufacturer": manufacturer,
+        "name": name,
+        "material": material,
+        "density": 1.24,
+        "weight": 1000,
+        "diameter": 1.75,
+    }
+
+
+@pytest.fixture(autouse=True)
+def catalog(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    rows = [filament(i, "Elegoo", f"White {i}") for i in range(25)]
+    rows.insert(10, filament(100, "Polymaker", "PolyLite Black", "PETG"))
+    path = tmp_path / "filaments.json"
+    path.write_text(json.dumps(rows))
+    monkeypatch.setattr(externaldb, "get_filaments_file", lambda: path)
+    monkeypatch.setattr(externaldb, "_filaments_cache", None)
+
+
+def test_pages_cover_every_match_once_in_catalog_order():
+    ids = []
+    for offset in (0, 10, 20):
+        items, total = externaldb.search_filaments("elegoo white", limit=10, offset=offset)
+        assert total == 25
+        ids += [f.id for f in items]
+    assert ids == [f"filament_{i}" for i in range(25)]
+
+
+def test_an_offset_past_the_end_is_empty_but_still_counts():
+    assert externaldb.search_filaments("elegoo", limit=10, offset=30) == ([], 25)
+
+
+def test_every_word_must_match_case_insensitively():
+    items, total = externaldb.search_filaments("POLYMAKER petg", limit=10)
+    assert ([f.id for f in items], total) == (["filament_100"], 1)
+
+
+def test_a_blank_query_matches_nothing():
+    assert externaldb.search_filaments("   ", limit=10) == ([], 0)