Merge pull request #1161 from Donkie/feat-1136-paginated-external-search

Page through SpoolmanDB search results in the filament pickers

client_v2/src/lib/api/externalSearch.svelte.ts

@@ -0,0 +1,94 @@
+import { spoolSource } from './spoolSource';
+import type { ExternalFilament } from './external';
+
+/** Rows per request. Enough to overflow a picker's result list, so there is something to scroll. */
+const PAGE = 20;
+/** Distance from the bottom of the result list at which the next page is fetched. */
+const SCROLL_MARGIN = 120;
+
+/**
+ * The external-catalog half of a filament picker's search (#1136): `search` loads the
+ * first page of matches, and scrolling the result list towards its end appends the next.
+ * Shared by the add-spools and change-filament dialogs.
+ */
+export class ExternalSearch {
+	items = $state<ExternalFilament[]>([]);
+	error = $state(false);
+	loadingMore = $state(false);
+	#total = 0;
+	#query = '';
+	#loading = false;
+	// Bumped by every new search, so a page that lands after the query changed is dropped.
+	#req = 0;
+	#ctrl: AbortController | undefined;
+	// The result list, remembered from its first scroll event.
+	#list: HTMLElement | undefined;
+
+	async search(query: string): Promise<void> {
+		const req = this.#restart();
+		this.#query = query.trim();
+		this.#loading = true;
+		this.error = false;
+		try {
+			const page = await spoolSource.searchExternalFilaments(this.#query, PAGE, 0, this.#ctrl!.signal);
+			if (req !== this.#req) return;
+			this.items = page.items;
+			this.#total = page.total;
+			// A new query starts at the top: left scrolled down, the list would fetch
+			// its second page straight away.
+			this.#list?.scrollTo(0, 0);
+		} catch {
+			if (req !== this.#req) return;
+			this.items = [];
+			this.#total = 0;
+			this.error = true;
+		} finally {
+			if (req === this.#req) this.#loading = false;
+		}
+	}
+
+	async more(): Promise<void> {
+		if (this.#loading || this.loadingMore || this.error || this.items.length >= this.#total) return;
+		const req = this.#req;
+		this.loadingMore = true;
+		try {
+			const page = await spoolSource.searchExternalFilaments(
+				this.#query,
+				PAGE,
+				this.items.length,
+				this.#ctrl!.signal
+			);
+			if (req !== this.#req) return;
+			// A catalog sync between two pages shifts the offsets and can hand back a row
+			// that is already listed; a repeated id would break the keyed list.
+			const fresh = page.items.filter((f) => !this.items.some((e) => e.id === f.id));
+			this.items = [...this.items, ...fresh];
+			this.#total = page.total;
+		} catch (e) {
+			if (req === this.#req) console.error('Failed to load more external filaments', e);
+		} finally {
+			if (req === this.#req) this.loadingMore = false;
+		}
+	}
+
+	/** Forget the results, dropping any request still in flight. */
+	clear(): void {
+		this.#restart();
+		this.items = [];
+		this.#total = 0;
+	}
+
+	/** Scroll handler for the result list. An arrow function so it can be passed as-is. */
+	onscroll = (e: Event) => {
+		const el = (this.#list = e.currentTarget as HTMLElement);
+		if (el.scrollTop + el.clientHeight >= el.scrollHeight - SCROLL_MARGIN) void this.more();
+	};
+
+	#restart(): number {
+		this.#ctrl?.abort();
+		this.#ctrl = new AbortController();
+		this.#loading = false;
+		this.loadingMore = false;
+		return ++this.#req;
+	}
+}

client_v2/src/lib/api/spoolSource.ts

@@ -234,12 +234,18 @@ class HttpSpoolSource {
 		return filaments.map((m) => m.entity).slice(0, limit);
 	}
 
-	async searchExternalFilaments(query: string, limit = 8): Promise<ExternalFilament[]> {
+	async searchExternalFilaments(
+		query: string,
+		limit: number,
+		offset = 0,
+		signal?: AbortSignal
+	): Promise<Page<ExternalFilament>> {
 		// Filtering happens server-side (/external/filament/search) so the whole catalog
 		// — thousands of entries — never has to be downloaded to the client.
 		const q = query.trim();
-		if (!q) return [];
-		return getJson<ExternalFilament[]>('/external/filament/search', { query: q, limit });
+		if (!q) return { items: [], total: 0 };
+		const { items, total } = await getList('/external/filament/search', { query: q, limit, offset }, signal);
+		return { items: items as ExternalFilament[], total };
 	}
 
 	/**

client_v2/src/lib/components/AddSpoolModal.svelte

@@ -17,6 +17,7 @@
 	import { fields } from '$lib/stores/fields.svelte';
 	import type { EntityType } from '$lib/api/fields';
 	import { externalColors, externalDirection, type ExternalFilament } from '$lib/api/external';
+	import { ExternalSearch } from '$lib/api/externalSearch.svelte';
 	import { roundGrams, weightAuto } from '$lib/utils/format';
 	import { numErr } from '$lib/utils/validate';
 	import {
@@ -49,9 +50,8 @@
 	let query = $state('');
 	let searchInput = $state<HTMLInputElement | undefined>();
 	let localResults = $state<Filament[]>([]);
-	let externalResults = $state<ExternalFilament[]>([]);
+	const extSearch = new ExternalSearch();
 	let searching = $state(false);
-	let extError = $state(false);
 	let chosen = $state<Choice | null>(null);
 	let creating = $state(false);
 	let submitting = $state(false);
@@ -123,17 +123,11 @@
 	}
 	async function runSearch() {
 		searching = true;
-		extError = false;
-		const [local, external] = await Promise.allSettled([
+		const [local] = await Promise.allSettled([
 			spoolSource.searchFilaments(query.trim()),
-			spoolSource.searchExternalFilaments(query.trim())
+			extSearch.search(query)
 		]);
 		localResults = local.status === 'fulfilled' ? local.value : [];
-		if (external.status === 'fulfilled') externalResults = external.value;
-		else {
-			externalResults = [];
-			extError = true;
-		}
 		searching = false;
 	}
 
@@ -339,7 +333,7 @@
 		step = 1;
 		query = '';
 		localResults = [];
-		externalResults = [];
+		extSearch.clear();
 		chosen = null;
 		creating = false;
 		cloneSource = null;
@@ -536,7 +530,7 @@
 						oninput={(e) => onSearch(e.currentTarget.value)}
 						placeholder={m['add.searchPlaceholder']({ name: serverInfo.externalDbName })}
 					/>
-					<div class="results">
+					<div class="results" onscroll={extSearch.onscroll}>
 						<div class="res-hdr">{m['add.yourCatalog']()}</div>
 						{#if searching && localResults.length === 0}
 							<div class="res-note">{m['add.searching']()}</div>
@@ -562,16 +556,16 @@
 						{/if}
 
 						<div class="res-hdr"><span class="hdr-note">{serverInfo.externalDbName}</span></div>
-						{#if extError}
+						{#if extSearch.error}
 							<div class="res-note">{m['add.dbUnavailable']({ name: serverInfo.externalDbName })}</div>
-						{:else if searching && externalResults.length === 0}
+						{:else if searching && extSearch.items.length === 0}
 							<div class="res-note">{m['add.searching']()}</div>
-						{:else if externalResults.length === 0}
+						{:else if extSearch.items.length === 0}
 							<div class="res-note">
 								{query.trim() ? m['add.typeToSearchMatches']() : m['add.typeToSearchAll']()}
 							</div>
 						{:else}
-							{#each externalResults as ext (ext.id)}
+							{#each extSearch.items as ext (ext.id)}
 								<button class="res-item" onclick={() => choose({ source: 'external', ext })}>
 									<Swatch
 										colors={externalColors(ext)}
@@ -593,6 +587,9 @@
 									<span class="tag external">{serverInfo.externalDbName}</span>
 								</button>
 							{/each}
+							{#if extSearch.loadingMore}
+								<div class="res-note">{m['add.searching']()}</div>
+							{/if}
 						{/if}
 					</div>
 

client_v2/src/lib/components/library/ChangeFilamentModal.svelte

@@ -20,6 +20,7 @@
 	import { spoolSource } from '$lib/api/spoolSource';
 	import { fields } from '$lib/stores/fields.svelte';
 	import { externalColors, externalDirection, type ExternalFilament } from '$lib/api/external';
+	import { ExternalSearch } from '$lib/api/externalSearch.svelte';
 	import { toasts } from '$lib/stores/toasts.svelte';
 	import { weightAuto } from '$lib/utils/format';
 	import { parseDecimal } from '$lib/utils/numeric';
@@ -53,9 +54,8 @@
 	let query = $state('');
 	let searchInput = $state<HTMLInputElement | undefined>();
 	let localResults = $state<Filament[]>([]);
-	let externalResults = $state<ExternalFilament[]>([]);
+	const extSearch = new ExternalSearch();
 	let searching = $state(false);
-	let extError = $state(false);
 	let chosen = $state<Choice | null>(null);
 	let busy = $state(false);
 
@@ -187,17 +187,11 @@
 	}
 	async function runSearch() {
 		searching = true;
-		extError = false;
-		const [local, external] = await Promise.allSettled([
+		const [local] = await Promise.allSettled([
 			spoolSource.searchFilaments(query.trim()),
-			spoolSource.searchExternalFilaments(query.trim())
+			extSearch.search(query)
 		]);
 		localResults = local.status === 'fulfilled' ? local.value : [];
-		if (external.status === 'fulfilled') externalResults = external.value;
-		else {
-			externalResults = [];
-			extError = true;
-		}
 		searching = false;
 	}
 
@@ -434,7 +428,7 @@
 						placeholder={m['add.searchPlaceholder']({ name: serverInfo.externalDbName })}
 						aria-label={m['add.searchPlaceholder']({ name: serverInfo.externalDbName })}
 					/>
-					<div class="results">
+					<div class="results" onscroll={extSearch.onscroll}>
 						<div class="res-hdr">{m['add.yourCatalog']()}</div>
 						{#if searching && localResults.length === 0}
 							<div class="res-note">{m['add.searching']()}</div>
@@ -465,16 +459,16 @@
 						{/if}
 
 						<div class="res-hdr"><span class="hdr-note">{serverInfo.externalDbName}</span></div>
-						{#if extError}
+						{#if extSearch.error}
 							<div class="res-note">{m['add.dbUnavailable']({ name: serverInfo.externalDbName })}</div>
-						{:else if searching && externalResults.length === 0}
+						{:else if searching && extSearch.items.length === 0}
 							<div class="res-note">{m['add.searching']()}</div>
-						{:else if externalResults.length === 0}
+						{:else if extSearch.items.length === 0}
 							<div class="res-note">
 								{query.trim() ? m['add.typeToSearchMatches']() : m['add.typeToSearchAll']()}
 							</div>
 						{:else}
-							{#each externalResults as ext (ext.id)}
+							{#each extSearch.items as ext (ext.id)}
 								<button
 									class="res-item"
 									class:sel={chosen?.source === 'external' && chosen.ext.id === ext.id}
@@ -496,6 +490,9 @@
 									<span class="tag external">{serverInfo.externalDbName}</span>
 								</button>
 							{/each}
+							{#if extSearch.loadingMore}
+								<div class="res-note">{m['add.searching']()}</div>
+							{/if}
 						{/if}
 					</div>
 					<button class="create-new" onclick={startCreate}>

spoolman/api/v1/externaldb.py

@@ -3,7 +3,7 @@
 import logging
 from typing import Annotated
 
-from fastapi import APIRouter, Query
+from fastapi import APIRouter, Query, Response
 from fastapi.responses import FileResponse
 
 from spoolman.externaldb import (
@@ -40,6 +40,7 @@ async def filaments() -> FileResponse:
     response_model_exclude_none=True,
 )
 async def search_external_filaments(
+    response: Response,
     query: Annotated[
         str,
         Query(
@@ -51,13 +52,19 @@ async def search_external_filaments(
         int,
         Query(ge=1, le=100, description="Maximum number of results to return."),
     ] = 20,
+    offset: Annotated[
+        int,
+        Query(ge=0, description="Number of matches to skip, for paging through the results."),
+    ] = 0,
 ) -> list[ExternalFilament]:
     """Search the external filament catalog.
 
     Filters server-side so clients don't have to download the entire catalog just to
-    search it.
+    search it. The total number of matches is returned in the x-total-count header.
     """
-    return search_filaments(query, limit)
+    items, total = search_filaments(query, limit, offset)
+    response.headers["x-total-count"] = str(total)
+    return items
 
 
 @router.get(

spoolman/externaldb.py

@@ -183,11 +183,16 @@ def get_materials_file() -> Path:
 # because mtime alone can miss a rewrite: filesystems with 1-second mtime granularity
 # report the same stamp for two writes within the same second, which would leave us
 # serving the previous catalog until the next sync.
-_filaments_cache: tuple[tuple[float, int], list[ExternalFilament]] | None = None
+_filaments_cache: tuple[tuple[float, int], list[tuple[str, ExternalFilament]]] | None = None
 
 
-def _load_filaments() -> list[ExternalFilament]:
-    """Load and parse the cached filament catalog, memoized by the file's mtime and size."""
+def _load_filaments() -> list[tuple[str, ExternalFilament]]:
+    """Load and parse the cached filament catalog, memoized by the file's mtime and size.
+
+    Each filament is paired with its lowercased search text, built once here instead of on
+    every search: on a large catalog (the community one has ~50k entries) that halves the
+    cost of a scan, and paging means a scan per page.
+    """
     global _filaments_cache  # noqa: PLW0603
     path = get_filaments_file()
     if not path.exists():
@@ -195,30 +200,26 @@ def _load_filaments() -> list[ExternalFilament]:
     stat = path.stat()
     key = (stat.st_mtime, stat.st_size)
     if _filaments_cache is None or _filaments_cache[0] != key:
-        _filaments_cache = (key, _parse_filaments_from_bytes(path.read_bytes()).root)
+        filaments = _parse_filaments_from_bytes(path.read_bytes()).root
+        _filaments_cache = (key, [(f"{f.manufacturer} {f.name} {f.material}".lower(), f) for f in filaments])
     return _filaments_cache[1]
 
 
-def search_filaments(query: str, limit: int) -> list[ExternalFilament]:
+def search_filaments(query: str, limit: int, offset: int = 0) -> tuple[list[ExternalFilament], int]:
     """Search the external filament catalog server-side.
 
     Keeps the same semantics as the client-side search it replaces: the query is split
     into whitespace-separated words and every word must appear (case-insensitively) as a
     substring of the filament's "manufacturer name material" text. Results preserve
-    catalog order and are capped at `limit`, so the entire catalog never has to be sent
-    to the client.
+    catalog order, so `offset` and `limit` page through them consistently.
+
+    Returns the requested page of matches and the total number of matches.
     """
     words = query.lower().split()
     if not words:
-        return []
-    results: list[ExternalFilament] = []
-    for filament in _load_filaments():
-        haystack = f"{filament.manufacturer} {filament.name} {filament.material}".lower()
-        if all(word in haystack for word in words):
-            results.append(filament)
-            if len(results) >= limit:
-                break
-    return results
+        return [], 0
+    matches = [filament for haystack, filament in _load_filaments() if all(word in haystack for word in words)]
+    return matches[offset : offset + limit], len(matches)
 
 
 async def _sync() -> None:

tests/test_externaldb_search.py

@@ -0,0 +1,52 @@
+"""Tests for searching and paging through the external filament catalog."""
+
+import json
+from pathlib import Path
+
+import pytest
+
+from spoolman import externaldb
+
+
+def filament(i: int, manufacturer: str, name: str, material: str = "PLA") -> dict:
+    return {
+        "id": f"filament_{i}",
+        "manufacturer": manufacturer,
+        "name": name,
+        "material": material,
+        "density": 1.24,
+        "weight": 1000,
+        "diameter": 1.75,
+    }
+
+
+@pytest.fixture(autouse=True)
+def catalog(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    rows = [filament(i, "Elegoo", f"White {i}") for i in range(25)]
+    rows.insert(10, filament(100, "Polymaker", "PolyLite Black", "PETG"))
+    path = tmp_path / "filaments.json"
+    path.write_text(json.dumps(rows))
+    monkeypatch.setattr(externaldb, "get_filaments_file", lambda: path)
+    monkeypatch.setattr(externaldb, "_filaments_cache", None)
+
+
+def test_pages_cover_every_match_once_in_catalog_order():
+    ids = []
+    for offset in (0, 10, 20):
+        items, total = externaldb.search_filaments("elegoo white", limit=10, offset=offset)
+        assert total == 25
+        ids += [f.id for f in items]
+    assert ids == [f"filament_{i}" for i in range(25)]
+
+
+def test_an_offset_past_the_end_is_empty_but_still_counts():
+    assert externaldb.search_filaments("elegoo", limit=10, offset=30) == ([], 25)
+
+
+def test_every_word_must_match_case_insensitively():
+    items, total = externaldb.search_filaments("POLYMAKER petg", limit=10)
+    assert ([f.id for f in items], total) == (["filament_100"], 1)
+
+
+def test_a_blank_query_matches_nothing():
+    assert externaldb.search_filaments("   ", limit=10) == ([], 0)