From 7ea89dc347b4a69bbb93883d57adeedfa8152f81 Mon Sep 17 00:00:00 2001 From: Donkie Date: Mon, 13 Jul 2026 19:33:52 +0200 Subject: [PATCH] Add cross-entity search to v2 client Implements the search feature for client_v2: a new backend search endpoint plus a results-panel UI, replacing the old query box that only filtered the library list by filament name. Backend (additive, API v1 backward compatibility preserved): - New GET /api/v1/search returning categorized spool/filament/vendor results, each annotated with the field that matched, plus an is_color_query flag. - Case-insensitive text search over native text fields and text/choice extra fields; exact spool-id match for numeric queries; color-similarity search (CSS color name or hex) via find_by_color, ordered closest first. - spoolman/colors.py resolves the CSS4 color names / hex codes to hex. - Integration tests (run across SQLite/PostgreSQL/MySQL/CockroachDB in CI). Frontend (client_v2): - SearchBox overlay with Spools/Filaments/Manufacturers sections, per-result match badges, keyboard navigation, a color-similarity threshold slider (persisted to localStorage), and an English-only color-name hint. - Clicking a result opens the matching inspector on the Library page. - Removes the old q -> filament.name inline list filtering. Claude-Session: https://claude.ai/code/session_01Ne7RsfHejKZX97uAq3aVMn --- client_v2/src/lib/api/query.ts | 3 - client_v2/src/lib/api/search.ts | 64 +++ client_v2/src/lib/api/spoolSource.ts | 2 - client_v2/src/lib/api/types.ts | 28 +- .../src/lib/components/SearchInput.svelte | 6 +- client_v2/src/lib/components/TopBar.svelte | 21 +- .../lib/components/library/SearchBox.svelte | 390 ++++++++++++++++++ client_v2/src/lib/library/params.ts | 27 +- client_v2/static/locales/en/common.json | 22 + spoolman/api/v1/models.py | 51 +++ spoolman/api/v1/router.py | 3 +- spoolman/api/v1/search.py | 96 +++++ spoolman/colors.py | 181 ++++++++ spoolman/database/search.py | 321 ++++++++++++++ tests_integration/tests/search/__init__.py | 1 + tests_integration/tests/search/test_search.py | 166 ++++++++ 16 files changed, 1345 insertions(+), 37 deletions(-) create mode 100644 client_v2/src/lib/api/search.ts create mode 100644 client_v2/src/lib/components/library/SearchBox.svelte create mode 100644 spoolman/api/v1/search.py create mode 100644 spoolman/colors.py create mode 100644 spoolman/database/search.py create mode 100644 tests_integration/tests/search/__init__.py create mode 100644 tests_integration/tests/search/test_search.py diff --git a/client_v2/src/lib/api/query.ts b/client_v2/src/lib/api/query.ts index 25d5e031..c9d2efe9 100644 --- a/client_v2/src/lib/api/query.ts +++ b/client_v2/src/lib/api/query.ts @@ -44,7 +44,6 @@ export function buildGroupQuery(state: LibraryState): GroupQuery { return { field, filters: currentFilters(state), - search: state.query.trim() || undefined, sort: [{ field: groupField, dir }], limit: state.pageSize, offset: (state.page - 1) * state.pageSize, @@ -62,7 +61,6 @@ export function buildScopedSpoolQuery(state: LibraryState, group: GroupSummary, return { filters: currentFilters(state), sort, - search: state.query.trim() || undefined, groupScope: { field: group.field, key: group.key }, limit, offset: 0, @@ -79,7 +77,6 @@ export function buildFlatSpoolQuery(state: LibraryState): SpoolQuery { return { filters: currentFilters(state), sort, - search: state.query.trim() || undefined, limit: state.pageSize, offset: (state.page - 1) * state.pageSize, lowThreshold: settings.lowThreshold diff --git a/client_v2/src/lib/api/search.ts b/client_v2/src/lib/api/search.ts new file mode 100644 index 00000000..5a37dbc6 --- /dev/null +++ b/client_v2/src/lib/api/search.ts @@ -0,0 +1,64 @@ +import type { SearchMatch, SearchResults } from './types'; +import type { Filament, Spool, Vendor } from '$lib/types'; +import { API_BASE } from './config'; +import { mapFilament, mapSpool, mapVendor } from './map'; +import { inventory } from '$lib/stores/inventory.svelte'; + +// Cross-entity search against GET /api/v1/search. Like the spool source, every +// entity it fetches is upserted into the reactive cache so that clicking a result +// (which opens its inspector) has the data ready immediately. + +/* eslint-disable @typescript-eslint/no-explicit-any */ +type Json = Record; + +const EMPTY: SearchResults = { spools: [], filaments: [], vendors: [], isColorQuery: false }; + +function mapSpoolHit(m: Json): SearchMatch { + const raw: Json = m.spool ?? {}; + if (raw.filament) { + inventory.upsertFilament(mapFilament(raw.filament)); + if (raw.filament.vendor) inventory.upsertVendor(mapVendor(raw.filament.vendor)); + } + const spool = mapSpool(raw); + inventory.upsertSpool(spool); + return { entity: spool, matchField: m.match_field }; +} + +function mapFilamentHit(m: Json): SearchMatch { + const raw: Json = m.filament ?? {}; + if (raw.vendor) inventory.upsertVendor(mapVendor(raw.vendor)); + const filament = mapFilament(raw); + inventory.upsertFilament(filament); + return { entity: filament, matchField: m.match_field }; +} + +function mapVendorHit(m: Json): SearchMatch { + const vendor = mapVendor(m.vendor ?? {}); + inventory.upsertVendor(vendor); + return { entity: vendor, matchField: m.match_field }; +} + +/** + * Run a cross-entity search. `threshold` is the color-similarity threshold used + * when the query is a color. Pass an AbortSignal to cancel a superseded request. + */ +export async function searchAll( + query: string, + threshold: number, + signal?: AbortSignal +): Promise { + const q = query.trim(); + if (!q) return EMPTY; + + const params = new URLSearchParams({ q, color_similarity_threshold: String(threshold) }); + const res = await fetch(`${API_BASE}/search?${params.toString()}`, { signal }); + if (!res.ok) throw new Error(`GET /search → ${res.status}`); + const data = (await res.json()) as Json; + + return { + spools: (data.spools ?? []).map(mapSpoolHit), + filaments: (data.filaments ?? []).map(mapFilamentHit), + vendors: (data.vendors ?? []).map(mapVendorHit), + isColorQuery: !!data.is_color_query + }; +} diff --git a/client_v2/src/lib/api/spoolSource.ts b/client_v2/src/lib/api/spoolSource.ts index df7c0202..ef78b166 100644 --- a/client_v2/src/lib/api/spoolSource.ts +++ b/client_v2/src/lib/api/spoolSource.ts @@ -98,7 +98,6 @@ class HttpSpoolSource { offset: query.offset }; applyFilters(params, query.filters); - if (query.search) params['filament.name'] = query.search; const { items, total } = await getList('/spool/group', params); for (const g of items as Json[]) { @@ -119,7 +118,6 @@ class HttpSpoolSource { }; applyFilters(params, query.filters); scopeParams(params, query.groupScope); - if (query.search) params['filament.name'] = query.search; const { items, total } = await getList('/spool', params); return { items: cacheSpools(items), total }; diff --git a/client_v2/src/lib/api/types.ts b/client_v2/src/lib/api/types.ts index b35aeb49..6cafc213 100644 --- a/client_v2/src/lib/api/types.ts +++ b/client_v2/src/lib/api/types.ts @@ -1,4 +1,4 @@ -import type { Spool } from '$lib/types'; +import type { Filament, Spool, Vendor } from '$lib/types'; // The query/data-source contract. Filtering, sorting, aggregation and // pagination are the SERVER's job. Two shapes of list: @@ -47,9 +47,6 @@ export interface SpoolQuery { /** field path -> allowed values (OR within a field, AND across fields). */ filters: Record; sort: SortField[]; - search?: string; - /** Hex color for similarity ordering (server-side vector search). */ - color?: string; /** Restrict to one group's spools (for lazy per-group loading). */ groupScope?: GroupScope; limit: number; @@ -60,7 +57,6 @@ export interface SpoolQuery { export interface GroupQuery { field: GroupField; filters: Record; - search?: string; /** Group-level ordering (by aggregate: group.total_remaining, group.last_used, group.title). */ sort: SortField[]; limit: number; @@ -74,6 +70,28 @@ export interface Page { total: number; } +// --- cross-entity search (GET /search) ----------------------------------- + +/** One search hit: the matched entity plus which field matched it. */ +export interface SearchMatch { + entity: T; + /** + * Which field matched: a native field name ("name", "comment", "location", + * "lot_nr", "material", "article_number"), "id" for an exact spool id, "color" + * for a color-similarity match, or "extra." for an extra field. + */ + matchField: string; +} + +/** Categorized results of a cross-entity search. */ +export interface SearchResults { + spools: SearchMatch[]; + filaments: SearchMatch[]; + vendors: SearchMatch[]; + /** True when the query was recognized as a color (a threshold slider is relevant). */ + isColorQuery: boolean; +} + export interface SpoolDataSource { listGroups(query: GroupQuery): Promise>; listSpools(query: SpoolQuery): Promise>; diff --git a/client_v2/src/lib/components/SearchInput.svelte b/client_v2/src/lib/components/SearchInput.svelte index ef6921f2..831def45 100644 --- a/client_v2/src/lib/components/SearchInput.svelte +++ b/client_v2/src/lib/components/SearchInput.svelte @@ -5,10 +5,12 @@ value: string; placeholder?: string; oninput?: (value: string) => void; + onkeydown?: (e: KeyboardEvent) => void; + onfocus?: () => void; fullWidth?: boolean; } - let { value, placeholder = '', oninput, fullWidth = false }: Props = $props(); + let { value, placeholder = '', oninput, onkeydown, onfocus, fullWidth = false }: Props = $props();
@@ -17,6 +19,8 @@ {value} {placeholder} oninput={(e) => oninput?.(e.currentTarget.value)} + onkeydown={(e) => onkeydown?.(e)} + onfocus={() => onfocus?.()} aria-label={$_('common.search')} />
diff --git a/client_v2/src/lib/components/TopBar.svelte b/client_v2/src/lib/components/TopBar.svelte index eaa5d7f7..c9db86e5 100644 --- a/client_v2/src/lib/components/TopBar.svelte +++ b/client_v2/src/lib/components/TopBar.svelte @@ -1,10 +1,8 @@
@@ -44,11 +38,7 @@
- setQuery(v)} - /> +
@@ -57,12 +47,7 @@
- setQuery(v)} - fullWidth - /> +
diff --git a/client_v2/src/lib/components/library/SearchBox.svelte b/client_v2/src/lib/components/library/SearchBox.svelte new file mode 100644 index 00000000..00d06029 --- /dev/null +++ b/client_v2/src/lib/components/library/SearchBox.svelte @@ -0,0 +1,390 @@ + + +
+ { + query = v; + open = true; + }} + onfocus={() => (open = true)} + onkeydown={onKeydown} + {fullWidth} + /> + + {#if showPanel} +
+ {#if results?.isColorQuery} +
+ +

{$_('search.color_hint')}

+
+ {/if} + + {#if loading && !results} +
{$_('search.searching')}
+ {:else if errored} +
{$_('library.api_error')}
+ {:else if !hasResults} +
{$_('search.no_results')}
+ {:else} + {#if results && results.spools.length} +
{$_('search.section.spools')}
+ {#each results.spools as m (m.entity.id)} + {@const filament = inventory.filamentById(m.entity.filamentId)} + + {/each} + {/if} + + {#if results && results.filaments.length} +
{$_('search.section.filaments')}
+ {#each results.filaments as m (m.entity.id)} + {@const vendor = inventory.vendorById(m.entity.vendorId)} + + {/each} + {/if} + + {#if results && results.vendors.length} +
{$_('search.section.vendors')}
+ {#each results.vendors as m (m.entity.id)} + + {/each} + {/if} + {/if} +
+ {/if} +
+ + diff --git a/client_v2/src/lib/library/params.ts b/client_v2/src/lib/library/params.ts index a87df691..c5fb9f99 100644 --- a/client_v2/src/lib/library/params.ts +++ b/client_v2/src/lib/library/params.ts @@ -1,4 +1,5 @@ import { goto } from '$app/navigation'; +import { base } from '$app/paths'; import type { EntityKind, Selection } from '$lib/types'; // The Library view's entire query state lives in the URL — this module is the @@ -19,7 +20,6 @@ export interface FilterChip { } export interface LibraryState { - query: string; selection: Selection | null; group: GroupMode; sortKey: string; @@ -68,7 +68,6 @@ export function parseLibraryState(params: URLSearchParams): LibraryState { const selection = kind && ENTITY_KINDS.includes(kind) ? { kind, id: sel!.slice(si + 1) } : null; return { - query: params.get('q') ?? '', selection, group: group && GROUP_MODES.includes(group) ? group : DEFAULTS.group, sortKey: params.get('sort') ?? DEFAULTS.sortKey, @@ -91,7 +90,6 @@ export function isSelected(params: URLSearchParams, kind: EntityKind, id: string /** Encode Library state back to a canonical query string (no leading `?`). */ function serializeState(s: LibraryState): string { const p = new URLSearchParams(); - if (s.query) p.set('q', s.query); if (s.group !== DEFAULTS.group) p.set('group', s.group); if (s.sortKey !== DEFAULTS.sortKey) p.set('sort', s.sortKey); if (s.sortAsc !== DEFAULTS.sortAsc) p.set('dir', s.sortAsc ? 'asc' : 'desc'); @@ -128,10 +126,6 @@ function navigate(next: LibraryState, replace = false): void { // --- mutators (each preserves the old ui-store semantics) ------------------ -export function setQuery(query: string): void { - navigate({ ...currentState(), query, page: DEFAULTS.page }, true); -} - export function setGroup(group: GroupMode): void { navigate({ ...currentState(), group, page: DEFAULTS.page }); } @@ -173,3 +167,22 @@ export function select(kind: EntityKind, id: string): void { export function clearSelection(): void { navigate({ ...currentState(), selection: null }); } + +/** + * Open a search result's inspector on the Library page. The search box lives in + * the layout, so a result can be picked from any route: when already on the + * Library page we merge the selection into the current view (preserving its + * group/sort/filters); otherwise we navigate to the Library root with just the + * selection. + */ +export function openSearchResult(kind: EntityKind, id: string): void { + const libraryPath = `${base}/`; + const path = window.location.pathname; + if (path === libraryPath || path === base || path === '/') { + select(kind, id); + return; + } + const p = new URLSearchParams(); + p.set('sel', `${kind}:${id}`); + goto(`${libraryPath}?${p.toString()}`, { noScroll: true }); +} diff --git a/client_v2/static/locales/en/common.json b/client_v2/static/locales/en/common.json index d75a82ce..c42ffbd9 100644 --- a/client_v2/static/locales/en/common.json +++ b/client_v2/static/locales/en/common.json @@ -455,6 +455,28 @@ "add_spools": "Add spools", "search_placeholder": "Search — try \"red\" or \"pla\"" }, + "search": { + "searching": "Searching…", + "no_results": "No matches found", + "unknown_filament": "Unknown filament", + "threshold": "Color match", + "color_hint": "Color names must be entered in English.", + "section": { + "spools": "Spools", + "filaments": "Filaments", + "vendors": "Manufacturers" + }, + "match": { + "name": "name", + "material": "material", + "article_number": "article number", + "comment": "comment", + "location": "location", + "lot_nr": "lot number", + "id": "ID", + "color": "color" + } + }, "footer": { "documentation": "Documentation", "report_issue": "Report an issue" diff --git a/spoolman/api/v1/models.py b/spoolman/api/v1/models.py index 109ac3b7..a46417a6 100644 --- a/spoolman/api/v1/models.py +++ b/spoolman/api/v1/models.py @@ -411,6 +411,57 @@ class SpoolGroup(BaseModel): ) +class SearchResultSpool(BaseModel): + """A spool that matched a search, with which field matched.""" + + spool: Spool = Field(description="The matching spool.") + match_field: str = Field( + description=( + "Which field matched the query: a native field name (e.g. 'comment', 'location', " + "'lot_nr'), 'id' for an exact spool-id match, or 'extra.' for an extra field." + ), + examples=["comment"], + ) + + +class SearchResultFilament(BaseModel): + """A filament that matched a search, with which field matched.""" + + filament: Filament = Field(description="The matching filament.") + match_field: str = Field( + description=( + "Which field matched the query: a native field name (e.g. 'name', 'material', " + "'article_number', 'comment'), 'color' for a color-similarity match, or 'extra.'." + ), + examples=["color"], + ) + + +class SearchResultVendor(BaseModel): + """A vendor that matched a search, with which field matched.""" + + vendor: Vendor = Field(description="The matching vendor.") + match_field: str = Field( + description="Which field matched the query: 'name', 'comment', or 'extra.'.", + examples=["name"], + ) + + +class SearchResults(BaseModel): + """Categorized results of a cross-entity search.""" + + spools: list[SearchResultSpool] = Field(description="Matching spools, best matches first.") + filaments: list[SearchResultFilament] = Field(description="Matching filaments, best matches first.") + vendors: list[SearchResultVendor] = Field(description="Matching vendors, best matches first.") + is_color_query: bool = Field( + description=( + "Whether the query was recognized as a color (hex code or CSS color name), in which case " + "the filament results include color-similarity matches and a threshold slider is relevant." + ), + examples=[False], + ) + + class Info(BaseModel): version: str = Field(examples=["0.7.0"]) debug_mode: bool = Field(examples=[False]) diff --git a/spoolman/api/v1/router.py b/spoolman/api/v1/router.py index a58e9539..9f9d5815 100644 --- a/spoolman/api/v1/router.py +++ b/spoolman/api/v1/router.py @@ -15,7 +15,7 @@ from spoolman.database.database import backup_global_db from spoolman.exceptions import ItemNotFoundError from spoolman.ws import websocket_manager -from . import export, externaldb, field, filament, models, other, setting, spool, vendor +from . import export, externaldb, field, filament, models, other, search, setting, spool, vendor logger = logging.getLogger(__name__) @@ -112,3 +112,4 @@ app.include_router(field.router) app.include_router(other.router) app.include_router(externaldb.router) app.include_router(export.router) +app.include_router(search.router) diff --git a/spoolman/api/v1/search.py b/spoolman/api/v1/search.py new file mode 100644 index 00000000..13a2befb --- /dev/null +++ b/spoolman/api/v1/search.py @@ -0,0 +1,96 @@ +"""Cross-entity search endpoint.""" + +import logging +from typing import Annotated + +from fastapi import APIRouter, Depends, Query +from sqlalchemy.ext.asyncio import AsyncSession + +from spoolman.api.v1.models import ( + Filament, + SearchResultFilament, + SearchResults, + SearchResultSpool, + SearchResultVendor, + Spool, + Vendor, +) +from spoolman.database import search +from spoolman.database.database import get_db_session + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix="/search", + tags=["search"], +) + +# ruff: noqa: D103 + + +@router.get( + "", + name="Search", + description=( + "Search across spools, filaments and vendors in a single request. The query is matched " + "case-insensitively against the text fields of each entity (and their text/choice extra " + "fields). A purely numeric query additionally matches a spool by its exact id. A query that " + "is a hex code (e.g. '#ff0000') or a CSS color name (e.g. 'red') additionally runs a " + "color-similarity search over filaments. Results are categorized by entity and each result " + "reports which field matched." + ), + response_model_exclude_none=True, +) +async def search_endpoint( + *, + db: Annotated[AsyncSession, Depends(get_db_session)], + query: Annotated[ + str, + Query( + alias="q", + title="Query", + description="The search query.", + min_length=1, + examples=["red"], + ), + ], + color_similarity_threshold: Annotated[ + float, + Query( + title="Color Similarity Threshold", + description=( + "The similarity threshold for color matching, when the query is a color. " + "A value between 0.0-100.0, where 0 means match only exactly the same color." + ), + ge=0.0, + le=100.0, + examples=[20.0], + ), + ] = 20.0, + limit: Annotated[ + int, + Query( + title="Limit", + description="Maximum number of results per category (spools, filaments, vendors).", + ge=1, + le=100, + ), + ] = 20, +) -> SearchResults: + result = await search.search( + db=db, + query=query, + color_similarity_threshold=color_similarity_threshold, + limit=limit, + ) + return SearchResults( + spools=[SearchResultSpool(spool=Spool.from_db(m.spool), match_field=m.match_field) for m in result.spools], + filaments=[ + SearchResultFilament(filament=Filament.from_db(m.filament), match_field=m.match_field) + for m in result.filaments + ], + vendors=[ + SearchResultVendor(vendor=Vendor.from_db(m.vendor), match_field=m.match_field) for m in result.vendors + ], + is_color_query=result.is_color_query, + ) diff --git a/spoolman/colors.py b/spoolman/colors.py new file mode 100644 index 00000000..32178276 --- /dev/null +++ b/spoolman/colors.py @@ -0,0 +1,181 @@ +"""CSS named colors and color-string resolution for the search feature. + +Color names are the standard CSS Color Module Level 4 keywords, which are +English-only. ``resolve_color`` turns a user-supplied color string (a hex code or +one of these names) into a canonical ``RRGGBB`` hex string suitable for +``spoolman.math.hex_to_rgb``; it returns ``None`` when the string is not a color. +""" + +import re + +# CSS Color Module Level 4 named colors (148 keywords), lowercased name -> RRGGBB. +CSS_COLORS: dict[str, str] = { + "aliceblue": "f0f8ff", + "antiquewhite": "faebd7", + "aqua": "00ffff", + "aquamarine": "7fffd4", + "azure": "f0ffff", + "beige": "f5f5dc", + "bisque": "ffe4c4", + "black": "000000", + "blanchedalmond": "ffebcd", + "blue": "0000ff", + "blueviolet": "8a2be2", + "brown": "a52a2a", + "burlywood": "deb887", + "cadetblue": "5f9ea0", + "chartreuse": "7fff00", + "chocolate": "d2691e", + "coral": "ff7f50", + "cornflowerblue": "6495ed", + "cornsilk": "fff8dc", + "crimson": "dc143c", + "cyan": "00ffff", + "darkblue": "00008b", + "darkcyan": "008b8b", + "darkgoldenrod": "b8860b", + "darkgray": "a9a9a9", + "darkgreen": "006400", + "darkgrey": "a9a9a9", + "darkkhaki": "bdb76b", + "darkmagenta": "8b008b", + "darkolivegreen": "556b2f", + "darkorange": "ff8c00", + "darkorchid": "9932cc", + "darkred": "8b0000", + "darksalmon": "e9967a", + "darkseagreen": "8fbc8f", + "darkslateblue": "483d8b", + "darkslategray": "2f4f4f", + "darkslategrey": "2f4f4f", + "darkturquoise": "00ced1", + "darkviolet": "9400d3", + "deeppink": "ff1493", + "deepskyblue": "00bfff", + "dimgray": "696969", + "dimgrey": "696969", + "dodgerblue": "1e90ff", + "firebrick": "b22222", + "floralwhite": "fffaf0", + "forestgreen": "228b22", + "fuchsia": "ff00ff", + "gainsboro": "dcdcdc", + "ghostwhite": "f8f8ff", + "gold": "ffd700", + "goldenrod": "daa520", + "gray": "808080", + "green": "008000", + "greenyellow": "adff2f", + "grey": "808080", + "honeydew": "f0fff0", + "hotpink": "ff69b4", + "indianred": "cd5c5c", + "indigo": "4b0082", + "ivory": "fffff0", + "khaki": "f0e68c", + "lavender": "e6e6fa", + "lavenderblush": "fff0f5", + "lawngreen": "7cfc00", + "lemonchiffon": "fffacd", + "lightblue": "add8e6", + "lightcoral": "f08080", + "lightcyan": "e0ffff", + "lightgoldenrodyellow": "fafad2", + "lightgray": "d3d3d3", + "lightgreen": "90ee90", + "lightgrey": "d3d3d3", + "lightpink": "ffb6c1", + "lightsalmon": "ffa07a", + "lightseagreen": "20b2aa", + "lightskyblue": "87cefa", + "lightslategray": "778899", + "lightslategrey": "778899", + "lightsteelblue": "b0c4de", + "lightyellow": "ffffe0", + "lime": "00ff00", + "limegreen": "32cd32", + "linen": "faf0e6", + "magenta": "ff00ff", + "maroon": "800000", + "mediumaquamarine": "66cdaa", + "mediumblue": "0000cd", + "mediumorchid": "ba55d3", + "mediumpurple": "9370db", + "mediumseagreen": "3cb371", + "mediumslateblue": "7b68ee", + "mediumspringgreen": "00fa9a", + "mediumturquoise": "48d1cc", + "mediumvioletred": "c71585", + "midnightblue": "191970", + "mintcream": "f5fffa", + "mistyrose": "ffe4e1", + "moccasin": "ffe4b5", + "navajowhite": "ffdead", + "navy": "000080", + "oldlace": "fdf5e6", + "olive": "808000", + "olivedrab": "6b8e23", + "orange": "ffa500", + "orangered": "ff4500", + "orchid": "da70d6", + "palegoldenrod": "eee8aa", + "palegreen": "98fb98", + "paleturquoise": "afeeee", + "palevioletred": "db7093", + "papayawhip": "ffefd5", + "peachpuff": "ffdab9", + "peru": "cd853f", + "pink": "ffc0cb", + "plum": "dda0dd", + "powderblue": "b0e0e6", + "purple": "800080", + "rebeccapurple": "663399", + "red": "ff0000", + "rosybrown": "bc8f8f", + "royalblue": "4169e1", + "saddlebrown": "8b4513", + "salmon": "fa8072", + "sandybrown": "f4a460", + "seagreen": "2e8b57", + "seashell": "fff5ee", + "sienna": "a0522d", + "silver": "c0c0c0", + "skyblue": "87ceeb", + "slateblue": "6a5acd", + "slategray": "708090", + "slategrey": "708090", + "snow": "fffafa", + "springgreen": "00ff7f", + "steelblue": "4682b4", + "tan": "d2b48c", + "teal": "008080", + "thistle": "d8bfd8", + "tomato": "ff6347", + "turquoise": "40e0d0", + "violet": "ee82ee", + "wheat": "f5deb3", + "white": "ffffff", + "whitesmoke": "f5f5f5", + "yellow": "ffff00", + "yellowgreen": "9acd32", +} + +_HEX_RE = re.compile(r"^#?([0-9a-f]{3}|[0-9a-f]{6})$") + + +def resolve_color(query: str) -> str | None: + """Resolve a color string to a canonical ``RRGGBB`` hex, or ``None``. + + Accepts a hex code (``#rgb``, ``#rrggbb`` or the same without ``#``) or a CSS + color name (case-insensitive, surrounding/internal whitespace ignored). + """ + normalized = query.strip().lower() + + match = _HEX_RE.match(normalized.replace(" ", "")) + if match is not None: + hex_digits = match.group(1) + if len(hex_digits) == 3: # noqa: PLR2004 - expand shorthand #rgb -> #rrggbb + hex_digits = "".join(ch * 2 for ch in hex_digits) + return hex_digits + + return CSS_COLORS.get(normalized.replace(" ", "")) diff --git a/spoolman/database/search.py b/spoolman/database/search.py new file mode 100644 index 00000000..5b18482c --- /dev/null +++ b/spoolman/database/search.py @@ -0,0 +1,321 @@ +"""Cross-entity free-text / id / color search backing the /search endpoint. + +Searches spools, filaments and vendors in one call and reports, per result, which +field matched. Text matching uses case-insensitive ``ilike`` so it works on all four +supported databases. A purely numeric query also matches a spool by its id, and a +query that is a hex code or a CSS color name runs a color-similarity search over +filaments (reusing :func:`spoolman.database.filament.find_by_color`). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from sqlalchemy import or_, select +from sqlalchemy.orm import joinedload + +from spoolman.colors import resolve_color +from spoolman.database import filament as filament_db +from spoolman.database import models +from spoolman.extra_field_registry import EntityType, ExtraFieldType, get_extra_fields +from spoolman.math import delta_e, hex_to_rgb, rgb_to_lab + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + +# How many candidate rows to pull per entity before ranking and trimming to `limit`. +# The search box only needs the top handful; this bound keeps ranking cheap while +# still letting prefix matches win over substring ones within a generous window. +_CANDIDATE_CAP = 200 + +# Extra-field types whose stored value is human-readable text worth searching. +_TEXT_EXTRA_TYPES = (ExtraFieldType.text, ExtraFieldType.choice) + +# Rank buckets (lower sorts first). +_RANK_ID_OR_COLOR = -1 # exact id / color-similarity match +_RANK_STRONG = 0 # native field prefix/exact text match +_RANK_WEAK = 1 # native field substring text match +_RANK_EXTRA = 2 # extra-field text match + + +@dataclass +class SpoolMatch: + spool: models.Spool + match_field: str + + +@dataclass +class FilamentMatch: + filament: models.Filament + match_field: str + + +@dataclass +class VendorMatch: + vendor: models.Vendor + match_field: str + + +@dataclass +class SearchResult: + spools: list[SpoolMatch] + filaments: list[FilamentMatch] + vendors: list[VendorMatch] + is_color_query: bool + + +def _classify_match(q_lower: str, fields: list[tuple[str, str | None]]) -> tuple[str, int] | None: + """Given ordered ``(label, value)`` pairs, return ``(label, rank)`` for the best match. + + A prefix/exact hit outranks a mere substring hit. Returns ``None`` if nothing matched + (shouldn't happen for rows the SQL ``ilike`` already selected, but stays defensive). + """ + weak: tuple[str, int] | None = None + for label, value in fields: + if not value: + continue + v = value.lower() + if q_lower in v: + if v == q_lower or v.startswith(q_lower): + return label, _RANK_STRONG + if weak is None: + weak = (label, _RANK_WEAK) + return weak + + +async def _text_extra_keys(db: AsyncSession, entity_type: EntityType) -> list[str]: + """Keys of the entity's text/choice extra fields (the ones worth text-searching).""" + fields = await get_extra_fields(db, entity_type) + return [f.key for f in fields if f.field_type in _TEXT_EXTRA_TYPES] + + +async def _search_spools(db: AsyncSession, q: str, limit: int) -> list[SpoolMatch]: + q_lower = q.lower() + # id -> (match_field, rank); spool object filled in afterwards. + hits: dict[int, tuple[str, int]] = {} + spools: dict[int, models.Spool] = {} + + load = (joinedload(models.Spool.filament).joinedload(models.Filament.vendor),) + + native_fields = ( + ("comment", models.Spool.comment), + ("location", models.Spool.location), + ("lot_nr", models.Spool.lot_nr), + ) + stmt = ( + select(models.Spool) + .where(or_(*(col.ilike(f"%{q}%") for _, col in native_fields))) + .options(*load) + .order_by(models.Spool.id) + .limit(_CANDIDATE_CAP) + ) + rows = (await db.execute(stmt)).unique().scalars().all() + for spool in rows: + classified = _classify_match( + q_lower, + [(label, getattr(spool, attr.key)) for label, attr in native_fields], + ) + if classified is None: + continue + spools[spool.id] = spool + hits[spool.id] = classified + + # Extra-field (text/choice) matches. + keys = await _text_extra_keys(db, EntityType.spool) + if keys: + estmt = ( + select(models.SpoolField.spool_id, models.SpoolField.key) + .where(models.SpoolField.key.in_(keys)) + .where(models.SpoolField.value.ilike(f"%{q}%")) + .limit(_CANDIDATE_CAP) + ) + need: list[int] = [] + for spool_id, key in (await db.execute(estmt)).all(): + if spool_id not in hits: + hits[spool_id] = (f"extra.{key}", _RANK_EXTRA) + need.append(spool_id) + if need: + fetch = select(models.Spool).where(models.Spool.id.in_(need)).options(*load) + for spool in (await db.execute(fetch)).unique().scalars().all(): + spools[spool.id] = spool + + ordered = sorted(hits.items(), key=lambda kv: (kv[1][1], kv[0]))[:limit] + return [SpoolMatch(spool=spools[sid], match_field=field) for sid, (field, _rank) in ordered] + + +async def _search_filaments( + db: AsyncSession, + q: str, + limit: int, + color_hits: list[FilamentMatch], +) -> list[FilamentMatch]: + q_lower = q.lower() + hits: dict[int, tuple[str, int]] = {} + filaments: dict[int, models.Filament] = {} + + # Color matches (already ordered by closeness) claim the top rank; their relative + # order is preserved below via `color_order`. + for match in color_hits: + fid = match.filament.id + filaments[fid] = match.filament + hits[fid] = (match.match_field, _RANK_ID_OR_COLOR) + + load = (joinedload(models.Filament.vendor),) + native_fields = ( + ("name", models.Filament.name), + ("material", models.Filament.material), + ("article_number", models.Filament.article_number), + ("comment", models.Filament.comment), + ) + stmt = ( + select(models.Filament) + .where(or_(*(col.ilike(f"%{q}%") for _, col in native_fields))) + .options(*load) + .order_by(models.Filament.id) + .limit(_CANDIDATE_CAP) + ) + for filament in (await db.execute(stmt)).unique().scalars().all(): + if filament.id in hits: + continue # color match already annotated this one + classified = _classify_match( + q_lower, + [(label, getattr(filament, attr.key)) for label, attr in native_fields], + ) + if classified is None: + continue + filaments[filament.id] = filament + hits[filament.id] = classified + + keys = await _text_extra_keys(db, EntityType.filament) + if keys: + estmt = ( + select(models.FilamentField.filament_id, models.FilamentField.key) + .where(models.FilamentField.key.in_(keys)) + .where(models.FilamentField.value.ilike(f"%{q}%")) + .limit(_CANDIDATE_CAP) + ) + need: list[int] = [] + for filament_id, key in (await db.execute(estmt)).all(): + if filament_id not in hits: + hits[filament_id] = (f"extra.{key}", _RANK_EXTRA) + need.append(filament_id) + if need: + fetch = select(models.Filament).where(models.Filament.id.in_(need)).options(*load) + for filament in (await db.execute(fetch)).unique().scalars().all(): + filaments[filament.id] = filament + + # Color hits keep their supplied order; everyone else sorts by (rank, id). + color_order = {m.filament.id: i for i, m in enumerate(color_hits)} + ordered = sorted( + hits.items(), + key=lambda kv: (kv[1][1], color_order.get(kv[0], 0), kv[0]), + )[:limit] + return [FilamentMatch(filament=filaments[fid], match_field=field) for fid, (field, _rank) in ordered] + + +async def _search_vendors(db: AsyncSession, q: str, limit: int) -> list[VendorMatch]: + q_lower = q.lower() + hits: dict[int, tuple[str, int]] = {} + vendors: dict[int, models.Vendor] = {} + + native_fields = ( + ("name", models.Vendor.name), + ("comment", models.Vendor.comment), + ) + stmt = ( + select(models.Vendor) + .where(or_(*(col.ilike(f"%{q}%") for _, col in native_fields))) + .order_by(models.Vendor.id) + .limit(_CANDIDATE_CAP) + ) + for vendor in (await db.execute(stmt)).unique().scalars().all(): + classified = _classify_match( + q_lower, + [(label, getattr(vendor, attr.key)) for label, attr in native_fields], + ) + if classified is None: + continue + vendors[vendor.id] = vendor + hits[vendor.id] = classified + + keys = await _text_extra_keys(db, EntityType.vendor) + if keys: + estmt = ( + select(models.VendorField.vendor_id, models.VendorField.key) + .where(models.VendorField.key.in_(keys)) + .where(models.VendorField.value.ilike(f"%{q}%")) + .limit(_CANDIDATE_CAP) + ) + need: list[int] = [] + for vendor_id, key in (await db.execute(estmt)).all(): + if vendor_id not in hits: + hits[vendor_id] = (f"extra.{key}", _RANK_EXTRA) + need.append(vendor_id) + if need: + fetch = select(models.Vendor).where(models.Vendor.id.in_(need)) + for vendor in (await db.execute(fetch)).unique().scalars().all(): + vendors[vendor.id] = vendor + + ordered = sorted(hits.items(), key=lambda kv: (kv[1][1], kv[0]))[:limit] + return [VendorMatch(vendor=vendors[vid], match_field=field) for vid, (field, _rank) in ordered] + + +def _filament_color_distance(filament: models.Filament, query_lab: list[float]) -> float: + """Smallest CIELAB delta-E between the query color and any of the filament's colors.""" + if filament.color_hex is not None: + colors = [filament.color_hex] + elif filament.multi_color_hexes is not None: + colors = filament.multi_color_hexes.split(",") + else: + return float("inf") + return min(delta_e(query_lab, rgb_to_lab(hex_to_rgb(color))) for color in colors) + + +async def _color_matches(db: AsyncSession, color_hex: str, threshold: float, limit: int) -> list[FilamentMatch]: + """Color-similar filaments, closest first.""" + matched = await filament_db.find_by_color( + db=db, + color_query_hex=color_hex, + similarity_threshold=threshold, + ) + query_lab = rgb_to_lab(hex_to_rgb(color_hex)) + matched.sort(key=lambda f: _filament_color_distance(f, query_lab)) + return [FilamentMatch(filament=f, match_field="color") for f in matched[:limit]] + + +async def search( + *, + db: AsyncSession, + query: str, + color_similarity_threshold: float = 20.0, + limit: int = 20, +) -> SearchResult: + """Run a cross-entity search for ``query`` and return categorized, annotated results.""" + q = query.strip() + if not q: + return SearchResult(spools=[], filaments=[], vendors=[], is_color_query=False) + + color_hex = resolve_color(q) + is_color_query = color_hex is not None + + color_hits = await _color_matches(db, color_hex, color_similarity_threshold, limit) if color_hex else [] + + spools = await _search_spools(db, q, limit) + filaments = await _search_filaments(db, q, limit, color_hits) + vendors = await _search_vendors(db, q, limit) + + # Numeric query: surface the spool with that exact id at the very top. + if q.isdigit(): + spool_id = int(q) + if not any(m.spool.id == spool_id for m in spools): + spool = await db.get( + models.Spool, + spool_id, + options=[joinedload(models.Spool.filament).joinedload(models.Filament.vendor)], + ) + if spool is not None: + spools.insert(0, SpoolMatch(spool=spool, match_field="id")) + spools = spools[:limit] + + return SearchResult(spools=spools, filaments=filaments, vendors=vendors, is_color_query=is_color_query) diff --git a/tests_integration/tests/search/__init__.py b/tests_integration/tests/search/__init__.py new file mode 100644 index 00000000..83cab6bb --- /dev/null +++ b/tests_integration/tests/search/__init__.py @@ -0,0 +1 @@ +"""Integration tests for the search endpoint.""" diff --git a/tests_integration/tests/search/test_search.py b/tests_integration/tests/search/test_search.py new file mode 100644 index 00000000..74cbc5c6 --- /dev/null +++ b/tests_integration/tests/search/test_search.py @@ -0,0 +1,166 @@ +"""Integration tests for the /search endpoint.""" + +import json +import uuid +from collections.abc import Iterable +from dataclasses import dataclass +from typing import Any + +import httpx +import pytest + +from ..conftest import URL, assert_httpx_success + +# Unique per test run so our queries can't collide with data from other tests. +SFX = uuid.uuid4().hex[:8] + +VENDOR_NAME = f"SearchVendor{SFX}" +FILAMENT_NAME = f"SearchFilament{SFX}" +FILAMENT_COMMENT = f"fcomment{SFX}" +SPOOL_LOCATION = f"SearchLoc{SFX}" +SPOOL_COMMENT = f"scomment{SFX}" +TEXT_VALUE = f"TextVal{SFX}" +CHOICE_VALUE = f"ChoiceVal{SFX}" + + +@dataclass +class Fixture: + vendor: dict[str, Any] + filament: dict[str, Any] + spool: dict[str, Any] + + +@pytest.fixture(scope="module") +def data() -> Iterable[Fixture]: + """Seed one vendor + one red filament + one spool with text/choice extra fields.""" + # Extra field definitions on the spool. + assert_httpx_success( + httpx.post( + f"{URL}/api/v1/field/spool/searchtext", + json={"name": "Search Text", "field_type": "text"}, + ), + ) + assert_httpx_success( + httpx.post( + f"{URL}/api/v1/field/spool/searchchoice", + json={"name": "Search Choice", "field_type": "choice", "choices": [CHOICE_VALUE, "other"]}, + ), + ) + + result = httpx.post(f"{URL}/api/v1/vendor", json={"name": VENDOR_NAME}) + result.raise_for_status() + vendor = result.json() + + result = httpx.post( + f"{URL}/api/v1/filament", + json={ + "name": FILAMENT_NAME, + "vendor_id": vendor["id"], + "material": "PLA", + "density": 1.25, + "diameter": 1.75, + "color_hex": "ff0000", + "comment": FILAMENT_COMMENT, + }, + ) + result.raise_for_status() + filament = result.json() + + result = httpx.post( + f"{URL}/api/v1/spool", + json={ + "filament_id": filament["id"], + "remaining_weight": 1000, + "location": SPOOL_LOCATION, + "comment": SPOOL_COMMENT, + "extra": { + "searchtext": json.dumps(TEXT_VALUE), + "searchchoice": json.dumps(CHOICE_VALUE), + }, + }, + ) + result.raise_for_status() + spool = result.json() + + yield Fixture(vendor=vendor, filament=filament, spool=spool) + + httpx.delete(f"{URL}/api/v1/spool/{spool['id']}").raise_for_status() + httpx.delete(f"{URL}/api/v1/filament/{filament['id']}").raise_for_status() + httpx.delete(f"{URL}/api/v1/vendor/{vendor['id']}").raise_for_status() + httpx.delete(f"{URL}/api/v1/field/spool/searchtext").raise_for_status() + httpx.delete(f"{URL}/api/v1/field/spool/searchchoice").raise_for_status() + + +def _search(query: str, color_similarity_threshold: float | None = None) -> dict[str, Any]: + params: dict[str, Any] = {"q": query} + if color_similarity_threshold is not None: + params["color_similarity_threshold"] = color_similarity_threshold + result = httpx.get(f"{URL}/api/v1/search", params=params) + result.raise_for_status() + return result.json() + + +def _has(items: list[dict[str, Any]], entity_key: str, entity_id: int, match_field: str) -> bool: + return any(i[entity_key]["id"] == entity_id and i["match_field"] == match_field for i in items) + + +def test_search_filament_name(data: Fixture): + body = _search(FILAMENT_NAME) + assert body["is_color_query"] is False + assert _has(body["filaments"], "filament", data.filament["id"], "name") + + +def test_search_vendor_name(data: Fixture): + body = _search(VENDOR_NAME) + assert _has(body["vendors"], "vendor", data.vendor["id"], "name") + + +def test_search_filament_comment(data: Fixture): + body = _search(FILAMENT_COMMENT) + assert _has(body["filaments"], "filament", data.filament["id"], "comment") + + +def test_search_spool_location(data: Fixture): + body = _search(SPOOL_LOCATION) + assert _has(body["spools"], "spool", data.spool["id"], "location") + + +def test_search_spool_comment(data: Fixture): + body = _search(SPOOL_COMMENT) + assert _has(body["spools"], "spool", data.spool["id"], "comment") + + +def test_search_extra_text_field(data: Fixture): + body = _search(TEXT_VALUE) + assert _has(body["spools"], "spool", data.spool["id"], "extra.searchtext") + + +def test_search_extra_choice_field(data: Fixture): + body = _search(CHOICE_VALUE) + assert _has(body["spools"], "spool", data.spool["id"], "extra.searchchoice") + + +def test_search_spool_by_id(data: Fixture): + body = _search(str(data.spool["id"])) + assert _has(body["spools"], "spool", data.spool["id"], "id") + + +def test_search_color_name(data: Fixture): + body = _search("red", color_similarity_threshold=20.0) + assert body["is_color_query"] is True + assert _has(body["filaments"], "filament", data.filament["id"], "color") + + +def test_search_color_hex(data: Fixture): + body = _search("#ff0000", color_similarity_threshold=5.0) + assert body["is_color_query"] is True + assert _has(body["filaments"], "filament", data.filament["id"], "color") + + +def test_search_no_match(data: Fixture): + assert data.spool["id"] # ensure the fixture data exists so "no match" is meaningful + body = _search(f"zzznomatch{SFX}") + assert body["is_color_query"] is False + assert body["spools"] == [] + assert body["filaments"] == [] + assert body["vendors"] == []