diff --git a/.agents/feature-map/components/llm-providers.md b/.agents/feature-map/components/llm-providers.md index 82da9d5c85..5ce22a572c 100644 --- a/.agents/feature-map/components/llm-providers.md +++ b/.agents/feature-map/components/llm-providers.md @@ -62,6 +62,7 @@ Admin router, prefix `/admin/llm`: | GET | `/admin/llm/built-in/options`, `/built-in/options/{provider_name}` | `fetch_llm_options`, `fetch_llm_provider_options` | The well-known-provider catalogue. | | GET | `/admin/llm/custom-provider-names` | `fetch_custom_provider_names` | Names for the custom provider option. | | GET | `/admin/llm/auto-config` | `get_auto_config` | | +| GET | `/admin/llm/router-models` | `list_router_model_providers` | Providers that own `is_router` model configurations. Options for the admin "Router Model" selector; hidden and persona-restricted routers included. | | GET | `/admin/llm/vision-providers` | `get_vision_capable_providers` | | | GET | `/admin/llm/provider-contextual-cost` | `get_provider_contextual_cost` | | | POST | `/admin/llm/{bedrock,ollama,openrouter,lm-studio,litellm,bifrost,nebius-tokenfactory,openai-compatible,vercel-ai-gateway,portkey}/available-models` | per-vendor model discovery | Each calls the vendor's model-list API live. Several also sync the discovered models into the DB when the request carries a `provider_id` (`sync_model_configurations`). | @@ -124,6 +125,11 @@ edit surface. It is a thin client over the endpoints above. `is_visible` gates end-user selectability, `max_input_tokens` is an optional override, `display_name` / `custom_display_name` control the UI label, and `reasoning_effort_default` / `reasoning_effort_max` bound reasoning requests. + `is_router` marks a virtual router entry (e.g. `openrouter/auto`) instead of + a real upstream model. Router rows are hidden from the normal model list and + can back the picker's synthesized "Auto" item when the admin enables model + routing (`Settings.model_routing_enabled` plus + `model_routing_model_configuration_id`). `supports_image_input` is **deprecated in favour of `LLMModelFlow` with the `VISION` flow type**; see §9. - **`LLMModelFlow`** (`db/models.py`): joins a `model_configuration_id` to an diff --git a/backend/alembic/versions/47af2651bf8b_add_is_router_to_model_configuration.py b/backend/alembic/versions/47af2651bf8b_add_is_router_to_model_configuration.py new file mode 100644 index 0000000000..90be8a7b35 --- /dev/null +++ b/backend/alembic/versions/47af2651bf8b_add_is_router_to_model_configuration.py @@ -0,0 +1,33 @@ +"""add is_router to model_configuration + +Revision ID: 47af2651bf8b +Revises: 1b26b1dfdc54 +Create Date: 2026-10-06 00:00:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "47af2651bf8b" +down_revision = "078171a436f9" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "model_configuration", + sa.Column( + "is_router", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + ) + + +def downgrade() -> None: + op.drop_column("model_configuration", "is_router") diff --git a/backend/onyx/db/llm.py b/backend/onyx/db/llm.py index 932a1b0896..491f39f2ee 100644 --- a/backend/onyx/db/llm.py +++ b/backend/onyx/db/llm.py @@ -554,6 +554,8 @@ def upsert_llm_provider( else existing.temperature_default ) ensure_default_within_max(merged_reasoning_default, merged_reasoning_max) + # Router status is additive like the capability flags. + existing.is_router = existing.is_router or model_config.is_router update_model_configuration__no_commit( db_session=db_session, model_configuration_id=existing.id, @@ -589,6 +591,7 @@ def upsert_llm_provider( reasoning_effort_max=model_config.reasoning_effort_max, reasoning_effort_default=model_config.reasoning_effort_default, temperature_default=model_config.temperature_default, + is_router=model_config.is_router, ) ) insert_new_model_configurations__no_commit( @@ -651,7 +654,7 @@ def sync_model_configurations( existing_by_name = {mc.name: mc for mc in provider.model_configurations} new_models: list[NewModelConfiguration] = [] - upgraded_flow_count = 0 + upgraded_count: int = 0 for model in models: existing = existing_by_name.get(model.name) if existing is None: @@ -669,10 +672,16 @@ def sync_model_configurations( is_visible=False, max_input_tokens=model.max_input_tokens, display_name=model.display_name, + is_router=model.is_router, ) ) continue + # Router status is additive like the capability flags. + if model.is_router and not existing.is_router: + existing.is_router = True + upgraded_count += 1 + # Existing model: add newly-reported capability flags (additive only). # TODO(ENG-4233): durable admin flow removals. existing_flows = set(existing.llm_model_flow_types) @@ -691,12 +700,12 @@ def sync_model_configurations( model_configuration_id=existing.id, flow_type=flow_type, ) - upgraded_flow_count += 1 + upgraded_count += 1 insert_new_model_configurations__no_commit(db_session, provider.id, new_models) new_count = len(new_models) - if new_count > 0 or upgraded_flow_count > 0: + if new_count > 0 or upgraded_count > 0: db_session.commit() return new_count @@ -820,9 +829,10 @@ def fetch_model_configurations_page( name_query: str | None = None, ) -> dict[int, ModelConfigurationWindow]: """The same LLM_PROVIDER_MODEL_PAGE_SIZE window of every provider's models, - visible first then by name. `name_query` narrows each provider to models - whose name or display names contain it, so a picker can search models it - has not paged in yet.""" + visible first (routers before other visible models, so a picker's router + tab sees them all on page one) then by name. `name_query` narrows each + provider to models whose name or display names contain it, so a picker can + search models it has not paged in yet.""" if not provider_ids: return {} @@ -832,6 +842,7 @@ def fetch_model_configurations_page( partition_by=ModelConfiguration.llm_provider_id, order_by=( ModelConfiguration.is_visible.desc(), + ModelConfiguration.is_router.desc(), ModelConfiguration.name, ModelConfiguration.id, ), @@ -1195,6 +1206,59 @@ def fetch_model_configuration_by_id( ) +def fetch_llm_providers_with_router_models( + db_session: Session, +) -> list[LLMProviderModel]: + """Providers that own at least one router model configuration, each with + only its router rows loaded (`provider.model_configurations` is routers + only). For pickers offering a routing target without paging a whole + catalog.""" + return list( + db_session.scalars( + select(LLMProviderModel) + .where( + LLMProviderModel.model_configurations.any( + ModelConfiguration.is_router == True # noqa: E712 + ) + ) + .options( + selectinload( + LLMProviderModel.model_configurations.and_( + ModelConfiguration.is_router == True # noqa: E712 + ) + ) + ) + .order_by(LLMProviderModel.id) + ) + ) + + +def mark_model_configuration_visible( + db_session: Session, model_configuration: ModelConfiguration +) -> bool: + """Flip `is_visible` on and commit. False when it was already visible, so + the caller can skip downstream work like cache invalidation.""" + if model_configuration.is_visible: + return False + model_configuration.is_visible = True + db_session.commit() + return True + + +def require_router_model_configuration( + db_session: Session, model_configuration_id: int +) -> ModelConfiguration: + model_configuration: ModelConfiguration | None = fetch_model_configuration_by_id( + db_session, model_configuration_id + ) + if model_configuration is None or not model_configuration.is_router: + raise OnyxError( + OnyxErrorCode.INVALID_INPUT, + "Model routing requires a router model configuration.", + ) + return model_configuration + + def fetch_llm_provider_view( db_session: Session, provider_name: str ) -> LLMProviderView | None: @@ -1645,6 +1709,7 @@ class NewModelConfiguration(BaseModel): reasoning_effort_max: ReasoningEffort | None = None reasoning_effort_default: ReasoningEffort | None = None temperature_default: float | None = None + is_router: bool = False # Rows per INSERT. Keeps bind-parameter counts modest while tens of @@ -1680,6 +1745,7 @@ def insert_new_model_configurations__no_commit( "reasoning_effort_max": model.reasoning_effort_max, "reasoning_effort_default": model.reasoning_effort_default, "temperature_default": model.temperature_default, + "is_router": model.is_router, } for model in chunk ] diff --git a/backend/onyx/db/models.py b/backend/onyx/db/models.py index 2fedc602af..dd634f8ae0 100644 --- a/backend/onyx/db/models.py +++ b/backend/onyx/db/models.py @@ -3728,6 +3728,11 @@ class ModelConfiguration(Base): ) temperature_default: Mapped[float | None] = mapped_column(Float, nullable=True) + # Virtual entry delegating model selection to a routing layer. + is_router: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False, server_default=text("false") + ) + llm_provider: Mapped["LLMProvider"] = relationship( "LLMProvider", back_populates="model_configurations", diff --git a/backend/onyx/server/manage/llm/api.py b/backend/onyx/server/manage/llm/api.py index c2ab0db2cf..b7c989ff8f 100644 --- a/backend/onyx/server/manage/llm/api.py +++ b/backend/onyx/server/manage/llm/api.py @@ -26,11 +26,14 @@ from onyx.db.llm import ( fetch_existing_llm_provider_by_id, fetch_existing_llm_providers, fetch_existing_models, + fetch_llm_providers_with_router_models, fetch_model_configuration_by_id, fetch_model_configurations_page, fetch_persona_with_groups, fetch_user_group_ids, + mark_model_configuration_visible, remove_llm_provider, + require_router_model_configuration, sync_model_configurations, update_default_chat_naming_provider, update_default_craft_provider, @@ -102,6 +105,7 @@ from onyx.server.manage.llm.models import ( LMStudioModelsRequest, ModelConfigurationPage, ModelConfigurationUpsertRequest, + ModelRoutingUpdateRequest, NebiusTokenfactoryFinalModelResponse, NebiusTokenfactoryModelsRequest, OllamaFinalModelResponse, @@ -135,6 +139,12 @@ from onyx.server.manage.llm.utils import ( lm_studio_capability_enabled, strip_openrouter_vendor_prefix, ) +from onyx.server.settings.models import Settings +from onyx.server.settings.store import ( + load_settings, + settings_write_lock, + store_settings, +) from onyx.utils.audit import ( AuditAction, AuditOutcome, @@ -625,6 +635,53 @@ def get_llm_provider( return provider_view +@admin_router.get("/router-models") +def list_router_model_providers( + _: User = Depends(require_permission(Permission.MANAGE_LLMS)), + db_session: Session = Depends(get_session), +) -> list[LLMProviderView]: + """Every router model configuration, grouped under its provider. Unlike + the paged listings this never truncates: pickers offering a routing + target must see hidden routers too, so no visibility or page filter.""" + views: list[LLMProviderView] = [ + LLMProviderView.from_model( + provider, + include_api_key=False, + model_configurations=provider.model_configurations, + ) + for provider in fetch_llm_providers_with_router_models(db_session) + ] + for view in views: + _mask_provider_credentials(view) + return views + + +@admin_router.patch("/model-routing") +def update_model_routing( + update: ModelRoutingUpdateRequest, + _: User = Depends(require_permission(Permission.MANAGE_LLMS)), + db_session: Session = Depends(get_session), +) -> Settings: + """Same write path as PATCH /admin/settings for the routing fields, but + scoped to MANAGE_LLMS so delegated model managers can save them.""" + with settings_write_lock(): + settings: Settings = load_settings(raise_on_error=True) + if update.model_routing_enabled is not None: + settings.model_routing_enabled = update.model_routing_enabled + if "model_routing_model_configuration_id" in update.model_fields_set: + routing_id: int | None = update.model_routing_model_configuration_id + settings.model_routing_model_configuration_id = routing_id + if routing_id is not None: + routing_model: ModelConfiguration = require_router_model_configuration( + db_session, routing_id + ) + # Hidden routers don't reach the picker's provider payload. + if mark_model_configuration_visible(db_session, routing_model): + invalidate_provider_listing_cache() + store_settings(settings) + return settings + + @admin_router.put("/provider") def put_llm_provider( llm_provider_upsert_request: LLMProviderUpsertRequest, @@ -1757,6 +1814,7 @@ def get_openrouter_available_models( display_name=display_name, max_input_tokens=context_length, supports_image_input=model_details.supports_image_input, + is_router=model_details.is_router, ) ) except Exception as e: @@ -1784,6 +1842,7 @@ def get_openrouter_available_models( display_name=r.display_name, max_input_tokens=r.max_input_tokens, supports_image_input=r.supports_image_input, + is_router=r.is_router, ) for r in sorted_results ], diff --git a/backend/onyx/server/manage/llm/models.py b/backend/onyx/server/manage/llm/models.py index 8e5c8aa0fa..1580baaa89 100644 --- a/backend/onyx/server/manage/llm/models.py +++ b/backend/onyx/server/manage/llm/models.py @@ -274,6 +274,7 @@ class ModelConfigurationUpsertRequest(BaseModel): supports_reasoning: bool | None = None display_name: str | None = None # For dynamic providers, from source API custom_display_name: str | None = None # Admin-specified override + is_router: bool = False reasoning_effort_max: ReasoningEffort | None = None reasoning_effort_default: ReasoningEffort | None = None temperature_default: float | None = None @@ -337,6 +338,7 @@ class ModelConfigurationUpsertRequest(BaseModel): ), display_name=model_configuration_model.display_name, custom_display_name=model_configuration_model.custom_display_name, + is_router=model_configuration_model.is_router, reasoning_effort_max=model_configuration_model.reasoning_effort_max, reasoning_effort_default=model_configuration_model.reasoning_effort_default, temperature_default=model_configuration_model.temperature_default, @@ -363,6 +365,8 @@ class ModelConfigurationView(BaseModel): reasoning_effort_max: ReasoningEffort | None = None reasoning_effort_default: ReasoningEffort | None = None temperature_default: float | None = None + # Virtual entry delegating model selection to a routing layer. + is_router: bool = False # True when this is the provider's recommended default model. is_recommended_default: bool = False display_name: str | None = None @@ -443,6 +447,7 @@ class ModelConfigurationView(BaseModel): reasoning_effort_max=model_configuration_model.reasoning_effort_max, reasoning_effort_default=model_configuration_model.reasoning_effort_default, temperature_default=model_configuration_model.temperature_default, + is_router=model_configuration_model.is_router, display_name=model_configuration_model.display_name, custom_display_name=model_configuration_model.custom_display_name, provider_display_name=None, # Not needed for dynamic providers @@ -507,6 +512,7 @@ class ModelConfigurationView(BaseModel): reasoning_effort_max=model_configuration_model.reasoning_effort_max, reasoning_effort_default=model_configuration_model.reasoning_effort_default, temperature_default=model_configuration_model.temperature_default, + is_router=model_configuration_model.is_router, # Populate display fields from parsed model name display_name=display_name, custom_display_name=model_configuration_model.custom_display_name, @@ -615,6 +621,8 @@ class OpenRouterModelDetails(BaseModel): # context_length may be missing or 0 for some models context_length: int | None = None architecture: dict[str, Any] = {} # Contains 'input_modalities' key + # Router entries bill at the routed upstream's rate, reported as "-1". + pricing: dict[str, Any] | None = None @property def supports_image_input(self) -> bool: @@ -626,6 +634,15 @@ class OpenRouterModelDetails(BaseModel): output_modalities = self.architecture.get("output_modalities", []) return isinstance(output_modalities, list) and "embeddings" in output_modalities + @property + def is_router(self) -> bool: + """Virtual models that delegate to an upstream (openrouter/auto, fusion, ...). + OpenRouter marks them with tokenizer "Router" and prices them at -1.""" + if self.architecture.get("tokenizer") == "Router": + return True + pricing: dict[str, Any] = self.pricing or {} + return pricing.get("prompt") == "-1" or pricing.get("completion") == "-1" + class OpenRouterFinalModelResponse(BaseModel): name: str # Model ID (e.g., "openai/gpt-5-pro") @@ -634,6 +651,7 @@ class OpenRouterFinalModelResponse(BaseModel): int | None ) # From OpenRouter API context_length (may be missing for some models) supports_image_input: bool + is_router: bool = False # LM Studio dynamic models fetch @@ -725,6 +743,7 @@ class SyncModelEntry(BaseModel): max_input_tokens: int | None = None supports_image_input: bool = False supports_reasoning: bool = False + is_router: bool = False class LitellmModelsRequest(BaseModel): @@ -894,3 +913,8 @@ class PortkeyFinalModelResponse(BaseModel): max_input_tokens: int | None supports_image_input: bool supports_reasoning: bool + + +class ModelRoutingUpdateRequest(BaseModel): + model_routing_enabled: bool | None = None + model_routing_model_configuration_id: int | None = None diff --git a/backend/onyx/server/settings/api.py b/backend/onyx/server/settings/api.py index c98f099e1b..3a7a954ebf 100644 --- a/backend/onyx/server/settings/api.py +++ b/backend/onyx/server/settings/api.py @@ -18,6 +18,10 @@ from onyx.configs.app_configs import ( from onyx.configs.constants import KV_REINDEX_KEY, NotificationType from onyx.db.engine.sql_engine import get_session from onyx.db.enums import Permission +from onyx.db.llm import ( + mark_model_configuration_visible, + require_router_model_configuration, +) from onyx.db.models import User from onyx.db.notification import ( dismiss_all_notifications, @@ -34,6 +38,7 @@ from onyx.server.features.build.utils import ( is_craft_enabled_for_user, ) from onyx.server.features.notifications.models import NotificationResponse +from onyx.server.manage.llm.provider_cache import invalidate_provider_listing_cache from onyx.server.settings.models import ( DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_NO_VECTOR_DB, DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_VECTOR_DB, @@ -73,6 +78,7 @@ def admin_patch_settings( current_user: User = Depends( require_permission(Permission.FULL_ADMIN_PANEL_ACCESS) ), + db_session: Session = Depends(get_session), ) -> Settings: if global_version.is_ee_version(): from ee.onyx.utils.tier import get_tier @@ -135,6 +141,16 @@ def admin_patch_settings( "The LLM gateway requires the Business or Enterprise plan.", ) + routing_id = merged.model_routing_model_configuration_id + if ( + "model_routing_model_configuration_id" in settings.model_fields_set + and routing_id is not None + ): + routing_model = require_router_model_configuration(db_session, routing_id) + # Hidden routers don't reach the picker's provider payload. + if mark_model_configuration_visible(db_session, routing_model): + invalidate_provider_listing_cache() + store_settings(merged) if merged.craft_default_enabled != existing.craft_default_enabled: diff --git a/backend/onyx/server/settings/models.py b/backend/onyx/server/settings/models.py index c113e98a0d..3473dea4f6 100644 --- a/backend/onyx/server/settings/models.py +++ b/backend/onyx/server/settings/models.py @@ -67,6 +67,10 @@ class Settings(BaseModel): reasoning_override_enabled: bool | None = True # Model selector shows one flat list instead of per-provider groups. hide_provider_grouping: bool = False + + # "Auto" picker item backed by the chosen router configuration. + model_routing_enabled: bool = False + model_routing_model_configuration_id: int | None = None auto_scroll: bool | None = False query_history_type: QueryHistoryType | None = None diff --git a/backend/tests/integration/common_utils/managers/llm_provider.py b/backend/tests/integration/common_utils/managers/llm_provider.py index 9312b3cd56..bbd9441f78 100644 --- a/backend/tests/integration/common_utils/managers/llm_provider.py +++ b/backend/tests/integration/common_utils/managers/llm_provider.py @@ -28,8 +28,10 @@ class LLMProviderManager: is_public: bool | None = None, set_as_default: bool = True, model_names: list[str] | None = None, + router_model_names: list[str] | None = None, ) -> DATestLLMProvider: - """`model_names` adds visible models beside the default one.""" + """`model_names` adds visible models beside the default one. + `router_model_names` adds visible router models.""" print(f"Seeding LLM Providers for {user_performing_action.email}...") llm_provider = LLMProviderUpsertRequest( @@ -43,17 +45,30 @@ class LLMProviderManager: groups=groups or [], personas=personas or [], model_configurations=[ - ModelConfigurationUpsertRequest( - name=model_name, - is_visible=True, - max_input_tokens=None, - display_name=model_name, - supports_image_input=True, - ) - for model_name in [ - default_model_name or "gpt-4o-mini", - *(model_names or []), - ] + *( + ModelConfigurationUpsertRequest( + name=model_name, + is_visible=True, + max_input_tokens=None, + display_name=model_name, + supports_image_input=True, + ) + for model_name in [ + default_model_name or "gpt-4o-mini", + *(model_names or []), + ] + ), + *( + ModelConfigurationUpsertRequest( + name=model_name, + is_visible=True, + max_input_tokens=None, + display_name=model_name, + supports_image_input=True, + is_router=True, + ) + for model_name in (router_model_names or []) + ), ], api_key_changed=True, ) diff --git a/backend/tests/integration/tests/llm_provider/test_llm_provider_model_paging.py b/backend/tests/integration/tests/llm_provider/test_llm_provider_model_paging.py index 7c91cd1c54..d8cd495e90 100644 --- a/backend/tests/integration/tests/llm_provider/test_llm_provider_model_paging.py +++ b/backend/tests/integration/tests/llm_provider/test_llm_provider_model_paging.py @@ -85,6 +85,27 @@ def test_listing_marks_a_small_provider_complete( assert listed["next_model_configuration_offset"] is None +def test_routers_always_land_on_the_first_page( + new_admin_user: DATestUser, +) -> None: + """The picker only loads page one up front, so visible routers must sort + ahead of other visible models even when their names would sort last.""" + model_names = [f"aaa-model-{i:05d}" for i in range(LLM_PROVIDER_MODEL_PAGE_SIZE)] + provider = LLMProviderManager.create( + user_performing_action=new_admin_user, + default_model_name="aaa-default", + model_names=model_names, + router_model_names=["zzz-router-auto"], + set_as_default=False, + ) + + listed = _listed_provider(new_admin_user, provider.id) + router = next( + mc for mc in listed["model_configurations"] if mc["name"] == "zzz-router-auto" + ) + assert router["is_router"] is True + + def test_model_search_matches_unloaded_models(new_admin_user: DATestUser) -> None: model_names = [f"filler-{i:05d}" for i in range(LLM_PROVIDER_MODEL_PAGE_SIZE)] + [ "zz-needle-alpha", diff --git a/backend/tests/unit/onyx/db/test_llm_sync.py b/backend/tests/unit/onyx/db/test_llm_sync.py index e717cfe2cc..aa1d116d95 100644 --- a/backend/tests/unit/onyx/db/test_llm_sync.py +++ b/backend/tests/unit/onyx/db/test_llm_sync.py @@ -264,6 +264,42 @@ class TestSyncModelConfigurations: assert mock_session.execute.call_count == 1 # One VISION flow insert mock_session.commit.assert_called_once() + def test_marks_existing_model_as_router(self) -> None: + """An existing row gains is_router when the source newly reports it + (e.g. a row synced before router detection existed). Returns 0 new + models but commits the flag.""" + mock_existing_model = _make_existing_model( + "openrouter/auto", [LLMModelFlowType.CHAT] + ) + mock_existing_model.is_router = False + + mock_provider = MagicMock() + mock_provider.id = 1 + mock_provider.model_configurations = [mock_existing_model] + + mock_session = MagicMock() + + with patch( + "onyx.db.llm.fetch_existing_llm_provider_by_id", return_value=mock_provider + ): + models = [ + SyncModelEntry( + name="openrouter/auto", + display_name="Auto Router", + is_router=True, + ), + ] + + result = sync_model_configurations( + db_session=mock_session, + provider_id=1, + models=models, + ) + + assert result == 0 + assert mock_existing_model.is_router is True + mock_session.commit.assert_called_once() + def test_does_not_remove_flows(self) -> None: """Capability flags are only added, never removed: a model that already has VISION keeps it even if the source omits it this fetch.""" diff --git a/backend/tests/unit/onyx/server/manage/llm/test_fetch_models_api.py b/backend/tests/unit/onyx/server/manage/llm/test_fetch_models_api.py index 4384a380b2..89b342b926 100644 --- a/backend/tests/unit/onyx/server/manage/llm/test_fetch_models_api.py +++ b/backend/tests/unit/onyx/server/manage/llm/test_fetch_models_api.py @@ -383,6 +383,52 @@ class TestGetOpenRouterAvailableModels: assert claude.supports_image_input is True assert llama.supports_image_input is False + def test_marks_router_models(self, mock_openrouter_response: dict) -> None: + """Router entries are detected via tokenizer "Router" or -1 pricing.""" + from onyx.server.manage.llm.api import get_openrouter_available_models + + mock_openrouter_response["data"].extend( + [ + { + "id": "openrouter/auto", + "name": "Auto Router", + "architecture": { + "tokenizer": "Router", + "input_modalities": ["text", "image"], + }, + "pricing": {"prompt": "-1", "completion": "-1"}, + }, + { + "id": "openrouter/fusion", + "name": "Fusion", + "architecture": {"input_modalities": ["text"]}, + "pricing": {"prompt": "-1", "completion": "-1"}, + }, + ] + ) + + mock_session = MagicMock() + + with patch("onyx.server.manage.llm.api.httpx.get") as mock_get: + mock_response = MagicMock() + mock_response.json.return_value = mock_openrouter_response + mock_response.raise_for_status = MagicMock() + mock_get.return_value = mock_response + + request = OpenRouterModelsRequest( + api_base="https://openrouter.ai/api/v1", + api_key="test-key", + ) + results = get_openrouter_available_models( + request, MagicMock(), mock_session + ) + + by_name = {r.name: r for r in results} + assert by_name["openrouter/auto"].is_router is True + assert by_name["openrouter/fusion"].is_router is True + assert by_name["anthropic/claude-3.5-sonnet"].is_router is False + assert by_name["openai/gpt-4o"].is_router is False + def test_syncs_to_db_when_provider_id_specified( self, mock_openrouter_response: dict ) -> None: diff --git a/backend/tests/unit/onyx/server/settings/test_admin_patch_settings.py b/backend/tests/unit/onyx/server/settings/test_admin_patch_settings.py index d49f099ffd..9d3d0b6f12 100644 --- a/backend/tests/unit/onyx/server/settings/test_admin_patch_settings.py +++ b/backend/tests/unit/onyx/server/settings/test_admin_patch_settings.py @@ -28,6 +28,7 @@ def _patch_settings( *, ee: bool = False, tier: Tier = Tier.COMMUNITY, + db_session: Any = None, ) -> Settings: stored: list[Settings] = [] monkeypatch.setattr( @@ -44,7 +45,9 @@ def _patch_settings( monkeypatch.setattr(tier_module, "get_tier", lambda: tier) settings_api.admin_patch_settings( - Settings.model_validate(payload), current_user=MagicMock() + Settings.model_validate(payload), + current_user=MagicMock(), + db_session=db_session or MagicMock(), ) assert len(stored) == 1 return stored[0] @@ -121,3 +124,48 @@ def test_llm_gateway_enabled_updates_on_business_tier( tier=Tier.BUSINESS, ) assert result.llm_gateway_enabled is False + + +def test_routing_target_must_be_a_router( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "onyx.db.llm.fetch_model_configuration_by_id", + lambda *_a, **_k: MagicMock(is_router=False), + ) + with pytest.raises(OnyxError) as exc_info: + _patch_settings( + {"model_routing_model_configuration_id": 3}, + Settings(), + monkeypatch, + ) + assert exc_info.value.error_code == OnyxErrorCode.INVALID_INPUT + + +def test_routing_target_marks_hidden_router_visible( + monkeypatch: pytest.MonkeyPatch, +) -> None: + marked: list[MagicMock] = [] + invalidated: list[bool] = [] + monkeypatch.setattr( + "onyx.db.llm.fetch_model_configuration_by_id", + lambda *_a, **_k: MagicMock(is_router=True), + ) + monkeypatch.setattr( + settings_api, + "mark_model_configuration_visible", + lambda _s, mc: marked.append(mc) or True, + ) + monkeypatch.setattr( + settings_api, + "invalidate_provider_listing_cache", + lambda: invalidated.append(True), + ) + result = _patch_settings( + {"model_routing_model_configuration_id": 3}, + Settings(), + monkeypatch, + ) + assert result.model_routing_model_configuration_id == 3 + assert len(marked) == 1 + assert invalidated == [True] diff --git a/web/src/i18n/messages/ar.json b/web/src/i18n/messages/ar.json index 6bfbe6922f..5c9ffccabd 100644 --- a/web/src/i18n/messages/ar.json +++ b/web/src/i18n/messages/ar.json @@ -6212,6 +6212,19 @@ "settingsUpdateFailed": "فشل تحديث الإعدادات", "settingsUpdated": "تم تحديث الإعدادات", "unknownError": "خطأ غير معروف" + }, + "modelRouting": { + "autoMode": { + "title": "الوضع التلقائي", + "description": "أضف خيارًا تلقائيًا إلى منتقي النموذج لاختيار النموذج الأنسب لكل رسالة." + }, + "target": { + "title": "نموذج التوجيه", + "description": "يتولى توجيه النماذج للوضع التلقائي.", + "empty": "لم يتم العثور على نماذج توجيه. قم بتمكين نموذج توجيه على موفر بوابة أولاً.", + "error": "تعذر تحميل نماذج التوجيه.", + "retry": "إعادة المحاولة" + } } }, "mcpActions": { @@ -11236,6 +11249,10 @@ }, "unsupportedSetting": { "tooltip": "تعديل هذا الإعداد غير مدعوم لهذا النموذج." + }, + "autoItem": { + "label": "تلقائي", + "description": "يختار أفضل نموذج لكل رسالة" } }, "nrf": { diff --git a/web/src/i18n/messages/de.json b/web/src/i18n/messages/de.json index 92a57f85e9..d9f765083b 100644 --- a/web/src/i18n/messages/de.json +++ b/web/src/i18n/messages/de.json @@ -6212,6 +6212,19 @@ "settingsUpdateFailed": "Die Einstellungen konnten nicht aktualisiert werden", "settingsUpdated": "Einstellungen aktualisiert", "unknownError": "Unbekannter Fehler" + }, + "modelRouting": { + "autoMode": { + "title": "Auto-Modus", + "description": "Fügt der Modellauswahl eine Auto-Option hinzu, die für jede Nachricht das am besten passende Modell wählt." + }, + "target": { + "title": "Router-Modell", + "description": "Übernimmt das Modell-Routing für den Auto-Modus.", + "empty": "Keine Router-Modelle gefunden. Aktivieren Sie zuerst ein Router-Modell bei einem Gateway-Anbieter.", + "error": "Router-Modelle konnten nicht geladen werden.", + "retry": "Erneut versuchen" + } } }, "mcpActions": { @@ -11236,6 +11249,10 @@ }, "unsupportedSetting": { "tooltip": "Das Ändern dieser Einstellung wird für dieses Modell nicht unterstützt." + }, + "autoItem": { + "label": "Auto", + "description": "Wählt für jede Nachricht das beste Modell" } }, "nrf": { diff --git a/web/src/i18n/messages/en.json b/web/src/i18n/messages/en.json index 85f2928bd0..645d8af4b9 100644 --- a/web/src/i18n/messages/en.json +++ b/web/src/i18n/messages/en.json @@ -6212,6 +6212,19 @@ "settingsUpdateFailed": "Failed to update settings", "settingsUpdated": "Settings updated", "unknownError": "Unknown error" + }, + "modelRouting": { + "autoMode": { + "title": "Auto Mode", + "description": "Add an Auto option to the model picker that picks the best-fit model for each message." + }, + "target": { + "title": "Router Model", + "description": "Handles model routing for Auto mode.", + "empty": "No router models found. Enable a router model on a gateway provider first.", + "error": "Failed to load router models.", + "retry": "Retry" + } } }, "mcpActions": { @@ -11236,6 +11249,10 @@ }, "unsupportedSetting": { "tooltip": "Modifying this setting is not supported for this model." + }, + "autoItem": { + "label": "Auto", + "description": "Picks the best model for each message" } }, "nrf": { diff --git a/web/src/i18n/messages/es.json b/web/src/i18n/messages/es.json index b2afd9085e..98721fe271 100644 --- a/web/src/i18n/messages/es.json +++ b/web/src/i18n/messages/es.json @@ -6212,6 +6212,19 @@ "settingsUpdateFailed": "No se pudo actualizar la configuración", "settingsUpdated": "Configuración actualizada", "unknownError": "Error desconocido" + }, + "modelRouting": { + "autoMode": { + "title": "Modo automático", + "description": "Agrega una opción Auto al selector de modelos que elige el modelo más adecuado para cada mensaje." + }, + "target": { + "title": "Modelo de enrutamiento", + "description": "Gestiona el enrutamiento de modelos para el modo Auto.", + "empty": "No se encontraron modelos de router. Primero habilite un modelo de router en un proveedor de puerta de enlace.", + "error": "No se pudieron cargar los modelos de enrutamiento.", + "retry": "Reintentar" + } } }, "mcpActions": { @@ -11236,6 +11249,10 @@ }, "unsupportedSetting": { "tooltip": "Modificar esta configuración no es compatible con este modelo." + }, + "autoItem": { + "label": "Auto", + "description": "Elige el mejor modelo para cada mensaje" } }, "nrf": { diff --git a/web/src/i18n/messages/fr.json b/web/src/i18n/messages/fr.json index 5fce1fb313..b2707f9b42 100644 --- a/web/src/i18n/messages/fr.json +++ b/web/src/i18n/messages/fr.json @@ -6212,6 +6212,19 @@ "settingsUpdateFailed": "Échec de la mise à jour des paramètres", "settingsUpdated": "Paramètres mis à jour", "unknownError": "Erreur inconnue" + }, + "modelRouting": { + "autoMode": { + "title": "Mode auto", + "description": "Ajoute une option Auto au sélecteur de modèle qui choisit le modèle le plus adapté à chaque message." + }, + "target": { + "title": "Modèle de routage", + "description": "Gère le routage des modèles pour le mode Auto.", + "empty": "Aucun modèle routeur trouvé. Activez d'abord un modèle routeur sur un fournisseur de passerelle.", + "error": "Impossible de charger les modèles de routage.", + "retry": "Réessayer" + } } }, "mcpActions": { @@ -11236,6 +11249,10 @@ }, "unsupportedSetting": { "tooltip": "La modification de ce paramètre n'est pas prise en charge pour ce modèle." + }, + "autoItem": { + "label": "Auto", + "description": "Choisit le meilleur modèle pour chaque message" } }, "nrf": { diff --git a/web/src/i18n/messages/ja.json b/web/src/i18n/messages/ja.json index 3adada7b19..a89fccb87b 100644 --- a/web/src/i18n/messages/ja.json +++ b/web/src/i18n/messages/ja.json @@ -6212,6 +6212,19 @@ "settingsUpdateFailed": "設定の更新に失敗しました", "settingsUpdated": "設定を更新しました", "unknownError": "不明なエラー" + }, + "modelRouting": { + "autoMode": { + "title": "オートモード", + "description": "各メッセージに最適なモデルを選ぶ Auto 項目をモデルピッカーに追加します。" + }, + "target": { + "title": "ルーターモデル", + "description": "Auto モードのモデルルーティングを担当します。", + "empty": "ルーターモデルが見つかりません。先にゲートウェイプロバイダーでルーターモデルを有効にしてください。", + "error": "ルーターモデルを読み込めませんでした。", + "retry": "再試行" + } } }, "mcpActions": { @@ -11236,6 +11249,10 @@ }, "unsupportedSetting": { "tooltip": "このモデルでは、この設定の変更に対応していません。" + }, + "autoItem": { + "label": "オート", + "description": "各メッセージに最適なモデルを選択します" } }, "nrf": { diff --git a/web/src/i18n/messages/ko.json b/web/src/i18n/messages/ko.json index 39bf636003..33cb9ac941 100644 --- a/web/src/i18n/messages/ko.json +++ b/web/src/i18n/messages/ko.json @@ -6212,6 +6212,19 @@ "settingsUpdateFailed": "설정을 업데이트하지 못했습니다", "settingsUpdated": "설정을 업데이트했습니다", "unknownError": "알 수 없는 오류" + }, + "modelRouting": { + "autoMode": { + "title": "자동 모드", + "description": "각 메시지에 가장 적합한 모델을 선택하는 Auto 항목을 모델 선택기에 추가합니다." + }, + "target": { + "title": "라우터 모델", + "description": "Auto 모드의 모델 라우팅을 처리합니다.", + "empty": "라우터 모델을 찾을 수 없습니다. 먼저 게이트웨이 제공자에서 라우터 모델을 활성화하세요.", + "error": "라우터 모델을 불러오지 못했습니다.", + "retry": "다시 시도" + } } }, "mcpActions": { @@ -11236,6 +11249,10 @@ }, "unsupportedSetting": { "tooltip": "이 모델은 이 설정의 변경을 지원하지 않습니다." + }, + "autoItem": { + "label": "자동", + "description": "각 메시지에 가장 적합한 모델을 선택합니다" } }, "nrf": { diff --git a/web/src/i18n/messages/pt.json b/web/src/i18n/messages/pt.json index 1ae765bd16..218b832e5e 100644 --- a/web/src/i18n/messages/pt.json +++ b/web/src/i18n/messages/pt.json @@ -6212,6 +6212,19 @@ "settingsUpdateFailed": "Falha ao atualizar as configurações", "settingsUpdated": "Configurações atualizadas", "unknownError": "Erro desconhecido" + }, + "modelRouting": { + "autoMode": { + "title": "Modo automático", + "description": "Adiciona uma opção Auto ao seletor de modelos que escolhe o modelo mais adequado para cada mensagem." + }, + "target": { + "title": "Modelo de roteamento", + "description": "Gerencia o roteamento de modelos para o modo Auto.", + "empty": "Nenhum modelo de roteador encontrado. Primeiro habilite um modelo de roteador em um provedor de gateway.", + "error": "Falha ao carregar os modelos de roteamento.", + "retry": "Tentar novamente" + } } }, "mcpActions": { @@ -11236,6 +11249,10 @@ }, "unsupportedSetting": { "tooltip": "Modificar esta configuração não é compatível com este modelo." + }, + "autoItem": { + "label": "Auto", + "description": "Escolhe o melhor modelo para cada mensagem" } }, "nrf": { diff --git a/web/src/i18n/messages/zh.json b/web/src/i18n/messages/zh.json index b0ae9c14a1..ba1c308a58 100644 --- a/web/src/i18n/messages/zh.json +++ b/web/src/i18n/messages/zh.json @@ -6212,6 +6212,19 @@ "settingsUpdateFailed": "更新设置失败", "settingsUpdated": "设置已更新", "unknownError": "未知错误" + }, + "modelRouting": { + "autoMode": { + "title": "自动模式", + "description": "在模型选择器中添加 Auto 项，为每条消息选择最合适的模型。" + }, + "target": { + "title": "路由器模型", + "description": "负责 Auto 模式的模型路由。", + "empty": "未找到路由器模型。请先在网关提供商上启用一个路由器模型。", + "error": "无法加载路由器模型。", + "retry": "重试" + } } }, "mcpActions": { @@ -11236,6 +11249,10 @@ }, "unsupportedSetting": { "tooltip": "此模型不支持修改该设置。" + }, + "autoItem": { + "label": "自动", + "description": "为每条消息选择最佳模型" } }, "nrf": { diff --git a/web/src/lib/languageModels/cache.ts b/web/src/lib/languageModels/cache.ts index b419575bdb..c30b11770c 100644 --- a/web/src/lib/languageModels/cache.ts +++ b/web/src/lib/languageModels/cache.ts @@ -6,13 +6,13 @@ import { setDefaultLlmModel } from "@/lib/languageModels/svc"; const PERSONA_PROVIDER_ENDPOINT_PATTERN = /^\/api\/llm\/persona\/\d+\/providers$/; const ADMIN_PROVIDER_ENDPOINT_PATTERN = /^\/api\/admin\/llm\/provider\/\d+$/; - export async function refreshLlmProviderCaches( mutate: ScopedMutator ): Promise { await Promise.all([ mutate(SWR_KEYS.adminLlmProvidersPaged), mutate(SWR_KEYS.llmProviders), + mutate(SWR_KEYS.llmRouterModels), mutate( (key) => typeof key === "string" && diff --git a/web/src/lib/languageModels/options.ts b/web/src/lib/languageModels/options.ts index de10668064..64a1c7425b 100644 --- a/web/src/lib/languageModels/options.ts +++ b/web/src/lib/languageModels/options.ts @@ -107,6 +107,7 @@ export function buildLlmOptions( reasoningEffortDefault: mc.reasoning_effort_default, temperatureDefault: mc.temperature_default, supportsImageInput: mc.supports_image_input || false, + isRouter: mc.is_router || false, }); }); }); diff --git a/web/src/lib/languageModels/svc.ts b/web/src/lib/languageModels/svc.ts index 3081d73c80..44eac79d03 100644 --- a/web/src/lib/languageModels/svc.ts +++ b/web/src/lib/languageModels/svc.ts @@ -326,6 +326,7 @@ export const fetchOpenRouterModels = async ( max_input_tokens: modelData.max_input_tokens, supports_image_input: modelData.supports_image_input, supports_reasoning: false, + is_router: modelData.is_router, effectiveDisplayName: modelData.display_name || modelData.name, })); @@ -848,3 +849,22 @@ export const fetchPortkeyModels = async ( return { models: [], error: errorMessage }; } }; + +export async function updateModelRouting(update: { + model_routing_enabled?: boolean; + model_routing_model_configuration_id?: number | null; +}): Promise { + const response: Response = await fetch("/api/admin/llm/model-routing", { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(update), + }); + if (!response.ok) { + const errorData: ErrorResponseBody = await response + .json() + .catch(() => ({})); + throw new Error( + errorData.detail || errorData.message || "Failed to update model routing" + ); + } +} diff --git a/web/src/lib/languageModels/types.ts b/web/src/lib/languageModels/types.ts index 17a19523af..82dfcbb73c 100644 --- a/web/src/lib/languageModels/types.ts +++ b/web/src/lib/languageModels/types.ts @@ -39,6 +39,9 @@ export interface ModelConfiguration { supported_features?: string[]; /** True when this is the provider's recommended default model. */ is_recommended_default?: boolean; + /** Virtual entry that delegates model selection to a routing layer + * (openrouter/auto, a Portkey config) rather than naming a model. */ + is_router?: boolean; display_name?: string; /** Admin-set override that takes precedence over display_name everywhere in the UI. */ custom_display_name?: string; @@ -167,6 +170,7 @@ export interface OpenRouterModelResponse { display_name: string; max_input_tokens: number | null; supports_image_input: boolean; + is_router: boolean; } export interface BedrockModelResponse { @@ -381,6 +385,10 @@ export interface LLMOption { reasoningEffortDefault?: ReasoningEffortOverride | null; temperatureDefault?: number | null; supportsImageInput?: boolean; + /** See ModelConfiguration.is_router. */ + isRouter?: boolean; + /** Synthesized picker item for model routing; selects the backing router. */ + isAuto?: boolean; } export interface LLMOptionGroup { diff --git a/web/src/lib/settings/types.ts b/web/src/lib/settings/types.ts index e00c8d89b4..1d58c68d50 100644 --- a/web/src/lib/settings/types.ts +++ b/web/src/lib/settings/types.ts @@ -35,6 +35,11 @@ export interface Settings { reasoning_override_enabled?: boolean; // Model selector shows one flat list instead of per-provider groups. hide_provider_grouping?: boolean; + + // "Auto" picker item backed by the admin-chosen router. + // router model configuration (e.g. openrouter/auto). + model_routing_enabled?: boolean; + model_routing_model_configuration_id?: number | null; query_history_type: QueryHistoryType; // Visibility-only: hides the sidebar page; query-history APIs + recording stay on. diff --git a/web/src/lib/swr-keys.ts b/web/src/lib/swr-keys.ts index 8e1b80e30e..bb96a1b2c1 100644 --- a/web/src/lib/swr-keys.ts +++ b/web/src/lib/swr-keys.ts @@ -51,6 +51,7 @@ export const SWR_KEYS = { adminLlmProvidersPaged: "/api/admin/llm/provider?page_models=true", adminLlmProvider: (providerId: number) => `/api/admin/llm/provider/${providerId}`, + llmRouterModels: "/api/admin/llm/router-models", llmProvidersWithImageGen: "/api/admin/llm/provider?include_image_gen=true", customProviderNames: "/api/admin/llm/custom-provider-names", wellKnownLlmProviders: "/api/admin/llm/built-in/options", diff --git a/web/src/sections/model-selector/ModelSelector.tsx b/web/src/sections/model-selector/ModelSelector.tsx index 9227d8e936..8eb2f9c1d7 100644 --- a/web/src/sections/model-selector/ModelSelector.tsx +++ b/web/src/sections/model-selector/ModelSelector.tsx @@ -5,6 +5,7 @@ import { useTranslations } from "next-intl"; import { Popover, OpenButton } from "@opal/components"; import { getModelIcon } from "@/lib/languageModels/utils"; import { GLOBAL_DEFAULT_LLM_OPTION } from "@/lib/languageModels/options"; +import { useSettings } from "@/lib/settings/hooks"; import { useLanguageModels, useLanguageModelsForAgent, @@ -92,6 +93,10 @@ export default function ModelSelector({ llmProviders: globalProviderOptions, defaultText: globalDefaultText, } = useLanguageModels(); + const { + model_routing_enabled: modelRoutingEnabled, + model_routing_model_configuration_id: routingModelId, + } = useSettings(); const llmProviders = providerOptions ?? fetchedProviderOptions ?? []; const isLoading = providerOptions === undefined && providersLoading; const [open, setOpen] = useState(false); @@ -129,8 +134,13 @@ export default function ModelSelector({ }, [defaultText, llmProviders]); const effectiveOption = currentOption ?? defaultModelOption; - const currentDisplayName = - effectiveOption?.displayName ?? t("trigger.noSelection.label"); + const isAutoSelected: boolean = + (modelRoutingEnabled ?? false) && + routingModelId != null && + value === routingModelId; + const currentDisplayName: string = isAutoSelected + ? t("autoItem.label") + : (effectiveOption?.displayName ?? t("trigger.noSelection.label")); const globalDefaultDisplayName = useMemo(() => { if (!globalDefaultText || !globalProviderOptions) return null; const provider = globalProviderOptions.find( diff --git a/web/src/sections/model-selector/ModelSelectorContent.tsx b/web/src/sections/model-selector/ModelSelectorContent.tsx index be9cf337ae..c7222a390b 100644 --- a/web/src/sections/model-selector/ModelSelectorContent.tsx +++ b/web/src/sections/model-selector/ModelSelectorContent.tsx @@ -18,6 +18,7 @@ import { SvgChevronRight, SvgCode, SvgSliders, + SvgSparkle, SvgThermometer, } from "@opal/icons"; import { ContentAction, Section } from "@opal/layouts"; @@ -439,7 +440,11 @@ export default function ModelSelectorContent({ onDetailSelect, }: ModelSelectorContentProps) { const t = useTranslations("chat.modelSelector"); - const { hide_provider_grouping: hideProviderGrouping } = useSettings(); + const { + hide_provider_grouping: hideProviderGrouping, + model_routing_enabled: modelRoutingEnabled, + model_routing_model_configuration_id: routingModelId, + } = useSettings(); const [detailOption, setDetailOption] = useState(null); const { llmProviders: currentAgentProviderOptions, @@ -461,8 +466,18 @@ export default function ModelSelectorContent({ [llmProviders, currentModelName, includeHiddenModels] ); + // "Auto" selects the admin-configured router; router rows never list. + const autoOption = useMemo(() => { + if (!modelRoutingEnabled || routingModelId == null) return undefined; + const backing: LLMOption | undefined = llmOptions.find( + (opt) => opt.modelConfigurationId === routingModelId + ); + if (!backing) return undefined; + return { ...backing, displayName: t("autoItem.label"), isAuto: true }; + }, [llmOptions, modelRoutingEnabled, routingModelId, t]); + const filteredOptions = useMemo(() => { - let result = llmOptions; + let result = llmOptions.filter((opt) => !opt.isRouter); if (requiresImageInput) { result = result.filter((opt) => opt.supportsImageInput); } @@ -612,8 +627,11 @@ export default function ModelSelectorContent({ const disabled = isDisabled?.(option) ?? false; // Skip the model-id description when it would just repeat the display name. - const description = - option.modelName !== option.displayName ? option.modelName : undefined; + const description: string | undefined = option.isAuto + ? t("autoItem.description") + : option.modelName !== option.displayName + ? option.modelName + : undefined; return ( @@ -621,7 +639,9 @@ export default function ModelSelectorContent({ onSelect(option)} @@ -692,6 +712,9 @@ export default function ModelSelectorContent({ />, ] : []), + ...(autoOption && !requiresImageInput && !isLoading + ? [renderModelItem(autoOption)] + : []), null, ...(isLoading ? [ diff --git a/web/src/sections/model-selector/MultiModelSelector.tsx b/web/src/sections/model-selector/MultiModelSelector.tsx index 79cf4336be..c7e0268cf9 100644 --- a/web/src/sections/model-selector/MultiModelSelector.tsx +++ b/web/src/sections/model-selector/MultiModelSelector.tsx @@ -10,7 +10,8 @@ import { Divider, Tooltip, } from "@opal/components"; -import { SvgPlusCircle, SvgX } from "@opal/icons"; +import { SvgPlusCircle, SvgSparkle, SvgX } from "@opal/icons"; +import type { IconFunctionComponent } from "@opal/types"; import { cn } from "@opal/utils"; import { useSettings } from "@/lib/settings/hooks"; import { buildLlmOptions, llmOptionKey } from "@/lib/languageModels/options"; @@ -58,6 +59,11 @@ export default function MultiModelSelector({ const settings = useSettings(); const multiModelAllowed = settings.multi_model_chat_enabled ?? true; + const routingEnabled: boolean = + (settings.model_routing_enabled ?? false) && + settings.model_routing_model_configuration_id != null; + const routingModelId: number | null | undefined = + settings.model_routing_model_configuration_id; const modelDetail = useModelDetailManagers( temperatureManager, @@ -217,10 +223,12 @@ export default function MultiModelSelector({ )}
{selectedModels.map((model, index) => { - const ProviderIcon = getModelIcon( - model.provider, - model.modelName - ); + const isAutoModel: boolean = + routingEnabled && + model.modelConfigurationId === routingModelId; + const ProviderIcon: IconFunctionComponent = isAutoModel + ? SvgSparkle + : getModelIcon(model.provider, model.modelName); return (
- {model.displayName} + {isAutoModel ? t("autoItem.label") : model.displayName}
); diff --git a/web/src/views/admin/LanguageModelsPage.tsx b/web/src/views/admin/LanguageModelsPage.tsx index 233ccd8e1f..eac8bb8a9b 100644 --- a/web/src/views/admin/LanguageModelsPage.tsx +++ b/web/src/views/admin/LanguageModelsPage.tsx @@ -3,7 +3,8 @@ import { useAdminRouteTitle } from "@/lib/adminNavLabels"; import { useState, useMemo, useEffect } from "react"; import { useTranslations } from "next-intl"; -import { useSWRConfig } from "swr"; +import useSWR, { useSWRConfig } from "swr"; +import { errorHandlingFetcher } from "@/lib/fetcher"; import { useAdminLanguageModel, useAdminLanguageModels, @@ -18,6 +19,7 @@ import { InputSwitch, Text, Card, + OnyxLoader, } from "@opal/components"; import { Hoverable, Disabled } from "@opal/core"; import { SvgArrowExchange, SvgSettings, SvgTrash } from "@opal/icons"; @@ -29,7 +31,10 @@ import { refreshLlmProviderCaches, setDefaultLlmModelAndRefresh, } from "@/lib/languageModels/cache"; -import { deleteLlmProvider } from "@/lib/languageModels/svc"; +import { + deleteLlmProvider, + updateModelRouting, +} from "@/lib/languageModels/svc"; import { buildLlmOptions, groupLlmOptions } from "@/lib/languageModels/options"; import { findProviderOwningModelConfig } from "@/lib/languageModels/utils"; import { @@ -366,6 +371,12 @@ export default function LanguageModelsPage() { const [pendingHideGrouping, setPendingHideGrouping] = useState< boolean | null >(null); + const [pendingRoutingEnabled, setPendingRoutingEnabled] = useState< + boolean | null + >(null); + const [pendingRoutingTarget, setPendingRoutingTarget] = useState< + number | null + >(null); const { llmProviders: existingLlmProviders, defaultText, @@ -382,6 +393,26 @@ export default function LanguageModelsPage() { [existingLlmProviders] ); + // Includes hidden and persona-restricted routers, unlike the paged listings. + const { + data: routerProviders, + error: routerProvidersError, + mutate: mutateRouterProviders, + } = useSWR( + SWR_KEYS.llmRouterModels, + async (url: string) => { + const providers = await errorHandlingFetcher(url); + return providers.map((provider) => ({ + ...provider, + model_configurations: provider.model_configurations.map((mc) => ({ + ...mc, + effectiveDisplayName: + mc.custom_display_name || mc.display_name || mc.name, + })), + })); + } + ); + // Resolve the current default to a model_configuration_id for the select const defaultModelConfigId = useMemo(() => { if (!defaultText || !existingLlmProviders) return null; @@ -490,6 +521,47 @@ export default function LanguageModelsPage() { } } + const routingEnabled = + pendingRoutingEnabled ?? settings.model_routing_enabled ?? false; + + async function handleRoutingEnabledChange(checked: boolean) { + if (pendingRoutingEnabled !== null) return; + setPendingRoutingEnabled(checked); + try { + await updateModelRouting({ model_routing_enabled: checked }); + await mutate(SWR_KEYS.settings); + toast.success(t("toasts.settingsUpdated")); + } catch (e) { + toast.error( + e instanceof Error ? e.message : t("toasts.settingsUpdateFailed") + ); + } finally { + setPendingRoutingEnabled(null); + } + } + + async function handleRoutingTargetChange(modelConfigurationId: number) { + if (pendingRoutingTarget !== null) return; + setPendingRoutingTarget(modelConfigurationId); + try { + await updateModelRouting({ + model_routing_model_configuration_id: modelConfigurationId, + }); + // The backend may have flipped the router's visibility. + await Promise.all([ + mutate(SWR_KEYS.settings), + refreshLlmProviderCaches(mutate), + ]); + toast.success(t("toasts.settingsUpdated")); + } catch (e) { + toast.error( + e instanceof Error ? e.message : t("toasts.settingsUpdateFailed") + ); + } finally { + setPendingRoutingTarget(null); + } + } + return ( + + { + void handleRoutingEnabledChange(checked); + }} + /> + + {routingEnabled && ( + + {routerProvidersError ? ( +
+ + {t("modelRouting.target.error")} + + +
+ ) : routerProviders === undefined ? ( + + ) : routerProviders.length > 0 ? ( + { + void handleRoutingTargetChange(modelConfigurationId); + }} + /> + ) : ( + + {t("modelRouting.target.empty")} + + )} + + )} {hasProviderGrouping && (