starred/onyx
1
0 Fork 0
mirror of https://github.com/onyx-dot-app/onyx.git synced 2026-10-22 19:32:55 -05:00
Code Issues Packages Projects Releases 10 Wiki Activity

feat(llm): Auto mode in the model picker backed by an admin-chosen router (#15697)

Browse Source
This commit is contained in:
roshan authored and GitHub committed 2026-10-09 17:14:35 +00:00
1 parent 1f07c56921
commit 4b8d1dfd51
32 files changed
+774 -35
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.agents/feature-map/components/llm-providers.md
+6
View File
@@ -62,6 +62,7 @@ Admin router, prefix `/admin/llm`:
| GET | `/admin/llm/built-in/options`, `/built-in/options/{provider_name}` | `fetch_llm_options`, `fetch_llm_provider_options` | The well-known-provider catalogue. |
| GET | `/admin/llm/custom-provider-names` | `fetch_custom_provider_names` | Names for the custom provider option. |
| GET | `/admin/llm/auto-config` | `get_auto_config` | |
| GET | `/admin/llm/router-models` | `list_router_model_providers` | Providers that own `is_router` model configurations. Options for the admin "Router Model" selector; hidden and persona-restricted routers included. |
| GET | `/admin/llm/vision-providers` | `get_vision_capable_providers` | |
| GET | `/admin/llm/provider-contextual-cost` | `get_provider_contextual_cost` | |
| POST | `/admin/llm/{bedrock,ollama,openrouter,lm-studio,litellm,bifrost,nebius-tokenfactory,openai-compatible,vercel-ai-gateway,portkey}/available-models` | per-vendor model discovery | Each calls the vendor's model-list API live. Several also sync the discovered models into the DB when the request carries a `provider_id` (`sync_model_configurations`). |
@@ -124,6 +125,11 @@ edit surface. It is a thin client over the endpoints above.
`is_visible` gates end-user selectability, `max_input_tokens` is an optional
override, `display_name` / `custom_display_name` control the UI label, and
`reasoning_effort_default` / `reasoning_effort_max` bound reasoning requests.
`is_router` marks a virtual router entry (e.g. `openrouter/auto`) instead of
a real upstream model. Router rows are hidden from the normal model list and
can back the picker's synthesized "Auto" item when the admin enables model
routing (`Settings.model_routing_enabled` plus
`model_routing_model_configuration_id`).
`supports_image_input` is **deprecated in favour of `LLMModelFlow` with the
`VISION` flow type**; see §9.
- **`LLMModelFlow`** (`db/models.py`): joins a `model_configuration_id` to an
backend/alembic/versions/47af2651bf8b_add_is_router_to_model_configuration.py
+33
View File
@@ -0,0 +1,33 @@
"""add is_router to model_configuration
Revision ID: 47af2651bf8b
Revises: 1b26b1dfdc54
Create Date: 2026-10-06 00:00:00.000000
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "47af2651bf8b"
down_revision = "078171a436f9"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"model_configuration",
sa.Column(
"is_router",
sa.Boolean(),
nullable=False,
server_default=sa.false(),
),
)
def downgrade() -> None:
op.drop_column("model_configuration", "is_router")
backend/onyx/db/llm.py
+72 -6
View File
@@ -554,6 +554,8 @@ def upsert_llm_provider(
else existing.temperature_default
)
ensure_default_within_max(merged_reasoning_default, merged_reasoning_max)
# Router status is additive like the capability flags.
existing.is_router = existing.is_router or model_config.is_router
update_model_configuration__no_commit(
db_session=db_session,
model_configuration_id=existing.id,
@@ -589,6 +591,7 @@ def upsert_llm_provider(
reasoning_effort_max=model_config.reasoning_effort_max,
reasoning_effort_default=model_config.reasoning_effort_default,
temperature_default=model_config.temperature_default,
is_router=model_config.is_router,
)
)
insert_new_model_configurations__no_commit(
@@ -651,7 +654,7 @@ def sync_model_configurations(
existing_by_name = {mc.name: mc for mc in provider.model_configurations}
new_models: list[NewModelConfiguration] = []
upgraded_flow_count = 0
upgraded_count: int = 0
for model in models:
existing = existing_by_name.get(model.name)
if existing is None:
@@ -669,10 +672,16 @@ def sync_model_configurations(
is_visible=False,
max_input_tokens=model.max_input_tokens,
display_name=model.display_name,
is_router=model.is_router,
)
)
continue
# Router status is additive like the capability flags.
if model.is_router and not existing.is_router:
existing.is_router = True
upgraded_count += 1
# Existing model: add newly-reported capability flags (additive only).
# TODO(ENG-4233): durable admin flow removals.
existing_flows = set(existing.llm_model_flow_types)
@@ -691,12 +700,12 @@ def sync_model_configurations(
model_configuration_id=existing.id,
flow_type=flow_type,
)
upgraded_flow_count += 1
upgraded_count += 1
insert_new_model_configurations__no_commit(db_session, provider.id, new_models)
new_count = len(new_models)
if new_count > 0 or upgraded_flow_count > 0:
if new_count > 0 or upgraded_count > 0:
db_session.commit()
return new_count
@@ -820,9 +829,10 @@ def fetch_model_configurations_page(
name_query: str | None = None,
) -> dict[int, ModelConfigurationWindow]:
"""The same LLM_PROVIDER_MODEL_PAGE_SIZE window of every provider's models,
visible first then by name. `name_query` narrows each provider to models
whose name or display names contain it, so a picker can search models it
has not paged in yet."""
visible first (routers before other visible models, so a picker's router
tab sees them all on page one) then by name. `name_query` narrows each
provider to models whose name or display names contain it, so a picker can
search models it has not paged in yet."""
if not provider_ids:
return {}
@@ -832,6 +842,7 @@ def fetch_model_configurations_page(
partition_by=ModelConfiguration.llm_provider_id,
order_by=(
ModelConfiguration.is_visible.desc(),
ModelConfiguration.is_router.desc(),
ModelConfiguration.name,
ModelConfiguration.id,
),
@@ -1195,6 +1206,59 @@ def fetch_model_configuration_by_id(
)
def fetch_llm_providers_with_router_models(
db_session: Session,
) -> list[LLMProviderModel]:
"""Providers that own at least one router model configuration, each with
only its router rows loaded (`provider.model_configurations` is routers
only). For pickers offering a routing target without paging a whole
catalog."""
return list(
db_session.scalars(
select(LLMProviderModel)
.where(
LLMProviderModel.model_configurations.any(
ModelConfiguration.is_router == True # noqa: E712
)
)
.options(
selectinload(
LLMProviderModel.model_configurations.and_(
ModelConfiguration.is_router == True # noqa: E712
)
)
)
.order_by(LLMProviderModel.id)
)
)
def mark_model_configuration_visible(
db_session: Session, model_configuration: ModelConfiguration
) -> bool:
"""Flip `is_visible` on and commit. False when it was already visible, so
the caller can skip downstream work like cache invalidation."""
if model_configuration.is_visible:
return False
model_configuration.is_visible = True
db_session.commit()
return True
def require_router_model_configuration(
db_session: Session, model_configuration_id: int
) -> ModelConfiguration:
model_configuration: ModelConfiguration | None = fetch_model_configuration_by_id(
db_session, model_configuration_id
)
if model_configuration is None or not model_configuration.is_router:
raise OnyxError(
OnyxErrorCode.INVALID_INPUT,
"Model routing requires a router model configuration.",
)
return model_configuration
def fetch_llm_provider_view(
db_session: Session, provider_name: str
) -> LLMProviderView | None:
@@ -1645,6 +1709,7 @@ class NewModelConfiguration(BaseModel):
reasoning_effort_max: ReasoningEffort | None = None
reasoning_effort_default: ReasoningEffort | None = None
temperature_default: float | None = None
is_router: bool = False
# Rows per INSERT. Keeps bind-parameter counts modest while tens of
@@ -1680,6 +1745,7 @@ def insert_new_model_configurations__no_commit(
"reasoning_effort_max": model.reasoning_effort_max,
"reasoning_effort_default": model.reasoning_effort_default,
"temperature_default": model.temperature_default,
"is_router": model.is_router,
}
for model in chunk
]
backend/onyx/db/models.py
+5
View File
@@ -3728,6 +3728,11 @@ class ModelConfiguration(Base):
)
temperature_default: Mapped[float | None] = mapped_column(Float, nullable=True)
# Virtual entry delegating model selection to a routing layer.
is_router: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default=text("false")
)
llm_provider: Mapped["LLMProvider"] = relationship(
"LLMProvider",
back_populates="model_configurations",
backend/onyx/server/manage/llm/api.py
+59
View File
@@ -26,11 +26,14 @@ from onyx.db.llm import (
fetch_existing_llm_provider_by_id,
fetch_existing_llm_providers,
fetch_existing_models,
fetch_llm_providers_with_router_models,
fetch_model_configuration_by_id,
fetch_model_configurations_page,
fetch_persona_with_groups,
fetch_user_group_ids,
mark_model_configuration_visible,
remove_llm_provider,
require_router_model_configuration,
sync_model_configurations,
update_default_chat_naming_provider,
update_default_craft_provider,
@@ -102,6 +105,7 @@ from onyx.server.manage.llm.models import (
LMStudioModelsRequest,
ModelConfigurationPage,
ModelConfigurationUpsertRequest,
ModelRoutingUpdateRequest,
NebiusTokenfactoryFinalModelResponse,
NebiusTokenfactoryModelsRequest,
OllamaFinalModelResponse,
@@ -135,6 +139,12 @@ from onyx.server.manage.llm.utils import (
lm_studio_capability_enabled,
strip_openrouter_vendor_prefix,
)
from onyx.server.settings.models import Settings
from onyx.server.settings.store import (
load_settings,
settings_write_lock,
store_settings,
)
from onyx.utils.audit import (
AuditAction,
AuditOutcome,
@@ -625,6 +635,53 @@ def get_llm_provider(
return provider_view
@admin_router.get("/router-models")
def list_router_model_providers(
_: User = Depends(require_permission(Permission.MANAGE_LLMS)),
db_session: Session = Depends(get_session),
) -> list[LLMProviderView]:
"""Every router model configuration, grouped under its provider. Unlike
the paged listings this never truncates: pickers offering a routing
target must see hidden routers too, so no visibility or page filter."""
views: list[LLMProviderView] = [
LLMProviderView.from_model(
provider,
include_api_key=False,
model_configurations=provider.model_configurations,
)
for provider in fetch_llm_providers_with_router_models(db_session)
]
for view in views:
_mask_provider_credentials(view)
return views
@admin_router.patch("/model-routing")
def update_model_routing(
update: ModelRoutingUpdateRequest,
_: User = Depends(require_permission(Permission.MANAGE_LLMS)),
db_session: Session = Depends(get_session),
) -> Settings:
"""Same write path as PATCH /admin/settings for the routing fields, but
scoped to MANAGE_LLMS so delegated model managers can save them."""
with settings_write_lock():
settings: Settings = load_settings(raise_on_error=True)
if update.model_routing_enabled is not None:
settings.model_routing_enabled = update.model_routing_enabled
if "model_routing_model_configuration_id" in update.model_fields_set:
routing_id: int | None = update.model_routing_model_configuration_id
settings.model_routing_model_configuration_id = routing_id
if routing_id is not None:
routing_model: ModelConfiguration = require_router_model_configuration(
db_session, routing_id
)
# Hidden routers don't reach the picker's provider payload.
if mark_model_configuration_visible(db_session, routing_model):
invalidate_provider_listing_cache()
store_settings(settings)
return settings
@admin_router.put("/provider")
def put_llm_provider(
llm_provider_upsert_request: LLMProviderUpsertRequest,
@@ -1757,6 +1814,7 @@ def get_openrouter_available_models(
display_name=display_name,
max_input_tokens=context_length,
supports_image_input=model_details.supports_image_input,
is_router=model_details.is_router,
)
)
except Exception as e:
@@ -1784,6 +1842,7 @@ def get_openrouter_available_models(
display_name=r.display_name,
max_input_tokens=r.max_input_tokens,
supports_image_input=r.supports_image_input,
is_router=r.is_router,
)
for r in sorted_results
],
backend/onyx/server/manage/llm/models.py
+24
View File
@@ -274,6 +274,7 @@ class ModelConfigurationUpsertRequest(BaseModel):
supports_reasoning: bool | None = None
display_name: str | None = None # For dynamic providers, from source API
custom_display_name: str | None = None # Admin-specified override
is_router: bool = False
reasoning_effort_max: ReasoningEffort | None = None
reasoning_effort_default: ReasoningEffort | None = None
temperature_default: float | None = None
@@ -337,6 +338,7 @@ class ModelConfigurationUpsertRequest(BaseModel):
),
display_name=model_configuration_model.display_name,
custom_display_name=model_configuration_model.custom_display_name,
is_router=model_configuration_model.is_router,
reasoning_effort_max=model_configuration_model.reasoning_effort_max,
reasoning_effort_default=model_configuration_model.reasoning_effort_default,
temperature_default=model_configuration_model.temperature_default,
@@ -363,6 +365,8 @@ class ModelConfigurationView(BaseModel):
reasoning_effort_max: ReasoningEffort | None = None
reasoning_effort_default: ReasoningEffort | None = None
temperature_default: float | None = None
# Virtual entry delegating model selection to a routing layer.
is_router: bool = False
# True when this is the provider's recommended default model.
is_recommended_default: bool = False
display_name: str | None = None
@@ -443,6 +447,7 @@ class ModelConfigurationView(BaseModel):
reasoning_effort_max=model_configuration_model.reasoning_effort_max,
reasoning_effort_default=model_configuration_model.reasoning_effort_default,
temperature_default=model_configuration_model.temperature_default,
is_router=model_configuration_model.is_router,
display_name=model_configuration_model.display_name,
custom_display_name=model_configuration_model.custom_display_name,
provider_display_name=None, # Not needed for dynamic providers
@@ -507,6 +512,7 @@ class ModelConfigurationView(BaseModel):
reasoning_effort_max=model_configuration_model.reasoning_effort_max,
reasoning_effort_default=model_configuration_model.reasoning_effort_default,
temperature_default=model_configuration_model.temperature_default,
is_router=model_configuration_model.is_router,
# Populate display fields from parsed model name
display_name=display_name,
custom_display_name=model_configuration_model.custom_display_name,
@@ -615,6 +621,8 @@ class OpenRouterModelDetails(BaseModel):
# context_length may be missing or 0 for some models
context_length: int | None = None
architecture: dict[str, Any] = {} # Contains 'input_modalities' key
# Router entries bill at the routed upstream's rate, reported as "-1".
pricing: dict[str, Any] | None = None
@property
def supports_image_input(self) -> bool:
@@ -626,6 +634,15 @@ class OpenRouterModelDetails(BaseModel):
output_modalities = self.architecture.get("output_modalities", [])
return isinstance(output_modalities, list) and "embeddings" in output_modalities
@property
def is_router(self) -> bool:
"""Virtual models that delegate to an upstream (openrouter/auto, fusion, ...).
OpenRouter marks them with tokenizer "Router" and prices them at -1."""
if self.architecture.get("tokenizer") == "Router":
return True
pricing: dict[str, Any] = self.pricing or {}
return pricing.get("prompt") == "-1" or pricing.get("completion") == "-1"
class OpenRouterFinalModelResponse(BaseModel):
name: str # Model ID (e.g., "openai/gpt-5-pro")
@@ -634,6 +651,7 @@ class OpenRouterFinalModelResponse(BaseModel):
int | None
) # From OpenRouter API context_length (may be missing for some models)
supports_image_input: bool
is_router: bool = False
# LM Studio dynamic models fetch
@@ -725,6 +743,7 @@ class SyncModelEntry(BaseModel):
max_input_tokens: int | None = None
supports_image_input: bool = False
supports_reasoning: bool = False
is_router: bool = False
class LitellmModelsRequest(BaseModel):
@@ -894,3 +913,8 @@ class PortkeyFinalModelResponse(BaseModel):
max_input_tokens: int | None
supports_image_input: bool
supports_reasoning: bool
class ModelRoutingUpdateRequest(BaseModel):
model_routing_enabled: bool | None = None
model_routing_model_configuration_id: int | None = None
backend/onyx/server/settings/api.py
+16
View File
@@ -18,6 +18,10 @@ from onyx.configs.app_configs import (
from onyx.configs.constants import KV_REINDEX_KEY, NotificationType
from onyx.db.engine.sql_engine import get_session
from onyx.db.enums import Permission
from onyx.db.llm import (
mark_model_configuration_visible,
require_router_model_configuration,
)
from onyx.db.models import User
from onyx.db.notification import (
dismiss_all_notifications,
@@ -34,6 +38,7 @@ from onyx.server.features.build.utils import (
is_craft_enabled_for_user,
)
from onyx.server.features.notifications.models import NotificationResponse
from onyx.server.manage.llm.provider_cache import invalidate_provider_listing_cache
from onyx.server.settings.models import (
DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_NO_VECTOR_DB,
DEFAULT_FILE_TOKEN_COUNT_THRESHOLD_K_VECTOR_DB,
@@ -73,6 +78,7 @@ def admin_patch_settings(
current_user: User = Depends(
require_permission(Permission.FULL_ADMIN_PANEL_ACCESS)
),
db_session: Session = Depends(get_session),
) -> Settings:
if global_version.is_ee_version():
from ee.onyx.utils.tier import get_tier
@@ -135,6 +141,16 @@ def admin_patch_settings(
"The LLM gateway requires the Business or Enterprise plan.",
)
routing_id = merged.model_routing_model_configuration_id
if (
"model_routing_model_configuration_id" in settings.model_fields_set
and routing_id is not None
):
routing_model = require_router_model_configuration(db_session, routing_id)
# Hidden routers don't reach the picker's provider payload.
if mark_model_configuration_visible(db_session, routing_model):
invalidate_provider_listing_cache()
store_settings(merged)
if merged.craft_default_enabled != existing.craft_default_enabled:
backend/onyx/server/settings/models.py
+4
View File
@@ -67,6 +67,10 @@ class Settings(BaseModel):
reasoning_override_enabled: bool | None = True
# Model selector shows one flat list instead of per-provider groups.
hide_provider_grouping: bool = False
# "Auto" picker item backed by the chosen router configuration.
model_routing_enabled: bool = False
model_routing_model_configuration_id: int | None = None
auto_scroll: bool | None = False
query_history_type: QueryHistoryType | None = None
backend/tests/integration/common_utils/managers/llm_provider.py
+27 -12
View File
@@ -28,8 +28,10 @@ class LLMProviderManager:
is_public: bool | None = None,
set_as_default: bool = True,
model_names: list[str] | None = None,
router_model_names: list[str] | None = None,
) -> DATestLLMProvider:
"""`model_names` adds visible models beside the default one."""
"""`model_names` adds visible models beside the default one.
`router_model_names` adds visible router models."""
print(f"Seeding LLM Providers for {user_performing_action.email}...")
llm_provider = LLMProviderUpsertRequest(
@@ -43,17 +45,30 @@ class LLMProviderManager:
groups=groups or [],
personas=personas or [],
model_configurations=[
ModelConfigurationUpsertRequest(
name=model_name,
is_visible=True,
max_input_tokens=None,
display_name=model_name,
supports_image_input=True,
)
for model_name in [
default_model_name or "gpt-4o-mini",
*(model_names or []),
]
*(
ModelConfigurationUpsertRequest(
name=model_name,
is_visible=True,
max_input_tokens=None,
display_name=model_name,
supports_image_input=True,
)
for model_name in [
default_model_name or "gpt-4o-mini",
*(model_names or []),
]
),
*(
ModelConfigurationUpsertRequest(
name=model_name,
is_visible=True,
max_input_tokens=None,
display_name=model_name,
supports_image_input=True,
is_router=True,
)
for model_name in (router_model_names or [])
),
],
api_key_changed=True,
)
backend/tests/integration/tests/llm_provider/test_llm_provider_model_paging.py
+21
View File
@@ -85,6 +85,27 @@ def test_listing_marks_a_small_provider_complete(
assert listed["next_model_configuration_offset"] is None
def test_routers_always_land_on_the_first_page(
new_admin_user: DATestUser,
) -> None:
"""The picker only loads page one up front, so visible routers must sort
ahead of other visible models even when their names would sort last."""
model_names = [f"aaa-model-{i:05d}" for i in range(LLM_PROVIDER_MODEL_PAGE_SIZE)]
provider = LLMProviderManager.create(
user_performing_action=new_admin_user,
default_model_name="aaa-default",
model_names=model_names,
router_model_names=["zzz-router-auto"],
set_as_default=False,
)
listed = _listed_provider(new_admin_user, provider.id)
router = next(
mc for mc in listed["model_configurations"] if mc["name"] == "zzz-router-auto"
)
assert router["is_router"] is True
def test_model_search_matches_unloaded_models(new_admin_user: DATestUser) -> None:
model_names = [f"filler-{i:05d}" for i in range(LLM_PROVIDER_MODEL_PAGE_SIZE)] + [
"zz-needle-alpha",
backend/tests/unit/onyx/db/test_llm_sync.py
+36
View File
@@ -264,6 +264,42 @@ class TestSyncModelConfigurations:
assert mock_session.execute.call_count == 1 # One VISION flow insert
mock_session.commit.assert_called_once()
def test_marks_existing_model_as_router(self) -> None:
"""An existing row gains is_router when the source newly reports it
(e.g. a row synced before router detection existed). Returns 0 new
models but commits the flag."""
mock_existing_model = _make_existing_model(
"openrouter/auto", [LLMModelFlowType.CHAT]
)
mock_existing_model.is_router = False
mock_provider = MagicMock()
mock_provider.id = 1
mock_provider.model_configurations = [mock_existing_model]
mock_session = MagicMock()
with patch(
"onyx.db.llm.fetch_existing_llm_provider_by_id", return_value=mock_provider
):
models = [
SyncModelEntry(
name="openrouter/auto",
display_name="Auto Router",
is_router=True,
),
]
result = sync_model_configurations(
db_session=mock_session,
provider_id=1,
models=models,
)
assert result == 0
assert mock_existing_model.is_router is True
mock_session.commit.assert_called_once()
def test_does_not_remove_flows(self) -> None:
"""Capability flags are only added, never removed: a model that already
has VISION keeps it even if the source omits it this fetch."""
backend/tests/unit/onyx/server/manage/llm/test_fetch_models_api.py
+46
View File
@@ -383,6 +383,52 @@ class TestGetOpenRouterAvailableModels:
assert claude.supports_image_input is True
assert llama.supports_image_input is False
def test_marks_router_models(self, mock_openrouter_response: dict) -> None:
"""Router entries are detected via tokenizer "Router" or -1 pricing."""
from onyx.server.manage.llm.api import get_openrouter_available_models
mock_openrouter_response["data"].extend(
[
{
"id": "openrouter/auto",
"name": "Auto Router",
"architecture": {
"tokenizer": "Router",
"input_modalities": ["text", "image"],
},
"pricing": {"prompt": "-1", "completion": "-1"},
},
{
"id": "openrouter/fusion",
"name": "Fusion",
"architecture": {"input_modalities": ["text"]},
"pricing": {"prompt": "-1", "completion": "-1"},
},
]
)
mock_session = MagicMock()
with patch("onyx.server.manage.llm.api.httpx.get") as mock_get:
mock_response = MagicMock()
mock_response.json.return_value = mock_openrouter_response
mock_response.raise_for_status = MagicMock()
mock_get.return_value = mock_response
request = OpenRouterModelsRequest(
api_base="https://openrouter.ai/api/v1",
api_key="test-key",
)
results = get_openrouter_available_models(
request, MagicMock(), mock_session
)
by_name = {r.name: r for r in results}
assert by_name["openrouter/auto"].is_router is True
assert by_name["openrouter/fusion"].is_router is True
assert by_name["anthropic/claude-3.5-sonnet"].is_router is False
assert by_name["openai/gpt-4o"].is_router is False
def test_syncs_to_db_when_provider_id_specified(
self, mock_openrouter_response: dict
) -> None:
backend/tests/unit/onyx/server/settings/test_admin_patch_settings.py
+49 -1
View File
@@ -28,6 +28,7 @@ def _patch_settings(
*,
ee: bool = False,
tier: Tier = Tier.COMMUNITY,
db_session: Any = None,
) -> Settings:
stored: list[Settings] = []
monkeypatch.setattr(
@@ -44,7 +45,9 @@ def _patch_settings(
monkeypatch.setattr(tier_module, "get_tier", lambda: tier)
settings_api.admin_patch_settings(
Settings.model_validate(payload), current_user=MagicMock()
Settings.model_validate(payload),
current_user=MagicMock(),
db_session=db_session or MagicMock(),
)
assert len(stored) == 1
return stored[0]
@@ -121,3 +124,48 @@ def test_llm_gateway_enabled_updates_on_business_tier(
tier=Tier.BUSINESS,
)
assert result.llm_gateway_enabled is False
def test_routing_target_must_be_a_router(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
"onyx.db.llm.fetch_model_configuration_by_id",
lambda *_a, **_k: MagicMock(is_router=False),
)
with pytest.raises(OnyxError) as exc_info:
_patch_settings(
{"model_routing_model_configuration_id": 3},
Settings(),
monkeypatch,
)
assert exc_info.value.error_code == OnyxErrorCode.INVALID_INPUT
def test_routing_target_marks_hidden_router_visible(
monkeypatch: pytest.MonkeyPatch,
) -> None:
marked: list[MagicMock] = []
invalidated: list[bool] = []
monkeypatch.setattr(
"onyx.db.llm.fetch_model_configuration_by_id",
lambda *_a, **_k: MagicMock(is_router=True),
)
monkeypatch.setattr(
settings_api,
"mark_model_configuration_visible",
lambda _s, mc: marked.append(mc) or True,
)
monkeypatch.setattr(
settings_api,
"invalidate_provider_listing_cache",
lambda: invalidated.append(True),
)
result = _patch_settings(
{"model_routing_model_configuration_id": 3},
Settings(),
monkeypatch,
)
assert result.model_routing_model_configuration_id == 3
assert len(marked) == 1
assert invalidated == [True]
web/src/i18n/messages/ar.json
+17
View File
@@ -6212,6 +6212,19 @@
"settingsUpdateFailed": "فشل تحديث الإعدادات",
"settingsUpdated": "تم تحديث الإعدادات",
"unknownError": "خطأ غير معروف"
},
"modelRouting": {
"autoMode": {
"title": "الوضع التلقائي",
"description": "أضف خيارًا تلقائيًا إلى منتقي النموذج لاختيار النموذج الأنسب لكل رسالة."
},
"target": {
"title": "نموذج التوجيه",
"description": "يتولى توجيه النماذج للوضع التلقائي.",
"empty": "لم يتم العثور على نماذج توجيه. قم بتمكين نموذج توجيه على موفر بوابة أولاً.",
"error": "تعذر تحميل نماذج التوجيه.",
"retry": "إعادة المحاولة"
}
}
},
"mcpActions": {
@@ -11236,6 +11249,10 @@
},
"unsupportedSetting": {
"tooltip": "تعديل هذا الإعداد غير مدعوم لهذا النموذج."
},
"autoItem": {
"label": "تلقائي",
"description": "يختار أفضل نموذج لكل رسالة"
}
},
"nrf": {
web/src/i18n/messages/de.json
+17
View File
@@ -6212,6 +6212,19 @@
"settingsUpdateFailed": "Die Einstellungen konnten nicht aktualisiert werden",
"settingsUpdated": "Einstellungen aktualisiert",
"unknownError": "Unbekannter Fehler"
},
"modelRouting": {
"autoMode": {
"title": "Auto-Modus",
"description": "Fügt der Modellauswahl eine Auto-Option hinzu, die für jede Nachricht das am besten passende Modell wählt."
},
"target": {
"title": "Router-Modell",
"description": "Übernimmt das Modell-Routing für den Auto-Modus.",
"empty": "Keine Router-Modelle gefunden. Aktivieren Sie zuerst ein Router-Modell bei einem Gateway-Anbieter.",
"error": "Router-Modelle konnten nicht geladen werden.",
"retry": "Erneut versuchen"
}
}
},
"mcpActions": {
@@ -11236,6 +11249,10 @@
},
"unsupportedSetting": {
"tooltip": "Das Ändern dieser Einstellung wird für dieses Modell nicht unterstützt."
},
"autoItem": {
"label": "Auto",
"description": "Wählt für jede Nachricht das beste Modell"
}
},
"nrf": {
web/src/i18n/messages/en.json
+17
View File
@@ -6212,6 +6212,19 @@
"settingsUpdateFailed": "Failed to update settings",
"settingsUpdated": "Settings updated",
"unknownError": "Unknown error"
},
"modelRouting": {
"autoMode": {
"title": "Auto Mode",
"description": "Add an Auto option to the model picker that picks the best-fit model for each message."
},
"target": {
"title": "Router Model",
"description": "Handles model routing for Auto mode.",
"empty": "No router models found. Enable a router model on a gateway provider first.",
"error": "Failed to load router models.",
"retry": "Retry"
}
}
},
"mcpActions": {
@@ -11236,6 +11249,10 @@
},
"unsupportedSetting": {
"tooltip": "Modifying this setting is not supported for this model."
},
"autoItem": {
"label": "Auto",
"description": "Picks the best model for each message"
}
},
"nrf": {
web/src/i18n/messages/es.json
+17
View File
@@ -6212,6 +6212,19 @@
"settingsUpdateFailed": "No se pudo actualizar la configuración",
"settingsUpdated": "Configuración actualizada",
"unknownError": "Error desconocido"
},
"modelRouting": {
"autoMode": {
"title": "Modo automático",
"description": "Agrega una opción Auto al selector de modelos que elige el modelo más adecuado para cada mensaje."
},
"target": {
"title": "Modelo de enrutamiento",
"description": "Gestiona el enrutamiento de modelos para el modo Auto.",
"empty": "No se encontraron modelos de router. Primero habilite un modelo de router en un proveedor de puerta de enlace.",
"error": "No se pudieron cargar los modelos de enrutamiento.",
"retry": "Reintentar"
}
}
},
"mcpActions": {
@@ -11236,6 +11249,10 @@
},
"unsupportedSetting": {
"tooltip": "Modificar esta configuración no es compatible con este modelo."
},
"autoItem": {
"label": "Auto",
"description": "Elige el mejor modelo para cada mensaje"
}
},
"nrf": {
web/src/i18n/messages/fr.json
+17
View File
@@ -6212,6 +6212,19 @@
"settingsUpdateFailed": "Échec de la mise à jour des paramètres",
"settingsUpdated": "Paramètres mis à jour",
"unknownError": "Erreur inconnue"
},
"modelRouting": {
"autoMode": {
"title": "Mode auto",
"description": "Ajoute une option Auto au sélecteur de modèle qui choisit le modèle le plus adapté à chaque message."
},
"target": {
"title": "Modèle de routage",
"description": "Gère le routage des modèles pour le mode Auto.",
"empty": "Aucun modèle routeur trouvé. Activez d'abord un modèle routeur sur un fournisseur de passerelle.",
"error": "Impossible de charger les modèles de routage.",
"retry": "Réessayer"
}
}
},
"mcpActions": {
@@ -11236,6 +11249,10 @@
},
"unsupportedSetting": {
"tooltip": "La modification de ce paramètre n'est pas prise en charge pour ce modèle."
},
"autoItem": {
"label": "Auto",
"description": "Choisit le meilleur modèle pour chaque message"
}
},
"nrf": {
web/src/i18n/messages/ja.json
+17
View File
@@ -6212,6 +6212,19 @@
"settingsUpdateFailed": "設定の更新に失敗しました",
"settingsUpdated": "設定を更新しました",
"unknownError": "不明なエラー"
},
"modelRouting": {
"autoMode": {
"title": "オートモード",
"description": "各メッセージに最適なモデルを選ぶ Auto 項目をモデルピッカーに追加します。"
},
"target": {
"title": "ルーターモデル",
"description": "Auto モードのモデルルーティングを担当します。",
"empty": "ルーターモデルが見つかりません。先にゲートウェイプロバイダーでルーターモデルを有効にしてください。",
"error": "ルーターモデルを読み込めませんでした。",
"retry": "再試行"
}
}
},
"mcpActions": {
@@ -11236,6 +11249,10 @@
},
"unsupportedSetting": {
"tooltip": "このモデルでは、この設定の変更に対応していません。"
},
"autoItem": {
"label": "オート",
"description": "各メッセージに最適なモデルを選択します"
}
},
"nrf": {
web/src/i18n/messages/ko.json
+17
View File
@@ -6212,6 +6212,19 @@
"settingsUpdateFailed": "설정을 업데이트하지 못했습니다",
"settingsUpdated": "설정을 업데이트했습니다",
"unknownError": "알 수 없는 오류"
},
"modelRouting": {
"autoMode": {
"title": "자동 모드",
"description": "각 메시지에 가장 적합한 모델을 선택하는 Auto 항목을 모델 선택기에 추가합니다."
},
"target": {
"title": "라우터 모델",
"description": "Auto 모드의 모델 라우팅을 처리합니다.",
"empty": "라우터 모델을 찾을 수 없습니다. 먼저 게이트웨이 제공자에서 라우터 모델을 활성화하세요.",
"error": "라우터 모델을 불러오지 못했습니다.",
"retry": "다시 시도"
}
}
},
"mcpActions": {
@@ -11236,6 +11249,10 @@
},
"unsupportedSetting": {
"tooltip": "이 모델은 이 설정의 변경을 지원하지 않습니다."
},
"autoItem": {
"label": "자동",
"description": "각 메시지에 가장 적합한 모델을 선택합니다"
}
},
"nrf": {
web/src/i18n/messages/pt.json
+17
View File
@@ -6212,6 +6212,19 @@
"settingsUpdateFailed": "Falha ao atualizar as configurações",
"settingsUpdated": "Configurações atualizadas",
"unknownError": "Erro desconhecido"
},
"modelRouting": {
"autoMode": {
"title": "Modo automático",
"description": "Adiciona uma opção Auto ao seletor de modelos que escolhe o modelo mais adequado para cada mensagem."
},
"target": {
"title": "Modelo de roteamento",
"description": "Gerencia o roteamento de modelos para o modo Auto.",
"empty": "Nenhum modelo de roteador encontrado. Primeiro habilite um modelo de roteador em um provedor de gateway.",
"error": "Falha ao carregar os modelos de roteamento.",
"retry": "Tentar novamente"
}
}
},
"mcpActions": {
@@ -11236,6 +11249,10 @@
},
"unsupportedSetting": {
"tooltip": "Modificar esta configuração não é compatível com este modelo."
},
"autoItem": {
"label": "Auto",
"description": "Escolhe o melhor modelo para cada mensagem"
}
},
"nrf": {
web/src/i18n/messages/zh.json
+17
View File
@@ -6212,6 +6212,19 @@
"settingsUpdateFailed": "更新设置失败",
"settingsUpdated": "设置已更新",
"unknownError": "未知错误"
},
"modelRouting": {
"autoMode": {
"title": "自动模式",
"description": "在模型选择器中添加 Auto 项，为每条消息选择最合适的模型。"
},
"target": {
"title": "路由器模型",
"description": "负责 Auto 模式的模型路由。",
"empty": "未找到路由器模型。请先在网关提供商上启用一个路由器模型。",
"error": "无法加载路由器模型。",
"retry": "重试"
}
}
},
"mcpActions": {
@@ -11236,6 +11249,10 @@
},
"unsupportedSetting": {
"tooltip": "此模型不支持修改该设置。"
},
"autoItem": {
"label": "自动",
"description": "为每条消息选择最佳模型"
}
},
"nrf": {
web/src/lib/languageModels/cache.ts
+1 -1
View File
@@ -6,13 +6,13 @@ import { setDefaultLlmModel } from "@/lib/languageModels/svc";
const PERSONA_PROVIDER_ENDPOINT_PATTERN =
/^\/api\/llm\/persona\/\d+\/providers$/;
const ADMIN_PROVIDER_ENDPOINT_PATTERN = /^\/api\/admin\/llm\/provider\/\d+$/;
export async function refreshLlmProviderCaches(
mutate: ScopedMutator
): Promise<void> {
await Promise.all([
mutate(SWR_KEYS.adminLlmProvidersPaged),
mutate(SWR_KEYS.llmProviders),
mutate(SWR_KEYS.llmRouterModels),
mutate(
(key) =>
typeof key === "string" &&
web/src/lib/languageModels/options.ts
+1
View File
@@ -107,6 +107,7 @@ export function buildLlmOptions(
reasoningEffortDefault: mc.reasoning_effort_default,
temperatureDefault: mc.temperature_default,
supportsImageInput: mc.supports_image_input || false,
isRouter: mc.is_router || false,
});
});
});
web/src/lib/languageModels/svc.ts
+20
View File
@@ -326,6 +326,7 @@ export const fetchOpenRouterModels = async (
max_input_tokens: modelData.max_input_tokens,
supports_image_input: modelData.supports_image_input,
supports_reasoning: false,
is_router: modelData.is_router,
effectiveDisplayName: modelData.display_name || modelData.name,
}));
@@ -848,3 +849,22 @@ export const fetchPortkeyModels = async (
return { models: [], error: errorMessage };
}
};
export async function updateModelRouting(update: {
model_routing_enabled?: boolean;
model_routing_model_configuration_id?: number | null;
}): Promise<void> {
const response: Response = await fetch("/api/admin/llm/model-routing", {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(update),
});
if (!response.ok) {
const errorData: ErrorResponseBody = await response
.json()
.catch(() => ({}));
throw new Error(
errorData.detail || errorData.message || "Failed to update model routing"
);
}
}
web/src/lib/languageModels/types.ts
+8
View File
@@ -39,6 +39,9 @@ export interface ModelConfiguration {
supported_features?: string[];
/** True when this is the provider's recommended default model. */
is_recommended_default?: boolean;
/** Virtual entry that delegates model selection to a routing layer
* (openrouter/auto, a Portkey config) rather than naming a model. */
is_router?: boolean;
display_name?: string;
/** Admin-set override that takes precedence over display_name everywhere in the UI. */
custom_display_name?: string;
@@ -167,6 +170,7 @@ export interface OpenRouterModelResponse {
display_name: string;
max_input_tokens: number | null;
supports_image_input: boolean;
is_router: boolean;
}
export interface BedrockModelResponse {
@@ -381,6 +385,10 @@ export interface LLMOption {
reasoningEffortDefault?: ReasoningEffortOverride | null;
temperatureDefault?: number | null;
supportsImageInput?: boolean;
/** See ModelConfiguration.is_router. */
isRouter?: boolean;
/** Synthesized picker item for model routing; selects the backing router. */
isAuto?: boolean;
}
export interface LLMOptionGroup {
web/src/lib/settings/types.ts
+5
View File
@@ -35,6 +35,11 @@ export interface Settings {
reasoning_override_enabled?: boolean;
// Model selector shows one flat list instead of per-provider groups.
hide_provider_grouping?: boolean;
// "Auto" picker item backed by the admin-chosen router.
// router model configuration (e.g. openrouter/auto).
model_routing_enabled?: boolean;
model_routing_model_configuration_id?: number | null;
query_history_type: QueryHistoryType;
// Visibility-only: hides the sidebar page; query-history APIs + recording stay on.
web/src/lib/swr-keys.ts
+1
View File
@@ -51,6 +51,7 @@ export const SWR_KEYS = {
adminLlmProvidersPaged: "/api/admin/llm/provider?page_models=true",
adminLlmProvider: (providerId: number) =>
`/api/admin/llm/provider/${providerId}`,
llmRouterModels: "/api/admin/llm/router-models",
llmProvidersWithImageGen: "/api/admin/llm/provider?include_image_gen=true",
customProviderNames: "/api/admin/llm/custom-provider-names",
wellKnownLlmProviders: "/api/admin/llm/built-in/options",
web/src/sections/model-selector/ModelSelector.tsx
+12 -2
View File
@@ -5,6 +5,7 @@ import { useTranslations } from "next-intl";
import { Popover, OpenButton } from "@opal/components";
import { getModelIcon } from "@/lib/languageModels/utils";
import { GLOBAL_DEFAULT_LLM_OPTION } from "@/lib/languageModels/options";
import { useSettings } from "@/lib/settings/hooks";
import {
useLanguageModels,
useLanguageModelsForAgent,
@@ -92,6 +93,10 @@ export default function ModelSelector({
llmProviders: globalProviderOptions,
defaultText: globalDefaultText,
} = useLanguageModels();
const {
model_routing_enabled: modelRoutingEnabled,
model_routing_model_configuration_id: routingModelId,
} = useSettings();
const llmProviders = providerOptions ?? fetchedProviderOptions ?? [];
const isLoading = providerOptions === undefined && providersLoading;
const [open, setOpen] = useState(false);
@@ -129,8 +134,13 @@ export default function ModelSelector({
}, [defaultText, llmProviders]);
const effectiveOption = currentOption ?? defaultModelOption;
const currentDisplayName =
effectiveOption?.displayName ?? t("trigger.noSelection.label");
const isAutoSelected: boolean =
(modelRoutingEnabled ?? false) &&
routingModelId != null &&
value === routingModelId;
const currentDisplayName: string = isAutoSelected
? t("autoItem.label")
: (effectiveOption?.displayName ?? t("trigger.noSelection.label"));
const globalDefaultDisplayName = useMemo(() => {
if (!globalDefaultText || !globalProviderOptions) return null;
const provider = globalProviderOptions.find(
web/src/sections/model-selector/ModelSelectorContent.tsx
+28 -5
View File
@@ -18,6 +18,7 @@ import {
SvgChevronRight,
SvgCode,
SvgSliders,
SvgSparkle,
SvgThermometer,
} from "@opal/icons";
import { ContentAction, Section } from "@opal/layouts";
@@ -439,7 +440,11 @@ export default function ModelSelectorContent({
onDetailSelect,
}: ModelSelectorContentProps) {
const t = useTranslations("chat.modelSelector");
const { hide_provider_grouping: hideProviderGrouping } = useSettings();
const {
hide_provider_grouping: hideProviderGrouping,
model_routing_enabled: modelRoutingEnabled,
model_routing_model_configuration_id: routingModelId,
} = useSettings();
const [detailOption, setDetailOption] = useState<LLMOption | null>(null);
const {
llmProviders: currentAgentProviderOptions,
@@ -461,8 +466,18 @@ export default function ModelSelectorContent({
[llmProviders, currentModelName, includeHiddenModels]
);
// "Auto" selects the admin-configured router; router rows never list.
const autoOption = useMemo<LLMOption | undefined>(() => {
if (!modelRoutingEnabled || routingModelId == null) return undefined;
const backing: LLMOption | undefined = llmOptions.find(
(opt) => opt.modelConfigurationId === routingModelId
);
if (!backing) return undefined;
return { ...backing, displayName: t("autoItem.label"), isAuto: true };
}, [llmOptions, modelRoutingEnabled, routingModelId, t]);
const filteredOptions = useMemo(() => {
let result = llmOptions;
let result = llmOptions.filter((opt) => !opt.isRouter);
if (requiresImageInput) {
result = result.filter((opt) => opt.supportsImageInput);
}
@@ -612,8 +627,11 @@ export default function ModelSelectorContent({
const disabled = isDisabled?.(option) ?? false;
// Skip the model-id description when it would just repeat the display name.
const description =
option.modelName !== option.displayName ? option.modelName : undefined;
const description: string | undefined = option.isAuto
? t("autoItem.description")
: option.modelName !== option.displayName
? option.modelName
: undefined;
return (
<Disabled key={llmOptionKey(option)} disabled={disabled}>
@@ -621,7 +639,9 @@ export default function ModelSelectorContent({
<LineItemButton
selectVariant="select-heavy"
state={selected ? "selected" : "empty"}
icon={selectionIcon(selected)}
icon={
option.isAuto && !selected ? SvgSparkle : selectionIcon(selected)
}
title={option.displayName}
description={description}
onClick={() => onSelect(option)}
@@ -692,6 +712,9 @@ export default function ModelSelectorContent({
/>,
]
: []),
...(autoOption && !requiresImageInput && !isLoading
? [renderModelItem(autoOption)]
: []),
null,
...(isLoading
? [
web/src/sections/model-selector/MultiModelSelector.tsx
+14 -6
View File
@@ -10,7 +10,8 @@ import {
Divider,
Tooltip,
} from "@opal/components";
import { SvgPlusCircle, SvgX } from "@opal/icons";
import { SvgPlusCircle, SvgSparkle, SvgX } from "@opal/icons";
import type { IconFunctionComponent } from "@opal/types";
import { cn } from "@opal/utils";
import { useSettings } from "@/lib/settings/hooks";
import { buildLlmOptions, llmOptionKey } from "@/lib/languageModels/options";
@@ -58,6 +59,11 @@ export default function MultiModelSelector({
const settings = useSettings();
const multiModelAllowed = settings.multi_model_chat_enabled ?? true;
const routingEnabled: boolean =
(settings.model_routing_enabled ?? false) &&
settings.model_routing_model_configuration_id != null;
const routingModelId: number | null | undefined =
settings.model_routing_model_configuration_id;
const modelDetail = useModelDetailManagers(
temperatureManager,
@@ -217,10 +223,12 @@ export default function MultiModelSelector({
)}
<div className="flex items-center shrink-0">
{selectedModels.map((model, index) => {
const ProviderIcon = getModelIcon(
model.provider,
model.modelName
);
const isAutoModel: boolean =
routingEnabled &&
model.modelConfigurationId === routingModelId;
const ProviderIcon: IconFunctionComponent = isAutoModel
? SvgSparkle
: getModelIcon(model.provider, model.modelName);
return (
<div
@@ -261,7 +269,7 @@ export default function MultiModelSelector({
);
}}
>
{model.displayName}
{isAutoModel ? t("autoItem.label") : model.displayName}
</SelectButton>
</div>
);
web/src/views/admin/LanguageModelsPage.tsx
+133 -2
View File
@@ -3,7 +3,8 @@
import { useAdminRouteTitle } from "@/lib/adminNavLabels";
import { useState, useMemo, useEffect } from "react";
import { useTranslations } from "next-intl";
import { useSWRConfig } from "swr";
import useSWR, { useSWRConfig } from "swr";
import { errorHandlingFetcher } from "@/lib/fetcher";
import {
useAdminLanguageModel,
useAdminLanguageModels,
@@ -18,6 +19,7 @@ import {
InputSwitch,
Text,
Card,
OnyxLoader,
} from "@opal/components";
import { Hoverable, Disabled } from "@opal/core";
import { SvgArrowExchange, SvgSettings, SvgTrash } from "@opal/icons";
@@ -29,7 +31,10 @@ import {
refreshLlmProviderCaches,
setDefaultLlmModelAndRefresh,
} from "@/lib/languageModels/cache";
import { deleteLlmProvider } from "@/lib/languageModels/svc";
import {
deleteLlmProvider,
updateModelRouting,
} from "@/lib/languageModels/svc";
import { buildLlmOptions, groupLlmOptions } from "@/lib/languageModels/options";
import { findProviderOwningModelConfig } from "@/lib/languageModels/utils";
import {
@@ -366,6 +371,12 @@ export default function LanguageModelsPage() {
const [pendingHideGrouping, setPendingHideGrouping] = useState<
boolean | null
>(null);
const [pendingRoutingEnabled, setPendingRoutingEnabled] = useState<
boolean | null
>(null);
const [pendingRoutingTarget, setPendingRoutingTarget] = useState<
number | null
>(null);
const {
llmProviders: existingLlmProviders,
defaultText,
@@ -382,6 +393,26 @@ export default function LanguageModelsPage() {
[existingLlmProviders]
);
// Includes hidden and persona-restricted routers, unlike the paged listings.
const {
data: routerProviders,
error: routerProvidersError,
mutate: mutateRouterProviders,
} = useSWR<LLMProviderView[]>(
SWR_KEYS.llmRouterModels,
async (url: string) => {
const providers = await errorHandlingFetcher<LLMProviderView[]>(url);
return providers.map((provider) => ({
...provider,
model_configurations: provider.model_configurations.map((mc) => ({
...mc,
effectiveDisplayName:
mc.custom_display_name || mc.display_name || mc.name,
})),
}));
}
);
// Resolve the current default to a model_configuration_id for the select
const defaultModelConfigId = useMemo(() => {
if (!defaultText || !existingLlmProviders) return null;
@@ -490,6 +521,47 @@ export default function LanguageModelsPage() {
}
}
const routingEnabled =
pendingRoutingEnabled ?? settings.model_routing_enabled ?? false;
async function handleRoutingEnabledChange(checked: boolean) {
if (pendingRoutingEnabled !== null) return;
setPendingRoutingEnabled(checked);
try {
await updateModelRouting({ model_routing_enabled: checked });
await mutate(SWR_KEYS.settings);
toast.success(t("toasts.settingsUpdated"));
} catch (e) {
toast.error(
e instanceof Error ? e.message : t("toasts.settingsUpdateFailed")
);
} finally {
setPendingRoutingEnabled(null);
}
}
async function handleRoutingTargetChange(modelConfigurationId: number) {
if (pendingRoutingTarget !== null) return;
setPendingRoutingTarget(modelConfigurationId);
try {
await updateModelRouting({
model_routing_model_configuration_id: modelConfigurationId,
});
// The backend may have flipped the router's visibility.
await Promise.all([
mutate(SWR_KEYS.settings),
refreshLlmProviderCaches(mutate),
]);
toast.success(t("toasts.settingsUpdated"));
} catch (e) {
toast.error(
e instanceof Error ? e.message : t("toasts.settingsUpdateFailed")
);
} finally {
setPendingRoutingTarget(null);
}
}
return (
<SettingsLayouts.Root>
<SettingsLayouts.Header
@@ -543,6 +615,65 @@ export default function LanguageModelsPage() {
}}
/>
</InputHorizontal>
<InputHorizontal
title={t("modelRouting.autoMode.title")}
description={t("modelRouting.autoMode.description")}
withLabel
>
<InputSwitch
checked={routingEnabled}
disabled={pendingRoutingEnabled !== null}
onCheckedChange={(checked) => {
void handleRoutingEnabledChange(checked);
}}
/>
</InputHorizontal>
{routingEnabled && (
<InputHorizontal
title={t("modelRouting.target.title")}
description={t("modelRouting.target.description")}
withLabel
>
{routerProvidersError ? (
<Section flexDirection="row" alignItems="center" gap={2}>
<Text font="secondary-body" color="text-03">
{t("modelRouting.target.error")}
</Text>
<Button
prominence="tertiary"
size="sm"
onClick={() => void mutateRouterProviders()}
>
{t("modelRouting.target.retry")}
</Button>
</Section>
) : routerProviders === undefined ? (
<OnyxLoader size={24} />
) : routerProviders.length > 0 ? (
<SimpleModelSelector
providers={routerProviders}
value={
pendingRoutingTarget ??
settings.model_routing_model_configuration_id ??
null
}
disabled={pendingRoutingTarget !== null}
grouped={
!(
pendingHideGrouping ?? settings.hide_provider_grouping
)
}
onChange={(modelConfigurationId) => {
void handleRoutingTargetChange(modelConfigurationId);
}}
/>
) : (
<Text font="secondary-body" color="text-03">
{t("modelRouting.target.empty")}
</Text>
)}
</InputHorizontal>
)}
{hasProviderGrouping && (
<InputHorizontal
title={t("hideProviderGrouping.title")}