diff --git a/backend/ee/onyx/configs/license_enforcement_config.py b/backend/ee/onyx/configs/license_enforcement_config.py index 5fecee55bc..68c05ddc7d 100644 --- a/backend/ee/onyx/configs/license_enforcement_config.py +++ b/backend/ee/onyx/configs/license_enforcement_config.py @@ -72,6 +72,7 @@ PATH_PREFIX_MIN_TIER: dict[str, Tier] = { "/admin/enterprise-settings": Tier.BUSINESS, # admin writes; public /enterprise-settings stays open "/manage/admin/user-group": Tier.BUSINESS, # groups + RBAC (Curator roles, group-scoped access) # ----- ENTERPRISE ----- + "/gateway": Tier.ENTERPRISE, # external LLM gateway API "/admin/enterprise-settings/custom-analytics-script": Tier.ENTERPRISE, # JS injection "/admin/enterprise-settings/scim": Tier.ENTERPRISE, # SCIM token mgmt "/manage/admin/standard-answer": Tier.ENTERPRISE, diff --git a/backend/ee/onyx/main.py b/backend/ee/onyx/main.py index 7fac88612d..7d644789a3 100644 --- a/backend/ee/onyx/main.py +++ b/backend/ee/onyx/main.py @@ -16,6 +16,7 @@ from ee.onyx.server.enterprise_settings.api import ( ) from ee.onyx.server.evals.api import router as evals_router from ee.onyx.server.features.hooks.api import router as hook_router +from ee.onyx.server.gateway.api import router as llm_gateway_router from ee.onyx.server.license.api import router as license_router from ee.onyx.server.log_export.api import router as log_export_router from ee.onyx.server.manage.standard_answer import router as standard_answer_router @@ -142,6 +143,7 @@ def get_application() -> FastAPI: include_router_with_global_prefix_prepended(application, ee_document_cc_pair_router) include_router_with_global_prefix_prepended(application, evals_router) include_router_with_global_prefix_prepended(application, hook_router) + include_router_with_global_prefix_prepended(application, llm_gateway_router) # Enterprise-only global settings include_router_with_global_prefix_prepended( diff --git a/backend/ee/onyx/server/gateway/__init__.py b/backend/ee/onyx/server/gateway/__init__.py new file mode 100644 index 0000000000..cae80cfecd --- /dev/null +++ b/backend/ee/onyx/server/gateway/__init__.py @@ -0,0 +1 @@ +"""Enterprise LLM gateway API implementation.""" diff --git a/backend/onyx/server/gateway/anthropic_passthrough.py b/backend/ee/onyx/server/gateway/anthropic_passthrough.py similarity index 99% rename from backend/onyx/server/gateway/anthropic_passthrough.py rename to backend/ee/onyx/server/gateway/anthropic_passthrough.py index e4ad105e61..d2c62b5211 100644 --- a/backend/onyx/server/gateway/anthropic_passthrough.py +++ b/backend/ee/onyx/server/gateway/anthropic_passthrough.py @@ -20,6 +20,12 @@ import httpx from fastapi import Request from fastapi.responses import JSONResponse, StreamingResponse +from ee.onyx.server.gateway.stream_bridge import ( + _put_stream_item, + _sse_response, + _stream_worker_guard, + _StreamAccumulator, +) from onyx.db.models import User from onyx.error_handling.error_codes import OnyxErrorCode from onyx.error_handling.exceptions import OnyxError @@ -37,12 +43,6 @@ from onyx.server.gateway.models import ( AnthropicErrorEvent, AnthropicMessagesRequest, ) -from onyx.server.gateway.stream_bridge import ( - _put_stream_item, - _sse_response, - _stream_worker_guard, - _StreamAccumulator, -) from onyx.server.manage.llm.models import LLMProviderView, ModelConfigurationView from onyx.tracing.flows import LLMFlow from onyx.tracing.framework.create import trace diff --git a/backend/onyx/server/gateway/api.py b/backend/ee/onyx/server/gateway/api.py similarity index 99% rename from backend/onyx/server/gateway/api.py rename to backend/ee/onyx/server/gateway/api.py index c9c676b16f..73b926f5b8 100644 --- a/backend/onyx/server/gateway/api.py +++ b/backend/ee/onyx/server/gateway/api.py @@ -13,6 +13,23 @@ from fastapi.responses import JSONResponse, StreamingResponse from pydantic import TypeAdapter, ValidationError from sqlalchemy.orm import Session +from ee.onyx.server.gateway.anthropic_passthrough import ( + AnthropicPassthroughUnavailable, + handle_anthropic_count_tokens_passthrough, + handle_anthropic_passthrough, + is_anthropic_passthrough_eligible, +) +from ee.onyx.server.gateway.openai_passthrough import ( + handle_openai_responses_passthrough, + is_openai_passthrough_eligible, +) +from ee.onyx.server.gateway.stream_bridge import ( + _RATE_LIMIT_ERROR, + _put_stream_item, + _sse_response, + _stream_worker_guard, + _StreamAccumulator, +) from onyx.auth.permissions import require_permission from onyx.db.engine.sql_engine import get_session from onyx.db.enums import Permission @@ -44,12 +61,6 @@ from onyx.llm.multi_llm import LLMRateLimitError, LLMTimeoutError from onyx.llm.prompt_cache.processor import process_with_prompt_cache from onyx.llm.tracing_wrap import _finalize_tool_calls from onyx.server.features.build.craft_gateway import gateway_request_flow -from onyx.server.gateway.anthropic_passthrough import ( - AnthropicPassthroughUnavailable, - handle_anthropic_count_tokens_passthrough, - handle_anthropic_passthrough, - is_anthropic_passthrough_eligible, -) from onyx.server.gateway.configs import GATEWAY_PATH_PREFIX from onyx.server.gateway.model_catalog import build_gateway_model_catalog from onyx.server.gateway.models import ( @@ -98,17 +109,6 @@ from onyx.server.gateway.models import ( ResponsesOutputTextPart, ResponsesRequest, ) -from onyx.server.gateway.openai_passthrough import ( - handle_openai_responses_passthrough, - is_openai_passthrough_eligible, -) -from onyx.server.gateway.stream_bridge import ( - _RATE_LIMIT_ERROR, - _put_stream_item, - _sse_response, - _stream_worker_guard, - _StreamAccumulator, -) from onyx.server.manage.llm.models import LLMProviderView, ModelConfigurationView from onyx.server.query_and_chat.token_limit import check_token_rate_limits from onyx.tracing.flows import LLMFlow diff --git a/backend/onyx/server/gateway/openai_passthrough.py b/backend/ee/onyx/server/gateway/openai_passthrough.py similarity index 99% rename from backend/onyx/server/gateway/openai_passthrough.py rename to backend/ee/onyx/server/gateway/openai_passthrough.py index 9ff51b4d23..d51925f234 100644 --- a/backend/onyx/server/gateway/openai_passthrough.py +++ b/backend/ee/onyx/server/gateway/openai_passthrough.py @@ -15,6 +15,12 @@ from typing import Any import httpx from fastapi.responses import JSONResponse, StreamingResponse +from ee.onyx.server.gateway.stream_bridge import ( + _put_stream_item, + _sse_response, + _stream_worker_guard, + _StreamAccumulator, +) from onyx.db.models import User from onyx.error_handling.error_codes import OnyxErrorCode from onyx.error_handling.exceptions import OnyxError @@ -35,12 +41,6 @@ from onyx.server.gateway.models import ( ResponsesObjectPayload, ResponsesRequest, ) -from onyx.server.gateway.stream_bridge import ( - _put_stream_item, - _sse_response, - _stream_worker_guard, - _StreamAccumulator, -) from onyx.server.manage.llm.models import LLMProviderView, ModelConfigurationView from onyx.tracing.flows import LLMFlow from onyx.tracing.framework.create import trace diff --git a/backend/onyx/server/gateway/stream_bridge.py b/backend/ee/onyx/server/gateway/stream_bridge.py similarity index 96% rename from backend/onyx/server/gateway/stream_bridge.py rename to backend/ee/onyx/server/gateway/stream_bridge.py index afbaa6a7f5..d065775fff 100644 --- a/backend/onyx/server/gateway/stream_bridge.py +++ b/backend/ee/onyx/server/gateway/stream_bridge.py @@ -1,8 +1,7 @@ -"""Shared SSE-over-thread streaming primitives for the LLM gateway. +"""Shared SSE-over-thread streaming primitives for the EE LLM gateway. -Moved verbatim out of ``onyx.server.gateway.api`` so both the translation -path (api.py) and the Anthropic native passthrough path -(anthropic_passthrough.py) can share them without a circular import. +The translation and native passthrough paths share these helpers to avoid a +circular import. """ import queue diff --git a/backend/onyx/main.py b/backend/onyx/main.py index e9c2aa84b6..7372a29920 100644 --- a/backend/onyx/main.py +++ b/backend/onyx/main.py @@ -110,7 +110,6 @@ from onyx.server.features.usage.api import router as cost_override_router from onyx.server.features.user_oauth_token.api import router as user_oauth_token_router from onyx.server.features.web_search.api import router as web_search_router from onyx.server.federated.api import router as federated_router -from onyx.server.gateway.api import router as llm_gateway_router from onyx.server.kg.api import admin_router as kg_admin_router from onyx.server.manage.administrative import router as admin_router from onyx.server.manage.code_interpreter.api import ( @@ -544,7 +543,6 @@ def get_application(lifespan_override: Lifespan | None = None) -> FastAPI: include_router_with_global_prefix_prepended(application, public_build_router) include_router_with_global_prefix_prepended(application, build_router) include_router_with_global_prefix_prepended(application, build_admin_router) - include_router_with_global_prefix_prepended(application, llm_gateway_router) include_router_with_global_prefix_prepended(application, image_generation_router) include_router_with_global_prefix_prepended(application, document_set_router) include_router_with_global_prefix_prepended(application, hierarchy_router) diff --git a/backend/tests/integration/tests/gateway_clients/README.md b/backend/tests/integration/tests/gateway_clients/README.md index f5b22fe2d8..c04cebc308 100644 --- a/backend/tests/integration/tests/gateway_clients/README.md +++ b/backend/tests/integration/tests/gateway_clients/README.md @@ -1,7 +1,7 @@ # Gateway Client Integration Tests -Exercises the Onyx AI gateway's Anthropic (`onyx/server/gateway/anthropic_passthrough.py`) -and OpenAI (`onyx/server/gateway/openai_passthrough.py`) passthrough endpoints with +Exercises the Onyx AI gateway's Anthropic (`ee/onyx/server/gateway/anthropic_passthrough.py`) +and OpenAI (`ee/onyx/server/gateway/openai_passthrough.py`) passthrough endpoints with REAL coding-agent CLIs: `@anthropic-ai/claude-code` and `@openai/codex`. ## Why this directory is different diff --git a/backend/tests/integration/tests/gateway_clients/test_claude_code_gateway.py b/backend/tests/integration/tests/gateway_clients/test_claude_code_gateway.py index 528c46c7c9..84a9153655 100644 --- a/backend/tests/integration/tests/gateway_clients/test_claude_code_gateway.py +++ b/backend/tests/integration/tests/gateway_clients/test_claude_code_gateway.py @@ -1,5 +1,5 @@ """Integration tests for the Claude Code CLI against the gateway's -Anthropic Messages API passthrough (onyx.server.gateway.anthropic_passthrough). +Anthropic Messages API passthrough (ee.onyx.server.gateway.anthropic_passthrough). Test Suite: 1. test_basic_turn_answers_correctly - `claude -p` answers correctly through the gateway diff --git a/backend/tests/integration/tests/gateway_clients/test_codex_gateway.py b/backend/tests/integration/tests/gateway_clients/test_codex_gateway.py index 4315d28eab..f9282b4cb5 100644 --- a/backend/tests/integration/tests/gateway_clients/test_codex_gateway.py +++ b/backend/tests/integration/tests/gateway_clients/test_codex_gateway.py @@ -1,5 +1,5 @@ """Integration tests for the Codex CLI against the gateway's Responses API -passthrough (onyx.server.gateway.openai_passthrough). +passthrough (ee.onyx.server.gateway.openai_passthrough). Test Suite: 1. test_basic_codex_turn - `codex exec` answers correctly through the gateway diff --git a/backend/tests/unit/onyx/server/gateway/fixtures/codex_responses_request.json b/backend/tests/unit/ee/onyx/server/gateway/fixtures/codex_responses_request.json similarity index 100% rename from backend/tests/unit/onyx/server/gateway/fixtures/codex_responses_request.json rename to backend/tests/unit/ee/onyx/server/gateway/fixtures/codex_responses_request.json diff --git a/backend/tests/unit/onyx/server/gateway/test_anthropic_messages_api.py b/backend/tests/unit/ee/onyx/server/gateway/test_anthropic_messages_api.py similarity index 99% rename from backend/tests/unit/onyx/server/gateway/test_anthropic_messages_api.py rename to backend/tests/unit/ee/onyx/server/gateway/test_anthropic_messages_api.py index 3101cf825b..187b3cd37a 100644 --- a/backend/tests/unit/onyx/server/gateway/test_anthropic_messages_api.py +++ b/backend/tests/unit/ee/onyx/server/gateway/test_anthropic_messages_api.py @@ -11,6 +11,9 @@ from fastapi import Request from fastapi.responses import StreamingResponse from sqlalchemy.orm import Session +from ee.onyx.server.gateway import api as gateway_api +from ee.onyx.server.gateway import stream_bridge +from ee.onyx.server.gateway.api import _MESSAGES_ADAPTER from onyx.error_handling.error_codes import OnyxErrorCode from onyx.error_handling.exceptions import OnyxError from onyx.llm.interfaces import LLM @@ -39,16 +42,13 @@ from onyx.llm.models import ( UserMessage, ) from onyx.llm.multi_llm import LLMRateLimitError, LLMTimeoutError -from onyx.server.gateway import api as gateway_api -from onyx.server.gateway import stream_bridge -from onyx.server.gateway.api import _MESSAGES_ADAPTER from onyx.server.gateway.models import ( AnthropicCountTokensRequest, AnthropicMessageResponse, AnthropicMessagesRequest, ) from onyx.tracing.flows import LLMFlow -from tests.unit.onyx.server.gateway.test_llm_gateway_api import ( +from tests.unit.ee.onyx.server.gateway.test_llm_gateway_api import ( _ChunkStreamLLM, _InvokeLLM, _model, diff --git a/backend/tests/unit/onyx/server/gateway/test_anthropic_passthrough.py b/backend/tests/unit/ee/onyx/server/gateway/test_anthropic_passthrough.py similarity index 99% rename from backend/tests/unit/onyx/server/gateway/test_anthropic_passthrough.py rename to backend/tests/unit/ee/onyx/server/gateway/test_anthropic_passthrough.py index 850407907a..bfee9cd422 100644 --- a/backend/tests/unit/onyx/server/gateway/test_anthropic_passthrough.py +++ b/backend/tests/unit/ee/onyx/server/gateway/test_anthropic_passthrough.py @@ -11,12 +11,9 @@ import pytest from fastapi import Request from fastapi.responses import JSONResponse -from onyx.error_handling.error_codes import OnyxErrorCode -from onyx.error_handling.exceptions import OnyxError -from onyx.llm.interfaces import LLMConfig -from onyx.server.gateway import anthropic_passthrough, stream_bridge -from onyx.server.gateway import api as gateway_api -from onyx.server.gateway.anthropic_passthrough import ( +from ee.onyx.server.gateway import anthropic_passthrough, stream_bridge +from ee.onyx.server.gateway import api as gateway_api +from ee.onyx.server.gateway.anthropic_passthrough import ( AnthropicPassthroughUnavailable, _build_upstream_headers, _build_upstream_request, @@ -29,13 +26,16 @@ from onyx.server.gateway.anthropic_passthrough import ( handle_anthropic_passthrough, is_anthropic_passthrough_eligible, ) +from onyx.error_handling.error_codes import OnyxErrorCode +from onyx.error_handling.exceptions import OnyxError +from onyx.llm.interfaces import LLMConfig from onyx.server.gateway.models import ( AnthropicCountTokensRequest, AnthropicMessagesRequest, AnthropicUsagePayload, ) from onyx.tracing.flows import LLMFlow -from tests.unit.onyx.server.gateway.test_llm_gateway_api import ( +from tests.unit.ee.onyx.server.gateway.test_llm_gateway_api import ( _ConfigOnlyLLM, _model, _provider, diff --git a/backend/tests/unit/onyx/server/gateway/test_llm_gateway_api.py b/backend/tests/unit/ee/onyx/server/gateway/test_llm_gateway_api.py similarity index 99% rename from backend/tests/unit/onyx/server/gateway/test_llm_gateway_api.py rename to backend/tests/unit/ee/onyx/server/gateway/test_llm_gateway_api.py index 1953a7c7c7..5da90199d0 100644 --- a/backend/tests/unit/onyx/server/gateway/test_llm_gateway_api.py +++ b/backend/tests/unit/ee/onyx/server/gateway/test_llm_gateway_api.py @@ -14,6 +14,9 @@ from fastapi.responses import StreamingResponse from fastapi.routing import APIRoute from sqlalchemy.orm import Session +from ee.onyx.server.gateway import api as gateway_api +from ee.onyx.server.gateway import stream_bridge +from ee.onyx.server.gateway.api import _MESSAGES_ADAPTER from onyx.db.enums import Permission from onyx.db.models import User from onyx.error_handling.error_codes import OnyxErrorCode @@ -50,9 +53,6 @@ from onyx.llm.multi_llm import LLMRateLimitError, LLMTimeoutError from onyx.server.auth_check import check_router_auth from onyx.server.features.build import craft_gateway from onyx.server.features.build.craft_gateway import gateway_request_flow -from onyx.server.gateway import api as gateway_api -from onyx.server.gateway import stream_bridge -from onyx.server.gateway.api import _MESSAGES_ADAPTER from onyx.server.gateway.configs import GATEWAY_PATH_PREFIX from onyx.server.gateway.models import ( ChatCompletionRequest, diff --git a/backend/tests/unit/onyx/server/gateway/test_openai_passthrough.py b/backend/tests/unit/ee/onyx/server/gateway/test_openai_passthrough.py similarity index 99% rename from backend/tests/unit/onyx/server/gateway/test_openai_passthrough.py rename to backend/tests/unit/ee/onyx/server/gateway/test_openai_passthrough.py index 7aca37f4fd..8924d18491 100644 --- a/backend/tests/unit/onyx/server/gateway/test_openai_passthrough.py +++ b/backend/tests/unit/ee/onyx/server/gateway/test_openai_passthrough.py @@ -12,14 +12,9 @@ import pytest from fastapi import Request from fastapi.responses import JSONResponse -from onyx.error_handling.error_codes import OnyxErrorCode -from onyx.error_handling.exceptions import OnyxError -from onyx.llm.interfaces import LLMConfig -from onyx.llm.multi_llm import LitellmLLM -from onyx.server.gateway import api as gateway_api -from onyx.server.gateway import openai_passthrough, stream_bridge -from onyx.server.gateway.models import ResponsesRequest -from onyx.server.gateway.openai_passthrough import ( +from ee.onyx.server.gateway import api as gateway_api +from ee.onyx.server.gateway import openai_passthrough, stream_bridge +from ee.onyx.server.gateway.openai_passthrough import ( _SANITIZED_ERROR, _base_url, _build_upstream_headers, @@ -33,8 +28,13 @@ from onyx.server.gateway.openai_passthrough import ( handle_openai_responses_passthrough, is_openai_passthrough_eligible, ) +from onyx.error_handling.error_codes import OnyxErrorCode +from onyx.error_handling.exceptions import OnyxError +from onyx.llm.interfaces import LLMConfig +from onyx.llm.multi_llm import LitellmLLM +from onyx.server.gateway.models import ResponsesRequest from onyx.tracing.flows import LLMFlow -from tests.unit.onyx.server.gateway.test_llm_gateway_api import ( +from tests.unit.ee.onyx.server.gateway.test_llm_gateway_api import ( _ConfigOnlyLLM, _model, _provider,