feat(ee): move the LLM gateway into EE (#13746)

Co-authored-by: OmX <omx@oh-my-codex.dev>

backend/ee/onyx/configs/license_enforcement_config.py

@@ -72,6 +72,7 @@ PATH_PREFIX_MIN_TIER: dict[str, Tier] = {
     "/admin/enterprise-settings": Tier.BUSINESS,  # admin writes; public /enterprise-settings stays open
     "/manage/admin/user-group": Tier.BUSINESS,  # groups + RBAC (Curator roles, group-scoped access)
     # ----- ENTERPRISE -----
+    "/gateway": Tier.ENTERPRISE,  # external LLM gateway API
     "/admin/enterprise-settings/custom-analytics-script": Tier.ENTERPRISE,  # JS injection
     "/admin/enterprise-settings/scim": Tier.ENTERPRISE,  # SCIM token mgmt
     "/manage/admin/standard-answer": Tier.ENTERPRISE,

backend/ee/onyx/main.py

@@ -16,6 +16,7 @@ from ee.onyx.server.enterprise_settings.api import (
 )
 from ee.onyx.server.evals.api import router as evals_router
 from ee.onyx.server.features.hooks.api import router as hook_router
+from ee.onyx.server.gateway.api import router as llm_gateway_router
 from ee.onyx.server.license.api import router as license_router
 from ee.onyx.server.log_export.api import router as log_export_router
 from ee.onyx.server.manage.standard_answer import router as standard_answer_router
@@ -142,6 +143,7 @@ def get_application() -> FastAPI:
     include_router_with_global_prefix_prepended(application, ee_document_cc_pair_router)
     include_router_with_global_prefix_prepended(application, evals_router)
     include_router_with_global_prefix_prepended(application, hook_router)
+    include_router_with_global_prefix_prepended(application, llm_gateway_router)
 
     # Enterprise-only global settings
     include_router_with_global_prefix_prepended(

backend/ee/onyx/server/gateway/__init__.py

@@ -0,0 +1 @@
+"""Enterprise LLM gateway API implementation."""

backend/ee/onyx/server/gateway/anthropic_passthrough.py

@@ -20,6 +20,12 @@ import httpx
 from fastapi import Request
 from fastapi.responses import JSONResponse, StreamingResponse
 
+from ee.onyx.server.gateway.stream_bridge import (
+    _put_stream_item,
+    _sse_response,
+    _stream_worker_guard,
+    _StreamAccumulator,
+)
 from onyx.db.models import User
 from onyx.error_handling.error_codes import OnyxErrorCode
 from onyx.error_handling.exceptions import OnyxError
@@ -37,12 +43,6 @@ from onyx.server.gateway.models import (
     AnthropicErrorEvent,
     AnthropicMessagesRequest,
 )
-from onyx.server.gateway.stream_bridge import (
-    _put_stream_item,
-    _sse_response,
-    _stream_worker_guard,
-    _StreamAccumulator,
-)
 from onyx.server.manage.llm.models import LLMProviderView, ModelConfigurationView
 from onyx.tracing.flows import LLMFlow
 from onyx.tracing.framework.create import trace

backend/ee/onyx/server/gateway/api.py

@@ -13,6 +13,23 @@ from fastapi.responses import JSONResponse, StreamingResponse
 from pydantic import TypeAdapter, ValidationError
 from sqlalchemy.orm import Session
 
+from ee.onyx.server.gateway.anthropic_passthrough import (
+    AnthropicPassthroughUnavailable,
+    handle_anthropic_count_tokens_passthrough,
+    handle_anthropic_passthrough,
+    is_anthropic_passthrough_eligible,
+)
+from ee.onyx.server.gateway.openai_passthrough import (
+    handle_openai_responses_passthrough,
+    is_openai_passthrough_eligible,
+)
+from ee.onyx.server.gateway.stream_bridge import (
+    _RATE_LIMIT_ERROR,
+    _put_stream_item,
+    _sse_response,
+    _stream_worker_guard,
+    _StreamAccumulator,
+)
 from onyx.auth.permissions import require_permission
 from onyx.db.engine.sql_engine import get_session
 from onyx.db.enums import Permission
@@ -44,12 +61,6 @@ from onyx.llm.multi_llm import LLMRateLimitError, LLMTimeoutError
 from onyx.llm.prompt_cache.processor import process_with_prompt_cache
 from onyx.llm.tracing_wrap import _finalize_tool_calls
 from onyx.server.features.build.craft_gateway import gateway_request_flow
-from onyx.server.gateway.anthropic_passthrough import (
-    AnthropicPassthroughUnavailable,
-    handle_anthropic_count_tokens_passthrough,
-    handle_anthropic_passthrough,
-    is_anthropic_passthrough_eligible,
-)
 from onyx.server.gateway.configs import GATEWAY_PATH_PREFIX
 from onyx.server.gateway.model_catalog import build_gateway_model_catalog
 from onyx.server.gateway.models import (
@@ -98,17 +109,6 @@ from onyx.server.gateway.models import (
     ResponsesOutputTextPart,
     ResponsesRequest,
 )
-from onyx.server.gateway.openai_passthrough import (
-    handle_openai_responses_passthrough,
-    is_openai_passthrough_eligible,
-)
-from onyx.server.gateway.stream_bridge import (
-    _RATE_LIMIT_ERROR,
-    _put_stream_item,
-    _sse_response,
-    _stream_worker_guard,
-    _StreamAccumulator,
-)
 from onyx.server.manage.llm.models import LLMProviderView, ModelConfigurationView
 from onyx.server.query_and_chat.token_limit import check_token_rate_limits
 from onyx.tracing.flows import LLMFlow

backend/ee/onyx/server/gateway/openai_passthrough.py

@@ -15,6 +15,12 @@ from typing import Any
 import httpx
 from fastapi.responses import JSONResponse, StreamingResponse
 
+from ee.onyx.server.gateway.stream_bridge import (
+    _put_stream_item,
+    _sse_response,
+    _stream_worker_guard,
+    _StreamAccumulator,
+)
 from onyx.db.models import User
 from onyx.error_handling.error_codes import OnyxErrorCode
 from onyx.error_handling.exceptions import OnyxError
@@ -35,12 +41,6 @@ from onyx.server.gateway.models import (
     ResponsesObjectPayload,
     ResponsesRequest,
 )
-from onyx.server.gateway.stream_bridge import (
-    _put_stream_item,
-    _sse_response,
-    _stream_worker_guard,
-    _StreamAccumulator,
-)
 from onyx.server.manage.llm.models import LLMProviderView, ModelConfigurationView
 from onyx.tracing.flows import LLMFlow
 from onyx.tracing.framework.create import trace

backend/ee/onyx/server/gateway/stream_bridge.py

@@ -1,8 +1,7 @@
-"""Shared SSE-over-thread streaming primitives for the LLM gateway.
+"""Shared SSE-over-thread streaming primitives for the EE LLM gateway.
 
-Moved verbatim out of ``onyx.server.gateway.api`` so both the translation
-path (api.py) and the Anthropic native passthrough path
-(anthropic_passthrough.py) can share them without a circular import.
+The translation and native passthrough paths share these helpers to avoid a
+circular import.
 """
 
 import queue

backend/onyx/main.py

@@ -110,7 +110,6 @@ from onyx.server.features.usage.api import router as cost_override_router
 from onyx.server.features.user_oauth_token.api import router as user_oauth_token_router
 from onyx.server.features.web_search.api import router as web_search_router
 from onyx.server.federated.api import router as federated_router
-from onyx.server.gateway.api import router as llm_gateway_router
 from onyx.server.kg.api import admin_router as kg_admin_router
 from onyx.server.manage.administrative import router as admin_router
 from onyx.server.manage.code_interpreter.api import (
@@ -544,7 +543,6 @@ def get_application(lifespan_override: Lifespan | None = None) -> FastAPI:
     include_router_with_global_prefix_prepended(application, public_build_router)
     include_router_with_global_prefix_prepended(application, build_router)
     include_router_with_global_prefix_prepended(application, build_admin_router)
-    include_router_with_global_prefix_prepended(application, llm_gateway_router)
     include_router_with_global_prefix_prepended(application, image_generation_router)
     include_router_with_global_prefix_prepended(application, document_set_router)
     include_router_with_global_prefix_prepended(application, hierarchy_router)

backend/tests/integration/tests/gateway_clients/README.md

@@ -1,7 +1,7 @@
 # Gateway Client Integration Tests
 
-Exercises the Onyx AI gateway's Anthropic (`onyx/server/gateway/anthropic_passthrough.py`)
-and OpenAI (`onyx/server/gateway/openai_passthrough.py`) passthrough endpoints with
+Exercises the Onyx AI gateway's Anthropic (`ee/onyx/server/gateway/anthropic_passthrough.py`)
+and OpenAI (`ee/onyx/server/gateway/openai_passthrough.py`) passthrough endpoints with
 REAL coding-agent CLIs: `@anthropic-ai/claude-code` and `@openai/codex`.
 
 ## Why this directory is different

backend/tests/integration/tests/gateway_clients/test_claude_code_gateway.py

@@ -1,5 +1,5 @@
 """Integration tests for the Claude Code CLI against the gateway's
-Anthropic Messages API passthrough (onyx.server.gateway.anthropic_passthrough).
+Anthropic Messages API passthrough (ee.onyx.server.gateway.anthropic_passthrough).
 
 Test Suite:
 1. test_basic_turn_answers_correctly - `claude -p` answers correctly through the gateway

backend/tests/integration/tests/gateway_clients/test_codex_gateway.py

@@ -1,5 +1,5 @@
 """Integration tests for the Codex CLI against the gateway's Responses API
-passthrough (onyx.server.gateway.openai_passthrough).
+passthrough (ee.onyx.server.gateway.openai_passthrough).
 
 Test Suite:
 1. test_basic_codex_turn - `codex exec` answers correctly through the gateway

backend/tests/unit/ee/onyx/server/gateway/test_anthropic_messages_api.py

@@ -11,6 +11,9 @@ from fastapi import Request
 from fastapi.responses import StreamingResponse
 from sqlalchemy.orm import Session
 
+from ee.onyx.server.gateway import api as gateway_api
+from ee.onyx.server.gateway import stream_bridge
+from ee.onyx.server.gateway.api import _MESSAGES_ADAPTER
 from onyx.error_handling.error_codes import OnyxErrorCode
 from onyx.error_handling.exceptions import OnyxError
 from onyx.llm.interfaces import LLM
@@ -39,16 +42,13 @@ from onyx.llm.models import (
     UserMessage,
 )
 from onyx.llm.multi_llm import LLMRateLimitError, LLMTimeoutError
-from onyx.server.gateway import api as gateway_api
-from onyx.server.gateway import stream_bridge
-from onyx.server.gateway.api import _MESSAGES_ADAPTER
 from onyx.server.gateway.models import (
     AnthropicCountTokensRequest,
     AnthropicMessageResponse,
     AnthropicMessagesRequest,
 )
 from onyx.tracing.flows import LLMFlow
-from tests.unit.onyx.server.gateway.test_llm_gateway_api import (
+from tests.unit.ee.onyx.server.gateway.test_llm_gateway_api import (
     _ChunkStreamLLM,
     _InvokeLLM,
     _model,

backend/tests/unit/ee/onyx/server/gateway/test_anthropic_passthrough.py

@@ -11,12 +11,9 @@ import pytest
 from fastapi import Request
 from fastapi.responses import JSONResponse
 
-from onyx.error_handling.error_codes import OnyxErrorCode
-from onyx.error_handling.exceptions import OnyxError
-from onyx.llm.interfaces import LLMConfig
-from onyx.server.gateway import anthropic_passthrough, stream_bridge
-from onyx.server.gateway import api as gateway_api
-from onyx.server.gateway.anthropic_passthrough import (
+from ee.onyx.server.gateway import anthropic_passthrough, stream_bridge
+from ee.onyx.server.gateway import api as gateway_api
+from ee.onyx.server.gateway.anthropic_passthrough import (
     AnthropicPassthroughUnavailable,
     _build_upstream_headers,
     _build_upstream_request,
@@ -29,13 +26,16 @@ from onyx.server.gateway.anthropic_passthrough import (
     handle_anthropic_passthrough,
     is_anthropic_passthrough_eligible,
 )
+from onyx.error_handling.error_codes import OnyxErrorCode
+from onyx.error_handling.exceptions import OnyxError
+from onyx.llm.interfaces import LLMConfig
 from onyx.server.gateway.models import (
     AnthropicCountTokensRequest,
     AnthropicMessagesRequest,
     AnthropicUsagePayload,
 )
 from onyx.tracing.flows import LLMFlow
-from tests.unit.onyx.server.gateway.test_llm_gateway_api import (
+from tests.unit.ee.onyx.server.gateway.test_llm_gateway_api import (
     _ConfigOnlyLLM,
     _model,
     _provider,

backend/tests/unit/ee/onyx/server/gateway/test_llm_gateway_api.py

@@ -14,6 +14,9 @@ from fastapi.responses import StreamingResponse
 from fastapi.routing import APIRoute
 from sqlalchemy.orm import Session
 
+from ee.onyx.server.gateway import api as gateway_api
+from ee.onyx.server.gateway import stream_bridge
+from ee.onyx.server.gateway.api import _MESSAGES_ADAPTER
 from onyx.db.enums import Permission
 from onyx.db.models import User
 from onyx.error_handling.error_codes import OnyxErrorCode
@@ -50,9 +53,6 @@ from onyx.llm.multi_llm import LLMRateLimitError, LLMTimeoutError
 from onyx.server.auth_check import check_router_auth
 from onyx.server.features.build import craft_gateway
 from onyx.server.features.build.craft_gateway import gateway_request_flow
-from onyx.server.gateway import api as gateway_api
-from onyx.server.gateway import stream_bridge
-from onyx.server.gateway.api import _MESSAGES_ADAPTER
 from onyx.server.gateway.configs import GATEWAY_PATH_PREFIX
 from onyx.server.gateway.models import (
     ChatCompletionRequest,

backend/tests/unit/ee/onyx/server/gateway/test_openai_passthrough.py

@@ -12,14 +12,9 @@ import pytest
 from fastapi import Request
 from fastapi.responses import JSONResponse
 
-from onyx.error_handling.error_codes import OnyxErrorCode
-from onyx.error_handling.exceptions import OnyxError
-from onyx.llm.interfaces import LLMConfig
-from onyx.llm.multi_llm import LitellmLLM
-from onyx.server.gateway import api as gateway_api
-from onyx.server.gateway import openai_passthrough, stream_bridge
-from onyx.server.gateway.models import ResponsesRequest
-from onyx.server.gateway.openai_passthrough import (
+from ee.onyx.server.gateway import api as gateway_api
+from ee.onyx.server.gateway import openai_passthrough, stream_bridge
+from ee.onyx.server.gateway.openai_passthrough import (
     _SANITIZED_ERROR,
     _base_url,
     _build_upstream_headers,
@@ -33,8 +28,13 @@ from onyx.server.gateway.openai_passthrough import (
     handle_openai_responses_passthrough,
     is_openai_passthrough_eligible,
 )
+from onyx.error_handling.error_codes import OnyxErrorCode
+from onyx.error_handling.exceptions import OnyxError
+from onyx.llm.interfaces import LLMConfig
+from onyx.llm.multi_llm import LitellmLLM
+from onyx.server.gateway.models import ResponsesRequest
 from onyx.tracing.flows import LLMFlow
-from tests.unit.onyx.server.gateway.test_llm_gateway_api import (
+from tests.unit.ee.onyx.server.gateway.test_llm_gateway_api import (
     _ConfigOnlyLLM,
     _model,
     _provider,