diff --git a/backend/Dockerfile.model_server b/backend/Dockerfile.model_server index f2345be9ef..1c959e8ed6 100644 --- a/backend/Dockerfile.model_server +++ b/backend/Dockerfile.model_server @@ -82,4 +82,4 @@ ENV PYTHONPATH=/app ARG ONYX_VERSION=0.0.0-dev ENV ONYX_VERSION=${ONYX_VERSION} -CMD ["uvicorn", "model_server.main:app", "--host", "0.0.0.0", "--port", "9000"] +CMD ["python", "-m", "model_server"] diff --git a/backend/model_server/__main__.py b/backend/model_server/__main__.py new file mode 100644 index 0000000000..52780a487c --- /dev/null +++ b/backend/model_server/__main__.py @@ -0,0 +1,29 @@ +"""Process entry point for the model server (`python -m model_server`). + +The `DISABLE_MODEL_SERVER` gate runs here, ahead of `model_server.main`'s heavy ML +imports, so a disabled container exits without loading torch / the model stack. +""" + +import sys + +from onyx.utils.logger import setup_logger +from shared_configs.configs import DISABLE_MODEL_SERVER + +logger = setup_logger() + + +def main() -> None: + if DISABLE_MODEL_SERVER: + # The deployment points inference/indexing at an external model server, so this + # container has nothing to run. Exit cleanly instead of starting uvicorn. + logger.notice("DISABLE_MODEL_SERVER is set; skipping model server startup.") + sys.exit(0) + + # Imported lazily so the disabled path above stays free of the heavy ML imports. + from model_server.main import run_server + + run_server() + + +if __name__ == "__main__": + main() diff --git a/backend/model_server/main.py b/backend/model_server/main.py index b8c80a38f2..cfcd7df11b 100644 --- a/backend/model_server/main.py +++ b/backend/model_server/main.py @@ -24,7 +24,6 @@ from onyx.utils.middleware import add_onyx_request_id_middleware from onyx.utils.middleware import add_onyx_tenant_id_middleware from shared_configs.configs import INDEXING_ONLY from shared_configs.configs import MIN_THREADS_ML_MODELS -from shared_configs.configs import MODEL_SERVER_ALLOWED_HOST from shared_configs.configs import MODEL_SERVER_PORT from shared_configs.configs import SENTRY_DSN from shared_configs.configs import SENTRY_TRACES_SAMPLE_RATE @@ -141,11 +140,20 @@ def get_model_app() -> FastAPI: app = get_model_app() -if __name__ == "__main__": +def run_server() -> None: + # Bind all interfaces (loopback included) so the container healthcheck's + # localhost probe reaches the server. Mirrors the old shell command's + # `--host 0.0.0.0`; MODEL_SERVER_HOST is a client-side address and must not + # drive the bind host. + host = "0.0.0.0" # noqa: S104 logger.notice( "Starting Onyx Model Server on http://%s:%s/", - MODEL_SERVER_ALLOWED_HOST, + host, str(MODEL_SERVER_PORT), ) logger.notice("Model Server Version: %s", __version__) - uvicorn.run(app, host=MODEL_SERVER_ALLOWED_HOST, port=MODEL_SERVER_PORT) + uvicorn.run(app, host=host, port=MODEL_SERVER_PORT) + + +if __name__ == "__main__": + run_server() diff --git a/deployment/aws_ecs_fargate/cloudformation/services/onyx_model_server_indexing_service_template.yaml b/deployment/aws_ecs_fargate/cloudformation/services/onyx_model_server_indexing_service_template.yaml index b1975fac9a..580c17bb14 100644 --- a/deployment/aws_ecs_fargate/cloudformation/services/onyx_model_server_indexing_service_template.yaml +++ b/deployment/aws_ecs_fargate/cloudformation/services/onyx_model_server_indexing_service_template.yaml @@ -120,12 +120,6 @@ Resources: Image: onyxdotapp/onyx-model-server:latest Cpu: 0 Essential: true - Command: - - "/bin/sh" - - "-c" - - > - if [ "${DISABLE_MODEL_SERVER}" = "True" ] || [ "${DISABLE_MODEL_SERVER}" = "true" ]; then echo 'Skipping service...'; - exit 0; else exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000; fi PortMappings: - Name: model-server ContainerPort: 9000 diff --git a/deployment/aws_ecs_fargate/cloudformation/services/onyx_model_server_inference_service_template.yaml b/deployment/aws_ecs_fargate/cloudformation/services/onyx_model_server_inference_service_template.yaml index 98658aa4c5..5256a60a65 100644 --- a/deployment/aws_ecs_fargate/cloudformation/services/onyx_model_server_inference_service_template.yaml +++ b/deployment/aws_ecs_fargate/cloudformation/services/onyx_model_server_inference_service_template.yaml @@ -120,12 +120,6 @@ Resources: Image: onyxdotapp/onyx-model-server:latest Cpu: 0 Essential: true - Command: - - "/bin/sh" - - "-c" - - > - if [ "${DISABLE_MODEL_SERVER}" = "True" ] || [ "${DISABLE_MODEL_SERVER}" = "true" ]; then echo 'Skipping service...'; - exit 0; else exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000; fi PortMappings: - Name: model-server ContainerPort: 9000 diff --git a/deployment/docker_compose/docker-compose.multitenant-dev.yml b/deployment/docker_compose/docker-compose.multitenant-dev.yml index d8d6a4e258..60404bb434 100644 --- a/deployment/docker_compose/docker-compose.multitenant-dev.yml +++ b/deployment/docker_compose/docker-compose.multitenant-dev.yml @@ -350,13 +350,6 @@ services: build: context: ../../backend dockerfile: Dockerfile.model_server - command: > - /bin/sh -c "if [ \"${DISABLE_MODEL_SERVER:-}\" = \"True\" ] || [ \"${DISABLE_MODEL_SERVER:-}\" = \"true\" ]; then - echo 'Skipping service...'; - exit 0; - else - exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000; - fi" restart: on-failure environment: - MIN_THREADS_ML_MODELS=${MIN_THREADS_ML_MODELS:-} @@ -379,13 +372,6 @@ services: build: context: ../../backend dockerfile: Dockerfile.model_server - command: > - /bin/sh -c "if [ \"${DISABLE_MODEL_SERVER:-}\" = \"True\" ] || [ \"${DISABLE_MODEL_SERVER:-}\" = \"true\" ]; then - echo 'Skipping service...'; - exit 0; - else - exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000; - fi" restart: on-failure environment: - INDEX_BATCH_SIZE=${INDEX_BATCH_SIZE:-} diff --git a/deployment/docker_compose/docker-compose.prod-cloud.yml b/deployment/docker_compose/docker-compose.prod-cloud.yml index 6938b18c74..ce52a306f2 100644 --- a/deployment/docker_compose/docker-compose.prod-cloud.yml +++ b/deployment/docker_compose/docker-compose.prod-cloud.yml @@ -164,13 +164,6 @@ services: build: context: ../../backend dockerfile: Dockerfile.model_server - command: > - /bin/sh -c "if [ \"${DISABLE_MODEL_SERVER:-}\" = \"True\" ] || [ \"${DISABLE_MODEL_SERVER:-}\" = \"true\" ]; then - echo 'Skipping service...'; - exit 0; - else - exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000; - fi" restart: on-failure environment: - MIN_THREADS_ML_MODELS=${MIN_THREADS_ML_MODELS:-} @@ -190,13 +183,6 @@ services: build: context: ../../backend dockerfile: Dockerfile.model_server - command: > - /bin/sh -c "if [ \"${DISABLE_MODEL_SERVER:-}\" = \"True\" ] || [ \"${DISABLE_MODEL_SERVER:-}\" = \"true\" ]; then - echo 'Skipping service...'; - exit 0; - else - exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000; - fi" restart: on-failure environment: - MIN_THREADS_ML_MODELS=${MIN_THREADS_ML_MODELS:-} diff --git a/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml b/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml index 7aeec49176..6c8d17efda 100644 --- a/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml +++ b/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml @@ -165,13 +165,6 @@ services: build: context: ../../backend dockerfile: Dockerfile.model_server - command: > - /bin/sh -c "if [ \"${DISABLE_MODEL_SERVER:-}\" = \"True\" ] || [ \"${DISABLE_MODEL_SERVER:-}\" = \"true\" ]; then - echo 'Skipping service...'; - exit 0; - else - exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000; - fi" restart: on-failure environment: - MIN_THREADS_ML_MODELS=${MIN_THREADS_ML_MODELS:-} @@ -193,13 +186,6 @@ services: build: context: ../../backend dockerfile: Dockerfile.model_server - command: > - /bin/sh -c "if [ \"${DISABLE_MODEL_SERVER:-}\" = \"True\" ] || [ \"${DISABLE_MODEL_SERVER:-}\" = \"true\" ]; then - echo 'Skipping service...'; - exit 0; - else - exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000; - fi" restart: on-failure environment: - MIN_THREADS_ML_MODELS=${MIN_THREADS_ML_MODELS:-} diff --git a/deployment/docker_compose/docker-compose.prod.yml b/deployment/docker_compose/docker-compose.prod.yml index e99706953a..df41829c5c 100644 --- a/deployment/docker_compose/docker-compose.prod.yml +++ b/deployment/docker_compose/docker-compose.prod.yml @@ -201,13 +201,6 @@ services: build: context: ../../backend dockerfile: Dockerfile.model_server - command: > - /bin/sh -c "if [ \"${DISABLE_MODEL_SERVER:-}\" = \"True\" ] || [ \"${DISABLE_MODEL_SERVER:-}\" = \"true\" ]; then - echo 'Skipping service...'; - exit 0; - else - exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000; - fi" restart: unless-stopped environment: - MIN_THREADS_ML_MODELS=${MIN_THREADS_ML_MODELS:-} @@ -229,13 +222,6 @@ services: build: context: ../../backend dockerfile: Dockerfile.model_server - command: > - /bin/sh -c "if [ \"${DISABLE_MODEL_SERVER:-}\" = \"True\" ] || [ \"${DISABLE_MODEL_SERVER:-}\" = \"true\" ]; then - echo 'Skipping service...'; - exit 0; - else - exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000; - fi" restart: unless-stopped environment: - MIN_THREADS_ML_MODELS=${MIN_THREADS_ML_MODELS:-} diff --git a/deployment/docker_compose/docker-compose.search-testing.yml b/deployment/docker_compose/docker-compose.search-testing.yml index 17430ce065..227d55b198 100644 --- a/deployment/docker_compose/docker-compose.search-testing.yml +++ b/deployment/docker_compose/docker-compose.search-testing.yml @@ -113,13 +113,6 @@ services: build: context: ../../backend dockerfile: Dockerfile.model_server - command: > - /bin/sh -c "if [ \"${DISABLE_MODEL_SERVER:-}\" = \"True\" ] || [ \"${DISABLE_MODEL_SERVER:-}\" = \"true\" ]; then - echo 'Skipping service...'; - exit 0; - else - exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000; - fi" restart: on-failure environment: - MIN_THREADS_ML_MODELS=${MIN_THREADS_ML_MODELS:-} @@ -137,13 +130,6 @@ services: build: context: ../../backend dockerfile: Dockerfile.model_server - command: > - /bin/sh -c "if [ \"${DISABLE_MODEL_SERVER:-}\" = \"True\" ] || [ \"${DISABLE_MODEL_SERVER:-}\" = \"true\" ]; then - echo 'Skipping service...'; - exit 0; - else - exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000; - fi" restart: on-failure environment: - MIN_THREADS_ML_MODELS=${MIN_THREADS_ML_MODELS:-} diff --git a/deployment/docker_compose/docker-compose.yml b/deployment/docker_compose/docker-compose.yml index f318bf2e97..6b279ac4ad 100644 --- a/deployment/docker_compose/docker-compose.yml +++ b/deployment/docker_compose/docker-compose.yml @@ -274,13 +274,6 @@ services: # - driver: nvidia # count: all # capabilities: [gpu] - command: > - /bin/sh -c "if [ \"${DISABLE_MODEL_SERVER:-}\" = \"True\" ] || [ \"${DISABLE_MODEL_SERVER:-}\" = \"true\" ]; then - echo 'Skipping service...'; - exit 0; - else - exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000; - fi" env_file: - path: .env required: false @@ -324,13 +317,6 @@ services: # - driver: nvidia # count: all # capabilities: [gpu] - command: > - /bin/sh -c "if [ \"${DISABLE_MODEL_SERVER:-}\" = \"True\" ] || [ \"${DISABLE_MODEL_SERVER:-}\" = \"true\" ]; then - echo 'Skipping service...'; - exit 0; - else - exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000; - fi" env_file: - path: .env required: false