starred/onyx
1
0 Fork 0
mirror of https://github.com/onyx-dot-app/onyx.git synced 2026-10-19 01:50:51 -05:00
Code Issues Packages Projects Releases 10 Wiki Activity

feat(ods): sync onyx-cli's embedded deployment files via generate-compose (#13524)

Browse Source 
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jamison LahmanandClaude Fable 5 authored and GitHub committed 2026-07-28 22:27:02 +00:00
1 parent be8b602fcc
commit 9fd104598b
15 files changed
+1922 -13
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitattributes
+2
View File
@@ -0,0 +1,2 @@
# Byte-for-byte copies of deployment files synced by `ods generate-compose --write`.
cli/internal/deploy/deployfiles/embedded/** linguist-generated=true
.pre-commit-config.yaml
+19 -7
View File
@@ -68,18 +68,30 @@ repos:
alias: docker-compose-sync
name: sync docker compose files from template
# Regenerates docker-compose.yml / docker-compose.prod.yml /
# docker-compose.prod-no-letsencrypt.yml from docker-compose.template.yml.
# docker-compose.prod-no-letsencrypt.yml from docker-compose.template.yml,
# then refreshes the copies of the guided-install deployment files that
# onyx-cli embeds (cli/internal/deploy/deployfiles/embedded/).
# `ods generate-compose` ships in onyx-devtools (tools/ods); changes to the
# generator need an ods release before this hook picks them up.
# generator need an ods release before this hook picks them up — until the
# release that adds the embedded-copy sync, the extra trigger paths below
# are harmless no-ops (the cli module's drift test still gates staleness).
args:
["--no-project", "--with=onyx-devtools", "ods", "generate-compose", "--write"]
pass_filenames: false
files: >-
(?x)^deployment/docker_compose/(
docker-compose\.template\.yml
|docker-compose\.yml
|docker-compose\.prod\.yml
|docker-compose\.prod-no-letsencrypt\.yml
(?x)^(
deployment/docker_compose/(
docker-compose\.template\.yml
|docker-compose\.yml
|docker-compose\.prod\.yml
|docker-compose\.prod-no-letsencrypt\.yml
|docker-compose\.onyx-lite\.yml
|docker-compose\.craft\.yml
|env\.template
|README\.md
)
|deployment/data/nginx/(app\.conf\.template|run-nginx\.sh)
|cli/internal/deploy/deployfiles/embedded/.+
)$
- id: uv-run
alias: ty
cli/internal/deploy/deployfiles/embedded/data/nginx/app.conf.template
Generated
+136
View File
@@ -0,0 +1,136 @@
# Don't expose the nginx version in the Server header or on error pages
server_tokens off;
# Log format to include request latency
log_format custom_main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for" '
'rt=$request_time';
upstream api_server {
# fail_timeout=0 means we always retry an upstream even if it failed
# to return a good HTTP response
# for UNIX domain socket setups
#server unix:/tmp/gunicorn.sock fail_timeout=0;
# for a TCP configuration
# TODO: use gunicorn to manage multiple processes
server ${ONYX_BACKEND_API_HOST}:8080 fail_timeout=0;
}
upstream web_server {
server ${ONYX_WEB_SERVER_HOST}:3000 fail_timeout=0;
}
# Conditionally include MCP upstream configuration
include /etc/nginx/conf.d/mcp_upstream.conf.inc;
# WebSocket support: only set Connection "upgrade" for actual upgrade requests
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
server {
listen 80 default_server;
client_max_body_size 5G; # Maximum upload size
access_log /var/log/nginx/access.log custom_main;
# Conditionally include MCP location configuration
include /etc/nginx/conf.d/mcp.conf.inc;
# First-party health probe served directly by nginx (no upstreams),
# used by the container healthcheck.
location = /nginx-health {
access_log off;
return 200 "ok\n";
}
# SAML login and IdP callbacks are backend routes under /auth/saml (legacy
# single-config and the DB-backed multi-provider router). Proxy them to the
# API like /scim, otherwise the IdP POST falls through to the web server.
location ~ ^/auth/saml(/.*)?$ {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Port $server_port;
proxy_set_header Host $host;
proxy_http_version 1.1;
proxy_buffering off;
proxy_redirect off;
proxy_pass http://api_server;
}
location ~ ^/scim(/.*)?$ {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Port $server_port;
proxy_set_header Host $host;
proxy_http_version 1.1;
proxy_buffering off;
proxy_redirect off;
proxy_connect_timeout ${NGINX_PROXY_CONNECT_TIMEOUT}s;
proxy_send_timeout ${NGINX_PROXY_SEND_TIMEOUT}s;
proxy_read_timeout ${NGINX_PROXY_READ_TIMEOUT}s;
proxy_pass http://api_server;
}
# Match both /api/* and /openapi.json in a single rule
location ~ ^/(api|openapi.json)(/.*)?$ {
# Rewrite /api prefixed matched paths
rewrite ^/api(/.*)$ $1 break;
# misc headers
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Port $server_port;
proxy_set_header Host $host;
# need to use 1.1 to support chunked transfers and WebSocket
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_buffering off;
# timeout settings
proxy_connect_timeout ${NGINX_PROXY_CONNECT_TIMEOUT}s;
proxy_send_timeout ${NGINX_PROXY_SEND_TIMEOUT}s;
proxy_read_timeout ${NGINX_PROXY_READ_TIMEOUT}s;
# we don't want nginx trying to do something clever with
# redirects, we set the Host: header above already.
proxy_redirect off;
proxy_pass http://api_server;
}
location / {
# misc headers
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Port $server_port;
proxy_set_header Host $host;
proxy_http_version 1.1;
# timeout settings
proxy_connect_timeout ${NGINX_PROXY_CONNECT_TIMEOUT}s;
proxy_send_timeout ${NGINX_PROXY_SEND_TIMEOUT}s;
proxy_read_timeout ${NGINX_PROXY_READ_TIMEOUT}s;
# we don't want nginx trying to do something clever with
# redirects, we set the Host: header above already.
proxy_redirect off;
proxy_pass http://web_server;
}
}
cli/internal/deploy/deployfiles/embedded/data/nginx/run-nginx.sh
Generated
+39
View File
@@ -0,0 +1,39 @@
#!/bin/sh
# fill in the template
export ONYX_BACKEND_API_HOST="${ONYX_BACKEND_API_HOST:-api_server}"
export ONYX_WEB_SERVER_HOST="${ONYX_WEB_SERVER_HOST:-web_server}"
export ONYX_MCP_SERVER_HOST="${ONYX_MCP_SERVER_HOST:-mcp_server}"
export SSL_CERT_FILE_NAME="${SSL_CERT_FILE_NAME:-ssl.crt}"
export SSL_CERT_KEY_FILE_NAME="${SSL_CERT_KEY_FILE_NAME:-ssl.key}"
# Nginx timeout settings (in seconds)
export NGINX_PROXY_CONNECT_TIMEOUT="${NGINX_PROXY_CONNECT_TIMEOUT:-300}"
export NGINX_PROXY_SEND_TIMEOUT="${NGINX_PROXY_SEND_TIMEOUT:-300}"
export NGINX_PROXY_READ_TIMEOUT="${NGINX_PROXY_READ_TIMEOUT:-300}"
echo "Using API server host: $ONYX_BACKEND_API_HOST"
echo "Using web server host: $ONYX_WEB_SERVER_HOST"
echo "Using MCP server host: $ONYX_MCP_SERVER_HOST"
echo "Using nginx proxy timeouts - connect: ${NGINX_PROXY_CONNECT_TIMEOUT}s, send: ${NGINX_PROXY_SEND_TIMEOUT}s, read: ${NGINX_PROXY_READ_TIMEOUT}s"
# shellcheck disable=SC2016
envsubst '$DOMAIN $SSL_CERT_FILE_NAME $SSL_CERT_KEY_FILE_NAME $ONYX_BACKEND_API_HOST $ONYX_WEB_SERVER_HOST $ONYX_MCP_SERVER_HOST $NGINX_PROXY_CONNECT_TIMEOUT $NGINX_PROXY_SEND_TIMEOUT $NGINX_PROXY_READ_TIMEOUT' < "/etc/nginx/conf.d/$1" > /etc/nginx/conf.d/app.conf
# Conditionally create MCP server configuration
if [ "${MCP_SERVER_ENABLED}" = "True" ] || [ "${MCP_SERVER_ENABLED}" = "true" ]; then
echo "MCP server is enabled, creating MCP configuration..."
# shellcheck disable=SC2016
envsubst '$ONYX_MCP_SERVER_HOST' < "/etc/nginx/conf.d/mcp_upstream.conf.inc.template" > /etc/nginx/conf.d/mcp_upstream.conf.inc
# shellcheck disable=SC2016
envsubst '$ONYX_MCP_SERVER_HOST' < "/etc/nginx/conf.d/mcp.conf.inc.template" > /etc/nginx/conf.d/mcp.conf.inc
else
echo "MCP server is disabled, removing MCP configuration..."
# Leave empty placeholder files so nginx includes do not fail
# These files are empty because MCP server is disabled
echo "# Empty file - MCP server is disabled" > /etc/nginx/conf.d/mcp_upstream.conf.inc
echo "# Empty file - MCP server is disabled" > /etc/nginx/conf.d/mcp.conf.inc
fi
# Start nginx and reload every 6 hours
while :; do sleep 6h & wait; nginx -s reload; done & nginx -g "daemon off;"
cli/internal/deploy/deployfiles/embedded/docker_compose/README.md
Generated
+76
View File
@@ -0,0 +1,76 @@
# Welcome to Onyx
To set up Onyx there are several options, Onyx supports the following for deployment:
1. Quick guided install via the install.sh script
2. Pulling the repo and running `docker compose up -d` from the deployment/docker_compose directory
- Note, it is recommended to copy over the env.template file to .env and edit the necessary values
3. For large scale deployments leveraging Kubernetes, there are two options, Helm or Terraform.
This README focuses on the easiest guided deployment which is via install.sh.
**For more detailed guides, please refer to the documentation: https://docs.onyx.app/deployment/overview**
## install.sh script
```
curl -fsSL https://raw.githubusercontent.com/onyx-dot-app/onyx/main/deployment/docker_compose/install.sh > install.sh && chmod +x install.sh && ./install.sh
```
This provides a guided installation of Onyx via Docker Compose. It will deploy the latest version of Onyx
and set up the volumes to ensure data is persisted across deployments or upgrades.
The script will create an onyx_data directory, all necessary files for the deployment will be stored in
there. Note that no application critical data is stored in that directory so even if you delete it, the
data needed to restore the app will not be destroyed.
The data about chats, users, etc. are instead stored as named Docker Volumes. This is managed by Docker
and where it is stored will depend on your Docker setup. You can always delete these as well by running
the install.sh script with --delete-data.
To shut down the deployment without deleting, use install.sh --shutdown.
### Upgrading the deployment
Onyx maintains backwards compatibility across all minor versions following SemVer. If following the install.sh script (or through Docker Compose), you can
upgrade it by first bringing down the containers. To do this, use `install.sh --shutdown`
(or `docker compose down` from the directory with the docker-compose.yml file).
After the containers are stopped, you can safely upgrade by either re-running the `install.sh` script (if you left the values as default which is latest,
then it will automatically update to latest each time the script is run). If you are more comfortable running docker compose commands, you can also run
commands directly from the directory with the docker-compose.yml file. First verify the version you want in the environment file (see below),
(if using `latest` tag, be sure to run `docker compose pull`) and run `docker compose up` to restart the services on the latest version
### Environment variables
The Docker Compose files try to look for a .env file in the same directory. The `install.sh` script sets it up from a file called env.template which is
downloaded during the initial setup. Feel free to edit the .env file to customize your deployment. The most important / common changed values are
located near the top of the file.
IMAGE_TAG is the version of Onyx to run. It is recommended to leave it as latest to get all updates with each redeployment.
Every image publishes a `-dev` twin for each of its tags (e.g. `latest-dev`, `v1.2.3-dev`), so a single
`IMAGE_TAG=latest-dev` selects the dev variant of the whole deployment. Today only the backend image actually differs:
its `-dev` twin adds interactive debugging tools (vim, nano, curl, ps, psql) that the default image leaves out to stay
minimal. The web-server, model-server, and sandbox `-dev` tags are identical to their plain counterparts and exist so
that one version string covers every image.
## Maintaining the compose files (contributors)
`docker-compose.yml`, `docker-compose.prod.yml` and `docker-compose.prod-no-letsencrypt.yml` are
generated from the single source of truth `docker-compose.template.yml` — do not hand-edit them.
To change any of the three, edit the template (per-variant differences are expressed with `#!for` /
`#!only` / `#!value` directives, documented in `ods generate-compose --help`) and regenerate:
```
ods generate-compose --write
```
The generator lives in `tools/ods` (`internal/composegen`) and ships with the `onyx-devtools`
package. The `docker-compose-sync` pre-commit hook runs it automatically for commits touching the
template or the generated files, so a stray edit to a generated file gets reverted on the next
commit.
onyx-cli embeds copies of the guided-install deployment files (the generated
`docker-compose.yml`, the lite/craft overlays, `env.template`, the nginx config, and this README)
under `cli/internal/deploy/deployfiles/embedded/`. The same `ods generate-compose --write` run
refreshes them after rendering the variants, and a drift test in the cli module (`go test ./...`)
gates staleness. If you change any of those files, re-run the generator and commit the refreshed
embedded copies.
cli/internal/deploy/deployfiles/embedded/docker_compose/docker-compose.craft.yml
Generated
+142
View File
@@ -0,0 +1,142 @@
# Onyx Craft Docker-sandbox overlay. Layer on top of docker-compose.yml.
# install.sh --include-craft wires this in automatically.
#
# Trust boundary: api_server, background, and sandbox-proxy all bind-mount
# /var/run/docker.sock (root-equivalent on host). Only run on hosts you fully
# control.
services:
api_server:
environment:
- SANDBOX_BACKEND=${SANDBOX_BACKEND:-docker}
- ONYX_SERVER_URL=${ONYX_SERVER_URL:-http://onyx-craft-api:8080}
- SANDBOX_CONTAINER_IMAGE=${SANDBOX_CONTAINER_IMAGE:-onyxdotapp/sandbox:${IMAGE_TAG:-latest}}
# Pinned: Must match the compose `networks:` block + install.sh literal.
# An env_file leak would point Python at a non-existent network.
- SANDBOX_DOCKER_NETWORK=onyx_craft_sandbox
- SANDBOX_DOCKER_MEMORY_LIMIT=${SANDBOX_DOCKER_MEMORY_LIMIT:-2g}
- SANDBOX_DOCKER_CPU_LIMIT=${SANDBOX_DOCKER_CPU_LIMIT:-1.0}
- ENABLE_OPENCODE_DEBUGGING=${ENABLE_OPENCODE_DEBUGGING:-false}
# Pinned: Container-side bind target. Overrides host-path values that
# the operator's .env may inject via env_file.
- SANDBOX_DOCKER_SOCKET=/var/run/docker.sock
# Pinned: Must match the sandbox container's entrypoint.sh hardcode.
- OPENCODE_SERVE_PORT=4096
# Proxy is mandatory; an explicit empty SANDBOX_PROXY_HOST raises at
# api_server startup (see DockerSandboxManager._initialize).
- SANDBOX_PROXY_HOST=${SANDBOX_PROXY_HOST-sandbox-proxy}
- SANDBOX_PROXY_PORT=${SANDBOX_PROXY_PORT:-8080}
volumes:
- ${SANDBOX_DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock
networks:
default:
onyx_craft_sandbox:
aliases:
- onyx-craft-api
# service_healthy, not service_started: The proxy binds its listener only
# after initial sync, so service_started would race startup provisions.
depends_on:
sandbox-proxy:
condition: service_healthy
background:
environment:
# Must match api_server so the idle-cleanup celery task uses the same
# backend to snapshot + terminate sandboxes.
- SANDBOX_BACKEND=${SANDBOX_BACKEND:-docker}
- ONYX_SERVER_URL=${ONYX_SERVER_URL:-http://onyx-craft-api:8080}
- SANDBOX_CONTAINER_IMAGE=${SANDBOX_CONTAINER_IMAGE:-onyxdotapp/sandbox:${IMAGE_TAG:-latest}}
# See api_server above re: Pinned env_file-override vars.
- SANDBOX_DOCKER_NETWORK=onyx_craft_sandbox
- SANDBOX_DOCKER_MEMORY_LIMIT=${SANDBOX_DOCKER_MEMORY_LIMIT:-2g}
- SANDBOX_DOCKER_CPU_LIMIT=${SANDBOX_DOCKER_CPU_LIMIT:-1.0}
- ENABLE_OPENCODE_DEBUGGING=${ENABLE_OPENCODE_DEBUGGING:-false}
- SANDBOX_DOCKER_SOCKET=/var/run/docker.sock
- OPENCODE_SERVE_PORT=4096
- SANDBOX_PROXY_HOST=${SANDBOX_PROXY_HOST-sandbox-proxy}
- SANDBOX_PROXY_PORT=${SANDBOX_PROXY_PORT:-8080}
volumes:
- ${SANDBOX_DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock
networks:
- default
- onyx_craft_sandbox
# See api_server above for the service_healthy rationale.
depends_on:
sandbox-proxy:
condition: service_healthy
# Egress proxy: MITMs sandbox HTTPS traffic and gates action approvals.
sandbox-proxy:
image: ${ONYX_BACKEND_IMAGE:-onyxdotapp/onyx-backend:${IMAGE_TAG:-latest}}
command: ["python", "-m", "onyx.sandbox_proxy.server"]
env_file:
- path: .env
required: false
environment:
- SANDBOX_BACKEND=docker
- POSTGRES_HOST=${POSTGRES_HOST:-relational_db}
- REDIS_HOST=${REDIS_HOST:-cache}
# Pinned to defeat env_file leaks. See api_server above.
- SANDBOX_DOCKER_SOCKET=/var/run/docker.sock
- SANDBOX_DOCKER_NETWORK=onyx_craft_sandbox
- SANDBOX_PROXY_LISTEN_PORT=${SANDBOX_PROXY_PORT:-8080}
# Matches the healthcheck URL below.
- SANDBOX_PROXY_HEALTHZ_PORT=8081
# OnyxPatResolver reads this at boot to know which host to claim for PAT
# injection. Without it, /me requests pass through bare with the
# placeholder bearer and api_server 403s.
- ONYX_SERVER_URL=${ONYX_SERVER_URL:-http://onyx-craft-api:8080}
volumes:
- sandbox_proxy_ca:/var/lib/sandbox-proxy/ca
# RO: the proxy only inspects (events watcher + initial sync).
- ${SANDBOX_DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock:ro
depends_on:
relational_db:
condition: service_started
cache:
condition: service_started
networks:
# `default` reaches relational_db + cache by compose-DNS; the sandbox
# bridge join makes the service resolvable as `sandbox-proxy` from inside
# each sandbox container.
- default
- onyx_craft_sandbox
restart: unless-stopped
# >= _DRAIN_TIMEOUT_S (10s in server.py) plus margin so the SIGTERM drain
# can terminalize parked approvals (claim EXPIRED + wake BLPOP listeners,
# see gate._terminalize_after_unhandled_error) before docker tears the
# container down. Matches K8s terminationGracePeriodSeconds.
stop_grace_period: 20s
healthcheck:
test:
[
"CMD",
"python",
"-c",
"import urllib.request; urllib.request.urlopen('http://localhost:8081/healthz')",
]
interval: 10s
timeout: 3s
retries: 3
# Matches server._LOOKUP_INITIAL_SYNC_TIMEOUT_S so a slow initial sync
# doesn't race the healthcheck: if sync wins, next probe flips healthy; if
# it loses, the process exits and docker restarts.
start_period: 60s
logging:
driver: json-file
options:
max-size: "50m"
max-file: "6"
networks:
# Pre-created by install.sh --include-craft.
onyx_craft_sandbox:
name: onyx_craft_sandbox
external: true
volumes:
# Pre-created by install.sh --include-craft. external+name avoids the compose
# project prefix so DockerSandboxManager can mount the same name when it
# provisions sandbox containers from outside compose's project scope.
sandbox_proxy_ca:
name: sandbox_proxy_ca
external: true
cli/internal/deploy/deployfiles/embedded/docker_compose/docker-compose.onyx-lite.yml
Generated
+88
View File
@@ -0,0 +1,88 @@
# =============================================================================
# ONYX LITE — MINIMAL DEPLOYMENT OVERLAY
# =============================================================================
# Overlay to run Onyx in a minimal configuration: no vector database, no Redis,
# no model servers, and no background workers. Only PostgreSQL is required. In
# this mode, connectors and RAG search are disabled, but the core chat
# experience (LLM conversations, tools, user file uploads, Projects, Agent
# knowledge, code interpreter) still works.
#
# Usage:
# docker compose -f docker-compose.yml -f docker-compose.onyx-lite.yml up -d
#
# With dev ports:
# docker compose -f docker-compose.yml -f docker-compose.onyx-lite.yml \
# -f docker-compose.dev.yml up -d --wait
#
# This overlay:
# - Moves both model servers, OpenSearch, MinIO, Redis (cache), and the
# background worker to profiles so they do not start by default
# - Makes depends_on references to removed services optional
# - Sets DISABLE_VECTOR_DB=true on the api_server
# - Uses PostgreSQL for caching and auth instead of Redis
# - Uses PostgreSQL for file storage instead of S3/MinIO
#
# To selectively bring services back:
# --profile vectordb Indexing model server
# --profile inference Inference model server
# --profile background Background worker (Celery) — also needs redis
# --profile redis Redis cache
# --profile opensearch OpenSearch
# --profile s3-filestore MinIO (S3-compatible file store)
# =============================================================================
name: onyx
services:
api_server:
depends_on:
opensearch:
condition: service_started
required: false
cache:
condition: service_started
required: false
inference_model_server:
condition: service_started
required: false
minio:
condition: service_started
required: false
environment:
- DISABLE_VECTOR_DB=true
- FILE_STORE_BACKEND=postgres
- CACHE_BACKEND=postgres
- AUTH_BACKEND=postgres
# Move the background worker to a profile so it does not start by default.
# The API server handles all background work in lite mode.
background:
profiles: ["background"]
depends_on:
inference_model_server:
condition: service_started
required: false
indexing_model_server:
condition: service_started
required: false
# Move Redis to a profile so it does not start by default.
# The Postgres cache backend replaces Redis in lite mode.
cache:
profiles: ["redis"]
# Move indexing model server to a profile so it does not start.
indexing_model_server:
profiles: ["vectordb"]
# Inference model server is only needed for local embeddings, not for LLM chat.
inference_model_server:
profiles: ["inference"]
# OpenSearch is not needed in lite mode (no indexing).
opensearch:
profiles: ["opensearch"]
# MinIO is not needed in lite mode (Postgres handles file storage).
minio:
profiles: ["s3-filestore"]
cli/internal/deploy/deployfiles/embedded/docker_compose/docker-compose.yml
Generated
+590
View File
@@ -0,0 +1,590 @@
# =============================================================================
# THIS FILE IS GENERATED - DO NOT EDIT DIRECTLY
# Source of truth: deployment/docker_compose/docker-compose.template.yml
# Regenerate: ods generate-compose --write
# =============================================================================
# =============================================================================
# ONYX DOCKER COMPOSE
# =============================================================================
# This is the default configuration for Onyx. This file is fairly configurable,
# also see env.template for possible settings.
#
# PRODUCTION DEPLOYMENT CHECKLIST:
# To convert this setup to a production deployment following best practices,
# follow the checklist below. Note that there are other ways to secure the Onyx
# deployment so these are not strictly necessary for all teams.
#
# 1. SECURITY HARDENING:
# - Remove all port exposures except nginx (80/443)
# - Comment out ports for: api_server, relational_db, cache, minio
# - Set strong MinIO/S3 credentials (MINIO_ROOT_* + S3_AWS_*), not minioadmin
#
# 2. SSL/TLS SETUP:
# - Uncomment the certbot service (see below)
# - Add SSL certificate volumes to nginx service
# - Change nginx command from app.conf.template to app.conf.template.prod
#
# 3. ENVIRONMENT CONFIGURATION:
# - Replace env_file with explicit environment variables
#
# 4. AUTHENTICATION:
# - Email/password auth works out of the box. SSO (Google, OIDC, SAML) is
# configured from the admin panel: Admin Panel > Organization > SSO Providers
#
# 5. CA CERTIFICATES:
# - Uncomment custom CA certificate volumes if needed
#
# 6. DOMAIN CONFIGURATION:
# - Set proper DOMAIN environment variable for nginx
# - Configure DNS and SSL certificates
#
# For a complete production setup, refer to docker-compose.prod.yml
# =============================================================================
name: onyx
services:
api_server:
image: ${ONYX_BACKEND_IMAGE:-onyxdotapp/onyx-backend:${IMAGE_TAG:-latest}}
build:
context: ../../backend
dockerfile: Dockerfile
# The shipped image; the Dockerfile's default (last) stage is the dev variant
# with debugging tools (published with a -dev tag suffix).
target: runtime
command: >
/bin/sh -c "alembic upgrade head &&
echo \"Starting Onyx Api Server\" &&
uvicorn onyx.main:app --host 0.0.0.0 --port 8080"
# Check env.template and copy to .env for env vars
env_file:
- path: .env
required: false
depends_on:
relational_db:
condition: service_started
opensearch:
condition: service_started
cache:
condition: service_started
inference_model_server:
condition: service_started
minio:
condition: service_started
required: false
restart: unless-stopped
# DEV: To expose ports, either:
# 1. Use docker-compose.dev.yml: docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --wait
# 2. Uncomment the ports below
# ports:
# - "8080:8080"
environment:
# Auth Settings
- AUTH_TYPE=${AUTH_TYPE:-basic}
- FILE_STORE_BACKEND=${FILE_STORE_BACKEND:-s3}
- POSTGRES_HOST=${POSTGRES_HOST:-relational_db}
- OPENSEARCH_HOST=${OPENSEARCH_HOST:-opensearch}
- OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-StrongPassword123!}
- REDIS_HOST=${REDIS_HOST:-cache}
- MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
- CODE_INTERPRETER_BASE_URL=${CODE_INTERPRETER_BASE_URL:-http://code-interpreter:8000}
- S3_ENDPOINT_URL=${S3_ENDPOINT_URL:-http://minio:9000}
- S3_AWS_ACCESS_KEY_ID=${S3_AWS_ACCESS_KEY_ID:-minioadmin}
- S3_AWS_SECRET_ACCESS_KEY=${S3_AWS_SECRET_ACCESS_KEY:-minioadmin}
# Onyx Craft configuration (disabled by default, set ENABLE_CRAFT=true in .env to enable)
# Use --include-craft with install script, or manually set in .env file
- ENABLE_CRAFT=${ENABLE_CRAFT:-false}
# PRODUCTION: Uncomment the line below to use if IAM_AUTH is true and you are using iam auth for postgres
# volumes:
# - ./bundle.pem:/app/bundle.pem:ro
extra_hosts:
- "host.docker.internal:host-gateway"
logging:
driver: json-file
options:
max-size: "50m"
max-file: "6"
healthcheck:
test:
[
"CMD",
"python",
"-c",
"import urllib.request; urllib.request.urlopen('http://localhost:8080/health')",
]
interval: 30s
timeout: 20s
retries: 3
# Generous start_period so that `docker compose up --wait` does not flag
# the container unhealthy while alembic migrations run on a fresh DB.
# Healthy is reported as soon as /health responds, so this does not slow
# down fast boots.
start_period: 600s
# Optional, only for debugging purposes
volumes:
- api_server_logs:/var/log/onyx
# Shared volume for persistent document storage (Craft file-system mode)
- file-system:/app/file-system
background:
image: ${ONYX_BACKEND_IMAGE:-onyxdotapp/onyx-backend:${IMAGE_TAG:-latest}}
build:
context: ../../backend
dockerfile: Dockerfile
# The shipped image; the Dockerfile's default (last) stage is the dev variant
# with debugging tools (published with a -dev tag suffix).
target: runtime
command: >
/bin/sh -c "
if [ -f /etc/ssl/certs/custom-ca.crt ]; then
update-ca-certificates;
fi &&
/app/scripts/supervisord_entrypoint.sh"
env_file:
- path: .env
required: false
depends_on:
relational_db:
condition: service_started
opensearch:
condition: service_started
cache:
condition: service_started
inference_model_server:
condition: service_started
indexing_model_server:
condition: service_started
restart: unless-stopped
environment:
- FILE_STORE_BACKEND=${FILE_STORE_BACKEND:-s3}
- POSTGRES_HOST=${POSTGRES_HOST:-relational_db}
- OPENSEARCH_HOST=${OPENSEARCH_HOST:-opensearch}
- OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-StrongPassword123!}
- REDIS_HOST=${REDIS_HOST:-cache}
- MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
- INDEXING_MODEL_SERVER_HOST=${INDEXING_MODEL_SERVER_HOST:-indexing_model_server}
- S3_ENDPOINT_URL=${S3_ENDPOINT_URL:-http://minio:9000}
- S3_AWS_ACCESS_KEY_ID=${S3_AWS_ACCESS_KEY_ID:-minioadmin}
- S3_AWS_SECRET_ACCESS_KEY=${S3_AWS_SECRET_ACCESS_KEY:-minioadmin}
- DISCORD_BOT_TOKEN=${DISCORD_BOT_TOKEN:-}
- DISCORD_BOT_INVOKE_CHAR=${DISCORD_BOT_INVOKE_CHAR:-!}
# API Server connection for Discord bot message processing
- API_SERVER_PROTOCOL=${API_SERVER_PROTOCOL:-http}
- API_SERVER_HOST=${API_SERVER_HOST:-api_server}
# Onyx Craft configuration (set up automatically on container startup)
- ENABLE_CRAFT=${ENABLE_CRAFT:-false}
# PRODUCTION: Uncomment the line below to use if IAM_AUTH is true and you are using iam auth for postgres
# volumes:
# - ./bundle.pem:/app/bundle.pem:ro
extra_hosts:
- "host.docker.internal:host-gateway"
# Optional, only for debugging purposes
volumes:
- background_logs:/var/log/onyx
# Shared volume for persistent document storage (Craft file-system mode)
- file-system:/app/file-system
logging:
driver: json-file
options:
max-size: "50m"
max-file: "6"
# PRODUCTION: Uncomment the following lines if you need to include a custom CA certificate
# This section enables the use of a custom CA certificate
# If present, the custom CA certificate is mounted as a volume
# The container checks for its existence and updates the system's CA certificates
# This allows for secure communication with services using custom SSL certificates
# Optional volume mount for CA certificate
# volumes:
# # Maps to the CA_CERT_PATH environment variable in the Dockerfile
# - ${CA_CERT_PATH:-./custom-ca.crt}:/etc/ssl/certs/custom-ca.crt:ro
web_server:
image: ${ONYX_WEB_SERVER_IMAGE:-onyxdotapp/onyx-web-server:${IMAGE_TAG:-latest}}
build:
context: ../../web
dockerfile: Dockerfile
args:
- NEXT_PUBLIC_DISABLE_LOGOUT=${NEXT_PUBLIC_DISABLE_LOGOUT:-}
- NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED=${NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED:-}
# Enterprise Edition only
- NEXT_PUBLIC_THEME=${NEXT_PUBLIC_THEME:-}
# DO NOT TURN ON unless you have EXPLICIT PERMISSION from Onyx.
- NEXT_PUBLIC_DO_NOT_USE_TOGGLE_OFF_DANSWER_POWERED=${NEXT_PUBLIC_DO_NOT_USE_TOGGLE_OFF_DANSWER_POWERED:-false}
- NODE_OPTIONS=${NODE_OPTIONS:-"--max-old-space-size=4096"}
env_file:
- path: .env
required: false
depends_on:
- api_server
restart: unless-stopped
environment:
- INTERNAL_URL=${INTERNAL_URL:-http://api_server:8080}
logging:
driver: json-file
options:
max-size: "50m"
max-file: "6"
healthcheck:
test:
[
"CMD",
"node",
"-e",
"require('http').get('http://127.0.0.1:3000/', (r) => process.exit(r.statusCode < 500 ? 0 : 1)).on('error', () => process.exit(1))",
]
interval: 30s
timeout: 10s
retries: 5
start_period: 30s
# Uncomment the block below to enable the MCP server for Onyx.
# mcp_server:
# image: ${ONYX_BACKEND_IMAGE:-onyxdotapp/onyx-backend:${IMAGE_TAG:-latest}}
# build:
# context: ../../backend
# dockerfile: Dockerfile
# target: runtime
# command: >
# /bin/sh -c "if [ \"${MCP_SERVER_ENABLED:-}\" != \"True\" ] && [ \"${MCP_SERVER_ENABLED:-}\" != \"true\" ]; then
# echo 'MCP server is disabled (MCP_SERVER_ENABLED=false), skipping...';
# exit 0;
# else
# exec python -m onyx.mcp_server_main;
# fi"
# env_file:
# - path: .env
# required: false
# depends_on:
# - relational_db
# - cache
# restart: "no"
# environment:
# - POSTGRES_HOST=${POSTGRES_HOST:-relational_db}
# - REDIS_HOST=${REDIS_HOST:-cache}
# # MCP Server Configuration
# - MCP_SERVER_ENABLED=${MCP_SERVER_ENABLED:-false}
# - MCP_SERVER_PORT=${MCP_SERVER_PORT:-8090}
# - MCP_SERVER_CORS_ORIGINS=${MCP_SERVER_CORS_ORIGINS:-}
# - API_SERVER_PROTOCOL=${API_SERVER_PROTOCOL:-http}
# - API_SERVER_HOST=${API_SERVER_HOST:-api_server}
# extra_hosts:
# - "host.docker.internal:host-gateway"
# logging:
# driver: json-file
# options:
# max-size: "50m"
# max-file: "6"
# # Optional, only for debugging purposes
# volumes:
# - mcp_server_logs:/var/log/onyx
inference_model_server:
image: ${ONYX_MODEL_SERVER_IMAGE:-onyxdotapp/onyx-model-server:${IMAGE_TAG:-latest}}
build:
context: ../../backend
dockerfile: Dockerfile.model_server
# GPU Support: Uncomment the following lines to enable GPU support
# Requires nvidia-container-toolkit to be installed on the host
# deploy:
# resources:
# reservations:
# devices:
# - driver: nvidia
# count: all
# capabilities: [gpu]
env_file:
- path: .env
required: false
restart: unless-stopped
volumes:
# Not necessary, this is just to reduce download time during startup
- model_cache_huggingface:/app/.cache/huggingface/
# Optional, only for debugging purposes
- inference_model_server_logs:/var/log/onyx
logging:
driver: json-file
options:
max-size: "50m"
max-file: "6"
healthcheck:
test:
[
"CMD",
"python",
"-c",
"import urllib.request; urllib.request.urlopen('http://localhost:9000/api/health')",
]
interval: 20s
timeout: 5s
retries: 3
# Generous start_period to absorb HuggingFace model downloads on first
# boot. Healthy is reported as soon as /api/health responds.
start_period: 600s
indexing_model_server:
image: ${ONYX_MODEL_SERVER_IMAGE:-onyxdotapp/onyx-model-server:${IMAGE_TAG:-latest}}
build:
context: ../../backend
dockerfile: Dockerfile.model_server
# GPU Support: Uncomment the following lines to enable GPU support
# Requires nvidia-container-toolkit to be installed on the host
# deploy:
# resources:
# reservations:
# devices:
# - driver: nvidia
# count: all
# capabilities: [gpu]
env_file:
- path: .env
required: false
restart: unless-stopped
environment:
- INDEXING_ONLY=True
volumes:
# Not necessary, this is just to reduce download time during startup
- indexing_huggingface_model_cache:/app/.cache/huggingface/
# Optional, only for debugging purposes
- indexing_model_server_logs:/var/log/onyx
logging:
driver: json-file
options:
max-size: "50m"
max-file: "6"
healthcheck:
test:
[
"CMD",
"python",
"-c",
"import urllib.request; urllib.request.urlopen('http://localhost:9000/api/health')",
]
interval: 20s
timeout: 5s
retries: 3
# Generous start_period to absorb HuggingFace model downloads on first
# boot. Healthy is reported as soon as /api/health responds.
start_period: 600s
relational_db:
image: ${BASE_IMAGE_REGISTRY:-docker.io}/library/postgres:15.2-alpine
shm_size: 1g
command: -c 'max_connections=250'
env_file:
- path: .env
required: false
restart: unless-stopped
# PRODUCTION: Override the defaults by passing in the environment variables
environment:
- POSTGRES_USER=${POSTGRES_USER:-postgres}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-password}
# DEV: To expose ports, either:
# 1. Use docker-compose.dev.yml: docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --wait
# 2. Uncomment the ports below
# ports:
# - "5432:5432"
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres}"]
interval: 10s
timeout: 5s
retries: 5
volumes:
- db_volume:/var/lib/postgresql/data
logging:
driver: json-file
options:
max-size: "50m"
max-file: "6"
opensearch:
image: ${BASE_IMAGE_REGISTRY:-docker.io}/opensearchproject/opensearch:3.6.0
restart: unless-stopped
# OpenSearch is the search backend and is enabled by default. To run against
# an external OpenSearch instance, set OPENSEARCH_HOST in your env and
# remove this service from the compose file (or skip it via the service list
# when running `docker compose up`).
environment:
# We need discovery.type=single-node so that OpenSearch doesn't try
# forming a cluster and waiting for other nodes to become live.
- discovery.type=single-node
- OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-StrongPassword123!}
# This and the JVM config below come from the example in https://docs.opensearch.org/latest/install-and-configure/install-opensearch/docker/
# We do this to avoid unstable performance from page swaps.
- bootstrap.memory_lock=true # Disable JVM heap memory swapping.
# Java heap should be ~50% of memory limit. For now we assume a limit of
# 4g although in practice the container can request more than this.
# See https://opster.com/guides/opensearch/opensearch-basics/opensearch-heap-size-usage-and-jvm-garbage-collection/
# Xms is the starting size, Xmx is the maximum size. These should be the
# same.
- "OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g"
volumes:
- opensearch-data:/usr/share/opensearch/data
# These come from the example in https://docs.opensearch.org/latest/install-and-configure/install-opensearch/docker/
ulimits:
# Similarly to bootstrap.memory_lock, we don't want to impose limits on
# how much memory a process can lock from being swapped.
memlock:
soft: -1 # Set memlock to unlimited (no soft or hard limit).
hard: -1
nofile:
soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536.
hard: 65536
logging:
driver: json-file
options:
max-size: "50m"
max-file: "6"
nginx:
image: ${BASE_IMAGE_REGISTRY:-docker.io}/library/nginx:1.25.5-alpine
restart: unless-stopped
# nginx will immediately crash with `nginx: [emerg] host not found in upstream`
# if api_server / web_server are not up
depends_on:
api_server:
condition: service_healthy
web_server:
condition: service_healthy
env_file:
- path: .env
required: false
environment:
- DOMAIN=localhost
# Nginx proxy timeout settings (in seconds)
- NGINX_PROXY_CONNECT_TIMEOUT=${NGINX_PROXY_CONNECT_TIMEOUT:-300}
- NGINX_PROXY_SEND_TIMEOUT=${NGINX_PROXY_SEND_TIMEOUT:-300}
- NGINX_PROXY_READ_TIMEOUT=${NGINX_PROXY_READ_TIMEOUT:-300}
ports:
- "${HOST_PORT_80:-80}:80"
- "${HOST_PORT:-3000}:80" # allow for localhost:3000 usage, since that is the norm
volumes:
# Mount templates read-only; the startup command copies them into
# the writable /etc/nginx/conf.d/ inside the container. This avoids
# "Permission denied" errors on Windows Docker bind mounts.
- ../data/nginx:/nginx-templates:ro
# PRODUCTION: Add SSL certificate volumes for HTTPS support:
# - ../data/certbot/conf:/etc/letsencrypt
# - ../data/certbot/www:/var/www/certbot
logging:
driver: json-file
options:
max-size: "50m"
max-file: "6"
# The specified script waits for the api_server to start up.
# Without this we've seen issues where nginx shows no error logs but
# does not receive any traffic
# PRODUCTION: Change to app.conf.template.prod for production nginx config
command: >
/bin/sh -c "rm -f /etc/nginx/conf.d/default.conf
&& cp -a /nginx-templates/. /etc/nginx/conf.d/
&& sed 's/\r$//' /etc/nginx/conf.d/run-nginx.sh > /tmp/run-nginx.sh
&& chmod +x /tmp/run-nginx.sh
&& /tmp/run-nginx.sh app.conf.template"
healthcheck:
test:
[
"CMD",
"wget",
"--quiet",
"--tries=1",
"--spider",
"http://127.0.0.1/nginx-health",
]
interval: 30s
timeout: 10s
retries: 5
start_period: 30s
cache:
image: ${BASE_IMAGE_REGISTRY:-docker.io}/library/redis:7.4-alpine
restart: unless-stopped
# DEV: To expose ports, either:
# 1. Use docker-compose.dev.yml: docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --wait
# 2. Uncomment the ports below
# ports:
# - "6379:6379"
# docker silently mounts /data even without an explicit volume mount, which enables
# persistence. explicitly setting save and appendonly forces ephemeral behavior.
command: redis-server --save "" --appendonly no
env_file:
- path: .env
required: false
# Use tmpfs to prevent creation of anonymous volumes for /data
tmpfs:
- /data
minio:
image: ${BASE_IMAGE_REGISTRY:-docker.io}/minio/minio:RELEASE.2025-07-23T15-54-02Z-cpuv1
profiles: ["s3-filestore"]
restart: unless-stopped
# DEV: To expose ports, either:
# 1. Use docker-compose.dev.yml: docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --wait
# 2. Uncomment the ports below
# ports:
# - "9004:9000"
# - "9005:9001"
env_file:
- path: .env
required: false
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin}
# Note: we've seen the default bucket creation logic not work in some cases
MINIO_DEFAULT_BUCKETS: ${S3_FILE_STORE_BUCKET_NAME:-onyx-file-store-bucket}
volumes:
- minio_data:/data
command: server /data --console-address ":9001"
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 30s
timeout: 20s
retries: 3
code-interpreter:
image: onyxdotapp/code-interpreter:${CODE_INTERPRETER_IMAGE_TAG:-latest}
command: ["bash", "./entrypoint.sh", "code-interpreter-api"]
restart: unless-stopped
env_file:
- path: .env
required: false
# Below is needed for the `docker-out-of-docker` execution mode
# For Linux rootless Docker, set DOCKER_SOCK_PATH=${XDG_RUNTIME_DIR}/docker.sock
user: root
volumes:
- ${DOCKER_SOCK_PATH:-/var/run/docker.sock}:/var/run/docker.sock
# uncomment below + comment out the above to use the `docker-in-docker` execution mode
# privileged: true
# PRODUCTION: Uncomment the following certbot service for SSL certificate management
# certbot:
# image: certbot/certbot
# restart: unless-stopped
# volumes:
# - ../data/certbot/conf:/etc/letsencrypt
# - ../data/certbot/www:/var/www/certbot
# logging:
# driver: json-file
# options:
# max-size: "50m"
# max-file: "6"
# entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'"
volumes:
# Necessary for persisting data for use
db_volume:
minio_data:
# Caches to prevent re-downloading models, not strictly necessary
model_cache_huggingface:
indexing_huggingface_model_cache:
# Logs preserved across container restarts
api_server_logs:
background_logs:
# mcp_server_logs:
inference_model_server_logs:
indexing_model_server_logs:
# Shared volume for persistent document storage (Craft file-system mode)
file-system:
# Persistent data for OpenSearch.
opensearch-data:
cli/internal/deploy/deployfiles/embedded/docker_compose/env.template
Generated
+418
View File
@@ -0,0 +1,418 @@
# Copy this file to .env so it's picked up by the docker compose yaml files
# Uncomment the values you would like to set
# No edits necessary, works out of the box
################################################################################
## COMMONLY MODIFIED CONFIGURATIONS
################################################################################
## Version of Onyx to deploy, default is latest (main built nightly).
## Craft uses this same tag for the sandbox image; do not set a separate
## sandbox image for normal deployments.
## Every image publishes a -dev twin for each of its tags (e.g. latest-dev,
## v1.2.3-dev), so IMAGE_TAG=latest-dev selects the dev variant of the whole
## deployment. Today only the backend image actually differs: its -dev twin adds
## debugging tools (vim, nano, curl, ps, psql) that the default image leaves out.
IMAGE_TAG=latest
## Onyx Craft Configuration (off by default).
## When enabling Craft through install.sh --include-craft or by manually adding
## docker-compose.craft.yml, the sandbox image follows IMAGE_TAG above.
# ENABLE_CRAFT=true
#
## docker backend drives sandboxes via the host Docker socket (root-equivalent;
## trusted hosts only) and needs v4.0.6+ images. install.sh picks docker vs
## kubernetes by image tag.
# SANDBOX_BACKEND=docker
#
## ONYX_SERVER_URL is the complete API base URL used by Onyx clients. The Craft
## compose overlay defaults it to its private onyx-craft-api network alias,
## which works on Docker Desktop and Linux without exposing another host port.
## Override only when routing sandbox API traffic through a public reverse
## proxy, including the /api path prefix served by that proxy.
# ONYX_SERVER_URL=https://onyx.your-org.example/api
#
## Sandbox bridge network. Created by install.sh (or manually:
## docker network create onyx_craft_sandbox
## Sandboxes join only this network. The firewall forces API and public egress
## through sandbox-proxy; postgres, redis, and minio stay off the bridge.
# SANDBOX_DOCKER_NETWORK=onyx_craft_sandbox
#
## Per-sandbox container resource limits. Defaults match K8s pod *requests*
## (1 CPU / 2Gi), not its limits (2 CPU / 10Gi), since single-VM compose
## deployments rarely have headroom to over-commit every sandbox.
# SANDBOX_DOCKER_MEMORY_LIMIT=2g
# SANDBOX_DOCKER_CPU_LIMIT=1.0
## EC2 IMDS note: on EC2 the Docker daemon's default bridge allows traffic to
## 169.254.169.254 (Instance Metadata Service), which can hand out IAM role
## credentials. The fix is a host-level DOCKER-USER iptables rule that drops
## bridge traffic to that address. install.sh --include-craft installs this
## rule automatically when it has sudo/iptables access, and prints manual
## instructions otherwise.
## Auth Settings
### https://docs.onyx.app/deployment/authentication
### Authentication is always enabled. SSO (Google / OIDC / SAML) is configured
### in the admin panel (Admin Panel > Organization > SSO Providers).
# SESSION_EXPIRE_TIME_SECONDS=
### Signs password reset, email verification, OAuth login state, and captcha cookies.
### REQUIRED: the API server refuses to start with an empty value (secure by default).
### If using install.sh this is auto-generated; if setting manually, run: openssl rand -hex 32
USER_AUTH_SECRET=""
### Recommend to set this for security
# ENCRYPTION_KEY_SECRET=
### Bearer token for the API server's /metrics endpoint. Auth is REQUIRED by
### default: scrapers must send `Authorization: Bearer <token>` (Prometheus
### scrape config: `authorization`). If you set neither this token nor
### DISABLE_METRICS_AUTH, /metrics is locked (returns 401).
# METRICS_AUTH_TOKEN=
### Set to "true" to expose /metrics with NO authentication (explicit opt-out).
# DISABLE_METRICS_AUTH=
### Set to "true" to expose the interactive API docs and schema (/openapi.json,
### /docs, /redoc) publicly with NO authentication. Off by default so the API
### surface is not exposed; when off these routes are not registered (404).
# ENABLE_PUBLIC_DOCS=
### Optional
# API_KEY_HASH_ROUNDS=
### You can add a comma separated list of domains like onyx.app, only those domains will be allowed to signup/log in
# VALID_EMAIL_DOMAINS=
### Set to "true" to restrict GET /users to admins. Non-admins can't enumerate
### accounts or share agents with individual users; group sharing still works.
# USER_DIRECTORY_ADMIN_ONLY=
## Chat Configuration
# HARD_DELETE_CHATS=
# MAX_ALLOWED_UPLOAD_SIZE_MB=250
# Default per-user upload size limit (MB) when no admin value is set.
# Automatically clamped to MAX_ALLOWED_UPLOAD_SIZE_MB at runtime.
# DEFAULT_USER_FILE_MAX_UPLOAD_SIZE_MB=100
## Base URL for redirects
# WEB_DOMAIN=
## Set to false to let any origin embed Onyx in an <iframe>, disabling
## clickjacking protection. Protected by default.
# WEB_FRAME_PROTECTION_ENABLED=false
## Set to true to add strict Content-Security-Policy directives
## (default-src/script-src/connect-src/etc.) restricting where scripts load
## from and where data can be sent. Off by default.
# WEB_STRICT_CSP_ENABLED=true
## Enterprise Features, requires a paid plan and licenses
ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=false
################################################################################
## SERVICES CONFIGURATIONS
################################################################################
## Database Configuration
POSTGRES_USER=postgres
POSTGRES_PASSWORD=password
# POSTGRES_DB=
# POSTGRES_DEFAULT_SCHEMA=
# POSTGRES_USE_NULL_POOL=
# POSTGRES_API_SERVER_POOL_SIZE=
# POSTGRES_API_SERVER_POOL_OVERFLOW
# POSTGRES_POOL_RECYCLE=
# Client-side TCP keepalives for the sync engine (sent on idle sockets to
# stop PgBouncer / NLB from silently closing them). Enabled by default.
# POSTGRES_TCP_KEEPALIVES=
# POSTGRES_TCP_KEEPALIVES_IDLE=
# POSTGRES_TCP_KEEPALIVES_INTERVAL=
# POSTGRES_TCP_KEEPALIVES_COUNT=
# --- Tenant sharding across multiple databases (multi-tenant cloud only) ---
# Leave all of these unset for the normal single-database deployment.
#
# JSON object of shard name -> connection overrides. Unspecified keys fall back
# to the POSTGRES_* values above. Adding a shard splits the connection pool
# budget rather than multiplying it.
# ONYX_DB_SHARDS={"shard-b":{"host":"other.example.com","db":"danswer"}}
# ONYX_DB_SHARDS=
# Shard hosting tenants that have no explicit mapping. Default: "default".
# ONYX_DB_DEFAULT_SHARD=
# Shard holding the shared `public` catalog tables (user_tenant_mapping,
# tenant_shard, ...). Must be resolvable without a lookup. Defaults to
# ONYX_DB_DEFAULT_SHARD.
# ONYX_DB_CATALOG_SHARD=
# Static tenant_id -> shard overrides, consulted before the catalog table.
# Incident-response escape hatch, not for routine use.
# ONYX_DB_SHARD_OVERRIDES={"tenant_abc":"shard-b"}
# ONYX_DB_SHARD_OVERRIDES=
# Seconds a resolved tenant -> shard mapping is cached in-process. This is the
# staleness backstop used when Redis is unreachable. Default 60.
# ONYX_DB_SHARD_MAP_TTL_SECONDS=
# Seconds between re-reads of the shared shard-map version from Redis. Bounds how
# long a tenant migration's map flip takes to reach every process, and therefore
# the minimum a migrated tenant stays frozen after the flip. Default 5.
# ONYX_DB_SHARD_MAP_VERSION_POLL_SECONDS=
# --- Database TLS (libpq sslmode vocabulary; ignored when USE_IAM_AUTH=true) ---
# Pick the level you need; each builds on the one above. Default (unset) = no TLS.
#
# require encrypt only — does NOT detect a MITM (server is not authenticated)
# verify-ca + verify the server cert chains to POSTGRES_SSLROOTCERT
# verify-full + also verify the server hostname <- recommended for managed DBs
#
# verify-ca / verify-full REQUIRE POSTGRES_SSLROOTCERT (CA bundle path; use the
# system bundle for publicly-trusted server certs).
# POSTGRES_SSLMODE= # disable | allow | prefer | require | verify-ca | verify-full
# POSTGRES_SSLROOTCERT= # e.g. /etc/ssl/certs/rds-ca-bundle.pem
#
# Mutual TLS (optional, rarely needed): also present a CLIENT cert so the server
# can authenticate us. Most deployments do NOT set these. Requires a POSTGRES_SSLMODE
# that negotiates SSL (require / verify-ca / verify-full); cert and key must both be set.
# POSTGRES_SSLCERT= # client certificate path
# POSTGRES_SSLKEY= # client private key path
# POSTGRES_SSLKEY_PASSWORD= # optional, only if the key file is encrypted
# DB_READONLY_USER=
# DB_READONLY_PASSWORD=
## File Store Backend: "s3" (default, uses MinIO) or "postgres" (no extra services needed)
## COMPOSE_PROFILES activates the MinIO service. To use PostgreSQL file storage instead,
## remove s3-filestore from COMPOSE_PROFILES and set FILE_STORE_BACKEND=postgres.
COMPOSE_PROFILES=s3-filestore
FILE_STORE_BACKEND=s3
## MinIO/S3 Configuration (only needed when FILE_STORE_BACKEND=s3)
## SECURITY: minioadmin is a local-dev default. install.sh randomizes it; if you
## edit .env by hand, change all four values before production. Keep the S3_AWS_*
## pair identical to MINIO_ROOT_* (the app uses MinIO's root creds).
S3_ENDPOINT_URL=http://minio:9000
S3_AWS_ACCESS_KEY_ID=minioadmin
S3_AWS_SECRET_ACCESS_KEY=minioadmin
S3_FILE_STORE_BUCKET_NAME=onyx-file-store-bucket
MINIO_ROOT_USER=minioadmin
MINIO_ROOT_PASSWORD=minioadmin
## GCS Configuration (only needed when FILE_STORE_BACKEND=gcs)
## No sidecar container needed — GCS is a managed service.
## Remove s3-filestore from COMPOSE_PROFILES when using FILE_STORE_BACKEND=gcs.
## By default, Application Default Credentials (ADC) are used, which supports
## GKE Workload Identity, Compute Engine instance SA, and gcloud CLI auth.
# FILE_STORE_BACKEND=gcs
# GCS_FILE_STORE_BUCKET_NAME=your-gcs-bucket-name
# GCS_FILE_STORE_PREFIX=onyx-files
# GCS_PROJECT_ID=your-gcp-project
# GCS_SERVICE_ACCOUNT_KEY_PATH=/path/to/service-account-key.json
# GCS_SERVICE_ACCOUNT_KEY_JSON=
## Azure Blob Storage Configuration (only needed when FILE_STORE_BACKEND=azure)
## No sidecar container needed — Azure Blob Storage is a managed service.
## Remove s3-filestore from COMPOSE_PROFILES when using FILE_STORE_BACKEND=azure.
## Auth priority: connection string, then account key, then DefaultAzureCredential
## (AKS Workload Identity, managed identity, or az CLI auth).
# FILE_STORE_BACKEND=azure
# AZURE_FILE_STORE_CONTAINER_NAME=your-container-name
# AZURE_FILE_STORE_PREFIX=onyx-files
# AZURE_STORAGE_ACCOUNT_NAME=yourstorageaccount
## Endpoint override — derived from the account name when unset. Set explicitly
## for Azurite or sovereign clouds (e.g. *.blob.core.usgovcloudapi.net).
# AZURE_STORAGE_ACCOUNT_URL=
# AZURE_STORAGE_CONNECTION_STRING=
# AZURE_STORAGE_ACCOUNT_KEY=
## Nginx Proxy Timeout Configuration (in seconds)
## These settings control how long nginx waits for upstream servers (api_server/web_server)
## Increase these values if you experience timeout errors with long-running requests
# NGINX_PROXY_CONNECT_TIMEOUT=300
# NGINX_PROXY_SEND_TIMEOUT=300
# NGINX_PROXY_READ_TIMEOUT=300
## MCP Server Configuration
## The MCP (Model Context Protocol) server allows external MCP clients to interact with Onyx
## Set to true to enable the MCP server (disabled by default)
# MCP_SERVER_ENABLED=false
## Port for the MCP server (defaults to 8090)
# MCP_SERVER_PORT=8090
## CORS origins for MCP clients (comma-separated list)
# MCP_SERVER_CORS_ORIGINS=
## Discord Bot Configuration
## The Discord bot allows users to interact with Onyx from Discord servers
## Bot token from Discord Developer Portal (required to enable the bot)
# DISCORD_BOT_TOKEN=
## Command prefix for bot commands (default: "!")
# DISCORD_BOT_INVOKE_CHAR=!
## Celery Configuration
# CELERY_BROKER_POOL_LIMIT=
# CELERY_WORKER_DOCFETCHING_CONCURRENCY=
# CELERY_WORKER_DOCPROCESSING_CONCURRENCY=
# CELERY_WORKER_LIGHT_CONCURRENCY=
# CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER=
## AWS Configuration
# AWS_ACCESS_KEY_ID=
# AWS_SECRET_ACCESS_KEY=
# AWS_REGION_NAME=
# Set to true when using IAM authentication for Postgres connections.
USE_IAM_AUTH=false
################################################################################
## DEVELOPER, DEBUGGING, AND LOGGING
################################################################################
## Logging and Telemetry
# LOG_LEVEL is the global knob for log verbosity across every Onyx process:
# api_server, web_server, model servers, slack/discord bots, AND all Celery
# workers (primary, light, heavy, docprocessing, docfetching, monitoring,
# scheduled_tasks, user_file_processing, beat). For per-worker overrides,
# explicitly add `--loglevel=<value>` to that worker's command in
# backend/supervisord.conf — an explicit CLI flag takes precedence over
# LOG_LEVEL inside the worker process.
LOG_LEVEL=info
LOG_ONYX_MODEL_INTERACTIONS=False
# LOG_VESPA_TIMING_INFORMATION=
# LOG_ENDPOINT_LATENCY=
# LOG_POSTGRES_LATENCY=
# LOG_POSTGRES_CONN_COUNTS=
# DISABLE_TELEMETRY=
## Feature Flags
# SHOW_EXTRA_CONNECTORS=true
# DISABLE_MODEL_SERVER=false
## Analytics
# SENTRY_DSN=
## Demo/Testing
# MOCK_CONNECTOR_FILE_PATH=
################################################################################
## ADVANCED CONFIGURATIONS
################################################################################
## SlackBot Configuration
# ONYX_BOT_DISABLE_DOCS_ONLY_ANSWER=
# ONYX_BOT_FEEDBACK_VISIBILITY=
# ONYX_BOT_DISPLAY_ERROR_MSGS=
# NOTIFY_SLACKBOT_NO_ANSWER=
# ONYX_BOT_MAX_QPM=
# ONYX_BOT_MAX_WAIT_TIME=
## Advanced Auth Settings
# REQUIRE_EMAIL_VERIFICATION=
# SMTP_SERVER=
# SMTP_PORT=
# SMTP_USER= # Optional - omit for unauthenticated relays
# SMTP_PASS= # Optional - omit for unauthenticated relays
# SMTP_STARTTLS=true # Set to "false" for servers that don't support TLS
# ENABLE_EMAIL_INVITES=
# EMAIL_FROM= # Required when SMTP_USER is not set
# EMAIL_ARCHIVE_BCC_ADDRESSES= # Optional comma-separated BCC archive recipients for all emails
# TRACK_EXTERNAL_IDP_EXPIRY=
# Opt-in: capture IdP directory claims at OAuth/OIDC login to enrich chat
# prompts and resolve {{user.<key>}} agent placeholders. Sends directory data
# to the configured LLM, so off by default.
# IDP_PROFILE_ENRICHMENT_ENABLED=
# Optional JSON overriding directory claim aliases per field, e.g.
# '{"department": ["dept", "division"], "country": ["c"]}'.
# IDP_PROFILE_CLAIM_MAP=
### Legacy pre-v4.4 SSO vars. Imported into an SSO provider entry on upgrade.
### Planned for removal in v4.5.
# AUTH_TYPE=
# GOOGLE_OAUTH_CLIENT_ID=
# GOOGLE_OAUTH_CLIENT_SECRET=
# OAUTH_CLIENT_ID=
# OAUTH_CLIENT_SECRET=
# OPENID_CONFIG_URL=
# SAML_CONF_DIR=
# Comma-separated list of origins allowed to make cross-origin (CORS) requests,
# e.g. "https://onyx.example.com". When unset, all origins are allowed but
# credentialed (cookie-authenticated) cross-origin requests are disabled.
# CORS_ALLOWED_ORIGIN=
# INTEGRATION_TESTS_MODE=
# JWT_PUBLIC_KEY_URL=
## Gen AI Settings
# GEN_AI_MAX_TOKENS=
# LLM_SOCKET_READ_TIMEOUT=
# MAX_CHUNKS_FED_TO_CHAT=
# DISABLE_LITELLM_STREAMING=
# LITELLM_EXTRA_HEADERS=
# GEN_AI_API_KEY=
# GENERATIVE_MODEL_ACCESS_CHECK_FREQ=
# LITELLM_CUSTOM_ERROR_MESSAGE_MAPPINGS=
## Usage / cost accounting
# Records priced generation spans into the per-user usage ledger (default on).
# Set to "false" to disable the recording processor entirely.
# USER_USAGE_TRACKING_ENABLED=true
# Flat per-image cost in cents when litellm has no price for an image model (default 4).
# DEFAULT_IMAGE_COST_CENTS=4
# Fallback USD per million tokens when litellm can't price a model (default 0 = free).
# DEFAULT_LLM_INPUT_COST_PER_MTOK=0
# DEFAULT_LLM_OUTPUT_COST_PER_MTOK=0
## Query Options
# DOC_TIME_DECAY=
# HYBRID_ALPHA=
# USE_SEMANTIC_KEYWORD_EXPANSIONS_BASIC_SEARCH=
## Model Configuration
# EMBEDDING_BATCH_SIZE=
# DOCUMENT_ENCODER_MODEL=
# DOC_EMBEDDING_DIM=
# NORMALIZE_EMBEDDINGS=
# ASYM_QUERY_PREFIX=
# ASYM_PASSAGE_PREFIX=
# DISABLE_RERANK_FOR_STREAMING=
# MODEL_SERVER_PORT=
# INDEX_BATCH_SIZE=
# MIN_THREADS_ML_MODELS=
## Indexing Configuration
# VESPA_SEARCHER_THREADS=
# ENABLED_CONNECTOR_TYPES=
# DISABLE_INDEX_UPDATE_ON_SWAP=
# OpenSearch Point-In-Time keep-alive for the reindex/port scan (default 5m)
# PIT_KEEP_ALIVE=5m
# CONTINUE_ON_CONNECTOR_FAILURE=
# CONFLUENCE_CONNECTOR_LABELS_TO_SKIP=
# JIRA_CONNECTOR_LABELS_TO_SKIP=
# JIRA_SERVER_API_VERSION=
# JIRA_CLOUD_API_VERSION=
# CANVAS_PERMISSION_DOC_SYNC_FREQUENCY=
# CANVAS_PERMISSION_GROUP_SYNC_FREQUENCY=
# BOX_CONNECTOR_SIZE_THRESHOLD=
# BOX_PERMISSION_DOC_SYNC_FREQUENCY=
# BOX_PERMISSION_GROUP_SYNC_FREQUENCY=
# GONG_CONNECTOR_START_TIME=
# NOTION_CONNECTOR_DISABLE_RECURSIVE_PAGE_LOOKUP=
# GITHUB_CONNECTOR_BASE_URL=
# MAX_DOCUMENT_CHARS=
# MAX_FILE_SIZE_BYTES=
# Push every successfully indexed public-connector document to an external HTTP
# endpoint (failures are logged and never fail indexing). Intended for non-EE
# deployments — EE users should configure the Document Push hook in the admin
# UI instead. Not supported in multi-tenant deployments.
# DOCUMENT_PUSH_ENDPOINT_URL=
# DOCUMENT_PUSH_API_KEY=
# DOCUMENT_PUSH_TIMEOUT_SECONDS=30
## OAuth Connector Configs
# EGNYTE_CLIENT_ID=
# EGNYTE_CLIENT_SECRET=
# LINEAR_CLIENT_ID=
# LINEAR_CLIENT_SECRET=
## Miscellaneous
# ONYX_QUERY_HISTORY_TYPE=
# Hide the Query History page from the admin sidebar (visibility-only; the history
# APIs + recording keep working, unlike ONYX_QUERY_HISTORY_TYPE=disabled).
# HIDE_QUERY_HISTORY_FROM_ADMIN_PANEL=true
# CHECK_TTL_MANAGEMENT_TASK_FREQUENCY_IN_HOURS=
# VESPA_LANGUAGE_OVERRIDE=
## Frontend Configs
# NEXT_PUBLIC_DISABLE_LOGOUT=
# NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED=
# NEXT_PUBLIC_THEME=
# NEXT_PUBLIC_DO_NOT_USE_TOGGLE_OFF_DANSWER_POWERED=
# NEXT_PUBLIC_CUSTOM_REFRESH_URL=
## Pointer to services
POSTGRES_HOST=relational_db
REDIS_HOST=cache
MODEL_SERVER_HOST=inference_model_server
INDEXING_MODEL_SERVER_HOST=indexing_model_server
INTERNAL_URL=http://api_server:8080
cli/internal/deploy/deployfiles/files.go
+94
View File
@@ -0,0 +1,94 @@
// Package deployfiles provides the deployment files onyx-cli ships for the
// guided docker compose install, embedded into the binary.
//
// The copies under embedded/ are synced from deployment/ in the onyx repo by
// `ods generate-compose --write` (tools/ods); TestEmbeddedFilesMatchSource
// fails when they drift.
package deployfiles
import (
"embed"
"io/fs"
)
//go:embed embedded
var embeddedFS embed.FS
// File describes one managed deployment file.
type File struct {
// EmbedPath is the file's path inside the embedded FS.
EmbedPath string
// RepoPath is the file's path in the onyx repo, used when fetching the
// version matching a pinned release tag from raw.githubusercontent.com.
RepoPath string
// DestRel is the file's destination relative to the install root, using
// slash separators.
DestRel string
// Mode is the permission bits applied when writing the file to disk
// (embed.FS does not preserve them).
Mode fs.FileMode
}
// Content returns the embedded copy of the file.
func (f File) Content() ([]byte, error) {
return embeddedFS.ReadFile(f.EmbedPath)
}
// The managed files. DestRel mirrors the layout install.sh creates: compose
// files and .env under deployment/, nginx config under data/nginx/ (the
// compose file bind-mounts ../data/nginx), README at the root.
var (
Compose = File{
EmbedPath: "embedded/docker_compose/docker-compose.yml",
RepoPath: "deployment/docker_compose/docker-compose.yml",
DestRel: "deployment/docker-compose.yml",
Mode: 0644,
}
LiteOverlay = File{
EmbedPath: "embedded/docker_compose/docker-compose.onyx-lite.yml",
RepoPath: "deployment/docker_compose/docker-compose.onyx-lite.yml",
DestRel: "deployment/docker-compose.onyx-lite.yml",
Mode: 0644,
}
CraftOverlay = File{
EmbedPath: "embedded/docker_compose/docker-compose.craft.yml",
RepoPath: "deployment/docker_compose/docker-compose.craft.yml",
DestRel: "deployment/docker-compose.craft.yml",
Mode: 0644,
}
EnvTemplate = File{
EmbedPath: "embedded/docker_compose/env.template",
RepoPath: "deployment/docker_compose/env.template",
DestRel: "deployment/env.template",
Mode: 0644,
}
Readme = File{
EmbedPath: "embedded/docker_compose/README.md",
RepoPath: "deployment/docker_compose/README.md",
DestRel: "README.md",
Mode: 0644,
}
NginxAppConf = File{
EmbedPath: "embedded/data/nginx/app.conf.template",
RepoPath: "deployment/data/nginx/app.conf.template",
DestRel: "data/nginx/app.conf.template",
Mode: 0644,
}
NginxRunScript = File{
EmbedPath: "embedded/data/nginx/run-nginx.sh",
RepoPath: "deployment/data/nginx/run-nginx.sh",
DestRel: "data/nginx/run-nginx.sh",
Mode: 0755,
}
)
// All lists every managed file.
var All = []File{
Compose,
LiteOverlay,
CraftOverlay,
EnvTemplate,
Readme,
NginxAppConf,
NginxRunScript,
}
cli/internal/deploy/deployfiles/sync_test.go
+79
View File
@@ -0,0 +1,79 @@
package deployfiles
import (
"io/fs"
"os"
"path/filepath"
"testing"
)
// repoRoot points at the onyx repo checkout containing this package
// (cli/internal/deploy/deployfiles -> four levels up).
var repoRoot = filepath.Join("..", "..", "..", "..")
// TestEmbeddedFilesMatchSource byte-compares every embedded copy against its
// source under deployment/, making `go test` in the cli module a drift gate
// independent of the docker-compose-sync pre-commit hook.
func TestEmbeddedFilesMatchSource(t *testing.T) {
for _, f := range All {
embedded, err := f.Content()
if err != nil {
t.Fatalf("failed to read embedded %s: %v", f.EmbedPath, err)
}
source, err := os.ReadFile(filepath.Join(repoRoot, filepath.FromSlash(f.RepoPath)))
if err != nil {
t.Fatalf("failed to read source %s: %v", f.RepoPath, err)
}
if string(embedded) != string(source) {
t.Errorf("%s does not match %s: run `ods generate-compose --write` and commit the result",
f.EmbedPath, f.RepoPath)
}
}
}
// TestManifestCoversEmbeddedFS ensures every file in the embedded FS has a
// manifest entry (and vice versa), so nothing ships unmanaged.
func TestManifestCoversEmbeddedFS(t *testing.T) {
manifest := make(map[string]bool, len(All))
for _, f := range All {
manifest[f.EmbedPath] = true
}
var embeddedPaths []string
err := fs.WalkDir(embeddedFS, "embedded", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if !d.IsDir() {
embeddedPaths = append(embeddedPaths, path)
}
return nil
})
if err != nil {
t.Fatalf("failed to walk embedded FS: %v", err)
}
for _, path := range embeddedPaths {
if !manifest[path] {
t.Errorf("embedded file %s has no entry in deployfiles.All", path)
}
}
if len(embeddedPaths) != len(All) {
t.Errorf("embedded FS has %d files but deployfiles.All lists %d", len(embeddedPaths), len(All))
}
}
// TestManifestInvariants checks each entry is internally consistent.
func TestManifestInvariants(t *testing.T) {
for _, f := range All {
if f.Mode == 0 {
t.Errorf("%s: zero Mode", f.EmbedPath)
}
if f.DestRel == "" || f.RepoPath == "" {
t.Errorf("%s: missing DestRel or RepoPath", f.EmbedPath)
}
}
if NginxRunScript.Mode != 0755 {
t.Errorf("run-nginx.sh must stay executable (got mode %o)", NginxRunScript.Mode)
}
}
deployment/docker_compose/README.md
+7
View File
@@ -67,3 +67,10 @@ The generator lives in `tools/ods` (`internal/composegen`) and ships with the `o
package. The `docker-compose-sync` pre-commit hook runs it automatically for commits touching the
template or the generated files, so a stray edit to a generated file gets reverted on the next
commit.
onyx-cli embeds copies of the guided-install deployment files (the generated
`docker-compose.yml`, the lite/craft overlays, `env.template`, the nginx config, and this README)
under `cli/internal/deploy/deployfiles/embedded/`. The same `ods generate-compose --write` run
refreshes them after rendering the variants, and a drift test in the cli module (`go test ./...`)
gates staleness. If you change any of those files, re-run the generator and commit the refreshed
embedded copies.
tools/ods/cmd/generate_compose.go
+42 -6
View File
@@ -11,6 +11,8 @@ import (
"github.com/spf13/cobra"
"github.com/onyx-dot-app/onyx/tools/ods/internal/composegen"
"github.com/onyx-dot-app/onyx/tools/ods/internal/deployfilessync"
"github.com/onyx-dot-app/onyx/tools/ods/internal/paths"
)
// NewGenerateComposeCommand creates the generate-compose command.
@@ -28,10 +30,17 @@ docker-compose.template.yml instead, then regenerate with:
ods generate-compose --write
The command also syncs the deployment files that onyx-cli embeds via
go:embed (the generated docker-compose.yml, the lite/craft overlays,
env.template, the nginx config, and the install-root README) into
byte-identical copies under cli/internal/deploy/deployfiles/embedded/ —
go:embed cannot reference files outside the cli module, and the copies must
always be refreshed together with the generated compose files.
Without --write the command runs in check mode: it prints a diff and exits
non-zero if any generated file is out of date. The docker-compose-sync
pre-commit hook runs --write automatically for commits touching the template
or the generated files.
non-zero if any generated file or embedded copy is out of date. The
docker-compose-sync pre-commit hook runs --write automatically for commits
touching any of the source or output files.
Template directives are line comments starting with the sentinel "#!". They
are always stripped from the output. <variants> is a comma-separated subset
@@ -95,11 +104,38 @@ func runGenerateCompose(write bool) {
}
fmt.Printf("regenerated %s\n", filename)
} else {
fmt.Fprint(os.Stderr, lineDiff(filename, string(current), content))
fmt.Fprint(os.Stderr, lineDiff(filename, composegen.TemplateName, string(current), content))
stale = append(stale, filename)
}
}
// Sync the embedded copies for onyx-cli after the variants, so the copy
// of docker-compose.yml reflects the freshly rendered output.
repoRoot, err := paths.GitRoot()
if err != nil {
log.Fatalf("Failed to find git root: %v", err)
}
var results []deployfilessync.Result
if write {
results, err = deployfilessync.Write(repoRoot)
} else {
results, err = deployfilessync.Check(repoRoot)
}
if err != nil {
log.Fatalf("%v", err)
}
for _, r := range results {
if !r.Stale {
continue
}
if write {
fmt.Printf("synced embedded copy of %s\n", r.RelPath)
} else {
fmt.Fprint(os.Stderr, lineDiff("embedded/"+r.RelPath, "deployment/"+r.RelPath, r.Dest, r.Source))
stale = append(stale, "embedded copy of "+r.RelPath)
}
}
if len(stale) > 0 {
for _, filename := range stale {
fmt.Fprintf(os.Stderr, "%s is stale (re-run with --write).\n", filename)
@@ -110,13 +146,13 @@ func runGenerateCompose(write bool) {
// lineDiff returns a compact line-based diff (changed lines only) between the
// on-disk file and the freshly rendered content.
func lineDiff(filename, current, generated string) string {
func lineDiff(filename, source, current, generated string) string {
dmp := diffmatchpatch.New()
a, b, lineArray := dmp.DiffLinesToChars(current, generated)
diffs := dmp.DiffCharsToLines(dmp.DiffMain(a, b, false), lineArray)
var sb strings.Builder
fmt.Fprintf(&sb, "--- %s\n+++ %s (from %s)\n", filename, filename, composegen.TemplateName)
fmt.Fprintf(&sb, "--- %s\n+++ %s (from %s)\n", filename, filename, source)
for _, d := range diffs {
var prefix string
switch d.Type {
tools/ods/internal/deployfilessync/deployfilessync.go
+92
View File
@@ -0,0 +1,92 @@
// Package deployfilessync mirrors the deployment files that onyx-cli embeds
// (via go:embed in cli/internal/deploy/deployfiles) from their source of truth
// under deployment/. go:embed cannot reference files outside the cli module,
// so byte-identical copies are checked in and kept in sync by
// `ods generate-compose --write` (which renders the compose variants and then
// syncs these copies) and its docker-compose-sync pre-commit hook.
package deployfilessync
import (
"fmt"
"os"
"path/filepath"
)
// RelPaths lists the synced files relative to <repo>/deployment. They are the
// exact set the guided installer ships (base compose + lite/craft overlays +
// env template + nginx config + install-root README) and are mirrored at the
// same relative paths under cli/internal/deploy/deployfiles/embedded/.
//
// docker-compose.yml is itself generated from docker-compose.template.yml,
// which is why the sync runs as the second half of `ods generate-compose`.
var RelPaths = []string{
"docker_compose/docker-compose.yml",
"docker_compose/docker-compose.onyx-lite.yml",
"docker_compose/docker-compose.craft.yml",
"docker_compose/env.template",
"docker_compose/README.md",
"data/nginx/app.conf.template",
"data/nginx/run-nginx.sh",
}
// SourceDir returns the directory the files are copied from.
func SourceDir(repoRoot string) string {
return filepath.Join(repoRoot, "deployment")
}
// DestDir returns the directory the files are copied to.
func DestDir(repoRoot string) string {
return filepath.Join(repoRoot, "cli", "internal", "deploy", "deployfiles", "embedded")
}
// Result describes one synced file.
type Result struct {
RelPath string
Stale bool
Source string // content under deployment/ (the source of truth)
Dest string // embedded copy content ("" if missing)
}
// Check compares every embedded copy against its source file.
func Check(repoRoot string) ([]Result, error) {
results := make([]Result, 0, len(RelPaths))
for _, rel := range RelPaths {
source, err := os.ReadFile(filepath.Join(SourceDir(repoRoot), filepath.FromSlash(rel)))
if err != nil {
return nil, fmt.Errorf("failed to read source %s: %w", rel, err)
}
dest, err := os.ReadFile(filepath.Join(DestDir(repoRoot), filepath.FromSlash(rel)))
if err != nil && !os.IsNotExist(err) {
return nil, fmt.Errorf("failed to read embedded copy %s: %w", rel, err)
}
results = append(results, Result{
RelPath: rel,
Stale: string(source) != string(dest),
Source: string(source),
Dest: string(dest),
})
}
return results, nil
}
// Write refreshes every stale embedded copy from its source file and returns
// the check results (with Stale reflecting the pre-write state).
func Write(repoRoot string) ([]Result, error) {
results, err := Check(repoRoot)
if err != nil {
return nil, err
}
for _, r := range results {
if !r.Stale {
continue
}
dest := filepath.Join(DestDir(repoRoot), filepath.FromSlash(r.RelPath))
if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
return nil, fmt.Errorf("failed to create directory for %s: %w", r.RelPath, err)
}
if err := os.WriteFile(dest, []byte(r.Source), 0644); err != nil {
return nil, fmt.Errorf("failed to write %s: %w", r.RelPath, err)
}
}
return results, nil
}
tools/ods/internal/deployfilessync/deployfilessync_test.go
+98
View File
@@ -0,0 +1,98 @@
package deployfilessync
import (
"os"
"path/filepath"
"testing"
)
// seedSources writes a fake deployment/ tree containing every synced file.
func seedSources(t *testing.T, repoRoot string) {
t.Helper()
for _, rel := range RelPaths {
path := filepath.Join(SourceDir(repoRoot), filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
t.Fatalf("mkdir for %s: %v", rel, err)
}
if err := os.WriteFile(path, []byte("content of "+rel+"\n"), 0644); err != nil {
t.Fatalf("write %s: %v", rel, err)
}
}
}
func TestCheckReportsMissingCopiesAsStale(t *testing.T) {
repoRoot := t.TempDir()
seedSources(t, repoRoot)
results, err := Check(repoRoot)
if err != nil {
t.Fatalf("Check failed: %v", err)
}
if len(results) != len(RelPaths) {
t.Fatalf("got %d results, want %d", len(results), len(RelPaths))
}
for _, r := range results {
if !r.Stale {
t.Errorf("%s: expected stale (no embedded copy exists)", r.RelPath)
}
}
}
func TestWriteSyncsAndCheckTurnsClean(t *testing.T) {
repoRoot := t.TempDir()
seedSources(t, repoRoot)
if _, err := Write(repoRoot); err != nil {
t.Fatalf("Write failed: %v", err)
}
results, err := Check(repoRoot)
if err != nil {
t.Fatalf("Check failed: %v", err)
}
for _, r := range results {
if r.Stale {
t.Errorf("%s: still stale after Write", r.RelPath)
}
if r.Source != r.Dest {
t.Errorf("%s: embedded copy differs from source", r.RelPath)
}
}
}
func TestSourceEditMakesCopyStaleAgain(t *testing.T) {
repoRoot := t.TempDir()
seedSources(t, repoRoot)
if _, err := Write(repoRoot); err != nil {
t.Fatalf("Write failed: %v", err)
}
edited := filepath.Join(SourceDir(repoRoot), "docker_compose", "env.template")
if err := os.WriteFile(edited, []byte("IMAGE_TAG=v9.9.9\n"), 0644); err != nil {
t.Fatalf("edit source: %v", err)
}
results, err := Check(repoRoot)
if err != nil {
t.Fatalf("Check failed: %v", err)
}
staleCount := 0
for _, r := range results {
if r.Stale {
staleCount++
if r.RelPath != "docker_compose/env.template" {
t.Errorf("unexpected stale file %s", r.RelPath)
}
}
}
if staleCount != 1 {
t.Errorf("got %d stale files, want 1", staleCount)
}
}
func TestCheckFailsWhenSourceMissing(t *testing.T) {
repoRoot := t.TempDir()
if _, err := Check(repoRoot); err == nil {
t.Fatal("expected error for missing source files")
}
}