feat(deployment): add the SeaweedFS object-store service beside MinIO (#15248)

cli/cmd/deploy_install.go

@@ -55,7 +55,7 @@ every prompt, including Lite mode).`,
 	cmd.Flags().BoolVar(&opts.Lite, "lite", false, "Deploy Onyx Lite (no OpenSearch, Redis, or model servers)")
 	cmd.Flags().BoolVar(&opts.IncludeCraft, "include-craft", false, "Enable Onyx Craft (AI-powered web app building)")
 	cmd.Flags().BoolVar(&opts.Prod, "prod", false, "Restart an existing prod deployment (the standalone docker-compose.prod.yml); fresh prod installs are not created here")
-	cmd.Flags().BoolVar(&opts.Dev, "dev", false, "Stack docker-compose.dev.yml on the deployment: publish the API, Postgres, Redis, OpenSearch, MinIO and model server ports on this host (development and testing)")
+	cmd.Flags().BoolVar(&opts.Dev, "dev", false, "Stack docker-compose.dev.yml on the deployment: publish the API, Postgres, Redis, OpenSearch, object store and model server ports on this host (development and testing)")
 	cmd.Flags().StringVar(&opts.Project, "project", "", `Docker compose project name (default: recorded in the manifest, else "onyx")`)
 	cmd.Flags().StringVar(&opts.Tag, "tag", "", "Image tag to deploy (default: the latest Onyx release)")
 	cmd.Flags().BoolVar(&opts.Local, "local", false, "Use existing config files on disk instead of downloading")

cli/internal/deploy/deployfiles/embedded/docker_compose/docker-compose.dev.yml

@@ -92,17 +92,16 @@ services:
     ports:
       - "${REDIS_HOST_PORT:-6379}:6379"
 
-  minio:
-    # Use different ports to avoid conflicts with model servers.
+  object-store:
+    # Host port 9004 keeps the S3_ENDPOINT_URL dev and CI setups already use.
     # TODO(security): prefix 127.0.0.1: to bind loopback, not 0.0.0.0 (LAN-exposed).
     ports:
-      - "${MINIO_API_HOST_PORT:-9004}:9000"
-      - "${MINIO_CONSOLE_HOST_PORT:-9005}:9001"
-    healthcheck:
-      # The base healthcheck has no start_period, so one is needed here for
-      # start_interval to apply at all.
-      start_period: 30s
-      start_interval: 1s
+      - "${OBJECT_STORE_HOST_PORT:-9004}:8333"
+
+  minio:
+    # TODO(security): prefix 127.0.0.1: to bind loopback, not 0.0.0.0 (LAN-exposed).
+    ports:
+      - "${MINIO_API_HOST_PORT:-9005}:9000"
 
   code-interpreter:
     ports:

cli/internal/deploy/deployfiles/embedded/docker_compose/docker-compose.onyx-lite.yml

@@ -15,12 +15,12 @@
 #                  -f docker-compose.dev.yml up -d --wait
 #
 # This overlay:
-#   - Moves both model servers, OpenSearch, MinIO, Redis (cache), and the
-#     background worker to profiles so they do not start by default
+#   - Moves both model servers, OpenSearch, the object store, Redis (cache),
+#     and the background worker to profiles so they do not start by default
 #   - Makes depends_on references to removed services optional
 #   - Sets DISABLE_VECTOR_DB=true on the api_server
 #   - Uses PostgreSQL for caching and auth instead of Redis
-#   - Uses PostgreSQL for file storage instead of S3/MinIO
+#   - Uses PostgreSQL for file storage instead of the S3 object store
 #
 # To selectively bring services back:
 #   --profile vectordb          Indexing model server
@@ -28,7 +28,7 @@
 #   --profile background        Background worker (Celery) — also needs redis
 #   --profile redis             Redis cache
 #   --profile opensearch        OpenSearch
-#   --profile s3-filestore      MinIO (S3-compatible file store)
+#   --profile s3-filestore      Object store and the legacy MinIO
 # =============================================================================
 
 name: onyx
@@ -45,6 +45,9 @@ services:
       inference_model_server:
         condition: service_started
         required: false
+      object-store:
+        condition: service_healthy
+        required: false
       minio:
         condition: service_started
         required: false
@@ -83,6 +86,9 @@ services:
   opensearch:
     profiles: ["opensearch"]
 
-  # MinIO is not needed in lite mode (Postgres handles file storage).
+  # The object store is not needed in lite mode (Postgres handles file storage).
+  object-store:
+    profiles: ["s3-filestore"]
+
   minio:
     profiles: ["s3-filestore"]

cli/internal/deploy/deployfiles/embedded/docker_compose/docker-compose.prod.yml

@@ -31,8 +31,14 @@ services:
         condition: service_started
       inference_model_server:
         condition: service_started
+      # Optional, since installs on external storage never use it.
+      object-store:
+        condition: service_healthy
+        required: false
+      # Only the legacy store, so a MinIO that fails to start never blocks the app.
       minio:
         condition: service_started
+        required: false
     restart: unless-stopped
     environment:
       # Auth Settings
@@ -47,7 +53,8 @@ services:
       - AWS_REGION_NAME=${AWS_REGION_NAME-}
       - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID-}
       - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY-}
-      # MinIO configuration
+      # Object store configuration
+      # A .env that names no endpoint predates object-store, so it keeps MinIO.
       - S3_ENDPOINT_URL=${S3_ENDPOINT_URL:-http://minio:9000}
       - S3_AWS_ACCESS_KEY_ID=${S3_AWS_ACCESS_KEY_ID:?set a strong value in .env, not minioadmin}
       - S3_AWS_SECRET_ACCESS_KEY=${S3_AWS_SECRET_ACCESS_KEY:?set a strong value in .env, not minioadmin}
@@ -111,6 +118,14 @@ services:
         condition: service_started
       indexing_model_server:
         condition: service_started
+      # Optional, since installs on external storage never use it.
+      object-store:
+        condition: service_healthy
+        required: false
+      # Only the legacy store, so a MinIO that fails to start never blocks the app.
+      minio:
+        condition: service_started
+        required: false
     restart: unless-stopped
     environment:
       - AUTH_TYPE=${AUTH_TYPE:-oidc}
@@ -124,7 +139,8 @@ services:
       - AWS_REGION_NAME=${AWS_REGION_NAME-}
       - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID-}
       - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY-}
-      # MinIO configuration
+      # Object store configuration
+      # A .env that names no endpoint predates object-store, so it keeps MinIO.
       - S3_ENDPOINT_URL=${S3_ENDPOINT_URL:-http://minio:9000}
       - S3_AWS_ACCESS_KEY_ID=${S3_AWS_ACCESS_KEY_ID:?set a strong value in .env, not minioadmin}
       - S3_AWS_SECRET_ACCESS_KEY=${S3_AWS_SECRET_ACCESS_KEY:?set a strong value in .env, not minioadmin}
@@ -466,24 +482,60 @@ services:
     tmpfs:
       - /data
 
+  object-store:
+    # SeaweedFS behind an S3 API. Only the S3 gateway listens beyond loopback,
+    # because the filer, master and volume HTTP APIs take no credentials.
+    image: ${BASE_IMAGE_REGISTRY:-docker.io}/chrislusf/seaweedfs:4.47@sha256:ce9e796f1fe6f06968f4c04bdaf8f678dad9c8acdfef3d244133d71bfa6bf882
+    restart: unless-stopped
+    environment:
+      # SeaweedFS makes this pair its admin identity.
+      AWS_ACCESS_KEY_ID: ${S3_AWS_ACCESS_KEY_ID:?set a strong value in .env, not minioadmin}
+      AWS_SECRET_ACCESS_KEY: ${S3_AWS_SECRET_ACCESS_KEY:?set a strong value in .env, not minioadmin}
+      # File store keys can hold a name segment longer than the 255-byte default.
+      WEED_FILER_OPTIONS_MAX_FILE_NAME_LENGTH: "1024"
+      # A bucket grows one volume at a time, not seven, so it can write with as
+      # little as one volume of free disk.
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: "1"
+    volumes:
+      - object_store_data:/data
+    # SeaweedFS does not fsync bucket writes by default, so a filer rule turns
+    # it on for /buckets/ once the server is up. The healthcheck waits for it.
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - |
+        /entrypoint.sh server -s3 -ip=127.0.0.1 -ip.bind=127.0.0.1 -s3.ip.bind=0.0.0.0 &
+        pid=$$!
+        trap 'kill -TERM $$pid; wait $$pid' TERM INT
+        until echo 'fs.configure -locationPrefix=/buckets/ -fsync -apply' | weed shell -master=127.0.0.1:9333 >/dev/null 2>&1; do
+          kill -0 $$pid 2>/dev/null || exit 1
+          sleep 1
+        done
+        touch /tmp/fsync-configured
+        wait $$pid
+    healthcheck:
+      test: ["CMD-SHELL", "test -f /tmp/fsync-configured && wget -q -O /dev/null http://127.0.0.1:8333/healthz"]
+      interval: 5s
+      timeout: 5s
+      retries: 12
+
   minio:
-    # MinIO archived its community edition, and Docker Hub and quay.io no longer
-    # serve minio/minio, so this is our mirror of the newest multi-arch build we
-    # kept. -cpuv1 also runs on x86-64-v1 CPUs. Pinned by digest.
+    # Earlier releases' store. A .env that still names it keeps using it.
+    # MinIO no longer publishes images, so this is our digest-pinned mirror of the last multi-arch build.
+    # -cpuv1 also runs on x86-64-v1 CPUs.
     image: ${BASE_IMAGE_REGISTRY:-docker.io}/onyxdotapp/minio:RELEASE.2025-07-23T15-54-02Z-cpuv1@sha256:7330be2e7320a7a699b36f72bd06a94d8a21ec9f6f397345d9c38fc6751141f5
     restart: unless-stopped
     environment:
-      MINIO_ROOT_USER: ${MINIO_ROOT_USER:?set a strong value in .env, not minioadmin}
-      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?set a strong value in .env, not minioadmin}
-      MINIO_DEFAULT_BUCKETS: ${S3_FILE_STORE_BUCKET_NAME:-onyx-file-store-bucket}
+      # A .env without MINIO_ROOT_* falls back to the S3_AWS_* pair.
+      MINIO_ROOT_USER: ${MINIO_ROOT_USER:-${S3_AWS_ACCESS_KEY_ID:?set a strong value in .env, not minioadmin}}
+      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-${S3_AWS_SECRET_ACCESS_KEY:?set a strong value in .env, not minioadmin}}
     volumes:
       - minio_data:/data
     command: server /data --console-address ":9001"
     healthcheck:
       test: ["CMD", "mc", "ready", "local"]
-      interval: 30s
-      timeout: 20s
-      retries: 3
+      interval: 5s
+      timeout: 5s
+      retries: 12
 
   code-interpreter:
     # The sandbox ships on its own release line, so IMAGE_TAG does not cover it.
@@ -527,6 +579,7 @@ services:
 volumes:
   # Necessary for persisting data for use
   db_volume:
+  object_store_data:
   minio_data:
   # Caches to prevent re-downloading models, not strictly necessary
   model_cache_huggingface:

cli/internal/deploy/deployfiles/embedded/docker_compose/docker-compose.yml

@@ -16,8 +16,8 @@
 #
 # 1. SECURITY HARDENING:
 #    - Remove all port exposures except nginx (80/443)
-#    - Comment out ports for: api_server, relational_db, cache, minio
-#    - Set strong MinIO/S3 credentials (MINIO_ROOT_* + S3_AWS_*), not minioadmin
+#    - Comment out ports for: api_server, relational_db, cache, object-store
+#    - Set strong object store credentials (S3_AWS_*), not minioadmin
 #
 # 2. SSL/TLS SETUP:
 #    - Uncomment the certbot service (see below)
@@ -69,6 +69,11 @@ services:
         condition: service_started
       inference_model_server:
         condition: service_started
+      # Optional, since installs on external storage never use it.
+      object-store:
+        condition: service_healthy
+        required: false
+      # Only the legacy store, so a MinIO that fails to start never blocks the app.
       minio:
         condition: service_started
         required: false
@@ -88,6 +93,7 @@ services:
       - REDIS_HOST=${REDIS_HOST:-cache}
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       - CODE_INTERPRETER_BASE_URL=${CODE_INTERPRETER_BASE_URL:-http://code-interpreter:8000}
+      # A .env that names no endpoint predates object-store, so it keeps MinIO.
       - S3_ENDPOINT_URL=${S3_ENDPOINT_URL:-http://minio:9000}
       - S3_AWS_ACCESS_KEY_ID=${S3_AWS_ACCESS_KEY_ID:-minioadmin}
       - S3_AWS_SECRET_ACCESS_KEY=${S3_AWS_SECRET_ACCESS_KEY:-minioadmin}
@@ -154,6 +160,14 @@ services:
         condition: service_started
       indexing_model_server:
         condition: service_started
+      # Optional, since installs on external storage never use it.
+      object-store:
+        condition: service_healthy
+        required: false
+      # Only the legacy store, so a MinIO that fails to start never blocks the app.
+      minio:
+        condition: service_started
+        required: false
     restart: unless-stopped
     environment:
       - FILE_STORE_BACKEND=${FILE_STORE_BACKEND:-s3}
@@ -163,6 +177,7 @@ services:
       - REDIS_HOST=${REDIS_HOST:-cache}
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       - INDEXING_MODEL_SERVER_HOST=${INDEXING_MODEL_SERVER_HOST:-indexing_model_server}
+      # A .env that names no endpoint predates object-store, so it keeps MinIO.
       - S3_ENDPOINT_URL=${S3_ENDPOINT_URL:-http://minio:9000}
       - S3_AWS_ACCESS_KEY_ID=${S3_AWS_ACCESS_KEY_ID:-minioadmin}
       - S3_AWS_SECRET_ACCESS_KEY=${S3_AWS_SECRET_ACCESS_KEY:-minioadmin}
@@ -511,35 +526,73 @@ services:
     tmpfs:
       - /data
 
-  minio:
-    # MinIO archived its community edition, and Docker Hub and quay.io no longer
-    # serve minio/minio, so this is our mirror of the newest multi-arch build we
-    # kept. -cpuv1 also runs on x86-64-v1 CPUs. Pinned by digest.
-    image: ${BASE_IMAGE_REGISTRY:-docker.io}/onyxdotapp/minio:RELEASE.2025-07-23T15-54-02Z-cpuv1@sha256:7330be2e7320a7a699b36f72bd06a94d8a21ec9f6f397345d9c38fc6751141f5
+  object-store:
+    # SeaweedFS behind an S3 API. Only the S3 gateway listens beyond loopback,
+    # because the filer, master and volume HTTP APIs take no credentials.
+    image: ${BASE_IMAGE_REGISTRY:-docker.io}/chrislusf/seaweedfs:4.47@sha256:ce9e796f1fe6f06968f4c04bdaf8f678dad9c8acdfef3d244133d71bfa6bf882
     profiles: ["s3-filestore"]
     restart: unless-stopped
     # DEV: To expose ports, either:
     # 1. Use docker-compose.dev.yml: docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --wait
     # 2. Uncomment the ports below
     # ports:
-    #   - "9004:9000"
-    #   - "9005:9001"
+    #   - "9004:8333"
     env_file:
       - path: .env
         required: false
     environment:
-      MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
-      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin}
-      # Note: we've seen the default bucket creation logic not work in some cases
-      MINIO_DEFAULT_BUCKETS: ${S3_FILE_STORE_BUCKET_NAME:-onyx-file-store-bucket}
+      # SeaweedFS makes this pair its admin identity.
+      AWS_ACCESS_KEY_ID: ${S3_AWS_ACCESS_KEY_ID:-minioadmin}
+      AWS_SECRET_ACCESS_KEY: ${S3_AWS_SECRET_ACCESS_KEY:-minioadmin}
+      # File store keys can hold a name segment longer than the 255-byte default.
+      WEED_FILER_OPTIONS_MAX_FILE_NAME_LENGTH: "1024"
+      # A bucket grows one volume at a time, not seven, so it can write with as
+      # little as one volume of free disk.
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: "1"
+    volumes:
+      - object_store_data:/data
+    # SeaweedFS does not fsync bucket writes by default, so a filer rule turns
+    # it on for /buckets/ once the server is up. The healthcheck waits for it.
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - |
+        /entrypoint.sh server -s3 -ip=127.0.0.1 -ip.bind=127.0.0.1 -s3.ip.bind=0.0.0.0 &
+        pid=$$!
+        trap 'kill -TERM $$pid; wait $$pid' TERM INT
+        until echo 'fs.configure -locationPrefix=/buckets/ -fsync -apply' | weed shell -master=127.0.0.1:9333 >/dev/null 2>&1; do
+          kill -0 $$pid 2>/dev/null || exit 1
+          sleep 1
+        done
+        touch /tmp/fsync-configured
+        wait $$pid
+    healthcheck:
+      test: ["CMD-SHELL", "test -f /tmp/fsync-configured && wget -q -O /dev/null http://127.0.0.1:8333/healthz"]
+      interval: 5s
+      timeout: 5s
+      retries: 12
+
+  minio:
+    # Earlier releases' store. A .env that still names it keeps using it.
+    # MinIO no longer publishes images, so this is our digest-pinned mirror of the last multi-arch build.
+    # -cpuv1 also runs on x86-64-v1 CPUs.
+    image: ${BASE_IMAGE_REGISTRY:-docker.io}/onyxdotapp/minio:RELEASE.2025-07-23T15-54-02Z-cpuv1@sha256:7330be2e7320a7a699b36f72bd06a94d8a21ec9f6f397345d9c38fc6751141f5
+    profiles: ["s3-filestore"]
+    restart: unless-stopped
+    env_file:
+      - path: .env
+        required: false
+    environment:
+      # A .env without MINIO_ROOT_* falls back to the S3_AWS_* pair.
+      MINIO_ROOT_USER: ${MINIO_ROOT_USER:-${S3_AWS_ACCESS_KEY_ID:-minioadmin}}
+      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-${S3_AWS_SECRET_ACCESS_KEY:-minioadmin}}
     volumes:
       - minio_data:/data
     command: server /data --console-address ":9001"
     healthcheck:
       test: ["CMD", "mc", "ready", "local"]
-      interval: 30s
-      timeout: 20s
-      retries: 3
+      interval: 5s
+      timeout: 5s
+      retries: 12
 
   code-interpreter:
     # The sandbox ships on its own release line, so IMAGE_TAG does not cover it.
@@ -597,6 +650,7 @@ services:
 volumes:
   # Necessary for persisting data for use
   db_volume:
+  object_store_data:
   minio_data:
   # Caches to prevent re-downloading models, not strictly necessary
   model_cache_huggingface:

cli/internal/deploy/deployfiles/embedded/docker_compose/env.template

@@ -40,7 +40,7 @@ IMAGE_TAG=latest
 ## Sandbox bridge network. Created by the installer (or manually:
 ##   docker network create onyx_craft_sandbox
 ## Sandboxes join only this network. The firewall forces API and public egress
-## through sandbox-proxy; postgres, redis, and minio stay off the bridge.
+## through sandbox-proxy. Postgres, redis, and the object store stay off the bridge.
 # SANDBOX_DOCKER_NETWORK=onyx_craft_sandbox
 #
 ## Per-sandbox container resource limits. Defaults match K8s pod *requests*
@@ -161,17 +161,18 @@ POSTGRES_PASSWORD=password
 # DB_READONLY_USER=
 # DB_READONLY_PASSWORD=
 
-## File Store Backend: "s3" (default, uses MinIO) or "postgres" (no extra services needed)
-## COMPOSE_PROFILES activates the MinIO service. To use PostgreSQL file storage instead,
+## File Store Backend: "s3" (default, uses the bundled object store) or "postgres" (no extra services needed)
+## COMPOSE_PROFILES activates the object store and the legacy MinIO services. To use PostgreSQL file storage instead,
 ## remove s3-filestore from COMPOSE_PROFILES and set FILE_STORE_BACKEND=postgres.
 COMPOSE_PROFILES=s3-filestore
 FILE_STORE_BACKEND=s3
 
-## MinIO/S3 Configuration (only needed when FILE_STORE_BACKEND=s3)
-## SECURITY: minioadmin is a local-dev default. The installer randomizes it; if you
-## edit .env by hand, change all four values before production. Keep the S3_AWS_*
-## pair identical to MINIO_ROOT_* (the app uses MinIO's root creds).
-S3_ENDPOINT_URL=http://minio:9000
+## Object store configuration (only needed when FILE_STORE_BACKEND=s3)
+## The bundled object store is SeaweedFS. It uses the S3_AWS_* pair as its admin credentials.
+## SECURITY: minioadmin is a local-dev default. The installer randomizes it. If you
+## edit .env by hand, change all four values before production. Keep MINIO_ROOT_*
+## equal to the S3_AWS_* pair, since earlier releases start MinIO from them.
+S3_ENDPOINT_URL=http://object-store:8333
 S3_AWS_ACCESS_KEY_ID=minioadmin
 S3_AWS_SECRET_ACCESS_KEY=minioadmin
 S3_FILE_STORE_BUCKET_NAME=onyx-file-store-bucket

cli/internal/deploy/deployfiles/files.go

@@ -57,7 +57,7 @@ var (
 		Mode:      0644,
 	}
 	// DevOverlay publishes the service ports (API, Postgres, Redis,
-	// OpenSearch, model server, MinIO, code interpreter) on the host. Stacked
+	// OpenSearch, model server, object store, MinIO, code interpreter) on the host. Stacked
 	// on Compose like LiteOverlay, but flag-only (--dev): it takes the
 	// deployment's internals out from behind nginx.
 	DevOverlay = File{