diff --git a/.agents/feature-map/components/craft-sandboxes.md b/.agents/feature-map/components/craft-sandboxes.md index eb1962dd6a..658f509b10 100644 --- a/.agents/feature-map/components/craft-sandboxes.md +++ b/.agents/feature-map/components/craft-sandboxes.md @@ -229,6 +229,17 @@ that creates `/workspace/sessions/$id/{outputs,attachments}`, symlinks `.opencod `user_library` to the sandbox-wide managed directories, writes `AGENTS.md` and `opencode.json`, and (if the session has a port) writes `start-webapp.sh`. It never scaffolds or starts the dev server itself (`[[craft-webapp-proxy]]` owns that). +New session config directories copy the image's preinstalled OpenCode plugin SDK +from `/workspace/templates/opencode`. Existing dependencies and package manifests +remain intact. Older images without this template use OpenCode's install fallback. +The image-owned script handles SDK copies +(`backend/onyx/server/features/build/sandbox/image/seed-opencode-dependencies.sh`). +Configuration regeneration also calls this script after snapshot restore. +A session seed lock serializes copies. Dependencies publish by rename after copying. +Kubernetes verifies a completion sentinel before reporting configuration regeneration success. +SDK files stay outside snapshots, which contain outputs and attachments. +Global configuration directories hardlink template dependencies to save image space. +Their manifests and all session dependency copies remain independent. A completion sentinel (`ONYX_WORKSPACE_SETUP_COMPLETE`) is the only reliable success signal, because the K8s exec client returns buffered output without raising on a nonzero exit or timeout (`session_workspace.py` module docstring). diff --git a/backend/onyx/server/features/build/sandbox/docker/docker_sandbox_manager.py b/backend/onyx/server/features/build/sandbox/docker/docker_sandbox_manager.py index b81921ee3e..c0e112f961 100644 --- a/backend/onyx/server/features/build/sandbox/docker/docker_sandbox_manager.py +++ b/backend/onyx/server/features/build/sandbox/docker/docker_sandbox_manager.py @@ -141,6 +141,7 @@ from onyx.server.features.build.sandbox.session_workspace import ( MANAGED_SKILLS_PATH, MANAGED_USER_LIBRARY_PATH, SESSIONS_ROOT, + build_opencode_dependency_setup_command, build_session_workspace_setup_script, build_workspace_exists_check_script, ) @@ -1597,6 +1598,7 @@ fi script = f""" set -e mkdir -p {session_path}/.opencode +{build_opencode_dependency_setup_command(session_path)} ln -sfn {MANAGED_SKILLS_PATH} {session_path}/.opencode/skills ln -sfn {MANAGED_USER_LIBRARY_PATH} {session_path}/user_library printf '%s' {shlex.quote(agents_md)} > {session_path}/AGENTS.md diff --git a/backend/onyx/server/features/build/sandbox/image/Dockerfile b/backend/onyx/server/features/build/sandbox/image/Dockerfile index 00cdf98daa..1abbd32afc 100644 --- a/backend/onyx/server/features/build/sandbox/image/Dockerfile +++ b/backend/onyx/server/features/build/sandbox/image/Dockerfile @@ -182,6 +182,15 @@ USER sandbox RUN curl -fsSL -o /tmp/opencode-install.sh https://opencode.ai/install && \ bash /tmp/opencode-install.sh --version "${OPENCODE_VERSION}" --no-modify-path && \ rm /tmp/opencode-install.sh +# OpenCode installs this SDK in each config directory during cold initialization. +RUN npm install --prefix /workspace/templates/opencode --ignore-scripts --no-audit --no-fund \ + --save-exact "@opencode-ai/plugin@${OPENCODE_VERSION}" && \ + for config_dir in /home/sandbox/.config/opencode /home/sandbox/.opencode; do \ + mkdir -p "$config_dir" && \ + cp -al /workspace/templates/opencode/node_modules "$config_dir/" && \ + cp -a /workspace/templates/opencode/package.json \ + /workspace/templates/opencode/package-lock.json "$config_dir/" || exit 1; \ + done # Pre-warm Bun's tarball cache so per-session installs hit a local store. RUN cd /workspace/templates/outputs/web && \ bun install --frozen-lockfile && \ @@ -214,6 +223,8 @@ RUN chmod +x /workspace/firewall-init.sh # kubernetes_sandbox_manager._OPENCODE_SESSION_TAG_PLUGIN_PATH. COPY opencode-plugins/ /workspace/opencode-plugins/ +COPY --chmod=0755 seed-opencode-dependencies.sh /usr/local/bin/seed-opencode-dependencies + COPY browser-cli.sh /usr/local/bin/browser RUN chmod 0755 /usr/local/bin/browser diff --git a/backend/onyx/server/features/build/sandbox/image/seed-opencode-dependencies.sh b/backend/onyx/server/features/build/sandbox/image/seed-opencode-dependencies.sh new file mode 100755 index 0000000000..5c7e603ef4 --- /dev/null +++ b/backend/onyx/server/features/build/sandbox/image/seed-opencode-dependencies.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# The caller holds the session seed lock until this script exits. +set -eu + +session_path="${1:?Usage: seed-opencode-dependencies SESSION_PATH [TEMPLATE_PATH]}" +template="${2:-/workspace/templates/opencode}" +config_dir="$session_path/.opencode" + +if [ ! -d "$template/node_modules" ] || [ -e "$config_dir/node_modules" ]; then + exit 0 +fi + +mkdir -p "$config_dir" +staging="$(mktemp -d "$config_dir/.onyx-sdk.XXXXXX")" +trap 'rm -rf "$staging"' EXIT +trap 'exit 1' HUP INT TERM + +cp -a "$template/node_modules" "$staging/" +for manifest in package.json package-lock.json; do + if [ ! -e "$config_dir/$manifest" ]; then + cp "$template/$manifest" "$staging/$manifest" + mv "$staging/$manifest" "$config_dir/$manifest" + fi +done +# Publish dependencies last, only after all copies completed. +mv "$staging/node_modules" "$config_dir/node_modules" diff --git a/backend/onyx/server/features/build/sandbox/kubernetes/kubernetes_sandbox_manager.py b/backend/onyx/server/features/build/sandbox/kubernetes/kubernetes_sandbox_manager.py index c56f324de8..f0dea379f3 100644 --- a/backend/onyx/server/features/build/sandbox/kubernetes/kubernetes_sandbox_manager.py +++ b/backend/onyx/server/features/build/sandbox/kubernetes/kubernetes_sandbox_manager.py @@ -124,8 +124,10 @@ from onyx.server.features.build.sandbox.nextjs_dev import ( ) from onyx.server.features.build.sandbox.serve_transport import ServeConnectionInfo from onyx.server.features.build.sandbox.session_workspace import ( + SESSION_CONFIG_COMPLETE_SENTINEL, SESSIONS_ROOT, WORKSPACE_SETUP_COMPLETE_SENTINEL, + build_opencode_dependency_setup_command, build_session_workspace_setup_script, build_workspace_exists_check_script, ) @@ -1936,6 +1938,7 @@ echo "Session cleanup complete" config_script = f""" set -e mkdir -p {session_path}/.opencode +{build_opencode_dependency_setup_command(session_path)} ln -sfn /workspace/managed/skills {session_path}/.opencode/skills ln -sfn /workspace/managed/user_library {session_path}/user_library printf '%s' '{agent_instructions_escaped}' > {session_path}/AGENTS.md @@ -1944,10 +1947,11 @@ if [ -n "$(find {session_path}/attachments -mindepth 1 -maxdepth 1 -print -quit printf '\n\n' >> {session_path}/AGENTS.md echo '{attachments_content_b64}' | base64 -d >> {session_path}/AGENTS.md fi +echo "{SESSION_CONFIG_COMPLETE_SENTINEL}" """ logger.info("Regenerating session configuration files") - k8s_stream( + exec_response = k8s_stream( self._stream_core_api.connect_get_namespaced_pod_exec, name=pod_name, namespace=self._namespace, @@ -1957,7 +1961,13 @@ fi stdin=False, stdout=True, tty=False, + _request_timeout=WORKSPACE_SETUP_DEADLINE_SECONDS, ) + if SESSION_CONFIG_COMPLETE_SENTINEL not in exec_response.splitlines(): + raise RuntimeError( + f"Session configuration regeneration for session {session_id} " + f"did not complete (output tail: {exec_response[-500:]!r})" + ) logger.info("Session configuration files regenerated") def health_check(self, sandbox_id: UUID, timeout: float) -> bool: diff --git a/backend/onyx/server/features/build/sandbox/session_workspace.py b/backend/onyx/server/features/build/sandbox/session_workspace.py index 3e9a2a4c88..ca727dcb36 100644 --- a/backend/onyx/server/features/build/sandbox/session_workspace.py +++ b/backend/onyx/server/features/build/sandbox/session_workspace.py @@ -31,6 +31,7 @@ SETUP_IN_PROGRESS_MARKER = ".setup-in-progress" # when its timeout lapses (and never raises on a nonzero exit), so the # sentinel is the only reliable success signal. WORKSPACE_SETUP_COMPLETE_SENTINEL = "ONYX_WORKSPACE_SETUP_COMPLETE" +SESSION_CONFIG_COMPLETE_SENTINEL = "ONYX_SESSION_CONFIG_COMPLETE" def build_workspace_exists_check_script(session_path: str) -> str: @@ -43,6 +44,16 @@ def build_workspace_exists_check_script(session_path: str) -> str: ) +def build_opencode_dependency_setup_command(session_path: str) -> str: + """Older sandbox images use OpenCode's runtime-install fallback.""" + script = "/usr/local/bin/seed-opencode-dependencies" + return ( + f"if [ -x {script} ]; then flock -x " + f"{shlex.quote(session_path + '.opencode-seed.lock')} " + f"{script} {shlex.quote(session_path)}; fi" + ) + + def build_session_workspace_setup_script( session_path: str, agents_md: str, @@ -84,6 +95,7 @@ mkdir -p {session_path}/attachments # which fails if the mount is a real directory. Dangling until the first # push lands is fine; nothing reads these during the rest of setup. mkdir -p {session_path}/.opencode +{build_opencode_dependency_setup_command(session_path)} ln -sfn {MANAGED_SKILLS_PATH} {session_path}/.opencode/skills echo "Linked skills to {MANAGED_SKILLS_PATH}" ln -sfn {MANAGED_USER_LIBRARY_PATH} {session_path}/user_library diff --git a/backend/tests/unit/onyx/server/features/craft/sandbox/test_k8s_workspace_setup_sentinel.py b/backend/tests/unit/onyx/server/features/craft/sandbox/test_k8s_workspace_setup_sentinel.py index 80d1fb96a2..2fc41254ef 100644 --- a/backend/tests/unit/onyx/server/features/craft/sandbox/test_k8s_workspace_setup_sentinel.py +++ b/backend/tests/unit/onyx/server/features/craft/sandbox/test_k8s_workspace_setup_sentinel.py @@ -79,3 +79,42 @@ def test_setup_raises_when_output_lacks_sentinel( # output with no sentinel. with pytest.raises(RuntimeError, match="did not complete"): _run_setup(monkeypatch, "Copying outputs template\nbun install ...") + + +@pytest.mark.parametrize("complete", [False, True]) +def test_regenerate_config_requires_completion_sentinel( + monkeypatch: pytest.MonkeyPatch, complete: bool +) -> None: + from onyx.server.features.build.sandbox.session_workspace import ( + SESSION_CONFIG_COMPLETE_SENTINEL, + ) + + mgr = _make_manager(monkeypatch) + stream = MagicMock( + return_value=( + f"{SESSION_CONFIG_COMPLETE_SENTINEL}\n" if complete else "cp: disk full\n" + ) + ) + monkeypatch.setattr(ksm, "k8s_stream", stream) + + def regenerate() -> None: + mgr.regenerate_session_config( + sandbox_id=uuid4(), + session_id=uuid4(), + agent_provider=None, + agent_model=None, + nextjs_port=None, + connectable_apps_section="", + llm_config=_LLM_CONFIG, + ) + + if complete: + regenerate() + else: + with pytest.raises(RuntimeError, match="did not complete"): + regenerate() + assert ( + stream.call_args.kwargs["_request_timeout"] == WORKSPACE_SETUP_DEADLINE_SECONDS + ) + script = stream.call_args.kwargs["command"][-1] + assert script.rstrip().endswith(f'echo "{SESSION_CONFIG_COMPLETE_SENTINEL}"') diff --git a/backend/tests/unit/onyx/server/features/craft/sandbox/test_opencode_dependency_seed.py b/backend/tests/unit/onyx/server/features/craft/sandbox/test_opencode_dependency_seed.py new file mode 100644 index 0000000000..64a74bd30c --- /dev/null +++ b/backend/tests/unit/onyx/server/features/craft/sandbox/test_opencode_dependency_seed.py @@ -0,0 +1,94 @@ +"""Run SDK seeding against real temporary session directories.""" + +import os +import subprocess +from pathlib import Path + +import pytest + +_SCRIPT = ( + Path(__file__).resolve().parents[7] + / "onyx/server/features/build/sandbox/image/seed-opencode-dependencies.sh" +) + + +@pytest.fixture +def sdk_template(tmp_path: Path) -> Path: + template = tmp_path / "template" + dependencies = template / "node_modules" / "plugin" + dependencies.mkdir(parents=True) + (dependencies / "index.js").write_text("sdk") + (template / "package.json").write_text('{"dependencies":{"plugin":"1.0.0"}}') + (template / "package-lock.json").write_text("{}") + return template + + +def _seed(session: Path, template: Path) -> None: + subprocess.run(["sh", str(_SCRIPT), str(session), str(template)], check=True) + + +def test_fresh_seed_replay_and_session_isolation( + tmp_path: Path, sdk_template: Path +) -> None: + first = tmp_path / "first session" + second = tmp_path / "second session" + _seed(first, sdk_template) + _seed(second, sdk_template) + sdk_file = Path(".opencode/node_modules/plugin/index.js") + assert (first / sdk_file).read_text() == "sdk" + assert (first / ".opencode/package-lock.json").read_text() == "{}" + (first / sdk_file).write_text("session change") + _seed(first, sdk_template) + assert (first / sdk_file).read_text() == "session change" + assert (second / sdk_file).read_text() == "sdk" + assert (sdk_template / "node_modules/plugin/index.js").read_text() == "sdk" + + +def test_existing_manifests_remain_intact(tmp_path: Path, sdk_template: Path) -> None: + config = tmp_path / "session/.opencode" + config.mkdir(parents=True) + for name in ("package.json", "package-lock.json"): + (config / name).write_text("session-owned") + _seed(config.parent, sdk_template) + for name in ("package.json", "package-lock.json"): + assert (config / name).read_text() == "session-owned" + assert (config / "node_modules/plugin/index.js").read_text() == "sdk" + + +def test_existing_dependencies_are_not_replaced( + tmp_path: Path, sdk_template: Path +) -> None: + config = tmp_path / "session/.opencode" + (config / "node_modules").mkdir(parents=True) + (config / "node_modules/custom").write_text("existing") + _seed(config.parent, sdk_template) + assert (config / "node_modules/custom").read_text() == "existing" + assert not (config / "package.json").exists() + + +def test_missing_template_leaves_runtime_install_fallback(tmp_path: Path) -> None: + session = tmp_path / "session" + _seed(session, tmp_path / "missing-template") + assert not session.exists() + + +def test_failed_copy_does_not_publish_partial_dependencies_and_retry_succeeds( + tmp_path: Path, sdk_template: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + commands = tmp_path / "commands" + commands.mkdir() + copy = commands / "cp" + copy.write_text( + '#!/bin/sh\nmkdir -p "$3/node_modules/plugin"\n' + 'echo partial > "$3/node_modules/plugin/index.js"\nexit 1\n' + ) + copy.chmod(0o755) + session = tmp_path / "session" + with monkeypatch.context() as patched: + patched.setenv("PATH", f"{commands}{os.pathsep}{os.environ['PATH']}") + with pytest.raises(subprocess.CalledProcessError): + _seed(session, sdk_template) + assert not (session / ".opencode/node_modules").exists() + assert list((session / ".opencode").iterdir()) == [] + _seed(session, sdk_template) + assert (session / ".opencode/node_modules/plugin/index.js").read_text() == "sdk" diff --git a/deployment/helm/dev/values-localdev.yaml b/deployment/helm/dev/values-localdev.yaml index f5478f10b8..76cbace051 100644 --- a/deployment/helm/dev/values-localdev.yaml +++ b/deployment/helm/dev/values-localdev.yaml @@ -1,7 +1,7 @@ # Onyx local-dev overlay for kind/minikube/k3d. # -# Shrinks resources and disables prod-only features so the chart fits on a -# laptop. Pairs with telepresence intercept so api_server / celery / web run +# Reserves resources for responsive development and disables prod-only features. +# Pairs with telepresence intercept so api_server / celery / web run # in your vscode debugger while the rest of the stack runs in-cluster. # # See docs/craft/dev/local-kubernetes.md for the full workflow. @@ -16,7 +16,21 @@ global: # ---- Persistence (kept on, sized for a laptop) ---- postgresql: + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "1" + memory: 512Mi cluster: + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi storage: size: 2Gi @@ -26,11 +40,11 @@ opensearch: size: 5Gi resources: requests: - cpu: 250m - memory: 1Gi - limits: - cpu: 1000m + cpu: 1 memory: 2Gi + limits: + cpu: 2 + memory: 4Gi opensearchJavaOpts: "-Xmx1g -Xms1g" minio: @@ -40,12 +54,21 @@ minio: # Upstream chart defaults to a 16 GiB memory request. resources: requests: - cpu: 100m + cpu: 250m memory: 256Mi limits: - cpu: 500m + cpu: 1 memory: 1Gi +objectStore: + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: "1" + memory: 2Gi + # ---- Networking (port-forward instead of ingress) ---- ingress: @@ -88,11 +111,11 @@ sandboxProxy: replicaCount: 1 resources: requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 250m + cpu: 500m memory: 512Mi + limits: + cpu: 2 + memory: 1Gi # The sandbox image is `kind load`ed with pullPolicy IfNotPresent, so there is # no registry pull to front-run — and without the local image overrides the @@ -191,28 +214,29 @@ inferenceCapability: replicaCount: 1 resources: requests: - cpu: 250m + cpu: 1 memory: 1Gi limits: - cpu: 1000m + cpu: 2 memory: 2Gi indexCapability: replicaCount: 1 resources: requests: - cpu: 250m + cpu: 1 memory: 1Gi limits: - cpu: 1000m + cpu: 2 memory: 2Gi redis: redisStandalone: resources: requests: - cpu: 50m - memory: 64Mi + cpu: 100m + memory: 128Mi + # The chart sets Redis maxmemory to 400 MB; leave room for process overhead. limits: - cpu: 250m - memory: 256Mi + cpu: "1" + memory: 512Mi diff --git a/docs/craft/dev/local-kubernetes.md b/docs/craft/dev/local-kubernetes.md index 4afdc070a9..418f8a08f4 100644 --- a/docs/craft/dev/local-kubernetes.md +++ b/docs/craft/dev/local-kubernetes.md @@ -20,7 +20,7 @@ touching `backend/onyx/sandbox_proxy/` or the docker manager — see Builds on the CONTRIBUTING.md prereqs (Python 3.13, uv, Node.js 22, the venv, `.vscode/.env`). Docker Desktop must be running with at least 8 CPU / 16 GB -allocated. +allocated. For three active Craft sandboxes, allocate at least 12 CPU / 24 GB. Craft sandbox pods use Kubernetes native restartable init sidecar containers, so the cluster must run Kubernetes `>= 1.33`. The local `k8s-up.sh` script pins @@ -54,6 +54,40 @@ One sudo prompt at session start; the daemon stays alive afterward: telepresence connect -n onyx ``` +## Local resource budgets + +`deployment/helm/dev/values-localdev.yaml` defines local service resources. +CPU requests reserve capacity. CPU limits allow short bursts above the request. +Memory limits remain separate from requests. + +| Service | CPU request / limit | Memory request / limit | +| --- | --- | --- | +| API (Telepresence target) | 0.1 / 0.5 | 256 MiB / 1 GiB | +| Sandbox proxy | 0.5 / 2 | 512 MiB / 1 GiB | +| Each model server | 1 / 2 | 1 GiB / 2 GiB | +| OpenSearch | 1 / 2 | 2 GiB / 4 GiB | +| PostgreSQL | 0.5 / 2 | 512 MiB / 2 GiB | +| PostgreSQL operator | 0.1 / 1 | 128 MiB / 512 MiB | +| MinIO | 0.25 / 1 | 256 MiB / 1 GiB | +| Object store | 0.25 / 1 | 256 MiB / 2 GiB | +| Redis | 0.1 / 1 | 128 MiB / 512 MiB | + +Redis has a 400 MB cache budget. Its memory limit also covers process overhead. +OpenSearch keeps its 1 GiB heap and has memory for other allocations. + +The API serves as a Telepresence target. The developer's local Python process handles intercepted API requests. +Its small resource budget is deliberate; local API startup can take longer. + +These services request 4.8 CPU and about 6 GiB before other operators and Kubernetes system pods. +Each sandbox adds a 1 CPU / 2 GiB request, plus its sidecar's 0.1 CPU / 256 MiB request. +Sandbox limits remain 2 CPU / 10 GiB; its sidecar has a 1 CPU / 1 GiB limit. +These values support development. They do not guarantee performance for large indexes or models. + +Run `make craft-up` to apply the local overlay. Keep any local image overrides when upgrading an existing release. +Changing PostgreSQL resources restarts its single instance. Existing connections can delay shutdown for up to three minutes. +New database connections fail during this restart. +Check node capacity with `kubectl describe node`; check usage with `kubectl top pods` when metrics are available. + ## kubectl context Every script in this doc — and the `make craft-up` wrapper — refuses to diff --git a/docs/craft/sandbox/image-and-spinup.md b/docs/craft/sandbox/image-and-spinup.md index c31a88a720..f517d62e07 100644 --- a/docs/craft/sandbox/image-and-spinup.md +++ b/docs/craft/sandbox/image-and-spinup.md @@ -14,6 +14,32 @@ Related files: - `backend/onyx/server/features/build/sandbox/kubernetes/scripts/bench-sandbox-spinup.sh` - `deployment/helm/charts/onyx/templates/sandbox-namespace.yaml` +## OpenCode plugin SDK during cold startup + +OpenCode 1.18.19 installs `@opencode-ai/plugin` in each configuration directory. +Session initialization waits for these installs before it returns. +The SDK includes about 63 MiB of dependencies. An observed cold setup took +three minutes and exceeded the API's 90-second session initialization timeout. +With a warm download cache, a new directory still took 74 seconds. +The same configuration with installed dependencies took 0.58 seconds. + +The sandbox image installs the matching SDK version during its build. +Both global configuration directories hardlink the template dependencies within the same image layer. +Their package manifests remain separate files. This avoids two extra copies of SDK data. +In-place dependency edits in either global directory also affect the template. +The image keeps the SDK in +`/workspace/templates/opencode`. Session workspace setup copies these dependencies +into a new `.opencode` directory. It preserves existing dependencies and manifests. +Each session has its own copy, so sessions cannot modify each other's SDK files. +Older images without the template retain OpenCode's runtime install fallback. + +The image-owned `seed-opencode-dependencies.sh` seeds new and restored sessions. +Python invokes the script under a session seed lock. +The script copies dependencies into a temporary directory and publishes them by rename. +A failed copy leaves no visible dependency directory, so the next attempt can retry. +Kubernetes verifies a completion sentinel after configuration regeneration. +The SDK stays outside snapshots, which contain outputs and attachments. + ## SHA-pinned base + helper images `python:3.13-slim`, `node:24-trixie-slim`, and `oven/bun:1.3.14` are