test(e2e): run Playwright against the mock LLM server (#15198)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

.github/workflows/pr-playwright-tests.yml

@@ -25,7 +25,6 @@ env:
   # Test Environment Variables
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
-  GEN_AI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
   FIRECRAWL_API_KEY: ${{ secrets.FIRECRAWL_API_KEY }}
   GOOGLE_PSE_API_KEY: ${{ secrets.GOOGLE_PSE_API_KEY }}
@@ -65,7 +64,6 @@ env:
   MCP_PER_USER_KEY_SERVER_HOST: 0.0.0.0
   MCP_PER_USER_KEY_SERVER_PUBLIC_HOST: host.docker.internal
 
-  MOCK_LLM_RESPONSE: true
   MCP_TEST_SERVER_PORT: 8004
   MCP_TEST_SERVER_URL: http://host.docker.internal:8004/mcp
   MCP_TEST_SERVER_PUBLIC_URL: http://host.docker.internal:8004/mcp
@@ -533,7 +531,6 @@ jobs:
 
       - name: Create .env file for Docker Compose
         env:
-          OPENAI_API_KEY_VALUE: ${{ env.OPENAI_API_KEY }}
           EXA_API_KEY_VALUE: ${{ env.EXA_API_KEY }}
           ECR_CACHE: ${{ env.RUNS_ON_ECR_CACHE }}
           RUN_ID: ${{ github.run_id }}
@@ -545,7 +542,6 @@ jobs:
           ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true
           AUTH_TYPE=basic
           INTEGRATION_TESTS_MODE=true
-          GEN_AI_API_KEY=${OPENAI_API_KEY_VALUE}
           EXA_API_KEY=${EXA_API_KEY_VALUE}
           REQUIRE_EMAIL_VERIFICATION=false
           DISABLE_TELEMETRY=true
@@ -595,7 +591,7 @@ jobs:
       - name: Start Docker containers
         run: |
           cd deployment/docker_compose
-          docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.mcp-oauth-test.yml -f docker-compose.mcp-api-key-test.yml -f docker-compose.mcp-per-user-key-test.yml up -d --wait --wait-timeout 300
+          docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.mcp-oauth-test.yml -f docker-compose.mcp-api-key-test.yml -f docker-compose.mcp-per-user-key-test.yml -f docker-compose.mock-llm-test.yml up -d --wait --wait-timeout 300
         id: start_docker
 
       - name: Seed dev license
@@ -642,7 +638,7 @@ jobs:
           cd deployment/docker_compose
           # Pass the same -f overrides as `up` so the mock MCP/IdP services
           # (defined only in the override files) are included in the logs.
-          docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.mcp-oauth-test.yml -f docker-compose.mcp-api-key-test.yml -f docker-compose.mcp-per-user-key-test.yml logs > docker-compose.log
+          docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.mcp-oauth-test.yml -f docker-compose.mcp-api-key-test.yml -f docker-compose.mcp-per-user-key-test.yml -f docker-compose.mock-llm-test.yml logs > docker-compose.log
           mv docker-compose.log ${WORKSPACE}/docker-compose.log
 
       - name: Upload logs
@@ -716,7 +712,6 @@ jobs:
 
       - name: Create .env file for Docker Compose
         env:
-          OPENAI_API_KEY_VALUE: ${{ env.OPENAI_API_KEY }}
           EXA_API_KEY_VALUE: ${{ env.EXA_API_KEY }}
           ECR_CACHE: ${{ env.RUNS_ON_ECR_CACHE }}
           RUN_ID: ${{ github.run_id }}
@@ -728,7 +723,6 @@ jobs:
           ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true
           AUTH_TYPE=basic
           INTEGRATION_TESTS_MODE=true
-          GEN_AI_API_KEY=${OPENAI_API_KEY_VALUE}
           EXA_API_KEY=${EXA_API_KEY_VALUE}
           REQUIRE_EMAIL_VERIFICATION=false
           DISABLE_TELEMETRY=true
@@ -784,7 +778,7 @@ jobs:
       - name: Start Docker containers
         run: |
           cd deployment/docker_compose
-          docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.mcp-oauth-test.yml -f docker-compose.mcp-api-key-test.yml -f docker-compose.mcp-per-user-key-test.yml up -d --wait --wait-timeout 300
+          docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.mcp-oauth-test.yml -f docker-compose.mcp-api-key-test.yml -f docker-compose.mcp-per-user-key-test.yml -f docker-compose.mock-llm-test.yml up -d --wait --wait-timeout 300
         id: start_docker
 
       - name: Seed dev license
@@ -817,7 +811,7 @@ jobs:
           WORKSPACE: ${{ github.workspace }}
         run: |
           cd deployment/docker_compose
-          docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.mcp-oauth-test.yml -f docker-compose.mcp-api-key-test.yml -f docker-compose.mcp-per-user-key-test.yml logs > docker-compose.log
+          docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.mcp-oauth-test.yml -f docker-compose.mcp-api-key-test.yml -f docker-compose.mcp-per-user-key-test.yml -f docker-compose.mock-llm-test.yml logs > docker-compose.log
           mv docker-compose.log ${WORKSPACE}/docker-compose.log
 
       - name: Upload logs
@@ -876,7 +870,6 @@ jobs:
 
       - name: Create .env file for Docker Compose
         env:
-          OPENAI_API_KEY_VALUE: ${{ env.OPENAI_API_KEY }}
           ECR_CACHE: ${{ env.RUNS_ON_ECR_CACHE }}
           RUN_ID: ${{ github.run_id }}
         run: |
@@ -884,8 +877,6 @@ jobs:
           ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true
           AUTH_TYPE=basic
           INTEGRATION_TESTS_MODE=true
-          GEN_AI_API_KEY=${OPENAI_API_KEY_VALUE}
-          MOCK_LLM_RESPONSE=true
           REQUIRE_EMAIL_VERIFICATION=false
           DISABLE_TELEMETRY=true
           # MCP E2E tests register mock servers on host.docker.internal (a private
@@ -904,7 +895,7 @@ jobs:
       - name: Start Docker containers (lite)
         run: |
           cd deployment/docker_compose
-          docker compose -f docker-compose.yml -f docker-compose.onyx-lite.yml -f docker-compose.dev.yml up -d --wait
+          docker compose -f docker-compose.yml -f docker-compose.onyx-lite.yml -f docker-compose.dev.yml -f docker-compose.mock-llm-test.yml up -d --wait
         id: start_docker
 
       - name: Seed dev license
@@ -929,7 +920,7 @@ jobs:
           WORKSPACE: ${{ github.workspace }}
         run: |
           cd deployment/docker_compose
-          docker compose logs > docker-compose.log
+          docker compose -f docker-compose.yml -f docker-compose.onyx-lite.yml -f docker-compose.dev.yml -f docker-compose.mock-llm-test.yml logs > docker-compose.log
           mv docker-compose.log ${WORKSPACE}/docker-compose.log
 
       - name: Upload logs

backend/tests/integration/README.md

@@ -120,6 +120,9 @@ def test_search_then_answer(admin_user: DATestUser, mock_llm: MockLLMScript) ->
 See `tests/llm_workflows/test_mock_llm_server.py`. Tests that must patch Onyx or control the stream very finely
 belong in `tests/external_dependency_unit/` with `MockLLM`.
 
+The Playwright stack runs the same server as a compose service
+(`deployment/docker_compose/docker-compose.mock-llm-test.yml`). See `web/tests/e2e/README.md`.
+
 ## Current Testing Limitations
 
 ### Test coverage

deployment/docker_compose/docker-compose.mock-llm-test.yml

@@ -0,0 +1,31 @@
+name: onyx
+
+services:
+  # Scripted OpenAI-compatible LLM for the Playwright stack. The backend reaches
+  # it at http://mock_llm_server:8095; the Playwright runner scripts it through
+  # the port published on host loopback (see web/tests/e2e/utils/mockLlm.ts).
+  mock_llm_server:
+    image: ${ONYX_BACKEND_IMAGE:-onyxdotapp/onyx-backend:latest}
+    restart: on-failure
+    working_dir: /workspace/backend
+    environment:
+      - PYTHONPATH=/workspace/backend
+      - MOCK_LLM_SERVER_HOST=0.0.0.0
+      - MOCK_LLM_SERVER_PORT=8095
+    command: python -m tests.integration.mock_services.mock_llm_server.server
+    ports:
+      - "127.0.0.1:8095:8095"
+    volumes:
+      - ../..:/workspace:ro
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "python",
+          "-c",
+          "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8095/health')",
+        ]
+      interval: 5s
+      timeout: 5s
+      retries: 12
+      start_period: 10s

web/tests/e2e/README.md

@@ -73,3 +73,44 @@ expect(selected).toBe(true);
 ```
 
 `getAttribute` / `evaluate` / `textContent` / `count` are still appropriate when you need the value for control flow inside the spec (e.g. branching on it, logging it). They are not appropriate as the basis of an assertion on async state.
+
+## 3. The LLM is the scripted mock LLM server
+
+Every LLM call of the Playwright deployment goes to the scripted mock LLM server
+(`backend/tests/integration/mock_services/mock_llm_server/server.py`). Global setup registers its `default` script and
+makes the public `openai_compatible` provider "PW Mock LLM" the default. The script's default reply answers every
+request that no conversation serves with `This is a mock LLM response.`
+
+To script a turn, use `tests/e2e/utils/mockLlm.ts`. Add a conversation to the default script that matches a nonce,
+and put the nonce in the chat message. Do not change the default provider to script a spec: all workers share it.
+
+```typescript
+const nonce = mockLlmNonce();
+await addMockLlmConversation({
+  name: nonce,
+  conditions: { prompt_contains: [nonce] },
+  replies: [
+    { tool_calls: [{ id: `call-${nonce}`, name: "tool_0", arguments: {} }] },
+    { text: "Done.", conditions: { has_results_for: [`call-${nonce}`] } },
+  ],
+});
+await sendMessage(page, `Run tool_0. ${nonce}`);
+```
+
+A tool-free request (no tools and no tool results), such as session naming, gets the default reply unless a
+conversation or reply `conditions` set `has_tools: false`. The web app names a new chat after the first answer without
+waiting for it, so its naming request can overlap the next turn. Set `has_tools: false` only on a reply that must
+answer a tool-free chat turn.
+
+To run the specs locally, start the server with the rest of the stack:
+
+```bash
+cd deployment/docker_compose
+docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.mock-llm-test.yml up -d
+```
+
+You can also run it on the host with
+`cd backend && MOCK_LLM_SERVER_PORT=8095 python -m tests.integration.mock_services.mock_llm_server.server`.
+The runner reaches the server at `MOCK_LLM_SERVER_URL` (default `http://localhost:8095`), and the backend reaches it
+at `MOCK_LLM_BACKEND_URL` (default `http://mock_llm_server:8095`). When the backend runs on the host, set
+`MOCK_LLM_BACKEND_URL=http://localhost:8095`. Global setup replaces the default LLM of the target deployment.

web/tests/e2e/chat/message_edit_regenerate.spec.ts

@@ -1,6 +1,10 @@
 import { test, expect } from "@playwright/test";
 import { loginAsRandomUser } from "@tests/e2e/utils/auth";
 import { sendMessage, switchModel } from "@tests/e2e/utils/chatActions";
+import {
+  MOCK_LLM_DEFAULT_MODEL,
+  MOCK_LLM_MODELS,
+} from "@tests/e2e/utils/mockLlm";
 
 test.describe("Message Edit and Regenerate Tests", () => {
   test.beforeEach(async ({ page }) => {
@@ -140,9 +144,9 @@ test.describe("Message Edit and Regenerate Tests", () => {
   });
 
   test("Message regeneration with model selection", async ({ page }) => {
-    // make sure we're using something other than GPT-4o Mini, otherwise the below
-    // will fail since we need to switch to a different model for the test
-    await switchModel(page, "GPT-4.1");
+    // make sure we're using something other than the default mock model, otherwise
+    // the below will fail since we need to switch to a different model for the test
+    await switchModel(page, MOCK_LLM_MODELS[1].displayName);
 
     // Send initial message
     await sendMessage(page, "hi! Respond with no more than a sentence");
@@ -160,15 +164,14 @@ test.describe("Message Edit and Regenerate Tests", () => {
     const regenerateButton = aiMessage.getByTestId("AgentMessage/regenerate");
     await regenerateButton.click();
 
-    // Wait for dropdown to appear and select GPT-4o Mini
+    // Wait for dropdown to appear and select the default mock model
     await page.waitForSelector('[role="dialog"]', { state: "visible" });
 
-    // Look for the GPT-4o Mini option in the dropdown
-    const gpt4oMiniOption = page
+    const defaultModelOption = page
       .locator('[role="dialog"]')
-      .getByText("GPT-4o Mini", { exact: true })
+      .getByText(MOCK_LLM_DEFAULT_MODEL.displayName, { exact: true })
       .first();
-    await gpt4oMiniOption.click();
+    await defaultModelOption.click();
 
     // Wait for regeneration to complete by waiting for feedback buttons to appear
     // The feedback buttons (copy, like, dislike, regenerate) appear when streaming is complete

web/tests/e2e/fixtures/llmProvider.ts

@@ -1,5 +1,5 @@
 /**
- * Playwright fixture that ensures a public LLM provider is available.
+ * Playwright fixture that ensures the public mock LLM provider is available.
  *
  * Usage:
  * ```ts
@@ -8,13 +8,14 @@
  *
  * test("my test that needs an LLM provider", async ({ page, llmProviderId }) => {
  *   // llmProviderId is the ID of the provider that was created (or null if
- *   // one already existed). The fixture handles cleanup automatically.
+ *   // it already existed). The fixture handles cleanup automatically.
  * });
  * ```
  *
  * The fixture:
  * - Authenticates as admin
- * - Creates a public LLM provider if none exists
+ * - Creates the public mock LLM provider if it does not exist, and makes it
+ *   the default
  * - Provides the created provider ID to the test
  * - Cleans up the provider after all tests in the file complete
  */
@@ -25,8 +26,8 @@ import { OnyxApiClient } from "@tests/e2e/utils/onyxApiClient";
 
 export const test = base.extend<{
   /**
-   * The ID of the public LLM provider created by this fixture, or `null`
-   * if a public provider already existed.
+   * The ID of the mock LLM provider created by this fixture, or `null`
+   * if it already existed.
    */
   llmProviderId: number | null;
 }>({

web/tests/e2e/global-setup.ts

@@ -6,6 +6,10 @@ import {
   workerUserCredentials,
 } from "@tests/e2e/constants";
 import { OnyxApiClient } from "@tests/e2e/utils/onyxApiClient";
+import {
+  putMockLlmDefaultScript,
+  waitForMockLlmServer,
+} from "@tests/e2e/utils/mockLlm";
 
 const PREFLIGHT_TIMEOUT_MS = 60_000;
 const PREFLIGHT_POLL_INTERVAL_MS = 2_000;
@@ -295,9 +299,11 @@ async function globalSetup(config: FullConfig) {
     await setDisplayName(baseURL, storageStatePath, "worker");
   }
 
-  // ── Ensure a public LLM provider exists ───────────────────────────
+  // ── Register the mock LLM script and make its provider the default ─
   // Many tests depend on a default LLM being configured (file uploads,
   // assistant creation, etc.).  Re-use the admin session we just saved.
+  await waitForMockLlmServer();
+  await putMockLlmDefaultScript();
   const adminCtx = await request.newContext({
     baseURL,
     storageState: "admin_auth.json",

web/tests/e2e/mcp/mcpToolInvocation.ts

@@ -9,28 +9,41 @@ import {
   getToolPacketCounts,
   sendMessageAndCaptureStreamPackets,
 } from "@tests/e2e/utils/chatStream";
+import { addMockLlmConversation, mockLlmNonce } from "@tests/e2e/utils/mockLlm";
 
 /**
  * Send a chat message that forces the given MCP tool to be called and return
- * the per-tool invocation packet counts (start / delta / debug).
+ * the per-tool invocation packet counts (start / delta / debug). The mock LLM
+ * calls the tool only when the request offers it.
  */
 export async function sendForcedMcpToolCall(
   page: Page,
   toolName: string,
   forcedToolId?: number | null
 ): Promise<{ start: number; delta: number; debug: number }> {
-  const argName = `playwright-${Date.now()}`;
+  const nonce = mockLlmNonce();
+  const callId = `call-${nonce}`;
+  await addMockLlmConversation({
+    name: `mcp-${nonce}`,
+    conditions: { prompt_contains: [nonce] },
+    replies: [
+      {
+        tool_calls: [
+          { id: callId, name: toolName, arguments: { name: nonce } },
+        ],
+        conditions: { offers: [toolName] },
+      },
+      { text: "Done.", conditions: { has_results_for: [callId] } },
+    ],
+  });
+
   const prompt = [
     `Call the MCP tool "${toolName}" now.`,
-    `Pass {"name":"${argName}"} as the arguments.`,
+    `Pass {"name":"${nonce}"} as the arguments.`,
     "Return the exact tool output.",
   ].join(" ");
 
   const packets = await sendMessageAndCaptureStreamPackets(page, prompt, {
-    mockLlmResponse: JSON.stringify({
-      name: toolName,
-      arguments: { name: argName },
-    }),
     payloadOverrides:
       forcedToolId != null
         ? { forced_tool_id: forcedToolId, forced_tool_ids: [forcedToolId] }

web/tests/e2e/utils/chatStream.ts

@@ -78,29 +78,22 @@ export async function sendMessageAndCaptureStreamPackets(
   page: Page,
   message: string,
   options?: {
-    mockLlmResponse?: string;
     payloadOverrides?: Record<string, unknown>;
     waitForAiMessage?: boolean;
   }
 ): Promise<ChatStreamPacket[]> {
   const requestUrlPattern = "**/api/chat/send-chat-message";
-  const mockLlmResponse = options?.mockLlmResponse;
   const payloadOverrides = options?.payloadOverrides;
   const waitForAiMessage = options?.waitForAiMessage ?? true;
   const routeHandler = async (route: Route) => {
-    if (!mockLlmResponse && !payloadOverrides) {
+    if (!payloadOverrides) {
       await route.continue();
       return;
     }
 
     const request = route.request();
     const payload = request.postDataJSON() as Record<string, unknown>;
-    if (payloadOverrides) {
-      Object.assign(payload, payloadOverrides);
-    }
-    if (mockLlmResponse) {
-      payload.mock_llm_response = mockLlmResponse;
-    }
+    Object.assign(payload, payloadOverrides);
 
     await route.continue({
       postData: JSON.stringify(payload),

web/tests/e2e/utils/mockLlm.ts

@@ -0,0 +1,172 @@
+/**
+ * Client for the scripted mock LLM server
+ * (`backend/tests/integration/mock_services/mock_llm_server/server.py`),
+ * which answers every LLM call of the Playwright deployment.
+ *
+ * Global setup registers the `default` script. Its default reply answers
+ * every request that no conversation serves, including all tool-free
+ * secondary calls (session naming, query expansion, filters). To script a
+ * turn, add a conversation to the default script that matches a nonce, and put
+ * that nonce in the chat message:
+ *
+ * ```ts
+ * const nonce = mockLlmNonce();
+ * await addMockLlmConversation({
+ *   name: nonce,
+ *   conditions: { prompt_contains: [nonce] },
+ *   replies: [{ tool_calls: [{ id: `call-${nonce}`, name: "tool_0" }] }],
+ * });
+ * await sendMessage(page, `Run the tool. ${nonce}`);
+ * ```
+ *
+ * The nonce keeps the conversation out of other workers' chat turns, so specs never
+ * change the shared default provider.
+ */
+
+import { randomUUID } from "crypto";
+
+/** Where the Playwright runner reaches the mock LLM server. */
+export const MOCK_LLM_SERVER_URL =
+  process.env.MOCK_LLM_SERVER_URL || "http://localhost:8095";
+/** Where the Onyx backend reaches the mock LLM server. */
+export const MOCK_LLM_BACKEND_URL =
+  process.env.MOCK_LLM_BACKEND_URL || "http://mock_llm_server:8095";
+
+export const MOCK_LLM_DEFAULT_SCRIPT_ID = "default";
+export const MOCK_LLM_DEFAULT_RESPONSE = "This is a mock LLM response.";
+export const MOCK_LLM_PROVIDER_NAME = "PW Mock LLM";
+export const MOCK_LLM_API_KEY = "sk-mock-llm-server";
+// Deep Research needs at least 50000 input tokens.
+export const MOCK_LLM_MAX_INPUT_TOKENS = 200000;
+// Neutral names: Onyx special-cases catalog names and names that contain
+// claude, qwen, glm, or gpt-5.x.
+export const MOCK_LLM_MODELS = [
+  { name: "mock-model", displayName: "Mock Model" },
+  { name: "mock-model-alt", displayName: "Mock Alt Model" },
+] as const;
+export const MOCK_LLM_DEFAULT_MODEL = MOCK_LLM_MODELS[0];
+
+const HEALTH_TIMEOUT_MS = 30_000;
+const HEALTH_POLL_INTERVAL_MS = 1_000;
+
+export interface MockLlmToolCall {
+  id: string;
+  name: string;
+  arguments?: Record<string, unknown>;
+}
+
+/**
+ * Every set field must hold. A tool-free request (no tools and no tool
+ * results, for example session naming) goes only to a reply whose
+ * conversation or reply `conditions` set `has_tools: false`; otherwise it gets the
+ * default reply.
+ */
+export interface MockLlmRequestConditions {
+  has_tools?: boolean;
+  offers?: string[];
+  does_not_offer?: string[];
+  has_results_for?: string[];
+  tool_choice?: "auto" | "required" | "none";
+  prompt_contains?: string[];
+}
+
+export interface MockLlmReply {
+  reasoning?: string;
+  text?: string;
+  tool_calls?: MockLlmToolCall[];
+  conditions?: MockLlmRequestConditions;
+  required?: boolean;
+}
+
+export interface MockLlmConversation {
+  name: string;
+  conditions?: MockLlmRequestConditions;
+  replies: MockLlmReply[];
+}
+
+export interface MockLlmScript {
+  conversations?: MockLlmConversation[];
+  default_reply?: MockLlmReply | null;
+}
+
+/** The `api_base` of a provider that the given script answers. */
+export function mockLlmApiBase(
+  scriptId: string = MOCK_LLM_DEFAULT_SCRIPT_ID
+): string {
+  return `${MOCK_LLM_BACKEND_URL}/scripts/${scriptId}/v1`;
+}
+
+/** A unique token to put in a chat message and match with `prompt_contains`. */
+export function mockLlmNonce(): string {
+  return `pw-${randomUUID()}`;
+}
+
+/** Wait until the mock LLM server answers its health check. */
+export async function waitForMockLlmServer(): Promise<void> {
+  const deadline = Date.now() + HEALTH_TIMEOUT_MS;
+  while (Date.now() < deadline) {
+    try {
+      const response = await fetch(`${MOCK_LLM_SERVER_URL}/health`);
+      if (response.ok) {
+        return;
+      }
+    } catch {
+      // Not up yet.
+    }
+    await new Promise((resolve) =>
+      setTimeout(resolve, HEALTH_POLL_INTERVAL_MS)
+    );
+  }
+  throw new Error(
+    `The mock LLM server is not reachable at ${MOCK_LLM_SERVER_URL}. Start it ` +
+      "with the docker-compose.mock-llm-test.yml overlay (see web/tests/e2e/README.md)."
+  );
+}
+
+async function sendControlRequest(
+  method: "PUT" | "POST",
+  path: string,
+  body: MockLlmScript | MockLlmConversation,
+  description: string
+): Promise<void> {
+  const response = await fetch(`${MOCK_LLM_SERVER_URL}${path}`, {
+    method,
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify(body),
+  });
+  if (!response.ok) {
+    throw new Error(
+      `Failed to ${description}: ${response.status} ${await response.text()}`
+    );
+  }
+}
+
+/**
+ * Create or replace the default script, with no conversations and a default
+ * reply that answers every other request. Replacing it drops the
+ * conversations of running specs, so only global setup calls this.
+ */
+export async function putMockLlmDefaultScript(): Promise<void> {
+  await sendControlRequest(
+    "PUT",
+    `/scripts/${MOCK_LLM_DEFAULT_SCRIPT_ID}`,
+    { conversations: [], default_reply: { text: MOCK_LLM_DEFAULT_RESPONSE } },
+    "register the default mock LLM script"
+  );
+}
+
+/**
+ * Add a conversation to a script. Conversations are served before the
+ * script's default reply.
+ */
+export async function addMockLlmConversation(
+  conversation: MockLlmConversation,
+  scriptId: string = MOCK_LLM_DEFAULT_SCRIPT_ID
+): Promise<void> {
+  await sendControlRequest(
+    "POST",
+    `/scripts/${scriptId}/conversations`,
+    conversation,
+    `add mock LLM conversation ${conversation.name}`
+  );
+}

web/tests/e2e/utils/onyxApiClient.ts

@@ -1,4 +1,12 @@
 import { APIRequestContext, expect, APIResponse } from "@playwright/test";
+import {
+  MOCK_LLM_API_KEY,
+  MOCK_LLM_DEFAULT_MODEL,
+  MOCK_LLM_MAX_INPUT_TOKENS,
+  MOCK_LLM_MODELS,
+  MOCK_LLM_PROVIDER_NAME,
+  mockLlmApiBase,
+} from "@tests/e2e/utils/mockLlm";
 
 const E2E_LLM_PROVIDER_API_KEY =
   process.env.E2E_LLM_PROVIDER_API_KEY ||
@@ -76,7 +84,7 @@ export interface CreateAgentOptions {
  *
  * **LLM Providers:**
  * - `listLlmProviders()` - Lists LLM providers (admin endpoint, includes is_public)
- * - `ensurePublicProvider(name?)` - Idempotently creates a public default LLM provider
+ * - `ensurePublicProvider()` - Idempotently makes the public mock LLM provider the default
  * - `createRestrictedProvider(name, groupId)` - Creates a restricted LLM provider assigned to a group
  * - `setProviderAsDefault(id)` - Sets an LLM provider as the default for chat
  * - `deleteProvider(id)` - Deletes an LLM provider
@@ -657,66 +665,75 @@ export class OnyxApiClient {
   /**
    * Lists LLM providers visible to the admin (includes `is_public`).
    *
-   * @returns Array of LLM providers with id and is_public fields
+   * @returns Array of LLM providers with id, name, and is_public fields
    */
   async listLlmProviders(): Promise<
     Array<{
       id: number;
+      name: string;
       is_public?: boolean;
     }>
   > {
     const response = await this.get("/admin/llm/provider");
     const data = await this.handleResponse<{
-      providers: Array<{ id: number; is_public?: boolean }>;
+      providers: Array<{ id: number; name: string; is_public?: boolean }>;
     }>(response, "Failed to list LLM providers");
     return data.providers;
   }
 
   /**
-   * Ensure at least one public LLM provider exists and is set as default.
+   * Ensure the public mock LLM provider exists, points at the mock LLM
+   * server's default script, and is the default for chat.
    *
-   * Idempotent — returns `null` if a public provider already exists,
+   * Idempotent — returns `null` if the provider already existed,
    * or the new provider ID if one was created.
    *
-   * @param providerName - Name for the provider (default: "PW Default Provider")
    * @returns The provider ID if one was created, or `null` if already present
    */
-  async ensurePublicProvider(
-    providerName: string = "PW Default Provider"
-  ): Promise<number | null> {
+  async ensurePublicProvider(): Promise<number | null> {
     const providers = await this.listLlmProviders();
-    const hasPublic = providers.some((p) => p.is_public);
+    const existing = providers.find((p) => p.name === MOCK_LLM_PROVIDER_NAME);
 
-    if (hasPublic) {
-      return null;
-    }
-
-    const defaultModelName = "gpt-4o";
     const response = await this.request.put(
-      `${this.baseUrl}/admin/llm/provider?is_creation=true`,
+      `${this.baseUrl}/admin/llm/provider${existing ? "" : "?is_creation=true"}`,
       {
         data: {
-          name: providerName,
-          provider: "openai",
-          api_key: E2E_LLM_PROVIDER_API_KEY,
+          id: existing?.id,
+          name: MOCK_LLM_PROVIDER_NAME,
+          provider: "openai_compatible",
+          api_key: MOCK_LLM_API_KEY,
+          api_key_changed: true,
+          api_base: mockLlmApiBase(),
           is_public: true,
           groups: [],
           personas: [],
-          model_configurations: [{ name: defaultModelName, is_visible: true }],
+          model_configurations: MOCK_LLM_MODELS.map((model) => ({
+            name: model.name,
+            custom_display_name: model.displayName,
+            is_visible: true,
+            max_input_tokens: MOCK_LLM_MAX_INPUT_TOKENS,
+            supports_image_input: true,
+          })),
         },
       }
     );
 
     const responseData = await this.handleResponse<{ id: number }>(
       response,
-      "Failed to create public provider"
+      "Failed to upsert the mock LLM provider"
     );
 
     // Set as default so get_default_llm() works (needed for tokenization, etc.)
-    await this.setProviderAsDefault(responseData.id, defaultModelName);
+    await this.setProviderAsDefault(
+      responseData.id,
+      MOCK_LLM_DEFAULT_MODEL.name
+    );
 
+    if (existing) {
+      return null;
+    }
     this.log(
-      `Created public LLM provider: ${providerName} (ID: ${responseData.id})`
+      `Created mock LLM provider: ${MOCK_LLM_PROVIDER_NAME} (ID: ${responseData.id})`
     );
     return responseData.id;
   }