From 1333ed68c6a9532611fec28568ee5db0c3394099 Mon Sep 17 00:00:00 2001 From: "dayuan.jiang" Date: Mon, 12 Oct 2026 09:11:25 +0900 Subject: [PATCH] test: shared e2e helpers, vi.stubEnv, drop placeholder specs - tests/e2e/lib: openApp(page, { path, config }), MODEL_CONFIGS_KEY, savedModelConfigs, settingsDialog, chunkedReplies, sse() and toolCallEvents() replace about 70 copies of goto + wait for iframe, three local openApp functions, 14 window.fetch overrides and the Node-side SSE serialisers; unused fixtures removed - delete theme, save, history, model-config, upload and copy-paste specs (the copy-button test moves to chat.spec.ts); trim iframe-interaction, history-restore, keyboard and smoke to tests that can fail; file-upload asserts on the preview image - settings-review-fixes.spec.ts split by topic into settings-bedrock, settings-tabs-sync and settings-providers - unit tests use vi.stubEnv with unstubEnvs: true instead of hand-written process.env save and restore loops --- .../tests/custom-instructions.test.ts | 16 +- packages/mcp-server/tests/persistence.test.ts | 16 +- packages/mcp-server/vitest.config.ts | 2 + tests/e2e/chat-errors.spec.ts | 5 +- tests/e2e/chat.spec.ts | 112 ++- tests/e2e/copy-paste.spec.ts | 137 --- tests/e2e/diagram-content.spec.ts | 323 +++---- tests/e2e/diagram-generation.spec.ts | 16 +- tests/e2e/error-handling.spec.ts | 28 +- tests/e2e/file-upload.spec.ts | 87 +- tests/e2e/fixtures/diagrams.ts | 5 - tests/e2e/history-restore.spec.ts | 199 +---- tests/e2e/history.spec.ts | 18 - tests/e2e/iframe-interaction.spec.ts | 88 +- tests/e2e/keyboard.spec.ts | 13 +- tests/e2e/language.spec.ts | 22 +- tests/e2e/lib/fixtures.ts | 193 ++-- tests/e2e/lib/helpers.ts | 88 +- tests/e2e/model-config.spec.ts | 19 - tests/e2e/model-test.spec.ts | 20 +- tests/e2e/multi-turn.spec.ts | 16 +- tests/e2e/provider-models.spec.ts | 93 +- tests/e2e/save.spec.ts | 16 - tests/e2e/settings-bedrock.spec.ts | 341 +++++++ tests/e2e/settings-providers.spec.ts | 507 +++++++---- tests/e2e/settings-review-fixes.spec.ts | 830 ------------------ tests/e2e/settings-tabs-sync.spec.ts | 292 ++++++ tests/e2e/settings.spec.ts | 5 +- tests/e2e/smoke.spec.ts | 9 +- tests/e2e/theme.spec.ts | 88 -- tests/e2e/upload.spec.ts | 20 - tests/e2e/workspace.spec.ts | 487 +++++----- tests/unit/admin-providers-route.test.ts | 14 +- tests/unit/admin-providers.test.ts | 73 +- tests/unit/admin-settings.test.ts | 16 +- tests/unit/admin-test-model.test.ts | 36 +- tests/unit/ai-providers-cache.test.ts | 11 +- tests/unit/ai-providers-credentials.test.ts | 113 +-- tests/unit/ai-providers-google.test.ts | 37 +- tests/unit/ai-providers.test.ts | 72 +- tests/unit/api-access-code.test.ts | 8 +- tests/unit/bedrock-api-key.test.ts | 1 - tests/unit/chat-route-edgeone.test.ts | 59 +- tests/unit/chat-route-errors.test.ts | 61 +- tests/unit/chat-route-quota.test.ts | 54 +- tests/unit/client-ip.test.ts | 15 +- tests/unit/cross-site-requests.test.ts | 8 +- tests/unit/output-token-limit.test.ts | 46 +- tests/unit/provider-models.test.ts | 51 +- tests/unit/server-model-config.test.ts | 59 +- tests/unit/ssrf-protection.test.ts | 7 +- tests/unit/validate-diagram-route.test.ts | 3 +- tests/unit/validate-model-route.test.ts | 156 ++-- vitest.config.mts | 2 + 54 files changed, 2025 insertions(+), 2988 deletions(-) delete mode 100644 tests/e2e/copy-paste.spec.ts delete mode 100644 tests/e2e/history.spec.ts delete mode 100644 tests/e2e/model-config.spec.ts delete mode 100644 tests/e2e/save.spec.ts create mode 100644 tests/e2e/settings-bedrock.spec.ts delete mode 100644 tests/e2e/settings-review-fixes.spec.ts create mode 100644 tests/e2e/settings-tabs-sync.spec.ts delete mode 100644 tests/e2e/theme.spec.ts delete mode 100644 tests/e2e/upload.spec.ts diff --git a/packages/mcp-server/tests/custom-instructions.test.ts b/packages/mcp-server/tests/custom-instructions.test.ts index a7b7bc4b..8d493cc8 100644 --- a/packages/mcp-server/tests/custom-instructions.test.ts +++ b/packages/mcp-server/tests/custom-instructions.test.ts @@ -6,7 +6,7 @@ import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs" import { homedir, tmpdir } from "node:os" import { join } from "node:path" -import { afterEach, describe, expect, it } from "vitest" +import { describe, expect, it, vi } from "vitest" import { CUSTOM_INSTRUCTIONS_FILE, customInstructionsDir, @@ -62,16 +62,10 @@ describe("readCustomInstructions", () => { }) describe("customInstructionsDir", () => { - const original = process.env.DRAWIO_DATA_DIR - afterEach(() => { - if (original === undefined) delete process.env.DRAWIO_DATA_DIR - else process.env.DRAWIO_DATA_DIR = original - }) - it("honours DRAWIO_DATA_DIR", () => { const dir = tempDir() writeRules(dir, "Use blue fill #dae8fc.") - process.env.DRAWIO_DATA_DIR = dir + vi.stubEnv("DRAWIO_DATA_DIR", dir) expect(customInstructionsDir()).toBe(dir) expect(guideWithCustomInstructions(GUIDE)).toBe( `${GUIDE}\n\n## Custom Instructions\nUse blue fill #dae8fc.`, @@ -79,15 +73,15 @@ describe("customInstructionsDir", () => { }) it("expands ~, which JSON configs pass on as it is", () => { - process.env.DRAWIO_DATA_DIR = "~/drawio-saves" + vi.stubEnv("DRAWIO_DATA_DIR", "~/drawio-saves") expect(customInstructionsDir()).toBe(join(homedir(), "drawio-saves")) }) it("falls back to the home folder when DRAWIO_DATA_DIR is off or unset", () => { const home = join(homedir(), ".next-ai-drawio") - process.env.DRAWIO_DATA_DIR = "off" + vi.stubEnv("DRAWIO_DATA_DIR", "off") expect(customInstructionsDir()).toBe(home) - delete process.env.DRAWIO_DATA_DIR + vi.stubEnv("DRAWIO_DATA_DIR", undefined) expect(customInstructionsDir()).toBe(home) }) }) diff --git a/packages/mcp-server/tests/persistence.test.ts b/packages/mcp-server/tests/persistence.test.ts index 6a86664e..697f22d2 100644 --- a/packages/mcp-server/tests/persistence.test.ts +++ b/packages/mcp-server/tests/persistence.test.ts @@ -15,7 +15,7 @@ import { } from "node:fs" import { homedir, tmpdir } from "node:os" import { join } from "node:path" -import { afterEach, describe, expect, it } from "vitest" +import { describe, expect, it, vi } from "vitest" import { Autosaver, defaultDataDir } from "../src/persistence.ts" const DIAGRAM = `` @@ -360,23 +360,17 @@ describe("Autosaver History", () => { }) describe("defaultDataDir", () => { - const original = process.env.DRAWIO_DATA_DIR - afterEach(() => { - if (original === undefined) delete process.env.DRAWIO_DATA_DIR - else process.env.DRAWIO_DATA_DIR = original - }) - it("reads DRAWIO_DATA_DIR, where off disables saving", () => { - process.env.DRAWIO_DATA_DIR = "off" + vi.stubEnv("DRAWIO_DATA_DIR", "off") expect(defaultDataDir()).toBeNull() - process.env.DRAWIO_DATA_DIR = "/tmp/x" + vi.stubEnv("DRAWIO_DATA_DIR", "/tmp/x") expect(defaultDataDir()).toBe("/tmp/x") - delete process.env.DRAWIO_DATA_DIR + vi.stubEnv("DRAWIO_DATA_DIR", undefined) expect(defaultDataDir()).toMatch(/\.next-ai-drawio$/) }) it("expands ~, which JSON configs pass on as it is", () => { - process.env.DRAWIO_DATA_DIR = "~/drawio-saves" + vi.stubEnv("DRAWIO_DATA_DIR", "~/drawio-saves") expect(defaultDataDir()).toBe(join(homedir(), "drawio-saves")) }) }) diff --git a/packages/mcp-server/vitest.config.ts b/packages/mcp-server/vitest.config.ts index a5a49d3f..e690f7c5 100644 --- a/packages/mcp-server/vitest.config.ts +++ b/packages/mcp-server/vitest.config.ts @@ -4,6 +4,8 @@ export default defineConfig({ test: { include: ["tests/**/*.test.ts"], environment: "node", + // vi.stubEnv values are undone after each test + unstubEnvs: true, setupFiles: ["tests/setup.ts"], // The package source uses Node16 module resolution with explicit .js // extensions in imports. Vitest+esbuild handles the .ts→.js mapping diff --git a/tests/e2e/chat-errors.spec.ts b/tests/e2e/chat-errors.spec.ts index 7d0c7176..c6bbb8ee 100644 --- a/tests/e2e/chat-errors.spec.ts +++ b/tests/e2e/chat-errors.spec.ts @@ -1,5 +1,5 @@ import { expect, type Page, test } from "@playwright/test" -import { getIframe, sendMessage } from "./lib/fixtures" +import { openApp, sendMessage } from "./lib/fixtures" /** Answer the chat request with the given response and send a message */ async function chatWith( @@ -7,8 +7,7 @@ async function chatWith( response: { status: number; contentType: string; body: string }, ) { await page.route("**/api/chat", (route) => route.fulfill(response)) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) await sendMessage(page, "Draw a box") } diff --git a/tests/e2e/chat.spec.ts b/tests/e2e/chat.spec.ts index 61726309..b6b0e92d 100644 --- a/tests/e2e/chat.spec.ts +++ b/tests/e2e/chat.spec.ts @@ -1,73 +1,38 @@ import { + chunkedReplies, expect, getChatInput, - getIframe, + openApp, sendMessage, test, } from "./lib/fixtures" -import { createTextOnlyResponse } from "./lib/helpers" - -test.describe("Chat Panel", () => { - test.beforeEach(async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - }) - - test("page has interactive elements", async ({ page }) => { - const buttons = page.locator("button") - const count = await buttons.count() - expect(count).toBeGreaterThan(0) - }) - - test("draw.io iframe is interactive", async ({ page }) => { - const iframe = getIframe(page) - await expect(iframe).toBeVisible() - - const src = await iframe.getAttribute("src") - expect(src).toBeTruthy() - }) -}) +import { createTextOnlyResponse, sse } from "./lib/helpers" test.describe("Crossing the mobile breakpoint", () => { test.beforeEach(async ({ page }) => { // A text answer that arrives in parts over a few seconds - await page.addInitScript(() => { - const realFetch = window.fetch - window.fetch = async (input, init) => { - const url = input instanceof Request ? input.url : String(input) - if (!url.endsWith("/api/chat")) return realFetch(input, init) - const events = [ - { type: "start" }, - { type: "text-start", id: "t" }, - { type: "text-delta", id: "t", delta: "Once upon" }, - { type: "text-delta", id: "t", delta: " a time." }, - { type: "text-end", id: "t" }, - { type: "finish" }, - ] - const body = new ReadableStream({ - async start(controller) { - for (const event of events) { - controller.enqueue( - new TextEncoder().encode( - `data: ${JSON.stringify(event)}\n\n`, - ), - ) - await new Promise((r) => setTimeout(r, 1500)) - } - controller.enqueue( - new TextEncoder().encode("data: [DONE]\n\n"), - ) - controller.close() - }, - }) - return new Response(body, { - headers: { "content-type": "text/event-stream" }, - }) - } - }) + await chunkedReplies( + page, + [ + [ + sse([{ type: "start" }], false), + sse([{ type: "text-start", id: "t" }], false), + sse( + [{ type: "text-delta", id: "t", delta: "Once upon" }], + false, + ), + sse( + [{ type: "text-delta", id: "t", delta: " a time." }], + false, + ), + sse([{ type: "text-end", id: "t" }], false), + sse([{ type: "finish" }]), + ], + ], + { gapMs: 1500 }, + ) await page.setViewportSize({ width: 1280, height: 800 }) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) }) test("keeps the chat and its streaming answer", async ({ page }) => { @@ -115,8 +80,7 @@ test.describe("Sending", () => { body: createTextOnlyResponse("Hello there."), }) }) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) const input = getChatInput(page) await input.fill("Hi") // The second press comes while the diagram is being exported @@ -130,3 +94,29 @@ test.describe("Sending", () => { await expect(page.getByText("Hello there.")).toHaveCount(1) }) }) + +test.describe("Answers", () => { + test("copy button copies response text", async ({ page }) => { + await page.route("**/api/chat", async (route) => { + await route.fulfill({ + status: 200, + contentType: "text/event-stream", + body: createTextOnlyResponse( + "Here is your diagram with a test box.", + ), + }) + }) + await openApp(page) + await sendMessage(page, "Create a test box") + await expect( + page.getByText("Here is your diagram with a test box."), + ).toBeVisible({ timeout: 15000 }) + + // The user's message has one too: the answer's is the last + await page.getByRole("button", { name: "Copy response" }).last().click() + // The button shows a check mark for a moment + await expect(page.getByRole("button", { name: "Copied!" })).toBeVisible( + { timeout: 3000 }, + ) + }) +}) diff --git a/tests/e2e/copy-paste.spec.ts b/tests/e2e/copy-paste.spec.ts deleted file mode 100644 index bfe8124d..00000000 --- a/tests/e2e/copy-paste.spec.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { SINGLE_BOX_XML } from "./fixtures/diagrams" -import { - expect, - getChatInput, - getIframe, - sendMessage, - test, -} from "./lib/fixtures" -import { createMockSSEResponse } from "./lib/helpers" - -test.describe("Copy/Paste Functionality", () => { - test("can paste text into chat input", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - const chatInput = getChatInput(page) - await expect(chatInput).toBeVisible({ timeout: 10000 }) - - await chatInput.focus() - await page.keyboard.insertText("Create a flowchart diagram") - - await expect(chatInput).toHaveValue("Create a flowchart diagram") - }) - - test("can paste multiline text into chat input", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - const chatInput = getChatInput(page) - await expect(chatInput).toBeVisible({ timeout: 10000 }) - - await chatInput.focus() - const multilineText = "Line 1\nLine 2\nLine 3" - await page.keyboard.insertText(multilineText) - - await expect(chatInput).toHaveValue(multilineText) - }) - - test("copy button copies response text", async ({ page }) => { - await page.route("**/api/chat", async (route) => { - await route.fulfill({ - status: 200, - contentType: "text/event-stream", - body: createMockSSEResponse( - SINGLE_BOX_XML, - "Here is your diagram with a test box.", - ), - }) - }) - - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - await sendMessage(page, "Create a test box") - - // Wait for response - await expect( - page.locator('text="Here is your diagram with a test box."'), - ).toBeVisible({ timeout: 15000 }) - - // Find copy button in message - const copyButton = page.locator( - '[data-testid="copy-button"], button[aria-label*="Copy"], button:has(svg.lucide-copy), button:has(svg.lucide-clipboard)', - ) - - // Copy button feature may not exist - skip if not available - const buttonCount = await copyButton.count() - if (buttonCount === 0) { - test.skip() - return - } - - await copyButton.first().click() - await expect( - page.locator('text="Copied"').or(page.locator("svg.lucide-check")), - ).toBeVisible({ timeout: 3000 }) - }) - - test("keyboard shortcuts work in chat input", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - const chatInput = getChatInput(page) - await expect(chatInput).toBeVisible({ timeout: 10000 }) - - await chatInput.fill("Hello world") - await chatInput.press("ControlOrMeta+a") - await chatInput.fill("New text") - - await expect(chatInput).toHaveValue("New text") - }) - - test("can undo/redo in chat input", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - const chatInput = getChatInput(page) - await expect(chatInput).toBeVisible({ timeout: 10000 }) - - await chatInput.fill("First text") - await chatInput.press("Tab") - - await chatInput.focus() - await chatInput.fill("Second text") - await chatInput.press("ControlOrMeta+z") - - // Verify page is still functional after undo - await expect(chatInput).toBeVisible() - }) - - test("chat input handles special characters", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - const chatInput = getChatInput(page) - await expect(chatInput).toBeVisible({ timeout: 10000 }) - - const specialText = "Test <>&\"' special chars 日本語 中文 🎉" - await chatInput.fill(specialText) - - await expect(chatInput).toHaveValue(specialText) - }) - - test("long text in chat input scrolls", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - const chatInput = getChatInput(page) - await expect(chatInput).toBeVisible({ timeout: 10000 }) - - const longText = "This is a very long text. ".repeat(50) - await chatInput.fill(longText) - - const value = await chatInput.inputValue() - expect(value.length).toBeGreaterThan(500) - }) -}) diff --git a/tests/e2e/diagram-content.spec.ts b/tests/e2e/diagram-content.spec.ts index c097e7f8..7167cbb3 100644 --- a/tests/e2e/diagram-content.spec.ts +++ b/tests/e2e/diagram-content.spec.ts @@ -1,11 +1,13 @@ import zlib from "node:zlib" import { expect, type Page, test } from "@playwright/test" import { - getIframe, + chunkedReplies as mockChunkedReplies, + openApp, openDrawioFile, sendMessage, waitForCompleteCount, } from "./lib/fixtures" +import { sse, toolCallEvents } from "./lib/helpers" /** * Checks what draw.io actually shows after the diagram tools, not only the @@ -13,20 +15,15 @@ import { * the browser tool handler (not the server) completes the tool call. */ function streamedToolCall(toolName: string, input: unknown) { - const toolCallId = `call_${Math.random().toString(36).slice(2)}` - const chunks = JSON.stringify(input).match(/[\s\S]{1,40}/g) ?? [] - const events = [ - { type: "start", messageId: `msg_${toolCallId}` }, - { type: "tool-input-start", toolCallId, toolName }, - ...chunks.map((inputTextDelta) => ({ - type: "tool-input-delta", - toolCallId, - inputTextDelta, - })), - { type: "tool-input-available", toolCallId, toolName, input }, + const id = `call_${Math.random().toString(36).slice(2)}` + const call = toolCallEvents(id, toolName, input) + return sse([ + { type: "start", messageId: `msg_${id}` }, + call.start, + ...call.deltas, + call.done, { type: "finish" }, - ] - return `${events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join("")}data: [DONE]\n\n` + ]) } const END_TURN = @@ -41,8 +38,7 @@ async function mockReplies(p: Page, replies: string[]) { body: replies.shift() ?? END_TURN, }) }) - await p.goto("/", { waitUntil: "networkidle" }) - await getIframe(p).waitFor({ state: "visible", timeout: 30000 }) + await openApp(p) return p.frameLocator("iframe") } @@ -60,65 +56,18 @@ const FAILED_CHECK = { suggestions: ["Move them apart"], } -const sse = (events: object[]) => - events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join("") const EDIT_GAMMA = { operations: [ { operation: "add", cell_id: "c", new_xml: cell("c", "Gamma", 400) }, ], } -const editStart = (id: string) => ({ - type: "tool-input-start", - toolCallId: id, - toolName: "edit_diagram", -}) -const editDeltas = (id: string) => - (JSON.stringify(EDIT_GAMMA).match(/[\s\S]{1,40}/g) ?? []).map((d) => ({ - type: "tool-input-delta", - toolCallId: id, - inputTextDelta: d, - })) +/** A streamed edit_diagram call that adds Gamma */ +const editGamma = (id: string) => toolCallEvents(id, "edit_diagram", EDIT_GAMMA) -/** - * Answer each chat request with the next reply. Each string in a reply is - * one network chunk, sent 300 ms apart, so the throttled UI renders between - * chunks like with a real model. A number in a reply sets the wait before - * the next chunk instead. - */ +/** Streams the replies in chunks, opens the app and returns draw.io's frame */ async function chunkedReplies(p: Page, replies: (string | number)[][]) { - await p.addInitScript((replies) => { - const realFetch = window.fetch - let n = 0 - window.fetch = async (input, init) => { - // Next.js passes URL objects for its own requests - const url = input instanceof Request ? input.url : String(input) - if (!url.endsWith("/api/chat")) return realFetch(input, init) - const chunks = replies[n++] ?? [ - 'data: {"type":"start"}\n\ndata: {"type":"finish"}\n\ndata: [DONE]\n\n', - ] - const body = new ReadableStream({ - async start(controller) { - for (const [i, chunk] of chunks.entries()) { - if (typeof chunk === "number") continue - controller.enqueue(new TextEncoder().encode(chunk)) - const next = chunks[i + 1] - await new Promise((r) => - setTimeout( - r, - typeof next === "number" ? next : 300, - ), - ) - } - controller.close() - }, - }) - return new Response(body, { - headers: { "content-type": "text/event-stream" }, - }) - } - }, replies) - await p.goto("/", { waitUntil: "networkidle" }) - await getIframe(p).waitFor({ state: "visible", timeout: 30000 }) + await mockChunkedReplies(p, replies) + await openApp(p) return p.frameLocator("iframe") } @@ -260,8 +209,7 @@ test("a built-in example draws its diagram", async ({ page: p }) => { requests++ return route.fulfill({ status: 500, body: "{}" }) }) - await p.goto("/", { waitUntil: "networkidle" }) - await getIframe(p).waitFor({ state: "visible", timeout: 30000 }) + await openApp(p) await sendMessage( p, "Give me a **animated connector** diagram of transformer's architecture", @@ -288,11 +236,10 @@ test("the thinking header is in the page language", async ({ page: p }) => { route.fulfill({ status: 200, contentType: "text/event-stream", - body: `${events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join("")}data: [DONE]\n\n`, + body: sse(events), }), ) - await p.goto("/zh", { waitUntil: "networkidle" }) - await getIframe(p).waitFor({ state: "visible", timeout: 30000 }) + await openApp(p, { path: "/zh" }) await sendMessage(p, "画两个框") await expect(p.getByText("Plan the boxes")).toBeAttached({ timeout: 15000, @@ -305,13 +252,14 @@ test("blank text before a tool call shows no empty bubble", async ({ page: p, }) => { // Kimi K2.6 sends a lone space before calling the tool - const blankText = [ - { type: "text-start", id: "t1" }, - { type: "text-delta", id: "t1", delta: " " }, - { type: "text-end", id: "t1" }, - ] - .map((e) => `data: ${JSON.stringify(e)}\n\n`) - .join("") + const blankText = sse( + [ + { type: "text-start", id: "t1" }, + { type: "text-delta", id: "t1", delta: " " }, + { type: "text-end", id: "t1" }, + ], + false, + ) const reply = streamedToolCall("display_diagram", { xml: cell("2", "Alpha", 40), }).replace( @@ -331,46 +279,45 @@ test("blank text before a tool call shows no empty bubble", async ({ test("an edit right after a broken edit call starts from the real diagram", async ({ page: p, }) => { + const e1 = editGamma("e1") + const e2 = editGamma("e2") // Seen with Claude Opus 5.5: the first edit call had invalid JSON, the // server rejected it, and the model sent the same edit again at once. // The second edit must not see the first one's streamed preview. const replies = [ [streamedToolCall("display_diagram", { xml: cell("a", "Alpha", 40) })], [ - sse([ - { type: "start" }, - { type: "start-step" }, - editStart("e1"), - ...editDeltas("e1"), - ]), - sse([ - { - type: "tool-input-error", - toolCallId: "e1", - toolName: "edit_diagram", - input: "{broken", - errorText: "JSON parsing failed", - }, - { - type: "tool-output-error", - toolCallId: "e1", - errorText: "JSON parsing failed", - }, - { type: "finish-step" }, - { type: "start-step" }, - editStart("e2"), - ...editDeltas("e2"), - ]), - `${sse([ - { - type: "tool-input-available", - toolCallId: "e2", - toolName: "edit_diagram", - input: EDIT_GAMMA, - }, - { type: "finish-step" }, - { type: "finish" }, - ])}data: [DONE]\n\n`, + sse( + [ + { type: "start" }, + { type: "start-step" }, + e1.start, + ...e1.deltas, + ], + false, + ), + sse( + [ + { + type: "tool-input-error", + toolCallId: "e1", + toolName: "edit_diagram", + input: "{broken", + errorText: "JSON parsing failed", + }, + { + type: "tool-output-error", + toolCallId: "e1", + errorText: "JSON parsing failed", + }, + { type: "finish-step" }, + { type: "start-step" }, + e2.start, + ...e2.deltas, + ], + false, + ), + sse([e2.done, { type: "finish-step" }, { type: "finish" }]), ], ] const canvas = await chunkedReplies(p, replies) @@ -388,16 +335,20 @@ test("an edit right after a broken edit call starts from the real diagram", asyn test("a request that fails during an edit undoes its preview", async ({ page: p, }) => { + const e1 = editGamma("e1") const canvas = await chunkedReplies(p, [ [streamedToolCall("display_diagram", { xml: cell("a", "Alpha", 40) })], [ - sse([ - { type: "start" }, - { type: "start-step" }, - editStart("e1"), - ...editDeltas("e1"), - ]), - `${sse([{ type: "error", errorText: "Upstream connection lost" }])}data: [DONE]\n\n`, + sse( + [ + { type: "start" }, + { type: "start-step" }, + e1.start, + ...e1.deltas, + ], + false, + ), + sse([{ type: "error", errorText: "Upstream connection lost" }]), ], ]) await sendMessage(p, "Draw a box") @@ -414,57 +365,59 @@ test("a request that fails during an edit undoes its preview", async ({ await expect(canvas.getByText("Alpha", { exact: true })).toBeVisible() }) -/** A streamed tool call split into its start, input deltas and finished input */ -function toolCallEvents(id: string, toolName: string, input: unknown) { - const deltas = (JSON.stringify(input).match(/[\s\S]{1,40}/g) ?? []).map( - (d) => ({ - type: "tool-input-delta", - toolCallId: id, - inputTextDelta: d, - }), - ) - return { - start: { type: "tool-input-start", toolCallId: id, toolName }, - deltas, - done: { type: "tool-input-available", toolCallId: id, toolName, input }, - } -} const drawReply = (id: string, xml: string) => { const call = toolCallEvents(id, "display_diagram", { xml }) - return `${sse([{ type: "start" }, call.start, ...call.deltas, call.done, { type: "finish" }])}data: [DONE]\n\n` + return sse([ + { type: "start" }, + call.start, + ...call.deltas, + call.done, + { type: "finish" }, + ]) } const textReply = (text: string) => - `${sse([ + sse([ { type: "start" }, { type: "text-start", id: "t" }, { type: "text-delta", id: "t", delta: text }, { type: "text-end", id: "t" }, { type: "finish" }, - ])}data: [DONE]\n\n` + ]) // SSE comments keep a stream open without sending anything const KEEP_OPEN = Array(20).fill(":\n\n") test("an error after a finished edit keeps the current diagram", async ({ page: p, }) => { - const edit = toolCallEvents("e1", "edit_diagram", EDIT_GAMMA) + const edit = editGamma("e1") const half = Math.ceil(edit.deltas.length / 2) const canvas = await chunkedReplies(p, [ [drawReply("d1", cell("a", "Alpha", 40))], [ - sse([ - { type: "start" }, - { type: "start-step" }, - edit.start, - ...edit.deltas.slice(0, half), - ]), + sse( + [ + { type: "start" }, + { type: "start-step" }, + edit.start, + ...edit.deltas.slice(0, half), + ], + false, + ), // The last input and the finished call arrive together, so the // tool handler runs before the UI shows the call as finished - `${sse([...edit.deltas.slice(half), edit.done, { type: "finish-step" }, { type: "finish" }])}data: [DONE]\n\n`, + sse([ + ...edit.deltas.slice(half), + edit.done, + { type: "finish-step" }, + { type: "finish" }, + ]), ], [drawReply("d2", cell("b", "Beta", 40))], [ - `${sse([{ type: "start" }, { type: "error", errorText: "Upstream down" }])}data: [DONE]\n\n`, + sse([ + { type: "start" }, + { type: "error", errorText: "Upstream down" }, + ]), ], ]) await sendMessage(p, "Draw a box") @@ -495,6 +448,7 @@ for (const gap of [220, 600]) { test(`an edit that arrives with a broken edit's error keeps its change (${gap} ms)`, async ({ page: p, }) => { + const e1 = editGamma("e1") const delta = toolCallEvents("e2", "edit_diagram", { operations: [ { @@ -507,10 +461,10 @@ for (const gap of [220, 600]) { const canvas = await chunkedReplies(p, [ [drawReply("d1", cell("a", "Alpha", 40))], [ - sse([{ type: "start" }, { type: "start-step" }]), - sse([editStart("e1"), ...editDeltas("e1")]), + sse([{ type: "start" }, { type: "start-step" }], false), + sse([e1.start, ...e1.deltas], false), gap, - `${sse([ + sse([ { type: "tool-input-error", toolCallId: "e1", @@ -525,7 +479,7 @@ for (const gap of [220, 600]) { delta.done, { type: "finish-step" }, { type: "finish" }, - ])}data: [DONE]\n\n`, + ]), ], ]) await sendMessage(p, "Draw a box") @@ -551,8 +505,8 @@ test("a request that fails while drawing undoes the half drawn diagram", async ( const canvas = await chunkedReplies(p, [ [drawReply("d1", cell("a", "Alpha", 40))], [ - sse([{ type: "start" }, redraw.start, ...redraw.deltas]), - `${sse([{ type: "error", errorText: "Upstream connection lost" }])}data: [DONE]\n\n`, + sse([{ type: "start" }, redraw.start, ...redraw.deltas], false), + sse([{ type: "error", errorText: "Upstream connection lost" }]), ], ]) await sendMessage(p, "Draw a box") @@ -577,7 +531,7 @@ test("stopping while drawing undoes the half drawn diagram", async ({ const canvas = await chunkedReplies(p, [ [drawReply("d1", cell("a", "Alpha", 40))], [ - sse([{ type: "start" }, redraw.start, ...redraw.deltas]), + sse([{ type: "start" }, redraw.start, ...redraw.deltas], false), ...KEEP_OPEN, ], ]) @@ -595,6 +549,7 @@ test("stopping while drawing undoes the half drawn diagram", async ({ test("a broken edit's preview is undone after a shape library call", async ({ page: p, }) => { + const e1 = editGamma("e1") // The server runs get_shape_library, but its call still reaches the // browser's tool handler, before the UI shows the broken edit's error const library = toolCallEvents("s1", "get_shape_library", { @@ -603,10 +558,10 @@ test("a broken edit's preview is undone after a shape library call", async ({ const canvas = await chunkedReplies(p, [ [drawReply("d1", cell("a", "Alpha", 40))], [ - sse([{ type: "start" }, { type: "start-step" }]), - sse([editStart("e1"), ...editDeltas("e1")]), + sse([{ type: "start" }, { type: "start-step" }], false), + sse([e1.start, ...e1.deltas], false), 220, - `${sse([ + sse([ { type: "tool-input-error", toolCallId: "e1", @@ -626,7 +581,7 @@ test("a broken edit's preview is undone after a shape library call", async ({ }, { type: "finish-step" }, { type: "finish" }, - ])}data: [DONE]\n\n`, + ]), ], ]) await sendMessage(p, "Draw a box") @@ -654,8 +609,8 @@ test("a drawing rejected by the checks undoes its preview", async ({ const canvas = await chunkedReplies(p, [ [drawReply("d1", cell("a", "Alpha", 40))], [ - sse([{ type: "start" }, redraw.start, ...redraw.deltas]), - `${sse([redraw.done, { type: "finish" }])}data: [DONE]\n\n`, + sse([{ type: "start" }, redraw.start, ...redraw.deltas], false), + sse([redraw.done, { type: "finish" }]), ], ]) await sendMessage(p, "Draw a box") @@ -711,8 +666,7 @@ test("stopping during the screenshot check starts no new request", async ({ }), }) }) - await p.goto("/", { waitUntil: "networkidle" }) - await getIframe(p).waitFor({ state: "visible", timeout: 30000 }) + await openApp(p) await sendMessage(p, "Draw a box") await expect.poll(() => checking, { timeout: 15000 }).toBe(true) await p.getByRole("button", { name: "Stop generation" }).click() @@ -748,8 +702,7 @@ test("stopping during the screenshot check lets the next message go at once", as await new Promise((r) => setTimeout(r, 30000)) await route.fulfill({ status: 200, body: "{}" }).catch(() => {}) }) - await p.goto("/", { waitUntil: "networkidle" }) - await getIframe(p).waitFor({ state: "visible", timeout: 30000 }) + await openApp(p) await sendMessage(p, "Draw a box") await expect.poll(() => checking, { timeout: 15000 }).toBe(true) await p.getByRole("button", { name: "Stop generation" }).click() @@ -796,8 +749,7 @@ test("the request after a failed screenshot check keeps the model and diagram", ), }) }) - await p.goto("/", { waitUntil: "networkidle" }) - await getIframe(p).waitFor({ state: "visible", timeout: 30000 }) + await openApp(p) await sendMessage(p, "Draw a box") await expect.poll(() => requests.length, { timeout: 20000 }).toBe(2) const [first, retry] = requests @@ -816,9 +768,9 @@ const CUT = toolCallEvents("d2", "display_diagram", { xml: `${cell("b", "Beta", 40)}${cell("a", "Packed", 40)}` const packed = zlib @@ -1052,8 +1011,7 @@ test("a rejected drawing stops saying it tries again once retries end", async ({ ), }) }) - await p.goto("/", { waitUntil: "networkidle" }) - await getIframe(p).waitFor({ state: "visible", timeout: 30000 }) + await openApp(p) await sendMessage(p, "Draw a box") await expect.poll(() => calls, { timeout: 20000 }).toBe(4) await expect(p.locator('[data-testid="new-chat-button"]')).toBeEnabled({ @@ -1094,8 +1052,7 @@ test("the version strip stays away while the first drawing is checked", async ({ }) .catch(() => {}) }) - await p.goto("/", { waitUntil: "networkidle" }) - await getIframe(p).waitFor({ state: "visible", timeout: 30000 }) + await openApp(p) await sendMessage(p, "Draw a box") await expect.poll(() => checking, { timeout: 15000 }).toBe(true) // The version exists already; its card comes when the check is done diff --git a/tests/e2e/diagram-generation.spec.ts b/tests/e2e/diagram-generation.spec.ts index 29c09a04..252fbffe 100644 --- a/tests/e2e/diagram-generation.spec.ts +++ b/tests/e2e/diagram-generation.spec.ts @@ -7,6 +7,7 @@ import { createMultiTurnMock, expect, getChatInput, + openApp, sendMessage, test, waitForComplete, @@ -27,10 +28,7 @@ test.describe("Diagram Generation", () => { }) }) - await page.goto("/", { waitUntil: "networkidle" }) - await page - .locator("iframe") - .waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) }) test("generates and displays a diagram", async ({ page }) => { @@ -77,10 +75,7 @@ test.describe("Diagram Edit", () => { ]), ) - await page.goto("/", { waitUntil: "networkidle" }) - await page - .locator("iframe") - .waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) }) test("can edit an existing diagram", async ({ page }) => { @@ -108,10 +103,7 @@ test.describe("Diagram Append", () => { ]), ) - await page.goto("/", { waitUntil: "networkidle" }) - await page - .locator("iframe") - .waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) }) test("can append to an existing diagram", async ({ page }) => { diff --git a/tests/e2e/error-handling.spec.ts b/tests/e2e/error-handling.spec.ts index 36e75a03..5e39880c 100644 --- a/tests/e2e/error-handling.spec.ts +++ b/tests/e2e/error-handling.spec.ts @@ -3,10 +3,11 @@ import { createErrorMock, expect, getChatInput, - getIframe, + openApp, sendMessage, test, } from "./lib/fixtures" +import { sse } from "./lib/helpers" test.describe("Error Handling", () => { test("displays error message when API returns 500", async ({ page }) => { @@ -15,8 +16,7 @@ test.describe("Error Handling", () => { createErrorMock(500, "Internal server error"), ) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) await sendMessage(page, "Draw a cat") @@ -41,8 +41,7 @@ test.describe("Error Handling", () => { createErrorMock(429, "Rate limit exceeded"), ) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) await sendMessage(page, "Draw a cat") @@ -65,8 +64,7 @@ test.describe("Error Handling", () => { await route.abort("timedout") }) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) await sendMessage(page, "Draw a cat") @@ -116,15 +114,11 @@ test.describe("Error Handling", () => { await route.fulfill({ status: 200, contentType: "text/event-stream", - body: - events - .map((e) => `data: ${JSON.stringify(e)}\n\n`) - .join("") + "data: [DONE]\n\n", + body: sse(events), }) }) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) await sendMessage(page, "Draw something") @@ -159,14 +153,10 @@ test.describe("Error Handling", () => { await route.fulfill({ status: 200, contentType: "text/event-stream", - body: - events - .map((e) => `data: ${JSON.stringify(e)}\n\n`) - .join("") + "data: [DONE]\n\n", + body: sse(events), }) }) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) await sendMessage(page, "Draw something") // The failed call's row opens to show the error diff --git a/tests/e2e/file-upload.spec.ts b/tests/e2e/file-upload.spec.ts index d2b4b994..f43066a8 100644 --- a/tests/e2e/file-upload.spec.ts +++ b/tests/e2e/file-upload.spec.ts @@ -2,17 +2,20 @@ import { SINGLE_BOX_XML } from "./fixtures/diagrams" import { expect, getAttachmentInput, - getChatInput, - getIframe, + openApp, sendMessage, test, } from "./lib/fixtures" import { createMockSSEResponse } from "./lib/helpers" +const ONE_PIXEL_PNG = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", + "base64", +) + test.describe("File Upload", () => { test("upload button opens file picker", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) // The "Add" menu holds the upload entry const addButton = page.locator('[data-testid="composer-add"]') @@ -25,57 +28,24 @@ test.describe("File Upload", () => { expect(chooser.isMultiple()).toBe(true) }) - test("shows file preview after selecting image", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - const fileInput = getAttachmentInput(page) - - await fileInput.setInputFiles({ - name: "test-image.png", - mimeType: "image/png", - buffer: Buffer.from( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", - "base64", - ), - }) - - await expect( - page.locator('[role="alert"][data-type="error"]'), - ).not.toBeVisible({ timeout: 2000 }) - }) - test("can remove uploaded file", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) - const fileInput = getAttachmentInput(page) - - await fileInput.setInputFiles({ + await getAttachmentInput(page).setInputFiles({ name: "test-image.png", mimeType: "image/png", - buffer: Buffer.from( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", - "base64", - ), + buffer: ONE_PIXEL_PNG, }) - await expect( - page.locator('[role="alert"][data-type="error"]'), - ).not.toBeVisible({ timeout: 2000 }) - - const removeButton = page.locator( - '[data-testid="remove-file-button"], button[aria-label*="Remove"], button:has(svg.lucide-x)', - ) - - const removeButtonCount = await removeButton.count() - if (removeButtonCount === 0) { - test.skip() - return - } + // The image shows as a preview + await expect(page.getByAltText("test-image.png")).toBeVisible({ + timeout: 3000, + }) + const removeButton = page.getByRole("button", { name: "Remove file" }) await removeButton.first().click() - await expect(removeButton.first()).not.toBeVisible({ timeout: 2000 }) + await expect(removeButton).toHaveCount(0, { timeout: 2000 }) + await expect(page.getByAltText("test-image.png")).toHaveCount(0) }) test("sends file with message to API", async ({ page }) => { @@ -93,18 +63,12 @@ test.describe("File Upload", () => { }) }) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) - const fileInput = getAttachmentInput(page) - - await fileInput.setInputFiles({ + await getAttachmentInput(page).setInputFiles({ name: "architecture.png", mimeType: "image/png", - buffer: Buffer.from( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", - "base64", - ), + buffer: ONE_PIXEL_PNG, }) await sendMessage(page, "Convert this to a diagram") @@ -117,8 +81,7 @@ test.describe("File Upload", () => { }) test("shows error for oversized file", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) const fileInput = getAttachmentInput(page) const largeBuffer = Buffer.alloc(3 * 1024 * 1024, "x") @@ -135,8 +98,7 @@ test.describe("File Upload", () => { }) test("drag and drop file upload works", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) const chatForm = page.locator("form").first() @@ -152,6 +114,9 @@ test.describe("File Upload", () => { await chatForm.dispatchEvent("dragover", { dataTransfer }) await chatForm.dispatchEvent("drop", { dataTransfer }) - await expect(getChatInput(page)).toBeVisible({ timeout: 3000 }) + // The dropped file shows as a preview + await expect(page.getByAltText("dropped-image.png")).toBeVisible({ + timeout: 3000, + }) }) }) diff --git a/tests/e2e/fixtures/diagrams.ts b/tests/e2e/fixtures/diagrams.ts index 70092532..d369fb97 100644 --- a/tests/e2e/fixtures/diagrams.ts +++ b/tests/e2e/fixtures/diagrams.ts @@ -26,11 +26,6 @@ export const SINGLE_BOX_XML = ` ` -// Test node for iframe interaction tests -export const TEST_NODE_XML = ` - -` - // Architecture box export const ARCHITECTURE_XML = ` diff --git a/tests/e2e/history-restore.spec.ts b/tests/e2e/history-restore.spec.ts index ad33ff89..2071142a 100644 --- a/tests/e2e/history-restore.spec.ts +++ b/tests/e2e/history-restore.spec.ts @@ -2,16 +2,13 @@ import type { Page } from "@playwright/test" import { SINGLE_BOX_XML } from "./fixtures/diagrams" import { expect, - getChatInput, - getIframe, getIframeContent, - openSettings, + openApp, sendMessage, test, - waitForComplete, waitForText, } from "./lib/fixtures" -import { createMockSSEResponse } from "./lib/helpers" +import { createMockSSEResponse, sse } from "./lib/helpers" test.describe("History and Session Restore", () => { test("new chat button clears conversation", async ({ page }) => { @@ -26,8 +23,7 @@ test.describe("History and Session Restore", () => { }) }) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) await test.step("create a conversation", async () => { await sendMessage(page, "Create a test diagram") @@ -62,8 +58,7 @@ test.describe("History and Session Restore", () => { ), }) }) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) await sendMessage(page, "Create a test diagram") await waitForText(page, "Created your test diagram.") @@ -101,8 +96,7 @@ test.describe("History and Session Restore", () => { ), }) }) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) await sendMessage(page, "Create a test diagram") await waitForText(page, "Created your test diagram.") await page.evaluate(() => { @@ -132,8 +126,7 @@ test.describe("History and Session Restore", () => { body: createMockSSEResponse(SINGLE_BOX_XML, `Answer ${n}.`), }) }) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) await sendMessage(page, "First chat") await waitForText(page, "Answer 1.") await page.waitForTimeout(1500) @@ -196,8 +189,7 @@ test.describe("History and Session Restore", () => { test("new chat keeps a diagram without messages that could not be saved", async ({ page, }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) // Leave the start screen to draw on the canvas await page.getByTestId("draw-yourself").click() await drawByHand(page, "Hand drawn") @@ -227,8 +219,7 @@ test.describe("History and Session Restore", () => { ), }) }) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) await sendMessage(page, "Create a test diagram") await waitForText(page, "Created your test diagram.") await page.waitForTimeout(1500) @@ -256,168 +247,6 @@ test.describe("History and Session Restore", () => { getIframeContent(page).getByText("Hand drawn", { exact: true }), ).toBeVisible() }) - - test("chat history sidebar shows past conversations", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - const historyButton = page.locator( - 'button[aria-label*="History"]:not([disabled]), button:has(svg.lucide-history):not([disabled]), button:has(svg.lucide-menu):not([disabled]), button:has(svg.lucide-sidebar):not([disabled]), button:has(svg.lucide-panel-left):not([disabled])', - ) - - const buttonCount = await historyButton.count() - if (buttonCount === 0) { - test.skip() - return - } - - await historyButton.first().click() - await expect(getChatInput(page)).toBeVisible({ timeout: 3000 }) - }) - - test("conversation persists after page reload", async ({ page }) => { - await page.route("**/api/chat", async (route) => { - await route.fulfill({ - status: 200, - contentType: "text/event-stream", - body: createMockSSEResponse( - SINGLE_BOX_XML, - "This message should persist.", - ), - }) - }) - - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - await test.step("create conversation", async () => { - await sendMessage(page, "Create persistent diagram") - await waitForText(page, "This message should persist.") - }) - - await test.step("verify message appears before reload", async () => { - await expect(getChatInput(page)).toBeVisible({ timeout: 10000 }) - await expect( - page.locator('text="This message should persist."'), - ).toBeVisible({ timeout: 10000 }) - }) - - // Note: After reload, mocked responses won't persist since we're not - // testing with real localStorage. We just verify the app loads correctly. - await test.step("verify app loads after reload", async () => { - await page.reload({ waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - await expect(getChatInput(page)).toBeVisible({ timeout: 10000 }) - }) - }) - - test("diagram state persists after reload", async ({ page }) => { - await page.route("**/api/chat", async (route) => { - await route.fulfill({ - status: 200, - contentType: "text/event-stream", - body: createMockSSEResponse( - SINGLE_BOX_XML, - "Created a diagram that should be saved.", - ), - }) - }) - - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - await sendMessage(page, "Create saveable diagram") - await waitForComplete(page) - - await page.reload({ waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - const frame = getIframeContent(page) - await expect(frame.locator(".geDiagramContainer").first()).toBeVisible({ - timeout: 30000, - }) - }) - - test("can restore from browser back/forward", async ({ page }) => { - await page.route("**/api/chat", async (route) => { - await route.fulfill({ - status: 200, - contentType: "text/event-stream", - body: createMockSSEResponse( - SINGLE_BOX_XML, - "Testing browser navigation.", - ), - }) - }) - - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - await sendMessage(page, "Test navigation") - await waitForText(page, "Testing browser navigation.") - - await page.goto("/about", { waitUntil: "networkidle" }) - await page.goBack({ waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - await expect(getChatInput(page)).toBeVisible({ timeout: 10000 }) - }) - - test("settings are restored after reload", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - await openSettings(page) - await page.keyboard.press("Escape") - - await page.reload({ waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - await openSettings(page) - }) - - test("model selection persists", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - const modelSelector = page.locator( - 'button[aria-label*="Model"], [data-testid="model-selector"], button:has-text("Claude")', - ) - - const selectorCount = await modelSelector.count() - if (selectorCount === 0) { - test.skip() - return - } - - const initialModel = await modelSelector.first().textContent() - - await page.reload({ waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - const modelAfterReload = await modelSelector.first().textContent() - expect(modelAfterReload).toBe(initialModel) - }) - - test("handles localStorage quota exceeded gracefully", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - await page.evaluate(() => { - try { - const largeData = "x".repeat(5 * 1024 * 1024) - localStorage.setItem("test-large-data", largeData) - } catch { - // Expected to fail on some browsers - } - }) - - await expect(getChatInput(page)).toBeVisible({ timeout: 10000 }) - - await page.evaluate(() => { - localStorage.removeItem("test-large-data") - }) - }) }) /** Number of chats stored in this origin's IndexedDB */ @@ -456,8 +285,7 @@ test("new chat right after an answer saves that chat once", async ({ body: createMockSSEResponse(SINGLE_BOX_XML, "Drew the box."), }) }) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) const newChat = page.locator('[data-testid="new-chat-button"]') // The auto-save runs a second after the answer; New Chat around then // waits for its thumbnail while the auto-save starts @@ -479,7 +307,7 @@ test("an idle chat is not saved again and again", async ({ page }) => { status: 200, contentType: "text/event-stream", // The tool result comes from the page, as with a real model - body: `${[ + body: sse([ { type: "start" }, { type: "text-start", id: "t" }, { type: "text-delta", id: "t", delta: "Drew the box." }, @@ -499,9 +327,7 @@ test("an idle chat is not saved again and again", async ({ page }) => { }, }, { type: "finish" }, - ] - .map((e) => `data: ${JSON.stringify(e)}\n\n`) - .join("")}data: [DONE]\n\n`, + ]), }) }) // Every auto-save takes a thumbnail: count draw.io's thumbnail exports @@ -520,8 +346,7 @@ test("an idle chat is not saved again and again", async ({ page }) => { } catch {} }) }) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) await sendMessage(page, "Draw a box") await waitForText(page, "Drew the box.") // Let the auto-save after the answer finish diff --git a/tests/e2e/history.spec.ts b/tests/e2e/history.spec.ts deleted file mode 100644 index bbd55e6a..00000000 --- a/tests/e2e/history.spec.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { expect, getIframe, test } from "./lib/fixtures" - -test.describe("History Dialog", () => { - test.beforeEach(async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - }) - - test("history button exists in UI", async ({ page }) => { - // History button may be disabled initially (no history) - // Just verify it exists in the DOM - const historyButton = page - .locator("button") - .filter({ has: page.locator("svg") }) - const count = await historyButton.count() - expect(count).toBeGreaterThan(0) - }) -}) diff --git a/tests/e2e/iframe-interaction.spec.ts b/tests/e2e/iframe-interaction.spec.ts index a11609df..27480ac1 100644 --- a/tests/e2e/iframe-interaction.spec.ts +++ b/tests/e2e/iframe-interaction.spec.ts @@ -1,13 +1,10 @@ -import { TEST_NODE_XML } from "./fixtures/diagrams" import { expect, getIframe, getIframeContent, - sendMessage, + openApp, test, - waitForComplete, } from "./lib/fixtures" -import { createMockSSEResponse } from "./lib/helpers" test.describe("Iframe Interaction", () => { test("draw.io iframe loads successfully", async ({ page }) => { @@ -24,8 +21,7 @@ test.describe("Iframe Interaction", () => { }) test("can interact with draw.io toolbar", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) const frame = getIframeContent(page) @@ -38,84 +34,4 @@ test.describe("Iframe Interaction", () => { .first(), ).toBeVisible({ timeout: 30000 }) }) - - test("diagram XML is rendered in iframe after generation", async ({ - page, - }) => { - await page.route("**/api/chat", async (route) => { - await route.fulfill({ - status: 200, - contentType: "text/event-stream", - body: createMockSSEResponse( - TEST_NODE_XML, - "Here is your diagram:", - ), - }) - }) - - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - await sendMessage(page, "Create a test node") - await waitForComplete(page) - - // Give draw.io time to render - await page.waitForTimeout(1000) - }) - - test("zoom controls work in draw.io", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - const frame = getIframeContent(page) - - // draw.io should be loaded and functional - check for diagram container - await expect( - frame.locator(".geDiagramContainer, canvas").first(), - ).toBeVisible({ timeout: 10000 }) - }) - - test("can resize the panel divider", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - // Find the resizer/divider between panels - const resizer = page.locator( - '[role="separator"], [data-panel-resize-handle-id], .resize-handle', - ) - - if ((await resizer.count()) > 0) { - await expect(resizer.first()).toBeVisible() - - const box = await resizer.first().boundingBox() - if (box) { - await page.mouse.move( - box.x + box.width / 2, - box.y + box.height / 2, - ) - await page.mouse.down() - await page.mouse.move(box.x + 50, box.y + box.height / 2) - await page.mouse.up() - } - } - }) - - test("iframe responds to window resize", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - const iframe = getIframe(page) - const initialBox = await iframe.boundingBox() - - // Resize window - await page.setViewportSize({ width: 800, height: 600 }) - await page.waitForTimeout(500) - - const newBox = await iframe.boundingBox() - - expect(newBox).toBeDefined() - if (initialBox && newBox) { - expect(newBox.width).toBeLessThanOrEqual(800) - } - }) }) diff --git a/tests/e2e/keyboard.spec.ts b/tests/e2e/keyboard.spec.ts index 236b2983..5d506f7e 100644 --- a/tests/e2e/keyboard.spec.ts +++ b/tests/e2e/keyboard.spec.ts @@ -1,9 +1,8 @@ -import { expect, getIframe, openSettings, test } from "./lib/fixtures" +import { expect, openApp, openSettings, test } from "./lib/fixtures" test.describe("Keyboard Interactions", () => { test.beforeEach(async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) }) test("Escape closes settings dialog", async ({ page }) => { @@ -15,12 +14,4 @@ test.describe("Keyboard Interactions", () => { await page.keyboard.press("Escape") await expect(dialog).not.toBeVisible({ timeout: 2000 }) }) - - test("page is keyboard accessible", async ({ page }) => { - const focusableElements = page.locator( - 'button, [tabindex="0"], input, textarea, a[href]', - ) - const count = await focusableElements.count() - expect(count).toBeGreaterThan(0) - }) }) diff --git a/tests/e2e/language.spec.ts b/tests/e2e/language.spec.ts index d90da205..de88b660 100644 --- a/tests/e2e/language.spec.ts +++ b/tests/e2e/language.spec.ts @@ -5,6 +5,7 @@ import { getChatInput, getIframe, getSendButton, + openApp, openSettingsTab, sendMessage, sleep, @@ -23,8 +24,7 @@ async function pickLanguage(page: Page, name: string) { test.describe("Language Switching", () => { test("loads English by default", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) const chatInput = getChatInput(page) await expect(chatInput).toBeVisible({ timeout: 10000 }) @@ -33,8 +33,7 @@ test.describe("Language Switching", () => { }) test("can switch to Japanese", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) await test.step("open settings and select Japanese", async () => { await pickLanguage(page, "日本語") @@ -48,8 +47,7 @@ test.describe("Language Switching", () => { }) test("can switch to Chinese", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) await test.step("open settings and select Chinese", async () => { await pickLanguage(page, "中文") @@ -63,8 +61,7 @@ test.describe("Language Switching", () => { }) test("language persists after reload", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) await test.step("switch to Japanese", async () => { await pickLanguage(page, "日本語") @@ -89,8 +86,7 @@ test.describe("Language Switching", () => { }) test("Japanese locale URL works", async ({ page }) => { - await page.goto("/ja", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page, { path: "/ja" }) await expect(getSendButton(page, "送信")).toBeVisible({ timeout: 10000, @@ -98,8 +94,7 @@ test.describe("Language Switching", () => { }) test("Chinese locale URL works", async ({ page }) => { - await page.goto("/zh", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page, { path: "/zh" }) await expect(getSendButton(page, "发送")).toBeVisible({ timeout: 10000, @@ -129,8 +124,7 @@ test.describe("Language Switching", () => { body: createMockSSEResponse(SINGLE_BOX_XML, "Drew the box."), }), ) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) await sendMessage(page, "Draw a box") await waitForComplete(page) // Storage is full: switching the language offers to go on unsaved diff --git a/tests/e2e/lib/fixtures.ts b/tests/e2e/lib/fixtures.ts index 1a3ab53e..dcb1d895 100644 --- a/tests/e2e/lib/fixtures.ts +++ b/tests/e2e/lib/fixtures.ts @@ -1,28 +1,11 @@ /** - * Playwright test fixtures for E2E tests - * Uses test.extend to provide common setup and helpers + * Shared helpers for the E2E tests: locators, actions and chat API mocks */ -import { test as base, expect, type Page, type Route } from "@playwright/test" +import { expect, type Locator, type Page, type Route } from "@playwright/test" import { createMockSSEResponse, createTextOnlyResponse } from "./helpers" -/** - * Extended test with common fixtures - */ -export const test = base.extend<{ - /** Page with iframe already loaded */ - appPage: Page -}>({ - appPage: async ({ page }, use) => { - await page.goto("/", { waitUntil: "networkidle" }) - await page - .locator("iframe") - .waitFor({ state: "visible", timeout: 30000 }) - await use(page) - }, -}) - -export { expect } +export { expect, test } from "@playwright/test" // ============================================ // Locator helpers @@ -56,14 +39,75 @@ export async function openDrawioFile(page: Page, name: string, xml: string) { } /** Get the settings button */ -export function getSettingsButton(page: Page) { +function getSettingsButton(page: Page) { return page.locator('[data-testid="settings-button"]') } +/** The settings dialog */ +export const settingsDialog = (page: Page) => + page.getByRole("dialog", { name: "Settings" }) + +/** Back from a provider's page to the list of the models tab */ +export const backToModels = (dialog: Locator) => + dialog.locator("header button", { hasText: /^Models$/ }).click() + // ============================================ // Action helpers // ============================================ +/** The key under which the app saves the user's own model configs */ +export const MODEL_CONFIGS_KEY = "next-ai-draw-io-model-configs" + +/** + * Open the app and wait for draw.io's frame. A config is saved as the + * user's model configs on the first load only: a reload keeps what the + * test did. + */ +export async function openApp( + page: Page, + { path = "/", config }: { path?: string; config?: object } = {}, +) { + if (config) { + await page.addInitScript( + ([key, config]) => { + if (!sessionStorage.getItem("seeded")) { + localStorage.setItem(key, JSON.stringify(config)) + sessionStorage.setItem("seeded", "1") + } + }, + [MODEL_CONFIGS_KEY, config] as const, + ) + } + await page.goto(path, { waitUntil: "networkidle" }) + await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) +} + +/** The model configs as saved in the browser */ +export const savedModelConfigs = (page: Page) => + page.evaluate( + (key) => JSON.parse(localStorage.getItem(key) ?? "{}"), + MODEL_CONFIGS_KEY, + ) + +/** What another browser tab saving the config looks like to this one */ +export async function changeInAnotherTab( + page: Page, + change: (config: any) => void, +) { + await page.evaluate( + ([key, body]) => { + const config = JSON.parse(localStorage.getItem(key) ?? "{}") + new Function("config", body)(config) + const value = JSON.stringify(config) + localStorage.setItem(key, value) + window.dispatchEvent( + new StorageEvent("storage", { key, newValue: value }), + ) + }, + [MODEL_CONFIGS_KEY, `(${change.toString()})(config)`] as const, + ) +} + /** Send a message in the chat input */ export async function sendMessage(page: Page, message: string) { const chatInput = getChatInput(page) @@ -73,7 +117,7 @@ export async function sendMessage(page: Page, message: string) { } /** Tool calls that finished (version cards and tool rows) */ -export function getCompletedTools(page: Page) { +function getCompletedTools(page: Page) { return page.locator('[data-tool-state="output-available"]') } @@ -121,6 +165,29 @@ export async function openSettingsTab( await page.locator(`[data-testid="settings-tab-${tab}"]`).click() } +// ============================================ +// Test data +// ============================================ + +/** Two providers with one model each, for the settings dialog's tests */ +export const TWO_PROVIDERS = { + version: 1, + providers: [ + { + id: "p1", + provider: "glm", + apiKey: "k1", + models: [{ id: "a1", modelId: "glm-a" }], + }, + { + id: "p2", + provider: "kimi", + apiKey: "k2", + models: [{ id: "b1", modelId: "kimi-b" }], + }, + ], +} + // ============================================ // Mock helpers // ============================================ @@ -153,22 +220,6 @@ export function createMultiTurnMock(responses: MockResponse[]) { } } -/** - * Create a mock that returns text-only responses - */ -export function createTextOnlyMock(responses: string[]) { - let requestCount = 0 - return async (route: Route) => { - const text = responses[requestCount] || responses[responses.length - 1] - requestCount++ - await route.fulfill({ - status: 200, - contentType: "text/event-stream", - body: createTextOnlyResponse(text), - }) - } -} - /** * Create a mock that alternates between text and diagram responses */ @@ -212,28 +263,58 @@ export function createErrorMock(status: number, error: string) { } } -// ============================================ -// Persistence helpers -// ============================================ - /** - * Test that state persists across page reload. - * Runs assertions before reload, reloads page, then runs assertions again. - * Keep assertions narrow and explicit - test one specific thing. - * - * @param page - Playwright page - * @param description - What persistence is being tested (for debugging) - * @param assertion - Async function with expect() calls + * Answer each chat request with the next reply, in the browser. Each string + * in a reply is one network chunk, sent gapMs apart, so the throttled UI + * renders between chunks like with a real model. A number in a reply sets + * the wait before the next chunk instead. With holdLast, the last reply + * never ends. Requests past the list get an answer with nothing in it. */ -export async function expectBeforeAndAfterReload( +export async function chunkedReplies( page: Page, - description: string, - assertion: () => Promise, + replies: (string | number)[][], + { + gapMs = 300, + holdLast = false, + }: { gapMs?: number; holdLast?: boolean } = {}, ) { - await test.step(`verify ${description} before reload`, assertion) - await page.reload({ waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - await test.step(`verify ${description} after reload`, assertion) + await page.addInitScript( + ({ replies, gapMs, holdLast }) => { + const realFetch = window.fetch + let n = 0 + window.fetch = async (input, init) => { + // Next.js passes URL objects for its own requests + const url = input instanceof Request ? input.url : String(input) + if (!url.endsWith("/api/chat")) return realFetch(input, init) + const index = n++ + const chunks = replies[index] ?? [ + 'data: {"type":"start"}\n\ndata: {"type":"finish"}\n\ndata: [DONE]\n\n', + ] + const body = new ReadableStream({ + async start(controller) { + for (const [i, chunk] of chunks.entries()) { + if (typeof chunk === "number") continue + controller.enqueue(new TextEncoder().encode(chunk)) + const next = chunks[i + 1] + await new Promise((r) => + setTimeout( + r, + typeof next === "number" ? next : gapMs, + ), + ) + } + if (!(holdLast && index === replies.length - 1)) { + controller.close() + } + }, + }) + return new Response(body, { + headers: { "content-type": "text/event-stream" }, + }) + } + }, + { replies, gapMs, holdLast }, + ) } /** Simple sleep helper */ diff --git a/tests/e2e/lib/helpers.ts b/tests/e2e/lib/helpers.ts index 70cbaedf..f3eebeac 100644 --- a/tests/e2e/lib/helpers.ts +++ b/tests/e2e/lib/helpers.ts @@ -2,6 +2,40 @@ * Shared test helpers for E2E tests */ +/** Serializes events as an SSE body; with done, the [DONE] line ends it */ +export function sse(events: object[], done = true) { + return ( + events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join("") + + (done ? "data: [DONE]\n\n" : "") + ) +} + +/** + * A streamed tool call split into its start, its input deltas (the input + * JSON cut into chunks, like a real model sends it) and its finished input + */ +export function toolCallEvents( + id: string, + toolName: string, + input: unknown, + chunk = 40, +) { + const pattern = new RegExp(`[\\s\\S]{1,${chunk}}`, "g") + const deltas = (JSON.stringify(input).match(pattern) ?? []).map((d) => ({ + type: "tool-input-delta", + toolCallId: id, + inputTextDelta: d, + })) + return { + start: { type: "tool-input-start", toolCallId: id, toolName }, + deltas, + done: { type: "tool-input-available", toolCallId: id, toolName, input }, + } +} + +// The app skips a tool call whose id it has seen: every response gets new ids +let nextId = 0 + /** * Creates a mock SSE response for the chat API * Format matches AI SDK UI message stream protocol @@ -11,12 +45,12 @@ export function createMockSSEResponse( text: string, toolName = "display_diagram", ) { - const messageId = `msg_${Date.now()}` - const toolCallId = `call_${Date.now()}` - const textId = `text_${Date.now()}` + const n = ++nextId + const toolCallId = `call_${n}` + const textId = `text_${n}` const events = [ - { type: "start", messageId }, + { type: "start", messageId: `msg_${n}` }, { type: "text-start", id: textId }, { type: "text-delta", id: textId, delta: text }, { type: "text-end", id: textId }, @@ -30,59 +64,23 @@ export function createMockSSEResponse( { type: "finish" }, ] - return ( - events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join("") + - "data: [DONE]\n\n" - ) + return sse(events) } /** * Creates a text-only SSE response (no tool call) */ export function createTextOnlyResponse(text: string) { - const messageId = `msg_${Date.now()}` - const textId = `text_${Date.now()}` + const n = ++nextId + const textId = `text_${n}` const events = [ - { type: "start", messageId }, + { type: "start", messageId: `msg_${n}` }, { type: "text-start", id: textId }, { type: "text-delta", id: textId, delta: text }, { type: "text-end", id: textId }, { type: "finish" }, ] - return ( - events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join("") + - "data: [DONE]\n\n" - ) -} - -/** - * Creates a mock SSE response with a tool error - */ -export function createToolErrorResponse(text: string, errorMessage: string) { - const messageId = `msg_${Date.now()}` - const toolCallId = `call_${Date.now()}` - const textId = `text_${Date.now()}` - - const events = [ - { type: "start", messageId }, - { type: "text-start", id: textId }, - { type: "text-delta", id: textId, delta: text }, - { type: "text-end", id: textId }, - { type: "tool-input-start", toolCallId, toolName: "display_diagram" }, - { - type: "tool-input-available", - toolCallId, - toolName: "display_diagram", - input: { xml: "" }, - }, - { type: "tool-output-error", toolCallId, error: errorMessage }, - { type: "finish" }, - ] - - return ( - events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join("") + - "data: [DONE]\n\n" - ) + return sse(events) } diff --git a/tests/e2e/model-config.spec.ts b/tests/e2e/model-config.spec.ts deleted file mode 100644 index 9b974ecf..00000000 --- a/tests/e2e/model-config.spec.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { expect, getIframe, openSettings, test } from "./lib/fixtures" - -test.describe("Model Configuration", () => { - test.beforeEach(async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - }) - - test("settings dialog opens and shows configuration options", async ({ - page, - }) => { - await openSettings(page) - - const dialog = page.locator('[role="dialog"]') - const buttons = dialog.locator("button") - const buttonCount = await buttons.count() - expect(buttonCount).toBeGreaterThan(0) - }) -}) diff --git a/tests/e2e/model-test.spec.ts b/tests/e2e/model-test.spec.ts index b059b5da..6a754a48 100644 --- a/tests/e2e/model-test.spec.ts +++ b/tests/e2e/model-test.spec.ts @@ -1,5 +1,5 @@ import { expect, test } from "@playwright/test" -import { getIframe } from "./lib/fixtures" +import { openApp } from "./lib/fixtures" // One provider with three models; the test endpoint answers each differently const CONFIG = { @@ -21,12 +21,6 @@ const CONFIG = { test("the Test button checks all models at once and shows each result", async ({ page, }) => { - await page.addInitScript((config) => { - localStorage.setItem( - "next-ai-draw-io-model-configs", - JSON.stringify(config), - ) - }, CONFIG) const started: string[] = [] await page.route("**/api/validate-model", async (route) => { const { modelId } = route.request().postDataJSON() @@ -45,8 +39,7 @@ test("the Test button checks all models at once and shows each result", async ({ } await route.fulfill({ json: answers[modelId] }) }) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page, { config: CONFIG }) await page.getByTestId("model-selector").click() await page.getByText("Configure Models...").click() @@ -67,14 +60,7 @@ test("the Test button checks all models at once and shows each result", async ({ }) test("the key link and the base URL cleanup", async ({ page }) => { - await page.addInitScript((config) => { - localStorage.setItem( - "next-ai-draw-io-model-configs", - JSON.stringify(config), - ) - }, CONFIG) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page, { config: CONFIG }) await page.getByTestId("model-selector").click() await page.getByText("Configure Models...").click() const dialog = page.locator('[role="dialog"]') diff --git a/tests/e2e/multi-turn.spec.ts b/tests/e2e/multi-turn.spec.ts index f7308b91..3bdc2492 100644 --- a/tests/e2e/multi-turn.spec.ts +++ b/tests/e2e/multi-turn.spec.ts @@ -3,6 +3,7 @@ import { createMixedMock, createMultiTurnMock, expect, + openApp, sendMessage, test, waitForComplete, @@ -26,10 +27,7 @@ test.describe("Multi-turn Conversation", () => { ]), ) - await page.goto("/", { waitUntil: "networkidle" }) - await page - .locator("iframe") - .waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) // First request await sendMessage(page, "Draw first box") @@ -63,10 +61,7 @@ test.describe("Multi-turn Conversation", () => { }) }) - await page.goto("/", { waitUntil: "networkidle" }) - await page - .locator("iframe") - .waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) // First message await sendMessage(page, "Hello") @@ -93,10 +88,7 @@ test.describe("Multi-turn Conversation", () => { ]), ) - await page.goto("/", { waitUntil: "networkidle" }) - await page - .locator("iframe") - .waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) // Ask for explanation first await sendMessage(page, "Explain the architecture") diff --git a/tests/e2e/provider-models.spec.ts b/tests/e2e/provider-models.spec.ts index 7a9ab885..24d38bc6 100644 --- a/tests/e2e/provider-models.spec.ts +++ b/tests/e2e/provider-models.spec.ts @@ -1,5 +1,5 @@ import { expect, type Locator, type Page, test } from "@playwright/test" -import { getIframe } from "./lib/fixtures" +import { changeInAnotherTab, openApp, savedModelConfigs } from "./lib/fixtures" // qwen-mt-plus is a translation model; models.dev lists no tool calls for it const CONFIG = { @@ -22,14 +22,7 @@ async function showProvider(dialog: Locator, id: string) { } async function openQwenSettings(page: Page, config: object = CONFIG) { - await page.addInitScript((config) => { - localStorage.setItem( - "next-ai-draw-io-model-configs", - JSON.stringify(config), - ) - }, config) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page, { config }) await page.getByTestId("model-selector").click() await page.getByText("Configure Models...").click() const dialog = page.locator('[role="dialog"]') @@ -255,15 +248,8 @@ test("a test result for an old API key is dropped", async ({ page }) => { const dialog = await openQwenSettings(page, TWO_PROVIDERS) await testButton(dialog).click() // Another tab saves a different key for this provider - await page.evaluate(() => { - const key = "next-ai-draw-io-model-configs" - const config = JSON.parse(localStorage.getItem(key) ?? "{}") + await changeInAnotherTab(page, (config) => { config.providers[0].apiKey = "key-from-another-tab" - const value = JSON.stringify(config) - localStorage.setItem(key, value) - window.dispatchEvent( - new StorageEvent("storage", { key, newValue: value }), - ) }) release() await page.waitForTimeout(500) @@ -337,15 +323,8 @@ test("no spinner stays after another tab's change while elsewhere", async ({ await testButton(dialog).click() // The user looks at the other provider while another tab changes the key await showProvider(dialog, "p2") - await page.evaluate(() => { - const key = "next-ai-draw-io-model-configs" - const config = JSON.parse(localStorage.getItem(key) ?? "{}") + await changeInAnotherTab(page, (config) => { config.providers[0].apiKey = "key-from-another-tab" - const value = JSON.stringify(config) - localStorage.setItem(key, value) - window.dispatchEvent( - new StorageEvent("storage", { key, newValue: value }), - ) }) release() await page.waitForTimeout(500) @@ -353,13 +332,6 @@ test("no spinner stays after another tab's change while elsewhere", async ({ await expect(dialog.locator(".animate-spin")).toHaveCount(0) }) -const savedConfig = (page: Page) => - page.evaluate(() => - JSON.parse( - localStorage.getItem("next-ai-draw-io-model-configs") ?? "{}", - ), - ) - test("a test result that arrives after the settings closed is kept", async ({ page, }) => { @@ -375,7 +347,8 @@ test("a test result that arrives after the settings closed is kept", async ({ await expect .poll( async () => - (await savedConfig(page)).providers[0].models[0].validated, + (await savedModelConfigs(page)).providers[0].models[0] + .validated, ) .toBe(true) }) @@ -396,7 +369,7 @@ test("a test result for a key changed on another settings tab is dropped", async await dialog.locator("input[type=password]").first().fill("new-key") release() await page.waitForTimeout(800) - const saved = await savedConfig(page) + const saved = await savedModelConfigs(page) expect(saved.providers[0].apiKey).toBe("new-key") expect(saved.providers[0].models[0].validated).not.toBe(true) }) @@ -419,22 +392,14 @@ test("a test result for a model renamed on another settings tab is dropped", asy await input.blur() release() await page.waitForTimeout(800) - const saved = await savedConfig(page) + const saved = await savedModelConfigs(page) expect(saved.providers[0].models[0].modelId).toBe("qwen-plus") expect(saved.providers[0].models[0].validated).not.toBe(true) }) test("a failed test speaks the page's language", async ({ page }) => { - await page.addInitScript((config) => { - localStorage.setItem("next-ai-draw-io-locale", "zh") - localStorage.setItem( - "next-ai-draw-io-model-configs", - JSON.stringify(config), - ) - }, TWO_PROVIDERS) await page.route("**/api/validate-model", (route) => route.abort()) - await page.goto("/zh", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page, { path: "/zh", config: TWO_PROVIDERS }) await page.getByTestId("settings-button").first().click() const dialog = page.locator('[role="dialog"]') await showProvider(dialog, "p1") @@ -453,21 +418,13 @@ test("a failed test speaks the page's language", async ({ page }) => { test("testing a provider without models speaks the page's language", async ({ page, }) => { - await page.addInitScript( - (config) => { - localStorage.setItem("next-ai-draw-io-locale", "zh") - localStorage.setItem( - "next-ai-draw-io-model-configs", - JSON.stringify(config), - ) - }, - { + await openApp(page, { + path: "/zh", + config: { version: 1, providers: [{ ...CONFIG.providers[0], models: [] }], }, - ) - await page.goto("/zh", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + }) await page.getByTestId("settings-button").first().click() const dialog = page.locator('[role="dialog"]') await showProvider(dialog, "p1") @@ -479,15 +436,7 @@ test("testing a provider without models speaks the page's language", async ({ test("adding a model id twice is refused in the page's language", async ({ page, }) => { - await page.addInitScript((config) => { - localStorage.setItem("next-ai-draw-io-locale", "zh") - localStorage.setItem( - "next-ai-draw-io-model-configs", - JSON.stringify(config), - ) - }, CONFIG) - await page.goto("/zh", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page, { path: "/zh", config: CONFIG }) await page.getByTestId("settings-button").first().click() const dialog = page.locator('[role="dialog"]') await showProvider(dialog, "p1") @@ -508,14 +457,8 @@ test("Bedrock can be tested and used with an API key alone", async ({ json: { valid: true, responseTime: 900 }, }) }) - await page.addInitScript( - (config) => { - localStorage.setItem( - "next-ai-draw-io-model-configs", - JSON.stringify(config), - ) - }, - { + await openApp(page, { + config: { version: 1, providers: [ { @@ -526,9 +469,7 @@ test("Bedrock can be tested and used with an API key alone", async ({ }, ], }, - ) - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + }) await page.getByTestId("model-selector").click() await page.getByText("Configure Models...").click() const dialog = page.locator('[role="dialog"]') diff --git a/tests/e2e/save.spec.ts b/tests/e2e/save.spec.ts deleted file mode 100644 index 995f3389..00000000 --- a/tests/e2e/save.spec.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { expect, getIframe, test } from "./lib/fixtures" - -test.describe("Save Dialog", () => { - test.beforeEach(async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - }) - - test("save/download buttons exist", async ({ page }) => { - const buttons = page - .locator("button") - .filter({ has: page.locator("svg") }) - const count = await buttons.count() - expect(count).toBeGreaterThan(0) - }) -}) diff --git a/tests/e2e/settings-bedrock.spec.ts b/tests/e2e/settings-bedrock.spec.ts new file mode 100644 index 00000000..2eeb698c --- /dev/null +++ b/tests/e2e/settings-bedrock.spec.ts @@ -0,0 +1,341 @@ +// Bedrock in the settings: an API key or AWS access keys, one at a time +import { + changeInAnotherTab, + expect, + openApp, + openSettings, + savedModelConfigs, + settingsDialog, + test, +} from "./lib/fixtures" + +test("Bedrock shows one way of signing in and empties the other", async ({ + page, +}) => { + await openApp(page, { + config: { + version: 1, + providers: [ + { + id: "b1", + provider: "bedrock", + apiKey: "bedrock-key", + awsRegion: "us-east-1", + // Left by an older version: belongs to the access keys + awsSessionToken: "session-token", + models: [ + { + id: "m1", + modelId: "amazon.nova-lite-v1:0", + validated: true, + }, + ], + }, + ], + }, + }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-b1").click() + await expect(dialog.getByLabel("Bedrock API key")).toHaveValue( + "bedrock-key", + ) + await expect(dialog.locator("#aws-access-key-id")).toHaveCount(0) + + await dialog + .getByRole("button", { name: "Access keys", exact: true }) + .click() + await expect(dialog.locator("#bedrock-api-key")).toHaveCount(0) + // The key that was tested is gone, and so is its test result + await expect + .poll( + async () => + (await savedModelConfigs(page)).providers[0].models[0] + .validated, + ) + .toBeUndefined() + await dialog.locator("#aws-access-key-id").fill("AKIATEST") + await dialog.locator("#aws-secret-access-key").fill("secret") + await expect + .poll(async () => (await savedModelConfigs(page)).providers[0]) + .toMatchObject({ + apiKey: "", + awsAccessKeyId: "AKIATEST", + awsSecretAccessKey: "secret", + }) + // Reopened, it shows the way that is filled in + await page.keyboard.press("Escape") + await openSettings(page) + await dialog.getByTestId("provider-row-b1").click() + await expect( + dialog.getByRole("button", { name: "Access keys", exact: true }), + ).toHaveAttribute("aria-pressed", "true") + + await dialog + .getByRole("button", { name: "Bedrock API key", exact: true }) + .click() + await expect + .poll(async () => (await savedModelConfigs(page)).providers[0]) + .toMatchObject({ + awsAccessKeyId: "", + awsSecretAccessKey: "", + awsSessionToken: "", + }) +}) + +test("switching Bedrock's sign-in with nothing filled in keeps the test results", async ({ + page, +}) => { + // Older configs may hold a result without the key that earned it + await openApp(page, { + config: { + version: 1, + providers: [ + { + id: "b1", + provider: "bedrock", + apiKey: "", + awsRegion: "us-east-1", + models: [ + { + id: "m1", + modelId: "amazon.nova-lite-v1:0", + validated: true, + }, + ], + }, + ], + }, + }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-b1").click() + await dialog + .getByRole("button", { name: "Access keys", exact: true }) + .click() + await expect(dialog.locator("#aws-access-key-id")).toBeVisible() + await page.waitForTimeout(300) + expect( + (await savedModelConfigs(page)).providers[0].models[0].validated, + ).toBe(true) +}) + +test("Bedrock shows the sign-in that is saved, also after another tab", async ({ + page, +}) => { + await openApp(page, { + config: { + version: 1, + providers: [ + { + id: "b1", + provider: "bedrock", + apiKey: "", + awsAccessKeyId: "AKIAOLD", + awsSecretAccessKey: "old-secret", + awsRegion: "us-east-1", + models: [{ id: "m1", modelId: "amazon.nova-lite-v1:0" }], + }, + ], + }, + }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-b1").click() + await expect(dialog.locator("#aws-access-key-id")).toHaveValue("AKIAOLD") + // Another tab signs the provider in with an API key instead + await changeInAnotherTab(page, (config) => { + config.providers[0].apiKey = "bedrock-key" + config.providers[0].awsAccessKeyId = "" + config.providers[0].awsSecretAccessKey = "" + }) + await expect(dialog.locator("#bedrock-api-key")).toHaveValue("bedrock-key") + await expect(dialog.locator("#aws-access-key-id")).toHaveCount(0) +}) + +test("emptying Bedrock's access keys to type new ones keeps their fields", async ({ + page, +}) => { + await openApp(page, { + config: { + version: 1, + providers: [ + { + id: "b1", + provider: "bedrock", + apiKey: "", + awsAccessKeyId: "AKIAOLD", + awsSecretAccessKey: "old-secret", + awsRegion: "us-east-1", + models: [{ id: "m1", modelId: "amazon.nova-lite-v1:0" }], + }, + ], + }, + }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-b1").click() + await dialog.locator("#aws-secret-access-key").fill("") + await dialog.locator("#aws-access-key-id").fill("") + await expect(dialog.locator("#bedrock-api-key")).toHaveCount(0) + await dialog.locator("#aws-access-key-id").fill("AKIANEW") + await expect + .poll(async () => (await savedModelConfigs(page)).providers[0]) + .toMatchObject({ apiKey: "", awsAccessKeyId: "AKIANEW" }) +}) + +test("refused access keys are named as such", async ({ page }) => { + await page.route("**/api/validate-model", (route) => + route.fulfill({ + json: { valid: false, code: "invalid_api_key", error: "Bad keys" }, + }), + ) + await openApp(page, { + config: { + version: 1, + providers: [ + { + id: "b1", + provider: "bedrock", + apiKey: "", + awsAccessKeyId: "AKIA1", + awsSecretAccessKey: "s", + awsRegion: "us-east-1", + models: [{ id: "m1", modelId: "amazon.nova-lite-v1:0" }], + }, + ], + }, + }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-b1").click() + await dialog.getByTestId("test-models").click() + await expect( + dialog.getByText( + "AWS rejected these access keys. Check that both were copied in full. Bad keys", + ), + ).toBeVisible() + for (const id of ["#aws-access-key-id", "#aws-secret-access-key"]) { + await expect(dialog.locator(id)).toHaveAttribute("aria-invalid", "true") + } +}) + +test("temporary Bedrock access keys can carry their session token", async ({ + page, +}) => { + const bodies: any[] = [] + await page.route("**/api/validate-model", async (route) => { + bodies.push(route.request().postDataJSON()) + await route.fulfill({ json: { valid: true, responseTime: 900 } }) + }) + await openApp(page, { + config: { + version: 1, + providers: [ + { + id: "b1", + provider: "bedrock", + apiKey: "", + awsAccessKeyId: "ASIATEMP", + awsSecretAccessKey: "secret", + awsRegion: "us-east-1", + models: [{ id: "m1", modelId: "amazon.nova-lite-v1:0" }], + }, + ], + }, + }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-b1").click() + await dialog.getByLabel("AWS session token (optional)").fill("session-1") + await dialog.getByTestId("test-models").click() + await expect.poll(() => bodies.length).toBe(1) + expect(bodies[0]).toMatchObject({ + awsAccessKeyId: "ASIATEMP", + awsSessionToken: "session-1", + }) + expect((await savedModelConfigs(page)).providers[0].awsSessionToken).toBe( + "session-1", + ) +}) + +test("Bedrock with both kinds of credentials offers to remove the access keys", async ({ + page, +}) => { + await openApp(page, { + config: { + version: 1, + providers: [ + { + id: "b1", + provider: "bedrock", + apiKey: "bedrock-key", + awsAccessKeyId: "AKIAOLD", + awsSecretAccessKey: "old-secret", + awsSessionToken: "old-token", + awsRegion: "us-east-1", + models: [{ id: "m1", modelId: "amazon.nova-lite-v1:0" }], + }, + ], + }, + }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-b1").click() + const notice = dialog.getByText(/also keeps an access key pair/) + await expect(notice).toBeVisible() + await dialog.getByRole("button", { name: "Remove the access keys" }).click() + await expect(notice).toHaveCount(0) + await expect + .poll(async () => (await savedModelConfigs(page)).providers[0]) + .toMatchObject({ + apiKey: "bedrock-key", + awsAccessKeyId: "", + awsSecretAccessKey: "", + awsSessionToken: "", + }) + await expect(dialog.locator("#bedrock-api-key")).toHaveValue("bedrock-key") +}) + +test("removing access keys kept behind an API key keeps the test results", async ({ + page, +}) => { + await openApp(page, { + config: { + version: 1, + providers: [ + { + id: "b1", + provider: "bedrock", + apiKey: "bedrock-key", + awsAccessKeyId: "AKIAOLD", + awsSecretAccessKey: "old-secret", + awsRegion: "us-east-1", + models: [ + { + id: "m1", + modelId: "amazon.nova-lite-v1:0", + validated: true, + }, + ], + }, + ], + }, + }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-b1").click() + await dialog.getByRole("button", { name: "Remove the access keys" }).focus() + await page.keyboard.press("Enter") + await expect + .poll( + async () => + (await savedModelConfigs(page)).providers[0].awsAccessKeyId, + ) + .toBe("") + // The API key was the one in use: its result still holds + expect( + (await savedModelConfigs(page)).providers[0].models[0].validated, + ).toBe(true) + await expect(dialog.locator("#bedrock-api-key")).toBeFocused() +}) diff --git a/tests/e2e/settings-providers.spec.ts b/tests/e2e/settings-providers.spec.ts index d23ed594..e417ab9d 100644 --- a/tests/e2e/settings-providers.spec.ts +++ b/tests/e2e/settings-providers.spec.ts @@ -1,37 +1,18 @@ -import type { Page } from "@playwright/test" +// The models tab of the settings: providers, their keys and models +import type { Locator } from "@playwright/test" import { + backToModels, expect, - getIframe, + openApp, openSettings, openSettingsTab, + savedModelConfigs, sendMessage, + settingsDialog, + TWO_PROVIDERS, test, } from "./lib/fixtures" -const KEY = "next-ai-draw-io-model-configs" - -async function openApp(page: Page, config?: object) { - if (config) { - await page.addInitScript( - ([key, config]) => { - // Only on the first load: a reload keeps what the test did - if (!sessionStorage.getItem("seeded")) { - localStorage.setItem(key, JSON.stringify(config)) - sessionStorage.setItem("seeded", "1") - } - }, - [KEY, config] as const, - ) - } - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) -} - -const saved = (page: Page) => - page.evaluate((key) => JSON.parse(localStorage.getItem(key) ?? "{}"), KEY) - -const dialogOf = (page: Page) => page.getByRole("dialog", { name: "Settings" }) - // Three providers: one with a failed model, one not set up, one working const PROVIDERS = { version: 1, @@ -72,7 +53,7 @@ test("a first provider: pick it, fill in the key, test, use it", async ({ }) await openApp(page) await openSettings(page) - const dialog = dialogOf(page) + const dialog = settingsDialog(page) await expect(dialog.getByText("Use your own API key")).toBeVisible() await dialog.getByTestId("pick-provider-deepseek").click() @@ -109,18 +90,18 @@ test("a first provider: pick it, fill in the key, test, use it", async ({ await expect(page.getByTestId("model-selector")).toContainText( "deepseek-v4-pro", ) - const config = await saved(page) + const config = await savedModelConfigs(page) expect(config.selectedModelId).toBe(config.providers[0].models[0].id) }) test("the provider picker searches, marks added ones, and Enter picks", async ({ page, }) => { - await openApp(page, PROVIDERS) + await openApp(page, { config: PROVIDERS }) await page.getByTestId("model-selector").click() // The model picker opens the settings straight on the provider picker await page.getByText("Add a provider…").click() - const dialog = dialogOf(page) + const dialog = settingsDialog(page) const search = dialog.getByRole("searchbox") await expect(search).toBeFocused() await expect(dialog.getByTestId("pick-provider-openai")).toContainText( @@ -146,13 +127,15 @@ test("the provider picker searches, marks added ones, and Enter picks", async ({ await expect( dialog.getByRole("heading", { name: "Kimi (Moonshot)", exact: true }), ).toBeVisible() - expect((await saved(page)).providers.at(-1).provider).toBe("kimi") + expect((await savedModelConfigs(page)).providers.at(-1).provider).toBe( + "kimi", + ) }) test("the provider list shows what needs attention", async ({ page }) => { - await openApp(page, PROVIDERS) + await openApp(page, { config: PROVIDERS }) await openSettings(page) - const dialog = dialogOf(page) + const dialog = settingsDialog(page) await expect(dialog.getByTestId("provider-row-p1")).toContainText( "1 failed", ) @@ -170,7 +153,7 @@ test("the provider list shows what needs attention", async ({ page }) => { }) test("the model picker lists untested and failed models", async ({ page }) => { - await openApp(page, PROVIDERS) + await openApp(page, { config: PROVIDERS }) await page.getByTestId("model-selector").click() // By the model config's id: the server may offer a model of that name const option = (id: string) => @@ -184,7 +167,9 @@ test("the model picker lists untested and failed models", async ({ page }) => { }) test("an old setting to hide untested models is ignored", async ({ page }) => { - await openApp(page, { ...PROVIDERS, showUnvalidatedModels: false }) + await openApp(page, { + config: { ...PROVIDERS, showUnvalidatedModels: false }, + }) await page.getByTestId("model-selector").click() await expect(page.locator('[role="option"][data-value="m3"]')).toBeVisible() }) @@ -201,12 +186,12 @@ test("a refused key opens the settings on that provider", async ({ page }) => { }), }), ) - await openApp(page, { ...PROVIDERS, selectedModelId: "m1" }) + await openApp(page, { config: { ...PROVIDERS, selectedModelId: "m1" } }) await sendMessage(page, "Draw a box") await page .getByRole("button", { name: "Open model settings" }) .click({ timeout: 15000 }) - const dialog = dialogOf(page) + const dialog = settingsDialog(page) await expect( dialog.getByRole("heading", { name: "OpenAI", exact: true }), ).toBeVisible() @@ -232,7 +217,7 @@ test("an access code error opens the general tab with the code field", async ({ ) await openApp(page) await sendMessage(page, "Draw a box") - const dialog = dialogOf(page) + const dialog = settingsDialog(page) await expect(dialog.locator("#access-code")).toBeVisible({ timeout: 15000 }) await expect(dialog.getByTestId("settings-tab-general")).toHaveAttribute( "aria-current", @@ -250,9 +235,9 @@ test("a refused key marks the key field until it changes", async ({ page }) => { }, }), ) - await openApp(page, PROVIDERS) + await openApp(page, { config: PROVIDERS }) await openSettings(page) - const dialog = dialogOf(page) + const dialog = settingsDialog(page) await dialog.getByTestId("provider-row-p3").click() await dialog.getByTestId("test-models").click() const key = dialog.locator("#api-key") @@ -272,107 +257,6 @@ test("a refused key marks the key field until it changes", async ({ page }) => { await expect(key).not.toHaveAttribute("aria-invalid", "true") }) -test("Bedrock shows one way of signing in and empties the other", async ({ - page, -}) => { - await openApp(page, { - version: 1, - providers: [ - { - id: "b1", - provider: "bedrock", - apiKey: "bedrock-key", - awsRegion: "us-east-1", - // Left by an older version: belongs to the access keys - awsSessionToken: "session-token", - models: [ - { - id: "m1", - modelId: "amazon.nova-lite-v1:0", - validated: true, - }, - ], - }, - ], - }) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-b1").click() - await expect(dialog.getByLabel("Bedrock API key")).toHaveValue( - "bedrock-key", - ) - await expect(dialog.locator("#aws-access-key-id")).toHaveCount(0) - - await dialog - .getByRole("button", { name: "Access keys", exact: true }) - .click() - await expect(dialog.locator("#bedrock-api-key")).toHaveCount(0) - // The key that was tested is gone, and so is its test result - await expect - .poll(async () => (await saved(page)).providers[0].models[0].validated) - .toBeUndefined() - await dialog.locator("#aws-access-key-id").fill("AKIATEST") - await dialog.locator("#aws-secret-access-key").fill("secret") - await expect - .poll(async () => (await saved(page)).providers[0]) - .toMatchObject({ - apiKey: "", - awsAccessKeyId: "AKIATEST", - awsSecretAccessKey: "secret", - }) - // Reopened, it shows the way that is filled in - await page.keyboard.press("Escape") - await openSettings(page) - await dialog.getByTestId("provider-row-b1").click() - await expect( - dialog.getByRole("button", { name: "Access keys", exact: true }), - ).toHaveAttribute("aria-pressed", "true") - - await dialog - .getByRole("button", { name: "Bedrock API key", exact: true }) - .click() - await expect - .poll(async () => (await saved(page)).providers[0]) - .toMatchObject({ - awsAccessKeyId: "", - awsSecretAccessKey: "", - awsSessionToken: "", - }) -}) - -test("switching Bedrock's sign-in with nothing filled in keeps the test results", async ({ - page, -}) => { - // Older configs may hold a result without the key that earned it - await openApp(page, { - version: 1, - providers: [ - { - id: "b1", - provider: "bedrock", - apiKey: "", - awsRegion: "us-east-1", - models: [ - { - id: "m1", - modelId: "amazon.nova-lite-v1:0", - validated: true, - }, - ], - }, - ], - }) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-b1").click() - await dialog - .getByRole("button", { name: "Access keys", exact: true }) - .click() - await expect(dialog.locator("#aws-access-key-id")).toBeVisible() - await page.waitForTimeout(300) - expect((await saved(page)).providers[0].models[0].validated).toBe(true) -}) - test("Use in the chat waits while an answer runs", async ({ page }) => { // The chat request stays open until released let release!: () => void @@ -390,10 +274,10 @@ test("Use in the chat waits while an answer runs", async ({ page }) => { await page.route("**/api/validate-model", (route) => route.fulfill({ json: { valid: true, responseTime: 900 } }), ) - await openApp(page, PROVIDERS) + await openApp(page, { config: PROVIDERS }) await sendMessage(page, "Draw a box") await openSettings(page) - const dialog = dialogOf(page) + const dialog = settingsDialog(page) await dialog.getByTestId("provider-row-p3").click() await dialog.getByTestId("test-models").click() const use = dialog.getByRole("button", { @@ -408,21 +292,23 @@ test("more options open by themselves only when they matter", async ({ page, }) => { await openApp(page, { - version: 1, - providers: [ - { id: "a1", provider: "azure", apiKey: "k", models: [] }, - { id: "o1", provider: "openai", apiKey: "k", models: [] }, - { - id: "o2", - provider: "openai", - apiKey: "k", - baseUrl: "https://proxy.example.com/v1", - models: [], - }, - ], + config: { + version: 1, + providers: [ + { id: "a1", provider: "azure", apiKey: "k", models: [] }, + { id: "o1", provider: "openai", apiKey: "k", models: [] }, + { + id: "o2", + provider: "openai", + apiKey: "k", + baseUrl: "https://proxy.example.com/v1", + models: [], + }, + ], + }, }) await openSettings(page) - const dialog = dialogOf(page) + const dialog = settingsDialog(page) const back = () => dialog.locator("header button", { hasText: "Models" }).click() // Azure's default URL is only an example @@ -441,9 +327,9 @@ test("more options open by themselves only when they matter", async ({ test("the models tab keeps its page across tabs, not across openings", async ({ page, }) => { - await openApp(page, PROVIDERS) + await openApp(page, { config: PROVIDERS }) await openSettings(page) - const dialog = dialogOf(page) + const dialog = settingsDialog(page) await dialog.getByTestId("provider-row-p1").click() await dialog.getByTestId("settings-tab-drawing").click() await expect(dialog.locator("#max-output-tokens")).toBeVisible() @@ -457,9 +343,9 @@ test("the models tab keeps its page across tabs, not across openings", async ({ }) test("deleting a provider says what goes with it", async ({ page }) => { - await openApp(page, PROVIDERS) + await openApp(page, { config: PROVIDERS }) await openSettings(page) - const dialog = dialogOf(page) + const dialog = settingsDialog(page) await dialog.getByTestId("provider-row-p3").click() await dialog .getByRole("button", { name: "Delete DeepSeek and its model" }) @@ -470,13 +356,13 @@ test("deleting a provider says what goes with it", async ({ page }) => { .click() await expect(dialog.getByTestId("provider-row-p3")).toHaveCount(0) await expect(dialog.getByTestId("provider-row-p1")).toBeVisible() - expect((await saved(page)).providers).toHaveLength(2) + expect((await savedModelConfigs(page)).providers).toHaveLength(2) }) test("the general and drawing tabs hold their settings", async ({ page }) => { await openApp(page) await openSettingsTab(page, "general") - const dialog = dialogOf(page) + const dialog = settingsDialog(page) for (const id of ["#language-select", "#send-shortcut-select"]) { await expect(dialog.locator(id)).toBeVisible() } @@ -495,3 +381,300 @@ test("the general and drawing tabs hold their settings", async ({ page }) => { dialog.getByRole("radiogroup", { name: "Diagram style" }), ).toBeVisible() }) + +test("Add a provider… leaves the cursor in the provider search", async ({ + page, +}) => { + await openApp(page) + await page.getByTestId("model-selector").click() + await page.getByText("Add a provider…").click() + const search = settingsDialog(page).getByRole("searchbox") + await expect(search).toBeFocused() + // After the model picker has closed + await page.waitForTimeout(800) + await expect(search).toBeFocused() + await page.keyboard.type("kimi") + await expect(search).toHaveValue("kimi") +}) + +test("the model picker's footer is there while searching and reachable by Tab", async ({ + page, +}) => { + await openApp(page) + await page.getByTestId("model-selector").click() + await page.keyboard.type("zzzz") + await expect(page.getByText("Add a provider…")).toBeVisible() + const add = page.getByRole("button", { name: "Add a provider…" }) + for (let i = 0; i < 6; i++) { + if (await add.evaluate((el) => el === document.activeElement)) break + await page.keyboard.press("Tab") + } + await expect(add).toBeFocused() + await page.keyboard.press("Enter") + await expect(settingsDialog(page).getByRole("searchbox")).toBeVisible() +}) + +test("an Enter that confirms an IME candidate picks no provider", async ({ + page, +}) => { + await openApp(page) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByRole("button", { name: /See all/ }).click() + const search = dialog.getByRole("searchbox") + await search.fill("deep") + await search.evaluate((el) => + el.dispatchEvent( + new KeyboardEvent("keydown", { + key: "Enter", + bubbles: true, + isComposing: true, + }), + ), + ) + await page.waitForTimeout(300) + await expect(search).toBeVisible() + expect((await savedModelConfigs(page)).providers ?? []).toHaveLength(0) +}) + +test("the focus follows the page within the models tab", async ({ page }) => { + await openApp(page, { config: TWO_PROVIDERS }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-p1").focus() + await page.keyboard.press("Enter") + await expect( + dialog.getByRole("heading", { name: "GLM (Zhipu)", exact: true }), + ).toBeFocused() + await backToModels(dialog) + await expect( + dialog.getByRole("heading", { name: "Models", exact: true }), + ).toBeFocused() +}) + +test("a passing test hands the focus to its next step, a failing one keeps it", async ({ + page, +}) => { + let pass = false + await page.route("**/api/validate-model", (route) => + route.fulfill({ + json: pass + ? { valid: true, responseTime: 900 } + : { valid: false, error: "Model not found" }, + }), + ) + await openApp(page, { config: TWO_PROVIDERS }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-p1").click() + const testButton = dialog.getByTestId("test-models") + await testButton.focus() + await page.keyboard.press("Enter") + await expect(dialog.getByTestId("test-result")).toHaveText( + "1 model failed the test", + ) + await expect(testButton).toBeFocused() + + pass = true + await page.keyboard.press("Enter") + const use = dialog.getByRole("button", { name: "Use glm-a in the chat" }) + await expect(use).toBeFocused() + const resultId = await dialog.getByTestId("test-result").getAttribute("id") + await expect(use).toHaveAttribute("aria-describedby", resultId ?? "") +}) + +test("the test result goes to a live region that is there from the start", async ({ + page, +}) => { + await page.route("**/api/validate-model", (route) => + route.fulfill({ json: { valid: true, responseTime: 900 } }), + ) + await openApp(page, { config: TWO_PROVIDERS }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-p1").click() + const result = dialog.getByTestId("test-result") + await expect(result).toHaveAttribute("role", "status") + await expect(result).toHaveText("") + await dialog.getByTestId("test-models").click() + await expect(result).toHaveText("glm-a works") +}) + +test("long names stay inside the page", async ({ page }) => { + const long = `model-${"x".repeat(150)}` + await openApp(page, { + config: { + version: 1, + providers: [ + { + id: "p1", + provider: "openai", + name: `My ${"provider ".repeat(10)}name`, + apiKey: "k", + models: [{ id: "m1", modelId: long }], + }, + ], + }, + }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-p1").click() + const right = async (l: Locator) => { + const box = await l.boundingBox() + return box ? box.x + box.width : 0 + } + const pageRight = await right(dialog) + expect(await right(dialog.getByTestId("test-models"))).toBeLessThanOrEqual( + pageRight, + ) + expect( + await right( + dialog.getByRole("button", { name: /^Get all models from/ }), + ), + ).toBeLessThanOrEqual(pageRight) +}) + +test("a local server's base URL is shown from the start", async ({ page }) => { + await openApp(page, { + config: { + version: 1, + providers: [ + { id: "l1", provider: "ollama", apiKey: "", models: [] }, + ], + }, + }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-l1").click() + await expect(dialog.locator("#base-url")).toBeVisible() +}) + +test("Azure needs its resource URL before it counts as connected", async ({ + page, +}) => { + await openApp(page, { + config: { + version: 1, + providers: [ + { + id: "z1", + provider: "azure", + apiKey: "k", + baseUrl: "https://your-resource.openai.azure.com/openai", + models: [{ id: "m1", modelId: "gpt-6-astra" }], + }, + ], + }, + }) + await openSettings(page) + const dialog = settingsDialog(page) + await expect(dialog.getByTestId("provider-row-z1")).toContainText( + "Not set up yet", + ) + await dialog.getByTestId("provider-row-z1").click() + await expect(dialog.getByTestId("step-1-todo")).toBeVisible() + await expect(dialog.getByTestId("test-models")).toBeDisabled() + await expect( + dialog.getByText( + "Fill in the API key and your Azure resource's base URL first.", + ), + ).toBeVisible() + await dialog + .locator("#base-url") + .fill("https://contoso.openai.azure.com/openai") + await expect(dialog.getByTestId("test-models")).toBeEnabled() +}) + +test("a model id typed for one provider does not follow to the next", async ({ + page, +}) => { + await openApp(page, { config: TWO_PROVIDERS }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-p1").click() + const input = dialog.getByPlaceholder("Or type another model ID") + await input.fill("half-typed") + await backToModels(dialog) + await dialog.getByTestId("provider-row-p2").click() + await expect(input).toHaveValue("") +}) + +test("refused keys stay marked for each provider", async ({ page }) => { + await page.route("**/api/validate-model", (route) => + route.fulfill({ + json: { valid: false, code: "invalid_api_key", error: "Bad key" }, + }), + ) + await openApp(page, { config: TWO_PROVIDERS }) + await openSettings(page) + const dialog = settingsDialog(page) + const key = dialog.locator("#api-key") + await dialog.getByTestId("provider-row-p1").click() + await dialog.getByTestId("test-models").click() + await expect(key).toHaveAttribute("aria-invalid", "true") + await backToModels(dialog) + await dialog.getByTestId("provider-row-p2").click() + await dialog.getByTestId("test-models").click() + await expect(key).toHaveAttribute("aria-invalid", "true") + await backToModels(dialog) + await dialog.getByTestId("provider-row-p1").click() + await expect(key).toHaveAttribute("aria-invalid", "true") +}) + +test("a model list that loads leaves a refused test's mark", async ({ + page, +}) => { + // Some providers list their models without the key: a list proves nothing + await page.route("**/api/validate-model", (route) => + route.fulfill({ + json: { valid: false, code: "invalid_api_key", error: "Bad key" }, + }), + ) + await page.route("**/api/provider-models", (route) => + route.fulfill({ json: { models: [{ id: "listed-model" }] } }), + ) + await openApp(page, { config: TWO_PROVIDERS }) + await openSettings(page) + const dialog = settingsDialog(page) + const key = dialog.locator("#api-key") + await dialog.getByTestId("provider-row-p1").click() + await dialog.getByTestId("test-models").click() + await expect(key).toHaveAttribute("aria-invalid", "true") + await dialog.getByRole("button", { name: /^Get all models from/ }).click() + await expect( + page.getByRole("option", { name: "listed-model" }), + ).toBeVisible() + await page.keyboard.press("Escape") + await expect(page.getByRole("listbox")).toHaveCount(0) + await expect(key).toHaveAttribute("aria-invalid", "true") +}) + +test("an empty model list says so and opens nothing", async ({ page }) => { + await page.route("**/api/provider-models", (route) => + route.fulfill({ json: { models: [] } }), + ) + // OpenAI has suggested models: they come back once the key changes + await openApp(page, { + config: { + version: 1, + providers: [ + { id: "o1", provider: "openai", apiKey: "k", models: [] }, + ], + }, + }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-o1").click() + await dialog.getByRole("button", { name: /^Get all models from/ }).click() + await expect( + dialog.getByText("The provider returned no models."), + ).toBeVisible() + await dialog.locator("#api-key").fill("k-new") + await expect( + dialog.getByRole("button", { name: "Browse models" }), + ).toBeVisible() + await page.waitForTimeout(300) + // The picker did not open by itself + await expect(page.getByRole("listbox")).toHaveCount(0) + await expect(dialog.locator("#api-key")).toBeFocused() +}) diff --git a/tests/e2e/settings-review-fixes.spec.ts b/tests/e2e/settings-review-fixes.spec.ts deleted file mode 100644 index 742d2d41..00000000 --- a/tests/e2e/settings-review-fixes.spec.ts +++ /dev/null @@ -1,830 +0,0 @@ -// What the settings review found: each test fails without its fix -import type { Locator, Page } from "@playwright/test" -import { - expect, - getIframe, - openSettings, - sendMessage, - test, -} from "./lib/fixtures" - -const KEY = "next-ai-draw-io-model-configs" - -async function openApp(page: Page, config?: object) { - if (config) { - await page.addInitScript( - ([key, config]) => { - if (!sessionStorage.getItem("seeded")) { - localStorage.setItem(key, JSON.stringify(config)) - sessionStorage.setItem("seeded", "1") - } - }, - [KEY, config] as const, - ) - } - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) -} - -const dialogOf = (page: Page) => page.getByRole("dialog", { name: "Settings" }) -const saved = (page: Page) => - page.evaluate((key) => JSON.parse(localStorage.getItem(key) ?? "{}"), KEY) - -/** What another browser tab saving the config looks like to this one */ -async function changeInAnotherTab(page: Page, change: (config: any) => void) { - await page.evaluate( - ([key, body]) => { - const config = JSON.parse(localStorage.getItem(key) ?? "{}") - new Function("config", body)(config) - const value = JSON.stringify(config) - localStorage.setItem(key, value) - window.dispatchEvent( - new StorageEvent("storage", { key, newValue: value }), - ) - }, - [KEY, `(${change.toString()})(config)`] as const, - ) -} - -const back = (dialog: Locator) => - dialog.locator("header button", { hasText: /^Models$/ }).click() - -/** Hold the requests to an endpoint; each waits for its own release */ -async function holdEach( - page: Page, - url: string, - answer: (n: number) => object, -) { - const releases: Array<() => void> = [] - await page.route(url, async (route) => { - const n = releases.length - await new Promise((r) => releases.push(r)) - await route.fulfill({ json: answer(n) }) - }) - return releases -} - -const TWO = { - version: 1, - providers: [ - { - id: "p1", - provider: "glm", - apiKey: "k1", - models: [{ id: "a1", modelId: "glm-a" }], - }, - { - id: "p2", - provider: "kimi", - apiKey: "k2", - models: [{ id: "b1", modelId: "kimi-b" }], - }, - ], -} - -test("Add a provider… leaves the cursor in the provider search", async ({ - page, -}) => { - await openApp(page) - await page.getByTestId("model-selector").click() - await page.getByText("Add a provider…").click() - const search = dialogOf(page).getByRole("searchbox") - await expect(search).toBeFocused() - // After the model picker has closed - await page.waitForTimeout(800) - await expect(search).toBeFocused() - await page.keyboard.type("kimi") - await expect(search).toHaveValue("kimi") -}) - -test("the model picker's footer is there while searching and reachable by Tab", async ({ - page, -}) => { - await openApp(page) - await page.getByTestId("model-selector").click() - await page.keyboard.type("zzzz") - await expect(page.getByText("Add a provider…")).toBeVisible() - const add = page.getByRole("button", { name: "Add a provider…" }) - for (let i = 0; i < 6; i++) { - if (await add.evaluate((el) => el === document.activeElement)) break - await page.keyboard.press("Tab") - } - await expect(add).toBeFocused() - await page.keyboard.press("Enter") - await expect(dialogOf(page).getByRole("searchbox")).toBeVisible() -}) - -test("an Enter that confirms an IME candidate picks no provider", async ({ - page, -}) => { - await openApp(page) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByRole("button", { name: /See all/ }).click() - const search = dialog.getByRole("searchbox") - await search.fill("deep") - await search.evaluate((el) => - el.dispatchEvent( - new KeyboardEvent("keydown", { - key: "Enter", - bubbles: true, - isComposing: true, - }), - ), - ) - await page.waitForTimeout(300) - await expect(search).toBeVisible() - expect((await saved(page)).providers ?? []).toHaveLength(0) -}) - -test("the focus follows the page within the models tab", async ({ page }) => { - await openApp(page, TWO) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-p1").focus() - await page.keyboard.press("Enter") - await expect( - dialog.getByRole("heading", { name: "GLM (Zhipu)", exact: true }), - ).toBeFocused() - await back(dialog) - await expect( - dialog.getByRole("heading", { name: "Models", exact: true }), - ).toBeFocused() -}) - -test("long names stay inside the page", async ({ page }) => { - const long = `model-${"x".repeat(150)}` - await openApp(page, { - version: 1, - providers: [ - { - id: "p1", - provider: "openai", - name: `My ${"provider ".repeat(10)}name`, - apiKey: "k", - models: [{ id: "m1", modelId: long }], - }, - ], - }) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-p1").click() - const right = async (l: Locator) => { - const box = await l.boundingBox() - return box ? box.x + box.width : 0 - } - const pageRight = await right(dialog) - expect(await right(dialog.getByTestId("test-models"))).toBeLessThanOrEqual( - pageRight, - ) - expect( - await right( - dialog.getByRole("button", { name: /^Get all models from/ }), - ), - ).toBeLessThanOrEqual(pageRight) -}) - -test("Azure needs its resource URL before it counts as connected", async ({ - page, -}) => { - await openApp(page, { - version: 1, - providers: [ - { - id: "z1", - provider: "azure", - apiKey: "k", - baseUrl: "https://your-resource.openai.azure.com/openai", - models: [{ id: "m1", modelId: "gpt-6-astra" }], - }, - ], - }) - await openSettings(page) - const dialog = dialogOf(page) - await expect(dialog.getByTestId("provider-row-z1")).toContainText( - "Not set up yet", - ) - await dialog.getByTestId("provider-row-z1").click() - await expect(dialog.getByTestId("step-1-todo")).toBeVisible() - await expect(dialog.getByTestId("test-models")).toBeDisabled() - await expect( - dialog.getByText( - "Fill in the API key and your Azure resource's base URL first.", - ), - ).toBeVisible() - await dialog - .locator("#base-url") - .fill("https://contoso.openai.azure.com/openai") - await expect(dialog.getByTestId("test-models")).toBeEnabled() -}) - -test("a refused key unchecks step 1, and a fix in another tab clears it", async ({ - page, -}) => { - await page.route("**/api/validate-model", (route) => - route.fulfill({ - json: { valid: false, code: "invalid_api_key", error: "Bad key" }, - }), - ) - await openApp(page, TWO) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-p1").click() - await dialog.getByTestId("test-models").click() - const key = dialog.locator("#api-key") - await expect(key).toHaveAttribute("aria-invalid", "true") - await expect(dialog.getByTestId("step-1-todo")).toBeVisible() - await changeInAnotherTab(page, (config) => { - config.providers[0].apiKey = "fixed-key" - config.providers[0].models[0].validated = undefined - config.providers[0].models[0].validationError = undefined - }) - await expect(key).toHaveValue("fixed-key") - await expect(key).not.toHaveAttribute("aria-invalid", "true") - await expect(dialog.getByTestId("step-1-done")).toBeVisible() - await expect(dialog.getByText("1 model failed the test")).toHaveCount(0) -}) - -test("testing another provider keeps the first one's results", async ({ - page, -}) => { - const releases = await holdEach(page, "**/api/validate-model", () => ({ - valid: true, - responseTime: 1000, - })) - await openApp(page, TWO) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-p1").click() - await dialog.getByTestId("test-models").click() - await expect.poll(() => releases.length).toBe(1) - await back(dialog) - await dialog.getByTestId("provider-row-p2").click() - await dialog.getByTestId("test-models").click() - await expect.poll(() => releases.length).toBe(2) - for (const release of releases) release() - await expect - .poll(async () => - (await saved(page)).providers.map( - (p: any) => p.models[0].validated, - ), - ) - .toEqual([true, true]) -}) - -test("an older test from before the tabs changed does not undo a newer one", async ({ - page, -}) => { - // The first test fails, the second passes; the first answers last - const releases = await holdEach(page, "**/api/validate-model", (n) => - n === 0 - ? { valid: false, error: "Old failure" } - : { valid: true, responseTime: 1000 }, - ) - await openApp(page, TWO) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-p1").click() - await dialog.getByTestId("test-models").click() - await expect.poll(() => releases.length).toBe(1) - await dialog.getByTestId("settings-tab-general").click() - await dialog.getByTestId("settings-tab-models").click() - await dialog.getByTestId("test-models").click() - await expect.poll(() => releases.length).toBe(2) - releases[1]() - await expect(dialog.getByText("Works · 1.0 s")).toBeVisible() - releases[0]() - await page.waitForTimeout(500) - await expect(dialog.getByText("Works · 1.0 s")).toBeVisible() - expect((await saved(page)).providers[0].models[0].validated).toBe(true) -}) - -test("a model list fetched with a key another tab replaced is not offered", async ({ - page, -}) => { - await page.route("**/api/provider-models", (route) => - route.fulfill({ json: { models: [{ id: "model-of-old-key" }] } }), - ) - await openApp(page, TWO) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-p1").click() - await dialog.getByRole("button", { name: /^Get all models from/ }).click() - await expect( - page.getByRole("option", { name: "model-of-old-key" }), - ).toBeVisible() - await page.keyboard.press("Escape") - await changeInAnotherTab(page, (config) => { - config.providers[0].apiKey = "another-key" - }) - await expect(dialog.locator("#api-key")).toHaveValue("another-key") - // GLM has no suggested models: nothing to browse until fetched again - await expect( - dialog.getByRole("button", { name: "Browse models" }), - ).toHaveCount(0) -}) - -test("a model list request keeps only its own provider busy", async ({ - page, -}) => { - const releases = await holdEach(page, "**/api/provider-models", () => ({ - models: [], - })) - await openApp(page, TWO) - await openSettings(page) - const dialog = dialogOf(page) - const fetchButton = dialog.getByRole("button", { - name: /^Get all models from/, - }) - await dialog.getByTestId("provider-row-p1").click() - await fetchButton.click() - await expect.poll(() => releases.length).toBe(1) - await expect(fetchButton).toBeDisabled() - await back(dialog) - await dialog.getByTestId("provider-row-p2").click() - await expect(fetchButton).toBeEnabled() - for (const release of releases) release() -}) - -test("a model id typed for one provider does not follow to the next", async ({ - page, -}) => { - await openApp(page, TWO) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-p1").click() - const input = dialog.getByPlaceholder("Or type another model ID") - await input.fill("half-typed") - await back(dialog) - await dialog.getByTestId("provider-row-p2").click() - await expect(input).toHaveValue("") -}) - -test("Bedrock shows the sign-in that is saved, also after another tab", async ({ - page, -}) => { - await openApp(page, { - version: 1, - providers: [ - { - id: "b1", - provider: "bedrock", - apiKey: "", - awsAccessKeyId: "AKIAOLD", - awsSecretAccessKey: "old-secret", - awsRegion: "us-east-1", - models: [{ id: "m1", modelId: "amazon.nova-lite-v1:0" }], - }, - ], - }) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-b1").click() - await expect(dialog.locator("#aws-access-key-id")).toHaveValue("AKIAOLD") - // Another tab signs the provider in with an API key instead - await changeInAnotherTab(page, (config) => { - config.providers[0].apiKey = "bedrock-key" - config.providers[0].awsAccessKeyId = "" - config.providers[0].awsSecretAccessKey = "" - }) - await expect(dialog.locator("#bedrock-api-key")).toHaveValue("bedrock-key") - await expect(dialog.locator("#aws-access-key-id")).toHaveCount(0) -}) - -test("a refused key opens the provider the request went to", async ({ - page, -}) => { - // The request waits while another tab selects another provider's model - let release!: () => void - const released = new Promise((r) => { - release = r - }) - const sentWith: string[] = [] - await page.route("**/api/chat", async (route) => { - sentWith.push(route.request().headers()["x-ai-provider"] ?? "") - await released - await route.fulfill({ - status: 401, - contentType: "application/json", - body: JSON.stringify({ - type: "provider", - code: "invalid_api_key", - message: "Incorrect API key", - }), - }) - }) - await openApp(page, { ...TWO, selectedModelId: "a1" }) - await sendMessage(page, "Draw a box") - // Sent to GLM; only then does the other tab pick Kimi's model - await expect.poll(() => sentWith).toEqual(["glm"]) - await changeInAnotherTab(page, (config) => { - config.selectedModelId = "b1" - }) - await expect(page.getByTestId("model-selector")).toContainText("kimi-b") - release() - await page - .getByRole("button", { name: "Open model settings" }) - .click({ timeout: 15000 }) - await expect( - dialogOf(page).getByRole("heading", { - name: "GLM (Zhipu)", - exact: true, - }), - ).toBeVisible() -}) - -test("a local server's base URL is shown from the start", async ({ page }) => { - await openApp(page, { - version: 1, - providers: [{ id: "l1", provider: "ollama", apiKey: "", models: [] }], - }) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-l1").click() - await expect(dialog.locator("#base-url")).toBeVisible() -}) - -test("refused access keys are named as such", async ({ page }) => { - await page.route("**/api/validate-model", (route) => - route.fulfill({ - json: { valid: false, code: "invalid_api_key", error: "Bad keys" }, - }), - ) - await openApp(page, { - version: 1, - providers: [ - { - id: "b1", - provider: "bedrock", - apiKey: "", - awsAccessKeyId: "AKIA1", - awsSecretAccessKey: "s", - awsRegion: "us-east-1", - models: [{ id: "m1", modelId: "amazon.nova-lite-v1:0" }], - }, - ], - }) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-b1").click() - await dialog.getByTestId("test-models").click() - await expect( - dialog.getByText( - "AWS rejected these access keys. Check that both were copied in full. Bad keys", - ), - ).toBeVisible() - for (const id of ["#aws-access-key-id", "#aws-secret-access-key"]) { - await expect(dialog.locator(id)).toHaveAttribute("aria-invalid", "true") - } -}) - -// Found by the review of the fixes above - -test("a slow model list keeps its provider busy while another loads", async ({ - page, -}) => { - const releases = await holdEach(page, "**/api/provider-models", () => ({ - models: [{ id: "listed-model" }], - })) - await openApp(page, TWO) - await openSettings(page) - const dialog = dialogOf(page) - const fetchButton = dialog.getByRole("button", { - name: /^Get all models from/, - }) - await dialog.getByTestId("provider-row-p1").click() - await fetchButton.click() - await expect.poll(() => releases.length).toBe(1) - await back(dialog) - await dialog.getByTestId("provider-row-p2").click() - await fetchButton.click() - await expect.poll(() => releases.length).toBe(2) - releases[1]() - // B's list opens its picker; close it - await expect( - page.getByRole("option", { name: "listed-model" }), - ).toBeVisible() - await page.keyboard.press("Escape") - await expect(page.getByRole("listbox")).toHaveCount(0) - await back(dialog) - await dialog.getByTestId("provider-row-p1").click() - // The first request is still out - await expect(fetchButton).toBeDisabled() - releases[0]() - // A's list arrives and opens its picker; afterwards A can fetch again - await expect( - page.getByRole("option", { name: "listed-model" }), - ).toBeVisible() - await page.keyboard.press("Escape") - await expect(fetchButton).toBeEnabled() -}) - -test("refused keys stay marked for each provider", async ({ page }) => { - await page.route("**/api/validate-model", (route) => - route.fulfill({ - json: { valid: false, code: "invalid_api_key", error: "Bad key" }, - }), - ) - await openApp(page, TWO) - await openSettings(page) - const dialog = dialogOf(page) - const key = dialog.locator("#api-key") - await dialog.getByTestId("provider-row-p1").click() - await dialog.getByTestId("test-models").click() - await expect(key).toHaveAttribute("aria-invalid", "true") - await back(dialog) - await dialog.getByTestId("provider-row-p2").click() - await dialog.getByTestId("test-models").click() - await expect(key).toHaveAttribute("aria-invalid", "true") - await back(dialog) - await dialog.getByTestId("provider-row-p1").click() - await expect(key).toHaveAttribute("aria-invalid", "true") -}) - -test("a model list that loads leaves a refused test's mark", async ({ - page, -}) => { - // Some providers list their models without the key: a list proves nothing - await page.route("**/api/validate-model", (route) => - route.fulfill({ - json: { valid: false, code: "invalid_api_key", error: "Bad key" }, - }), - ) - await page.route("**/api/provider-models", (route) => - route.fulfill({ json: { models: [{ id: "listed-model" }] } }), - ) - await openApp(page, TWO) - await openSettings(page) - const dialog = dialogOf(page) - const key = dialog.locator("#api-key") - await dialog.getByTestId("provider-row-p1").click() - await dialog.getByTestId("test-models").click() - await expect(key).toHaveAttribute("aria-invalid", "true") - await dialog.getByRole("button", { name: /^Get all models from/ }).click() - await expect( - page.getByRole("option", { name: "listed-model" }), - ).toBeVisible() - await page.keyboard.press("Escape") - await expect(page.getByRole("listbox")).toHaveCount(0) - await expect(key).toHaveAttribute("aria-invalid", "true") -}) - -test("an empty model list says so and opens nothing", async ({ page }) => { - await page.route("**/api/provider-models", (route) => - route.fulfill({ json: { models: [] } }), - ) - // OpenAI has suggested models: they come back once the key changes - await openApp(page, { - version: 1, - providers: [{ id: "o1", provider: "openai", apiKey: "k", models: [] }], - }) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-o1").click() - await dialog.getByRole("button", { name: /^Get all models from/ }).click() - await expect( - dialog.getByText("The provider returned no models."), - ).toBeVisible() - await dialog.locator("#api-key").fill("k-new") - await expect( - dialog.getByRole("button", { name: "Browse models" }), - ).toBeVisible() - await page.waitForTimeout(300) - // The picker did not open by itself - await expect(page.getByRole("listbox")).toHaveCount(0) - await expect(dialog.locator("#api-key")).toBeFocused() -}) - -test("emptying Bedrock's access keys to type new ones keeps their fields", async ({ - page, -}) => { - await openApp(page, { - version: 1, - providers: [ - { - id: "b1", - provider: "bedrock", - apiKey: "", - awsAccessKeyId: "AKIAOLD", - awsSecretAccessKey: "old-secret", - awsRegion: "us-east-1", - models: [{ id: "m1", modelId: "amazon.nova-lite-v1:0" }], - }, - ], - }) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-b1").click() - await dialog.locator("#aws-secret-access-key").fill("") - await dialog.locator("#aws-access-key-id").fill("") - await expect(dialog.locator("#bedrock-api-key")).toHaveCount(0) - await dialog.locator("#aws-access-key-id").fill("AKIANEW") - await expect - .poll(async () => (await saved(page)).providers[0]) - .toMatchObject({ apiKey: "", awsAccessKeyId: "AKIANEW" }) -}) - -test("a provider added in another tab while sending is the one its error opens", async ({ - page, -}) => { - const sentWith: string[] = [] - await page.route("**/api/chat", async (route) => { - sentWith.push(route.request().headers()["x-ai-provider"] ?? "") - await route.fulfill({ - status: 401, - contentType: "application/json", - body: JSON.stringify({ - type: "provider", - code: "invalid_api_key", - message: "Incorrect API key", - }), - }) - }) - await openApp(page, { ...TWO, selectedModelId: "a1" }) - await page.getByTestId("chat-input").fill("Draw a box") - // Send, and right away another tab adds DeepSeek and picks its model, - // while this tab still prepares the request (it exports the diagram) - await page.evaluate((key) => { - const send = document.querySelector( - 'button[aria-label="Send"]', - ) - send?.click() - const config = JSON.parse(localStorage.getItem(key) ?? "{}") - config.providers.push({ - id: "p3", - provider: "deepseek", - apiKey: "k3", - models: [{ id: "c1", modelId: "deepseek-chat" }], - }) - config.selectedModelId = "c1" - const value = JSON.stringify(config) - localStorage.setItem(key, value) - window.dispatchEvent( - new StorageEvent("storage", { key, newValue: value }), - ) - }, KEY) - await expect.poll(() => sentWith).toEqual(["deepseek"]) - await page - .getByRole("button", { name: "Open model settings" }) - .click({ timeout: 15000 }) - await expect( - dialogOf(page).getByRole("heading", { name: "DeepSeek", exact: true }), - ).toBeVisible() -}) - -// After #957 merged - -test("temporary Bedrock access keys can carry their session token", async ({ - page, -}) => { - const bodies: any[] = [] - await page.route("**/api/validate-model", async (route) => { - bodies.push(route.request().postDataJSON()) - await route.fulfill({ json: { valid: true, responseTime: 900 } }) - }) - await openApp(page, { - version: 1, - providers: [ - { - id: "b1", - provider: "bedrock", - apiKey: "", - awsAccessKeyId: "ASIATEMP", - awsSecretAccessKey: "secret", - awsRegion: "us-east-1", - models: [{ id: "m1", modelId: "amazon.nova-lite-v1:0" }], - }, - ], - }) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-b1").click() - await dialog.getByLabel("AWS session token (optional)").fill("session-1") - await dialog.getByTestId("test-models").click() - await expect.poll(() => bodies.length).toBe(1) - expect(bodies[0]).toMatchObject({ - awsAccessKeyId: "ASIATEMP", - awsSessionToken: "session-1", - }) - expect((await saved(page)).providers[0].awsSessionToken).toBe("session-1") -}) - -test("Bedrock with both kinds of credentials offers to remove the access keys", async ({ - page, -}) => { - await openApp(page, { - version: 1, - providers: [ - { - id: "b1", - provider: "bedrock", - apiKey: "bedrock-key", - awsAccessKeyId: "AKIAOLD", - awsSecretAccessKey: "old-secret", - awsSessionToken: "old-token", - awsRegion: "us-east-1", - models: [{ id: "m1", modelId: "amazon.nova-lite-v1:0" }], - }, - ], - }) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-b1").click() - const notice = dialog.getByText(/also keeps an access key pair/) - await expect(notice).toBeVisible() - await dialog.getByRole("button", { name: "Remove the access keys" }).click() - await expect(notice).toHaveCount(0) - await expect - .poll(async () => (await saved(page)).providers[0]) - .toMatchObject({ - apiKey: "bedrock-key", - awsAccessKeyId: "", - awsSecretAccessKey: "", - awsSessionToken: "", - }) - await expect(dialog.locator("#bedrock-api-key")).toHaveValue("bedrock-key") -}) - -test("the test result goes to a live region that is there from the start", async ({ - page, -}) => { - await page.route("**/api/validate-model", (route) => - route.fulfill({ json: { valid: true, responseTime: 900 } }), - ) - await openApp(page, TWO) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-p1").click() - const result = dialog.getByTestId("test-result") - await expect(result).toHaveAttribute("role", "status") - await expect(result).toHaveText("") - await dialog.getByTestId("test-models").click() - await expect(result).toHaveText("glm-a works") -}) - -test("removing access keys kept behind an API key keeps the test results", async ({ - page, -}) => { - await openApp(page, { - version: 1, - providers: [ - { - id: "b1", - provider: "bedrock", - apiKey: "bedrock-key", - awsAccessKeyId: "AKIAOLD", - awsSecretAccessKey: "old-secret", - awsRegion: "us-east-1", - models: [ - { - id: "m1", - modelId: "amazon.nova-lite-v1:0", - validated: true, - }, - ], - }, - ], - }) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-b1").click() - await dialog.getByRole("button", { name: "Remove the access keys" }).focus() - await page.keyboard.press("Enter") - await expect - .poll(async () => (await saved(page)).providers[0].awsAccessKeyId) - .toBe("") - // The API key was the one in use: its result still holds - expect((await saved(page)).providers[0].models[0].validated).toBe(true) - await expect(dialog.locator("#bedrock-api-key")).toBeFocused() -}) - -test("a passing test hands the focus to its next step, a failing one keeps it", async ({ - page, -}) => { - let pass = false - await page.route("**/api/validate-model", (route) => - route.fulfill({ - json: pass - ? { valid: true, responseTime: 900 } - : { valid: false, error: "Model not found" }, - }), - ) - await openApp(page, TWO) - await openSettings(page) - const dialog = dialogOf(page) - await dialog.getByTestId("provider-row-p1").click() - const testButton = dialog.getByTestId("test-models") - await testButton.focus() - await page.keyboard.press("Enter") - await expect(dialog.getByTestId("test-result")).toHaveText( - "1 model failed the test", - ) - await expect(testButton).toBeFocused() - - pass = true - await page.keyboard.press("Enter") - const use = dialog.getByRole("button", { name: "Use glm-a in the chat" }) - await expect(use).toBeFocused() - const resultId = await dialog.getByTestId("test-result").getAttribute("id") - await expect(use).toHaveAttribute("aria-describedby", resultId ?? "") -}) diff --git a/tests/e2e/settings-tabs-sync.spec.ts b/tests/e2e/settings-tabs-sync.spec.ts new file mode 100644 index 00000000..9ff320cc --- /dev/null +++ b/tests/e2e/settings-tabs-sync.spec.ts @@ -0,0 +1,292 @@ +// Settings changed by another browser tab, and answers that arrive late +import type { Page } from "@playwright/test" +import { + backToModels, + changeInAnotherTab, + expect, + MODEL_CONFIGS_KEY, + openApp, + openSettings, + savedModelConfigs, + sendMessage, + settingsDialog, + TWO_PROVIDERS, + test, +} from "./lib/fixtures" + +/** Hold the requests to an endpoint; each waits for its own release */ +async function holdEach( + page: Page, + url: string, + answer: (n: number) => object, +) { + const releases: Array<() => void> = [] + await page.route(url, async (route) => { + const n = releases.length + await new Promise((r) => releases.push(r)) + await route.fulfill({ json: answer(n) }) + }) + return releases +} + +test("a refused key unchecks step 1, and a fix in another tab clears it", async ({ + page, +}) => { + await page.route("**/api/validate-model", (route) => + route.fulfill({ + json: { valid: false, code: "invalid_api_key", error: "Bad key" }, + }), + ) + await openApp(page, { config: TWO_PROVIDERS }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-p1").click() + await dialog.getByTestId("test-models").click() + const key = dialog.locator("#api-key") + await expect(key).toHaveAttribute("aria-invalid", "true") + await expect(dialog.getByTestId("step-1-todo")).toBeVisible() + await changeInAnotherTab(page, (config) => { + config.providers[0].apiKey = "fixed-key" + config.providers[0].models[0].validated = undefined + config.providers[0].models[0].validationError = undefined + }) + await expect(key).toHaveValue("fixed-key") + await expect(key).not.toHaveAttribute("aria-invalid", "true") + await expect(dialog.getByTestId("step-1-done")).toBeVisible() + await expect(dialog.getByText("1 model failed the test")).toHaveCount(0) +}) + +test("a model list fetched with a key another tab replaced is not offered", async ({ + page, +}) => { + await page.route("**/api/provider-models", (route) => + route.fulfill({ json: { models: [{ id: "model-of-old-key" }] } }), + ) + await openApp(page, { config: TWO_PROVIDERS }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-p1").click() + await dialog.getByRole("button", { name: /^Get all models from/ }).click() + await expect( + page.getByRole("option", { name: "model-of-old-key" }), + ).toBeVisible() + await page.keyboard.press("Escape") + await changeInAnotherTab(page, (config) => { + config.providers[0].apiKey = "another-key" + }) + await expect(dialog.locator("#api-key")).toHaveValue("another-key") + // GLM has no suggested models: nothing to browse until fetched again + await expect( + dialog.getByRole("button", { name: "Browse models" }), + ).toHaveCount(0) +}) + +test("a refused key opens the provider the request went to", async ({ + page, +}) => { + // The request waits while another tab selects another provider's model + let release!: () => void + const released = new Promise((r) => { + release = r + }) + const sentWith: string[] = [] + await page.route("**/api/chat", async (route) => { + sentWith.push(route.request().headers()["x-ai-provider"] ?? "") + await released + await route.fulfill({ + status: 401, + contentType: "application/json", + body: JSON.stringify({ + type: "provider", + code: "invalid_api_key", + message: "Incorrect API key", + }), + }) + }) + await openApp(page, { config: { ...TWO_PROVIDERS, selectedModelId: "a1" } }) + await sendMessage(page, "Draw a box") + // Sent to GLM; only then does the other tab pick Kimi's model + await expect.poll(() => sentWith).toEqual(["glm"]) + await changeInAnotherTab(page, (config) => { + config.selectedModelId = "b1" + }) + await expect(page.getByTestId("model-selector")).toContainText("kimi-b") + release() + await page + .getByRole("button", { name: "Open model settings" }) + .click({ timeout: 15000 }) + await expect( + settingsDialog(page).getByRole("heading", { + name: "GLM (Zhipu)", + exact: true, + }), + ).toBeVisible() +}) + +test("a provider added in another tab while sending is the one its error opens", async ({ + page, +}) => { + const sentWith: string[] = [] + await page.route("**/api/chat", async (route) => { + sentWith.push(route.request().headers()["x-ai-provider"] ?? "") + await route.fulfill({ + status: 401, + contentType: "application/json", + body: JSON.stringify({ + type: "provider", + code: "invalid_api_key", + message: "Incorrect API key", + }), + }) + }) + await openApp(page, { config: { ...TWO_PROVIDERS, selectedModelId: "a1" } }) + await page.getByTestId("chat-input").fill("Draw a box") + // Send, and right away another tab adds DeepSeek and picks its model, + // while this tab still prepares the request (it exports the diagram) + await page.evaluate((key) => { + const send = document.querySelector( + 'button[aria-label="Send"]', + ) + send?.click() + const config = JSON.parse(localStorage.getItem(key) ?? "{}") + config.providers.push({ + id: "p3", + provider: "deepseek", + apiKey: "k3", + models: [{ id: "c1", modelId: "deepseek-chat" }], + }) + config.selectedModelId = "c1" + const value = JSON.stringify(config) + localStorage.setItem(key, value) + window.dispatchEvent( + new StorageEvent("storage", { key, newValue: value }), + ) + }, MODEL_CONFIGS_KEY) + await expect.poll(() => sentWith).toEqual(["deepseek"]) + await page + .getByRole("button", { name: "Open model settings" }) + .click({ timeout: 15000 }) + await expect( + settingsDialog(page).getByRole("heading", { + name: "DeepSeek", + exact: true, + }), + ).toBeVisible() +}) + +test("testing another provider keeps the first one's results", async ({ + page, +}) => { + const releases = await holdEach(page, "**/api/validate-model", () => ({ + valid: true, + responseTime: 1000, + })) + await openApp(page, { config: TWO_PROVIDERS }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-p1").click() + await dialog.getByTestId("test-models").click() + await expect.poll(() => releases.length).toBe(1) + await backToModels(dialog) + await dialog.getByTestId("provider-row-p2").click() + await dialog.getByTestId("test-models").click() + await expect.poll(() => releases.length).toBe(2) + for (const release of releases) release() + await expect + .poll(async () => + (await savedModelConfigs(page)).providers.map( + (p: any) => p.models[0].validated, + ), + ) + .toEqual([true, true]) +}) + +test("an older test from before the tabs changed does not undo a newer one", async ({ + page, +}) => { + // The first test fails, the second passes; the first answers last + const releases = await holdEach(page, "**/api/validate-model", (n) => + n === 0 + ? { valid: false, error: "Old failure" } + : { valid: true, responseTime: 1000 }, + ) + await openApp(page, { config: TWO_PROVIDERS }) + await openSettings(page) + const dialog = settingsDialog(page) + await dialog.getByTestId("provider-row-p1").click() + await dialog.getByTestId("test-models").click() + await expect.poll(() => releases.length).toBe(1) + await dialog.getByTestId("settings-tab-general").click() + await dialog.getByTestId("settings-tab-models").click() + await dialog.getByTestId("test-models").click() + await expect.poll(() => releases.length).toBe(2) + releases[1]() + await expect(dialog.getByText("Works · 1.0 s")).toBeVisible() + releases[0]() + await page.waitForTimeout(500) + await expect(dialog.getByText("Works · 1.0 s")).toBeVisible() + expect( + (await savedModelConfigs(page)).providers[0].models[0].validated, + ).toBe(true) +}) + +test("a model list request keeps only its own provider busy", async ({ + page, +}) => { + const releases = await holdEach(page, "**/api/provider-models", () => ({ + models: [], + })) + await openApp(page, { config: TWO_PROVIDERS }) + await openSettings(page) + const dialog = settingsDialog(page) + const fetchButton = dialog.getByRole("button", { + name: /^Get all models from/, + }) + await dialog.getByTestId("provider-row-p1").click() + await fetchButton.click() + await expect.poll(() => releases.length).toBe(1) + await expect(fetchButton).toBeDisabled() + await backToModels(dialog) + await dialog.getByTestId("provider-row-p2").click() + await expect(fetchButton).toBeEnabled() + for (const release of releases) release() +}) + +test("a slow model list keeps its provider busy while another loads", async ({ + page, +}) => { + const releases = await holdEach(page, "**/api/provider-models", () => ({ + models: [{ id: "listed-model" }], + })) + await openApp(page, { config: TWO_PROVIDERS }) + await openSettings(page) + const dialog = settingsDialog(page) + const fetchButton = dialog.getByRole("button", { + name: /^Get all models from/, + }) + await dialog.getByTestId("provider-row-p1").click() + await fetchButton.click() + await expect.poll(() => releases.length).toBe(1) + await backToModels(dialog) + await dialog.getByTestId("provider-row-p2").click() + await fetchButton.click() + await expect.poll(() => releases.length).toBe(2) + releases[1]() + // B's list opens its picker; close it + await expect( + page.getByRole("option", { name: "listed-model" }), + ).toBeVisible() + await page.keyboard.press("Escape") + await expect(page.getByRole("listbox")).toHaveCount(0) + await backToModels(dialog) + await dialog.getByTestId("provider-row-p1").click() + // The first request is still out + await expect(fetchButton).toBeDisabled() + releases[0]() + // A's list arrives and opens its picker; afterwards A can fetch again + await expect( + page.getByRole("option", { name: "listed-model" }), + ).toBeVisible() + await page.keyboard.press("Escape") + await expect(fetchButton).toBeEnabled() +}) diff --git a/tests/e2e/settings.spec.ts b/tests/e2e/settings.spec.ts index 57286608..070dbdfd 100644 --- a/tests/e2e/settings.spec.ts +++ b/tests/e2e/settings.spec.ts @@ -1,6 +1,6 @@ import { expect, - getIframe, + openApp, openSettings, openSettingsTab, test, @@ -8,8 +8,7 @@ import { test.describe("Settings", () => { test.beforeEach(async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openApp(page) }) test("settings dialog opens", async ({ page }) => { diff --git a/tests/e2e/smoke.spec.ts b/tests/e2e/smoke.spec.ts index ffc4795a..be851e83 100644 --- a/tests/e2e/smoke.spec.ts +++ b/tests/e2e/smoke.spec.ts @@ -1,4 +1,4 @@ -import { expect, getIframe, openSettings, test } from "./lib/fixtures" +import { expect, getIframe, test } from "./lib/fixtures" test.describe("Smoke Tests", () => { test("homepage loads without errors", async ({ page }) => { @@ -26,11 +26,4 @@ test.describe("Smoke Tests", () => { expect(errors).toEqual([]) }) - - test("settings dialog opens", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - await openSettings(page) - }) }) diff --git a/tests/e2e/theme.spec.ts b/tests/e2e/theme.spec.ts deleted file mode 100644 index 2c7a8a43..00000000 --- a/tests/e2e/theme.spec.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { expect, getIframe, openSettings, sleep, test } from "./lib/fixtures" - -test.describe("Theme Switching", () => { - test("can toggle app dark mode", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - await openSettings(page) - - const html = page.locator("html") - const initialClass = await html.getAttribute("class") - - const themeButton = page.locator( - "button:has(svg.lucide-sun), button:has(svg.lucide-moon)", - ) - - if ((await themeButton.count()) > 0) { - await test.step("toggle theme", async () => { - await themeButton.first().click() - await sleep(500) - }) - - await test.step("verify theme changed", async () => { - const newClass = await html.getAttribute("class") - expect(newClass).not.toBe(initialClass) - }) - } - }) - - test("theme persists after page reload", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - await openSettings(page) - - const themeButton = page.locator( - "button:has(svg.lucide-sun), button:has(svg.lucide-moon)", - ) - - if ((await themeButton.count()) > 0) { - let themeClass: string | null - - await test.step("change theme", async () => { - await themeButton.first().click() - await sleep(300) - themeClass = await page.locator("html").getAttribute("class") - await page.keyboard.press("Escape") - }) - - await test.step("reload page", async () => { - await page.reload({ waitUntil: "networkidle" }) - await getIframe(page).waitFor({ - state: "visible", - timeout: 30000, - }) - }) - - await test.step("verify theme persisted", async () => { - const reloadedClass = await page - .locator("html") - .getAttribute("class") - expect(reloadedClass).toBe(themeClass) - }) - } - }) - - test("draw.io theme toggle exists", async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - await openSettings(page) - - await expect( - page.locator('[role="dialog"], [role="menu"], form').first(), - ).toBeVisible({ timeout: 5000 }) - }) - - test("system theme preference is respected", async ({ page }) => { - await page.emulateMedia({ colorScheme: "dark" }) - - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - - const html = page.locator("html") - const classes = await html.getAttribute("class") - expect(classes).toBeDefined() - }) -}) diff --git a/tests/e2e/upload.spec.ts b/tests/e2e/upload.spec.ts deleted file mode 100644 index ec2a3395..00000000 --- a/tests/e2e/upload.spec.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { expect, getIframe, test } from "./lib/fixtures" - -test.describe("File Upload Area", () => { - test.beforeEach(async ({ page }) => { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) - }) - - test("page loads without console errors", async ({ page }) => { - const errors: string[] = [] - page.on("pageerror", (err) => errors.push(err.message)) - - await page.waitForTimeout(1000) - - const criticalErrors = errors.filter( - (e) => !e.includes("ResizeObserver") && !e.includes("Script error"), - ) - expect(criticalErrors).toEqual([]) - }) -}) diff --git a/tests/e2e/workspace.spec.ts b/tests/e2e/workspace.spec.ts index bd1f8c6d..13f1838a 100644 --- a/tests/e2e/workspace.spec.ts +++ b/tests/e2e/workspace.spec.ts @@ -1,6 +1,7 @@ import type { Locator, Page, Route } from "@playwright/test" import { FLOWCHART_XML } from "./fixtures/diagrams" import { + chunkedReplies, expect, getAttachmentInput, getChatInput, @@ -8,13 +9,14 @@ import { getIframeContent, getSendButton, openDrawioFile, + openApp as openPage, openSettingsTab, sendMessage, test, waitForComplete, waitForCompleteCount, } from "./lib/fixtures" -import { createMockSSEResponse } from "./lib/helpers" +import { createMockSSEResponse, sse, toolCallEvents } from "./lib/helpers" /** An edit_diagram response that adds one node next to "Start" */ function createEditResponse() { @@ -35,10 +37,7 @@ function createEditResponse() { }, { type: "finish" }, ] - return ( - events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join("") + - "data: [DONE]\n\n" - ) + return sse(events) } /** First request draws the flowchart, later ones add a node */ @@ -60,8 +59,7 @@ function drawThenEdit(onRequest?: (body: any) => void) { } async function openApp(page: Page) { - await page.goto("/", { waitUntil: "networkidle" }) - await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await openPage(page) // The cover goes away once draw.io has loaded and been styled await expect(page.locator('[data-testid="canvas-loading"]')).toHaveCount( 0, @@ -900,9 +898,7 @@ test.describe("Workspace", () => { route.fulfill({ status: 200, contentType: "text/event-stream", - body: events - .map((e) => `data: ${JSON.stringify(e)}\n\n`) - .join(""), + body: sse(events, false), }), ) await openApp(page) @@ -2003,46 +1999,9 @@ test.describe("Workspace", () => { // Edge cases found in review // --------------------------------------------------------------------------- -const sseOf = (events: object[], done = true) => - events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join("") + - (done ? "data: [DONE]\n\n" : "") const boxCell = (id: string, label: string, x: number, parent = "1") => `` -/** - * Replies to /api/chat in order, each a list of chunks sent 800 ms apart. - * With holdLast, the last reply never ends. - */ -async function slowReplies(page: Page, replies: string[][], holdLast = false) { - await page.addInitScript( - ({ replies, holdLast }) => { - const realFetch = window.fetch - let n = 0 - window.fetch = async (input, init) => { - const url = input instanceof Request ? input.url : String(input) - if (!url.endsWith("/api/chat")) return realFetch(input, init) - const index = n++ - const chunks = replies[index] ?? [] - const body = new ReadableStream({ - async start(c) { - for (const chunk of chunks) { - c.enqueue(new TextEncoder().encode(chunk)) - await new Promise((r) => setTimeout(r, 800)) - } - if (!(holdLast && index === replies.length - 1)) { - c.close() - } - }, - }) - return new Response(body, { - headers: { "content-type": "text/event-stream" }, - }) - } - }, - { replies, holdLast }, - ) -} - /** Puts a saved chat into the browser's storage */ async function seedSession(page: Page, session: object) { await page.evaluate(async (session) => { @@ -2169,27 +2128,13 @@ async function twoChats(page: Page) { /** Streamed display_diagram deltas for this input, the last one held back */ function streamedDisplay(id: string, input: object) { - const deltas = (JSON.stringify(input).match(/[\s\S]{1,30}/g) ?? []).map( - (d) => ({ - type: "tool-input-delta", - toolCallId: id, - inputTextDelta: d, - }), - ) + const call = toolCallEvents(id, "display_diagram", input, 30) return { - head: sseOf( - [ - { type: "start" }, - { - type: "tool-input-start", - toolCallId: id, - toolName: "display_diagram", - }, - ...deltas.slice(0, -1), - ], + head: sse( + [{ type: "start" }, call.start, ...call.deltas.slice(0, -1)], false, ), - last: deltas.at(-1) as object, + last: call.deltas.at(-1) as object, } } @@ -2331,22 +2276,26 @@ test.describe("Edge cases", () => { xml: boxCell("a", "OldOne", 40) + boxCell("b", "NewOne", 240), } const stream = streamedDisplay("d2", input) - await slowReplies(page, [ - [createMockSSEResponse(boxCell("a", "OldOne", 40), "One.")], + await chunkedReplies( + page, [ - stream.head, - sseOf([ - stream.last, - { - type: "tool-input-available", - toolCallId: "d2", - toolName: "display_diagram", - input, - }, - { type: "finish" }, - ]), + [createMockSSEResponse(boxCell("a", "OldOne", 40), "One.")], + [ + stream.head, + sse([ + stream.last, + { + type: "tool-input-available", + toolCallId: "d2", + toolName: "display_diagram", + input, + }, + { type: "finish" }, + ]), + ], ], - ]) + { gapMs: 800 }, + ) await openApp(page) await sendMessage(page, "One") await waitForComplete(page) @@ -2372,10 +2321,14 @@ test.describe("Edge cases", () => { xml: boxCell("b", "Beta", 240) + boxCell("c", "Gamma", 440), } const stream = streamedDisplay("d2", input) - await slowReplies(page, [ - [createMockSSEResponse(boxCell("a", "Alpha", 40), "One.")], - [stream.head], - ]) + await chunkedReplies( + page, + [ + [createMockSSEResponse(boxCell("a", "Alpha", 40), "One.")], + [stream.head], + ], + { gapMs: 800 }, + ) await openApp(page) await sendMessage(page, "One") await waitForComplete(page) @@ -2477,7 +2430,7 @@ test.describe("Edge cases", () => { route.fulfill({ status: 200, contentType: "text/event-stream", - body: sseOf([ + body: sse([ { type: "start" }, { type: "tool-input-start", @@ -3073,7 +3026,7 @@ test.describe("Edge cases", () => { { type: "text-end", id }, ] const replies = [ - sseOf([ + sse([ { type: "start", messageId: "m1" }, { type: "start-step" }, ...call("d1", bad), @@ -3081,7 +3034,7 @@ test.describe("Edge cases", () => { { type: "finish-step" }, { type: "finish" }, ]), - sseOf([ + sse([ { type: "start", messageId: "m1" }, { type: "start-step" }, ...text("t2", "Second words."), @@ -3097,7 +3050,7 @@ test.describe("Edge cases", () => { contentType: "text/event-stream", body: replies[n++] ?? - sseOf([{ type: "start" }, { type: "finish" }]), + sse([{ type: "start" }, { type: "finish" }]), }), ) await openApp(page) @@ -3494,33 +3447,37 @@ test.describe("Edge cases", () => { toolCallId: "d1", inputTextDelta, }) - await slowReplies(page, [ + await chunkedReplies( + page, [ - sseOf( - [ - { type: "start" }, + [ + sse( + [ + { type: "start" }, + { + type: "tool-input-start", + toolCallId: "d1", + toolName: "display_diagram", + }, + ], + false, + ), + // The first preview shows only the near shape + sse([delta(json.slice(0, cut))], false), + sse([delta(json.slice(cut))], false), + sse([ { - type: "tool-input-start", + type: "tool-input-available", toolCallId: "d1", toolName: "display_diagram", + input: { xml: near + far }, }, - ], - false, - ), - // The first preview shows only the near shape - sseOf([delta(json.slice(0, cut))], false), - sseOf([delta(json.slice(cut))], false), - sseOf([ - { - type: "tool-input-available", - toolCallId: "d1", - toolName: "display_diagram", - input: { xml: near + far }, - }, - { type: "finish" }, - ]), + { type: "finish" }, + ]), + ], ], - ]) + { gapMs: 800 }, + ) await openApp(page) await openDrawioFile( page, @@ -3540,11 +3497,15 @@ test.describe("Edge cases", () => { const stream = streamedDisplay("d2", { xml: boxCell("b", "Beta", 240) + boxCell("c", "Gamma", 440), }) - await slowReplies(page, [ - [createMockSSEResponse(boxCell("a", "Alpha", 40), "One.")], - [stream.head], - [createMockSSEResponse(boxCell("d", "Delta", 40), "Again.")], - ]) + await chunkedReplies( + page, + [ + [createMockSSEResponse(boxCell("a", "Alpha", 40), "One.")], + [stream.head], + [createMockSSEResponse(boxCell("d", "Delta", 40), "Again.")], + ], + { gapMs: 800 }, + ) await openApp(page) await sendMessage(page, "One") await waitForComplete(page) @@ -3564,13 +3525,13 @@ test.describe("Edge cases", () => { const stream = streamedDisplay("d2", { xml: boxCell("b", "Beta", 240) + boxCell("c", "Gamma", 440), }) - await slowReplies( + await chunkedReplies( page, [ [createMockSSEResponse(boxCell("a", "Alpha", 40), "One.")], [stream.head], ], - true, + { gapMs: 800, holdLast: true }, ) await openApp(page) await sendMessage(page, "One") @@ -3612,39 +3573,43 @@ test.describe("Edge cases", () => { toolCallId: "e1", inputTextDelta, }) - await slowReplies(page, [ - [createMockSSEResponse(boxCell("a", "Alpha", 40), "One.")], + await chunkedReplies( + page, [ - sseOf( - [ - { type: "start" }, + [createMockSSEResponse(boxCell("a", "Alpha", 40), "One.")], + [ + sse( + [ + { type: "start" }, + { + type: "tool-input-start", + toolCallId: "e1", + toolName: "edit_diagram", + }, + ], + false, + ), + sse([delta(json.slice(0, split))], false), + // The operation's XML is complete; the call is not + sse([delta(json.slice(split, end))], false), + ": wait\n\n", + ": wait\n\n", + ": wait\n\n", + ": wait\n\n", + sse([ + delta(json.slice(end)), { - type: "tool-input-start", + type: "tool-input-available", toolCallId: "e1", toolName: "edit_diagram", + input, }, - ], - false, - ), - sseOf([delta(json.slice(0, split))], false), - // The operation's XML is complete; the call is not - sseOf([delta(json.slice(split, end))], false), - ": wait\n\n", - ": wait\n\n", - ": wait\n\n", - ": wait\n\n", - sseOf([ - delta(json.slice(end)), - { - type: "tool-input-available", - toolCallId: "e1", - toolName: "edit_diagram", - input, - }, - { type: "finish" }, - ]), + { type: "finish" }, + ]), + ], ], - ]) + { gapMs: 800 }, + ) await openApp(page) await sendMessage(page, "One") await waitForComplete(page) @@ -3915,14 +3880,14 @@ test.describe("Edge cases", () => { const stream = streamedDisplay("d3", { xml: boxCell("p", "PrevOne", 240) + boxCell("q", "QueueOne", 440), }) - await slowReplies( + await chunkedReplies( page, [ [createMockSSEResponse(boxCell("a", "BeeOne", 40), "B.")], [createMockSSEResponse(boxCell("a", "AyOne", 40), "A.")], [stream.head], ], - true, + { gapMs: 800, holdLast: true }, ) await openApp(page) const chatB = await twoChats(page) @@ -3954,27 +3919,31 @@ test.describe("Edge cases", () => { test("a reply of the chat left that ends late adds nothing to the chat on screen", async ({ page, }) => { - await slowReplies(page, [ - [createMockSSEResponse(boxCell("a", "BeeOne", 40), "B.")], - [createMockSSEResponse(boxCell("a", "AyOne", 40), "A.")], + await chunkedReplies( + page, [ - sseOf( - [ - { type: "start" }, - { - type: "tool-input-start", - toolCallId: "d3", - toolName: "display_diagram", - }, - ], - false, - ), - ": wait\n\n", - ": wait\n\n", - ": wait\n\n", - ": wait\n\n", + [createMockSSEResponse(boxCell("a", "BeeOne", 40), "B.")], + [createMockSSEResponse(boxCell("a", "AyOne", 40), "A.")], + [ + sse( + [ + { type: "start" }, + { + type: "tool-input-start", + toolCallId: "d3", + toolName: "display_diagram", + }, + ], + false, + ), + ": wait\n\n", + ": wait\n\n", + ": wait\n\n", + ": wait\n\n", + ], ], - ]) + { gapMs: 800 }, + ) await openApp(page) const chatB = await twoChats(page) await sendMessage(page, "Chat A two") @@ -4269,35 +4238,27 @@ test.describe("Edge cases", () => { test("Stop in one chat does not label another chat's cut off drawing", async ({ page, }) => { - await page.addInitScript(() => { - const sse = (events: object[]) => - events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join("") - const realFetch = window.fetch - window.fetch = async (input, init) => { - const url = input instanceof Request ? input.url : String(input) - if (!url.endsWith("/api/chat")) return realFetch(input, init) - const body = new ReadableStream({ - start(c) { - c.enqueue( - new TextEncoder().encode( - sse([ - { type: "start" }, - { type: "text-start", id: "t1" }, - { - type: "text-delta", - id: "t1", - delta: "Thinking about it.", - }, - ]), - ), - ) - }, - }) - return new Response(body, { - headers: { "content-type": "text/event-stream" }, - }) - } - }) + // The answer starts and never ends + await chunkedReplies( + page, + [ + [ + sse( + [ + { type: "start" }, + { type: "text-start", id: "t1" }, + { + type: "text-delta", + id: "t1", + delta: "Thinking about it.", + }, + ], + false, + ), + ], + ], + { holdLast: true }, + ) const doc = `${boxCell("a", "BeeOne", 40)}` await openApp(page) // Chat B was saved while a drawing streamed (its tab was closed) @@ -4349,77 +4310,48 @@ test.describe("Edge cases", () => { test("a drawing of the chat left that ends late leaves the canvas on screen as it was", async ({ page, }) => { - await page.addInitScript(() => { - const sse = (events: object[]) => - events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join("") - const cell = (id: string, label: string, x: number) => - `` - const realFetch = window.fetch - let n = 0 - window.fetch = async (input, init) => { - const url = input instanceof Request ? input.url : String(input) - if (!url.endsWith("/api/chat")) return realFetch(input, init) - const i = n++ - const body = new ReadableStream({ - async start(c) { - const send = (events: object[]) => - c.enqueue(new TextEncoder().encode(sse(events))) - if (i < 2) { - send([ - { type: "start" }, - { - type: "tool-input-start", - toolCallId: `x${i}`, - toolName: "display_diagram", - }, - { - type: "tool-input-available", - toolCallId: `x${i}`, - toolName: "display_diagram", - input: { - xml: cell( - "a", - i === 0 ? "BeeOne" : "AyOne", - 40, - ), - }, - }, - { type: "finish" }, - ]) - c.close() - return - } - send([ - { type: "start" }, - { - type: "tool-input-start", - toolCallId: "d3", - toolName: "display_diagram", - }, - ]) - // The chat is left meanwhile; then the drawing streams on - await new Promise((r) => setTimeout(r, 4000)) - const json = JSON.stringify({ - xml: - cell("p", "PrevOne", 240) + - cell("q", "QueueOne", 440), - }) - send([ - { - type: "tool-input-delta", - toolCallId: "d3", - inputTextDelta: json.slice(0, json.length - 30), - }, - ]) - await new Promise((r) => setTimeout(r, 2000)) - c.close() - }, - }) - return new Response(body, { - headers: { "content-type": "text/event-stream" }, - }) - } + const drawn = (id: string, label: string) => { + const call = toolCallEvents(id, "display_diagram", { + xml: boxCell("a", label, 40), + }) + return sse( + [{ type: "start" }, call.start, call.done, { type: "finish" }], + false, + ) + } + const json = JSON.stringify({ + xml: boxCell("p", "PrevOne", 240) + boxCell("q", "QueueOne", 440), }) + await chunkedReplies(page, [ + [drawn("x0", "BeeOne")], + [drawn("x1", "AyOne")], + [ + sse( + [ + { type: "start" }, + { + type: "tool-input-start", + toolCallId: "d3", + toolName: "display_diagram", + }, + ], + false, + ), + // The chat is left meanwhile; then the drawing streams on + 4000, + sse( + [ + { + type: "tool-input-delta", + toolCallId: "d3", + inputTextDelta: json.slice(0, json.length - 30), + }, + ], + false, + ), + 2000, + ], + ]) await openApp(page) const chatB = await twoChats(page) await sendMessage(page, "Chat A two") @@ -4670,7 +4602,7 @@ test.describe("Edge cases", () => { held.push(event) } }) - // Keeps the saved prompts' store busy, so the panel's click count waits + // Keeps the saved prompts' store busy, so the panel's run count waits w.__blockTemplates = () => new Promise((started) => { const request = indexedDB.open("next-ai-drawio-templates") @@ -4922,7 +4854,7 @@ test.describe("Edge cases", () => { ).toBeVisible() }) - test("a saved prompt goes out at once, also while its click count waits for storage", async ({ + test("a saved prompt goes out at once, also while its run count waits for storage", async ({ page, }) => { await page.addInitScript(() => { @@ -5137,7 +5069,7 @@ test.describe("Edge cases", () => { page, }) => { const first = `${boxCell("a", "FirstOne", 40)}` - const body = sseOf([ + const body = sse([ { type: "start" }, { type: "tool-input-start", @@ -5171,7 +5103,7 @@ test.describe("Edge cases", () => { body: n++ === 0 ? body - : sseOf([{ type: "start" }, { type: "finish" }]), + : sse([{ type: "start" }, { type: "finish" }]), }), ) await openApp(page) @@ -5495,19 +5427,18 @@ test.describe("Edge cases", () => { }) => { // Cut off in the middle of its second shape const cut = `${boxCell("p", "PartOne", 240)} ({ - type: "tool-input-delta", - toolCallId: "cut", - inputTextDelta: d, - })) - await slowReplies( + const { deltas } = toolCallEvents( + "cut", + "display_diagram", + { xml: cut }, + 30, + ) + await chunkedReplies( page, [ [createMockSSEResponse(boxCell("a", "AyOne", 40), "A.")], [ - sseOf( + sse( [ { type: "start" }, { @@ -5519,7 +5450,7 @@ test.describe("Edge cases", () => { ], false, ), - sseOf([ + sse([ { type: "tool-input-available", toolCallId: "cut", @@ -5530,9 +5461,9 @@ test.describe("Edge cases", () => { ]), ], // The continuation never comes - [sseOf([{ type: "start" }], false)], + [sse([{ type: "start" }], false)], ], - true, + { gapMs: 800, holdLast: true }, ) const doc = `${boxCell("b", "BeeOne", 40)}` await openApp(page) diff --git a/tests/unit/admin-providers-route.test.ts b/tests/unit/admin-providers-route.test.ts index c0ad3d18..51a39f68 100644 --- a/tests/unit/admin-providers-route.test.ts +++ b/tests/unit/admin-providers-route.test.ts @@ -2,7 +2,7 @@ import fs from "fs" import os from "os" import path from "path" -import { afterEach, beforeEach, describe, expect, it } from "vitest" +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" import { GET, PUT } from "@/app/api/admin/providers/route" import { _resetForTests } from "@/lib/admin/settings" @@ -10,17 +10,15 @@ let tmpDir: string beforeEach(() => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "admin-providers-route-")) - process.env.SETTINGS_FILE = path.join(tmpDir, "settings.json") - process.env.ADMIN_PASSWORD = "pw" - process.env.AI_MODELS_CONFIG_PATH = path.join(tmpDir, "none.json") + vi.stubEnv("SETTINGS_FILE", path.join(tmpDir, "settings.json")) + vi.stubEnv("ADMIN_PASSWORD", "pw") + vi.stubEnv("AI_MODELS_CONFIG_PATH", path.join(tmpDir, "none.json")) _resetForTests() }) afterEach(() => { _resetForTests() - delete process.env.SETTINGS_FILE - delete process.env.ADMIN_PASSWORD - delete process.env.AI_MODELS_CONFIG_PATH + // Written by the panel default's save, not stubbed by a test delete process.env.AI_MODEL fs.rmSync(tmpDir, { recursive: true, force: true }) }) @@ -57,7 +55,7 @@ async function envHasDefaultModel(): Promise { describe("envHasDefaultModel", () => { it("is true when .env sets AI_MODEL, even after a panel default", async () => { - process.env.AI_MODEL = "gpt-env" + vi.stubEnv("AI_MODEL", "gpt-env") expect(await envHasDefaultModel()).toBe(true) await saveDefaultPanelProvider() expect(await envHasDefaultModel()).toBe(true) diff --git a/tests/unit/admin-providers.test.ts b/tests/unit/admin-providers.test.ts index 0f1296b0..9e985fd7 100644 --- a/tests/unit/admin-providers.test.ts +++ b/tests/unit/admin-providers.test.ts @@ -1,7 +1,7 @@ import fs from "fs" import os from "os" import path from "path" -import { afterEach, beforeEach, describe, expect, it } from "vitest" +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" import { ADMIN_PROVIDERS_KEY, adminProvidersToConfig, @@ -19,14 +19,12 @@ let tmpDir: string beforeEach(() => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "admin-providers-")) - process.env.SETTINGS_FILE = path.join(tmpDir, "settings.json") + vi.stubEnv("SETTINGS_FILE", path.join(tmpDir, "settings.json")) _resetForTests() }) afterEach(() => { _resetForTests() - delete process.env.SETTINGS_FILE - delete process.env.AI_MODELS_CONFIG fs.rmSync(tmpDir, { recursive: true, force: true }) }) @@ -227,16 +225,19 @@ describe("mergeSecrets", () => { describe("loadRawServerModelsConfig merge", () => { it("combines env AI_MODELS_CONFIG with panel providers", async () => { - process.env.AI_MODELS_CONFIG = JSON.stringify({ - providers: [ - { - name: "Env OpenAI", - provider: "openai", - models: ["gpt-from-env"], - default: true, - }, - ], - }) + vi.stubEnv( + "AI_MODELS_CONFIG", + JSON.stringify({ + providers: [ + { + name: "Env OpenAI", + provider: "openai", + models: ["gpt-from-env"], + default: true, + }, + ], + }), + ) saveSettings(deriveEnvUpdates([provider({ name: "Panel" })], [])) const merged = await loadRawServerModelsConfig() @@ -249,16 +250,19 @@ describe("loadRawServerModelsConfig merge", () => { }) it("panel default overrides the env default", async () => { - process.env.AI_MODELS_CONFIG = JSON.stringify({ - providers: [ - { - name: "Env OpenAI", - provider: "openai", - models: ["gpt-from-env"], - default: true, - }, - ], - }) + vi.stubEnv( + "AI_MODELS_CONFIG", + JSON.stringify({ + providers: [ + { + name: "Env OpenAI", + provider: "openai", + models: ["gpt-from-env"], + default: true, + }, + ], + }), + ) saveSettings( deriveEnvUpdates( [provider({ name: "Panel", isDefault: true })], @@ -272,15 +276,18 @@ describe("loadRawServerModelsConfig merge", () => { }) it("returns only env config when the panel has no providers", async () => { - process.env.AI_MODELS_CONFIG = JSON.stringify({ - providers: [ - { - name: "Env Only", - provider: "openai", - models: ["gpt-from-env"], - }, - ], - }) + vi.stubEnv( + "AI_MODELS_CONFIG", + JSON.stringify({ + providers: [ + { + name: "Env Only", + provider: "openai", + models: ["gpt-from-env"], + }, + ], + }), + ) const merged = await loadRawServerModelsConfig() expect(merged?.providers.map((p) => p.name)).toEqual(["Env Only"]) }) diff --git a/tests/unit/admin-settings.test.ts b/tests/unit/admin-settings.test.ts index c07e9808..65b63a42 100644 --- a/tests/unit/admin-settings.test.ts +++ b/tests/unit/admin-settings.test.ts @@ -16,14 +16,14 @@ let tmpDir: string beforeEach(() => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "admin-settings-")) - process.env.SETTINGS_FILE = path.join(tmpDir, "settings.json") + vi.stubEnv("SETTINGS_FILE", path.join(tmpDir, "settings.json")) _resetForTests() }) afterEach(() => { _resetForTests() - delete process.env.SETTINGS_FILE fs.rmSync(tmpDir, { recursive: true, force: true }) + // Written by saveSettings and applyToEnv, not stubbed by every test delete process.env.TEST_ADMIN_VAR }) @@ -82,13 +82,13 @@ describe("applyToEnv / saveSettings", () => { }) it("file value wins over pre-existing env value", () => { - process.env.TEST_ADMIN_VAR = "from-env" + vi.stubEnv("TEST_ADMIN_VAR", "from-env") saveSettings({ TEST_ADMIN_VAR: "from-file" }) expect(process.env.TEST_ADMIN_VAR).toBe("from-file") }) it("deleting a key restores the original env value", () => { - process.env.TEST_ADMIN_VAR = "from-env" + vi.stubEnv("TEST_ADMIN_VAR", "from-env") saveSettings({ TEST_ADMIN_VAR: "from-file" }) saveSettings({ TEST_ADMIN_VAR: null }) expect(process.env.TEST_ADMIN_VAR).toBe("from-env") @@ -102,7 +102,7 @@ describe("applyToEnv / saveSettings", () => { it("a second module instance can remove a key the first one overlaid", async () => { // instrumentation.ts and API routes load separate copies in a build - process.env.TEST_ADMIN_VAR = "from-env" + vi.stubEnv("TEST_ADMIN_VAR", "from-env") fs.writeFileSync( process.env.SETTINGS_FILE!, JSON.stringify({ version: 1, values: { TEST_ADMIN_VAR: "abc" } }), @@ -133,7 +133,7 @@ describe("getValueSource / getEnvFallback", () => { }) it("reports env source when only env is set", () => { - process.env.TEST_ADMIN_VAR = "from-env" + vi.stubEnv("TEST_ADMIN_VAR", "from-env") applyToEnv() expect(getValueSource("TEST_ADMIN_VAR")).toBe("env") }) @@ -143,7 +143,7 @@ describe("getValueSource / getEnvFallback", () => { }) it("returns the shadowed env value as fallback", () => { - process.env.TEST_ADMIN_VAR = "from-env" + vi.stubEnv("TEST_ADMIN_VAR", "from-env") saveSettings({ TEST_ADMIN_VAR: "from-file" }) expect(getEnvFallback("TEST_ADMIN_VAR")).toBe("from-env") }) @@ -156,7 +156,7 @@ describe("isSettingsWritable", () => { it("returns false when the settings path is a directory", () => { _resetForTests() - process.env.SETTINGS_FILE = tmpDir + vi.stubEnv("SETTINGS_FILE", tmpDir) expect(isSettingsWritable()).toBe(false) }) }) diff --git a/tests/unit/admin-test-model.test.ts b/tests/unit/admin-test-model.test.ts index cb463912..faacb982 100644 --- a/tests/unit/admin-test-model.test.ts +++ b/tests/unit/admin-test-model.test.ts @@ -1,5 +1,5 @@ // @vitest-environment node -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" +import { beforeEach, describe, expect, it, vi } from "vitest" // What the admin Test hands to testModelConnection const sent = vi.hoisted(() => ({ body: null as any, trusted: null as any })) @@ -27,19 +27,9 @@ const ENV = [ "AZURE_BASE_URL", "AZURE_RESOURCE_NAME", ] -const saved: Record = {} beforeEach(() => { envFallback.values = {} - for (const k of ENV) { - saved[k] = process.env[k] - delete process.env[k] - } -}) -afterEach(() => { - for (const k of ENV) { - if (saved[k] === undefined) delete process.env[k] - else process.env[k] = saved[k] - } + for (const k of ENV) vi.stubEnv(k, undefined) }) const test = (provider: Record) => @@ -82,19 +72,15 @@ describe("admin Test of an entry without a URL", () => { it("does not use Vertex's variable, which the panel writes itself", async () => { // Before a save it still holds the entry's previous URL - process.env.GOOGLE_VERTEX_BASE_URL = "https://old-proxy.example.com" - try { - await test({ provider: "vertexai", vertexApiKey: "new-key" }) - expect(sent.body.baseUrl).toBeUndefined() - // The environment's own URL, which chat uses once it is saved - envFallback.values.GOOGLE_VERTEX_BASE_URL = - "https://vertex-proxy.example.com" - await test({ provider: "vertexai", vertexApiKey: "new-key" }) - expect(sent.body.baseUrl).toBe("https://vertex-proxy.example.com") - expect(sent.trusted).toBe(true) - } finally { - delete process.env.GOOGLE_VERTEX_BASE_URL - } + vi.stubEnv("GOOGLE_VERTEX_BASE_URL", "https://old-proxy.example.com") + await test({ provider: "vertexai", vertexApiKey: "new-key" }) + expect(sent.body.baseUrl).toBeUndefined() + // The environment's own URL, which chat uses once it is saved + envFallback.values.GOOGLE_VERTEX_BASE_URL = + "https://vertex-proxy.example.com" + await test({ provider: "vertexai", vertexApiKey: "new-key" }) + expect(sent.body.baseUrl).toBe("https://vertex-proxy.example.com") + expect(sent.trusted).toBe(true) }) it("tests Azure set up by resource name where chat goes", async () => { diff --git a/tests/unit/ai-providers-cache.test.ts b/tests/unit/ai-providers-cache.test.ts index c67863da..e11f0cc3 100644 --- a/tests/unit/ai-providers-cache.test.ts +++ b/tests/unit/ai-providers-cache.test.ts @@ -3,14 +3,7 @@ import { generateText, type ModelMessage } from "ai" import { afterEach, describe, expect, it, vi } from "vitest" import { CACHE_POINT, getAIModel } from "@/lib/ai-providers" -const ENV = ["ANTHROPIC_API_KEY", "OPENROUTER_API_KEY"] -const saved = Object.fromEntries(ENV.map((k) => [k, process.env[k]])) - afterEach(() => { - for (const k of ENV) { - if (saved[k] === undefined) delete process.env[k] - else process.env[k] = saved[k] - } vi.unstubAllGlobals() }) @@ -48,7 +41,7 @@ async function requestBody( describe("prompt cache breakpoints", () => { it("reach the Anthropic API", async () => { - process.env.ANTHROPIC_API_KEY = "test-key" + vi.stubEnv("ANTHROPIC_API_KEY", "test-key") const body = await requestBody("anthropic", { id: "msg_1", type: "message", @@ -67,7 +60,7 @@ describe("prompt cache breakpoints", () => { }) it("reach OpenRouter", async () => { - process.env.OPENROUTER_API_KEY = "test-key" + vi.stubEnv("OPENROUTER_API_KEY", "test-key") const body = await requestBody("openrouter", { id: "gen-1", choices: [ diff --git a/tests/unit/ai-providers-credentials.test.ts b/tests/unit/ai-providers-credentials.test.ts index a0749385..07351f63 100644 --- a/tests/unit/ai-providers-credentials.test.ts +++ b/tests/unit/ai-providers-credentials.test.ts @@ -1,5 +1,5 @@ import { createOpenAI } from "@ai-sdk/openai" -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" +import { beforeEach, describe, expect, it, vi } from "vitest" import { AWS_REGIONS } from "@/components/provider-credentials-fields" import { getAIModel, @@ -78,24 +78,12 @@ const ENV_KEYS = [ "AZURE_RESOURCE_NAME", "OLLAMA_BASE_URL", ] -const savedEnv: Record = {} - beforeEach(() => { - for (const key of ENV_KEYS) { - savedEnv[key] = process.env[key] - delete process.env[key] - } + for (const key of ENV_KEYS) vi.stubEnv(key, undefined) settings.values = {} vi.clearAllMocks() }) -afterEach(() => { - for (const key of ENV_KEYS) { - if (savedEnv[key] === undefined) delete process.env[key] - else process.env[key] = savedEnv[key] - } -}) - describe("Vertex AI key security", () => { it("never sends the server key to a client base URL", () => { process.env.GOOGLE_VERTEX_API_KEY = "server-vertex-key" @@ -459,63 +447,56 @@ describe("whose keys a request uses", () => { it("uses a server model's own Ollama URL variable", async () => { process.env.OLLAMA_BASE_URL = "http://other.internal:11434/api" - process.env.MY_OLLAMA_URL = "https://ollama.proxy.example/api" - process.env.MY_OLLAMA_KEY = "proxy-key" - try { - const { createOllama } = await import("ollama-ai-provider-v2") - getAIModel({ - provider: "ollama", - modelId: "m", - apiKeyEnv: "MY_OLLAMA_KEY", - baseUrlEnv: "MY_OLLAMA_URL", - }) - expect(createOllama).toHaveBeenLastCalledWith( - expect.objectContaining({ - baseURL: "https://ollama.proxy.example/api", - headers: { Authorization: "Bearer proxy-key" }, - }), - ) - } finally { - delete process.env.MY_OLLAMA_URL - delete process.env.MY_OLLAMA_KEY - } + vi.stubEnv("MY_OLLAMA_URL", "https://ollama.proxy.example/api") + vi.stubEnv("MY_OLLAMA_KEY", "proxy-key") + const { createOllama } = await import("ollama-ai-provider-v2") + getAIModel({ + provider: "ollama", + modelId: "m", + apiKeyEnv: "MY_OLLAMA_KEY", + baseUrlEnv: "MY_OLLAMA_URL", + }) + expect(createOllama).toHaveBeenLastCalledWith( + expect.objectContaining({ + baseURL: "https://ollama.proxy.example/api", + headers: { Authorization: "Bearer proxy-key" }, + }), + ) }) it("runs an Azure entry set up only in the admin panel", async () => { // No AZURE_BASE_URL or AZURE_RESOURCE_NAME: the entry's own // variables hold the key and the resource URL - process.env.ADMIN_AZURE_API_KEY = "panel-key" - process.env.ADMIN_AZURE_BASE_URL = "https://res.openai.azure.com/openai" - try { - const { createAzure } = await import("@ai-sdk/azure") - expect(() => - getAIModel({ - provider: "azure", - modelId: "gpt-4o", - apiKeyEnv: "ADMIN_AZURE_API_KEY", - baseUrlEnv: "ADMIN_AZURE_BASE_URL", - }), - ).not.toThrow() - expect(createAzure).toHaveBeenLastCalledWith( - expect.objectContaining({ - apiKey: "panel-key", - baseURL: "https://res.openai.azure.com/openai", - }), - ) - // Without any URL it still says what is missing - delete process.env.ADMIN_AZURE_BASE_URL - expect(() => - getAIModel({ - provider: "azure", - modelId: "gpt-4o", - apiKeyEnv: "ADMIN_AZURE_API_KEY", - baseUrlEnv: "ADMIN_AZURE_BASE_URL", - }), - ).toThrow(/AZURE_BASE_URL/) - } finally { - delete process.env.ADMIN_AZURE_API_KEY - delete process.env.ADMIN_AZURE_BASE_URL - } + vi.stubEnv("ADMIN_AZURE_API_KEY", "panel-key") + vi.stubEnv( + "ADMIN_AZURE_BASE_URL", + "https://res.openai.azure.com/openai", + ) + const { createAzure } = await import("@ai-sdk/azure") + expect(() => + getAIModel({ + provider: "azure", + modelId: "gpt-4o", + apiKeyEnv: "ADMIN_AZURE_API_KEY", + baseUrlEnv: "ADMIN_AZURE_BASE_URL", + }), + ).not.toThrow() + expect(createAzure).toHaveBeenLastCalledWith( + expect.objectContaining({ + apiKey: "panel-key", + baseURL: "https://res.openai.azure.com/openai", + }), + ) + // Without any URL it still says what is missing + vi.stubEnv("ADMIN_AZURE_BASE_URL", undefined) + expect(() => + getAIModel({ + provider: "azure", + modelId: "gpt-4o", + apiKeyEnv: "ADMIN_AZURE_API_KEY", + baseUrlEnv: "ADMIN_AZURE_BASE_URL", + }), + ).toThrow(/AZURE_BASE_URL/) }) it("needs a base URL with a user's Azure key", () => { diff --git a/tests/unit/ai-providers-google.test.ts b/tests/unit/ai-providers-google.test.ts index 0633767e..048c3777 100644 --- a/tests/unit/ai-providers-google.test.ts +++ b/tests/unit/ai-providers-google.test.ts @@ -3,20 +3,7 @@ import { generateText } from "ai" import { afterEach, describe, expect, it, vi } from "vitest" import { getAIModel } from "@/lib/ai-providers" -const ENV = [ - "GOOGLE_GENERATIVE_AI_API_KEY", - "GOOGLE_TOP_K", - "GOOGLE_TOP_P", - "GOOGLE_THINKING_BUDGET", - "GOOGLE_THINKING_LEVEL", -] -const saved = Object.fromEntries(ENV.map((k) => [k, process.env[k]])) - afterEach(() => { - for (const k of ENV) { - if (saved[k] === undefined) delete process.env[k] - else process.env[k] = saved[k] - } vi.unstubAllGlobals() }) @@ -56,17 +43,17 @@ async function googleRequestBody(modelId = "gemini-2.5-flash") { describe("Google sampling settings", () => { it("sends GOOGLE_TOP_K and GOOGLE_TOP_P in the generation config", async () => { - process.env.GOOGLE_GENERATIVE_AI_API_KEY = "test-key" - process.env.GOOGLE_TOP_K = "40" - process.env.GOOGLE_TOP_P = "0.9" + vi.stubEnv("GOOGLE_GENERATIVE_AI_API_KEY", "test-key") + vi.stubEnv("GOOGLE_TOP_K", "40") + vi.stubEnv("GOOGLE_TOP_P", "0.9") const body = await googleRequestBody() expect(body.generationConfig).toMatchObject({ topK: 40, topP: 0.9 }) }) it("sends neither when they are not set", async () => { - process.env.GOOGLE_GENERATIVE_AI_API_KEY = "test-key" - delete process.env.GOOGLE_TOP_K - delete process.env.GOOGLE_TOP_P + vi.stubEnv("GOOGLE_GENERATIVE_AI_API_KEY", "test-key") + vi.stubEnv("GOOGLE_TOP_K", undefined) + vi.stubEnv("GOOGLE_TOP_P", undefined) const body = await googleRequestBody() expect(body.generationConfig?.topK).toBeUndefined() expect(body.generationConfig?.topP).toBeUndefined() @@ -75,9 +62,9 @@ describe("Google sampling settings", () => { describe("Gemini thinking settings", () => { it("sends GOOGLE_THINKING_BUDGET as the Gemini 2.5 thinking budget", async () => { - process.env.GOOGLE_GENERATIVE_AI_API_KEY = "test-key" - process.env.GOOGLE_THINKING_BUDGET = "8192" - delete process.env.GOOGLE_THINKING_LEVEL + vi.stubEnv("GOOGLE_GENERATIVE_AI_API_KEY", "test-key") + vi.stubEnv("GOOGLE_THINKING_BUDGET", "8192") + vi.stubEnv("GOOGLE_THINKING_LEVEL", undefined) const body = await googleRequestBody("gemini-2.5-flash") expect(body.generationConfig.thinkingConfig).toEqual({ includeThoughts: true, @@ -86,9 +73,9 @@ describe("Gemini thinking settings", () => { }) it("sends GOOGLE_THINKING_LEVEL as the Gemini 3 thinking level", async () => { - process.env.GOOGLE_GENERATIVE_AI_API_KEY = "test-key" - process.env.GOOGLE_THINKING_LEVEL = "high" - delete process.env.GOOGLE_THINKING_BUDGET + vi.stubEnv("GOOGLE_GENERATIVE_AI_API_KEY", "test-key") + vi.stubEnv("GOOGLE_THINKING_LEVEL", "high") + vi.stubEnv("GOOGLE_THINKING_BUDGET", undefined) const body = await googleRequestBody("gemini-3-pro-preview") expect(body.generationConfig.thinkingConfig).toEqual({ includeThoughts: true, diff --git a/tests/unit/ai-providers.test.ts b/tests/unit/ai-providers.test.ts index 2e9e75d2..ac1cfdb6 100644 --- a/tests/unit/ai-providers.test.ts +++ b/tests/unit/ai-providers.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" +import { beforeEach, describe, expect, it, vi } from "vitest" import { getAIModel, isAihubmixStandardBaseURL, @@ -131,25 +131,16 @@ vi.mock("@ai-sdk/openai", () => { describe("AIHubMix provider", () => { let createAihubmixMock: ReturnType - const savedEnv: Record = {} - beforeEach(async () => { - savedEnv.AIHUBMIX_API_KEY = process.env.AIHUBMIX_API_KEY - savedEnv.AIHUBMIX_BASE_URL = process.env.AIHUBMIX_BASE_URL - delete process.env.AIHUBMIX_BASE_URL + vi.stubEnv("AIHUBMIX_BASE_URL", undefined) const mod = await import("@aihubmix/ai-sdk-provider") createAihubmixMock = mod.createAihubmix as ReturnType createAihubmixMock.mockClear() }) - afterEach(() => { - process.env.AIHUBMIX_API_KEY = savedEnv.AIHUBMIX_API_KEY - process.env.AIHUBMIX_BASE_URL = savedEnv.AIHUBMIX_BASE_URL - }) - it("uses AIHUBMIX_API_KEY for server configured AIHubMix", () => { - process.env.AIHUBMIX_API_KEY = "server-aihubmix-key" + vi.stubEnv("AIHUBMIX_API_KEY", "server-aihubmix-key") getAIModel({ provider: "aihubmix", @@ -194,12 +185,8 @@ vi.mock("@ai-sdk/openai-compatible", () => { describe("Atlas Cloud provider", () => { let createCompatibleMock: ReturnType - const savedEnv: Record = {} - beforeEach(async () => { - savedEnv.ATLASCLOUD_API_KEY = process.env.ATLASCLOUD_API_KEY - savedEnv.ATLASCLOUD_BASE_URL = process.env.ATLASCLOUD_BASE_URL - delete process.env.ATLASCLOUD_BASE_URL + vi.stubEnv("ATLASCLOUD_BASE_URL", undefined) const mod = await import("@ai-sdk/openai-compatible") createCompatibleMock = mod.createOpenAICompatible as ReturnType< @@ -208,13 +195,8 @@ describe("Atlas Cloud provider", () => { createCompatibleMock.mockClear() }) - afterEach(() => { - process.env.ATLASCLOUD_API_KEY = savedEnv.ATLASCLOUD_API_KEY - process.env.ATLASCLOUD_BASE_URL = savedEnv.ATLASCLOUD_BASE_URL - }) - it("uses Atlas Cloud default endpoint with ATLASCLOUD_API_KEY", () => { - process.env.ATLASCLOUD_API_KEY = "server-atlas-key" + vi.stubEnv("ATLASCLOUD_API_KEY", "server-atlas-key") getAIModel({ provider: "atlascloud", @@ -264,25 +246,16 @@ describe("Atlas Cloud provider", () => { describe("Kimi provider uses createDeepSeek for reasoning_content support", () => { let createDeepSeekMock: ReturnType - const savedEnv: Record = {} - beforeEach(async () => { - savedEnv.KIMI_API_KEY = process.env.KIMI_API_KEY - savedEnv.KIMI_BASE_URL = process.env.KIMI_BASE_URL - delete process.env.KIMI_BASE_URL + vi.stubEnv("KIMI_BASE_URL", undefined) const mod = await import("@ai-sdk/deepseek") createDeepSeekMock = mod.createDeepSeek as ReturnType createDeepSeekMock.mockClear() }) - afterEach(() => { - process.env.KIMI_API_KEY = savedEnv.KIMI_API_KEY - process.env.KIMI_BASE_URL = savedEnv.KIMI_BASE_URL - }) - it("uses createDeepSeek with Kimi default base URL for reasoning_content support", () => { - process.env.KIMI_API_KEY = "test-kimi-key" + vi.stubEnv("KIMI_API_KEY", "test-kimi-key") getAIModel({ provider: "kimi", @@ -298,7 +271,7 @@ describe("Kimi provider uses createDeepSeek for reasoning_content support", () = }) it("uses custom base URL when provided for kimi provider", () => { - process.env.KIMI_API_KEY = "test-kimi-key" + vi.stubEnv("KIMI_API_KEY", "test-kimi-key") getAIModel({ provider: "kimi", @@ -317,25 +290,16 @@ describe("Kimi provider uses createDeepSeek for reasoning_content support", () = describe("Ollama API key security", () => { let createOllamaMock: ReturnType - const savedEnv: Record = {} - beforeEach(async () => { - savedEnv.OLLAMA_API_KEY = process.env.OLLAMA_API_KEY - savedEnv.OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL - delete process.env.OLLAMA_BASE_URL + vi.stubEnv("OLLAMA_BASE_URL", undefined) const mod = await import("ollama-ai-provider-v2") createOllamaMock = mod.createOllama as ReturnType createOllamaMock.mockClear() }) - afterEach(() => { - process.env.OLLAMA_API_KEY = savedEnv.OLLAMA_API_KEY - process.env.OLLAMA_BASE_URL = savedEnv.OLLAMA_BASE_URL - }) - it("applies server OLLAMA_API_KEY when no client baseUrl is provided", () => { - process.env.OLLAMA_API_KEY = "server-secret-key" + vi.stubEnv("OLLAMA_API_KEY", "server-secret-key") getAIModel({ provider: "ollama", modelId: "llama2" }) @@ -347,7 +311,7 @@ describe("Ollama API key security", () => { }) it("does NOT leak server OLLAMA_API_KEY when client provides a custom baseUrl", () => { - process.env.OLLAMA_API_KEY = "server-secret-key" + vi.stubEnv("OLLAMA_API_KEY", "server-secret-key") // When server has OLLAMA_API_KEY, the SSRF guard rejects // client-provided baseUrl without an apiKey outright @@ -361,7 +325,7 @@ describe("Ollama API key security", () => { }) it("uses client API key when client provides both baseUrl and apiKey", () => { - process.env.OLLAMA_API_KEY = "server-secret-key" + vi.stubEnv("OLLAMA_API_KEY", "server-secret-key") getAIModel({ provider: "ollama", @@ -379,8 +343,8 @@ describe("Ollama API key security", () => { }) it("applies both server OLLAMA_BASE_URL and OLLAMA_API_KEY when no client overrides", () => { - process.env.OLLAMA_BASE_URL = "https://cloud.ollama.com" - process.env.OLLAMA_API_KEY = "server-key" + vi.stubEnv("OLLAMA_BASE_URL", "https://cloud.ollama.com") + vi.stubEnv("OLLAMA_API_KEY", "server-key") getAIModel({ provider: "ollama", modelId: "llama2" }) @@ -393,8 +357,8 @@ describe("Ollama API key security", () => { }) it("works when OLLAMA_API_KEY is set but OLLAMA_BASE_URL is not", () => { - process.env.OLLAMA_API_KEY = "server-key" - delete process.env.OLLAMA_BASE_URL + vi.stubEnv("OLLAMA_API_KEY", "server-key") + vi.stubEnv("OLLAMA_BASE_URL", undefined) getAIModel({ provider: "ollama", modelId: "llama2" }) @@ -411,7 +375,7 @@ describe("Ollama API key security", () => { }) it("allows client custom baseUrl without apiKey when no server OLLAMA_API_KEY", () => { - delete process.env.OLLAMA_API_KEY + vi.stubEnv("OLLAMA_API_KEY", undefined) getAIModel({ provider: "ollama", @@ -426,7 +390,7 @@ describe("Ollama API key security", () => { }) it("sends chat to Ollama's /api for a server address or a /v1 URL", () => { - delete process.env.OLLAMA_API_KEY + vi.stubEnv("OLLAMA_API_KEY", undefined) for (const baseUrl of [ "http://localhost:11434", diff --git a/tests/unit/api-access-code.test.ts b/tests/unit/api-access-code.test.ts index 10432a71..c6f750ce 100644 --- a/tests/unit/api-access-code.test.ts +++ b/tests/unit/api-access-code.test.ts @@ -24,18 +24,16 @@ function post(path: string, body: unknown, accessCode?: string): Request { } beforeEach(() => { - process.env.ACCESS_CODE_LIST = "secret, other" + vi.stubEnv("ACCESS_CODE_LIST", "secret, other") }) afterEach(() => { - delete process.env.ACCESS_CODE_LIST - delete process.env.ALLOW_PRIVATE_URLS vi.unstubAllGlobals() }) describe("checkAccessCode", () => { it("passes when no access codes are configured", () => { - delete process.env.ACCESS_CODE_LIST + vi.stubEnv("ACCESS_CODE_LIST", undefined) expect(checkAccessCode(post("/x", {}))).toBeNull() }) @@ -124,7 +122,7 @@ describe("size limits", () => { describe("validate-model redirects", () => { it("refuses redirects when private URLs are blocked", async () => { - process.env.ALLOW_PRIVATE_URLS = "false" + vi.stubEnv("ALLOW_PRIVATE_URLS", "false") vi.stubGlobal( "fetch", vi.fn( diff --git a/tests/unit/bedrock-api-key.test.ts b/tests/unit/bedrock-api-key.test.ts index 95010d26..45302858 100644 --- a/tests/unit/bedrock-api-key.test.ts +++ b/tests/unit/bedrock-api-key.test.ts @@ -20,7 +20,6 @@ beforeEach(() => { }) afterEach(() => { vi.unstubAllGlobals() - vi.unstubAllEnvs() }) const run = async (apiKey: string) => { diff --git a/tests/unit/chat-route-edgeone.test.ts b/tests/unit/chat-route-edgeone.test.ts index 20991bad..9b359d86 100644 --- a/tests/unit/chat-route-edgeone.test.ts +++ b/tests/unit/chat-route-edgeone.test.ts @@ -9,14 +9,11 @@ vi.mock("@/lib/dynamo-quota-manager", () => ({ import { POST as chat } from "@/app/api/chat/route" -const ENV = ["AI_MODELS_CONFIG", "AI_PROVIDER", "AI_MODEL"] -const saved: Record = {} const calls: Array<{ url: string; headers: Headers }> = [] beforeEach(() => { - for (const k of ENV) saved[k] = process.env[k] - delete process.env.AI_PROVIDER - delete process.env.AI_MODEL + vi.stubEnv("AI_PROVIDER", undefined) + vi.stubEnv("AI_MODEL", undefined) calls.length = 0 vi.stubGlobal( "fetch", @@ -28,10 +25,6 @@ beforeEach(() => { }) afterEach(() => { - for (const k of ENV) { - if (saved[k] === undefined) delete process.env[k] - else process.env[k] = saved[k] - } vi.unstubAllGlobals() }) @@ -39,15 +32,18 @@ describe("EdgeOne as a server model", () => { it("calls the site's Edge AI function with the cookies", async () => { // An admin panel or ai-models.json provider: the client sends the // provider name's slug, not "edgeone" - process.env.AI_MODELS_CONFIG = JSON.stringify({ - providers: [ - { - name: "Edge Pages", - provider: "edgeone", - models: ["@tx/deepseek-ai/deepseek-v3-0324"], - }, - ], - }) + vi.stubEnv( + "AI_MODELS_CONFIG", + JSON.stringify({ + providers: [ + { + name: "Edge Pages", + provider: "edgeone", + models: ["@tx/deepseek-ai/deepseek-v3-0324"], + }, + ], + }), + ) const res = await chat( new Request("http://localhost/api/chat", { method: "POST", @@ -98,8 +94,8 @@ const send = (headers: Record) => describe("EdgeOne endpoints", () => { it("works when the deployment names EdgeOne only in AI_PROVIDER", async () => { - process.env.AI_PROVIDER = "edgeone" - process.env.AI_MODEL = "@tx/deepseek-ai/deepseek-v3-0324" + vi.stubEnv("AI_PROVIDER", "edgeone") + vi.stubEnv("AI_MODEL", "@tx/deepseek-ai/deepseek-v3-0324") await send({}) expect(calls[0]?.url).toBe( "http://localhost/api/edgeai/chat/completions", @@ -120,20 +116,13 @@ describe("EdgeOne endpoints", () => { }) it("calls the function at the site root, also with a base path", async () => { - const savedPath = process.env.NEXT_PUBLIC_BASE_PATH - process.env.NEXT_PUBLIC_BASE_PATH = "/draw" - try { - await send({ - "x-ai-provider": "edgeone", - "x-ai-model": "@tx/deepseek-ai/deepseek-v3-0324", - }) - expect(calls[0]?.url).toBe( - "http://localhost/api/edgeai/chat/completions", - ) - } finally { - if (savedPath === undefined) - delete process.env.NEXT_PUBLIC_BASE_PATH - else process.env.NEXT_PUBLIC_BASE_PATH = savedPath - } + vi.stubEnv("NEXT_PUBLIC_BASE_PATH", "/draw") + await send({ + "x-ai-provider": "edgeone", + "x-ai-model": "@tx/deepseek-ai/deepseek-v3-0324", + }) + expect(calls[0]?.url).toBe( + "http://localhost/api/edgeai/chat/completions", + ) }) }) diff --git a/tests/unit/chat-route-errors.test.ts b/tests/unit/chat-route-errors.test.ts index d0ef66ee..d6fed120 100644 --- a/tests/unit/chat-route-errors.test.ts +++ b/tests/unit/chat-route-errors.test.ts @@ -18,18 +18,11 @@ const ENV = [ "OLLAMA_API_KEY", "NEXT_AI_DRAWIO_DESKTOP", ] -const saved: Record = {} - beforeEach(() => { - for (const k of ENV) saved[k] = process.env[k] - for (const k of ENV) delete process.env[k] + for (const k of ENV) vi.stubEnv(k, undefined) }) afterEach(() => { - for (const k of ENV) { - if (saved[k] === undefined) delete process.env[k] - else process.env[k] = saved[k] - } vi.unstubAllGlobals() }) @@ -146,7 +139,7 @@ describe("provider error texts in the stream", () => { describe("the output cap", () => { it("holds for the server's own keyless endpoints too", async () => { - process.env.MAX_OUTPUT_TOKENS = "8000" + vi.stubEnv("MAX_OUTPUT_TOKENS", "8000") const sent: string[] = [] vi.stubGlobal( "fetch", @@ -155,34 +148,30 @@ describe("the output cap", () => { return new Response("{}", { status: 400 }) }), ) - try { - const res = await chat( - new Request("http://localhost/api/chat", { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-ai-provider": "ollama", - "x-ai-base-url": "http://127.0.0.1:11434/api", - "x-ai-model": "llama3", - "x-max-output-tokens": "200000", - }, - body: JSON.stringify({ - messages: [ - { - id: "u1", - role: "user", - parts: [{ type: "text", text: "Draw" }], - }, - ], - xml: "", - }), + const res = await chat( + new Request("http://localhost/api/chat", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-ai-provider": "ollama", + "x-ai-base-url": "http://127.0.0.1:11434/api", + "x-ai-model": "llama3", + "x-max-output-tokens": "200000", + }, + body: JSON.stringify({ + messages: [ + { + id: "u1", + role: "user", + parts: [{ type: "text", text: "Draw" }], + }, + ], + xml: "", }), - ) - await res.text() - expect(sent[0]).toContain('"max_output_tokens":8000') - } finally { - delete process.env.MAX_OUTPUT_TOKENS - } + }), + ) + await res.text() + expect(sent[0]).toContain('"max_output_tokens":8000') }) }) diff --git a/tests/unit/chat-route-quota.test.ts b/tests/unit/chat-route-quota.test.ts index 28a3fdac..276d3356 100644 --- a/tests/unit/chat-route-quota.test.ts +++ b/tests/unit/chat-route-quota.test.ts @@ -27,22 +27,10 @@ vi.mock("@/lib/ssrf-protection", async (importOriginal) => ({ import { POST as chat } from "@/app/api/chat/route" -const ENV = [ - "AI_PROVIDER", - "AI_MODEL", - "OPENAI_API_KEY", - "OLLAMA_BASE_URL", - "OLLAMA_API_KEY", - "AI_GATEWAY_API_KEY", - "ALLOW_PRIVATE_URLS", -] -const saved: Record = {} - beforeEach(() => { - for (const k of ENV) saved[k] = process.env[k] - process.env.AI_PROVIDER = "openai" - process.env.AI_MODEL = "gpt-5.5" - process.env.OPENAI_API_KEY = "server-key" + vi.stubEnv("AI_PROVIDER", "openai") + vi.stubEnv("AI_MODEL", "gpt-5.5") + vi.stubEnv("OPENAI_API_KEY", "server-key") quota.checks = 0 // No request may reach a provider vi.stubGlobal( @@ -54,10 +42,6 @@ beforeEach(() => { }) afterEach(() => { - for (const k of ENV) { - if (saved[k] === undefined) delete process.env[k] - else process.env[k] = saved[k] - } vi.unstubAllGlobals() }) @@ -105,9 +89,9 @@ describe("chat quota", () => { }) it("counts the server's keyless Ollama and EdgeOne", async () => { - process.env.AI_PROVIDER = "ollama" - process.env.AI_MODEL = "llama3.2" - process.env.OLLAMA_BASE_URL = "http://ollama.internal:11434/api" + vi.stubEnv("AI_PROVIDER", "ollama") + vi.stubEnv("AI_MODEL", "llama3.2") + vi.stubEnv("OLLAMA_BASE_URL", "http://ollama.internal:11434/api") expect((await send({})).status).toBe(429) expect( ( @@ -189,18 +173,14 @@ describe("chat quota", () => { }) it("never counts in the desktop app, where every endpoint is the user's", async () => { - process.env.NEXT_AI_DRAWIO_DESKTOP = "1" - try { - const res = await send({ - "x-ai-provider": "ollama", - "x-ai-base-url": "http://127.0.0.1:11434/api", - "x-ai-model": "llama3.2", - }) - expect(res.status).not.toBe(429) - expect(quota.checks).toBe(0) - } finally { - delete process.env.NEXT_AI_DRAWIO_DESKTOP - } + vi.stubEnv("NEXT_AI_DRAWIO_DESKTOP", "1") + const res = await send({ + "x-ai-provider": "ollama", + "x-ai-base-url": "http://127.0.0.1:11434/api", + "x-ai-model": "llama3.2", + }) + expect(res.status).not.toBe(429) + expect(quota.checks).toBe(0) }) it("does not count Ollama on the user's own server", async () => { @@ -216,8 +196,8 @@ describe("chat quota", () => { describe("request checks", () => { it("refuses an AWS region that is not a region name", async () => { - process.env.AI_PROVIDER = "bedrock" - process.env.AI_MODEL = "amazon.nova-lite-v1:0" + vi.stubEnv("AI_PROVIDER", "bedrock") + vi.stubEnv("AI_MODEL", "amazon.nova-lite-v1:0") const res = await send({ "x-aws-region": "us-east-1.attacker.example/", }) @@ -230,7 +210,7 @@ describe("request checks", () => { describe("server model allowlist", () => { it("runs AI_MODEL only on the server's AI_PROVIDER", async () => { // Another provider's server key must not run it - process.env.AI_GATEWAY_API_KEY = "server-gateway-key" + vi.stubEnv("AI_GATEWAY_API_KEY", "server-gateway-key") const res = await send({ "x-ai-provider": "gateway", "x-ai-model": "gpt-5.5", diff --git a/tests/unit/client-ip.test.ts b/tests/unit/client-ip.test.ts index caec89b3..129a1fcc 100644 --- a/tests/unit/client-ip.test.ts +++ b/tests/unit/client-ip.test.ts @@ -1,15 +1,10 @@ import { NextRequest } from "next/server" -import { afterEach, describe, expect, it } from "vitest" +import { describe, expect, it, vi } from "vitest" import { getUserIdFromRequest } from "@/lib/user-id" import { proxy } from "@/proxy" const idFor = (ip: string) => `user-${Buffer.from(ip).toString("base64url")}` -afterEach(() => { - delete process.env.CLIENT_IP_HEADER - delete process.env.ORIGIN_SECRET -}) - describe("getUserIdFromRequest", () => { const req = new Request("http://localhost/api/chat", { headers: { @@ -23,12 +18,12 @@ describe("getUserIdFromRequest", () => { }) it("uses CLIENT_IP_HEADER when set", () => { - process.env.CLIENT_IP_HEADER = "cf-connecting-ip" + vi.stubEnv("CLIENT_IP_HEADER", "cf-connecting-ip") expect(getUserIdFromRequest(req)).toBe(idFor("198.51.100.7")) }) it("is anonymous when the configured header is missing", () => { - process.env.CLIENT_IP_HEADER = "cf-connecting-ip" + vi.stubEnv("CLIENT_IP_HEADER", "cf-connecting-ip") expect( getUserIdFromRequest(new Request("http://localhost/api/chat")), ).toBe("anonymous") @@ -48,7 +43,7 @@ describe("ORIGIN_SECRET", () => { }) it("refuses API calls without the right header", async () => { - process.env.ORIGIN_SECRET = "s3cret" + vi.stubEnv("ORIGIN_SECRET", "s3cret") for (const res of [call("/api/chat"), call("/api/chat", "wrong")]) { expect(res?.status).toBe(403) } @@ -56,7 +51,7 @@ describe("ORIGIN_SECRET", () => { }) it("leaves pages alone, such as the health check on /", () => { - process.env.ORIGIN_SECRET = "s3cret" + vi.stubEnv("ORIGIN_SECRET", "s3cret") expect(call("/")?.status).toBe(307) expect(call("/en")).toBeUndefined() }) diff --git a/tests/unit/cross-site-requests.test.ts b/tests/unit/cross-site-requests.test.ts index 8053ae3b..ac7b1b34 100644 --- a/tests/unit/cross-site-requests.test.ts +++ b/tests/unit/cross-site-requests.test.ts @@ -27,8 +27,6 @@ const body = JSON.stringify({ imageData: "data:image/png;base64,AAAA", }) -const saved = process.env.NEXT_AI_DRAWIO_DESKTOP - beforeEach(() => { vi.stubGlobal( "fetch", @@ -39,8 +37,6 @@ beforeEach(() => { }) afterEach(() => { - if (saved === undefined) delete process.env.NEXT_AI_DRAWIO_DESKTOP - else process.env.NEXT_AI_DRAWIO_DESKTOP = saved vi.unstubAllGlobals() }) @@ -60,7 +56,7 @@ describe("requests another website could send", () => { }) it(`${name}: desktop app refuses a foreign Host (DNS rebinding)`, async () => { - process.env.NEXT_AI_DRAWIO_DESKTOP = "1" + vi.stubEnv("NEXT_AI_DRAWIO_DESKTOP", "1") const res = await post( new Request(`http://127.0.0.1:61337/api/${name}`, { method: "POST", @@ -77,7 +73,7 @@ describe("requests another website could send", () => { } it("lets the desktop window's own requests through", async () => { - process.env.NEXT_AI_DRAWIO_DESKTOP = "1" + vi.stubEnv("NEXT_AI_DRAWIO_DESKTOP", "1") const res = await validateModel( new Request("http://127.0.0.1:61337/api/validate-model", { method: "POST", diff --git a/tests/unit/output-token-limit.test.ts b/tests/unit/output-token-limit.test.ts index b16ed8ba..cf89c723 100644 --- a/tests/unit/output-token-limit.test.ts +++ b/tests/unit/output-token-limit.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest" +import { describe, expect, it, vi } from "vitest" import { DEFAULT_MAX_OUTPUT_TOKENS, readCeiling, @@ -273,39 +273,27 @@ describe("resolveMaxOutputTokens", () => { }) it("uses the env value when no header is sent, and validates it too", () => { - const original = process.env.MAX_OUTPUT_TOKENS - try { - process.env.MAX_OUTPUT_TOKENS = "24000" - expect(resolveMaxOutputTokens(null, true)).toBe(24000) - // A lower header still wins - expect(resolveMaxOutputTokens("8000", true)).toBe(8000) + vi.stubEnv("MAX_OUTPUT_TOKENS", "24000") + expect(resolveMaxOutputTokens(null, true)).toBe(24000) + // A lower header still wins + expect(resolveMaxOutputTokens("8000", true)).toBe(8000) - process.env.MAX_OUTPUT_TOKENS = "-1" - expect(resolveMaxOutputTokens(null, true)).toBe( - DEFAULT_MAX_OUTPUT_TOKENS, - ) - } finally { - if (original === undefined) delete process.env.MAX_OUTPUT_TOKENS - else process.env.MAX_OUTPUT_TOKENS = original - } + vi.stubEnv("MAX_OUTPUT_TOKENS", "-1") + expect(resolveMaxOutputTokens(null, true)).toBe( + DEFAULT_MAX_OUTPUT_TOKENS, + ) }) it("lets the header raise the budget only on the user's own credentials", () => { - const original = process.env.MAX_OUTPUT_TOKENS - try { - process.env.MAX_OUTPUT_TOKENS = "16000" - expect(resolveMaxOutputTokens("200000", true)).toBe(16000) - expect(resolveMaxOutputTokens("200000", false)).toBe(200000) + vi.stubEnv("MAX_OUTPUT_TOKENS", "16000") + expect(resolveMaxOutputTokens("200000", true)).toBe(16000) + expect(resolveMaxOutputTokens("200000", false)).toBe(200000) - // Without MAX_OUTPUT_TOKENS the default is the cap - delete process.env.MAX_OUTPUT_TOKENS - expect(resolveMaxOutputTokens("100000", true)).toBe( - DEFAULT_MAX_OUTPUT_TOKENS, - ) - } finally { - if (original === undefined) delete process.env.MAX_OUTPUT_TOKENS - else process.env.MAX_OUTPUT_TOKENS = original - } + // Without MAX_OUTPUT_TOKENS the default is the cap + vi.stubEnv("MAX_OUTPUT_TOKENS", undefined) + expect(resolveMaxOutputTokens("100000", true)).toBe( + DEFAULT_MAX_OUTPUT_TOKENS, + ) }) }) diff --git a/tests/unit/provider-models.test.ts b/tests/unit/provider-models.test.ts index 55468b0b..41965a9d 100644 --- a/tests/unit/provider-models.test.ts +++ b/tests/unit/provider-models.test.ts @@ -113,12 +113,9 @@ describe("listProviderModels", () => { it("lists Ollama from where chat goes without a base URL", async () => { const { fn, calls } = answer({ models: [{ name: "llama3.2" }] }) - process.env.OLLAMA_BASE_URL = "http://ollama.internal:11434" - try { - await listProviderModels("ollama", {}, fn) - } finally { - delete process.env.OLLAMA_BASE_URL - } + vi.stubEnv("OLLAMA_BASE_URL", "http://ollama.internal:11434") + await listProviderModels("ollama", {}, fn) + vi.stubEnv("OLLAMA_BASE_URL", undefined) await listProviderModels("ollama", {}, fn) expect(calls.map((c) => c.url)).toEqual([ "http://ollama.internal:11434/api/tags", @@ -129,12 +126,8 @@ describe("listProviderModels", () => { it("lists Ollama Cloud with a user's key, like chat", async () => { // The user's key must not go to the server's Ollama const { fn, calls } = answer({ models: [{ name: "gpt-oss:120b" }] }) - process.env.OLLAMA_BASE_URL = "http://ollama.internal:11434" - try { - await listProviderModels("ollama", { apiKey: "user-key" }, fn) - } finally { - delete process.env.OLLAMA_BASE_URL - } + vi.stubEnv("OLLAMA_BASE_URL", "http://ollama.internal:11434") + await listProviderModels("ollama", { apiKey: "user-key" }, fn) expect(calls[0].url).toBe("https://ollama.com/api/tags") }) @@ -272,25 +265,21 @@ describe("POST /api/provider-models", () => { }) it("explains a refused redirect", async () => { - process.env.ALLOW_PRIVATE_URLS = "false" - try { - vi.stubGlobal( - "fetch", - vi.fn( - async () => - new Response(null, { - status: 301, - headers: { location: "https://elsewhere.example" }, - }), - ), - ) - const data = await ( - await post({ provider: "ollama", baseUrl: "https://x.example" }) - ).json() - expect(data.error).toMatch(/Redirects are not allowed/) - } finally { - delete process.env.ALLOW_PRIVATE_URLS - } + vi.stubEnv("ALLOW_PRIVATE_URLS", "false") + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response(null, { + status: 301, + headers: { location: "https://elsewhere.example" }, + }), + ), + ) + const data = await ( + await post({ provider: "ollama", baseUrl: "https://x.example" }) + ).json() + expect(data.error).toMatch(/Redirects are not allowed/) }) it("keeps its own explanations and hides other error texts", async () => { diff --git a/tests/unit/server-model-config.test.ts b/tests/unit/server-model-config.test.ts index d94c87b1..13dda9ca 100644 --- a/tests/unit/server-model-config.test.ts +++ b/tests/unit/server-model-config.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it } from "vitest" +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" import { _resetForTests } from "@/lib/admin/settings" import { loadFlattenedServerModels, @@ -7,22 +7,15 @@ import { slugify, } from "@/lib/server-model-config" -const ORIGINAL_ENV = { ...process.env } - beforeEach(() => { // Isolate from any local data/settings.json (admin panel providers // are merged into the server models config) - process.env.SETTINGS_FILE = "/nonexistent/settings.json" + vi.stubEnv("SETTINGS_FILE", "/nonexistent/settings.json") _resetForTests() }) afterEach(() => { _resetForTests() - process.env.AI_PROVIDER = ORIGINAL_ENV.AI_PROVIDER - process.env.AI_MODEL = ORIGINAL_ENV.AI_MODEL - process.env.AI_MODELS_CONFIG_PATH = ORIGINAL_ENV.AI_MODELS_CONFIG_PATH - process.env.AI_MODELS_CONFIG = ORIGINAL_ENV.AI_MODELS_CONFIG - delete process.env.SETTINGS_FILE }) describe("ServerModelsConfigSchema", () => { @@ -143,7 +136,10 @@ describe("ServerModelsConfigSchema", () => { describe("loadFlattenedServerModels", () => { it("returns empty array when config file is missing", async () => { // Point to a non-existent config path so fs.readFile throws ENOENT - process.env.AI_MODELS_CONFIG_PATH = `non-existent-config-${Date.now()}.json` + vi.stubEnv( + "AI_MODELS_CONFIG_PATH", + `non-existent-config-${Date.now()}.json`, + ) const models = await loadFlattenedServerModels() expect(models).toEqual([]) @@ -161,8 +157,8 @@ describe("loadFlattenedServerModels", () => { }, ], } - process.env.AI_MODELS_CONFIG = JSON.stringify(config) - process.env.AI_MODELS_CONFIG_PATH = "" // Clear file path + vi.stubEnv("AI_MODELS_CONFIG", JSON.stringify(config)) + vi.stubEnv("AI_MODELS_CONFIG_PATH", "") // Clear file path const models = await loadFlattenedServerModels() @@ -177,10 +173,13 @@ describe("loadFlattenedServerModels", () => { }) it("falls back to comma-separated AI_MODEL when no other config is set", async () => { - process.env.AI_MODELS_CONFIG = "" - process.env.AI_MODELS_CONFIG_PATH = `non-existent-config-${Date.now()}.json` - process.env.AI_PROVIDER = "openai" - process.env.AI_MODEL = "gpt-4o, gpt-4o-mini, gpt-4o" + vi.stubEnv("AI_MODELS_CONFIG", "") + vi.stubEnv( + "AI_MODELS_CONFIG_PATH", + `non-existent-config-${Date.now()}.json`, + ) + vi.stubEnv("AI_PROVIDER", "openai") + vi.stubEnv("AI_MODEL", "gpt-4o, gpt-4o-mini, gpt-4o") const models = await loadFlattenedServerModels() @@ -195,20 +194,26 @@ describe("loadFlattenedServerModels", () => { }) it("does not synthesize when AI_MODEL has no comma", async () => { - process.env.AI_MODELS_CONFIG = "" - process.env.AI_MODELS_CONFIG_PATH = `non-existent-config-${Date.now()}.json` - process.env.AI_PROVIDER = "openai" - process.env.AI_MODEL = "gpt-4o" + vi.stubEnv("AI_MODELS_CONFIG", "") + vi.stubEnv( + "AI_MODELS_CONFIG_PATH", + `non-existent-config-${Date.now()}.json`, + ) + vi.stubEnv("AI_PROVIDER", "openai") + vi.stubEnv("AI_MODEL", "gpt-4o") const models = await loadFlattenedServerModels() expect(models).toEqual([]) }) it("does not synthesize when AI_PROVIDER is unset", async () => { - process.env.AI_MODELS_CONFIG = "" - process.env.AI_MODELS_CONFIG_PATH = `non-existent-config-${Date.now()}.json` - delete process.env.AI_PROVIDER - process.env.AI_MODEL = "gpt-4o, gpt-4o-mini" + vi.stubEnv("AI_MODELS_CONFIG", "") + vi.stubEnv( + "AI_MODELS_CONFIG_PATH", + `non-existent-config-${Date.now()}.json`, + ) + vi.stubEnv("AI_PROVIDER", undefined) + vi.stubEnv("AI_MODEL", "gpt-4o, gpt-4o-mini") const models = await loadFlattenedServerModels() expect(models).toEqual([]) @@ -225,8 +230,8 @@ describe("loadFlattenedServerModels", () => { }, ], } - process.env.AI_MODELS_CONFIG = JSON.stringify(config) - process.env.AI_MODELS_CONFIG_PATH = "" // Clear file path + vi.stubEnv("AI_MODELS_CONFIG", JSON.stringify(config)) + vi.stubEnv("AI_MODELS_CONFIG_PATH", "") // Clear file path const models = await loadFlattenedServerModels() @@ -272,7 +277,7 @@ describe("loadFlattenedServerModels id collisions", () => { }, ], } - process.env.AI_MODELS_CONFIG = JSON.stringify(config) + vi.stubEnv("AI_MODELS_CONFIG", JSON.stringify(config)) const models = await loadFlattenedServerModels() const ids = models.map((m) => m.id) diff --git a/tests/unit/ssrf-protection.test.ts b/tests/unit/ssrf-protection.test.ts index 80b3c1ae..5b618724 100644 --- a/tests/unit/ssrf-protection.test.ts +++ b/tests/unit/ssrf-protection.test.ts @@ -93,12 +93,11 @@ describe("redirectGuardedFetch with the quota on", () => { ? [{ address: "93.184.216.34", family: 4 }] : [], ) - process.env.DYNAMODB_QUOTA_TABLE = "quota" - delete process.env.ALLOW_PRIVATE_URLS + vi.stubEnv("DYNAMODB_QUOTA_TABLE", "quota") + vi.stubEnv("ALLOW_PRIVATE_URLS", undefined) }) afterEach(() => { - delete process.env.DYNAMODB_QUOTA_TABLE vi.unstubAllGlobals() }) @@ -202,7 +201,7 @@ describe("redirectGuardedFetch with the quota on", () => { }) it("is not used without the quota", () => { - delete process.env.DYNAMODB_QUOTA_TABLE + vi.stubEnv("DYNAMODB_QUOTA_TABLE", undefined) expect(redirectGuardedFetch()).toBeUndefined() }) }) diff --git a/tests/unit/validate-diagram-route.test.ts b/tests/unit/validate-diagram-route.test.ts index 9775ad66..579383d0 100644 --- a/tests/unit/validate-diagram-route.test.ts +++ b/tests/unit/validate-diagram-route.test.ts @@ -72,7 +72,6 @@ const post = () => }) afterEach(() => { - delete process.env.ENABLE_VLM_VALIDATION quota.enabled = false quota.allowed = true quota.checks = [] @@ -108,7 +107,7 @@ describe("POST /api/validate-diagram", () => { }) it("answers valid when the check is turned off", async () => { - process.env.ENABLE_VLM_VALIDATION = "false" + vi.stubEnv("ENABLE_VLM_VALIDATION", "false") const res = await validateDiagram(post()) expect(await res.json()).toEqual({ valid: true, diff --git a/tests/unit/validate-model-route.test.ts b/tests/unit/validate-model-route.test.ts index bfd1ec22..243a9771 100644 --- a/tests/unit/validate-model-route.test.ts +++ b/tests/unit/validate-model-route.test.ts @@ -30,7 +30,6 @@ vi.mock("@/lib/dynamo-quota-manager", () => ({ vi.mock("@/lib/user-id", () => ({ getUserIdFromRequest: () => "user-1" })) afterEach(() => { - delete process.env.ALLOW_PRIVATE_URLS quota.enabled = false quota.checks = 0 privateUrls.all = false @@ -121,23 +120,19 @@ describe("POST /api/validate-model", () => { }) it("does not run on the server's keys", async () => { - process.env.OLLAMA_API_KEY = "server-ollama-key" - try { - const res = await validateModel( - new Request("http://localhost/api/validate-model", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - provider: "ollama", - modelId: "any-cloud-model", - }), + vi.stubEnv("OLLAMA_API_KEY", "server-ollama-key") + const res = await validateModel( + new Request("http://localhost/api/validate-model", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + provider: "ollama", + modelId: "any-cloud-model", }), - ) - expect(res.status).toBe(400) - expect((await res.json()).error).toMatch(/API key/) - } finally { - delete process.env.OLLAMA_API_KEY - } + }), + ) + expect(res.status).toBe(400) + expect((await res.json()).error).toMatch(/API key/) }) it("warns when the model answers without a tool call", async () => { @@ -150,7 +145,7 @@ describe("POST /api/validate-model", () => { describe("chat requests to a client base URL", () => { it("refuse redirects when private URLs are blocked", async () => { - process.env.ALLOW_PRIVATE_URLS = "false" + vi.stubEnv("ALLOW_PRIVATE_URLS", "false") vi.stubGlobal( "fetch", vi.fn( @@ -222,37 +217,33 @@ describe("testing EdgeOne", () => { it("passes the admin's access code and cookies on", async () => { // The EdgeOne function checks the access code too - process.env.ADMIN_PASSWORD = "admin-pw" - try { - const calls = capture() - await testModel( - new Request("http://localhost/api/admin/test-model", { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-admin-password": "admin-pw", - "x-access-code": "visitor-code", - cookie: "eo_token=t", - origin: "https://draw.example", + vi.stubEnv("ADMIN_PASSWORD", "admin-pw") + const calls = capture() + await testModel( + new Request("http://localhost/api/admin/test-model", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-admin-password": "admin-pw", + "x-access-code": "visitor-code", + cookie: "eo_token=t", + origin: "https://draw.example", + }, + body: JSON.stringify({ + provider: { + id: "p1", + provider: "edgeone", + models: ["@tx/deepseek-ai/deepseek-v3-0324"], }, - body: JSON.stringify({ - provider: { - id: "p1", - provider: "edgeone", - models: ["@tx/deepseek-ai/deepseek-v3-0324"], - }, - modelId: "@tx/deepseek-ai/deepseek-v3-0324", - }), + modelId: "@tx/deepseek-ai/deepseek-v3-0324", }), - ) - expect(calls[0]?.url).toBe( - "https://draw.example/api/edgeai/chat/completions", - ) - expect(calls[0]?.headers.get("x-access-code")).toBe("visitor-code") - expect(calls[0]?.headers.get("cookie")).toBe("eo_token=t") - } finally { - delete process.env.ADMIN_PASSWORD - } + }), + ) + expect(calls[0]?.url).toBe( + "https://draw.example/api/edgeai/chat/completions", + ) + expect(calls[0]?.headers.get("x-access-code")).toBe("visitor-code") + expect(calls[0]?.headers.get("cookie")).toBe("eo_token=t") }) }) @@ -294,56 +285,45 @@ describe("the admin Test of the server's own base URL", () => { it("tests it as chat uses it: an internal address is allowed", async () => { // ALLOW_PRIVATE_URLS=false guards URLs users type, not the server's - process.env.ALLOW_PRIVATE_URLS = "false" - process.env.OPENAI_BASE_URL = "http://10.0.0.5:8000/v1" - process.env.ADMIN_PASSWORD = "admin-pw" + vi.stubEnv("ALLOW_PRIVATE_URLS", "false") + vi.stubEnv("OPENAI_BASE_URL", "http://10.0.0.5:8000/v1") + vi.stubEnv("ADMIN_PASSWORD", "admin-pw") privateUrls.all = true - try { - streamReply({ role: "assistant", content: "OK" }) - const admin = await (await adminTest()).json() - expect(admin.valid).toBe(true) - // A user typing the same address is still refused - const other = await userTest() - expect(other.status).toBe(400) - } finally { - privateUrls.all = false - delete process.env.OPENAI_BASE_URL - delete process.env.ADMIN_PASSWORD - } + streamReply({ role: "assistant", content: "OK" }) + const admin = await (await adminTest()).json() + expect(admin.valid).toBe(true) + // A user typing the same address is still refused + const other = await userTest() + expect(other.status).toBe(400) }) }) describe("the admin panel's Test button", () => { it("works when access codes are set", async () => { // The admin password stands in for the visitor access code - process.env.ACCESS_CODE_LIST = "visitor-code" - process.env.ADMIN_PASSWORD = "admin-pw" - try { - streamReply({ role: "assistant", content: "OK" }) - const res = await testModel( - new Request("http://localhost/api/admin/test-model", { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-admin-password": "admin-pw", + vi.stubEnv("ACCESS_CODE_LIST", "visitor-code") + vi.stubEnv("ADMIN_PASSWORD", "admin-pw") + streamReply({ role: "assistant", content: "OK" }) + const res = await testModel( + new Request("http://localhost/api/admin/test-model", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-admin-password": "admin-pw", + }, + body: JSON.stringify({ + provider: { + id: "p1", + provider: "glm", + apiKey: "key", + models: ["glm-5"], }, - body: JSON.stringify({ - provider: { - id: "p1", - provider: "glm", - apiKey: "key", - models: ["glm-5"], - }, - modelId: "glm-5", - }), + modelId: "glm-5", }), - ) - expect(res.status).toBe(200) - expect((await res.json()).valid).toBe(true) - } finally { - delete process.env.ACCESS_CODE_LIST - delete process.env.ADMIN_PASSWORD - } + }), + ) + expect(res.status).toBe(200) + expect((await res.json()).valid).toBe(true) }) }) diff --git a/vitest.config.mts b/vitest.config.mts index f454801c..e84633ed 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -7,5 +7,7 @@ export default defineConfig({ test: { environment: "jsdom", include: ["tests/**/*.test.{ts,tsx}"], + // vi.stubEnv values are undone after each test + unstubEnvs: true, }, })