Files
next-ai-draw-io/tests/e2e/provider-models.spec.ts
T
Dayuan Jiang c513d1a0c2 feat(ui): canvas-first redesign with a self-hosted draw.io (#957)
The canvas now fills the window and the chat floats on its right. draw.io is
served from our own origin (public/drawio, downloaded at build time and
checked against the release's SHA-256), so the app can drive the editor
directly:

- every AI change becomes a version card with a thumbnail; compare, restore
  or undo it, and Ctrl+Z on the canvas takes an AI change back in one step
- shapes the AI just changed are highlighted on the canvas
- selected shapes appear as a chip and are sent to the model
- draw.io's own simple UI: its toolbar, menus (auto layout, Mermaid, CSV
  import) and page tabs; the shape library starts closed
- new start screen, session rename and switcher, open .drawio files
- settings split into Models & keys, Appearance, Advanced and About
- Bedrock in the model settings also takes an API key instead of access keys
- phones switch between canvas and chat
- dark mode uses draw.io's own dark mode, with neutral greys around it

With an external draw.io (NEXT_PUBLIC_DRAWIO_BASE_URL) the features above
that need same origin are off.

The chat logic moved from the 1500-line chat panel into a chat engine plus
small UI components; zustand holds canvas, versions, settings and UI state.
2026-10-08 16:20:15 +09:00

570 lines
21 KiB
TypeScript

import { expect, type Page, test } from "@playwright/test"
import { getIframe } from "./lib/fixtures"
// qwen-mt-plus is a translation model; models.dev lists no tool calls for it
const CONFIG = {
version: 1,
providers: [
{
id: "p1",
provider: "qwen",
apiKey: "test-key",
models: [{ id: "m1", modelId: "qwen-mt-plus" }],
},
],
}
async function openQwenSettings(page: Page, config: object = CONFIG) {
await page.addInitScript((config) => {
localStorage.setItem(
"next-ai-draw-io-model-configs",
JSON.stringify(config),
)
}, config)
await page.goto("/", { waitUntil: "networkidle" })
await getIframe(page).waitFor({ state: "visible", timeout: 30000 })
await page.getByTestId("model-selector").click()
await page.getByText("Configure Models...").click()
const dialog = page.locator('[role="dialog"]')
await dialog.getByText("Qwen (Alibaba)").first().click()
return dialog
}
test("fetches the provider's models and adds one from the picker", async ({
page,
}) => {
let request: Record<string, unknown> | undefined
await page.route("**/api/provider-models", async (route) => {
request = route.request().postDataJSON()
await route.fulfill({
json: {
models: [
{ id: "qwen-new-max", tools: true },
{ id: "qwen-text-only", tools: false },
],
},
})
})
const dialog = await openQwenSettings(page)
await expect(
dialog.getByText("may not be able to draw").first(),
).toBeVisible()
await dialog
.getByRole("button", { name: "Fetch models from the provider" })
.click()
const picker = page.locator('[role="listbox"]')
await expect(picker.getByText("qwen-new-max")).toBeVisible()
await expect(
picker.getByRole("option", { name: /qwen-text-only/ }),
).toContainText("no tool calls")
expect(request).toMatchObject({ provider: "qwen", apiKey: "test-key" })
await page.getByPlaceholder("Search models...").fill("new-max")
await picker.getByText("qwen-new-max").click()
await expect(dialog.locator('input[title="qwen-new-max"]')).toBeVisible()
})
test("the model picker scrolls with the mouse wheel", async ({ page }) => {
// The picker sits in a popover above the settings dialog, which blocks
// wheel events outside itself
await page.route("**/api/provider-models", (route) =>
route.fulfill({
json: {
models: Array.from({ length: 60 }, (_, i) => ({
id: `qwen-model-${i}`,
})),
},
}),
)
const dialog = await openQwenSettings(page)
await dialog
.getByRole("button", { name: "Fetch models from the provider" })
.click()
const list = page.locator("[cmdk-list]")
await expect(list.getByText("qwen-model-0")).toBeVisible()
await list.hover()
await page.mouse.wheel(0, 400)
await expect
.poll(() => list.evaluate((el) => el.scrollTop), { timeout: 3000 })
.toBeGreaterThan(0)
})
test("shows a hint when the provider rejects the key", async ({ page }) => {
await page.route("**/api/provider-models", (route) =>
route.fulfill({
status: 401,
json: { code: "invalid_api_key", error: "Incorrect API key" },
}),
)
const dialog = await openQwenSettings(page)
await dialog
.getByRole("button", { name: "Fetch models from the provider" })
.click()
await expect(
dialog.getByText(
"The provider rejected the API key. Check it in model settings. Incorrect API key",
),
).toBeVisible()
})
test("a fetch error stays with its provider", async ({ page }) => {
await page.route("**/api/provider-models", (route) =>
route.fulfill({
status: 401,
json: { code: "invalid_api_key", error: "Incorrect API key" },
}),
)
const dialog = await openQwenSettings(page, {
version: 1,
providers: [
...CONFIG.providers,
{ id: "p2", provider: "glm", apiKey: "k", models: [] },
],
})
await dialog
.getByRole("button", { name: "Fetch models from the provider" })
.click()
const error = dialog.getByText("Incorrect API key")
await expect(error).toBeVisible()
await dialog.getByText("GLM (Zhipu)").first().click()
await expect(error).toHaveCount(0)
})
test("editing a model id clears the old test warning", async ({ page }) => {
const warning = "Connected, but the model answered without calling a tool."
const dialog = await openQwenSettings(page, {
version: 1,
providers: [
{
...CONFIG.providers[0],
models: [
{
id: "m1",
modelId: "qwen-max",
validated: true,
validationWarning: warning,
},
],
},
],
})
await expect(dialog.getByText(warning)).toBeVisible()
const input = dialog.locator('input[title="qwen-max"]')
await input.fill("qwen-mt-plus")
await input.blur()
await expect(dialog.getByText(warning)).toHaveCount(0)
await expect(dialog.getByText("may not be able to draw")).toBeVisible()
})
/** Hold requests to an endpoint until release() answers them with json */
async function holdRoute(page: Page, url: string, json: object) {
let release!: () => void
const released = new Promise<void>((r) => {
release = r
})
await page.route(url, async (route) => {
await released
await route.fulfill({ status: 200, json })
})
return release
}
const TWO_PROVIDERS = {
version: 1,
providers: [
{
...CONFIG.providers[0],
models: [{ id: "m1", modelId: "qwen-max" }],
},
{ id: "p2", provider: "glm", apiKey: "k", models: [] },
],
}
test("a model list that arrives after switching provider stays with its provider", async ({
page,
}) => {
const release = await holdRoute(page, "**/api/provider-models", {
code: "invalid_api_key",
error: "Incorrect API key",
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog
.getByRole("button", { name: "Fetch models from the provider" })
.click()
await dialog.getByText("GLM (Zhipu)").first().click()
release()
await page.waitForTimeout(500)
await expect(dialog.getByText("Incorrect API key")).toHaveCount(0)
})
test("a test result that arrives after switching provider stays with its provider", async ({
page,
}) => {
const release = await holdRoute(page, "**/api/validate-model", {
valid: false,
error: "Model not found",
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog.getByRole("button", { name: "Test", exact: true }).click()
await dialog.getByText("GLM (Zhipu)").first().click()
release()
await page.waitForTimeout(500)
await expect(dialog.getByText(/model\(s\) failed validation/)).toHaveCount(
0,
)
})
test("a test result does not count for a model id changed meanwhile", async ({
page,
}) => {
const release = await holdRoute(page, "**/api/validate-model", {
valid: true,
responseTime: 1000,
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog.getByRole("button", { name: "Test", exact: true }).click()
const input = dialog.locator('input[title="qwen-max"]')
await input.fill("qwen-plus")
await input.blur()
release()
await page.waitForTimeout(500)
await expect(dialog.locator('[title="1.0 s"]')).toHaveCount(0)
})
test("a model list fetched with an old API key is dropped", async ({
page,
}) => {
const release = await holdRoute(page, "**/api/provider-models", {
models: [{ id: "model-of-old-key", tools: true }],
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog
.getByRole("button", { name: "Fetch models from the provider" })
.click()
// The user corrects the key while the list is loading
await dialog.locator("#api-key").fill("new-key")
release()
await page.waitForTimeout(500)
await expect(page.getByText("model-of-old-key")).toHaveCount(0)
})
test("a test result for an old API key is dropped", async ({ page }) => {
const release = await holdRoute(page, "**/api/validate-model", {
valid: true,
responseTime: 1000,
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog.getByRole("button", { name: "Test", exact: true }).click()
// Another tab saves a different key for this provider
await page.evaluate(() => {
const key = "next-ai-draw-io-model-configs"
const config = JSON.parse(localStorage.getItem(key) ?? "{}")
config.providers[0].apiKey = "key-from-another-tab"
const value = JSON.stringify(config)
localStorage.setItem(key, value)
window.dispatchEvent(
new StorageEvent("storage", { key, newValue: value }),
)
})
release()
await page.waitForTimeout(500)
await expect(dialog.locator('[title="1.0 s"]')).toHaveCount(0)
// The test is over: the button works again and nothing spins
await expect(
dialog.getByRole("button", { name: "Test", exact: true }),
).toBeEnabled()
await expect(dialog.locator(".animate-spin")).toHaveCount(0)
})
test("an older test does not end a newer one's spinners", async ({ page }) => {
// Each validate request waits for its own release
const releases: Array<() => void> = []
await page.route("**/api/validate-model", async (route) => {
await new Promise<void>((r) => releases.push(r))
await route.fulfill({
status: 200,
json: { valid: true, responseTime: 1000 },
})
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog.getByRole("button", { name: "Test", exact: true }).click()
await expect.poll(() => releases.length).toBe(1)
// The user corrects the key and tests again
await dialog.locator("#api-key").fill("new-key")
await dialog.getByRole("button", { name: "Test", exact: true }).click()
await expect.poll(() => releases.length).toBe(2)
releases[0]()
await page.waitForTimeout(500)
await expect(dialog.locator(".animate-spin").first()).toBeVisible()
releases[1]()
await expect(dialog.locator('[title="1.0 s"]')).toHaveCount(1)
})
test("an older test touches nothing, also when the key came back", async ({
page,
}) => {
const releases: Array<() => void> = []
await page.route("**/api/validate-model", async (route) => {
const n = releases.length
await new Promise<void>((r) => releases.push(r))
await route.fulfill({
status: 200,
// The older test's result would say 9.0 s
json: { valid: true, responseTime: n === 0 ? 9000 : 1000 },
})
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog.getByRole("button", { name: "Test", exact: true }).click()
await expect.poll(() => releases.length).toBe(1)
// The key changes and comes back, and the user tests again
await dialog.locator("#api-key").fill("other-key")
await dialog.locator("#api-key").fill("test-key")
await dialog.getByRole("button", { name: "Test", exact: true }).click()
await expect.poll(() => releases.length).toBe(2)
releases[0]()
await page.waitForTimeout(500)
await expect(dialog.locator(".animate-spin").first()).toBeVisible()
await expect(dialog.locator('[title="9.0 s"]')).toHaveCount(0)
releases[1]()
await expect(dialog.locator('[title="1.0 s"]')).toHaveCount(1)
})
test("no spinner stays after another tab's change while elsewhere", async ({
page,
}) => {
const release = await holdRoute(page, "**/api/validate-model", {
valid: true,
responseTime: 1000,
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog.getByRole("button", { name: "Test", exact: true }).click()
// The user looks at the other provider while another tab changes the key
await dialog.getByText("GLM (Zhipu)").first().click()
await page.evaluate(() => {
const key = "next-ai-draw-io-model-configs"
const config = JSON.parse(localStorage.getItem(key) ?? "{}")
config.providers[0].apiKey = "key-from-another-tab"
const value = JSON.stringify(config)
localStorage.setItem(key, value)
window.dispatchEvent(
new StorageEvent("storage", { key, newValue: value }),
)
})
release()
await page.waitForTimeout(500)
await dialog.getByText("Qwen (Alibaba)").first().click()
await expect(dialog.locator(".animate-spin")).toHaveCount(0)
})
const savedConfig = (page: Page) =>
page.evaluate(() =>
JSON.parse(
localStorage.getItem("next-ai-draw-io-model-configs") ?? "{}",
),
)
test("a test result that arrives after the settings closed is kept", async ({
page,
}) => {
const release = await holdRoute(page, "**/api/validate-model", {
valid: true,
responseTime: 1000,
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog.getByRole("button", { name: "Test", exact: true }).click()
await page.keyboard.press("Escape")
await expect(dialog).toHaveCount(0)
release()
await expect
.poll(
async () =>
(await savedConfig(page)).providers[0].models[0].validated,
)
.toBe(true)
})
test("a test result for a key changed on another settings tab is dropped", async ({
page,
}) => {
const release = await holdRoute(page, "**/api/validate-model", {
valid: true,
responseTime: 1000,
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog.getByRole("button", { name: "Test", exact: true }).click()
// The panel goes away while the test runs, then the key changes
await page.locator('[data-testid="settings-tab-advanced"]').click()
await page.locator('[data-testid="settings-tab-models"]').click()
await dialog.getByText("Qwen (Alibaba)").first().click()
await dialog.locator("input[type=password]").first().fill("new-key")
release()
await page.waitForTimeout(800)
const saved = await savedConfig(page)
expect(saved.providers[0].apiKey).toBe("new-key")
expect(saved.providers[0].models[0].validated).not.toBe(true)
expect(saved.providers[0].validated).not.toBe(true)
})
test("a test result for a model renamed on another settings tab is dropped", async ({
page,
}) => {
const release = await holdRoute(page, "**/api/validate-model", {
valid: true,
responseTime: 1000,
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog.getByRole("button", { name: "Test", exact: true }).click()
// The panel goes away while the test runs, then the model id changes
await page.locator('[data-testid="settings-tab-advanced"]').click()
await page.locator('[data-testid="settings-tab-models"]').click()
await dialog.getByText("Qwen (Alibaba)").first().click()
const input = dialog.locator('input[title="qwen-max"]')
await input.fill("qwen-plus")
await input.blur()
release()
await page.waitForTimeout(800)
const saved = await savedConfig(page)
expect(saved.providers[0].models[0].modelId).toBe("qwen-plus")
expect(saved.providers[0].models[0].validated).not.toBe(true)
})
test("a failed test speaks the page's language", async ({ page }) => {
await page.addInitScript((config) => {
localStorage.setItem("next-ai-draw-io-locale", "zh")
localStorage.setItem(
"next-ai-draw-io-model-configs",
JSON.stringify(config),
)
}, TWO_PROVIDERS)
await page.route("**/api/validate-model", (route) => route.abort())
await page.goto("/zh", { waitUntil: "networkidle" })
await getIframe(page).waitFor({ state: "visible", timeout: 30000 })
await page.getByTestId("settings-button").first().click()
const dialog = page.locator('[role="dialog"]')
await dialog.getByText("Qwen (Alibaba)").first().click()
await dialog.getByRole("button", { name: "测试", exact: true }).click()
await expect(dialog.getByText("1 个模型验证失败")).toBeVisible()
await expect(
dialog.getByText("网络错误。请检查您的连接。").first(),
).toBeAttached()
})
test("testing a provider without models speaks the page's language", async ({
page,
}) => {
await page.addInitScript(
(config) => {
localStorage.setItem("next-ai-draw-io-locale", "zh")
localStorage.setItem(
"next-ai-draw-io-model-configs",
JSON.stringify(config),
)
},
{
version: 1,
providers: [{ ...CONFIG.providers[0], models: [] }],
},
)
await page.goto("/zh", { waitUntil: "networkidle" })
await getIframe(page).waitFor({ state: "visible", timeout: 30000 })
await page.getByTestId("settings-button").first().click()
const dialog = page.locator('[role="dialog"]')
await dialog.getByText("Qwen (Alibaba)").first().click()
await dialog.getByRole("button", { name: "测试", exact: true }).click()
await expect(
dialog.getByText("请先添加至少一个模型以进行验证"),
).toBeVisible()
})
test("adding a model id twice is refused in the page's language", async ({
page,
}) => {
await page.addInitScript((config) => {
localStorage.setItem("next-ai-draw-io-locale", "zh")
localStorage.setItem(
"next-ai-draw-io-model-configs",
JSON.stringify(config),
)
}, CONFIG)
await page.goto("/zh", { waitUntil: "networkidle" })
await getIframe(page).waitFor({ state: "visible", timeout: 30000 })
await page.getByTestId("settings-button").first().click()
const dialog = page.locator('[role="dialog"]')
await dialog.getByText("Qwen (Alibaba)").first().click()
const input = dialog.getByPlaceholder("自定义模型 ID...")
await input.fill("qwen-mt-plus")
await input.press("Enter")
await expect(dialog.getByText("此模型 ID 已存在")).toBeVisible()
})
test("Bedrock can be tested and used with an API key alone", async ({
page,
}) => {
const bodies: any[] = []
await page.route("**/api/validate-model", async (route) => {
bodies.push(route.request().postDataJSON())
await route.fulfill({
status: 200,
json: { valid: true, responseTime: 900 },
})
})
await page.addInitScript(
(config) => {
localStorage.setItem(
"next-ai-draw-io-model-configs",
JSON.stringify(config),
)
},
{
version: 1,
providers: [
{
id: "b1",
provider: "bedrock",
awsRegion: "us-east-1",
models: [{ id: "m1", modelId: "amazon.nova-lite-v1:0" }],
},
],
},
)
await page.goto("/", { waitUntil: "networkidle" })
await getIframe(page).waitFor({ state: "visible", timeout: 30000 })
await page.getByTestId("model-selector").click()
await page.getByText("Configure Models...").click()
const dialog = page.locator('[role="dialog"]')
await dialog.getByText("Amazon Bedrock").first().click()
const test = dialog.getByRole("button", { name: "Test", exact: true })
await expect(test).toBeDisabled()
await dialog.getByLabel("Bedrock API key").fill("bedrock-api-key")
await expect(test).toBeEnabled()
await test.click()
await expect.poll(() => bodies.length).toBe(1)
expect(bodies[0]).toMatchObject({
provider: "bedrock",
apiKey: "bedrock-api-key",
awsRegion: "us-east-1",
})
// The chat sends it with the request
let headers: Record<string, string> = {}
await page.route("**/api/chat", async (route) => {
headers = route.request().headers()
await route.fulfill({
status: 200,
contentType: "text/event-stream",
body: 'data: {"type":"start"}\n\ndata: {"type":"finish"}\n\ndata: [DONE]\n\n',
})
})
await page.keyboard.press("Escape")
await page.getByTestId("model-selector").click()
await page.getByText("amazon.nova-lite-v1:0").first().click()
const input = page.getByTestId("chat-input")
await input.fill("Hello")
await input.press("ControlOrMeta+Enter")
await expect.poll(() => headers["x-ai-api-key"]).toBe("bedrock-api-key")
expect(headers["x-ai-provider"]).toBe("bedrock")
expect(headers["x-aws-region"]).toBe("us-east-1")
})