i/next-ai-draw-io
1
0
Fork 0
mirror of https://github.com/DayuanJiang/next-ai-draw-io.git synced 2026-10-10 03:29:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(settings): reorganize the settings dialog and the model settings

Browse Source 
Tabs Models, General, Drawing and About, each with a fixed title row.

The models tab becomes three pages: the user's providers with one status
each and the server's models; a searchable picker of all 24 providers in
four groups; and a provider page in three steps (connection, models,
test). A new provider gets its first suggested model, and a passing test
offers to use the model in the chat. Bedrock asks for one way of signing
in, with an optional session token for temporary access keys.

The composer's model picker lists every model, marking untested and
failed ones, and can open the provider picker. A chat error that offers
model settings opens the provider the request went to.
This commit is contained in:
dayuan.jiang
2026-10-08 17:44:08 +09:00
parent c513d1a0c2
commit 69fb66804d
32 changed files with 4237 additions and 1730 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/e2e/chat-errors.spec.ts
+10 -4
View File
@@ -32,10 +32,13 @@ test("a rejected key shows a hint, the provider's words and a settings button",
// First match: the Next.js dev overlay at the end also lists the error
await expect(page.getByText("Authentication Fails").first()).toBeVisible()
await page.getByRole("button", { name: "Open model settings" }).click()
// Settings opens on its models tab
// A server model answered: settings opens on the models tab's list
const dialog = page.getByRole("dialog", { name: "Settings" })
await expect(dialog).toBeVisible()
await expect(dialog.getByText("Providers", { exact: true })).toBeVisible()
await expect(
dialog.getByRole("heading", { name: "Models", exact: true }),
).toBeVisible()
await expect(dialog.getByText("Use your own API key")).toBeVisible()
})
test("an error before the stream shows its text, not raw JSON", async ({
@@ -99,8 +102,11 @@ test("a refused server key shows only the quota hint and a settings button", asy
0,
)
await page.getByRole("button", { name: "Open model settings" }).click()
// Settings opens on its models tab
// A server model answered: settings opens on the models tab's list
const dialog = page.getByRole("dialog", { name: "Settings" })
await expect(dialog).toBeVisible()
await expect(dialog.getByText("Providers", { exact: true })).toBeVisible()
await expect(
dialog.getByRole("heading", { name: "Models", exact: true }),
).toBeVisible()
await expect(dialog.getByText("Use your own API key")).toBeVisible()
})
tests/e2e/language.spec.ts
+1 -1
View File
@@ -15,7 +15,7 @@ import { createMockSSEResponse } from "./lib/helpers"
/** Pick a language in the settings, then close them */
async function pickLanguage(page: Page, name: string) {
await openSettingsTab(page, "appearance")
await openSettingsTab(page, "general")
await page.locator("#language-select").click()
await page.getByRole("option", { name, exact: true }).click()
await page.keyboard.press("Escape")
tests/e2e/lib/fixtures.ts
+2 -2
View File
@@ -112,10 +112,10 @@ export async function openSettings(page: Page) {
await expect(page.locator('[role="dialog"]')).toBeVisible({ timeout: 5000 })
}
/** Open settings on one of its pages: models, appearance, advanced, about */
/** Open settings on one of its pages: models, general, drawing, about */
export async function openSettingsTab(
page: Page,
tab: "models" | "appearance" | "advanced" | "about",
tab: "models" | "general" | "drawing" | "about",
) {
await openSettings(page)
await page.locator(`[data-testid="settings-tab-${tab}"]`).click()
tests/e2e/model-test.spec.ts
+6 -4
View File
@@ -51,14 +51,14 @@ test("the Test button checks all models at once and shows each result", async ({
await page.getByTestId("model-selector").click()
await page.getByText("Configure Models...").click()
const dialog = page.locator('[role="dialog"]')
await dialog.getByText("GLM (Zhipu)").first().click()
await dialog.getByRole("button", { name: "Test", exact: true }).click()
await dialog.getByTestId("provider-row-p1").click()
await dialog.getByTestId("test-models").click()
await expect(
dialog.getByText("answered without calling a tool"),
).toBeVisible({ timeout: 15000 })
await expect(dialog.getByText("Model not found")).toBeVisible()
await expect(dialog.locator('[title="1.2 s"]')).toBeVisible()
await expect(dialog.getByText("Works · 1.2 s")).toBeVisible()
expect(started.sort()).toEqual([
"model-broken",
"model-no-tools",
@@ -78,7 +78,7 @@ test("the key link and the base URL cleanup", async ({ page }) => {
await page.getByTestId("model-selector").click()
await page.getByText("Configure Models...").click()
const dialog = page.locator('[role="dialog"]')
await dialog.getByText("GLM (Zhipu)").first().click()
await dialog.getByTestId("provider-row-p1").click()
await expect(
dialog.getByRole("link", { name: "Get API key" }),
@@ -86,6 +86,8 @@ test("the key link and the base URL cleanup", async ({ page }) => {
"href",
"https://open.bigmodel.cn/usercenter/proj-mgmt/apikeys",
)
// The base URL is under "More options"
await dialog.getByRole("button", { name: /^More options/ }).click()
const baseUrl = dialog.locator("#base-url")
await baseUrl.fill("https://proxy.example.com/v4/chat/completions/")
await baseUrl.blur()
tests/e2e/provider-models.spec.ts
+63 -64
View File
@@ -1,4 +1,4 @@
import { expect, type Page, test } from "@playwright/test"
import { expect, type Locator, type Page, test } from "@playwright/test"
import { getIframe } from "./lib/fixtures"
// qwen-mt-plus is a translation model; models.dev lists no tool calls for it
@@ -14,6 +14,13 @@ const CONFIG = {
],
}
/** Show a provider's page, going back to the provider list first */
async function showProvider(dialog: Locator, id: string) {
const back = dialog.locator("header button", { hasText: /^(Models|模型)$/ })
if (await back.count()) await back.click()
await dialog.getByTestId(`provider-row-${id}`).click()
}
async function openQwenSettings(page: Page, config: object = CONFIG) {
await page.addInitScript((config) => {
localStorage.setItem(
@@ -26,10 +33,14 @@ async function openQwenSettings(page: Page, config: object = CONFIG) {
await page.getByTestId("model-selector").click()
await page.getByText("Configure Models...").click()
const dialog = page.locator('[role="dialog"]')
await dialog.getByText("Qwen (Alibaba)").first().click()
await showProvider(dialog, "p1")
return dialog
}
const fetchButton = (dialog: Locator) =>
dialog.getByRole("button", { name: /^Get all models from/ })
const testButton = (dialog: Locator) => dialog.getByTestId("test-models")
test("fetches the provider's models and adds one from the picker", async ({
page,
}) => {
@@ -51,9 +62,7 @@ test("fetches the provider's models and adds one from the picker", async ({
dialog.getByText("may not be able to draw").first(),
).toBeVisible()
await dialog
.getByRole("button", { name: "Fetch models from the provider" })
.click()
await fetchButton(dialog).click()
const picker = page.locator('[role="listbox"]')
await expect(picker.getByText("qwen-new-max")).toBeVisible()
await expect(
@@ -79,9 +88,7 @@ test("the model picker scrolls with the mouse wheel", async ({ page }) => {
}),
)
const dialog = await openQwenSettings(page)
await dialog
.getByRole("button", { name: "Fetch models from the provider" })
.click()
await fetchButton(dialog).click()
const list = page.locator("[cmdk-list]")
await expect(list.getByText("qwen-model-0")).toBeVisible()
await list.hover()
@@ -99,12 +106,10 @@ test("shows a hint when the provider rejects the key", async ({ page }) => {
}),
)
const dialog = await openQwenSettings(page)
await dialog
.getByRole("button", { name: "Fetch models from the provider" })
.click()
await fetchButton(dialog).click()
await expect(
dialog.getByText(
"The provider rejected the API key. Check it in model settings. Incorrect API key",
"The provider rejected this API key. Check that the key above was copied in full. Incorrect API key",
),
).toBeVisible()
})
@@ -123,12 +128,10 @@ test("a fetch error stays with its provider", async ({ page }) => {
{ id: "p2", provider: "glm", apiKey: "k", models: [] },
],
})
await dialog
.getByRole("button", { name: "Fetch models from the provider" })
.click()
await fetchButton(dialog).click()
const error = dialog.getByText("Incorrect API key")
await expect(error).toBeVisible()
await dialog.getByText("GLM (Zhipu)").first().click()
await showProvider(dialog, "p2")
await expect(error).toHaveCount(0)
})
@@ -190,10 +193,8 @@ test("a model list that arrives after switching provider stays with its provider
error: "Incorrect API key",
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog
.getByRole("button", { name: "Fetch models from the provider" })
.click()
await dialog.getByText("GLM (Zhipu)").first().click()
await fetchButton(dialog).click()
await showProvider(dialog, "p2")
release()
await page.waitForTimeout(500)
await expect(dialog.getByText("Incorrect API key")).toHaveCount(0)
@@ -207,13 +208,11 @@ test("a test result that arrives after switching provider stays with its provide
error: "Model not found",
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog.getByRole("button", { name: "Test", exact: true }).click()
await dialog.getByText("GLM (Zhipu)").first().click()
await testButton(dialog).click()
await showProvider(dialog, "p2")
release()
await page.waitForTimeout(500)
await expect(dialog.getByText(/model\(s\) failed validation/)).toHaveCount(
0,
)
await expect(dialog.getByText(/failed the test/)).toHaveCount(0)
})
test("a test result does not count for a model id changed meanwhile", async ({
@@ -224,13 +223,13 @@ test("a test result does not count for a model id changed meanwhile", async ({
responseTime: 1000,
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog.getByRole("button", { name: "Test", exact: true }).click()
await testButton(dialog).click()
const input = dialog.locator('input[title="qwen-max"]')
await input.fill("qwen-plus")
await input.blur()
release()
await page.waitForTimeout(500)
await expect(dialog.locator('[title="1.0 s"]')).toHaveCount(0)
await expect(dialog.getByText("Works · 1.0 s")).toHaveCount(0)
})
test("a model list fetched with an old API key is dropped", async ({
@@ -240,9 +239,7 @@ test("a model list fetched with an old API key is dropped", async ({
models: [{ id: "model-of-old-key", tools: true }],
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog
.getByRole("button", { name: "Fetch models from the provider" })
.click()
await fetchButton(dialog).click()
// The user corrects the key while the list is loading
await dialog.locator("#api-key").fill("new-key")
release()
@@ -256,7 +253,7 @@ test("a test result for an old API key is dropped", async ({ page }) => {
responseTime: 1000,
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog.getByRole("button", { name: "Test", exact: true }).click()
await testButton(dialog).click()
// Another tab saves a different key for this provider
await page.evaluate(() => {
const key = "next-ai-draw-io-model-configs"
@@ -270,11 +267,9 @@ test("a test result for an old API key is dropped", async ({ page }) => {
})
release()
await page.waitForTimeout(500)
await expect(dialog.locator('[title="1.0 s"]')).toHaveCount(0)
await expect(dialog.getByText("Works · 1.0 s")).toHaveCount(0)
// The test is over: the button works again and nothing spins
await expect(
dialog.getByRole("button", { name: "Test", exact: true }),
).toBeEnabled()
await expect(testButton(dialog)).toBeEnabled()
await expect(dialog.locator(".animate-spin")).toHaveCount(0)
})
@@ -289,17 +284,17 @@ test("an older test does not end a newer one's spinners", async ({ page }) => {
})
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog.getByRole("button", { name: "Test", exact: true }).click()
await testButton(dialog).click()
await expect.poll(() => releases.length).toBe(1)
// The user corrects the key and tests again
await dialog.locator("#api-key").fill("new-key")
await dialog.getByRole("button", { name: "Test", exact: true }).click()
await testButton(dialog).click()
await expect.poll(() => releases.length).toBe(2)
releases[0]()
await page.waitForTimeout(500)
await expect(dialog.locator(".animate-spin").first()).toBeVisible()
releases[1]()
await expect(dialog.locator('[title="1.0 s"]')).toHaveCount(1)
await expect(dialog.getByText("Works · 1.0 s")).toHaveCount(1)
})
test("an older test touches nothing, also when the key came back", async ({
@@ -316,19 +311,19 @@ test("an older test touches nothing, also when the key came back", async ({
})
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog.getByRole("button", { name: "Test", exact: true }).click()
await testButton(dialog).click()
await expect.poll(() => releases.length).toBe(1)
// The key changes and comes back, and the user tests again
await dialog.locator("#api-key").fill("other-key")
await dialog.locator("#api-key").fill("test-key")
await dialog.getByRole("button", { name: "Test", exact: true }).click()
await testButton(dialog).click()
await expect.poll(() => releases.length).toBe(2)
releases[0]()
await page.waitForTimeout(500)
await expect(dialog.locator(".animate-spin").first()).toBeVisible()
await expect(dialog.locator('[title="9.0 s"]')).toHaveCount(0)
await expect(dialog.getByText("Works · 9.0 s")).toHaveCount(0)
releases[1]()
await expect(dialog.locator('[title="1.0 s"]')).toHaveCount(1)
await expect(dialog.getByText("Works · 1.0 s")).toHaveCount(1)
})
test("no spinner stays after another tab's change while elsewhere", async ({
@@ -339,9 +334,9 @@ test("no spinner stays after another tab's change while elsewhere", async ({
responseTime: 1000,
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog.getByRole("button", { name: "Test", exact: true }).click()
await testButton(dialog).click()
// The user looks at the other provider while another tab changes the key
await dialog.getByText("GLM (Zhipu)").first().click()
await showProvider(dialog, "p2")
await page.evaluate(() => {
const key = "next-ai-draw-io-model-configs"
const config = JSON.parse(localStorage.getItem(key) ?? "{}")
@@ -354,7 +349,7 @@ test("no spinner stays after another tab's change while elsewhere", async ({
})
release()
await page.waitForTimeout(500)
await dialog.getByText("Qwen (Alibaba)").first().click()
await showProvider(dialog, "p1")
await expect(dialog.locator(".animate-spin")).toHaveCount(0)
})
@@ -373,7 +368,7 @@ test("a test result that arrives after the settings closed is kept", async ({
responseTime: 1000,
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog.getByRole("button", { name: "Test", exact: true }).click()
await testButton(dialog).click()
await page.keyboard.press("Escape")
await expect(dialog).toHaveCount(0)
release()
@@ -393,11 +388,11 @@ test("a test result for a key changed on another settings tab is dropped", async
responseTime: 1000,
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog.getByRole("button", { name: "Test", exact: true }).click()
await testButton(dialog).click()
// The panel goes away while the test runs, then the key changes
await page.locator('[data-testid="settings-tab-advanced"]').click()
await page.locator('[data-testid="settings-tab-general"]').click()
await page.locator('[data-testid="settings-tab-models"]').click()
await dialog.getByText("Qwen (Alibaba)").first().click()
await showProvider(dialog, "p1")
await dialog.locator("input[type=password]").first().fill("new-key")
release()
await page.waitForTimeout(800)
@@ -415,11 +410,11 @@ test("a test result for a model renamed on another settings tab is dropped", asy
responseTime: 1000,
})
const dialog = await openQwenSettings(page, TWO_PROVIDERS)
await dialog.getByRole("button", { name: "Test", exact: true }).click()
await testButton(dialog).click()
// The panel goes away while the test runs, then the model id changes
await page.locator('[data-testid="settings-tab-advanced"]').click()
await page.locator('[data-testid="settings-tab-general"]').click()
await page.locator('[data-testid="settings-tab-models"]').click()
await dialog.getByText("Qwen (Alibaba)").first().click()
await showProvider(dialog, "p1")
const input = dialog.locator('input[title="qwen-max"]')
await input.fill("qwen-plus")
await input.blur()
@@ -443,9 +438,14 @@ test("a failed test speaks the page's language", async ({ page }) => {
await getIframe(page).waitFor({ state: "visible", timeout: 30000 })
await page.getByTestId("settings-button").first().click()
const dialog = page.locator('[role="dialog"]')
await dialog.getByText("Qwen (Alibaba)").first().click()
await dialog.getByRole("button", { name: "测试", exact: true }).click()
await expect(dialog.getByText("1 个模型验证失败")).toBeVisible()
await showProvider(dialog, "p1")
await testButton(dialog).click()
// The visible line, not the screen reader's copy
await expect(
dialog
.getByText("1 个模型没通过测试")
.and(dialog.locator(':not([role="status"])')),
).toBeVisible()
await expect(
dialog.getByText("网络错误。请检查您的连接。").first(),
).toBeAttached()
@@ -471,11 +471,10 @@ test("testing a provider without models speaks the page's language", async ({
await getIframe(page).waitFor({ state: "visible", timeout: 30000 })
await page.getByTestId("settings-button").first().click()
const dialog = page.locator('[role="dialog"]')
await dialog.getByText("Qwen (Alibaba)").first().click()
await dialog.getByRole("button", { name: "测试", exact: true }).click()
await expect(
dialog.getByText("请先添加至少一个模型以进行验证"),
).toBeVisible()
await showProvider(dialog, "p1")
// Nothing to test yet: the button says what is missing
await expect(testButton(dialog)).toBeDisabled()
await expect(dialog.getByText("先添加一个模型。")).toBeVisible()
})
test("adding a model id twice is refused in the page's language", async ({
@@ -492,8 +491,8 @@ test("adding a model id twice is refused in the page's language", async ({
await getIframe(page).waitFor({ state: "visible", timeout: 30000 })
await page.getByTestId("settings-button").first().click()
const dialog = page.locator('[role="dialog"]')
await dialog.getByText("Qwen (Alibaba)").first().click()
const input = dialog.getByPlaceholder("自定义模型 ID...")
await showProvider(dialog, "p1")
const input = dialog.getByPlaceholder("或者输入其他模型 ID")
await input.fill("qwen-mt-plus")
await input.press("Enter")
await expect(dialog.getByText("此模型 ID 已存在")).toBeVisible()
@@ -534,8 +533,8 @@ test("Bedrock can be tested and used with an API key alone", async ({
await page.getByTestId("model-selector").click()
await page.getByText("Configure Models...").click()
const dialog = page.locator('[role="dialog"]')
await dialog.getByText("Amazon Bedrock").first().click()
const test = dialog.getByRole("button", { name: "Test", exact: true })
await showProvider(dialog, "b1")
const test = testButton(dialog)
await expect(test).toBeDisabled()
await dialog.getByLabel("Bedrock API key").fill("bedrock-api-key")
await expect(test).toBeEnabled()
tests/e2e/settings-providers.spec.ts
+504
View File
@@ -0,0 +1,504 @@
import type { Page } from "@playwright/test"
import {
expect,
getIframe,
openSettings,
openSettingsTab,
sendMessage,
test,
} from "./lib/fixtures"
const KEY = "next-ai-draw-io-model-configs"
async function openApp(page: Page, config?: object) {
if (config) {
await page.addInitScript(
([key, config]) => {
// Only on the first load: a reload keeps what the test did
if (!sessionStorage.getItem("seeded")) {
localStorage.setItem(key, JSON.stringify(config))
sessionStorage.setItem("seeded", "1")
}
},
[KEY, config] as const,
)
}
await page.goto("/", { waitUntil: "networkidle" })
await getIframe(page).waitFor({ state: "visible", timeout: 30000 })
}
const saved = (page: Page) =>
page.evaluate((key) => JSON.parse(localStorage.getItem(key) ?? "{}"), KEY)
const dialogOf = (page: Page) => page.getByRole("dialog", { name: "Settings" })
// Three providers: one with a failed model, one not set up, one working
const PROVIDERS = {
version: 1,
selectedModelId: "m4",
providers: [
{
id: "p1",
provider: "openai",
apiKey: "sk-test",
models: [
{ id: "m1", modelId: "gpt-6-astra", validated: true },
{
id: "m2",
modelId: "gpt-6.1-sol",
validated: false,
validationError: "Model not found",
},
{ id: "m3", modelId: "gpt-6-luna" },
],
},
{ id: "p2", provider: "bedrock", apiKey: "", models: [] },
{
id: "p3",
provider: "deepseek",
apiKey: "sk-ds",
models: [{ id: "m4", modelId: "deepseek-chat", validated: true }],
},
],
}
test("a first provider: pick it, fill in the key, test, use it", async ({
page,
}) => {
const tested: string[] = []
await page.route("**/api/validate-model", async (route) => {
tested.push(route.request().postDataJSON().modelId)
await route.fulfill({ json: { valid: true, responseTime: 1400 } })
})
await openApp(page)
await openSettings(page)
const dialog = dialogOf(page)
await expect(dialog.getByText("Use your own API key")).toBeVisible()
await dialog.getByTestId("pick-provider-deepseek").click()
// The first suggested model is added, and the key field has the cursor
await expect(
dialog.getByRole("heading", { name: "DeepSeek", exact: true }),
).toBeVisible()
await expect(dialog.locator('input[title="deepseek-v4-pro"]')).toBeVisible()
await expect(dialog.locator("#api-key")).toBeFocused()
await expect(dialog.getByTestId("step-1-todo")).toBeVisible()
await expect(dialog.getByTestId("step-2-done")).toBeVisible()
const testButton = dialog.getByTestId("test-models")
await expect(testButton).toBeDisabled()
await expect(dialog.getByText("Fill in the API key first.")).toBeVisible()
await dialog.locator("#api-key").fill("sk-1234")
await expect(dialog.getByTestId("step-1-done")).toBeVisible()
await expect(testButton).toHaveText("Test deepseek-v4-pro")
await testButton.click()
// The visible panel, not the screen reader's copy
await expect(
dialog
.getByText("deepseek-v4-pro works")
.and(dialog.locator(':not([role="status"])')),
).toBeVisible()
await expect(dialog.getByTestId("step-3-done")).toBeVisible()
expect(tested).toEqual(["deepseek-v4-pro"])
// The next step: use it in the chat
await dialog
.getByRole("button", { name: "Use deepseek-v4-pro in the chat" })
.click()
await expect(dialog).toHaveCount(0)
await expect(page.getByTestId("model-selector")).toContainText(
"deepseek-v4-pro",
)
const config = await saved(page)
expect(config.selectedModelId).toBe(config.providers[0].models[0].id)
})
test("the provider picker searches, marks added ones, and Enter picks", async ({
page,
}) => {
await openApp(page, PROVIDERS)
await page.getByTestId("model-selector").click()
// The model picker opens the settings straight on the provider picker
await page.getByText("Add a provider…").click()
const dialog = dialogOf(page)
const search = dialog.getByRole("searchbox")
await expect(search).toBeFocused()
await expect(dialog.getByTestId("pick-provider-openai")).toContainText(
"Added",
)
await expect(dialog.getByTestId("pick-provider-bedrock")).toContainText(
"AWS credentials · Added",
)
await expect(
dialog.getByTestId("pick-provider-anthropic"),
).not.toContainText("Added")
await expect(dialog.locator('[data-testid^="pick-provider-"]')).toHaveCount(
24,
)
await search.fill("zzz")
await expect(dialog.getByText("No provider matches.")).toBeVisible()
await search.fill("kimi")
await expect(dialog.locator('[data-testid^="pick-provider-"]')).toHaveCount(
1,
)
await search.press("Enter")
await expect(
dialog.getByRole("heading", { name: "Kimi (Moonshot)", exact: true }),
).toBeVisible()
expect((await saved(page)).providers.at(-1).provider).toBe("kimi")
})
test("the provider list shows what needs attention", async ({ page }) => {
await openApp(page, PROVIDERS)
await openSettings(page)
const dialog = dialogOf(page)
await expect(dialog.getByTestId("provider-row-p1")).toContainText(
"1 failed",
)
await expect(dialog.getByTestId("provider-row-p2")).toContainText(
"Not set up yet",
)
await expect(dialog.getByTestId("provider-row-p3")).toContainText(
"1 working",
)
// The selected model's provider is the one in use
await expect(dialog.getByTestId("provider-row-p3")).toContainText("In use")
await expect(dialog.getByTestId("provider-row-p1")).not.toContainText(
"In use",
)
})
test("the model picker lists untested and failed models", async ({ page }) => {
await openApp(page, PROVIDERS)
await page.getByTestId("model-selector").click()
// By the model config's id: the server may offer a model of that name
const option = (id: string) =>
page.locator(`[role="option"][data-value="${id}"]`)
await expect(option("m3")).toContainText("Untested")
await expect(option("m2")).toContainText("Failed")
await expect(option("m1")).not.toContainText(/Untested|Failed/)
// An untested model can be picked
await option("m3").click()
await expect(page.getByTestId("model-selector")).toContainText("gpt-6-luna")
})
test("an old setting to hide untested models is ignored", async ({ page }) => {
await openApp(page, { ...PROVIDERS, showUnvalidatedModels: false })
await page.getByTestId("model-selector").click()
await expect(page.locator('[role="option"][data-value="m3"]')).toBeVisible()
})
test("a refused key opens the settings on that provider", async ({ page }) => {
await page.route("**/api/chat", (route) =>
route.fulfill({
status: 401,
contentType: "application/json",
body: JSON.stringify({
type: "provider",
code: "invalid_api_key",
message: "Incorrect API key",
}),
}),
)
await openApp(page, { ...PROVIDERS, selectedModelId: "m1" })
await sendMessage(page, "Draw a box")
await page
.getByRole("button", { name: "Open model settings" })
.click({ timeout: 15000 })
const dialog = dialogOf(page)
await expect(
dialog.getByRole("heading", { name: "OpenAI", exact: true }),
).toBeVisible()
// Back to the list from there
await dialog.locator("header button", { hasText: "Models" }).click()
await expect(dialog.getByTestId("provider-row-p1")).toBeVisible()
})
test("an access code error opens the general tab with the code field", async ({
page,
}) => {
await page.route("**/api/config", (route) =>
route.fulfill({
json: {
accessCodeRequired: true,
dailyRequestLimit: 0,
dailyTokenLimit: 0,
tpmLimit: 0,
},
}),
)
await page.route("**/api/chat", (route) =>
route.fulfill({
status: 401,
contentType: "application/json",
body: JSON.stringify({
error: "Invalid or missing access code. Please configure it in Settings.",
}),
}),
)
await openApp(page)
await sendMessage(page, "Draw a box")
const dialog = dialogOf(page)
await expect(dialog.locator("#access-code")).toBeVisible({ timeout: 15000 })
await expect(dialog.getByTestId("settings-tab-general")).toHaveAttribute(
"aria-current",
"page",
)
})
test("a refused key marks the key field until it changes", async ({ page }) => {
await page.route("**/api/validate-model", (route) =>
route.fulfill({
json: {
valid: false,
code: "invalid_api_key",
error: "Incorrect API key",
},
}),
)
await openApp(page, PROVIDERS)
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-p3").click()
await dialog.getByTestId("test-models").click()
const key = dialog.locator("#api-key")
await expect(key).toHaveAttribute("aria-invalid", "true")
await expect(
dialog.getByText(
"The provider rejected this API key. Check that the key above was copied in full. Incorrect API key",
),
).toBeVisible()
// The visible line, not the screen reader's copy
await expect(
dialog
.getByText("1 model failed the test")
.and(dialog.locator(':not([role="status"])')),
).toBeVisible()
await key.fill("sk-new")
await expect(key).not.toHaveAttribute("aria-invalid", "true")
})
test("Bedrock shows one way of signing in and empties the other", async ({
page,
}) => {
await openApp(page, {
version: 1,
providers: [
{
id: "b1",
provider: "bedrock",
apiKey: "bedrock-key",
awsRegion: "us-east-1",
// Left by an older version: belongs to the access keys
awsSessionToken: "session-token",
models: [
{
id: "m1",
modelId: "amazon.nova-lite-v1:0",
validated: true,
},
],
},
],
})
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-b1").click()
await expect(dialog.getByLabel("Bedrock API key")).toHaveValue(
"bedrock-key",
)
await expect(dialog.locator("#aws-access-key-id")).toHaveCount(0)
await dialog
.getByRole("button", { name: "Access keys", exact: true })
.click()
await expect(dialog.locator("#bedrock-api-key")).toHaveCount(0)
// The key that was tested is gone, and so is its test result
await expect
.poll(async () => (await saved(page)).providers[0].models[0].validated)
.toBeUndefined()
await dialog.locator("#aws-access-key-id").fill("AKIATEST")
await dialog.locator("#aws-secret-access-key").fill("secret")
await expect
.poll(async () => (await saved(page)).providers[0])
.toMatchObject({
apiKey: "",
awsAccessKeyId: "AKIATEST",
awsSecretAccessKey: "secret",
})
// Reopened, it shows the way that is filled in
await page.keyboard.press("Escape")
await openSettings(page)
await dialog.getByTestId("provider-row-b1").click()
await expect(
dialog.getByRole("button", { name: "Access keys", exact: true }),
).toHaveAttribute("aria-pressed", "true")
await dialog
.getByRole("button", { name: "Bedrock API key", exact: true })
.click()
await expect
.poll(async () => (await saved(page)).providers[0])
.toMatchObject({
awsAccessKeyId: "",
awsSecretAccessKey: "",
awsSessionToken: "",
})
})
test("switching Bedrock's sign-in with nothing filled in keeps the test results", async ({
page,
}) => {
// Older configs may hold a result without the key that earned it
await openApp(page, {
version: 1,
providers: [
{
id: "b1",
provider: "bedrock",
apiKey: "",
awsRegion: "us-east-1",
models: [
{
id: "m1",
modelId: "amazon.nova-lite-v1:0",
validated: true,
},
],
},
],
})
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-b1").click()
await dialog
.getByRole("button", { name: "Access keys", exact: true })
.click()
await expect(dialog.locator("#aws-access-key-id")).toBeVisible()
await page.waitForTimeout(300)
expect((await saved(page)).providers[0].models[0].validated).toBe(true)
})
test("Use in the chat waits while an answer runs", async ({ page }) => {
// The chat request stays open until released
let release!: () => void
const released = new Promise<void>((r) => {
release = r
})
await page.route("**/api/chat", async (route) => {
await released
await route.fulfill({
status: 200,
contentType: "text/event-stream",
body: 'data: {"type":"start"}\n\ndata: {"type":"finish"}\n\ndata: [DONE]\n\n',
})
})
await page.route("**/api/validate-model", (route) =>
route.fulfill({ json: { valid: true, responseTime: 900 } }),
)
await openApp(page, PROVIDERS)
await sendMessage(page, "Draw a box")
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-p3").click()
await dialog.getByTestId("test-models").click()
const use = dialog.getByRole("button", {
name: "Use deepseek-chat in the chat",
})
await expect(use).toBeDisabled()
release()
await expect(use).toBeEnabled({ timeout: 15000 })
})
test("more options open by themselves only when they matter", async ({
page,
}) => {
await openApp(page, {
version: 1,
providers: [
{ id: "a1", provider: "azure", apiKey: "k", models: [] },
{ id: "o1", provider: "openai", apiKey: "k", models: [] },
{
id: "o2",
provider: "openai",
apiKey: "k",
baseUrl: "https://proxy.example.com/v1",
models: [],
},
],
})
await openSettings(page)
const dialog = dialogOf(page)
const back = () =>
dialog.locator("header button", { hasText: "Models" }).click()
// Azure's default URL is only an example
await dialog.getByTestId("provider-row-a1").click()
await expect(dialog.locator("#base-url")).toBeVisible()
await back()
await dialog.getByTestId("provider-row-o1").click()
await expect(dialog.locator("#base-url")).toHaveCount(0)
await back()
await dialog.getByTestId("provider-row-o2").click()
await expect(dialog.locator("#base-url")).toHaveValue(
"https://proxy.example.com/v1",
)
})
test("the models tab keeps its page across tabs, not across openings", async ({
page,
}) => {
await openApp(page, PROVIDERS)
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-p1").click()
await dialog.getByTestId("settings-tab-drawing").click()
await expect(dialog.locator("#max-output-tokens")).toBeVisible()
await dialog.getByTestId("settings-tab-models").click()
await expect(
dialog.getByRole("heading", { name: "OpenAI", exact: true }),
).toBeVisible()
await page.keyboard.press("Escape")
await openSettings(page)
await expect(dialog.getByTestId("provider-row-p1")).toBeVisible()
})
test("deleting a provider says what goes with it", async ({ page }) => {
await openApp(page, PROVIDERS)
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-p3").click()
await dialog
.getByRole("button", { name: "Delete DeepSeek and its model" })
.click()
await page
.getByRole("alertdialog")
.getByRole("button", { name: "Delete" })
.click()
await expect(dialog.getByTestId("provider-row-p3")).toHaveCount(0)
await expect(dialog.getByTestId("provider-row-p1")).toBeVisible()
expect((await saved(page)).providers).toHaveLength(2)
})
test("the general and drawing tabs hold their settings", async ({ page }) => {
await openApp(page)
await openSettingsTab(page, "general")
const dialog = dialogOf(page)
for (const id of ["#language-select", "#send-shortcut-select"]) {
await expect(dialog.locator(id)).toBeVisible()
}
await expect(
dialog.getByRole("radiogroup", { name: "Theme" }),
).toBeVisible()
await dialog.getByTestId("settings-tab-drawing").click()
for (const id of [
"#custom-system-message",
"#vlm-validation",
"#max-output-tokens",
]) {
await expect(dialog.locator(id)).toBeVisible()
}
await expect(
dialog.getByRole("radiogroup", { name: "Diagram style" }),
).toBeVisible()
})
tests/e2e/settings-review-fixes.spec.ts
+830
View File
@@ -0,0 +1,830 @@
// What the settings review found: each test fails without its fix
import type { Locator, Page } from "@playwright/test"
import {
expect,
getIframe,
openSettings,
sendMessage,
test,
} from "./lib/fixtures"
const KEY = "next-ai-draw-io-model-configs"
async function openApp(page: Page, config?: object) {
if (config) {
await page.addInitScript(
([key, config]) => {
if (!sessionStorage.getItem("seeded")) {
localStorage.setItem(key, JSON.stringify(config))
sessionStorage.setItem("seeded", "1")
}
},
[KEY, config] as const,
)
}
await page.goto("/", { waitUntil: "networkidle" })
await getIframe(page).waitFor({ state: "visible", timeout: 30000 })
}
const dialogOf = (page: Page) => page.getByRole("dialog", { name: "Settings" })
const saved = (page: Page) =>
page.evaluate((key) => JSON.parse(localStorage.getItem(key) ?? "{}"), KEY)
/** What another browser tab saving the config looks like to this one */
async function changeInAnotherTab(page: Page, change: (config: any) => void) {
await page.evaluate(
([key, body]) => {
const config = JSON.parse(localStorage.getItem(key) ?? "{}")
new Function("config", body)(config)
const value = JSON.stringify(config)
localStorage.setItem(key, value)
window.dispatchEvent(
new StorageEvent("storage", { key, newValue: value }),
)
},
[KEY, `(${change.toString()})(config)`] as const,
)
}
const back = (dialog: Locator) =>
dialog.locator("header button", { hasText: /^Models$/ }).click()
/** Hold the requests to an endpoint; each waits for its own release */
async function holdEach(
page: Page,
url: string,
answer: (n: number) => object,
) {
const releases: Array<() => void> = []
await page.route(url, async (route) => {
const n = releases.length
await new Promise<void>((r) => releases.push(r))
await route.fulfill({ json: answer(n) })
})
return releases
}
const TWO = {
version: 1,
providers: [
{
id: "p1",
provider: "glm",
apiKey: "k1",
models: [{ id: "a1", modelId: "glm-a" }],
},
{
id: "p2",
provider: "kimi",
apiKey: "k2",
models: [{ id: "b1", modelId: "kimi-b" }],
},
],
}
test("Add a provider… leaves the cursor in the provider search", async ({
page,
}) => {
await openApp(page)
await page.getByTestId("model-selector").click()
await page.getByText("Add a provider…").click()
const search = dialogOf(page).getByRole("searchbox")
await expect(search).toBeFocused()
// After the model picker has closed
await page.waitForTimeout(800)
await expect(search).toBeFocused()
await page.keyboard.type("kimi")
await expect(search).toHaveValue("kimi")
})
test("the model picker's footer is there while searching and reachable by Tab", async ({
page,
}) => {
await openApp(page)
await page.getByTestId("model-selector").click()
await page.keyboard.type("zzzz")
await expect(page.getByText("Add a provider…")).toBeVisible()
const add = page.getByRole("button", { name: "Add a provider…" })
for (let i = 0; i < 6; i++) {
if (await add.evaluate((el) => el === document.activeElement)) break
await page.keyboard.press("Tab")
}
await expect(add).toBeFocused()
await page.keyboard.press("Enter")
await expect(dialogOf(page).getByRole("searchbox")).toBeVisible()
})
test("an Enter that confirms an IME candidate picks no provider", async ({
page,
}) => {
await openApp(page)
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByRole("button", { name: /See all/ }).click()
const search = dialog.getByRole("searchbox")
await search.fill("deep")
await search.evaluate((el) =>
el.dispatchEvent(
new KeyboardEvent("keydown", {
key: "Enter",
bubbles: true,
isComposing: true,
}),
),
)
await page.waitForTimeout(300)
await expect(search).toBeVisible()
expect((await saved(page)).providers ?? []).toHaveLength(0)
})
test("the focus follows the page within the models tab", async ({ page }) => {
await openApp(page, TWO)
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-p1").focus()
await page.keyboard.press("Enter")
await expect(
dialog.getByRole("heading", { name: "GLM (Zhipu)", exact: true }),
).toBeFocused()
await back(dialog)
await expect(
dialog.getByRole("heading", { name: "Models", exact: true }),
).toBeFocused()
})
test("long names stay inside the page", async ({ page }) => {
const long = `model-${"x".repeat(150)}`
await openApp(page, {
version: 1,
providers: [
{
id: "p1",
provider: "openai",
name: `My ${"provider ".repeat(10)}name`,
apiKey: "k",
models: [{ id: "m1", modelId: long }],
},
],
})
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-p1").click()
const right = async (l: Locator) => {
const box = await l.boundingBox()
return box ? box.x + box.width : 0
}
const pageRight = await right(dialog)
expect(await right(dialog.getByTestId("test-models"))).toBeLessThanOrEqual(
pageRight,
)
expect(
await right(
dialog.getByRole("button", { name: /^Get all models from/ }),
),
).toBeLessThanOrEqual(pageRight)
})
test("Azure needs its resource URL before it counts as connected", async ({
page,
}) => {
await openApp(page, {
version: 1,
providers: [
{
id: "z1",
provider: "azure",
apiKey: "k",
baseUrl: "https://your-resource.openai.azure.com/openai",
models: [{ id: "m1", modelId: "gpt-6-astra" }],
},
],
})
await openSettings(page)
const dialog = dialogOf(page)
await expect(dialog.getByTestId("provider-row-z1")).toContainText(
"Not set up yet",
)
await dialog.getByTestId("provider-row-z1").click()
await expect(dialog.getByTestId("step-1-todo")).toBeVisible()
await expect(dialog.getByTestId("test-models")).toBeDisabled()
await expect(
dialog.getByText(
"Fill in the API key and your Azure resource's base URL first.",
),
).toBeVisible()
await dialog
.locator("#base-url")
.fill("https://contoso.openai.azure.com/openai")
await expect(dialog.getByTestId("test-models")).toBeEnabled()
})
test("a refused key unchecks step 1, and a fix in another tab clears it", async ({
page,
}) => {
await page.route("**/api/validate-model", (route) =>
route.fulfill({
json: { valid: false, code: "invalid_api_key", error: "Bad key" },
}),
)
await openApp(page, TWO)
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-p1").click()
await dialog.getByTestId("test-models").click()
const key = dialog.locator("#api-key")
await expect(key).toHaveAttribute("aria-invalid", "true")
await expect(dialog.getByTestId("step-1-todo")).toBeVisible()
await changeInAnotherTab(page, (config) => {
config.providers[0].apiKey = "fixed-key"
config.providers[0].models[0].validated = undefined
config.providers[0].models[0].validationError = undefined
})
await expect(key).toHaveValue("fixed-key")
await expect(key).not.toHaveAttribute("aria-invalid", "true")
await expect(dialog.getByTestId("step-1-done")).toBeVisible()
await expect(dialog.getByText("1 model failed the test")).toHaveCount(0)
})
test("testing another provider keeps the first one's results", async ({
page,
}) => {
const releases = await holdEach(page, "**/api/validate-model", () => ({
valid: true,
responseTime: 1000,
}))
await openApp(page, TWO)
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-p1").click()
await dialog.getByTestId("test-models").click()
await expect.poll(() => releases.length).toBe(1)
await back(dialog)
await dialog.getByTestId("provider-row-p2").click()
await dialog.getByTestId("test-models").click()
await expect.poll(() => releases.length).toBe(2)
for (const release of releases) release()
await expect
.poll(async () =>
(await saved(page)).providers.map(
(p: any) => p.models[0].validated,
),
)
.toEqual([true, true])
})
test("an older test from before the tabs changed does not undo a newer one", async ({
page,
}) => {
// The first test fails, the second passes; the first answers last
const releases = await holdEach(page, "**/api/validate-model", (n) =>
n === 0
? { valid: false, error: "Old failure" }
: { valid: true, responseTime: 1000 },
)
await openApp(page, TWO)
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-p1").click()
await dialog.getByTestId("test-models").click()
await expect.poll(() => releases.length).toBe(1)
await dialog.getByTestId("settings-tab-general").click()
await dialog.getByTestId("settings-tab-models").click()
await dialog.getByTestId("test-models").click()
await expect.poll(() => releases.length).toBe(2)
releases[1]()
await expect(dialog.getByText("Works · 1.0 s")).toBeVisible()
releases[0]()
await page.waitForTimeout(500)
await expect(dialog.getByText("Works · 1.0 s")).toBeVisible()
expect((await saved(page)).providers[0].models[0].validated).toBe(true)
})
test("a model list fetched with a key another tab replaced is not offered", async ({
page,
}) => {
await page.route("**/api/provider-models", (route) =>
route.fulfill({ json: { models: [{ id: "model-of-old-key" }] } }),
)
await openApp(page, TWO)
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-p1").click()
await dialog.getByRole("button", { name: /^Get all models from/ }).click()
await expect(
page.getByRole("option", { name: "model-of-old-key" }),
).toBeVisible()
await page.keyboard.press("Escape")
await changeInAnotherTab(page, (config) => {
config.providers[0].apiKey = "another-key"
})
await expect(dialog.locator("#api-key")).toHaveValue("another-key")
// GLM has no suggested models: nothing to browse until fetched again
await expect(
dialog.getByRole("button", { name: "Browse models" }),
).toHaveCount(0)
})
test("a model list request keeps only its own provider busy", async ({
page,
}) => {
const releases = await holdEach(page, "**/api/provider-models", () => ({
models: [],
}))
await openApp(page, TWO)
await openSettings(page)
const dialog = dialogOf(page)
const fetchButton = dialog.getByRole("button", {
name: /^Get all models from/,
})
await dialog.getByTestId("provider-row-p1").click()
await fetchButton.click()
await expect.poll(() => releases.length).toBe(1)
await expect(fetchButton).toBeDisabled()
await back(dialog)
await dialog.getByTestId("provider-row-p2").click()
await expect(fetchButton).toBeEnabled()
for (const release of releases) release()
})
test("a model id typed for one provider does not follow to the next", async ({
page,
}) => {
await openApp(page, TWO)
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-p1").click()
const input = dialog.getByPlaceholder("Or type another model ID")
await input.fill("half-typed")
await back(dialog)
await dialog.getByTestId("provider-row-p2").click()
await expect(input).toHaveValue("")
})
test("Bedrock shows the sign-in that is saved, also after another tab", async ({
page,
}) => {
await openApp(page, {
version: 1,
providers: [
{
id: "b1",
provider: "bedrock",
apiKey: "",
awsAccessKeyId: "AKIAOLD",
awsSecretAccessKey: "old-secret",
awsRegion: "us-east-1",
models: [{ id: "m1", modelId: "amazon.nova-lite-v1:0" }],
},
],
})
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-b1").click()
await expect(dialog.locator("#aws-access-key-id")).toHaveValue("AKIAOLD")
// Another tab signs the provider in with an API key instead
await changeInAnotherTab(page, (config) => {
config.providers[0].apiKey = "bedrock-key"
config.providers[0].awsAccessKeyId = ""
config.providers[0].awsSecretAccessKey = ""
})
await expect(dialog.locator("#bedrock-api-key")).toHaveValue("bedrock-key")
await expect(dialog.locator("#aws-access-key-id")).toHaveCount(0)
})
test("a refused key opens the provider the request went to", async ({
page,
}) => {
// The request waits while another tab selects another provider's model
let release!: () => void
const released = new Promise<void>((r) => {
release = r
})
const sentWith: string[] = []
await page.route("**/api/chat", async (route) => {
sentWith.push(route.request().headers()["x-ai-provider"] ?? "")
await released
await route.fulfill({
status: 401,
contentType: "application/json",
body: JSON.stringify({
type: "provider",
code: "invalid_api_key",
message: "Incorrect API key",
}),
})
})
await openApp(page, { ...TWO, selectedModelId: "a1" })
await sendMessage(page, "Draw a box")
// Sent to GLM; only then does the other tab pick Kimi's model
await expect.poll(() => sentWith).toEqual(["glm"])
await changeInAnotherTab(page, (config) => {
config.selectedModelId = "b1"
})
await expect(page.getByTestId("model-selector")).toContainText("kimi-b")
release()
await page
.getByRole("button", { name: "Open model settings" })
.click({ timeout: 15000 })
await expect(
dialogOf(page).getByRole("heading", {
name: "GLM (Zhipu)",
exact: true,
}),
).toBeVisible()
})
test("a local server's base URL is shown from the start", async ({ page }) => {
await openApp(page, {
version: 1,
providers: [{ id: "l1", provider: "ollama", apiKey: "", models: [] }],
})
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-l1").click()
await expect(dialog.locator("#base-url")).toBeVisible()
})
test("refused access keys are named as such", async ({ page }) => {
await page.route("**/api/validate-model", (route) =>
route.fulfill({
json: { valid: false, code: "invalid_api_key", error: "Bad keys" },
}),
)
await openApp(page, {
version: 1,
providers: [
{
id: "b1",
provider: "bedrock",
apiKey: "",
awsAccessKeyId: "AKIA1",
awsSecretAccessKey: "s",
awsRegion: "us-east-1",
models: [{ id: "m1", modelId: "amazon.nova-lite-v1:0" }],
},
],
})
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-b1").click()
await dialog.getByTestId("test-models").click()
await expect(
dialog.getByText(
"AWS rejected these access keys. Check that both were copied in full. Bad keys",
),
).toBeVisible()
for (const id of ["#aws-access-key-id", "#aws-secret-access-key"]) {
await expect(dialog.locator(id)).toHaveAttribute("aria-invalid", "true")
}
})
// Found by the review of the fixes above
test("a slow model list keeps its provider busy while another loads", async ({
page,
}) => {
const releases = await holdEach(page, "**/api/provider-models", () => ({
models: [{ id: "listed-model" }],
}))
await openApp(page, TWO)
await openSettings(page)
const dialog = dialogOf(page)
const fetchButton = dialog.getByRole("button", {
name: /^Get all models from/,
})
await dialog.getByTestId("provider-row-p1").click()
await fetchButton.click()
await expect.poll(() => releases.length).toBe(1)
await back(dialog)
await dialog.getByTestId("provider-row-p2").click()
await fetchButton.click()
await expect.poll(() => releases.length).toBe(2)
releases[1]()
// B's list opens its picker; close it
await expect(
page.getByRole("option", { name: "listed-model" }),
).toBeVisible()
await page.keyboard.press("Escape")
await expect(page.getByRole("listbox")).toHaveCount(0)
await back(dialog)
await dialog.getByTestId("provider-row-p1").click()
// The first request is still out
await expect(fetchButton).toBeDisabled()
releases[0]()
// A's list arrives and opens its picker; afterwards A can fetch again
await expect(
page.getByRole("option", { name: "listed-model" }),
).toBeVisible()
await page.keyboard.press("Escape")
await expect(fetchButton).toBeEnabled()
})
test("refused keys stay marked for each provider", async ({ page }) => {
await page.route("**/api/validate-model", (route) =>
route.fulfill({
json: { valid: false, code: "invalid_api_key", error: "Bad key" },
}),
)
await openApp(page, TWO)
await openSettings(page)
const dialog = dialogOf(page)
const key = dialog.locator("#api-key")
await dialog.getByTestId("provider-row-p1").click()
await dialog.getByTestId("test-models").click()
await expect(key).toHaveAttribute("aria-invalid", "true")
await back(dialog)
await dialog.getByTestId("provider-row-p2").click()
await dialog.getByTestId("test-models").click()
await expect(key).toHaveAttribute("aria-invalid", "true")
await back(dialog)
await dialog.getByTestId("provider-row-p1").click()
await expect(key).toHaveAttribute("aria-invalid", "true")
})
test("a model list that loads leaves a refused test's mark", async ({
page,
}) => {
// Some providers list their models without the key: a list proves nothing
await page.route("**/api/validate-model", (route) =>
route.fulfill({
json: { valid: false, code: "invalid_api_key", error: "Bad key" },
}),
)
await page.route("**/api/provider-models", (route) =>
route.fulfill({ json: { models: [{ id: "listed-model" }] } }),
)
await openApp(page, TWO)
await openSettings(page)
const dialog = dialogOf(page)
const key = dialog.locator("#api-key")
await dialog.getByTestId("provider-row-p1").click()
await dialog.getByTestId("test-models").click()
await expect(key).toHaveAttribute("aria-invalid", "true")
await dialog.getByRole("button", { name: /^Get all models from/ }).click()
await expect(
page.getByRole("option", { name: "listed-model" }),
).toBeVisible()
await page.keyboard.press("Escape")
await expect(page.getByRole("listbox")).toHaveCount(0)
await expect(key).toHaveAttribute("aria-invalid", "true")
})
test("an empty model list says so and opens nothing", async ({ page }) => {
await page.route("**/api/provider-models", (route) =>
route.fulfill({ json: { models: [] } }),
)
// OpenAI has suggested models: they come back once the key changes
await openApp(page, {
version: 1,
providers: [{ id: "o1", provider: "openai", apiKey: "k", models: [] }],
})
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-o1").click()
await dialog.getByRole("button", { name: /^Get all models from/ }).click()
await expect(
dialog.getByText("The provider returned no models."),
).toBeVisible()
await dialog.locator("#api-key").fill("k-new")
await expect(
dialog.getByRole("button", { name: "Browse models" }),
).toBeVisible()
await page.waitForTimeout(300)
// The picker did not open by itself
await expect(page.getByRole("listbox")).toHaveCount(0)
await expect(dialog.locator("#api-key")).toBeFocused()
})
test("emptying Bedrock's access keys to type new ones keeps their fields", async ({
page,
}) => {
await openApp(page, {
version: 1,
providers: [
{
id: "b1",
provider: "bedrock",
apiKey: "",
awsAccessKeyId: "AKIAOLD",
awsSecretAccessKey: "old-secret",
awsRegion: "us-east-1",
models: [{ id: "m1", modelId: "amazon.nova-lite-v1:0" }],
},
],
})
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-b1").click()
await dialog.locator("#aws-secret-access-key").fill("")
await dialog.locator("#aws-access-key-id").fill("")
await expect(dialog.locator("#bedrock-api-key")).toHaveCount(0)
await dialog.locator("#aws-access-key-id").fill("AKIANEW")
await expect
.poll(async () => (await saved(page)).providers[0])
.toMatchObject({ apiKey: "", awsAccessKeyId: "AKIANEW" })
})
test("a provider added in another tab while sending is the one its error opens", async ({
page,
}) => {
const sentWith: string[] = []
await page.route("**/api/chat", async (route) => {
sentWith.push(route.request().headers()["x-ai-provider"] ?? "")
await route.fulfill({
status: 401,
contentType: "application/json",
body: JSON.stringify({
type: "provider",
code: "invalid_api_key",
message: "Incorrect API key",
}),
})
})
await openApp(page, { ...TWO, selectedModelId: "a1" })
await page.getByTestId("chat-input").fill("Draw a box")
// Send, and right away another tab adds DeepSeek and picks its model,
// while this tab still prepares the request (it exports the diagram)
await page.evaluate((key) => {
const send = document.querySelector<HTMLButtonElement>(
'button[aria-label="Send"]',
)
send?.click()
const config = JSON.parse(localStorage.getItem(key) ?? "{}")
config.providers.push({
id: "p3",
provider: "deepseek",
apiKey: "k3",
models: [{ id: "c1", modelId: "deepseek-chat" }],
})
config.selectedModelId = "c1"
const value = JSON.stringify(config)
localStorage.setItem(key, value)
window.dispatchEvent(
new StorageEvent("storage", { key, newValue: value }),
)
}, KEY)
await expect.poll(() => sentWith).toEqual(["deepseek"])
await page
.getByRole("button", { name: "Open model settings" })
.click({ timeout: 15000 })
await expect(
dialogOf(page).getByRole("heading", { name: "DeepSeek", exact: true }),
).toBeVisible()
})
// After #957 merged
test("temporary Bedrock access keys can carry their session token", async ({
page,
}) => {
const bodies: any[] = []
await page.route("**/api/validate-model", async (route) => {
bodies.push(route.request().postDataJSON())
await route.fulfill({ json: { valid: true, responseTime: 900 } })
})
await openApp(page, {
version: 1,
providers: [
{
id: "b1",
provider: "bedrock",
apiKey: "",
awsAccessKeyId: "ASIATEMP",
awsSecretAccessKey: "secret",
awsRegion: "us-east-1",
models: [{ id: "m1", modelId: "amazon.nova-lite-v1:0" }],
},
],
})
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-b1").click()
await dialog.getByLabel("AWS session token (optional)").fill("session-1")
await dialog.getByTestId("test-models").click()
await expect.poll(() => bodies.length).toBe(1)
expect(bodies[0]).toMatchObject({
awsAccessKeyId: "ASIATEMP",
awsSessionToken: "session-1",
})
expect((await saved(page)).providers[0].awsSessionToken).toBe("session-1")
})
test("Bedrock with both kinds of credentials offers to remove the access keys", async ({
page,
}) => {
await openApp(page, {
version: 1,
providers: [
{
id: "b1",
provider: "bedrock",
apiKey: "bedrock-key",
awsAccessKeyId: "AKIAOLD",
awsSecretAccessKey: "old-secret",
awsSessionToken: "old-token",
awsRegion: "us-east-1",
models: [{ id: "m1", modelId: "amazon.nova-lite-v1:0" }],
},
],
})
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-b1").click()
const notice = dialog.getByText(/also keeps an access key pair/)
await expect(notice).toBeVisible()
await dialog.getByRole("button", { name: "Remove the access keys" }).click()
await expect(notice).toHaveCount(0)
await expect
.poll(async () => (await saved(page)).providers[0])
.toMatchObject({
apiKey: "bedrock-key",
awsAccessKeyId: "",
awsSecretAccessKey: "",
awsSessionToken: "",
})
await expect(dialog.locator("#bedrock-api-key")).toHaveValue("bedrock-key")
})
test("the test result goes to a live region that is there from the start", async ({
page,
}) => {
await page.route("**/api/validate-model", (route) =>
route.fulfill({ json: { valid: true, responseTime: 900 } }),
)
await openApp(page, TWO)
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-p1").click()
const result = dialog.getByTestId("test-result")
await expect(result).toHaveAttribute("role", "status")
await expect(result).toHaveText("")
await dialog.getByTestId("test-models").click()
await expect(result).toHaveText("glm-a works")
})
test("removing access keys kept behind an API key keeps the test results", async ({
page,
}) => {
await openApp(page, {
version: 1,
providers: [
{
id: "b1",
provider: "bedrock",
apiKey: "bedrock-key",
awsAccessKeyId: "AKIAOLD",
awsSecretAccessKey: "old-secret",
awsRegion: "us-east-1",
models: [
{
id: "m1",
modelId: "amazon.nova-lite-v1:0",
validated: true,
},
],
},
],
})
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-b1").click()
await dialog.getByRole("button", { name: "Remove the access keys" }).focus()
await page.keyboard.press("Enter")
await expect
.poll(async () => (await saved(page)).providers[0].awsAccessKeyId)
.toBe("")
// The API key was the one in use: its result still holds
expect((await saved(page)).providers[0].models[0].validated).toBe(true)
await expect(dialog.locator("#bedrock-api-key")).toBeFocused()
})
test("a passing test hands the focus to its next step, a failing one keeps it", async ({
page,
}) => {
let pass = false
await page.route("**/api/validate-model", (route) =>
route.fulfill({
json: pass
? { valid: true, responseTime: 900 }
: { valid: false, error: "Model not found" },
}),
)
await openApp(page, TWO)
await openSettings(page)
const dialog = dialogOf(page)
await dialog.getByTestId("provider-row-p1").click()
const testButton = dialog.getByTestId("test-models")
await testButton.focus()
await page.keyboard.press("Enter")
await expect(dialog.getByTestId("test-result")).toHaveText(
"1 model failed the test",
)
await expect(testButton).toBeFocused()
pass = true
await page.keyboard.press("Enter")
const use = dialog.getByRole("button", { name: "Use glm-a in the chat" })
await expect(use).toBeFocused()
const resultId = await dialog.getByTestId("test-result").getAttribute("id")
await expect(use).toHaveAttribute("aria-describedby", resultId ?? "")
})
tests/e2e/settings.spec.ts
+3 -3
View File
@@ -18,14 +18,14 @@ test.describe("Settings", () => {
})
test("language selection is available", async ({ page }) => {
await openSettingsTab(page, "appearance")
await openSettingsTab(page, "general")
const dialog = page.locator('[role="dialog"]')
await expect(dialog.locator("#language-select")).toHaveText(/English/)
})
test("max output tokens is editable and persists", async ({ page }) => {
await openSettingsTab(page, "advanced")
await openSettingsTab(page, "drawing")
const input = page.locator("#max-output-tokens")
await expect(input).toBeVisible()
@@ -45,7 +45,7 @@ test.describe("Settings", () => {
})
test("theme can be light, dark or follow the system", async ({ page }) => {
await openSettingsTab(page, "appearance")
await openSettingsTab(page, "general")
const dialog = page.locator('[role="dialog"]')
for (const name of ["Light", "Dark", "System"]) {
tests/e2e/workspace.spec.ts
+21 -15
View File
@@ -471,7 +471,7 @@ test.describe("Workspace", () => {
return { dark: w.Editor.isDarkMode(), mark: w.__naiMark }
}, mark)
expect((await drawio(true)).dark).toBe(true)
await openSettingsTab(page, "appearance")
await openSettingsTab(page, "general")
const dialog = page.getByRole("dialog")
await dialog.getByRole("radio", { name: "Light" }).click()
await expect.poll(() => drawio()).toEqual({ dark: false, mark: 1 })
@@ -494,7 +494,7 @@ test.describe("Workspace", () => {
.locator(".geDiagramContainer")
.evaluate((el) => getComputedStyle(el).backgroundColor)
const light = await table()
await openSettingsTab(page, "appearance")
await openSettingsTab(page, "general")
const dialog = page.getByRole("dialog")
await dialog.getByRole("radio", { name: "Dark" }).click()
// draw.io's own dark table
@@ -511,7 +511,7 @@ test.describe("Workspace", () => {
page.locator('[data-testid="canvas-loading"]'),
).toHaveCount(0, { timeout: 30000 })
await expect.poll(table).toBe("rgb(27, 29, 30)")
await openSettingsTab(page, "appearance")
await openSettingsTab(page, "general")
await page
.getByRole("dialog")
.getByRole("radio", { name: "Light" })
@@ -916,23 +916,29 @@ test.describe("Workspace", () => {
expect(send.y + send.height).toBeLessThanOrEqual(320)
})
test("settings put their sections on top in a narrow window", async ({
test("settings put their sections on top in the phone layout", async ({
page,
}) => {
await page.setViewportSize({ width: 800, height: 800 })
await openApp(page)
// Below 768 px the app shows the phone layout, with the canvas hidden
await page.setViewportSize({ width: 720, height: 800 })
await page.goto("/", { waitUntil: "networkidle" })
await page.locator('[data-testid="settings-button"]').first().click()
const models = page.locator('[data-testid="settings-tab-models"]')
const appearance = page.locator(
'[data-testid="settings-tab-appearance"]',
)
const general = page.locator('[data-testid="settings-tab-general"]')
await expect(models).toBeVisible()
// Once fonts and the dialog's opening are done
await expect
.poll(async () =>
Math.abs((await boxOf(models)).y - (await boxOf(appearance)).y),
Math.abs((await boxOf(models)).y - (await boxOf(general)).y),
)
.toBeLessThan(2)
// From 768 px on they are beside the page, one under the other
await page.setViewportSize({ width: 900, height: 800 })
await expect
.poll(
async () => (await boxOf(general)).y - (await boxOf(models)).y,
)
.toBeGreaterThan(20)
})
test("New chat brings the start screen back", async ({ page }) => {
@@ -1206,7 +1212,7 @@ test.describe("Workspace", () => {
await sendMessage(page, "Create a flowchart")
await waitForComplete(page)
// Right after the answer: its auto-save may not have run yet
await openSettingsTab(page, "appearance")
await openSettingsTab(page, "general")
await page.locator("#language-select").click()
await page.getByRole("option", { name: "日本語", exact: true }).click()
await expect(page).toHaveURL(/\/ja\?session=/, { timeout: 15000 })
@@ -1231,7 +1237,7 @@ test.describe("Workspace", () => {
})
await openApp(page)
await sendMessage(page, "Create a flowchart")
await openSettingsTab(page, "appearance")
await openSettingsTab(page, "general")
await expect(page.locator("#language-select")).toBeDisabled()
})
@@ -1634,7 +1640,7 @@ test.describe("Workspace", () => {
const tabs = drawioTabs(page)
await expect(tabs).toContainText("Sheet")
// Right away, before the chat is saved on its own
await openSettingsTab(page, "appearance")
await openSettingsTab(page, "general")
await page.locator("#language-select").click()
await page.getByRole("option", { name: "日本語", exact: true }).click()
await expect(page).toHaveURL(/\/ja\?session=/, { timeout: 15000 })
@@ -2669,7 +2675,7 @@ test.describe("Edge cases", () => {
await openApp(page)
await sendMessage(page, "Create a flowchart")
await waitForComplete(page)
await openSettingsTab(page, "appearance")
await openSettingsTab(page, "general")
await page.locator("#language-select").click()
await page.getByRole("option", { name: "日本語", exact: true }).click()
await expect(page).toHaveURL(/\/ja/, { timeout: 15000 })
@@ -4708,7 +4714,7 @@ test.describe("Edge cases", () => {
}
})
await openApp(page)
await openSettingsTab(page, "appearance")
await openSettingsTab(page, "general")
await page.locator("#language-select").click()
await page.getByRole("option", { name: "日本語", exact: true }).click()
await expect(page).toHaveURL(/\/ja/, { timeout: 15000 })
tests/unit/provider-setup.test.ts
+154
View File
@@ -0,0 +1,154 @@
import { describe, expect, it } from "vitest"
import enDict from "@/lib/i18n/dictionaries/en.json"
import jaDict from "@/lib/i18n/dictionaries/ja.json"
import zhDict from "@/lib/i18n/dictionaries/zh.json"
import zhHantDict from "@/lib/i18n/dictionaries/zh-Hant.json"
import {
filterProviderGroups,
hasCredentials,
POPULAR_PROVIDERS,
PROVIDER_GROUPS,
PROVIDER_NOTE_KEYS,
providerStatus,
} from "@/lib/provider-setup"
import { PROVIDER_INFO, type ProviderConfig } from "@/lib/types/model-config"
import { useUiStore } from "@/stores/ui-store"
const provider = (p: Partial<ProviderConfig>): ProviderConfig => ({
id: "p",
provider: "openai",
apiKey: "",
models: [],
...p,
})
describe("provider groups", () => {
it("hold every provider exactly once", () => {
const grouped = PROVIDER_GROUPS.flatMap((g) => g.providers)
expect([...grouped].sort()).toEqual(Object.keys(PROVIDER_INFO).sort())
expect(new Set(grouped).size).toBe(grouped.length)
})
it("offer known providers as popular ones", () => {
for (const p of POPULAR_PROVIDERS) expect(PROVIDER_INFO[p]).toBeTruthy()
})
it("have their notes in every language", () => {
for (const dict of [enDict, zhDict, jaDict, zhHantDict]) {
const t = dict.modelConfig as Record<string, string>
for (const key of Object.values(PROVIDER_NOTE_KEYS)) {
expect(t[key as string]).toBeTruthy()
}
}
})
it("are searched by label or name, ignoring case and spaces", () => {
expect(filterProviderGroups(" ")).toBe(PROVIDER_GROUPS)
const kimi = filterProviderGroups("MOONSHOT")
expect(kimi.map((g) => g.providers)).toEqual([["kimi"]])
// By the provider's own name too
expect(
filterProviderGroups("vertexai").flatMap((g) => g.providers),
).toEqual(["vertexai"])
expect(filterProviderGroups("no such provider")).toEqual([])
})
})
describe("hasCredentials", () => {
it("asks each provider for what it signs in with", () => {
const cases: [Partial<ProviderConfig>, boolean][] = [
[{ provider: "openai" }, false],
[{ provider: "openai", apiKey: "k" }, true],
[{ provider: "ollama" }, true],
[{ provider: "edgeone" }, true],
[{ provider: "vertexai", apiKey: "k" }, false],
[{ provider: "vertexai", vertexApiKey: "k" }, true],
[{ provider: "bedrock", apiKey: "k" }, false],
[
{ provider: "bedrock", apiKey: "k", awsRegion: "us-east-1" },
true,
],
[
{
provider: "bedrock",
awsAccessKeyId: "a",
awsRegion: "us-east-1",
},
false,
],
[
{
provider: "bedrock",
awsAccessKeyId: "a",
awsSecretAccessKey: "s",
awsRegion: "us-east-1",
},
true,
],
]
for (const [p, expected] of cases) {
expect([p, hasCredentials(provider(p))]).toEqual([p, expected])
}
})
})
describe("providerStatus", () => {
const models = (...validated: (boolean | undefined)[]) =>
validated.map((v, i) => ({
id: `m${i}`,
modelId: `m${i}`,
validated: v,
}))
it("puts a failed model first", () => {
expect(
providerStatus(provider({ models: models(false, false, true) })),
).toEqual({ kind: "error", count: 2 })
// Even without credentials
expect(providerStatus(provider({ models: models(false) }))).toEqual({
kind: "error",
count: 1,
})
})
it("then a missing key or no model", () => {
expect(providerStatus(provider({ models: models(true) }))).toEqual({
kind: "incomplete",
})
expect(providerStatus(provider({ apiKey: "k" }))).toEqual({
kind: "incomplete",
})
})
it("then untested models, else all working", () => {
expect(
providerStatus(
provider({ apiKey: "k", models: models(true, undefined) }),
),
).toEqual({ kind: "untested", count: 1 })
expect(
providerStatus(
provider({ apiKey: "k", models: models(true, true) }),
),
).toEqual({ kind: "ok", count: 2 })
})
})
describe("openSettings", () => {
it("opens the models tab's list unless a page is given", () => {
const store = useUiStore.getState()
store.openSettings("models", { providerId: "p1" })
expect(useUiStore.getState().modelsPage).toEqual({ providerId: "p1" })
store.setSettingsOpen(false)
// Opened again some other way: back on the list, on the last tab
store.setSettingsTab("drawing")
store.openSettings()
expect(useUiStore.getState()).toMatchObject({
settingsOpen: true,
settingsTab: "drawing",
modelsPage: "list",
})
store.openSettings("models", "picker")
expect(useUiStore.getState().modelsPage).toBe("picker")
})
})
tests/unit/use-model-config.test.tsx
+42
View File
@@ -260,3 +260,45 @@ describe("useModelConfig across tabs", () => {
expect(result.current.config.providers).toHaveLength(2)
})
})
describe("useModelConfig migration", () => {
const OLD = {
provider: "next-ai-draw-io-ai-provider",
baseUrl: "next-ai-draw-io-ai-base-url",
apiKey: "next-ai-draw-io-ai-api-key",
model: "next-ai-draw-io-ai-model",
}
it("moves an old keyless Ollama setup into the new format", async () => {
localStorage.setItem(OLD.provider, "ollama")
localStorage.setItem(OLD.baseUrl, "http://localhost:11434/api")
localStorage.setItem(OLD.apiKey, "")
localStorage.setItem(OLD.model, "llama3.2")
const { result } = await renderLoaded()
const [provider] = result.current.config.providers
expect(provider).toMatchObject({
provider: "ollama",
apiKey: "",
baseUrl: "http://localhost:11434/api",
models: [{ modelId: "llama3.2" }],
})
expect(result.current.selectedModelId).toBe(provider.models[0].id)
expect(localStorage.getItem(OLD.model)).toBeNull()
})
it("keeps an old Ollama without a URL on the server's Ollama", async () => {
localStorage.setItem(OLD.provider, "ollama")
localStorage.setItem(OLD.apiKey, "")
localStorage.setItem(OLD.model, "llama3.2")
const { result } = await renderLoaded()
// No cloud address: the request takes OLLAMA_BASE_URL or this machine
expect(result.current.config.providers[0].baseUrl).toBe("")
})
it("still needs a key for other providers", async () => {
localStorage.setItem(OLD.provider, "openai")
localStorage.setItem(OLD.model, "gpt-4o")
const { result } = await renderLoaded()
expect(result.current.config.providers).toEqual([])
})
})