feat(errors): classify provider errors and show a hint the user can act on

- lib/llm-errors.ts sorts an error into about a dozen kinds (key
  rejected, no access, unknown model, no credit, rate limited, context
  too long, no image input, no tool calls, output cut off, provider down,
  cannot connect, timeout): first texts that name the cause precisely,
  then the HTTP status code, then general texts. It unwraps RetryError and
  hides keys and Bearer tokens in the provider's message
- The chat route uses it for errors before the stream and, through
  toUIMessageStreamResponse's onError, for errors in the stream. Errors
  of the model's own tool call stay as they are: the same text goes back
  to the model so it can fix the call
- The chat shows the hint in the user's language, then the provider's
  message; a rejected key, missing access or unknown model adds an "Open
  model settings" button. The Test button shows the same hints
- Fixes: our message "API key is required when using a custom base URL"
  was replaced by "Authentication failed" because it contains "key"; a
  provider's "Rate limit exceeded" opened this site's quota toast; an
  error body like {"error": ...} was shown as raw JSON; the Test button
  matched "401" in the message, where providers rarely put it
- Remove the string matching fallbacks in the chat panel

tests/e2e/chat-errors.spec.ts

@@ -0,0 +1,77 @@
+import { expect, type Page, test } from "@playwright/test"
+import { getIframe, sendMessage } from "./lib/fixtures"
+
+/** Answer the chat request with the given response and send a message */
+async function chatWith(
+    page: Page,
+    response: { status: number; contentType: string; body: string },
+) {
+    await page.route("**/api/chat", (route) => route.fulfill(response))
+    await page.goto("/", { waitUntil: "networkidle" })
+    await getIframe(page).waitFor({ state: "visible", timeout: 30000 })
+    await sendMessage(page, "Draw a box")
+}
+
+test("a rejected key shows a hint, the provider's words and a settings button", async ({
+    page,
+}) => {
+    await chatWith(page, {
+        status: 401,
+        contentType: "application/json",
+        body: JSON.stringify({
+            type: "provider",
+            code: "invalid_api_key",
+            message: "Authentication Fails, Your api key is invalid",
+        }),
+    })
+    await expect(
+        page.getByText("The provider rejected the API key"),
+    ).toBeVisible({
+        timeout: 15000,
+    })
+    // First match: the Next.js dev overlay at the end also lists the error
+    await expect(page.getByText("Authentication Fails").first()).toBeVisible()
+    await page.getByRole("button", { name: "Open model settings" }).click()
+    await expect(
+        page.getByRole("dialog", { name: "AI Model Configuration" }),
+    ).toBeVisible()
+})
+
+test("an error before the stream shows its text, not raw JSON", async ({
+    page,
+}) => {
+    await chatWith(page, {
+        status: 400,
+        contentType: "application/json",
+        body: JSON.stringify({
+            error: 'Model "x" is not available on the server',
+        }),
+    })
+    // The whole message, not the JSON around it (the Next.js dev overlay
+    // also lists the raw console error; it is not part of the app)
+    await expect(
+        page.getByText('Model "x" is not available on the server', {
+            exact: true,
+        }),
+    ).toBeVisible({ timeout: 15000 })
+})
+
+test("a provider rate limit is not shown as this site's quota", async ({
+    page,
+}) => {
+    const errorText = JSON.stringify({
+        type: "provider",
+        code: "rate_limited",
+        message: "Rate limit exceeded for your organization",
+    })
+    await chatWith(page, {
+        status: 200,
+        contentType: "text/event-stream",
+        body: `data: {"type":"start"}\n\ndata: ${JSON.stringify({ type: "error", errorText })}\n\ndata: [DONE]\n\n`,
+    })
+    await expect(
+        page.getByText("The provider is limiting requests"),
+    ).toBeVisible({ timeout: 15000 })
+    // The site's own tokens-per-minute toast
+    await expect(page.getByText("Rate limit reached")).toHaveCount(0)
+})

tests/unit/llm-errors.test.ts

@@ -0,0 +1,103 @@
+// @vitest-environment node
+import { APICallError, InvalidToolInputError, RetryError } from "ai"
+import { describe, expect, it } from "vitest"
+import { classifyLLMError, isToolCallError } from "@/lib/llm-errors"
+
+const apiError = (statusCode: number, message: string, responseBody = "") =>
+    new APICallError({
+        message,
+        url: "https://api.example.com/v1/chat/completions",
+        requestBodyValues: {},
+        statusCode,
+        responseBody,
+    })
+
+describe("classifyLLMError", () => {
+    it("reads the status code, not the message", () => {
+        // Providers rarely put the number in their message
+        expect(
+            classifyLLMError(apiError(401, "Authentication Fails")).code,
+        ).toBe("invalid_api_key")
+        expect(classifyLLMError(apiError(404, "Unknown")).code).toBe(
+            "model_not_found",
+        )
+        expect(classifyLLMError(apiError(503, "busy")).code).toBe(
+            "provider_unavailable",
+        )
+    })
+
+    it("lets a specific text win over the status code", () => {
+        expect(
+            classifyLLMError(
+                apiError(429, "You exceeded your current quota, check billing"),
+            ).code,
+        ).toBe("insufficient_quota")
+        expect(
+            classifyLLMError(
+                apiError(400, "This model's maximum context length is 128000"),
+            ).code,
+        ).toBe("context_too_long")
+        expect(
+            classifyLLMError(
+                apiError(400, "bad", '{"message":"toolUse.input is invalid"}'),
+            ).code,
+        ).toBe("output_truncated")
+    })
+
+    it("does not call a 403 an invalid key", () => {
+        expect(classifyLLMError(apiError(403, "Forbidden")).code).toBe(
+            "forbidden",
+        )
+    })
+
+    it("uses the last attempt after retries", () => {
+        const retry = new RetryError({
+            message: "Failed after 3 attempts",
+            reason: "maxRetriesExceeded",
+            errors: [apiError(500, "x"), apiError(429, "slow down")],
+        })
+        expect(classifyLLMError(retry).code).toBe("rate_limited")
+    })
+
+    it("keeps the message but hides secrets in it", () => {
+        const { code, message } = classifyLLMError(
+            apiError(
+                401,
+                "Incorrect API key provided: sk-proj-abcdefghijklmnop. Header Bearer abc.def",
+            ),
+        )
+        expect(code).toBe("invalid_api_key")
+        expect(message).toContain("Incorrect API key provided")
+        expect(message).not.toContain("abcdefghijklmnop")
+        expect(message).not.toContain("abc.def")
+    })
+
+    it("leaves our own messages readable", () => {
+        // This one used to be replaced by "Authentication failed" for
+        // containing the word key
+        const { message } = classifyLLMError(
+            new Error(
+                "API key is required when using a custom base URL. Please provide your own API key in Settings.",
+            ),
+        )
+        expect(message).toContain("API key is required when using a custom")
+    })
+
+    it("names a timeout", () => {
+        const timeout = new Error("The operation was aborted due to timeout")
+        timeout.name = "TimeoutError"
+        expect(classifyLLMError(timeout).code).toBe("timeout")
+    })
+})
+
+describe("isToolCallError", () => {
+    it("spots errors the model must see unchanged", () => {
+        const invalid = new InvalidToolInputError({
+            toolName: "display_diagram",
+            toolInput: "{",
+            cause: new Error("bad JSON"),
+        })
+        expect(isToolCallError(invalid)).toBe(true)
+        expect(isToolCallError(apiError(500, "x"))).toBe(false)
+    })
+})