feat(errors): classify provider errors and show a hint the user can act on

- lib/llm-errors.ts sorts an error into about a dozen kinds (key rejected, no access, unknown model, no credit, rate limited, context too long, no image input, no tool calls, output cut off, provider down, cannot connect, timeout): first texts that name the cause precisely, then the HTTP status code, then general texts. It unwraps RetryError and hides keys and Bearer tokens in the provider's message - The chat route uses it for errors before the stream and, through toUIMessageStreamResponse's onError, for errors in the stream. Errors of the model's own tool call stay as they are: the same text goes back to the model so it can fix the call - The chat shows the hint in the user's language, then the provider's message; a rejected key, missing access or unknown model adds an "Open model settings" button. The Test button shows the same hints - Fixes: our message "API key is required when using a custom base URL" was replaced by "Authentication failed" because it contains "key"; a provider's "Rate limit exceeded" opened this site's quota toast; an error body like {"error": ...} was shown as raw JSON; the Test button matched "401" in the message, where providers rarely put it - Remove the string matching fallbacks in the chat panel
2026-10-07 02:07:47 +08:00 · 2026-10-04 13:49:34 +09:00
parent 99890e9e37
commit ab1a58f999
12 changed files with 487 additions and 142 deletions
@@ -4,7 +4,6 @@ import {
    createUIMessageStream,
    createUIMessageStreamResponse,
    InvalidToolInputError,
-    LoadAPIKeyError,
    stepCountIs,
    streamText,
 } from "ai"
@@ -39,6 +38,7 @@ import {
    setTraceOutput,
    wrapWithObserve,
 } from "@/lib/langfuse"
+import { classifyLLMError, isToolCallError } from "@/lib/llm-errors"
 import {
    resolveMaxOutputTokens,
    withOutputTokenLimitFallback,
@@ -720,6 +720,12 @@ Call this tool to get shape names and usage syntax for a specific library.`,

    const response = result.toUIMessageStreamResponse({
        sendReasoning: true,
+        // The same text goes back to the model when its tool call was
+        // invalid, so it can fix it: keep that one as it is
+        onError: (error) =>
+            isToolCallError(error)
+                ? (error as Error).message
+                : JSON.stringify(classifyLLMError(error)),
        messageMetadata: ({ part }) => {
            if (part.type === "finish") {
                const usage = (part as any).totalUsage
@@ -736,61 +742,24 @@ Call this tool to get shape names and usage syntax for a specific library.`,
    return response
 }

-// Helper to categorize errors and return appropriate response
+// Errors before the stream starts, as JSON the chat panel reads
 function handleError(error: unknown): Response {
    console.error("Error in chat route:", error)

    const isDev = process.env.NODE_ENV === "development"
-
-    // Check for specific AI SDK error types
-    if (APICallError.isInstance(error)) {
-        return Response.json(
-            {
-                error: error.message,
-                ...(isDev && {
-                    details: error.responseBody,
-                    stack: error.stack,
-                }),
-            },
-            { status: error.statusCode || 500 },
-        )
-    }
-
-    if (LoadAPIKeyError.isInstance(error)) {
-        return Response.json(
-            {
-                error: "Authentication failed. Please check your API key.",
-                ...(isDev && {
-                    stack: error.stack,
-                }),
-            },
-            { status: 401 },
-        )
-    }
-
-    // Fallback for other errors with safety filter
-    const message =
-        error instanceof Error ? error.message : "An unexpected error occurred"
-    const status = (error as any)?.statusCode || (error as any)?.status || 500
-
-    // Prevent leaking API keys, tokens, or other sensitive data
-    const lowerMessage = message.toLowerCase()
-    const safeMessage =
-        lowerMessage.includes("key") ||
-        lowerMessage.includes("token") ||
-        lowerMessage.includes("sig") ||
-        lowerMessage.includes("signature") ||
-        lowerMessage.includes("secret") ||
-        lowerMessage.includes("password") ||
-        lowerMessage.includes("credential")
-            ? "Authentication failed. Please check your credentials."
-            : message
+    const classified = classifyLLMError(error)
+    const status =
+        (error as { statusCode?: number })?.statusCode ||
+        (error as { status?: number })?.status ||
+        (classified.code === "invalid_api_key" ? 401 : 500)

    return Response.json(
        {
-            error: safeMessage,
+            ...classified,
            ...(isDev && {
-                details: message,
+                details: APICallError.isInstance(error)
+                    ? error.responseBody
+                    : undefined,
                stack: error instanceof Error ? error.stack : undefined,
            }),
        },
@@ -3,6 +3,7 @@ import { NextResponse } from "next/server"
 import { z } from "zod"
 import { checkAccessCode } from "@/lib/access-code"
 import { getAIModel } from "@/lib/ai-providers"
+import { classifyLLMError } from "@/lib/llm-errors"
 import { allowPrivateUrls, isPrivateUrl } from "@/lib/ssrf-protection"

 export const runtime = "nodejs"
@@ -146,35 +147,9 @@ export async function POST(req: Request) {
    } catch (error) {
        console.error("[validate-model] Error:", error)

-        let errorMessage = "Validation failed"
-        if (error instanceof Error) {
-            // Extract meaningful error message
-            if (error.name === "TimeoutError") {
-                errorMessage = `No answer within ${TEST_TIMEOUT_MS / 1000} seconds`
-            } else if (
-                error.message.includes("401") ||
-                error.message.includes("Unauthorized")
-            ) {
-                errorMessage = "Invalid API key"
-            } else if (
-                error.message.includes("404") ||
-                error.message.includes("not found")
-            ) {
-                errorMessage = "Model not found"
-            } else if (
-                error.message.includes("429") ||
-                error.message.includes("rate limit")
-            ) {
-                errorMessage = "Rate limited - try again later"
-            } else if (error.message.includes("ECONNREFUSED")) {
-                errorMessage = "Cannot connect to server"
-            } else {
-                errorMessage = error.message.slice(0, 100)
-            }
-        }
-
+        const { code, message } = classifyLLMError(error)
        return NextResponse.json(
-            { valid: false, error: errorMessage },
+            { valid: false, code, error: message },
            { status: 200 }, // Return 200 so client can read error message
        )
    }
@@ -148,6 +148,8 @@ interface SessionMetadata {

 interface ChatMessageDisplayProps {
    messages: UIMessage[]
+    // Shown on an error that a model setting can fix (bad key, unknown model)
+    onOpenModelConfig?: () => void
    setInput: (input: string) => void
    setFiles: (files: File[]) => void
    processedToolCallsRef: MutableRefObject<Set<string>>
@@ -171,6 +173,7 @@ interface ChatMessageDisplayProps {

 export function ChatMessageDisplay({
    messages,
+    onOpenModelConfig,
    setInput,
    setFiles,
    processedToolCallsRef,
@@ -1193,6 +1196,32 @@ export function ChatMessageDisplay({
                                                                    return null
                                                                },
                                                            )}
+                                                            {message.role ===
+                                                                "system" &&
+                                                                (
+                                                                    message.metadata as
+                                                                        | {
+                                                                              openModelConfig?: boolean
+                                                                          }
+                                                                        | undefined
+                                                                )
+                                                                    ?.openModelConfig &&
+                                                                onOpenModelConfig && (
+                                                                    <button
+                                                                        type="button"
+                                                                        onClick={
+                                                                            onOpenModelConfig
+                                                                        }
+                                                                        className="mt-2 text-xs font-medium underline underline-offset-2 hover:opacity-80"
+                                                                    >
+                                                                        {
+                                                                            dict
+                                                                                .errors
+                                                                                .llm
+                                                                                .openModelSettings
+                                                                        }
+                                                                    </button>
+                                                                )}
                                                        </div>
                                                    )
                                                },
@@ -363,84 +363,69 @@ export default function ChatPanel({
            await handleToolCall({ toolCall }, addToolOutput)
        },
        onError: (error) => {
-            // Handle server-side quota limit (429 response)
-            // AI SDK puts the full response body in error.message for non-OK responses
+            // Server errors are JSON: a quota limit ({type: request, token or
+            // tpm}), a provider error ({type: "provider", code, message}) or
+            // {error}. The SDK puts the response body in error.message.
+            let data: any = null
            try {
-                const data = JSON.parse(error.message)
-                if (data.type === "request") {
-                    quotaManager.showQuotaLimitToast(data.used, data.limit)
-                    return
-                }
-                if (data.type === "token") {
-                    quotaManager.showTokenLimitToast(data.used, data.limit)
-                    return
-                }
-                if (data.type === "tpm") {
-                    quotaManager.showTPMLimitToast(data.limit)
-                    return
-                }
+                data = JSON.parse(error.message)
            } catch {
-                // Not JSON, fall through to string matching for backwards compatibility
+                // Plain text, e.g. a network failure in the browser
            }
-
-            // Fallback to string matching
-            if (error.message.includes("Daily request limit")) {
-                quotaManager.showQuotaLimitToast()
+            if (data?.type === "request") {
+                quotaManager.showQuotaLimitToast(data.used, data.limit)
                return
            }
-            if (error.message.includes("Daily token limit")) {
-                quotaManager.showTokenLimitToast()
+            if (data?.type === "token") {
+                quotaManager.showTokenLimitToast(data.used, data.limit)
                return
            }
-            if (
-                error.message.includes("Rate limit exceeded") ||
-                error.message.includes("tokens per minute")
-            ) {
-                quotaManager.showTPMLimitToast()
+            if (data?.type === "tpm") {
+                quotaManager.showTPMLimitToast(data.limit)
                return
            }

+            const isAccessCodeError = String(
+                data?.error ?? error.message,
+            ).includes("Invalid or missing access code")
            // Silence access code error in console since it's handled by UI
-            if (!error.message.includes("Invalid or missing access code")) {
-                console.error("Chat error:", error)
-            }
+            if (!isAccessCodeError) console.error("Chat error:", error)

-            // Translate technical errors into user-friendly messages
-            // The server now handles detailed error messages, so we can display them directly.
-            // But we still handle connection/network errors that happen before reaching the server.
-            let friendlyMessage = error.message
-
-            // Simple check for network errors if message is generic
-            if (friendlyMessage === "Failed to fetch") {
-                friendlyMessage = "Network error. Please check your connection."
-            }
-
-            // Truncated tool input error (model output limit too low)
-            if (friendlyMessage.includes("toolUse.input is invalid")) {
-                friendlyMessage =
-                    "Output was truncated before the diagram could be generated. Try a simpler request or increase the maxOutputLength."
-            }
-
-            // Translate image not supported error
-            if (
-                friendlyMessage.includes("image content block") ||
-                friendlyMessage.toLowerCase().includes("image_url")
-            ) {
-                friendlyMessage = "This model doesn't support image input."
+            // A hint the user can act on, then the provider's own words
+            let text: string = error.message
+            let openModelConfig = false
+            if (data?.type === "provider") {
+                const hints = dict.errors.llm as Record<string, string>
+                text = hints[data.code]
+                    ? `${hints[data.code]}\n\n${data.message}`
+                    : data.message
+                openModelConfig = [
+                    "invalid_api_key",
+                    "forbidden",
+                    "model_not_found",
+                ].includes(data.code)
+            } else if (typeof data?.error === "string") {
+                text = data.error
+            } else if (error.message === "Failed to fetch") {
+                text = dict.errors.networkError
            }

            // Add system message for error so it can be cleared
-            setMessages((currentMessages) => {
-                const errorMessage = {
+            setMessages((currentMessages) => [
+                ...currentMessages,
+                {
                    id: `error-${Date.now()}`,
                    role: "system" as const,
-                    content: friendlyMessage,
-                    parts: [{ type: "text" as const, text: friendlyMessage }],
-                }
-                return [...currentMessages, errorMessage]
-            })
+                    content: text,
+                    parts: [{ type: "text" as const, text }],
+                    // The message shows a button that opens model settings
+                    ...(openModelConfig && {
+                        metadata: { openModelConfig: true },
+                    }),
+                },
+            ])

-            if (error.message.includes("Invalid or missing access code")) {
+            if (isAccessCodeError) {
                // Show settings dialog to help user fix it
                setShowSettingsDialog(true)
            }
@@ -1417,6 +1402,7 @@ export default function ChatPanel({
            {/* Messages */}
            <main className="flex-1 w-full overflow-hidden">
                <ChatMessageDisplay
+                    onOpenModelConfig={() => setShowModelConfigDialog(true)}
                    messages={messages}
                    setInput={setInput}
                    setFiles={handleFileChange}
@@ -414,8 +414,20 @@ export function ModelConfigDialog({
                          }
                        : {
                              validated: false,
+                              // The hint for the error's kind, then the
+                              // provider's own message
                              validationError:
-                                  data.error ||
+                                  [
+                                      (
+                                          dict.errors.llm as Record<
+                                              string,
+                                              string
+                                          >
+                                      )[data.code],
+                                      data.error,
+                                  ]
+                                      .filter(Boolean)
+                                      .join(" ") ||
                                  (response.ok
                                      ? "Validation failed"
                                      : `Request failed (${response.status})`),
@@ -459,7 +471,13 @@ export function ModelConfigDialog({
            setValidationStatus("error")
            setValidationError(`${errorCount} model(s) failed validation`)
        }
-    }, [selectedProvider, selectedProviderId, updateProvider, updateModel])
+    }, [
+        selectedProvider,
+        selectedProviderId,
+        updateProvider,
+        updateModel,
+        dict,
+    ])

    // Get all available provider types
    const availableProviders = Object.keys(PROVIDER_INFO) as ProviderName[]
@@ -186,7 +186,22 @@
        "failedToLoadExample": "Error loading example image",
        "failedToRecordFeedback": "Failed to record your feedback. Please try again.",
        "storageUpdateFailed": "Chat cleared but browser storage could not be updated",
-        "sessionSaveFailed": "Could not save this chat. Browser storage may be full: delete old chats from history and try again."
+        "sessionSaveFailed": "Could not save this chat. Browser storage may be full: delete old chats from history and try again.",
+        "llm": {
+            "invalid_api_key": "The provider rejected the API key. Check it in model settings.",
+            "forbidden": "The provider refused the request. The key may not have access to this model or region.",
+            "model_not_found": "The provider does not know this model. Check the model ID in model settings.",
+            "insufficient_quota": "The provider account has no credit or quota left.",
+            "rate_limited": "The provider is limiting requests. Wait a moment and try again.",
+            "context_too_long": "The conversation is too long for this model. Start a new chat or pick a model with a larger context.",
+            "images_unsupported": "This model doesn't support image input.",
+            "tools_unsupported": "This model doesn't support tool calls, which drawing needs. Pick another model.",
+            "output_truncated": "The output was cut off before the diagram was complete. Try a simpler request or raise the output limit in settings.",
+            "provider_unavailable": "The provider is having problems. Try again later.",
+            "cannot_connect": "Could not reach the provider. Check the base URL and your network.",
+            "timeout": "The provider did not answer in time.",
+            "openModelSettings": "Open model settings"
+        }
    },
    "quota": {
        "dailyLimit": "Daily Quota Reached",
@@ -186,7 +186,22 @@
        "failedToLoadExample": "例の画像の読み込みエラー",
        "failedToRecordFeedback": "フィードバックの記録に失敗しました。もう一度お試しください。",
        "storageUpdateFailed": "チャットはクリアされましたが、ブラウザストレージを更新できませんでした",
-        "sessionSaveFailed": "このチャットを保存できませんでした。ブラウザのストレージがいっぱいの可能性があります。履歴から古いチャットを削除して、もう一度お試しください。"
+        "sessionSaveFailed": "このチャットを保存できませんでした。ブラウザのストレージがいっぱいの可能性があります。履歴から古いチャットを削除して、もう一度お試しください。",
+        "llm": {
+            "invalid_api_key": "プロバイダーが API キーを拒否しました。モデル設定で確認してください。",
+            "forbidden": "プロバイダーがリクエストを拒否しました。このキーにはこのモデルまたはリージョンの利用権限がない可能性があります。",
+            "model_not_found": "プロバイダーがこのモデルを認識できません。モデル設定でモデル ID を確認してください。",
+            "insufficient_quota": "プロバイダーのアカウントの残高または利用枠がなくなりました。",
+            "rate_limited": "プロバイダーがリクエスト数を制限しています。少し待ってから再試行してください。",
+            "context_too_long": "会話がこのモデルで扱える長さを超えています。新しいチャットを始めるか、より長いコンテキストに対応したモデルを選んでください。",
+            "images_unsupported": "このモデルは画像入力に対応していません。",
+            "tools_unsupported": "このモデルはツール呼び出しに対応していません。作図にはツール呼び出しが必要です。別のモデルを選んでください。",
+            "output_truncated": "ダイアグラムが完成する前に出力が途中で切れました。リクエストを簡単にするか、設定で出力上限を上げてください。",
+            "provider_unavailable": "プロバイダーで問題が発生しています。しばらくしてから再試行してください。",
+            "cannot_connect": "プロバイダーに接続できません。Base URL とネットワークを確認してください。",
+            "timeout": "プロバイダーから時間内に応答がありませんでした。",
+            "openModelSettings": "モデル設定を開く"
+        }
    },
    "quota": {
        "dailyLimit": "1日の割当量に達しました",
@@ -186,7 +186,22 @@
        "failedToLoadExample": "載入範例圖片時出錯",
        "failedToRecordFeedback": "記錄您的回饋失敗。請重試。",
        "storageUpdateFailed": "聊天已清除，但無法更新瀏覽器儲存空間",
-        "sessionSaveFailed": "無法儲存這個對話。瀏覽器儲存空間可能已滿，請在歷史紀錄裡刪除舊對話後重試。"
+        "sessionSaveFailed": "無法儲存這個對話。瀏覽器儲存空間可能已滿，請在歷史紀錄裡刪除舊對話後重試。",
+        "llm": {
+            "invalid_api_key": "服務商拒絕了這個 API Key，請在模型設定中檢查。",
+            "forbidden": "服務商拒絕了這次請求。這個 Key 可能沒有使用該模型或該地區的權限。",
+            "model_not_found": "服務商找不到這個模型，請在模型設定中檢查模型 ID。",
+            "insufficient_quota": "服務商帳戶的餘額或額度已經用完。",
+            "rate_limited": "服務商正在限制請求頻率，請稍候再試。",
+            "context_too_long": "對話內容超過了這個模型能處理的長度。請開啟新的對話，或換一個上下文更長的模型。",
+            "images_unsupported": "這個模型不支援圖片輸入。",
+            "tools_unsupported": "這個模型不支援工具呼叫，而繪圖需要工具呼叫。請換一個模型。",
+            "output_truncated": "輸出在圖表完成之前就被截斷了。請簡化請求，或在設定中調高輸出上限。",
+            "provider_unavailable": "服務商發生問題，請稍後再試。",
+            "cannot_connect": "無法連線到服務商，請檢查 Base URL 和網路。",
+            "timeout": "服務商沒有及時回應。",
+            "openModelSettings": "開啟模型設定"
+        }
    },
    "quota": {
        "dailyLimit": "已達每日配額",
@@ -186,7 +186,22 @@
        "failedToLoadExample": "加载示例图片时出错",
        "failedToRecordFeedback": "记录您的反馈失败。请重试。",
        "storageUpdateFailed": "聊天已清除，但无法更新浏览器存储",
-        "sessionSaveFailed": "无法保存这个对话。浏览器存储空间可能已满，请在历史记录里删除旧对话后重试。"
+        "sessionSaveFailed": "无法保存这个对话。浏览器存储空间可能已满，请在历史记录里删除旧对话后重试。",
+        "llm": {
+            "invalid_api_key": "服务商拒绝了这个 API Key，请在模型设置里检查。",
+            "forbidden": "服务商拒绝了这次请求。这个 Key 可能没有使用该模型或该地区的权限。",
+            "model_not_found": "服务商找不到这个模型，请在模型设置里检查模型 ID。",
+            "insufficient_quota": "服务商账户的余额或额度已经用完。",
+            "rate_limited": "服务商正在限制请求频率，请稍等片刻再试。",
+            "context_too_long": "对话内容超过了这个模型能处理的长度。请新开一个对话，或换一个上下文更长的模型。",
+            "images_unsupported": "这个模型不支持图片输入。",
+            "tools_unsupported": "这个模型不支持工具调用，而画图需要工具调用。请换一个模型。",
+            "output_truncated": "输出在图画完之前就被截断了。请简化请求，或在设置里调高输出上限。",
+            "provider_unavailable": "服务商出了问题，请稍后再试。",
+            "cannot_connect": "连接不上服务商，请检查 Base URL 和网络。",
+            "timeout": "服务商没有及时响应。",
+            "openModelSettings": "打开模型设置"
+        }
    },
    "quota": {
        "dailyLimit": "已达每日配额",
@@ -0,0 +1,128 @@
+import {
+    APICallError,
+    InvalidToolInputError,
+    LoadAPIKeyError,
+    NoSuchToolError,
+    RetryError,
+    ToolCallRepairError,
+} from "ai"
+
+/**
+ * What went wrong with a model call, for a hint the user can act on. The
+ * provider's own message always goes along, because a guess can be wrong.
+ */
+export type LLMErrorCode =
+    | "invalid_api_key"
+    | "forbidden"
+    | "model_not_found"
+    | "insufficient_quota"
+    | "rate_limited"
+    | "context_too_long"
+    | "images_unsupported"
+    | "tools_unsupported"
+    | "output_truncated"
+    | "provider_unavailable"
+    | "cannot_connect"
+    | "timeout"
+    | "unknown"
+
+export interface LLMError {
+    type: "provider"
+    code: LLMErrorCode
+    message: string
+}
+
+// Texts that name the cause more precisely than the status code: a quota
+// error can come as 403 or 429, a context or image error as a plain 400
+const SPECIFIC_TEXTS: Array<[RegExp, LLMErrorCode]> = [
+    [
+        /context length|context window|maximum context|prompt is too long|input is too long|too many (?:input )?tokens/i,
+        "context_too_long",
+    ],
+    [
+        /image content block|image_url|does not support image|image input is not supported/i,
+        "images_unsupported",
+    ],
+    [
+        /does not support tools|tool use is not supported|tools? (?:are|is) not supported|function calling is not supported/i,
+        "tools_unsupported",
+    ],
+    // Bedrock, when the output limit cut the tool call's JSON short
+    [/toolUse\.input is invalid/i, "output_truncated"],
+    [
+        /insufficient[_ ]quota|insufficient balance|exceeded your current quota|credit balance is too low|余额不足/i,
+        "insufficient_quota",
+    ],
+]
+
+const STATUS_CODES: Record<number, LLMErrorCode> = {
+    401: "invalid_api_key",
+    402: "insufficient_quota",
+    // Not "invalid key": a valid key can lack access to a model or region
+    403: "forbidden",
+    404: "model_not_found",
+    408: "timeout",
+    413: "context_too_long",
+    429: "rate_limited",
+}
+
+const GENERAL_TEXTS: Array<[RegExp, LLMErrorCode]> = [
+    [
+        /model[_ ]not[_ ]found|model .*does not exist|unknown model|no such model/i,
+        "model_not_found",
+    ],
+    [
+        /invalid[_ ]api[_ ]key|incorrect api key|unauthorized/i,
+        "invalid_api_key",
+    ],
+    [/rate limit|too many requests/i, "rate_limited"],
+    [/ECONNREFUSED|ENOTFOUND|ECONNRESET|fetch failed/i, "cannot_connect"],
+]
+
+/** Secrets a provider may echo back: API keys, Bearer tokens, key=value */
+function redact(text: string): string {
+    return text
+        .replace(/\b(sk|pk|rk|ak)-[A-Za-z0-9_-]{8,}/g, "$1-[redacted]")
+        .replace(/\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi, "Bearer [redacted]")
+        .replace(/\bAKIA[0-9A-Z]{16}\b/g, "[redacted]")
+        .replace(
+            /\b(api[_-]?key|access[_-]?key|secret|token|password|signature)(["']?\s*[:=]\s*["']?)[^\s"',&}]+/gi,
+            "$1$2[redacted]",
+        )
+}
+
+/**
+ * Model and tool errors the SDK sends back to the model as the tool result,
+ * so it can fix its call. Their text has to stay as it is.
+ */
+export function isToolCallError(error: unknown): boolean {
+    return (
+        InvalidToolInputError.isInstance(error) ||
+        NoSuchToolError.isInstance(error) ||
+        ToolCallRepairError.isInstance(error)
+    )
+}
+
+export function classifyLLMError(error: unknown): LLMError {
+    // After the SDK's retries, the last attempt says what happened
+    const e = RetryError.isInstance(error) ? error.lastError : error
+    const raw = e instanceof Error ? e.message : String(e)
+    const message = redact(raw).slice(0, 500)
+    const body = APICallError.isInstance(e) ? (e.responseBody ?? "") : ""
+    const text = `${raw} ${body}`
+    const status = APICallError.isInstance(e)
+        ? e.statusCode
+        : (e as { statusCode?: number })?.statusCode
+
+    const find = (rules: Array<[RegExp, LLMErrorCode]>) =>
+        rules.find(([pattern]) => pattern.test(text))?.[1]
+    const code =
+        (e instanceof Error && e.name === "TimeoutError" && "timeout") ||
+        (LoadAPIKeyError.isInstance(e) && "invalid_api_key") ||
+        find(SPECIFIC_TEXTS) ||
+        (status && STATUS_CODES[status]) ||
+        (status && status >= 500 && "provider_unavailable") ||
+        find(GENERAL_TEXTS) ||
+        "unknown"
+    return { type: "provider", code, message }
+}
@@ -0,0 +1,77 @@
+import { expect, type Page, test } from "@playwright/test"
+import { getIframe, sendMessage } from "./lib/fixtures"
+
+/** Answer the chat request with the given response and send a message */
+async function chatWith(
+    page: Page,
+    response: { status: number; contentType: string; body: string },
+) {
+    await page.route("**/api/chat", (route) => route.fulfill(response))
+    await page.goto("/", { waitUntil: "networkidle" })
+    await getIframe(page).waitFor({ state: "visible", timeout: 30000 })
+    await sendMessage(page, "Draw a box")
+}
+
+test("a rejected key shows a hint, the provider's words and a settings button", async ({
+    page,
+}) => {
+    await chatWith(page, {
+        status: 401,
+        contentType: "application/json",
+        body: JSON.stringify({
+            type: "provider",
+            code: "invalid_api_key",
+            message: "Authentication Fails, Your api key is invalid",
+        }),
+    })
+    await expect(
+        page.getByText("The provider rejected the API key"),
+    ).toBeVisible({
+        timeout: 15000,
+    })
+    // First match: the Next.js dev overlay at the end also lists the error
+    await expect(page.getByText("Authentication Fails").first()).toBeVisible()
+    await page.getByRole("button", { name: "Open model settings" }).click()
+    await expect(
+        page.getByRole("dialog", { name: "AI Model Configuration" }),
+    ).toBeVisible()
+})
+
+test("an error before the stream shows its text, not raw JSON", async ({
+    page,
+}) => {
+    await chatWith(page, {
+        status: 400,
+        contentType: "application/json",
+        body: JSON.stringify({
+            error: 'Model "x" is not available on the server',
+        }),
+    })
+    // The whole message, not the JSON around it (the Next.js dev overlay
+    // also lists the raw console error; it is not part of the app)
+    await expect(
+        page.getByText('Model "x" is not available on the server', {
+            exact: true,
+        }),
+    ).toBeVisible({ timeout: 15000 })
+})
+
+test("a provider rate limit is not shown as this site's quota", async ({
+    page,
+}) => {
+    const errorText = JSON.stringify({
+        type: "provider",
+        code: "rate_limited",
+        message: "Rate limit exceeded for your organization",
+    })
+    await chatWith(page, {
+        status: 200,
+        contentType: "text/event-stream",
+        body: `data: {"type":"start"}\n\ndata: ${JSON.stringify({ type: "error", errorText })}\n\ndata: [DONE]\n\n`,
+    })
+    await expect(
+        page.getByText("The provider is limiting requests"),
+    ).toBeVisible({ timeout: 15000 })
+    // The site's own tokens-per-minute toast
+    await expect(page.getByText("Rate limit reached")).toHaveCount(0)
+})
@@ -0,0 +1,103 @@
+// @vitest-environment node
+import { APICallError, InvalidToolInputError, RetryError } from "ai"
+import { describe, expect, it } from "vitest"
+import { classifyLLMError, isToolCallError } from "@/lib/llm-errors"
+
+const apiError = (statusCode: number, message: string, responseBody = "") =>
+    new APICallError({
+        message,
+        url: "https://api.example.com/v1/chat/completions",
+        requestBodyValues: {},
+        statusCode,
+        responseBody,
+    })
+
+describe("classifyLLMError", () => {
+    it("reads the status code, not the message", () => {
+        // Providers rarely put the number in their message
+        expect(
+            classifyLLMError(apiError(401, "Authentication Fails")).code,
+        ).toBe("invalid_api_key")
+        expect(classifyLLMError(apiError(404, "Unknown")).code).toBe(
+            "model_not_found",
+        )
+        expect(classifyLLMError(apiError(503, "busy")).code).toBe(
+            "provider_unavailable",
+        )
+    })
+
+    it("lets a specific text win over the status code", () => {
+        expect(
+            classifyLLMError(
+                apiError(429, "You exceeded your current quota, check billing"),
+            ).code,
+        ).toBe("insufficient_quota")
+        expect(
+            classifyLLMError(
+                apiError(400, "This model's maximum context length is 128000"),
+            ).code,
+        ).toBe("context_too_long")
+        expect(
+            classifyLLMError(
+                apiError(400, "bad", '{"message":"toolUse.input is invalid"}'),
+            ).code,
+        ).toBe("output_truncated")
+    })
+
+    it("does not call a 403 an invalid key", () => {
+        expect(classifyLLMError(apiError(403, "Forbidden")).code).toBe(
+            "forbidden",
+        )
+    })
+
+    it("uses the last attempt after retries", () => {
+        const retry = new RetryError({
+            message: "Failed after 3 attempts",
+            reason: "maxRetriesExceeded",
+            errors: [apiError(500, "x"), apiError(429, "slow down")],
+        })
+        expect(classifyLLMError(retry).code).toBe("rate_limited")
+    })
+
+    it("keeps the message but hides secrets in it", () => {
+        const { code, message } = classifyLLMError(
+            apiError(
+                401,
+                "Incorrect API key provided: sk-proj-abcdefghijklmnop. Header Bearer abc.def",
+            ),
+        )
+        expect(code).toBe("invalid_api_key")
+        expect(message).toContain("Incorrect API key provided")
+        expect(message).not.toContain("abcdefghijklmnop")
+        expect(message).not.toContain("abc.def")
+    })
+
+    it("leaves our own messages readable", () => {
+        // This one used to be replaced by "Authentication failed" for
+        // containing the word key
+        const { message } = classifyLLMError(
+            new Error(
+                "API key is required when using a custom base URL. Please provide your own API key in Settings.",
+            ),
+        )
+        expect(message).toContain("API key is required when using a custom")
+    })
+
+    it("names a timeout", () => {
+        const timeout = new Error("The operation was aborted due to timeout")
+        timeout.name = "TimeoutError"
+        expect(classifyLLMError(timeout).code).toBe("timeout")
+    })
+})
+
+describe("isToolCallError", () => {
+    it("spots errors the model must see unchanged", () => {
+        const invalid = new InvalidToolInputError({
+            toolName: "display_diagram",
+            toolInput: "{",
+            cause: new Error("bad JSON"),
+        })
+        expect(isToolCallError(invalid)).toBe(true)
+        expect(isToolCallError(apiError(500, "x"))).toBe(false)
+    })
+})