From ab1a58f999020151ecb2897dbf70a24fe1296729 Mon Sep 17 00:00:00 2001 From: "dayuan.jiang" Date: Sun, 4 Oct 2026 13:49:34 +0900 Subject: [PATCH] feat(errors): classify provider errors and show a hint the user can act on - lib/llm-errors.ts sorts an error into about a dozen kinds (key rejected, no access, unknown model, no credit, rate limited, context too long, no image input, no tool calls, output cut off, provider down, cannot connect, timeout): first texts that name the cause precisely, then the HTTP status code, then general texts. It unwraps RetryError and hides keys and Bearer tokens in the provider's message - The chat route uses it for errors before the stream and, through toUIMessageStreamResponse's onError, for errors in the stream. Errors of the model's own tool call stay as they are: the same text goes back to the model so it can fix the call - The chat shows the hint in the user's language, then the provider's message; a rejected key, missing access or unknown model adds an "Open model settings" button. The Test button shows the same hints - Fixes: our message "API key is required when using a custom base URL" was replaced by "Authentication failed" because it contains "key"; a provider's "Rate limit exceeded" opened this site's quota toast; an error body like {"error": ...} was shown as raw JSON; the Test button matched "401" in the message, where providers rarely put it - Remove the string matching fallbacks in the chat panel --- app/api/chat/route.ts | 65 ++++---------- app/api/validate-model/route.ts | 31 +------ components/chat-message-display.tsx | 29 +++++++ components/chat-panel.tsx | 106 ++++++++++------------- components/model-config-dialog.tsx | 22 ++++- lib/i18n/dictionaries/en.json | 17 +++- lib/i18n/dictionaries/ja.json | 17 +++- lib/i18n/dictionaries/zh-Hant.json | 17 +++- lib/i18n/dictionaries/zh.json | 17 +++- lib/llm-errors.ts | 128 ++++++++++++++++++++++++++++ tests/e2e/chat-errors.spec.ts | 77 +++++++++++++++++ tests/unit/llm-errors.test.ts | 103 ++++++++++++++++++++++ 12 files changed, 487 insertions(+), 142 deletions(-) create mode 100644 lib/llm-errors.ts create mode 100644 tests/e2e/chat-errors.spec.ts create mode 100644 tests/unit/llm-errors.test.ts diff --git a/app/api/chat/route.ts b/app/api/chat/route.ts index 3ad90654..8d743ed6 100644 --- a/app/api/chat/route.ts +++ b/app/api/chat/route.ts @@ -4,7 +4,6 @@ import { createUIMessageStream, createUIMessageStreamResponse, InvalidToolInputError, - LoadAPIKeyError, stepCountIs, streamText, } from "ai" @@ -39,6 +38,7 @@ import { setTraceOutput, wrapWithObserve, } from "@/lib/langfuse" +import { classifyLLMError, isToolCallError } from "@/lib/llm-errors" import { resolveMaxOutputTokens, withOutputTokenLimitFallback, @@ -720,6 +720,12 @@ Call this tool to get shape names and usage syntax for a specific library.`, const response = result.toUIMessageStreamResponse({ sendReasoning: true, + // The same text goes back to the model when its tool call was + // invalid, so it can fix it: keep that one as it is + onError: (error) => + isToolCallError(error) + ? (error as Error).message + : JSON.stringify(classifyLLMError(error)), messageMetadata: ({ part }) => { if (part.type === "finish") { const usage = (part as any).totalUsage @@ -736,61 +742,24 @@ Call this tool to get shape names and usage syntax for a specific library.`, return response } -// Helper to categorize errors and return appropriate response +// Errors before the stream starts, as JSON the chat panel reads function handleError(error: unknown): Response { console.error("Error in chat route:", error) const isDev = process.env.NODE_ENV === "development" - - // Check for specific AI SDK error types - if (APICallError.isInstance(error)) { - return Response.json( - { - error: error.message, - ...(isDev && { - details: error.responseBody, - stack: error.stack, - }), - }, - { status: error.statusCode || 500 }, - ) - } - - if (LoadAPIKeyError.isInstance(error)) { - return Response.json( - { - error: "Authentication failed. Please check your API key.", - ...(isDev && { - stack: error.stack, - }), - }, - { status: 401 }, - ) - } - - // Fallback for other errors with safety filter - const message = - error instanceof Error ? error.message : "An unexpected error occurred" - const status = (error as any)?.statusCode || (error as any)?.status || 500 - - // Prevent leaking API keys, tokens, or other sensitive data - const lowerMessage = message.toLowerCase() - const safeMessage = - lowerMessage.includes("key") || - lowerMessage.includes("token") || - lowerMessage.includes("sig") || - lowerMessage.includes("signature") || - lowerMessage.includes("secret") || - lowerMessage.includes("password") || - lowerMessage.includes("credential") - ? "Authentication failed. Please check your credentials." - : message + const classified = classifyLLMError(error) + const status = + (error as { statusCode?: number })?.statusCode || + (error as { status?: number })?.status || + (classified.code === "invalid_api_key" ? 401 : 500) return Response.json( { - error: safeMessage, + ...classified, ...(isDev && { - details: message, + details: APICallError.isInstance(error) + ? error.responseBody + : undefined, stack: error instanceof Error ? error.stack : undefined, }), }, diff --git a/app/api/validate-model/route.ts b/app/api/validate-model/route.ts index 1bebcb14..1f103781 100644 --- a/app/api/validate-model/route.ts +++ b/app/api/validate-model/route.ts @@ -3,6 +3,7 @@ import { NextResponse } from "next/server" import { z } from "zod" import { checkAccessCode } from "@/lib/access-code" import { getAIModel } from "@/lib/ai-providers" +import { classifyLLMError } from "@/lib/llm-errors" import { allowPrivateUrls, isPrivateUrl } from "@/lib/ssrf-protection" export const runtime = "nodejs" @@ -146,35 +147,9 @@ export async function POST(req: Request) { } catch (error) { console.error("[validate-model] Error:", error) - let errorMessage = "Validation failed" - if (error instanceof Error) { - // Extract meaningful error message - if (error.name === "TimeoutError") { - errorMessage = `No answer within ${TEST_TIMEOUT_MS / 1000} seconds` - } else if ( - error.message.includes("401") || - error.message.includes("Unauthorized") - ) { - errorMessage = "Invalid API key" - } else if ( - error.message.includes("404") || - error.message.includes("not found") - ) { - errorMessage = "Model not found" - } else if ( - error.message.includes("429") || - error.message.includes("rate limit") - ) { - errorMessage = "Rate limited - try again later" - } else if (error.message.includes("ECONNREFUSED")) { - errorMessage = "Cannot connect to server" - } else { - errorMessage = error.message.slice(0, 100) - } - } - + const { code, message } = classifyLLMError(error) return NextResponse.json( - { valid: false, error: errorMessage }, + { valid: false, code, error: message }, { status: 200 }, // Return 200 so client can read error message ) } diff --git a/components/chat-message-display.tsx b/components/chat-message-display.tsx index 0d78ad78..ef120afe 100644 --- a/components/chat-message-display.tsx +++ b/components/chat-message-display.tsx @@ -148,6 +148,8 @@ interface SessionMetadata { interface ChatMessageDisplayProps { messages: UIMessage[] + // Shown on an error that a model setting can fix (bad key, unknown model) + onOpenModelConfig?: () => void setInput: (input: string) => void setFiles: (files: File[]) => void processedToolCallsRef: MutableRefObject> @@ -171,6 +173,7 @@ interface ChatMessageDisplayProps { export function ChatMessageDisplay({ messages, + onOpenModelConfig, setInput, setFiles, processedToolCallsRef, @@ -1193,6 +1196,32 @@ export function ChatMessageDisplay({ return null }, )} + {message.role === + "system" && + ( + message.metadata as + | { + openModelConfig?: boolean + } + | undefined + ) + ?.openModelConfig && + onOpenModelConfig && ( + + )} ) }, diff --git a/components/chat-panel.tsx b/components/chat-panel.tsx index 6722a20f..d0d938de 100644 --- a/components/chat-panel.tsx +++ b/components/chat-panel.tsx @@ -363,84 +363,69 @@ export default function ChatPanel({ await handleToolCall({ toolCall }, addToolOutput) }, onError: (error) => { - // Handle server-side quota limit (429 response) - // AI SDK puts the full response body in error.message for non-OK responses + // Server errors are JSON: a quota limit ({type: request, token or + // tpm}), a provider error ({type: "provider", code, message}) or + // {error}. The SDK puts the response body in error.message. + let data: any = null try { - const data = JSON.parse(error.message) - if (data.type === "request") { - quotaManager.showQuotaLimitToast(data.used, data.limit) - return - } - if (data.type === "token") { - quotaManager.showTokenLimitToast(data.used, data.limit) - return - } - if (data.type === "tpm") { - quotaManager.showTPMLimitToast(data.limit) - return - } + data = JSON.parse(error.message) } catch { - // Not JSON, fall through to string matching for backwards compatibility + // Plain text, e.g. a network failure in the browser } - - // Fallback to string matching - if (error.message.includes("Daily request limit")) { - quotaManager.showQuotaLimitToast() + if (data?.type === "request") { + quotaManager.showQuotaLimitToast(data.used, data.limit) return } - if (error.message.includes("Daily token limit")) { - quotaManager.showTokenLimitToast() + if (data?.type === "token") { + quotaManager.showTokenLimitToast(data.used, data.limit) return } - if ( - error.message.includes("Rate limit exceeded") || - error.message.includes("tokens per minute") - ) { - quotaManager.showTPMLimitToast() + if (data?.type === "tpm") { + quotaManager.showTPMLimitToast(data.limit) return } + const isAccessCodeError = String( + data?.error ?? error.message, + ).includes("Invalid or missing access code") // Silence access code error in console since it's handled by UI - if (!error.message.includes("Invalid or missing access code")) { - console.error("Chat error:", error) - } + if (!isAccessCodeError) console.error("Chat error:", error) - // Translate technical errors into user-friendly messages - // The server now handles detailed error messages, so we can display them directly. - // But we still handle connection/network errors that happen before reaching the server. - let friendlyMessage = error.message - - // Simple check for network errors if message is generic - if (friendlyMessage === "Failed to fetch") { - friendlyMessage = "Network error. Please check your connection." - } - - // Truncated tool input error (model output limit too low) - if (friendlyMessage.includes("toolUse.input is invalid")) { - friendlyMessage = - "Output was truncated before the diagram could be generated. Try a simpler request or increase the maxOutputLength." - } - - // Translate image not supported error - if ( - friendlyMessage.includes("image content block") || - friendlyMessage.toLowerCase().includes("image_url") - ) { - friendlyMessage = "This model doesn't support image input." + // A hint the user can act on, then the provider's own words + let text: string = error.message + let openModelConfig = false + if (data?.type === "provider") { + const hints = dict.errors.llm as Record + text = hints[data.code] + ? `${hints[data.code]}\n\n${data.message}` + : data.message + openModelConfig = [ + "invalid_api_key", + "forbidden", + "model_not_found", + ].includes(data.code) + } else if (typeof data?.error === "string") { + text = data.error + } else if (error.message === "Failed to fetch") { + text = dict.errors.networkError } // Add system message for error so it can be cleared - setMessages((currentMessages) => { - const errorMessage = { + setMessages((currentMessages) => [ + ...currentMessages, + { id: `error-${Date.now()}`, role: "system" as const, - content: friendlyMessage, - parts: [{ type: "text" as const, text: friendlyMessage }], - } - return [...currentMessages, errorMessage] - }) + content: text, + parts: [{ type: "text" as const, text }], + // The message shows a button that opens model settings + ...(openModelConfig && { + metadata: { openModelConfig: true }, + }), + }, + ]) - if (error.message.includes("Invalid or missing access code")) { + if (isAccessCodeError) { // Show settings dialog to help user fix it setShowSettingsDialog(true) } @@ -1417,6 +1402,7 @@ export default function ChatPanel({ {/* Messages */}
setShowModelConfigDialog(true)} messages={messages} setInput={setInput} setFiles={handleFileChange} diff --git a/components/model-config-dialog.tsx b/components/model-config-dialog.tsx index b1c2a35a..ead86ffa 100644 --- a/components/model-config-dialog.tsx +++ b/components/model-config-dialog.tsx @@ -414,8 +414,20 @@ export function ModelConfigDialog({ } : { validated: false, + // The hint for the error's kind, then the + // provider's own message validationError: - data.error || + [ + ( + dict.errors.llm as Record< + string, + string + > + )[data.code], + data.error, + ] + .filter(Boolean) + .join(" ") || (response.ok ? "Validation failed" : `Request failed (${response.status})`), @@ -459,7 +471,13 @@ export function ModelConfigDialog({ setValidationStatus("error") setValidationError(`${errorCount} model(s) failed validation`) } - }, [selectedProvider, selectedProviderId, updateProvider, updateModel]) + }, [ + selectedProvider, + selectedProviderId, + updateProvider, + updateModel, + dict, + ]) // Get all available provider types const availableProviders = Object.keys(PROVIDER_INFO) as ProviderName[] diff --git a/lib/i18n/dictionaries/en.json b/lib/i18n/dictionaries/en.json index 1e4f0e7f..62fdb080 100644 --- a/lib/i18n/dictionaries/en.json +++ b/lib/i18n/dictionaries/en.json @@ -186,7 +186,22 @@ "failedToLoadExample": "Error loading example image", "failedToRecordFeedback": "Failed to record your feedback. Please try again.", "storageUpdateFailed": "Chat cleared but browser storage could not be updated", - "sessionSaveFailed": "Could not save this chat. Browser storage may be full: delete old chats from history and try again." + "sessionSaveFailed": "Could not save this chat. Browser storage may be full: delete old chats from history and try again.", + "llm": { + "invalid_api_key": "The provider rejected the API key. Check it in model settings.", + "forbidden": "The provider refused the request. The key may not have access to this model or region.", + "model_not_found": "The provider does not know this model. Check the model ID in model settings.", + "insufficient_quota": "The provider account has no credit or quota left.", + "rate_limited": "The provider is limiting requests. Wait a moment and try again.", + "context_too_long": "The conversation is too long for this model. Start a new chat or pick a model with a larger context.", + "images_unsupported": "This model doesn't support image input.", + "tools_unsupported": "This model doesn't support tool calls, which drawing needs. Pick another model.", + "output_truncated": "The output was cut off before the diagram was complete. Try a simpler request or raise the output limit in settings.", + "provider_unavailable": "The provider is having problems. Try again later.", + "cannot_connect": "Could not reach the provider. Check the base URL and your network.", + "timeout": "The provider did not answer in time.", + "openModelSettings": "Open model settings" + } }, "quota": { "dailyLimit": "Daily Quota Reached", diff --git a/lib/i18n/dictionaries/ja.json b/lib/i18n/dictionaries/ja.json index 6fc0ba52..feced64c 100644 --- a/lib/i18n/dictionaries/ja.json +++ b/lib/i18n/dictionaries/ja.json @@ -186,7 +186,22 @@ "failedToLoadExample": "例の画像の読み込みエラー", "failedToRecordFeedback": "フィードバックの記録に失敗しました。もう一度お試しください。", "storageUpdateFailed": "チャットはクリアされましたが、ブラウザストレージを更新できませんでした", - "sessionSaveFailed": "このチャットを保存できませんでした。ブラウザのストレージがいっぱいの可能性があります。履歴から古いチャットを削除して、もう一度お試しください。" + "sessionSaveFailed": "このチャットを保存できませんでした。ブラウザのストレージがいっぱいの可能性があります。履歴から古いチャットを削除して、もう一度お試しください。", + "llm": { + "invalid_api_key": "プロバイダーが API キーを拒否しました。モデル設定で確認してください。", + "forbidden": "プロバイダーがリクエストを拒否しました。このキーにはこのモデルまたはリージョンの利用権限がない可能性があります。", + "model_not_found": "プロバイダーがこのモデルを認識できません。モデル設定でモデル ID を確認してください。", + "insufficient_quota": "プロバイダーのアカウントの残高または利用枠がなくなりました。", + "rate_limited": "プロバイダーがリクエスト数を制限しています。少し待ってから再試行してください。", + "context_too_long": "会話がこのモデルで扱える長さを超えています。新しいチャットを始めるか、より長いコンテキストに対応したモデルを選んでください。", + "images_unsupported": "このモデルは画像入力に対応していません。", + "tools_unsupported": "このモデルはツール呼び出しに対応していません。作図にはツール呼び出しが必要です。別のモデルを選んでください。", + "output_truncated": "ダイアグラムが完成する前に出力が途中で切れました。リクエストを簡単にするか、設定で出力上限を上げてください。", + "provider_unavailable": "プロバイダーで問題が発生しています。しばらくしてから再試行してください。", + "cannot_connect": "プロバイダーに接続できません。Base URL とネットワークを確認してください。", + "timeout": "プロバイダーから時間内に応答がありませんでした。", + "openModelSettings": "モデル設定を開く" + } }, "quota": { "dailyLimit": "1日の割当量に達しました", diff --git a/lib/i18n/dictionaries/zh-Hant.json b/lib/i18n/dictionaries/zh-Hant.json index 9412ee04..7227ef5f 100644 --- a/lib/i18n/dictionaries/zh-Hant.json +++ b/lib/i18n/dictionaries/zh-Hant.json @@ -186,7 +186,22 @@ "failedToLoadExample": "載入範例圖片時出錯", "failedToRecordFeedback": "記錄您的回饋失敗。請重試。", "storageUpdateFailed": "聊天已清除，但無法更新瀏覽器儲存空間", - "sessionSaveFailed": "無法儲存這個對話。瀏覽器儲存空間可能已滿，請在歷史紀錄裡刪除舊對話後重試。" + "sessionSaveFailed": "無法儲存這個對話。瀏覽器儲存空間可能已滿，請在歷史紀錄裡刪除舊對話後重試。", + "llm": { + "invalid_api_key": "服務商拒絕了這個 API Key，請在模型設定中檢查。", + "forbidden": "服務商拒絕了這次請求。這個 Key 可能沒有使用該模型或該地區的權限。", + "model_not_found": "服務商找不到這個模型，請在模型設定中檢查模型 ID。", + "insufficient_quota": "服務商帳戶的餘額或額度已經用完。", + "rate_limited": "服務商正在限制請求頻率，請稍候再試。", + "context_too_long": "對話內容超過了這個模型能處理的長度。請開啟新的對話，或換一個上下文更長的模型。", + "images_unsupported": "這個模型不支援圖片輸入。", + "tools_unsupported": "這個模型不支援工具呼叫，而繪圖需要工具呼叫。請換一個模型。", + "output_truncated": "輸出在圖表完成之前就被截斷了。請簡化請求，或在設定中調高輸出上限。", + "provider_unavailable": "服務商發生問題，請稍後再試。", + "cannot_connect": "無法連線到服務商，請檢查 Base URL 和網路。", + "timeout": "服務商沒有及時回應。", + "openModelSettings": "開啟模型設定" + } }, "quota": { "dailyLimit": "已達每日配額", diff --git a/lib/i18n/dictionaries/zh.json b/lib/i18n/dictionaries/zh.json index abdf9f64..93c905be 100644 --- a/lib/i18n/dictionaries/zh.json +++ b/lib/i18n/dictionaries/zh.json @@ -186,7 +186,22 @@ "failedToLoadExample": "加载示例图片时出错", "failedToRecordFeedback": "记录您的反馈失败。请重试。", "storageUpdateFailed": "聊天已清除，但无法更新浏览器存储", - "sessionSaveFailed": "无法保存这个对话。浏览器存储空间可能已满，请在历史记录里删除旧对话后重试。" + "sessionSaveFailed": "无法保存这个对话。浏览器存储空间可能已满，请在历史记录里删除旧对话后重试。", + "llm": { + "invalid_api_key": "服务商拒绝了这个 API Key，请在模型设置里检查。", + "forbidden": "服务商拒绝了这次请求。这个 Key 可能没有使用该模型或该地区的权限。", + "model_not_found": "服务商找不到这个模型，请在模型设置里检查模型 ID。", + "insufficient_quota": "服务商账户的余额或额度已经用完。", + "rate_limited": "服务商正在限制请求频率，请稍等片刻再试。", + "context_too_long": "对话内容超过了这个模型能处理的长度。请新开一个对话，或换一个上下文更长的模型。", + "images_unsupported": "这个模型不支持图片输入。", + "tools_unsupported": "这个模型不支持工具调用，而画图需要工具调用。请换一个模型。", + "output_truncated": "输出在图画完之前就被截断了。请简化请求，或在设置里调高输出上限。", + "provider_unavailable": "服务商出了问题，请稍后再试。", + "cannot_connect": "连接不上服务商，请检查 Base URL 和网络。", + "timeout": "服务商没有及时响应。", + "openModelSettings": "打开模型设置" + } }, "quota": { "dailyLimit": "已达每日配额", diff --git a/lib/llm-errors.ts b/lib/llm-errors.ts new file mode 100644 index 00000000..1cc15118 --- /dev/null +++ b/lib/llm-errors.ts @@ -0,0 +1,128 @@ +import { + APICallError, + InvalidToolInputError, + LoadAPIKeyError, + NoSuchToolError, + RetryError, + ToolCallRepairError, +} from "ai" + +/** + * What went wrong with a model call, for a hint the user can act on. The + * provider's own message always goes along, because a guess can be wrong. + */ +export type LLMErrorCode = + | "invalid_api_key" + | "forbidden" + | "model_not_found" + | "insufficient_quota" + | "rate_limited" + | "context_too_long" + | "images_unsupported" + | "tools_unsupported" + | "output_truncated" + | "provider_unavailable" + | "cannot_connect" + | "timeout" + | "unknown" + +export interface LLMError { + type: "provider" + code: LLMErrorCode + message: string +} + +// Texts that name the cause more precisely than the status code: a quota +// error can come as 403 or 429, a context or image error as a plain 400 +const SPECIFIC_TEXTS: Array<[RegExp, LLMErrorCode]> = [ + [ + /context length|context window|maximum context|prompt is too long|input is too long|too many (?:input )?tokens/i, + "context_too_long", + ], + [ + /image content block|image_url|does not support image|image input is not supported/i, + "images_unsupported", + ], + [ + /does not support tools|tool use is not supported|tools? (?:are|is) not supported|function calling is not supported/i, + "tools_unsupported", + ], + // Bedrock, when the output limit cut the tool call's JSON short + [/toolUse\.input is invalid/i, "output_truncated"], + [ + /insufficient[_ ]quota|insufficient balance|exceeded your current quota|credit balance is too low|余额不足/i, + "insufficient_quota", + ], +] + +const STATUS_CODES: Record = { + 401: "invalid_api_key", + 402: "insufficient_quota", + // Not "invalid key": a valid key can lack access to a model or region + 403: "forbidden", + 404: "model_not_found", + 408: "timeout", + 413: "context_too_long", + 429: "rate_limited", +} + +const GENERAL_TEXTS: Array<[RegExp, LLMErrorCode]> = [ + [ + /model[_ ]not[_ ]found|model .*does not exist|unknown model|no such model/i, + "model_not_found", + ], + [ + /invalid[_ ]api[_ ]key|incorrect api key|unauthorized/i, + "invalid_api_key", + ], + [/rate limit|too many requests/i, "rate_limited"], + [/ECONNREFUSED|ENOTFOUND|ECONNRESET|fetch failed/i, "cannot_connect"], +] + +/** Secrets a provider may echo back: API keys, Bearer tokens, key=value */ +function redact(text: string): string { + return text + .replace(/\b(sk|pk|rk|ak)-[A-Za-z0-9_-]{8,}/g, "$1-[redacted]") + .replace(/\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi, "Bearer [redacted]") + .replace(/\bAKIA[0-9A-Z]{16}\b/g, "[redacted]") + .replace( + /\b(api[_-]?key|access[_-]?key|secret|token|password|signature)(["']?\s*[:=]\s*["']?)[^\s"',&}]+/gi, + "$1$2[redacted]", + ) +} + +/** + * Model and tool errors the SDK sends back to the model as the tool result, + * so it can fix its call. Their text has to stay as it is. + */ +export function isToolCallError(error: unknown): boolean { + return ( + InvalidToolInputError.isInstance(error) || + NoSuchToolError.isInstance(error) || + ToolCallRepairError.isInstance(error) + ) +} + +export function classifyLLMError(error: unknown): LLMError { + // After the SDK's retries, the last attempt says what happened + const e = RetryError.isInstance(error) ? error.lastError : error + const raw = e instanceof Error ? e.message : String(e) + const message = redact(raw).slice(0, 500) + const body = APICallError.isInstance(e) ? (e.responseBody ?? "") : "" + const text = `${raw} ${body}` + const status = APICallError.isInstance(e) + ? e.statusCode + : (e as { statusCode?: number })?.statusCode + + const find = (rules: Array<[RegExp, LLMErrorCode]>) => + rules.find(([pattern]) => pattern.test(text))?.[1] + const code = + (e instanceof Error && e.name === "TimeoutError" && "timeout") || + (LoadAPIKeyError.isInstance(e) && "invalid_api_key") || + find(SPECIFIC_TEXTS) || + (status && STATUS_CODES[status]) || + (status && status >= 500 && "provider_unavailable") || + find(GENERAL_TEXTS) || + "unknown" + return { type: "provider", code, message } +} diff --git a/tests/e2e/chat-errors.spec.ts b/tests/e2e/chat-errors.spec.ts new file mode 100644 index 00000000..ff9813f5 --- /dev/null +++ b/tests/e2e/chat-errors.spec.ts @@ -0,0 +1,77 @@ +import { expect, type Page, test } from "@playwright/test" +import { getIframe, sendMessage } from "./lib/fixtures" + +/** Answer the chat request with the given response and send a message */ +async function chatWith( + page: Page, + response: { status: number; contentType: string; body: string }, +) { + await page.route("**/api/chat", (route) => route.fulfill(response)) + await page.goto("/", { waitUntil: "networkidle" }) + await getIframe(page).waitFor({ state: "visible", timeout: 30000 }) + await sendMessage(page, "Draw a box") +} + +test("a rejected key shows a hint, the provider's words and a settings button", async ({ + page, +}) => { + await chatWith(page, { + status: 401, + contentType: "application/json", + body: JSON.stringify({ + type: "provider", + code: "invalid_api_key", + message: "Authentication Fails, Your api key is invalid", + }), + }) + await expect( + page.getByText("The provider rejected the API key"), + ).toBeVisible({ + timeout: 15000, + }) + // First match: the Next.js dev overlay at the end also lists the error + await expect(page.getByText("Authentication Fails").first()).toBeVisible() + await page.getByRole("button", { name: "Open model settings" }).click() + await expect( + page.getByRole("dialog", { name: "AI Model Configuration" }), + ).toBeVisible() +}) + +test("an error before the stream shows its text, not raw JSON", async ({ + page, +}) => { + await chatWith(page, { + status: 400, + contentType: "application/json", + body: JSON.stringify({ + error: 'Model "x" is not available on the server', + }), + }) + // The whole message, not the JSON around it (the Next.js dev overlay + // also lists the raw console error; it is not part of the app) + await expect( + page.getByText('Model "x" is not available on the server', { + exact: true, + }), + ).toBeVisible({ timeout: 15000 }) +}) + +test("a provider rate limit is not shown as this site's quota", async ({ + page, +}) => { + const errorText = JSON.stringify({ + type: "provider", + code: "rate_limited", + message: "Rate limit exceeded for your organization", + }) + await chatWith(page, { + status: 200, + contentType: "text/event-stream", + body: `data: {"type":"start"}\n\ndata: ${JSON.stringify({ type: "error", errorText })}\n\ndata: [DONE]\n\n`, + }) + await expect( + page.getByText("The provider is limiting requests"), + ).toBeVisible({ timeout: 15000 }) + // The site's own tokens-per-minute toast + await expect(page.getByText("Rate limit reached")).toHaveCount(0) +}) diff --git a/tests/unit/llm-errors.test.ts b/tests/unit/llm-errors.test.ts new file mode 100644 index 00000000..a0326922 --- /dev/null +++ b/tests/unit/llm-errors.test.ts @@ -0,0 +1,103 @@ +// @vitest-environment node +import { APICallError, InvalidToolInputError, RetryError } from "ai" +import { describe, expect, it } from "vitest" +import { classifyLLMError, isToolCallError } from "@/lib/llm-errors" + +const apiError = (statusCode: number, message: string, responseBody = "") => + new APICallError({ + message, + url: "https://api.example.com/v1/chat/completions", + requestBodyValues: {}, + statusCode, + responseBody, + }) + +describe("classifyLLMError", () => { + it("reads the status code, not the message", () => { + // Providers rarely put the number in their message + expect( + classifyLLMError(apiError(401, "Authentication Fails")).code, + ).toBe("invalid_api_key") + expect(classifyLLMError(apiError(404, "Unknown")).code).toBe( + "model_not_found", + ) + expect(classifyLLMError(apiError(503, "busy")).code).toBe( + "provider_unavailable", + ) + }) + + it("lets a specific text win over the status code", () => { + expect( + classifyLLMError( + apiError(429, "You exceeded your current quota, check billing"), + ).code, + ).toBe("insufficient_quota") + expect( + classifyLLMError( + apiError(400, "This model's maximum context length is 128000"), + ).code, + ).toBe("context_too_long") + expect( + classifyLLMError( + apiError(400, "bad", '{"message":"toolUse.input is invalid"}'), + ).code, + ).toBe("output_truncated") + }) + + it("does not call a 403 an invalid key", () => { + expect(classifyLLMError(apiError(403, "Forbidden")).code).toBe( + "forbidden", + ) + }) + + it("uses the last attempt after retries", () => { + const retry = new RetryError({ + message: "Failed after 3 attempts", + reason: "maxRetriesExceeded", + errors: [apiError(500, "x"), apiError(429, "slow down")], + }) + expect(classifyLLMError(retry).code).toBe("rate_limited") + }) + + it("keeps the message but hides secrets in it", () => { + const { code, message } = classifyLLMError( + apiError( + 401, + "Incorrect API key provided: sk-proj-abcdefghijklmnop. Header Bearer abc.def", + ), + ) + expect(code).toBe("invalid_api_key") + expect(message).toContain("Incorrect API key provided") + expect(message).not.toContain("abcdefghijklmnop") + expect(message).not.toContain("abc.def") + }) + + it("leaves our own messages readable", () => { + // This one used to be replaced by "Authentication failed" for + // containing the word key + const { message } = classifyLLMError( + new Error( + "API key is required when using a custom base URL. Please provide your own API key in Settings.", + ), + ) + expect(message).toContain("API key is required when using a custom") + }) + + it("names a timeout", () => { + const timeout = new Error("The operation was aborted due to timeout") + timeout.name = "TimeoutError" + expect(classifyLLMError(timeout).code).toBe("timeout") + }) +}) + +describe("isToolCallError", () => { + it("spots errors the model must see unchanged", () => { + const invalid = new InvalidToolInputError({ + toolName: "display_diagram", + toolInput: "{", + cause: new Error("bad JSON"), + }) + expect(isToolCallError(invalid)).toBe(true) + expect(isToolCallError(apiError(500, "x"))).toBe(false) + }) +})