diff --git a/app/api/chat/route.ts b/app/api/chat/route.ts index 8f987f34..ef79738a 100644 --- a/app/api/chat/route.ts +++ b/app/api/chat/route.ts @@ -13,6 +13,7 @@ import path from "path" import { z } from "zod" import { checkAccessCode } from "@/lib/access-code" import { + CACHE_POINT, getAIModel, SINGLE_SYSTEM_PROVIDERS, supportsPromptCaching, @@ -25,6 +26,7 @@ import { replaceHistoricalToolInputs, validateFileParts, } from "@/lib/chat-helpers" +import { withDeprecatedParamsFallback } from "@/lib/deprecated-params" import { checkAndIncrementRequest, isQuotaEnabled, @@ -266,7 +268,6 @@ async function handleChatRequest(req: Request): Promise { const { model: baseModel, providerOptions, - headers, modelId, provider: resolvedProvider, } = getAIModel(clientOverrides) @@ -289,8 +290,11 @@ async function handleChatRequest(req: Request): Promise { ) } - // Retry with a smaller budget if the provider rejects the requested one - const model = withOutputTokenLimitFallback(baseModel) + // Retry once if the provider rejects the requested budget, or (newer + // Claude models) the sampling or thinking settings + const model = withOutputTokenLimitFallback( + withDeprecatedParamsFallback(baseModel), + ) // The user setting can raise the budget only on their own key (desktop users // can still raise it themselves); on the server's keys it can only lower it @@ -444,9 +448,7 @@ ${userInputText} if (enhancedMessages[i].role === "assistant") { enhancedMessages[i] = { ...enhancedMessages[i], - providerOptions: { - bedrock: { cachePoint: { type: "default" } }, - }, + providerOptions: CACHE_POINT, } break // Only cache the last assistant message } @@ -499,21 +501,13 @@ IMPORTANT: The "Current diagram XML" is the SINGLE SOURCE OF TRUTH for what's on { role: "system" as const, content: finalSystemMessage, - ...(shouldCache && { - providerOptions: { - bedrock: { cachePoint: { type: "default" } }, - }, - }), + ...(shouldCache && { providerOptions: CACHE_POINT }), }, // Cache breakpoint 2: Previous and Current diagram XML context { role: "system" as const, content: xmlContext, - ...(shouldCache && { - providerOptions: { - bedrock: { cachePoint: { type: "default" } }, - }, - }), + ...(shouldCache && { providerOptions: CACHE_POINT }), }, ] @@ -566,7 +560,6 @@ IMPORTANT: The "Current diagram XML" is the SINGLE SOURCE OF TRUTH for what's on }, messages: allMessages, ...(providerOptions && { providerOptions }), // This now includes all reasoning configs - ...(headers && { headers }), // Langfuse telemetry config (returns undefined if not configured) ...(getTelemetryConfig({ sessionId: validSessionId, userId }) && { experimental_telemetry: getTelemetryConfig({ diff --git a/env.example b/env.example index c8abbc9a..fb5850ce 100644 --- a/env.example +++ b/env.example @@ -44,7 +44,6 @@ AI_MODEL=global.anthropic.claude-sonnet-4-5-20250929-v1:0 # Google Generative AI Configuration # GOOGLE_GENERATIVE_AI_API_KEY=... # GOOGLE_BASE_URL=https://generativelanguage.googleapis.com/v1beta # Optional: Custom endpoint -# GOOGLE_CANDIDATE_COUNT=1 # Optional: Number of candidates to generate # GOOGLE_TOP_K=40 # Optional: Top K sampling parameter # GOOGLE_TOP_P=0.95 # Optional: Nucleus sampling parameter # Note: Gemini 2.5/3 models automatically enable reasoning display (includeThoughts: true) diff --git a/lib/ai-providers.ts b/lib/ai-providers.ts index e3ae5823..b3a60d27 100644 --- a/lib/ai-providers.ts +++ b/lib/ai-providers.ts @@ -9,6 +9,7 @@ import { createOpenAI, openai } from "@ai-sdk/openai" import { aihubmix, createAihubmix } from "@aihubmix/ai-sdk-provider" import { fromNodeProviderChain } from "@aws-sdk/credential-providers" import { createOpenRouter } from "@openrouter/ai-sdk-provider" +import { defaultSettingsMiddleware, wrapLanguageModel } from "ai" import { createOllama, ollama } from "ollama-ai-provider-v2" import { adminProvidersToConfig, @@ -23,7 +24,6 @@ export const AIHUBMIX_APP_CODE = "MSBS9675" interface ModelConfig { model: any providerOptions?: any - headers?: Record modelId: string provider: ProviderName } @@ -132,11 +132,6 @@ const BEDROCK_ANTHROPIC_BETA = { }, } -// Direct Anthropic API headers for beta features -const ANTHROPIC_BETA_HEADERS = { - "anthropic-beta": "fine-grained-tool-streaming-2025-05-14", -} - /** * Resolve baseURL based on whether user is providing their own API key. * When user provides their own API key, we should NOT fall back to server's @@ -241,6 +236,26 @@ function parseIntSafe( return parsed } +/** + * GOOGLE_TOP_K and GOOGLE_TOP_P. They are call settings, so they go on the + * model through a middleware: as Google provider options they were dropped. + */ +function googleSamplingSettings(): { topK?: number; topP?: number } { + const settings: { topK?: number; topP?: number } = {} + const topK = parseIntSafe(process.env.GOOGLE_TOP_K, "GOOGLE_TOP_K", 1, 100) + if (topK) settings.topK = topK + if (process.env.GOOGLE_TOP_P) { + const topP = Number.parseFloat(process.env.GOOGLE_TOP_P) + if (Number.isNaN(topP) || topP < 0 || topP > 1) { + throw new Error( + `GOOGLE_TOP_P must be a number between 0 and 1, got: ${process.env.GOOGLE_TOP_P}`, + ) + } + settings.topP = topP + } + return settings +} + /** * Build provider-specific options from environment variables * Supports various AI SDK providers with their unique configuration options @@ -335,7 +350,6 @@ function buildProviderOptions( } case "google": { - const reasoningEffort = process.env.GOOGLE_REASONING_EFFORT const thinkingBudgetVal = parseIntSafe( process.env.GOOGLE_THINKING_BUDGET, "GOOGLE_THINKING_BUDGET", @@ -374,47 +388,6 @@ function buildProviderOptions( } options.google = { thinkingConfig } - } else if (reasoningEffort) { - options.google = { - reasoningEffort: reasoningEffort as - | "low" - | "medium" - | "high", - } - } - - // Keep existing Google options - const options_obj: Record = {} - const candidateCount = parseIntSafe( - process.env.GOOGLE_CANDIDATE_COUNT, - "GOOGLE_CANDIDATE_COUNT", - 1, - 8, - ) - if (candidateCount) { - options_obj.candidateCount = candidateCount - } - const topK = parseIntSafe( - process.env.GOOGLE_TOP_K, - "GOOGLE_TOP_K", - 1, - 100, - ) - if (topK) { - options_obj.topK = topK - } - if (process.env.GOOGLE_TOP_P) { - const topP = Number.parseFloat(process.env.GOOGLE_TOP_P) - if (Number.isNaN(topP) || topP < 0 || topP > 1) { - throw new Error( - `GOOGLE_TOP_P must be a number between 0 and 1, got: ${process.env.GOOGLE_TOP_P}`, - ) - } - options_obj.topP = topP - } - - if (Object.keys(options_obj).length > 0) { - options.google = { ...options.google, ...options_obj } } break } @@ -818,7 +791,6 @@ export function getAIModel(overrides?: ClientOverrides): ModelConfig { let model: any let providerOptions: any - let headers: Record | undefined // Build provider-specific options from environment variables const customProviderOptions = buildProviderOptions(provider, modelId) @@ -922,14 +894,13 @@ export function getAIModel(overrides?: ClientOverrides): ModelConfig { const authToken = !apiKey ? process.env.ANTHROPIC_AUTH_TOKEN : undefined + // The provider streams tool input per tool (eager_input_streaming), + // which replaced the fine-grained-tool-streaming beta header const customProvider = createAnthropic({ ...(authToken ? { authToken } : { apiKey }), baseURL, - headers: ANTHROPIC_BETA_HEADERS, }) model = customProvider(modelId) - // Add beta headers for fine-grained tool streaming - headers = ANTHROPIC_BETA_HEADERS break } @@ -958,6 +929,15 @@ export function getAIModel(overrides?: ClientOverrides): ModelConfig { } else { model = google(modelId) } + const sampling = googleSamplingSettings() + if (Object.keys(sampling).length > 0) { + model = wrapLanguageModel({ + model, + middleware: defaultSettingsMiddleware({ + settings: sampling, + }), + }) + } break } case "vertexai": { @@ -1455,7 +1435,7 @@ export function getAIModel(overrides?: ClientOverrides): ModelConfig { providerOptions = customProviderOptions } - return { model, providerOptions, headers, modelId, provider } + return { model, providerOptions, modelId, provider } } /** @@ -1489,11 +1469,20 @@ export function usesServerCredentials( } /** - * Check if a model supports prompt caching. - * Currently only Claude models on Bedrock support prompt caching. + * Prompt cache breakpoint for Claude, set on a message's providerOptions. + * Each provider reads only its own key; OpenRouter also reads the + * anthropic one. + */ +export const CACHE_POINT = { + bedrock: { cachePoint: { type: "default" } }, + anthropic: { cacheControl: { type: "ephemeral" } }, +} + +/** + * Check if a model supports prompt caching: Claude models, on Bedrock, + * the Anthropic API or OpenRouter (see CACHE_POINT). */ export function supportsPromptCaching(modelId: string): boolean { - // Bedrock prompt caching is supported for Claude models return ( modelId.includes("claude") || modelId.includes("anthropic") || diff --git a/lib/deprecated-params.ts b/lib/deprecated-params.ts new file mode 100644 index 00000000..612338b2 --- /dev/null +++ b/lib/deprecated-params.ts @@ -0,0 +1,90 @@ +import { wrapLanguageModel } from "ai" +import { rejectionText } from "@/lib/output-token-limit" + +type WrappedModel = ReturnType + +/** + * Claude 4.7 and later answer a non-default temperature, top_p or top_k, + * and the extended thinking budget (thinking type "enabled"), with a 400. + * TEMPERATURE and the *_THINKING_BUDGET_TOKENS settings send exactly these. + */ +const DEPRECATED_PARAM = + /`?(?:temperature|top_p|top_k)`? is deprecated for this model|"?thinking\.type\.enabled"? is not supported/i + +interface CallParams { + temperature?: number + topP?: number + topK?: number + providerOptions?: Record | undefined> +} + +/** Drop a thinking config of type "enabled" stored under key, if any. */ +function withoutEnabledThinking( + options: Record | undefined, + key: string, +): Record | undefined { + const config = options?.[key] as { type?: string } | undefined + if (config?.type !== "enabled") return options + const { [key]: _, ...rest } = options as Record + return rest +} + +/** + * The params without the settings newer Claude models reject, or null when + * the error is about something else or there is nothing to drop. The model + * then runs with its own default sampling and thinking. + */ +export function withoutDeprecatedParams( + error: unknown, + params: T, +): T | null { + const text = rejectionText(error) + if (!text || !DEPRECATED_PARAM.test(text)) return null + + const { temperature, topP, topK, ...rest } = params + const options = params.providerOptions + const anthropic = withoutEnabledThinking(options?.anthropic, "thinking") + const bedrock = withoutEnabledThinking(options?.bedrock, "reasoningConfig") + const changed = + temperature !== undefined || + topP !== undefined || + topK !== undefined || + anthropic !== options?.anthropic || + bedrock !== options?.bedrock + if (!changed) return null + + return { + ...rest, + ...(options && { + providerOptions: { + ...options, + ...(anthropic && { anthropic }), + ...(bedrock && { bedrock }), + }, + }), + } as T +} + +/** Retry the stream once without the settings newer Claude models reject. */ +export function withDeprecatedParamsFallback( + model: WrappedModel, +): WrappedModel { + return wrapLanguageModel({ + model, + middleware: { + specificationVersion: "v3", + async wrapStream({ doStream, params, model: inner }) { + try { + return await doStream() + } catch (error) { + const retry = withoutDeprecatedParams(error, params) + if (!retry) throw error + console.warn( + "[model params] Rejected sampling or thinking settings, retrying with the model defaults", + ) + return await inner.doStream(retry) + } + }, + }, + }) +} diff --git a/lib/output-token-limit.ts b/lib/output-token-limit.ts index 5aa1c4ed..28119eea 100644 --- a/lib/output-token-limit.ts +++ b/lib/output-token-limit.ts @@ -36,7 +36,7 @@ function usableLimit(value: number): number | null { } /** Message and body of an error that may be about the budget, or null. */ -function rejectionText(error: unknown): string | null { +export function rejectionText(error: unknown): string | null { const err = error as { message?: unknown responseBody?: unknown diff --git a/package-lock.json b/package-lock.json index e2106634..33bbe267 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,16 +9,17 @@ "version": "0.4.16", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/amazon-bedrock": "^4.0.1", - "@ai-sdk/anthropic": "^3.0.0", - "@ai-sdk/azure": "^3.0.0", - "@ai-sdk/deepseek": "^2.0.0", - "@ai-sdk/gateway": "^3.0.0", - "@ai-sdk/google": "^3.0.0", - "@ai-sdk/google-vertex": "^4.0.16", - "@ai-sdk/openai": "^3.0.0", - "@ai-sdk/react": "^3.0.1", - "@aihubmix/ai-sdk-provider": "^2.1.0", + "@ai-sdk/amazon-bedrock": "^4.0.191", + "@ai-sdk/anthropic": "^3.0.127", + "@ai-sdk/azure": "^3.0.133", + "@ai-sdk/deepseek": "^2.0.71", + "@ai-sdk/gateway": "^3.0.209", + "@ai-sdk/google": "^3.0.130", + "@ai-sdk/google-vertex": "^4.0.210", + "@ai-sdk/openai": "^3.0.124", + "@ai-sdk/openai-compatible": "^2.0.81", + "@ai-sdk/react": "^3.0.303", + "@aihubmix/ai-sdk-provider": "^2.2.1", "@aws-sdk/client-dynamodb": "^3.957.0", "@aws-sdk/credential-providers": "^3.943.0", "@extractus/article-extractor": "^8.0.18", @@ -28,7 +29,7 @@ "@langfuse/tracing": "^4.4.9", "@next/third-parties": "^16.0.6", "@opennextjs/cloudflare": "^1.17.1", - "@openrouter/ai-sdk-provider": "^2.0.0", + "@openrouter/ai-sdk-provider": "^2.10.0", "@opentelemetry/api": "^1.9.0", "@opentelemetry/exporter-trace-otlp-http": "^0.222.0", "@opentelemetry/sdk-trace-node": "^2.2.0", @@ -44,7 +45,7 @@ "@radix-ui/react-tooltip": "^1.1.8", "@radix-ui/react-use-controllable-state": "^1.2.2", "@xmldom/xmldom": "^0.9.8", - "ai": "^6.0.1", + "ai": "^6.0.300", "base-64": "^1.0.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", @@ -56,7 +57,7 @@ "nanoid": "^5.0.0", "negotiator": "^1.0.0", "next": "^16.0.7", - "ollama-ai-provider-v2": "^3.0.0", + "ollama-ai-provider-v2": "^3.6.0", "pako": "^2.1.0", "prism-react-renderer": "^2.4.1", "react": "^19.1.2", @@ -124,15 +125,15 @@ "license": "MIT" }, "node_modules/@ai-sdk/amazon-bedrock": { - "version": "4.0.113", - "resolved": "https://registry.npmjs.org/@ai-sdk/amazon-bedrock/-/amazon-bedrock-4.0.113.tgz", - "integrity": "sha512-qoeF2ghkYqHY4u68rasZopYsRuGnRwgqSfe6rhC/jM6W73Z7TU5v9QcfDYKt9dgp19YHY9EqiX3SXVC+uPGkRQ==", + "version": "4.0.191", + "resolved": "https://registry.npmjs.org/@ai-sdk/amazon-bedrock/-/amazon-bedrock-4.0.191.tgz", + "integrity": "sha512-7wgtq8On8oqmofGIGtYl8EOEbawnfiPbIbWpu2ksGNF065J8mbpE6DhkHNv0dpPbvnn4txL4sflUwUPwHw7/SQ==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/anthropic": "3.0.81", - "@ai-sdk/openai": "3.0.68", - "@ai-sdk/provider": "3.0.10", - "@ai-sdk/provider-utils": "4.0.27", + "@ai-sdk/anthropic": "3.0.127", + "@ai-sdk/openai": "3.0.124", + "@ai-sdk/provider": "3.0.18", + "@ai-sdk/provider-utils": "4.0.57", "@smithy/eventstream-codec": "^4.0.1", "@smithy/util-utf8": "^4.0.0", "aws4fetch": "^1.0.20" @@ -144,75 +145,14 @@ "zod": "^3.25.76 || ^4.1.8" } }, - "node_modules/@ai-sdk/amazon-bedrock/node_modules/@ai-sdk/anthropic": { - "version": "3.0.81", - "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-3.0.81.tgz", - "integrity": "sha512-B1JDd9Ugq9R5AgIaW3674lhGCMMYJcPUxnrZh8fzbGojgg4QvHFRv6eZahGQAUsmGHbcf74G9bdSBDLWQGY2GA==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/provider": "3.0.10", - "@ai-sdk/provider-utils": "4.0.27" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.25.76 || ^4.1.8" - } - }, - "node_modules/@ai-sdk/amazon-bedrock/node_modules/@ai-sdk/openai": { - "version": "3.0.68", - "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-3.0.68.tgz", - "integrity": "sha512-FCs/DPr4M95UyZ/ABHJmTmCEYRCka/4J0Bna0nsd78QCdGIS0X/zhn+fVzB7mZJo7464uOWYUjROx9PGNGOb0w==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/provider": "3.0.10", - "@ai-sdk/provider-utils": "4.0.27" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.25.76 || ^4.1.8" - } - }, - "node_modules/@ai-sdk/amazon-bedrock/node_modules/@ai-sdk/provider": { - "version": "3.0.10", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.10.tgz", - "integrity": "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw==", - "license": "Apache-2.0", - "dependencies": { - "json-schema": "^0.4.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@ai-sdk/amazon-bedrock/node_modules/@ai-sdk/provider-utils": { - "version": "4.0.27", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.27.tgz", - "integrity": "sha512-ubkAJ+xODouwtmN1tYlvTPphH1hPOBfZaEQe8U7skGvFAnIRs9PPpsq57bC2+Ky/MB4yzhd6YOsxTAx9sGpazw==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/provider": "3.0.10", - "@standard-schema/spec": "^1.1.0", - "eventsource-parser": "^3.0.8" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.25.76 || ^4.1.8" - } - }, "node_modules/@ai-sdk/anthropic": { - "version": "3.0.47", - "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-3.0.47.tgz", - "integrity": "sha512-E6Z3i/xvxGDxRskMMbuX9+xDK4l5LesrP2O7YQ0CcbAkYP25qTo/kYGf/AsJrLkNIY23HeO/kheUWtG1XZllDA==", + "version": "3.0.127", + "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-3.0.127.tgz", + "integrity": "sha512-Inff1DmPRVWi6QGr9YL2wiA4maILZuve9I8id/O4keQEoq6fJv6EfvkrSFw/nNvm2jiTcIUfDrpczeHTuhl9mg==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "3.0.8", - "@ai-sdk/provider-utils": "4.0.15" + "@ai-sdk/provider": "3.0.18", + "@ai-sdk/provider-utils": "4.0.57" }, "engines": { "node": ">=18" @@ -222,14 +162,15 @@ } }, "node_modules/@ai-sdk/azure": { - "version": "3.0.34", - "resolved": "https://registry.npmjs.org/@ai-sdk/azure/-/azure-3.0.34.tgz", - "integrity": "sha512-nnOFtgvZYOa6XIeAm18i56NX77Yu4Bd+Tnbt85LGUEqwJFR54kFTlR1nm3BAJCphHrmQteJd1P3QErtyoXig8A==", + "version": "3.0.133", + "resolved": "https://registry.npmjs.org/@ai-sdk/azure/-/azure-3.0.133.tgz", + "integrity": "sha512-iVCT1q6bRPWjzisu7Xy1bc1yWtegd3WSZzmi4+jkiFTKTxir703F5jH8mWPG0hPnqW/OjdkOrSzcQ063j7T/ng==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/openai": "3.0.33", - "@ai-sdk/provider": "3.0.8", - "@ai-sdk/provider-utils": "4.0.15" + "@ai-sdk/deepseek": "2.0.71", + "@ai-sdk/openai": "3.0.124", + "@ai-sdk/provider": "3.0.18", + "@ai-sdk/provider-utils": "4.0.57" }, "engines": { "node": ">=18" @@ -239,13 +180,13 @@ } }, "node_modules/@ai-sdk/deepseek": { - "version": "2.0.20", - "resolved": "https://registry.npmjs.org/@ai-sdk/deepseek/-/deepseek-2.0.20.tgz", - "integrity": "sha512-MAL04sDTOWUiBjAGWaVgyeE4bYRb9QpKYRlIeCTZFga6I8yQs50XakhWEssrmvVihdpHGkqpDtCHsFqCydsWLA==", + "version": "2.0.71", + "resolved": "https://registry.npmjs.org/@ai-sdk/deepseek/-/deepseek-2.0.71.tgz", + "integrity": "sha512-2uLtZBgONfzEP7tZOg23sBzi33n1/gcW1zLNNw96/PLt9vdqDYIkGQnKRd6F+/0Wwy77jVulU+gzpiudk3f0UQ==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "3.0.8", - "@ai-sdk/provider-utils": "4.0.15" + "@ai-sdk/provider": "3.0.18", + "@ai-sdk/provider-utils": "4.0.57" }, "engines": { "node": ">=18" @@ -255,14 +196,14 @@ } }, "node_modules/@ai-sdk/gateway": { - "version": "3.0.55", - "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.55.tgz", - "integrity": "sha512-7xMeTJnCjwRwXKVCiv4Ly4qzWvDuW3+W1WIV0X1EFu6W83d4mEhV9bFArto10MeTw40ewuDjrbrZd21mXKohkw==", + "version": "3.0.209", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.209.tgz", + "integrity": "sha512-CjCBzC35lRZ0LnUYLDfTB92p0Fr1Fu9Wrivc+LI0u3ox/DyZ0wPOHjt8DKBKHD/R4aZ86+JCFFuD9f8MFjAQFg==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "3.0.8", - "@ai-sdk/provider-utils": "4.0.15", - "@vercel/oidc": "3.1.0" + "@ai-sdk/provider": "3.0.18", + "@ai-sdk/provider-utils": "4.0.57", + "@vercel/oidc": "3.2.0" }, "engines": { "node": ">=18" @@ -272,13 +213,13 @@ } }, "node_modules/@ai-sdk/google": { - "version": "3.0.31", - "resolved": "https://registry.npmjs.org/@ai-sdk/google/-/google-3.0.31.tgz", - "integrity": "sha512-RVNz8WFSIRbXbYDBE6JvlE2escWPJimBCs22LzKEYH7DNfl/X7cHNa1LFho4PsY6Ib0JmbzB8s2+i0wHs/wNCg==", + "version": "3.0.130", + "resolved": "https://registry.npmjs.org/@ai-sdk/google/-/google-3.0.130.tgz", + "integrity": "sha512-DOhGfFT667LbopxMZU7kIiT4K8F2RNN8ziwmOxAfMbp3xTvAaxQ4ynPKNWBVXcCkGYDmzPb7QiazN1IRX8y3lw==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "3.0.8", - "@ai-sdk/provider-utils": "4.0.15" + "@ai-sdk/provider": "3.0.18", + "@ai-sdk/provider-utils": "4.0.57" }, "engines": { "node": ">=18" @@ -288,15 +229,16 @@ } }, "node_modules/@ai-sdk/google-vertex": { - "version": "4.0.63", - "resolved": "https://registry.npmjs.org/@ai-sdk/google-vertex/-/google-vertex-4.0.63.tgz", - "integrity": "sha512-/RNi6KSB4162DDYeXHUKQc5jLPmiJMkhTswLwbfPUEPyyFjbxpBWgeAk/vS/u8jxT4IPyYQ8cD/uyvJibmtmww==", + "version": "4.0.210", + "resolved": "https://registry.npmjs.org/@ai-sdk/google-vertex/-/google-vertex-4.0.210.tgz", + "integrity": "sha512-7Jj41iWsTFUcv6pzji5AlW2QaMJt/PEMU/zwqji2H7U52WvZE345kyo+nKRJMFtHgSVqFmzmMUIM9gyeaXKRtw==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/anthropic": "3.0.47", - "@ai-sdk/google": "3.0.31", - "@ai-sdk/provider": "3.0.8", - "@ai-sdk/provider-utils": "4.0.15", + "@ai-sdk/anthropic": "3.0.127", + "@ai-sdk/google": "3.0.130", + "@ai-sdk/openai-compatible": "2.0.81", + "@ai-sdk/provider": "3.0.18", + "@ai-sdk/provider-utils": "4.0.57", "google-auth-library": "^10.5.0" }, "engines": { @@ -307,13 +249,13 @@ } }, "node_modules/@ai-sdk/openai": { - "version": "3.0.33", - "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-3.0.33.tgz", - "integrity": "sha512-O/8SVKAiwFHkGAUfBnrLb7L2IjbpP9ySWbmOktOfa0KtzutZkmKNrJ5CtB5dj+lwuENbOuZeRsnsZdOjar7hig==", + "version": "3.0.124", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-3.0.124.tgz", + "integrity": "sha512-7DpRUPXzJ+S6XasEJBojzFCs04J3rNPs0FZFF3PEeix49IggZAwsMrqscUwB/+05+87r01SS/jeopQGOOL4AEQ==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "3.0.8", - "@ai-sdk/provider-utils": "4.0.15" + "@ai-sdk/provider": "3.0.18", + "@ai-sdk/provider-utils": "4.0.57" }, "engines": { "node": ">=18" @@ -323,42 +265,13 @@ } }, "node_modules/@ai-sdk/openai-compatible": { - "version": "2.0.48", - "resolved": "https://registry.npmjs.org/@ai-sdk/openai-compatible/-/openai-compatible-2.0.48.tgz", - "integrity": "sha512-z9MC6M4Oh/yUY/F/eszOtO8wc2nMz99XmZQKd2gWTtyIfe716xTfrKe3aYZKg20NZDtyjqPPKPSR+wqz7q1T7Q==", + "version": "2.0.81", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai-compatible/-/openai-compatible-2.0.81.tgz", + "integrity": "sha512-L14Jd0lAFKNM42l9TxlpwkNiyMlSvSvGySLc9GN4FB0qOo4/WmYOeLyn/CPhghIr0K4O6vH1y0nChxGljq2ccg==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "3.0.10", - "@ai-sdk/provider-utils": "4.0.27" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.25.76 || ^4.1.8" - } - }, - "node_modules/@ai-sdk/openai-compatible/node_modules/@ai-sdk/provider": { - "version": "3.0.10", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.10.tgz", - "integrity": "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw==", - "license": "Apache-2.0", - "dependencies": { - "json-schema": "^0.4.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@ai-sdk/openai-compatible/node_modules/@ai-sdk/provider-utils": { - "version": "4.0.27", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.27.tgz", - "integrity": "sha512-ubkAJ+xODouwtmN1tYlvTPphH1hPOBfZaEQe8U7skGvFAnIRs9PPpsq57bC2+Ky/MB4yzhd6YOsxTAx9sGpazw==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/provider": "3.0.10", - "@standard-schema/spec": "^1.1.0", - "eventsource-parser": "^3.0.8" + "@ai-sdk/provider": "3.0.18", + "@ai-sdk/provider-utils": "4.0.57" }, "engines": { "node": ">=18" @@ -368,9 +281,9 @@ } }, "node_modules/@ai-sdk/provider": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.8.tgz", - "integrity": "sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ==", + "version": "3.0.18", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.18.tgz", + "integrity": "sha512-IpefF5ssZVOZCD4Wu7zxt6PwZoGT2HLVjxpJ10Bjx6EAmRfLYJPjyrfxb0RYDlwvXnDb9nkQ7Do/oJVDuRFzew==", "license": "Apache-2.0", "dependencies": { "json-schema": "^0.4.0" @@ -380,30 +293,40 @@ } }, "node_modules/@ai-sdk/provider-utils": { - "version": "4.0.15", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.15.tgz", - "integrity": "sha512-8XiKWbemmCbvNN0CLR9u3PQiet4gtEVIrX4zzLxnCj06AwsEDJwJVBbKrEI4t6qE8XRSIvU2irka0dcpziKW6w==", + "version": "4.0.57", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.57.tgz", + "integrity": "sha512-89a7sPvZqXsP3PigcAQLWntu/duIVEcN/RyChAB+j6p3C7NwW767sVttvL8r0+BetTg63uAXQoYHCZZueQKBSw==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "3.0.8", + "@ai-sdk/provider": "3.0.18", "@standard-schema/spec": "^1.1.0", - "eventsource-parser": "^3.0.6" + "eventsource-parser": "^3.0.8", + "undici": "^6.28.0" }, "engines": { - "node": ">=18" + "node": ">=18.17" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, + "node_modules/@ai-sdk/provider-utils/node_modules/undici": { + "version": "6.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.29.0.tgz", + "integrity": "sha512-R+RODBqp6i2pPflGdq+xIOUkl+RNfGgHwoinecKu/JCuf2uO06cOKoDbI2P7Dn6KcswdKwrczbU6IYJ6K8X+wg==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, "node_modules/@ai-sdk/react": { - "version": "3.0.102", - "resolved": "https://registry.npmjs.org/@ai-sdk/react/-/react-3.0.102.tgz", - "integrity": "sha512-WPSYJxk/HM3SWhhxE+WrLhIMbaRLpDHWWYzUlrptXowwEoyliYBYZAkzip/gV8hybT2NyrWnsUh5KRO5iBdsQA==", + "version": "3.0.303", + "resolved": "https://registry.npmjs.org/@ai-sdk/react/-/react-3.0.303.tgz", + "integrity": "sha512-PeTBn25x8QxfIIpdUDOpWHfiIZxMRJCw04PhWNHI9hl+gtu+zGwAxWU+sy3PH/f/oEBBAp7eDDUDDjiuqwcw6g==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider-utils": "4.0.15", - "ai": "6.0.100", + "@ai-sdk/provider-utils": "4.0.57", + "ai": "6.0.300", "swr": "^2.2.5", "throttleit": "2.1.0" }, @@ -415,9 +338,9 @@ } }, "node_modules/@aihubmix/ai-sdk-provider": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@aihubmix/ai-sdk-provider/-/ai-sdk-provider-2.1.0.tgz", - "integrity": "sha512-AqK10PV5B4zWFBav5PRUhrWGYTHjC0s6cIbd4v9hBo6D9SqKv5o41B+dkl6IeIPrnZrMGLEE9Mn5R+VaEkhbdg==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@aihubmix/ai-sdk-provider/-/ai-sdk-provider-2.2.1.tgz", + "integrity": "sha512-VUkSYFbrigs5ZF44LRSHEI/q1fZyLkExWvrgBG4w0ysGEE/XXt7fi9NgMG9UH1PQVBgnc/PP03CUhTM1nhmGxQ==", "license": "Apache-2.0", "dependencies": { "@ai-sdk/anthropic": "^3.0.0", @@ -5891,9 +5814,9 @@ } }, "node_modules/@openrouter/ai-sdk-provider": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/@openrouter/ai-sdk-provider/-/ai-sdk-provider-2.8.1.tgz", - "integrity": "sha512-Y6j3yivgoEUf/kutD/k5GX/mzZfioRFoSx0gbQ+mIOzMaH/vJv1rCkztiuvlLw5xRYQil7oxHUZvmSfXqOx1NQ==", + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@openrouter/ai-sdk-provider/-/ai-sdk-provider-2.10.0.tgz", + "integrity": "sha512-FMsAEjLUt5pWuRE2LDC/LCvVrFjLlrEzUITH5+5SZtfq7KZ2wrOHjQVxzz92sju8S9ltpzW87CLW8/b0oBXVCw==", "license": "Apache-2.0", "engines": { "node": ">=18" @@ -9327,9 +9250,9 @@ ] }, "node_modules/@vercel/oidc": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.1.0.tgz", - "integrity": "sha512-Fw28YZpRnA3cAHHDlkt7xQHiJ0fcL+NRcIqsocZQUSmbzeIKRpwttJjik5ZGanXP+vlA4SbTg+AbA3bP363l+w==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz", + "integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==", "license": "Apache-2.0", "engines": { "node": ">= 20" @@ -9595,15 +9518,15 @@ } }, "node_modules/ai": { - "version": "6.0.100", - "resolved": "https://registry.npmjs.org/ai/-/ai-6.0.100.tgz", - "integrity": "sha512-BIxhG7M7wvcWCF+IEnZi7WpkRLOM3jR2vJ0mMuohl2UB2i1R/ZUa1cHFel1xI8nWvyUpOoQXKqsM0BAH50EYSQ==", + "version": "6.0.300", + "resolved": "https://registry.npmjs.org/ai/-/ai-6.0.300.tgz", + "integrity": "sha512-ZBT30eQTy6exP+EWJJxm3DMb34m9td8STy+hfnvKdjhyEQoM8vQN5YfFLTRVVTCONHoM+y5K+Tudb6PTFrpikA==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/gateway": "3.0.55", - "@ai-sdk/provider": "3.0.8", - "@ai-sdk/provider-utils": "4.0.15", - "@opentelemetry/api": "1.9.0" + "@ai-sdk/gateway": "3.0.209", + "@ai-sdk/provider": "3.0.18", + "@ai-sdk/provider-utils": "4.0.57", + "@opentelemetry/api": "^1.9.0" }, "engines": { "node": ">=18" @@ -9612,15 +9535,6 @@ "zod": "^3.25.76 || ^4.1.8" } }, - "node_modules/ai/node_modules/@opentelemetry/api": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", - "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", - "license": "Apache-2.0", - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/ajv": { "version": "6.14.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", @@ -19590,39 +19504,26 @@ "license": "MIT" }, "node_modules/ollama-ai-provider-v2": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/ollama-ai-provider-v2/-/ollama-ai-provider-v2-3.5.0.tgz", - "integrity": "sha512-+s/aYIYa91z2Vk3AkGAz3BaPAQ0flS2eFZD3BN2mD/N6W6YQbcookyu6pc2cbc8SP5VGpNB857WJ0eHDjKXsXw==", + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/ollama-ai-provider-v2/-/ollama-ai-provider-v2-3.6.0.tgz", + "integrity": "sha512-1Om3FVJYhBwkAr5kQ+BX1s/tdVdtVdoFQWrX4PBQHDHPISyGt24CjhtggEjUYpy5ait0YeVfZwEpIYjgD8Ih7Q==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "^3.0.8", - "@ai-sdk/provider-utils": "^4.0.19" + "@ai-sdk/provider": "^3.0.10", + "@ai-sdk/provider-utils": "^4.0.27" }, "engines": { - "node": ">=18" + "node": ">=20" + }, + "funding": { + "type": "Buy Me a Coffee", + "url": "https://buymeacoffee.com/nordwestt" }, "peerDependencies": { "ai": "^5.0.0 || ^6.0.0", "zod": "^4.0.16" } }, - "node_modules/ollama-ai-provider-v2/node_modules/@ai-sdk/provider-utils": { - "version": "4.0.23", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.23.tgz", - "integrity": "sha512-z8GlDaCmRSDlqkMF2f4/RFgWxdarvIbyuk+m6WXT1LYgsnGiXRJGTD2Z1+SDl3LqtFuRtGX1aghYvQLoHL/9pg==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/provider": "3.0.8", - "@standard-schema/spec": "^1.1.0", - "eventsource-parser": "^3.0.6" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.25.76 || ^4.1.8" - } - }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", diff --git a/package.json b/package.json index 205055ab..15da0185 100644 --- a/package.json +++ b/package.json @@ -31,16 +31,17 @@ "test:e2e": "playwright test" }, "dependencies": { - "@ai-sdk/amazon-bedrock": "^4.0.1", - "@ai-sdk/anthropic": "^3.0.0", - "@ai-sdk/azure": "^3.0.0", - "@ai-sdk/deepseek": "^2.0.0", - "@ai-sdk/gateway": "^3.0.0", - "@ai-sdk/google": "^3.0.0", - "@ai-sdk/google-vertex": "^4.0.16", - "@ai-sdk/openai": "^3.0.0", - "@ai-sdk/react": "^3.0.1", - "@aihubmix/ai-sdk-provider": "^2.1.0", + "@ai-sdk/amazon-bedrock": "^4.0.191", + "@ai-sdk/anthropic": "^3.0.127", + "@ai-sdk/azure": "^3.0.133", + "@ai-sdk/deepseek": "^2.0.71", + "@ai-sdk/gateway": "^3.0.209", + "@ai-sdk/google": "^3.0.130", + "@ai-sdk/google-vertex": "^4.0.210", + "@ai-sdk/openai": "^3.0.124", + "@ai-sdk/openai-compatible": "^2.0.81", + "@ai-sdk/react": "^3.0.303", + "@aihubmix/ai-sdk-provider": "^2.2.1", "@aws-sdk/client-dynamodb": "^3.957.0", "@aws-sdk/credential-providers": "^3.943.0", "@extractus/article-extractor": "^8.0.18", @@ -50,7 +51,7 @@ "@langfuse/tracing": "^4.4.9", "@next/third-parties": "^16.0.6", "@opennextjs/cloudflare": "^1.17.1", - "@openrouter/ai-sdk-provider": "^2.0.0", + "@openrouter/ai-sdk-provider": "^2.10.0", "@opentelemetry/api": "^1.9.0", "@opentelemetry/exporter-trace-otlp-http": "^0.222.0", "@opentelemetry/sdk-trace-node": "^2.2.0", @@ -66,7 +67,7 @@ "@radix-ui/react-tooltip": "^1.1.8", "@radix-ui/react-use-controllable-state": "^1.2.2", "@xmldom/xmldom": "^0.9.8", - "ai": "^6.0.1", + "ai": "^6.0.300", "base-64": "^1.0.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", @@ -78,7 +79,7 @@ "nanoid": "^5.0.0", "negotiator": "^1.0.0", "next": "^16.0.7", - "ollama-ai-provider-v2": "^3.0.0", + "ollama-ai-provider-v2": "^3.6.0", "pako": "^2.1.0", "prism-react-renderer": "^2.4.1", "react": "^19.1.2", diff --git a/tests/unit/ai-providers-cache.test.ts b/tests/unit/ai-providers-cache.test.ts new file mode 100644 index 00000000..c67863da --- /dev/null +++ b/tests/unit/ai-providers-cache.test.ts @@ -0,0 +1,85 @@ +// @vitest-environment node +import { generateText, type ModelMessage } from "ai" +import { afterEach, describe, expect, it, vi } from "vitest" +import { CACHE_POINT, getAIModel } from "@/lib/ai-providers" + +const ENV = ["ANTHROPIC_API_KEY", "OPENROUTER_API_KEY"] +const saved = Object.fromEntries(ENV.map((k) => [k, process.env[k]])) + +afterEach(() => { + for (const k of ENV) { + if (saved[k] === undefined) delete process.env[k] + else process.env[k] = saved[k] + } + vi.unstubAllGlobals() +}) + +// The chat route marks its system messages like this +const messages: ModelMessage[] = [ + { role: "system", content: "Instructions", providerOptions: CACHE_POINT }, + { role: "user", content: "hi" }, +] + +/** Send the messages through a real provider and return the request body */ +async function requestBody( + provider: "anthropic" | "openrouter", + reply: unknown, +) { + let body: any + vi.stubGlobal( + "fetch", + vi.fn(async (_url: string, init: RequestInit) => { + body = JSON.parse(init.body as string) + return new Response(JSON.stringify(reply), { + headers: { "content-type": "application/json" }, + }) + }), + ) + const { model } = getAIModel({ + provider, + modelId: + provider === "anthropic" + ? "claude-sonnet-4-5" + : "anthropic/claude-sonnet-4.5", + }) + await generateText({ model, messages, maxRetries: 0 }).catch(() => {}) + return body +} + +describe("prompt cache breakpoints", () => { + it("reach the Anthropic API", async () => { + process.env.ANTHROPIC_API_KEY = "test-key" + const body = await requestBody("anthropic", { + id: "msg_1", + type: "message", + role: "assistant", + content: [{ type: "text", text: "ok" }], + stop_reason: "end_turn", + usage: { input_tokens: 1, output_tokens: 1 }, + }) + expect(body.system).toEqual([ + { + type: "text", + text: "Instructions", + cache_control: { type: "ephemeral" }, + }, + ]) + }) + + it("reach OpenRouter", async () => { + process.env.OPENROUTER_API_KEY = "test-key" + const body = await requestBody("openrouter", { + id: "gen-1", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + }) + expect(JSON.stringify(body.messages[0])).toContain( + '"cache_control":{"type":"ephemeral"}', + ) + }) +}) diff --git a/tests/unit/ai-providers-google.test.ts b/tests/unit/ai-providers-google.test.ts new file mode 100644 index 00000000..77b53637 --- /dev/null +++ b/tests/unit/ai-providers-google.test.ts @@ -0,0 +1,62 @@ +// @vitest-environment node +import { generateText } from "ai" +import { afterEach, describe, expect, it, vi } from "vitest" +import { getAIModel } from "@/lib/ai-providers" + +const ENV = ["GOOGLE_GENERATIVE_AI_API_KEY", "GOOGLE_TOP_K", "GOOGLE_TOP_P"] +const saved = Object.fromEntries(ENV.map((k) => [k, process.env[k]])) + +afterEach(() => { + for (const k of ENV) { + if (saved[k] === undefined) delete process.env[k] + else process.env[k] = saved[k] + } + vi.unstubAllGlobals() +}) + +/** Send one request through the real Google provider and return its body */ +async function googleRequestBody() { + let body: any + vi.stubGlobal( + "fetch", + vi.fn(async (_url: string, init: RequestInit) => { + body = JSON.parse(init.body as string) + return new Response( + JSON.stringify({ + candidates: [ + { + content: { role: "model", parts: [{ text: "ok" }] }, + finishReason: "STOP", + }, + ], + }), + { headers: { "content-type": "application/json" } }, + ) + }), + ) + const { model } = getAIModel({ + provider: "google", + modelId: "gemini-2.5-flash", + }) + await generateText({ model, prompt: "hi", maxRetries: 0 }) + return body +} + +describe("Google sampling settings", () => { + it("sends GOOGLE_TOP_K and GOOGLE_TOP_P in the generation config", async () => { + process.env.GOOGLE_GENERATIVE_AI_API_KEY = "test-key" + process.env.GOOGLE_TOP_K = "40" + process.env.GOOGLE_TOP_P = "0.9" + const body = await googleRequestBody() + expect(body.generationConfig).toMatchObject({ topK: 40, topP: 0.9 }) + }) + + it("sends neither when they are not set", async () => { + process.env.GOOGLE_GENERATIVE_AI_API_KEY = "test-key" + delete process.env.GOOGLE_TOP_K + delete process.env.GOOGLE_TOP_P + const body = await googleRequestBody() + expect(body.generationConfig?.topK).toBeUndefined() + expect(body.generationConfig?.topP).toBeUndefined() + }) +}) diff --git a/tests/unit/deprecated-params.test.ts b/tests/unit/deprecated-params.test.ts new file mode 100644 index 00000000..13fff590 --- /dev/null +++ b/tests/unit/deprecated-params.test.ts @@ -0,0 +1,112 @@ +// @vitest-environment node +import { simulateReadableStream, streamText } from "ai" +import { MockLanguageModelV3 } from "ai/test" +import { describe, expect, it } from "vitest" +import { + withDeprecatedParamsFallback, + withoutDeprecatedParams, +} from "@/lib/deprecated-params" + +// The error texts Claude 4.7 and later return (the Anthropic API and Bedrock) +const rejection = (message: string) => ({ + statusCode: 400, + message, + responseBody: JSON.stringify({ error: { message } }), +}) +const TEMPERATURE = rejection("`temperature` is deprecated for this model.") +const THINKING = rejection( + '"thinking.type.enabled" is not supported for this model. Use "thinking.type.adaptive" and "output_config.effort" to control thinking behavior.', +) + +describe("withoutDeprecatedParams", () => { + const params = { + temperature: 0.2, + topP: 0.9, + maxOutputTokens: 1000, + providerOptions: { + anthropic: { + thinking: { type: "enabled", budgetTokens: 4000 }, + cacheControl: { type: "ephemeral" }, + }, + }, + } + + it("drops sampling settings and the thinking budget", () => { + for (const error of [TEMPERATURE, THINKING]) { + expect(withoutDeprecatedParams(error, params)).toEqual({ + maxOutputTokens: 1000, + providerOptions: { + anthropic: { cacheControl: { type: "ephemeral" } }, + }, + }) + } + }) + + it("drops a Bedrock thinking budget", () => { + const bedrock = { + providerOptions: { + bedrock: { + reasoningConfig: { type: "enabled", budgetTokens: 4000 }, + }, + }, + } + expect(withoutDeprecatedParams(THINKING, bedrock)).toEqual({ + providerOptions: { bedrock: {} }, + }) + }) + + it("leaves other errors and requests with nothing to drop alone", () => { + expect(withoutDeprecatedParams(rejection("bad key"), params)).toBeNull() + expect( + withoutDeprecatedParams( + { ...TEMPERATURE, statusCode: 401 }, + params, + ), + ).toBeNull() + const nothingToDrop = { + providerOptions: { + anthropic: { cacheControl: { type: "ephemeral" } }, + }, + } + expect(withoutDeprecatedParams(TEMPERATURE, nothingToDrop)).toBeNull() + }) +}) + +describe("withDeprecatedParamsFallback", () => { + it("retries the stream once without the rejected settings", async () => { + const calls: any[] = [] + const model = new MockLanguageModelV3({ + // The test stream only has the parts this check needs + doStream: (async (options: any) => { + calls.push(options) + if (options.temperature !== undefined) throw TEMPERATURE + return { + stream: simulateReadableStream({ + chunks: [ + { type: "text-start", id: "t" }, + { type: "text-delta", id: "t", delta: "ok" }, + { type: "text-end", id: "t" }, + { + type: "finish", + finishReason: { unified: "stop", raw: "stop" }, + usage: { + inputTokens: { total: 1 }, + outputTokens: { total: 1 }, + }, + }, + ], + }), + } + }) as any, + }) + const result = streamText({ + model: withDeprecatedParamsFallback(model as any), + prompt: "hi", + temperature: 0.2, + maxRetries: 0, + }) + expect(await result.text).toBe("ok") + expect(calls).toHaveLength(2) + expect(calls[1].temperature).toBeUndefined() + }) +})