mirror of
https://github.com/DayuanJiang/next-ai-draw-io.git
synced 2026-09-02 01:20:23 +08:00
8fb9ef20bd
A reasoning model spends the output budget in order: thinking first, then prose, then the tool call. With 16000 the thinking alone can consume all of it, so the turn ends with finishReason "length" before display_diagram is ever called. The canvas stays empty and nothing surfaces in the UI, because no tool call means no tool error, and the client never reads finishReason. Measured on openrouter deepseek/deepseek-v4-flash, the model from the report: - max_tokens=800 with reasoning on returns reasoning_tokens=800, empty content, finish_reason length. So reasoning is billed against this budget, not exempt. - refining an existing diagram (19k chars of XML in the input) produced 49142 chars of reasoning, zero tool calls, finishReason "length" at 16000 - the same request at 40000 finished and called edit_diagram with 12 operations 64000 cannot just be sent to every model: bedrock claude-3-haiku caps at 4096, nova-lite at 10000, and the openrouter deepseek-r1 endpoint counts input and output against one 64000 ceiling. All three name the real limit in the 400, so parse it and retry once. Verified: nova-lite logs "64000 rejected, retrying with 10000" and then completes its tool call. Also expose the budget in Settings. It is sent as a header rather than read from env only, so desktop users can raise it themselves without an env file. vercel.json goes back to the 300s it had before #238 traded it for $2-4/month. That is now Vercel's own default, and billing pauses while the function waits on the model, so the saving that motivated 120s no longer applies. edgeone.json is left alone: its 120 may be that platform's actual ceiling.
111 lines
4.1 KiB
TypeScript
111 lines
4.1 KiB
TypeScript
import { wrapLanguageModel } from "ai"
|
|
|
|
type WrappedModel = ReturnType<typeof wrapLanguageModel>
|
|
|
|
/**
|
|
* Default output budget for a chat turn.
|
|
*
|
|
* This has to cover thinking + prose + the tool call, because reasoning models
|
|
* spend it in that order. Measured on deepseek-v4-flash: refining an existing
|
|
* diagram burned 16000 tokens on thinking alone and the request ended with
|
|
* finishReason "length" before display_diagram was ever called (issue #924).
|
|
* 64000 leaves room for the plan and the XML in one turn.
|
|
*/
|
|
export const DEFAULT_MAX_OUTPUT_TOKENS = 64000
|
|
|
|
/** Ceiling for the user-supplied override, to catch typos like an extra zero. */
|
|
export const MAX_OUTPUT_TOKENS_LIMIT = 200000
|
|
|
|
/**
|
|
* A budget this large exceeds what some models accept. Providers reject it with a
|
|
* 400 that names the real limit, so we parse the number out and retry once
|
|
* instead of failing the turn.
|
|
*
|
|
* Formats seen in the wild:
|
|
* - Bedrock: "The maximum tokens you requested exceeds the model limit of 4096."
|
|
* - OpenRouter: "This endpoint's maximum context length is 64000 tokens. However,
|
|
* you requested about 64025 tokens (25 of text input, 64000 in the output)."
|
|
* Note this one is an input+output ceiling, so the input has to be subtracted.
|
|
* - Anthropic: "max_tokens: 200000 > 64000, which is the maximum allowed..."
|
|
* - OpenAI: "This model supports at most 16384 completion tokens"
|
|
*/
|
|
export function parseOutputTokenLimit(error: unknown): number | null {
|
|
const err = error as { message?: unknown; responseBody?: unknown }
|
|
const text = [
|
|
typeof err?.message === "string" ? err.message : "",
|
|
typeof err?.responseBody === "string" ? err.responseBody : "",
|
|
].join(" ")
|
|
|
|
if (!text) return null
|
|
|
|
// Combined input+output ceiling: subtract the input the provider counted,
|
|
// plus a small margin because its estimate is approximate.
|
|
const context = text.match(/maximum context length is (\d+)/i)
|
|
if (context) {
|
|
const input = text.match(/(\d+) of text input/i)
|
|
const budget =
|
|
Number(context[1]) - (input ? Number(input[1]) : 0) - 1024
|
|
return budget > 0 ? budget : null
|
|
}
|
|
|
|
const output =
|
|
text.match(/model limit of (\d+)/i) ||
|
|
text.match(/> (\d+), which is the maximum/i) ||
|
|
text.match(/at most (\d+) completion tokens/i) ||
|
|
text.match(/lower than (\d+)/i)
|
|
|
|
return output ? Number(output[1]) : null
|
|
}
|
|
|
|
/**
|
|
* Retry the stream once with a smaller budget when the provider rejects the
|
|
* requested one. Without this, raising the default breaks every model whose
|
|
* ceiling is below it (measured: bedrock claude-3-haiku 4096, nova-lite 10000,
|
|
* openrouter deepseek-r1 64000 shared with the input).
|
|
*/
|
|
export function withOutputTokenLimitFallback(
|
|
model: WrappedModel,
|
|
): WrappedModel {
|
|
return wrapLanguageModel({
|
|
model,
|
|
middleware: {
|
|
specificationVersion: "v3",
|
|
async wrapStream({ doStream, params, model: inner }) {
|
|
try {
|
|
return await doStream()
|
|
} catch (error) {
|
|
const limit = parseOutputTokenLimit(error)
|
|
const requested = params.maxOutputTokens
|
|
|
|
if (!limit || !requested || limit >= requested) throw error
|
|
|
|
console.warn(
|
|
`[maxOutputTokens] ${requested} rejected, retrying with ${limit}`,
|
|
)
|
|
return await inner.doStream({
|
|
...params,
|
|
maxOutputTokens: limit,
|
|
})
|
|
}
|
|
},
|
|
},
|
|
})
|
|
}
|
|
|
|
/**
|
|
* Resolve the output budget: user setting (sent as a header so it works in the
|
|
* desktop app too), then server env, then the default.
|
|
*/
|
|
export function resolveMaxOutputTokens(headerValue: string | null): number {
|
|
const fromHeader = Number(headerValue)
|
|
if (
|
|
Number.isInteger(fromHeader) &&
|
|
fromHeader > 0 &&
|
|
fromHeader <= MAX_OUTPUT_TOKENS_LIMIT
|
|
) {
|
|
return fromHeader
|
|
}
|
|
|
|
return Number(process.env.MAX_OUTPUT_TOKENS) || DEFAULT_MAX_OUTPUT_TOKENS
|
|
}
|