feat(routing): move sticky-key retries into routing policy with lazy attempts

Replace the provider/endpoint max_retries fields as the source of same-key
retries with a routing policy setting, sticky_key_attempts (default 2). Only
the first-ranked candidate is retried on the same key; every failover
candidate gets a single attempt so failover keeps advancing instead of
retrying each fallback key.

Materialize exactly one attempt per candidate and derive same-key retries in
the attempt loop after a candidate-scoped failure, so the retry budget no
longer inflates up-front materialization and needs no upper bound. The budget
travels in the report context; retries reuse the plan with a fresh candidate
id and incremented retry index. Pool groups only retry their first key within
the retry-index stride.

Expose the setting in the routing profile editor and the set_scheduling rule
action, and drop the max_retries input from the provider form.
This commit is contained in:
elky
2026-09-02 20:48:40 +08:00
parent 415b2da81b
commit 7323d41fbe
40 changed files with 851 additions and 570 deletions
@@ -155,17 +155,6 @@
</SelectContent>
</Select>
</div>
<div class="space-y-1.5">
<Label>{{ legacyT('最大重试次数') }}</Label>
<Input
:model-value="form.max_retries ?? ''"
type="number"
min="0"
max="999"
:placeholder="legacyT('默认 2')"
@update:model-value="(v) => form.max_retries = parseNumberInput(v)"
/>
</div>
</div>
<!-- 超时配置 -->
@@ -12,6 +12,7 @@ import {
getModelScheduling,
modelSchedulingRuleId,
normalizeRoutingGroupConfig,
normalizeStickyKeyAttempts,
parseAllowedModelsInput,
removePerModelRoutingConfig,
resolveModelKeyPriorityOverride,
@@ -77,6 +78,19 @@ describe('routingPolicy', () => {
expect(policy.key_priority_overrides).toEqual({})
})
it('defaults sticky key attempts to 2 and normalizes invalid values', () => {
expect(createEmptyRoutingGroupConfig().default_policy.sticky_key_attempts).toBe(2)
expect(normalizeRoutingGroupConfig({}).default_policy.sticky_key_attempts).toBe(2)
expect(normalizeRoutingGroupConfig({
default_policy: { priority_mode: 'provider', scheduling_mode: 'cache_affinity', keep_priority_on_conversion: false, sticky_key_attempts: 3 },
}).default_policy.sticky_key_attempts).toBe(3)
expect(normalizeStickyKeyAttempts('5')).toBe(5)
expect(normalizeStickyKeyAttempts(-1)).toBe(2)
expect(normalizeStickyKeyAttempts('abc')).toBe(2)
expect(normalizeStickyKeyAttempts(500)).toBe(99)
expect(getModelScheduling(createEmptyRoutingGroupConfig(), 'gpt-5').sticky_key_attempts).toBe(2)
})
it('keeps key priority overrides independent per api format', () => {
let config = setModelKeyPriorityOverridesForFormat(
createEmptyRoutingGroupConfig(),
@@ -3,10 +3,15 @@ export type RoutingSchedulingMode = 'fixed_order' | 'cache_affinity' | 'load_bal
export type RoutingRulePhase = 'client_request' | 'provider_request'
export type RoutingSortingScope = 'unified' | 'per_model'
/** 首个候选(粘性 Key)的总尝试次数默认值:失败后同 Key 重试 1 次 */
export const DEFAULT_STICKY_KEY_ATTEMPTS = 2
export interface RoutingDefaultPolicy {
priority_mode: RoutingPriorityMode
scheduling_mode: RoutingSchedulingMode
keep_priority_on_conversion: boolean
/** 首个候选的总尝试次数;后续候选始终只尝试 1 次。0 或 1 表示不重试 */
sticky_key_attempts: number
}
export interface RoutingPoolSchedulingPreset {
@@ -51,6 +56,7 @@ export interface RoutingSetSchedulingAction {
type: 'set_scheduling'
priority_mode: RoutingPriorityMode
scheduling_mode: RoutingSchedulingMode
sticky_key_attempts?: number
}
export interface RoutingGroupConfig {
@@ -70,12 +76,19 @@ export function createEmptyRoutingGroupConfig(): RoutingGroupConfig {
priority_mode: 'provider',
scheduling_mode: 'cache_affinity',
keep_priority_on_conversion: false,
sticky_key_attempts: DEFAULT_STICKY_KEY_ATTEMPTS,
},
model_policies: [],
rules: [],
}
}
export function normalizeStickyKeyAttempts(value: unknown): number {
const parsed = Math.trunc(Number(value))
if (!Number.isFinite(parsed) || parsed < 0) return DEFAULT_STICKY_KEY_ATTEMPTS
return Math.min(parsed, 99)
}
export function createEmptyModelPolicy(model = ''): RoutingModelPolicy {
return {
model,
@@ -97,6 +110,9 @@ export function normalizeRoutingGroupConfig(value: Partial<RoutingGroupConfig> |
default_policy: {
...base.default_policy,
...(value?.default_policy ?? {}),
sticky_key_attempts: normalizeStickyKeyAttempts(
value?.default_policy?.sticky_key_attempts ?? DEFAULT_STICKY_KEY_ATTEMPTS,
),
},
model_policies: Array.isArray(value?.model_policies)
? value.model_policies.map(policy => ({
@@ -411,6 +427,7 @@ export function getModelScheduling(
priority_mode: action?.priority_mode ?? normalized.default_policy.priority_mode,
scheduling_mode: action?.scheduling_mode ?? normalized.default_policy.scheduling_mode,
keep_priority_on_conversion: normalized.default_policy.keep_priority_on_conversion,
sticky_key_attempts: action?.sticky_key_attempts ?? normalized.default_policy.sticky_key_attempts,
}
}