fix(pool): honor quota exhaustion toggle and default ignore to off

This commit is contained in:
elky
2026-10-10 10:04:41 +08:00
parent 29ba6c8cb1
commit 6bafef672f
8 changed files with 99 additions and 37 deletions
@@ -326,7 +326,9 @@ fn active_probe_member_is_unschedulable_for_request(
return true; return true;
} }
key_context.is_some_and(|context| { key_context.is_some_and(|context| {
context.account_blocked || context.quota_exhausted || context.quota_hard_blocked context.account_blocked
|| (pool_config.skip_exhausted_accounts && context.quota_exhausted)
|| context.quota_hard_blocked
}) })
} }
@@ -1529,12 +1531,14 @@ async fn read_pool_catalog_key_contexts_by_id(
provider_type, provider_type,
provider_model_name, provider_model_name,
); );
context.quota_exhausted |= reserve_minimum_quota_key_ids.contains(&key.id) let reserve_reached = reserve_minimum_quota_key_ids.contains(&key.id)
&& admin_provider_pool_pure::admin_pool_key_minimum_quota_reached( && admin_provider_pool_pure::admin_pool_key_minimum_quota_reached(
&key, &key,
provider_type, provider_type,
provider_model_name, provider_model_name,
); );
context.quota_exhausted |= reserve_reached;
context.quota_hard_blocked |= reserve_reached;
(key.id.clone(), context) (key.id.clone(), context)
}) })
.collect::<BTreeMap<_, _>>(); .collect::<BTreeMap<_, _>>();
@@ -2303,6 +2307,31 @@ mod tests {
); );
} }
#[test]
fn pool_scheduler_allows_quota_exhausted_key_when_ignored() {
let exhausted = sample_eligible_candidate(
"provider-pool",
"endpoint-1",
"key-exhausted",
10,
Some(json!({ "pool_advanced": { "ignore_exhausted_accounts": true } })),
);
let contexts = BTreeMap::from([(
"key-exhausted".to_string(),
PoolCatalogKeyContext {
quota_exhausted: true,
..PoolCatalogKeyContext::default()
},
)]);
let (scheduled, skipped) = apply_local_execution_pool_scheduler_with_runtime_map(
vec![exhausted],
&BTreeMap::new(),
&contexts,
);
assert_eq!(scheduled.len(), 1);
assert!(skipped.is_empty());
}
#[test] #[test]
fn pool_scheduler_skips_quota_exhausted_key_when_flag_is_false() { fn pool_scheduler_skips_quota_exhausted_key_when_flag_is_false() {
let ready = sample_eligible_candidate( let ready = sample_eligible_candidate(
@@ -410,7 +410,7 @@ pub(crate) fn admin_provider_pool_config_from_config_value(
}], }],
unschedulable_rules: Vec::new(), unschedulable_rules: Vec::new(),
lru_enabled: false, lru_enabled: false,
skip_exhausted_accounts: false, skip_exhausted_accounts: true,
reserve_minimum_quota: false, reserve_minimum_quota: false,
sticky_session_ttl_seconds: 3600, sticky_session_ttl_seconds: 3600,
latency_window_seconds: 3600, latency_window_seconds: 3600,
@@ -444,9 +444,10 @@ pub(crate) fn admin_provider_pool_config_from_config_value(
scheduling_presets, scheduling_presets,
unschedulable_rules, unschedulable_rules,
skip_exhausted_accounts: pool_advanced skip_exhausted_accounts: pool_advanced
.get("skip_exhausted_accounts") .get("ignore_exhausted_accounts")
.and_then(Value::as_bool) .and_then(Value::as_bool)
.unwrap_or(false), .map(|ignore| !ignore)
.unwrap_or(true),
reserve_minimum_quota: pool_advanced reserve_minimum_quota: pool_advanced
.get("reserve_minimum_quota") .get("reserve_minimum_quota")
.and_then(Value::as_bool) .and_then(Value::as_bool)
@@ -574,11 +575,25 @@ mod tests {
} }
#[test] #[test]
fn defaults_skip_exhausted_accounts_to_false() { fn parses_ignore_exhausted_accounts_with_default_off() {
for ignore in [false, true] {
let provider = sample_provider(json!({
"pool_advanced": {
"ignore_exhausted_accounts": ignore,
"skip_exhausted_accounts": true
}
}));
let config = admin_provider_pool_config(&provider).expect("pool config should exist");
assert_eq!(config.skip_exhausted_accounts, !ignore);
}
}
#[test]
fn defaults_skip_exhausted_accounts_to_true() {
let provider = sample_provider(json!({ "pool_advanced": {} })); let provider = sample_provider(json!({ "pool_advanced": {} }));
let config = admin_provider_pool_config(&provider).expect("pool config should exist"); let config = admin_provider_pool_config(&provider).expect("pool config should exist");
assert!(!config.skip_exhausted_accounts); assert!(config.skip_exhausted_accounts);
assert!(!config.reserve_minimum_quota); assert!(!config.reserve_minimum_quota);
} }
@@ -593,7 +608,7 @@ mod tests {
})); }));
let config = admin_provider_pool_config(&provider).expect("pool config should exist"); let config = admin_provider_pool_config(&provider).expect("pool config should exist");
assert_eq!(config.reserve_minimum_quota, enabled); assert_eq!(config.reserve_minimum_quota, enabled);
assert!(!config.skip_exhausted_accounts); assert!(config.skip_exhausted_accounts);
} }
} }
@@ -385,9 +385,10 @@ async fn read_provider_pool_state_map(
.and_then(|value| value.get("pool_advanced")); .and_then(|value| value.get("pool_advanced"));
let skip_exhausted_accounts = pool_advanced let skip_exhausted_accounts = pool_advanced
.and_then(serde_json::Value::as_object) .and_then(serde_json::Value::as_object)
.and_then(|value| value.get("skip_exhausted_accounts")) .and_then(|value| value.get("ignore_exhausted_accounts"))
.and_then(serde_json::Value::as_bool) .and_then(serde_json::Value::as_bool)
.unwrap_or(false); .map(|ignore| !ignore)
.unwrap_or(true);
let reserve_minimum_quota = pool_advanced let reserve_minimum_quota = pool_advanced
.and_then(serde_json::Value::as_object) .and_then(serde_json::Value::as_object)
.and_then(|value| value.get("reserve_minimum_quota")) .and_then(|value| value.get("reserve_minimum_quota"))
+28 -12
View File
@@ -18,8 +18,6 @@ pub struct PoolSchedulingPreset {
pub struct PoolSchedulingConfig { pub struct PoolSchedulingConfig {
pub scheduling_presets: Vec<PoolSchedulingPreset>, pub scheduling_presets: Vec<PoolSchedulingPreset>,
pub lru_enabled: bool, pub lru_enabled: bool,
/// Retained for configuration/API compatibility. Active quota exhaustion is
/// always an admission block; reset-aware adapters decide when it clears.
pub skip_exhausted_accounts: bool, pub skip_exhausted_accounts: bool,
pub cost_limit_per_key_tokens: Option<u64>, pub cost_limit_per_key_tokens: Option<u64>,
} }
@@ -227,14 +225,9 @@ fn schedule_pool_group<Candidate>(
continue; continue;
} }
// A quota snapshot is an account-level admission signal, not merely a if item.key_context.quota_hard_blocked
// ranking hint. Continuing to schedule a member whose quota is known to || (pool_config.skip_exhausted_accounts && item.key_context.quota_exhausted)
// be exhausted causes a request-wide retry storm (the upstream returns {
// 429 for every attempt). `quota_hard_blocked` remains available for
// providers that can distinguish an explicit permanent block, but every
// active exhaustion must be removed from the request's candidate set;
// reset-aware provider adapters clear the signal once capacity returns.
if item.key_context.quota_hard_blocked || item.key_context.quota_exhausted {
skipped.push(PoolSkippedCandidate { skipped.push(PoolSkippedCandidate {
candidate: item.candidate, candidate: item.candidate,
skip_reason: POOL_ACCOUNT_EXHAUSTED_SKIP_REASON, skip_reason: POOL_ACCOUNT_EXHAUSTED_SKIP_REASON,
@@ -936,7 +929,30 @@ mod tests {
} }
#[test] #[test]
fn pool_scheduler_skips_exhausted_accounts_even_when_legacy_flag_is_false() { fn pool_scheduler_allows_exhausted_accounts_when_ignored_but_keeps_hard_blocks() {
let mut exhausted =
sample_candidate("provider-pool", "endpoint-1", "key-exhausted", 10, true);
exhausted.key_context.quota_exhausted = true;
exhausted
.pool_config
.as_mut()
.unwrap()
.skip_exhausted_accounts = false;
let mut blocked = exhausted.clone();
blocked.candidate = "key-blocked".to_string();
blocked.facts.key_id = "key-blocked".to_string();
blocked.key_context.quota_hard_blocked = true;
let outcome = run_pool_scheduler(vec![exhausted, blocked], &BTreeMap::new(), "seed");
assert_eq!(outcome.candidates.len(), 1);
assert_eq!(outcome.candidates[0].candidate, "key-exhausted");
assert_eq!(outcome.skipped_candidates.len(), 1);
assert_eq!(outcome.skipped_candidates[0].candidate, "key-blocked");
}
#[test]
fn pool_scheduler_skips_exhausted_accounts_by_default() {
let ready = sample_candidate("provider-pool", "endpoint-1", "key-ready", 10, true); let ready = sample_candidate("provider-pool", "endpoint-1", "key-ready", 10, true);
let mut exhausted = let mut exhausted =
sample_candidate("provider-pool", "endpoint-1", "key-exhausted", 10, true); sample_candidate("provider-pool", "endpoint-1", "key-exhausted", 10, true);
@@ -1535,7 +1551,7 @@ mod tests {
let pool_config = pool_enabled.then(|| PoolSchedulingConfig { let pool_config = pool_enabled.then(|| PoolSchedulingConfig {
scheduling_presets: Vec::new(), scheduling_presets: Vec::new(),
lru_enabled: true, lru_enabled: true,
skip_exhausted_accounts: false, skip_exhausted_accounts: true,
cost_limit_per_key_tokens: None, cost_limit_per_key_tokens: None,
}); });
PoolCandidateInput { PoolCandidateInput {
@@ -836,6 +836,7 @@ export interface PoolAdvancedConfig {
sticky_session_ttl_seconds?: number | null sticky_session_ttl_seconds?: number | null
load_threshold_percent?: number | null load_threshold_percent?: number | null
skip_exhausted_accounts?: boolean | null skip_exhausted_accounts?: boolean | null
ignore_exhausted_accounts?: boolean | null
// Codex only: treat remaining quota <= 1% as exhausted (default false). // Codex only: treat remaining quota <= 1% as exhausted (default false).
reserve_minimum_quota?: boolean reserve_minimum_quota?: boolean
// 旧字段(兼容读取) // 旧字段(兼容读取)
@@ -591,7 +591,7 @@ const form = ref({
account_self_check_concurrency: null as number | null | undefined, account_self_check_concurrency: null as number | null | undefined,
auto_remove_banned_keys: false, auto_remove_banned_keys: false,
auto_remove_quota_exhausted_keys: false, auto_remove_quota_exhausted_keys: false,
skip_exhausted_accounts: false, ignore_exhausted_accounts: false,
}) })
interface ClaudeFormState { interface ClaudeFormState {
@@ -630,8 +630,8 @@ function getHealthToggleValue(key: PoolHealthToggleKey): boolean {
return form.value.auto_remove_banned_keys return form.value.auto_remove_banned_keys
case 'auto_remove_quota_exhausted_keys': case 'auto_remove_quota_exhausted_keys':
return form.value.auto_remove_quota_exhausted_keys return form.value.auto_remove_quota_exhausted_keys
case 'skip_exhausted_accounts': case 'ignore_exhausted_accounts':
return form.value.skip_exhausted_accounts return form.value.ignore_exhausted_accounts
} }
} }
@@ -649,8 +649,8 @@ function updateHealthToggleValue(key: PoolHealthToggleKey, value: boolean): void
case 'auto_remove_quota_exhausted_keys': case 'auto_remove_quota_exhausted_keys':
form.value.auto_remove_quota_exhausted_keys = value form.value.auto_remove_quota_exhausted_keys = value
return return
case 'skip_exhausted_accounts': case 'ignore_exhausted_accounts':
form.value.skip_exhausted_accounts = value form.value.ignore_exhausted_accounts = value
} }
} }
@@ -686,7 +686,7 @@ watch([() => props.modelValue, () => props.providerId], ([open]) => {
account_self_check_concurrency: cfg?.account_self_check_concurrency ?? null, account_self_check_concurrency: cfg?.account_self_check_concurrency ?? null,
auto_remove_banned_keys: cfg?.auto_remove_banned_keys ?? false, auto_remove_banned_keys: cfg?.auto_remove_banned_keys ?? false,
auto_remove_quota_exhausted_keys: cfg?.auto_remove_quota_exhausted_keys ?? false, auto_remove_quota_exhausted_keys: cfg?.auto_remove_quota_exhausted_keys ?? false,
skip_exhausted_accounts: cfg?.skip_exhausted_accounts ?? false, ignore_exhausted_accounts: cfg?.ignore_exhausted_accounts ?? false,
} }
const cc = props.currentClaudeConfig const cc = props.currentClaudeConfig
@@ -754,7 +754,7 @@ async function handleSave() {
: undefined, : undefined,
auto_remove_banned_keys: form.value.auto_remove_banned_keys, auto_remove_banned_keys: form.value.auto_remove_banned_keys,
auto_remove_quota_exhausted_keys: form.value.auto_remove_quota_exhausted_keys, auto_remove_quota_exhausted_keys: form.value.auto_remove_quota_exhausted_keys,
skip_exhausted_accounts: form.value.skip_exhausted_accounts, ignore_exhausted_accounts: form.value.ignore_exhausted_accounts,
}) })
const payload: Parameters<typeof updateProvider>[1] = { const payload: Parameters<typeof updateProvider>[1] = {
@@ -12,7 +12,7 @@ describe('poolAdvancedDialog', () => {
'account_self_check_enabled', 'account_self_check_enabled',
'auto_remove_banned_keys', 'auto_remove_banned_keys',
'auto_remove_quota_exhausted_keys', 'auto_remove_quota_exhausted_keys',
'skip_exhausted_accounts', 'ignore_exhausted_accounts',
]) ])
}) })
@@ -39,9 +39,9 @@ describe('poolAdvancedDialog', () => {
description: '探测到黑色“额度耗尽”账号后自动从号池移除。', description: '探测到黑色“额度耗尽”账号后自动从号池移除。',
}, },
{ {
key: 'skip_exhausted_accounts', key: 'ignore_exhausted_accounts',
label: '跳过额度耗尽账号', label: '忽略额度耗尽',
description: '当 Codex / Kiro 账号额度已耗尽时,直接标记为不可调度并在请求侧跳过。', description: '默认关闭,额度耗尽账号不参与调度。开启后忽略额度耗尽状态,仍保留封禁、冷却和最低额度保留等限制。',
}, },
]) ])
}) })
@@ -3,7 +3,7 @@ export type PoolHealthToggleKey =
| 'account_self_check_enabled' | 'account_self_check_enabled'
| 'auto_remove_banned_keys' | 'auto_remove_banned_keys'
| 'auto_remove_quota_exhausted_keys' | 'auto_remove_quota_exhausted_keys'
| 'skip_exhausted_accounts' | 'ignore_exhausted_accounts'
export interface PoolHealthToggleCard { export interface PoolHealthToggleCard {
key: PoolHealthToggleKey key: PoolHealthToggleKey
@@ -39,9 +39,9 @@ export function buildPoolHealthToggleCards(): PoolHealthToggleCard[] {
description: '探测到黑色“额度耗尽”账号后自动从号池移除。', description: '探测到黑色“额度耗尽”账号后自动从号池移除。',
}, },
{ {
key: 'skip_exhausted_accounts', key: 'ignore_exhausted_accounts',
label: '跳过额度耗尽账号', label: '忽略额度耗尽',
description: '当 Codex / Kiro 账号额度已耗尽时,直接标记为不可调度并在请求侧跳过。', description: '默认关闭,额度耗尽账号不参与调度。开启后忽略额度耗尽状态,仍保留封禁、冷却和最低额度保留等限制。',
}, },
] ]
} }