From 6bafef672fcfeda669612403e607e071ea10e58a Mon Sep 17 00:00:00 2001 From: elky Date: Sat, 10 Oct 2026 10:04:41 +0800 Subject: [PATCH] fix(pool): honor quota exhaustion toggle and default ignore to off --- .../src/dispatch/pool_scheduler.rs | 33 ++++++++++++++- .../handlers/admin/provider/pool/config.rs | 27 ++++++++++--- .../src/scheduler/candidate/runtime.rs | 5 ++- crates/aether-pool-core/src/scheduler.rs | 40 +++++++++++++------ frontend/src/api/endpoints/types/provider.ts | 1 + .../pool/components/PoolAdvancedDialog.vue | 14 +++---- .../__tests__/poolAdvancedDialog.spec.ts | 8 ++-- .../features/pool/utils/poolAdvancedDialog.ts | 8 ++-- 8 files changed, 99 insertions(+), 37 deletions(-) diff --git a/apps/aether-gateway/src/dispatch/pool_scheduler.rs b/apps/aether-gateway/src/dispatch/pool_scheduler.rs index 28bd8ae78..1b512168e 100644 --- a/apps/aether-gateway/src/dispatch/pool_scheduler.rs +++ b/apps/aether-gateway/src/dispatch/pool_scheduler.rs @@ -326,7 +326,9 @@ fn active_probe_member_is_unschedulable_for_request( return true; } key_context.is_some_and(|context| { - context.account_blocked || context.quota_exhausted || context.quota_hard_blocked + context.account_blocked + || (pool_config.skip_exhausted_accounts && context.quota_exhausted) + || context.quota_hard_blocked }) } @@ -1529,12 +1531,14 @@ async fn read_pool_catalog_key_contexts_by_id( provider_type, provider_model_name, ); - context.quota_exhausted |= reserve_minimum_quota_key_ids.contains(&key.id) + let reserve_reached = reserve_minimum_quota_key_ids.contains(&key.id) && admin_provider_pool_pure::admin_pool_key_minimum_quota_reached( &key, provider_type, provider_model_name, ); + context.quota_exhausted |= reserve_reached; + context.quota_hard_blocked |= reserve_reached; (key.id.clone(), context) }) .collect::>(); @@ -2303,6 +2307,31 @@ mod tests { ); } + #[test] + fn pool_scheduler_allows_quota_exhausted_key_when_ignored() { + let exhausted = sample_eligible_candidate( + "provider-pool", + "endpoint-1", + "key-exhausted", + 10, + Some(json!({ "pool_advanced": { "ignore_exhausted_accounts": true } })), + ); + let contexts = BTreeMap::from([( + "key-exhausted".to_string(), + PoolCatalogKeyContext { + quota_exhausted: true, + ..PoolCatalogKeyContext::default() + }, + )]); + let (scheduled, skipped) = apply_local_execution_pool_scheduler_with_runtime_map( + vec![exhausted], + &BTreeMap::new(), + &contexts, + ); + assert_eq!(scheduled.len(), 1); + assert!(skipped.is_empty()); + } + #[test] fn pool_scheduler_skips_quota_exhausted_key_when_flag_is_false() { let ready = sample_eligible_candidate( diff --git a/apps/aether-gateway/src/handlers/admin/provider/pool/config.rs b/apps/aether-gateway/src/handlers/admin/provider/pool/config.rs index 779589534..f95f9165c 100644 --- a/apps/aether-gateway/src/handlers/admin/provider/pool/config.rs +++ b/apps/aether-gateway/src/handlers/admin/provider/pool/config.rs @@ -410,7 +410,7 @@ pub(crate) fn admin_provider_pool_config_from_config_value( }], unschedulable_rules: Vec::new(), lru_enabled: false, - skip_exhausted_accounts: false, + skip_exhausted_accounts: true, reserve_minimum_quota: false, sticky_session_ttl_seconds: 3600, latency_window_seconds: 3600, @@ -444,9 +444,10 @@ pub(crate) fn admin_provider_pool_config_from_config_value( scheduling_presets, unschedulable_rules, skip_exhausted_accounts: pool_advanced - .get("skip_exhausted_accounts") + .get("ignore_exhausted_accounts") .and_then(Value::as_bool) - .unwrap_or(false), + .map(|ignore| !ignore) + .unwrap_or(true), reserve_minimum_quota: pool_advanced .get("reserve_minimum_quota") .and_then(Value::as_bool) @@ -574,11 +575,25 @@ mod tests { } #[test] - fn defaults_skip_exhausted_accounts_to_false() { + fn parses_ignore_exhausted_accounts_with_default_off() { + for ignore in [false, true] { + let provider = sample_provider(json!({ + "pool_advanced": { + "ignore_exhausted_accounts": ignore, + "skip_exhausted_accounts": true + } + })); + let config = admin_provider_pool_config(&provider).expect("pool config should exist"); + assert_eq!(config.skip_exhausted_accounts, !ignore); + } + } + + #[test] + fn defaults_skip_exhausted_accounts_to_true() { let provider = sample_provider(json!({ "pool_advanced": {} })); let config = admin_provider_pool_config(&provider).expect("pool config should exist"); - assert!(!config.skip_exhausted_accounts); + assert!(config.skip_exhausted_accounts); assert!(!config.reserve_minimum_quota); } @@ -593,7 +608,7 @@ mod tests { })); let config = admin_provider_pool_config(&provider).expect("pool config should exist"); assert_eq!(config.reserve_minimum_quota, enabled); - assert!(!config.skip_exhausted_accounts); + assert!(config.skip_exhausted_accounts); } } diff --git a/apps/aether-gateway/src/scheduler/candidate/runtime.rs b/apps/aether-gateway/src/scheduler/candidate/runtime.rs index c6837019f..15eae9f6c 100644 --- a/apps/aether-gateway/src/scheduler/candidate/runtime.rs +++ b/apps/aether-gateway/src/scheduler/candidate/runtime.rs @@ -385,9 +385,10 @@ async fn read_provider_pool_state_map( .and_then(|value| value.get("pool_advanced")); let skip_exhausted_accounts = pool_advanced .and_then(serde_json::Value::as_object) - .and_then(|value| value.get("skip_exhausted_accounts")) + .and_then(|value| value.get("ignore_exhausted_accounts")) .and_then(serde_json::Value::as_bool) - .unwrap_or(false); + .map(|ignore| !ignore) + .unwrap_or(true); let reserve_minimum_quota = pool_advanced .and_then(serde_json::Value::as_object) .and_then(|value| value.get("reserve_minimum_quota")) diff --git a/crates/aether-pool-core/src/scheduler.rs b/crates/aether-pool-core/src/scheduler.rs index 8cf3ebfc3..d947157f4 100644 --- a/crates/aether-pool-core/src/scheduler.rs +++ b/crates/aether-pool-core/src/scheduler.rs @@ -18,8 +18,6 @@ pub struct PoolSchedulingPreset { pub struct PoolSchedulingConfig { pub scheduling_presets: Vec, pub lru_enabled: bool, - /// Retained for configuration/API compatibility. Active quota exhaustion is - /// always an admission block; reset-aware adapters decide when it clears. pub skip_exhausted_accounts: bool, pub cost_limit_per_key_tokens: Option, } @@ -227,14 +225,9 @@ fn schedule_pool_group( continue; } - // A quota snapshot is an account-level admission signal, not merely a - // ranking hint. Continuing to schedule a member whose quota is known to - // be exhausted causes a request-wide retry storm (the upstream returns - // 429 for every attempt). `quota_hard_blocked` remains available for - // providers that can distinguish an explicit permanent block, but every - // active exhaustion must be removed from the request's candidate set; - // reset-aware provider adapters clear the signal once capacity returns. - if item.key_context.quota_hard_blocked || item.key_context.quota_exhausted { + if item.key_context.quota_hard_blocked + || (pool_config.skip_exhausted_accounts && item.key_context.quota_exhausted) + { skipped.push(PoolSkippedCandidate { candidate: item.candidate, skip_reason: POOL_ACCOUNT_EXHAUSTED_SKIP_REASON, @@ -936,7 +929,30 @@ mod tests { } #[test] - fn pool_scheduler_skips_exhausted_accounts_even_when_legacy_flag_is_false() { + fn pool_scheduler_allows_exhausted_accounts_when_ignored_but_keeps_hard_blocks() { + let mut exhausted = + sample_candidate("provider-pool", "endpoint-1", "key-exhausted", 10, true); + exhausted.key_context.quota_exhausted = true; + exhausted + .pool_config + .as_mut() + .unwrap() + .skip_exhausted_accounts = false; + let mut blocked = exhausted.clone(); + blocked.candidate = "key-blocked".to_string(); + blocked.facts.key_id = "key-blocked".to_string(); + blocked.key_context.quota_hard_blocked = true; + + let outcome = run_pool_scheduler(vec![exhausted, blocked], &BTreeMap::new(), "seed"); + + assert_eq!(outcome.candidates.len(), 1); + assert_eq!(outcome.candidates[0].candidate, "key-exhausted"); + assert_eq!(outcome.skipped_candidates.len(), 1); + assert_eq!(outcome.skipped_candidates[0].candidate, "key-blocked"); + } + + #[test] + fn pool_scheduler_skips_exhausted_accounts_by_default() { let ready = sample_candidate("provider-pool", "endpoint-1", "key-ready", 10, true); let mut exhausted = sample_candidate("provider-pool", "endpoint-1", "key-exhausted", 10, true); @@ -1535,7 +1551,7 @@ mod tests { let pool_config = pool_enabled.then(|| PoolSchedulingConfig { scheduling_presets: Vec::new(), lru_enabled: true, - skip_exhausted_accounts: false, + skip_exhausted_accounts: true, cost_limit_per_key_tokens: None, }); PoolCandidateInput { diff --git a/frontend/src/api/endpoints/types/provider.ts b/frontend/src/api/endpoints/types/provider.ts index ee6f53fd2..36c86e0de 100644 --- a/frontend/src/api/endpoints/types/provider.ts +++ b/frontend/src/api/endpoints/types/provider.ts @@ -836,6 +836,7 @@ export interface PoolAdvancedConfig { sticky_session_ttl_seconds?: number | null load_threshold_percent?: number | null skip_exhausted_accounts?: boolean | null + ignore_exhausted_accounts?: boolean | null // Codex only: treat remaining quota <= 1% as exhausted (default false). reserve_minimum_quota?: boolean // 旧字段（兼容读取） diff --git a/frontend/src/features/pool/components/PoolAdvancedDialog.vue b/frontend/src/features/pool/components/PoolAdvancedDialog.vue index 6118fc79f..da9e51356 100644 --- a/frontend/src/features/pool/components/PoolAdvancedDialog.vue +++ b/frontend/src/features/pool/components/PoolAdvancedDialog.vue @@ -591,7 +591,7 @@ const form = ref({ account_self_check_concurrency: null as number | null | undefined, auto_remove_banned_keys: false, auto_remove_quota_exhausted_keys: false, - skip_exhausted_accounts: false, + ignore_exhausted_accounts: false, }) interface ClaudeFormState { @@ -630,8 +630,8 @@ function getHealthToggleValue(key: PoolHealthToggleKey): boolean { return form.value.auto_remove_banned_keys case 'auto_remove_quota_exhausted_keys': return form.value.auto_remove_quota_exhausted_keys - case 'skip_exhausted_accounts': - return form.value.skip_exhausted_accounts + case 'ignore_exhausted_accounts': + return form.value.ignore_exhausted_accounts } } @@ -649,8 +649,8 @@ function updateHealthToggleValue(key: PoolHealthToggleKey, value: boolean): void case 'auto_remove_quota_exhausted_keys': form.value.auto_remove_quota_exhausted_keys = value return - case 'skip_exhausted_accounts': - form.value.skip_exhausted_accounts = value + case 'ignore_exhausted_accounts': + form.value.ignore_exhausted_accounts = value } } @@ -686,7 +686,7 @@ watch([() => props.modelValue, () => props.providerId], ([open]) => { account_self_check_concurrency: cfg?.account_self_check_concurrency ?? null, auto_remove_banned_keys: cfg?.auto_remove_banned_keys ?? false, auto_remove_quota_exhausted_keys: cfg?.auto_remove_quota_exhausted_keys ?? false, - skip_exhausted_accounts: cfg?.skip_exhausted_accounts ?? false, + ignore_exhausted_accounts: cfg?.ignore_exhausted_accounts ?? false, } const cc = props.currentClaudeConfig @@ -754,7 +754,7 @@ async function handleSave() { : undefined, auto_remove_banned_keys: form.value.auto_remove_banned_keys, auto_remove_quota_exhausted_keys: form.value.auto_remove_quota_exhausted_keys, - skip_exhausted_accounts: form.value.skip_exhausted_accounts, + ignore_exhausted_accounts: form.value.ignore_exhausted_accounts, }) const payload: Parameters[1] = { diff --git a/frontend/src/features/pool/utils/__tests__/poolAdvancedDialog.spec.ts b/frontend/src/features/pool/utils/__tests__/poolAdvancedDialog.spec.ts index 1b45bc88e..9ed829331 100644 --- a/frontend/src/features/pool/utils/__tests__/poolAdvancedDialog.spec.ts +++ b/frontend/src/features/pool/utils/__tests__/poolAdvancedDialog.spec.ts @@ -12,7 +12,7 @@ describe('poolAdvancedDialog', () => { 'account_self_check_enabled', 'auto_remove_banned_keys', 'auto_remove_quota_exhausted_keys', - 'skip_exhausted_accounts', + 'ignore_exhausted_accounts', ]) }) @@ -39,9 +39,9 @@ describe('poolAdvancedDialog', () => { description: '探测到黑色“额度耗尽”账号后自动从号池移除。', }, { - key: 'skip_exhausted_accounts', - label: '跳过额度耗尽账号', - description: '当 Codex / Kiro 账号额度已耗尽时，直接标记为不可调度并在请求侧跳过。', + key: 'ignore_exhausted_accounts', + label: '忽略额度耗尽', + description: '默认关闭，额度耗尽账号不参与调度。开启后忽略额度耗尽状态，仍保留封禁、冷却和最低额度保留等限制。', }, ]) }) diff --git a/frontend/src/features/pool/utils/poolAdvancedDialog.ts b/frontend/src/features/pool/utils/poolAdvancedDialog.ts index 88da8a136..6044f7c90 100644 --- a/frontend/src/features/pool/utils/poolAdvancedDialog.ts +++ b/frontend/src/features/pool/utils/poolAdvancedDialog.ts @@ -3,7 +3,7 @@ export type PoolHealthToggleKey = | 'account_self_check_enabled' | 'auto_remove_banned_keys' | 'auto_remove_quota_exhausted_keys' - | 'skip_exhausted_accounts' + | 'ignore_exhausted_accounts' export interface PoolHealthToggleCard { key: PoolHealthToggleKey @@ -39,9 +39,9 @@ export function buildPoolHealthToggleCards(): PoolHealthToggleCard[] { description: '探测到黑色“额度耗尽”账号后自动从号池移除。', }, { - key: 'skip_exhausted_accounts', - label: '跳过额度耗尽账号', - description: '当 Codex / Kiro 账号额度已耗尽时，直接标记为不可调度并在请求侧跳过。', + key: 'ignore_exhausted_accounts', + label: '忽略额度耗尽', + description: '默认关闭，额度耗尽账号不参与调度。开启后忽略额度耗尽状态，仍保留封禁、冷却和最低额度保留等限制。', }, ] }