mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-12 04:09:48 +08:00
fix(pool): honor quota exhaustion toggle and default ignore to off
This commit is contained in:
@@ -326,7 +326,9 @@ fn active_probe_member_is_unschedulable_for_request(
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
key_context.is_some_and(|context| {
|
key_context.is_some_and(|context| {
|
||||||
context.account_blocked || context.quota_exhausted || context.quota_hard_blocked
|
context.account_blocked
|
||||||
|
|| (pool_config.skip_exhausted_accounts && context.quota_exhausted)
|
||||||
|
|| context.quota_hard_blocked
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1529,12 +1531,14 @@ async fn read_pool_catalog_key_contexts_by_id(
|
|||||||
provider_type,
|
provider_type,
|
||||||
provider_model_name,
|
provider_model_name,
|
||||||
);
|
);
|
||||||
context.quota_exhausted |= reserve_minimum_quota_key_ids.contains(&key.id)
|
let reserve_reached = reserve_minimum_quota_key_ids.contains(&key.id)
|
||||||
&& admin_provider_pool_pure::admin_pool_key_minimum_quota_reached(
|
&& admin_provider_pool_pure::admin_pool_key_minimum_quota_reached(
|
||||||
&key,
|
&key,
|
||||||
provider_type,
|
provider_type,
|
||||||
provider_model_name,
|
provider_model_name,
|
||||||
);
|
);
|
||||||
|
context.quota_exhausted |= reserve_reached;
|
||||||
|
context.quota_hard_blocked |= reserve_reached;
|
||||||
(key.id.clone(), context)
|
(key.id.clone(), context)
|
||||||
})
|
})
|
||||||
.collect::<BTreeMap<_, _>>();
|
.collect::<BTreeMap<_, _>>();
|
||||||
@@ -2303,6 +2307,31 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pool_scheduler_allows_quota_exhausted_key_when_ignored() {
|
||||||
|
let exhausted = sample_eligible_candidate(
|
||||||
|
"provider-pool",
|
||||||
|
"endpoint-1",
|
||||||
|
"key-exhausted",
|
||||||
|
10,
|
||||||
|
Some(json!({ "pool_advanced": { "ignore_exhausted_accounts": true } })),
|
||||||
|
);
|
||||||
|
let contexts = BTreeMap::from([(
|
||||||
|
"key-exhausted".to_string(),
|
||||||
|
PoolCatalogKeyContext {
|
||||||
|
quota_exhausted: true,
|
||||||
|
..PoolCatalogKeyContext::default()
|
||||||
|
},
|
||||||
|
)]);
|
||||||
|
let (scheduled, skipped) = apply_local_execution_pool_scheduler_with_runtime_map(
|
||||||
|
vec![exhausted],
|
||||||
|
&BTreeMap::new(),
|
||||||
|
&contexts,
|
||||||
|
);
|
||||||
|
assert_eq!(scheduled.len(), 1);
|
||||||
|
assert!(skipped.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn pool_scheduler_skips_quota_exhausted_key_when_flag_is_false() {
|
fn pool_scheduler_skips_quota_exhausted_key_when_flag_is_false() {
|
||||||
let ready = sample_eligible_candidate(
|
let ready = sample_eligible_candidate(
|
||||||
|
|||||||
@@ -410,7 +410,7 @@ pub(crate) fn admin_provider_pool_config_from_config_value(
|
|||||||
}],
|
}],
|
||||||
unschedulable_rules: Vec::new(),
|
unschedulable_rules: Vec::new(),
|
||||||
lru_enabled: false,
|
lru_enabled: false,
|
||||||
skip_exhausted_accounts: false,
|
skip_exhausted_accounts: true,
|
||||||
reserve_minimum_quota: false,
|
reserve_minimum_quota: false,
|
||||||
sticky_session_ttl_seconds: 3600,
|
sticky_session_ttl_seconds: 3600,
|
||||||
latency_window_seconds: 3600,
|
latency_window_seconds: 3600,
|
||||||
@@ -444,9 +444,10 @@ pub(crate) fn admin_provider_pool_config_from_config_value(
|
|||||||
scheduling_presets,
|
scheduling_presets,
|
||||||
unschedulable_rules,
|
unschedulable_rules,
|
||||||
skip_exhausted_accounts: pool_advanced
|
skip_exhausted_accounts: pool_advanced
|
||||||
.get("skip_exhausted_accounts")
|
.get("ignore_exhausted_accounts")
|
||||||
.and_then(Value::as_bool)
|
.and_then(Value::as_bool)
|
||||||
.unwrap_or(false),
|
.map(|ignore| !ignore)
|
||||||
|
.unwrap_or(true),
|
||||||
reserve_minimum_quota: pool_advanced
|
reserve_minimum_quota: pool_advanced
|
||||||
.get("reserve_minimum_quota")
|
.get("reserve_minimum_quota")
|
||||||
.and_then(Value::as_bool)
|
.and_then(Value::as_bool)
|
||||||
@@ -574,11 +575,25 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn defaults_skip_exhausted_accounts_to_false() {
|
fn parses_ignore_exhausted_accounts_with_default_off() {
|
||||||
|
for ignore in [false, true] {
|
||||||
|
let provider = sample_provider(json!({
|
||||||
|
"pool_advanced": {
|
||||||
|
"ignore_exhausted_accounts": ignore,
|
||||||
|
"skip_exhausted_accounts": true
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
let config = admin_provider_pool_config(&provider).expect("pool config should exist");
|
||||||
|
assert_eq!(config.skip_exhausted_accounts, !ignore);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn defaults_skip_exhausted_accounts_to_true() {
|
||||||
let provider = sample_provider(json!({ "pool_advanced": {} }));
|
let provider = sample_provider(json!({ "pool_advanced": {} }));
|
||||||
let config = admin_provider_pool_config(&provider).expect("pool config should exist");
|
let config = admin_provider_pool_config(&provider).expect("pool config should exist");
|
||||||
|
|
||||||
assert!(!config.skip_exhausted_accounts);
|
assert!(config.skip_exhausted_accounts);
|
||||||
assert!(!config.reserve_minimum_quota);
|
assert!(!config.reserve_minimum_quota);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -593,7 +608,7 @@ mod tests {
|
|||||||
}));
|
}));
|
||||||
let config = admin_provider_pool_config(&provider).expect("pool config should exist");
|
let config = admin_provider_pool_config(&provider).expect("pool config should exist");
|
||||||
assert_eq!(config.reserve_minimum_quota, enabled);
|
assert_eq!(config.reserve_minimum_quota, enabled);
|
||||||
assert!(!config.skip_exhausted_accounts);
|
assert!(config.skip_exhausted_accounts);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -385,9 +385,10 @@ async fn read_provider_pool_state_map(
|
|||||||
.and_then(|value| value.get("pool_advanced"));
|
.and_then(|value| value.get("pool_advanced"));
|
||||||
let skip_exhausted_accounts = pool_advanced
|
let skip_exhausted_accounts = pool_advanced
|
||||||
.and_then(serde_json::Value::as_object)
|
.and_then(serde_json::Value::as_object)
|
||||||
.and_then(|value| value.get("skip_exhausted_accounts"))
|
.and_then(|value| value.get("ignore_exhausted_accounts"))
|
||||||
.and_then(serde_json::Value::as_bool)
|
.and_then(serde_json::Value::as_bool)
|
||||||
.unwrap_or(false);
|
.map(|ignore| !ignore)
|
||||||
|
.unwrap_or(true);
|
||||||
let reserve_minimum_quota = pool_advanced
|
let reserve_minimum_quota = pool_advanced
|
||||||
.and_then(serde_json::Value::as_object)
|
.and_then(serde_json::Value::as_object)
|
||||||
.and_then(|value| value.get("reserve_minimum_quota"))
|
.and_then(|value| value.get("reserve_minimum_quota"))
|
||||||
|
|||||||
@@ -18,8 +18,6 @@ pub struct PoolSchedulingPreset {
|
|||||||
pub struct PoolSchedulingConfig {
|
pub struct PoolSchedulingConfig {
|
||||||
pub scheduling_presets: Vec<PoolSchedulingPreset>,
|
pub scheduling_presets: Vec<PoolSchedulingPreset>,
|
||||||
pub lru_enabled: bool,
|
pub lru_enabled: bool,
|
||||||
/// Retained for configuration/API compatibility. Active quota exhaustion is
|
|
||||||
/// always an admission block; reset-aware adapters decide when it clears.
|
|
||||||
pub skip_exhausted_accounts: bool,
|
pub skip_exhausted_accounts: bool,
|
||||||
pub cost_limit_per_key_tokens: Option<u64>,
|
pub cost_limit_per_key_tokens: Option<u64>,
|
||||||
}
|
}
|
||||||
@@ -227,14 +225,9 @@ fn schedule_pool_group<Candidate>(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// A quota snapshot is an account-level admission signal, not merely a
|
if item.key_context.quota_hard_blocked
|
||||||
// ranking hint. Continuing to schedule a member whose quota is known to
|
|| (pool_config.skip_exhausted_accounts && item.key_context.quota_exhausted)
|
||||||
// be exhausted causes a request-wide retry storm (the upstream returns
|
{
|
||||||
// 429 for every attempt). `quota_hard_blocked` remains available for
|
|
||||||
// providers that can distinguish an explicit permanent block, but every
|
|
||||||
// active exhaustion must be removed from the request's candidate set;
|
|
||||||
// reset-aware provider adapters clear the signal once capacity returns.
|
|
||||||
if item.key_context.quota_hard_blocked || item.key_context.quota_exhausted {
|
|
||||||
skipped.push(PoolSkippedCandidate {
|
skipped.push(PoolSkippedCandidate {
|
||||||
candidate: item.candidate,
|
candidate: item.candidate,
|
||||||
skip_reason: POOL_ACCOUNT_EXHAUSTED_SKIP_REASON,
|
skip_reason: POOL_ACCOUNT_EXHAUSTED_SKIP_REASON,
|
||||||
@@ -936,7 +929,30 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn pool_scheduler_skips_exhausted_accounts_even_when_legacy_flag_is_false() {
|
fn pool_scheduler_allows_exhausted_accounts_when_ignored_but_keeps_hard_blocks() {
|
||||||
|
let mut exhausted =
|
||||||
|
sample_candidate("provider-pool", "endpoint-1", "key-exhausted", 10, true);
|
||||||
|
exhausted.key_context.quota_exhausted = true;
|
||||||
|
exhausted
|
||||||
|
.pool_config
|
||||||
|
.as_mut()
|
||||||
|
.unwrap()
|
||||||
|
.skip_exhausted_accounts = false;
|
||||||
|
let mut blocked = exhausted.clone();
|
||||||
|
blocked.candidate = "key-blocked".to_string();
|
||||||
|
blocked.facts.key_id = "key-blocked".to_string();
|
||||||
|
blocked.key_context.quota_hard_blocked = true;
|
||||||
|
|
||||||
|
let outcome = run_pool_scheduler(vec![exhausted, blocked], &BTreeMap::new(), "seed");
|
||||||
|
|
||||||
|
assert_eq!(outcome.candidates.len(), 1);
|
||||||
|
assert_eq!(outcome.candidates[0].candidate, "key-exhausted");
|
||||||
|
assert_eq!(outcome.skipped_candidates.len(), 1);
|
||||||
|
assert_eq!(outcome.skipped_candidates[0].candidate, "key-blocked");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pool_scheduler_skips_exhausted_accounts_by_default() {
|
||||||
let ready = sample_candidate("provider-pool", "endpoint-1", "key-ready", 10, true);
|
let ready = sample_candidate("provider-pool", "endpoint-1", "key-ready", 10, true);
|
||||||
let mut exhausted =
|
let mut exhausted =
|
||||||
sample_candidate("provider-pool", "endpoint-1", "key-exhausted", 10, true);
|
sample_candidate("provider-pool", "endpoint-1", "key-exhausted", 10, true);
|
||||||
@@ -1535,7 +1551,7 @@ mod tests {
|
|||||||
let pool_config = pool_enabled.then(|| PoolSchedulingConfig {
|
let pool_config = pool_enabled.then(|| PoolSchedulingConfig {
|
||||||
scheduling_presets: Vec::new(),
|
scheduling_presets: Vec::new(),
|
||||||
lru_enabled: true,
|
lru_enabled: true,
|
||||||
skip_exhausted_accounts: false,
|
skip_exhausted_accounts: true,
|
||||||
cost_limit_per_key_tokens: None,
|
cost_limit_per_key_tokens: None,
|
||||||
});
|
});
|
||||||
PoolCandidateInput {
|
PoolCandidateInput {
|
||||||
|
|||||||
@@ -836,6 +836,7 @@ export interface PoolAdvancedConfig {
|
|||||||
sticky_session_ttl_seconds?: number | null
|
sticky_session_ttl_seconds?: number | null
|
||||||
load_threshold_percent?: number | null
|
load_threshold_percent?: number | null
|
||||||
skip_exhausted_accounts?: boolean | null
|
skip_exhausted_accounts?: boolean | null
|
||||||
|
ignore_exhausted_accounts?: boolean | null
|
||||||
// Codex only: treat remaining quota <= 1% as exhausted (default false).
|
// Codex only: treat remaining quota <= 1% as exhausted (default false).
|
||||||
reserve_minimum_quota?: boolean
|
reserve_minimum_quota?: boolean
|
||||||
// 旧字段(兼容读取)
|
// 旧字段(兼容读取)
|
||||||
|
|||||||
@@ -591,7 +591,7 @@ const form = ref({
|
|||||||
account_self_check_concurrency: null as number | null | undefined,
|
account_self_check_concurrency: null as number | null | undefined,
|
||||||
auto_remove_banned_keys: false,
|
auto_remove_banned_keys: false,
|
||||||
auto_remove_quota_exhausted_keys: false,
|
auto_remove_quota_exhausted_keys: false,
|
||||||
skip_exhausted_accounts: false,
|
ignore_exhausted_accounts: false,
|
||||||
})
|
})
|
||||||
|
|
||||||
interface ClaudeFormState {
|
interface ClaudeFormState {
|
||||||
@@ -630,8 +630,8 @@ function getHealthToggleValue(key: PoolHealthToggleKey): boolean {
|
|||||||
return form.value.auto_remove_banned_keys
|
return form.value.auto_remove_banned_keys
|
||||||
case 'auto_remove_quota_exhausted_keys':
|
case 'auto_remove_quota_exhausted_keys':
|
||||||
return form.value.auto_remove_quota_exhausted_keys
|
return form.value.auto_remove_quota_exhausted_keys
|
||||||
case 'skip_exhausted_accounts':
|
case 'ignore_exhausted_accounts':
|
||||||
return form.value.skip_exhausted_accounts
|
return form.value.ignore_exhausted_accounts
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -649,8 +649,8 @@ function updateHealthToggleValue(key: PoolHealthToggleKey, value: boolean): void
|
|||||||
case 'auto_remove_quota_exhausted_keys':
|
case 'auto_remove_quota_exhausted_keys':
|
||||||
form.value.auto_remove_quota_exhausted_keys = value
|
form.value.auto_remove_quota_exhausted_keys = value
|
||||||
return
|
return
|
||||||
case 'skip_exhausted_accounts':
|
case 'ignore_exhausted_accounts':
|
||||||
form.value.skip_exhausted_accounts = value
|
form.value.ignore_exhausted_accounts = value
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -686,7 +686,7 @@ watch([() => props.modelValue, () => props.providerId], ([open]) => {
|
|||||||
account_self_check_concurrency: cfg?.account_self_check_concurrency ?? null,
|
account_self_check_concurrency: cfg?.account_self_check_concurrency ?? null,
|
||||||
auto_remove_banned_keys: cfg?.auto_remove_banned_keys ?? false,
|
auto_remove_banned_keys: cfg?.auto_remove_banned_keys ?? false,
|
||||||
auto_remove_quota_exhausted_keys: cfg?.auto_remove_quota_exhausted_keys ?? false,
|
auto_remove_quota_exhausted_keys: cfg?.auto_remove_quota_exhausted_keys ?? false,
|
||||||
skip_exhausted_accounts: cfg?.skip_exhausted_accounts ?? false,
|
ignore_exhausted_accounts: cfg?.ignore_exhausted_accounts ?? false,
|
||||||
}
|
}
|
||||||
|
|
||||||
const cc = props.currentClaudeConfig
|
const cc = props.currentClaudeConfig
|
||||||
@@ -754,7 +754,7 @@ async function handleSave() {
|
|||||||
: undefined,
|
: undefined,
|
||||||
auto_remove_banned_keys: form.value.auto_remove_banned_keys,
|
auto_remove_banned_keys: form.value.auto_remove_banned_keys,
|
||||||
auto_remove_quota_exhausted_keys: form.value.auto_remove_quota_exhausted_keys,
|
auto_remove_quota_exhausted_keys: form.value.auto_remove_quota_exhausted_keys,
|
||||||
skip_exhausted_accounts: form.value.skip_exhausted_accounts,
|
ignore_exhausted_accounts: form.value.ignore_exhausted_accounts,
|
||||||
})
|
})
|
||||||
|
|
||||||
const payload: Parameters<typeof updateProvider>[1] = {
|
const payload: Parameters<typeof updateProvider>[1] = {
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ describe('poolAdvancedDialog', () => {
|
|||||||
'account_self_check_enabled',
|
'account_self_check_enabled',
|
||||||
'auto_remove_banned_keys',
|
'auto_remove_banned_keys',
|
||||||
'auto_remove_quota_exhausted_keys',
|
'auto_remove_quota_exhausted_keys',
|
||||||
'skip_exhausted_accounts',
|
'ignore_exhausted_accounts',
|
||||||
])
|
])
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -39,9 +39,9 @@ describe('poolAdvancedDialog', () => {
|
|||||||
description: '探测到黑色“额度耗尽”账号后自动从号池移除。',
|
description: '探测到黑色“额度耗尽”账号后自动从号池移除。',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
key: 'skip_exhausted_accounts',
|
key: 'ignore_exhausted_accounts',
|
||||||
label: '跳过额度耗尽账号',
|
label: '忽略额度耗尽',
|
||||||
description: '当 Codex / Kiro 账号额度已耗尽时,直接标记为不可调度并在请求侧跳过。',
|
description: '默认关闭,额度耗尽账号不参与调度。开启后忽略额度耗尽状态,仍保留封禁、冷却和最低额度保留等限制。',
|
||||||
},
|
},
|
||||||
])
|
])
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ export type PoolHealthToggleKey =
|
|||||||
| 'account_self_check_enabled'
|
| 'account_self_check_enabled'
|
||||||
| 'auto_remove_banned_keys'
|
| 'auto_remove_banned_keys'
|
||||||
| 'auto_remove_quota_exhausted_keys'
|
| 'auto_remove_quota_exhausted_keys'
|
||||||
| 'skip_exhausted_accounts'
|
| 'ignore_exhausted_accounts'
|
||||||
|
|
||||||
export interface PoolHealthToggleCard {
|
export interface PoolHealthToggleCard {
|
||||||
key: PoolHealthToggleKey
|
key: PoolHealthToggleKey
|
||||||
@@ -39,9 +39,9 @@ export function buildPoolHealthToggleCards(): PoolHealthToggleCard[] {
|
|||||||
description: '探测到黑色“额度耗尽”账号后自动从号池移除。',
|
description: '探测到黑色“额度耗尽”账号后自动从号池移除。',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
key: 'skip_exhausted_accounts',
|
key: 'ignore_exhausted_accounts',
|
||||||
label: '跳过额度耗尽账号',
|
label: '忽略额度耗尽',
|
||||||
description: '当 Codex / Kiro 账号额度已耗尽时,直接标记为不可调度并在请求侧跳过。',
|
description: '默认关闭,额度耗尽账号不参与调度。开启后忽略额度耗尽状态,仍保留封禁、冷却和最低额度保留等限制。',
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user