fix(pool): isolate dynamic model quota buckets and 429 scheduling

This commit is contained in:
zhefox
2026-09-02 15:23:23 +08:00
parent 2cd20da1ec
commit dbbe7b22ab
37 changed files with 1491 additions and 78 deletions
+33 -3
View File
@@ -18,6 +18,8 @@ pub struct PoolSchedulingPreset {
pub struct PoolSchedulingConfig {
pub scheduling_presets: Vec<PoolSchedulingPreset>,
pub lru_enabled: bool,
/// Retained for configuration/API compatibility. Active quota exhaustion is
/// always an admission block; reset-aware adapters decide when it clears.
pub skip_exhausted_accounts: bool,
pub cost_limit_per_key_tokens: Option<u64>,
}
@@ -225,9 +227,14 @@ fn schedule_pool_group<Candidate>(
continue;
}
if item.key_context.quota_hard_blocked
|| (pool_config.skip_exhausted_accounts && item.key_context.quota_exhausted)
{
// A quota snapshot is an account-level admission signal, not merely a
// ranking hint. Continuing to schedule a member whose quota is known to
// be exhausted causes a request-wide retry storm (the upstream returns
// 429 for every attempt). `quota_hard_blocked` remains available for
// providers that can distinguish an explicit permanent block, but every
// active exhaustion must be removed from the request's candidate set;
// reset-aware provider adapters clear the signal once capacity returns.
if item.key_context.quota_hard_blocked || item.key_context.quota_exhausted {
skipped.push(PoolSkippedCandidate {
candidate: item.candidate,
skip_reason: POOL_ACCOUNT_EXHAUSTED_SKIP_REASON,
@@ -928,6 +935,29 @@ mod tests {
);
}
#[test]
fn pool_scheduler_skips_exhausted_accounts_even_when_legacy_flag_is_false() {
let ready = sample_candidate("provider-pool", "endpoint-1", "key-ready", 10, true);
let mut exhausted =
sample_candidate("provider-pool", "endpoint-1", "key-exhausted", 10, true);
exhausted.key_context.quota_exhausted = true;
let outcome = run_pool_scheduler(vec![ready, exhausted], &BTreeMap::new(), "seed");
assert_eq!(
outcome
.candidates
.iter()
.map(|item| item.candidate.as_str())
.collect::<Vec<_>>(),
vec!["key-ready"]
);
assert_eq!(
outcome.skipped_candidates[0].skip_reason,
POOL_ACCOUNT_EXHAUSTED_SKIP_REASON
);
}
#[test]
fn pool_scheduler_promotes_sticky_hit_before_other_sorted_keys() {
let key_a = sample_candidate("provider-pool", "endpoint-1", "key-a", 10, true)