i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-03 16:07:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(codex): add optional minimum quota reserve for pool scheduling

Browse Source
This commit is contained in:
ZheFox
2026-09-23 14:44:29 +08:00
parent 2a9d8d3b25
commit 1a4eba1005
16 changed files with 951 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/aether-gateway/src/dispatch/pool_scheduler.rs
+133 -11
View File
@@ -131,8 +131,13 @@ async fn schedule_pool_page_candidates(
entry.1.insert(candidate.candidate.key_id.clone());
}
let key_context_by_id =
read_pool_catalog_key_contexts_by_id(state, &candidates, provider_model_name).await;
let key_context_by_id = read_pool_catalog_key_contexts_by_id(
state,
&candidates,
provider_model_name,
effective_pool_config,
)
.await;
let mut runtime_by_provider = BTreeMap::new();
let mut pool_config_by_provider = BTreeMap::new();
@@ -1029,6 +1034,26 @@ impl<'a> PoolKeyCursor<'a> {
return None;
}
if pool_config.reserve_minimum_quota
&& admin_provider_pool_pure::admin_pool_key_minimum_quota_reached(
&key,
self.group.candidate.provider_type.as_str(),
Some(self.group.candidate.selected_provider_model_name.as_str()),
)
{
self.seen_key_ids.insert(key.id.clone());
self.record_skip_reason(POOL_ACCOUNT_EXHAUSTED_SKIP_REASON);
self.skipped_candidates
.push(SkippedLocalExecutionCandidate {
candidate: pool_candidate_from_catalog_key(&self.group, key),
skip_reason: POOL_ACCOUNT_EXHAUSTED_SKIP_REASON,
transport: None,
ranking: self.group.ranking.clone(),
extra_data: None,
});
return None;
}
let candidate = pool_candidate_from_catalog_key(&self.group, key);
self.build_eligible_candidate(candidate).await
}
@@ -1429,15 +1454,23 @@ async fn read_pool_catalog_key_contexts_by_id(
state: PlannerAppState<'_>,
candidates: &[EligibleLocalExecutionCandidate],
provider_model_name: Option<&str>,
effective_pool_config: Option<&AdminProviderPoolConfig>,
) -> BTreeMap<String, PoolCatalogKeyContext> {
let mut key_ids = Vec::new();
let mut provider_type_by_key_id = BTreeMap::<String, String>::new();
let mut reserve_minimum_quota_key_ids = BTreeSet::new();
for candidate in candidates {
if pool_config_for_candidate(candidate).is_none() {
let Some(pool_config) = effective_pool_config
.cloned()
.or_else(|| pool_config_for_candidate(candidate))
else {
continue;
}
};
let key_id = candidate.candidate.key_id.clone();
if pool_config.reserve_minimum_quota {
reserve_minimum_quota_key_ids.insert(key_id.clone());
}
if let Entry::Vacant(entry) = provider_type_by_key_id.entry(key_id.clone()) {
entry.insert(candidate.transport.provider.provider_type.clone());
key_ids.push(key_id);
@@ -1489,16 +1522,20 @@ async fn read_pool_catalog_key_contexts_by_id(
.get(&key.id)
.map(String::as_str)
.unwrap_or_default();
(
key.id.clone(),
build_pool_catalog_key_context(
state,
&provider_pool_service,
let mut context = build_pool_catalog_key_context(
state,
&provider_pool_service,
&key,
provider_type,
provider_model_name,
);
context.quota_exhausted |= reserve_minimum_quota_key_ids.contains(&key.id)
&& admin_provider_pool_pure::admin_pool_key_minimum_quota_reached(
&key,
provider_type,
provider_model_name,
),
)
);
(key.id.clone(), context)
})
.collect::<BTreeMap<_, _>>();
// A key can disappear between the candidate-row and catalog reads. Keep
@@ -3964,6 +4001,91 @@ mod tests {
}));
}
#[tokio::test]
async fn pool_key_cursor_reserve_minimum_quota_filters_pages_and_sticky_hits() {
for reserve_enabled in [false, true] {
for sticky in [false, true] {
for used_percent in [99.0, 98.0] {
let provider_config = Some(json!({
"pool_advanced": {
"reserve_minimum_quota": reserve_enabled,
"skip_exhausted_accounts": false
}
}));
let provider =
sample_codex_pool_provider("provider-pool", 0, provider_config.clone());
let endpoint = sample_codex_pool_endpoint("provider-pool", "endpoint-1");
let mut reserved = sample_codex_pool_key("provider-pool", "key-low");
reserved.upstream_metadata = Some(json!({
"codex": {"primary_used_percent": used_percent}
}));
let ready = sample_codex_pool_key("provider-pool", "key-ready");
let rows = vec![
sample_codex_pool_row("provider-pool", "endpoint-1", "key-low", 0),
sample_codex_pool_row("provider-pool", "endpoint-1", "key-ready", 0),
];
let data_state = GatewayDataState::with_provider_catalog_and_minimal_candidate_selection_for_tests(
Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![provider], vec![endpoint], vec![reserved, ready],
)),
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(rows)),
)
.with_encryption_key_for_tests(aether_crypto::DEVELOPMENT_ENCRYPTION_KEY);
let app = AppState::new()
.expect("state should build")
.with_data_state_for_tests(data_state);
let group =
sample_codex_pool_group("provider-pool", "endpoint-1", 0, provider_config);
let pool_config =
pool_config_for_candidate(&group).expect("pool config should parse");
let sticky_token = sticky.then_some("reserve-session");
if sticky {
record_admin_provider_pool_success(
app.runtime_state.as_ref(),
"provider-pool",
"key-low",
&pool_config,
sticky_token,
0,
None,
)
.await;
}
let mut cursor = PoolKeyCursor::new(
PlannerAppState::new(&app),
group,
sticky_token,
None,
None,
);
cursor.window_size = 1;
cursor.page_size = 1;
let mut returned = Vec::new();
while let Some(candidate) = cursor.next_key().await {
returned.push(candidate.candidate.key_id);
}
let reserve_reached = reserve_enabled && used_percent >= 99.0;
assert_eq!(
returned.contains(&"key-low".to_string()),
!reserve_reached,
"reserve={reserve_enabled}, sticky={sticky}, used={used_percent}"
);
assert!(returned.contains(&"key-ready".to_string()));
if reserve_reached {
assert_eq!(
cursor
.skip_reason_counts
.get(POOL_ACCOUNT_EXHAUSTED_SKIP_REASON),
Some(&1)
);
} else if sticky {
assert_eq!(returned.first().map(String::as_str), Some("key-low"));
}
}
}
}
}
#[tokio::test]
async fn pool_key_cursor_does_not_spend_effective_scan_budget_on_exhausted_accounts() {
let provider_config = Some(json!({
apps/aether-gateway/src/handlers/admin/provider/pool/config.rs
+21
View File
@@ -411,6 +411,7 @@ pub(crate) fn admin_provider_pool_config_from_config_value(
unschedulable_rules: Vec::new(),
lru_enabled: false,
skip_exhausted_accounts: false,
reserve_minimum_quota: false,
sticky_session_ttl_seconds: 3600,
latency_window_seconds: 3600,
latency_sample_limit: 50,
@@ -446,6 +447,10 @@ pub(crate) fn admin_provider_pool_config_from_config_value(
.get("skip_exhausted_accounts")
.and_then(Value::as_bool)
.unwrap_or(false),
reserve_minimum_quota: pool_advanced
.get("reserve_minimum_quota")
.and_then(Value::as_bool)
.unwrap_or(false),
sticky_session_ttl_seconds: pool_advanced
.get("sticky_session_ttl_seconds")
.and_then(json_u64)
@@ -574,6 +579,22 @@ mod tests {
let config = admin_provider_pool_config(&provider).expect("pool config should exist");
assert!(!config.skip_exhausted_accounts);
assert!(!config.reserve_minimum_quota);
}
#[test]
fn parses_reserve_minimum_quota_independently_of_skip_exhausted_accounts() {
for enabled in [false, true] {
let provider = sample_provider(json!({
"pool_advanced": {
"reserve_minimum_quota": enabled,
"skip_exhausted_accounts": false
}
}));
let config = admin_provider_pool_config(&provider).expect("pool config should exist");
assert_eq!(config.reserve_minimum_quota, enabled);
assert!(!config.skip_exhausted_accounts);
}
}
#[test]
apps/aether-gateway/src/handlers/admin/provider/pool/runtime/writes.rs
+1
View File
@@ -651,6 +651,7 @@ mod tests {
unschedulable_rules: Vec::new(),
lru_enabled: true,
skip_exhausted_accounts: false,
reserve_minimum_quota: false,
sticky_session_ttl_seconds: 120,
latency_window_seconds: 600,
latency_sample_limit: 10,
apps/aether-gateway/src/handlers/admin/provider/pool_admin/payloads.rs
+10 -4
View File
@@ -1117,10 +1117,16 @@ pub(super) fn build_admin_pool_key_payload(
let health_score = admin_pool_health_score(key);
let circuit_breaker_open = false;
let auth_semantics = provider_key_auth_semantics(key, provider_type);
let account_quota_exhausted = pool_config
.as_ref()
.is_some_and(|config| config.skip_exhausted_accounts)
&& admin_provider_pool_pure::admin_pool_key_account_quota_exhausted(key, provider_type);
let account_quota_exhausted = pool_config.as_ref().is_some_and(|config| {
(config.skip_exhausted_accounts
&& admin_provider_pool_pure::admin_pool_key_account_quota_exhausted(key, provider_type))
|| (config.reserve_minimum_quota
&& admin_provider_pool_pure::admin_pool_key_minimum_quota_reached(
key,
provider_type,
None,
))
});
let auth_config = state.parse_catalog_auth_config_json(key);
let oauth_expires_at =
admin_pool_derive_oauth_expires_at(provider_type, key, auth_config.as_ref());
apps/aether-gateway/src/handlers/admin/provider/pool_admin/read_routes/keys.rs
+10 -3
View File
@@ -488,9 +488,16 @@ pub(super) fn admin_pool_key_visible_status_filter(
) {
return status;
}
if pool_config.is_some_and(|config| config.skip_exhausted_accounts)
&& admin_provider_pool_pure::admin_pool_key_account_quota_exhausted(key, provider_type)
{
if pool_config.is_some_and(|config| {
(config.skip_exhausted_accounts
&& admin_provider_pool_pure::admin_pool_key_account_quota_exhausted(key, provider_type))
|| (config.reserve_minimum_quota
&& admin_provider_pool_pure::admin_pool_key_minimum_quota_reached(
key,
provider_type,
None,
))
}) {
return "quota_exhausted";
}
if !key.is_active {
apps/aether-gateway/src/handlers/admin/provider/query/models/model_test.rs
+10 -5
View File
@@ -1340,6 +1340,7 @@ async fn provider_query_apply_pool_scheduler_to_test_candidates(
provider.id.clone(),
provider_query_ai_pool_runtime_state(&runtime),
);
let reserve_minimum_quota = pool_config.reserve_minimum_quota;
let pool_config =
provider_query_ai_pool_scheduling_config(pool_config, provider.provider_type.as_str());
let inputs = keys
@@ -1351,6 +1352,14 @@ async fn provider_query_apply_pool_scheduler_to_test_candidates(
effective_model: effective_model.to_string(),
scheduler_skip_reason: None,
};
let mut key_context =
provider_query_pool_catalog_key_context(state, &key, &provider.provider_type);
key_context.quota_exhausted |= reserve_minimum_quota
&& admin_provider_pool_pure::admin_pool_key_minimum_quota_reached(
&key,
&provider.provider_type,
Some(effective_model),
);
AiPoolCandidateInput {
facts: AiPoolCandidateFacts {
provider_id: provider.id.clone(),
@@ -1362,11 +1371,7 @@ async fn provider_query_apply_pool_scheduler_to_test_candidates(
key_internal_priority: key.internal_priority,
},
pool_config: Some(pool_config.clone()),
key_context: provider_query_pool_catalog_key_context(
state,
&key,
&provider.provider_type,
),
key_context,
candidate,
}
})
apps/aether-gateway/src/handlers/admin/provider/shared/support.rs
+1
View File
@@ -65,6 +65,7 @@ pub(crate) struct AdminProviderPoolConfig {
pub(crate) unschedulable_rules: Vec<AdminProviderPoolUnschedulableRule>,
pub(crate) lru_enabled: bool,
pub(crate) skip_exhausted_accounts: bool,
pub(crate) reserve_minimum_quota: bool,
pub(crate) sticky_session_ttl_seconds: u64,
pub(crate) latency_window_seconds: u64,
pub(crate) latency_sample_limit: u64,
apps/aether-gateway/src/scheduler/candidate/runtime.rs
+81 -9
View File
@@ -40,10 +40,6 @@ pub(super) async fn read_candidate_runtime_selection_snapshot(
) -> Result<CandidateRuntimeSelectionSnapshot, GatewayError> {
let provider_concurrent_limits = read_provider_concurrent_limits(state, candidates).await?;
let provider_pool_state = read_provider_pool_state_map(state, candidates).await?;
let provider_skip_exhausted_accounts = provider_pool_state
.iter()
.map(|(provider_id, state)| (provider_id.clone(), state.skip_exhausted_accounts))
.collect::<BTreeMap<_, _>>();
let pool_provider_ids = provider_pool_state
.iter()
.filter_map(|(provider_id, state)| state.pool_enabled.then_some(provider_id.clone()))
@@ -62,7 +58,7 @@ pub(super) async fn read_candidate_runtime_selection_snapshot(
let key_account_quota_exhausted = read_key_account_quota_exhaustion_map(
candidates,
&provider_key_rpm_states,
&provider_skip_exhausted_accounts,
&provider_pool_state,
);
let key_oauth_invalid =
read_key_oauth_invalid_map(candidates, &provider_key_rpm_states, now_unix_secs);
@@ -360,6 +356,7 @@ async fn read_provider_quota_block_map(
struct ProviderPoolState {
pool_enabled: bool,
skip_exhausted_accounts: bool,
reserve_minimum_quota: bool,
}
async fn read_provider_pool_state_map(
@@ -391,11 +388,17 @@ async fn read_provider_pool_state_map(
.and_then(|value| value.get("skip_exhausted_accounts"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false);
let reserve_minimum_quota = pool_advanced
.and_then(serde_json::Value::as_object)
.and_then(|value| value.get("reserve_minimum_quota"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false);
(
provider.id,
ProviderPoolState {
pool_enabled: pool_advanced.is_some(),
skip_exhausted_accounts,
reserve_minimum_quota,
},
)
})
@@ -405,7 +408,7 @@ async fn read_provider_pool_state_map(
fn read_key_account_quota_exhaustion_map(
candidates: &[SchedulerMinimalCandidateSelectionCandidate],
provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
provider_skip_exhausted_accounts: &BTreeMap<String, bool>,
provider_pool_state: &BTreeMap<String, ProviderPoolState>,
) -> BTreeMap<String, bool> {
candidates
.iter()
@@ -431,11 +434,19 @@ fn read_key_account_quota_exhaustion_map(
candidate.provider_type.as_str(),
candidate.selected_provider_model_name.as_str(),
);
let skip_configured = provider_skip_exhausted_accounts
let pool_state = provider_pool_state
.get(candidate.provider_id.as_str())
.copied()
.unwrap_or(false);
hard_blocked || (skip_configured && account_exhausted)
.unwrap_or_default();
let reserve_reached = pool_state.reserve_minimum_quota
&& admin_provider_pool_pure::admin_pool_key_minimum_quota_reached(
key,
candidate.provider_type.as_str(),
Some(candidate.selected_provider_model_name.as_str()),
);
hard_blocked
|| reserve_reached
|| (pool_state.skip_exhausted_accounts && account_exhausted)
});
(candidate.key_id.clone(), exhausted)
})
@@ -566,3 +577,64 @@ fn read_provider_key_rpm_reset_at_map(
})
.collect::<BTreeMap<_, _>>()
}
#[cfg(test)]
mod reserve_minimum_quota_tests {
use super::*;
use serde_json::json;
#[test]
fn reserve_minimum_quota_is_independent_of_skip_exhausted_accounts() {
let candidate = SchedulerMinimalCandidateSelectionCandidate {
provider_id: "provider-codex".to_string(),
provider_name: "codex".to_string(),
provider_type: "codex".to_string(),
provider_priority: 0,
endpoint_id: "endpoint-codex".to_string(),
endpoint_api_format: "openai:responses".to_string(),
key_id: "key-codex".to_string(),
key_name: "codex".to_string(),
key_auth_type: "oauth".to_string(),
key_internal_priority: 0,
key_global_priority_for_format: None,
key_capabilities: None,
model_id: "model-codex".to_string(),
global_model_id: "global-model-codex".to_string(),
global_model_name: "gpt-5".to_string(),
selected_provider_model_name: "gpt-5".to_string(),
supports_streaming: true,
mapping_matched_model: None,
};
let mut key = StoredProviderCatalogKey::new(
candidate.key_id.clone(),
candidate.provider_id.clone(),
"codex".to_string(),
"oauth".to_string(),
None,
true,
)
.expect("key should build");
for reserve_enabled in [false, true] {
for used_percent in [99.0, 98.0] {
key.upstream_metadata =
Some(json!({"codex": {"primary_used_percent": used_percent}}));
let exhausted = read_key_account_quota_exhaustion_map(
std::slice::from_ref(&candidate),
&BTreeMap::from([(key.id.clone(), key.clone())]),
&BTreeMap::from([(
candidate.provider_id.clone(),
ProviderPoolState {
pool_enabled: true,
reserve_minimum_quota: reserve_enabled,
skip_exhausted_accounts: false,
},
)]),
);
assert_eq!(
exhausted.get(&key.id),
Some(&(reserve_enabled && used_percent >= 99.0))
);
}
}
}
}
apps/aether-gateway/src/tests/control/admin/pool.rs
+72
View File
@@ -2364,6 +2364,78 @@ async fn gateway_marks_exhausted_codex_pool_key_as_blocked_when_flag_enabled() {
assert_eq!(keys[0]["account_quota"], json!("5H剩余 0.0%"));
}
#[tokio::test]
async fn gateway_reserve_minimum_quota_marks_and_filters_codex_pool_keys() {
for (reserve_enabled, used_percent, exhausted) in [
(true, 99.0, true),
(false, 99.0, false),
(true, 98.0, false),
] {
let mut provider = sample_provider("provider-codex", "codex", 10);
provider.provider_type = "codex".to_string();
provider.config = Some(json!({
"pool_advanced": {
"reserve_minimum_quota": reserve_enabled,
"skip_exhausted_accounts": false,
"auto_remove_quota_exhausted_keys": true
}
}));
let mut key = sample_key(
"key-codex-reserved",
"provider-codex",
"openai:responses",
"oauth-placeholder",
);
key.auth_type = "oauth".to_string();
key.upstream_metadata = Some(json!({
"codex": {"primary_used_percent": used_percent}
}));
let state = AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests(
Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![provider],
Vec::new(),
vec![key],
)),
));
for status in ["all", "quota_exhausted", "available"] {
let response = local_admin_pool_response(
&state,
http::Method::GET,
&format!("/api/admin/pool/provider-codex/keys?page=1&page_size=50&status={status}"),
None,
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = serde_json::from_slice(
&to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read"),
)
.expect("json body should parse");
let keys = payload["keys"].as_array().expect("keys should be array");
let visible = status == "all" || (status == "quota_exhausted") == exhausted;
assert_eq!(
keys.len(),
usize::from(visible),
"reserve={reserve_enabled}, used={used_percent}, status={status}"
);
if visible {
assert_eq!(keys[0]["key_id"], "key-codex-reserved");
assert_eq!(
keys[0]["scheduling_reason"] == "account_quota_exhausted",
exhausted
);
if exhausted {
assert_eq!(keys[0]["scheduling_status"], "blocked");
assert_eq!(keys[0]["scheduling_label"], "额度耗尽");
}
}
}
}
}
#[tokio::test]
async fn gateway_lists_inherited_fixed_provider_api_formats_for_pool_keys() {
let mut provider = sample_provider("provider-codex", "codex", 10).with_transport_fields(
crates/aether-admin/src/provider/pool.rs
+12
View File
@@ -169,6 +169,18 @@ pub fn admin_pool_key_account_quota_exhausted(
aether_provider_pool::provider_pool_key_account_quota_exhausted(key, provider_type)
}
pub fn admin_pool_key_minimum_quota_reached(
key: &StoredProviderCatalogKey,
provider_type: &str,
provider_model_name: Option<&str>,
) -> bool {
aether_provider_pool::provider_pool_key_minimum_quota_reached(
key,
provider_type,
provider_model_name,
)
}
pub fn admin_pool_key_quota_hard_blocked(
key: &StoredProviderCatalogKey,
provider_type: &str,
crates/aether-provider/pool/src/lib.rs
+274 -5
View File
@@ -38,11 +38,11 @@ pub use providers::{
XAI_BILLING_PATH, XAI_USER_PATH,
};
pub use quota::{
provider_pool_key_account_quota_exhausted, provider_pool_key_model_quota_exhausted,
provider_pool_key_model_quota_hard_blocked, provider_pool_key_quota_hard_blocked,
provider_pool_key_scheduling_label, provider_pool_member_quota_snapshot,
provider_pool_quota_metadata_provider_type, provider_pool_quota_metadata_updated_at,
provider_pool_quota_snapshot_updated_at,
provider_pool_key_account_quota_exhausted, provider_pool_key_minimum_quota_reached,
provider_pool_key_model_quota_exhausted, provider_pool_key_model_quota_hard_blocked,
provider_pool_key_quota_hard_blocked, provider_pool_key_scheduling_label,
provider_pool_member_quota_snapshot, provider_pool_quota_metadata_provider_type,
provider_pool_quota_metadata_updated_at, provider_pool_quota_snapshot_updated_at,
};
pub use quota_refresh::ProviderPoolQuotaRequestSpec;
pub use service::ProviderPoolService;
@@ -727,6 +727,275 @@ mod tests {
assert!(unsupported.is_empty());
}
#[test]
fn codex_minimum_quota_respects_boundary_and_provider() {
for (used_percent, expected) in [
(98.9, false),
(98.99999, false),
(99.0, true),
(99.5, true),
(100.0, true),
] {
let key = sample_key(Some(json!({
"codex": { "primary_used_percent": used_percent }
})));
assert_eq!(
provider_pool_key_minimum_quota_reached(&key, "codex", None),
expected,
"used_percent={used_percent}"
);
assert!(!provider_pool_key_minimum_quota_reached(&key, "kiro", None));
}
assert!(!provider_pool_key_minimum_quota_reached(
&sample_key(None),
"codex",
None
));
}
#[test]
fn codex_minimum_quota_short_model_names_still_use_account_windows() {
for (used_percent, expected) in [(98.0, false), (99.0, true)] {
let key = sample_key(Some(json!({
"codex": { "primary_used_percent": used_percent }
})));
for model in ["o1", "o3", "", " "] {
assert_eq!(
provider_pool_key_minimum_quota_reached(&key, "codex", Some(model)),
expected,
"model={model:?}, used_percent={used_percent}"
);
}
}
}
#[test]
fn codex_minimum_quota_respects_windows_and_reset() {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("system time should be after unix epoch")
.as_secs();
for (reset_at, expected) in [(now + 3600, true), (now - 60, false)] {
let mut key = sample_key(None);
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"updated_at": now - 600,
"windows": [
{ "code": "5h", "used_ratio": 0.2 },
{ "code": "weekly", "used_ratio": 0.99, "reset_at": reset_at }
]
}
}));
for model in [None, Some("gpt-5.4")] {
assert_eq!(
provider_pool_key_minimum_quota_reached(&key, "codex", model),
expected
);
}
}
}
#[test]
fn codex_minimum_quota_checks_either_window_and_relative_resets() {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("system time should be after unix epoch")
.as_secs();
for prefix in ["primary", "secondary"] {
for (reset_seconds, expected) in [(3600, true), (60, false)] {
let key = sample_key(Some(json!({
"codex": {
"updated_at": now - 600,
"allowed": true,
"limit_reached": false,
format!("{prefix}_used_percent"): 99.0,
format!("{prefix}_reset_after_seconds"): reset_seconds
}
})));
for model in [None, Some("gpt-5.4")] {
assert_eq!(
provider_pool_key_minimum_quota_reached(&key, "codex", model),
expected,
"prefix={prefix}, reset_seconds={reset_seconds}, model={model:?}"
);
}
assert!(!provider_pool_key_account_quota_exhausted(&key, "codex"));
}
}
}
#[test]
fn codex_minimum_quota_uses_newest_source_with_or_without_model() {
for (snapshot_at, metadata_at, metadata_wins) in [
(Some(200), Some(100), false),
(Some(100), Some(200), true),
(None, None, false),
(Some(100), None, false),
(None, Some(100), true),
] {
for snapshot_reached in [false, true] {
let mut key = sample_key(Some(json!({
"codex": {
"updated_at": metadata_at,
"primary_used_percent": if snapshot_reached { 20.0 } else { 99.0 }
}
})));
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"observed_at": snapshot_at,
"windows": [{
"code": "5h",
"used_ratio": if snapshot_reached { 0.99 } else { 0.2 },
"is_exhausted": false
}]
}
}));
for model in [None, Some("gpt-5.4")] {
assert_eq!(
provider_pool_key_minimum_quota_reached(&key, "codex", model),
snapshot_reached != metadata_wins,
"snapshot_at={snapshot_at:?}, metadata_at={metadata_at:?}, model={model:?}"
);
}
}
}
}
#[test]
fn codex_minimum_quota_isolates_model_buckets_and_checks_each_model_window() {
for explicit_model in [false, true] {
let mut key = sample_key(None);
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"windows": [
{ "code": "5h", "used_ratio": 0.2 },
{
"code": "spark_5h", "used_ratio": 0.99,
"model": if explicit_model { Some("spark") } else { None },
"is_exhausted": false
},
{
"code": "spark_weekly", "used_ratio": 0.2,
"model": if explicit_model { Some("spark") } else { None },
"is_exhausted": false
}
]
}
}));
assert!(provider_pool_key_minimum_quota_reached(
&key,
"codex",
Some("gpt-5.3-codex-spark")
));
for model in [None, Some("gpt-5.4")] {
assert!(!provider_pool_key_minimum_quota_reached(
&key, "codex", model
));
}
assert_eq!(
provider_pool_key_model_quota_exhausted(&key, "codex", "gpt-5.3-codex-spark"),
Some(false)
);
}
for (account_percent, spark_percent) in [(99.0, 20.0), (20.0, 99.0)] {
let key = sample_key(Some(json!({
"codex": {
"primary_used_percent": account_percent,
"spark_primary_used_percent": spark_percent,
"spark_secondary_used_percent": 20.0
}
})));
assert_eq!(
provider_pool_key_minimum_quota_reached(&key, "codex", Some("gpt-5.3-codex-spark")),
spark_percent == 99.0
);
assert_eq!(
provider_pool_key_minimum_quota_reached(&key, "codex", Some("gpt-5.4")),
account_percent == 99.0
);
}
}
#[test]
fn codex_minimum_quota_keeps_model_bucket_when_account_metadata_is_newer() {
for (snapshot_ratio, account_percent) in [(0.2, 99.0), (0.99, 20.0)] {
let mut key = sample_key(Some(json!({
"codex": { "updated_at": 200, "primary_used_percent": account_percent }
})));
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"observed_at": 100,
"windows": [{ "code": "spark_5h", "used_ratio": snapshot_ratio }]
}
}));
assert_eq!(
provider_pool_key_minimum_quota_reached(&key, "codex", Some("gpt-5.3-codex-spark")),
snapshot_ratio == 0.99
);
assert_eq!(
provider_pool_key_minimum_quota_reached(&key, "codex", Some("gpt-5.4")),
account_percent == 99.0
);
key.upstream_metadata.as_mut().unwrap()["codex"]["spark_primary_used_percent"] =
json!(account_percent);
assert_eq!(
provider_pool_key_minimum_quota_reached(&key, "codex", Some("gpt-5.3-codex-spark")),
account_percent == 99.0,
"the newer observation for the same model bucket must win"
);
}
}
#[test]
fn codex_minimum_quota_supports_remaining_values_and_ignores_unknown_data() {
for (metrics, expected) in [
(json!({ "remaining_ratio": 0.01 }), true),
(json!({ "remaining_percent": "1" }), true),
(json!({ "remaining": 1, "limit": 100 }), true),
(json!({ "used_percent": "99" }), true),
(json!({ "used_ratio": null }), false),
(json!({ "used_ratio": "NaN" }), false),
(json!({ "remaining": 0, "limit": 0 }), false),
(json!({ "remaining_ratio": 0.01001 }), false),
] {
let key = sample_key(Some(json!({
"codex": { "quota_by_model": { "spark": metrics } }
})));
assert_eq!(
provider_pool_key_minimum_quota_reached(&key, "codex", Some("gpt-5.3-codex-spark")),
expected,
"metrics={metrics}"
);
assert!(!provider_pool_key_minimum_quota_reached(
&key, "codex", None
));
}
let disabled = sample_key(Some(json!({
"codex": { "primary_used_percent": 99.0, "primary_window_minutes": 0 }
})));
assert!(!provider_pool_key_minimum_quota_reached(
&disabled, "codex", None
));
let mut mismatched = sample_key(None);
mismatched.status_snapshot = Some(json!({
"quota": {
"provider_type": "kiro",
"windows": [{ "code": "weekly", "used_ratio": 0.99 }]
}
}));
assert!(!provider_pool_key_minimum_quota_reached(
&mismatched,
"codex",
None
));
}
#[test]
fn provider_quota_exhaustion_is_adapter_owned() {
assert!(provider_pool_key_account_quota_exhausted(
crates/aether-provider/pool/src/quota.rs
+125 -10
View File
@@ -78,8 +78,17 @@ pub(crate) fn provider_pool_model_quota_exhausted(
provider_type: &str,
provider_model_name: &str,
) -> Option<bool> {
let requested = provider_pool_identifier_tokens(provider_model_name);
if requested.is_empty() {
provider_pool_quota_reaches_reserve(key, provider_type, Some(provider_model_name), 0.0)
}
fn provider_pool_quota_reaches_reserve(
key: &StoredProviderCatalogKey,
provider_type: &str,
provider_model_name: Option<&str>,
reserve_ratio: f64,
) -> Option<bool> {
let requested = provider_model_name.map(provider_pool_identifier_tokens);
if reserve_ratio <= 0.0 && requested.as_ref().is_some_and(|tokens| tokens.is_empty()) {
return None;
}
@@ -92,8 +101,38 @@ pub(crate) fn provider_pool_model_quota_exhausted(
provider_pool_metadata_bucket(key.upstream_metadata.as_ref(), provider_type),
];
let mut resolved = None::<(Option<u64>, bool)>;
let mut resolved_specific_bucket = false;
for source in sources.into_iter().flatten() {
let windows = provider_pool_collect_quota_windows(source);
let mut windows = provider_pool_collect_quota_windows(source);
if reserve_ratio > 0.0 && provider_type.trim().eq_ignore_ascii_case("codex") {
// Raw refresh metadata can be newer than the materialized windows.
// Include all four legacy slots before selecting the model bucket.
for prefix in ["primary", "secondary", "spark_primary", "spark_secondary"] {
let Some(used_percent) = source.get(&format!("{prefix}_used_percent")) else {
continue;
};
if provider_pool_json_f64(source.get(&format!("{prefix}_window_minutes")))
== Some(0.0)
{
continue;
}
let mut window = Map::from_iter([
("code".to_string(), json!(prefix)),
("used_percent".to_string(), used_percent.clone()),
]);
for field in [
"reset_at",
"next_reset_at",
"reset_seconds",
"reset_after_seconds",
] {
if let Some(value) = source.get(&format!("{prefix}_{field}")) {
window.insert(field.to_string(), value.clone());
}
}
windows.push(window);
}
}
if windows.is_empty() {
continue;
}
@@ -102,19 +141,28 @@ pub(crate) fn provider_pool_model_quota_exhausted(
let model_matches = windows
.iter()
.filter(|window| {
provider_pool_window_explicitly_matches_model(window, provider_model_name)
provider_model_name.is_some_and(|model| {
provider_pool_window_explicitly_matches_model(window, model)
})
})
.collect::<Vec<_>>();
if !model_matches.is_empty() {
let exhausted =
provider_pool_explicit_model_windows_exhausted(model_matches, observed_at);
let exhausted = if reserve_ratio > 0.0 {
// Reserving quota protects every applicable rate-limit window,
// including short and weekly limits for an explicit model.
provider_pool_any_window_exhausted(model_matches, observed_at, reserve_ratio)
} else {
provider_pool_explicit_model_windows_exhausted(model_matches, observed_at)
};
if resolved.is_none()
|| (reserve_ratio > 0.0 && !resolved_specific_bucket)
|| provider_pool_should_replace_model_quota_resolution(
resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
observed_at,
)
{
resolved = Some((observed_at, exhausted));
resolved_specific_bucket = true;
}
continue;
}
@@ -125,21 +173,34 @@ pub(crate) fn provider_pool_model_quota_exhausted(
// windows isolated without baking in names such as "spark".
let family_matches = windows
.iter()
.filter(|window| provider_pool_window_family_matches_model(window, &requested))
.filter(|window| {
requested
.as_ref()
.is_some_and(|tokens| provider_pool_window_family_matches_model(window, tokens))
})
.collect::<Vec<_>>();
if !family_matches.is_empty() {
let exhausted = provider_pool_any_window_exhausted(family_matches, observed_at);
let exhausted =
provider_pool_any_window_exhausted(family_matches, observed_at, reserve_ratio);
if resolved.is_none()
|| (reserve_ratio > 0.0 && !resolved_specific_bucket)
|| provider_pool_should_replace_model_quota_resolution(
resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
observed_at,
)
{
resolved = Some((observed_at, exhausted));
resolved_specific_bucket = true;
}
continue;
}
// A newer account observation does not update an independent model
// bucket. Only compare freshness between applicable model sources.
if reserve_ratio > 0.0 && resolved_specific_bucket {
continue;
}
// Account-scoped windows (for example the ordinary weekly and short
// windows emitted by Codex) apply to every model that has no more
// specific family. Restrict this fallback to well-known structural
@@ -150,7 +211,8 @@ pub(crate) fn provider_pool_model_quota_exhausted(
.filter(|window| provider_pool_window_is_generic(window))
.collect::<Vec<_>>();
if !generic_matches.is_empty() {
let exhausted = provider_pool_any_window_exhausted(generic_matches, observed_at);
let exhausted =
provider_pool_any_window_exhausted(generic_matches, observed_at, reserve_ratio);
if resolved.is_none()
|| provider_pool_should_replace_model_quota_resolution(
resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
@@ -209,16 +271,69 @@ fn provider_pool_explicit_model_windows_exhausted(
fn provider_pool_any_window_exhausted(
windows: Vec<&Map<String, Value>>,
snapshot_observed_at: Option<u64>,
reserve_ratio: f64,
) -> bool {
let now_unix_secs = provider_pool_current_unix_secs();
windows.iter().any(|window| {
provider_pool_quota_window_is_exhausted(window)
provider_pool_quota_window_reaches_reserve(window, reserve_ratio)
&& !now_unix_secs.is_some_and(|now| {
provider_pool_reset_deadline_elapsed(window, snapshot_observed_at, now)
})
})
}
/// Whether an applicable Codex window has at most 1% remaining. Missing quota
/// data and windows whose reset has elapsed do not trigger this opt-in guard.
pub fn provider_pool_key_minimum_quota_reached(
key: &StoredProviderCatalogKey,
provider_type: &str,
provider_model_name: Option<&str>,
) -> bool {
if !provider_type.trim().eq_ignore_ascii_case("codex") {
return false;
}
provider_pool_quota_reaches_reserve(key, provider_type, provider_model_name, 0.01)
.unwrap_or(false)
}
fn provider_pool_quota_window_reaches_reserve(
window: &Map<String, Value>,
reserve_ratio: f64,
) -> bool {
if provider_pool_quota_window_is_exhausted(window) {
return true;
}
if reserve_ratio <= 0.0 {
return false;
}
let used_ratio = provider_pool_json_f64(window.get("used_ratio"))
.or_else(|| provider_pool_json_f64(window.get("usage_ratio")))
.or_else(|| provider_pool_json_f64(window.get("used_percent")).map(|value| value / 100.0))
.or_else(|| {
provider_pool_json_f64(window.get("remaining_ratio"))
.or_else(|| provider_pool_json_f64(window.get("remaining_fraction")))
.map(|value| 1.0 - value)
})
.or_else(|| {
provider_pool_json_f64(window.get("remaining_percent")).map(|value| 1.0 - value / 100.0)
})
.or_else(|| {
let remaining = provider_pool_json_f64(
window
.get("remaining")
.or_else(|| window.get("remaining_value")),
)?;
let limit = provider_pool_json_f64(
window
.get("limit")
.or_else(|| window.get("limit_value"))
.or_else(|| window.get("total")),
)?;
(limit > 0.0).then_some(1.0 - remaining / limit)
});
used_ratio.is_some_and(|used_ratio| used_ratio >= 1.0 - reserve_ratio)
}
fn provider_pool_window_is_generic(window: &Map<String, Value>) -> bool {
if provider_pool_window_has_explicit_model(window)
|| window
frontend/src/api/endpoints/types/provider.ts
+2
View File
@@ -822,6 +822,8 @@ export interface PoolAdvancedConfig {
sticky_session_ttl_seconds?: number | null
load_threshold_percent?: number | null
skip_exhausted_accounts?: boolean | null
// Codex only: treat remaining quota <= 1% as exhausted (default false).
reserve_minimum_quota?: boolean
// 旧字段（兼容读取）
lru_enabled?: boolean
scheduling_mode?: 'lru' | 'multi_score' | null
frontend/src/features/pool/components/PoolSchedulingDialog.vue
+31
View File
@@ -72,6 +72,33 @@
</div>
</div>
<div
v-if="isCodex"
class="flex items-start justify-between gap-4 rounded-2xl border border-border/60 bg-card/70 p-4"
>
<div class="space-y-1">
<label
for="pool-reserve-minimum-quota"
class="text-sm font-medium"
>
保留最低额度
</label>
<p
id="pool-reserve-minimum-quota-description"
class="text-xs leading-5 text-muted-foreground"
>
账号剩余额度不高于 1% 时提前标记为额度耗尽并停止调度，待额度恢复后再使用。
</p>
</div>
<Switch
id="pool-reserve-minimum-quota"
v-model="reserveMinimumQuota"
aria-describedby="pool-reserve-minimum-quota-description"
:disabled="loading"
class="mt-0.5 shrink-0"
/>
</div>
<!-- Section 2: 策略调度 (非互斥, 可叠加组合 + 拖拽排序) -->
<div class="space-y-4 rounded-2xl border border-border/60 bg-card/70 p-4">
<div class="space-y-1">
@@ -437,6 +464,8 @@ const FALLBACK_PRESET_DEFS: PoolPresetMeta[] = [
const { success, error: showError } = useToast()
const loading = ref(false)
const reserveMinimumQuota = ref(false)
const isCodex = computed(() => normalizeProviderType(props.providerType) === 'codex')
let dialogRevision = 0
const presetDefs = ref<PoolPresetMeta[]>([])
const presetDefsLoaded = ref(false)
@@ -850,6 +879,7 @@ watch([() => props.modelValue, () => props.providerId], async ([open]) => {
const revision = ++dialogRevision
loading.value = false
if (!open) return
reserveMinimumQuota.value = props.currentConfig?.reserve_minimum_quota === true
await ensurePresetDefsLoaded()
if (!props.modelValue || dialogRevision !== revision) return
presetList.value = normalizeMutexSelection(loadFromConfig(props.currentConfig))
@@ -883,6 +913,7 @@ async function handleSave() {
const latestAdvanced = latestProvider.pool_advanced
const mergedAdvanced = mergePoolAdvancedPatch(latestAdvanced, {
scheduling_presets: schedulingPresets,
...(isCodex.value ? { reserve_minimum_quota: reserveMinimumQuota.value } : {}),
})
const payload: Parameters<typeof updateProvider>[1] = {
pool_advanced: mergedAdvanced as PoolAdvancedConfig,
frontend/src/features/pool/components/__tests__/PoolSchedulingDialog.minimum-quota.spec.ts
+166
View File
@@ -0,0 +1,166 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { createApp, defineComponent, h, nextTick, ref, type App } from 'vue'
import type { PoolAdvancedConfig } from '@/api/endpoints/types/provider'
import PoolSchedulingDialog from '../PoolSchedulingDialog.vue'
const endpointMocks = vi.hoisted(() => ({
getPoolSchedulingPresets: vi.fn(),
getProvider: vi.fn(),
updateProvider: vi.fn(),
}))
vi.mock('@/api/endpoints', () => ({
getProvider: endpointMocks.getProvider,
updateProvider: endpointMocks.updateProvider,
}))
vi.mock('@/api/endpoints/pool', () => ({
getPoolSchedulingPresets: endpointMocks.getPoolSchedulingPresets,
}))
vi.mock('@/composables/useToast', () => ({
useToast: () => ({ success: vi.fn(), error: vi.fn() }),
}))
const mountedApps: Array<{ app: App, root: HTMLElement }> = []
async function settle(): Promise<void> {
for (let index = 0; index < 4; index += 1) {
await Promise.resolve()
await nextTick()
}
}
function mountDialog(providerType = 'codex', currentConfig: PoolAdvancedConfig | null = null) {
const root = document.createElement('div')
document.body.appendChild(root)
const open = ref(false)
const TestHost = defineComponent({
setup() {
void nextTick(() => { open.value = true })
return () => h(PoolSchedulingDialog, {
modelValue: open.value,
providerId: 'provider-1',
providerType,
currentConfig,
'onUpdate:modelValue': (value: boolean) => { open.value = value },
})
},
})
const app = createApp(TestHost)
app.mount(root)
mountedApps.push({ app, root })
return open
}
function quotaSwitch() {
return document.body.querySelector<HTMLButtonElement>('#pool-reserve-minimum-quota')
}
async function saveDialog() {
const saveButton = [...document.body.querySelectorAll<HTMLButtonElement>('button')]
.find(button => button.textContent?.trim() === '保存')
expect(saveButton).toBeDefined()
saveButton?.click()
await settle()
}
beforeEach(() => {
endpointMocks.getPoolSchedulingPresets.mockReset()
endpointMocks.getProvider.mockReset()
endpointMocks.updateProvider.mockReset()
endpointMocks.getPoolSchedulingPresets.mockResolvedValue([{
name: 'lru',
label: 'LRU 轮转',
description: '最久未使用的 Key 优先',
providers: [],
default_enabled: true,
modes: null,
default_mode: null,
mutex_group: 'distribution_mode',
}])
endpointMocks.getProvider.mockResolvedValue({ id: 'provider-1', pool_advanced: {} })
endpointMocks.updateProvider.mockResolvedValue({ id: 'provider-1' })
})
afterEach(() => {
for (const { app, root } of mountedApps.splice(0)) {
app.unmount()
root.remove()
}
document.body.innerHTML = ''
})
describe('PoolSchedulingDialog minimum quota reserve', () => {
it('defaults to disabled and saves enabling it while preserving the latest settings', async () => {
endpointMocks.getProvider.mockResolvedValue({
id: 'provider-1',
pool_advanced: { rate_limit_cooldown_seconds: 900, skip_exhausted_accounts: false },
})
mountDialog('codex', { rate_limit_cooldown_seconds: 300 })
await settle()
expect(quotaSwitch()?.getAttribute('aria-checked')).toBe('false')
expect(document.body.textContent).toContain('剩余额度不高于 1%')
quotaSwitch()?.click()
await nextTick()
await saveDialog()
expect(endpointMocks.updateProvider).toHaveBeenCalledWith('provider-1', {
pool_advanced: {
rate_limit_cooldown_seconds: 900,
skip_exhausted_accounts: false,
reserve_minimum_quota: true,
scheduling_presets: [{ preset: 'lru', enabled: true }],
},
})
})
it('loads an enabled reserve and saves disabling it', async () => {
mountDialog('codex', { reserve_minimum_quota: true })
await settle()
expect(quotaSwitch()?.getAttribute('aria-checked')).toBe('true')
quotaSwitch()?.click()
await nextTick()
await saveDialog()
expect(endpointMocks.updateProvider).toHaveBeenCalledWith('provider-1', {
pool_advanced: {
reserve_minimum_quota: false,
scheduling_presets: [{ preset: 'lru', enabled: true }],
},
})
})
it('discards unsaved reserve changes when the dialog is reopened', async () => {
const open = mountDialog()
await settle()
quotaSwitch()?.click()
await nextTick()
expect(quotaSwitch()?.getAttribute('aria-checked')).toBe('true')
open.value = false
await settle()
open.value = true
await settle()
expect(quotaSwitch()?.getAttribute('aria-checked')).toBe('false')
expect(endpointMocks.updateProvider).not.toHaveBeenCalled()
})
it.each(['openai', 'kiro'])('does not expose or add the reserve setting for %s', async (providerType) => {
mountDialog(providerType)
await settle()
expect(quotaSwitch()).toBeNull()
await saveDialog()
expect(endpointMocks.updateProvider).toHaveBeenCalledWith('provider-1', {
pool_advanced: {
scheduling_presets: [{ preset: 'lru', enabled: true }],
},
})
})
})
frontend/src/i18n/legacy-ui-messages.ts
+2
View File
@@ -1662,6 +1662,8 @@ export const legacyUiEnglishMessages: Record<string, string> = {
'依据窗口成本/Token 用量，缺失时回退配额使用率': 'Use window cost or token usage, falling back to quota utilization if unavailable',
'调度策略元数据加载失败，请重试': 'Failed to load scheduling policy metadata. Try again.',
'号池调度已保存': 'Pool scheduling saved',
'保留最低额度': 'Reserve minimum quota',
'账号剩余额度不高于 1% 时提前标记为额度耗尽并停止调度，待额度恢复后再使用。': 'Mark accounts as exhausted and stop scheduling when remaining quota is 1% or less, until quota recovers.',
'自适应热池': 'Adaptive warm pool',
'自动维护热池，缺口时异步补位。': 'Maintain a warm pool and replenish missing capacity asynchronously.',
'账号自检': 'Account self-check',