i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-03 16:07:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #852 from zhefox/main

Browse Source 
fix(codex): restore recovered quota status and stabilize lifecycle tests
This commit is contained in:
ZheFox
2026-09-24 11:16:05 +08:00
committed by GitHub
parent 57f53903f5 31d5e2d172
commit e3333f1aef
14 changed files with 955 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+1
View File
@@ -524,6 +524,7 @@ dependencies = [
"aether-data-contracts",
"aether-pool-core",
"aether-provider-transport",
"chrono",
"serde_json",
"url",
"uuid",
apps/aether-gateway/src/dispatch/pool_scheduler.rs
+21 -2
View File
@@ -4005,7 +4005,7 @@ mod tests {
async fn pool_key_cursor_reserve_minimum_quota_filters_pages_and_sticky_hits() {
for reserve_enabled in [false, true] {
for sticky in [false, true] {
for used_percent in [99.0, 98.0] {
for used_percent in [99.0, 98.0, 83.0] {
let provider_config = Some(json!({
"pool_advanced": {
"reserve_minimum_quota": reserve_enabled,
@@ -4016,8 +4016,27 @@ mod tests {
sample_codex_pool_provider("provider-pool", 0, provider_config.clone());
let endpoint = sample_codex_pool_endpoint("provider-pool", "endpoint-1");
let mut reserved = sample_codex_pool_key("provider-pool", "key-low");
reserved.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"updated_at": 100,
"allowed": false,
"exhausted": true,
"code": "exhausted",
"windows": [{
"code": "weekly",
"scope": "account",
"used_ratio": 1.0,
"reset_at": 4_102_444_800u64
}]
}
}));
reserved.upstream_metadata = Some(json!({
"codex": {"primary_used_percent": used_percent}
"codex": {
"updated_at": 200,
"primary_used_percent": used_percent,
"primary_reset_at": 4_102_444_800u64
}
}));
let ready = sample_codex_pool_key("provider-pool", "key-ready");
let rows = vec![
apps/aether-gateway/src/handlers/shared/catalog.rs
+135 -10
View File
@@ -21,6 +21,7 @@ use aether_crypto::{
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
use aether_provider_pool::{
grok_pool_tier_from_quota_bucket, grok_supported_quota_windows_for_tier,
provider_pool_codex_metadata_has_account_quota,
};
use aether_scheduler_core::provider_key_circuit_payload_is_active_open_at;
use serde_json::{json, Map, Value};
@@ -1111,9 +1112,8 @@ fn build_codex_quota_status_snapshot(
source: &str,
) -> Option<Value> {
let metadata = provider_quota_metadata_bucket(upstream_metadata, "codex")?;
let observed_at_unix_secs = metadata
.get("updated_at")
.and_then(admin_provider_quota_pure::coerce_json_u64);
let observed_at_unix_secs = provider_quota_timestamp_unix_secs(metadata.get("observed_at"))
.or_else(|| provider_quota_timestamp_unix_secs(metadata.get("updated_at")));
let plan_type = metadata
.get("plan_type")
.and_then(Value::as_str)
@@ -2502,17 +2502,19 @@ fn codex_upstream_metadata_is_at_least_as_fresh(
let Some(metadata) = provider_quota_metadata_bucket(upstream_metadata, "codex") else {
return false;
};
let Some(metadata_updated_at) = metadata
.get("updated_at")
.and_then(admin_provider_quota_pure::coerce_json_u64)
// Identity, reset-credit, and model-only updates do not replace the
// account's quota observation, even when their timestamp is newer.
if !provider_pool_codex_metadata_has_account_quota(metadata) {
return false;
}
let Some(metadata_updated_at) = provider_quota_timestamp_unix_secs(metadata.get("observed_at"))
.or_else(|| provider_quota_timestamp_unix_secs(metadata.get("updated_at")))
else {
return false;
};
let snapshot_updated_at = quota_snapshot.and_then(|quota| {
quota
.get("updated_at")
.or_else(|| quota.get("observed_at"))
.and_then(admin_provider_quota_pure::coerce_json_u64)
provider_quota_timestamp_unix_secs(quota.get("observed_at"))
.or_else(|| provider_quota_timestamp_unix_secs(quota.get("updated_at")))
});
snapshot_updated_at.is_none_or(|updated_at| metadata_updated_at >= updated_at)
@@ -2611,6 +2613,19 @@ pub(crate) fn provider_key_status_snapshot_payload(
let mut snapshot = provider_key_status_snapshot_object(Some(&payload))
.or_else(|| default_provider_key_status_snapshot().as_object().cloned())
.unwrap_or_default();
// Legacy snapshots can retain an exhausted summary after a window reset or
// newer quota observation. Use the same decision as scheduling so the
// account list and its status filter do not keep displaying that stale block.
if provider_type.trim().eq_ignore_ascii_case("codex")
&& !aether_provider_pool::provider_pool_key_account_quota_exhausted(key, provider_type)
{
if let Some(quota) = snapshot.get_mut("quota").and_then(Value::as_object_mut) {
quota.insert("exhausted".to_string(), json!(false));
if quota.get("code").and_then(Value::as_str) == Some("exhausted") {
quota.insert("code".to_string(), json!("ok"));
}
}
}
snapshot.insert(
"oauth".to_string(),
build_provider_key_oauth_status_snapshot(key),
@@ -4213,6 +4228,116 @@ mod tests {
);
}
#[test]
fn provider_key_status_snapshot_payload_clears_stale_codex_exhaustion_summary() {
let mut key = sample_catalog_key();
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"code": "exhausted",
"exhausted": true,
"windows": [{
"code": "weekly",
"scope": "account",
"used_ratio": 0.83,
"remaining_ratio": 0.17,
"reset_at": 4_102_444_800u64
}]
}
}));
let payload = provider_key_status_snapshot_payload(&key, "codex");
assert_eq!(payload["quota"]["code"], "ok");
assert_eq!(payload["quota"]["exhausted"], false);
assert!(payload["quota"]["label"].is_null());
// An explicit current upstream refusal is not a stale percentage summary.
key.status_snapshot.as_mut().unwrap()["quota"]["allowed"] = json!(false);
let payload = provider_key_status_snapshot_payload(&key, "codex");
assert_eq!(payload["quota"]["code"], "exhausted");
assert_eq!(payload["quota"]["exhausted"], true);
// Missing capacity evidence must not clear an exhausted summary either.
key.status_snapshot = Some(json!({"quota": {
"provider_type": "codex", "code": "exhausted", "exhausted": true
}}));
let payload = provider_key_status_snapshot_payload(&key, "codex");
assert_eq!(payload["quota"]["code"], "exhausted");
assert_eq!(payload["quota"]["exhausted"], true);
}
#[test]
fn provider_key_status_snapshot_payload_refreshes_codex_timestamp_formats() {
for updated_at in [
json!(1_900_000_000u64),
json!(1_900_000_000_000u64),
json!("2030-03-17T17:46:40Z"),
] {
let mut key = sample_catalog_key();
key.upstream_metadata = Some(json!({
"codex": {
"updated_at": updated_at,
"primary_used_percent": 83.0,
"primary_reset_at": 4_102_444_800u64
}
}));
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"updated_at": 1_899_999_000u64,
"code": "exhausted",
"exhausted": true,
"allowed": false,
"windows": [{
"code": "weekly",
"scope": "account",
"used_ratio": 1.0,
"remaining_ratio": 0.0,
"reset_at": 4_102_444_800u64
}]
}
}));
let payload = provider_key_status_snapshot_payload(&key, "codex");
assert_eq!(payload["quota"]["code"], "ok");
assert_eq!(payload["quota"]["updated_at"], 1_900_000_000u64);
assert_eq!(payload["quota"]["windows"][0]["used_ratio"], 0.83);
assert!(payload["quota"]["allowed"].is_null());
}
}
#[test]
fn provider_key_status_snapshot_payload_preserves_codex_account_quota_on_unrelated_updates() {
for patch in [
json!({"plan_type": "pro"}),
json!({"spark_primary_used_percent": 83.0}),
json!({"credits_unlimited": false}),
json!({"windows": [{"code": "weekly", "reset_at": 4_102_444_800u64}]}),
] {
let mut metadata = patch;
metadata["updated_at"] = json!("2030-03-17T17:46:40Z");
let mut key = sample_catalog_key();
key.upstream_metadata = Some(json!({"codex": metadata}));
key.status_snapshot = Some(json!({"quota": {
"provider_type": "codex", "updated_at": 200,
"code": "exhausted", "exhausted": true, "allowed": false,
"windows": [{"code": "weekly", "scope": "account", "used_ratio": 1.0,
"remaining_ratio": 0.0, "reset_at": 4_102_444_800u64}]
}}));
let payload = provider_key_status_snapshot_payload(&key, "codex");
assert_eq!(payload["quota"]["code"], "exhausted", "{metadata}");
assert_eq!(payload["quota"]["exhausted"], true, "{metadata}");
assert_eq!(payload["quota"]["allowed"], false, "{metadata}");
assert_eq!(payload["quota"]["updated_at"], 200, "{metadata}");
assert_eq!(
payload["quota"]["windows"][0]["code"], "weekly",
"{metadata}"
);
assert_eq!(
payload["quota"]["windows"][0]["used_ratio"], 1.0,
"{metadata}"
);
}
}
#[test]
fn provider_key_status_snapshot_payload_restores_complete_codex_cache() {
let mut key = sample_catalog_key();
apps/aether-gateway/src/tests/control/admin/pool.rs
+131 -1
View File
@@ -2436,6 +2436,134 @@ async fn gateway_reserve_minimum_quota_marks_and_filters_codex_pool_keys() {
}
}
#[tokio::test]
async fn gateway_reserve_minimum_quota_recovers_after_config_update_with_cached_catalog() {
let mut provider = sample_provider("provider-codex", "codex", 10);
provider.provider_type = "codex".to_string();
provider.config = Some(json!({
"pool_advanced": {"reserve_minimum_quota": true, "skip_exhausted_accounts": true}
}));
let mut key = sample_key(
"key-codex",
"provider-codex",
"openai:responses",
"oauth-placeholder",
);
key.auth_type = "oauth".to_string();
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex", "updated_at": 100,
"code": "exhausted", "exhausted": true, "allowed": false,
"windows": [{
"code": "weekly", "scope": "account", "used_ratio": 1.0,
"remaining_ratio": 0.0, "reset_at": 4_102_444_800u64
}]
}
}));
key.upstream_metadata = Some(json!({
"codex": {"updated_at": 200, "primary_used_percent": 99.0,
"primary_reset_at": 4_102_444_800u64}
}));
let repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![provider.clone()],
Vec::new(),
vec![key],
));
let state = AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(
GatewayDataState::with_provider_catalog_repository_for_tests(Arc::clone(&repository))
.with_cached_provider_catalog_reader_for_tests(repository),
);
for reserve_enabled in [true, false, true] {
provider.config.as_mut().unwrap()["pool_advanced"]["reserve_minimum_quota"] =
json!(reserve_enabled);
state
.update_provider_catalog_provider(&provider)
.await
.expect("provider should update");
for status in ["all", "quota_exhausted", "available"] {
let response = local_admin_pool_response(
&state,
http::Method::GET,
&format!("/api/admin/pool/provider-codex/keys?status={status}"),
None,
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = serde_json::from_slice(
&to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read"),
)
.expect("json should parse");
let keys = payload["keys"].as_array().expect("keys should be array");
let visible = status == "all" || (status == "quota_exhausted") == reserve_enabled;
assert_eq!(keys.len(), usize::from(visible));
if visible {
assert_eq!(
keys[0]["scheduling_reason"] == "account_quota_exhausted",
reserve_enabled
);
assert_eq!(keys[0]["status_snapshot"]["quota"]["code"], "ok");
}
}
}
}
#[tokio::test]
async fn gateway_codex_pool_filter_clears_stale_exhausted_summary_with_remaining_quota() {
let mut provider = sample_provider("provider-codex", "codex", 10);
provider.provider_type = "codex".to_string();
provider.config = Some(json!({"pool_advanced": {
"reserve_minimum_quota": false, "skip_exhausted_accounts": true
}}));
let mut key = sample_key(
"key-codex",
"provider-codex",
"openai:responses",
"oauth-placeholder",
);
key.auth_type = "oauth".to_string();
key.status_snapshot = Some(json!({"quota": {
"provider_type": "codex", "code": "exhausted", "exhausted": true,
"windows": [{"code": "weekly", "scope": "account", "used_ratio": 0.83,
"remaining_ratio": 0.17, "reset_at": 4_102_444_800u64}]
}}));
let state = AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests(
Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![provider],
Vec::new(),
vec![key],
)),
));
for status in ["all", "available", "quota_exhausted"] {
let response = local_admin_pool_response(
&state,
http::Method::GET,
&format!("/api/admin/pool/provider-codex/keys?status={status}"),
None,
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = serde_json::from_slice(
&to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read"),
)
.expect("json should parse");
let keys = payload["keys"].as_array().expect("keys should be array");
assert_eq!(keys.len(), usize::from(status != "quota_exhausted"));
if let Some(key) = keys.first() {
assert_eq!(key["scheduling_status"], "available");
assert_eq!(key["status_snapshot"]["quota"]["code"], "ok");
assert!(key["account_quota"].as_str().unwrap().contains("17.0%"));
}
}
}
#[tokio::test]
async fn gateway_lists_inherited_fixed_provider_api_formats_for_pool_keys() {
let mut provider = sample_provider("provider-codex", "codex", 10).with_transport_fields(
@@ -2738,8 +2866,10 @@ async fn gateway_treats_stale_codex_exhausted_snapshot_as_available_when_windows
assert_eq!(keys[0]["scheduling_reason"], json!("available"));
assert_eq!(
keys[0]["account_quota"],
json!("周剩余 100.0% (7天0小时后重置) | 5H剩余 100.0% (5小时0分钟后重置)")
json!("周剩余 100.0% | 5H剩余 100.0%")
);
assert_eq!(keys[0]["status_snapshot"]["quota"]["code"], "ok");
assert_eq!(keys[0]["status_snapshot"]["quota"]["exhausted"], false);
}
#[tokio::test]
crates/aether-provider/pool/Cargo.toml
+1
View File
@@ -11,6 +11,7 @@ aether-contracts.workspace = true
aether-data-contracts.workspace = true
aether-pool-core.workspace = true
aether-provider-transport.workspace = true
chrono.workspace = true
serde_json.workspace = true
url.workspace = true
uuid.workspace = true
crates/aether-provider/pool/src/lib.rs
+245 -5
View File
@@ -38,11 +38,12 @@ pub use providers::{
XAI_BILLING_PATH, XAI_USER_PATH,
};
pub use quota::{
provider_pool_key_account_quota_exhausted, provider_pool_key_minimum_quota_reached,
provider_pool_key_model_quota_exhausted, provider_pool_key_model_quota_hard_blocked,
provider_pool_key_quota_hard_blocked, provider_pool_key_scheduling_label,
provider_pool_member_quota_snapshot, provider_pool_quota_metadata_provider_type,
provider_pool_quota_metadata_updated_at, provider_pool_quota_snapshot_updated_at,
provider_pool_codex_metadata_has_account_quota, provider_pool_key_account_quota_exhausted,
provider_pool_key_minimum_quota_reached, provider_pool_key_model_quota_exhausted,
provider_pool_key_model_quota_hard_blocked, provider_pool_key_quota_hard_blocked,
provider_pool_key_scheduling_label, provider_pool_member_quota_snapshot,
provider_pool_quota_metadata_provider_type, provider_pool_quota_metadata_updated_at,
provider_pool_quota_snapshot_updated_at,
};
pub use quota_refresh::ProviderPoolQuotaRequestSpec;
pub use service::ProviderPoolService;
@@ -1600,6 +1601,245 @@ mod tests {
assert!(!signals.quota_exhausted);
}
#[test]
fn codex_newer_flat_quota_metadata_clears_stale_exhausted_snapshot() {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("system time should be after unix epoch")
.as_secs();
let service = ProviderPoolService::with_builtin_adapters();
// Headers and quota refreshes persist the flat metadata shape while the
// derived snapshot may still describe the previous quota observation.
for used_percent in [83.0, 99.0, 100.0] {
let mut key = sample_key(Some(json!({
"codex": {
"updated_at": now,
"primary_used_percent": used_percent,
"primary_reset_at": now + 3600
}
})));
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"observed_at": now - 60,
"updated_at": now - 60,
"code": "exhausted",
"exhausted": true,
"allowed": false,
"limit_reached": true,
"reset_at": now + 3600,
"windows": [{
"code": "weekly",
"scope": "account",
"used_ratio": 1.0,
"remaining_ratio": 0.0,
"is_exhausted": true,
"reset_at": now + 3600
}]
}
}));
for model in [None, Some("gpt-5.4"), Some("o3")] {
let signals = service.member_signals("codex", &key, None, model);
assert_eq!(signals.quota_exhausted, used_percent >= 100.0);
assert!(!signals.quota_hard_blocked);
assert_eq!(
provider_pool_key_minimum_quota_reached(&key, "codex", model),
used_percent >= 99.0
);
}
}
}
#[test]
fn codex_account_and_model_quota_agree_on_explicit_allow_and_deny_flags() {
let service = ProviderPoolService::with_builtin_adapters();
for (flags, exhausted) in [
(json!({ "allowed": true }), false),
(json!({ "limit_reached": false }), false),
(json!({ "allowed": true, "limit_reached": true }), true),
(json!({ "allowed": false, "limit_reached": false }), true),
] {
for use_windows in [false, true] {
let mut metadata = flags.clone();
metadata["updated_at"] = json!(300);
if use_windows {
metadata["windows"] = json!([{ "code": "weekly", "used_ratio": 1.0 }]);
} else {
metadata["primary_used_percent"] = json!(100.0);
}
let key = sample_key(Some(json!({ "codex": metadata })));
for model in [None, Some("gpt-5.4"), Some("o3")] {
assert_eq!(
service
.member_signals("codex", &key, None, model)
.quota_exhausted,
exhausted,
"flags={flags}, use_windows={use_windows}, model={model:?}"
);
// The opt-in reserve still protects a numerically full
// window even when the upstream reports it as allowed.
assert!(provider_pool_key_minimum_quota_reached(
&key, "codex", model
));
}
}
}
}
#[test]
fn codex_model_quota_honors_latest_account_refusal_without_blocking_spark() {
let service = ProviderPoolService::with_builtin_adapters();
for include_window in [false, true] {
let mut metadata = json!({
"updated_at": 300,
"allowed": false,
"limit_reached": true,
"spark_primary_used_percent": 17.0
});
if include_window {
metadata["primary_used_percent"] = json!(83.0);
}
let mut key = sample_key(Some(json!({ "codex": metadata })));
for include_snapshot in [false, true] {
if include_snapshot {
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"observed_at": 200,
"exhausted": false,
"windows": [{ "code": "weekly", "used_ratio": 0.83 }]
}
}));
}
assert!(
service
.member_signals("codex", &key, None, Some("gpt-5.4"))
.quota_exhausted
);
assert!(
!service
.member_signals("codex", &key, None, Some("gpt-5.3-codex-spark"))
.quota_exhausted
);
}
}
}
#[test]
fn codex_quota_source_freshness_supports_all_timestamp_formats() {
let service = ProviderPoolService::with_builtin_adapters();
for observed_at in [
json!(1_700_000_200_u64),
json!(1_700_000_200_000_u64),
json!("1700000200000"),
json!("2023-11-14T22:16:40Z"),
] {
for use_windows in [false, true] {
let metadata = if use_windows {
json!({
"updated_at": observed_at,
"windows": [{ "code": "weekly", "used_ratio": 0.83 }]
})
} else {
json!({ "updated_at": observed_at, "primary_used_percent": 83.0 })
};
let mut key = sample_key(Some(json!({ "codex": metadata })));
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"observed_at": "2023-11-14T22:15:00Z",
"exhausted": true,
"allowed": false,
"windows": [{ "code": "weekly", "used_ratio": 1.0 }]
}
}));
for model in [None, Some("gpt-5.4")] {
let signals = service.member_signals("codex", &key, None, model);
assert!(!signals.quota_exhausted, "timestamp={observed_at}");
assert!(!signals.quota_hard_blocked, "timestamp={observed_at}");
assert!(!provider_pool_key_minimum_quota_reached(
&key, "codex", model
));
}
// Swap freshness while retaining the same observations. An
// older usable bucket cannot erase a later exhausted snapshot.
key.status_snapshot.as_mut().unwrap()["quota"]["observed_at"] =
json!("2023-11-14T22:18:20Z");
assert!(provider_pool_key_account_quota_exhausted(&key, "codex"));
assert!(provider_pool_key_quota_hard_blocked(&key, "codex"));
assert_eq!(
provider_pool_key_model_quota_exhausted(&key, "codex", "gpt-5.4"),
Some(true)
);
}
}
}
#[test]
fn codex_account_quota_recovery_requires_new_account_observation() {
for metadata in [
json!({ "updated_at": 100, "primary_used_percent": 83.0 }),
json!({ "primary_used_percent": 83.0 }),
json!({ "updated_at": 300, "plan_type": "plus" }),
json!({ "updated_at": 300, "credits_unlimited": false }),
json!({ "updated_at": 300, "spark_primary_used_percent": 83.0 }),
json!({ "updated_at": 300, "windows": [{ "code": "weekly" }] }),
] {
let mut key = sample_key(Some(json!({ "codex": metadata })));
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"observed_at": 200,
"exhausted": true,
"allowed": false,
"limit_reached": true,
"windows": [{ "code": "weekly", "used_ratio": 1.0 }]
}
}));
assert!(provider_pool_key_account_quota_exhausted(&key, "codex"));
assert!(provider_pool_key_quota_hard_blocked(&key, "codex"));
assert_eq!(
provider_pool_key_model_quota_exhausted(&key, "codex", "gpt-5.4"),
Some(true)
);
}
let mut key = sample_key(Some(json!({
"codex": {
"updated_at": 300,
"primary_used_percent": 83.0,
"allowed": false,
"limit_reached": true
}
})));
key.status_snapshot = Some(json!({
"quota": { "provider_type": "codex", "updated_at": 200, "exhausted": false }
}));
assert!(provider_pool_key_account_quota_exhausted(&key, "codex"));
assert!(provider_pool_key_quota_hard_blocked(&key, "codex"));
}
#[test]
fn codex_account_updates_do_not_override_independent_model_quota() {
for (spark_ratio, account_percent) in [(0.83, 100.0), (1.0, 83.0)] {
let mut key = sample_key(Some(json!({
"codex": { "updated_at": 300, "primary_used_percent": account_percent }
})));
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"updated_at": 200,
"windows": [{ "code": "spark_5h", "used_ratio": spark_ratio }]
}
}));
assert_eq!(
provider_pool_key_model_quota_exhausted(&key, "codex", "gpt-5.3-codex-spark"),
Some(spark_ratio >= 1.0)
);
}
}
#[test]
fn codex_explicit_quota_block_is_hard_until_reset() {
let now = std::time::SystemTime::now()
crates/aether-provider/pool/src/providers/codex.rs
+46 -13
View File
@@ -10,10 +10,11 @@ use crate::provider::{
ProviderPoolMemberInput,
};
use crate::quota::{
provider_pool_current_unix_secs, provider_pool_json_bool, provider_pool_json_f64,
provider_pool_member_quota_snapshot, provider_pool_metadata_bucket,
provider_pool_model_quota_exhausted, provider_pool_quota_snapshot_exhausted_decision,
provider_pool_reset_deadline_elapsed, provider_pool_timestamp_unix_secs,
provider_pool_codex_metadata_has_account_quota, provider_pool_current_unix_secs,
provider_pool_json_bool, provider_pool_json_f64, provider_pool_member_quota_snapshot,
provider_pool_metadata_bucket, provider_pool_model_quota_exhausted,
provider_pool_quota_snapshot_exhausted_decision, provider_pool_reset_deadline_elapsed,
provider_pool_source_account_quota_exhausted, provider_pool_timestamp_unix_secs,
};
use crate::quota_refresh::ProviderPoolQuotaRequestSpec;
@@ -54,6 +55,9 @@ impl ProviderPoolAdapter for CodexProviderPoolAdapter {
}) {
return exhausted;
}
if let Some(bucket) = codex_newer_account_quota_metadata(input.key, input.provider_type) {
return quota_exhausted_from_bucket(bucket);
}
if let Some(quota_snapshot) =
provider_pool_member_quota_snapshot(input.key, input.provider_type)
{
@@ -108,15 +112,12 @@ fn codex_explicit_quota_block_active(
key: &aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey,
provider_type: &str,
) -> bool {
if let Some(bucket) = codex_newer_account_quota_metadata(key, provider_type) {
return codex_explicit_quota_block_from_bucket(bucket);
}
let Some(quota_snapshot) = provider_pool_member_quota_snapshot(key, provider_type) else {
return provider_pool_metadata_bucket(key.upstream_metadata.as_ref(), provider_type)
.is_some_and(|bucket| {
(provider_pool_json_bool(bucket.get("allowed")) == Some(false)
|| provider_pool_json_bool(bucket.get("limit_reached")) == Some(true))
&& !["primary", "secondary"]
.into_iter()
.any(|prefix| codex_window_reset_elapsed(bucket, prefix))
});
.is_some_and(codex_explicit_quota_block_from_bucket);
};
let explicitly_blocked = provider_pool_json_bool(quota_snapshot.get("allowed")) == Some(false)
|| provider_pool_json_bool(quota_snapshot.get("limit_reached")) == Some(true);
@@ -133,6 +134,35 @@ fn codex_explicit_quota_block_active(
})
}
fn codex_explicit_quota_block_from_bucket(bucket: &Map<String, Value>) -> bool {
(provider_pool_json_bool(bucket.get("allowed")) == Some(false)
|| provider_pool_json_bool(bucket.get("limit_reached")) == Some(true))
&& !["primary", "secondary"]
.into_iter()
.any(|prefix| codex_window_reset_elapsed(bucket, prefix))
}
/// A successful refresh can update raw metadata before the status snapshot.
/// Do not keep an old account-level block once a newer quota observation exists.
/// Identity-only or model-only updates cannot clear an account quota decision.
fn codex_newer_account_quota_metadata<'a>(
key: &'a aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey,
provider_type: &str,
) -> Option<&'a Map<String, Value>> {
let snapshot = provider_pool_member_quota_snapshot(key, provider_type)?;
let metadata = provider_pool_metadata_bucket(key.upstream_metadata.as_ref(), provider_type)?;
if !provider_pool_codex_metadata_has_account_quota(metadata) {
return None;
}
let metadata_observed_at = provider_pool_timestamp_unix_secs(metadata.get("observed_at"))
.or_else(|| provider_pool_timestamp_unix_secs(metadata.get("updated_at")))?;
let snapshot_observed_at = provider_pool_timestamp_unix_secs(snapshot.get("observed_at"))
.or_else(|| provider_pool_timestamp_unix_secs(snapshot.get("updated_at")));
snapshot_observed_at
.is_none_or(|observed_at| metadata_observed_at >= observed_at)
.then_some(metadata)
}
fn build_codex_wham_headers(
resolved_oauth_auth: Option<(String, String)>,
decrypted_api_key: Option<&str>,
@@ -322,11 +352,14 @@ pub(crate) fn quota_exhausted_from_bucket(bucket: &Map<String, Value>) -> bool {
if provider_pool_json_bool(bucket.get("credits_unlimited")) == Some(true) {
return false;
}
let account_windows_exhausted = provider_pool_source_account_quota_exhausted(bucket);
let has_window_data = provider_pool_json_f64(bucket.get("primary_used_percent")).is_some()
|| provider_pool_json_f64(bucket.get("secondary_used_percent")).is_some();
|| provider_pool_json_f64(bucket.get("secondary_used_percent")).is_some()
|| account_windows_exhausted.is_some();
if !has_window_data && provider_pool_json_bool(bucket.get("has_credits")) == Some(false) {
return true;
}
codex_window_used_percent_exhausted(bucket, "primary")
account_windows_exhausted == Some(true)
|| codex_window_used_percent_exhausted(bucket, "primary")
|| codex_window_used_percent_exhausted(bucket, "secondary")
}
crates/aether-provider/pool/src/quota.rs
+132 -11
View File
@@ -87,6 +87,7 @@ fn provider_pool_quota_reaches_reserve(
provider_model_name: Option<&str>,
reserve_ratio: f64,
) -> Option<bool> {
let is_codex = provider_type.trim().eq_ignore_ascii_case("codex");
let requested = provider_model_name.map(provider_pool_identifier_tokens);
if reserve_ratio <= 0.0 && requested.as_ref().is_some_and(|tokens| tokens.is_empty()) {
return None;
@@ -103,12 +104,19 @@ fn provider_pool_quota_reaches_reserve(
let mut resolved = None::<(Option<u64>, bool)>;
let mut resolved_specific_bucket = false;
for source in sources.into_iter().flatten() {
let observed_at = provider_pool_timestamp_unix_secs(source.get("observed_at"))
.or_else(|| provider_pool_timestamp_unix_secs(source.get("updated_at")));
let account_signal = (is_codex && reserve_ratio <= 0.0)
.then(|| provider_pool_codex_account_quota_signal(source, observed_at))
.flatten();
let mut windows = provider_pool_collect_quota_windows(source);
if reserve_ratio > 0.0 && provider_type.trim().eq_ignore_ascii_case("codex") {
if is_codex {
// Raw refresh metadata can be newer than the materialized windows.
// Include all four legacy slots before selecting the model bucket.
for prefix in ["primary", "secondary", "spark_primary", "spark_secondary"] {
let Some(used_percent) = source.get(&format!("{prefix}_used_percent")) else {
let Some(used_percent) =
provider_pool_json_f64(source.get(&format!("{prefix}_used_percent")))
else {
continue;
};
if provider_pool_json_f64(source.get(&format!("{prefix}_window_minutes")))
@@ -118,7 +126,7 @@ fn provider_pool_quota_reaches_reserve(
}
let mut window = Map::from_iter([
("code".to_string(), json!(prefix)),
("used_percent".to_string(), used_percent.clone()),
("used_percent".to_string(), json!(used_percent)),
]);
for field in [
"reset_at",
@@ -132,12 +140,21 @@ fn provider_pool_quota_reaches_reserve(
}
windows.push(window);
}
windows.retain(provider_pool_quota_window_has_observation);
if let Some(exhausted) = account_signal {
if !windows.iter().any(provider_pool_window_is_generic) {
// A flags-only quota refresh is still a newer account
// observation, but must not override a model-specific bucket.
windows.push(Map::from_iter([
("code".to_string(), json!("account")),
("is_exhausted".to_string(), json!(exhausted)),
]));
}
}
}
if windows.is_empty() {
continue;
}
let observed_at = provider_pool_timestamp_unix_secs(source.get("observed_at"))
.or_else(|| provider_pool_timestamp_unix_secs(source.get("updated_at")));
let model_matches = windows
.iter()
.filter(|window| {
@@ -155,7 +172,7 @@ fn provider_pool_quota_reaches_reserve(
provider_pool_explicit_model_windows_exhausted(model_matches, observed_at)
};
if resolved.is_none()
|| (reserve_ratio > 0.0 && !resolved_specific_bucket)
|| (is_codex && !resolved_specific_bucket)
|| provider_pool_should_replace_model_quota_resolution(
resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
observed_at,
@@ -183,7 +200,7 @@ fn provider_pool_quota_reaches_reserve(
let exhausted =
provider_pool_any_window_exhausted(family_matches, observed_at, reserve_ratio);
if resolved.is_none()
|| (reserve_ratio > 0.0 && !resolved_specific_bucket)
|| (is_codex && !resolved_specific_bucket)
|| provider_pool_should_replace_model_quota_resolution(
resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
observed_at,
@@ -197,7 +214,7 @@ fn provider_pool_quota_reaches_reserve(
// A newer account observation does not update an independent model
// bucket. Only compare freshness between applicable model sources.
if reserve_ratio > 0.0 && resolved_specific_bucket {
if is_codex && resolved_specific_bucket {
continue;
}
@@ -211,8 +228,9 @@ fn provider_pool_quota_reaches_reserve(
.filter(|window| provider_pool_window_is_generic(window))
.collect::<Vec<_>>();
if !generic_matches.is_empty() {
let exhausted =
provider_pool_any_window_exhausted(generic_matches, observed_at, reserve_ratio);
let exhausted = account_signal.unwrap_or_else(|| {
provider_pool_any_window_exhausted(generic_matches, observed_at, reserve_ratio)
});
if resolved.is_none()
|| provider_pool_should_replace_model_quota_resolution(
resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
@@ -227,6 +245,37 @@ fn provider_pool_quota_reaches_reserve(
resolved.map(|(_, exhausted)| exhausted)
}
fn provider_pool_codex_account_quota_signal(
source: &Map<String, Value>,
observed_at: Option<u64>,
) -> Option<bool> {
let allowed = provider_pool_json_bool(source.get("allowed"));
let limit_reached = provider_pool_json_bool(source.get("limit_reached"));
if allowed == Some(false) || limit_reached == Some(true) {
let reset_elapsed = provider_pool_current_unix_secs().is_some_and(|now| {
provider_pool_reset_deadline_elapsed(source, observed_at, now)
|| ["primary", "secondary"].into_iter().any(|prefix| {
let window = [
"reset_at",
"next_reset_at",
"reset_seconds",
"reset_after_seconds",
]
.into_iter()
.filter_map(|field| {
source
.get(&format!("{prefix}_{field}"))
.map(|value| (field.to_string(), value.clone()))
})
.collect::<Map<_, _>>();
provider_pool_reset_deadline_elapsed(&window, observed_at, now)
})
});
return (!reset_elapsed).then_some(true);
}
(allowed == Some(true) || limit_reached == Some(false)).then_some(false)
}
fn provider_pool_should_replace_model_quota_resolution(
previous_observed_at: Option<u64>,
next_observed_at: Option<u64>,
@@ -282,6 +331,69 @@ fn provider_pool_any_window_exhausted(
})
}
fn provider_pool_quota_window_has_observation(window: &Map<String, Value>) -> bool {
["is_exhausted", "exhausted"]
.into_iter()
.any(|field| provider_pool_json_bool(window.get(field)).is_some())
|| [
"used_ratio",
"usage_ratio",
"used_percent",
"remaining_ratio",
"remaining_fraction",
"remaining_percent",
]
.into_iter()
.any(|field| provider_pool_json_f64(window.get(field)).is_some())
|| (provider_pool_json_f64(
window
.get("remaining")
.or_else(|| window.get("remaining_value")),
)
.is_some()
&& provider_pool_json_f64(
window
.get("limit")
.or_else(|| window.get("limit_value"))
.or_else(|| window.get("total")),
)
.is_some_and(|limit| limit > 0.0))
}
pub(crate) fn provider_pool_source_account_quota_exhausted(
source: &Map<String, Value>,
) -> Option<bool> {
let windows = provider_pool_collect_quota_windows(source);
let account_windows = windows
.iter()
.filter(|window| provider_pool_window_is_generic(window))
.filter(|window| provider_pool_quota_window_has_observation(window))
.collect::<Vec<_>>();
if account_windows.is_empty() {
return None;
}
let observed_at = provider_pool_timestamp_unix_secs(source.get("observed_at"))
.or_else(|| provider_pool_timestamp_unix_secs(source.get("updated_at")));
Some(provider_pool_any_window_exhausted(
account_windows,
observed_at,
0.0,
))
}
/// Whether Codex metadata contains an account quota observation rather than an
/// identity-only update or an independent model quota bucket.
pub fn provider_pool_codex_metadata_has_account_quota(source: &Map<String, Value>) -> bool {
["primary_used_percent", "secondary_used_percent"]
.into_iter()
.any(|field| provider_pool_json_f64(source.get(field)).is_some())
|| provider_pool_source_account_quota_exhausted(source).is_some()
|| ["allowed", "limit_reached", "has_credits"]
.into_iter()
.any(|field| provider_pool_json_bool(source.get(field)).is_some())
|| provider_pool_json_bool(source.get("credits_unlimited")) == Some(true)
}
/// Whether an applicable Codex window has at most 1% remaining. Missing quota
/// data and windows whose reset has elapsed do not trigger this opt-in guard.
pub fn provider_pool_key_minimum_quota_reached(
@@ -700,7 +812,16 @@ pub(crate) fn provider_pool_json_f64(value: Option<&Value>) -> Option<f64> {
}
pub(crate) fn provider_pool_timestamp_unix_secs(value: Option<&Value>) -> Option<u64> {
let mut timestamp = provider_pool_json_f64(value)?;
let mut timestamp = match provider_pool_json_f64(value) {
Some(timestamp) => timestamp,
None => {
return chrono::DateTime::parse_from_rfc3339(value?.as_str()?.trim())
.ok()?
.timestamp()
.try_into()
.ok();
}
};
if timestamp <= 0.0 {
return None;
}
crates/aether-usage/runtime/src/runtime.rs
+55 -18
View File
@@ -10055,6 +10055,24 @@ mod tests {
2,
"the later direct caller should receive its own bounded write attempt"
);
// Direct persistence can finish before the submission worker joins the
// barrier handoff and accounts for its completed slot.
timeout(Duration::from_secs(1), async {
loop {
let snapshot = runtime.metrics_snapshot();
let submission = &runtime.lifecycle_submission.state;
if snapshot.terminal_submission_pending == 0
&& snapshot.ordered_lifecycle_pending == 0
&& snapshot.lifecycle_submission_pending == 0
&& submission.admission.available_permits() == submission.capacity
{
break;
}
sleep(Duration::from_millis(1)).await;
}
})
.await
.expect("failed terminal submission accounting and admission should drain");
let snapshot = runtime.metrics_snapshot();
assert_eq!(snapshot.terminal_submission_pending, 0);
assert_eq!(snapshot.ordered_lifecycle_pending, 0);
@@ -10162,24 +10180,43 @@ mod tests {
.await;
assert_eq!(remaining_policy_panics.load(Ordering::Acquire), 0);
let records = records.lock().expect("records lock");
assert_eq!(
records
.iter()
.filter(|record| record.request_id == healthy_request_id)
.count(),
2,
"the same terminal shard should continue processing healthy requests"
);
assert!(
records
.iter()
.filter(|record| record.request_id == failed_request_id)
.count()
== 1,
"only the later healthy attempt should persist for the panicked request"
);
drop(records);
{
let records = records.lock().expect("records lock");
assert_eq!(
records
.iter()
.filter(|record| record.request_id == healthy_request_id)
.count(),
2,
"the same terminal shard should continue processing healthy requests"
);
assert!(
records
.iter()
.filter(|record| record.request_id == failed_request_id)
.count()
== 1,
"only the later healthy attempt should persist for the panicked request"
);
}
// The final direct attempt also submits a barrier whose worker may
// account for completion after the persistence call has returned.
timeout(Duration::from_secs(1), async {
loop {
let snapshot = runtime.metrics_snapshot();
let submission = &runtime.lifecycle_submission.state;
if snapshot.terminal_submission_pending == 0
&& snapshot.ordered_lifecycle_pending == 0
&& snapshot.lifecycle_submission_pending == 0
&& submission.admission.available_permits() == submission.capacity
{
break;
}
sleep(Duration::from_millis(1)).await;
}
})
.await
.expect("panicked terminal submission accounting and admission should drain");
let snapshot = runtime.metrics_snapshot();
assert_eq!(snapshot.terminal_submission_pending, 0);
assert_eq!(snapshot.ordered_lifecycle_pending, 0);
frontend/src/api/__tests__/providers-pool-cache.spec.ts
+76
View File
@@ -0,0 +1,76 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
const { getMock, patchMock } = vi.hoisted(() => ({ getMock: vi.fn(), patchMock: vi.fn() }))
vi.mock('@/api/client', () => ({ default: { get: getMock, patch: patchMock } }))
import { getPoolOverview, listPoolKeys, listPoolScores } from '@/api/endpoints/pool'
import { getProvider, updateProvider } from '@/api/endpoints/providers'
import { cache } from '@/utils/cache'
const options = { cacheTtlMs: 30_000 }
const provider = { id: 'codex', pool_advanced: { reserve_minimum_quota: false } }
function deferred<T>() {
let resolve!: (value: T) => void
const promise = new Promise<T>((resolvePromise) => { resolve = resolvePromise })
return { promise, resolve }
}
beforeEach(() => {
cache.clear()
getMock.mockReset()
patchMock.mockReset()
patchMock.mockResolvedValue({ data: provider })
})
describe('provider pool settings cache invalidation', () => {
it('refreshes every cached key page, score page and overview after changing pool settings', async () => {
getMock.mockResolvedValue({ data: { version: 'old' } })
await getPoolOverview(options)
await listPoolKeys('codex', {}, options)
await listPoolKeys('codex', { page: 2, status: 'quota_exhausted' }, options)
await listPoolScores('codex', {}, options)
await listPoolKeys('codex-other', {}, options)
await updateProvider('codex', { pool_advanced: { reserve_minimum_quota: false } })
getMock.mockResolvedValue({ data: { version: 'new' } })
await expect(getPoolOverview(options)).resolves.toEqual({ version: 'new' })
await expect(listPoolKeys('codex', {}, options)).resolves.toEqual({ version: 'new' })
await expect(listPoolKeys('codex', { page: 2, status: 'quota_exhausted' }, options))
.resolves.toEqual({ version: 'new' })
await expect(listPoolScores('codex', {}, options)).resolves.toEqual({ version: 'new' })
await expect(listPoolKeys('codex-other', {}, options)).resolves.toEqual({ version: 'old' })
})
it('does not reuse or cache a key request started before the settings were saved', async () => {
const oldResponse = deferred<{ data: { version: string } }>()
const newResponse = deferred<{ data: { version: string } }>()
getMock.mockReturnValueOnce(oldResponse.promise).mockReturnValueOnce(newResponse.promise)
const oldRequest = listPoolKeys('codex', { page: 2 }, options)
await updateProvider('codex', { pool_advanced: { reserve_minimum_quota: false } })
const newRequest = listPoolKeys('codex', { page: 2 }, options)
expect(getMock).toHaveBeenCalledTimes(2)
oldResponse.resolve({ data: { version: 'old' } })
await oldRequest
const deduped = listPoolKeys('codex', { page: 2 }, options)
expect(getMock).toHaveBeenCalledTimes(2)
newResponse.resolve({ data: { version: 'new' } })
await expect(newRequest).resolves.toEqual({ version: 'new' })
await expect(deduped).resolves.toEqual({ version: 'new' })
await expect(listPoolKeys('codex', { page: 2 }, options)).resolves.toEqual({ version: 'new' })
})
it('does not reuse a provider detail request started before a successful save', async () => {
const oldResponse = deferred<{ data: typeof provider }>()
getMock.mockReturnValueOnce(oldResponse.promise).mockResolvedValueOnce({ data: provider })
const oldRequest = getProvider('codex')
await updateProvider('codex', { pool_advanced: { reserve_minimum_quota: false } })
const newRequest = getProvider('codex')
expect(getMock).toHaveBeenCalledTimes(2)
oldResponse.resolve({ data: { ...provider, pool_advanced: { reserve_minimum_quota: true } } })
await oldRequest
await expect(newRequest).resolves.toMatchObject(provider)
})
})
frontend/src/api/endpoints/providers.ts
+10 -1
View File
@@ -1,5 +1,5 @@
import client from '../client'
import { buildCacheKey, cachedRequest, dedupedRequest } from '@/utils/cache'
import { buildCacheKey, cache, cachedRequest, dedupedRequest } from '@/utils/cache'
import type {
ClaudeCodeAdvancedConfig,
FailoverRulesConfig,
@@ -142,6 +142,15 @@ export async function updateProvider(
requestOptions?: ProviderRequestOptions,
): Promise<ProviderWithEndpointsSummary> {
const response = await client.patch<ProviderWithEndpointsSummary>(`/api/admin/providers/${providerId}`, data, requestOptions)
cache.delete(`providers:detail:${providerId}`)
if ('pool_advanced' in data) {
cache.delete('pool:overview')
for (const kind of ['keys', 'scores']) {
const prefix = `pool:${kind}:${providerId}`
cache.delete(prefix)
cache.deleteByPrefix(`${prefix}:`)
}
}
return normalizeProviderSummary(response.data)
}
frontend/src/utils/cache.ts
+8
View File
@@ -61,6 +61,14 @@ class MemoryCache {
this.keyVersions.set(key, (this.keyVersions.get(key) ?? 0) + 1)
}
/** 删除匹配前缀的缓存及进行中请求，旧响应不再回填缓存。 */
deleteByPrefix(prefix: string): void {
const keys = new Set([...this.cache.keys(), ...this.inFlight.keys()])
for (const key of keys) {
if (key.startsWith(prefix)) this.delete(key)
}
}
/**
* 清空所有缓存
*/
frontend/src/views/admin/PoolManagement.vue
+3 -1
View File
@@ -1395,6 +1395,8 @@ async function loadOverview(options: { cacheTtlMs?: number, silent?: boolean } =
async function handleSchedulingSaved(updatedProvider: ProviderWithEndpointsSummary) {
if (!selectedProviderId.value || updatedProvider.id !== selectedProviderId.value) return
// 保存前发出的详情读取不得覆盖这次保存返回的新配置。
providerDataRequestId += 1
// 优先回写保存接口返回值，避免弹窗立即重开时读到旧配置。
if (selectedProviderData.value) {
Object.assign(selectedProviderData.value, updatedProvider)
@@ -1403,7 +1405,7 @@ async function handleSchedulingSaved(updatedProvider: ProviderWithEndpointsSumma
}
showSchedulingDialog.value = false
showAdvancedDialog.value = false
await loadOverview({ silent: true })
await Promise.all([loadKeys({ silent: true }), loadOverview({ silent: true })])
}
// --- Provider Selection ---
frontend/src/views/admin/__tests__/PoolManagement.codex-cycle-stats.spec.ts
+91 -3
View File
@@ -340,12 +340,24 @@ vi.mock('@/components/ui/refresh-button.vue', async () => {
})
vi.mock('@/features/pool/components/PoolSchedulingDialog.vue', async () => {
const { defineComponent } = await import('vue')
const { defineComponent, h } = await import('vue')
return {
default: defineComponent({
name: 'PoolSchedulingDialogStub',
setup() {
return () => null
props: {
providerId: { type: String, required: true },
currentConfig: { type: Object, default: null },
},
emits: ['saved'],
setup(props, { emit }) {
return () => h('button', {
'data-testid': 'save-pool-scheduling',
'data-reserve': String(props.currentConfig?.reserve_minimum_quota ?? false),
onClick: () => emit('saved', {
id: props.providerId,
pool_advanced: { ...props.currentConfig, reserve_minimum_quota: false },
}),
}, '保存调度测试')
},
}),
}
@@ -1278,4 +1290,80 @@ describe('PoolManagement Codex cycle stats mode', () => {
expect(disabledRoot.querySelector('[data-testid="pool-demand-metrics-button"]')).toBeNull()
})
it('reloads account scheduling state immediately after disabling minimum quota reserve', async () => {
const key = createPoolKey('codex', {
scheduling_status: 'blocked',
scheduling_reason: 'account_quota_exhausted',
scheduling_label: '额度耗尽',
})
endpointMocks.getPoolOverview.mockResolvedValue({ items: [createOverview('codex')] })
endpointMocks.listPoolKeys.mockResolvedValue(createKeyPage(key))
endpointMocks.getProvider.mockResolvedValue(createProvider('codex', {
pool_advanced: { reserve_minimum_quota: true },
}))
const root = mountPoolManagement()
await settle()
expect(root.querySelector('tbody')?.textContent).toContain('额度耗尽')
endpointMocks.listPoolKeys.mockClear()
endpointMocks.listPoolKeys.mockResolvedValue(createKeyPage({
...key,
scheduling_status: 'available',
scheduling_reason: 'available',
scheduling_label: '可用',
}))
root.querySelector<HTMLButtonElement>('[data-testid="save-pool-scheduling"]')?.click()
await settle()
expect(endpointMocks.listPoolKeys).toHaveBeenCalledWith(
'codex-provider', expect.anything(), { cacheTtlMs: 0 },
)
expect(root.querySelector('tbody')?.textContent).not.toContain('额度耗尽')
expect(root.querySelector('tbody')?.textContent).toContain('可用')
})
it('shows 17% remaining without marking a schedulable account exhausted when reserve is enabled', async () => {
endpointMocks.getPoolOverview.mockResolvedValue({ items: [createOverview('codex')] })
endpointMocks.getProvider.mockResolvedValue(createProvider('codex', {
pool_advanced: { reserve_minimum_quota: true },
}))
endpointMocks.listPoolKeys.mockResolvedValue(createKeyPage(createPoolKey('codex', {
scheduling_status: 'available',
scheduling_reason: 'available',
scheduling_label: '可用',
status_snapshot: {
account: { code: 'ok', blocked: false },
quota: {
code: 'ok',
exhausted: false,
provider_type: 'codex',
windows: [{ code: '5h', used_ratio: 0.83, remaining_ratio: 0.17 }],
},
oauth: { code: 'none' },
},
})))
const root = mountPoolManagement()
await settle()
expect(root.querySelector('[data-testid="pool-quota-meter-text"]')?.textContent).toBe('17.0%')
expect(root.querySelector('tbody')?.textContent).toContain('可用')
expect(root.querySelector('tbody')?.textContent).not.toContain('额度耗尽')
})
it('does not replace saved scheduling settings with an older provider detail response', async () => {
let resolveOldProvider!: (provider: ReturnType<typeof createProvider>) => void
endpointMocks.getProvider.mockReturnValue(new Promise((resolve) => { resolveOldProvider = resolve }))
endpointMocks.getPoolOverview.mockResolvedValue({ items: [createOverview('codex')] })
endpointMocks.listPoolKeys.mockResolvedValue(createKeyPage(createPoolKey()))
const root = mountPoolManagement()
await settle()
root.querySelector<HTMLButtonElement>('[data-testid="save-pool-scheduling"]')?.click()
await settle()
resolveOldProvider(createProvider('codex', { pool_advanced: { reserve_minimum_quota: true } }))
await settle()
expect(root.querySelector('[data-testid="save-pool-scheduling"]')?.getAttribute('data-reserve')).toBe('false')
})
})