diff --git a/Cargo.lock b/Cargo.lock index e475509d2..991a6330a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -524,6 +524,7 @@ dependencies = [ "aether-data-contracts", "aether-pool-core", "aether-provider-transport", + "chrono", "serde_json", "url", "uuid", diff --git a/apps/aether-gateway/src/dispatch/pool_scheduler.rs b/apps/aether-gateway/src/dispatch/pool_scheduler.rs index cdfde206c..5a86b5590 100644 --- a/apps/aether-gateway/src/dispatch/pool_scheduler.rs +++ b/apps/aether-gateway/src/dispatch/pool_scheduler.rs @@ -4005,7 +4005,7 @@ mod tests { async fn pool_key_cursor_reserve_minimum_quota_filters_pages_and_sticky_hits() { for reserve_enabled in [false, true] { for sticky in [false, true] { - for used_percent in [99.0, 98.0] { + for used_percent in [99.0, 98.0, 83.0] { let provider_config = Some(json!({ "pool_advanced": { "reserve_minimum_quota": reserve_enabled, @@ -4016,8 +4016,27 @@ mod tests { sample_codex_pool_provider("provider-pool", 0, provider_config.clone()); let endpoint = sample_codex_pool_endpoint("provider-pool", "endpoint-1"); let mut reserved = sample_codex_pool_key("provider-pool", "key-low"); + reserved.status_snapshot = Some(json!({ + "quota": { + "provider_type": "codex", + "updated_at": 100, + "allowed": false, + "exhausted": true, + "code": "exhausted", + "windows": [{ + "code": "weekly", + "scope": "account", + "used_ratio": 1.0, + "reset_at": 4_102_444_800u64 + }] + } + })); reserved.upstream_metadata = Some(json!({ - "codex": {"primary_used_percent": used_percent} + "codex": { + "updated_at": 200, + "primary_used_percent": used_percent, + "primary_reset_at": 4_102_444_800u64 + } })); let ready = sample_codex_pool_key("provider-pool", "key-ready"); let rows = vec![ diff --git a/apps/aether-gateway/src/handlers/shared/catalog.rs b/apps/aether-gateway/src/handlers/shared/catalog.rs index b8c585c4c..96ab2f78c 100644 --- a/apps/aether-gateway/src/handlers/shared/catalog.rs +++ b/apps/aether-gateway/src/handlers/shared/catalog.rs @@ -21,6 +21,7 @@ use aether_crypto::{ use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey; use aether_provider_pool::{ grok_pool_tier_from_quota_bucket, grok_supported_quota_windows_for_tier, + provider_pool_codex_metadata_has_account_quota, }; use aether_scheduler_core::provider_key_circuit_payload_is_active_open_at; use serde_json::{json, Map, Value}; @@ -1111,9 +1112,8 @@ fn build_codex_quota_status_snapshot( source: &str, ) -> Option { let metadata = provider_quota_metadata_bucket(upstream_metadata, "codex")?; - let observed_at_unix_secs = metadata - .get("updated_at") - .and_then(admin_provider_quota_pure::coerce_json_u64); + let observed_at_unix_secs = provider_quota_timestamp_unix_secs(metadata.get("observed_at")) + .or_else(|| provider_quota_timestamp_unix_secs(metadata.get("updated_at"))); let plan_type = metadata .get("plan_type") .and_then(Value::as_str) @@ -2502,17 +2502,19 @@ fn codex_upstream_metadata_is_at_least_as_fresh( let Some(metadata) = provider_quota_metadata_bucket(upstream_metadata, "codex") else { return false; }; - let Some(metadata_updated_at) = metadata - .get("updated_at") - .and_then(admin_provider_quota_pure::coerce_json_u64) + // Identity, reset-credit, and model-only updates do not replace the + // account's quota observation, even when their timestamp is newer. + if !provider_pool_codex_metadata_has_account_quota(metadata) { + return false; + } + let Some(metadata_updated_at) = provider_quota_timestamp_unix_secs(metadata.get("observed_at")) + .or_else(|| provider_quota_timestamp_unix_secs(metadata.get("updated_at"))) else { return false; }; let snapshot_updated_at = quota_snapshot.and_then(|quota| { - quota - .get("updated_at") - .or_else(|| quota.get("observed_at")) - .and_then(admin_provider_quota_pure::coerce_json_u64) + provider_quota_timestamp_unix_secs(quota.get("observed_at")) + .or_else(|| provider_quota_timestamp_unix_secs(quota.get("updated_at"))) }); snapshot_updated_at.is_none_or(|updated_at| metadata_updated_at >= updated_at) @@ -2611,6 +2613,19 @@ pub(crate) fn provider_key_status_snapshot_payload( let mut snapshot = provider_key_status_snapshot_object(Some(&payload)) .or_else(|| default_provider_key_status_snapshot().as_object().cloned()) .unwrap_or_default(); + // Legacy snapshots can retain an exhausted summary after a window reset or + // newer quota observation. Use the same decision as scheduling so the + // account list and its status filter do not keep displaying that stale block. + if provider_type.trim().eq_ignore_ascii_case("codex") + && !aether_provider_pool::provider_pool_key_account_quota_exhausted(key, provider_type) + { + if let Some(quota) = snapshot.get_mut("quota").and_then(Value::as_object_mut) { + quota.insert("exhausted".to_string(), json!(false)); + if quota.get("code").and_then(Value::as_str) == Some("exhausted") { + quota.insert("code".to_string(), json!("ok")); + } + } + } snapshot.insert( "oauth".to_string(), build_provider_key_oauth_status_snapshot(key), @@ -4213,6 +4228,116 @@ mod tests { ); } + #[test] + fn provider_key_status_snapshot_payload_clears_stale_codex_exhaustion_summary() { + let mut key = sample_catalog_key(); + key.status_snapshot = Some(json!({ + "quota": { + "provider_type": "codex", + "code": "exhausted", + "exhausted": true, + "windows": [{ + "code": "weekly", + "scope": "account", + "used_ratio": 0.83, + "remaining_ratio": 0.17, + "reset_at": 4_102_444_800u64 + }] + } + })); + let payload = provider_key_status_snapshot_payload(&key, "codex"); + assert_eq!(payload["quota"]["code"], "ok"); + assert_eq!(payload["quota"]["exhausted"], false); + assert!(payload["quota"]["label"].is_null()); + + // An explicit current upstream refusal is not a stale percentage summary. + key.status_snapshot.as_mut().unwrap()["quota"]["allowed"] = json!(false); + let payload = provider_key_status_snapshot_payload(&key, "codex"); + assert_eq!(payload["quota"]["code"], "exhausted"); + assert_eq!(payload["quota"]["exhausted"], true); + + // Missing capacity evidence must not clear an exhausted summary either. + key.status_snapshot = Some(json!({"quota": { + "provider_type": "codex", "code": "exhausted", "exhausted": true + }})); + let payload = provider_key_status_snapshot_payload(&key, "codex"); + assert_eq!(payload["quota"]["code"], "exhausted"); + assert_eq!(payload["quota"]["exhausted"], true); + } + + #[test] + fn provider_key_status_snapshot_payload_refreshes_codex_timestamp_formats() { + for updated_at in [ + json!(1_900_000_000u64), + json!(1_900_000_000_000u64), + json!("2030-03-17T17:46:40Z"), + ] { + let mut key = sample_catalog_key(); + key.upstream_metadata = Some(json!({ + "codex": { + "updated_at": updated_at, + "primary_used_percent": 83.0, + "primary_reset_at": 4_102_444_800u64 + } + })); + key.status_snapshot = Some(json!({ + "quota": { + "provider_type": "codex", + "updated_at": 1_899_999_000u64, + "code": "exhausted", + "exhausted": true, + "allowed": false, + "windows": [{ + "code": "weekly", + "scope": "account", + "used_ratio": 1.0, + "remaining_ratio": 0.0, + "reset_at": 4_102_444_800u64 + }] + } + })); + let payload = provider_key_status_snapshot_payload(&key, "codex"); + assert_eq!(payload["quota"]["code"], "ok"); + assert_eq!(payload["quota"]["updated_at"], 1_900_000_000u64); + assert_eq!(payload["quota"]["windows"][0]["used_ratio"], 0.83); + assert!(payload["quota"]["allowed"].is_null()); + } + } + + #[test] + fn provider_key_status_snapshot_payload_preserves_codex_account_quota_on_unrelated_updates() { + for patch in [ + json!({"plan_type": "pro"}), + json!({"spark_primary_used_percent": 83.0}), + json!({"credits_unlimited": false}), + json!({"windows": [{"code": "weekly", "reset_at": 4_102_444_800u64}]}), + ] { + let mut metadata = patch; + metadata["updated_at"] = json!("2030-03-17T17:46:40Z"); + let mut key = sample_catalog_key(); + key.upstream_metadata = Some(json!({"codex": metadata})); + key.status_snapshot = Some(json!({"quota": { + "provider_type": "codex", "updated_at": 200, + "code": "exhausted", "exhausted": true, "allowed": false, + "windows": [{"code": "weekly", "scope": "account", "used_ratio": 1.0, + "remaining_ratio": 0.0, "reset_at": 4_102_444_800u64}] + }})); + let payload = provider_key_status_snapshot_payload(&key, "codex"); + assert_eq!(payload["quota"]["code"], "exhausted", "{metadata}"); + assert_eq!(payload["quota"]["exhausted"], true, "{metadata}"); + assert_eq!(payload["quota"]["allowed"], false, "{metadata}"); + assert_eq!(payload["quota"]["updated_at"], 200, "{metadata}"); + assert_eq!( + payload["quota"]["windows"][0]["code"], "weekly", + "{metadata}" + ); + assert_eq!( + payload["quota"]["windows"][0]["used_ratio"], 1.0, + "{metadata}" + ); + } + } + #[test] fn provider_key_status_snapshot_payload_restores_complete_codex_cache() { let mut key = sample_catalog_key(); diff --git a/apps/aether-gateway/src/tests/control/admin/pool.rs b/apps/aether-gateway/src/tests/control/admin/pool.rs index 8b8e4ae4a..21edb44b2 100644 --- a/apps/aether-gateway/src/tests/control/admin/pool.rs +++ b/apps/aether-gateway/src/tests/control/admin/pool.rs @@ -2436,6 +2436,134 @@ async fn gateway_reserve_minimum_quota_marks_and_filters_codex_pool_keys() { } } +#[tokio::test] +async fn gateway_reserve_minimum_quota_recovers_after_config_update_with_cached_catalog() { + let mut provider = sample_provider("provider-codex", "codex", 10); + provider.provider_type = "codex".to_string(); + provider.config = Some(json!({ + "pool_advanced": {"reserve_minimum_quota": true, "skip_exhausted_accounts": true} + })); + let mut key = sample_key( + "key-codex", + "provider-codex", + "openai:responses", + "oauth-placeholder", + ); + key.auth_type = "oauth".to_string(); + key.status_snapshot = Some(json!({ + "quota": { + "provider_type": "codex", "updated_at": 100, + "code": "exhausted", "exhausted": true, "allowed": false, + "windows": [{ + "code": "weekly", "scope": "account", "used_ratio": 1.0, + "remaining_ratio": 0.0, "reset_at": 4_102_444_800u64 + }] + } + })); + key.upstream_metadata = Some(json!({ + "codex": {"updated_at": 200, "primary_used_percent": 99.0, + "primary_reset_at": 4_102_444_800u64} + })); + let repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( + vec![provider.clone()], + Vec::new(), + vec![key], + )); + let state = AppState::new() + .expect("gateway should build") + .with_data_state_for_tests( + GatewayDataState::with_provider_catalog_repository_for_tests(Arc::clone(&repository)) + .with_cached_provider_catalog_reader_for_tests(repository), + ); + for reserve_enabled in [true, false, true] { + provider.config.as_mut().unwrap()["pool_advanced"]["reserve_minimum_quota"] = + json!(reserve_enabled); + state + .update_provider_catalog_provider(&provider) + .await + .expect("provider should update"); + for status in ["all", "quota_exhausted", "available"] { + let response = local_admin_pool_response( + &state, + http::Method::GET, + &format!("/api/admin/pool/provider-codex/keys?status={status}"), + None, + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let payload: serde_json::Value = serde_json::from_slice( + &to_bytes(response.into_body(), usize::MAX) + .await + .expect("body should read"), + ) + .expect("json should parse"); + let keys = payload["keys"].as_array().expect("keys should be array"); + let visible = status == "all" || (status == "quota_exhausted") == reserve_enabled; + assert_eq!(keys.len(), usize::from(visible)); + if visible { + assert_eq!( + keys[0]["scheduling_reason"] == "account_quota_exhausted", + reserve_enabled + ); + assert_eq!(keys[0]["status_snapshot"]["quota"]["code"], "ok"); + } + } + } +} + +#[tokio::test] +async fn gateway_codex_pool_filter_clears_stale_exhausted_summary_with_remaining_quota() { + let mut provider = sample_provider("provider-codex", "codex", 10); + provider.provider_type = "codex".to_string(); + provider.config = Some(json!({"pool_advanced": { + "reserve_minimum_quota": false, "skip_exhausted_accounts": true + }})); + let mut key = sample_key( + "key-codex", + "provider-codex", + "openai:responses", + "oauth-placeholder", + ); + key.auth_type = "oauth".to_string(); + key.status_snapshot = Some(json!({"quota": { + "provider_type": "codex", "code": "exhausted", "exhausted": true, + "windows": [{"code": "weekly", "scope": "account", "used_ratio": 0.83, + "remaining_ratio": 0.17, "reset_at": 4_102_444_800u64}] + }})); + let state = AppState::new() + .expect("gateway should build") + .with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests( + Arc::new(InMemoryProviderCatalogReadRepository::seed( + vec![provider], + Vec::new(), + vec![key], + )), + )); + for status in ["all", "available", "quota_exhausted"] { + let response = local_admin_pool_response( + &state, + http::Method::GET, + &format!("/api/admin/pool/provider-codex/keys?status={status}"), + None, + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let payload: serde_json::Value = serde_json::from_slice( + &to_bytes(response.into_body(), usize::MAX) + .await + .expect("body should read"), + ) + .expect("json should parse"); + let keys = payload["keys"].as_array().expect("keys should be array"); + assert_eq!(keys.len(), usize::from(status != "quota_exhausted")); + if let Some(key) = keys.first() { + assert_eq!(key["scheduling_status"], "available"); + assert_eq!(key["status_snapshot"]["quota"]["code"], "ok"); + assert!(key["account_quota"].as_str().unwrap().contains("17.0%")); + } + } +} + #[tokio::test] async fn gateway_lists_inherited_fixed_provider_api_formats_for_pool_keys() { let mut provider = sample_provider("provider-codex", "codex", 10).with_transport_fields( @@ -2738,8 +2866,10 @@ async fn gateway_treats_stale_codex_exhausted_snapshot_as_available_when_windows assert_eq!(keys[0]["scheduling_reason"], json!("available")); assert_eq!( keys[0]["account_quota"], - json!("周剩余 100.0% (7天0小时后重置) | 5H剩余 100.0% (5小时0分钟后重置)") + json!("周剩余 100.0% | 5H剩余 100.0%") ); + assert_eq!(keys[0]["status_snapshot"]["quota"]["code"], "ok"); + assert_eq!(keys[0]["status_snapshot"]["quota"]["exhausted"], false); } #[tokio::test] diff --git a/crates/aether-provider/pool/Cargo.toml b/crates/aether-provider/pool/Cargo.toml index f08919450..ac389e6d9 100644 --- a/crates/aether-provider/pool/Cargo.toml +++ b/crates/aether-provider/pool/Cargo.toml @@ -11,6 +11,7 @@ aether-contracts.workspace = true aether-data-contracts.workspace = true aether-pool-core.workspace = true aether-provider-transport.workspace = true +chrono.workspace = true serde_json.workspace = true url.workspace = true uuid.workspace = true diff --git a/crates/aether-provider/pool/src/lib.rs b/crates/aether-provider/pool/src/lib.rs index 91f9b01be..088987fff 100644 --- a/crates/aether-provider/pool/src/lib.rs +++ b/crates/aether-provider/pool/src/lib.rs @@ -38,11 +38,12 @@ pub use providers::{ XAI_BILLING_PATH, XAI_USER_PATH, }; pub use quota::{ - provider_pool_key_account_quota_exhausted, provider_pool_key_minimum_quota_reached, - provider_pool_key_model_quota_exhausted, provider_pool_key_model_quota_hard_blocked, - provider_pool_key_quota_hard_blocked, provider_pool_key_scheduling_label, - provider_pool_member_quota_snapshot, provider_pool_quota_metadata_provider_type, - provider_pool_quota_metadata_updated_at, provider_pool_quota_snapshot_updated_at, + provider_pool_codex_metadata_has_account_quota, provider_pool_key_account_quota_exhausted, + provider_pool_key_minimum_quota_reached, provider_pool_key_model_quota_exhausted, + provider_pool_key_model_quota_hard_blocked, provider_pool_key_quota_hard_blocked, + provider_pool_key_scheduling_label, provider_pool_member_quota_snapshot, + provider_pool_quota_metadata_provider_type, provider_pool_quota_metadata_updated_at, + provider_pool_quota_snapshot_updated_at, }; pub use quota_refresh::ProviderPoolQuotaRequestSpec; pub use service::ProviderPoolService; @@ -1600,6 +1601,245 @@ mod tests { assert!(!signals.quota_exhausted); } + #[test] + fn codex_newer_flat_quota_metadata_clears_stale_exhausted_snapshot() { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system time should be after unix epoch") + .as_secs(); + let service = ProviderPoolService::with_builtin_adapters(); + // Headers and quota refreshes persist the flat metadata shape while the + // derived snapshot may still describe the previous quota observation. + for used_percent in [83.0, 99.0, 100.0] { + let mut key = sample_key(Some(json!({ + "codex": { + "updated_at": now, + "primary_used_percent": used_percent, + "primary_reset_at": now + 3600 + } + }))); + key.status_snapshot = Some(json!({ + "quota": { + "provider_type": "codex", + "observed_at": now - 60, + "updated_at": now - 60, + "code": "exhausted", + "exhausted": true, + "allowed": false, + "limit_reached": true, + "reset_at": now + 3600, + "windows": [{ + "code": "weekly", + "scope": "account", + "used_ratio": 1.0, + "remaining_ratio": 0.0, + "is_exhausted": true, + "reset_at": now + 3600 + }] + } + })); + for model in [None, Some("gpt-5.4"), Some("o3")] { + let signals = service.member_signals("codex", &key, None, model); + assert_eq!(signals.quota_exhausted, used_percent >= 100.0); + assert!(!signals.quota_hard_blocked); + assert_eq!( + provider_pool_key_minimum_quota_reached(&key, "codex", model), + used_percent >= 99.0 + ); + } + } + } + + #[test] + fn codex_account_and_model_quota_agree_on_explicit_allow_and_deny_flags() { + let service = ProviderPoolService::with_builtin_adapters(); + for (flags, exhausted) in [ + (json!({ "allowed": true }), false), + (json!({ "limit_reached": false }), false), + (json!({ "allowed": true, "limit_reached": true }), true), + (json!({ "allowed": false, "limit_reached": false }), true), + ] { + for use_windows in [false, true] { + let mut metadata = flags.clone(); + metadata["updated_at"] = json!(300); + if use_windows { + metadata["windows"] = json!([{ "code": "weekly", "used_ratio": 1.0 }]); + } else { + metadata["primary_used_percent"] = json!(100.0); + } + let key = sample_key(Some(json!({ "codex": metadata }))); + for model in [None, Some("gpt-5.4"), Some("o3")] { + assert_eq!( + service + .member_signals("codex", &key, None, model) + .quota_exhausted, + exhausted, + "flags={flags}, use_windows={use_windows}, model={model:?}" + ); + // The opt-in reserve still protects a numerically full + // window even when the upstream reports it as allowed. + assert!(provider_pool_key_minimum_quota_reached( + &key, "codex", model + )); + } + } + } + } + + #[test] + fn codex_model_quota_honors_latest_account_refusal_without_blocking_spark() { + let service = ProviderPoolService::with_builtin_adapters(); + for include_window in [false, true] { + let mut metadata = json!({ + "updated_at": 300, + "allowed": false, + "limit_reached": true, + "spark_primary_used_percent": 17.0 + }); + if include_window { + metadata["primary_used_percent"] = json!(83.0); + } + let mut key = sample_key(Some(json!({ "codex": metadata }))); + for include_snapshot in [false, true] { + if include_snapshot { + key.status_snapshot = Some(json!({ + "quota": { + "provider_type": "codex", + "observed_at": 200, + "exhausted": false, + "windows": [{ "code": "weekly", "used_ratio": 0.83 }] + } + })); + } + assert!( + service + .member_signals("codex", &key, None, Some("gpt-5.4")) + .quota_exhausted + ); + assert!( + !service + .member_signals("codex", &key, None, Some("gpt-5.3-codex-spark")) + .quota_exhausted + ); + } + } + } + + #[test] + fn codex_quota_source_freshness_supports_all_timestamp_formats() { + let service = ProviderPoolService::with_builtin_adapters(); + for observed_at in [ + json!(1_700_000_200_u64), + json!(1_700_000_200_000_u64), + json!("1700000200000"), + json!("2023-11-14T22:16:40Z"), + ] { + for use_windows in [false, true] { + let metadata = if use_windows { + json!({ + "updated_at": observed_at, + "windows": [{ "code": "weekly", "used_ratio": 0.83 }] + }) + } else { + json!({ "updated_at": observed_at, "primary_used_percent": 83.0 }) + }; + let mut key = sample_key(Some(json!({ "codex": metadata }))); + key.status_snapshot = Some(json!({ + "quota": { + "provider_type": "codex", + "observed_at": "2023-11-14T22:15:00Z", + "exhausted": true, + "allowed": false, + "windows": [{ "code": "weekly", "used_ratio": 1.0 }] + } + })); + for model in [None, Some("gpt-5.4")] { + let signals = service.member_signals("codex", &key, None, model); + assert!(!signals.quota_exhausted, "timestamp={observed_at}"); + assert!(!signals.quota_hard_blocked, "timestamp={observed_at}"); + assert!(!provider_pool_key_minimum_quota_reached( + &key, "codex", model + )); + } + + // Swap freshness while retaining the same observations. An + // older usable bucket cannot erase a later exhausted snapshot. + key.status_snapshot.as_mut().unwrap()["quota"]["observed_at"] = + json!("2023-11-14T22:18:20Z"); + assert!(provider_pool_key_account_quota_exhausted(&key, "codex")); + assert!(provider_pool_key_quota_hard_blocked(&key, "codex")); + assert_eq!( + provider_pool_key_model_quota_exhausted(&key, "codex", "gpt-5.4"), + Some(true) + ); + } + } + } + + #[test] + fn codex_account_quota_recovery_requires_new_account_observation() { + for metadata in [ + json!({ "updated_at": 100, "primary_used_percent": 83.0 }), + json!({ "primary_used_percent": 83.0 }), + json!({ "updated_at": 300, "plan_type": "plus" }), + json!({ "updated_at": 300, "credits_unlimited": false }), + json!({ "updated_at": 300, "spark_primary_used_percent": 83.0 }), + json!({ "updated_at": 300, "windows": [{ "code": "weekly" }] }), + ] { + let mut key = sample_key(Some(json!({ "codex": metadata }))); + key.status_snapshot = Some(json!({ + "quota": { + "provider_type": "codex", + "observed_at": 200, + "exhausted": true, + "allowed": false, + "limit_reached": true, + "windows": [{ "code": "weekly", "used_ratio": 1.0 }] + } + })); + assert!(provider_pool_key_account_quota_exhausted(&key, "codex")); + assert!(provider_pool_key_quota_hard_blocked(&key, "codex")); + assert_eq!( + provider_pool_key_model_quota_exhausted(&key, "codex", "gpt-5.4"), + Some(true) + ); + } + + let mut key = sample_key(Some(json!({ + "codex": { + "updated_at": 300, + "primary_used_percent": 83.0, + "allowed": false, + "limit_reached": true + } + }))); + key.status_snapshot = Some(json!({ + "quota": { "provider_type": "codex", "updated_at": 200, "exhausted": false } + })); + assert!(provider_pool_key_account_quota_exhausted(&key, "codex")); + assert!(provider_pool_key_quota_hard_blocked(&key, "codex")); + } + + #[test] + fn codex_account_updates_do_not_override_independent_model_quota() { + for (spark_ratio, account_percent) in [(0.83, 100.0), (1.0, 83.0)] { + let mut key = sample_key(Some(json!({ + "codex": { "updated_at": 300, "primary_used_percent": account_percent } + }))); + key.status_snapshot = Some(json!({ + "quota": { + "provider_type": "codex", + "updated_at": 200, + "windows": [{ "code": "spark_5h", "used_ratio": spark_ratio }] + } + })); + assert_eq!( + provider_pool_key_model_quota_exhausted(&key, "codex", "gpt-5.3-codex-spark"), + Some(spark_ratio >= 1.0) + ); + } + } + #[test] fn codex_explicit_quota_block_is_hard_until_reset() { let now = std::time::SystemTime::now() diff --git a/crates/aether-provider/pool/src/providers/codex.rs b/crates/aether-provider/pool/src/providers/codex.rs index f7133d281..86145f350 100644 --- a/crates/aether-provider/pool/src/providers/codex.rs +++ b/crates/aether-provider/pool/src/providers/codex.rs @@ -10,10 +10,11 @@ use crate::provider::{ ProviderPoolMemberInput, }; use crate::quota::{ - provider_pool_current_unix_secs, provider_pool_json_bool, provider_pool_json_f64, - provider_pool_member_quota_snapshot, provider_pool_metadata_bucket, - provider_pool_model_quota_exhausted, provider_pool_quota_snapshot_exhausted_decision, - provider_pool_reset_deadline_elapsed, provider_pool_timestamp_unix_secs, + provider_pool_codex_metadata_has_account_quota, provider_pool_current_unix_secs, + provider_pool_json_bool, provider_pool_json_f64, provider_pool_member_quota_snapshot, + provider_pool_metadata_bucket, provider_pool_model_quota_exhausted, + provider_pool_quota_snapshot_exhausted_decision, provider_pool_reset_deadline_elapsed, + provider_pool_source_account_quota_exhausted, provider_pool_timestamp_unix_secs, }; use crate::quota_refresh::ProviderPoolQuotaRequestSpec; @@ -54,6 +55,9 @@ impl ProviderPoolAdapter for CodexProviderPoolAdapter { }) { return exhausted; } + if let Some(bucket) = codex_newer_account_quota_metadata(input.key, input.provider_type) { + return quota_exhausted_from_bucket(bucket); + } if let Some(quota_snapshot) = provider_pool_member_quota_snapshot(input.key, input.provider_type) { @@ -108,15 +112,12 @@ fn codex_explicit_quota_block_active( key: &aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey, provider_type: &str, ) -> bool { + if let Some(bucket) = codex_newer_account_quota_metadata(key, provider_type) { + return codex_explicit_quota_block_from_bucket(bucket); + } let Some(quota_snapshot) = provider_pool_member_quota_snapshot(key, provider_type) else { return provider_pool_metadata_bucket(key.upstream_metadata.as_ref(), provider_type) - .is_some_and(|bucket| { - (provider_pool_json_bool(bucket.get("allowed")) == Some(false) - || provider_pool_json_bool(bucket.get("limit_reached")) == Some(true)) - && !["primary", "secondary"] - .into_iter() - .any(|prefix| codex_window_reset_elapsed(bucket, prefix)) - }); + .is_some_and(codex_explicit_quota_block_from_bucket); }; let explicitly_blocked = provider_pool_json_bool(quota_snapshot.get("allowed")) == Some(false) || provider_pool_json_bool(quota_snapshot.get("limit_reached")) == Some(true); @@ -133,6 +134,35 @@ fn codex_explicit_quota_block_active( }) } +fn codex_explicit_quota_block_from_bucket(bucket: &Map) -> bool { + (provider_pool_json_bool(bucket.get("allowed")) == Some(false) + || provider_pool_json_bool(bucket.get("limit_reached")) == Some(true)) + && !["primary", "secondary"] + .into_iter() + .any(|prefix| codex_window_reset_elapsed(bucket, prefix)) +} + +/// A successful refresh can update raw metadata before the status snapshot. +/// Do not keep an old account-level block once a newer quota observation exists. +/// Identity-only or model-only updates cannot clear an account quota decision. +fn codex_newer_account_quota_metadata<'a>( + key: &'a aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey, + provider_type: &str, +) -> Option<&'a Map> { + let snapshot = provider_pool_member_quota_snapshot(key, provider_type)?; + let metadata = provider_pool_metadata_bucket(key.upstream_metadata.as_ref(), provider_type)?; + if !provider_pool_codex_metadata_has_account_quota(metadata) { + return None; + } + let metadata_observed_at = provider_pool_timestamp_unix_secs(metadata.get("observed_at")) + .or_else(|| provider_pool_timestamp_unix_secs(metadata.get("updated_at")))?; + let snapshot_observed_at = provider_pool_timestamp_unix_secs(snapshot.get("observed_at")) + .or_else(|| provider_pool_timestamp_unix_secs(snapshot.get("updated_at"))); + snapshot_observed_at + .is_none_or(|observed_at| metadata_observed_at >= observed_at) + .then_some(metadata) +} + fn build_codex_wham_headers( resolved_oauth_auth: Option<(String, String)>, decrypted_api_key: Option<&str>, @@ -322,11 +352,14 @@ pub(crate) fn quota_exhausted_from_bucket(bucket: &Map) -> bool { if provider_pool_json_bool(bucket.get("credits_unlimited")) == Some(true) { return false; } + let account_windows_exhausted = provider_pool_source_account_quota_exhausted(bucket); let has_window_data = provider_pool_json_f64(bucket.get("primary_used_percent")).is_some() - || provider_pool_json_f64(bucket.get("secondary_used_percent")).is_some(); + || provider_pool_json_f64(bucket.get("secondary_used_percent")).is_some() + || account_windows_exhausted.is_some(); if !has_window_data && provider_pool_json_bool(bucket.get("has_credits")) == Some(false) { return true; } - codex_window_used_percent_exhausted(bucket, "primary") + account_windows_exhausted == Some(true) + || codex_window_used_percent_exhausted(bucket, "primary") || codex_window_used_percent_exhausted(bucket, "secondary") } diff --git a/crates/aether-provider/pool/src/quota.rs b/crates/aether-provider/pool/src/quota.rs index 609739a42..af49f3f63 100644 --- a/crates/aether-provider/pool/src/quota.rs +++ b/crates/aether-provider/pool/src/quota.rs @@ -87,6 +87,7 @@ fn provider_pool_quota_reaches_reserve( provider_model_name: Option<&str>, reserve_ratio: f64, ) -> Option { + let is_codex = provider_type.trim().eq_ignore_ascii_case("codex"); let requested = provider_model_name.map(provider_pool_identifier_tokens); if reserve_ratio <= 0.0 && requested.as_ref().is_some_and(|tokens| tokens.is_empty()) { return None; @@ -103,12 +104,19 @@ fn provider_pool_quota_reaches_reserve( let mut resolved = None::<(Option, bool)>; let mut resolved_specific_bucket = false; for source in sources.into_iter().flatten() { + let observed_at = provider_pool_timestamp_unix_secs(source.get("observed_at")) + .or_else(|| provider_pool_timestamp_unix_secs(source.get("updated_at"))); + let account_signal = (is_codex && reserve_ratio <= 0.0) + .then(|| provider_pool_codex_account_quota_signal(source, observed_at)) + .flatten(); let mut windows = provider_pool_collect_quota_windows(source); - if reserve_ratio > 0.0 && provider_type.trim().eq_ignore_ascii_case("codex") { + if is_codex { // Raw refresh metadata can be newer than the materialized windows. // Include all four legacy slots before selecting the model bucket. for prefix in ["primary", "secondary", "spark_primary", "spark_secondary"] { - let Some(used_percent) = source.get(&format!("{prefix}_used_percent")) else { + let Some(used_percent) = + provider_pool_json_f64(source.get(&format!("{prefix}_used_percent"))) + else { continue; }; if provider_pool_json_f64(source.get(&format!("{prefix}_window_minutes"))) @@ -118,7 +126,7 @@ fn provider_pool_quota_reaches_reserve( } let mut window = Map::from_iter([ ("code".to_string(), json!(prefix)), - ("used_percent".to_string(), used_percent.clone()), + ("used_percent".to_string(), json!(used_percent)), ]); for field in [ "reset_at", @@ -132,12 +140,21 @@ fn provider_pool_quota_reaches_reserve( } windows.push(window); } + windows.retain(provider_pool_quota_window_has_observation); + if let Some(exhausted) = account_signal { + if !windows.iter().any(provider_pool_window_is_generic) { + // A flags-only quota refresh is still a newer account + // observation, but must not override a model-specific bucket. + windows.push(Map::from_iter([ + ("code".to_string(), json!("account")), + ("is_exhausted".to_string(), json!(exhausted)), + ])); + } + } } if windows.is_empty() { continue; } - let observed_at = provider_pool_timestamp_unix_secs(source.get("observed_at")) - .or_else(|| provider_pool_timestamp_unix_secs(source.get("updated_at"))); let model_matches = windows .iter() .filter(|window| { @@ -155,7 +172,7 @@ fn provider_pool_quota_reaches_reserve( provider_pool_explicit_model_windows_exhausted(model_matches, observed_at) }; if resolved.is_none() - || (reserve_ratio > 0.0 && !resolved_specific_bucket) + || (is_codex && !resolved_specific_bucket) || provider_pool_should_replace_model_quota_resolution( resolved.as_ref().and_then(|(observed_at, _)| *observed_at), observed_at, @@ -183,7 +200,7 @@ fn provider_pool_quota_reaches_reserve( let exhausted = provider_pool_any_window_exhausted(family_matches, observed_at, reserve_ratio); if resolved.is_none() - || (reserve_ratio > 0.0 && !resolved_specific_bucket) + || (is_codex && !resolved_specific_bucket) || provider_pool_should_replace_model_quota_resolution( resolved.as_ref().and_then(|(observed_at, _)| *observed_at), observed_at, @@ -197,7 +214,7 @@ fn provider_pool_quota_reaches_reserve( // A newer account observation does not update an independent model // bucket. Only compare freshness between applicable model sources. - if reserve_ratio > 0.0 && resolved_specific_bucket { + if is_codex && resolved_specific_bucket { continue; } @@ -211,8 +228,9 @@ fn provider_pool_quota_reaches_reserve( .filter(|window| provider_pool_window_is_generic(window)) .collect::>(); if !generic_matches.is_empty() { - let exhausted = - provider_pool_any_window_exhausted(generic_matches, observed_at, reserve_ratio); + let exhausted = account_signal.unwrap_or_else(|| { + provider_pool_any_window_exhausted(generic_matches, observed_at, reserve_ratio) + }); if resolved.is_none() || provider_pool_should_replace_model_quota_resolution( resolved.as_ref().and_then(|(observed_at, _)| *observed_at), @@ -227,6 +245,37 @@ fn provider_pool_quota_reaches_reserve( resolved.map(|(_, exhausted)| exhausted) } +fn provider_pool_codex_account_quota_signal( + source: &Map, + observed_at: Option, +) -> Option { + let allowed = provider_pool_json_bool(source.get("allowed")); + let limit_reached = provider_pool_json_bool(source.get("limit_reached")); + if allowed == Some(false) || limit_reached == Some(true) { + let reset_elapsed = provider_pool_current_unix_secs().is_some_and(|now| { + provider_pool_reset_deadline_elapsed(source, observed_at, now) + || ["primary", "secondary"].into_iter().any(|prefix| { + let window = [ + "reset_at", + "next_reset_at", + "reset_seconds", + "reset_after_seconds", + ] + .into_iter() + .filter_map(|field| { + source + .get(&format!("{prefix}_{field}")) + .map(|value| (field.to_string(), value.clone())) + }) + .collect::>(); + provider_pool_reset_deadline_elapsed(&window, observed_at, now) + }) + }); + return (!reset_elapsed).then_some(true); + } + (allowed == Some(true) || limit_reached == Some(false)).then_some(false) +} + fn provider_pool_should_replace_model_quota_resolution( previous_observed_at: Option, next_observed_at: Option, @@ -282,6 +331,69 @@ fn provider_pool_any_window_exhausted( }) } +fn provider_pool_quota_window_has_observation(window: &Map) -> bool { + ["is_exhausted", "exhausted"] + .into_iter() + .any(|field| provider_pool_json_bool(window.get(field)).is_some()) + || [ + "used_ratio", + "usage_ratio", + "used_percent", + "remaining_ratio", + "remaining_fraction", + "remaining_percent", + ] + .into_iter() + .any(|field| provider_pool_json_f64(window.get(field)).is_some()) + || (provider_pool_json_f64( + window + .get("remaining") + .or_else(|| window.get("remaining_value")), + ) + .is_some() + && provider_pool_json_f64( + window + .get("limit") + .or_else(|| window.get("limit_value")) + .or_else(|| window.get("total")), + ) + .is_some_and(|limit| limit > 0.0)) +} + +pub(crate) fn provider_pool_source_account_quota_exhausted( + source: &Map, +) -> Option { + let windows = provider_pool_collect_quota_windows(source); + let account_windows = windows + .iter() + .filter(|window| provider_pool_window_is_generic(window)) + .filter(|window| provider_pool_quota_window_has_observation(window)) + .collect::>(); + if account_windows.is_empty() { + return None; + } + let observed_at = provider_pool_timestamp_unix_secs(source.get("observed_at")) + .or_else(|| provider_pool_timestamp_unix_secs(source.get("updated_at"))); + Some(provider_pool_any_window_exhausted( + account_windows, + observed_at, + 0.0, + )) +} + +/// Whether Codex metadata contains an account quota observation rather than an +/// identity-only update or an independent model quota bucket. +pub fn provider_pool_codex_metadata_has_account_quota(source: &Map) -> bool { + ["primary_used_percent", "secondary_used_percent"] + .into_iter() + .any(|field| provider_pool_json_f64(source.get(field)).is_some()) + || provider_pool_source_account_quota_exhausted(source).is_some() + || ["allowed", "limit_reached", "has_credits"] + .into_iter() + .any(|field| provider_pool_json_bool(source.get(field)).is_some()) + || provider_pool_json_bool(source.get("credits_unlimited")) == Some(true) +} + /// Whether an applicable Codex window has at most 1% remaining. Missing quota /// data and windows whose reset has elapsed do not trigger this opt-in guard. pub fn provider_pool_key_minimum_quota_reached( @@ -700,7 +812,16 @@ pub(crate) fn provider_pool_json_f64(value: Option<&Value>) -> Option { } pub(crate) fn provider_pool_timestamp_unix_secs(value: Option<&Value>) -> Option { - let mut timestamp = provider_pool_json_f64(value)?; + let mut timestamp = match provider_pool_json_f64(value) { + Some(timestamp) => timestamp, + None => { + return chrono::DateTime::parse_from_rfc3339(value?.as_str()?.trim()) + .ok()? + .timestamp() + .try_into() + .ok(); + } + }; if timestamp <= 0.0 { return None; } diff --git a/crates/aether-usage/runtime/src/runtime.rs b/crates/aether-usage/runtime/src/runtime.rs index 3a571be5e..48c1354f9 100644 --- a/crates/aether-usage/runtime/src/runtime.rs +++ b/crates/aether-usage/runtime/src/runtime.rs @@ -10055,6 +10055,24 @@ mod tests { 2, "the later direct caller should receive its own bounded write attempt" ); + // Direct persistence can finish before the submission worker joins the + // barrier handoff and accounts for its completed slot. + timeout(Duration::from_secs(1), async { + loop { + let snapshot = runtime.metrics_snapshot(); + let submission = &runtime.lifecycle_submission.state; + if snapshot.terminal_submission_pending == 0 + && snapshot.ordered_lifecycle_pending == 0 + && snapshot.lifecycle_submission_pending == 0 + && submission.admission.available_permits() == submission.capacity + { + break; + } + sleep(Duration::from_millis(1)).await; + } + }) + .await + .expect("failed terminal submission accounting and admission should drain"); let snapshot = runtime.metrics_snapshot(); assert_eq!(snapshot.terminal_submission_pending, 0); assert_eq!(snapshot.ordered_lifecycle_pending, 0); @@ -10162,24 +10180,43 @@ mod tests { .await; assert_eq!(remaining_policy_panics.load(Ordering::Acquire), 0); - let records = records.lock().expect("records lock"); - assert_eq!( - records - .iter() - .filter(|record| record.request_id == healthy_request_id) - .count(), - 2, - "the same terminal shard should continue processing healthy requests" - ); - assert!( - records - .iter() - .filter(|record| record.request_id == failed_request_id) - .count() - == 1, - "only the later healthy attempt should persist for the panicked request" - ); - drop(records); + { + let records = records.lock().expect("records lock"); + assert_eq!( + records + .iter() + .filter(|record| record.request_id == healthy_request_id) + .count(), + 2, + "the same terminal shard should continue processing healthy requests" + ); + assert!( + records + .iter() + .filter(|record| record.request_id == failed_request_id) + .count() + == 1, + "only the later healthy attempt should persist for the panicked request" + ); + } + // The final direct attempt also submits a barrier whose worker may + // account for completion after the persistence call has returned. + timeout(Duration::from_secs(1), async { + loop { + let snapshot = runtime.metrics_snapshot(); + let submission = &runtime.lifecycle_submission.state; + if snapshot.terminal_submission_pending == 0 + && snapshot.ordered_lifecycle_pending == 0 + && snapshot.lifecycle_submission_pending == 0 + && submission.admission.available_permits() == submission.capacity + { + break; + } + sleep(Duration::from_millis(1)).await; + } + }) + .await + .expect("panicked terminal submission accounting and admission should drain"); let snapshot = runtime.metrics_snapshot(); assert_eq!(snapshot.terminal_submission_pending, 0); assert_eq!(snapshot.ordered_lifecycle_pending, 0); diff --git a/frontend/src/api/__tests__/providers-pool-cache.spec.ts b/frontend/src/api/__tests__/providers-pool-cache.spec.ts new file mode 100644 index 000000000..745444bc5 --- /dev/null +++ b/frontend/src/api/__tests__/providers-pool-cache.spec.ts @@ -0,0 +1,76 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { getMock, patchMock } = vi.hoisted(() => ({ getMock: vi.fn(), patchMock: vi.fn() })) + +vi.mock('@/api/client', () => ({ default: { get: getMock, patch: patchMock } })) + +import { getPoolOverview, listPoolKeys, listPoolScores } from '@/api/endpoints/pool' +import { getProvider, updateProvider } from '@/api/endpoints/providers' +import { cache } from '@/utils/cache' + +const options = { cacheTtlMs: 30_000 } +const provider = { id: 'codex', pool_advanced: { reserve_minimum_quota: false } } + +function deferred() { + let resolve!: (value: T) => void + const promise = new Promise((resolvePromise) => { resolve = resolvePromise }) + return { promise, resolve } +} + +beforeEach(() => { + cache.clear() + getMock.mockReset() + patchMock.mockReset() + patchMock.mockResolvedValue({ data: provider }) +}) + +describe('provider pool settings cache invalidation', () => { + it('refreshes every cached key page, score page and overview after changing pool settings', async () => { + getMock.mockResolvedValue({ data: { version: 'old' } }) + await getPoolOverview(options) + await listPoolKeys('codex', {}, options) + await listPoolKeys('codex', { page: 2, status: 'quota_exhausted' }, options) + await listPoolScores('codex', {}, options) + await listPoolKeys('codex-other', {}, options) + await updateProvider('codex', { pool_advanced: { reserve_minimum_quota: false } }) + getMock.mockResolvedValue({ data: { version: 'new' } }) + + await expect(getPoolOverview(options)).resolves.toEqual({ version: 'new' }) + await expect(listPoolKeys('codex', {}, options)).resolves.toEqual({ version: 'new' }) + await expect(listPoolKeys('codex', { page: 2, status: 'quota_exhausted' }, options)) + .resolves.toEqual({ version: 'new' }) + await expect(listPoolScores('codex', {}, options)).resolves.toEqual({ version: 'new' }) + await expect(listPoolKeys('codex-other', {}, options)).resolves.toEqual({ version: 'old' }) + }) + + it('does not reuse or cache a key request started before the settings were saved', async () => { + const oldResponse = deferred<{ data: { version: string } }>() + const newResponse = deferred<{ data: { version: string } }>() + getMock.mockReturnValueOnce(oldResponse.promise).mockReturnValueOnce(newResponse.promise) + const oldRequest = listPoolKeys('codex', { page: 2 }, options) + await updateProvider('codex', { pool_advanced: { reserve_minimum_quota: false } }) + const newRequest = listPoolKeys('codex', { page: 2 }, options) + expect(getMock).toHaveBeenCalledTimes(2) + + oldResponse.resolve({ data: { version: 'old' } }) + await oldRequest + const deduped = listPoolKeys('codex', { page: 2 }, options) + expect(getMock).toHaveBeenCalledTimes(2) + newResponse.resolve({ data: { version: 'new' } }) + await expect(newRequest).resolves.toEqual({ version: 'new' }) + await expect(deduped).resolves.toEqual({ version: 'new' }) + await expect(listPoolKeys('codex', { page: 2 }, options)).resolves.toEqual({ version: 'new' }) + }) + + it('does not reuse a provider detail request started before a successful save', async () => { + const oldResponse = deferred<{ data: typeof provider }>() + getMock.mockReturnValueOnce(oldResponse.promise).mockResolvedValueOnce({ data: provider }) + const oldRequest = getProvider('codex') + await updateProvider('codex', { pool_advanced: { reserve_minimum_quota: false } }) + const newRequest = getProvider('codex') + expect(getMock).toHaveBeenCalledTimes(2) + oldResponse.resolve({ data: { ...provider, pool_advanced: { reserve_minimum_quota: true } } }) + await oldRequest + await expect(newRequest).resolves.toMatchObject(provider) + }) +}) diff --git a/frontend/src/api/endpoints/providers.ts b/frontend/src/api/endpoints/providers.ts index ab70f598b..d73206ebf 100644 --- a/frontend/src/api/endpoints/providers.ts +++ b/frontend/src/api/endpoints/providers.ts @@ -1,5 +1,5 @@ import client from '../client' -import { buildCacheKey, cachedRequest, dedupedRequest } from '@/utils/cache' +import { buildCacheKey, cache, cachedRequest, dedupedRequest } from '@/utils/cache' import type { ClaudeCodeAdvancedConfig, FailoverRulesConfig, @@ -142,6 +142,15 @@ export async function updateProvider( requestOptions?: ProviderRequestOptions, ): Promise { const response = await client.patch(`/api/admin/providers/${providerId}`, data, requestOptions) + cache.delete(`providers:detail:${providerId}`) + if ('pool_advanced' in data) { + cache.delete('pool:overview') + for (const kind of ['keys', 'scores']) { + const prefix = `pool:${kind}:${providerId}` + cache.delete(prefix) + cache.deleteByPrefix(`${prefix}:`) + } + } return normalizeProviderSummary(response.data) } diff --git a/frontend/src/utils/cache.ts b/frontend/src/utils/cache.ts index 842325e10..33ddf9c2c 100644 --- a/frontend/src/utils/cache.ts +++ b/frontend/src/utils/cache.ts @@ -61,6 +61,14 @@ class MemoryCache { this.keyVersions.set(key, (this.keyVersions.get(key) ?? 0) + 1) } + /** 删除匹配前缀的缓存及进行中请求,旧响应不再回填缓存。 */ + deleteByPrefix(prefix: string): void { + const keys = new Set([...this.cache.keys(), ...this.inFlight.keys()]) + for (const key of keys) { + if (key.startsWith(prefix)) this.delete(key) + } + } + /** * 清空所有缓存 */ diff --git a/frontend/src/views/admin/PoolManagement.vue b/frontend/src/views/admin/PoolManagement.vue index 1e05aac93..17a8bf2b4 100644 --- a/frontend/src/views/admin/PoolManagement.vue +++ b/frontend/src/views/admin/PoolManagement.vue @@ -1395,6 +1395,8 @@ async function loadOverview(options: { cacheTtlMs?: number, silent?: boolean } = async function handleSchedulingSaved(updatedProvider: ProviderWithEndpointsSummary) { if (!selectedProviderId.value || updatedProvider.id !== selectedProviderId.value) return + // 保存前发出的详情读取不得覆盖这次保存返回的新配置。 + providerDataRequestId += 1 // 优先回写保存接口返回值,避免弹窗立即重开时读到旧配置。 if (selectedProviderData.value) { Object.assign(selectedProviderData.value, updatedProvider) @@ -1403,7 +1405,7 @@ async function handleSchedulingSaved(updatedProvider: ProviderWithEndpointsSumma } showSchedulingDialog.value = false showAdvancedDialog.value = false - await loadOverview({ silent: true }) + await Promise.all([loadKeys({ silent: true }), loadOverview({ silent: true })]) } // --- Provider Selection --- diff --git a/frontend/src/views/admin/__tests__/PoolManagement.codex-cycle-stats.spec.ts b/frontend/src/views/admin/__tests__/PoolManagement.codex-cycle-stats.spec.ts index 392c70b44..2f7991b57 100644 --- a/frontend/src/views/admin/__tests__/PoolManagement.codex-cycle-stats.spec.ts +++ b/frontend/src/views/admin/__tests__/PoolManagement.codex-cycle-stats.spec.ts @@ -340,12 +340,24 @@ vi.mock('@/components/ui/refresh-button.vue', async () => { }) vi.mock('@/features/pool/components/PoolSchedulingDialog.vue', async () => { - const { defineComponent } = await import('vue') + const { defineComponent, h } = await import('vue') return { default: defineComponent({ name: 'PoolSchedulingDialogStub', - setup() { - return () => null + props: { + providerId: { type: String, required: true }, + currentConfig: { type: Object, default: null }, + }, + emits: ['saved'], + setup(props, { emit }) { + return () => h('button', { + 'data-testid': 'save-pool-scheduling', + 'data-reserve': String(props.currentConfig?.reserve_minimum_quota ?? false), + onClick: () => emit('saved', { + id: props.providerId, + pool_advanced: { ...props.currentConfig, reserve_minimum_quota: false }, + }), + }, '保存调度测试') }, }), } @@ -1278,4 +1290,80 @@ describe('PoolManagement Codex cycle stats mode', () => { expect(disabledRoot.querySelector('[data-testid="pool-demand-metrics-button"]')).toBeNull() }) + + it('reloads account scheduling state immediately after disabling minimum quota reserve', async () => { + const key = createPoolKey('codex', { + scheduling_status: 'blocked', + scheduling_reason: 'account_quota_exhausted', + scheduling_label: '额度耗尽', + }) + endpointMocks.getPoolOverview.mockResolvedValue({ items: [createOverview('codex')] }) + endpointMocks.listPoolKeys.mockResolvedValue(createKeyPage(key)) + endpointMocks.getProvider.mockResolvedValue(createProvider('codex', { + pool_advanced: { reserve_minimum_quota: true }, + })) + const root = mountPoolManagement() + await settle() + expect(root.querySelector('tbody')?.textContent).toContain('额度耗尽') + endpointMocks.listPoolKeys.mockClear() + endpointMocks.listPoolKeys.mockResolvedValue(createKeyPage({ + ...key, + scheduling_status: 'available', + scheduling_reason: 'available', + scheduling_label: '可用', + })) + + root.querySelector('[data-testid="save-pool-scheduling"]')?.click() + await settle() + + expect(endpointMocks.listPoolKeys).toHaveBeenCalledWith( + 'codex-provider', expect.anything(), { cacheTtlMs: 0 }, + ) + expect(root.querySelector('tbody')?.textContent).not.toContain('额度耗尽') + expect(root.querySelector('tbody')?.textContent).toContain('可用') + }) + + it('shows 17% remaining without marking a schedulable account exhausted when reserve is enabled', async () => { + endpointMocks.getPoolOverview.mockResolvedValue({ items: [createOverview('codex')] }) + endpointMocks.getProvider.mockResolvedValue(createProvider('codex', { + pool_advanced: { reserve_minimum_quota: true }, + })) + endpointMocks.listPoolKeys.mockResolvedValue(createKeyPage(createPoolKey('codex', { + scheduling_status: 'available', + scheduling_reason: 'available', + scheduling_label: '可用', + status_snapshot: { + account: { code: 'ok', blocked: false }, + quota: { + code: 'ok', + exhausted: false, + provider_type: 'codex', + windows: [{ code: '5h', used_ratio: 0.83, remaining_ratio: 0.17 }], + }, + oauth: { code: 'none' }, + }, + }))) + const root = mountPoolManagement() + await settle() + + expect(root.querySelector('[data-testid="pool-quota-meter-text"]')?.textContent).toBe('17.0%') + expect(root.querySelector('tbody')?.textContent).toContain('可用') + expect(root.querySelector('tbody')?.textContent).not.toContain('额度耗尽') + }) + + it('does not replace saved scheduling settings with an older provider detail response', async () => { + let resolveOldProvider!: (provider: ReturnType) => void + endpointMocks.getProvider.mockReturnValue(new Promise((resolve) => { resolveOldProvider = resolve })) + endpointMocks.getPoolOverview.mockResolvedValue({ items: [createOverview('codex')] }) + endpointMocks.listPoolKeys.mockResolvedValue(createKeyPage(createPoolKey())) + const root = mountPoolManagement() + await settle() + + root.querySelector('[data-testid="save-pool-scheduling"]')?.click() + await settle() + resolveOldProvider(createProvider('codex', { pool_advanced: { reserve_minimum_quota: true } })) + await settle() + + expect(root.querySelector('[data-testid="save-pool-scheduling"]')?.getAttribute('data-reserve')).toBe('false') + }) })