fix(codex): restore recovered quota status and stabilize lifecycle tests

This commit is contained in:
ZheFox
2026-09-24 10:33:45 +08:00
parent 57f53903f5
commit 31d5e2d172
14 changed files with 955 additions and 65 deletions
Generated
+1
View File
@@ -524,6 +524,7 @@ dependencies = [
"aether-data-contracts", "aether-data-contracts",
"aether-pool-core", "aether-pool-core",
"aether-provider-transport", "aether-provider-transport",
"chrono",
"serde_json", "serde_json",
"url", "url",
"uuid", "uuid",
@@ -4005,7 +4005,7 @@ mod tests {
async fn pool_key_cursor_reserve_minimum_quota_filters_pages_and_sticky_hits() { async fn pool_key_cursor_reserve_minimum_quota_filters_pages_and_sticky_hits() {
for reserve_enabled in [false, true] { for reserve_enabled in [false, true] {
for sticky in [false, true] { for sticky in [false, true] {
for used_percent in [99.0, 98.0] { for used_percent in [99.0, 98.0, 83.0] {
let provider_config = Some(json!({ let provider_config = Some(json!({
"pool_advanced": { "pool_advanced": {
"reserve_minimum_quota": reserve_enabled, "reserve_minimum_quota": reserve_enabled,
@@ -4016,8 +4016,27 @@ mod tests {
sample_codex_pool_provider("provider-pool", 0, provider_config.clone()); sample_codex_pool_provider("provider-pool", 0, provider_config.clone());
let endpoint = sample_codex_pool_endpoint("provider-pool", "endpoint-1"); let endpoint = sample_codex_pool_endpoint("provider-pool", "endpoint-1");
let mut reserved = sample_codex_pool_key("provider-pool", "key-low"); let mut reserved = sample_codex_pool_key("provider-pool", "key-low");
reserved.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"updated_at": 100,
"allowed": false,
"exhausted": true,
"code": "exhausted",
"windows": [{
"code": "weekly",
"scope": "account",
"used_ratio": 1.0,
"reset_at": 4_102_444_800u64
}]
}
}));
reserved.upstream_metadata = Some(json!({ reserved.upstream_metadata = Some(json!({
"codex": {"primary_used_percent": used_percent} "codex": {
"updated_at": 200,
"primary_used_percent": used_percent,
"primary_reset_at": 4_102_444_800u64
}
})); }));
let ready = sample_codex_pool_key("provider-pool", "key-ready"); let ready = sample_codex_pool_key("provider-pool", "key-ready");
let rows = vec![ let rows = vec![
@@ -21,6 +21,7 @@ use aether_crypto::{
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey; use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
use aether_provider_pool::{ use aether_provider_pool::{
grok_pool_tier_from_quota_bucket, grok_supported_quota_windows_for_tier, grok_pool_tier_from_quota_bucket, grok_supported_quota_windows_for_tier,
provider_pool_codex_metadata_has_account_quota,
}; };
use aether_scheduler_core::provider_key_circuit_payload_is_active_open_at; use aether_scheduler_core::provider_key_circuit_payload_is_active_open_at;
use serde_json::{json, Map, Value}; use serde_json::{json, Map, Value};
@@ -1111,9 +1112,8 @@ fn build_codex_quota_status_snapshot(
source: &str, source: &str,
) -> Option<Value> { ) -> Option<Value> {
let metadata = provider_quota_metadata_bucket(upstream_metadata, "codex")?; let metadata = provider_quota_metadata_bucket(upstream_metadata, "codex")?;
let observed_at_unix_secs = metadata let observed_at_unix_secs = provider_quota_timestamp_unix_secs(metadata.get("observed_at"))
.get("updated_at") .or_else(|| provider_quota_timestamp_unix_secs(metadata.get("updated_at")));
.and_then(admin_provider_quota_pure::coerce_json_u64);
let plan_type = metadata let plan_type = metadata
.get("plan_type") .get("plan_type")
.and_then(Value::as_str) .and_then(Value::as_str)
@@ -2502,17 +2502,19 @@ fn codex_upstream_metadata_is_at_least_as_fresh(
let Some(metadata) = provider_quota_metadata_bucket(upstream_metadata, "codex") else { let Some(metadata) = provider_quota_metadata_bucket(upstream_metadata, "codex") else {
return false; return false;
}; };
let Some(metadata_updated_at) = metadata // Identity, reset-credit, and model-only updates do not replace the
.get("updated_at") // account's quota observation, even when their timestamp is newer.
.and_then(admin_provider_quota_pure::coerce_json_u64) if !provider_pool_codex_metadata_has_account_quota(metadata) {
return false;
}
let Some(metadata_updated_at) = provider_quota_timestamp_unix_secs(metadata.get("observed_at"))
.or_else(|| provider_quota_timestamp_unix_secs(metadata.get("updated_at")))
else { else {
return false; return false;
}; };
let snapshot_updated_at = quota_snapshot.and_then(|quota| { let snapshot_updated_at = quota_snapshot.and_then(|quota| {
quota provider_quota_timestamp_unix_secs(quota.get("observed_at"))
.get("updated_at") .or_else(|| provider_quota_timestamp_unix_secs(quota.get("updated_at")))
.or_else(|| quota.get("observed_at"))
.and_then(admin_provider_quota_pure::coerce_json_u64)
}); });
snapshot_updated_at.is_none_or(|updated_at| metadata_updated_at >= updated_at) snapshot_updated_at.is_none_or(|updated_at| metadata_updated_at >= updated_at)
@@ -2611,6 +2613,19 @@ pub(crate) fn provider_key_status_snapshot_payload(
let mut snapshot = provider_key_status_snapshot_object(Some(&payload)) let mut snapshot = provider_key_status_snapshot_object(Some(&payload))
.or_else(|| default_provider_key_status_snapshot().as_object().cloned()) .or_else(|| default_provider_key_status_snapshot().as_object().cloned())
.unwrap_or_default(); .unwrap_or_default();
// Legacy snapshots can retain an exhausted summary after a window reset or
// newer quota observation. Use the same decision as scheduling so the
// account list and its status filter do not keep displaying that stale block.
if provider_type.trim().eq_ignore_ascii_case("codex")
&& !aether_provider_pool::provider_pool_key_account_quota_exhausted(key, provider_type)
{
if let Some(quota) = snapshot.get_mut("quota").and_then(Value::as_object_mut) {
quota.insert("exhausted".to_string(), json!(false));
if quota.get("code").and_then(Value::as_str) == Some("exhausted") {
quota.insert("code".to_string(), json!("ok"));
}
}
}
snapshot.insert( snapshot.insert(
"oauth".to_string(), "oauth".to_string(),
build_provider_key_oauth_status_snapshot(key), build_provider_key_oauth_status_snapshot(key),
@@ -4213,6 +4228,116 @@ mod tests {
); );
} }
#[test]
fn provider_key_status_snapshot_payload_clears_stale_codex_exhaustion_summary() {
let mut key = sample_catalog_key();
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"code": "exhausted",
"exhausted": true,
"windows": [{
"code": "weekly",
"scope": "account",
"used_ratio": 0.83,
"remaining_ratio": 0.17,
"reset_at": 4_102_444_800u64
}]
}
}));
let payload = provider_key_status_snapshot_payload(&key, "codex");
assert_eq!(payload["quota"]["code"], "ok");
assert_eq!(payload["quota"]["exhausted"], false);
assert!(payload["quota"]["label"].is_null());
// An explicit current upstream refusal is not a stale percentage summary.
key.status_snapshot.as_mut().unwrap()["quota"]["allowed"] = json!(false);
let payload = provider_key_status_snapshot_payload(&key, "codex");
assert_eq!(payload["quota"]["code"], "exhausted");
assert_eq!(payload["quota"]["exhausted"], true);
// Missing capacity evidence must not clear an exhausted summary either.
key.status_snapshot = Some(json!({"quota": {
"provider_type": "codex", "code": "exhausted", "exhausted": true
}}));
let payload = provider_key_status_snapshot_payload(&key, "codex");
assert_eq!(payload["quota"]["code"], "exhausted");
assert_eq!(payload["quota"]["exhausted"], true);
}
#[test]
fn provider_key_status_snapshot_payload_refreshes_codex_timestamp_formats() {
for updated_at in [
json!(1_900_000_000u64),
json!(1_900_000_000_000u64),
json!("2030-03-17T17:46:40Z"),
] {
let mut key = sample_catalog_key();
key.upstream_metadata = Some(json!({
"codex": {
"updated_at": updated_at,
"primary_used_percent": 83.0,
"primary_reset_at": 4_102_444_800u64
}
}));
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"updated_at": 1_899_999_000u64,
"code": "exhausted",
"exhausted": true,
"allowed": false,
"windows": [{
"code": "weekly",
"scope": "account",
"used_ratio": 1.0,
"remaining_ratio": 0.0,
"reset_at": 4_102_444_800u64
}]
}
}));
let payload = provider_key_status_snapshot_payload(&key, "codex");
assert_eq!(payload["quota"]["code"], "ok");
assert_eq!(payload["quota"]["updated_at"], 1_900_000_000u64);
assert_eq!(payload["quota"]["windows"][0]["used_ratio"], 0.83);
assert!(payload["quota"]["allowed"].is_null());
}
}
#[test]
fn provider_key_status_snapshot_payload_preserves_codex_account_quota_on_unrelated_updates() {
for patch in [
json!({"plan_type": "pro"}),
json!({"spark_primary_used_percent": 83.0}),
json!({"credits_unlimited": false}),
json!({"windows": [{"code": "weekly", "reset_at": 4_102_444_800u64}]}),
] {
let mut metadata = patch;
metadata["updated_at"] = json!("2030-03-17T17:46:40Z");
let mut key = sample_catalog_key();
key.upstream_metadata = Some(json!({"codex": metadata}));
key.status_snapshot = Some(json!({"quota": {
"provider_type": "codex", "updated_at": 200,
"code": "exhausted", "exhausted": true, "allowed": false,
"windows": [{"code": "weekly", "scope": "account", "used_ratio": 1.0,
"remaining_ratio": 0.0, "reset_at": 4_102_444_800u64}]
}}));
let payload = provider_key_status_snapshot_payload(&key, "codex");
assert_eq!(payload["quota"]["code"], "exhausted", "{metadata}");
assert_eq!(payload["quota"]["exhausted"], true, "{metadata}");
assert_eq!(payload["quota"]["allowed"], false, "{metadata}");
assert_eq!(payload["quota"]["updated_at"], 200, "{metadata}");
assert_eq!(
payload["quota"]["windows"][0]["code"], "weekly",
"{metadata}"
);
assert_eq!(
payload["quota"]["windows"][0]["used_ratio"], 1.0,
"{metadata}"
);
}
}
#[test] #[test]
fn provider_key_status_snapshot_payload_restores_complete_codex_cache() { fn provider_key_status_snapshot_payload_restores_complete_codex_cache() {
let mut key = sample_catalog_key(); let mut key = sample_catalog_key();
@@ -2436,6 +2436,134 @@ async fn gateway_reserve_minimum_quota_marks_and_filters_codex_pool_keys() {
} }
} }
#[tokio::test]
async fn gateway_reserve_minimum_quota_recovers_after_config_update_with_cached_catalog() {
let mut provider = sample_provider("provider-codex", "codex", 10);
provider.provider_type = "codex".to_string();
provider.config = Some(json!({
"pool_advanced": {"reserve_minimum_quota": true, "skip_exhausted_accounts": true}
}));
let mut key = sample_key(
"key-codex",
"provider-codex",
"openai:responses",
"oauth-placeholder",
);
key.auth_type = "oauth".to_string();
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex", "updated_at": 100,
"code": "exhausted", "exhausted": true, "allowed": false,
"windows": [{
"code": "weekly", "scope": "account", "used_ratio": 1.0,
"remaining_ratio": 0.0, "reset_at": 4_102_444_800u64
}]
}
}));
key.upstream_metadata = Some(json!({
"codex": {"updated_at": 200, "primary_used_percent": 99.0,
"primary_reset_at": 4_102_444_800u64}
}));
let repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![provider.clone()],
Vec::new(),
vec![key],
));
let state = AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(
GatewayDataState::with_provider_catalog_repository_for_tests(Arc::clone(&repository))
.with_cached_provider_catalog_reader_for_tests(repository),
);
for reserve_enabled in [true, false, true] {
provider.config.as_mut().unwrap()["pool_advanced"]["reserve_minimum_quota"] =
json!(reserve_enabled);
state
.update_provider_catalog_provider(&provider)
.await
.expect("provider should update");
for status in ["all", "quota_exhausted", "available"] {
let response = local_admin_pool_response(
&state,
http::Method::GET,
&format!("/api/admin/pool/provider-codex/keys?status={status}"),
None,
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = serde_json::from_slice(
&to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read"),
)
.expect("json should parse");
let keys = payload["keys"].as_array().expect("keys should be array");
let visible = status == "all" || (status == "quota_exhausted") == reserve_enabled;
assert_eq!(keys.len(), usize::from(visible));
if visible {
assert_eq!(
keys[0]["scheduling_reason"] == "account_quota_exhausted",
reserve_enabled
);
assert_eq!(keys[0]["status_snapshot"]["quota"]["code"], "ok");
}
}
}
}
#[tokio::test]
async fn gateway_codex_pool_filter_clears_stale_exhausted_summary_with_remaining_quota() {
let mut provider = sample_provider("provider-codex", "codex", 10);
provider.provider_type = "codex".to_string();
provider.config = Some(json!({"pool_advanced": {
"reserve_minimum_quota": false, "skip_exhausted_accounts": true
}}));
let mut key = sample_key(
"key-codex",
"provider-codex",
"openai:responses",
"oauth-placeholder",
);
key.auth_type = "oauth".to_string();
key.status_snapshot = Some(json!({"quota": {
"provider_type": "codex", "code": "exhausted", "exhausted": true,
"windows": [{"code": "weekly", "scope": "account", "used_ratio": 0.83,
"remaining_ratio": 0.17, "reset_at": 4_102_444_800u64}]
}}));
let state = AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests(
Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![provider],
Vec::new(),
vec![key],
)),
));
for status in ["all", "available", "quota_exhausted"] {
let response = local_admin_pool_response(
&state,
http::Method::GET,
&format!("/api/admin/pool/provider-codex/keys?status={status}"),
None,
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = serde_json::from_slice(
&to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read"),
)
.expect("json should parse");
let keys = payload["keys"].as_array().expect("keys should be array");
assert_eq!(keys.len(), usize::from(status != "quota_exhausted"));
if let Some(key) = keys.first() {
assert_eq!(key["scheduling_status"], "available");
assert_eq!(key["status_snapshot"]["quota"]["code"], "ok");
assert!(key["account_quota"].as_str().unwrap().contains("17.0%"));
}
}
}
#[tokio::test] #[tokio::test]
async fn gateway_lists_inherited_fixed_provider_api_formats_for_pool_keys() { async fn gateway_lists_inherited_fixed_provider_api_formats_for_pool_keys() {
let mut provider = sample_provider("provider-codex", "codex", 10).with_transport_fields( let mut provider = sample_provider("provider-codex", "codex", 10).with_transport_fields(
@@ -2738,8 +2866,10 @@ async fn gateway_treats_stale_codex_exhausted_snapshot_as_available_when_windows
assert_eq!(keys[0]["scheduling_reason"], json!("available")); assert_eq!(keys[0]["scheduling_reason"], json!("available"));
assert_eq!( assert_eq!(
keys[0]["account_quota"], keys[0]["account_quota"],
json!("周剩余 100.0% (7天0小时后重置) | 5H剩余 100.0% (5小时0分钟后重置)") json!("周剩余 100.0% | 5H剩余 100.0%")
); );
assert_eq!(keys[0]["status_snapshot"]["quota"]["code"], "ok");
assert_eq!(keys[0]["status_snapshot"]["quota"]["exhausted"], false);
} }
#[tokio::test] #[tokio::test]
+1
View File
@@ -11,6 +11,7 @@ aether-contracts.workspace = true
aether-data-contracts.workspace = true aether-data-contracts.workspace = true
aether-pool-core.workspace = true aether-pool-core.workspace = true
aether-provider-transport.workspace = true aether-provider-transport.workspace = true
chrono.workspace = true
serde_json.workspace = true serde_json.workspace = true
url.workspace = true url.workspace = true
uuid.workspace = true uuid.workspace = true
+245 -5
View File
@@ -38,11 +38,12 @@ pub use providers::{
XAI_BILLING_PATH, XAI_USER_PATH, XAI_BILLING_PATH, XAI_USER_PATH,
}; };
pub use quota::{ pub use quota::{
provider_pool_key_account_quota_exhausted, provider_pool_key_minimum_quota_reached, provider_pool_codex_metadata_has_account_quota, provider_pool_key_account_quota_exhausted,
provider_pool_key_model_quota_exhausted, provider_pool_key_model_quota_hard_blocked, provider_pool_key_minimum_quota_reached, provider_pool_key_model_quota_exhausted,
provider_pool_key_quota_hard_blocked, provider_pool_key_scheduling_label, provider_pool_key_model_quota_hard_blocked, provider_pool_key_quota_hard_blocked,
provider_pool_member_quota_snapshot, provider_pool_quota_metadata_provider_type, provider_pool_key_scheduling_label, provider_pool_member_quota_snapshot,
provider_pool_quota_metadata_updated_at, provider_pool_quota_snapshot_updated_at, provider_pool_quota_metadata_provider_type, provider_pool_quota_metadata_updated_at,
provider_pool_quota_snapshot_updated_at,
}; };
pub use quota_refresh::ProviderPoolQuotaRequestSpec; pub use quota_refresh::ProviderPoolQuotaRequestSpec;
pub use service::ProviderPoolService; pub use service::ProviderPoolService;
@@ -1600,6 +1601,245 @@ mod tests {
assert!(!signals.quota_exhausted); assert!(!signals.quota_exhausted);
} }
#[test]
fn codex_newer_flat_quota_metadata_clears_stale_exhausted_snapshot() {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("system time should be after unix epoch")
.as_secs();
let service = ProviderPoolService::with_builtin_adapters();
// Headers and quota refreshes persist the flat metadata shape while the
// derived snapshot may still describe the previous quota observation.
for used_percent in [83.0, 99.0, 100.0] {
let mut key = sample_key(Some(json!({
"codex": {
"updated_at": now,
"primary_used_percent": used_percent,
"primary_reset_at": now + 3600
}
})));
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"observed_at": now - 60,
"updated_at": now - 60,
"code": "exhausted",
"exhausted": true,
"allowed": false,
"limit_reached": true,
"reset_at": now + 3600,
"windows": [{
"code": "weekly",
"scope": "account",
"used_ratio": 1.0,
"remaining_ratio": 0.0,
"is_exhausted": true,
"reset_at": now + 3600
}]
}
}));
for model in [None, Some("gpt-5.4"), Some("o3")] {
let signals = service.member_signals("codex", &key, None, model);
assert_eq!(signals.quota_exhausted, used_percent >= 100.0);
assert!(!signals.quota_hard_blocked);
assert_eq!(
provider_pool_key_minimum_quota_reached(&key, "codex", model),
used_percent >= 99.0
);
}
}
}
#[test]
fn codex_account_and_model_quota_agree_on_explicit_allow_and_deny_flags() {
let service = ProviderPoolService::with_builtin_adapters();
for (flags, exhausted) in [
(json!({ "allowed": true }), false),
(json!({ "limit_reached": false }), false),
(json!({ "allowed": true, "limit_reached": true }), true),
(json!({ "allowed": false, "limit_reached": false }), true),
] {
for use_windows in [false, true] {
let mut metadata = flags.clone();
metadata["updated_at"] = json!(300);
if use_windows {
metadata["windows"] = json!([{ "code": "weekly", "used_ratio": 1.0 }]);
} else {
metadata["primary_used_percent"] = json!(100.0);
}
let key = sample_key(Some(json!({ "codex": metadata })));
for model in [None, Some("gpt-5.4"), Some("o3")] {
assert_eq!(
service
.member_signals("codex", &key, None, model)
.quota_exhausted,
exhausted,
"flags={flags}, use_windows={use_windows}, model={model:?}"
);
// The opt-in reserve still protects a numerically full
// window even when the upstream reports it as allowed.
assert!(provider_pool_key_minimum_quota_reached(
&key, "codex", model
));
}
}
}
}
#[test]
fn codex_model_quota_honors_latest_account_refusal_without_blocking_spark() {
let service = ProviderPoolService::with_builtin_adapters();
for include_window in [false, true] {
let mut metadata = json!({
"updated_at": 300,
"allowed": false,
"limit_reached": true,
"spark_primary_used_percent": 17.0
});
if include_window {
metadata["primary_used_percent"] = json!(83.0);
}
let mut key = sample_key(Some(json!({ "codex": metadata })));
for include_snapshot in [false, true] {
if include_snapshot {
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"observed_at": 200,
"exhausted": false,
"windows": [{ "code": "weekly", "used_ratio": 0.83 }]
}
}));
}
assert!(
service
.member_signals("codex", &key, None, Some("gpt-5.4"))
.quota_exhausted
);
assert!(
!service
.member_signals("codex", &key, None, Some("gpt-5.3-codex-spark"))
.quota_exhausted
);
}
}
}
#[test]
fn codex_quota_source_freshness_supports_all_timestamp_formats() {
let service = ProviderPoolService::with_builtin_adapters();
for observed_at in [
json!(1_700_000_200_u64),
json!(1_700_000_200_000_u64),
json!("1700000200000"),
json!("2023-11-14T22:16:40Z"),
] {
for use_windows in [false, true] {
let metadata = if use_windows {
json!({
"updated_at": observed_at,
"windows": [{ "code": "weekly", "used_ratio": 0.83 }]
})
} else {
json!({ "updated_at": observed_at, "primary_used_percent": 83.0 })
};
let mut key = sample_key(Some(json!({ "codex": metadata })));
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"observed_at": "2023-11-14T22:15:00Z",
"exhausted": true,
"allowed": false,
"windows": [{ "code": "weekly", "used_ratio": 1.0 }]
}
}));
for model in [None, Some("gpt-5.4")] {
let signals = service.member_signals("codex", &key, None, model);
assert!(!signals.quota_exhausted, "timestamp={observed_at}");
assert!(!signals.quota_hard_blocked, "timestamp={observed_at}");
assert!(!provider_pool_key_minimum_quota_reached(
&key, "codex", model
));
}
// Swap freshness while retaining the same observations. An
// older usable bucket cannot erase a later exhausted snapshot.
key.status_snapshot.as_mut().unwrap()["quota"]["observed_at"] =
json!("2023-11-14T22:18:20Z");
assert!(provider_pool_key_account_quota_exhausted(&key, "codex"));
assert!(provider_pool_key_quota_hard_blocked(&key, "codex"));
assert_eq!(
provider_pool_key_model_quota_exhausted(&key, "codex", "gpt-5.4"),
Some(true)
);
}
}
}
#[test]
fn codex_account_quota_recovery_requires_new_account_observation() {
for metadata in [
json!({ "updated_at": 100, "primary_used_percent": 83.0 }),
json!({ "primary_used_percent": 83.0 }),
json!({ "updated_at": 300, "plan_type": "plus" }),
json!({ "updated_at": 300, "credits_unlimited": false }),
json!({ "updated_at": 300, "spark_primary_used_percent": 83.0 }),
json!({ "updated_at": 300, "windows": [{ "code": "weekly" }] }),
] {
let mut key = sample_key(Some(json!({ "codex": metadata })));
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"observed_at": 200,
"exhausted": true,
"allowed": false,
"limit_reached": true,
"windows": [{ "code": "weekly", "used_ratio": 1.0 }]
}
}));
assert!(provider_pool_key_account_quota_exhausted(&key, "codex"));
assert!(provider_pool_key_quota_hard_blocked(&key, "codex"));
assert_eq!(
provider_pool_key_model_quota_exhausted(&key, "codex", "gpt-5.4"),
Some(true)
);
}
let mut key = sample_key(Some(json!({
"codex": {
"updated_at": 300,
"primary_used_percent": 83.0,
"allowed": false,
"limit_reached": true
}
})));
key.status_snapshot = Some(json!({
"quota": { "provider_type": "codex", "updated_at": 200, "exhausted": false }
}));
assert!(provider_pool_key_account_quota_exhausted(&key, "codex"));
assert!(provider_pool_key_quota_hard_blocked(&key, "codex"));
}
#[test]
fn codex_account_updates_do_not_override_independent_model_quota() {
for (spark_ratio, account_percent) in [(0.83, 100.0), (1.0, 83.0)] {
let mut key = sample_key(Some(json!({
"codex": { "updated_at": 300, "primary_used_percent": account_percent }
})));
key.status_snapshot = Some(json!({
"quota": {
"provider_type": "codex",
"updated_at": 200,
"windows": [{ "code": "spark_5h", "used_ratio": spark_ratio }]
}
}));
assert_eq!(
provider_pool_key_model_quota_exhausted(&key, "codex", "gpt-5.3-codex-spark"),
Some(spark_ratio >= 1.0)
);
}
}
#[test] #[test]
fn codex_explicit_quota_block_is_hard_until_reset() { fn codex_explicit_quota_block_is_hard_until_reset() {
let now = std::time::SystemTime::now() let now = std::time::SystemTime::now()
@@ -10,10 +10,11 @@ use crate::provider::{
ProviderPoolMemberInput, ProviderPoolMemberInput,
}; };
use crate::quota::{ use crate::quota::{
provider_pool_current_unix_secs, provider_pool_json_bool, provider_pool_json_f64, provider_pool_codex_metadata_has_account_quota, provider_pool_current_unix_secs,
provider_pool_member_quota_snapshot, provider_pool_metadata_bucket, provider_pool_json_bool, provider_pool_json_f64, provider_pool_member_quota_snapshot,
provider_pool_model_quota_exhausted, provider_pool_quota_snapshot_exhausted_decision, provider_pool_metadata_bucket, provider_pool_model_quota_exhausted,
provider_pool_reset_deadline_elapsed, provider_pool_timestamp_unix_secs, provider_pool_quota_snapshot_exhausted_decision, provider_pool_reset_deadline_elapsed,
provider_pool_source_account_quota_exhausted, provider_pool_timestamp_unix_secs,
}; };
use crate::quota_refresh::ProviderPoolQuotaRequestSpec; use crate::quota_refresh::ProviderPoolQuotaRequestSpec;
@@ -54,6 +55,9 @@ impl ProviderPoolAdapter for CodexProviderPoolAdapter {
}) { }) {
return exhausted; return exhausted;
} }
if let Some(bucket) = codex_newer_account_quota_metadata(input.key, input.provider_type) {
return quota_exhausted_from_bucket(bucket);
}
if let Some(quota_snapshot) = if let Some(quota_snapshot) =
provider_pool_member_quota_snapshot(input.key, input.provider_type) provider_pool_member_quota_snapshot(input.key, input.provider_type)
{ {
@@ -108,15 +112,12 @@ fn codex_explicit_quota_block_active(
key: &aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey, key: &aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey,
provider_type: &str, provider_type: &str,
) -> bool { ) -> bool {
if let Some(bucket) = codex_newer_account_quota_metadata(key, provider_type) {
return codex_explicit_quota_block_from_bucket(bucket);
}
let Some(quota_snapshot) = provider_pool_member_quota_snapshot(key, provider_type) else { let Some(quota_snapshot) = provider_pool_member_quota_snapshot(key, provider_type) else {
return provider_pool_metadata_bucket(key.upstream_metadata.as_ref(), provider_type) return provider_pool_metadata_bucket(key.upstream_metadata.as_ref(), provider_type)
.is_some_and(|bucket| { .is_some_and(codex_explicit_quota_block_from_bucket);
(provider_pool_json_bool(bucket.get("allowed")) == Some(false)
|| provider_pool_json_bool(bucket.get("limit_reached")) == Some(true))
&& !["primary", "secondary"]
.into_iter()
.any(|prefix| codex_window_reset_elapsed(bucket, prefix))
});
}; };
let explicitly_blocked = provider_pool_json_bool(quota_snapshot.get("allowed")) == Some(false) let explicitly_blocked = provider_pool_json_bool(quota_snapshot.get("allowed")) == Some(false)
|| provider_pool_json_bool(quota_snapshot.get("limit_reached")) == Some(true); || provider_pool_json_bool(quota_snapshot.get("limit_reached")) == Some(true);
@@ -133,6 +134,35 @@ fn codex_explicit_quota_block_active(
}) })
} }
fn codex_explicit_quota_block_from_bucket(bucket: &Map<String, Value>) -> bool {
(provider_pool_json_bool(bucket.get("allowed")) == Some(false)
|| provider_pool_json_bool(bucket.get("limit_reached")) == Some(true))
&& !["primary", "secondary"]
.into_iter()
.any(|prefix| codex_window_reset_elapsed(bucket, prefix))
}
/// A successful refresh can update raw metadata before the status snapshot.
/// Do not keep an old account-level block once a newer quota observation exists.
/// Identity-only or model-only updates cannot clear an account quota decision.
fn codex_newer_account_quota_metadata<'a>(
key: &'a aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey,
provider_type: &str,
) -> Option<&'a Map<String, Value>> {
let snapshot = provider_pool_member_quota_snapshot(key, provider_type)?;
let metadata = provider_pool_metadata_bucket(key.upstream_metadata.as_ref(), provider_type)?;
if !provider_pool_codex_metadata_has_account_quota(metadata) {
return None;
}
let metadata_observed_at = provider_pool_timestamp_unix_secs(metadata.get("observed_at"))
.or_else(|| provider_pool_timestamp_unix_secs(metadata.get("updated_at")))?;
let snapshot_observed_at = provider_pool_timestamp_unix_secs(snapshot.get("observed_at"))
.or_else(|| provider_pool_timestamp_unix_secs(snapshot.get("updated_at")));
snapshot_observed_at
.is_none_or(|observed_at| metadata_observed_at >= observed_at)
.then_some(metadata)
}
fn build_codex_wham_headers( fn build_codex_wham_headers(
resolved_oauth_auth: Option<(String, String)>, resolved_oauth_auth: Option<(String, String)>,
decrypted_api_key: Option<&str>, decrypted_api_key: Option<&str>,
@@ -322,11 +352,14 @@ pub(crate) fn quota_exhausted_from_bucket(bucket: &Map<String, Value>) -> bool {
if provider_pool_json_bool(bucket.get("credits_unlimited")) == Some(true) { if provider_pool_json_bool(bucket.get("credits_unlimited")) == Some(true) {
return false; return false;
} }
let account_windows_exhausted = provider_pool_source_account_quota_exhausted(bucket);
let has_window_data = provider_pool_json_f64(bucket.get("primary_used_percent")).is_some() let has_window_data = provider_pool_json_f64(bucket.get("primary_used_percent")).is_some()
|| provider_pool_json_f64(bucket.get("secondary_used_percent")).is_some(); || provider_pool_json_f64(bucket.get("secondary_used_percent")).is_some()
|| account_windows_exhausted.is_some();
if !has_window_data && provider_pool_json_bool(bucket.get("has_credits")) == Some(false) { if !has_window_data && provider_pool_json_bool(bucket.get("has_credits")) == Some(false) {
return true; return true;
} }
codex_window_used_percent_exhausted(bucket, "primary") account_windows_exhausted == Some(true)
|| codex_window_used_percent_exhausted(bucket, "primary")
|| codex_window_used_percent_exhausted(bucket, "secondary") || codex_window_used_percent_exhausted(bucket, "secondary")
} }
+132 -11
View File
@@ -87,6 +87,7 @@ fn provider_pool_quota_reaches_reserve(
provider_model_name: Option<&str>, provider_model_name: Option<&str>,
reserve_ratio: f64, reserve_ratio: f64,
) -> Option<bool> { ) -> Option<bool> {
let is_codex = provider_type.trim().eq_ignore_ascii_case("codex");
let requested = provider_model_name.map(provider_pool_identifier_tokens); let requested = provider_model_name.map(provider_pool_identifier_tokens);
if reserve_ratio <= 0.0 && requested.as_ref().is_some_and(|tokens| tokens.is_empty()) { if reserve_ratio <= 0.0 && requested.as_ref().is_some_and(|tokens| tokens.is_empty()) {
return None; return None;
@@ -103,12 +104,19 @@ fn provider_pool_quota_reaches_reserve(
let mut resolved = None::<(Option<u64>, bool)>; let mut resolved = None::<(Option<u64>, bool)>;
let mut resolved_specific_bucket = false; let mut resolved_specific_bucket = false;
for source in sources.into_iter().flatten() { for source in sources.into_iter().flatten() {
let observed_at = provider_pool_timestamp_unix_secs(source.get("observed_at"))
.or_else(|| provider_pool_timestamp_unix_secs(source.get("updated_at")));
let account_signal = (is_codex && reserve_ratio <= 0.0)
.then(|| provider_pool_codex_account_quota_signal(source, observed_at))
.flatten();
let mut windows = provider_pool_collect_quota_windows(source); let mut windows = provider_pool_collect_quota_windows(source);
if reserve_ratio > 0.0 && provider_type.trim().eq_ignore_ascii_case("codex") { if is_codex {
// Raw refresh metadata can be newer than the materialized windows. // Raw refresh metadata can be newer than the materialized windows.
// Include all four legacy slots before selecting the model bucket. // Include all four legacy slots before selecting the model bucket.
for prefix in ["primary", "secondary", "spark_primary", "spark_secondary"] { for prefix in ["primary", "secondary", "spark_primary", "spark_secondary"] {
let Some(used_percent) = source.get(&format!("{prefix}_used_percent")) else { let Some(used_percent) =
provider_pool_json_f64(source.get(&format!("{prefix}_used_percent")))
else {
continue; continue;
}; };
if provider_pool_json_f64(source.get(&format!("{prefix}_window_minutes"))) if provider_pool_json_f64(source.get(&format!("{prefix}_window_minutes")))
@@ -118,7 +126,7 @@ fn provider_pool_quota_reaches_reserve(
} }
let mut window = Map::from_iter([ let mut window = Map::from_iter([
("code".to_string(), json!(prefix)), ("code".to_string(), json!(prefix)),
("used_percent".to_string(), used_percent.clone()), ("used_percent".to_string(), json!(used_percent)),
]); ]);
for field in [ for field in [
"reset_at", "reset_at",
@@ -132,12 +140,21 @@ fn provider_pool_quota_reaches_reserve(
} }
windows.push(window); windows.push(window);
} }
windows.retain(provider_pool_quota_window_has_observation);
if let Some(exhausted) = account_signal {
if !windows.iter().any(provider_pool_window_is_generic) {
// A flags-only quota refresh is still a newer account
// observation, but must not override a model-specific bucket.
windows.push(Map::from_iter([
("code".to_string(), json!("account")),
("is_exhausted".to_string(), json!(exhausted)),
]));
}
}
} }
if windows.is_empty() { if windows.is_empty() {
continue; continue;
} }
let observed_at = provider_pool_timestamp_unix_secs(source.get("observed_at"))
.or_else(|| provider_pool_timestamp_unix_secs(source.get("updated_at")));
let model_matches = windows let model_matches = windows
.iter() .iter()
.filter(|window| { .filter(|window| {
@@ -155,7 +172,7 @@ fn provider_pool_quota_reaches_reserve(
provider_pool_explicit_model_windows_exhausted(model_matches, observed_at) provider_pool_explicit_model_windows_exhausted(model_matches, observed_at)
}; };
if resolved.is_none() if resolved.is_none()
|| (reserve_ratio > 0.0 && !resolved_specific_bucket) || (is_codex && !resolved_specific_bucket)
|| provider_pool_should_replace_model_quota_resolution( || provider_pool_should_replace_model_quota_resolution(
resolved.as_ref().and_then(|(observed_at, _)| *observed_at), resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
observed_at, observed_at,
@@ -183,7 +200,7 @@ fn provider_pool_quota_reaches_reserve(
let exhausted = let exhausted =
provider_pool_any_window_exhausted(family_matches, observed_at, reserve_ratio); provider_pool_any_window_exhausted(family_matches, observed_at, reserve_ratio);
if resolved.is_none() if resolved.is_none()
|| (reserve_ratio > 0.0 && !resolved_specific_bucket) || (is_codex && !resolved_specific_bucket)
|| provider_pool_should_replace_model_quota_resolution( || provider_pool_should_replace_model_quota_resolution(
resolved.as_ref().and_then(|(observed_at, _)| *observed_at), resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
observed_at, observed_at,
@@ -197,7 +214,7 @@ fn provider_pool_quota_reaches_reserve(
// A newer account observation does not update an independent model // A newer account observation does not update an independent model
// bucket. Only compare freshness between applicable model sources. // bucket. Only compare freshness between applicable model sources.
if reserve_ratio > 0.0 && resolved_specific_bucket { if is_codex && resolved_specific_bucket {
continue; continue;
} }
@@ -211,8 +228,9 @@ fn provider_pool_quota_reaches_reserve(
.filter(|window| provider_pool_window_is_generic(window)) .filter(|window| provider_pool_window_is_generic(window))
.collect::<Vec<_>>(); .collect::<Vec<_>>();
if !generic_matches.is_empty() { if !generic_matches.is_empty() {
let exhausted = let exhausted = account_signal.unwrap_or_else(|| {
provider_pool_any_window_exhausted(generic_matches, observed_at, reserve_ratio); provider_pool_any_window_exhausted(generic_matches, observed_at, reserve_ratio)
});
if resolved.is_none() if resolved.is_none()
|| provider_pool_should_replace_model_quota_resolution( || provider_pool_should_replace_model_quota_resolution(
resolved.as_ref().and_then(|(observed_at, _)| *observed_at), resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
@@ -227,6 +245,37 @@ fn provider_pool_quota_reaches_reserve(
resolved.map(|(_, exhausted)| exhausted) resolved.map(|(_, exhausted)| exhausted)
} }
fn provider_pool_codex_account_quota_signal(
source: &Map<String, Value>,
observed_at: Option<u64>,
) -> Option<bool> {
let allowed = provider_pool_json_bool(source.get("allowed"));
let limit_reached = provider_pool_json_bool(source.get("limit_reached"));
if allowed == Some(false) || limit_reached == Some(true) {
let reset_elapsed = provider_pool_current_unix_secs().is_some_and(|now| {
provider_pool_reset_deadline_elapsed(source, observed_at, now)
|| ["primary", "secondary"].into_iter().any(|prefix| {
let window = [
"reset_at",
"next_reset_at",
"reset_seconds",
"reset_after_seconds",
]
.into_iter()
.filter_map(|field| {
source
.get(&format!("{prefix}_{field}"))
.map(|value| (field.to_string(), value.clone()))
})
.collect::<Map<_, _>>();
provider_pool_reset_deadline_elapsed(&window, observed_at, now)
})
});
return (!reset_elapsed).then_some(true);
}
(allowed == Some(true) || limit_reached == Some(false)).then_some(false)
}
fn provider_pool_should_replace_model_quota_resolution( fn provider_pool_should_replace_model_quota_resolution(
previous_observed_at: Option<u64>, previous_observed_at: Option<u64>,
next_observed_at: Option<u64>, next_observed_at: Option<u64>,
@@ -282,6 +331,69 @@ fn provider_pool_any_window_exhausted(
}) })
} }
fn provider_pool_quota_window_has_observation(window: &Map<String, Value>) -> bool {
["is_exhausted", "exhausted"]
.into_iter()
.any(|field| provider_pool_json_bool(window.get(field)).is_some())
|| [
"used_ratio",
"usage_ratio",
"used_percent",
"remaining_ratio",
"remaining_fraction",
"remaining_percent",
]
.into_iter()
.any(|field| provider_pool_json_f64(window.get(field)).is_some())
|| (provider_pool_json_f64(
window
.get("remaining")
.or_else(|| window.get("remaining_value")),
)
.is_some()
&& provider_pool_json_f64(
window
.get("limit")
.or_else(|| window.get("limit_value"))
.or_else(|| window.get("total")),
)
.is_some_and(|limit| limit > 0.0))
}
pub(crate) fn provider_pool_source_account_quota_exhausted(
source: &Map<String, Value>,
) -> Option<bool> {
let windows = provider_pool_collect_quota_windows(source);
let account_windows = windows
.iter()
.filter(|window| provider_pool_window_is_generic(window))
.filter(|window| provider_pool_quota_window_has_observation(window))
.collect::<Vec<_>>();
if account_windows.is_empty() {
return None;
}
let observed_at = provider_pool_timestamp_unix_secs(source.get("observed_at"))
.or_else(|| provider_pool_timestamp_unix_secs(source.get("updated_at")));
Some(provider_pool_any_window_exhausted(
account_windows,
observed_at,
0.0,
))
}
/// Whether Codex metadata contains an account quota observation rather than an
/// identity-only update or an independent model quota bucket.
pub fn provider_pool_codex_metadata_has_account_quota(source: &Map<String, Value>) -> bool {
["primary_used_percent", "secondary_used_percent"]
.into_iter()
.any(|field| provider_pool_json_f64(source.get(field)).is_some())
|| provider_pool_source_account_quota_exhausted(source).is_some()
|| ["allowed", "limit_reached", "has_credits"]
.into_iter()
.any(|field| provider_pool_json_bool(source.get(field)).is_some())
|| provider_pool_json_bool(source.get("credits_unlimited")) == Some(true)
}
/// Whether an applicable Codex window has at most 1% remaining. Missing quota /// Whether an applicable Codex window has at most 1% remaining. Missing quota
/// data and windows whose reset has elapsed do not trigger this opt-in guard. /// data and windows whose reset has elapsed do not trigger this opt-in guard.
pub fn provider_pool_key_minimum_quota_reached( pub fn provider_pool_key_minimum_quota_reached(
@@ -700,7 +812,16 @@ pub(crate) fn provider_pool_json_f64(value: Option<&Value>) -> Option<f64> {
} }
pub(crate) fn provider_pool_timestamp_unix_secs(value: Option<&Value>) -> Option<u64> { pub(crate) fn provider_pool_timestamp_unix_secs(value: Option<&Value>) -> Option<u64> {
let mut timestamp = provider_pool_json_f64(value)?; let mut timestamp = match provider_pool_json_f64(value) {
Some(timestamp) => timestamp,
None => {
return chrono::DateTime::parse_from_rfc3339(value?.as_str()?.trim())
.ok()?
.timestamp()
.try_into()
.ok();
}
};
if timestamp <= 0.0 { if timestamp <= 0.0 {
return None; return None;
} }
+55 -18
View File
@@ -10055,6 +10055,24 @@ mod tests {
2, 2,
"the later direct caller should receive its own bounded write attempt" "the later direct caller should receive its own bounded write attempt"
); );
// Direct persistence can finish before the submission worker joins the
// barrier handoff and accounts for its completed slot.
timeout(Duration::from_secs(1), async {
loop {
let snapshot = runtime.metrics_snapshot();
let submission = &runtime.lifecycle_submission.state;
if snapshot.terminal_submission_pending == 0
&& snapshot.ordered_lifecycle_pending == 0
&& snapshot.lifecycle_submission_pending == 0
&& submission.admission.available_permits() == submission.capacity
{
break;
}
sleep(Duration::from_millis(1)).await;
}
})
.await
.expect("failed terminal submission accounting and admission should drain");
let snapshot = runtime.metrics_snapshot(); let snapshot = runtime.metrics_snapshot();
assert_eq!(snapshot.terminal_submission_pending, 0); assert_eq!(snapshot.terminal_submission_pending, 0);
assert_eq!(snapshot.ordered_lifecycle_pending, 0); assert_eq!(snapshot.ordered_lifecycle_pending, 0);
@@ -10162,24 +10180,43 @@ mod tests {
.await; .await;
assert_eq!(remaining_policy_panics.load(Ordering::Acquire), 0); assert_eq!(remaining_policy_panics.load(Ordering::Acquire), 0);
let records = records.lock().expect("records lock"); {
assert_eq!( let records = records.lock().expect("records lock");
records assert_eq!(
.iter() records
.filter(|record| record.request_id == healthy_request_id) .iter()
.count(), .filter(|record| record.request_id == healthy_request_id)
2, .count(),
"the same terminal shard should continue processing healthy requests" 2,
); "the same terminal shard should continue processing healthy requests"
assert!( );
records assert!(
.iter() records
.filter(|record| record.request_id == failed_request_id) .iter()
.count() .filter(|record| record.request_id == failed_request_id)
== 1, .count()
"only the later healthy attempt should persist for the panicked request" == 1,
); "only the later healthy attempt should persist for the panicked request"
drop(records); );
}
// The final direct attempt also submits a barrier whose worker may
// account for completion after the persistence call has returned.
timeout(Duration::from_secs(1), async {
loop {
let snapshot = runtime.metrics_snapshot();
let submission = &runtime.lifecycle_submission.state;
if snapshot.terminal_submission_pending == 0
&& snapshot.ordered_lifecycle_pending == 0
&& snapshot.lifecycle_submission_pending == 0
&& submission.admission.available_permits() == submission.capacity
{
break;
}
sleep(Duration::from_millis(1)).await;
}
})
.await
.expect("panicked terminal submission accounting and admission should drain");
let snapshot = runtime.metrics_snapshot(); let snapshot = runtime.metrics_snapshot();
assert_eq!(snapshot.terminal_submission_pending, 0); assert_eq!(snapshot.terminal_submission_pending, 0);
assert_eq!(snapshot.ordered_lifecycle_pending, 0); assert_eq!(snapshot.ordered_lifecycle_pending, 0);
@@ -0,0 +1,76 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
const { getMock, patchMock } = vi.hoisted(() => ({ getMock: vi.fn(), patchMock: vi.fn() }))
vi.mock('@/api/client', () => ({ default: { get: getMock, patch: patchMock } }))
import { getPoolOverview, listPoolKeys, listPoolScores } from '@/api/endpoints/pool'
import { getProvider, updateProvider } from '@/api/endpoints/providers'
import { cache } from '@/utils/cache'
const options = { cacheTtlMs: 30_000 }
const provider = { id: 'codex', pool_advanced: { reserve_minimum_quota: false } }
function deferred<T>() {
let resolve!: (value: T) => void
const promise = new Promise<T>((resolvePromise) => { resolve = resolvePromise })
return { promise, resolve }
}
beforeEach(() => {
cache.clear()
getMock.mockReset()
patchMock.mockReset()
patchMock.mockResolvedValue({ data: provider })
})
describe('provider pool settings cache invalidation', () => {
it('refreshes every cached key page, score page and overview after changing pool settings', async () => {
getMock.mockResolvedValue({ data: { version: 'old' } })
await getPoolOverview(options)
await listPoolKeys('codex', {}, options)
await listPoolKeys('codex', { page: 2, status: 'quota_exhausted' }, options)
await listPoolScores('codex', {}, options)
await listPoolKeys('codex-other', {}, options)
await updateProvider('codex', { pool_advanced: { reserve_minimum_quota: false } })
getMock.mockResolvedValue({ data: { version: 'new' } })
await expect(getPoolOverview(options)).resolves.toEqual({ version: 'new' })
await expect(listPoolKeys('codex', {}, options)).resolves.toEqual({ version: 'new' })
await expect(listPoolKeys('codex', { page: 2, status: 'quota_exhausted' }, options))
.resolves.toEqual({ version: 'new' })
await expect(listPoolScores('codex', {}, options)).resolves.toEqual({ version: 'new' })
await expect(listPoolKeys('codex-other', {}, options)).resolves.toEqual({ version: 'old' })
})
it('does not reuse or cache a key request started before the settings were saved', async () => {
const oldResponse = deferred<{ data: { version: string } }>()
const newResponse = deferred<{ data: { version: string } }>()
getMock.mockReturnValueOnce(oldResponse.promise).mockReturnValueOnce(newResponse.promise)
const oldRequest = listPoolKeys('codex', { page: 2 }, options)
await updateProvider('codex', { pool_advanced: { reserve_minimum_quota: false } })
const newRequest = listPoolKeys('codex', { page: 2 }, options)
expect(getMock).toHaveBeenCalledTimes(2)
oldResponse.resolve({ data: { version: 'old' } })
await oldRequest
const deduped = listPoolKeys('codex', { page: 2 }, options)
expect(getMock).toHaveBeenCalledTimes(2)
newResponse.resolve({ data: { version: 'new' } })
await expect(newRequest).resolves.toEqual({ version: 'new' })
await expect(deduped).resolves.toEqual({ version: 'new' })
await expect(listPoolKeys('codex', { page: 2 }, options)).resolves.toEqual({ version: 'new' })
})
it('does not reuse a provider detail request started before a successful save', async () => {
const oldResponse = deferred<{ data: typeof provider }>()
getMock.mockReturnValueOnce(oldResponse.promise).mockResolvedValueOnce({ data: provider })
const oldRequest = getProvider('codex')
await updateProvider('codex', { pool_advanced: { reserve_minimum_quota: false } })
const newRequest = getProvider('codex')
expect(getMock).toHaveBeenCalledTimes(2)
oldResponse.resolve({ data: { ...provider, pool_advanced: { reserve_minimum_quota: true } } })
await oldRequest
await expect(newRequest).resolves.toMatchObject(provider)
})
})
+10 -1
View File
@@ -1,5 +1,5 @@
import client from '../client' import client from '../client'
import { buildCacheKey, cachedRequest, dedupedRequest } from '@/utils/cache' import { buildCacheKey, cache, cachedRequest, dedupedRequest } from '@/utils/cache'
import type { import type {
ClaudeCodeAdvancedConfig, ClaudeCodeAdvancedConfig,
FailoverRulesConfig, FailoverRulesConfig,
@@ -142,6 +142,15 @@ export async function updateProvider(
requestOptions?: ProviderRequestOptions, requestOptions?: ProviderRequestOptions,
): Promise<ProviderWithEndpointsSummary> { ): Promise<ProviderWithEndpointsSummary> {
const response = await client.patch<ProviderWithEndpointsSummary>(`/api/admin/providers/${providerId}`, data, requestOptions) const response = await client.patch<ProviderWithEndpointsSummary>(`/api/admin/providers/${providerId}`, data, requestOptions)
cache.delete(`providers:detail:${providerId}`)
if ('pool_advanced' in data) {
cache.delete('pool:overview')
for (const kind of ['keys', 'scores']) {
const prefix = `pool:${kind}:${providerId}`
cache.delete(prefix)
cache.deleteByPrefix(`${prefix}:`)
}
}
return normalizeProviderSummary(response.data) return normalizeProviderSummary(response.data)
} }
+8
View File
@@ -61,6 +61,14 @@ class MemoryCache {
this.keyVersions.set(key, (this.keyVersions.get(key) ?? 0) + 1) this.keyVersions.set(key, (this.keyVersions.get(key) ?? 0) + 1)
} }
/** 删除匹配前缀的缓存及进行中请求,旧响应不再回填缓存。 */
deleteByPrefix(prefix: string): void {
const keys = new Set([...this.cache.keys(), ...this.inFlight.keys()])
for (const key of keys) {
if (key.startsWith(prefix)) this.delete(key)
}
}
/** /**
* 清空所有缓存 * 清空所有缓存
*/ */
+3 -1
View File
@@ -1395,6 +1395,8 @@ async function loadOverview(options: { cacheTtlMs?: number, silent?: boolean } =
async function handleSchedulingSaved(updatedProvider: ProviderWithEndpointsSummary) { async function handleSchedulingSaved(updatedProvider: ProviderWithEndpointsSummary) {
if (!selectedProviderId.value || updatedProvider.id !== selectedProviderId.value) return if (!selectedProviderId.value || updatedProvider.id !== selectedProviderId.value) return
// 保存前发出的详情读取不得覆盖这次保存返回的新配置。
providerDataRequestId += 1
// 优先回写保存接口返回值,避免弹窗立即重开时读到旧配置。 // 优先回写保存接口返回值,避免弹窗立即重开时读到旧配置。
if (selectedProviderData.value) { if (selectedProviderData.value) {
Object.assign(selectedProviderData.value, updatedProvider) Object.assign(selectedProviderData.value, updatedProvider)
@@ -1403,7 +1405,7 @@ async function handleSchedulingSaved(updatedProvider: ProviderWithEndpointsSumma
} }
showSchedulingDialog.value = false showSchedulingDialog.value = false
showAdvancedDialog.value = false showAdvancedDialog.value = false
await loadOverview({ silent: true }) await Promise.all([loadKeys({ silent: true }), loadOverview({ silent: true })])
} }
// --- Provider Selection --- // --- Provider Selection ---
@@ -340,12 +340,24 @@ vi.mock('@/components/ui/refresh-button.vue', async () => {
}) })
vi.mock('@/features/pool/components/PoolSchedulingDialog.vue', async () => { vi.mock('@/features/pool/components/PoolSchedulingDialog.vue', async () => {
const { defineComponent } = await import('vue') const { defineComponent, h } = await import('vue')
return { return {
default: defineComponent({ default: defineComponent({
name: 'PoolSchedulingDialogStub', name: 'PoolSchedulingDialogStub',
setup() { props: {
return () => null providerId: { type: String, required: true },
currentConfig: { type: Object, default: null },
},
emits: ['saved'],
setup(props, { emit }) {
return () => h('button', {
'data-testid': 'save-pool-scheduling',
'data-reserve': String(props.currentConfig?.reserve_minimum_quota ?? false),
onClick: () => emit('saved', {
id: props.providerId,
pool_advanced: { ...props.currentConfig, reserve_minimum_quota: false },
}),
}, '保存调度测试')
}, },
}), }),
} }
@@ -1278,4 +1290,80 @@ describe('PoolManagement Codex cycle stats mode', () => {
expect(disabledRoot.querySelector('[data-testid="pool-demand-metrics-button"]')).toBeNull() expect(disabledRoot.querySelector('[data-testid="pool-demand-metrics-button"]')).toBeNull()
}) })
it('reloads account scheduling state immediately after disabling minimum quota reserve', async () => {
const key = createPoolKey('codex', {
scheduling_status: 'blocked',
scheduling_reason: 'account_quota_exhausted',
scheduling_label: '额度耗尽',
})
endpointMocks.getPoolOverview.mockResolvedValue({ items: [createOverview('codex')] })
endpointMocks.listPoolKeys.mockResolvedValue(createKeyPage(key))
endpointMocks.getProvider.mockResolvedValue(createProvider('codex', {
pool_advanced: { reserve_minimum_quota: true },
}))
const root = mountPoolManagement()
await settle()
expect(root.querySelector('tbody')?.textContent).toContain('额度耗尽')
endpointMocks.listPoolKeys.mockClear()
endpointMocks.listPoolKeys.mockResolvedValue(createKeyPage({
...key,
scheduling_status: 'available',
scheduling_reason: 'available',
scheduling_label: '可用',
}))
root.querySelector<HTMLButtonElement>('[data-testid="save-pool-scheduling"]')?.click()
await settle()
expect(endpointMocks.listPoolKeys).toHaveBeenCalledWith(
'codex-provider', expect.anything(), { cacheTtlMs: 0 },
)
expect(root.querySelector('tbody')?.textContent).not.toContain('额度耗尽')
expect(root.querySelector('tbody')?.textContent).toContain('可用')
})
it('shows 17% remaining without marking a schedulable account exhausted when reserve is enabled', async () => {
endpointMocks.getPoolOverview.mockResolvedValue({ items: [createOverview('codex')] })
endpointMocks.getProvider.mockResolvedValue(createProvider('codex', {
pool_advanced: { reserve_minimum_quota: true },
}))
endpointMocks.listPoolKeys.mockResolvedValue(createKeyPage(createPoolKey('codex', {
scheduling_status: 'available',
scheduling_reason: 'available',
scheduling_label: '可用',
status_snapshot: {
account: { code: 'ok', blocked: false },
quota: {
code: 'ok',
exhausted: false,
provider_type: 'codex',
windows: [{ code: '5h', used_ratio: 0.83, remaining_ratio: 0.17 }],
},
oauth: { code: 'none' },
},
})))
const root = mountPoolManagement()
await settle()
expect(root.querySelector('[data-testid="pool-quota-meter-text"]')?.textContent).toBe('17.0%')
expect(root.querySelector('tbody')?.textContent).toContain('可用')
expect(root.querySelector('tbody')?.textContent).not.toContain('额度耗尽')
})
it('does not replace saved scheduling settings with an older provider detail response', async () => {
let resolveOldProvider!: (provider: ReturnType<typeof createProvider>) => void
endpointMocks.getProvider.mockReturnValue(new Promise((resolve) => { resolveOldProvider = resolve }))
endpointMocks.getPoolOverview.mockResolvedValue({ items: [createOverview('codex')] })
endpointMocks.listPoolKeys.mockResolvedValue(createKeyPage(createPoolKey()))
const root = mountPoolManagement()
await settle()
root.querySelector<HTMLButtonElement>('[data-testid="save-pool-scheduling"]')?.click()
await settle()
resolveOldProvider(createProvider('codex', { pool_advanced: { reserve_minimum_quota: true } }))
await settle()
expect(root.querySelector('[data-testid="save-pool-scheduling"]')?.getAttribute('data-reserve')).toBe('false')
})
}) })