mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-04 00:17:45 +08:00
Merge pull request #852 from zhefox/main
fix(codex): restore recovered quota status and stabilize lifecycle tests
This commit is contained in:
Generated
+1
@@ -524,6 +524,7 @@ dependencies = [
|
|||||||
"aether-data-contracts",
|
"aether-data-contracts",
|
||||||
"aether-pool-core",
|
"aether-pool-core",
|
||||||
"aether-provider-transport",
|
"aether-provider-transport",
|
||||||
|
"chrono",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"url",
|
"url",
|
||||||
"uuid",
|
"uuid",
|
||||||
|
|||||||
@@ -4005,7 +4005,7 @@ mod tests {
|
|||||||
async fn pool_key_cursor_reserve_minimum_quota_filters_pages_and_sticky_hits() {
|
async fn pool_key_cursor_reserve_minimum_quota_filters_pages_and_sticky_hits() {
|
||||||
for reserve_enabled in [false, true] {
|
for reserve_enabled in [false, true] {
|
||||||
for sticky in [false, true] {
|
for sticky in [false, true] {
|
||||||
for used_percent in [99.0, 98.0] {
|
for used_percent in [99.0, 98.0, 83.0] {
|
||||||
let provider_config = Some(json!({
|
let provider_config = Some(json!({
|
||||||
"pool_advanced": {
|
"pool_advanced": {
|
||||||
"reserve_minimum_quota": reserve_enabled,
|
"reserve_minimum_quota": reserve_enabled,
|
||||||
@@ -4016,8 +4016,27 @@ mod tests {
|
|||||||
sample_codex_pool_provider("provider-pool", 0, provider_config.clone());
|
sample_codex_pool_provider("provider-pool", 0, provider_config.clone());
|
||||||
let endpoint = sample_codex_pool_endpoint("provider-pool", "endpoint-1");
|
let endpoint = sample_codex_pool_endpoint("provider-pool", "endpoint-1");
|
||||||
let mut reserved = sample_codex_pool_key("provider-pool", "key-low");
|
let mut reserved = sample_codex_pool_key("provider-pool", "key-low");
|
||||||
|
reserved.status_snapshot = Some(json!({
|
||||||
|
"quota": {
|
||||||
|
"provider_type": "codex",
|
||||||
|
"updated_at": 100,
|
||||||
|
"allowed": false,
|
||||||
|
"exhausted": true,
|
||||||
|
"code": "exhausted",
|
||||||
|
"windows": [{
|
||||||
|
"code": "weekly",
|
||||||
|
"scope": "account",
|
||||||
|
"used_ratio": 1.0,
|
||||||
|
"reset_at": 4_102_444_800u64
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
}));
|
||||||
reserved.upstream_metadata = Some(json!({
|
reserved.upstream_metadata = Some(json!({
|
||||||
"codex": {"primary_used_percent": used_percent}
|
"codex": {
|
||||||
|
"updated_at": 200,
|
||||||
|
"primary_used_percent": used_percent,
|
||||||
|
"primary_reset_at": 4_102_444_800u64
|
||||||
|
}
|
||||||
}));
|
}));
|
||||||
let ready = sample_codex_pool_key("provider-pool", "key-ready");
|
let ready = sample_codex_pool_key("provider-pool", "key-ready");
|
||||||
let rows = vec![
|
let rows = vec![
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ use aether_crypto::{
|
|||||||
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
|
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
|
||||||
use aether_provider_pool::{
|
use aether_provider_pool::{
|
||||||
grok_pool_tier_from_quota_bucket, grok_supported_quota_windows_for_tier,
|
grok_pool_tier_from_quota_bucket, grok_supported_quota_windows_for_tier,
|
||||||
|
provider_pool_codex_metadata_has_account_quota,
|
||||||
};
|
};
|
||||||
use aether_scheduler_core::provider_key_circuit_payload_is_active_open_at;
|
use aether_scheduler_core::provider_key_circuit_payload_is_active_open_at;
|
||||||
use serde_json::{json, Map, Value};
|
use serde_json::{json, Map, Value};
|
||||||
@@ -1111,9 +1112,8 @@ fn build_codex_quota_status_snapshot(
|
|||||||
source: &str,
|
source: &str,
|
||||||
) -> Option<Value> {
|
) -> Option<Value> {
|
||||||
let metadata = provider_quota_metadata_bucket(upstream_metadata, "codex")?;
|
let metadata = provider_quota_metadata_bucket(upstream_metadata, "codex")?;
|
||||||
let observed_at_unix_secs = metadata
|
let observed_at_unix_secs = provider_quota_timestamp_unix_secs(metadata.get("observed_at"))
|
||||||
.get("updated_at")
|
.or_else(|| provider_quota_timestamp_unix_secs(metadata.get("updated_at")));
|
||||||
.and_then(admin_provider_quota_pure::coerce_json_u64);
|
|
||||||
let plan_type = metadata
|
let plan_type = metadata
|
||||||
.get("plan_type")
|
.get("plan_type")
|
||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
@@ -2502,17 +2502,19 @@ fn codex_upstream_metadata_is_at_least_as_fresh(
|
|||||||
let Some(metadata) = provider_quota_metadata_bucket(upstream_metadata, "codex") else {
|
let Some(metadata) = provider_quota_metadata_bucket(upstream_metadata, "codex") else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
let Some(metadata_updated_at) = metadata
|
// Identity, reset-credit, and model-only updates do not replace the
|
||||||
.get("updated_at")
|
// account's quota observation, even when their timestamp is newer.
|
||||||
.and_then(admin_provider_quota_pure::coerce_json_u64)
|
if !provider_pool_codex_metadata_has_account_quota(metadata) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let Some(metadata_updated_at) = provider_quota_timestamp_unix_secs(metadata.get("observed_at"))
|
||||||
|
.or_else(|| provider_quota_timestamp_unix_secs(metadata.get("updated_at")))
|
||||||
else {
|
else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
let snapshot_updated_at = quota_snapshot.and_then(|quota| {
|
let snapshot_updated_at = quota_snapshot.and_then(|quota| {
|
||||||
quota
|
provider_quota_timestamp_unix_secs(quota.get("observed_at"))
|
||||||
.get("updated_at")
|
.or_else(|| provider_quota_timestamp_unix_secs(quota.get("updated_at")))
|
||||||
.or_else(|| quota.get("observed_at"))
|
|
||||||
.and_then(admin_provider_quota_pure::coerce_json_u64)
|
|
||||||
});
|
});
|
||||||
|
|
||||||
snapshot_updated_at.is_none_or(|updated_at| metadata_updated_at >= updated_at)
|
snapshot_updated_at.is_none_or(|updated_at| metadata_updated_at >= updated_at)
|
||||||
@@ -2611,6 +2613,19 @@ pub(crate) fn provider_key_status_snapshot_payload(
|
|||||||
let mut snapshot = provider_key_status_snapshot_object(Some(&payload))
|
let mut snapshot = provider_key_status_snapshot_object(Some(&payload))
|
||||||
.or_else(|| default_provider_key_status_snapshot().as_object().cloned())
|
.or_else(|| default_provider_key_status_snapshot().as_object().cloned())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
// Legacy snapshots can retain an exhausted summary after a window reset or
|
||||||
|
// newer quota observation. Use the same decision as scheduling so the
|
||||||
|
// account list and its status filter do not keep displaying that stale block.
|
||||||
|
if provider_type.trim().eq_ignore_ascii_case("codex")
|
||||||
|
&& !aether_provider_pool::provider_pool_key_account_quota_exhausted(key, provider_type)
|
||||||
|
{
|
||||||
|
if let Some(quota) = snapshot.get_mut("quota").and_then(Value::as_object_mut) {
|
||||||
|
quota.insert("exhausted".to_string(), json!(false));
|
||||||
|
if quota.get("code").and_then(Value::as_str) == Some("exhausted") {
|
||||||
|
quota.insert("code".to_string(), json!("ok"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
snapshot.insert(
|
snapshot.insert(
|
||||||
"oauth".to_string(),
|
"oauth".to_string(),
|
||||||
build_provider_key_oauth_status_snapshot(key),
|
build_provider_key_oauth_status_snapshot(key),
|
||||||
@@ -4213,6 +4228,116 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn provider_key_status_snapshot_payload_clears_stale_codex_exhaustion_summary() {
|
||||||
|
let mut key = sample_catalog_key();
|
||||||
|
key.status_snapshot = Some(json!({
|
||||||
|
"quota": {
|
||||||
|
"provider_type": "codex",
|
||||||
|
"code": "exhausted",
|
||||||
|
"exhausted": true,
|
||||||
|
"windows": [{
|
||||||
|
"code": "weekly",
|
||||||
|
"scope": "account",
|
||||||
|
"used_ratio": 0.83,
|
||||||
|
"remaining_ratio": 0.17,
|
||||||
|
"reset_at": 4_102_444_800u64
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
let payload = provider_key_status_snapshot_payload(&key, "codex");
|
||||||
|
assert_eq!(payload["quota"]["code"], "ok");
|
||||||
|
assert_eq!(payload["quota"]["exhausted"], false);
|
||||||
|
assert!(payload["quota"]["label"].is_null());
|
||||||
|
|
||||||
|
// An explicit current upstream refusal is not a stale percentage summary.
|
||||||
|
key.status_snapshot.as_mut().unwrap()["quota"]["allowed"] = json!(false);
|
||||||
|
let payload = provider_key_status_snapshot_payload(&key, "codex");
|
||||||
|
assert_eq!(payload["quota"]["code"], "exhausted");
|
||||||
|
assert_eq!(payload["quota"]["exhausted"], true);
|
||||||
|
|
||||||
|
// Missing capacity evidence must not clear an exhausted summary either.
|
||||||
|
key.status_snapshot = Some(json!({"quota": {
|
||||||
|
"provider_type": "codex", "code": "exhausted", "exhausted": true
|
||||||
|
}}));
|
||||||
|
let payload = provider_key_status_snapshot_payload(&key, "codex");
|
||||||
|
assert_eq!(payload["quota"]["code"], "exhausted");
|
||||||
|
assert_eq!(payload["quota"]["exhausted"], true);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn provider_key_status_snapshot_payload_refreshes_codex_timestamp_formats() {
|
||||||
|
for updated_at in [
|
||||||
|
json!(1_900_000_000u64),
|
||||||
|
json!(1_900_000_000_000u64),
|
||||||
|
json!("2030-03-17T17:46:40Z"),
|
||||||
|
] {
|
||||||
|
let mut key = sample_catalog_key();
|
||||||
|
key.upstream_metadata = Some(json!({
|
||||||
|
"codex": {
|
||||||
|
"updated_at": updated_at,
|
||||||
|
"primary_used_percent": 83.0,
|
||||||
|
"primary_reset_at": 4_102_444_800u64
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
key.status_snapshot = Some(json!({
|
||||||
|
"quota": {
|
||||||
|
"provider_type": "codex",
|
||||||
|
"updated_at": 1_899_999_000u64,
|
||||||
|
"code": "exhausted",
|
||||||
|
"exhausted": true,
|
||||||
|
"allowed": false,
|
||||||
|
"windows": [{
|
||||||
|
"code": "weekly",
|
||||||
|
"scope": "account",
|
||||||
|
"used_ratio": 1.0,
|
||||||
|
"remaining_ratio": 0.0,
|
||||||
|
"reset_at": 4_102_444_800u64
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
let payload = provider_key_status_snapshot_payload(&key, "codex");
|
||||||
|
assert_eq!(payload["quota"]["code"], "ok");
|
||||||
|
assert_eq!(payload["quota"]["updated_at"], 1_900_000_000u64);
|
||||||
|
assert_eq!(payload["quota"]["windows"][0]["used_ratio"], 0.83);
|
||||||
|
assert!(payload["quota"]["allowed"].is_null());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn provider_key_status_snapshot_payload_preserves_codex_account_quota_on_unrelated_updates() {
|
||||||
|
for patch in [
|
||||||
|
json!({"plan_type": "pro"}),
|
||||||
|
json!({"spark_primary_used_percent": 83.0}),
|
||||||
|
json!({"credits_unlimited": false}),
|
||||||
|
json!({"windows": [{"code": "weekly", "reset_at": 4_102_444_800u64}]}),
|
||||||
|
] {
|
||||||
|
let mut metadata = patch;
|
||||||
|
metadata["updated_at"] = json!("2030-03-17T17:46:40Z");
|
||||||
|
let mut key = sample_catalog_key();
|
||||||
|
key.upstream_metadata = Some(json!({"codex": metadata}));
|
||||||
|
key.status_snapshot = Some(json!({"quota": {
|
||||||
|
"provider_type": "codex", "updated_at": 200,
|
||||||
|
"code": "exhausted", "exhausted": true, "allowed": false,
|
||||||
|
"windows": [{"code": "weekly", "scope": "account", "used_ratio": 1.0,
|
||||||
|
"remaining_ratio": 0.0, "reset_at": 4_102_444_800u64}]
|
||||||
|
}}));
|
||||||
|
let payload = provider_key_status_snapshot_payload(&key, "codex");
|
||||||
|
assert_eq!(payload["quota"]["code"], "exhausted", "{metadata}");
|
||||||
|
assert_eq!(payload["quota"]["exhausted"], true, "{metadata}");
|
||||||
|
assert_eq!(payload["quota"]["allowed"], false, "{metadata}");
|
||||||
|
assert_eq!(payload["quota"]["updated_at"], 200, "{metadata}");
|
||||||
|
assert_eq!(
|
||||||
|
payload["quota"]["windows"][0]["code"], "weekly",
|
||||||
|
"{metadata}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
payload["quota"]["windows"][0]["used_ratio"], 1.0,
|
||||||
|
"{metadata}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn provider_key_status_snapshot_payload_restores_complete_codex_cache() {
|
fn provider_key_status_snapshot_payload_restores_complete_codex_cache() {
|
||||||
let mut key = sample_catalog_key();
|
let mut key = sample_catalog_key();
|
||||||
|
|||||||
@@ -2436,6 +2436,134 @@ async fn gateway_reserve_minimum_quota_marks_and_filters_codex_pool_keys() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn gateway_reserve_minimum_quota_recovers_after_config_update_with_cached_catalog() {
|
||||||
|
let mut provider = sample_provider("provider-codex", "codex", 10);
|
||||||
|
provider.provider_type = "codex".to_string();
|
||||||
|
provider.config = Some(json!({
|
||||||
|
"pool_advanced": {"reserve_minimum_quota": true, "skip_exhausted_accounts": true}
|
||||||
|
}));
|
||||||
|
let mut key = sample_key(
|
||||||
|
"key-codex",
|
||||||
|
"provider-codex",
|
||||||
|
"openai:responses",
|
||||||
|
"oauth-placeholder",
|
||||||
|
);
|
||||||
|
key.auth_type = "oauth".to_string();
|
||||||
|
key.status_snapshot = Some(json!({
|
||||||
|
"quota": {
|
||||||
|
"provider_type": "codex", "updated_at": 100,
|
||||||
|
"code": "exhausted", "exhausted": true, "allowed": false,
|
||||||
|
"windows": [{
|
||||||
|
"code": "weekly", "scope": "account", "used_ratio": 1.0,
|
||||||
|
"remaining_ratio": 0.0, "reset_at": 4_102_444_800u64
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
key.upstream_metadata = Some(json!({
|
||||||
|
"codex": {"updated_at": 200, "primary_used_percent": 99.0,
|
||||||
|
"primary_reset_at": 4_102_444_800u64}
|
||||||
|
}));
|
||||||
|
let repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||||
|
vec![provider.clone()],
|
||||||
|
Vec::new(),
|
||||||
|
vec![key],
|
||||||
|
));
|
||||||
|
let state = AppState::new()
|
||||||
|
.expect("gateway should build")
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
GatewayDataState::with_provider_catalog_repository_for_tests(Arc::clone(&repository))
|
||||||
|
.with_cached_provider_catalog_reader_for_tests(repository),
|
||||||
|
);
|
||||||
|
for reserve_enabled in [true, false, true] {
|
||||||
|
provider.config.as_mut().unwrap()["pool_advanced"]["reserve_minimum_quota"] =
|
||||||
|
json!(reserve_enabled);
|
||||||
|
state
|
||||||
|
.update_provider_catalog_provider(&provider)
|
||||||
|
.await
|
||||||
|
.expect("provider should update");
|
||||||
|
for status in ["all", "quota_exhausted", "available"] {
|
||||||
|
let response = local_admin_pool_response(
|
||||||
|
&state,
|
||||||
|
http::Method::GET,
|
||||||
|
&format!("/api/admin/pool/provider-codex/keys?status={status}"),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
let payload: serde_json::Value = serde_json::from_slice(
|
||||||
|
&to_bytes(response.into_body(), usize::MAX)
|
||||||
|
.await
|
||||||
|
.expect("body should read"),
|
||||||
|
)
|
||||||
|
.expect("json should parse");
|
||||||
|
let keys = payload["keys"].as_array().expect("keys should be array");
|
||||||
|
let visible = status == "all" || (status == "quota_exhausted") == reserve_enabled;
|
||||||
|
assert_eq!(keys.len(), usize::from(visible));
|
||||||
|
if visible {
|
||||||
|
assert_eq!(
|
||||||
|
keys[0]["scheduling_reason"] == "account_quota_exhausted",
|
||||||
|
reserve_enabled
|
||||||
|
);
|
||||||
|
assert_eq!(keys[0]["status_snapshot"]["quota"]["code"], "ok");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn gateway_codex_pool_filter_clears_stale_exhausted_summary_with_remaining_quota() {
|
||||||
|
let mut provider = sample_provider("provider-codex", "codex", 10);
|
||||||
|
provider.provider_type = "codex".to_string();
|
||||||
|
provider.config = Some(json!({"pool_advanced": {
|
||||||
|
"reserve_minimum_quota": false, "skip_exhausted_accounts": true
|
||||||
|
}}));
|
||||||
|
let mut key = sample_key(
|
||||||
|
"key-codex",
|
||||||
|
"provider-codex",
|
||||||
|
"openai:responses",
|
||||||
|
"oauth-placeholder",
|
||||||
|
);
|
||||||
|
key.auth_type = "oauth".to_string();
|
||||||
|
key.status_snapshot = Some(json!({"quota": {
|
||||||
|
"provider_type": "codex", "code": "exhausted", "exhausted": true,
|
||||||
|
"windows": [{"code": "weekly", "scope": "account", "used_ratio": 0.83,
|
||||||
|
"remaining_ratio": 0.17, "reset_at": 4_102_444_800u64}]
|
||||||
|
}}));
|
||||||
|
let state = AppState::new()
|
||||||
|
.expect("gateway should build")
|
||||||
|
.with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests(
|
||||||
|
Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||||
|
vec![provider],
|
||||||
|
Vec::new(),
|
||||||
|
vec![key],
|
||||||
|
)),
|
||||||
|
));
|
||||||
|
for status in ["all", "available", "quota_exhausted"] {
|
||||||
|
let response = local_admin_pool_response(
|
||||||
|
&state,
|
||||||
|
http::Method::GET,
|
||||||
|
&format!("/api/admin/pool/provider-codex/keys?status={status}"),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
let payload: serde_json::Value = serde_json::from_slice(
|
||||||
|
&to_bytes(response.into_body(), usize::MAX)
|
||||||
|
.await
|
||||||
|
.expect("body should read"),
|
||||||
|
)
|
||||||
|
.expect("json should parse");
|
||||||
|
let keys = payload["keys"].as_array().expect("keys should be array");
|
||||||
|
assert_eq!(keys.len(), usize::from(status != "quota_exhausted"));
|
||||||
|
if let Some(key) = keys.first() {
|
||||||
|
assert_eq!(key["scheduling_status"], "available");
|
||||||
|
assert_eq!(key["status_snapshot"]["quota"]["code"], "ok");
|
||||||
|
assert!(key["account_quota"].as_str().unwrap().contains("17.0%"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn gateway_lists_inherited_fixed_provider_api_formats_for_pool_keys() {
|
async fn gateway_lists_inherited_fixed_provider_api_formats_for_pool_keys() {
|
||||||
let mut provider = sample_provider("provider-codex", "codex", 10).with_transport_fields(
|
let mut provider = sample_provider("provider-codex", "codex", 10).with_transport_fields(
|
||||||
@@ -2738,8 +2866,10 @@ async fn gateway_treats_stale_codex_exhausted_snapshot_as_available_when_windows
|
|||||||
assert_eq!(keys[0]["scheduling_reason"], json!("available"));
|
assert_eq!(keys[0]["scheduling_reason"], json!("available"));
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
keys[0]["account_quota"],
|
keys[0]["account_quota"],
|
||||||
json!("周剩余 100.0% (7天0小时后重置) | 5H剩余 100.0% (5小时0分钟后重置)")
|
json!("周剩余 100.0% | 5H剩余 100.0%")
|
||||||
);
|
);
|
||||||
|
assert_eq!(keys[0]["status_snapshot"]["quota"]["code"], "ok");
|
||||||
|
assert_eq!(keys[0]["status_snapshot"]["quota"]["exhausted"], false);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ aether-contracts.workspace = true
|
|||||||
aether-data-contracts.workspace = true
|
aether-data-contracts.workspace = true
|
||||||
aether-pool-core.workspace = true
|
aether-pool-core.workspace = true
|
||||||
aether-provider-transport.workspace = true
|
aether-provider-transport.workspace = true
|
||||||
|
chrono.workspace = true
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
url.workspace = true
|
url.workspace = true
|
||||||
uuid.workspace = true
|
uuid.workspace = true
|
||||||
|
|||||||
@@ -38,11 +38,12 @@ pub use providers::{
|
|||||||
XAI_BILLING_PATH, XAI_USER_PATH,
|
XAI_BILLING_PATH, XAI_USER_PATH,
|
||||||
};
|
};
|
||||||
pub use quota::{
|
pub use quota::{
|
||||||
provider_pool_key_account_quota_exhausted, provider_pool_key_minimum_quota_reached,
|
provider_pool_codex_metadata_has_account_quota, provider_pool_key_account_quota_exhausted,
|
||||||
provider_pool_key_model_quota_exhausted, provider_pool_key_model_quota_hard_blocked,
|
provider_pool_key_minimum_quota_reached, provider_pool_key_model_quota_exhausted,
|
||||||
provider_pool_key_quota_hard_blocked, provider_pool_key_scheduling_label,
|
provider_pool_key_model_quota_hard_blocked, provider_pool_key_quota_hard_blocked,
|
||||||
provider_pool_member_quota_snapshot, provider_pool_quota_metadata_provider_type,
|
provider_pool_key_scheduling_label, provider_pool_member_quota_snapshot,
|
||||||
provider_pool_quota_metadata_updated_at, provider_pool_quota_snapshot_updated_at,
|
provider_pool_quota_metadata_provider_type, provider_pool_quota_metadata_updated_at,
|
||||||
|
provider_pool_quota_snapshot_updated_at,
|
||||||
};
|
};
|
||||||
pub use quota_refresh::ProviderPoolQuotaRequestSpec;
|
pub use quota_refresh::ProviderPoolQuotaRequestSpec;
|
||||||
pub use service::ProviderPoolService;
|
pub use service::ProviderPoolService;
|
||||||
@@ -1600,6 +1601,245 @@ mod tests {
|
|||||||
assert!(!signals.quota_exhausted);
|
assert!(!signals.quota_exhausted);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn codex_newer_flat_quota_metadata_clears_stale_exhausted_snapshot() {
|
||||||
|
let now = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.expect("system time should be after unix epoch")
|
||||||
|
.as_secs();
|
||||||
|
let service = ProviderPoolService::with_builtin_adapters();
|
||||||
|
// Headers and quota refreshes persist the flat metadata shape while the
|
||||||
|
// derived snapshot may still describe the previous quota observation.
|
||||||
|
for used_percent in [83.0, 99.0, 100.0] {
|
||||||
|
let mut key = sample_key(Some(json!({
|
||||||
|
"codex": {
|
||||||
|
"updated_at": now,
|
||||||
|
"primary_used_percent": used_percent,
|
||||||
|
"primary_reset_at": now + 3600
|
||||||
|
}
|
||||||
|
})));
|
||||||
|
key.status_snapshot = Some(json!({
|
||||||
|
"quota": {
|
||||||
|
"provider_type": "codex",
|
||||||
|
"observed_at": now - 60,
|
||||||
|
"updated_at": now - 60,
|
||||||
|
"code": "exhausted",
|
||||||
|
"exhausted": true,
|
||||||
|
"allowed": false,
|
||||||
|
"limit_reached": true,
|
||||||
|
"reset_at": now + 3600,
|
||||||
|
"windows": [{
|
||||||
|
"code": "weekly",
|
||||||
|
"scope": "account",
|
||||||
|
"used_ratio": 1.0,
|
||||||
|
"remaining_ratio": 0.0,
|
||||||
|
"is_exhausted": true,
|
||||||
|
"reset_at": now + 3600
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
for model in [None, Some("gpt-5.4"), Some("o3")] {
|
||||||
|
let signals = service.member_signals("codex", &key, None, model);
|
||||||
|
assert_eq!(signals.quota_exhausted, used_percent >= 100.0);
|
||||||
|
assert!(!signals.quota_hard_blocked);
|
||||||
|
assert_eq!(
|
||||||
|
provider_pool_key_minimum_quota_reached(&key, "codex", model),
|
||||||
|
used_percent >= 99.0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn codex_account_and_model_quota_agree_on_explicit_allow_and_deny_flags() {
|
||||||
|
let service = ProviderPoolService::with_builtin_adapters();
|
||||||
|
for (flags, exhausted) in [
|
||||||
|
(json!({ "allowed": true }), false),
|
||||||
|
(json!({ "limit_reached": false }), false),
|
||||||
|
(json!({ "allowed": true, "limit_reached": true }), true),
|
||||||
|
(json!({ "allowed": false, "limit_reached": false }), true),
|
||||||
|
] {
|
||||||
|
for use_windows in [false, true] {
|
||||||
|
let mut metadata = flags.clone();
|
||||||
|
metadata["updated_at"] = json!(300);
|
||||||
|
if use_windows {
|
||||||
|
metadata["windows"] = json!([{ "code": "weekly", "used_ratio": 1.0 }]);
|
||||||
|
} else {
|
||||||
|
metadata["primary_used_percent"] = json!(100.0);
|
||||||
|
}
|
||||||
|
let key = sample_key(Some(json!({ "codex": metadata })));
|
||||||
|
for model in [None, Some("gpt-5.4"), Some("o3")] {
|
||||||
|
assert_eq!(
|
||||||
|
service
|
||||||
|
.member_signals("codex", &key, None, model)
|
||||||
|
.quota_exhausted,
|
||||||
|
exhausted,
|
||||||
|
"flags={flags}, use_windows={use_windows}, model={model:?}"
|
||||||
|
);
|
||||||
|
// The opt-in reserve still protects a numerically full
|
||||||
|
// window even when the upstream reports it as allowed.
|
||||||
|
assert!(provider_pool_key_minimum_quota_reached(
|
||||||
|
&key, "codex", model
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn codex_model_quota_honors_latest_account_refusal_without_blocking_spark() {
|
||||||
|
let service = ProviderPoolService::with_builtin_adapters();
|
||||||
|
for include_window in [false, true] {
|
||||||
|
let mut metadata = json!({
|
||||||
|
"updated_at": 300,
|
||||||
|
"allowed": false,
|
||||||
|
"limit_reached": true,
|
||||||
|
"spark_primary_used_percent": 17.0
|
||||||
|
});
|
||||||
|
if include_window {
|
||||||
|
metadata["primary_used_percent"] = json!(83.0);
|
||||||
|
}
|
||||||
|
let mut key = sample_key(Some(json!({ "codex": metadata })));
|
||||||
|
for include_snapshot in [false, true] {
|
||||||
|
if include_snapshot {
|
||||||
|
key.status_snapshot = Some(json!({
|
||||||
|
"quota": {
|
||||||
|
"provider_type": "codex",
|
||||||
|
"observed_at": 200,
|
||||||
|
"exhausted": false,
|
||||||
|
"windows": [{ "code": "weekly", "used_ratio": 0.83 }]
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
service
|
||||||
|
.member_signals("codex", &key, None, Some("gpt-5.4"))
|
||||||
|
.quota_exhausted
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!service
|
||||||
|
.member_signals("codex", &key, None, Some("gpt-5.3-codex-spark"))
|
||||||
|
.quota_exhausted
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn codex_quota_source_freshness_supports_all_timestamp_formats() {
|
||||||
|
let service = ProviderPoolService::with_builtin_adapters();
|
||||||
|
for observed_at in [
|
||||||
|
json!(1_700_000_200_u64),
|
||||||
|
json!(1_700_000_200_000_u64),
|
||||||
|
json!("1700000200000"),
|
||||||
|
json!("2023-11-14T22:16:40Z"),
|
||||||
|
] {
|
||||||
|
for use_windows in [false, true] {
|
||||||
|
let metadata = if use_windows {
|
||||||
|
json!({
|
||||||
|
"updated_at": observed_at,
|
||||||
|
"windows": [{ "code": "weekly", "used_ratio": 0.83 }]
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
json!({ "updated_at": observed_at, "primary_used_percent": 83.0 })
|
||||||
|
};
|
||||||
|
let mut key = sample_key(Some(json!({ "codex": metadata })));
|
||||||
|
key.status_snapshot = Some(json!({
|
||||||
|
"quota": {
|
||||||
|
"provider_type": "codex",
|
||||||
|
"observed_at": "2023-11-14T22:15:00Z",
|
||||||
|
"exhausted": true,
|
||||||
|
"allowed": false,
|
||||||
|
"windows": [{ "code": "weekly", "used_ratio": 1.0 }]
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
for model in [None, Some("gpt-5.4")] {
|
||||||
|
let signals = service.member_signals("codex", &key, None, model);
|
||||||
|
assert!(!signals.quota_exhausted, "timestamp={observed_at}");
|
||||||
|
assert!(!signals.quota_hard_blocked, "timestamp={observed_at}");
|
||||||
|
assert!(!provider_pool_key_minimum_quota_reached(
|
||||||
|
&key, "codex", model
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Swap freshness while retaining the same observations. An
|
||||||
|
// older usable bucket cannot erase a later exhausted snapshot.
|
||||||
|
key.status_snapshot.as_mut().unwrap()["quota"]["observed_at"] =
|
||||||
|
json!("2023-11-14T22:18:20Z");
|
||||||
|
assert!(provider_pool_key_account_quota_exhausted(&key, "codex"));
|
||||||
|
assert!(provider_pool_key_quota_hard_blocked(&key, "codex"));
|
||||||
|
assert_eq!(
|
||||||
|
provider_pool_key_model_quota_exhausted(&key, "codex", "gpt-5.4"),
|
||||||
|
Some(true)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn codex_account_quota_recovery_requires_new_account_observation() {
|
||||||
|
for metadata in [
|
||||||
|
json!({ "updated_at": 100, "primary_used_percent": 83.0 }),
|
||||||
|
json!({ "primary_used_percent": 83.0 }),
|
||||||
|
json!({ "updated_at": 300, "plan_type": "plus" }),
|
||||||
|
json!({ "updated_at": 300, "credits_unlimited": false }),
|
||||||
|
json!({ "updated_at": 300, "spark_primary_used_percent": 83.0 }),
|
||||||
|
json!({ "updated_at": 300, "windows": [{ "code": "weekly" }] }),
|
||||||
|
] {
|
||||||
|
let mut key = sample_key(Some(json!({ "codex": metadata })));
|
||||||
|
key.status_snapshot = Some(json!({
|
||||||
|
"quota": {
|
||||||
|
"provider_type": "codex",
|
||||||
|
"observed_at": 200,
|
||||||
|
"exhausted": true,
|
||||||
|
"allowed": false,
|
||||||
|
"limit_reached": true,
|
||||||
|
"windows": [{ "code": "weekly", "used_ratio": 1.0 }]
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
assert!(provider_pool_key_account_quota_exhausted(&key, "codex"));
|
||||||
|
assert!(provider_pool_key_quota_hard_blocked(&key, "codex"));
|
||||||
|
assert_eq!(
|
||||||
|
provider_pool_key_model_quota_exhausted(&key, "codex", "gpt-5.4"),
|
||||||
|
Some(true)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut key = sample_key(Some(json!({
|
||||||
|
"codex": {
|
||||||
|
"updated_at": 300,
|
||||||
|
"primary_used_percent": 83.0,
|
||||||
|
"allowed": false,
|
||||||
|
"limit_reached": true
|
||||||
|
}
|
||||||
|
})));
|
||||||
|
key.status_snapshot = Some(json!({
|
||||||
|
"quota": { "provider_type": "codex", "updated_at": 200, "exhausted": false }
|
||||||
|
}));
|
||||||
|
assert!(provider_pool_key_account_quota_exhausted(&key, "codex"));
|
||||||
|
assert!(provider_pool_key_quota_hard_blocked(&key, "codex"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn codex_account_updates_do_not_override_independent_model_quota() {
|
||||||
|
for (spark_ratio, account_percent) in [(0.83, 100.0), (1.0, 83.0)] {
|
||||||
|
let mut key = sample_key(Some(json!({
|
||||||
|
"codex": { "updated_at": 300, "primary_used_percent": account_percent }
|
||||||
|
})));
|
||||||
|
key.status_snapshot = Some(json!({
|
||||||
|
"quota": {
|
||||||
|
"provider_type": "codex",
|
||||||
|
"updated_at": 200,
|
||||||
|
"windows": [{ "code": "spark_5h", "used_ratio": spark_ratio }]
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
assert_eq!(
|
||||||
|
provider_pool_key_model_quota_exhausted(&key, "codex", "gpt-5.3-codex-spark"),
|
||||||
|
Some(spark_ratio >= 1.0)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn codex_explicit_quota_block_is_hard_until_reset() {
|
fn codex_explicit_quota_block_is_hard_until_reset() {
|
||||||
let now = std::time::SystemTime::now()
|
let now = std::time::SystemTime::now()
|
||||||
|
|||||||
@@ -10,10 +10,11 @@ use crate::provider::{
|
|||||||
ProviderPoolMemberInput,
|
ProviderPoolMemberInput,
|
||||||
};
|
};
|
||||||
use crate::quota::{
|
use crate::quota::{
|
||||||
provider_pool_current_unix_secs, provider_pool_json_bool, provider_pool_json_f64,
|
provider_pool_codex_metadata_has_account_quota, provider_pool_current_unix_secs,
|
||||||
provider_pool_member_quota_snapshot, provider_pool_metadata_bucket,
|
provider_pool_json_bool, provider_pool_json_f64, provider_pool_member_quota_snapshot,
|
||||||
provider_pool_model_quota_exhausted, provider_pool_quota_snapshot_exhausted_decision,
|
provider_pool_metadata_bucket, provider_pool_model_quota_exhausted,
|
||||||
provider_pool_reset_deadline_elapsed, provider_pool_timestamp_unix_secs,
|
provider_pool_quota_snapshot_exhausted_decision, provider_pool_reset_deadline_elapsed,
|
||||||
|
provider_pool_source_account_quota_exhausted, provider_pool_timestamp_unix_secs,
|
||||||
};
|
};
|
||||||
use crate::quota_refresh::ProviderPoolQuotaRequestSpec;
|
use crate::quota_refresh::ProviderPoolQuotaRequestSpec;
|
||||||
|
|
||||||
@@ -54,6 +55,9 @@ impl ProviderPoolAdapter for CodexProviderPoolAdapter {
|
|||||||
}) {
|
}) {
|
||||||
return exhausted;
|
return exhausted;
|
||||||
}
|
}
|
||||||
|
if let Some(bucket) = codex_newer_account_quota_metadata(input.key, input.provider_type) {
|
||||||
|
return quota_exhausted_from_bucket(bucket);
|
||||||
|
}
|
||||||
if let Some(quota_snapshot) =
|
if let Some(quota_snapshot) =
|
||||||
provider_pool_member_quota_snapshot(input.key, input.provider_type)
|
provider_pool_member_quota_snapshot(input.key, input.provider_type)
|
||||||
{
|
{
|
||||||
@@ -108,15 +112,12 @@ fn codex_explicit_quota_block_active(
|
|||||||
key: &aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey,
|
key: &aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey,
|
||||||
provider_type: &str,
|
provider_type: &str,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
|
if let Some(bucket) = codex_newer_account_quota_metadata(key, provider_type) {
|
||||||
|
return codex_explicit_quota_block_from_bucket(bucket);
|
||||||
|
}
|
||||||
let Some(quota_snapshot) = provider_pool_member_quota_snapshot(key, provider_type) else {
|
let Some(quota_snapshot) = provider_pool_member_quota_snapshot(key, provider_type) else {
|
||||||
return provider_pool_metadata_bucket(key.upstream_metadata.as_ref(), provider_type)
|
return provider_pool_metadata_bucket(key.upstream_metadata.as_ref(), provider_type)
|
||||||
.is_some_and(|bucket| {
|
.is_some_and(codex_explicit_quota_block_from_bucket);
|
||||||
(provider_pool_json_bool(bucket.get("allowed")) == Some(false)
|
|
||||||
|| provider_pool_json_bool(bucket.get("limit_reached")) == Some(true))
|
|
||||||
&& !["primary", "secondary"]
|
|
||||||
.into_iter()
|
|
||||||
.any(|prefix| codex_window_reset_elapsed(bucket, prefix))
|
|
||||||
});
|
|
||||||
};
|
};
|
||||||
let explicitly_blocked = provider_pool_json_bool(quota_snapshot.get("allowed")) == Some(false)
|
let explicitly_blocked = provider_pool_json_bool(quota_snapshot.get("allowed")) == Some(false)
|
||||||
|| provider_pool_json_bool(quota_snapshot.get("limit_reached")) == Some(true);
|
|| provider_pool_json_bool(quota_snapshot.get("limit_reached")) == Some(true);
|
||||||
@@ -133,6 +134,35 @@ fn codex_explicit_quota_block_active(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn codex_explicit_quota_block_from_bucket(bucket: &Map<String, Value>) -> bool {
|
||||||
|
(provider_pool_json_bool(bucket.get("allowed")) == Some(false)
|
||||||
|
|| provider_pool_json_bool(bucket.get("limit_reached")) == Some(true))
|
||||||
|
&& !["primary", "secondary"]
|
||||||
|
.into_iter()
|
||||||
|
.any(|prefix| codex_window_reset_elapsed(bucket, prefix))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A successful refresh can update raw metadata before the status snapshot.
|
||||||
|
/// Do not keep an old account-level block once a newer quota observation exists.
|
||||||
|
/// Identity-only or model-only updates cannot clear an account quota decision.
|
||||||
|
fn codex_newer_account_quota_metadata<'a>(
|
||||||
|
key: &'a aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey,
|
||||||
|
provider_type: &str,
|
||||||
|
) -> Option<&'a Map<String, Value>> {
|
||||||
|
let snapshot = provider_pool_member_quota_snapshot(key, provider_type)?;
|
||||||
|
let metadata = provider_pool_metadata_bucket(key.upstream_metadata.as_ref(), provider_type)?;
|
||||||
|
if !provider_pool_codex_metadata_has_account_quota(metadata) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let metadata_observed_at = provider_pool_timestamp_unix_secs(metadata.get("observed_at"))
|
||||||
|
.or_else(|| provider_pool_timestamp_unix_secs(metadata.get("updated_at")))?;
|
||||||
|
let snapshot_observed_at = provider_pool_timestamp_unix_secs(snapshot.get("observed_at"))
|
||||||
|
.or_else(|| provider_pool_timestamp_unix_secs(snapshot.get("updated_at")));
|
||||||
|
snapshot_observed_at
|
||||||
|
.is_none_or(|observed_at| metadata_observed_at >= observed_at)
|
||||||
|
.then_some(metadata)
|
||||||
|
}
|
||||||
|
|
||||||
fn build_codex_wham_headers(
|
fn build_codex_wham_headers(
|
||||||
resolved_oauth_auth: Option<(String, String)>,
|
resolved_oauth_auth: Option<(String, String)>,
|
||||||
decrypted_api_key: Option<&str>,
|
decrypted_api_key: Option<&str>,
|
||||||
@@ -322,11 +352,14 @@ pub(crate) fn quota_exhausted_from_bucket(bucket: &Map<String, Value>) -> bool {
|
|||||||
if provider_pool_json_bool(bucket.get("credits_unlimited")) == Some(true) {
|
if provider_pool_json_bool(bucket.get("credits_unlimited")) == Some(true) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
let account_windows_exhausted = provider_pool_source_account_quota_exhausted(bucket);
|
||||||
let has_window_data = provider_pool_json_f64(bucket.get("primary_used_percent")).is_some()
|
let has_window_data = provider_pool_json_f64(bucket.get("primary_used_percent")).is_some()
|
||||||
|| provider_pool_json_f64(bucket.get("secondary_used_percent")).is_some();
|
|| provider_pool_json_f64(bucket.get("secondary_used_percent")).is_some()
|
||||||
|
|| account_windows_exhausted.is_some();
|
||||||
if !has_window_data && provider_pool_json_bool(bucket.get("has_credits")) == Some(false) {
|
if !has_window_data && provider_pool_json_bool(bucket.get("has_credits")) == Some(false) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
codex_window_used_percent_exhausted(bucket, "primary")
|
account_windows_exhausted == Some(true)
|
||||||
|
|| codex_window_used_percent_exhausted(bucket, "primary")
|
||||||
|| codex_window_used_percent_exhausted(bucket, "secondary")
|
|| codex_window_used_percent_exhausted(bucket, "secondary")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -87,6 +87,7 @@ fn provider_pool_quota_reaches_reserve(
|
|||||||
provider_model_name: Option<&str>,
|
provider_model_name: Option<&str>,
|
||||||
reserve_ratio: f64,
|
reserve_ratio: f64,
|
||||||
) -> Option<bool> {
|
) -> Option<bool> {
|
||||||
|
let is_codex = provider_type.trim().eq_ignore_ascii_case("codex");
|
||||||
let requested = provider_model_name.map(provider_pool_identifier_tokens);
|
let requested = provider_model_name.map(provider_pool_identifier_tokens);
|
||||||
if reserve_ratio <= 0.0 && requested.as_ref().is_some_and(|tokens| tokens.is_empty()) {
|
if reserve_ratio <= 0.0 && requested.as_ref().is_some_and(|tokens| tokens.is_empty()) {
|
||||||
return None;
|
return None;
|
||||||
@@ -103,12 +104,19 @@ fn provider_pool_quota_reaches_reserve(
|
|||||||
let mut resolved = None::<(Option<u64>, bool)>;
|
let mut resolved = None::<(Option<u64>, bool)>;
|
||||||
let mut resolved_specific_bucket = false;
|
let mut resolved_specific_bucket = false;
|
||||||
for source in sources.into_iter().flatten() {
|
for source in sources.into_iter().flatten() {
|
||||||
|
let observed_at = provider_pool_timestamp_unix_secs(source.get("observed_at"))
|
||||||
|
.or_else(|| provider_pool_timestamp_unix_secs(source.get("updated_at")));
|
||||||
|
let account_signal = (is_codex && reserve_ratio <= 0.0)
|
||||||
|
.then(|| provider_pool_codex_account_quota_signal(source, observed_at))
|
||||||
|
.flatten();
|
||||||
let mut windows = provider_pool_collect_quota_windows(source);
|
let mut windows = provider_pool_collect_quota_windows(source);
|
||||||
if reserve_ratio > 0.0 && provider_type.trim().eq_ignore_ascii_case("codex") {
|
if is_codex {
|
||||||
// Raw refresh metadata can be newer than the materialized windows.
|
// Raw refresh metadata can be newer than the materialized windows.
|
||||||
// Include all four legacy slots before selecting the model bucket.
|
// Include all four legacy slots before selecting the model bucket.
|
||||||
for prefix in ["primary", "secondary", "spark_primary", "spark_secondary"] {
|
for prefix in ["primary", "secondary", "spark_primary", "spark_secondary"] {
|
||||||
let Some(used_percent) = source.get(&format!("{prefix}_used_percent")) else {
|
let Some(used_percent) =
|
||||||
|
provider_pool_json_f64(source.get(&format!("{prefix}_used_percent")))
|
||||||
|
else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
if provider_pool_json_f64(source.get(&format!("{prefix}_window_minutes")))
|
if provider_pool_json_f64(source.get(&format!("{prefix}_window_minutes")))
|
||||||
@@ -118,7 +126,7 @@ fn provider_pool_quota_reaches_reserve(
|
|||||||
}
|
}
|
||||||
let mut window = Map::from_iter([
|
let mut window = Map::from_iter([
|
||||||
("code".to_string(), json!(prefix)),
|
("code".to_string(), json!(prefix)),
|
||||||
("used_percent".to_string(), used_percent.clone()),
|
("used_percent".to_string(), json!(used_percent)),
|
||||||
]);
|
]);
|
||||||
for field in [
|
for field in [
|
||||||
"reset_at",
|
"reset_at",
|
||||||
@@ -132,12 +140,21 @@ fn provider_pool_quota_reaches_reserve(
|
|||||||
}
|
}
|
||||||
windows.push(window);
|
windows.push(window);
|
||||||
}
|
}
|
||||||
|
windows.retain(provider_pool_quota_window_has_observation);
|
||||||
|
if let Some(exhausted) = account_signal {
|
||||||
|
if !windows.iter().any(provider_pool_window_is_generic) {
|
||||||
|
// A flags-only quota refresh is still a newer account
|
||||||
|
// observation, but must not override a model-specific bucket.
|
||||||
|
windows.push(Map::from_iter([
|
||||||
|
("code".to_string(), json!("account")),
|
||||||
|
("is_exhausted".to_string(), json!(exhausted)),
|
||||||
|
]));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if windows.is_empty() {
|
if windows.is_empty() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let observed_at = provider_pool_timestamp_unix_secs(source.get("observed_at"))
|
|
||||||
.or_else(|| provider_pool_timestamp_unix_secs(source.get("updated_at")));
|
|
||||||
let model_matches = windows
|
let model_matches = windows
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|window| {
|
.filter(|window| {
|
||||||
@@ -155,7 +172,7 @@ fn provider_pool_quota_reaches_reserve(
|
|||||||
provider_pool_explicit_model_windows_exhausted(model_matches, observed_at)
|
provider_pool_explicit_model_windows_exhausted(model_matches, observed_at)
|
||||||
};
|
};
|
||||||
if resolved.is_none()
|
if resolved.is_none()
|
||||||
|| (reserve_ratio > 0.0 && !resolved_specific_bucket)
|
|| (is_codex && !resolved_specific_bucket)
|
||||||
|| provider_pool_should_replace_model_quota_resolution(
|
|| provider_pool_should_replace_model_quota_resolution(
|
||||||
resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
|
resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
|
||||||
observed_at,
|
observed_at,
|
||||||
@@ -183,7 +200,7 @@ fn provider_pool_quota_reaches_reserve(
|
|||||||
let exhausted =
|
let exhausted =
|
||||||
provider_pool_any_window_exhausted(family_matches, observed_at, reserve_ratio);
|
provider_pool_any_window_exhausted(family_matches, observed_at, reserve_ratio);
|
||||||
if resolved.is_none()
|
if resolved.is_none()
|
||||||
|| (reserve_ratio > 0.0 && !resolved_specific_bucket)
|
|| (is_codex && !resolved_specific_bucket)
|
||||||
|| provider_pool_should_replace_model_quota_resolution(
|
|| provider_pool_should_replace_model_quota_resolution(
|
||||||
resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
|
resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
|
||||||
observed_at,
|
observed_at,
|
||||||
@@ -197,7 +214,7 @@ fn provider_pool_quota_reaches_reserve(
|
|||||||
|
|
||||||
// A newer account observation does not update an independent model
|
// A newer account observation does not update an independent model
|
||||||
// bucket. Only compare freshness between applicable model sources.
|
// bucket. Only compare freshness between applicable model sources.
|
||||||
if reserve_ratio > 0.0 && resolved_specific_bucket {
|
if is_codex && resolved_specific_bucket {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -211,8 +228,9 @@ fn provider_pool_quota_reaches_reserve(
|
|||||||
.filter(|window| provider_pool_window_is_generic(window))
|
.filter(|window| provider_pool_window_is_generic(window))
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
if !generic_matches.is_empty() {
|
if !generic_matches.is_empty() {
|
||||||
let exhausted =
|
let exhausted = account_signal.unwrap_or_else(|| {
|
||||||
provider_pool_any_window_exhausted(generic_matches, observed_at, reserve_ratio);
|
provider_pool_any_window_exhausted(generic_matches, observed_at, reserve_ratio)
|
||||||
|
});
|
||||||
if resolved.is_none()
|
if resolved.is_none()
|
||||||
|| provider_pool_should_replace_model_quota_resolution(
|
|| provider_pool_should_replace_model_quota_resolution(
|
||||||
resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
|
resolved.as_ref().and_then(|(observed_at, _)| *observed_at),
|
||||||
@@ -227,6 +245,37 @@ fn provider_pool_quota_reaches_reserve(
|
|||||||
resolved.map(|(_, exhausted)| exhausted)
|
resolved.map(|(_, exhausted)| exhausted)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn provider_pool_codex_account_quota_signal(
|
||||||
|
source: &Map<String, Value>,
|
||||||
|
observed_at: Option<u64>,
|
||||||
|
) -> Option<bool> {
|
||||||
|
let allowed = provider_pool_json_bool(source.get("allowed"));
|
||||||
|
let limit_reached = provider_pool_json_bool(source.get("limit_reached"));
|
||||||
|
if allowed == Some(false) || limit_reached == Some(true) {
|
||||||
|
let reset_elapsed = provider_pool_current_unix_secs().is_some_and(|now| {
|
||||||
|
provider_pool_reset_deadline_elapsed(source, observed_at, now)
|
||||||
|
|| ["primary", "secondary"].into_iter().any(|prefix| {
|
||||||
|
let window = [
|
||||||
|
"reset_at",
|
||||||
|
"next_reset_at",
|
||||||
|
"reset_seconds",
|
||||||
|
"reset_after_seconds",
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|field| {
|
||||||
|
source
|
||||||
|
.get(&format!("{prefix}_{field}"))
|
||||||
|
.map(|value| (field.to_string(), value.clone()))
|
||||||
|
})
|
||||||
|
.collect::<Map<_, _>>();
|
||||||
|
provider_pool_reset_deadline_elapsed(&window, observed_at, now)
|
||||||
|
})
|
||||||
|
});
|
||||||
|
return (!reset_elapsed).then_some(true);
|
||||||
|
}
|
||||||
|
(allowed == Some(true) || limit_reached == Some(false)).then_some(false)
|
||||||
|
}
|
||||||
|
|
||||||
fn provider_pool_should_replace_model_quota_resolution(
|
fn provider_pool_should_replace_model_quota_resolution(
|
||||||
previous_observed_at: Option<u64>,
|
previous_observed_at: Option<u64>,
|
||||||
next_observed_at: Option<u64>,
|
next_observed_at: Option<u64>,
|
||||||
@@ -282,6 +331,69 @@ fn provider_pool_any_window_exhausted(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn provider_pool_quota_window_has_observation(window: &Map<String, Value>) -> bool {
|
||||||
|
["is_exhausted", "exhausted"]
|
||||||
|
.into_iter()
|
||||||
|
.any(|field| provider_pool_json_bool(window.get(field)).is_some())
|
||||||
|
|| [
|
||||||
|
"used_ratio",
|
||||||
|
"usage_ratio",
|
||||||
|
"used_percent",
|
||||||
|
"remaining_ratio",
|
||||||
|
"remaining_fraction",
|
||||||
|
"remaining_percent",
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.any(|field| provider_pool_json_f64(window.get(field)).is_some())
|
||||||
|
|| (provider_pool_json_f64(
|
||||||
|
window
|
||||||
|
.get("remaining")
|
||||||
|
.or_else(|| window.get("remaining_value")),
|
||||||
|
)
|
||||||
|
.is_some()
|
||||||
|
&& provider_pool_json_f64(
|
||||||
|
window
|
||||||
|
.get("limit")
|
||||||
|
.or_else(|| window.get("limit_value"))
|
||||||
|
.or_else(|| window.get("total")),
|
||||||
|
)
|
||||||
|
.is_some_and(|limit| limit > 0.0))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn provider_pool_source_account_quota_exhausted(
|
||||||
|
source: &Map<String, Value>,
|
||||||
|
) -> Option<bool> {
|
||||||
|
let windows = provider_pool_collect_quota_windows(source);
|
||||||
|
let account_windows = windows
|
||||||
|
.iter()
|
||||||
|
.filter(|window| provider_pool_window_is_generic(window))
|
||||||
|
.filter(|window| provider_pool_quota_window_has_observation(window))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if account_windows.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let observed_at = provider_pool_timestamp_unix_secs(source.get("observed_at"))
|
||||||
|
.or_else(|| provider_pool_timestamp_unix_secs(source.get("updated_at")));
|
||||||
|
Some(provider_pool_any_window_exhausted(
|
||||||
|
account_windows,
|
||||||
|
observed_at,
|
||||||
|
0.0,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether Codex metadata contains an account quota observation rather than an
|
||||||
|
/// identity-only update or an independent model quota bucket.
|
||||||
|
pub fn provider_pool_codex_metadata_has_account_quota(source: &Map<String, Value>) -> bool {
|
||||||
|
["primary_used_percent", "secondary_used_percent"]
|
||||||
|
.into_iter()
|
||||||
|
.any(|field| provider_pool_json_f64(source.get(field)).is_some())
|
||||||
|
|| provider_pool_source_account_quota_exhausted(source).is_some()
|
||||||
|
|| ["allowed", "limit_reached", "has_credits"]
|
||||||
|
.into_iter()
|
||||||
|
.any(|field| provider_pool_json_bool(source.get(field)).is_some())
|
||||||
|
|| provider_pool_json_bool(source.get("credits_unlimited")) == Some(true)
|
||||||
|
}
|
||||||
|
|
||||||
/// Whether an applicable Codex window has at most 1% remaining. Missing quota
|
/// Whether an applicable Codex window has at most 1% remaining. Missing quota
|
||||||
/// data and windows whose reset has elapsed do not trigger this opt-in guard.
|
/// data and windows whose reset has elapsed do not trigger this opt-in guard.
|
||||||
pub fn provider_pool_key_minimum_quota_reached(
|
pub fn provider_pool_key_minimum_quota_reached(
|
||||||
@@ -700,7 +812,16 @@ pub(crate) fn provider_pool_json_f64(value: Option<&Value>) -> Option<f64> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn provider_pool_timestamp_unix_secs(value: Option<&Value>) -> Option<u64> {
|
pub(crate) fn provider_pool_timestamp_unix_secs(value: Option<&Value>) -> Option<u64> {
|
||||||
let mut timestamp = provider_pool_json_f64(value)?;
|
let mut timestamp = match provider_pool_json_f64(value) {
|
||||||
|
Some(timestamp) => timestamp,
|
||||||
|
None => {
|
||||||
|
return chrono::DateTime::parse_from_rfc3339(value?.as_str()?.trim())
|
||||||
|
.ok()?
|
||||||
|
.timestamp()
|
||||||
|
.try_into()
|
||||||
|
.ok();
|
||||||
|
}
|
||||||
|
};
|
||||||
if timestamp <= 0.0 {
|
if timestamp <= 0.0 {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10055,6 +10055,24 @@ mod tests {
|
|||||||
2,
|
2,
|
||||||
"the later direct caller should receive its own bounded write attempt"
|
"the later direct caller should receive its own bounded write attempt"
|
||||||
);
|
);
|
||||||
|
// Direct persistence can finish before the submission worker joins the
|
||||||
|
// barrier handoff and accounts for its completed slot.
|
||||||
|
timeout(Duration::from_secs(1), async {
|
||||||
|
loop {
|
||||||
|
let snapshot = runtime.metrics_snapshot();
|
||||||
|
let submission = &runtime.lifecycle_submission.state;
|
||||||
|
if snapshot.terminal_submission_pending == 0
|
||||||
|
&& snapshot.ordered_lifecycle_pending == 0
|
||||||
|
&& snapshot.lifecycle_submission_pending == 0
|
||||||
|
&& submission.admission.available_permits() == submission.capacity
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
sleep(Duration::from_millis(1)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("failed terminal submission accounting and admission should drain");
|
||||||
let snapshot = runtime.metrics_snapshot();
|
let snapshot = runtime.metrics_snapshot();
|
||||||
assert_eq!(snapshot.terminal_submission_pending, 0);
|
assert_eq!(snapshot.terminal_submission_pending, 0);
|
||||||
assert_eq!(snapshot.ordered_lifecycle_pending, 0);
|
assert_eq!(snapshot.ordered_lifecycle_pending, 0);
|
||||||
@@ -10162,6 +10180,7 @@ mod tests {
|
|||||||
.await;
|
.await;
|
||||||
|
|
||||||
assert_eq!(remaining_policy_panics.load(Ordering::Acquire), 0);
|
assert_eq!(remaining_policy_panics.load(Ordering::Acquire), 0);
|
||||||
|
{
|
||||||
let records = records.lock().expect("records lock");
|
let records = records.lock().expect("records lock");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
records
|
records
|
||||||
@@ -10179,7 +10198,25 @@ mod tests {
|
|||||||
== 1,
|
== 1,
|
||||||
"only the later healthy attempt should persist for the panicked request"
|
"only the later healthy attempt should persist for the panicked request"
|
||||||
);
|
);
|
||||||
drop(records);
|
}
|
||||||
|
// The final direct attempt also submits a barrier whose worker may
|
||||||
|
// account for completion after the persistence call has returned.
|
||||||
|
timeout(Duration::from_secs(1), async {
|
||||||
|
loop {
|
||||||
|
let snapshot = runtime.metrics_snapshot();
|
||||||
|
let submission = &runtime.lifecycle_submission.state;
|
||||||
|
if snapshot.terminal_submission_pending == 0
|
||||||
|
&& snapshot.ordered_lifecycle_pending == 0
|
||||||
|
&& snapshot.lifecycle_submission_pending == 0
|
||||||
|
&& submission.admission.available_permits() == submission.capacity
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
sleep(Duration::from_millis(1)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("panicked terminal submission accounting and admission should drain");
|
||||||
let snapshot = runtime.metrics_snapshot();
|
let snapshot = runtime.metrics_snapshot();
|
||||||
assert_eq!(snapshot.terminal_submission_pending, 0);
|
assert_eq!(snapshot.terminal_submission_pending, 0);
|
||||||
assert_eq!(snapshot.ordered_lifecycle_pending, 0);
|
assert_eq!(snapshot.ordered_lifecycle_pending, 0);
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||||
|
|
||||||
|
const { getMock, patchMock } = vi.hoisted(() => ({ getMock: vi.fn(), patchMock: vi.fn() }))
|
||||||
|
|
||||||
|
vi.mock('@/api/client', () => ({ default: { get: getMock, patch: patchMock } }))
|
||||||
|
|
||||||
|
import { getPoolOverview, listPoolKeys, listPoolScores } from '@/api/endpoints/pool'
|
||||||
|
import { getProvider, updateProvider } from '@/api/endpoints/providers'
|
||||||
|
import { cache } from '@/utils/cache'
|
||||||
|
|
||||||
|
const options = { cacheTtlMs: 30_000 }
|
||||||
|
const provider = { id: 'codex', pool_advanced: { reserve_minimum_quota: false } }
|
||||||
|
|
||||||
|
function deferred<T>() {
|
||||||
|
let resolve!: (value: T) => void
|
||||||
|
const promise = new Promise<T>((resolvePromise) => { resolve = resolvePromise })
|
||||||
|
return { promise, resolve }
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
cache.clear()
|
||||||
|
getMock.mockReset()
|
||||||
|
patchMock.mockReset()
|
||||||
|
patchMock.mockResolvedValue({ data: provider })
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('provider pool settings cache invalidation', () => {
|
||||||
|
it('refreshes every cached key page, score page and overview after changing pool settings', async () => {
|
||||||
|
getMock.mockResolvedValue({ data: { version: 'old' } })
|
||||||
|
await getPoolOverview(options)
|
||||||
|
await listPoolKeys('codex', {}, options)
|
||||||
|
await listPoolKeys('codex', { page: 2, status: 'quota_exhausted' }, options)
|
||||||
|
await listPoolScores('codex', {}, options)
|
||||||
|
await listPoolKeys('codex-other', {}, options)
|
||||||
|
await updateProvider('codex', { pool_advanced: { reserve_minimum_quota: false } })
|
||||||
|
getMock.mockResolvedValue({ data: { version: 'new' } })
|
||||||
|
|
||||||
|
await expect(getPoolOverview(options)).resolves.toEqual({ version: 'new' })
|
||||||
|
await expect(listPoolKeys('codex', {}, options)).resolves.toEqual({ version: 'new' })
|
||||||
|
await expect(listPoolKeys('codex', { page: 2, status: 'quota_exhausted' }, options))
|
||||||
|
.resolves.toEqual({ version: 'new' })
|
||||||
|
await expect(listPoolScores('codex', {}, options)).resolves.toEqual({ version: 'new' })
|
||||||
|
await expect(listPoolKeys('codex-other', {}, options)).resolves.toEqual({ version: 'old' })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('does not reuse or cache a key request started before the settings were saved', async () => {
|
||||||
|
const oldResponse = deferred<{ data: { version: string } }>()
|
||||||
|
const newResponse = deferred<{ data: { version: string } }>()
|
||||||
|
getMock.mockReturnValueOnce(oldResponse.promise).mockReturnValueOnce(newResponse.promise)
|
||||||
|
const oldRequest = listPoolKeys('codex', { page: 2 }, options)
|
||||||
|
await updateProvider('codex', { pool_advanced: { reserve_minimum_quota: false } })
|
||||||
|
const newRequest = listPoolKeys('codex', { page: 2 }, options)
|
||||||
|
expect(getMock).toHaveBeenCalledTimes(2)
|
||||||
|
|
||||||
|
oldResponse.resolve({ data: { version: 'old' } })
|
||||||
|
await oldRequest
|
||||||
|
const deduped = listPoolKeys('codex', { page: 2 }, options)
|
||||||
|
expect(getMock).toHaveBeenCalledTimes(2)
|
||||||
|
newResponse.resolve({ data: { version: 'new' } })
|
||||||
|
await expect(newRequest).resolves.toEqual({ version: 'new' })
|
||||||
|
await expect(deduped).resolves.toEqual({ version: 'new' })
|
||||||
|
await expect(listPoolKeys('codex', { page: 2 }, options)).resolves.toEqual({ version: 'new' })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('does not reuse a provider detail request started before a successful save', async () => {
|
||||||
|
const oldResponse = deferred<{ data: typeof provider }>()
|
||||||
|
getMock.mockReturnValueOnce(oldResponse.promise).mockResolvedValueOnce({ data: provider })
|
||||||
|
const oldRequest = getProvider('codex')
|
||||||
|
await updateProvider('codex', { pool_advanced: { reserve_minimum_quota: false } })
|
||||||
|
const newRequest = getProvider('codex')
|
||||||
|
expect(getMock).toHaveBeenCalledTimes(2)
|
||||||
|
oldResponse.resolve({ data: { ...provider, pool_advanced: { reserve_minimum_quota: true } } })
|
||||||
|
await oldRequest
|
||||||
|
await expect(newRequest).resolves.toMatchObject(provider)
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
import client from '../client'
|
import client from '../client'
|
||||||
import { buildCacheKey, cachedRequest, dedupedRequest } from '@/utils/cache'
|
import { buildCacheKey, cache, cachedRequest, dedupedRequest } from '@/utils/cache'
|
||||||
import type {
|
import type {
|
||||||
ClaudeCodeAdvancedConfig,
|
ClaudeCodeAdvancedConfig,
|
||||||
FailoverRulesConfig,
|
FailoverRulesConfig,
|
||||||
@@ -142,6 +142,15 @@ export async function updateProvider(
|
|||||||
requestOptions?: ProviderRequestOptions,
|
requestOptions?: ProviderRequestOptions,
|
||||||
): Promise<ProviderWithEndpointsSummary> {
|
): Promise<ProviderWithEndpointsSummary> {
|
||||||
const response = await client.patch<ProviderWithEndpointsSummary>(`/api/admin/providers/${providerId}`, data, requestOptions)
|
const response = await client.patch<ProviderWithEndpointsSummary>(`/api/admin/providers/${providerId}`, data, requestOptions)
|
||||||
|
cache.delete(`providers:detail:${providerId}`)
|
||||||
|
if ('pool_advanced' in data) {
|
||||||
|
cache.delete('pool:overview')
|
||||||
|
for (const kind of ['keys', 'scores']) {
|
||||||
|
const prefix = `pool:${kind}:${providerId}`
|
||||||
|
cache.delete(prefix)
|
||||||
|
cache.deleteByPrefix(`${prefix}:`)
|
||||||
|
}
|
||||||
|
}
|
||||||
return normalizeProviderSummary(response.data)
|
return normalizeProviderSummary(response.data)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -61,6 +61,14 @@ class MemoryCache {
|
|||||||
this.keyVersions.set(key, (this.keyVersions.get(key) ?? 0) + 1)
|
this.keyVersions.set(key, (this.keyVersions.get(key) ?? 0) + 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 删除匹配前缀的缓存及进行中请求,旧响应不再回填缓存。 */
|
||||||
|
deleteByPrefix(prefix: string): void {
|
||||||
|
const keys = new Set([...this.cache.keys(), ...this.inFlight.keys()])
|
||||||
|
for (const key of keys) {
|
||||||
|
if (key.startsWith(prefix)) this.delete(key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 清空所有缓存
|
* 清空所有缓存
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -1395,6 +1395,8 @@ async function loadOverview(options: { cacheTtlMs?: number, silent?: boolean } =
|
|||||||
|
|
||||||
async function handleSchedulingSaved(updatedProvider: ProviderWithEndpointsSummary) {
|
async function handleSchedulingSaved(updatedProvider: ProviderWithEndpointsSummary) {
|
||||||
if (!selectedProviderId.value || updatedProvider.id !== selectedProviderId.value) return
|
if (!selectedProviderId.value || updatedProvider.id !== selectedProviderId.value) return
|
||||||
|
// 保存前发出的详情读取不得覆盖这次保存返回的新配置。
|
||||||
|
providerDataRequestId += 1
|
||||||
// 优先回写保存接口返回值,避免弹窗立即重开时读到旧配置。
|
// 优先回写保存接口返回值,避免弹窗立即重开时读到旧配置。
|
||||||
if (selectedProviderData.value) {
|
if (selectedProviderData.value) {
|
||||||
Object.assign(selectedProviderData.value, updatedProvider)
|
Object.assign(selectedProviderData.value, updatedProvider)
|
||||||
@@ -1403,7 +1405,7 @@ async function handleSchedulingSaved(updatedProvider: ProviderWithEndpointsSumma
|
|||||||
}
|
}
|
||||||
showSchedulingDialog.value = false
|
showSchedulingDialog.value = false
|
||||||
showAdvancedDialog.value = false
|
showAdvancedDialog.value = false
|
||||||
await loadOverview({ silent: true })
|
await Promise.all([loadKeys({ silent: true }), loadOverview({ silent: true })])
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- Provider Selection ---
|
// --- Provider Selection ---
|
||||||
|
|||||||
@@ -340,12 +340,24 @@ vi.mock('@/components/ui/refresh-button.vue', async () => {
|
|||||||
})
|
})
|
||||||
|
|
||||||
vi.mock('@/features/pool/components/PoolSchedulingDialog.vue', async () => {
|
vi.mock('@/features/pool/components/PoolSchedulingDialog.vue', async () => {
|
||||||
const { defineComponent } = await import('vue')
|
const { defineComponent, h } = await import('vue')
|
||||||
return {
|
return {
|
||||||
default: defineComponent({
|
default: defineComponent({
|
||||||
name: 'PoolSchedulingDialogStub',
|
name: 'PoolSchedulingDialogStub',
|
||||||
setup() {
|
props: {
|
||||||
return () => null
|
providerId: { type: String, required: true },
|
||||||
|
currentConfig: { type: Object, default: null },
|
||||||
|
},
|
||||||
|
emits: ['saved'],
|
||||||
|
setup(props, { emit }) {
|
||||||
|
return () => h('button', {
|
||||||
|
'data-testid': 'save-pool-scheduling',
|
||||||
|
'data-reserve': String(props.currentConfig?.reserve_minimum_quota ?? false),
|
||||||
|
onClick: () => emit('saved', {
|
||||||
|
id: props.providerId,
|
||||||
|
pool_advanced: { ...props.currentConfig, reserve_minimum_quota: false },
|
||||||
|
}),
|
||||||
|
}, '保存调度测试')
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
@@ -1278,4 +1290,80 @@ describe('PoolManagement Codex cycle stats mode', () => {
|
|||||||
|
|
||||||
expect(disabledRoot.querySelector('[data-testid="pool-demand-metrics-button"]')).toBeNull()
|
expect(disabledRoot.querySelector('[data-testid="pool-demand-metrics-button"]')).toBeNull()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('reloads account scheduling state immediately after disabling minimum quota reserve', async () => {
|
||||||
|
const key = createPoolKey('codex', {
|
||||||
|
scheduling_status: 'blocked',
|
||||||
|
scheduling_reason: 'account_quota_exhausted',
|
||||||
|
scheduling_label: '额度耗尽',
|
||||||
|
})
|
||||||
|
endpointMocks.getPoolOverview.mockResolvedValue({ items: [createOverview('codex')] })
|
||||||
|
endpointMocks.listPoolKeys.mockResolvedValue(createKeyPage(key))
|
||||||
|
endpointMocks.getProvider.mockResolvedValue(createProvider('codex', {
|
||||||
|
pool_advanced: { reserve_minimum_quota: true },
|
||||||
|
}))
|
||||||
|
const root = mountPoolManagement()
|
||||||
|
await settle()
|
||||||
|
expect(root.querySelector('tbody')?.textContent).toContain('额度耗尽')
|
||||||
|
endpointMocks.listPoolKeys.mockClear()
|
||||||
|
endpointMocks.listPoolKeys.mockResolvedValue(createKeyPage({
|
||||||
|
...key,
|
||||||
|
scheduling_status: 'available',
|
||||||
|
scheduling_reason: 'available',
|
||||||
|
scheduling_label: '可用',
|
||||||
|
}))
|
||||||
|
|
||||||
|
root.querySelector<HTMLButtonElement>('[data-testid="save-pool-scheduling"]')?.click()
|
||||||
|
await settle()
|
||||||
|
|
||||||
|
expect(endpointMocks.listPoolKeys).toHaveBeenCalledWith(
|
||||||
|
'codex-provider', expect.anything(), { cacheTtlMs: 0 },
|
||||||
|
)
|
||||||
|
expect(root.querySelector('tbody')?.textContent).not.toContain('额度耗尽')
|
||||||
|
expect(root.querySelector('tbody')?.textContent).toContain('可用')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('shows 17% remaining without marking a schedulable account exhausted when reserve is enabled', async () => {
|
||||||
|
endpointMocks.getPoolOverview.mockResolvedValue({ items: [createOverview('codex')] })
|
||||||
|
endpointMocks.getProvider.mockResolvedValue(createProvider('codex', {
|
||||||
|
pool_advanced: { reserve_minimum_quota: true },
|
||||||
|
}))
|
||||||
|
endpointMocks.listPoolKeys.mockResolvedValue(createKeyPage(createPoolKey('codex', {
|
||||||
|
scheduling_status: 'available',
|
||||||
|
scheduling_reason: 'available',
|
||||||
|
scheduling_label: '可用',
|
||||||
|
status_snapshot: {
|
||||||
|
account: { code: 'ok', blocked: false },
|
||||||
|
quota: {
|
||||||
|
code: 'ok',
|
||||||
|
exhausted: false,
|
||||||
|
provider_type: 'codex',
|
||||||
|
windows: [{ code: '5h', used_ratio: 0.83, remaining_ratio: 0.17 }],
|
||||||
|
},
|
||||||
|
oauth: { code: 'none' },
|
||||||
|
},
|
||||||
|
})))
|
||||||
|
const root = mountPoolManagement()
|
||||||
|
await settle()
|
||||||
|
|
||||||
|
expect(root.querySelector('[data-testid="pool-quota-meter-text"]')?.textContent).toBe('17.0%')
|
||||||
|
expect(root.querySelector('tbody')?.textContent).toContain('可用')
|
||||||
|
expect(root.querySelector('tbody')?.textContent).not.toContain('额度耗尽')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('does not replace saved scheduling settings with an older provider detail response', async () => {
|
||||||
|
let resolveOldProvider!: (provider: ReturnType<typeof createProvider>) => void
|
||||||
|
endpointMocks.getProvider.mockReturnValue(new Promise((resolve) => { resolveOldProvider = resolve }))
|
||||||
|
endpointMocks.getPoolOverview.mockResolvedValue({ items: [createOverview('codex')] })
|
||||||
|
endpointMocks.listPoolKeys.mockResolvedValue(createKeyPage(createPoolKey()))
|
||||||
|
const root = mountPoolManagement()
|
||||||
|
await settle()
|
||||||
|
|
||||||
|
root.querySelector<HTMLButtonElement>('[data-testid="save-pool-scheduling"]')?.click()
|
||||||
|
await settle()
|
||||||
|
resolveOldProvider(createProvider('codex', { pool_advanced: { reserve_minimum_quota: true } }))
|
||||||
|
await settle()
|
||||||
|
|
||||||
|
expect(root.querySelector('[data-testid="save-pool-scheduling"]')?.getAttribute('data-reserve')).toBe('false')
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
Reference in New Issue
Block a user