i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 01:10:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix provider pool quota status handling

Browse Source
This commit is contained in:
fawney19
2026-05-07 11:28:44 +08:00
parent 01c9f9be49
commit fd44906bb7
37 changed files with 1844 additions and 500 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
apps/aether-gateway/src/ai_serving/planner/candidate_materialization.rs
View File

@@ -35,6 +35,8 @@ use crate::handlers::shared::provider_pool::admin_provider_pool_config_from_conf
use crate::orchestration::{local_attempt_slot_count, ExecutionAttemptIdentity};
use crate::{AppState, GatewayError};
const POOL_KEY_RETRY_INDEX_STRIDE: u32 = 100;
#[derive(Debug, Clone)]
pub(crate) struct LocalExecutionCandidateAttempt {
pub(crate) eligible: EligibleLocalExecutionCandidate,
@@ -129,6 +131,16 @@ impl LocalExecutionCandidateAttempt {
}
}
fn effective_retry_index(retry_index: u32, pool_key_index: Option<u32>) -> u32 {
pool_key_index
.and_then(|index| {
index
.checked_mul(POOL_KEY_RETRY_INDEX_STRIDE)
.and_then(|base| base.checked_add(retry_index))
})
.unwrap_or(retry_index)
}
#[derive(Debug, Clone, Copy)]
pub(crate) struct LocalAvailableCandidatePersistenceContext<'a> {
pub(crate) user_id: &'a str,
@@ -326,7 +338,7 @@ where
self.api_key_id,
&candidate.candidate,
candidate_index,
retry_index,
effective_retry_index(retry_index, candidate.orchestration.pool_key_index),
generated_candidate_id,
self.required_capabilities,
extra_data,
@@ -343,6 +355,8 @@ where
retry_index: u32,
candidate_id: String,
) -> Self::Attempt {
let retry_index =
effective_retry_index(retry_index, candidate.orchestration.pool_key_index);
LocalExecutionCandidateAttempt {
eligible: candidate,
candidate_index,
@@ -908,7 +922,7 @@ where
context.api_key_id,
&candidate_ref.candidate,
candidate_index,
retry_index,
effective_retry_index(retry_index, candidate_ref.orchestration.pool_key_index),
generated_candidate_id.as_str(),
context.required_capabilities,
extra_data.clone(),
@@ -927,6 +941,8 @@ where
} else {
candidate_ref.clone()
};
let retry_index =
effective_retry_index(retry_index, candidate.orchestration.pool_key_index);
attempts.push(LocalExecutionCandidateAttempt {
eligible: candidate,
candidate_index,
@@ -957,6 +973,8 @@ fn build_unpersisted_local_execution_candidate_attempts(
.expect("candidate should remain available until final retry")
.clone()
};
let retry_index =
effective_retry_index(retry_index, candidate.orchestration.pool_key_index);
attempts.push_back(LocalExecutionCandidateAttempt {
eligible: candidate,
candidate_index,
@@ -1290,6 +1308,29 @@ mod tests {
assert_eq!(stored[0].candidate_index, 2);
}
#[test]
fn pool_key_attempts_use_distinct_effective_retry_indices() {
let first = build_unpersisted_local_execution_candidate_attempts(
sample_eligible("pool-key-1", Some(0)),
0,
)
.pop_front()
.expect("first pool key attempt");
let second = build_unpersisted_local_execution_candidate_attempts(
sample_eligible("pool-key-2", Some(1)),
0,
)
.pop_front()
.expect("second pool key attempt");
assert_eq!(first.retry_index, 0);
assert_eq!(second.retry_index, 100);
assert_eq!(first.attempt_identity().retry_index, 0);
assert_eq!(second.attempt_identity().retry_index, 100);
assert_eq!(first.attempt_identity().pool_key_index, Some(0));
assert_eq!(second.attempt_identity().pool_key_index, Some(1));
}
#[tokio::test]
async fn available_candidates_persist_ranking_metadata_in_extra_data() {
let repository = Arc::new(InMemoryRequestCandidateRepository::default());

4
apps/aether-gateway/src/executor/outcome.rs
View File

@@ -279,7 +279,7 @@ pub(crate) async fn record_failed_usage_for_exhausted_request(
state
.usage_runtime
.record_terminal_event(
.record_terminal_event_direct(
state.data.as_ref(),
UsageEvent::new(UsageEventType::Failed, request_id, data),
)
@@ -418,7 +418,7 @@ pub(crate) async fn record_failed_usage_for_runtime_miss_request(
state
.usage_runtime
.record_terminal_event(
.record_terminal_event_direct(
state.data.as_ref(),
UsageEvent::new(UsageEventType::Failed, request_id, data),
)

17
apps/aether-gateway/src/handlers/admin/provider/endpoint_keys/mutations/reset_cycle_stats.rs
View File

@@ -111,7 +111,14 @@ fn reset_codex_cycle_usage_windows(status_snapshot: &mut Value, now_unix_secs: u
}
window.insert("usage_reset_at".to_string(), json!(now_unix_secs));
window.remove("usage");
window.insert(
"usage".to_string(),
json!({
"request_count": 0,
"total_tokens": 0,
"total_cost_usd": "0.00000000",
}),
);
reset_count += 1;
}
@@ -173,9 +180,13 @@ mod tests {
assert_eq!(reset_codex_cycle_usage_windows(&mut snapshot, 1_234), 2);
let windows = snapshot["quota"]["windows"].as_array().expect("windows");
assert_eq!(windows[0]["usage_reset_at"], json!(1_234));
assert!(windows[0].get("usage").is_none());
assert_eq!(windows[0]["usage"]["request_count"], json!(0));
assert_eq!(windows[0]["usage"]["total_tokens"], json!(0));
assert_eq!(windows[0]["usage"]["total_cost_usd"], json!("0.00000000"));
assert_eq!(windows[1]["usage_reset_at"], json!(1_234));
assert!(windows[1].get("usage").is_none());
assert_eq!(windows[1]["usage"]["request_count"], json!(0));
assert_eq!(windows[1]["usage"]["total_tokens"], json!(0));
assert_eq!(windows[1]["usage"]["total_cost_usd"], json!("0.00000000"));
assert!(windows[2].get("usage_reset_at").is_none());
assert!(windows[2].get("usage").is_some());
}

6
apps/aether-gateway/src/handlers/admin/provider/oauth/errors.rs
View File

@@ -147,7 +147,7 @@ pub(crate) fn merge_provider_oauth_refresh_failure_reason(
return Some(refresh_reason.to_string());
}
if current_reason.starts_with(OAUTH_EXPIRED_PREFIX) {
return Some(refresh_reason.to_string());
return Some(current_reason.to_string());
}
if oauth_invalid_reason_is_account_level_block(Some(current_reason)) {
return None;
@@ -190,13 +190,13 @@ mod tests {
}
#[test]
fn refresh_failure_replaces_access_token_expired_marker() {
fn refresh_failure_does_not_replace_access_token_expired_marker() {
assert_eq!(
merge_provider_oauth_refresh_failure_reason(
Some("[OAUTH_EXPIRED] access token invalid"),
"[REFRESH_FAILED] Token 续期失败 (401): refresh_token 无效",
),
Some("[REFRESH_FAILED] Token 续期失败 (401): refresh_token 无效".to_string()),
Some("[OAUTH_EXPIRED] access token invalid".to_string()),
);
}
}

248
apps/aether-gateway/src/handlers/admin/provider/pool_admin/payloads.rs
View File

@@ -9,11 +9,8 @@ use aether_admin::provider::quota as admin_provider_quota_pure;
use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
};
use aether_data_contracts::repository::usage::{
ProviderApiKeyWindowUsageRequest, StoredProviderApiKeyWindowUsageSummary,
};
use serde_json::json;
use std::collections::BTreeMap;
use std::time::{SystemTime, UNIX_EPOCH};
fn admin_pool_string_list(value: Option<&serde_json::Value>) -> Option<Vec<String>> {
let values = value
@@ -293,128 +290,6 @@ fn admin_pool_quota_window<'a>(
})
}
pub(super) type AdminPoolCodexWindowUsageByKey =
BTreeMap<(String, String), StoredProviderApiKeyWindowUsageSummary>;
fn admin_pool_provider_type_is_codex(provider_type: &str) -> bool {
provider_type.trim().eq_ignore_ascii_case("codex")
}
fn admin_pool_codex_window_usage_code(
window: &serde_json::Map<String, serde_json::Value>,
) -> Option<&'static str> {
let code = window
.get("code")
.and_then(serde_json::Value::as_str)
.map(str::trim)?;
if code.eq_ignore_ascii_case("5h") {
Some("5h")
} else if code.eq_ignore_ascii_case("weekly") {
Some("weekly")
} else {
None
}
}
fn admin_pool_codex_window_usage_bounds(
window: &serde_json::Map<String, serde_json::Value>,
) -> Option<(u64, u64)> {
let reset_at = admin_pool_json_to_u64(window.get("reset_at"))?;
let window_minutes = admin_pool_json_to_u64(window.get("window_minutes"))?;
let window_seconds = window_minutes.checked_mul(60)?;
let window_start = reset_at.checked_sub(window_seconds)?;
let usage_reset_at = admin_pool_json_to_u64(window.get("usage_reset_at"))
.filter(|reset_at_override| *reset_at_override < reset_at)
.unwrap_or(window_start);
let start = window_start.max(usage_reset_at);
(start < reset_at).then_some((start, reset_at))
}
pub(super) fn build_admin_pool_codex_window_usage_requests(
provider_type: &str,
keys: &[StoredProviderCatalogKey],
) -> Vec<ProviderApiKeyWindowUsageRequest> {
if !admin_pool_provider_type_is_codex(provider_type) {
return Vec::new();
}
let mut requests = Vec::new();
for key in keys {
let status_snapshot = provider_key_status_snapshot_payload(key, provider_type);
let Some(windows) = status_snapshot
.get("quota")
.and_then(serde_json::Value::as_object)
.and_then(|quota| quota.get("windows"))
.and_then(serde_json::Value::as_array)
else {
continue;
};
for window in windows.iter().filter_map(serde_json::Value::as_object) {
let Some(window_code) = admin_pool_codex_window_usage_code(window) else {
continue;
};
let Some((start_unix_secs, end_unix_secs)) =
admin_pool_codex_window_usage_bounds(window)
else {
continue;
};
requests.push(ProviderApiKeyWindowUsageRequest {
provider_api_key_id: key.id.clone(),
window_code: window_code.to_string(),
start_unix_secs,
end_unix_secs,
});
}
}
requests
}
fn admin_pool_codex_window_usage_payload(
usage: &StoredProviderApiKeyWindowUsageSummary,
) -> serde_json::Value {
json!({
"request_count": usage.request_count,
"total_tokens": usage.total_tokens,
"total_cost_usd": format!("{:.8}", usage.total_cost_usd),
})
}
fn admin_pool_attach_codex_window_usage(
status_snapshot: &mut serde_json::Value,
key_id: &str,
usage_by_key: &AdminPoolCodexWindowUsageByKey,
) {
if usage_by_key.is_empty() {
return;
}
let Some(windows) = status_snapshot
.get_mut("quota")
.and_then(serde_json::Value::as_object_mut)
.and_then(|quota| quota.get_mut("windows"))
.and_then(serde_json::Value::as_array_mut)
else {
return;
};
for window in windows
.iter_mut()
.filter_map(serde_json::Value::as_object_mut)
{
let Some(window_code) = admin_pool_codex_window_usage_code(window) else {
continue;
};
let lookup_key = (key_id.to_string(), window_code.to_string());
if let Some(usage) = usage_by_key.get(&lookup_key) {
window.insert(
"usage".to_string(),
admin_pool_codex_window_usage_payload(usage),
);
}
}
}
fn admin_pool_quota_window_used_percent(
window: &serde_json::Map<String, serde_json::Value>,
) -> Option<f64> {
@@ -528,6 +403,54 @@ fn admin_pool_build_codex_account_quota_from_snapshot(
None
}
fn admin_pool_current_unix_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.ok()
.map(|duration| duration.as_secs())
.unwrap_or(0)
}
fn admin_pool_prune_expired_codex_window_usage(status_snapshot: &mut serde_json::Value) {
let now_unix_secs = admin_pool_current_unix_secs();
let Some(windows) = status_snapshot
.get_mut("quota")
.and_then(serde_json::Value::as_object_mut)
.and_then(|quota| quota.get_mut("windows"))
.and_then(serde_json::Value::as_array_mut)
else {
return;
};
for window in windows
.iter_mut()
.filter_map(serde_json::Value::as_object_mut)
{
let code = window
.get("code")
.and_then(serde_json::Value::as_str)
.map(str::trim)
.unwrap_or_default();
if !code.eq_ignore_ascii_case("5h") && !code.eq_ignore_ascii_case("weekly") {
continue;
}
let Some(reset_at) = admin_pool_json_to_u64(window.get("reset_at")) else {
continue;
};
if now_unix_secs < reset_at {
continue;
}
window.insert(
"usage".to_string(),
json!({
"request_count": 0,
"total_tokens": 0,
"total_cost_usd": "0.00000000",
}),
);
}
}
fn admin_pool_quota_windows<'a>(
quota_snapshot: &'a serde_json::Map<String, serde_json::Value>,
) -> Vec<&'a serde_json::Map<String, serde_json::Value>> {
@@ -930,7 +853,6 @@ pub(super) fn build_admin_pool_key_payload(
key: &StoredProviderCatalogKey,
runtime: &AdminProviderPoolRuntimeState,
pool_config: Option<AdminProviderPoolConfig>,
codex_window_usage_by_key: &AdminPoolCodexWindowUsageByKey,
) -> serde_json::Value {
let cooldown_reason = runtime.cooldown_reason_by_key.get(&key.id).cloned();
let cooldown_ttl_seconds = cooldown_reason
@@ -949,12 +871,8 @@ pub(super) fn build_admin_pool_key_payload(
let oauth_plan_type =
admin_pool_derive_oauth_plan_type(key, provider_type, auth_config.as_ref());
let mut status_snapshot = provider_key_status_snapshot_payload(key, provider_type);
if admin_pool_provider_type_is_codex(provider_type) {
admin_pool_attach_codex_window_usage(
&mut status_snapshot,
&key.id,
codex_window_usage_by_key,
);
if provider_type.trim().eq_ignore_ascii_case("codex") {
admin_pool_prune_expired_codex_window_usage(&mut status_snapshot);
}
let account_snapshot = status_snapshot
.get("account")
@@ -1237,34 +1155,54 @@ mod tests {
use serde_json::json;
#[test]
fn codex_window_usage_bounds_honor_manual_usage_reset_at() {
let window = json!({
"code": "5h",
"reset_at": 20_000,
"window_minutes": 300,
"usage_reset_at": 9_000,
fn expired_codex_window_usage_is_zeroed_for_display() {
let mut snapshot = json!({
"quota": {
"windows": [
{
"code": "5h",
"reset_at": 1,
"usage": {
"request_count": 7,
"total_tokens": 700,
"total_cost_usd": "7.00000000"
}
}
]
}
});
let window = window.as_object().expect("window object");
assert_eq!(
admin_pool_codex_window_usage_bounds(window),
Some((9_000, 20_000))
);
admin_pool_prune_expired_codex_window_usage(&mut snapshot);
let usage = &snapshot["quota"]["windows"][0]["usage"];
assert_eq!(usage["request_count"], json!(0));
assert_eq!(usage["total_tokens"], json!(0));
assert_eq!(usage["total_cost_usd"], json!("0.00000000"));
}
#[test]
fn codex_window_usage_bounds_ignore_reset_before_window_start() {
let window = json!({
"code": "weekly",
"reset_at": 700_000,
"window_minutes": 10_080,
"usage_reset_at": 1,
fn active_codex_window_usage_is_preserved_for_display() {
let mut snapshot = json!({
"quota": {
"windows": [
{
"code": "weekly",
"reset_at": 4_102_444_800u64,
"usage": {
"request_count": 3,
"total_tokens": 375,
"total_cost_usd": "0.60000000"
}
}
]
}
});
let window = window.as_object().expect("window object");
assert_eq!(
admin_pool_codex_window_usage_bounds(window),
Some((95_200, 700_000))
);
admin_pool_prune_expired_codex_window_usage(&mut snapshot);
let usage = &snapshot["quota"]["windows"][0]["usage"];
assert_eq!(usage["request_count"], json!(3));
assert_eq!(usage["total_tokens"], json!(375));
assert_eq!(usage["total_cost_usd"], json!("0.60000000"));
}
}

31
apps/aether-gateway/src/handlers/admin/provider/pool_admin/read_routes/keys.rs
View File

@@ -253,36 +253,6 @@ pub(super) async fn build_admin_pool_list_keys_response(
}
_ => AdminProviderPoolRuntimeState::default(),
};
let codex_window_usage_requests =
pool_payloads::build_admin_pool_codex_window_usage_requests(&provider.provider_type, &keys);
let mut codex_window_usage_by_key = codex_window_usage_requests
.iter()
.map(|request| {
(
(
request.provider_api_key_id.clone(),
request.window_code.clone(),
),
aether_data_contracts::repository::usage::StoredProviderApiKeyWindowUsageSummary {
provider_api_key_id: request.provider_api_key_id.clone(),
window_code: request.window_code.clone(),
..Default::default()
},
)
})
.collect::<pool_payloads::AdminPoolCodexWindowUsageByKey>();
if !codex_window_usage_requests.is_empty() {
for usage in state
.app()
.summarize_usage_by_provider_api_key_windows(&codex_window_usage_requests)
.await?
{
codex_window_usage_by_key.insert(
(usage.provider_api_key_id.clone(), usage.window_code.clone()),
usage,
);
}
}
let items = keys
.into_iter()
@@ -294,7 +264,6 @@ pub(super) async fn build_admin_pool_list_keys_response(
&key,
&runtime,
pool_config.clone(),
&codex_window_usage_by_key,
)
})
.collect::<Vec<_>>();

168
apps/aether-gateway/src/handlers/shared/catalog.rs
View File

@@ -240,6 +240,10 @@ fn tagged_oauth_invalid_reason(reason: Option<&str>, prefix: &str) -> Option<Str
})
}
fn oauth_access_token_expired(expires_at_unix_secs: Option<u64>, now_unix_secs: u64) -> bool {
expires_at_unix_secs.is_none_or(|expires_at| expires_at == 0 || expires_at <= now_unix_secs)
}
fn build_provider_key_oauth_status_snapshot(key: &StoredProviderCatalogKey) -> Value {
if !key.auth_type.trim().eq_ignore_ascii_case("oauth") {
return default_oauth_status_snapshot_value();
@@ -271,14 +275,16 @@ fn build_provider_key_oauth_status_snapshot(key: &StoredProviderCatalogKey) -> V
if let Some(reason) =
tagged_oauth_invalid_reason(invalid_reason.as_deref(), OAUTH_REFRESH_FAILED_PREFIX)
{
let access_token_expired = oauth_access_token_expired(expires_at_unix_secs, now_unix_secs);
return json!({
"code": "invalid",
"label": "已失效",
"code": if access_token_expired { "invalid" } else { "reauth_required" },
"label": if access_token_expired { "已失效" } else { "续期失败" },
"reason": reason,
"expires_at": expires_at_unix_secs,
"invalid_at": invalid_at_unix_secs,
"source": "oauth_refresh",
"requires_reauth": true,
"usable_until_expiry": !access_token_expired,
"expiring_soon": false,
});
}
@@ -320,11 +326,11 @@ fn build_provider_key_oauth_status_snapshot(key: &StoredProviderCatalogKey) -> V
return json!({
"code": "expired",
"label": "已过期",
"reason": "Token 已过期，请重新授权",
"reason": "Access Token 已过期，等待自动续期",
"expires_at": expires_at_unix_secs,
"invalid_at": Value::Null,
"source": "expires_at",
"requires_reauth": true,
"requires_reauth": false,
"expiring_soon": false,
});
}
@@ -556,10 +562,7 @@ fn quota_windows_all_exhausted(windows: &[Value]) -> bool {
total > 0 && exhausted == total
}
fn preserve_quota_window_usage_reset_at(
current_status_snapshot: Option<&Value>,
quota: &mut Value,
) {
fn preserve_quota_window_usage_state(current_status_snapshot: Option<&Value>, quota: &mut Value) {
let Some(current_windows) = current_status_snapshot
.and_then(Value::as_object)
.and_then(|snapshot| snapshot.get("quota"))
@@ -582,23 +585,39 @@ fn preserve_quota_window_usage_reset_at(
else {
continue;
};
let Some(usage_reset_at) = current_windows
.iter()
.filter_map(Value::as_object)
.find(|current_window| {
current_window
.get("code")
.and_then(Value::as_str)
.map(str::trim)
.is_some_and(|current_code| current_code.eq_ignore_ascii_case(code))
})
.and_then(|current_window| current_window.get("usage_reset_at"))
.and_then(admin_provider_quota_pure::coerce_json_u64)
else {
let current_window =
current_windows
.iter()
.filter_map(Value::as_object)
.find(|current_window| {
current_window
.get("code")
.and_then(Value::as_str)
.map(str::trim)
.is_some_and(|current_code| current_code.eq_ignore_ascii_case(code))
});
let Some(current_window) = current_window else {
continue;
};
let current_reset_at = current_window
.get("reset_at")
.and_then(admin_provider_quota_pure::coerce_json_u64);
let next_reset_at = next_window
.get("reset_at")
.and_then(admin_provider_quota_pure::coerce_json_u64);
if current_reset_at.is_none() || current_reset_at != next_reset_at {
continue;
}
next_window.insert("usage_reset_at".to_string(), json!(usage_reset_at));
if let Some(usage_reset_at) = current_window
.get("usage_reset_at")
.and_then(admin_provider_quota_pure::coerce_json_u64)
{
next_window.insert("usage_reset_at".to_string(), json!(usage_reset_at));
}
if let Some(usage) = current_window.get("usage") {
next_window.insert("usage".to_string(), usage.clone());
}
}
}
@@ -1162,7 +1181,7 @@ pub(crate) fn sync_provider_key_quota_status_snapshot(
_ => None,
}?;
if normalized_provider_type == "codex" {
preserve_quota_window_usage_reset_at(status_snapshot, &mut quota);
preserve_quota_window_usage_state(status_snapshot, &mut quota);
}
let default_snapshot = default_provider_key_status_snapshot();
@@ -2062,7 +2081,7 @@ mod tests {
}
#[test]
fn sync_provider_key_quota_status_snapshot_preserves_codex_usage_reset_at() {
fn sync_provider_key_quota_status_snapshot_preserves_codex_usage_state() {
let current_status_snapshot = json!({
"quota": {
"version": 2,
@@ -2071,12 +2090,22 @@ mod tests {
{
"code": "weekly",
"usage_reset_at": 1_775_600_000u64,
"usage": {
"request_count": 3,
"total_tokens": 375,
"total_cost_usd": "0.60000000"
},
"reset_at": 1_900_000_000u64,
"window_minutes": 10_080u64
},
{
"code": "5h",
"usage_reset_at": 1_775_700_000u64,
"usage": {
"request_count": 2,
"total_tokens": 225,
"total_cost_usd": "0.30000000"
},
"reset_at": 1_900_500_000u64,
"window_minutes": 300u64
}
@@ -2088,10 +2117,10 @@ mod tests {
"updated_at": 1_775_800_000u64,
"plan_type": "plus",
"primary_used_percent": 5.0,
"primary_reset_at": 1_901_000_000u64,
"primary_reset_at": 1_900_000_000u64,
"primary_window_minutes": 10_080u64,
"secondary_used_percent": 1.0,
"secondary_reset_at": 1_901_500_000u64,
"secondary_reset_at": 1_900_500_000u64,
"secondary_window_minutes": 300u64
}
});
@@ -2121,7 +2150,94 @@ mod tests {
.expect("5h window should exist");
assert_eq!(weekly.get("usage_reset_at"), Some(&json!(1_775_600_000u64)));
assert_eq!(
weekly
.get("usage")
.and_then(|usage| usage.get("request_count")),
Some(&json!(3))
);
assert_eq!(
weekly
.get("usage")
.and_then(|usage| usage.get("total_tokens")),
Some(&json!(375))
);
assert_eq!(
weekly
.get("usage")
.and_then(|usage| usage.get("total_cost_usd")),
Some(&json!("0.60000000"))
);
assert_eq!(five_h.get("usage_reset_at"), Some(&json!(1_775_700_000u64)));
assert_eq!(
five_h
.get("usage")
.and_then(|usage| usage.get("request_count")),
Some(&json!(2))
);
assert_eq!(
five_h
.get("usage")
.and_then(|usage| usage.get("total_tokens")),
Some(&json!(225))
);
assert_eq!(
five_h
.get("usage")
.and_then(|usage| usage.get("total_cost_usd")),
Some(&json!("0.30000000"))
);
}
#[test]
fn sync_provider_key_quota_status_snapshot_drops_codex_usage_state_when_window_resets() {
let current_status_snapshot = json!({
"quota": {
"version": 2,
"provider_type": "codex",
"windows": [
{
"code": "weekly",
"usage_reset_at": 1_775_600_000u64,
"usage": {
"request_count": 3,
"total_tokens": 375,
"total_cost_usd": "0.60000000"
},
"reset_at": 1_900_000_000u64,
"window_minutes": 10_080u64
}
]
}
});
let upstream_metadata = json!({
"codex": {
"updated_at": 1_900_000_100u64,
"plan_type": "plus",
"primary_used_percent": 0.0,
"primary_reset_at": 1_960_480_100u64,
"primary_window_minutes": 10_080u64
}
});
let payload = sync_provider_key_quota_status_snapshot(
Some(&current_status_snapshot),
"codex",
Some(&upstream_metadata),
"refresh_api",
)
.expect("quota snapshot should sync");
let weekly = payload["quota"]["windows"]
.as_array()
.expect("quota windows should exist")
.iter()
.filter_map(Value::as_object)
.find(|window| window.get("code") == Some(&json!("weekly")))
.expect("weekly window should exist");
assert_eq!(weekly.get("reset_at"), Some(&json!(1_960_480_100u64)));
assert!(weekly.get("usage_reset_at").is_none());
assert!(weekly.get("usage").is_none());
}
#[test]

52
apps/aether-gateway/src/scheduler/candidate/runtime.rs
View File

@@ -10,6 +10,7 @@ use aether_scheduler_core::{
candidate_is_selectable_with_runtime_state, candidate_runtime_skip_reason_with_state,
CandidateRuntimeSelectabilityInput,
};
use std::time::{SystemTime, UNIX_EPOCH};
use crate::data::auth::GatewayAuthApiKeySnapshot;
use crate::GatewayError;
@@ -343,7 +344,7 @@ fn read_key_oauth_invalid_map(
fn key_requires_oauth_reauth(
key: &StoredProviderCatalogKey,
provider_type: &str,
_now_unix_secs: u64,
now_unix_secs: u64,
) -> bool {
if !key.auth_type.trim().eq_ignore_ascii_case("oauth") {
return false;
@@ -355,34 +356,69 @@ fn key_requires_oauth_reauth(
.map(str::trim)
.unwrap_or_default();
if !invalid_reason.is_empty() {
return oauth_invalid_reason_is_hard_account_block(key, provider_type, invalid_reason);
return oauth_invalid_reason_blocks_scheduling(
key,
provider_type,
invalid_reason,
now_unix_secs,
);
}
false
}
fn oauth_invalid_reason_is_hard_account_block(
fn oauth_invalid_reason_blocks_scheduling(
key: &StoredProviderCatalogKey,
provider_type: &str,
invalid_reason: &str,
now_unix_secs: u64,
) -> bool {
if provider_type.trim().eq_ignore_ascii_case("kiro")
&& invalid_reason.trim().starts_with("[REFRESH_FAILED] ")
{
let trimmed_reason = invalid_reason.trim();
if oauth_invalid_reason_has_tag(trimmed_reason, "[OAUTH_EXPIRED]") {
return true;
}
let account_state = admin_provider_status_pure::resolve_pool_account_state(
Some(provider_type),
key.upstream_metadata.as_ref(),
Some(invalid_reason),
Some(trimmed_reason),
);
account_state.blocked
if account_state.blocked
&& !account_state.recoverable
&& account_state
.code
.as_deref()
.is_some_and(oauth_account_state_code_is_hard_block)
{
return true;
}
if oauth_invalid_reason_has_tag(trimmed_reason, "[REFRESH_FAILED]") {
return oauth_access_token_expired(key, now_unix_secs);
}
false
}
fn oauth_invalid_reason_has_tag(reason: &str, tag: &str) -> bool {
reason
.lines()
.map(str::trim)
.any(|line| line.starts_with(tag))
}
fn oauth_access_token_expired(key: &StoredProviderCatalogKey, now_unix_secs: u64) -> bool {
let now_unix_secs = if now_unix_secs == 0 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.ok()
.map(|duration| duration.as_secs())
.unwrap_or(0)
} else {
now_unix_secs
};
key.expires_at_unix_secs
.is_none_or(|expires_at| expires_at == 0 || expires_at <= now_unix_secs)
}
fn oauth_account_state_code_is_hard_block(code: &str) -> bool {

198
apps/aether-gateway/src/scheduler/candidate/tests/selection.rs
View File

@@ -1692,6 +1692,7 @@ async fn keeps_refresh_failed_oauth_candidate_selectable_before_local_auth_resol
{
let mut key = sample_key("key-codex", "provider-codex", Some(10));
key.auth_type = "oauth".to_string();
key.expires_at_unix_secs = Some(1_710_000_200);
key.oauth_invalid_at_unix_secs = Some(1_710_000_000);
key.oauth_invalid_reason = Some(
"[REFRESH_FAILED] Token 续期失败 (401): refresh_token 已被使用并轮换，请重新登录授权"
@@ -1733,6 +1734,136 @@ async fn keeps_refresh_failed_oauth_candidate_selectable_before_local_auth_resol
assert!(skipped.is_empty());
}
#[tokio::test]
async fn skips_refresh_failed_oauth_candidate_after_access_token_expiry() {
let mut row = sample_row();
row.provider_id = "provider-codex".to_string();
row.provider_name = "codex".to_string();
row.provider_type = "codex".to_string();
row.endpoint_id = "endpoint-codex".to_string();
row.endpoint_api_format = "openai:responses".to_string();
row.key_id = "key-codex".to_string();
row.key_name = "codex-refresh-failed-expired".to_string();
row.key_auth_type = "oauth".to_string();
row.key_api_formats = Some(vec!["openai:responses".to_string()]);
row.key_global_priority_by_format = Some(serde_json::json!({"openai:responses": 1}));
let candidates = Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
row,
]));
let mut provider = sample_provider("provider-codex", None);
provider.provider_type = "codex".to_string();
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![provider],
Vec::new(),
vec![{
let mut key = sample_key("key-codex", "provider-codex", Some(10));
key.auth_type = "oauth".to_string();
key.expires_at_unix_secs = Some(1_710_000_000);
key.oauth_invalid_at_unix_secs = Some(1_710_000_000);
key.oauth_invalid_reason = Some(
"[REFRESH_FAILED] Token 续期失败 (401): refresh_token 已被使用并轮换，请重新登录授权"
.to_string(),
);
key
}],
));
let quotas = Arc::new(InMemoryProviderQuotaRepository::seed(vec![]));
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![]));
let state = AppState::new()
.expect("state should build")
.with_data_state_for_tests(
GatewayDataState::with_candidate_selection_provider_catalog_quota_and_request_candidates_for_tests(
candidates,
provider_catalog,
quotas,
request_candidates,
),
);
let (selected, skipped) = collect_selectable_candidates_with_skip_reasons(
state.data.as_ref(),
&state,
"openai:responses",
"gpt-4.1",
false,
None,
1_710_000_100,
)
.await
.expect("selection should succeed");
assert!(selected.is_empty());
assert_eq!(skipped.len(), 1);
assert_eq!(skipped[0].candidate.key_id, "key-codex");
assert_eq!(skipped[0].skip_reason, "oauth_invalid");
}
#[tokio::test]
async fn skips_oauth_candidate_with_account_block_even_when_refresh_failed_is_present() {
let mut row = sample_row();
row.provider_id = "provider-codex".to_string();
row.provider_name = "codex".to_string();
row.provider_type = "codex".to_string();
row.endpoint_id = "endpoint-codex".to_string();
row.endpoint_api_format = "openai:responses".to_string();
row.key_id = "key-codex".to_string();
row.key_name = "codex-account-blocked-refresh-failed".to_string();
row.key_auth_type = "oauth".to_string();
row.key_api_formats = Some(vec!["openai:responses".to_string()]);
row.key_global_priority_by_format = Some(serde_json::json!({"openai:responses": 1}));
let candidates = Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
row,
]));
let mut provider = sample_provider("provider-codex", None);
provider.provider_type = "codex".to_string();
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![provider],
Vec::new(),
vec![{
let mut key = sample_key("key-codex", "provider-codex", Some(10));
key.auth_type = "oauth".to_string();
key.expires_at_unix_secs = Some(1_710_000_200);
key.oauth_invalid_at_unix_secs = Some(1_710_000_000);
key.oauth_invalid_reason = Some(
"[REFRESH_FAILED] Token 续期失败 (401): refresh_token 已被使用并轮换，请重新登录授权\n[ACCOUNT_BLOCK] account has been deactivated"
.to_string(),
);
key
}],
));
let quotas = Arc::new(InMemoryProviderQuotaRepository::seed(vec![]));
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![]));
let state = AppState::new()
.expect("state should build")
.with_data_state_for_tests(
GatewayDataState::with_candidate_selection_provider_catalog_quota_and_request_candidates_for_tests(
candidates,
provider_catalog,
quotas,
request_candidates,
),
);
let (selected, skipped) = collect_selectable_candidates_with_skip_reasons(
state.data.as_ref(),
&state,
"openai:responses",
"gpt-4.1",
false,
None,
1_710_000_100,
)
.await
.expect("selection should succeed");
assert!(selected.is_empty());
assert_eq!(skipped.len(), 1);
assert_eq!(skipped[0].candidate.key_id, "key-codex");
assert_eq!(skipped[0].skip_reason, "oauth_invalid");
}
#[tokio::test]
async fn keeps_request_failed_oauth_candidate_selectable() {
let mut row = sample_row();
@@ -1971,7 +2102,7 @@ async fn keeps_refreshable_kiro_candidate_selectable_when_oauth_token_expired()
}
#[tokio::test]
async fn skips_kiro_candidate_after_refresh_token_failure() {
async fn keeps_kiro_candidate_selectable_after_refresh_token_failure_until_access_token_expiry() {
let mut row = sample_row();
row.provider_id = "provider-kiro".to_string();
row.provider_name = "kiro".to_string();
@@ -1995,6 +2126,71 @@ async fn skips_kiro_candidate_after_refresh_token_failure() {
let mut key = sample_key("key-kiro", "provider-kiro", Some(10));
key.auth_type = "oauth".to_string();
key.encrypted_auth_config = Some("encrypted-refreshable-session".to_string());
key.expires_at_unix_secs = Some(1_710_000_200);
key.oauth_invalid_at_unix_secs = Some(1_710_000_000);
key.oauth_invalid_reason = Some(
"[REFRESH_FAILED] Token 续期失败 (401): refresh_token 无效、已过期或已撤销，请重新登录授权"
.to_string(),
);
key
}],
));
let quotas = Arc::new(InMemoryProviderQuotaRepository::seed(vec![]));
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![]));
let state = AppState::new()
.expect("state should build")
.with_data_state_for_tests(
GatewayDataState::with_candidate_selection_provider_catalog_quota_and_request_candidates_for_tests(
candidates,
provider_catalog,
quotas,
request_candidates,
),
);
let (selected, skipped) = collect_selectable_candidates_with_skip_reasons(
state.data.as_ref(),
&state,
"claude:messages",
"gpt-4.1",
false,
None,
1_710_000_100,
)
.await
.expect("selection should succeed");
assert_eq!(selected.len(), 1);
assert_eq!(selected[0].provider_id, "provider-kiro");
assert!(skipped.is_empty());
}
#[tokio::test]
async fn skips_kiro_candidate_after_refresh_token_failure_and_access_token_expiry() {
let mut row = sample_row();
row.provider_id = "provider-kiro".to_string();
row.provider_name = "kiro".to_string();
row.provider_type = "kiro".to_string();
row.endpoint_id = "endpoint-kiro".to_string();
row.endpoint_api_format = "claude:messages".to_string();
row.key_id = "key-kiro".to_string();
row.key_name = "kiro-refresh-failed-expired".to_string();
row.key_auth_type = "oauth".to_string();
row.key_api_formats = Some(vec!["claude:messages".to_string()]);
let candidates = Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
row,
]));
let mut provider = sample_provider("provider-kiro", None);
provider.provider_type = "kiro".to_string();
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![provider],
Vec::new(),
vec![{
let mut key = sample_key("key-kiro", "provider-kiro", Some(10));
key.auth_type = "oauth".to_string();
key.encrypted_auth_config = Some("encrypted-refreshable-session".to_string());
key.expires_at_unix_secs = Some(1_710_000_000);
key.oauth_invalid_at_unix_secs = Some(1_710_000_000);
key.oauth_invalid_reason = Some(
"[REFRESH_FAILED] Token 续期失败 (401): refresh_token 无效、已过期或已撤销，请重新登录授权"

20
apps/aether-gateway/src/state/oauth.rs
View File

@@ -53,6 +53,10 @@ fn tagged_reason(reason: Option<&str>, prefix: &str) -> Option<String> {
})
}
fn oauth_access_token_expired(expires_at_unix_secs: Option<u64>, now_unix_secs: u64) -> bool {
expires_at_unix_secs.is_none_or(|expires_at| expires_at == 0 || expires_at <= now_unix_secs)
}
fn oauth_auth_config_refresh_token_fingerprint(auth_config: Option<&str>) -> Option<String> {
let parsed = auth_config
.map(str::trim)
@@ -275,7 +279,7 @@ fn merge_local_oauth_refresh_failure_reason(
return Some(refresh_reason.to_string());
}
if current_reason.starts_with(OAUTH_EXPIRED_PREFIX) {
return Some(refresh_reason.to_string());
return Some(current_reason.to_string());
}
if oauth_invalid_reason_is_account_block(Some(current_reason)) {
return None;
@@ -345,14 +349,16 @@ fn build_oauth_status_snapshot_value(key: &StoredProviderCatalogKey) -> Value {
});
}
if let Some(reason) = tagged_reason(invalid_reason.as_deref(), OAUTH_REFRESH_FAILED_PREFIX) {
let access_token_expired = oauth_access_token_expired(expires_at_unix_secs, now_unix_secs);
return json!({
"code": "invalid",
"label": "已失效",
"code": if access_token_expired { "invalid" } else { "reauth_required" },
"label": if access_token_expired { "已失效" } else { "续期失败" },
"reason": reason,
"expires_at": expires_at_unix_secs,
"invalid_at": invalid_at_unix_secs,
"source": "oauth_refresh",
"requires_reauth": true,
"usable_until_expiry": !access_token_expired,
"expiring_soon": false,
});
}
@@ -392,11 +398,11 @@ fn build_oauth_status_snapshot_value(key: &StoredProviderCatalogKey) -> Value {
return json!({
"code": "expired",
"label": "已过期",
"reason": "Token 已过期，请重新授权",
"reason": "Access Token 已过期，等待自动续期",
"expires_at": expires_at_unix_secs,
"invalid_at": Value::Null,
"source": "expires_at",
"requires_reauth": true,
"requires_reauth": false,
"expiring_soon": false,
});
}
@@ -1688,13 +1694,13 @@ mod tests {
}
#[test]
fn local_refresh_failure_replaces_access_token_expired_marker() {
fn local_refresh_failure_does_not_replace_access_token_expired_marker() {
assert_eq!(
super::merge_local_oauth_refresh_failure_reason(
Some("[OAUTH_EXPIRED] access token invalid"),
"[REFRESH_FAILED] Token 续期失败 (401): refresh_token 无效",
),
Some("[REFRESH_FAILED] Token 续期失败 (401): refresh_token 无效".to_string()),
Some("[OAUTH_EXPIRED] access token invalid".to_string()),
);
}
}

16
apps/aether-gateway/src/tests/control/admin/oauth.rs
View File

@@ -4463,11 +4463,11 @@ async fn gateway_refreshes_admin_provider_oauth_key_locally_with_trusted_admin_p
"oauth": {
"code": "expired",
"label": "已过期",
"reason": "Token 已过期，请重新授权",
"reason": "Access Token 已过期，等待自动续期",
"expires_at": 1u64,
"invalid_at": 1_700_000_000u64,
"source": "expires_at",
"requires_reauth": true,
"requires_reauth": false,
"expiring_soon": false
},
"account": {
@@ -4754,13 +4754,13 @@ async fn gateway_marks_manual_oauth_refresh_failures_as_invalid_in_pool_payload(
"stale-codex-access-token",
);
key.auth_type = "oauth".to_string();
key.expires_at_unix_secs = Some(1_900_000_000);
key.expires_at_unix_secs = Some(4_102_444_800);
key.status_snapshot = Some(json!({
"oauth": {
"code": "valid",
"label": "有效",
"reason": serde_json::Value::Null,
"expires_at": 1_900_000_000u64,
"expires_at": 4_102_444_800u64,
"invalid_at": serde_json::Value::Null,
"source": "expires_at",
"requires_reauth": false,
@@ -4788,7 +4788,7 @@ async fn gateway_marks_manual_oauth_refresh_failures_as_invalid_in_pool_payload(
key.encrypted_auth_config = Some(
encrypt_python_fernet_plaintext(
DEVELOPMENT_ENCRYPTION_KEY,
r#"{"provider_type":"codex","refresh_token":"used-refresh-token","email":"alice@example.com","account_id":"acct-codex-123","plan_type":"plus","expires_at":1900000000}"#,
r#"{"provider_type":"codex","refresh_token":"used-refresh-token","email":"alice@example.com","account_id":"acct-codex-123","plan_type":"plus","expires_at":4102444800}"#,
)
.expect("auth config ciphertext should build"),
);
@@ -4873,7 +4873,7 @@ async fn gateway_marks_manual_oauth_refresh_failures_as_invalid_in_pool_payload(
);
assert_eq!(
keys[0]["status_snapshot"]["oauth"]["code"],
json!("invalid")
json!("reauth_required")
);
assert_eq!(
keys[0]["status_snapshot"]["oauth"]["reason"],
@@ -4887,6 +4887,10 @@ async fn gateway_marks_manual_oauth_refresh_failures_as_invalid_in_pool_payload(
keys[0]["status_snapshot"]["oauth"]["requires_reauth"],
json!(true)
);
assert_eq!(
keys[0]["status_snapshot"]["oauth"]["usable_until_expiry"],
json!(true)
);
gateway_handle.abort();
token_handle.abort();

177
apps/aether-gateway/src/tests/control/admin/pool.rs
View File

@@ -2,9 +2,7 @@ use std::sync::{Arc, Mutex};
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
use aether_data::repository::usage::InMemoryUsageReadRepository;
use aether_data_contracts::repository::provider_catalog::ProviderCatalogReadRepository;
use aether_data_contracts::repository::usage::StoredRequestUsageAudit;
use axum::body::{to_bytes, Body, Bytes};
use axum::routing::{any, get, post};
use axum::{extract::Request, Router};
@@ -23,54 +21,6 @@ use crate::constants::{
use crate::control::resolve_public_request_context;
use crate::data::GatewayDataState;
fn sample_pool_usage_row(
request_id: &str,
provider_api_key_id: &str,
created_at_unix_secs: i64,
total_tokens: i32,
total_cost_usd: f64,
) -> StoredRequestUsageAudit {
StoredRequestUsageAudit::new(
format!("usage-{request_id}"),
request_id.to_string(),
Some("user-codex".to_string()),
Some("api-key-codex".to_string()),
Some("codex-user".to_string()),
Some("codex-api-key".to_string()),
"codex".to_string(),
"gpt-5-codex".to_string(),
None,
Some("provider-codex".to_string()),
Some("endpoint-codex".to_string()),
Some(provider_api_key_id.to_string()),
Some("responses".to_string()),
Some("openai:responses".to_string()),
Some("openai".to_string()),
Some("responses".to_string()),
Some("openai:responses".to_string()),
Some("openai".to_string()),
Some("responses".to_string()),
false,
false,
total_tokens,
0,
total_tokens,
total_cost_usd,
total_cost_usd,
Some(200),
None,
None,
Some(240),
Some(80),
"completed".to_string(),
"settled".to_string(),
created_at_unix_secs,
created_at_unix_secs + 1,
Some(created_at_unix_secs + 2),
)
.expect("usage row should build")
}
fn trusted_admin_headers() -> HeaderMap {
let mut headers = HeaderMap::new();
headers.insert(GATEWAY_HEADER, HeaderValue::from_static("rust-phase3b"));
@@ -866,8 +816,8 @@ async fn gateway_sorts_admin_pool_keys_by_imported_and_last_used_time() {
}
#[tokio::test]
async fn gateway_pool_list_adds_codex_cycle_usage_to_quota_windows() {
const RESET_AT: u64 = 1_711_000_000;
async fn gateway_pool_list_reads_materialized_codex_cycle_usage_from_quota_windows() {
const RESET_AT: u64 = 4_102_444_800;
let mut provider = sample_provider("provider-codex", "codex", 10).with_transport_fields(
true,
@@ -898,21 +848,21 @@ async fn gateway_pool_list_adds_codex_cycle_usage_to_quota_windows() {
usage_key.total_tokens = 999;
usage_key.total_cost_usd = 9.99;
usage_key.status_snapshot = Some(json!({
"quota": {
"version": 2,
"provider_type": "codex",
"code": "ok",
"label": serde_json::Value::Null,
"reason": serde_json::Value::Null,
"freshness": "fresh",
"source": "response_headers",
"observed_at": RESET_AT,
"exhausted": false,
"usage_ratio": 0.0,
"updated_at": RESET_AT,
"reset_seconds": serde_json::Value::Null,
"plan_type": "plus",
"windows": [
"quota": {
"version": 2,
"provider_type": "codex",
"code": "ok",
"label": serde_json::Value::Null,
"reason": serde_json::Value::Null,
"freshness": "fresh",
"source": "response_headers",
"observed_at": RESET_AT,
"exhausted": false,
"usage_ratio": 0.0,
"updated_at": RESET_AT,
"reset_seconds": serde_json::Value::Null,
"plan_type": "plus",
"windows": [
{
"code": "weekly",
"label": "周",
@@ -922,7 +872,12 @@ async fn gateway_pool_list_adds_codex_cycle_usage_to_quota_windows() {
"remaining_ratio": 1.0,
"reset_at": RESET_AT,
"reset_seconds": 604_800,
"window_minutes": 10_080
"window_minutes": 10_080,
"usage": {
"request_count": 3,
"total_tokens": 375,
"total_cost_usd": "0.60000000"
}
},
{
"code": "5h",
@@ -933,7 +888,12 @@ async fn gateway_pool_list_adds_codex_cycle_usage_to_quota_windows() {
"remaining_ratio": 1.0,
"reset_at": RESET_AT,
"reset_seconds": 18_000,
"window_minutes": 300
"window_minutes": 300,
"usage": {
"request_count": 2,
"total_tokens": 225,
"total_cost_usd": "0.30000000"
}
}
]
}
@@ -947,7 +907,37 @@ async fn gateway_pool_list_adds_codex_cycle_usage_to_quota_windows() {
);
zero_key.name = "codex zero usage".to_string();
zero_key.auth_type = "oauth".to_string();
zero_key.status_snapshot = usage_key.status_snapshot.clone();
zero_key.status_snapshot = Some(json!({
"quota": {
"version": 2,
"provider_type": "codex",
"code": "ok",
"windows": [
{
"code": "weekly",
"label": "周",
"reset_at": RESET_AT,
"window_minutes": 10_080,
"usage": {
"request_count": 0,
"total_tokens": 0,
"total_cost_usd": "0.00000000"
}
},
{
"code": "5h",
"label": "5H",
"reset_at": RESET_AT,
"window_minutes": 300,
"usage": {
"request_count": 0,
"total_tokens": 0,
"total_cost_usd": "0.00000000"
}
}
]
}
}));
let mut invalid_key = sample_key(
"key-codex-invalid",
@@ -984,44 +974,11 @@ async fn gateway_pool_list_adds_codex_cycle_usage_to_quota_windows() {
Vec::new(),
vec![usage_key, zero_key, invalid_key],
));
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![
sample_pool_usage_row(
"req-5h-a",
"key-codex-cycle",
RESET_AT as i64 - 60,
100,
0.10,
),
sample_pool_usage_row(
"req-5h-b",
"key-codex-cycle",
RESET_AT as i64 - 17_999,
125,
0.20,
),
sample_pool_usage_row(
"req-weekly-only",
"key-codex-cycle",
RESET_AT as i64 - 18_001,
150,
0.30,
),
sample_pool_usage_row(
"req-before-weekly",
"key-codex-cycle",
RESET_AT as i64 - 604_801,
200,
0.40,
),
]));
let state = AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(
GatewayDataState::with_provider_catalog_and_usage_reader_for_tests(
provider_catalog_repository,
usage_repository,
),
);
.with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests(
provider_catalog_repository,
));
let response = local_admin_pool_response(
&state,
@@ -1302,11 +1259,11 @@ async fn gateway_handles_admin_pool_list_keys_with_quota_compatibility_fields()
"oauth": {
"code": "expired",
"label": "已过期",
"reason": "Token 已过期，请重新授权",
"reason": "Access Token 已过期，等待自动续期",
"expires_at": 1775556730u64,
"invalid_at": null,
"source": "expires_at",
"requires_reauth": true,
"requires_reauth": false,
"expiring_soon": false
},
"account": {
@@ -1441,11 +1398,11 @@ async fn gateway_includes_pool_quota_and_compat_fields_in_list_keys_response() {
"oauth": {
"code": "expired",
"label": "已过期",
"reason": "Token 已过期，请重新授权",
"reason": "Access Token 已过期，等待自动续期",
"expires_at": 1_775_556_730u64,
"invalid_at": serde_json::Value::Null,
"source": "expires_at",
"requires_reauth": true,
"requires_reauth": false,
"expiring_soon": false
},
"account": {