i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-03 01:40:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix provider pool quota status handling

Browse Source
This commit is contained in:
fawney19
2026-05-07 11:28:44 +08:00
parent 01c9f9be49
commit fd44906bb7
37 changed files with 1844 additions and 500 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
apps/aether-gateway/src/handlers/admin/provider/endpoint_keys/mutations/reset_cycle_stats.rs
View File

@@ -111,7 +111,14 @@ fn reset_codex_cycle_usage_windows(status_snapshot: &mut Value, now_unix_secs: u
}
window.insert("usage_reset_at".to_string(), json!(now_unix_secs));
window.remove("usage");
window.insert(
"usage".to_string(),
json!({
"request_count": 0,
"total_tokens": 0,
"total_cost_usd": "0.00000000",
}),
);
reset_count += 1;
}
@@ -173,9 +180,13 @@ mod tests {
assert_eq!(reset_codex_cycle_usage_windows(&mut snapshot, 1_234), 2);
let windows = snapshot["quota"]["windows"].as_array().expect("windows");
assert_eq!(windows[0]["usage_reset_at"], json!(1_234));
assert!(windows[0].get("usage").is_none());
assert_eq!(windows[0]["usage"]["request_count"], json!(0));
assert_eq!(windows[0]["usage"]["total_tokens"], json!(0));
assert_eq!(windows[0]["usage"]["total_cost_usd"], json!("0.00000000"));
assert_eq!(windows[1]["usage_reset_at"], json!(1_234));
assert!(windows[1].get("usage").is_none());
assert_eq!(windows[1]["usage"]["request_count"], json!(0));
assert_eq!(windows[1]["usage"]["total_tokens"], json!(0));
assert_eq!(windows[1]["usage"]["total_cost_usd"], json!("0.00000000"));
assert!(windows[2].get("usage_reset_at").is_none());
assert!(windows[2].get("usage").is_some());
}

6
apps/aether-gateway/src/handlers/admin/provider/oauth/errors.rs
View File

@@ -147,7 +147,7 @@ pub(crate) fn merge_provider_oauth_refresh_failure_reason(
return Some(refresh_reason.to_string());
}
if current_reason.starts_with(OAUTH_EXPIRED_PREFIX) {
return Some(refresh_reason.to_string());
return Some(current_reason.to_string());
}
if oauth_invalid_reason_is_account_level_block(Some(current_reason)) {
return None;
@@ -190,13 +190,13 @@ mod tests {
}
#[test]
fn refresh_failure_replaces_access_token_expired_marker() {
fn refresh_failure_does_not_replace_access_token_expired_marker() {
assert_eq!(
merge_provider_oauth_refresh_failure_reason(
Some("[OAUTH_EXPIRED] access token invalid"),
"[REFRESH_FAILED] Token 续期失败 (401): refresh_token 无效",
),
Some("[REFRESH_FAILED] Token 续期失败 (401): refresh_token 无效".to_string()),
Some("[OAUTH_EXPIRED] access token invalid".to_string()),
);
}
}

248
apps/aether-gateway/src/handlers/admin/provider/pool_admin/payloads.rs
View File

@@ -9,11 +9,8 @@ use aether_admin::provider::quota as admin_provider_quota_pure;
use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
};
use aether_data_contracts::repository::usage::{
ProviderApiKeyWindowUsageRequest, StoredProviderApiKeyWindowUsageSummary,
};
use serde_json::json;
use std::collections::BTreeMap;
use std::time::{SystemTime, UNIX_EPOCH};
fn admin_pool_string_list(value: Option<&serde_json::Value>) -> Option<Vec<String>> {
let values = value
@@ -293,128 +290,6 @@ fn admin_pool_quota_window<'a>(
})
}
pub(super) type AdminPoolCodexWindowUsageByKey =
BTreeMap<(String, String), StoredProviderApiKeyWindowUsageSummary>;
fn admin_pool_provider_type_is_codex(provider_type: &str) -> bool {
provider_type.trim().eq_ignore_ascii_case("codex")
}
fn admin_pool_codex_window_usage_code(
window: &serde_json::Map<String, serde_json::Value>,
) -> Option<&'static str> {
let code = window
.get("code")
.and_then(serde_json::Value::as_str)
.map(str::trim)?;
if code.eq_ignore_ascii_case("5h") {
Some("5h")
} else if code.eq_ignore_ascii_case("weekly") {
Some("weekly")
} else {
None
}
}
fn admin_pool_codex_window_usage_bounds(
window: &serde_json::Map<String, serde_json::Value>,
) -> Option<(u64, u64)> {
let reset_at = admin_pool_json_to_u64(window.get("reset_at"))?;
let window_minutes = admin_pool_json_to_u64(window.get("window_minutes"))?;
let window_seconds = window_minutes.checked_mul(60)?;
let window_start = reset_at.checked_sub(window_seconds)?;
let usage_reset_at = admin_pool_json_to_u64(window.get("usage_reset_at"))
.filter(|reset_at_override| *reset_at_override < reset_at)
.unwrap_or(window_start);
let start = window_start.max(usage_reset_at);
(start < reset_at).then_some((start, reset_at))
}
pub(super) fn build_admin_pool_codex_window_usage_requests(
provider_type: &str,
keys: &[StoredProviderCatalogKey],
) -> Vec<ProviderApiKeyWindowUsageRequest> {
if !admin_pool_provider_type_is_codex(provider_type) {
return Vec::new();
}
let mut requests = Vec::new();
for key in keys {
let status_snapshot = provider_key_status_snapshot_payload(key, provider_type);
let Some(windows) = status_snapshot
.get("quota")
.and_then(serde_json::Value::as_object)
.and_then(|quota| quota.get("windows"))
.and_then(serde_json::Value::as_array)
else {
continue;
};
for window in windows.iter().filter_map(serde_json::Value::as_object) {
let Some(window_code) = admin_pool_codex_window_usage_code(window) else {
continue;
};
let Some((start_unix_secs, end_unix_secs)) =
admin_pool_codex_window_usage_bounds(window)
else {
continue;
};
requests.push(ProviderApiKeyWindowUsageRequest {
provider_api_key_id: key.id.clone(),
window_code: window_code.to_string(),
start_unix_secs,
end_unix_secs,
});
}
}
requests
}
fn admin_pool_codex_window_usage_payload(
usage: &StoredProviderApiKeyWindowUsageSummary,
) -> serde_json::Value {
json!({
"request_count": usage.request_count,
"total_tokens": usage.total_tokens,
"total_cost_usd": format!("{:.8}", usage.total_cost_usd),
})
}
fn admin_pool_attach_codex_window_usage(
status_snapshot: &mut serde_json::Value,
key_id: &str,
usage_by_key: &AdminPoolCodexWindowUsageByKey,
) {
if usage_by_key.is_empty() {
return;
}
let Some(windows) = status_snapshot
.get_mut("quota")
.and_then(serde_json::Value::as_object_mut)
.and_then(|quota| quota.get_mut("windows"))
.and_then(serde_json::Value::as_array_mut)
else {
return;
};
for window in windows
.iter_mut()
.filter_map(serde_json::Value::as_object_mut)
{
let Some(window_code) = admin_pool_codex_window_usage_code(window) else {
continue;
};
let lookup_key = (key_id.to_string(), window_code.to_string());
if let Some(usage) = usage_by_key.get(&lookup_key) {
window.insert(
"usage".to_string(),
admin_pool_codex_window_usage_payload(usage),
);
}
}
}
fn admin_pool_quota_window_used_percent(
window: &serde_json::Map<String, serde_json::Value>,
) -> Option<f64> {
@@ -528,6 +403,54 @@ fn admin_pool_build_codex_account_quota_from_snapshot(
None
}
fn admin_pool_current_unix_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.ok()
.map(|duration| duration.as_secs())
.unwrap_or(0)
}
fn admin_pool_prune_expired_codex_window_usage(status_snapshot: &mut serde_json::Value) {
let now_unix_secs = admin_pool_current_unix_secs();
let Some(windows) = status_snapshot
.get_mut("quota")
.and_then(serde_json::Value::as_object_mut)
.and_then(|quota| quota.get_mut("windows"))
.and_then(serde_json::Value::as_array_mut)
else {
return;
};
for window in windows
.iter_mut()
.filter_map(serde_json::Value::as_object_mut)
{
let code = window
.get("code")
.and_then(serde_json::Value::as_str)
.map(str::trim)
.unwrap_or_default();
if !code.eq_ignore_ascii_case("5h") && !code.eq_ignore_ascii_case("weekly") {
continue;
}
let Some(reset_at) = admin_pool_json_to_u64(window.get("reset_at")) else {
continue;
};
if now_unix_secs < reset_at {
continue;
}
window.insert(
"usage".to_string(),
json!({
"request_count": 0,
"total_tokens": 0,
"total_cost_usd": "0.00000000",
}),
);
}
}
fn admin_pool_quota_windows<'a>(
quota_snapshot: &'a serde_json::Map<String, serde_json::Value>,
) -> Vec<&'a serde_json::Map<String, serde_json::Value>> {
@@ -930,7 +853,6 @@ pub(super) fn build_admin_pool_key_payload(
key: &StoredProviderCatalogKey,
runtime: &AdminProviderPoolRuntimeState,
pool_config: Option<AdminProviderPoolConfig>,
codex_window_usage_by_key: &AdminPoolCodexWindowUsageByKey,
) -> serde_json::Value {
let cooldown_reason = runtime.cooldown_reason_by_key.get(&key.id).cloned();
let cooldown_ttl_seconds = cooldown_reason
@@ -949,12 +871,8 @@ pub(super) fn build_admin_pool_key_payload(
let oauth_plan_type =
admin_pool_derive_oauth_plan_type(key, provider_type, auth_config.as_ref());
let mut status_snapshot = provider_key_status_snapshot_payload(key, provider_type);
if admin_pool_provider_type_is_codex(provider_type) {
admin_pool_attach_codex_window_usage(
&mut status_snapshot,
&key.id,
codex_window_usage_by_key,
);
if provider_type.trim().eq_ignore_ascii_case("codex") {
admin_pool_prune_expired_codex_window_usage(&mut status_snapshot);
}
let account_snapshot = status_snapshot
.get("account")
@@ -1237,34 +1155,54 @@ mod tests {
use serde_json::json;
#[test]
fn codex_window_usage_bounds_honor_manual_usage_reset_at() {
let window = json!({
"code": "5h",
"reset_at": 20_000,
"window_minutes": 300,
"usage_reset_at": 9_000,
fn expired_codex_window_usage_is_zeroed_for_display() {
let mut snapshot = json!({
"quota": {
"windows": [
{
"code": "5h",
"reset_at": 1,
"usage": {
"request_count": 7,
"total_tokens": 700,
"total_cost_usd": "7.00000000"
}
}
]
}
});
let window = window.as_object().expect("window object");
assert_eq!(
admin_pool_codex_window_usage_bounds(window),
Some((9_000, 20_000))
);
admin_pool_prune_expired_codex_window_usage(&mut snapshot);
let usage = &snapshot["quota"]["windows"][0]["usage"];
assert_eq!(usage["request_count"], json!(0));
assert_eq!(usage["total_tokens"], json!(0));
assert_eq!(usage["total_cost_usd"], json!("0.00000000"));
}
#[test]
fn codex_window_usage_bounds_ignore_reset_before_window_start() {
let window = json!({
"code": "weekly",
"reset_at": 700_000,
"window_minutes": 10_080,
"usage_reset_at": 1,
fn active_codex_window_usage_is_preserved_for_display() {
let mut snapshot = json!({
"quota": {
"windows": [
{
"code": "weekly",
"reset_at": 4_102_444_800u64,
"usage": {
"request_count": 3,
"total_tokens": 375,
"total_cost_usd": "0.60000000"
}
}
]
}
});
let window = window.as_object().expect("window object");
assert_eq!(
admin_pool_codex_window_usage_bounds(window),
Some((95_200, 700_000))
);
admin_pool_prune_expired_codex_window_usage(&mut snapshot);
let usage = &snapshot["quota"]["windows"][0]["usage"];
assert_eq!(usage["request_count"], json!(3));
assert_eq!(usage["total_tokens"], json!(375));
assert_eq!(usage["total_cost_usd"], json!("0.60000000"));
}
}

31
apps/aether-gateway/src/handlers/admin/provider/pool_admin/read_routes/keys.rs
View File

@@ -253,36 +253,6 @@ pub(super) async fn build_admin_pool_list_keys_response(
}
_ => AdminProviderPoolRuntimeState::default(),
};
let codex_window_usage_requests =
pool_payloads::build_admin_pool_codex_window_usage_requests(&provider.provider_type, &keys);
let mut codex_window_usage_by_key = codex_window_usage_requests
.iter()
.map(|request| {
(
(
request.provider_api_key_id.clone(),
request.window_code.clone(),
),
aether_data_contracts::repository::usage::StoredProviderApiKeyWindowUsageSummary {
provider_api_key_id: request.provider_api_key_id.clone(),
window_code: request.window_code.clone(),
..Default::default()
},
)
})
.collect::<pool_payloads::AdminPoolCodexWindowUsageByKey>();
if !codex_window_usage_requests.is_empty() {
for usage in state
.app()
.summarize_usage_by_provider_api_key_windows(&codex_window_usage_requests)
.await?
{
codex_window_usage_by_key.insert(
(usage.provider_api_key_id.clone(), usage.window_code.clone()),
usage,
);
}
}
let items = keys
.into_iter()
@@ -294,7 +264,6 @@ pub(super) async fn build_admin_pool_list_keys_response(
&key,
&runtime,
pool_config.clone(),
&codex_window_usage_by_key,
)
})
.collect::<Vec<_>>();