i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 01:10:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add provider key cycle stats reset handling

Browse Source
This commit is contained in:
fawney19
2026-05-07 03:23:20 +08:00
parent 6c0ac2e8da
commit 10e679bb2f
35 changed files with 1443 additions and 273 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
apps/aether-gateway/src/control/route/admin/endpoints_families.rs
View File

@@ -155,6 +155,17 @@ pub(super) fn classify_admin_endpoints_family_route(
"admin:endpoints_manage",
false,
))
} else if method == http::Method::POST
&& normalized_path.starts_with("/api/admin/endpoints/keys/")
&& normalized_path.ends_with("/reset-cycle-stats")
{
Some(classified(
"admin_proxy",
"endpoints_manage",
"reset_cycle_stats",
"admin:endpoints_manage",
false,
))
} else if method == http::Method::POST
&& normalized_path.starts_with("/api/admin/endpoints/providers/")
&& normalized_path.ends_with("/refresh-quota")

14
apps/aether-gateway/src/control/tests/admin_endpoints.rs
View File

@@ -275,6 +275,20 @@ fn classifies_admin_clear_oauth_invalid_as_admin_proxy_route() {
assert!(!decision.is_execution_runtime_candidate());
}
#[test]
fn classifies_admin_reset_key_cycle_stats_as_admin_proxy_route() {
let headers = http::HeaderMap::new();
let uri: Uri = "/api/admin/endpoints/keys/key-codex/reset-cycle-stats"
.parse()
.expect("uri should parse");
let decision = classify_control_route(&http::Method::POST, &uri, &headers)
.expect("decision should resolve");
assert_eq!(decision.route_class.as_deref(), Some("admin_proxy"));
assert_eq!(decision.route_family.as_deref(), Some("endpoints_manage"));
assert_eq!(decision.route_kind.as_deref(), Some("reset_cycle_stats"));
assert!(!decision.is_execution_runtime_candidate());
}
#[test]
fn classifies_admin_create_provider_key_as_admin_proxy_route() {
let headers = http::HeaderMap::new();

6
apps/aether-gateway/src/handlers/admin/provider/endpoint_keys/mutations/mod.rs
View File

@@ -2,6 +2,7 @@ mod batch;
mod create;
mod delete;
mod oauth_invalid;
mod reset_cycle_stats;
mod update;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
@@ -30,6 +31,11 @@ pub(super) async fn maybe_handle(
{
return Ok(Some(response));
}
if let Some(response) =
reset_cycle_stats::maybe_handle(state, request_context, request_body).await?
{
return Ok(Some(response));
}
if let Some(response) = create::maybe_handle(state, request_context, request_body).await? {
return Ok(Some(response));
}

182
apps/aether-gateway/src/handlers/admin/provider/endpoint_keys/mutations/reset_cycle_stats.rs Normal file
View File

@@ -0,0 +1,182 @@
use crate::handlers::admin::provider::shared::paths::admin_reset_cycle_stats_key_id;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::provider_key_status_snapshot_payload;
use crate::GatewayError;
use axum::{
body::{Body, Bytes},
http,
response::{IntoResponse, Response},
Json,
};
use serde_json::{json, Value};
use std::time::{SystemTime, UNIX_EPOCH};
pub(super) async fn maybe_handle(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
_request_body: Option<&Bytes>,
) -> Result<Option<Response<Body>>, GatewayError> {
let Some(decision) = request_context.decision() else {
return Ok(None);
};
if decision.route_family.as_deref() != Some("endpoints_manage")
|| decision.route_kind.as_deref() != Some("reset_cycle_stats")
|| request_context.method() != http::Method::POST
|| !request_context
.path()
.starts_with("/api/admin/endpoints/keys/")
|| !request_context.path().ends_with("/reset-cycle-stats")
{
return Ok(None);
}
let Some(key_id) = admin_reset_cycle_stats_key_id(request_context.path()) else {
return Ok(Some(not_found_response("Key 不存在")));
};
let Some(mut key) = state
.read_provider_catalog_keys_by_ids(std::slice::from_ref(&key_id))
.await?
.into_iter()
.next()
else {
return Ok(Some(not_found_response(format!("Key {key_id} 不存在"))));
};
let Some(provider) = state
.read_provider_catalog_providers_by_ids(std::slice::from_ref(&key.provider_id))
.await?
.into_iter()
.next()
else {
return Ok(Some(not_found_response(format!(
"Provider {} 不存在",
key.provider_id
))));
};
if !provider.provider_type.trim().eq_ignore_ascii_case("codex") {
return Ok(Some(bad_request_response(
"仅 Codex Provider 支持重置周期统计",
)));
}
let now_unix_secs = current_unix_secs();
let mut status_snapshot = provider_key_status_snapshot_payload(&key, &provider.provider_type);
let reset_windows = reset_codex_cycle_usage_windows(&mut status_snapshot, now_unix_secs);
if reset_windows == 0 {
return Ok(Some(bad_request_response("当前账号没有可重置的周期窗口")));
}
key.status_snapshot = Some(status_snapshot);
key.updated_at_unix_secs = Some(now_unix_secs);
let Some(_) = state.update_provider_catalog_key(&key).await? else {
return Ok(None);
};
Ok(Some(
Json(json!({
"message": "已重置周期统计",
"reset_at": now_unix_secs,
"windows": reset_windows,
}))
.into_response(),
))
}
fn current_unix_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.ok()
.map(|duration| duration.as_secs())
.unwrap_or(0)
}
fn reset_codex_cycle_usage_windows(status_snapshot: &mut Value, now_unix_secs: u64) -> usize {
let Some(windows) = status_snapshot
.get_mut("quota")
.and_then(Value::as_object_mut)
.and_then(|quota| quota.get_mut("windows"))
.and_then(Value::as_array_mut)
else {
return 0;
};
let mut reset_count = 0;
for window in windows.iter_mut().filter_map(Value::as_object_mut) {
let code = window
.get("code")
.and_then(Value::as_str)
.map(str::trim)
.unwrap_or_default();
if !code.eq_ignore_ascii_case("5h") && !code.eq_ignore_ascii_case("weekly") {
continue;
}
window.insert("usage_reset_at".to_string(), json!(now_unix_secs));
window.remove("usage");
reset_count += 1;
}
reset_count
}
fn bad_request_response(detail: impl Into<String>) -> Response<Body> {
(
http::StatusCode::BAD_REQUEST,
Json(json!({ "detail": detail.into() })),
)
.into_response()
}
fn not_found_response(detail: impl Into<String>) -> Response<Body> {
(
http::StatusCode::NOT_FOUND,
Json(json!({ "detail": detail.into() })),
)
.into_response()
}
#[cfg(test)]
mod tests {
use super::reset_codex_cycle_usage_windows;
use serde_json::json;
#[test]
fn reset_cycle_usage_marks_codex_windows_and_removes_stale_usage() {
let mut snapshot = json!({
"quota": {
"windows": [
{
"code": "5h",
"usage": {
"request_count": 9,
"total_tokens": 100,
"total_cost_usd": "1.00000000"
}
},
{
"code": "weekly",
"usage": {
"request_count": 10,
"total_tokens": 200,
"total_cost_usd": "2.00000000"
}
},
{
"code": "monthly",
"usage": {
"request_count": 11
}
}
]
}
});
assert_eq!(reset_codex_cycle_usage_windows(&mut snapshot, 1_234), 2);
let windows = snapshot["quota"]["windows"].as_array().expect("windows");
assert_eq!(windows[0]["usage_reset_at"], json!(1_234));
assert!(windows[0].get("usage").is_none());
assert_eq!(windows[1]["usage_reset_at"], json!(1_234));
assert!(windows[1].get("usage").is_none());
assert!(windows[2].get("usage_reset_at").is_none());
assert!(windows[2].get("usage").is_some());
}
}

24
apps/aether-gateway/src/handlers/admin/provider/endpoint_keys/quota.rs
View File

@@ -11,7 +11,7 @@ use axum::{
Json,
};
use serde_json::json;
use std::collections::BTreeSet;
use std::collections::{BTreeMap, BTreeSet};
use super::super::oauth::quota::antigravity::refresh_antigravity_provider_quota_locally;
use super::super::oauth::quota::chatgpt_web::refresh_chatgpt_web_provider_quota_locally;
@@ -138,20 +138,28 @@ pub(super) async fn maybe_handle(
));
};
let mut keys = state
.list_provider_catalog_keys_by_provider_ids(std::slice::from_ref(&provider_id))
.await?;
keys = if let Some(selected_key_ids) = selected_key_ids.as_ref() {
let keys = if let Some(selected_key_ids) = selected_key_ids.as_ref() {
if selected_key_ids.is_empty() {
Vec::new()
} else {
let selected = selected_key_ids.iter().cloned().collect::<BTreeSet<_>>();
keys.into_iter()
.filter(|key| selected.contains(&key.id))
let mut by_id = state
.read_provider_catalog_keys_by_ids(selected_key_ids)
.await?
.into_iter()
.filter(|key| key.provider_id == provider_id && selected.contains(&key.id))
.map(|key| (key.id.clone(), key))
.collect::<BTreeMap<_, _>>();
selected_key_ids
.iter()
.filter_map(|key_id| by_id.remove(key_id))
.collect()
}
} else {
keys.into_iter()
state
.list_provider_catalog_keys_by_provider_ids(std::slice::from_ref(&provider_id))
.await?
.into_iter()
.filter(|key| {
key.is_active
|| key

22
apps/aether-gateway/src/handlers/admin/provider/oauth/errors.rs
View File

@@ -16,8 +16,13 @@ fn oauth_invalid_reason_is_account_level_block(reason: Option<&str>) -> bool {
if reason.starts_with(OAUTH_ACCOUNT_BLOCK_PREFIX) {
return true;
}
aether_admin::provider::status::resolve_account_status_snapshot(None, None, Some(reason))
.blocked
let snapshot =
aether_admin::provider::status::resolve_account_status_snapshot(None, None, Some(reason));
snapshot.blocked
&& !matches!(
snapshot.code.trim().to_ascii_lowercase().as_str(),
"oauth_token_invalid" | "oauth_expired" | "oauth_refresh_failed"
)
}
pub(crate) fn build_internal_control_error_response(
@@ -142,7 +147,7 @@ pub(crate) fn merge_provider_oauth_refresh_failure_reason(
return Some(refresh_reason.to_string());
}
if current_reason.starts_with(OAUTH_EXPIRED_PREFIX) {
return None;
return Some(refresh_reason.to_string());
}
if oauth_invalid_reason_is_account_level_block(Some(current_reason)) {
return None;
@@ -183,4 +188,15 @@ mod tests {
None,
);
}
#[test]
fn refresh_failure_replaces_access_token_expired_marker() {
assert_eq!(
merge_provider_oauth_refresh_failure_reason(
Some("[OAUTH_EXPIRED] access token invalid"),
"[REFRESH_FAILED] Token 续期失败 (401): refresh_token 无效",
),
Some("[REFRESH_FAILED] Token 续期失败 (401): refresh_token 无效".to_string()),
);
}
}

138
apps/aether-gateway/src/handlers/admin/provider/pool/runtime/writes.rs
View File

@@ -16,22 +16,40 @@ const MAX_POOL_COOLDOWN_SECONDS: u64 = 32 * 60;
const ACCOUNT_DISABLE_PATTERNS: &[&str] = &[
"organization has been disabled",
"organization disabled",
"organization_disabled",
"account has been disabled",
"account disabled",
"account_disabled",
"account has been deactivated",
"account_deactivated",
"account deactivated",
];
const WORKSPACE_DISABLE_PATTERNS: &[&str] = &[
"deactivated_workspace",
"workspace has been disabled",
"workspace disabled",
"workspace has been deactivated",
"workspace deactivated",
"workspace is disabled",
"workspace is deactivated",
];
const FORBIDDEN_ACCOUNT_PATTERNS: &[&str] = &[
"account suspended",
"account suspend",
"account banned",
"account blocked",
"account forbidden",
"account deactivated",
"account access denied",
"subscription inactive",
"suspended",
"banned",
"blocked",
"deactivated",
"access denied",
];
fn current_unix_secs_f64() -> f64 {
@@ -178,10 +196,9 @@ fn extract_error_message(error_body: Option<&str>) -> String {
.as_object()
.and_then(|object| object.get("error").or_else(|| object.get("message")))
.and_then(|error| match error {
serde_json::Value::Object(object) => object
.get("message")
.and_then(serde_json::Value::as_str)
.map(ToOwned::to_owned),
serde_json::Value::Object(object) => {
first_error_text(object, &["message", "detail", "reason", "code", "status"])
}
serde_json::Value::String(text) => Some(text.clone()),
_ => None,
})
@@ -189,11 +206,32 @@ fn extract_error_message(error_body: Option<&str>) -> String {
.unwrap_or_else(|| error_body.chars().take(500).collect())
}
fn first_error_text(
object: &serde_json::Map<String, serde_json::Value>,
keys: &[&str],
) -> Option<String> {
keys.iter().find_map(|key| {
let text = object.get(*key).and_then(|value| match value {
serde_json::Value::String(text) => Some(text.trim().to_string()),
serde_json::Value::Number(number) => Some(number.to_string()),
_ => None,
})?;
(!text.is_empty()).then_some(text)
})
}
pub(crate) fn admin_provider_pool_key_circuit_breaker_reason(
status_code: u16,
error_body: Option<&str>,
) -> Option<String> {
let error_message = extract_error_message(error_body).to_ascii_lowercase();
if let Some(pattern) = WORKSPACE_DISABLE_PATTERNS
.iter()
.find(|pattern| error_message.contains(**pattern))
{
return Some(format!("workspace_deactivated_{status_code}:{pattern}"));
}
match status_code {
401 if ACCOUNT_DISABLE_PATTERNS
.iter()
@@ -212,6 +250,12 @@ pub(crate) fn admin_provider_pool_key_circuit_breaker_reason(
.iter()
.find(|pattern| error_message.contains(**pattern))
.map(|pattern| format!("account_disabled_400:{pattern}")),
423 if FORBIDDEN_ACCOUNT_PATTERNS
.iter()
.any(|pattern| error_message.contains(pattern)) =>
{
Some("account_locked_423".to_string())
}
_ => None,
}
}
@@ -449,9 +493,9 @@ pub(crate) async fn record_admin_provider_pool_error(
}
if status_code == 400 {
if admin_provider_pool_key_circuit_breaker_reason(status_code, error_body).is_some() {
return;
}
// Bad Request is usually attributable to the caller payload, not key health.
// Account-level 400s are handled by the orchestration circuit-breaker path.
return;
}
if let Some(rule) =
@@ -709,6 +753,46 @@ mod tests {
assert_eq!(message_cooldown, Some(2_700));
}
#[test]
fn circuit_reason_detects_workspace_deactivated_errors() {
assert_eq!(
admin_provider_pool_key_circuit_breaker_reason(
402,
Some(r#"{"error":{"message":"workspace has been deactivated"}}"#),
)
.as_deref(),
Some("workspace_deactivated_402:workspace has been deactivated")
);
assert_eq!(
admin_provider_pool_key_circuit_breaker_reason(
400,
Some(r#"{"error":{"message":"deactivated_workspace"}}"#),
)
.as_deref(),
Some("workspace_deactivated_400:deactivated_workspace")
);
}
#[test]
fn circuit_reason_detects_account_ban_errors() {
assert_eq!(
admin_provider_pool_key_circuit_breaker_reason(
403,
Some(r#"{"error":{"message":"AccountSuspendedException: account suspended"}}"#),
)
.as_deref(),
Some("forbidden_403")
);
assert_eq!(
admin_provider_pool_key_circuit_breaker_reason(
423,
Some(r#"{"error":{"message":"account access denied"}}"#),
)
.as_deref(),
Some("account_locked_423")
);
}
#[tokio::test]
async fn success_feedback_writes_sticky_lru_cost_and_latency() {
let Some(redis) = start_managed_redis_or_skip().await else {
@@ -1025,6 +1109,46 @@ mod tests {
.is_some_and(|ttl| *ttl <= 420 && *ttl >= 380));
}
#[tokio::test]
async fn error_feedback_ignores_client_bad_request_for_cooldown() {
let Some(redis) = start_managed_redis_or_skip().await else {
return;
};
let app = build_runner_app(redis.redis_url(), "pool_runtime_ignore_400");
let runner = app.redis_kv_runner().expect("redis runner should exist");
let mut pool_config = sample_pool_config();
pool_config.unschedulable_rules = vec![AdminProviderPoolUnschedulableRule {
keyword: "review required".to_string(),
duration_minutes: 7,
}];
let key_ids = vec!["key-client-400".to_string()];
record_admin_provider_pool_error(
&runner,
"provider-1",
"key-client-400",
&pool_config,
400,
Some(r#"{"error":{"message":"manual review required before reuse"}}"#),
None,
)
.await;
let runtime = read_admin_provider_pool_runtime_state(
&runner,
"provider-1",
&key_ids,
&pool_config,
None,
)
.await;
assert!(!runtime
.cooldown_reason_by_key
.contains_key("key-client-400"));
assert!(!runtime.cooldown_ttl_by_key.contains_key("key-client-400"));
}
#[tokio::test]
async fn stream_timeout_policy_cools_down_after_threshold() {
let Some(redis) = start_managed_redis_or_skip().await else {

44
apps/aether-gateway/src/handlers/admin/provider/pool_admin/payloads.rs
View File

@@ -322,7 +322,11 @@ fn admin_pool_codex_window_usage_bounds(
let reset_at = admin_pool_json_to_u64(window.get("reset_at"))?;
let window_minutes = admin_pool_json_to_u64(window.get("window_minutes"))?;
let window_seconds = window_minutes.checked_mul(60)?;
let start = reset_at.checked_sub(window_seconds)?;
let window_start = reset_at.checked_sub(window_seconds)?;
let usage_reset_at = admin_pool_json_to_u64(window.get("usage_reset_at"))
.filter(|reset_at_override| *reset_at_override < reset_at)
.unwrap_or(window_start);
let start = window_start.max(usage_reset_at);
(start < reset_at).then_some((start, reset_at))
}
@@ -1226,3 +1230,41 @@ pub(super) fn build_admin_pool_key_payload(
serde_json::Value::Object(payload)
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn codex_window_usage_bounds_honor_manual_usage_reset_at() {
let window = json!({
"code": "5h",
"reset_at": 20_000,
"window_minutes": 300,
"usage_reset_at": 9_000,
});
let window = window.as_object().expect("window object");
assert_eq!(
admin_pool_codex_window_usage_bounds(window),
Some((9_000, 20_000))
);
}
#[test]
fn codex_window_usage_bounds_ignore_reset_before_window_start() {
let window = json!({
"code": "weekly",
"reset_at": 700_000,
"window_minutes": 10_080,
"usage_reset_at": 1,
});
let window = window.as_object().expect("window object");
assert_eq!(
admin_pool_codex_window_usage_bounds(window),
Some((95_200, 700_000))
);
}
}

45
apps/aether-gateway/src/handlers/admin/provider/pool_admin/read_routes/keys.rs
View File

@@ -255,23 +255,34 @@ pub(super) async fn build_admin_pool_list_keys_response(
};
let codex_window_usage_requests =
pool_payloads::build_admin_pool_codex_window_usage_requests(&provider.provider_type, &keys);
let codex_window_usage_by_key: pool_payloads::AdminPoolCodexWindowUsageByKey =
if codex_window_usage_requests.is_empty() {
pool_payloads::AdminPoolCodexWindowUsageByKey::new()
} else {
state
.app()
.summarize_usage_by_provider_api_key_windows(&codex_window_usage_requests)
.await?
.into_iter()
.map(|usage| {
(
(usage.provider_api_key_id.clone(), usage.window_code.clone()),
usage,
)
})
.collect()
};
let mut codex_window_usage_by_key = codex_window_usage_requests
.iter()
.map(|request| {
(
(
request.provider_api_key_id.clone(),
request.window_code.clone(),
),
aether_data_contracts::repository::usage::StoredProviderApiKeyWindowUsageSummary {
provider_api_key_id: request.provider_api_key_id.clone(),
window_code: request.window_code.clone(),
..Default::default()
},
)
})
.collect::<pool_payloads::AdminPoolCodexWindowUsageByKey>();
if !codex_window_usage_requests.is_empty() {
for usage in state
.app()
.summarize_usage_by_provider_api_key_windows(&codex_window_usage_requests)
.await?
{
codex_window_usage_by_key.insert(
(usage.provider_api_key_id.clone(), usage.window_code.clone()),
usage,
);
}
}
let items = keys
.into_iter()

5
apps/aether-gateway/src/handlers/admin/provider/pool_admin/support.rs
View File

@@ -37,7 +37,7 @@ pub(crate) struct AdminPoolKeySort {
impl Default for AdminPoolKeySort {
fn default() -> Self {
Self {
field: AdminPoolKeySortField::Default,
field: AdminPoolKeySortField::ImportedAt,
direction: AdminPoolKeySortDirection::Desc,
}
}
@@ -117,7 +117,8 @@ pub(crate) fn parse_admin_pool_key_sort(query: Option<&str>) -> Result<AdminPool
.filter(|value| !value.is_empty())
.as_deref()
{
None | Some("default") | Some("name") => AdminPoolKeySortField::Default,
None | Some("default") => AdminPoolKeySortField::ImportedAt,
Some("name") => AdminPoolKeySortField::Default,
Some("imported_at") | Some("created_at") => AdminPoolKeySortField::ImportedAt,
Some("last_used_at") | Some("last_used") => AdminPoolKeySortField::LastUsedAt,
Some(_) => {

7
apps/aether-gateway/src/handlers/admin/provider/shared/paths/endpoint_keys.rs
View File

@@ -33,6 +33,13 @@ pub(crate) fn admin_clear_oauth_invalid_key_id(request_path: &str) -> Option<Str
.map(ToOwned::to_owned)
}
pub(crate) fn admin_reset_cycle_stats_key_id(request_path: &str) -> Option<String> {
request_path
.strip_prefix("/api/admin/endpoints/keys/")?
.strip_suffix("/reset-cycle-stats")
.map(ToOwned::to_owned)
}
pub(crate) fn admin_update_key_id(request_path: &str) -> Option<String> {
let key_id = request_path.strip_prefix("/api/admin/endpoints/keys/")?;
(!key_id.is_empty() && !key_id.contains('/')).then_some(key_id.to_string())

3
apps/aether-gateway/src/handlers/admin/provider/shared/paths/mod.rs
View File

@@ -16,7 +16,8 @@ pub(crate) use self::crud::{
};
pub(crate) use self::endpoint_keys::{
admin_clear_oauth_invalid_key_id, admin_export_key_id, admin_provider_id_for_keys,
admin_provider_id_for_refresh_quota, admin_reveal_key_id, admin_update_key_id,
admin_provider_id_for_refresh_quota, admin_reset_cycle_stats_key_id, admin_reveal_key_id,
admin_update_key_id,
};
pub(crate) use self::oauth::{
admin_provider_oauth_batch_import_provider_id, admin_provider_oauth_batch_import_task_path,

114
apps/aether-gateway/src/handlers/shared/catalog.rs
View File

@@ -556,6 +556,52 @@ fn quota_windows_all_exhausted(windows: &[Value]) -> bool {
total > 0 && exhausted == total
}
fn preserve_quota_window_usage_reset_at(
current_status_snapshot: Option<&Value>,
quota: &mut Value,
) {
let Some(current_windows) = current_status_snapshot
.and_then(Value::as_object)
.and_then(|snapshot| snapshot.get("quota"))
.and_then(Value::as_object)
.and_then(|quota| quota.get("windows"))
.and_then(Value::as_array)
else {
return;
};
let Some(next_windows) = quota.get_mut("windows").and_then(Value::as_array_mut) else {
return;
};
for next_window in next_windows.iter_mut().filter_map(Value::as_object_mut) {
let Some(code) = next_window
.get("code")
.and_then(Value::as_str)
.map(str::trim)
.filter(|code| !code.is_empty())
else {
continue;
};
let Some(usage_reset_at) = current_windows
.iter()
.filter_map(Value::as_object)
.find(|current_window| {
current_window
.get("code")
.and_then(Value::as_str)
.map(str::trim)
.is_some_and(|current_code| current_code.eq_ignore_ascii_case(code))
})
.and_then(|current_window| current_window.get("usage_reset_at"))
.and_then(admin_provider_quota_pure::coerce_json_u64)
else {
continue;
};
next_window.insert("usage_reset_at".to_string(), json!(usage_reset_at));
}
}
fn codex_quota_window_snapshot(
metadata: &Map<String, Value>,
prefix: &str,
@@ -1107,7 +1153,7 @@ pub(crate) fn sync_provider_key_quota_status_snapshot(
source: &str,
) -> Option<Value> {
let normalized_provider_type = provider_type.trim().to_ascii_lowercase();
let quota = match normalized_provider_type.as_str() {
let mut quota = match normalized_provider_type.as_str() {
"codex" => build_codex_quota_status_snapshot(upstream_metadata, source),
"kiro" => build_kiro_quota_status_snapshot(upstream_metadata, source),
"chatgpt_web" => build_chatgpt_web_quota_status_snapshot(upstream_metadata, source),
@@ -1115,6 +1161,9 @@ pub(crate) fn sync_provider_key_quota_status_snapshot(
"gemini_cli" => build_gemini_cli_quota_status_snapshot(upstream_metadata, source),
_ => None,
}?;
if normalized_provider_type == "codex" {
preserve_quota_window_usage_reset_at(status_snapshot, &mut quota);
}
let default_snapshot = default_provider_key_status_snapshot();
let mut snapshot = provider_key_status_snapshot_object(status_snapshot)
@@ -2012,6 +2061,69 @@ mod tests {
);
}
#[test]
fn sync_provider_key_quota_status_snapshot_preserves_codex_usage_reset_at() {
let current_status_snapshot = json!({
"quota": {
"version": 2,
"provider_type": "codex",
"windows": [
{
"code": "weekly",
"usage_reset_at": 1_775_600_000u64,
"reset_at": 1_900_000_000u64,
"window_minutes": 10_080u64
},
{
"code": "5h",
"usage_reset_at": 1_775_700_000u64,
"reset_at": 1_900_500_000u64,
"window_minutes": 300u64
}
]
}
});
let upstream_metadata = json!({
"codex": {
"updated_at": 1_775_800_000u64,
"plan_type": "plus",
"primary_used_percent": 5.0,
"primary_reset_at": 1_901_000_000u64,
"primary_window_minutes": 10_080u64,
"secondary_used_percent": 1.0,
"secondary_reset_at": 1_901_500_000u64,
"secondary_window_minutes": 300u64
}
});
let payload = sync_provider_key_quota_status_snapshot(
Some(&current_status_snapshot),
"codex",
Some(&upstream_metadata),
"refresh_api",
)
.expect("quota snapshot should sync");
let windows = payload
.get("quota")
.and_then(Value::as_object)
.and_then(|quota| quota.get("windows"))
.and_then(Value::as_array)
.expect("quota windows should exist");
let weekly = windows
.iter()
.filter_map(Value::as_object)
.find(|window| window.get("code") == Some(&json!("weekly")))
.expect("weekly window should exist");
let five_h = windows
.iter()
.filter_map(Value::as_object)
.find(|window| window.get("code") == Some(&json!("5h")))
.expect("5h window should exist");
assert_eq!(weekly.get("usage_reset_at"), Some(&json!(1_775_600_000u64)));
assert_eq!(five_h.get("usage_reset_at"), Some(&json!(1_775_700_000u64)));
}
#[test]
fn provider_key_status_snapshot_payload_backfills_thin_ok_snapshot_from_upstream_metadata() {
let mut key = sample_catalog_key();

8
apps/aether-gateway/src/orchestration/effects.rs
View File

@@ -731,6 +731,10 @@ fn local_candidate_failure_should_record_pool_error(
classification: LocalFailoverClassification,
status_code: u16,
) -> bool {
if status_code == 400 {
return false;
}
local_candidate_failure_should_invalidate_affinity(classification, status_code)
}
@@ -1441,6 +1445,10 @@ mod tests {
LocalFailoverClassification::StopErrorPattern,
400,
));
assert!(!local_candidate_failure_should_record_pool_error(
LocalFailoverClassification::RetryUpstreamFailure,
400,
));
assert!(local_candidate_failure_should_record_pool_error(
LocalFailoverClassification::RetryUpstreamFailure,
429,

15
apps/aether-gateway/src/orchestration/health.rs
View File

@@ -144,6 +144,9 @@ fn local_candidate_failure_should_project_health(
if status_code < 400 {
return false;
}
if status_code == 400 {
return false;
}
match classification {
LocalFailoverClassification::RetrySuccessPattern
@@ -216,6 +219,18 @@ mod tests {
.is_none());
}
#[test]
fn failure_projection_ignores_client_bad_request() {
assert!(project_local_failure_health(
None,
"openai:chat",
LocalFailoverClassification::RetryUpstreamFailure,
400,
1_760_000_000,
)
.is_none());
}
#[test]
fn success_projection_resets_only_target_format() {
let projected = project_local_success_health(

6
apps/aether-gateway/src/scheduler/candidate/runtime.rs
View File

@@ -366,6 +366,12 @@ fn oauth_invalid_reason_is_hard_account_block(
provider_type: &str,
invalid_reason: &str,
) -> bool {
if provider_type.trim().eq_ignore_ascii_case("kiro")
&& invalid_reason.trim().starts_with("[REFRESH_FAILED] ")
{
return true;
}
let account_state = admin_provider_status_pure::resolve_pool_account_state(
Some(provider_type),
key.upstream_metadata.as_ref(),

9
apps/aether-gateway/src/scheduler/candidate/tests/selection.rs
View File

@@ -1971,7 +1971,7 @@ async fn keeps_refreshable_kiro_candidate_selectable_when_oauth_token_expired()
}
#[tokio::test]
async fn keeps_kiro_candidate_selectable_after_refresh_failure() {
async fn skips_kiro_candidate_after_refresh_token_failure() {
let mut row = sample_row();
row.provider_id = "provider-kiro".to_string();
row.provider_name = "kiro".to_string();
@@ -2028,9 +2028,10 @@ async fn keeps_kiro_candidate_selectable_after_refresh_failure() {
.await
.expect("selection should succeed");
assert_eq!(selected.len(), 1);
assert_eq!(selected[0].provider_id, "provider-kiro");
assert!(skipped.is_empty());
assert!(selected.is_empty());
assert_eq!(skipped.len(), 1);
assert_eq!(skipped[0].candidate.key_id, "key-kiro");
assert_eq!(skipped[0].skip_reason, "oauth_invalid");
}
#[tokio::test]

22
apps/aether-gateway/src/state/oauth.rs
View File

@@ -151,8 +151,13 @@ fn oauth_invalid_reason_is_account_block(reason: Option<&str>) -> bool {
if reason.starts_with(OAUTH_ACCOUNT_BLOCK_PREFIX) {
return true;
}
aether_admin::provider::status::resolve_account_status_snapshot(None, None, Some(reason))
.blocked
let snapshot =
aether_admin::provider::status::resolve_account_status_snapshot(None, None, Some(reason));
snapshot.blocked
&& !matches!(
snapshot.code.trim().to_ascii_lowercase().as_str(),
"oauth_token_invalid" | "oauth_expired" | "oauth_refresh_failed"
)
}
fn normalize_local_oauth_refresh_error_message(
@@ -270,7 +275,7 @@ fn merge_local_oauth_refresh_failure_reason(
return Some(refresh_reason.to_string());
}
if current_reason.starts_with(OAUTH_EXPIRED_PREFIX) {
return None;
return Some(refresh_reason.to_string());
}
if oauth_invalid_reason_is_account_block(Some(current_reason)) {
return None;
@@ -1681,4 +1686,15 @@ mod tests {
"refresh_token 无效、已过期或已撤销，请重新登录授权"
);
}
#[test]
fn local_refresh_failure_replaces_access_token_expired_marker() {
assert_eq!(
super::merge_local_oauth_refresh_failure_reason(
Some("[OAUTH_EXPIRED] access token invalid"),
"[REFRESH_FAILED] Token 续期失败 (401): refresh_token 无效",
),
Some("[REFRESH_FAILED] Token 续期失败 (401): refresh_token 无效".to_string()),
);
}
}

22
apps/aether-gateway/src/tests/control/admin/pool.rs
View File

@@ -798,6 +798,28 @@ async fn gateway_sorts_admin_pool_keys_by_imported_and_last_used_time() {
provider_catalog_repository,
));
let default_response = local_admin_pool_response(
&state,
http::Method::GET,
"/api/admin/pool/provider-openai/keys?page=1&page_size=50&status=all",
None,
)
.await;
assert_eq!(default_response.status(), StatusCode::OK);
let default_payload: serde_json::Value = serde_json::from_slice(
&to_bytes(default_response.into_body(), usize::MAX)
.await
.expect("body should read"),
)
.expect("json body should parse");
let default_names = default_payload["keys"]
.as_array()
.expect("keys should be array")
.iter()
.map(|item| item["key_name"].as_str().unwrap_or_default())
.collect::<Vec<_>>();
assert_eq!(default_names, vec!["fresh", "active", "old"]);
let imported_response = local_admin_pool_response(
&state,
http::Method::GET,