i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 17:30:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(gateway): 修复 balance 刷新去重键冲突、usage 状态回退与测试竞态

Browse Source 
- balance_cache: 引入实例级 refresh key 防止多实例共享进程级 HashSet 冲突
- InMemoryUsageRepo: 阻止 pending/streaming 状态覆盖已终结(completed/failed/cancelled)记录
- usage 同步测试: 等待条件从 is_some() 改为检查 status=="completed" 避免竞态
- wallet 测试: 增加轮询等待 wallet 扣款完成
- 整理 import 语句与 tests 模块位置
This commit is contained in:
fawney19
2026-04-13 16:48:05 +08:00
parent 9a376e4223
commit 4fd2b4a014
13 changed files with 266 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
apps/aether-gateway/src/handlers/admin/observability/usage/analytics/mod.rs
View File

@@ -4,4 +4,5 @@ mod filters;
pub(super) use aggregations::admin_usage_aggregation_by_user_json;
pub(super) use cache_affinity::list_recent_completed_usage_for_cache_affinity;
pub(super) use filters::{admin_usage_api_key_names, admin_usage_provider_key_names};
pub(super) use filters::admin_usage_api_key_names;
pub(super) use filters::admin_usage_provider_key_names;

3
apps/aether-gateway/src/handlers/admin/observability/usage/detail_routes.rs
View File

@@ -1,4 +1,5 @@
use super::analytics::{admin_usage_api_key_names, admin_usage_provider_key_names};
use super::analytics::admin_usage_api_key_names;
use super::analytics::admin_usage_provider_key_names;
use super::replay::{
admin_usage_curl_headers, admin_usage_curl_url, admin_usage_headers_from_value,
admin_usage_id_from_action_path, admin_usage_id_from_detail_path,

68
apps/aether-gateway/src/handlers/admin/observability/usage/replay.rs
View File

@@ -107,40 +107,6 @@ pub(super) fn build_admin_usage_curl_response(
)
}
#[cfg(test)]
mod tests {
use super::admin_usage_body_value_from_sources;
use serde_json::json;
#[test]
fn resolved_reference_body_wins_over_inline_fallback() {
let inline_body = json!({
"truncated": true,
"reason": "usage_capture_limits_exceeded"
});
let ref_body = json!({
"messages": [{"role": "user", "content": "real request body"}]
});
assert_eq!(
admin_usage_body_value_from_sources(Some(ref_body.clone()), Some(&inline_body)),
Some(ref_body)
);
}
#[test]
fn inline_body_is_used_when_reference_body_is_unavailable() {
let inline_body = json!({
"messages": [{"role": "user", "content": "fallback inline body"}]
});
assert_eq!(
admin_usage_body_value_from_sources(None, Some(&inline_body)),
Some(inline_body)
);
}
}
pub(super) fn build_admin_usage_detail_payload(
item: &StoredRequestUsageAudit,
users_by_id: &BTreeMap<String, aether_data::repository::users::StoredUserSummary>,
@@ -415,3 +381,37 @@ pub(super) fn admin_usage_build_curl_command(
) -> String {
aether_admin::observability::usage::admin_usage_build_curl_command(url, headers, body)
}
#[cfg(test)]
mod tests {
use super::admin_usage_body_value_from_sources;
use serde_json::json;
#[test]
fn resolved_reference_body_wins_over_inline_fallback() {
let inline_body = json!({
"truncated": true,
"reason": "usage_capture_limits_exceeded"
});
let ref_body = json!({
"messages": [{"role": "user", "content": "real request body"}]
});
assert_eq!(
admin_usage_body_value_from_sources(Some(ref_body.clone()), Some(&inline_body)),
Some(ref_body)
);
}
#[test]
fn inline_body_is_used_when_reference_body_is_unavailable() {
let inline_body = json!({
"messages": [{"role": "user", "content": "fallback inline body"}]
});
assert_eq!(
admin_usage_body_value_from_sources(None, Some(&inline_body)),
Some(inline_body)
);
}
}

3
apps/aether-gateway/src/handlers/admin/observability/usage/summary_routes.rs
View File

@@ -1,5 +1,6 @@
use super::super::stats::{AdminStatsTimeRange, AdminStatsUsageFilter};
use super::analytics::{admin_usage_api_key_names, admin_usage_provider_key_names};
use super::analytics::admin_usage_api_key_names;
use super::analytics::admin_usage_provider_key_names;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::query_param_value;
use crate::GatewayError;

31
apps/aether-gateway/src/handlers/admin/provider/ops/providers/balance_cache.rs
View File

@@ -7,6 +7,7 @@ use tokio::sync::{Mutex, Semaphore};
use tracing::{debug, warn};
const ADMIN_PROVIDER_OPS_BALANCE_CACHE_PREFIX: &str = "provider_ops:balance:";
const ADMIN_PROVIDER_OPS_BALANCE_REFRESH_PREFIX: &str = "provider_ops:balance_refresh:";
const ADMIN_PROVIDER_OPS_BALANCE_CACHE_TTL_SECS: u64 = 86_400;
const ADMIN_PROVIDER_OPS_BALANCE_AUTH_FAILED_CACHE_TTL_SECS: u64 = 60;
const ADMIN_PROVIDER_OPS_BALANCE_REFRESH_CONCURRENCY: usize = 3;
@@ -139,8 +140,9 @@ pub(super) async fn spawn_admin_provider_ops_balance_refresh(
state: &AdminAppState<'_>,
provider_id: &str,
) {
let refresh_key = admin_provider_ops_balance_refresh_key(state, provider_id);
let mut guard = ADMIN_PROVIDER_OPS_REFRESHING_PROVIDERS.lock().await;
if !guard.insert(provider_id.to_string()) {
if !guard.insert(refresh_key.clone()) {
debug!(provider_id, "provider ops balance refresh already running");
return;
}
@@ -162,12 +164,12 @@ pub(super) async fn spawn_admin_provider_ops_balance_refresh(
error = %err,
"provider ops balance refresh semaphore closed"
);
finish_refresh_provider(&provider_id).await;
finish_refresh_provider(&refresh_key).await;
return;
}
Err(_) => {
debug!(provider_id = %provider_id, "provider ops balance refresh skipped by concurrency limit");
finish_refresh_provider(&provider_id).await;
finish_refresh_provider(&refresh_key).await;
return;
}
};
@@ -186,7 +188,7 @@ pub(super) async fn spawn_admin_provider_ops_balance_refresh(
"failed to load provider for balance refresh"
);
drop(permit);
finish_refresh_provider(&provider_id).await;
finish_refresh_provider(&refresh_key).await;
return;
}
};
@@ -204,7 +206,7 @@ pub(super) async fn spawn_admin_provider_ops_balance_refresh(
"failed to load endpoints for balance refresh"
);
drop(permit);
finish_refresh_provider(&provider_id).await;
finish_refresh_provider(&refresh_key).await;
return;
}
}
@@ -223,7 +225,7 @@ pub(super) async fn spawn_admin_provider_ops_balance_refresh(
.await;
store_admin_provider_ops_balance_cache(&admin_state, &provider_id, &payload).await;
drop(permit);
finish_refresh_provider(&provider_id).await;
finish_refresh_provider(&refresh_key).await;
});
}
@@ -239,11 +241,24 @@ fn balance_cache_ttl_seconds(payload: &Value) -> Option<u64> {
}
}
async fn finish_refresh_provider(provider_id: &str) {
fn admin_provider_ops_balance_refresh_key(state: &AdminAppState<'_>, provider_id: &str) -> String {
let raw_key = format!("{ADMIN_PROVIDER_OPS_BALANCE_REFRESH_PREFIX}{provider_id}");
if let Some(runner) = state.redis_kv_runner() {
format!(
"{:p}:{}",
state.app(),
runner.keyspace().key(raw_key.as_str())
)
} else {
format!("{:p}:{raw_key}", state.app())
}
}
async fn finish_refresh_provider(refresh_key: &str) {
ADMIN_PROVIDER_OPS_REFRESHING_PROVIDERS
.lock()
.await
.remove(provider_id);
.remove(refresh_key);
}
fn admin_provider_ops_action_response(

5
apps/aether-gateway/src/handlers/admin/provider/write/provider/create.rs
View File

@@ -2,9 +2,8 @@ use crate::handlers::admin::provider::shared::payloads::AdminProviderCreateReque
use crate::handlers::admin::provider::shared::support::{
normalize_provider_billing_type, parse_optional_rfc3339_unix_secs,
};
use crate::handlers::admin::provider::write::normalize::{
normalize_pool_advanced_config, normalize_provider_type_input,
};
use crate::handlers::admin::provider::write::normalize::normalize_pool_advanced_config;
use crate::handlers::admin::provider::write::normalize::normalize_provider_type_input;
use crate::handlers::admin::request::AdminAppState;
use crate::handlers::admin::shared::normalize_json_object;
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogProvider;

5
apps/aether-gateway/src/handlers/admin/provider/write/provider/update.rs
View File

@@ -2,9 +2,8 @@ use crate::handlers::admin::provider::shared::payloads::AdminProviderUpdatePatch
use crate::handlers::admin::provider::shared::support::{
normalize_provider_billing_type, parse_optional_rfc3339_unix_secs,
};
use crate::handlers::admin::provider::write::normalize::{
normalize_pool_advanced_config, normalize_provider_type_input,
};
use crate::handlers::admin::provider::write::normalize::normalize_pool_advanced_config;
use crate::handlers::admin::provider::write::normalize::normalize_provider_type_input;
use crate::handlers::admin::request::AdminAppState;
use crate::handlers::admin::shared::normalize_json_object;
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogProvider;