i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-03 01:40:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 大规模模块拆分与重组，新增 aether-admin crate

Browse Source 
- 新建独立 aether-admin crate 承载 admin 相关共享契约与纯辅助函数
- 拆分 ai_pipeline 下 kiro/private_envelope/conversion/planner 等大文件为子模块目录
- 重组 admin handlers 各业务域（billing/oauth/provider/system/users 等）为目录结构，移除 shared.rs/builders.rs 等反模式
- 移除 ai_pipeline runtime adapters 旧实现（claude/openai/gemini/kiro/vertex/antigravity 等），改由 provider transport 统一承载
- 移除 control_facade/execution_facade/auth_snapshot_facade 等冗余 facade 层
- 拆分 query/billing 与 query/monitoring 模块、state/runtime/payments 与 security 模块
- 扩展架构测试覆盖 admin_billing/admin_model/admin_users 等新模块
- 删除 docs/architecture/refactor-execution-plan.md 已完成的执行计划文档
This commit is contained in:
fawney19
2026-04-09 00:10:38 +08:00
parent 4fb9882b54
commit 4fc95adfb9
663 changed files with 48471 additions and 40232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
apps/aether-gateway/src/handlers/admin/observability/mod.rs
View File

@@ -1,7 +1,13 @@
mod monitoring;
pub(crate) mod stats;
mod routes;
mod stats;
mod usage;
pub(crate) use self::monitoring::maybe_build_local_admin_monitoring_response;
pub(crate) use self::stats::maybe_build_local_admin_stats_response;
pub(super) use self::monitoring::maybe_build_local_admin_monitoring_response;
pub(super) use self::routes::maybe_build_local_admin_observability_response;
pub(crate) use self::stats::{
admin_stats_bad_request_response, list_usage_for_optional_range, list_usage_for_range,
maybe_build_local_admin_stats_response, parse_bounded_u32, round_to,
};
pub(crate) use self::stats::{AdminStatsTimeRange, AdminStatsUsageFilter};
pub(crate) use self::usage::maybe_build_local_admin_usage_response;

185
apps/aether-gateway/src/handlers/admin/observability/monitoring/activity.rs
View File

@@ -1,4 +1,3 @@
use super::responses::admin_monitoring_bad_request_response;
use super::route_filters::{
admin_monitoring_escape_like_pattern, parse_admin_monitoring_days,
parse_admin_monitoring_event_type_filter, parse_admin_monitoring_hours,
@@ -7,102 +6,24 @@ use super::route_filters::{
};
use super::usage_helpers::admin_monitoring_usage_is_error;
use crate::constants::INTERNAL_GATEWAY_PATH_PREFIXES;
use crate::control::GatewayPublicRequestContext;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::query::monitoring as monitoring_query;
use crate::{AppState, GatewayError};
use aether_data_contracts::repository::usage::UsageAuditListQuery;
use axum::{
body::Body,
response::{IntoResponse, Response},
Json,
use crate::GatewayError;
use aether_admin::observability::monitoring::{
admin_monitoring_bad_request_response, admin_monitoring_user_behavior_user_id_from_path,
build_admin_monitoring_audit_logs_payload_response,
build_admin_monitoring_suspicious_activities_payload_response,
build_admin_monitoring_system_status_payload_response,
build_admin_monitoring_user_behavior_payload_response,
};
use serde_json::json;
fn build_admin_monitoring_audit_logs_payload(
items: Vec<serde_json::Value>,
total: usize,
limit: usize,
offset: usize,
username: Option<String>,
event_type: Option<String>,
days: i64,
) -> Response<Body> {
let count = items.len();
Json(json!({
"items": items,
"meta": {
"total": total,
"limit": limit,
"offset": offset,
"count": count,
},
"filters": {
"username": username,
"event_type": event_type,
"days": days,
},
}))
.into_response()
}
fn build_admin_monitoring_suspicious_activities_payload(
activities: Vec<serde_json::Value>,
hours: i64,
) -> Response<Body> {
let count = activities.len();
Json(json!({
"activities": activities,
"count": count,
"time_range_hours": hours,
}))
.into_response()
}
fn build_admin_monitoring_user_behavior_payload(
user_id: String,
days: i64,
event_counts: std::collections::BTreeMap<String, u64>,
failed_requests: u64,
success_requests: u64,
suspicious_activities: u64,
) -> Response<Body> {
let total_requests = success_requests.saturating_add(failed_requests);
let success_rate = if total_requests == 0 {
0.0
} else {
success_requests as f64 / total_requests as f64
};
Json(json!({
"user_id": user_id,
"period_days": days,
"event_counts": event_counts,
"failed_requests": failed_requests,
"success_requests": success_requests,
"success_rate": success_rate,
"suspicious_activities": suspicious_activities,
"analysis_time": chrono::Utc::now().to_rfc3339(),
}))
.into_response()
}
fn admin_monitoring_user_behavior_user_id_from_path(request_path: &str) -> Option<String> {
let value = request_path
.strip_prefix("/api/admin/monitoring/user-behavior/")?
.trim()
.trim_matches('/')
.to_string();
if value.is_empty() || value.contains('/') {
None
} else {
Some(value)
}
}
use aether_data_contracts::repository::usage::UsageAuditListQuery;
use axum::{body::Body, response::Response};
pub(super) async fn build_admin_monitoring_audit_logs_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
let state = state.as_ref();
let query = request_context.request_query_string.as_deref();
let username = parse_admin_monitoring_username_filter(query);
let event_type = parse_admin_monitoring_event_type_filter(query);
@@ -120,7 +41,7 @@ pub(super) async fn build_admin_monitoring_audit_logs_response(
};
let Some(pool) = state.postgres_pool() else {
return Ok(build_admin_monitoring_audit_logs_payload(
return Ok(build_admin_monitoring_audit_logs_payload_response(
Vec::new(),
0,
limit,
@@ -147,15 +68,16 @@ pub(super) async fn build_admin_monitoring_audit_logs_response(
)
.await?;
Ok(build_admin_monitoring_audit_logs_payload(
Ok(build_admin_monitoring_audit_logs_payload_response(
items, total, limit, offset, username, event_type, days,
))
}
pub(super) async fn build_admin_monitoring_suspicious_activities_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
let state = state.as_ref();
let query = request_context.request_query_string.as_deref();
let hours = match parse_admin_monitoring_hours(query) {
Ok(value) => value,
@@ -163,24 +85,22 @@ pub(super) async fn build_admin_monitoring_suspicious_activities_response(
};
let Some(pool) = state.postgres_pool() else {
return Ok(build_admin_monitoring_suspicious_activities_payload(
Vec::new(),
hours,
));
return Ok(
build_admin_monitoring_suspicious_activities_payload_response(Vec::new(), hours),
);
};
let cutoff_time = chrono::Utc::now() - chrono::Duration::hours(hours);
let activities = monitoring_query::list_admin_suspicious_activities(&pool, cutoff_time).await?;
Ok(build_admin_monitoring_suspicious_activities_payload(
activities, hours,
))
Ok(build_admin_monitoring_suspicious_activities_payload_response(activities, hours))
}
pub(super) async fn build_admin_monitoring_user_behavior_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
let state = state.as_ref();
let Some(user_id) =
admin_monitoring_user_behavior_user_id_from_path(&request_context.request_path)
else {
@@ -192,7 +112,7 @@ pub(super) async fn build_admin_monitoring_user_behavior_response(
};
let Some(pool) = state.postgres_pool() else {
return Ok(build_admin_monitoring_user_behavior_payload(
return Ok(build_admin_monitoring_user_behavior_payload_response(
user_id,
days,
std::collections::BTreeMap::new(),
@@ -227,7 +147,7 @@ pub(super) async fn build_admin_monitoring_user_behavior_response(
.unwrap_or_default(),
);
Ok(build_admin_monitoring_user_behavior_payload(
Ok(build_admin_monitoring_user_behavior_payload_response(
user_id,
days,
event_counts,
@@ -238,8 +158,9 @@ pub(super) async fn build_admin_monitoring_user_behavior_response(
}
pub(super) async fn build_admin_monitoring_system_status_response(
state: &AppState,
state: &AdminAppState<'_>,
) -> Result<Response<Body>, GatewayError> {
let state = state.as_ref();
let now = chrono::Utc::now();
let today_start = now
.date_naive()
@@ -301,35 +222,21 @@ pub(super) async fn build_admin_monitoring_system_status_response(
.count();
let tunnel = state.tunnel.stats();
Ok(Json(json!({
"timestamp": now.to_rfc3339(),
"users": {
"total": total_users,
"active": active_users,
},
"providers": {
"total": total_providers,
"active": active_providers,
},
"api_keys": {
"total": total_api_keys,
"active": active_api_keys,
},
"today_stats": {
"requests": today_requests,
"tokens": today_tokens,
"cost_usd": format!("${today_cost:.4}"),
},
"tunnel": {
"proxy_connections": tunnel.proxy_connections,
"nodes": tunnel.nodes,
"active_streams": tunnel.active_streams,
},
"internal_gateway": {
"status": "rust_native_control_plane",
"path_prefixes": INTERNAL_GATEWAY_PATH_PREFIXES,
},
"recent_errors": recent_errors,
}))
.into_response())
Ok(build_admin_monitoring_system_status_payload_response(
now,
total_users,
active_users,
total_providers,
active_providers,
total_api_keys,
active_api_keys,
today_requests,
today_tokens,
today_cost,
tunnel.proxy_connections,
tunnel.nodes,
tunnel.active_streams,
INTERNAL_GATEWAY_PATH_PREFIXES,
recent_errors,
))
}

7
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache.rs
View File

@@ -8,7 +8,8 @@ use super::cache_config::{
ADMIN_MONITORING_DYNAMIC_RESERVATION_STABLE_MIN_RESERVATION,
};
use super::cache_store::build_admin_monitoring_cache_snapshot;
use crate::{AppState, GatewayError};
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
use axum::{
body::Body,
http,
@@ -18,7 +19,7 @@ use axum::{
use serde_json::json;
pub(super) async fn build_admin_monitoring_cache_stats_response(
state: &AppState,
state: &AdminAppState<'_>,
) -> Result<Response<Body>, GatewayError> {
let snapshot = build_admin_monitoring_cache_snapshot(state).await?;
@@ -64,7 +65,7 @@ pub(super) async fn build_admin_monitoring_cache_stats_response(
}
pub(super) async fn build_admin_monitoring_cache_metrics_response(
state: &AppState,
state: &AdminAppState<'_>,
) -> Result<Response<Body>, GatewayError> {
let snapshot = build_admin_monitoring_cache_snapshot(state).await?;
let metrics = [

19
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_affinity.rs
View File

@@ -1,5 +1,6 @@
use super::cache_types::AdminMonitoringCacheAffinityRecord;
use crate::{AppState, GatewayError};
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
fn parse_admin_monitoring_cache_affinity_key(raw_key: &str) -> Option<(String, String, String)> {
let parts = raw_key.split(':').collect::<Vec<_>>();
@@ -92,7 +93,7 @@ pub(super) fn admin_monitoring_cache_affinity_record(
}
pub(super) fn clear_admin_monitoring_scheduler_affinity_entries(
state: &AppState,
state: &AdminAppState<'_>,
records: &[AdminMonitoringCacheAffinityRecord],
) {
let scheduler_keys = records
@@ -100,28 +101,32 @@ pub(super) fn clear_admin_monitoring_scheduler_affinity_entries(
.filter_map(admin_monitoring_scheduler_affinity_cache_key)
.collect::<std::collections::BTreeSet<_>>();
for scheduler_key in scheduler_keys {
let _ = state.remove_scheduler_affinity_cache_entry(&scheduler_key);
let _ = state
.as_ref()
.remove_scheduler_affinity_cache_entry(&scheduler_key);
}
}
#[cfg(test)]
pub(super) fn delete_admin_monitoring_cache_affinity_entries_for_tests(
state: &AppState,
state: &AdminAppState<'_>,
raw_keys: &[String],
) -> usize {
state.remove_admin_monitoring_cache_affinity_entries_for_tests(raw_keys)
state
.as_ref()
.remove_admin_monitoring_cache_affinity_entries_for_tests(raw_keys)
}
#[cfg(not(test))]
pub(super) fn delete_admin_monitoring_cache_affinity_entries_for_tests(
_state: &AppState,
_state: &AdminAppState<'_>,
_raw_keys: &[String],
) -> usize {
0
}
pub(super) async fn delete_admin_monitoring_cache_affinity_raw_keys(
state: &AppState,
state: &AdminAppState<'_>,
raw_keys: &[String],
) -> Result<usize, GatewayError> {
if raw_keys.is_empty() {

30
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_affinity_reads.rs
View File

@@ -15,10 +15,10 @@ use super::cache_store::{
list_admin_monitoring_cache_affinity_records,
list_admin_monitoring_cache_affinity_records_by_affinity_keys,
};
use super::responses::admin_monitoring_bad_request_response;
use super::route_filters::{parse_admin_monitoring_limit, parse_admin_monitoring_offset};
use crate::control::GatewayPublicRequestContext;
use crate::{AppState, GatewayError};
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use aether_admin::observability::monitoring::admin_monitoring_bad_request_response;
use axum::{
body::Body,
response::{IntoResponse, Response},
@@ -31,8 +31,8 @@ fn normalize_keyword<'a>(keyword: Option<&'a String>) -> Option<String> {
}
pub(super) async fn build_admin_monitoring_cache_affinities_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
let limit = match parse_admin_monitoring_limit(request_context.request_query_string.as_deref())
{
@@ -106,26 +106,20 @@ pub(super) async fn build_admin_monitoring_cache_affinities_response(
.collect::<Vec<_>>();
let provider_by_id = state
.data
.list_provider_catalog_providers_by_ids(&provider_ids)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
.read_provider_catalog_providers_by_ids(&provider_ids)
.await?
.into_iter()
.map(|item| (item.id.clone(), item))
.collect::<std::collections::BTreeMap<_, _>>();
let endpoint_by_id = state
.data
.list_provider_catalog_endpoints_by_ids(&endpoint_ids)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
.read_provider_catalog_endpoints_by_ids(&endpoint_ids)
.await?
.into_iter()
.map(|item| (item.id.clone(), item))
.collect::<std::collections::BTreeMap<_, _>>();
let key_by_id = state
.data
.list_provider_catalog_keys_by_ids(&key_ids)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
.await?
.into_iter()
.map(|item| (item.id.clone(), item))
.collect::<std::collections::BTreeMap<_, _>>();
@@ -242,8 +236,8 @@ pub(super) async fn build_admin_monitoring_cache_affinities_response(
}
pub(super) async fn build_admin_monitoring_cache_affinity_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
let Some(user_identifier) =
admin_monitoring_cache_affinity_user_identifier_from_path(&request_context.request_path)

11
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_identity.rs
View File

@@ -1,8 +1,9 @@
use super::cache_types::AdminMonitoringCacheAffinityRecord;
use crate::{AppState, GatewayError};
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
pub(super) async fn admin_monitoring_list_export_api_key_records_by_ids(
state: &AppState,
state: &AdminAppState<'_>,
api_key_ids: &[String],
) -> Result<
std::collections::BTreeMap<String, aether_data::repository::auth::StoredAuthApiKeyExportRecord>,
@@ -21,7 +22,7 @@ pub(super) async fn admin_monitoring_list_export_api_key_records_by_ids(
}
async fn admin_monitoring_list_user_summaries_by_ids(
state: &AppState,
state: &AdminAppState<'_>,
user_ids: &[String],
) -> Result<
std::collections::BTreeMap<String, aether_data::repository::users::StoredUserSummary>,
@@ -40,7 +41,7 @@ async fn admin_monitoring_list_user_summaries_by_ids(
}
pub(super) async fn admin_monitoring_load_affinity_identity_maps(
state: &AppState,
state: &AdminAppState<'_>,
affinities: &[AdminMonitoringCacheAffinityRecord],
) -> Result<
(
@@ -71,7 +72,7 @@ pub(super) async fn admin_monitoring_load_affinity_identity_maps(
}
pub(super) async fn admin_monitoring_find_user_summary_by_id(
state: &AppState,
state: &AdminAppState<'_>,
user_id: &str,
) -> Result<Option<aether_data::repository::users::StoredUserSummary>, GatewayError> {
if user_id.trim().is_empty() {

6
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_model_mapping.rs
View File

@@ -2,7 +2,7 @@ use super::cache_config::ADMIN_MONITORING_REDIS_CACHE_CATEGORIES;
use super::cache_store::{
admin_monitoring_has_test_redis_keys, list_admin_monitoring_namespaced_keys,
};
use crate::AppState;
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
use axum::{
body::Body,
@@ -12,7 +12,7 @@ use axum::{
use serde_json::json;
pub(super) async fn build_admin_monitoring_model_mapping_stats_response(
state: &AppState,
state: &AdminAppState<'_>,
) -> Result<Response<Body>, GatewayError> {
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
return Ok(Json(json!({
@@ -67,7 +67,7 @@ pub(super) async fn build_admin_monitoring_model_mapping_stats_response(
}
pub(super) async fn build_admin_monitoring_redis_cache_categories_response(
state: &AppState,
state: &AdminAppState<'_>,
) -> Result<Response<Body>, GatewayError> {
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
return Ok(Json(json!({

400
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_mutations.rs
View File

@@ -1,400 +0,0 @@
use super::cache_affinity::{
clear_admin_monitoring_scheduler_affinity_entries,
delete_admin_monitoring_cache_affinity_raw_keys,
};
use super::cache_config::ADMIN_MONITORING_REDIS_CACHE_CATEGORIES;
use super::cache_identity::{
admin_monitoring_find_user_summary_by_id, admin_monitoring_list_export_api_key_records_by_ids,
};
use super::cache_route_helpers::{
admin_monitoring_cache_affinity_delete_params_from_path,
admin_monitoring_cache_affinity_unavailable_response,
admin_monitoring_cache_model_mapping_provider_params_from_path,
admin_monitoring_cache_model_name_from_path, admin_monitoring_cache_provider_id_from_path,
admin_monitoring_cache_redis_category_from_path,
admin_monitoring_cache_users_not_found_response,
admin_monitoring_cache_users_user_identifier_from_path,
admin_monitoring_redis_unavailable_response,
};
use super::cache_store::{
admin_monitoring_has_test_redis_keys, delete_admin_monitoring_namespaced_keys,
list_admin_monitoring_cache_affinity_records,
list_admin_monitoring_cache_affinity_records_by_affinity_keys,
list_admin_monitoring_namespaced_keys, load_admin_monitoring_cache_affinity_entries_for_tests,
};
use super::responses::{
admin_monitoring_bad_request_response, admin_monitoring_not_found_response,
};
use crate::control::GatewayPublicRequestContext;
use crate::{AppState, GatewayError};
use axum::{
body::Body,
http,
response::{IntoResponse, Response},
Json,
};
use serde_json::json;
pub(super) async fn build_admin_monitoring_cache_users_delete_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
) -> Result<Response<Body>, GatewayError> {
let Some(user_identifier) =
admin_monitoring_cache_users_user_identifier_from_path(&request_context.request_path)
else {
return Ok(admin_monitoring_bad_request_response(
"缺少 user_identifier",
));
};
if state.redis_kv_runner().is_none()
&& load_admin_monitoring_cache_affinity_entries_for_tests(state).is_empty()
{
return Ok(admin_monitoring_cache_affinity_unavailable_response());
}
let direct_api_key_by_id =
admin_monitoring_list_export_api_key_records_by_ids(state, &[user_identifier.clone()])
.await?;
if let Some(api_key) = direct_api_key_by_id.get(&user_identifier) {
let target_affinity_keys =
std::iter::once(user_identifier.clone()).collect::<std::collections::BTreeSet<_>>();
let target_affinities = list_admin_monitoring_cache_affinity_records_by_affinity_keys(
state,
&target_affinity_keys,
)
.await?;
let raw_keys = target_affinities
.iter()
.map(|item| item.raw_key.clone())
.collect::<Vec<_>>();
let _ = delete_admin_monitoring_cache_affinity_raw_keys(state, &raw_keys).await?;
clear_admin_monitoring_scheduler_affinity_entries(state, &target_affinities);
let user = admin_monitoring_find_user_summary_by_id(state, &api_key.user_id).await?;
let api_key_name = api_key
.name
.clone()
.unwrap_or_else(|| user_identifier.clone());
return Ok(Json(json!({
"status": "ok",
"message": format!("已清除 API Key {api_key_name} 的缓存亲和性"),
"user_info": {
"user_id": Some(api_key.user_id.clone()),
"username": user.as_ref().map(|item| item.username.clone()),
"email": user.and_then(|item| item.email),
"api_key_id": user_identifier,
"api_key_name": api_key.name.clone(),
},
}))
.into_response());
}
let Some(user) = state.find_user_auth_by_identifier(&user_identifier).await? else {
return Ok(admin_monitoring_cache_users_not_found_response(
&user_identifier,
));
};
let user_api_key_ids = state
.list_auth_api_key_export_records_by_user_ids(std::slice::from_ref(&user.id))
.await?
.into_iter()
.map(|item| item.api_key_id.clone())
.collect::<std::collections::BTreeSet<_>>();
let target_affinities =
list_admin_monitoring_cache_affinity_records_by_affinity_keys(state, &user_api_key_ids)
.await?;
let raw_keys = target_affinities
.iter()
.map(|item| item.raw_key.clone())
.collect::<Vec<_>>();
let _ = delete_admin_monitoring_cache_affinity_raw_keys(state, &raw_keys).await?;
clear_admin_monitoring_scheduler_affinity_entries(state, &target_affinities);
Ok(Json(json!({
"status": "ok",
"message": format!("已清除用户 {} 的所有缓存亲和性", user.username),
"user_info": {
"user_id": user.id,
"username": user.username,
"email": user.email,
},
}))
.into_response())
}
pub(super) async fn build_admin_monitoring_cache_affinity_delete_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
) -> Result<Response<Body>, GatewayError> {
let Some((affinity_key, endpoint_id, model_id, api_format)) =
admin_monitoring_cache_affinity_delete_params_from_path(&request_context.request_path)
else {
return Ok(admin_monitoring_bad_request_response(
"缺少 affinity_key、endpoint_id、model_id 或 api_format",
));
};
if state.redis_kv_runner().is_none()
&& load_admin_monitoring_cache_affinity_entries_for_tests(state).is_empty()
{
return Ok(admin_monitoring_cache_affinity_unavailable_response());
}
let target_affinity_keys =
std::iter::once(affinity_key.clone()).collect::<std::collections::BTreeSet<_>>();
let target_affinity =
list_admin_monitoring_cache_affinity_records_by_affinity_keys(state, &target_affinity_keys)
.await?
.into_iter()
.find(|item| {
item.affinity_key == affinity_key
&& item.endpoint_id.as_deref() == Some(endpoint_id.as_str())
&& item.model_name == model_id
&& item.api_format.eq_ignore_ascii_case(&api_format)
});
let Some(target_affinity) = target_affinity else {
return Ok(admin_monitoring_not_found_response(
"未找到指定的缓存亲和性记录",
));
};
let _ = delete_admin_monitoring_cache_affinity_raw_keys(
state,
std::slice::from_ref(&target_affinity.raw_key),
)
.await?;
clear_admin_monitoring_scheduler_affinity_entries(
state,
std::slice::from_ref(&target_affinity),
);
let mut api_key_by_id = admin_monitoring_list_export_api_key_records_by_ids(
state,
std::slice::from_ref(&affinity_key),
)
.await?;
let api_key_name = api_key_by_id
.remove(&affinity_key)
.and_then(|item| item.name)
.unwrap_or_else(|| affinity_key.chars().take(8).collect::<String>());
Ok(Json(json!({
"status": "ok",
"message": format!("已清除缓存亲和性: {api_key_name}"),
"affinity_key": affinity_key,
"endpoint_id": endpoint_id,
"model_id": model_id,
}))
.into_response())
}
pub(super) async fn build_admin_monitoring_cache_flush_response(
state: &AppState,
) -> Result<Response<Body>, GatewayError> {
let raw_affinities = list_admin_monitoring_cache_affinity_records(state).await?;
if state.redis_kv_runner().is_none() && raw_affinities.is_empty() {
return Ok(admin_monitoring_cache_affinity_unavailable_response());
}
let raw_keys = raw_affinities
.iter()
.map(|item| item.raw_key.clone())
.collect::<Vec<_>>();
let deleted = delete_admin_monitoring_cache_affinity_raw_keys(state, &raw_keys).await?;
clear_admin_monitoring_scheduler_affinity_entries(state, &raw_affinities);
Ok(Json(json!({
"status": "ok",
"message": "已清除全部缓存亲和性",
"deleted_affinities": deleted,
}))
.into_response())
}
pub(super) async fn build_admin_monitoring_cache_provider_delete_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
) -> Result<Response<Body>, GatewayError> {
let Some(provider_id) =
admin_monitoring_cache_provider_id_from_path(&request_context.request_path)
else {
return Ok(admin_monitoring_bad_request_response("缺少 provider_id"));
};
let raw_affinities = list_admin_monitoring_cache_affinity_records(state).await?;
if state.redis_kv_runner().is_none() && raw_affinities.is_empty() {
return Ok(admin_monitoring_cache_affinity_unavailable_response());
}
let target_affinities = raw_affinities
.into_iter()
.filter(|item| item.provider_id.as_deref() == Some(provider_id.as_str()))
.collect::<Vec<_>>();
if target_affinities.is_empty() {
return Ok((
http::StatusCode::NOT_FOUND,
Json(json!({
"detail": format!("未找到 provider {provider_id} 的缓存亲和性记录")
})),
)
.into_response());
}
let raw_keys = target_affinities
.iter()
.map(|item| item.raw_key.clone())
.collect::<Vec<_>>();
let deleted = delete_admin_monitoring_cache_affinity_raw_keys(state, &raw_keys).await?;
clear_admin_monitoring_scheduler_affinity_entries(state, &target_affinities);
Ok(Json(json!({
"status": "ok",
"message": format!("已清除 provider {provider_id} 的缓存亲和性"),
"provider_id": provider_id,
"deleted_affinities": deleted,
}))
.into_response())
}
pub(super) async fn build_admin_monitoring_model_mapping_delete_response(
state: &AppState,
) -> Result<Response<Body>, GatewayError> {
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
return Ok(admin_monitoring_redis_unavailable_response());
}
let mut raw_keys = list_admin_monitoring_namespaced_keys(state, "model:*").await?;
raw_keys.extend(list_admin_monitoring_namespaced_keys(state, "global_model:*").await?);
raw_keys.sort();
raw_keys.dedup();
let deleted_count = delete_admin_monitoring_namespaced_keys(state, &raw_keys).await?;
Ok(Json(json!({
"status": "ok",
"message": "已清除所有模型映射缓存",
"deleted_count": deleted_count,
}))
.into_response())
}
pub(super) async fn build_admin_monitoring_model_mapping_delete_model_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
) -> Result<Response<Body>, GatewayError> {
let Some(model_name) =
admin_monitoring_cache_model_name_from_path(&request_context.request_path)
else {
return Ok(admin_monitoring_bad_request_response("缺少 model_name"));
};
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
return Ok(admin_monitoring_redis_unavailable_response());
}
let candidate_keys = [
format!("global_model:resolve:{model_name}"),
format!("global_model:name:{model_name}"),
];
let mut existing_keys = Vec::new();
for key in candidate_keys {
let matches = list_admin_monitoring_namespaced_keys(state, key.as_str()).await?;
existing_keys.extend(matches);
}
existing_keys.sort();
existing_keys.dedup();
let deleted_count = delete_admin_monitoring_namespaced_keys(state, &existing_keys).await?;
let deleted_keys = if deleted_count == 0 {
Vec::new()
} else {
existing_keys
};
Ok(Json(json!({
"status": "ok",
"message": format!("已清除模型 {model_name} 的映射缓存"),
"model_name": model_name,
"deleted_keys": deleted_keys,
}))
.into_response())
}
pub(super) async fn build_admin_monitoring_model_mapping_delete_provider_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
) -> Result<Response<Body>, GatewayError> {
let Some((provider_id, global_model_id)) =
admin_monitoring_cache_model_mapping_provider_params_from_path(
&request_context.request_path,
)
else {
return Ok(admin_monitoring_bad_request_response(
"缺少 provider_id 或 global_model_id",
));
};
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
return Ok(admin_monitoring_redis_unavailable_response());
}
let candidate_keys = [
format!("model:provider_global:{provider_id}:{global_model_id}"),
format!("model:provider_global:hits:{provider_id}:{global_model_id}"),
];
let mut existing_keys = Vec::new();
for key in candidate_keys {
let matches = list_admin_monitoring_namespaced_keys(state, key.as_str()).await?;
existing_keys.extend(matches);
}
existing_keys.sort();
existing_keys.dedup();
let _ = delete_admin_monitoring_namespaced_keys(state, &existing_keys).await?;
Ok(Json(json!({
"status": "ok",
"message": "已清除 Provider 模型映射缓存",
"provider_id": provider_id,
"global_model_id": global_model_id,
"deleted_keys": existing_keys,
}))
.into_response())
}
pub(super) async fn build_admin_monitoring_redis_keys_delete_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
) -> Result<Response<Body>, GatewayError> {
let Some(category) =
admin_monitoring_cache_redis_category_from_path(&request_context.request_path)
else {
return Ok(admin_monitoring_bad_request_response("缺少 category"));
};
let Some((cat_key, name, pattern, _description)) = ADMIN_MONITORING_REDIS_CACHE_CATEGORIES
.iter()
.find(|(cat_key, _, _, _)| *cat_key == category)
else {
return Ok((
http::StatusCode::NOT_FOUND,
Json(json!({ "detail": format!("未知的缓存分类: {category}") })),
)
.into_response());
};
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
return Ok(admin_monitoring_redis_unavailable_response());
}
let raw_keys = list_admin_monitoring_namespaced_keys(state, pattern).await?;
let deleted_count = delete_admin_monitoring_namespaced_keys(state, &raw_keys).await?;
Ok(Json(json!({
"status": "ok",
"message": format!("已清除 {name} 缓存"),
"category": cat_key,
"deleted_count": deleted_count,
}))
.into_response())
}

86
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_mutations/affinity.rs Normal file
View File

@@ -0,0 +1,86 @@
use super::super::cache_affinity::{
clear_admin_monitoring_scheduler_affinity_entries,
delete_admin_monitoring_cache_affinity_raw_keys,
};
use super::super::cache_identity::admin_monitoring_list_export_api_key_records_by_ids;
use super::super::cache_route_helpers::{
admin_monitoring_cache_affinity_delete_params_from_path,
admin_monitoring_cache_affinity_unavailable_response,
};
use super::super::cache_store::{
list_admin_monitoring_cache_affinity_records_by_affinity_keys,
load_admin_monitoring_cache_affinity_entries_for_tests,
};
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use aether_admin::observability::monitoring::{
admin_monitoring_bad_request_response, admin_monitoring_not_found_response,
build_admin_monitoring_cache_affinity_delete_success_response,
};
use axum::{body::Body, response::Response};
pub(in super::super) async fn build_admin_monitoring_cache_affinity_delete_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
let Some((affinity_key, endpoint_id, model_id, api_format)) =
admin_monitoring_cache_affinity_delete_params_from_path(&request_context.request_path)
else {
return Ok(admin_monitoring_bad_request_response(
"缺少 affinity_key、endpoint_id、model_id 或 api_format",
));
};
if state.redis_kv_runner().is_none()
&& load_admin_monitoring_cache_affinity_entries_for_tests(state).is_empty()
{
return Ok(admin_monitoring_cache_affinity_unavailable_response());
}
let target_affinity_keys =
std::iter::once(affinity_key.clone()).collect::<std::collections::BTreeSet<_>>();
let target_affinity =
list_admin_monitoring_cache_affinity_records_by_affinity_keys(state, &target_affinity_keys)
.await?
.into_iter()
.find(|item| {
item.affinity_key == affinity_key
&& item.endpoint_id.as_deref() == Some(endpoint_id.as_str())
&& item.model_name == model_id
&& item.api_format.eq_ignore_ascii_case(&api_format)
});
let Some(target_affinity) = target_affinity else {
return Ok(admin_monitoring_not_found_response(
"未找到指定的缓存亲和性记录",
));
};
let _ = delete_admin_monitoring_cache_affinity_raw_keys(
state,
std::slice::from_ref(&target_affinity.raw_key),
)
.await?;
clear_admin_monitoring_scheduler_affinity_entries(
state,
std::slice::from_ref(&target_affinity),
);
let mut api_key_by_id = admin_monitoring_list_export_api_key_records_by_ids(
state,
std::slice::from_ref(&affinity_key),
)
.await?;
let api_key_name = api_key_by_id
.remove(&affinity_key)
.and_then(|item| item.name)
.unwrap_or_else(|| affinity_key.chars().take(8).collect::<String>());
Ok(
build_admin_monitoring_cache_affinity_delete_success_response(
affinity_key,
endpoint_id,
model_id,
api_key_name,
),
)
}

28
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_mutations/flush.rs Normal file
View File

@@ -0,0 +1,28 @@
use super::super::cache_affinity::{
clear_admin_monitoring_scheduler_affinity_entries,
delete_admin_monitoring_cache_affinity_raw_keys,
};
use super::super::cache_route_helpers::admin_monitoring_cache_affinity_unavailable_response;
use super::super::cache_store::list_admin_monitoring_cache_affinity_records;
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
use aether_admin::observability::monitoring::build_admin_monitoring_cache_flush_success_response;
use axum::{body::Body, response::Response};
pub(in super::super) async fn build_admin_monitoring_cache_flush_response(
state: &AdminAppState<'_>,
) -> Result<Response<Body>, GatewayError> {
let raw_affinities = list_admin_monitoring_cache_affinity_records(state).await?;
if state.redis_kv_runner().is_none() && raw_affinities.is_empty() {
return Ok(admin_monitoring_cache_affinity_unavailable_response());
}
let raw_keys = raw_affinities
.iter()
.map(|item| item.raw_key.clone())
.collect::<Vec<_>>();
let deleted = delete_admin_monitoring_cache_affinity_raw_keys(state, &raw_keys).await?;
clear_admin_monitoring_scheduler_affinity_entries(state, &raw_affinities);
Ok(build_admin_monitoring_cache_flush_success_response(deleted))
}

17
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_mutations/mod.rs Normal file
View File

@@ -0,0 +1,17 @@
mod affinity;
mod flush;
mod model_mapping;
mod provider;
mod redis_keys;
mod users;
pub(super) use affinity::build_admin_monitoring_cache_affinity_delete_response;
pub(super) use flush::build_admin_monitoring_cache_flush_response;
pub(super) use model_mapping::{
build_admin_monitoring_model_mapping_delete_model_response,
build_admin_monitoring_model_mapping_delete_provider_response,
build_admin_monitoring_model_mapping_delete_response,
};
pub(super) use provider::build_admin_monitoring_cache_provider_delete_response;
pub(super) use redis_keys::build_admin_monitoring_redis_keys_delete_response;
pub(super) use users::build_admin_monitoring_cache_users_delete_response;

113
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_mutations/model_mapping.rs Normal file
View File

@@ -0,0 +1,113 @@
use super::super::cache_route_helpers::{
admin_monitoring_cache_model_mapping_provider_params_from_path,
admin_monitoring_cache_model_name_from_path, admin_monitoring_redis_unavailable_response,
};
use super::super::cache_store::{
admin_monitoring_has_test_redis_keys, delete_admin_monitoring_namespaced_keys,
list_admin_monitoring_namespaced_keys,
};
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use aether_admin::observability::monitoring::{
admin_monitoring_bad_request_response,
build_admin_monitoring_model_mapping_delete_model_success_response,
build_admin_monitoring_model_mapping_delete_provider_success_response,
build_admin_monitoring_model_mapping_delete_success_response,
};
use axum::{body::Body, response::Response};
pub(in super::super) async fn build_admin_monitoring_model_mapping_delete_response(
state: &AdminAppState<'_>,
) -> Result<Response<Body>, GatewayError> {
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
return Ok(admin_monitoring_redis_unavailable_response());
}
let mut raw_keys = list_admin_monitoring_namespaced_keys(state, "model:*").await?;
raw_keys.extend(list_admin_monitoring_namespaced_keys(state, "global_model:*").await?);
raw_keys.sort();
raw_keys.dedup();
let deleted_count = delete_admin_monitoring_namespaced_keys(state, &raw_keys).await?;
Ok(build_admin_monitoring_model_mapping_delete_success_response(deleted_count))
}
pub(in super::super) async fn build_admin_monitoring_model_mapping_delete_model_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
let Some(model_name) =
admin_monitoring_cache_model_name_from_path(&request_context.request_path)
else {
return Ok(admin_monitoring_bad_request_response("缺少 model_name"));
};
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
return Ok(admin_monitoring_redis_unavailable_response());
}
let candidate_keys = [
format!("global_model:resolve:{model_name}"),
format!("global_model:name:{model_name}"),
];
let mut existing_keys = Vec::new();
for key in candidate_keys {
let matches = list_admin_monitoring_namespaced_keys(state, key.as_str()).await?;
existing_keys.extend(matches);
}
existing_keys.sort();
existing_keys.dedup();
let deleted_count = delete_admin_monitoring_namespaced_keys(state, &existing_keys).await?;
let deleted_keys = if deleted_count == 0 {
Vec::new()
} else {
existing_keys
};
Ok(
build_admin_monitoring_model_mapping_delete_model_success_response(
model_name,
deleted_keys,
),
)
}
pub(in super::super) async fn build_admin_monitoring_model_mapping_delete_provider_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
let Some((provider_id, global_model_id)) =
admin_monitoring_cache_model_mapping_provider_params_from_path(
&request_context.request_path,
)
else {
return Ok(admin_monitoring_bad_request_response(
"缺少 provider_id 或 global_model_id",
));
};
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
return Ok(admin_monitoring_redis_unavailable_response());
}
let candidate_keys = [
format!("model:provider_global:{provider_id}:{global_model_id}"),
format!("model:provider_global:hits:{provider_id}:{global_model_id}"),
];
let mut existing_keys = Vec::new();
for key in candidate_keys {
let matches = list_admin_monitoring_namespaced_keys(state, key.as_str()).await?;
existing_keys.extend(matches);
}
existing_keys.sort();
existing_keys.dedup();
let _ = delete_admin_monitoring_namespaced_keys(state, &existing_keys).await?;
Ok(
build_admin_monitoring_model_mapping_delete_provider_success_response(
provider_id,
global_model_id,
existing_keys,
),
)
}

51
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_mutations/provider.rs Normal file
View File

@@ -0,0 +1,51 @@
use super::super::cache_affinity::{
clear_admin_monitoring_scheduler_affinity_entries,
delete_admin_monitoring_cache_affinity_raw_keys,
};
use super::super::cache_route_helpers::{
admin_monitoring_cache_affinity_unavailable_response,
admin_monitoring_cache_provider_id_from_path,
};
use super::super::cache_store::list_admin_monitoring_cache_affinity_records;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use aether_admin::observability::monitoring::{
admin_monitoring_bad_request_response, admin_monitoring_cache_provider_not_found_response,
build_admin_monitoring_cache_provider_delete_success_response,
};
use axum::{body::Body, response::Response};
pub(in super::super) async fn build_admin_monitoring_cache_provider_delete_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
let Some(provider_id) =
admin_monitoring_cache_provider_id_from_path(&request_context.request_path)
else {
return Ok(admin_monitoring_bad_request_response("缺少 provider_id"));
};
let raw_affinities = list_admin_monitoring_cache_affinity_records(state).await?;
if state.redis_kv_runner().is_none() && raw_affinities.is_empty() {
return Ok(admin_monitoring_cache_affinity_unavailable_response());
}
let target_affinities = raw_affinities
.into_iter()
.filter(|item| item.provider_id.as_deref() == Some(provider_id.as_str()))
.collect::<Vec<_>>();
if target_affinities.is_empty() {
return Ok(admin_monitoring_cache_provider_not_found_response(
&provider_id,
));
}
let raw_keys = target_affinities
.iter()
.map(|item| item.raw_key.clone())
.collect::<Vec<_>>();
let deleted = delete_admin_monitoring_cache_affinity_raw_keys(state, &raw_keys).await?;
clear_admin_monitoring_scheduler_affinity_entries(state, &target_affinities);
Ok(build_admin_monitoring_cache_provider_delete_success_response(provider_id, deleted))
}

46
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_mutations/redis_keys.rs Normal file
View File

@@ -0,0 +1,46 @@
use super::super::cache_config::ADMIN_MONITORING_REDIS_CACHE_CATEGORIES;
use super::super::cache_route_helpers::{
admin_monitoring_cache_redis_category_from_path, admin_monitoring_redis_unavailable_response,
};
use super::super::cache_store::{
admin_monitoring_has_test_redis_keys, delete_admin_monitoring_namespaced_keys,
list_admin_monitoring_namespaced_keys,
};
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use aether_admin::observability::monitoring::{
admin_monitoring_bad_request_response, admin_monitoring_unknown_cache_category_response,
build_admin_monitoring_redis_keys_delete_success_response,
};
use axum::{body::Body, response::Response};
pub(in super::super) async fn build_admin_monitoring_redis_keys_delete_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
let Some(category) =
admin_monitoring_cache_redis_category_from_path(&request_context.request_path)
else {
return Ok(admin_monitoring_bad_request_response("缺少 category"));
};
let Some((cat_key, name, pattern, _description)) = ADMIN_MONITORING_REDIS_CACHE_CATEGORIES
.iter()
.find(|(cat_key, _, _, _)| *cat_key == category)
else {
return Ok(admin_monitoring_unknown_cache_category_response(&category));
};
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
return Ok(admin_monitoring_redis_unavailable_response());
}
let raw_keys = list_admin_monitoring_namespaced_keys(state, pattern).await?;
let deleted_count = delete_admin_monitoring_namespaced_keys(state, &raw_keys).await?;
Ok(build_admin_monitoring_redis_keys_delete_success_response(
cat_key,
name,
deleted_count,
))
}

108
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_mutations/users.rs Normal file
View File

@@ -0,0 +1,108 @@
use super::super::cache_affinity::{
clear_admin_monitoring_scheduler_affinity_entries,
delete_admin_monitoring_cache_affinity_raw_keys,
};
use super::super::cache_identity::{
admin_monitoring_find_user_summary_by_id, admin_monitoring_list_export_api_key_records_by_ids,
};
use super::super::cache_route_helpers::{
admin_monitoring_cache_affinity_unavailable_response,
admin_monitoring_cache_users_not_found_response,
admin_monitoring_cache_users_user_identifier_from_path,
};
use super::super::cache_store::{
list_admin_monitoring_cache_affinity_records_by_affinity_keys,
load_admin_monitoring_cache_affinity_entries_for_tests,
};
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use aether_admin::observability::monitoring::{
admin_monitoring_bad_request_response,
build_admin_monitoring_cache_users_delete_api_key_success_response,
build_admin_monitoring_cache_users_delete_user_success_response,
};
use axum::{body::Body, response::Response};
pub(in super::super) async fn build_admin_monitoring_cache_users_delete_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
let Some(user_identifier) =
admin_monitoring_cache_users_user_identifier_from_path(&request_context.request_path)
else {
return Ok(admin_monitoring_bad_request_response(
"缺少 user_identifier",
));
};
if state.redis_kv_runner().is_none()
&& load_admin_monitoring_cache_affinity_entries_for_tests(state).is_empty()
{
return Ok(admin_monitoring_cache_affinity_unavailable_response());
}
let direct_api_key_by_id =
admin_monitoring_list_export_api_key_records_by_ids(state, &[user_identifier.clone()])
.await?;
if let Some(api_key) = direct_api_key_by_id.get(&user_identifier) {
let target_affinity_keys =
std::iter::once(user_identifier.clone()).collect::<std::collections::BTreeSet<_>>();
let target_affinities = list_admin_monitoring_cache_affinity_records_by_affinity_keys(
state,
&target_affinity_keys,
)
.await?;
let raw_keys = target_affinities
.iter()
.map(|item| item.raw_key.clone())
.collect::<Vec<_>>();
let _ = delete_admin_monitoring_cache_affinity_raw_keys(state, &raw_keys).await?;
clear_admin_monitoring_scheduler_affinity_entries(state, &target_affinities);
let user = admin_monitoring_find_user_summary_by_id(state, &api_key.user_id).await?;
let api_key_name = api_key
.name
.clone()
.unwrap_or_else(|| user_identifier.clone());
return Ok(
build_admin_monitoring_cache_users_delete_api_key_success_response(
api_key.user_id.clone(),
user.as_ref().map(|item| item.username.clone()),
user.and_then(|item| item.email),
user_identifier,
api_key.name.clone(),
),
);
}
let Some(user) = state.find_user_auth_by_identifier(&user_identifier).await? else {
return Ok(admin_monitoring_cache_users_not_found_response(
&user_identifier,
));
};
let user_api_key_ids = state
.list_auth_api_key_export_records_by_user_ids(std::slice::from_ref(&user.id))
.await?
.into_iter()
.map(|item| item.api_key_id.clone())
.collect::<std::collections::BTreeSet<_>>();
let target_affinities =
list_admin_monitoring_cache_affinity_records_by_affinity_keys(state, &user_api_key_ids)
.await?;
let raw_keys = target_affinities
.iter()
.map(|item| item.raw_key.clone())
.collect::<Vec<_>>();
let _ = delete_admin_monitoring_cache_affinity_raw_keys(state, &raw_keys).await?;
clear_admin_monitoring_scheduler_affinity_entries(state, &target_affinities);
Ok(
build_admin_monitoring_cache_users_delete_user_success_response(
user.id,
user.username,
user.email,
),
)
}

11
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_payloads.rs
View File

@@ -1,11 +1,11 @@
use crate::AppState;
use crate::handlers::admin::request::AdminAppState;
use aether_crypto::decrypt_python_fernet_ciphertext;
#[cfg(test)]
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
pub(super) fn admin_monitoring_masked_user_api_key_prefix(
state: &AppState,
state: &AdminAppState<'_>,
ciphertext: Option<&str>,
) -> Option<String> {
let Some(ciphertext) = ciphertext.map(str::trim).filter(|value| !value.is_empty()) else {
@@ -23,7 +23,7 @@ pub(super) fn admin_monitoring_masked_user_api_key_prefix(
}
pub(super) fn admin_monitoring_masked_provider_key_prefix(
state: &AppState,
state: &AdminAppState<'_>,
key: &StoredProviderCatalogKey,
) -> Option<String> {
match key.auth_type.trim() {
@@ -44,7 +44,10 @@ pub(super) fn admin_monitoring_masked_provider_key_prefix(
}
}
fn admin_monitoring_try_decrypt_secret(state: &AppState, ciphertext: &str) -> Option<String> {
fn admin_monitoring_try_decrypt_secret(
state: &AdminAppState<'_>,
ciphertext: &str,
) -> Option<String> {
let ciphertext = ciphertext.trim();
if ciphertext.is_empty() {
return None;

47
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_store.rs
View File

@@ -1,10 +1,11 @@
use super::cache_affinity::admin_monitoring_cache_affinity_record;
use super::cache_types::{AdminMonitoringCacheAffinityRecord, AdminMonitoringCacheSnapshot};
use crate::handlers::admin::observability::stats::round_to;
use crate::{AppState, GatewayError};
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
use aether_data_contracts::repository::usage::UsageAuditListQuery;
async fn count_admin_monitoring_cache_affinity_entries(state: &AppState) -> usize {
async fn count_admin_monitoring_cache_affinity_entries(state: &AdminAppState<'_>) -> usize {
let Some(runner) = state.redis_kv_runner() else {
return 0;
};
@@ -74,35 +75,45 @@ async fn scan_admin_monitoring_namespaced_keys(
#[cfg(test)]
pub(super) fn load_admin_monitoring_cache_affinity_entries_for_tests(
state: &AppState,
state: &AdminAppState<'_>,
) -> Vec<(String, String)> {
state.list_admin_monitoring_cache_affinity_entries_for_tests()
state
.as_ref()
.list_admin_monitoring_cache_affinity_entries_for_tests()
}
#[cfg(not(test))]
pub(super) fn load_admin_monitoring_cache_affinity_entries_for_tests(
_state: &AppState,
_state: &AdminAppState<'_>,
) -> Vec<(String, String)> {
Vec::new()
}
#[cfg(test)]
fn load_admin_monitoring_redis_keys_for_tests(state: &AppState) -> Vec<String> {
state.list_admin_monitoring_redis_keys_for_tests()
fn load_admin_monitoring_redis_keys_for_tests(state: &AdminAppState<'_>) -> Vec<String> {
state.as_ref().list_admin_monitoring_redis_keys_for_tests()
}
#[cfg(not(test))]
fn load_admin_monitoring_redis_keys_for_tests(_state: &AppState) -> Vec<String> {
fn load_admin_monitoring_redis_keys_for_tests(_state: &AdminAppState<'_>) -> Vec<String> {
Vec::new()
}
#[cfg(test)]
fn delete_admin_monitoring_redis_keys_for_tests(state: &AppState, raw_keys: &[String]) -> usize {
state.remove_admin_monitoring_redis_keys_for_tests(raw_keys)
fn delete_admin_monitoring_redis_keys_for_tests(
state: &AdminAppState<'_>,
raw_keys: &[String],
) -> usize {
state
.as_ref()
.remove_admin_monitoring_redis_keys_for_tests(raw_keys)
}
#[cfg(not(test))]
fn delete_admin_monitoring_redis_keys_for_tests(_state: &AppState, _raw_keys: &[String]) -> usize {
fn delete_admin_monitoring_redis_keys_for_tests(
_state: &AdminAppState<'_>,
_raw_keys: &[String],
) -> usize {
0
}
@@ -113,12 +124,12 @@ fn admin_monitoring_test_key_matches_pattern(key: &str, pattern: &str) -> bool {
}
}
pub(super) fn admin_monitoring_has_test_redis_keys(state: &AppState) -> bool {
pub(super) fn admin_monitoring_has_test_redis_keys(state: &AdminAppState<'_>) -> bool {
!load_admin_monitoring_redis_keys_for_tests(state).is_empty()
}
pub(super) async fn list_admin_monitoring_namespaced_keys(
state: &AppState,
state: &AdminAppState<'_>,
pattern: &str,
) -> Result<Vec<String>, GatewayError> {
if let Some(runner) = state.redis_kv_runner() {
@@ -134,7 +145,7 @@ pub(super) async fn list_admin_monitoring_namespaced_keys(
}
pub(super) async fn delete_admin_monitoring_namespaced_keys(
state: &AppState,
state: &AdminAppState<'_>,
raw_keys: &[String],
) -> Result<usize, GatewayError> {
if raw_keys.is_empty() {
@@ -165,13 +176,13 @@ pub(super) async fn delete_admin_monitoring_namespaced_keys(
}
pub(super) async fn list_admin_monitoring_cache_affinity_records(
state: &AppState,
state: &AdminAppState<'_>,
) -> Result<Vec<AdminMonitoringCacheAffinityRecord>, GatewayError> {
list_admin_monitoring_cache_affinity_records_matching(state, None).await
}
pub(super) async fn list_admin_monitoring_cache_affinity_records_by_affinity_keys(
state: &AppState,
state: &AdminAppState<'_>,
affinity_keys: &std::collections::BTreeSet<String>,
) -> Result<Vec<AdminMonitoringCacheAffinityRecord>, GatewayError> {
if affinity_keys.is_empty() {
@@ -181,7 +192,7 @@ pub(super) async fn list_admin_monitoring_cache_affinity_records_by_affinity_key
}
async fn list_admin_monitoring_cache_affinity_records_matching(
state: &AppState,
state: &AdminAppState<'_>,
affinity_keys: Option<&std::collections::BTreeSet<String>>,
) -> Result<Vec<AdminMonitoringCacheAffinityRecord>, GatewayError> {
let mut records = Vec::new();
@@ -272,7 +283,7 @@ async fn list_admin_monitoring_cache_affinity_records_matching(
}
pub(super) async fn build_admin_monitoring_cache_snapshot(
state: &AppState,
state: &AdminAppState<'_>,
) -> Result<AdminMonitoringCacheSnapshot, GatewayError> {
let scheduling_mode = state
.read_system_config_json_value("scheduling_mode")

9
apps/aether-gateway/src/handlers/admin/observability/monitoring/mod.rs
View File

@@ -1,5 +1,5 @@
use crate::control::GatewayPublicRequestContext;
use crate::{AppState, GatewayError};
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use axum::{body::Body, response::Response};
mod activity;
@@ -15,7 +15,6 @@ mod cache_route_helpers;
mod cache_store;
mod cache_types;
mod resilience;
mod responses;
mod route_filters;
mod routes;
#[cfg(test)]
@@ -24,8 +23,8 @@ mod trace;
mod usage_helpers;
pub(crate) async fn maybe_build_local_admin_monitoring_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Option<Response<Body>>, GatewayError> {
routes::maybe_build_local_admin_monitoring_response(state, request_context).await
}

488
apps/aether-gateway/src/handlers/admin/observability/monitoring/resilience.rs
View File

@@ -1,488 +0,0 @@
use super::responses::admin_monitoring_bad_request_response;
use super::usage_helpers::admin_monitoring_usage_is_error;
use crate::control::GatewayPublicRequestContext;
use crate::handlers::admin::shared::{
provider_key_health_summary, query_param_value, unix_secs_to_rfc3339,
};
use crate::{AppState, GatewayError};
use aether_data_contracts::repository::{
provider_catalog::StoredProviderCatalogKey, usage::UsageAuditListQuery,
};
use axum::{
body::Body,
response::{IntoResponse, Response},
Json,
};
use serde_json::json;
use std::collections::BTreeMap;
struct AdminMonitoringResilienceSnapshot {
timestamp: chrono::DateTime<chrono::Utc>,
health_score: i64,
status: &'static str,
error_statistics: serde_json::Value,
recent_errors: Vec<serde_json::Value>,
recommendations: Vec<String>,
previous_stats: serde_json::Value,
}
fn build_admin_monitoring_resilience_recommendations(
total_errors: usize,
health_score: i64,
open_breaker_labels: &[String],
) -> Vec<String> {
let mut recommendations = Vec::new();
if health_score < 50 {
recommendations.push("系统健康状况严重，请立即检查错误日志".to_string());
}
if total_errors > 100 {
recommendations.push("错误频率过高，建议检查系统配置和外部依赖".to_string());
}
if !open_breaker_labels.is_empty() {
recommendations.push(format!(
"以下服务熔断器已打开：{}",
open_breaker_labels.join(", ")
));
}
if health_score > 90 {
recommendations.push("系统运行良好".to_string());
}
recommendations
}
fn parse_admin_monitoring_circuit_history_limit(query: Option<&str>) -> Result<usize, String> {
match query_param_value(query, "limit") {
None => Ok(50),
Some(value) => {
let parsed = value
.parse::<usize>()
.map_err(|_| "limit must be an integer between 1 and 200".to_string())?;
if (1..=200).contains(&parsed) {
Ok(parsed)
} else {
Err("limit must be an integer between 1 and 200".to_string())
}
}
}
}
fn build_admin_monitoring_circuit_history_items(
keys: &[StoredProviderCatalogKey],
provider_name_by_id: &BTreeMap<String, String>,
limit: usize,
) -> Vec<serde_json::Value> {
let mut items = Vec::new();
for key in keys {
let health_by_format = key
.health_by_format
.as_ref()
.and_then(serde_json::Value::as_object)
.cloned()
.unwrap_or_default();
let circuit_by_format = key
.circuit_breaker_by_format
.as_ref()
.and_then(serde_json::Value::as_object)
.cloned()
.unwrap_or_default();
for (api_format, circuit_value) in circuit_by_format {
let Some(circuit) = circuit_value.as_object() else {
continue;
};
let is_open = circuit
.get("open")
.and_then(serde_json::Value::as_bool)
.unwrap_or(false);
let is_half_open = circuit
.get("half_open_until")
.and_then(serde_json::Value::as_str)
.is_some();
if !is_open && !is_half_open {
continue;
}
let health = health_by_format
.get(&api_format)
.and_then(serde_json::Value::as_object);
let timestamp = circuit
.get("open_at")
.and_then(serde_json::Value::as_str)
.or_else(|| {
circuit
.get("half_open_until")
.and_then(serde_json::Value::as_str)
})
.or_else(|| {
health.and_then(|value| {
value
.get("last_failure_at")
.and_then(serde_json::Value::as_str)
})
})
.map(ToOwned::to_owned);
let event = if is_half_open { "half_open" } else { "opened" };
let reason = circuit
.get("reason")
.and_then(serde_json::Value::as_str)
.map(ToOwned::to_owned)
.or_else(|| {
health
.and_then(|value| {
value
.get("consecutive_failures")
.and_then(serde_json::Value::as_i64)
})
.filter(|value| *value > 0)
.map(|value| format!("连续失败 {value} 次"))
})
.or_else(|| {
Some(if is_half_open {
"熔断器处于半开状态".to_string()
} else {
"熔断器处于打开状态".to_string()
})
});
let recovery_seconds = circuit
.get("recovery_seconds")
.and_then(serde_json::Value::as_i64)
.or_else(|| {
let open_at = circuit
.get("open_at")
.and_then(serde_json::Value::as_str)
.and_then(|value| chrono::DateTime::parse_from_rfc3339(value).ok());
let next_probe_at = circuit
.get("next_probe_at")
.and_then(serde_json::Value::as_str)
.and_then(|value| chrono::DateTime::parse_from_rfc3339(value).ok());
match (open_at, next_probe_at) {
(Some(open_at), Some(next_probe_at)) => {
Some((next_probe_at - open_at).num_seconds().max(0))
}
_ => None,
}
});
items.push(json!({
"event": event,
"key_id": key.id,
"provider_id": key.provider_id,
"provider_name": provider_name_by_id.get(&key.provider_id).cloned(),
"key_name": key.name,
"api_format": api_format,
"reason": reason,
"recovery_seconds": recovery_seconds,
"timestamp": timestamp,
}));
}
}
items.sort_by(|left, right| {
let left_ts = left
.get("timestamp")
.and_then(serde_json::Value::as_str)
.unwrap_or_default();
let right_ts = right
.get("timestamp")
.and_then(serde_json::Value::as_str)
.unwrap_or_default();
right_ts.cmp(left_ts)
});
items.truncate(limit);
items
}
pub(super) async fn build_admin_monitoring_resilience_circuit_history_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
) -> Result<Response<Body>, GatewayError> {
let limit = match parse_admin_monitoring_circuit_history_limit(
request_context.request_query_string.as_deref(),
) {
Ok(value) => value,
Err(detail) => return Ok(admin_monitoring_bad_request_response(detail)),
};
let providers = state.list_provider_catalog_providers(false).await?;
let provider_ids = providers
.iter()
.map(|item| item.id.clone())
.collect::<Vec<_>>();
let provider_name_by_id = providers
.iter()
.map(|item| (item.id.clone(), item.name.clone()))
.collect::<BTreeMap<_, _>>();
let keys = if provider_ids.is_empty() {
Vec::new()
} else {
state
.list_provider_catalog_keys_by_provider_ids(&provider_ids)
.await?
};
let items = build_admin_monitoring_circuit_history_items(&keys, &provider_name_by_id, limit);
let count = items.len();
Ok(Json(json!({
"items": items,
"count": count,
}))
.into_response())
}
pub(super) async fn build_admin_monitoring_resilience_status_response(
state: &AppState,
) -> Result<Response<Body>, GatewayError> {
let snapshot = build_admin_monitoring_resilience_snapshot(state).await?;
Ok(Json(json!({
"timestamp": snapshot.timestamp.to_rfc3339(),
"health_score": snapshot.health_score,
"status": snapshot.status,
"error_statistics": snapshot.error_statistics,
"recent_errors": snapshot.recent_errors,
"recommendations": snapshot.recommendations,
}))
.into_response())
}
async fn build_admin_monitoring_resilience_snapshot(
state: &AppState,
) -> Result<AdminMonitoringResilienceSnapshot, GatewayError> {
let now = chrono::Utc::now();
let recent_error_from = std::cmp::max(
now - chrono::Duration::hours(24),
chrono::DateTime::<chrono::Utc>::from_timestamp(
state
.admin_monitoring_error_stats_reset_at()
.unwrap_or_default() as i64,
0,
)
.unwrap_or_else(|| {
chrono::DateTime::<chrono::Utc>::from_timestamp(0, 0).expect("unix epoch should exist")
}),
);
let providers = state.list_provider_catalog_providers(false).await?;
let provider_ids = providers
.iter()
.map(|item| item.id.clone())
.collect::<Vec<_>>();
let provider_name_by_id = providers
.iter()
.map(|item| (item.id.clone(), item.name.clone()))
.collect::<BTreeMap<_, _>>();
let keys = if provider_ids.is_empty() {
Vec::new()
} else {
state
.list_provider_catalog_keys_by_provider_ids(&provider_ids)
.await?
};
let active_keys = keys.iter().filter(|item| item.is_active).count();
let mut degraded_keys = 0usize;
let mut unhealthy_keys = 0usize;
let mut open_circuit_breakers = 0usize;
let mut open_breaker_labels = Vec::new();
let mut circuit_breakers = serde_json::Map::new();
let mut previous_circuit_breakers = serde_json::Map::new();
for key in &keys {
let (
health_score,
consecutive_failures,
last_failure_at,
circuit_breaker_open,
circuit_by_format,
) = provider_key_health_summary(key);
if health_score < 0.8 {
degraded_keys += 1;
}
if health_score < 0.5 {
unhealthy_keys += 1;
}
let open_formats = circuit_by_format
.iter()
.filter_map(|(api_format, value)| {
value
.get("open")
.and_then(serde_json::Value::as_bool)
.filter(|open| *open)
.map(|_| api_format.clone())
})
.collect::<Vec<_>>();
if circuit_breaker_open {
open_circuit_breakers += 1;
let provider_label = provider_name_by_id
.get(&key.provider_id)
.cloned()
.unwrap_or_else(|| key.provider_id.clone());
open_breaker_labels.push(format!("{provider_label}/{}", key.name));
}
if circuit_breaker_open || consecutive_failures > 0 || health_score < 1.0 {
circuit_breakers.insert(
key.id.clone(),
json!({
"state": if circuit_breaker_open { "open" } else { "closed" },
"provider_id": key.provider_id,
"provider_name": provider_name_by_id.get(&key.provider_id).cloned(),
"key_name": key.name,
"health_score": health_score,
"consecutive_failures": consecutive_failures,
"last_failure_at": last_failure_at,
"open_formats": open_formats,
}),
);
previous_circuit_breakers.insert(
key.id.clone(),
json!({
"state": if circuit_breaker_open { "open" } else { "closed" },
"failure_count": consecutive_failures,
}),
);
}
}
let mut recent_usage_errors = state
.list_usage_audits(&UsageAuditListQuery {
created_from_unix_secs: Some(recent_error_from.timestamp().max(0) as u64),
..Default::default()
})
.await?
.into_iter()
.filter(admin_monitoring_usage_is_error)
.collect::<Vec<_>>();
recent_usage_errors
.sort_by(|left, right| right.created_at_unix_secs.cmp(&left.created_at_unix_secs));
let total_errors = recent_usage_errors.len();
let mut error_breakdown = std::collections::BTreeMap::<String, usize>::new();
for item in &recent_usage_errors {
let error_type = item
.error_category
.clone()
.unwrap_or_else(|| item.status.clone());
let operation = format!(
"{}:{}",
item.provider_name,
item.api_format
.clone()
.unwrap_or_else(|| item.model.clone())
);
*error_breakdown
.entry(format!("{error_type}:{operation}"))
.or_default() += 1;
}
let recent_errors = recent_usage_errors
.iter()
.take(10)
.map(|item| {
let error_type = item
.error_category
.clone()
.unwrap_or_else(|| item.status.clone());
let operation = format!(
"{}:{}",
item.provider_name,
item.api_format
.clone()
.unwrap_or_else(|| item.model.clone())
);
json!({
"error_id": item.id,
"error_type": error_type,
"operation": operation,
"timestamp": unix_secs_to_rfc3339(item.created_at_unix_secs),
"context": {
"request_id": item.request_id,
"provider_id": item.provider_id,
"provider_name": item.provider_name,
"model": item.model,
"api_format": item.api_format,
"status_code": item.status_code,
"error_message": item.error_message,
}
})
})
.collect::<Vec<_>>();
let health_score = (100_i64
- i64::try_from(total_errors)
.unwrap_or(i64::MAX)
.saturating_mul(2)
- i64::try_from(open_circuit_breakers)
.unwrap_or(i64::MAX)
.saturating_mul(20))
.clamp(0, 100);
let status = if health_score > 80 {
"healthy"
} else if health_score > 50 {
"degraded"
} else {
"critical"
};
let recommendations = build_admin_monitoring_resilience_recommendations(
total_errors,
health_score,
&open_breaker_labels,
);
Ok(AdminMonitoringResilienceSnapshot {
timestamp: now,
health_score,
status,
error_statistics: json!({
"total_errors": total_errors,
"active_keys": active_keys,
"degraded_keys": degraded_keys,
"unhealthy_keys": unhealthy_keys,
"open_circuit_breakers": open_circuit_breakers,
"circuit_breakers": circuit_breakers,
}),
recent_errors,
recommendations,
previous_stats: json!({
"total_errors": total_errors,
"error_breakdown": error_breakdown,
"recent_errors": total_errors,
"circuit_breakers": previous_circuit_breakers,
}),
})
}
pub(super) async fn build_admin_monitoring_reset_error_stats_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
) -> Result<Response<Body>, GatewayError> {
let snapshot = build_admin_monitoring_resilience_snapshot(state).await?;
let reset_at = chrono::Utc::now();
state.mark_admin_monitoring_error_stats_reset(reset_at.timestamp().max(0) as u64);
let reset_by = if let Some(user_id) = request_context
.control_decision
.as_ref()
.and_then(|decision| decision.admin_principal.as_ref())
.map(|principal| principal.user_id.clone())
{
state
.find_user_auth_by_id(&user_id)
.await?
.and_then(|user| user.email.or(Some(user.username)))
.or(Some(user_id))
} else {
None
};
Ok(Json(json!({
"message": "错误统计已重置",
"previous_stats": snapshot.previous_stats,
"reset_by": reset_by,
"reset_at": reset_at.to_rfc3339(),
}))
.into_response())
}

28
apps/aether-gateway/src/handlers/admin/observability/monitoring/resilience/history.rs Normal file
View File

@@ -0,0 +1,28 @@
use super::snapshot::build_admin_monitoring_provider_name_by_id_and_keys;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use aether_admin::observability::monitoring::{
admin_monitoring_bad_request_response, build_admin_monitoring_circuit_history_items,
build_admin_monitoring_circuit_history_payload_response,
parse_admin_monitoring_circuit_history_limit,
};
use axum::{body::Body, response::Response};
pub(in super::super) async fn build_admin_monitoring_resilience_circuit_history_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
let limit = match parse_admin_monitoring_circuit_history_limit(
request_context.request_query_string.as_deref(),
) {
Ok(value) => value,
Err(detail) => return Ok(admin_monitoring_bad_request_response(detail)),
};
let (provider_name_by_id, keys) =
build_admin_monitoring_provider_name_by_id_and_keys(state).await?;
let items = build_admin_monitoring_circuit_history_items(&keys, &provider_name_by_id, limit);
Ok(build_admin_monitoring_circuit_history_payload_response(
items,
))
}

8
apps/aether-gateway/src/handlers/admin/observability/monitoring/resilience/mod.rs Normal file
View File

@@ -0,0 +1,8 @@
mod history;
mod reset;
mod snapshot;
mod status;
pub(super) use history::build_admin_monitoring_resilience_circuit_history_response;
pub(super) use reset::build_admin_monitoring_reset_error_stats_response;
pub(super) use status::build_admin_monitoring_resilience_status_response;

35
apps/aether-gateway/src/handlers/admin/observability/monitoring/resilience/reset.rs Normal file
View File

@@ -0,0 +1,35 @@
use super::snapshot::build_admin_monitoring_resilience_snapshot;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use aether_admin::observability::monitoring::build_admin_monitoring_reset_error_stats_payload_response;
use axum::{body::Body, response::Response};
pub(in super::super) async fn build_admin_monitoring_reset_error_stats_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
let snapshot = build_admin_monitoring_resilience_snapshot(state).await?;
let reset_at = chrono::Utc::now();
state.mark_admin_monitoring_error_stats_reset(reset_at.timestamp().max(0) as u64);
let reset_by = if let Some(user_id) = request_context
.control_decision
.as_ref()
.and_then(|decision| decision.admin_principal.as_ref())
.map(|principal| principal.user_id.clone())
{
state
.find_user_auth_by_id(&user_id)
.await?
.and_then(|user| user.email.or(Some(user.username)))
.or(Some(user_id))
} else {
None
};
Ok(build_admin_monitoring_reset_error_stats_payload_response(
snapshot.previous_stats,
reset_by,
reset_at,
))
}

259
apps/aether-gateway/src/handlers/admin/observability/monitoring/resilience/snapshot.rs Normal file
View File

@@ -0,0 +1,259 @@
use super::super::usage_helpers::admin_monitoring_usage_is_error;
use crate::handlers::admin::request::AdminAppState;
use crate::handlers::admin::shared::{provider_key_health_summary, unix_secs_to_rfc3339};
use crate::GatewayError;
use aether_data_contracts::repository::{
provider_catalog::StoredProviderCatalogKey, usage::UsageAuditListQuery,
};
use serde_json::json;
use std::collections::BTreeMap;
pub(super) struct AdminMonitoringResilienceSnapshot {
pub(super) timestamp: chrono::DateTime<chrono::Utc>,
pub(super) health_score: i64,
pub(super) status: &'static str,
pub(super) error_statistics: serde_json::Value,
pub(super) recent_errors: Vec<serde_json::Value>,
pub(super) recommendations: Vec<String>,
pub(super) previous_stats: serde_json::Value,
}
fn build_admin_monitoring_resilience_recommendations(
total_errors: usize,
health_score: i64,
open_breaker_labels: &[String],
) -> Vec<String> {
let mut recommendations = Vec::new();
if health_score < 50 {
recommendations.push("系统健康状况严重，请立即检查错误日志".to_string());
}
if total_errors > 100 {
recommendations.push("错误频率过高，建议检查系统配置和外部依赖".to_string());
}
if !open_breaker_labels.is_empty() {
recommendations.push(format!(
"以下服务熔断器已打开：{}",
open_breaker_labels.join(", ")
));
}
if health_score > 90 {
recommendations.push("系统运行良好".to_string());
}
recommendations
}
pub(super) async fn build_admin_monitoring_provider_name_by_id_and_keys(
state: &AdminAppState<'_>,
) -> Result<(BTreeMap<String, String>, Vec<StoredProviderCatalogKey>), GatewayError> {
let providers = state.list_provider_catalog_providers(false).await?;
let provider_ids = providers
.iter()
.map(|item| item.id.clone())
.collect::<Vec<_>>();
let provider_name_by_id = providers
.iter()
.map(|item| (item.id.clone(), item.name.clone()))
.collect::<BTreeMap<_, _>>();
let keys = if provider_ids.is_empty() {
Vec::new()
} else {
state
.list_provider_catalog_keys_by_provider_ids(&provider_ids)
.await?
};
Ok((provider_name_by_id, keys))
}
pub(super) async fn build_admin_monitoring_resilience_snapshot(
state: &AdminAppState<'_>,
) -> Result<AdminMonitoringResilienceSnapshot, GatewayError> {
let now = chrono::Utc::now();
let recent_error_from = std::cmp::max(
now - chrono::Duration::hours(24),
chrono::DateTime::<chrono::Utc>::from_timestamp(
state
.admin_monitoring_error_stats_reset_at()
.unwrap_or_default() as i64,
0,
)
.unwrap_or_else(|| {
chrono::DateTime::<chrono::Utc>::from_timestamp(0, 0).expect("unix epoch should exist")
}),
);
let (provider_name_by_id, keys) =
build_admin_monitoring_provider_name_by_id_and_keys(state).await?;
let active_keys = keys.iter().filter(|item| item.is_active).count();
let mut degraded_keys = 0usize;
let mut unhealthy_keys = 0usize;
let mut open_circuit_breakers = 0usize;
let mut open_breaker_labels = Vec::new();
let mut circuit_breakers = serde_json::Map::new();
let mut previous_circuit_breakers = serde_json::Map::new();
for key in &keys {
let (
health_score,
consecutive_failures,
last_failure_at,
circuit_breaker_open,
circuit_by_format,
) = provider_key_health_summary(key);
if health_score < 0.8 {
degraded_keys += 1;
}
if health_score < 0.5 {
unhealthy_keys += 1;
}
let open_formats = circuit_by_format
.iter()
.filter_map(|(api_format, value)| {
value
.get("open")
.and_then(serde_json::Value::as_bool)
.filter(|open| *open)
.map(|_| api_format.clone())
})
.collect::<Vec<_>>();
if circuit_breaker_open {
open_circuit_breakers += 1;
let provider_label = provider_name_by_id
.get(&key.provider_id)
.cloned()
.unwrap_or_else(|| key.provider_id.clone());
open_breaker_labels.push(format!("{provider_label}/{}", key.name));
}
if circuit_breaker_open || consecutive_failures > 0 || health_score < 1.0 {
circuit_breakers.insert(
key.id.clone(),
json!({
"state": if circuit_breaker_open { "open" } else { "closed" },
"provider_id": key.provider_id,
"provider_name": provider_name_by_id.get(&key.provider_id).cloned(),
"key_name": key.name,
"health_score": health_score,
"consecutive_failures": consecutive_failures,
"last_failure_at": last_failure_at,
"open_formats": open_formats,
}),
);
previous_circuit_breakers.insert(
key.id.clone(),
json!({
"state": if circuit_breaker_open { "open" } else { "closed" },
"failure_count": consecutive_failures,
}),
);
}
}
let mut recent_usage_errors = state
.list_usage_audits(&UsageAuditListQuery {
created_from_unix_secs: Some(recent_error_from.timestamp().max(0) as u64),
..Default::default()
})
.await?
.into_iter()
.filter(admin_monitoring_usage_is_error)
.collect::<Vec<_>>();
recent_usage_errors
.sort_by(|left, right| right.created_at_unix_secs.cmp(&left.created_at_unix_secs));
let total_errors = recent_usage_errors.len();
let mut error_breakdown = BTreeMap::<String, usize>::new();
for item in &recent_usage_errors {
let error_type = item
.error_category
.clone()
.unwrap_or_else(|| item.status.clone());
let operation = format!(
"{}:{}",
item.provider_name,
item.api_format
.clone()
.unwrap_or_else(|| item.model.clone())
);
*error_breakdown
.entry(format!("{error_type}:{operation}"))
.or_default() += 1;
}
let recent_errors = recent_usage_errors
.iter()
.take(10)
.map(|item| {
let error_type = item
.error_category
.clone()
.unwrap_or_else(|| item.status.clone());
let operation = format!(
"{}:{}",
item.provider_name,
item.api_format
.clone()
.unwrap_or_else(|| item.model.clone())
);
json!({
"error_id": item.id,
"error_type": error_type,
"operation": operation,
"timestamp": unix_secs_to_rfc3339(item.created_at_unix_secs),
"context": {
"request_id": item.request_id,
"provider_id": item.provider_id,
"provider_name": item.provider_name,
"model": item.model,
"api_format": item.api_format,
"status_code": item.status_code,
"error_message": item.error_message,
}
})
})
.collect::<Vec<_>>();
let health_score = (100_i64
- i64::try_from(total_errors)
.unwrap_or(i64::MAX)
.saturating_mul(2)
- i64::try_from(open_circuit_breakers)
.unwrap_or(i64::MAX)
.saturating_mul(20))
.clamp(0, 100);
let status = if health_score > 80 {
"healthy"
} else if health_score > 50 {
"degraded"
} else {
"critical"
};
let recommendations = build_admin_monitoring_resilience_recommendations(
total_errors,
health_score,
&open_breaker_labels,
);
Ok(AdminMonitoringResilienceSnapshot {
timestamp: now,
health_score,
status,
error_statistics: json!({
"total_errors": total_errors,
"active_keys": active_keys,
"degraded_keys": degraded_keys,
"unhealthy_keys": unhealthy_keys,
"open_circuit_breakers": open_circuit_breakers,
"circuit_breakers": circuit_breakers,
}),
recent_errors,
recommendations,
previous_stats: json!({
"total_errors": total_errors,
"error_breakdown": error_breakdown,
"recent_errors": total_errors,
"circuit_breakers": previous_circuit_breakers,
}),
})
}

20
apps/aether-gateway/src/handlers/admin/observability/monitoring/resilience/status.rs Normal file
View File

@@ -0,0 +1,20 @@
use super::snapshot::build_admin_monitoring_resilience_snapshot;
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
use aether_admin::observability::monitoring::build_admin_monitoring_resilience_status_payload_response;
use axum::{body::Body, response::Response};
pub(in super::super) async fn build_admin_monitoring_resilience_status_response(
state: &AdminAppState<'_>,
) -> Result<Response<Body>, GatewayError> {
let snapshot = build_admin_monitoring_resilience_snapshot(state).await?;
Ok(build_admin_monitoring_resilience_status_payload_response(
snapshot.timestamp,
snapshot.health_score,
snapshot.status,
snapshot.error_statistics,
snapshot.recent_errors,
snapshot.recommendations,
))
}

23
apps/aether-gateway/src/handlers/admin/observability/monitoring/responses.rs
View File

@@ -1,23 +0,0 @@
use axum::{
body::Body,
http,
response::{IntoResponse, Response},
Json,
};
use serde_json::json;
pub(super) fn admin_monitoring_bad_request_response(detail: impl Into<String>) -> Response<Body> {
(
http::StatusCode::BAD_REQUEST,
Json(json!({ "detail": detail.into() })),
)
.into_response()
}
pub(super) fn admin_monitoring_not_found_response(detail: &'static str) -> Response<Body> {
(
http::StatusCode::NOT_FOUND,
Json(json!({ "detail": detail })),
)
.into_response()
}

148
apps/aether-gateway/src/handlers/admin/observability/monitoring/routes.rs
View File

@@ -34,42 +34,15 @@ use super::trace::{
build_admin_monitoring_trace_provider_stats_response,
build_admin_monitoring_trace_request_response,
};
use crate::control::GatewayPublicRequestContext;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::attach_admin_audit_response;
use crate::{AppState, GatewayError};
use crate::GatewayError;
use aether_admin::observability::monitoring::{match_admin_monitoring_route, AdminMonitoringRoute};
use axum::{body::Body, http, response::Response};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum AdminMonitoringRoute {
AuditLogs,
SystemStatus,
SuspiciousActivities,
UserBehavior,
ResilienceStatus,
ResilienceErrorStats,
ResilienceCircuitHistory,
TraceRequest,
TraceProviderStats,
CacheStats,
CacheAffinity,
CacheAffinities,
CacheUsersDelete,
CacheAffinityDelete,
CacheFlush,
CacheProviderDelete,
CacheConfig,
CacheMetrics,
CacheModelMappingStats,
CacheModelMappingDelete,
CacheModelMappingDeleteModel,
CacheModelMappingDeleteProvider,
CacheRedisKeys,
CacheRedisKeysDelete,
}
pub(crate) async fn maybe_build_local_admin_monitoring_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Option<Response<Body>>, GatewayError> {
let Some(route) = match_admin_monitoring_route(
&request_context.request_method,
@@ -185,7 +158,7 @@ pub(crate) async fn maybe_build_local_admin_monitoring_response(
}
}
fn admin_monitoring_audit_target_id(request_context: &GatewayPublicRequestContext) -> String {
fn admin_monitoring_audit_target_id(request_context: &AdminRequestContext<'_>) -> String {
match request_context.request_query_string.as_deref() {
Some(query) if !query.trim().is_empty() => {
format!("{}?{query}", request_context.request_path)
@@ -193,112 +166,3 @@ fn admin_monitoring_audit_target_id(request_context: &GatewayPublicRequestContex
_ => request_context.request_path.clone(),
}
}
pub(crate) fn match_admin_monitoring_route(
method: &http::Method,
path: &str,
) -> Option<AdminMonitoringRoute> {
let path = normalize_admin_monitoring_path(path);
match *method {
http::Method::GET => match path {
"/api/admin/monitoring/audit-logs" => Some(AdminMonitoringRoute::AuditLogs),
"/api/admin/monitoring/system-status" => Some(AdminMonitoringRoute::SystemStatus),
"/api/admin/monitoring/suspicious-activities" => {
Some(AdminMonitoringRoute::SuspiciousActivities)
}
"/api/admin/monitoring/resilience-status" => {
Some(AdminMonitoringRoute::ResilienceStatus)
}
"/api/admin/monitoring/resilience/circuit-history" => {
Some(AdminMonitoringRoute::ResilienceCircuitHistory)
}
"/api/admin/monitoring/cache/stats" => Some(AdminMonitoringRoute::CacheStats),
"/api/admin/monitoring/cache/affinities" => Some(AdminMonitoringRoute::CacheAffinities),
"/api/admin/monitoring/cache/config" => Some(AdminMonitoringRoute::CacheConfig),
"/api/admin/monitoring/cache/metrics" => Some(AdminMonitoringRoute::CacheMetrics),
"/api/admin/monitoring/cache/model-mapping/stats" => {
Some(AdminMonitoringRoute::CacheModelMappingStats)
}
"/api/admin/monitoring/cache/redis-keys" => Some(AdminMonitoringRoute::CacheRedisKeys),
_ if matches_dynamic_segments(path, "/api/admin/monitoring/user-behavior/", 1) => {
Some(AdminMonitoringRoute::UserBehavior)
}
_ if matches_dynamic_segments(
path,
"/api/admin/monitoring/trace/stats/provider/",
1,
) =>
{
Some(AdminMonitoringRoute::TraceProviderStats)
}
_ if matches_dynamic_segments(path, "/api/admin/monitoring/trace/", 1) => {
Some(AdminMonitoringRoute::TraceRequest)
}
_ if matches_dynamic_segments(path, "/api/admin/monitoring/cache/affinity/", 1) => {
Some(AdminMonitoringRoute::CacheAffinity)
}
_ => None,
},
http::Method::DELETE => match path {
"/api/admin/monitoring/resilience/error-stats" => {
Some(AdminMonitoringRoute::ResilienceErrorStats)
}
"/api/admin/monitoring/cache" => Some(AdminMonitoringRoute::CacheFlush),
"/api/admin/monitoring/cache/model-mapping" => {
Some(AdminMonitoringRoute::CacheModelMappingDelete)
}
_ if matches_dynamic_segments(path, "/api/admin/monitoring/cache/users/", 1) => {
Some(AdminMonitoringRoute::CacheUsersDelete)
}
_ if matches_dynamic_segments(path, "/api/admin/monitoring/cache/providers/", 1) => {
Some(AdminMonitoringRoute::CacheProviderDelete)
}
_ if matches_dynamic_segments(path, "/api/admin/monitoring/cache/redis-keys/", 1) => {
Some(AdminMonitoringRoute::CacheRedisKeysDelete)
}
_ if matches_dynamic_segments(
path,
"/api/admin/monitoring/cache/model-mapping/provider/",
2,
) =>
{
Some(AdminMonitoringRoute::CacheModelMappingDeleteProvider)
}
_ if matches_dynamic_segments(
path,
"/api/admin/monitoring/cache/model-mapping/",
1,
) =>
{
Some(AdminMonitoringRoute::CacheModelMappingDeleteModel)
}
_ if matches_dynamic_segments(path, "/api/admin/monitoring/cache/affinity/", 4) => {
Some(AdminMonitoringRoute::CacheAffinityDelete)
}
_ => None,
},
_ => None,
}
}
fn normalize_admin_monitoring_path(path: &str) -> &str {
let normalized = path.trim_end_matches('/');
if normalized.is_empty() {
"/"
} else {
normalized
}
}
fn matches_dynamic_segments(path: &str, prefix: &str, dynamic_segments: usize) -> bool {
let Some(suffix) = path.strip_prefix(prefix) else {
return false;
};
let segments = suffix
.split('/')
.filter(|segment| !segment.is_empty())
.collect::<Vec<_>>();
segments.len() == dynamic_segments
}

17
apps/aether-gateway/src/handlers/admin/observability/monitoring/tests/basics.rs
View File

@@ -1,9 +1,8 @@
use super::super::cache_config::ADMIN_MONITORING_REDIS_REQUIRED_DETAIL;
use super::super::routes::{
match_admin_monitoring_route, maybe_build_local_admin_monitoring_response, AdminMonitoringRoute,
};
use super::super::test_support::{request_context, sample_key, sample_provider, sample_usage};
use super::local_monitoring_response;
use crate::AppState;
use aether_admin::observability::monitoring::{match_admin_monitoring_route, AdminMonitoringRoute};
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
use aether_data::repository::usage::InMemoryUsageReadRepository;
use axum::body::to_bytes;
@@ -76,7 +75,7 @@ async fn admin_monitoring_model_mapping_delete_requires_redis_without_runtime_or
http::Method::DELETE,
"/api/admin/monitoring/cache/model-mapping",
);
let response = maybe_build_local_admin_monitoring_response(&state, &context)
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("monitoring route should be handled locally");
@@ -100,7 +99,7 @@ async fn admin_monitoring_user_behavior_returns_empty_local_payload_without_post
"/api/admin/monitoring/user-behavior/user-123?days=30",
);
let response = maybe_build_local_admin_monitoring_response(&state, &context)
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("user behavior route should be handled locally");
@@ -128,7 +127,7 @@ async fn admin_monitoring_audit_logs_returns_empty_local_payload_without_postgre
"/api/admin/monitoring/audit-logs?username=alice&event_type=login_failed&days=14&limit=20&offset=5",
);
let response = maybe_build_local_admin_monitoring_response(&state, &context)
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("monitoring route should be handled locally");
@@ -156,7 +155,7 @@ async fn admin_monitoring_suspicious_activities_returns_empty_local_payload_with
"/api/admin/monitoring/suspicious-activities?hours=48",
);
let response = maybe_build_local_admin_monitoring_response(&state, &context)
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("monitoring route should be handled locally");
@@ -224,7 +223,7 @@ async fn admin_monitoring_resilience_status_returns_local_payload() {
);
let context = request_context(http::Method::GET, "/api/admin/monitoring/resilience-status");
let response = maybe_build_local_admin_monitoring_response(&state, &context)
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
@@ -296,7 +295,7 @@ async fn admin_monitoring_cache_stats_returns_local_payload() {
);
let context = request_context(http::Method::GET, "/api/admin/monitoring/cache/stats");
let response = maybe_build_local_admin_monitoring_response(&state, &context)
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");

60
apps/aether-gateway/src/handlers/admin/observability/monitoring/tests/mod.rs
View File

@@ -1,5 +1,8 @@
use super::maybe_build_local_admin_monitoring_response;
use super::test_support::*;
use super::{maybe_build_local_admin_monitoring_response, AppState};
use crate::control::GatewayPublicRequestContext;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::AppState;
use aether_data_contracts::repository::{
candidates::{RequestCandidateStatus, StoredRequestCandidate},
provider_catalog::{
@@ -23,12 +26,23 @@ use aether_data::repository::users::{
mod basics;
pub(super) async fn local_monitoring_response(
state: &AppState,
context: &GatewayPublicRequestContext,
) -> crate::handlers::admin::request::AdminRouteResult {
maybe_build_local_admin_monitoring_response(
&AdminAppState::new(state),
&AdminRequestContext::new(context),
)
.await
}
#[tokio::test]
async fn admin_monitoring_cache_affinities_returns_empty_payload_without_runtime_or_test_entries() {
let state = AppState::new().expect("state should build");
let context = request_context(http::Method::GET, "/api/admin/monitoring/cache/affinities");
let response = maybe_build_local_admin_monitoring_response(&state, &context)
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
@@ -67,7 +81,7 @@ async fn admin_monitoring_cache_affinity_returns_not_found_without_runtime_or_te
"/api/admin/monitoring/cache/affinity/alice",
);
let response = maybe_build_local_admin_monitoring_response(&state, &context)
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
@@ -136,7 +150,7 @@ async fn admin_monitoring_cache_affinities_and_affinity_return_local_payload_fro
http::Method::GET,
"/api/admin/monitoring/cache/affinities?keyword=alice&limit=20&offset=0",
);
let list_response = maybe_build_local_admin_monitoring_response(&state, &list_context)
let list_response = local_monitoring_response(&state, &list_context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
@@ -172,7 +186,7 @@ async fn admin_monitoring_cache_affinities_and_affinity_return_local_payload_fro
http::Method::GET,
"/api/admin/monitoring/cache/affinity/alice",
);
let detail_response = maybe_build_local_admin_monitoring_response(&state, &detail_context)
let detail_response = local_monitoring_response(&state, &detail_context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
@@ -235,7 +249,7 @@ async fn admin_monitoring_cache_users_delete_returns_local_payload_from_test_sto
}),
);
let response = maybe_build_local_admin_monitoring_response(
let response = local_monitoring_response(
&state,
&request_context(
http::Method::DELETE,
@@ -280,7 +294,7 @@ async fn admin_monitoring_cache_users_delete_returns_not_found_for_unknown_ident
}),
);
let response = maybe_build_local_admin_monitoring_response(
let response = local_monitoring_response(
&state,
&request_context(
http::Method::DELETE,
@@ -329,7 +343,7 @@ async fn admin_monitoring_cache_flush_returns_local_payload_from_test_store() {
}),
);
let response = maybe_build_local_admin_monitoring_response(
let response = local_monitoring_response(
&state,
&request_context(http::Method::DELETE, "/api/admin/monitoring/cache"),
)
@@ -377,7 +391,7 @@ async fn admin_monitoring_cache_provider_delete_returns_local_payload_from_test_
}),
);
let response = maybe_build_local_admin_monitoring_response(
let response = local_monitoring_response(
&state,
&request_context(
http::Method::DELETE,
@@ -426,7 +440,7 @@ async fn admin_monitoring_model_mapping_delete_returns_local_payload_from_test_s
json!({"id": "model-alpha"}),
);
let response = maybe_build_local_admin_monitoring_response(
let response = local_monitoring_response(
&state,
&request_context(
http::Method::DELETE,
@@ -467,7 +481,7 @@ async fn admin_monitoring_model_mapping_delete_model_returns_local_payload_from_
json!({"name": "model-beta"}),
);
let response = maybe_build_local_admin_monitoring_response(
let response = local_monitoring_response(
&state,
&request_context(
http::Method::DELETE,
@@ -515,7 +529,7 @@ async fn admin_monitoring_model_mapping_delete_provider_returns_local_payload_fr
json!({"provider_id": "provider-2"}),
);
let response = maybe_build_local_admin_monitoring_response(
let response = local_monitoring_response(
&state,
&request_context(
http::Method::DELETE,
@@ -555,7 +569,7 @@ async fn admin_monitoring_redis_keys_delete_returns_local_payload_from_test_stor
.with_admin_monitoring_redis_key_for_tests("dashboard:stats:user-1", json!({"ok": true}))
.with_admin_monitoring_redis_key_for_tests("user:user-1", json!({"ok": true}));
let response = maybe_build_local_admin_monitoring_response(
let response = local_monitoring_response(
&state,
&request_context(
http::Method::DELETE,
@@ -621,7 +635,7 @@ async fn admin_monitoring_cache_affinity_delete_returns_local_payload_from_test_
}),
);
let response = maybe_build_local_admin_monitoring_response(
let response = local_monitoring_response(
&state,
&request_context(
http::Method::DELETE,
@@ -665,7 +679,7 @@ async fn admin_monitoring_cache_affinity_delete_returns_not_found_for_mismatched
}),
);
let response = maybe_build_local_admin_monitoring_response(
let response = local_monitoring_response(
&state,
&request_context(
http::Method::DELETE,
@@ -721,7 +735,7 @@ async fn admin_monitoring_cache_metrics_returns_local_payload() {
);
let context = request_context(http::Method::GET, "/api/admin/monitoring/cache/metrics");
let response = maybe_build_local_admin_monitoring_response(&state, &context)
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
@@ -752,7 +766,7 @@ async fn admin_monitoring_cache_config_returns_local_payload() {
let state = AppState::new().expect("state should build");
let context = request_context(http::Method::GET, "/api/admin/monitoring/cache/config");
let response = maybe_build_local_admin_monitoring_response(&state, &context)
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
@@ -791,7 +805,7 @@ async fn admin_monitoring_model_mapping_stats_returns_local_payload_without_redi
"/api/admin/monitoring/cache/model-mapping/stats",
);
let response = maybe_build_local_admin_monitoring_response(&state, &context)
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
@@ -853,7 +867,7 @@ async fn admin_monitoring_reset_error_stats_returns_local_payload_and_clears_fut
http::Method::DELETE,
"/api/admin/monitoring/resilience/error-stats",
);
let reset_response = maybe_build_local_admin_monitoring_response(&state, &reset_context)
let reset_response = local_monitoring_response(&state, &reset_context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
@@ -875,7 +889,7 @@ async fn admin_monitoring_reset_error_stats_returns_local_payload_and_clears_fut
let status_context =
request_context(http::Method::GET, "/api/admin/monitoring/resilience-status");
let status_response = maybe_build_local_admin_monitoring_response(&state, &status_context)
let status_response = local_monitoring_response(&state, &status_context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
@@ -898,7 +912,7 @@ async fn admin_monitoring_redis_keys_returns_local_payload_without_redis() {
let state = AppState::new().expect("state should build");
let context = request_context(http::Method::GET, "/api/admin/monitoring/cache/redis-keys");
let response = maybe_build_local_admin_monitoring_response(&state, &context)
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
@@ -921,7 +935,7 @@ async fn admin_monitoring_redis_keys_delete_returns_unavailable_without_redis()
"/api/admin/monitoring/cache/redis-keys/upstream_models",
);
let response = maybe_build_local_admin_monitoring_response(&state, &context)
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
@@ -967,7 +981,7 @@ async fn admin_monitoring_circuit_history_returns_local_payload() {
"/api/admin/monitoring/resilience/circuit-history?limit=10",
);
let response = maybe_build_local_admin_monitoring_response(&state, &context)
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");

8
apps/aether-gateway/src/handlers/admin/observability/monitoring/tests/trace.rs
View File

@@ -1,7 +1,7 @@
use super::super::routes::maybe_build_local_admin_monitoring_response;
use super::super::test_support::{
request_context, sample_candidate, sample_endpoint, sample_key, sample_provider,
};
use super::local_monitoring_response;
use crate::AppState;
use aether_data_contracts::repository::candidates::RequestCandidateStatus;
use axum::body::to_bytes;
@@ -46,7 +46,7 @@ async fn admin_monitoring_trace_request_returns_local_payload() {
"/api/admin/monitoring/trace/request-1?attempted_only=true",
);
let response = maybe_build_local_admin_monitoring_response(&state, &context)
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
@@ -132,7 +132,7 @@ async fn admin_monitoring_trace_provider_stats_returns_local_payload() {
"/api/admin/monitoring/trace/stats/provider/provider-1?limit=10",
);
let response = maybe_build_local_admin_monitoring_response(&state, &context)
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
@@ -163,7 +163,7 @@ async fn admin_monitoring_trace_request_returns_contextual_not_found_payload() {
"/api/admin/monitoring/trace/provider-test-missing?attempted_only=false",
);
let response = maybe_build_local_admin_monitoring_response(&state, &context)
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");

156
apps/aether-gateway/src/handlers/admin/observability/monitoring/trace.rs
View File

@@ -1,75 +1,25 @@
use super::responses::admin_monitoring_bad_request_response;
use super::route_filters::parse_admin_monitoring_limit;
use crate::control::GatewayPublicRequestContext;
use crate::handlers::admin::shared::{query_param_value, unix_secs_to_rfc3339};
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::log_ids::short_request_id;
use crate::{AppState, GatewayError};
use crate::GatewayError;
use aether_admin::observability::monitoring::{
admin_monitoring_bad_request_response, admin_monitoring_trace_not_found_response,
admin_monitoring_trace_provider_id_from_path, admin_monitoring_trace_request_id_from_path,
build_admin_monitoring_trace_provider_stats_payload_response,
build_admin_monitoring_trace_request_payload_response, parse_admin_monitoring_attempted_only,
};
use aether_data_contracts::repository::candidates::RequestCandidateStatus;
use axum::{
body::Body,
http,
response::{IntoResponse, Response},
Json,
};
use serde_json::json;
use tracing::warn;
fn admin_monitoring_trace_request_id_from_path(request_path: &str) -> Option<String> {
let value = request_path
.strip_prefix("/api/admin/monitoring/trace/")?
.trim()
.trim_matches('/')
.to_string();
if value.is_empty() || value.contains('/') {
None
} else {
Some(value)
}
}
fn admin_monitoring_trace_provider_id_from_path(request_path: &str) -> Option<String> {
let value = request_path
.strip_prefix("/api/admin/monitoring/trace/stats/provider/")?
.trim()
.trim_matches('/')
.to_string();
if value.is_empty() || value.contains('/') {
None
} else {
Some(value)
}
}
fn parse_admin_monitoring_attempted_only(query: Option<&str>) -> Result<bool, String> {
match query_param_value(query, "attempted_only") {
None => Ok(false),
Some(value) => match value.trim().to_ascii_lowercase().as_str() {
"true" | "1" | "yes" => Ok(true),
"false" | "0" | "no" => Ok(false),
_ => Err("attempted_only must be a boolean".to_string()),
},
}
}
fn admin_monitoring_trace_not_found_response(
request_id: &str,
attempted_only: bool,
) -> Response<Body> {
(
http::StatusCode::NOT_FOUND,
Json(json!({
"detail": "Request trace not found",
"request_id": request_id,
"attempted_only": attempted_only,
})),
)
.into_response()
}
pub(super) async fn build_admin_monitoring_trace_request_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
let state = state.as_ref();
let Some(request_id) =
admin_monitoring_trace_request_id_from_path(&request_context.request_path)
else {
@@ -102,59 +52,16 @@ pub(super) async fn build_admin_monitoring_trace_request_response(
));
};
let candidates = trace
.candidates
.iter()
.map(|item| {
let candidate = &item.candidate;
json!({
"id": candidate.id,
"request_id": candidate.request_id,
"candidate_index": candidate.candidate_index,
"retry_index": candidate.retry_index,
"provider_id": candidate.provider_id,
"provider_name": item.provider_name,
"provider_website": item.provider_website,
"endpoint_id": candidate.endpoint_id,
"endpoint_name": item.endpoint_api_format,
"key_id": candidate.key_id,
"key_name": item.provider_key_name,
"key_account_label": serde_json::Value::Null,
"key_preview": serde_json::Value::Null,
"key_auth_type": item.provider_key_auth_type,
"key_oauth_plan_type": serde_json::Value::Null,
"key_capabilities": item.provider_key_capabilities,
"required_capabilities": candidate.required_capabilities,
"status": candidate.status,
"skip_reason": candidate.skip_reason,
"is_cached": candidate.is_cached,
"status_code": candidate.status_code,
"error_type": candidate.error_type,
"error_message": candidate.error_message,
"latency_ms": candidate.latency_ms,
"concurrent_requests": candidate.concurrent_requests,
"extra_data": candidate.extra_data,
"created_at": unix_secs_to_rfc3339(candidate.created_at_unix_secs),
"started_at": candidate.started_at_unix_secs.and_then(unix_secs_to_rfc3339),
"finished_at": candidate.finished_at_unix_secs.and_then(unix_secs_to_rfc3339),
})
})
.collect::<Vec<_>>();
Ok(Json(json!({
"request_id": trace.request_id,
"total_candidates": trace.total_candidates,
"final_status": trace.final_status,
"total_latency_ms": trace.total_latency_ms,
"candidates": candidates,
}))
.into_response())
Ok(build_admin_monitoring_trace_request_payload_response(
&trace,
))
}
pub(super) async fn build_admin_monitoring_trace_provider_stats_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
let state = state.as_ref();
let Some(provider_id) =
admin_monitoring_trace_provider_id_from_path(&request_context.request_path)
else {
@@ -215,18 +122,19 @@ pub(super) async fn build_admin_monitoring_trace_provider_stats_response(
((total / latency_values.len() as f64) * 100.0).round() / 100.0
};
Ok(Json(json!({
"provider_id": provider_id,
"total_attempts": total_attempts,
"success_count": success_count,
"failed_count": failed_count,
"cancelled_count": cancelled_count,
"skipped_count": skipped_count,
"pending_count": pending_count,
"available_count": available_count,
"unused_count": unused_count,
"failure_rate": failure_rate,
"avg_latency_ms": avg_latency_ms,
}))
.into_response())
Ok(
build_admin_monitoring_trace_provider_stats_payload_response(
provider_id,
total_attempts,
success_count,
failed_count,
cancelled_count,
skipped_count,
pending_count,
available_count,
unused_count,
failure_rate,
avg_latency_ms,
),
)
}

34
apps/aether-gateway/src/handlers/admin/observability/routes.rs Normal file
View File

@@ -0,0 +1,34 @@
use super::{monitoring, stats, usage};
use crate::handlers::admin::request::{AdminRouteRequest, AdminRouteResult};
pub(crate) async fn maybe_build_local_admin_observability_response(
request: AdminRouteRequest<'_>,
) -> AdminRouteResult {
if let Some(response) =
stats::maybe_build_local_admin_stats_response(&request.state(), &request.request_context())
.await?
{
return Ok(Some(response));
}
if let Some(response) = monitoring::maybe_build_local_admin_monitoring_response(
&request.state(),
&request.request_context(),
)
.await?
{
return Ok(Some(response));
}
if let Some(response) = usage::maybe_build_local_admin_usage_response(
&request.state(),
&request.request_context(),
request.request_body(),
)
.await?
{
return Ok(Some(response));
}
Ok(None)
}

314
apps/aether-gateway/src/handlers/admin/observability/stats/analytics_routes.rs
View File

@@ -1,20 +1,15 @@
use super::helpers::{
round_to, AdminStatsComparisonType, AdminStatsGranularity, AdminStatsTimeRange,
AdminStatsUsageFilter,
};
use super::range::{build_comparison_range, list_usage_for_range};
use super::responses::{
use super::range::build_comparison_range;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::query_param_value;
use crate::GatewayError;
use aether_admin::observability::stats::{
admin_stats_bad_request_response, admin_stats_comparison_empty_response,
admin_stats_error_distribution_empty_response,
admin_stats_performance_percentiles_empty_response, admin_stats_time_series_empty_response,
build_admin_stats_comparison_response, build_admin_stats_error_distribution_response,
build_admin_stats_performance_percentiles_response, build_admin_stats_time_series_response,
AdminStatsComparisonType, AdminStatsGranularity, AdminStatsTimeRange, AdminStatsUsageFilter,
};
use super::timeseries::{
aggregate_usage_stats, build_time_series_payload, pct_change_value, percentile_cont,
};
use crate::control::GatewayControlDecision;
use crate::control::GatewayPublicRequestContext;
use crate::handlers::admin::shared::query_param_value;
use crate::{AppState, GatewayError};
use axum::{
body::Body,
http,
@@ -24,39 +19,34 @@ use axum::{
use serde_json::json;
pub(super) async fn maybe_build_local_admin_stats_analytics_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
decision: &GatewayControlDecision,
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Option<Response<Body>>, GatewayError> {
if decision.route_kind.as_deref() == Some("comparison")
&& request_context.request_method == http::Method::GET
if request_context.route_kind() == Some("comparison")
&& request_context.method() == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
request_context.path(),
"/api/admin/stats/comparison" | "/api/admin/stats/comparison/"
)
{
let current_range = match AdminStatsTimeRange::resolve_required(
request_context.request_query_string.as_deref(),
request_context.query_string(),
"current_start",
"current_end",
) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
let comparison_type = match query_param_value(
request_context.request_query_string.as_deref(),
"comparison_type",
)
.as_deref()
{
None | Some("period") => AdminStatsComparisonType::Period,
Some("year") => AdminStatsComparisonType::Year,
Some(_) => {
return Ok(Some(admin_stats_bad_request_response(
"comparison_type must be 'period' or 'year'".to_string(),
)));
}
};
let comparison_type =
match query_param_value(request_context.query_string(), "comparison_type").as_deref() {
None | Some("period") => AdminStatsComparisonType::Period,
Some("year") => AdminStatsComparisonType::Year,
Some(_) => {
return Ok(Some(admin_stats_bad_request_response(
"comparison_type must be 'period' or 'year'".to_string(),
)));
}
};
let comparison_range = match build_comparison_range(&current_range, comparison_type) {
Ok(value) => value,
@@ -68,62 +58,33 @@ pub(super) async fn maybe_build_local_admin_stats_analytics_response(
&comparison_range,
)));
}
let current_usage =
list_usage_for_range(state, &current_range, &AdminStatsUsageFilter::default()).await?;
let comparison_usage =
list_usage_for_range(state, &comparison_range, &AdminStatsUsageFilter::default())
.await?;
let current = aggregate_usage_stats(&current_usage);
let comparison = aggregate_usage_stats(&comparison_usage);
return Ok(Some(
Json(json!({
"current": {
"total_requests": current.total_requests,
"total_tokens": current.total_tokens,
"total_cost": round_to(current.total_cost, 6),
"actual_total_cost": round_to(current.actual_total_cost, 6),
"avg_response_time_ms": round_to(current.avg_response_time_ms(), 2),
"error_requests": current.error_requests,
},
"comparison": {
"total_requests": comparison.total_requests,
"total_tokens": comparison.total_tokens,
"total_cost": round_to(comparison.total_cost, 6),
"actual_total_cost": round_to(comparison.actual_total_cost, 6),
"avg_response_time_ms": round_to(comparison.avg_response_time_ms(), 2),
"error_requests": comparison.error_requests,
},
"change_percent": {
"total_requests": pct_change_value(current.total_requests as f64, comparison.total_requests as f64),
"total_tokens": pct_change_value(current.total_tokens as f64, comparison.total_tokens as f64),
"total_cost": pct_change_value(current.total_cost, comparison.total_cost),
"actual_total_cost": pct_change_value(current.actual_total_cost, comparison.actual_total_cost),
"avg_response_time_ms": pct_change_value(current.avg_response_time_ms(), comparison.avg_response_time_ms()),
"error_requests": pct_change_value(current.error_requests as f64, comparison.error_requests as f64),
},
"current_start": current_range.start_date.to_string(),
"current_end": current_range.end_date.to_string(),
"comparison_start": comparison_range.start_date.to_string(),
"comparison_end": comparison_range.end_date.to_string(),
}))
.into_response(),
));
let current_usage = state
.list_admin_usage_for_range(&current_range, &AdminStatsUsageFilter::default())
.await?;
let comparison_usage = state
.list_admin_usage_for_range(&comparison_range, &AdminStatsUsageFilter::default())
.await?;
return Ok(Some(build_admin_stats_comparison_response(
&current_usage,
&comparison_usage,
&current_range,
&comparison_range,
)));
}
if decision.route_kind.as_deref() == Some("error_distribution")
&& request_context.request_method == http::Method::GET
if request_context.route_kind() == Some("error_distribution")
&& request_context.method() == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
request_context.path(),
"/api/admin/stats/errors/distribution" | "/api/admin/stats/errors/distribution/"
)
{
let Some(time_range) = (match AdminStatsTimeRange::resolve_optional(
request_context.request_query_string.as_deref(),
) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
}) else {
let Some(time_range) =
(match AdminStatsTimeRange::resolve_optional(request_context.query_string()) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
})
else {
return Ok(Some(
Json(json!({
"distribution": [],
@@ -136,161 +97,61 @@ pub(super) async fn maybe_build_local_admin_stats_analytics_response(
return Ok(Some(admin_stats_error_distribution_empty_response()));
}
let usage =
list_usage_for_range(state, &time_range, &AdminStatsUsageFilter::default()).await?;
let mut distribution: std::collections::BTreeMap<String, u64> =
std::collections::BTreeMap::new();
let mut trend: std::collections::BTreeMap<String, std::collections::BTreeMap<String, u64>> =
std::collections::BTreeMap::new();
for item in usage {
let Some(category) = item
.error_category
.as_ref()
.filter(|value| !value.trim().is_empty())
else {
continue;
};
let Some(local_day) =
time_range.local_date_string_for_unix_secs(item.created_at_unix_secs)
else {
continue;
};
*distribution.entry(category.clone()).or_default() += 1;
*trend
.entry(local_day)
.or_default()
.entry(category.clone())
.or_default() += 1;
}
let mut distribution_items: Vec<_> = distribution
.into_iter()
.map(|(category, count)| json!({ "category": category, "count": count }))
.collect();
distribution_items.sort_by(|left, right| {
let left_count = left
.get("count")
.and_then(serde_json::Value::as_u64)
.unwrap_or(0);
let right_count = right
.get("count")
.and_then(serde_json::Value::as_u64)
.unwrap_or(0);
right_count.cmp(&left_count).then_with(|| {
left.get("category")
.and_then(serde_json::Value::as_str)
.cmp(&right.get("category").and_then(serde_json::Value::as_str))
})
});
let trend_items: Vec<_> = trend
.into_iter()
.map(|(date, categories)| {
let total: u64 = categories.values().copied().sum();
json!({
"date": date,
"total": total,
"categories": categories,
})
})
.collect();
return Ok(Some(
Json(json!({
"distribution": distribution_items,
"trend": trend_items,
}))
.into_response(),
));
let usage = state
.list_admin_usage_for_range(&time_range, &AdminStatsUsageFilter::default())
.await?;
return Ok(Some(build_admin_stats_error_distribution_response(
&time_range,
&usage,
)));
}
if decision.route_kind.as_deref() == Some("performance_percentiles")
&& request_context.request_method == http::Method::GET
if request_context.route_kind() == Some("performance_percentiles")
&& request_context.method() == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
request_context.path(),
"/api/admin/stats/performance/percentiles"
| "/api/admin/stats/performance/percentiles/"
)
{
let Some(time_range) = (match AdminStatsTimeRange::resolve_optional(
request_context.request_query_string.as_deref(),
) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
}) else {
let Some(time_range) =
(match AdminStatsTimeRange::resolve_optional(request_context.query_string()) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
})
else {
return Ok(Some(Json(json!([])).into_response()));
};
if !state.has_usage_data_reader() {
return Ok(Some(admin_stats_performance_percentiles_empty_response()));
}
let usage =
list_usage_for_range(state, &time_range, &AdminStatsUsageFilter::default()).await?;
let mut by_day: std::collections::BTreeMap<String, (Vec<u64>, Vec<u64>)> = time_range
.local_date_strings()
.into_iter()
.map(|date| (date, (Vec::new(), Vec::new())))
.collect();
for item in usage {
if item.status != "completed" {
continue;
}
let Some(local_day) =
time_range.local_date_string_for_unix_secs(item.created_at_unix_secs)
else {
continue;
};
let Some((response_times, first_byte_times)) = by_day.get_mut(&local_day) else {
continue;
};
if let Some(response_time_ms) = item.response_time_ms {
response_times.push(response_time_ms);
}
if let Some(first_byte_time_ms) = item.first_byte_time_ms {
first_byte_times.push(first_byte_time_ms);
}
}
let payload: Vec<_> = by_day
.into_iter()
.map(|(date, (mut response_times, mut first_byte_times))| {
json!({
"date": date,
"p50_response_time_ms": percentile_cont(&mut response_times, 0.5),
"p90_response_time_ms": percentile_cont(&mut response_times, 0.9),
"p99_response_time_ms": percentile_cont(&mut response_times, 0.99),
"p50_first_byte_time_ms": percentile_cont(&mut first_byte_times, 0.5),
"p90_first_byte_time_ms": percentile_cont(&mut first_byte_times, 0.9),
"p99_first_byte_time_ms": percentile_cont(&mut first_byte_times, 0.99),
})
})
.collect();
return Ok(Some(
Json(serde_json::Value::Array(payload)).into_response(),
));
let usage = state
.list_admin_usage_for_range(&time_range, &AdminStatsUsageFilter::default())
.await?;
return Ok(Some(build_admin_stats_performance_percentiles_response(
&time_range,
&usage,
)));
}
if decision.route_kind.as_deref() == Some("time_series")
&& request_context.request_method == http::Method::GET
if request_context.route_kind() == Some("time_series")
&& request_context.method() == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
request_context.path(),
"/api/admin/stats/time-series" | "/api/admin/stats/time-series/"
)
{
let granularity =
match AdminStatsGranularity::parse(request_context.request_query_string.as_deref()) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
let Some(time_range) = (match AdminStatsTimeRange::resolve_optional(
request_context.request_query_string.as_deref(),
) {
let granularity = match AdminStatsGranularity::parse(request_context.query_string()) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
}) else {
};
let Some(time_range) =
(match AdminStatsTimeRange::resolve_optional(request_context.query_string()) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
})
else {
return Ok(Some(Json(json!([])).into_response()));
};
if let Err(detail) = time_range.validate_for_time_series(granularity) {
@@ -300,14 +161,15 @@ pub(super) async fn maybe_build_local_admin_stats_analytics_response(
return Ok(Some(admin_stats_time_series_empty_response()));
}
let filters =
AdminStatsUsageFilter::from_query(request_context.request_query_string.as_deref());
let usage = list_usage_for_range(state, &time_range, &filters).await?;
let payload = build_time_series_payload(&time_range, granularity, &usage);
return Ok(Some(
Json(serde_json::Value::Array(payload)).into_response(),
));
let filters = AdminStatsUsageFilter::from_query(request_context.query_string());
let usage = state
.list_admin_usage_for_range(&time_range, &filters)
.await?;
return Ok(Some(build_admin_stats_time_series_response(
&time_range,
granularity,
&usage,
)));
}
Ok(None)

136
apps/aether-gateway/src/handlers/admin/observability/stats/cost_routes.rs
View File

@@ -1,40 +1,28 @@
use super::helpers::{
round_to, AdminStatsForecastPoint, AdminStatsGranularity, AdminStatsTimeRange,
use super::range::{build_time_range_from_days, parse_bounded_u32, parse_tz_offset_minutes};
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::query_param_value;
use crate::GatewayError;
use aether_admin::observability::stats::{
admin_stats_bad_request_response, admin_stats_cost_forecast_empty_response,
admin_stats_cost_savings_empty_response, build_admin_stats_cost_forecast_response,
build_admin_stats_cost_savings_response, AdminStatsGranularity, AdminStatsTimeRange,
AdminStatsUsageFilter,
};
use super::range::{
build_time_range_from_days, list_usage_for_range, parse_bounded_u32, parse_tz_offset_minutes,
};
use super::responses::{
admin_stats_bad_request_response, admin_stats_cost_forecast_empty_response,
admin_stats_cost_savings_empty_response,
};
use super::timeseries::{build_daily_time_series_buckets, linear_regression};
use crate::control::GatewayPublicRequestContext;
use crate::handlers::admin::shared::query_param_value;
use crate::{AppState, GatewayError};
use axum::{
body::Body,
http,
response::{IntoResponse, Response},
Json,
};
use serde_json::json;
use axum::{body::Body, http, response::Response};
pub(super) async fn maybe_build_local_admin_stats_cost_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Option<Response<Body>>, GatewayError> {
let query = request_context.request_query_string.as_deref();
let query = request_context.query_string();
if request_context
.control_decision
.as_ref()
.decision()
.and_then(|decision| decision.route_kind.as_deref())
== Some("cost_forecast")
&& request_context.request_method == http::Method::GET
&& request_context.method() == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
request_context.path(),
"/api/admin/stats/cost/forecast" | "/api/admin/stats/cost/forecast/"
)
{
@@ -76,60 +64,23 @@ pub(super) async fn maybe_build_local_admin_stats_cost_response(
return Ok(Some(admin_stats_bad_request_response(detail)));
}
let usage =
list_usage_for_range(state, &time_range, &AdminStatsUsageFilter::default()).await?;
let daily = build_daily_time_series_buckets(&time_range, &usage);
let history: Vec<AdminStatsForecastPoint> = daily
.into_iter()
.map(|(date, bucket)| AdminStatsForecastPoint {
date,
total_cost: bucket.total_cost,
})
.collect();
let values: Vec<f64> = history.iter().map(|item| item.total_cost).collect();
let (slope, intercept) = linear_regression(&values);
let last_date = history
.last()
.map(|item| item.date)
.unwrap_or(time_range.end_date);
let forecast: Vec<_> = (0..forecast_days)
.map(|index| {
let idx = values.len() + index as usize;
let predicted = (slope * idx as f64 + intercept).max(0.0);
json!({
"date": last_date
.checked_add_signed(chrono::Duration::days(i64::from(index + 1)))
.unwrap_or(last_date)
.to_string(),
"total_cost": round_to(predicted, 4),
})
})
.collect();
return Ok(Some(
Json(json!({
"history": history.into_iter().map(|item| json!({
"date": item.date.to_string(),
"total_cost": round_to(item.total_cost, 6),
})).collect::<Vec<_>>(),
"forecast": forecast,
"slope": round_to(slope, 6),
"intercept": round_to(intercept, 6),
"start_date": time_range.start_date.to_string(),
"end_date": time_range.end_date.to_string(),
}))
.into_response(),
));
let usage = state
.list_admin_usage_for_range(&time_range, &AdminStatsUsageFilter::default())
.await?;
return Ok(Some(build_admin_stats_cost_forecast_response(
&time_range,
forecast_days,
&usage,
)));
}
if request_context
.control_decision
.as_ref()
.decision()
.and_then(|decision| decision.route_kind.as_deref())
== Some("cost_savings")
&& request_context.request_method == http::Method::GET
&& request_context.method() == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
request_context.path(),
"/api/admin/stats/cost/savings" | "/api/admin/stats/cost/savings/"
)
{
@@ -149,38 +100,11 @@ pub(super) async fn maybe_build_local_admin_stats_cost_response(
provider_name: query_param_value(query, "provider_name"),
model: query_param_value(query, "model"),
};
let usage = list_usage_for_range(
state,
time_range.as_ref().expect("time range exists"),
&filters,
)
.await?;
let usage = state
.list_admin_usage_for_range(time_range.as_ref().expect("time range exists"), &filters)
.await?;
let cache_read_tokens: u64 = usage.iter().map(|item| item.cache_read_input_tokens).sum();
let cache_read_cost: f64 = usage.iter().map(|item| item.cache_read_cost_usd).sum();
let cache_creation_cost: f64 = usage.iter().map(|item| item.cache_creation_cost_usd).sum();
let mut estimated_full_cost: f64 = usage
.iter()
.map(|item| {
item.output_price_per_1m.unwrap_or(0.0) * item.cache_read_input_tokens as f64
/ 1_000_000.0
})
.sum();
if estimated_full_cost <= 0.0 && cache_read_cost > 0.0 {
estimated_full_cost = cache_read_cost * 10.0;
}
let cache_savings = estimated_full_cost - cache_read_cost;
return Ok(Some(
Json(json!({
"cache_read_tokens": cache_read_tokens,
"cache_read_cost": round_to(cache_read_cost, 6),
"cache_creation_cost": round_to(cache_creation_cost, 6),
"estimated_full_cost": round_to(estimated_full_cost, 6),
"cache_savings": round_to(cache_savings, 6),
}))
.into_response(),
));
return Ok(Some(build_admin_stats_cost_savings_response(&usage)));
}
Ok(None)

387
apps/aether-gateway/src/handlers/admin/observability/stats/helpers.rs
View File

@@ -1,387 +0,0 @@
use super::range::{
admin_usage_default_days, parse_naive_date, parse_tz_offset_minutes, resolve_preset_dates,
user_today,
};
use crate::handlers::admin::shared::query_param_value;
use aether_data_contracts::repository::usage::StoredRequestUsageAudit;
use chrono::Utc;
use serde_json::json;
pub(crate) const MIN_PERCENTILE_SAMPLES: usize = 10;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum AdminStatsComparisonType {
Period,
Year,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum AdminStatsGranularity {
Hour,
Day,
Week,
Month,
}
#[derive(Clone, Debug)]
pub(crate) struct AdminStatsTimeRange {
pub(crate) start_date: chrono::NaiveDate,
pub(crate) end_date: chrono::NaiveDate,
pub(crate) tz_offset_minutes: i32,
}
#[derive(Clone, Debug, Default)]
pub(crate) struct AdminStatsUsageFilter {
pub(crate) user_id: Option<String>,
pub(crate) provider_name: Option<String>,
pub(crate) model: Option<String>,
}
#[derive(Clone, Debug, Default)]
pub(crate) struct AdminStatsAggregate {
pub(crate) total_requests: u64,
pub(crate) total_tokens: u64,
pub(crate) total_cost: f64,
pub(crate) actual_total_cost: f64,
pub(crate) total_response_time_ms: f64,
pub(crate) error_requests: u64,
}
#[derive(Clone, Debug)]
pub(crate) struct AdminStatsForecastPoint {
pub(crate) date: chrono::NaiveDate,
pub(crate) total_cost: f64,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum AdminStatsLeaderboardMetric {
Requests,
Tokens,
Cost,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum AdminStatsSortOrder {
Asc,
Desc,
}
#[derive(Clone, Debug)]
pub(crate) struct AdminStatsLeaderboardItem {
pub(crate) id: String,
pub(crate) name: String,
pub(crate) requests: u64,
pub(crate) tokens: u64,
pub(crate) cost: f64,
}
#[derive(Clone, Debug)]
pub(crate) struct AdminStatsUserMetadata {
pub(crate) name: String,
pub(crate) role: String,
pub(crate) is_active: bool,
pub(crate) is_deleted: bool,
}
#[derive(Clone, Debug, Default)]
pub(crate) struct AdminStatsTimeSeriesBucket {
pub(crate) total_requests: u64,
pub(crate) input_tokens: u64,
pub(crate) output_tokens: u64,
pub(crate) cache_creation_tokens: u64,
pub(crate) cache_read_tokens: u64,
pub(crate) total_cost: f64,
pub(crate) total_response_time_ms: f64,
}
impl AdminStatsAggregate {
pub(crate) fn avg_response_time_ms(&self) -> f64 {
if self.total_requests == 0 {
0.0
} else {
self.total_response_time_ms / self.total_requests as f64
}
}
}
impl AdminStatsGranularity {
pub(crate) fn parse(query: Option<&str>) -> Result<Self, String> {
match query_param_value(query, "granularity").as_deref() {
None | Some("day") => Ok(Self::Day),
Some("hour") => Ok(Self::Hour),
Some("week") => Ok(Self::Week),
Some("month") => Ok(Self::Month),
Some(_) => Err("granularity must be one of: hour, day, week, month".to_string()),
}
}
}
impl AdminStatsLeaderboardMetric {
pub(crate) fn parse(query: Option<&str>) -> Result<Self, String> {
match query_param_value(query, "metric").as_deref() {
None | Some("requests") => Ok(Self::Requests),
Some("tokens") => Ok(Self::Tokens),
Some("cost") => Ok(Self::Cost),
Some(_) => Err("metric must be one of: requests, tokens, cost".to_string()),
}
}
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::Requests => "requests",
Self::Tokens => "tokens",
Self::Cost => "cost",
}
}
}
impl AdminStatsSortOrder {
pub(crate) fn parse(query: Option<&str>) -> Result<Self, String> {
match query_param_value(query, "order").as_deref() {
None | Some("desc") => Ok(Self::Desc),
Some("asc") => Ok(Self::Asc),
Some(_) => Err("order must be one of: asc, desc".to_string()),
}
}
}
impl AdminStatsUsageFilter {
pub(crate) fn from_query(query: Option<&str>) -> Self {
Self {
user_id: query_param_value(query, "user_id"),
provider_name: query_param_value(query, "provider_name"),
model: query_param_value(query, "model"),
}
}
}
impl AdminStatsTimeSeriesBucket {
pub(crate) fn add_usage(&mut self, item: &StoredRequestUsageAudit) {
self.total_requests = self.total_requests.saturating_add(1);
self.input_tokens = self.input_tokens.saturating_add(item.input_tokens);
self.output_tokens = self.output_tokens.saturating_add(item.output_tokens);
self.cache_creation_tokens = self
.cache_creation_tokens
.saturating_add(item.cache_creation_input_tokens);
self.cache_read_tokens = self
.cache_read_tokens
.saturating_add(item.cache_read_input_tokens);
self.total_cost += item.total_cost_usd;
self.total_response_time_ms += item.response_time_ms.unwrap_or(0) as f64;
}
pub(crate) fn merge(&mut self, other: &Self) {
self.total_requests = self.total_requests.saturating_add(other.total_requests);
self.input_tokens = self.input_tokens.saturating_add(other.input_tokens);
self.output_tokens = self.output_tokens.saturating_add(other.output_tokens);
self.cache_creation_tokens = self
.cache_creation_tokens
.saturating_add(other.cache_creation_tokens);
self.cache_read_tokens = self
.cache_read_tokens
.saturating_add(other.cache_read_tokens);
self.total_cost += other.total_cost;
self.total_response_time_ms += other.total_response_time_ms;
}
pub(crate) fn avg_response_time_ms(&self) -> f64 {
if self.total_requests == 0 {
0.0
} else {
self.total_response_time_ms / self.total_requests as f64
}
}
pub(crate) fn to_json_with_avg(&self, date: String) -> serde_json::Value {
json!({
"date": date,
"total_requests": self.total_requests,
"input_tokens": self.input_tokens,
"output_tokens": self.output_tokens,
"cache_creation_tokens": self.cache_creation_tokens,
"cache_read_tokens": self.cache_read_tokens,
"total_cost": round_to(self.total_cost, 6),
"avg_response_time_ms": round_to(self.avg_response_time_ms(), 2),
})
}
pub(crate) fn to_json_without_avg(&self, date: String) -> serde_json::Value {
json!({
"date": date,
"total_requests": self.total_requests,
"input_tokens": self.input_tokens,
"output_tokens": self.output_tokens,
"cache_creation_tokens": self.cache_creation_tokens,
"cache_read_tokens": self.cache_read_tokens,
"total_cost": round_to(self.total_cost, 6),
})
}
}
impl AdminStatsTimeRange {
pub(crate) fn resolve_optional(query: Option<&str>) -> Result<Option<Self>, String> {
let tz_offset_minutes = parse_tz_offset_minutes(query)?;
let start_date = query_param_value(query, "start_date")
.map(|value| parse_naive_date("start_date", &value))
.transpose()?;
let end_date = query_param_value(query, "end_date")
.map(|value| parse_naive_date("end_date", &value))
.transpose()?;
let preset = query_param_value(query, "preset");
if preset.is_none() && start_date.is_none() && end_date.is_none() {
let default_days = admin_usage_default_days();
if default_days == 0 {
return Ok(None);
}
let end_date = user_today(tz_offset_minutes);
let start_date = end_date
.checked_sub_signed(chrono::Duration::days(
i64::try_from(default_days.saturating_sub(1)).unwrap_or(0),
))
.unwrap_or(end_date);
return Ok(Some(Self {
start_date,
end_date,
tz_offset_minutes,
}));
}
let (start_date, end_date) = match (preset.as_deref(), start_date, end_date) {
(Some(preset), None, None) => resolve_preset_dates(preset, tz_offset_minutes)?,
(None, Some(start_date), Some(end_date)) => (start_date, end_date),
(Some(_), Some(_), _) | (Some(_), _, Some(_)) => {
return Err("preset cannot be combined with start_date or end_date".to_string());
}
_ => {
return Err(
"Either preset or both start_date and end_date must be provided".to_string(),
);
}
};
if start_date > end_date {
return Err("start_date must be <= end_date".to_string());
}
let days = (end_date - start_date).num_days();
if days > 365 {
return Err("Query range cannot exceed 365 days".to_string());
}
Ok(Some(Self {
start_date,
end_date,
tz_offset_minutes,
}))
}
pub(crate) fn resolve_required(
query: Option<&str>,
start_key: &str,
end_key: &str,
) -> Result<Self, String> {
let tz_offset_minutes = parse_tz_offset_minutes(query)?;
let start_date = query_param_value(query, start_key)
.ok_or_else(|| format!("{start_key} is required"))
.and_then(|value| parse_naive_date(start_key, &value))?;
let end_date = query_param_value(query, end_key)
.ok_or_else(|| format!("{end_key} is required"))
.and_then(|value| parse_naive_date(end_key, &value))?;
if start_date > end_date {
return Err(format!("{start_key} must be <= {end_key}"));
}
Ok(Self {
start_date,
end_date,
tz_offset_minutes,
})
}
pub(crate) fn to_unix_bounds(&self) -> Option<(u64, u64)> {
let offset = chrono::Duration::minutes(i64::from(self.tz_offset_minutes));
let start_local = self.start_date.and_hms_opt(0, 0, 0)?;
let end_local = self
.end_date
.checked_add_signed(chrono::Duration::days(1))?
.and_hms_opt(0, 0, 0)?;
let start_utc =
chrono::DateTime::<Utc>::from_naive_utc_and_offset(start_local - offset, Utc)
.timestamp();
let end_utc =
chrono::DateTime::<Utc>::from_naive_utc_and_offset(end_local - offset, Utc).timestamp();
if start_utc < 0 || end_utc <= 0 {
return None;
}
Some((start_utc as u64, end_utc as u64))
}
pub(crate) fn to_utc_datetime_bounds(
&self,
) -> Option<(chrono::DateTime<Utc>, chrono::DateTime<Utc>)> {
let offset = chrono::Duration::minutes(i64::from(self.tz_offset_minutes));
let start_local = self.start_date.and_hms_opt(0, 0, 0)?;
let end_local = self
.end_date
.checked_add_signed(chrono::Duration::days(1))?
.and_hms_opt(0, 0, 0)?;
Some((
chrono::DateTime::<Utc>::from_naive_utc_and_offset(start_local - offset, Utc),
chrono::DateTime::<Utc>::from_naive_utc_and_offset(end_local - offset, Utc),
))
}
pub(crate) fn validate_for_time_series(
&self,
granularity: AdminStatsGranularity,
) -> Result<(), String> {
if granularity == AdminStatsGranularity::Hour && self.start_date != self.end_date {
return Err("Hour granularity only supports single day query".to_string());
}
let days_inclusive = (self.end_date - self.start_date).num_days() + 1;
if days_inclusive > 90 {
return Err(format!(
"Time series query range cannot exceed 90 days (requested {days_inclusive} days). For longer ranges, use aggregated statistics instead."
));
}
Ok(())
}
pub(crate) fn local_dates(&self) -> Vec<chrono::NaiveDate> {
let mut current = self.start_date;
let mut dates = Vec::new();
while current <= self.end_date {
dates.push(current);
let Some(next) = current.checked_add_signed(chrono::Duration::days(1)) else {
break;
};
current = next;
}
dates
}
pub(crate) fn local_date_strings(&self) -> Vec<String> {
self.local_dates()
.into_iter()
.map(|date| date.to_string())
.collect()
}
pub(crate) fn local_date_for_unix_secs(&self, unix_secs: u64) -> Option<chrono::NaiveDate> {
let timestamp = chrono::DateTime::<Utc>::from_timestamp(i64::try_from(unix_secs).ok()?, 0)?;
let local = timestamp
.checked_add_signed(chrono::Duration::minutes(i64::from(self.tz_offset_minutes)))?;
Some(local.date_naive())
}
pub(crate) fn local_date_string_for_unix_secs(&self, unix_secs: u64) -> Option<String> {
Some(self.local_date_for_unix_secs(unix_secs)?.to_string())
}
}
pub(crate) fn round_to(value: f64, decimals: u32) -> f64 {
let factor = 10_f64.powi(i32::try_from(decimals).unwrap_or(0));
(value * factor).round() / factor
}

237
apps/aether-gateway/src/handlers/admin/observability/stats/leaderboard.rs
View File

@@ -1,181 +1,16 @@
use super::helpers::{
AdminStatsLeaderboardItem, AdminStatsLeaderboardMetric, AdminStatsSortOrder,
AdminStatsUserMetadata,
};
use crate::{AppState, GatewayError};
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
use aether_data_contracts::repository::usage::StoredRequestUsageAudit;
pub(super) fn build_model_leaderboard_items(
items: &[StoredRequestUsageAudit],
) -> Vec<AdminStatsLeaderboardItem> {
let mut grouped: std::collections::BTreeMap<String, AdminStatsLeaderboardItem> =
std::collections::BTreeMap::new();
for item in items {
if matches!(item.status.as_str(), "pending" | "streaming")
|| matches!(item.provider_name.as_str(), "unknown" | "pending")
{
continue;
}
let entry =
grouped
.entry(item.model.clone())
.or_insert_with(|| AdminStatsLeaderboardItem {
id: item.model.clone(),
name: item.model.clone(),
requests: 0,
tokens: 0,
cost: 0.0,
});
entry.requests = entry.requests.saturating_add(1);
entry.tokens = entry.tokens.saturating_add(
item.input_tokens
.saturating_add(item.output_tokens)
.saturating_add(item.cache_creation_input_tokens)
.saturating_add(item.cache_read_input_tokens),
);
entry.cost += item.total_cost_usd;
}
grouped.into_values().collect()
}
pub(super) fn build_api_key_leaderboard_items(
items: &[StoredRequestUsageAudit],
snapshots: Option<&[aether_data::repository::auth::StoredAuthApiKeySnapshot]>,
include_inactive: bool,
exclude_admin: bool,
) -> Vec<AdminStatsLeaderboardItem> {
let snapshot_by_api_key_id: std::collections::BTreeMap<_, _> = snapshots
.unwrap_or(&[])
.iter()
.map(|snapshot| (snapshot.api_key_id.as_str(), snapshot))
.collect();
let mut grouped: std::collections::BTreeMap<String, AdminStatsLeaderboardItem> =
std::collections::BTreeMap::new();
let snapshots_available = snapshots.is_some();
for item in items {
if matches!(item.status.as_str(), "pending" | "streaming")
|| matches!(item.provider_name.as_str(), "unknown" | "pending")
{
continue;
}
let Some(api_key_id) = item.api_key_id.as_deref() else {
continue;
};
let entry_name = if let Some(snapshot) = snapshot_by_api_key_id.get(api_key_id) {
if snapshot.user_is_deleted {
continue;
}
if !include_inactive && !snapshot.api_key_is_active {
continue;
}
if exclude_admin && snapshot.user_role.eq_ignore_ascii_case("admin") {
continue;
}
snapshot
.api_key_name
.clone()
.or_else(|| item.api_key_name.clone())
.unwrap_or_else(|| api_key_id.to_string())
} else {
if snapshots_available {
continue;
}
item.api_key_name
.clone()
.unwrap_or_else(|| api_key_id.to_string())
};
let entry =
grouped
.entry(api_key_id.to_string())
.or_insert_with(|| AdminStatsLeaderboardItem {
id: api_key_id.to_string(),
name: entry_name,
requests: 0,
tokens: 0,
cost: 0.0,
});
entry.requests = entry.requests.saturating_add(1);
entry.tokens = entry.tokens.saturating_add(
item.input_tokens
.saturating_add(item.output_tokens)
.saturating_add(item.cache_creation_input_tokens)
.saturating_add(item.cache_read_input_tokens),
);
entry.cost += item.total_cost_usd;
}
grouped.into_values().collect()
}
pub(super) fn build_user_leaderboard_items(
items: &[StoredRequestUsageAudit],
users: &std::collections::BTreeMap<String, AdminStatsUserMetadata>,
include_inactive: bool,
exclude_admin: bool,
) -> Vec<AdminStatsLeaderboardItem> {
let mut grouped: std::collections::BTreeMap<String, AdminStatsLeaderboardItem> =
std::collections::BTreeMap::new();
for item in items {
if matches!(item.status.as_str(), "pending" | "streaming")
|| matches!(item.provider_name.as_str(), "unknown" | "pending")
{
continue;
}
let Some(user_id) = item.user_id.as_deref() else {
continue;
};
let entry_name = if let Some(user) = users.get(user_id) {
if user.is_deleted {
continue;
}
if !include_inactive && !user.is_active {
continue;
}
if exclude_admin && user.role.eq_ignore_ascii_case("admin") {
continue;
}
user.name.clone()
} else {
if exclude_admin {
continue;
}
item.username
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.unwrap_or_else(|| user_id.to_string())
};
let entry =
grouped
.entry(user_id.to_string())
.or_insert_with(|| AdminStatsLeaderboardItem {
id: user_id.to_string(),
name: entry_name,
requests: 0,
tokens: 0,
cost: 0.0,
});
entry.requests = entry.requests.saturating_add(1);
entry.tokens = entry.tokens.saturating_add(
item.input_tokens
.saturating_add(item.output_tokens)
.saturating_add(item.cache_creation_input_tokens)
.saturating_add(item.cache_read_input_tokens),
);
entry.cost += item.total_cost_usd;
}
grouped.into_values().collect()
}
pub(super) use aether_admin::observability::stats::{
build_admin_stats_leaderboard_response, build_api_key_leaderboard_items,
build_model_leaderboard_items, build_user_leaderboard_items, compare_leaderboard_items,
compute_dense_rank, AdminStatsLeaderboardItem, AdminStatsLeaderboardMetric,
AdminStatsLeaderboardNameMode, AdminStatsSortOrder, AdminStatsUserMetadata,
};
pub(super) async fn load_user_leaderboard_metadata(
state: &AppState,
state: &AdminAppState<'_>,
user_ids: &[String],
) -> Result<std::collections::BTreeMap<String, AdminStatsUserMetadata>, GatewayError> {
let mut metadata = std::collections::BTreeMap::new();
@@ -222,57 +57,3 @@ pub(super) async fn load_user_leaderboard_metadata(
Ok(metadata)
}
pub(super) fn compare_leaderboard_items(
metric: AdminStatsLeaderboardMetric,
order: AdminStatsSortOrder,
left: &AdminStatsLeaderboardItem,
right: &AdminStatsLeaderboardItem,
) -> std::cmp::Ordering {
let metric_order = match metric {
AdminStatsLeaderboardMetric::Requests => left.requests.cmp(&right.requests),
AdminStatsLeaderboardMetric::Tokens => left.tokens.cmp(&right.tokens),
AdminStatsLeaderboardMetric::Cost => left
.cost
.partial_cmp(&right.cost)
.unwrap_or(std::cmp::Ordering::Equal),
};
let metric_order = match order {
AdminStatsSortOrder::Asc => metric_order,
AdminStatsSortOrder::Desc => metric_order.reverse(),
};
if metric_order == std::cmp::Ordering::Equal {
left.id.cmp(&right.id)
} else {
metric_order
}
}
fn leaderboard_metric_equal(
metric: AdminStatsLeaderboardMetric,
left: &AdminStatsLeaderboardItem,
right: &AdminStatsLeaderboardItem,
) -> bool {
match metric {
AdminStatsLeaderboardMetric::Requests => left.requests == right.requests,
AdminStatsLeaderboardMetric::Tokens => left.tokens == right.tokens,
AdminStatsLeaderboardMetric::Cost => (left.cost - right.cost).abs() < 1e-9,
}
}
pub(super) fn compute_dense_rank(
metric: AdminStatsLeaderboardMetric,
items: &[AdminStatsLeaderboardItem],
index: usize,
) -> usize {
if index == 0 {
return 1;
}
let mut rank = 1usize;
for current in 1..=index {
if !leaderboard_metric_equal(metric, &items[current - 1], &items[current]) {
rank = rank.saturating_add(1);
}
}
rank
}

198
apps/aether-gateway/src/handlers/admin/observability/stats/leaderboard_routes.rs
View File

@@ -1,38 +1,31 @@
use super::helpers::{
round_to, AdminStatsLeaderboardMetric, AdminStatsSortOrder, AdminStatsTimeRange,
AdminStatsUsageFilter,
};
use super::leaderboard::{
build_api_key_leaderboard_items, build_model_leaderboard_items, build_user_leaderboard_items,
compare_leaderboard_items, compute_dense_rank, load_user_leaderboard_metadata,
build_admin_stats_leaderboard_response, build_api_key_leaderboard_items,
build_model_leaderboard_items, build_user_leaderboard_items, compare_leaderboard_items,
load_user_leaderboard_metadata, AdminStatsLeaderboardNameMode,
};
use super::range::{list_usage_for_optional_range, parse_bounded_u32, parse_nonnegative_usize};
use super::responses::{admin_stats_bad_request_response, admin_stats_leaderboard_empty_response};
use crate::control::GatewayPublicRequestContext;
use super::range::{parse_bounded_u32, parse_nonnegative_usize};
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::{query_param_bool, query_param_value};
use crate::{AppState, GatewayError};
use axum::{
body::Body,
http,
response::{IntoResponse, Response},
Json,
use crate::GatewayError;
use aether_admin::observability::stats::{
admin_stats_bad_request_response, admin_stats_leaderboard_empty_response,
AdminStatsLeaderboardMetric, AdminStatsSortOrder, AdminStatsTimeRange, AdminStatsUsageFilter,
};
use serde_json::json;
use axum::{body::Body, http, response::Response};
pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Option<Response<Body>>, GatewayError> {
let query = request_context.request_query_string.as_deref();
let query = request_context.query_string();
if request_context
.control_decision
.as_ref()
.decision()
.and_then(|decision| decision.route_kind.as_deref())
== Some("leaderboard_models")
&& request_context.request_method == http::Method::GET
&& request_context.method() == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
request_context.path(),
"/api/admin/stats/leaderboard/models" | "/api/admin/stats/leaderboard/models/"
)
{
@@ -71,55 +64,29 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
)));
}
let filters = AdminStatsUsageFilter::from_query(query);
let usage = list_usage_for_optional_range(state, time_range.as_ref(), &filters).await?;
let usage = state
.list_admin_usage_for_optional_range(time_range.as_ref(), &filters)
.await?;
let mut leaderboard = build_model_leaderboard_items(&usage);
leaderboard.sort_by(|left, right| compare_leaderboard_items(metric, order, left, right));
let total = leaderboard.len();
let items: Vec<_> = leaderboard
.iter()
.enumerate()
.skip(offset)
.take(limit)
.map(|(index, item)| {
let rank = compute_dense_rank(metric, &leaderboard, index);
let value = match metric {
AdminStatsLeaderboardMetric::Requests => json!(item.requests),
AdminStatsLeaderboardMetric::Tokens => json!(item.tokens),
AdminStatsLeaderboardMetric::Cost => json!(round_to(item.cost, 6)),
};
json!({
"rank": rank,
"id": item.id,
"name": item.id,
"value": value,
"requests": item.requests,
"tokens": item.tokens,
"cost": round_to(item.cost, 6),
})
})
.collect();
return Ok(Some(
Json(json!({
"items": items,
"total": total,
"metric": metric.as_str(),
"start_date": time_range.as_ref().map(|value| value.start_date.to_string()),
"end_date": time_range.as_ref().map(|value| value.end_date.to_string()),
}))
.into_response(),
));
return Ok(Some(build_admin_stats_leaderboard_response(
metric,
time_range.as_ref(),
&leaderboard,
offset,
limit,
AdminStatsLeaderboardNameMode::Id,
)));
}
if request_context
.control_decision
.as_ref()
.decision()
.and_then(|decision| decision.route_kind.as_deref())
== Some("leaderboard_api_keys")
&& request_context.request_method == http::Method::GET
&& request_context.method() == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
request_context.path(),
"/api/admin/stats/leaderboard/api-keys" | "/api/admin/stats/leaderboard/api-keys/"
)
{
@@ -160,7 +127,9 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
let include_inactive = query_param_bool(query, "include_inactive", false);
let exclude_admin = query_param_bool(query, "exclude_admin", false);
let filters = AdminStatsUsageFilter::from_query(query);
let usage = list_usage_for_optional_range(state, time_range.as_ref(), &filters).await?;
let usage = state
.list_admin_usage_for_optional_range(time_range.as_ref(), &filters)
.await?;
let snapshots = if state.has_auth_api_key_data_reader() {
let api_key_ids: Vec<String> = usage
.iter()
@@ -170,10 +139,8 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
.collect();
Some(
state
.data
.list_auth_api_key_snapshots_by_ids(&api_key_ids)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?,
.await?,
)
} else {
None
@@ -186,51 +153,23 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
);
leaderboard.sort_by(|left, right| compare_leaderboard_items(metric, order, left, right));
let total = leaderboard.len();
let items: Vec<_> = leaderboard
.iter()
.enumerate()
.skip(offset)
.take(limit)
.map(|(index, item)| {
let rank = compute_dense_rank(metric, &leaderboard, index);
let value = match metric {
AdminStatsLeaderboardMetric::Requests => json!(item.requests),
AdminStatsLeaderboardMetric::Tokens => json!(item.tokens),
AdminStatsLeaderboardMetric::Cost => json!(round_to(item.cost, 6)),
};
json!({
"rank": rank,
"id": item.id,
"name": item.name,
"value": value,
"requests": item.requests,
"tokens": item.tokens,
"cost": round_to(item.cost, 6),
})
})
.collect();
return Ok(Some(
Json(json!({
"items": items,
"total": total,
"metric": metric.as_str(),
"start_date": time_range.as_ref().map(|value| value.start_date.to_string()),
"end_date": time_range.as_ref().map(|value| value.end_date.to_string()),
}))
.into_response(),
));
return Ok(Some(build_admin_stats_leaderboard_response(
metric,
time_range.as_ref(),
&leaderboard,
offset,
limit,
AdminStatsLeaderboardNameMode::Name,
)));
}
if request_context
.control_decision
.as_ref()
.decision()
.and_then(|decision| decision.route_kind.as_deref())
== Some("leaderboard_users")
&& request_context.request_method == http::Method::GET
&& request_context.method() == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
request_context.path(),
"/api/admin/stats/leaderboard/users" | "/api/admin/stats/leaderboard/users/"
)
{
@@ -271,7 +210,9 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
let include_inactive = query_param_bool(query, "include_inactive", false);
let exclude_admin = query_param_bool(query, "exclude_admin", false);
let filters = AdminStatsUsageFilter::from_query(query);
let usage = list_usage_for_optional_range(state, time_range.as_ref(), &filters).await?;
let usage = state
.list_admin_usage_for_optional_range(time_range.as_ref(), &filters)
.await?;
let user_ids: Vec<String> = usage
.iter()
.filter_map(|item| item.user_id.clone())
@@ -283,41 +224,14 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
build_user_leaderboard_items(&usage, &user_metadata, include_inactive, exclude_admin);
leaderboard.sort_by(|left, right| compare_leaderboard_items(metric, order, left, right));
let total = leaderboard.len();
let items: Vec<_> = leaderboard
.iter()
.enumerate()
.skip(offset)
.take(limit)
.map(|(index, item)| {
let rank = compute_dense_rank(metric, &leaderboard, index);
let value = match metric {
AdminStatsLeaderboardMetric::Requests => json!(item.requests),
AdminStatsLeaderboardMetric::Tokens => json!(item.tokens),
AdminStatsLeaderboardMetric::Cost => json!(round_to(item.cost, 6)),
};
json!({
"rank": rank,
"id": item.id,
"name": item.name,
"value": value,
"requests": item.requests,
"tokens": item.tokens,
"cost": round_to(item.cost, 6),
})
})
.collect();
return Ok(Some(
Json(json!({
"items": items,
"total": total,
"metric": metric.as_str(),
"start_date": time_range.as_ref().map(|value| value.start_date.to_string()),
"end_date": time_range.as_ref().map(|value| value.end_date.to_string()),
}))
.into_response(),
));
return Ok(Some(build_admin_stats_leaderboard_response(
metric,
time_range.as_ref(),
&leaderboard,
offset,
limit,
AdminStatsLeaderboardNameMode::Name,
)));
}
Ok(None)

37
apps/aether-gateway/src/handlers/admin/observability/stats/mod.rs
View File

@@ -1,29 +1,26 @@
use crate::control::GatewayPublicRequestContext;
use crate::{AppState, GatewayError};
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use axum::{body::Body, response::Response};
mod analytics_routes;
mod cost_routes;
mod helpers;
mod leaderboard;
mod leaderboard_routes;
mod provider_quota_routes;
mod range;
mod responses;
mod timeseries;
pub(crate) use self::helpers::{round_to, AdminStatsTimeRange, AdminStatsUsageFilter};
pub(crate) use self::range::{list_usage_for_optional_range, parse_bounded_u32};
pub(crate) use self::responses::admin_stats_bad_request_response;
pub(crate) use self::timeseries::aggregate_usage_stats;
pub(crate) use self::range::{
list_usage_for_optional_range, list_usage_for_range, parse_bounded_u32,
};
pub(crate) use aether_admin::observability::stats::{
admin_stats_bad_request_response, aggregate_usage_stats, round_to, AdminStatsTimeRange,
AdminStatsUsageFilter,
};
pub(crate) async fn maybe_build_local_admin_stats_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Option<Response<Body>>, GatewayError> {
let Some(decision) = request_context.control_decision.as_ref() else {
return Ok(None);
};
if decision.route_family.as_deref() != Some("stats_manage") {
if request_context.route_family() != Some("stats_manage") {
return Ok(None);
}
@@ -31,19 +28,15 @@ pub(crate) async fn maybe_build_local_admin_stats_response(
provider_quota_routes::maybe_build_local_admin_stats_provider_quota_response(
state,
request_context,
decision,
)
.await?
{
return Ok(Some(response));
}
if let Some(response) = analytics_routes::maybe_build_local_admin_stats_analytics_response(
state,
request_context,
decision,
)
.await?
if let Some(response) =
analytics_routes::maybe_build_local_admin_stats_analytics_response(state, request_context)
.await?
{
return Ok(Some(response));
}

105
apps/aether-gateway/src/handlers/admin/observability/stats/provider_quota_routes.rs
View File

@@ -1,26 +1,20 @@
use super::responses::admin_stats_provider_quota_usage_empty_response;
use crate::control::GatewayControlDecision;
use crate::control::GatewayPublicRequestContext;
use crate::handlers::admin::shared::unix_secs_to_rfc3339;
use crate::{AppState, GatewayError};
use axum::{
body::Body,
http,
response::{IntoResponse, Response},
Json,
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use aether_admin::observability::stats::{
admin_stats_provider_quota_usage_empty_response,
build_admin_stats_provider_quota_usage_response,
};
use chrono::{Datelike, Utc};
use serde_json::json;
use axum::{body::Body, http, response::Response};
use chrono::Utc;
pub(super) async fn maybe_build_local_admin_stats_provider_quota_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
decision: &GatewayControlDecision,
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Option<Response<Body>>, GatewayError> {
if decision.route_kind.as_deref() != Some("provider_quota_usage")
|| request_context.request_method != http::Method::GET
if request_context.route_kind() != Some("provider_quota_usage")
|| request_context.method() != http::Method::GET
|| !matches!(
request_context.request_path.as_str(),
request_context.path(),
"/api/admin/stats/providers/quota-usage" | "/api/admin/stats/providers/quota-usage/"
)
{
@@ -31,74 +25,9 @@ pub(super) async fn maybe_build_local_admin_stats_provider_quota_response(
return Ok(Some(admin_stats_provider_quota_usage_empty_response()));
}
let now = Utc::now();
let now_unix_secs = now.timestamp().max(0) as u64;
let now_day = u64::from(now.day());
let mut providers = state.list_provider_catalog_providers(false).await?;
providers.retain(|provider| {
provider.billing_type.as_deref() == Some("monthly_quota")
|| provider.monthly_quota_usd.is_some()
});
let mut payload: Vec<serde_json::Value> = providers
.into_iter()
.map(|provider| {
let quota = provider.monthly_quota_usd.unwrap_or(0.0);
let used = provider.monthly_used_usd.unwrap_or(0.0);
let remaining = (quota - used).max(0.0);
let usage_percent = if quota > 0.0 {
((used / quota) * 10_000.0).round() / 100.0
} else {
0.0
};
let days_elapsed = provider
.quota_last_reset_at_unix_secs
.map(|reset_at| ((now_unix_secs.saturating_sub(reset_at)) / 86_400).max(1))
.unwrap_or_else(|| now_day.saturating_sub(1).max(1));
let daily_rate = if used > 0.0 {
used / days_elapsed as f64
} else {
0.0
};
let estimated_exhaust_at_unix_secs = if daily_rate > 0.0 && remaining > 0.0 {
let estimated = now_unix_secs
.saturating_add(((remaining / daily_rate) * 86_400.0).max(0.0) as u64);
Some(
provider
.quota_expires_at_unix_secs
.map(|quota_expires_at| quota_expires_at.min(estimated))
.unwrap_or(estimated),
)
} else {
provider.quota_expires_at_unix_secs
};
json!({
"id": provider.id,
"name": provider.name,
"quota_usd": quota,
"used_usd": used,
"remaining_usd": remaining,
"usage_percent": usage_percent,
"quota_expires_at": provider.quota_expires_at_unix_secs.and_then(unix_secs_to_rfc3339),
"estimated_exhaust_at": estimated_exhaust_at_unix_secs.and_then(unix_secs_to_rfc3339),
})
})
.collect();
payload.sort_by(|left, right| {
let left_value = left
.get("usage_percent")
.and_then(serde_json::Value::as_f64)
.unwrap_or(0.0);
let right_value = right
.get("usage_percent")
.and_then(serde_json::Value::as_f64)
.unwrap_or(0.0);
right_value.total_cmp(&left_value)
});
Ok(Some(Json(json!({ "providers": payload })).into_response()))
let providers = state.list_provider_catalog_providers(false).await?;
Ok(Some(build_admin_stats_provider_quota_usage_response(
&providers,
Utc::now(),
)))
}

194
apps/aether-gateway/src/handlers/admin/observability/stats/range.rs
View File

@@ -1,187 +1,15 @@
use super::helpers::{AdminStatsComparisonType, AdminStatsTimeRange, AdminStatsUsageFilter};
use crate::handlers::admin::shared::query_param_value;
use crate::{AppState, GatewayError};
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
pub(crate) use aether_admin::observability::stats::parse_bounded_u32;
pub(super) use aether_admin::observability::stats::{
admin_usage_default_days, build_comparison_range, build_time_range_from_days, parse_naive_date,
parse_nonnegative_usize, parse_tz_offset_minutes, resolve_preset_dates, user_today,
};
use aether_admin::observability::stats::{AdminStatsTimeRange, AdminStatsUsageFilter};
use aether_data_contracts::repository::usage::{StoredRequestUsageAudit, UsageAuditListQuery};
use chrono::{Datelike, Utc};
pub(super) fn parse_tz_offset_minutes(query: Option<&str>) -> Result<i32, String> {
query_param_value(query, "tz_offset_minutes")
.map(|value| {
value
.parse::<i32>()
.map_err(|_| "tz_offset_minutes must be a valid integer".to_string())
})
.transpose()
.map(|value| value.unwrap_or(0))
}
pub(super) fn parse_naive_date(field: &str, value: &str) -> Result<chrono::NaiveDate, String> {
chrono::NaiveDate::parse_from_str(value, "%Y-%m-%d")
.map_err(|_| format!("{field} must be a valid date in YYYY-MM-DD format"))
}
pub(crate) fn parse_bounded_u32(
field: &str,
value: &str,
min: u32,
max: u32,
) -> Result<u32, String> {
let parsed = value
.parse::<u32>()
.map_err(|_| format!("{field} must be a valid integer"))?;
if parsed < min || parsed > max {
return Err(format!("{field} must be between {min} and {max}"));
}
Ok(parsed)
}
pub(super) fn parse_nonnegative_usize(field: &str, value: &str) -> Result<usize, String> {
value
.parse::<usize>()
.map_err(|_| format!("{field} must be a valid integer"))
}
pub(super) fn admin_usage_default_days() -> usize {
match std::env::var("ADMIN_USAGE_DEFAULT_DAYS") {
Ok(value) => value.parse::<usize>().ok().unwrap_or(0),
Err(_) => match std::env::var("ENVIRONMENT") {
Ok(value) if !matches!(value.as_str(), "development" | "test" | "testing") => 30,
_ => 0,
},
}
}
pub(super) fn user_today(tz_offset_minutes: i32) -> chrono::NaiveDate {
(Utc::now() + chrono::Duration::minutes(i64::from(tz_offset_minutes))).date_naive()
}
pub(super) fn resolve_preset_dates(
preset: &str,
tz_offset_minutes: i32,
) -> Result<(chrono::NaiveDate, chrono::NaiveDate), String> {
let user_today = user_today(tz_offset_minutes);
match preset {
"today" => Ok((user_today, user_today)),
"yesterday" => {
let value = user_today
.checked_sub_signed(chrono::Duration::days(1))
.unwrap_or(user_today);
Ok((value, value))
}
"last7days" => Ok((
user_today
.checked_sub_signed(chrono::Duration::days(6))
.unwrap_or(user_today),
user_today,
)),
"last30days" => Ok((
user_today
.checked_sub_signed(chrono::Duration::days(29))
.unwrap_or(user_today),
user_today,
)),
"last90days" => Ok((
user_today
.checked_sub_signed(chrono::Duration::days(89))
.unwrap_or(user_today),
user_today,
)),
"this_week" => {
let week_start = user_today
.checked_sub_signed(chrono::Duration::days(i64::from(
user_today.weekday().num_days_from_monday(),
)))
.unwrap_or(user_today);
Ok((week_start, user_today))
}
"last_week" => {
let this_week_start = user_today
.checked_sub_signed(chrono::Duration::days(i64::from(
user_today.weekday().num_days_from_monday(),
)))
.unwrap_or(user_today);
let last_week_end = this_week_start
.checked_sub_signed(chrono::Duration::days(1))
.unwrap_or(this_week_start);
let last_week_start = last_week_end
.checked_sub_signed(chrono::Duration::days(6))
.unwrap_or(last_week_end);
Ok((last_week_start, last_week_end))
}
"this_month" => {
let start = user_today.with_day(1).unwrap_or(user_today);
Ok((start, user_today))
}
"last_month" => {
let first_of_this_month = user_today.with_day(1).unwrap_or(user_today);
let last_month_end = first_of_this_month
.checked_sub_signed(chrono::Duration::days(1))
.unwrap_or(first_of_this_month);
let last_month_start = last_month_end.with_day(1).unwrap_or(last_month_end);
Ok((last_month_start, last_month_end))
}
"this_year" => {
let start =
chrono::NaiveDate::from_ymd_opt(user_today.year(), 1, 1).unwrap_or(user_today);
Ok((start, user_today))
}
_ => Err("Invalid preset".to_string()),
}
}
pub(super) fn build_time_range_from_days(
days: u32,
tz_offset_minutes: i32,
) -> Result<AdminStatsTimeRange, String> {
let end_date = user_today(tz_offset_minutes);
let start_date = end_date
.checked_sub_signed(chrono::Duration::days(i64::from(days.saturating_sub(1))))
.unwrap_or(end_date);
Ok(AdminStatsTimeRange {
start_date,
end_date,
tz_offset_minutes,
})
}
pub(super) fn build_comparison_range(
current: &AdminStatsTimeRange,
comparison_type: AdminStatsComparisonType,
) -> Result<AdminStatsTimeRange, String> {
let comparison = match comparison_type {
AdminStatsComparisonType::Period => {
let days = (current.end_date - current.start_date).num_days() + 1;
let comparison_end = current
.start_date
.checked_sub_signed(chrono::Duration::days(1))
.ok_or_else(|| "comparison range underflow".to_string())?;
let comparison_start = comparison_end
.checked_sub_signed(chrono::Duration::days(days - 1))
.ok_or_else(|| "comparison range underflow".to_string())?;
(comparison_start, comparison_end)
}
AdminStatsComparisonType::Year => (
safe_year_shift(current.start_date),
safe_year_shift(current.end_date),
),
};
Ok(AdminStatsTimeRange {
start_date: comparison.0,
end_date: comparison.1,
tz_offset_minutes: current.tz_offset_minutes,
})
}
fn safe_year_shift(value: chrono::NaiveDate) -> chrono::NaiveDate {
value
.with_year(value.year() - 1)
.or_else(|| chrono::NaiveDate::from_ymd_opt(value.year() - 1, value.month(), 28))
.unwrap_or(value)
}
pub(super) async fn list_usage_for_range(
state: &AppState,
pub(crate) async fn list_usage_for_range(
state: &AdminAppState<'_>,
time_range: &AdminStatsTimeRange,
filters: &AdminStatsUsageFilter,
) -> Result<Vec<StoredRequestUsageAudit>, GatewayError> {
@@ -202,7 +30,7 @@ pub(super) async fn list_usage_for_range(
}
pub(crate) async fn list_usage_for_optional_range(
state: &AppState,
state: &AdminAppState<'_>,
time_range: Option<&AdminStatsTimeRange>,
filters: &AdminStatsUsageFilter,
) -> Result<Vec<StoredRequestUsageAudit>, GatewayError> {

113
apps/aether-gateway/src/handlers/admin/observability/stats/responses.rs
View File

@@ -1,113 +0,0 @@
use super::helpers::{AdminStatsLeaderboardMetric, AdminStatsTimeRange};
use axum::{
body::Body,
http,
response::{IntoResponse, Response},
Json,
};
use serde_json::json;
pub(super) fn admin_stats_provider_quota_usage_empty_response() -> Response<Body> {
Json(json!({
"providers": [],
"data_source_available": false,
}))
.into_response()
}
pub(super) fn admin_stats_cost_forecast_empty_response() -> Response<Body> {
Json(json!({
"history": [],
"forecast": [],
"slope": 0.0,
"intercept": 0.0,
"data_source_available": false,
}))
.into_response()
}
pub(super) fn admin_stats_comparison_empty_response(
current_range: &AdminStatsTimeRange,
comparison_range: &AdminStatsTimeRange,
) -> Response<Body> {
Json(json!({
"current": {
"total_requests": 0,
"total_tokens": 0,
"total_cost": 0.0,
"actual_total_cost": 0.0,
"avg_response_time_ms": 0.0,
"error_requests": 0,
},
"comparison": {
"total_requests": 0,
"total_tokens": 0,
"total_cost": 0.0,
"actual_total_cost": 0.0,
"avg_response_time_ms": 0.0,
"error_requests": 0,
},
"change_percent": {
"total_requests": serde_json::Value::Null,
"total_tokens": serde_json::Value::Null,
"total_cost": serde_json::Value::Null,
"actual_total_cost": serde_json::Value::Null,
"avg_response_time_ms": serde_json::Value::Null,
"error_requests": serde_json::Value::Null,
},
"current_start": current_range.start_date.to_string(),
"current_end": current_range.end_date.to_string(),
"comparison_start": comparison_range.start_date.to_string(),
"comparison_end": comparison_range.end_date.to_string(),
}))
.into_response()
}
pub(super) fn admin_stats_error_distribution_empty_response() -> Response<Body> {
Json(json!({
"distribution": [],
"trend": [],
}))
.into_response()
}
pub(super) fn admin_stats_performance_percentiles_empty_response() -> Response<Body> {
Json(json!([])).into_response()
}
pub(super) fn admin_stats_cost_savings_empty_response() -> Response<Body> {
Json(json!({
"cache_read_tokens": 0,
"cache_read_cost": 0.0,
"cache_creation_cost": 0.0,
"estimated_full_cost": 0.0,
"cache_savings": 0.0,
}))
.into_response()
}
pub(super) fn admin_stats_leaderboard_empty_response(
metric: AdminStatsLeaderboardMetric,
time_range: Option<&AdminStatsTimeRange>,
) -> Response<Body> {
Json(json!({
"items": [],
"total": 0,
"metric": metric.as_str(),
"start_date": time_range.map(|value| value.start_date.to_string()),
"end_date": time_range.map(|value| value.end_date.to_string()),
}))
.into_response()
}
pub(super) fn admin_stats_time_series_empty_response() -> Response<Body> {
Json(json!([])).into_response()
}
pub(crate) fn admin_stats_bad_request_response(detail: String) -> Response<Body> {
(
http::StatusCode::BAD_REQUEST,
Json(json!({ "detail": detail })),
)
.into_response()
}

213
apps/aether-gateway/src/handlers/admin/observability/stats/timeseries.rs
View File

@@ -1,213 +0,0 @@
use super::helpers::{
round_to, AdminStatsAggregate, AdminStatsGranularity, AdminStatsTimeRange,
AdminStatsTimeSeriesBucket, MIN_PERCENTILE_SAMPLES,
};
use aether_data_contracts::repository::usage::StoredRequestUsageAudit;
use chrono::{Datelike, Utc};
use serde_json::json;
pub(super) fn build_time_series_payload(
time_range: &AdminStatsTimeRange,
granularity: AdminStatsGranularity,
items: &[StoredRequestUsageAudit],
) -> Vec<serde_json::Value> {
match granularity {
AdminStatsGranularity::Hour => build_hourly_time_series_payload(time_range, items),
AdminStatsGranularity::Day => build_daily_time_series_payload(time_range, items),
AdminStatsGranularity::Week => build_weekly_time_series_payload(time_range, items),
AdminStatsGranularity::Month => build_monthly_time_series_payload(time_range, items),
}
}
pub(super) fn build_daily_time_series_buckets(
time_range: &AdminStatsTimeRange,
items: &[StoredRequestUsageAudit],
) -> std::collections::BTreeMap<chrono::NaiveDate, AdminStatsTimeSeriesBucket> {
let mut buckets: std::collections::BTreeMap<chrono::NaiveDate, AdminStatsTimeSeriesBucket> =
time_range
.local_dates()
.into_iter()
.map(|date| (date, AdminStatsTimeSeriesBucket::default()))
.collect();
for item in items {
let Some(local_day) = time_range.local_date_for_unix_secs(item.created_at_unix_secs) else {
continue;
};
let Some(bucket) = buckets.get_mut(&local_day) else {
continue;
};
bucket.add_usage(item);
}
buckets
}
fn build_daily_time_series_payload(
time_range: &AdminStatsTimeRange,
items: &[StoredRequestUsageAudit],
) -> Vec<serde_json::Value> {
build_daily_time_series_buckets(time_range, items)
.into_iter()
.map(|(date, bucket)| bucket.to_json_with_avg(date.to_string()))
.collect()
}
fn build_weekly_time_series_payload(
time_range: &AdminStatsTimeRange,
items: &[StoredRequestUsageAudit],
) -> Vec<serde_json::Value> {
let mut weekly: std::collections::BTreeMap<
(i32, u32),
(chrono::NaiveDate, AdminStatsTimeSeriesBucket),
> = std::collections::BTreeMap::new();
for (date, bucket) in build_daily_time_series_buckets(time_range, items) {
let iso = date.iso_week();
let entry = weekly
.entry((iso.year(), iso.week()))
.or_insert_with(|| (date, AdminStatsTimeSeriesBucket::default()));
entry.0 = entry.0.min(date);
entry.1.merge(&bucket);
}
weekly
.into_values()
.map(|(date, bucket)| bucket.to_json_with_avg(date.to_string()))
.collect()
}
fn build_monthly_time_series_payload(
time_range: &AdminStatsTimeRange,
items: &[StoredRequestUsageAudit],
) -> Vec<serde_json::Value> {
let mut monthly: std::collections::BTreeMap<
(i32, u32),
(chrono::NaiveDate, AdminStatsTimeSeriesBucket),
> = std::collections::BTreeMap::new();
for (date, bucket) in build_daily_time_series_buckets(time_range, items) {
let Some(month_start) = chrono::NaiveDate::from_ymd_opt(date.year(), date.month(), 1)
else {
continue;
};
let entry = monthly
.entry((date.year(), date.month()))
.or_insert_with(|| (month_start, AdminStatsTimeSeriesBucket::default()));
entry.1.merge(&bucket);
}
monthly
.into_values()
.map(|(date, bucket)| bucket.to_json_with_avg(date.to_string()))
.collect()
}
fn build_hourly_time_series_payload(
time_range: &AdminStatsTimeRange,
items: &[StoredRequestUsageAudit],
) -> Vec<serde_json::Value> {
let Some((mut current, end)) = time_range.to_utc_datetime_bounds() else {
return Vec::new();
};
let offset = chrono::Duration::minutes(i64::from(time_range.tz_offset_minutes));
let mut buckets: std::collections::BTreeMap<String, AdminStatsTimeSeriesBucket> =
std::collections::BTreeMap::new();
while current < end {
let label = (current + offset)
.format("%Y-%m-%dT%H:00:00+00:00")
.to_string();
buckets.insert(label, AdminStatsTimeSeriesBucket::default());
let Some(next) = current.checked_add_signed(chrono::Duration::hours(1)) else {
break;
};
current = next;
}
for item in items {
let Some(unix_secs) = i64::try_from(item.created_at_unix_secs).ok() else {
continue;
};
let Some(timestamp) = chrono::DateTime::<Utc>::from_timestamp(unix_secs, 0) else {
continue;
};
let Some(local) = timestamp.checked_add_signed(offset) else {
continue;
};
let label = local.format("%Y-%m-%dT%H:00:00+00:00").to_string();
let Some(bucket) = buckets.get_mut(&label) else {
continue;
};
bucket.add_usage(item);
}
buckets
.into_iter()
.map(|(date, bucket)| bucket.to_json_without_avg(date))
.collect()
}
pub(crate) fn aggregate_usage_stats(items: &[StoredRequestUsageAudit]) -> AdminStatsAggregate {
let mut aggregate = AdminStatsAggregate::default();
for item in items {
aggregate.total_requests = aggregate.total_requests.saturating_add(1);
aggregate.total_tokens = aggregate.total_tokens.saturating_add(item.total_tokens);
aggregate.total_cost += item.total_cost_usd;
aggregate.actual_total_cost += item.actual_total_cost_usd;
aggregate.total_response_time_ms += item.response_time_ms.unwrap_or(0) as f64;
if item.status_code.is_some_and(|value| value >= 400) || item.error_message.is_some() {
aggregate.error_requests = aggregate.error_requests.saturating_add(1);
}
}
aggregate
}
pub(super) fn percentile_cont(values: &mut [u64], percentile: f64) -> Option<u64> {
if values.len() < MIN_PERCENTILE_SAMPLES {
return None;
}
values.sort_unstable();
let position = percentile * (values.len().saturating_sub(1)) as f64;
let lower = position.floor() as usize;
let upper = position.ceil() as usize;
let lower_value = values[lower] as f64;
let upper_value = values[upper] as f64;
Some((lower_value + (upper_value - lower_value) * (position - lower as f64)).trunc() as u64)
}
pub(super) fn pct_change_value(current: f64, previous: f64) -> serde_json::Value {
if previous == 0.0 {
if current == 0.0 {
json!(0.0)
} else {
serde_json::Value::Null
}
} else {
json!(round_to((current - previous) / previous * 100.0, 2))
}
}
pub(super) fn linear_regression(values: &[f64]) -> (f64, f64) {
let n = values.len();
if n <= 1 {
return (0.0, values.first().copied().unwrap_or(0.0));
}
let sum_x: f64 = (0..n).map(|value| value as f64).sum();
let sum_y: f64 = values.iter().sum();
let sum_x2: f64 = (0..n).map(|value| (value * value) as f64).sum();
let sum_xy: f64 = values
.iter()
.enumerate()
.map(|(index, value)| index as f64 * *value)
.sum();
let n = n as f64;
let denom = n * sum_x2 - sum_x * sum_x;
if denom == 0.0 {
return (0.0, values.last().copied().unwrap_or(0.0));
}
let slope = (n * sum_xy - sum_x * sum_y) / denom;
let intercept = (sum_y - slope * sum_x) / n;
(slope, intercept)
}

777
apps/aether-gateway/src/handlers/admin/observability/usage/analytics.rs
View File

@@ -1,777 +0,0 @@
use super::super::stats::{parse_bounded_u32, round_to};
use crate::handlers::admin::shared::{query_param_value, unix_secs_to_rfc3339};
use crate::{AppState, GatewayError};
use aether_data_contracts::repository::usage::{StoredRequestUsageAudit, UsageAuditListQuery};
use serde_json::{json, Value};
use std::collections::{BTreeMap, BTreeSet};
pub(super) fn admin_usage_total_tokens(item: &StoredRequestUsageAudit) -> u64 {
item.input_tokens
.saturating_add(item.output_tokens)
.saturating_add(item.cache_creation_input_tokens)
.saturating_add(item.cache_read_input_tokens)
}
pub(super) fn admin_usage_parse_limit(query: Option<&str>) -> Result<usize, String> {
match query_param_value(query, "limit") {
None => Ok(100),
Some(value) => {
let parsed = value
.parse::<usize>()
.map_err(|_| "limit must be a positive integer".to_string())?;
if parsed == 0 || parsed > 500 {
return Err("limit must be between 1 and 500".to_string());
}
Ok(parsed)
}
}
}
pub(super) fn admin_usage_parse_offset(query: Option<&str>) -> Result<usize, String> {
match query_param_value(query, "offset") {
None => Ok(0),
Some(value) => value
.parse::<usize>()
.map_err(|_| "offset must be a non-negative integer".to_string()),
}
}
pub(super) fn admin_usage_parse_ids(query: Option<&str>) -> Option<BTreeSet<String>> {
let ids = query_param_value(query, "ids")?;
let parsed: BTreeSet<String> = ids
.split(',')
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.collect();
Some(parsed)
}
pub(super) fn admin_usage_parse_recent_hours(
query: Option<&str>,
default: u32,
) -> Result<u32, String> {
match query_param_value(query, "hours") {
Some(value) => parse_bounded_u32("hours", &value, 1, 720),
None => Ok(default),
}
}
pub(super) fn admin_usage_parse_timeline_limit(query: Option<&str>) -> Result<usize, String> {
match query_param_value(query, "limit") {
Some(value) => {
let parsed = value
.parse::<usize>()
.map_err(|_| "limit must be an integer between 100 and 50000".to_string())?;
if (100..=50_000).contains(&parsed) {
Ok(parsed)
} else {
Err("limit must be an integer between 100 and 50000".to_string())
}
}
None => Ok(3_000),
}
}
pub(super) fn admin_usage_parse_aggregation_limit(query: Option<&str>) -> Result<usize, String> {
match query_param_value(query, "limit") {
Some(value) => {
let parsed = value
.parse::<usize>()
.map_err(|_| "limit must be an integer between 1 and 100".to_string())?;
if (1..=100).contains(&parsed) {
Ok(parsed)
} else {
Err("limit must be an integer between 1 and 100".to_string())
}
}
None => Ok(20),
}
}
pub(super) fn admin_usage_token_cache_hit_rate(input_tokens: u64, cache_read_tokens: u64) -> f64 {
let total_input_context = input_tokens.saturating_add(cache_read_tokens);
if total_input_context == 0 {
0.0
} else {
round_to(
cache_read_tokens as f64 / total_input_context as f64 * 100.0,
2,
)
}
}
pub(super) fn admin_usage_aggregation_by_model_json(
usage: &[StoredRequestUsageAudit],
limit: usize,
) -> serde_json::Value {
let mut grouped: BTreeMap<String, (u64, u64, u64, u64, f64, f64)> = BTreeMap::new();
for item in usage {
let key = item.model.clone();
let entry = grouped.entry(key).or_insert((0, 0, 0, 0, 0.0, 0.0));
entry.0 = entry.0.saturating_add(1);
entry.1 = entry.1.saturating_add(item.total_tokens);
entry.2 = entry.2.saturating_add(item.input_tokens);
entry.3 = entry.3.saturating_add(item.cache_read_input_tokens);
entry.4 += item.total_cost_usd;
entry.5 += item.actual_total_cost_usd;
}
let mut items: Vec<serde_json::Value> = grouped
.into_iter()
.map(
|(model, (request_count, total_tokens, input_tokens, cache_read_tokens, total_cost, actual_cost))| {
json!({
"model": model,
"request_count": request_count,
"total_tokens": total_tokens,
"total_input_context": input_tokens.saturating_add(cache_read_tokens),
"output_tokens": total_tokens.saturating_sub(input_tokens),
"total_cost": round_to(total_cost, 6),
"actual_cost": round_to(actual_cost, 6),
"cache_read_tokens": cache_read_tokens,
"cache_creation_tokens": 0,
"cache_hit_rate": admin_usage_token_cache_hit_rate(input_tokens, cache_read_tokens),
})
},
)
.collect();
items.sort_by(|left, right| {
right["request_count"]
.as_u64()
.unwrap_or_default()
.cmp(&left["request_count"].as_u64().unwrap_or_default())
.then_with(|| {
left["model"]
.as_str()
.unwrap_or_default()
.cmp(right["model"].as_str().unwrap_or_default())
})
});
items.truncate(limit);
json!(items)
}
pub(super) async fn admin_usage_aggregation_by_user_json(
state: &AppState,
usage: &[StoredRequestUsageAudit],
limit: usize,
) -> Result<serde_json::Value, GatewayError> {
let mut grouped: BTreeMap<String, (u64, u64, f64)> = BTreeMap::new();
for item in usage {
let Some(user_id) = item.user_id.as_ref() else {
continue;
};
let entry = grouped.entry(user_id.clone()).or_insert((0, 0, 0.0));
entry.0 = entry.0.saturating_add(1);
entry.1 = entry.1.saturating_add(item.total_tokens);
entry.2 += item.total_cost_usd;
}
let usernames = if state.has_user_data_reader() && !grouped.is_empty() {
state
.list_users_by_ids(&grouped.keys().cloned().collect::<Vec<_>>())
.await?
.into_iter()
.map(|user| (user.id, (user.email, user.username)))
.collect::<BTreeMap<_, _>>()
} else {
BTreeMap::new()
};
let mut items: Vec<serde_json::Value> = grouped
.into_iter()
.map(|(user_id, (request_count, total_tokens, total_cost))| {
let (email, username) = usernames
.get(&user_id)
.cloned()
.unwrap_or((None, String::new()));
json!({
"user_id": user_id,
"email": email,
"username": if username.is_empty() { serde_json::Value::Null } else { json!(username) },
"request_count": request_count,
"total_tokens": total_tokens,
"total_cost": round_to(total_cost, 6),
})
})
.collect();
items.sort_by(|left, right| {
right["request_count"]
.as_u64()
.unwrap_or_default()
.cmp(&left["request_count"].as_u64().unwrap_or_default())
.then_with(|| {
left["user_id"]
.as_str()
.unwrap_or_default()
.cmp(right["user_id"].as_str().unwrap_or_default())
})
});
items.truncate(limit);
Ok(json!(items))
}
pub(super) fn admin_usage_aggregation_by_provider_json(
usage: &[StoredRequestUsageAudit],
limit: usize,
) -> serde_json::Value {
let mut grouped: BTreeMap<String, (u64, u64, u64, u64, f64, f64, u64, u64)> = BTreeMap::new();
for item in usage {
let key = item
.provider_id
.clone()
.unwrap_or_else(|| "unknown".to_string());
let entry = grouped.entry(key).or_insert((0, 0, 0, 0, 0.0, 0.0, 0, 0));
entry.0 = entry.0.saturating_add(1);
entry.1 = entry.1.saturating_add(item.total_tokens);
entry.2 = entry.2.saturating_add(item.input_tokens);
entry.3 = entry.3.saturating_add(item.cache_read_input_tokens);
entry.4 += item.total_cost_usd;
entry.5 += item.actual_total_cost_usd;
entry.6 = entry
.6
.saturating_add(item.response_time_ms.unwrap_or_default());
entry.7 = entry
.7
.saturating_add(if admin_usage_is_success(item) { 1 } else { 0 });
}
let mut items: Vec<serde_json::Value> = grouped
.into_iter()
.map(
|(provider_id, (request_count, total_tokens, input_tokens, cache_read_tokens, total_cost, actual_cost, response_time_ms_sum, success_count))| {
let avg_response_time_ms = if request_count == 0 {
0.0
} else {
round_to(response_time_ms_sum as f64 / request_count as f64, 2)
};
let error_count = request_count.saturating_sub(success_count);
let success_rate = if request_count == 0 {
0.0
} else {
round_to(success_count as f64 / request_count as f64 * 100.0, 2)
};
json!({
"provider_id": provider_id,
"provider": serde_json::Value::Null,
"request_count": request_count,
"total_tokens": total_tokens,
"total_input_context": input_tokens.saturating_add(cache_read_tokens),
"output_tokens": total_tokens.saturating_sub(input_tokens),
"total_cost": round_to(total_cost, 6),
"actual_cost": round_to(actual_cost, 6),
"avg_response_time_ms": avg_response_time_ms,
"success_rate": success_rate,
"error_count": error_count,
"cache_read_tokens": cache_read_tokens,
"cache_creation_tokens": 0,
"cache_hit_rate": admin_usage_token_cache_hit_rate(input_tokens, cache_read_tokens),
})
},
)
.collect();
items.sort_by(|left, right| {
right["request_count"]
.as_u64()
.unwrap_or_default()
.cmp(&left["request_count"].as_u64().unwrap_or_default())
.then_with(|| {
left["provider_id"]
.as_str()
.unwrap_or_default()
.cmp(right["provider_id"].as_str().unwrap_or_default())
})
});
items.truncate(limit);
json!(items)
}
pub(super) fn admin_usage_aggregation_by_api_format_json(
usage: &[StoredRequestUsageAudit],
limit: usize,
) -> serde_json::Value {
let mut grouped: BTreeMap<String, (u64, u64, u64, u64, f64, f64, u64)> = BTreeMap::new();
for item in usage {
let key = item
.api_format
.clone()
.unwrap_or_else(|| "unknown".to_string());
let entry = grouped.entry(key).or_insert((0, 0, 0, 0, 0.0, 0.0, 0));
entry.0 = entry.0.saturating_add(1);
entry.1 = entry.1.saturating_add(item.total_tokens);
entry.2 = entry.2.saturating_add(item.input_tokens);
entry.3 = entry.3.saturating_add(item.cache_read_input_tokens);
entry.4 += item.total_cost_usd;
entry.5 += item.actual_total_cost_usd;
entry.6 = entry
.6
.saturating_add(item.response_time_ms.unwrap_or_default());
}
let mut items: Vec<serde_json::Value> = grouped
.into_iter()
.map(
|(api_format, (request_count, total_tokens, input_tokens, cache_read_tokens, total_cost, actual_cost, response_time_ms_sum))| {
let avg_response_time_ms = if request_count == 0 {
0.0
} else {
round_to(response_time_ms_sum as f64 / request_count as f64, 2)
};
json!({
"api_format": api_format,
"request_count": request_count,
"total_tokens": total_tokens,
"total_input_context": input_tokens.saturating_add(cache_read_tokens),
"output_tokens": total_tokens.saturating_sub(input_tokens),
"total_cost": round_to(total_cost, 6),
"actual_cost": round_to(actual_cost, 6),
"avg_response_time_ms": avg_response_time_ms,
"cache_read_tokens": cache_read_tokens,
"cache_creation_tokens": 0,
"cache_hit_rate": admin_usage_token_cache_hit_rate(input_tokens, cache_read_tokens),
})
},
)
.collect();
items.sort_by(|left, right| {
right["request_count"]
.as_u64()
.unwrap_or_default()
.cmp(&left["request_count"].as_u64().unwrap_or_default())
.then_with(|| {
left["api_format"]
.as_str()
.unwrap_or_default()
.cmp(right["api_format"].as_str().unwrap_or_default())
})
});
items.truncate(limit);
json!(items)
}
pub(super) fn admin_usage_heatmap_json(usage: &[StoredRequestUsageAudit]) -> serde_json::Value {
let today = chrono::Utc::now().date_naive();
let start_date = today
.checked_sub_signed(chrono::Duration::days(364))
.unwrap_or(today);
let mut grouped: BTreeMap<chrono::NaiveDate, (u64, u64, f64, f64)> = BTreeMap::new();
for item in usage {
let Ok(created_at_unix_secs) = i64::try_from(item.created_at_unix_secs) else {
continue;
};
let Some(created_at) =
chrono::DateTime::<chrono::Utc>::from_timestamp(created_at_unix_secs, 0)
else {
continue;
};
let date_key = created_at.date_naive();
if date_key < start_date || date_key > today {
continue;
}
let entry = grouped.entry(date_key).or_insert((0, 0, 0.0, 0.0));
entry.0 = entry.0.saturating_add(1);
entry.1 = entry.1.saturating_add(admin_usage_total_tokens(item));
entry.2 += item.total_cost_usd;
entry.3 += item.actual_total_cost_usd;
}
let mut max_requests = 0_u64;
let mut cursor = start_date;
let mut days = Vec::new();
while cursor <= today {
let (requests, total_tokens, total_cost, actual_total_cost) =
grouped.get(&cursor).copied().unwrap_or((0, 0, 0.0, 0.0));
max_requests = max_requests.max(requests);
days.push(json!({
"date": cursor.to_string(),
"requests": requests,
"total_tokens": total_tokens,
"total_cost": round_to(total_cost, 6),
"actual_total_cost": round_to(actual_total_cost, 6),
}));
cursor = cursor
.checked_add_signed(chrono::Duration::days(1))
.unwrap_or(today + chrono::Duration::days(1));
}
json!({
"start_date": start_date.to_string(),
"end_date": today.to_string(),
"total_days": days.len(),
"max_requests": max_requests,
"days": days,
})
}
pub(super) fn admin_usage_is_success(item: &StoredRequestUsageAudit) -> bool {
matches!(
item.status.as_str(),
"completed" | "success" | "ok" | "billed" | "settled"
) && item.status_code.is_none_or(|code| code < 400)
}
pub(super) fn admin_usage_matches_optional_id(value: Option<&str>, expected: Option<&str>) -> bool {
let Some(expected) = expected.map(str::trim).filter(|value| !value.is_empty()) else {
return true;
};
value.is_some_and(|candidate| candidate == expected)
}
pub(super) async fn list_recent_completed_usage_for_cache_affinity(
state: &AppState,
hours: u32,
user_id: Option<&str>,
) -> Result<Vec<StoredRequestUsageAudit>, GatewayError> {
let now_unix_secs = u64::try_from(chrono::Utc::now().timestamp()).unwrap_or_default();
let created_from_unix_secs = now_unix_secs.saturating_sub(u64::from(hours) * 3600);
let mut items = state
.list_usage_audits(&UsageAuditListQuery {
created_from_unix_secs: Some(created_from_unix_secs),
created_until_unix_secs: None,
user_id: user_id.map(ToOwned::to_owned),
provider_name: None,
model: None,
})
.await?;
items.retain(|item| item.status == "completed");
items.sort_by(|left, right| {
left.created_at_unix_secs
.cmp(&right.created_at_unix_secs)
.then_with(|| left.id.cmp(&right.id))
});
Ok(items)
}
pub(super) fn admin_usage_group_completed_by_user(
items: &[StoredRequestUsageAudit],
) -> BTreeMap<String, Vec<StoredRequestUsageAudit>> {
let mut grouped = BTreeMap::new();
for item in items.iter().filter(|item| item.user_id.is_some()) {
grouped
.entry(item.user_id.clone().unwrap_or_default())
.or_insert_with(Vec::new)
.push(item.clone());
}
grouped
}
pub(super) fn admin_usage_group_completed_by_api_key(
items: &[StoredRequestUsageAudit],
api_key_id: Option<&str>,
) -> BTreeMap<String, Vec<StoredRequestUsageAudit>> {
let mut grouped = BTreeMap::new();
for item in items.iter().filter(|item| item.api_key_id.is_some()) {
if !admin_usage_matches_optional_id(item.api_key_id.as_deref(), api_key_id) {
continue;
}
grouped
.entry(item.api_key_id.clone().unwrap_or_default())
.or_insert_with(Vec::new)
.push(item.clone());
}
grouped
}
pub(super) fn admin_usage_collect_request_intervals_minutes(
items: &[StoredRequestUsageAudit],
) -> Vec<f64> {
let mut previous_created_at_unix_secs = None;
let mut intervals = Vec::new();
for item in items {
if let Some(previous) = previous_created_at_unix_secs {
intervals.push(item.created_at_unix_secs.saturating_sub(previous) as f64 / 60.0);
}
previous_created_at_unix_secs = Some(item.created_at_unix_secs);
}
intervals
}
pub(super) fn admin_usage_percentile_cont(values: &[f64], percentile: f64) -> Option<f64> {
if values.is_empty() {
return None;
}
if values.len() == 1 {
return Some(values[0]);
}
let position = percentile.clamp(0.0, 1.0) * (values.len() - 1) as f64;
let lower_index = position.floor() as usize;
let upper_index = position.ceil() as usize;
let lower = values[lower_index];
let upper = values[upper_index];
Some(lower + (upper - lower) * (position - lower_index as f64))
}
pub(super) fn admin_usage_calculate_recommended_ttl(
p75_interval: Option<f64>,
p90_interval: Option<f64>,
) -> u64 {
let Some(p75_interval) = p75_interval else {
return 5;
};
let Some(p90_interval) = p90_interval else {
return 5;
};
if p90_interval <= 5.0 {
5
} else if p75_interval <= 15.0 {
15
} else if p75_interval <= 30.0 {
30
} else {
60
}
}
pub(super) fn admin_usage_ttl_recommendation_reason(
ttl: u64,
p75_interval: Option<f64>,
p90_interval: Option<f64>,
) -> String {
let Some(p75_interval) = p75_interval else {
return "数据不足，使用默认值".to_string();
};
let Some(p90_interval) = p90_interval else {
return "数据不足，使用默认值".to_string();
};
match ttl {
5 => format!("高频用户：90% 的请求间隔在 {:.1} 分钟内", p90_interval),
15 => format!("中高频用户：75% 的请求间隔在 {:.1} 分钟内", p75_interval),
30 => format!("中频用户：75% 的请求间隔在 {:.1} 分钟内", p75_interval),
_ => format!(
"低频用户：75% 的请求间隔为 {:.1} 分钟，建议使用长 TTL",
p75_interval
),
}
}
pub(super) fn admin_usage_proportional_limits(
grouped: &BTreeMap<String, Vec<serde_json::Value>>,
limit: usize,
total_points: usize,
) -> BTreeMap<String, usize> {
let mut limits = BTreeMap::new();
for (group_id, items) in grouped {
let computed = if total_points <= limit || total_points == 0 {
items.len()
} else {
let scaled =
((items.len() as f64 * limit as f64) / total_points as f64).ceil() as usize;
std::cmp::max(scaled, 1)
};
limits.insert(group_id.clone(), computed);
}
limits
}
pub(super) fn admin_usage_point_sort_key(
left: &serde_json::Value,
right: &serde_json::Value,
) -> std::cmp::Ordering {
left["x"]
.as_str()
.unwrap_or_default()
.cmp(right["x"].as_str().unwrap_or_default())
.then_with(|| {
left["user_id"]
.as_str()
.unwrap_or_default()
.cmp(right["user_id"].as_str().unwrap_or_default())
})
}
pub(super) fn admin_usage_matches_search(
item: &StoredRequestUsageAudit,
search: Option<&str>,
) -> bool {
let Some(search) = search.map(str::trim).filter(|value| !value.is_empty()) else {
return true;
};
let haystack = [
item.username.as_deref(),
item.api_key_name.as_deref(),
Some(item.model.as_str()),
Some(item.provider_name.as_str()),
];
search.split_whitespace().all(|keyword| {
let keyword = keyword.to_ascii_lowercase();
haystack
.iter()
.flatten()
.any(|value| value.to_ascii_lowercase().contains(keyword.as_str()))
})
}
pub(super) fn admin_usage_matches_username(
item: &StoredRequestUsageAudit,
username: Option<&str>,
) -> bool {
let Some(username) = username.map(str::trim).filter(|value| !value.is_empty()) else {
return true;
};
item.username
.as_deref()
.unwrap_or_default()
.to_ascii_lowercase()
.contains(username.to_ascii_lowercase().as_str())
}
pub(super) fn admin_usage_matches_eq(value: &str, query: Option<&str>) -> bool {
let Some(query) = query
.map(str::trim)
.filter(|candidate| !candidate.is_empty())
else {
return true;
};
value.eq_ignore_ascii_case(query)
}
pub(super) fn admin_usage_matches_api_format(
item: &StoredRequestUsageAudit,
api_format: Option<&str>,
) -> bool {
let Some(api_format) = api_format.map(str::trim).filter(|value| !value.is_empty()) else {
return true;
};
item.api_format
.as_deref()
.is_some_and(|value| value.eq_ignore_ascii_case(api_format))
}
pub(super) fn admin_usage_matches_status(
item: &StoredRequestUsageAudit,
status: Option<&str>,
) -> bool {
let Some(status) = status.map(str::trim).filter(|value| !value.is_empty()) else {
return true;
};
match status {
"stream" => item.is_stream,
"standard" => !item.is_stream,
"error" => {
item.status_code.is_some_and(|value| value >= 400) || item.error_message.is_some()
}
"pending" | "streaming" | "completed" | "cancelled" => item.status == status,
"failed" => {
item.status == "failed"
|| item.status_code.is_some_and(|value| value >= 400)
|| item.error_message.is_some()
}
"active" => matches!(item.status.as_str(), "pending" | "streaming"),
_ => true,
}
}
pub(super) async fn admin_usage_provider_key_names(
state: &AppState,
usage: &[StoredRequestUsageAudit],
) -> Result<BTreeMap<String, String>, GatewayError> {
if !state.has_provider_catalog_data_reader() {
return Ok(BTreeMap::new());
}
let key_ids = usage
.iter()
.filter_map(|item| item.provider_api_key_id.clone())
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
if key_ids.is_empty() {
return Ok(BTreeMap::new());
}
Ok(state
.list_provider_catalog_keys_by_ids(&key_ids)
.await?
.into_iter()
.map(|key| (key.id, key.name))
.collect())
}
fn admin_usage_request_metadata_string(
item: &StoredRequestUsageAudit,
key: &str,
) -> Option<String> {
item.request_metadata
.as_ref()
.and_then(Value::as_object)
.and_then(|metadata| metadata.get(key))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
pub(super) fn admin_usage_provider_key_name(
item: &StoredRequestUsageAudit,
provider_key_names: &BTreeMap<String, String>,
) -> Option<String> {
item.provider_api_key_id
.as_ref()
.and_then(|key_id| provider_key_names.get(key_id))
.cloned()
.or_else(|| admin_usage_request_metadata_string(item, "key_name"))
}
pub(super) fn admin_usage_record_json(
item: &StoredRequestUsageAudit,
users_by_id: &BTreeMap<String, aether_data::repository::users::StoredUserSummary>,
provider_key_name: Option<&str>,
) -> Value {
let user = item
.user_id
.as_ref()
.and_then(|user_id| users_by_id.get(user_id));
let username = user
.map(|value| value.username.clone())
.or_else(|| item.username.clone())
.unwrap_or_else(|| "已删除用户".to_string());
let user_email = user
.and_then(|value| value.email.clone())
.unwrap_or_else(|| "已删除用户".to_string());
json!({
"id": item.id,
"user_id": item.user_id,
"user_email": user_email,
"username": username,
"api_key": item.api_key_id.as_ref().map(|api_key_id| json!({
"id": api_key_id,
"name": item.api_key_name.clone(),
"display": item.api_key_name.clone().unwrap_or_else(|| api_key_id.clone()),
})),
"provider": item.provider_name,
"model": item.model,
"target_model": item.target_model,
"input_tokens": item.input_tokens,
"output_tokens": item.output_tokens,
"cache_creation_input_tokens": item.cache_creation_input_tokens,
"cache_read_input_tokens": item.cache_read_input_tokens,
"total_tokens": admin_usage_total_tokens(item),
"cost": round_to(item.total_cost_usd, 6),
"actual_cost": round_to(item.actual_total_cost_usd, 6),
"rate_multiplier": Value::Null,
"response_time_ms": item.response_time_ms,
"first_byte_time_ms": item.first_byte_time_ms,
"created_at": unix_secs_to_rfc3339(item.created_at_unix_secs),
"is_stream": item.is_stream,
"input_price_per_1m": Value::Null,
"output_price_per_1m": item.output_price_per_1m,
"cache_creation_price_per_1m": Value::Null,
"cache_read_price_per_1m": Value::Null,
"status_code": item.status_code,
"error_message": item.error_message,
"status": item.status,
"has_fallback": false,
"has_retry": false,
"has_rectified": false,
"api_format": item.api_format,
"endpoint_api_format": item.endpoint_api_format,
"has_format_conversion": item.has_format_conversion,
"api_key_name": item.api_key_name,
"provider_key_name": provider_key_name,
"model_version": Value::Null,
})
}

66
apps/aether-gateway/src/handlers/admin/observability/usage/analytics/aggregations.rs Normal file
View File

@@ -0,0 +1,66 @@
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
use aether_admin::observability::stats::round_to;
use aether_data_contracts::repository::usage::{StoredRequestUsageAudit, UsageAuditListQuery};
use serde_json::json;
use std::collections::BTreeMap;
pub(in super::super) async fn admin_usage_aggregation_by_user_json(
state: &AdminAppState<'_>,
usage: &[StoredRequestUsageAudit],
limit: usize,
) -> Result<serde_json::Value, GatewayError> {
let mut grouped: BTreeMap<String, (u64, u64, f64)> = BTreeMap::new();
for item in usage {
let Some(user_id) = item.user_id.as_ref() else {
continue;
};
let entry = grouped.entry(user_id.clone()).or_insert((0, 0, 0.0));
entry.0 = entry.0.saturating_add(1);
entry.1 = entry.1.saturating_add(item.total_tokens);
entry.2 += item.total_cost_usd;
}
let usernames = if state.has_user_data_reader() && !grouped.is_empty() {
state
.list_users_by_ids(&grouped.keys().cloned().collect::<Vec<_>>())
.await?
.into_iter()
.map(|user| (user.id, (user.email, user.username)))
.collect::<BTreeMap<_, _>>()
} else {
BTreeMap::new()
};
let mut items: Vec<serde_json::Value> = grouped
.into_iter()
.map(|(user_id, (request_count, total_tokens, total_cost))| {
let (email, username) = usernames
.get(&user_id)
.cloned()
.unwrap_or((None, String::new()));
json!({
"user_id": user_id,
"email": email,
"username": if username.is_empty() { serde_json::Value::Null } else { json!(username) },
"request_count": request_count,
"total_tokens": total_tokens,
"total_cost": round_to(total_cost, 6),
})
})
.collect();
items.sort_by(|left, right| {
right["request_count"]
.as_u64()
.unwrap_or_default()
.cmp(&left["request_count"].as_u64().unwrap_or_default())
.then_with(|| {
left["user_id"]
.as_str()
.unwrap_or_default()
.cmp(right["user_id"].as_str().unwrap_or_default())
})
});
items.truncate(limit);
Ok(json!(items))
}

28
apps/aether-gateway/src/handlers/admin/observability/usage/analytics/cache_affinity.rs Normal file
View File

@@ -0,0 +1,28 @@
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
use aether_data_contracts::repository::usage::{StoredRequestUsageAudit, UsageAuditListQuery};
pub(in super::super) async fn list_recent_completed_usage_for_cache_affinity(
state: &AdminAppState<'_>,
hours: u32,
user_id: Option<&str>,
) -> Result<Vec<StoredRequestUsageAudit>, GatewayError> {
let now_unix_secs = u64::try_from(chrono::Utc::now().timestamp()).unwrap_or_default();
let created_from_unix_secs = now_unix_secs.saturating_sub(u64::from(hours) * 3600);
let mut items = state
.list_usage_audits(&UsageAuditListQuery {
created_from_unix_secs: Some(created_from_unix_secs),
created_until_unix_secs: None,
user_id: user_id.map(ToOwned::to_owned),
provider_name: None,
model: None,
})
.await?;
items.retain(|item| item.status == "completed");
items.sort_by(|left, right| {
left.created_at_unix_secs
.cmp(&right.created_at_unix_secs)
.then_with(|| left.id.cmp(&right.id))
});
Ok(items)
}

30
apps/aether-gateway/src/handlers/admin/observability/usage/analytics/filters.rs Normal file
View File

@@ -0,0 +1,30 @@
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
use aether_data_contracts::repository::usage::StoredRequestUsageAudit;
use std::collections::{BTreeMap, BTreeSet};
pub(in super::super) async fn admin_usage_provider_key_names(
state: &AdminAppState<'_>,
usage: &[StoredRequestUsageAudit],
) -> Result<BTreeMap<String, String>, GatewayError> {
if !state.has_provider_catalog_data_reader() {
return Ok(BTreeMap::new());
}
let key_ids = usage
.iter()
.filter_map(|item| item.provider_api_key_id.clone())
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
if key_ids.is_empty() {
return Ok(BTreeMap::new());
}
Ok(state
.list_provider_catalog_keys_by_ids(&key_ids)
.await?
.into_iter()
.map(|key| (key.id, key.name))
.collect())
}

7
apps/aether-gateway/src/handlers/admin/observability/usage/analytics/mod.rs Normal file
View File

@@ -0,0 +1,7 @@
mod aggregations;
mod cache_affinity;
mod filters;
pub(super) use aggregations::admin_usage_aggregation_by_user_json;
pub(super) use cache_affinity::list_recent_completed_usage_for_cache_affinity;
pub(super) use filters::admin_usage_provider_key_names;

515
apps/aether-gateway/src/handlers/admin/observability/usage/analytics_routes.rs
View File

@@ -1,515 +0,0 @@
use super::super::stats::{
list_usage_for_optional_range, round_to, AdminStatsTimeRange, AdminStatsUsageFilter,
};
use super::analytics::{
admin_usage_aggregation_by_api_format_json, admin_usage_aggregation_by_model_json,
admin_usage_aggregation_by_provider_json, admin_usage_aggregation_by_user_json,
admin_usage_calculate_recommended_ttl, admin_usage_collect_request_intervals_minutes,
admin_usage_group_completed_by_api_key, admin_usage_group_completed_by_user,
admin_usage_heatmap_json, admin_usage_matches_optional_id, admin_usage_parse_aggregation_limit,
admin_usage_parse_recent_hours, admin_usage_parse_timeline_limit, admin_usage_percentile_cont,
admin_usage_point_sort_key, admin_usage_proportional_limits,
admin_usage_ttl_recommendation_reason, list_recent_completed_usage_for_cache_affinity,
};
use super::helpers::{
admin_usage_bad_request_response, admin_usage_data_unavailable_response,
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
};
use crate::control::GatewayPublicRequestContext;
use crate::handlers::admin::shared::{query_param_bool, query_param_value, unix_secs_to_rfc3339};
use crate::{AppState, GatewayError};
use aether_data_contracts::repository::usage::UsageAuditListQuery;
use axum::{
body::Body,
http,
response::{IntoResponse, Response},
Json,
};
use serde_json::{json, Value};
use std::collections::{BTreeMap, BTreeSet};
pub(super) async fn maybe_build_local_admin_usage_analytics_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
) -> Result<Option<Response<Body>>, GatewayError> {
let route_kind = request_context
.control_decision
.as_ref()
.and_then(|decision| decision.route_kind.as_deref());
match route_kind {
Some("aggregation_stats")
if request_context.request_method == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
"/api/admin/usage/aggregation/stats" | "/api/admin/usage/aggregation/stats/"
) =>
{
if !state.has_usage_data_reader() {
return Ok(Some(admin_usage_data_unavailable_response(
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
)));
}
let query = request_context.request_query_string.as_deref();
let group_by = query_param_value(query, "group_by")
.unwrap_or_default()
.trim()
.to_ascii_lowercase();
if !matches!(
group_by.as_str(),
"model" | "user" | "provider" | "api_format"
) {
return Ok(Some(admin_usage_bad_request_response(
"Invalid group_by value: must be one of model, user, provider, api_format",
)));
}
let limit = match admin_usage_parse_aggregation_limit(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
let mut usage = list_usage_for_optional_range(
state,
time_range.as_ref(),
&AdminStatsUsageFilter::default(),
)
.await?;
usage.retain(|item| item.status != "pending" && item.status != "streaming");
let response = match group_by.as_str() {
"model" => admin_usage_aggregation_by_model_json(&usage, limit),
"user" => admin_usage_aggregation_by_user_json(state, &usage, limit).await?,
"provider" => admin_usage_aggregation_by_provider_json(&usage, limit),
"api_format" => admin_usage_aggregation_by_api_format_json(&usage, limit),
_ => unreachable!(),
};
return Ok(Some(Json(response).into_response()));
}
Some("heatmap")
if request_context.request_method == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
"/api/admin/usage/heatmap" | "/api/admin/usage/heatmap/"
) =>
{
if !state.has_usage_data_reader() {
return Ok(Some(admin_usage_data_unavailable_response(
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
)));
}
let now_unix_secs = u64::try_from(chrono::Utc::now().timestamp()).unwrap_or_default();
let created_from_unix_secs = now_unix_secs.saturating_sub(365 * 24 * 3600);
let mut usage = state
.list_usage_audits(&UsageAuditListQuery {
created_from_unix_secs: Some(created_from_unix_secs),
..Default::default()
})
.await?;
usage.retain(|item| item.status != "pending" && item.status != "streaming");
return Ok(Some(Json(admin_usage_heatmap_json(&usage)).into_response()));
}
Some("cache_affinity_hit_analysis")
if request_context.request_method == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
"/api/admin/usage/cache-affinity/hit-analysis"
| "/api/admin/usage/cache-affinity/hit-analysis/"
) =>
{
if !state.has_usage_data_reader() {
return Ok(Some(admin_usage_data_unavailable_response(
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
)));
}
let query = request_context.request_query_string.as_deref();
let hours = match admin_usage_parse_recent_hours(query, 168) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
let user_id = query_param_value(query, "user_id");
let api_key_id = query_param_value(query, "api_key_id");
let usage =
list_recent_completed_usage_for_cache_affinity(state, hours, user_id.as_deref())
.await?;
let filtered: Vec<_> = usage
.into_iter()
.filter(|item| {
admin_usage_matches_optional_id(item.user_id.as_deref(), user_id.as_deref())
&& admin_usage_matches_optional_id(
item.api_key_id.as_deref(),
api_key_id.as_deref(),
)
})
.collect();
let total_requests = filtered.len();
let total_input_tokens: u64 = filtered.iter().map(|item| item.input_tokens).sum();
let total_cache_read_tokens: u64 = filtered
.iter()
.map(|item| item.cache_read_input_tokens)
.sum();
let total_cache_creation_tokens: u64 = filtered
.iter()
.map(|item| item.cache_creation_input_tokens)
.sum();
let total_cache_read_cost: f64 =
filtered.iter().map(|item| item.cache_read_cost_usd).sum();
let total_cache_creation_cost: f64 = filtered
.iter()
.map(|item| item.cache_creation_cost_usd)
.sum();
let requests_with_cache_hit = filtered
.iter()
.filter(|item| item.cache_read_input_tokens > 0)
.count();
let total_context_tokens = total_input_tokens.saturating_add(total_cache_read_tokens);
let token_cache_hit_rate = if total_context_tokens == 0 {
0.0
} else {
round_to(
total_cache_read_tokens as f64 / total_context_tokens as f64 * 100.0,
2,
)
};
let request_cache_hit_rate = if total_requests == 0 {
0.0
} else {
round_to(
requests_with_cache_hit as f64 / total_requests as f64 * 100.0,
2,
)
};
return Ok(Some(
Json(json!({
"analysis_period_hours": hours,
"total_requests": total_requests,
"requests_with_cache_hit": requests_with_cache_hit,
"request_cache_hit_rate": request_cache_hit_rate,
"total_input_tokens": total_input_tokens,
"total_cache_read_tokens": total_cache_read_tokens,
"total_cache_creation_tokens": total_cache_creation_tokens,
"token_cache_hit_rate": token_cache_hit_rate,
"total_cache_read_cost_usd": round_to(total_cache_read_cost, 4),
"total_cache_creation_cost_usd": round_to(total_cache_creation_cost, 4),
"estimated_savings_usd": round_to(total_cache_read_cost * 9.0, 4),
}))
.into_response(),
));
}
Some("cache_affinity_interval_timeline")
if request_context.request_method == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
"/api/admin/usage/cache-affinity/interval-timeline"
| "/api/admin/usage/cache-affinity/interval-timeline/"
) =>
{
if !state.has_usage_data_reader() {
return Ok(Some(admin_usage_data_unavailable_response(
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
)));
}
let query = request_context.request_query_string.as_deref();
let hours = match admin_usage_parse_recent_hours(query, 24) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
let limit = match admin_usage_parse_timeline_limit(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
let user_id = query_param_value(query, "user_id");
let include_user_info = query_param_bool(query, "include_user_info", false);
let usage =
list_recent_completed_usage_for_cache_affinity(state, hours, user_id.as_deref())
.await?;
let mut grouped: BTreeMap<String, Vec<serde_json::Value>> = BTreeMap::new();
let mut models = BTreeSet::new();
let mut usernames_by_user_id = BTreeMap::new();
for (group_user_id, items) in admin_usage_group_completed_by_user(&usage) {
if let Some(ref requested_user_id) = user_id {
if &group_user_id != requested_user_id {
continue;
}
}
let mut previous_created_at_unix_secs = None;
for item in items {
if let Some(previous) = previous_created_at_unix_secs {
let interval_minutes =
item.created_at_unix_secs.saturating_sub(previous) as f64 / 60.0;
if interval_minutes <= 120.0 {
let mut point = json!({
"x": unix_secs_to_rfc3339(item.created_at_unix_secs),
"y": round_to(interval_minutes, 2),
});
if !item.model.trim().is_empty() {
point["model"] = json!(item.model.clone());
models.insert(item.model.clone());
}
if include_user_info && user_id.is_none() {
point["user_id"] = json!(group_user_id.clone());
if let Some(username) = item.username.clone() {
usernames_by_user_id
.entry(group_user_id.clone())
.or_insert(username);
}
}
grouped
.entry(group_user_id.clone())
.or_default()
.push(point);
}
}
previous_created_at_unix_secs = Some(item.created_at_unix_secs);
}
}
if include_user_info && user_id.is_none() && state.has_user_data_reader() {
let user_ids: Vec<_> = grouped.keys().cloned().collect();
let user_map: BTreeMap<_, _> = state
.list_users_by_ids(&user_ids)
.await?
.into_iter()
.map(|user| (user.id, user.username))
.collect();
for (user_id, username) in user_map {
usernames_by_user_id.insert(user_id, username);
}
}
let total_points_before_limit: usize = grouped.values().map(Vec::len).sum();
let points: Vec<serde_json::Value> = if include_user_info && user_id.is_none() {
let user_limits =
admin_usage_proportional_limits(&grouped, limit, total_points_before_limit);
let mut selected = Vec::new();
for (group_user_id, mut items) in grouped {
let take = user_limits
.get(&group_user_id)
.copied()
.unwrap_or(items.len());
selected.extend(items.drain(..std::cmp::min(take, items.len())));
}
selected.sort_by(admin_usage_point_sort_key);
selected
} else {
let mut selected = grouped
.into_values()
.flatten()
.collect::<Vec<serde_json::Value>>();
selected.sort_by(admin_usage_point_sort_key);
if selected.len() > limit {
selected.truncate(limit);
}
selected
};
let mut response = json!({
"analysis_period_hours": hours,
"total_points": points.len(),
"points": points,
});
if include_user_info && user_id.is_none() {
response["users"] = json!(usernames_by_user_id);
}
if !models.is_empty() {
response["models"] = json!(models.into_iter().collect::<Vec<_>>());
}
return Ok(Some(Json(response).into_response()));
}
Some("cache_affinity_ttl_analysis")
if request_context.request_method == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
"/api/admin/usage/cache-affinity/ttl-analysis"
| "/api/admin/usage/cache-affinity/ttl-analysis/"
) =>
{
if !state.has_usage_data_reader() {
return Ok(Some(admin_usage_data_unavailable_response(
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
)));
}
let query = request_context.request_query_string.as_deref();
let hours = match admin_usage_parse_recent_hours(query, 168) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
let user_id = query_param_value(query, "user_id");
let api_key_id = query_param_value(query, "api_key_id");
let group_by_api_key = api_key_id.is_some();
let usage =
list_recent_completed_usage_for_cache_affinity(state, hours, user_id.as_deref())
.await?;
let grouped = if group_by_api_key {
admin_usage_group_completed_by_api_key(&usage, api_key_id.as_deref())
} else {
admin_usage_group_completed_by_user(&usage)
.into_iter()
.filter(|(group_user_id, _)| {
admin_usage_matches_optional_id(
Some(group_user_id.as_str()),
user_id.as_deref(),
)
})
.collect()
};
let user_map: BTreeMap<String, aether_data::repository::users::StoredUserSummary> =
if !group_by_api_key && state.has_user_data_reader() {
let user_ids = grouped.keys().cloned().collect::<Vec<_>>();
state
.list_users_by_ids(&user_ids)
.await?
.into_iter()
.map(|user| (user.id.clone(), user))
.collect()
} else {
BTreeMap::new()
};
let mut ttl_distribution = json!({
"5min": 0_u64,
"15min": 0_u64,
"30min": 0_u64,
"60min": 0_u64,
});
let mut users = Vec::new();
for (group_id, items) in grouped {
let intervals = admin_usage_collect_request_intervals_minutes(&items);
if intervals.len() < 2 {
continue;
}
let within_5min = intervals.iter().filter(|value| **value <= 5.0).count() as u64;
let within_15min = intervals
.iter()
.filter(|value| **value > 5.0 && **value <= 15.0)
.count() as u64;
let within_30min = intervals
.iter()
.filter(|value| **value > 15.0 && **value <= 30.0)
.count() as u64;
let within_60min = intervals
.iter()
.filter(|value| **value > 30.0 && **value <= 60.0)
.count() as u64;
let over_60min = intervals.iter().filter(|value| **value > 60.0).count() as u64;
let request_count = intervals.len() as u64;
let p50 = admin_usage_percentile_cont(&intervals, 0.5);
let p75 = admin_usage_percentile_cont(&intervals, 0.75);
let p90 = admin_usage_percentile_cont(&intervals, 0.90);
let avg_interval = intervals.iter().copied().sum::<f64>() / intervals.len() as f64;
let min_interval = intervals.iter().copied().reduce(f64::min);
let max_interval = intervals.iter().copied().reduce(f64::max);
let recommended_ttl = admin_usage_calculate_recommended_ttl(p75, p90);
match recommended_ttl {
0..=5 => {
ttl_distribution["5min"] = json!(ttl_distribution["5min"]
.as_u64()
.unwrap_or(0)
.saturating_add(1))
}
6..=15 => {
ttl_distribution["15min"] = json!(ttl_distribution["15min"]
.as_u64()
.unwrap_or(0)
.saturating_add(1))
}
16..=30 => {
ttl_distribution["30min"] = json!(ttl_distribution["30min"]
.as_u64()
.unwrap_or(0)
.saturating_add(1))
}
_ => {
ttl_distribution["60min"] = json!(ttl_distribution["60min"]
.as_u64()
.unwrap_or(0)
.saturating_add(1))
}
}
let (username, email) = if group_by_api_key {
(Value::Null, Value::Null)
} else if let Some(user) = user_map.get(&group_id) {
(
json!(user.username.clone()),
json!(user.email.clone().unwrap_or_default()),
)
} else {
(Value::Null, Value::Null)
};
users.push(json!({
"group_id": group_id,
"username": username,
"email": email,
"request_count": request_count,
"interval_distribution": {
"within_5min": within_5min,
"within_15min": within_15min,
"within_30min": within_30min,
"within_60min": within_60min,
"over_60min": over_60min,
},
"interval_percentages": {
"within_5min": round_to(within_5min as f64 / request_count as f64 * 100.0, 1),
"within_15min": round_to(within_15min as f64 / request_count as f64 * 100.0, 1),
"within_30min": round_to(within_30min as f64 / request_count as f64 * 100.0, 1),
"within_60min": round_to(within_60min as f64 / request_count as f64 * 100.0, 1),
"over_60min": round_to(over_60min as f64 / request_count as f64 * 100.0, 1),
},
"percentiles": {
"p50": p50.map(|value| round_to(value, 2)),
"p75": p75.map(|value| round_to(value, 2)),
"p90": p90.map(|value| round_to(value, 2)),
},
"avg_interval_minutes": round_to(avg_interval, 2),
"min_interval_minutes": min_interval.map(|value| round_to(value, 2)),
"max_interval_minutes": max_interval.map(|value| round_to(value, 2)),
"recommended_ttl_minutes": recommended_ttl,
"recommendation_reason": admin_usage_ttl_recommendation_reason(recommended_ttl, p75, p90),
}));
}
users.sort_by(|left, right| {
right["request_count"]
.as_u64()
.unwrap_or(0)
.cmp(&left["request_count"].as_u64().unwrap_or(0))
.then_with(|| {
left["group_id"]
.as_str()
.unwrap_or_default()
.cmp(right["group_id"].as_str().unwrap_or_default())
})
});
return Ok(Some(
Json(json!({
"analysis_period_hours": hours,
"total_users_analyzed": users.len(),
"ttl_distribution": ttl_distribution,
"users": users,
}))
.into_response(),
));
}
_ => {}
}
Ok(None)
}

64
apps/aether-gateway/src/handlers/admin/observability/usage/analytics_routes/aggregation.rs Normal file
View File

@@ -0,0 +1,64 @@
use super::super::super::stats::{AdminStatsTimeRange, AdminStatsUsageFilter};
use super::super::analytics::admin_usage_aggregation_by_user_json;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::query_param_value;
use crate::GatewayError;
use aether_admin::observability::usage::{
admin_usage_aggregation_by_api_format_json, admin_usage_aggregation_by_model_json,
admin_usage_aggregation_by_provider_json, admin_usage_bad_request_response,
admin_usage_data_unavailable_response, admin_usage_parse_aggregation_limit,
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
};
use axum::{
body::Body,
response::{IntoResponse, Response},
Json,
};
use serde_json::json;
pub(super) async fn build_admin_usage_aggregation_stats_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
if !state.has_usage_data_reader() {
return Ok(admin_usage_data_unavailable_response(
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
));
}
let query = request_context.request_query_string.as_deref();
let group_by = query_param_value(query, "group_by")
.unwrap_or_default()
.trim()
.to_ascii_lowercase();
if !matches!(
group_by.as_str(),
"model" | "user" | "provider" | "api_format"
) {
return Ok(admin_usage_bad_request_response(
"Invalid group_by value: must be one of model, user, provider, api_format",
));
}
let limit = match admin_usage_parse_aggregation_limit(query) {
Ok(value) => value,
Err(detail) => return Ok(admin_usage_bad_request_response(detail)),
};
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
Ok(value) => value,
Err(detail) => return Ok(admin_usage_bad_request_response(detail)),
};
let mut usage = state
.list_admin_usage_for_optional_range(time_range.as_ref(), &AdminStatsUsageFilter::default())
.await?;
usage.retain(|item| item.status != "pending" && item.status != "streaming");
let response = match group_by.as_str() {
"model" => admin_usage_aggregation_by_model_json(&usage, limit),
"user" => admin_usage_aggregation_by_user_json(state, &usage, limit).await?,
"provider" => admin_usage_aggregation_by_provider_json(&usage, limit),
"api_format" => admin_usage_aggregation_by_api_format_json(&usage, limit),
_ => unreachable!(),
};
Ok(Json(response).into_response())
}

98
apps/aether-gateway/src/handlers/admin/observability/usage/analytics_routes/cache_affinity_hit_analysis.rs Normal file
View File

@@ -0,0 +1,98 @@
use super::super::super::stats::round_to;
use super::super::analytics::list_recent_completed_usage_for_cache_affinity;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::query_param_value;
use crate::GatewayError;
use aether_admin::observability::usage::{
admin_usage_bad_request_response, admin_usage_data_unavailable_response,
admin_usage_matches_optional_id, admin_usage_parse_recent_hours,
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
};
use axum::{
body::Body,
response::{IntoResponse, Response},
Json,
};
use serde_json::json;
pub(super) async fn build_admin_usage_cache_affinity_hit_analysis_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
if !state.has_usage_data_reader() {
return Ok(admin_usage_data_unavailable_response(
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
));
}
let query = request_context.request_query_string.as_deref();
let hours = match admin_usage_parse_recent_hours(query, 168) {
Ok(value) => value,
Err(detail) => return Ok(admin_usage_bad_request_response(detail)),
};
let user_id = query_param_value(query, "user_id");
let api_key_id = query_param_value(query, "api_key_id");
let usage =
list_recent_completed_usage_for_cache_affinity(state, hours, user_id.as_deref()).await?;
let filtered: Vec<_> = usage
.into_iter()
.filter(|item| {
admin_usage_matches_optional_id(item.user_id.as_deref(), user_id.as_deref())
&& admin_usage_matches_optional_id(
item.api_key_id.as_deref(),
api_key_id.as_deref(),
)
})
.collect();
let total_requests = filtered.len();
let total_input_tokens: u64 = filtered.iter().map(|item| item.input_tokens).sum();
let total_cache_read_tokens: u64 = filtered
.iter()
.map(|item| item.cache_read_input_tokens)
.sum();
let total_cache_creation_tokens: u64 = filtered
.iter()
.map(|item| item.cache_creation_input_tokens)
.sum();
let total_cache_read_cost: f64 = filtered.iter().map(|item| item.cache_read_cost_usd).sum();
let total_cache_creation_cost: f64 = filtered
.iter()
.map(|item| item.cache_creation_cost_usd)
.sum();
let requests_with_cache_hit = filtered
.iter()
.filter(|item| item.cache_read_input_tokens > 0)
.count();
let total_context_tokens = total_input_tokens.saturating_add(total_cache_read_tokens);
let token_cache_hit_rate = if total_context_tokens == 0 {
0.0
} else {
round_to(
total_cache_read_tokens as f64 / total_context_tokens as f64 * 100.0,
2,
)
};
let request_cache_hit_rate = if total_requests == 0 {
0.0
} else {
round_to(
requests_with_cache_hit as f64 / total_requests as f64 * 100.0,
2,
)
};
Ok(Json(json!({
"analysis_period_hours": hours,
"total_requests": total_requests,
"requests_with_cache_hit": requests_with_cache_hit,
"request_cache_hit_rate": request_cache_hit_rate,
"total_input_tokens": total_input_tokens,
"total_cache_read_tokens": total_cache_read_tokens,
"total_cache_creation_tokens": total_cache_creation_tokens,
"token_cache_hit_rate": token_cache_hit_rate,
"total_cache_read_cost_usd": round_to(total_cache_read_cost, 4),
"total_cache_creation_cost_usd": round_to(total_cache_creation_cost, 4),
"estimated_savings_usd": round_to(total_cache_read_cost * 9.0, 4),
}))
.into_response())
}

134
apps/aether-gateway/src/handlers/admin/observability/usage/analytics_routes/cache_affinity_interval_timeline.rs Normal file
View File

@@ -0,0 +1,134 @@
use super::super::analytics::list_recent_completed_usage_for_cache_affinity;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::{query_param_bool, query_param_value, unix_secs_to_rfc3339};
use crate::GatewayError;
use aether_admin::observability::usage::{
admin_usage_bad_request_response, admin_usage_data_unavailable_response,
admin_usage_group_completed_by_user, admin_usage_parse_recent_hours,
admin_usage_parse_timeline_limit, admin_usage_point_sort_key, admin_usage_proportional_limits,
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
};
use axum::{
body::Body,
response::{IntoResponse, Response},
Json,
};
use serde_json::json;
use std::collections::{BTreeMap, BTreeSet};
pub(super) async fn build_admin_usage_cache_affinity_interval_timeline_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
if !state.has_usage_data_reader() {
return Ok(admin_usage_data_unavailable_response(
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
));
}
let query = request_context.request_query_string.as_deref();
let hours = match admin_usage_parse_recent_hours(query, 24) {
Ok(value) => value,
Err(detail) => return Ok(admin_usage_bad_request_response(detail)),
};
let limit = match admin_usage_parse_timeline_limit(query) {
Ok(value) => value,
Err(detail) => return Ok(admin_usage_bad_request_response(detail)),
};
let user_id = query_param_value(query, "user_id");
let include_user_info = query_param_bool(query, "include_user_info", false);
let usage =
list_recent_completed_usage_for_cache_affinity(state, hours, user_id.as_deref()).await?;
let mut grouped: BTreeMap<String, Vec<serde_json::Value>> = BTreeMap::new();
let mut models = BTreeSet::new();
let mut usernames_by_user_id = BTreeMap::new();
for (group_user_id, items) in admin_usage_group_completed_by_user(&usage) {
if let Some(ref requested_user_id) = user_id {
if &group_user_id != requested_user_id {
continue;
}
}
let mut previous_created_at_unix_secs = None;
for item in items {
if let Some(previous) = previous_created_at_unix_secs {
let interval_minutes =
item.created_at_unix_secs.saturating_sub(previous) as f64 / 60.0;
if interval_minutes <= 120.0 {
let mut point = json!({
"x": unix_secs_to_rfc3339(item.created_at_unix_secs),
"y": ((interval_minutes * 100.0).round()) / 100.0,
});
if !item.model.trim().is_empty() {
point["model"] = json!(item.model.clone());
models.insert(item.model.clone());
}
if include_user_info && user_id.is_none() {
point["user_id"] = json!(group_user_id.clone());
if let Some(username) = item.username.clone() {
usernames_by_user_id
.entry(group_user_id.clone())
.or_insert(username);
}
}
grouped
.entry(group_user_id.clone())
.or_default()
.push(point);
}
}
previous_created_at_unix_secs = Some(item.created_at_unix_secs);
}
}
if include_user_info && user_id.is_none() && state.has_user_data_reader() {
let user_ids: Vec<_> = grouped.keys().cloned().collect();
let user_map: BTreeMap<_, _> = state
.list_users_by_ids(&user_ids)
.await?
.into_iter()
.map(|user| (user.id, user.username))
.collect();
for (user_id, username) in user_map {
usernames_by_user_id.insert(user_id, username);
}
}
let total_points_before_limit: usize = grouped.values().map(Vec::len).sum();
let points: Vec<serde_json::Value> = if include_user_info && user_id.is_none() {
let user_limits =
admin_usage_proportional_limits(&grouped, limit, total_points_before_limit);
let mut selected = Vec::new();
for (group_user_id, mut items) in grouped {
let take = user_limits
.get(&group_user_id)
.copied()
.unwrap_or(items.len());
selected.extend(items.drain(..std::cmp::min(take, items.len())));
}
selected.sort_by(admin_usage_point_sort_key);
selected
} else {
let mut selected = grouped.into_values().flatten().collect::<Vec<_>>();
selected.sort_by(admin_usage_point_sort_key);
if selected.len() > limit {
selected.truncate(limit);
}
selected
};
let mut response = json!({
"analysis_period_hours": hours,
"total_points": points.len(),
"points": points,
});
if include_user_info && user_id.is_none() {
response["users"] = json!(usernames_by_user_id);
}
if !models.is_empty() {
response["models"] = json!(models.into_iter().collect::<Vec<_>>());
}
Ok(Json(response).into_response())
}

192
apps/aether-gateway/src/handlers/admin/observability/usage/analytics_routes/cache_affinity_ttl_analysis.rs Normal file
View File

@@ -0,0 +1,192 @@
use super::super::super::stats::round_to;
use super::super::analytics::list_recent_completed_usage_for_cache_affinity;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::query_param_value;
use crate::GatewayError;
use aether_admin::observability::usage::{
admin_usage_bad_request_response, admin_usage_calculate_recommended_ttl,
admin_usage_collect_request_intervals_minutes, admin_usage_data_unavailable_response,
admin_usage_group_completed_by_api_key, admin_usage_group_completed_by_user,
admin_usage_matches_optional_id, admin_usage_parse_recent_hours, admin_usage_percentile_cont,
admin_usage_ttl_recommendation_reason, ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
};
use axum::{
body::Body,
response::{IntoResponse, Response},
Json,
};
use serde_json::{json, Value};
use std::collections::BTreeMap;
pub(super) async fn build_admin_usage_cache_affinity_ttl_analysis_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
if !state.has_usage_data_reader() {
return Ok(admin_usage_data_unavailable_response(
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
));
}
let query = request_context.request_query_string.as_deref();
let hours = match admin_usage_parse_recent_hours(query, 168) {
Ok(value) => value,
Err(detail) => return Ok(admin_usage_bad_request_response(detail)),
};
let user_id = query_param_value(query, "user_id");
let api_key_id = query_param_value(query, "api_key_id");
let group_by_api_key = api_key_id.is_some();
let usage =
list_recent_completed_usage_for_cache_affinity(state, hours, user_id.as_deref()).await?;
let grouped = if group_by_api_key {
admin_usage_group_completed_by_api_key(&usage, api_key_id.as_deref())
} else {
admin_usage_group_completed_by_user(&usage)
.into_iter()
.filter(|(group_user_id, _)| {
admin_usage_matches_optional_id(Some(group_user_id.as_str()), user_id.as_deref())
})
.collect()
};
let user_map: BTreeMap<String, aether_data::repository::users::StoredUserSummary> =
if !group_by_api_key && state.has_user_data_reader() {
let user_ids = grouped.keys().cloned().collect::<Vec<_>>();
state
.list_users_by_ids(&user_ids)
.await?
.into_iter()
.map(|user| (user.id.clone(), user))
.collect()
} else {
BTreeMap::new()
};
let mut ttl_distribution = json!({
"5min": 0_u64,
"15min": 0_u64,
"30min": 0_u64,
"60min": 0_u64,
});
let mut users = Vec::new();
for (group_id, items) in grouped {
let intervals = admin_usage_collect_request_intervals_minutes(&items);
if intervals.len() < 2 {
continue;
}
let within_5min = intervals.iter().filter(|value| **value <= 5.0).count() as u64;
let within_15min = intervals
.iter()
.filter(|value| **value > 5.0 && **value <= 15.0)
.count() as u64;
let within_30min = intervals
.iter()
.filter(|value| **value > 15.0 && **value <= 30.0)
.count() as u64;
let within_60min = intervals
.iter()
.filter(|value| **value > 30.0 && **value <= 60.0)
.count() as u64;
let over_60min = intervals.iter().filter(|value| **value > 60.0).count() as u64;
let request_count = intervals.len() as u64;
let p50 = admin_usage_percentile_cont(&intervals, 0.5);
let p75 = admin_usage_percentile_cont(&intervals, 0.75);
let p90 = admin_usage_percentile_cont(&intervals, 0.90);
let avg_interval = intervals.iter().copied().sum::<f64>() / intervals.len() as f64;
let min_interval = intervals.iter().copied().reduce(f64::min);
let max_interval = intervals.iter().copied().reduce(f64::max);
let recommended_ttl = admin_usage_calculate_recommended_ttl(p75, p90);
match recommended_ttl {
0..=5 => {
ttl_distribution["5min"] = json!(ttl_distribution["5min"]
.as_u64()
.unwrap_or(0)
.saturating_add(1))
}
6..=15 => {
ttl_distribution["15min"] = json!(ttl_distribution["15min"]
.as_u64()
.unwrap_or(0)
.saturating_add(1))
}
16..=30 => {
ttl_distribution["30min"] = json!(ttl_distribution["30min"]
.as_u64()
.unwrap_or(0)
.saturating_add(1))
}
_ => {
ttl_distribution["60min"] = json!(ttl_distribution["60min"]
.as_u64()
.unwrap_or(0)
.saturating_add(1))
}
}
let (username, email) = if group_by_api_key {
(Value::Null, Value::Null)
} else if let Some(user) = user_map.get(&group_id) {
(
json!(user.username.clone()),
json!(user.email.clone().unwrap_or_default()),
)
} else {
(Value::Null, Value::Null)
};
users.push(json!({
"group_id": group_id,
"username": username,
"email": email,
"request_count": request_count,
"interval_distribution": {
"within_5min": within_5min,
"within_15min": within_15min,
"within_30min": within_30min,
"within_60min": within_60min,
"over_60min": over_60min,
},
"interval_percentages": {
"within_5min": round_to(within_5min as f64 / request_count as f64 * 100.0, 1),
"within_15min": round_to(within_15min as f64 / request_count as f64 * 100.0, 1),
"within_30min": round_to(within_30min as f64 / request_count as f64 * 100.0, 1),
"within_60min": round_to(within_60min as f64 / request_count as f64 * 100.0, 1),
"over_60min": round_to(over_60min as f64 / request_count as f64 * 100.0, 1),
},
"percentiles": {
"p50": p50.map(|value| round_to(value, 2)),
"p75": p75.map(|value| round_to(value, 2)),
"p90": p90.map(|value| round_to(value, 2)),
},
"avg_interval_minutes": round_to(avg_interval, 2),
"min_interval_minutes": min_interval.map(|value| round_to(value, 2)),
"max_interval_minutes": max_interval.map(|value| round_to(value, 2)),
"recommended_ttl_minutes": recommended_ttl,
"recommendation_reason": admin_usage_ttl_recommendation_reason(recommended_ttl, p75, p90),
}));
}
users.sort_by(|left, right| {
right["request_count"]
.as_u64()
.unwrap_or(0)
.cmp(&left["request_count"].as_u64().unwrap_or(0))
.then_with(|| {
left["group_id"]
.as_str()
.unwrap_or_default()
.cmp(right["group_id"].as_str().unwrap_or_default())
})
});
Ok(Json(json!({
"analysis_period_hours": hours,
"total_users_analyzed": users.len(),
"ttl_distribution": ttl_distribution,
"users": users,
}))
.into_response())
}

32
apps/aether-gateway/src/handlers/admin/observability/usage/analytics_routes/heatmap.rs Normal file
View File

@@ -0,0 +1,32 @@
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
use aether_admin::observability::usage::{
admin_usage_data_unavailable_response, admin_usage_heatmap_json,
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
};
use aether_data_contracts::repository::usage::UsageAuditListQuery;
use axum::{
body::Body,
response::{IntoResponse, Response},
Json,
};
pub(super) async fn build_admin_usage_heatmap_response(
state: &AdminAppState<'_>,
) -> Result<Response<Body>, GatewayError> {
if !state.has_usage_data_reader() {
return Ok(admin_usage_data_unavailable_response(
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
));
}
let now_unix_secs = u64::try_from(chrono::Utc::now().timestamp()).unwrap_or_default();
let created_from_unix_secs = now_unix_secs.saturating_sub(365 * 24 * 3600);
let mut usage = state
.list_usage_audits(&UsageAuditListQuery {
created_from_unix_secs: Some(created_from_unix_secs),
..Default::default()
})
.await?;
usage.retain(|item| item.status != "pending" && item.status != "streaming");
Ok(Json(admin_usage_heatmap_json(&usage)).into_response())
}

95
apps/aether-gateway/src/handlers/admin/observability/usage/analytics_routes/mod.rs Normal file
View File

@@ -0,0 +1,95 @@
mod aggregation;
mod cache_affinity_hit_analysis;
mod cache_affinity_interval_timeline;
mod cache_affinity_ttl_analysis;
mod heatmap;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use axum::{body::Body, http, response::Response};
pub(super) async fn maybe_build_local_admin_usage_analytics_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Option<Response<Body>>, GatewayError> {
let route_kind = request_context
.control_decision
.as_ref()
.and_then(|decision| decision.route_kind.as_deref());
match route_kind {
Some("aggregation_stats")
if request_context.request_method == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
"/api/admin/usage/aggregation/stats" | "/api/admin/usage/aggregation/stats/"
) =>
{
return aggregation::build_admin_usage_aggregation_stats_response(
state,
request_context,
)
.await
.map(Some);
}
Some("heatmap")
if request_context.request_method == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
"/api/admin/usage/heatmap" | "/api/admin/usage/heatmap/"
) =>
{
return heatmap::build_admin_usage_heatmap_response(state)
.await
.map(Some);
}
Some("cache_affinity_hit_analysis")
if request_context.request_method == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
"/api/admin/usage/cache-affinity/hit-analysis"
| "/api/admin/usage/cache-affinity/hit-analysis/"
) =>
{
return cache_affinity_hit_analysis::build_admin_usage_cache_affinity_hit_analysis_response(
state,
request_context,
)
.await
.map(Some);
}
Some("cache_affinity_interval_timeline")
if request_context.request_method == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
"/api/admin/usage/cache-affinity/interval-timeline"
| "/api/admin/usage/cache-affinity/interval-timeline/"
) =>
{
return cache_affinity_interval_timeline::build_admin_usage_cache_affinity_interval_timeline_response(
state,
request_context,
)
.await
.map(Some);
}
Some("cache_affinity_ttl_analysis")
if request_context.request_method == http::Method::GET
&& matches!(
request_context.request_path.as_str(),
"/api/admin/usage/cache-affinity/ttl-analysis"
| "/api/admin/usage/cache-affinity/ttl-analysis/"
) =>
{
return cache_affinity_ttl_analysis::build_admin_usage_cache_affinity_ttl_analysis_response(
state,
request_context,
)
.await
.map(Some);
}
_ => {}
}
Ok(None)
}

142
apps/aether-gateway/src/handlers/admin/observability/usage/detail_routes.rs
View File

@@ -1,32 +1,29 @@
use super::super::stats::round_to;
use super::analytics::{
admin_usage_provider_key_name, admin_usage_provider_key_names, admin_usage_record_json,
};
use super::helpers::{
admin_usage_bad_request_response, admin_usage_data_unavailable_response,
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
};
use super::analytics::admin_usage_provider_key_names;
use super::replay::{
admin_usage_build_curl_command, admin_usage_curl_headers, admin_usage_curl_url,
admin_usage_headers_from_value, admin_usage_id_from_action_path,
admin_usage_id_from_detail_path, admin_usage_resolve_request_preview_body,
build_admin_usage_replay_response,
admin_usage_curl_headers, admin_usage_curl_url, admin_usage_headers_from_value,
admin_usage_id_from_action_path, admin_usage_id_from_detail_path,
admin_usage_resolve_request_preview_body, build_admin_usage_curl_response,
build_admin_usage_detail_payload, build_admin_usage_replay_response,
};
use crate::control::GatewayPublicRequestContext;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::{attach_admin_audit_response, query_param_bool};
use crate::{AppState, GatewayError};
use crate::GatewayError;
use aether_admin::observability::usage::{
admin_usage_bad_request_response, admin_usage_data_unavailable_response,
admin_usage_provider_key_name, ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
};
use axum::{
body::Body,
http,
response::{IntoResponse, Response},
Json,
};
use serde_json::{json, Value};
use serde_json::json;
use std::collections::BTreeMap;
pub(super) async fn maybe_build_local_admin_usage_detail_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
request_body: Option<&axum::body::Bytes>,
) -> Result<Option<Response<Body>>, GatewayError> {
let route_kind = request_context
@@ -54,12 +51,7 @@ pub(super) async fn maybe_build_local_admin_usage_detail_response(
return Ok(Some(admin_usage_bad_request_response("usage_id 无效")));
};
let Some(item) = state
.data
.find_request_usage_by_id(&usage_id)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
else {
let Some(item) = state.find_request_usage_by_id(&usage_id).await? else {
return Ok(Some(
(
http::StatusCode::NOT_FOUND,
@@ -80,7 +72,7 @@ pub(super) async fn maybe_build_local_admin_usage_detail_response(
};
let url = endpoint
.as_ref()
.map(|endpoint| admin_usage_curl_url(endpoint, &item));
.map(|endpoint| admin_usage_curl_url(state, endpoint, &item));
let headers_json = item
.provider_request_headers
.clone()
@@ -95,18 +87,8 @@ pub(super) async fn maybe_build_local_admin_usage_detail_response(
.clone()
.or_else(|| item.request_body.clone())
.unwrap_or_else(|| admin_usage_resolve_request_preview_body(&item, None));
let curl = admin_usage_build_curl_command(url.as_deref(), &headers, Some(&body));
return Ok(Some(attach_admin_audit_response(
Json(json!({
"url": url,
"method": "POST",
"headers": headers_json.unwrap_or_else(|| json!(headers.clone())),
"body": body,
"curl": curl,
"original_request_body_available": item.request_body.is_some() || item.provider_request_body.is_some(),
}))
.into_response(),
build_admin_usage_curl_response(&item, url, headers_json, &headers, &body),
"admin_usage_curl_viewed",
"view_usage_curl_replay",
"usage_record",
@@ -153,12 +135,7 @@ pub(super) async fn maybe_build_local_admin_usage_detail_response(
true,
);
let Some(item) = state
.data
.find_request_usage_by_id(&usage_id)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
else {
let Some(item) = state.find_request_usage_by_id(&usage_id).await? else {
return Ok(Some(
(
http::StatusCode::NOT_FOUND,
@@ -187,84 +164,19 @@ pub(super) async fn maybe_build_local_admin_usage_detail_response(
admin_usage_provider_key_names(state, std::slice::from_ref(&item)).await?;
let provider_key_name = admin_usage_provider_key_name(&item, &provider_key_names);
let mut payload =
admin_usage_record_json(&item, &users_by_id, provider_key_name.as_deref());
let request_body = item
.request_body
.clone()
.unwrap_or_else(|| admin_usage_resolve_request_preview_body(&item, None));
let request_preview_source = if item.request_body.is_some() {
"stored_original"
} else {
"local_reconstruction"
};
let mut metadata = match item.request_metadata.clone() {
Some(serde_json::Value::Object(object)) => serde_json::Value::Object(object),
Some(value) => json!({ "request_metadata": value }),
None => json!({}),
};
if let Some(object) = metadata.as_object_mut() {
object.insert(
"request_preview_source".to_string(),
json!(request_preview_source),
);
object.insert(
"original_request_body_available".to_string(),
json!(item.request_body.is_some()),
);
object.insert(
"original_response_body_available".to_string(),
json!(item.response_body.is_some() || item.client_response_body.is_some()),
);
}
payload["user"] = match item.user_id.as_ref() {
Some(user_id) => json!({
"id": user_id,
"email": payload["user_email"].clone(),
"username": payload["username"].clone(),
}),
None => Value::Null,
};
payload["request_id"] = json!(item.request_id);
payload["billing_status"] = json!(item.billing_status);
payload["request_type"] = json!(item.request_type);
payload["provider_id"] = json!(item.provider_id);
payload["provider_endpoint_id"] = json!(item.provider_endpoint_id);
payload["provider_api_key_id"] = json!(item.provider_api_key_id);
payload["error_category"] = json!(item.error_category);
payload["cache_creation_cost"] = json!(round_to(item.cache_creation_cost_usd, 6));
payload["cache_read_cost"] = json!(round_to(item.cache_read_cost_usd, 6));
payload["request_cost"] = json!(round_to(item.total_cost_usd, 6));
payload["request_headers"] = item
.request_headers
.clone()
.unwrap_or_else(|| json!(admin_usage_curl_headers()));
payload["provider_request_headers"] = item
.provider_request_headers
.clone()
.unwrap_or_else(|| json!(admin_usage_curl_headers()));
payload["response_headers"] = item.response_headers.clone().unwrap_or(Value::Null);
payload["client_response_headers"] =
item.client_response_headers.clone().unwrap_or(Value::Null);
payload["metadata"] = metadata;
payload["has_request_body"] = json!(true);
payload["has_provider_request_body"] = json!(item.provider_request_body.is_some());
payload["has_response_body"] = json!(item.response_body.is_some());
payload["has_client_response_body"] = json!(item.client_response_body.is_some());
payload["tiered_pricing"] = serde_json::Value::Null;
if include_bodies {
payload["request_body"] = request_body;
payload["provider_request_body"] =
item.provider_request_body.clone().unwrap_or(Value::Null);
payload["response_body"] = item.response_body.clone().unwrap_or(Value::Null);
payload["client_response_body"] =
item.client_response_body.clone().unwrap_or(Value::Null);
} else {
payload["request_body"] = serde_json::Value::Null;
payload["provider_request_body"] = serde_json::Value::Null;
payload["response_body"] = serde_json::Value::Null;
payload["client_response_body"] = serde_json::Value::Null;
}
let default_headers = admin_usage_curl_headers();
let payload = build_admin_usage_detail_payload(
&item,
&users_by_id,
provider_key_name.as_deref(),
include_bodies,
request_body,
&default_headers,
);
return Ok(Some(attach_admin_audit_response(
Json(payload).into_response(),

25
apps/aether-gateway/src/handlers/admin/observability/usage/helpers.rs
View File

@@ -1,25 +0,0 @@
use axum::{
body::Body,
http,
response::{IntoResponse, Response},
Json,
};
use serde_json::json;
pub(crate) const ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL: &str = "Admin usage data unavailable";
pub(crate) fn admin_usage_data_unavailable_response(detail: &'static str) -> Response<Body> {
(
http::StatusCode::SERVICE_UNAVAILABLE,
Json(json!({ "detail": detail })),
)
.into_response()
}
pub(crate) fn admin_usage_bad_request_response(detail: impl Into<String>) -> Response<Body> {
(
http::StatusCode::BAD_REQUEST,
Json(json!({ "detail": detail.into() })),
)
.into_response()
}

16
apps/aether-gateway/src/handlers/admin/observability/usage/mod.rs
View File

@@ -1,23 +1,19 @@
use crate::control::GatewayPublicRequestContext;
use crate::{AppState, GatewayError};
use axum::{
body::Body,
response::{IntoResponse, Response},
};
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use axum::{body::Body, response::Response};
mod analytics;
mod analytics_routes;
mod detail_routes;
mod helpers;
mod replay;
mod summary_routes;
pub(crate) async fn maybe_build_local_admin_usage_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
request_body: Option<&axum::body::Bytes>,
) -> Result<Option<Response<Body>>, GatewayError> {
let Some(decision) = request_context.control_decision.as_ref() else {
let Some(decision) = request_context.decision() else {
return Ok(None);
};

248
apps/aether-gateway/src/handlers/admin/observability/usage/replay.rs
View File

@@ -1,9 +1,9 @@
use super::helpers::{
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use aether_admin::observability::usage::{
admin_usage_bad_request_response, admin_usage_data_unavailable_response,
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
};
use crate::control::GatewayPublicRequestContext;
use crate::{AppState, GatewayError};
use aether_data_contracts::repository::{
provider_catalog::StoredProviderCatalogEndpoint, usage::StoredRequestUsageAudit,
};
@@ -16,28 +16,12 @@ use axum::{
use serde_json::{json, Value};
use std::collections::BTreeMap;
fn admin_usage_id_from_path_suffix(request_path: &str, suffix: Option<&str>) -> Option<String> {
let mut value = request_path
.strip_prefix("/api/admin/usage/")?
.trim()
.trim_matches('/')
.to_string();
if let Some(suffix) = suffix {
value = value.strip_suffix(suffix)?.trim_matches('/').to_string();
}
if value.is_empty() || value.contains('/') {
None
} else {
Some(value)
}
}
pub(super) fn admin_usage_id_from_detail_path(request_path: &str) -> Option<String> {
admin_usage_id_from_path_suffix(request_path, None)
aether_admin::observability::usage::admin_usage_id_from_detail_path(request_path)
}
pub(super) fn admin_usage_id_from_action_path(request_path: &str, action: &str) -> Option<String> {
admin_usage_id_from_path_suffix(request_path, Some(action))
aether_admin::observability::usage::admin_usage_id_from_action_path(request_path, action)
}
#[derive(Debug, Default, serde::Deserialize)]
@@ -52,66 +36,77 @@ struct AdminUsageReplayRequest {
body_override: Option<serde_json::Value>,
}
fn admin_usage_resolve_replay_mode(same_provider: bool, same_endpoint: bool) -> &'static str {
if same_provider && same_endpoint {
"same_endpoint_reuse"
} else if same_provider {
"same_provider_remap"
} else {
"cross_provider_remap"
}
}
pub(super) fn admin_usage_resolve_request_preview_body(
item: &StoredRequestUsageAudit,
body_override: Option<serde_json::Value>,
) -> serde_json::Value {
let resolved_model = item.model.clone();
let mut request_body = body_override
.or_else(|| item.request_body.clone())
.unwrap_or_else(|| {
json!({
"model": resolved_model,
"stream": item.is_stream,
})
});
if let Some(body) = request_body.as_object_mut() {
body.entry("model".to_string())
.or_insert_with(|| json!(resolved_model));
if !body.contains_key("stream") {
body.insert("stream".to_string(), json!(item.is_stream));
}
if let Some(target_model) = item
.target_model
.as_ref()
.filter(|value| !value.trim().is_empty())
{
body.entry("target_model".to_string())
.or_insert_with(|| json!(target_model));
}
if let Some(request_type) = item
.request_type
.as_ref()
.filter(|value| !value.trim().is_empty())
{
body.entry("request_type".to_string())
.or_insert_with(|| json!(request_type));
}
if let Some(api_format) = item
.api_format
.as_ref()
.filter(|value| !value.trim().is_empty())
{
body.entry("api_format".to_string())
.or_insert_with(|| json!(api_format));
}
}
request_body
aether_admin::observability::usage::admin_usage_resolve_request_preview_body(
item,
body_override,
)
}
pub(super) fn build_admin_usage_curl_response(
item: &StoredRequestUsageAudit,
url: Option<String>,
headers_json: Option<Value>,
headers: &BTreeMap<String, String>,
body: &Value,
) -> Response<Body> {
aether_admin::observability::usage::build_admin_usage_curl_response(
item,
url,
headers_json,
headers,
body,
)
}
pub(super) fn build_admin_usage_detail_payload(
item: &StoredRequestUsageAudit,
users_by_id: &BTreeMap<String, aether_data::repository::users::StoredUserSummary>,
provider_key_name: Option<&str>,
include_bodies: bool,
request_body: Value,
default_headers: &BTreeMap<String, String>,
) -> Value {
aether_admin::observability::usage::build_admin_usage_detail_payload(
item,
users_by_id,
provider_key_name,
include_bodies,
request_body,
default_headers,
)
}
pub(super) fn build_admin_usage_replay_plan_response(
item: &StoredRequestUsageAudit,
target_provider: &aether_data_contracts::repository::provider_catalog::StoredProviderCatalogProvider,
target_endpoint: &StoredProviderCatalogEndpoint,
target_api_key_id: Option<String>,
request_body: Value,
url: &str,
headers: &BTreeMap<String, String>,
same_provider: bool,
same_endpoint: bool,
) -> Response<Body> {
aether_admin::observability::usage::build_admin_usage_replay_plan_response(
item,
target_provider,
target_endpoint,
target_api_key_id,
request_body,
url,
headers,
same_provider,
same_endpoint,
)
}
pub(super) async fn build_admin_usage_replay_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
request_body: Option<&axum::body::Bytes>,
) -> Result<Response<Body>, GatewayError> {
if !state.has_usage_data_reader() || !state.has_provider_catalog_data_reader() {
@@ -132,12 +127,7 @@ pub(super) async fn build_admin_usage_replay_response(
_ => AdminUsageReplayRequest::default(),
};
let Some(item) = state
.data
.find_request_usage_by_id(&usage_id)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
else {
let Some(item) = state.find_request_usage_by_id(&usage_id).await? else {
return Ok((
http::StatusCode::NOT_FOUND,
Json(json!({ "detail": "Usage record not found" })),
@@ -264,59 +254,35 @@ pub(super) async fn build_admin_usage_replay_response(
let same_provider = item.provider_id.as_deref() == Some(target_provider.id.as_str());
let same_endpoint = item.provider_endpoint_id.as_deref() == Some(target_endpoint.id.as_str());
let resolved_model = item.model.clone();
let mapping_source = "none";
let request_body = admin_usage_resolve_request_preview_body(&item, payload.body_override);
let url = admin_usage_curl_url(&target_endpoint, &item);
let url = admin_usage_curl_url(state, &target_endpoint, &item);
let headers = admin_usage_curl_headers();
let curl = admin_usage_build_curl_command(Some(&url), &headers, Some(&request_body));
Ok(Json(json!({
"dry_run": true,
"usage_id": item.id,
"request_id": item.request_id,
"mode": admin_usage_resolve_replay_mode(same_provider, same_endpoint),
"target_provider_id": target_provider.id,
"target_provider_name": target_provider.name,
"target_endpoint_id": target_endpoint.id,
"target_api_key_id": payload.api_key_id.or(item.provider_api_key_id.clone()),
"target_api_format": target_endpoint.api_format,
"resolved_model": resolved_model,
"mapping_source": mapping_source,
"method": "POST",
"url": url,
"request_headers": headers,
"request_body": request_body,
"original_request_body_available": item.request_body.is_some(),
"note": "Rust local replay currently exposes a dry-run plan and does not dispatch upstream",
"curl": curl,
}))
.into_response())
Ok(build_admin_usage_replay_plan_response(
&item,
&target_provider,
&target_endpoint,
payload.api_key_id.or(item.provider_api_key_id.clone()),
request_body,
&url,
&headers,
same_provider,
same_endpoint,
))
}
pub(super) fn admin_usage_headers_from_value(
value: &serde_json::Value,
) -> Option<BTreeMap<String, String>> {
let object = value.as_object()?;
Some(BTreeMap::from_iter(object.iter().filter_map(
|(key, value)| {
if value.is_null() {
return None;
}
let value = value
.as_str()
.map(ToOwned::to_owned)
.unwrap_or_else(|| value.to_string());
Some((key.clone(), value))
},
)))
aether_admin::observability::usage::admin_usage_headers_from_value(value)
}
pub(super) fn admin_usage_curl_headers() -> BTreeMap<String, String> {
BTreeMap::from([("Content-Type".to_string(), "application/json".to_string())])
aether_admin::observability::usage::admin_usage_curl_headers()
}
pub(super) fn admin_usage_curl_url(
state: &AdminAppState<'_>,
endpoint: &StoredProviderCatalogEndpoint,
item: &StoredRequestUsageAudit,
) -> String {
@@ -331,60 +297,34 @@ pub(super) fn admin_usage_curl_url(
.as_deref()
.filter(|value| !value.is_empty())
{
return crate::provider_transport::url::build_passthrough_path_url(
&endpoint.base_url,
custom_path,
None,
&[],
)
.unwrap_or_else(|| endpoint.base_url.clone());
return state
.build_passthrough_path_url(&endpoint.base_url, custom_path, None, &[])
.unwrap_or_else(|| endpoint.base_url.clone());
}
match api_format {
value if value.starts_with("claude:") => {
crate::provider_transport::url::build_claude_messages_url(&endpoint.base_url, None)
state.build_claude_messages_url(&endpoint.base_url, None)
}
value if value.starts_with("gemini:") => {
crate::provider_transport::url::build_gemini_content_url(
value if value.starts_with("gemini:") => state
.build_gemini_content_url(
&endpoint.base_url,
item.target_model.as_deref().unwrap_or(item.model.as_str()),
item.is_stream,
None,
)
.unwrap_or_else(|| endpoint.base_url.clone())
}
.unwrap_or_else(|| endpoint.base_url.clone()),
value if value.starts_with("openai:") => {
crate::provider_transport::url::build_openai_chat_url(&endpoint.base_url, None)
state.build_openai_chat_url(&endpoint.base_url, None)
}
_ => endpoint.base_url.clone(),
}
}
fn admin_usage_curl_shell_quote(value: &str) -> String {
format!("'{}'", value.replace('\'', "'\"'\"'"))
}
pub(super) fn admin_usage_build_curl_command(
url: Option<&str>,
headers: &BTreeMap<String, String>,
body: Option<&serde_json::Value>,
) -> String {
let mut parts = vec!["curl".to_string()];
if let Some(url) = url {
parts.push(admin_usage_curl_shell_quote(url));
}
parts.push("-X POST".to_string());
for (key, value) in headers {
parts.push(format!(
"-H {}",
admin_usage_curl_shell_quote(&format!("{key}: {value}"))
));
}
if let Some(body) = body {
parts.push(format!(
"-d {}",
admin_usage_curl_shell_quote(&body.to_string())
));
}
parts.join(" \\\n ")
aether_admin::observability::usage::admin_usage_build_curl_command(url, headers, body)
}

154
apps/aether-gateway/src/handlers/admin/observability/usage/summary_routes.rs
View File

@@ -1,33 +1,23 @@
use super::super::stats::{
aggregate_usage_stats, list_usage_for_optional_range, round_to, AdminStatsTimeRange,
AdminStatsUsageFilter,
};
use super::analytics::{
use super::super::stats::{AdminStatsTimeRange, AdminStatsUsageFilter};
use super::analytics::admin_usage_provider_key_names;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::query_param_value;
use crate::GatewayError;
use aether_admin::observability::usage::{
admin_usage_bad_request_response, admin_usage_data_unavailable_response,
admin_usage_matches_api_format, admin_usage_matches_eq, admin_usage_matches_search,
admin_usage_matches_status, admin_usage_matches_username, admin_usage_parse_ids,
admin_usage_parse_limit, admin_usage_parse_offset, admin_usage_provider_key_name,
admin_usage_provider_key_names, admin_usage_record_json, admin_usage_total_tokens,
};
use super::helpers::{
admin_usage_bad_request_response, admin_usage_data_unavailable_response,
admin_usage_parse_limit, admin_usage_parse_offset, build_admin_usage_active_requests_response,
build_admin_usage_records_response, build_admin_usage_summary_stats_response,
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
};
use crate::control::GatewayPublicRequestContext;
use crate::handlers::admin::shared::query_param_value;
use crate::{AppState, GatewayError};
use aether_data_contracts::repository::usage::UsageAuditListQuery;
use axum::{
body::Body,
http,
response::{IntoResponse, Response},
Json,
};
use serde_json::json;
use axum::{body::Body, http, response::Response};
use std::collections::{BTreeMap, BTreeSet};
pub(super) async fn maybe_build_local_admin_usage_summary_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
) -> Result<Option<Response<Body>>, GatewayError> {
let route_kind = request_context
.control_decision
@@ -53,51 +43,13 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
let usage = list_usage_for_optional_range(
state,
time_range.as_ref(),
&AdminStatsUsageFilter::default(),
)
.await?;
let aggregate = aggregate_usage_stats(&usage);
let cache_creation_tokens: u64 = usage
.iter()
.map(|item| item.cache_creation_input_tokens)
.sum();
let cache_read_tokens: u64 =
usage.iter().map(|item| item.cache_read_input_tokens).sum();
let cache_creation_cost: f64 =
usage.iter().map(|item| item.cache_creation_cost_usd).sum();
let cache_read_cost: f64 = usage.iter().map(|item| item.cache_read_cost_usd).sum();
let total_tokens: u64 = usage.iter().map(admin_usage_total_tokens).sum();
let avg_response_time = round_to(aggregate.avg_response_time_ms() / 1000.0, 2);
let error_rate = if aggregate.total_requests == 0 {
0.0
} else {
round_to(
(aggregate.error_requests as f64 / aggregate.total_requests as f64) * 100.0,
2,
let usage = state
.list_admin_usage_for_optional_range(
time_range.as_ref(),
&AdminStatsUsageFilter::default(),
)
};
return Ok(Some(
Json(json!({
"total_requests": aggregate.total_requests,
"total_tokens": total_tokens,
"total_cost": round_to(aggregate.total_cost, 6),
"total_actual_cost": round_to(aggregate.actual_total_cost, 6),
"avg_response_time": avg_response_time,
"error_count": aggregate.error_requests,
"error_rate": error_rate,
"cache_stats": {
"cache_creation_tokens": cache_creation_tokens,
"cache_read_tokens": cache_read_tokens,
"cache_creation_cost": round_to(cache_creation_cost, 6),
"cache_read_cost": round_to(cache_read_cost, 6),
}
}))
.into_response(),
));
.await?;
return Ok(Some(build_admin_usage_summary_stats_response(&usage)));
}
Some("active")
if request_context.request_method == http::Method::GET
@@ -135,41 +87,10 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
}
let provider_key_names = admin_usage_provider_key_names(state, &items).await?;
let payload: Vec<_> = items
.into_iter()
.map(|item| {
let provider_key_name =
admin_usage_provider_key_name(&item, &provider_key_names);
let mut value = json!({
"id": item.id,
"status": item.status,
"input_tokens": item.input_tokens,
"output_tokens": item.output_tokens,
"cache_creation_input_tokens": item.cache_creation_input_tokens,
"cache_read_input_tokens": item.cache_read_input_tokens,
"cost": round_to(item.total_cost_usd, 6),
"actual_cost": round_to(item.actual_total_cost_usd, 6),
"response_time_ms": item.response_time_ms,
"first_byte_time_ms": item.first_byte_time_ms,
"provider": item.provider_name,
"api_key_name": item.api_key_name,
"provider_key_name": provider_key_name,
});
if let Some(api_format) = item.api_format {
value["api_format"] = json!(api_format);
}
if let Some(endpoint_api_format) = item.endpoint_api_format {
value["endpoint_api_format"] = json!(endpoint_api_format);
}
value["has_format_conversion"] = json!(item.has_format_conversion);
if let Some(target_model) = item.target_model {
value["target_model"] = json!(target_model);
}
value
})
.collect();
return Ok(Some(Json(json!({ "requests": payload })).into_response()));
return Ok(Some(build_admin_usage_active_requests_response(
&items,
&provider_key_names,
)));
}
Some("records")
if request_context.request_method == http::Method::GET
@@ -194,8 +115,9 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
provider_name: None,
model: None,
};
let mut usage =
list_usage_for_optional_range(state, time_range.as_ref(), &filters).await?;
let mut usage = state
.list_admin_usage_for_optional_range(time_range.as_ref(), &filters)
.await?;
let search = query_param_value(query, "search");
let username_filter = query_param_value(query, "username");
@@ -251,26 +173,20 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
let provider_key_names = admin_usage_provider_key_names(state, &usage).await?;
let records: Vec<_> = usage
let records = usage
.into_iter()
.skip(offset)
.take(limit)
.map(|item| {
let provider_key_name =
admin_usage_provider_key_name(&item, &provider_key_names);
admin_usage_record_json(&item, &users_by_id, provider_key_name.as_deref())
})
.collect();
.collect::<Vec<_>>();
return Ok(Some(
Json(json!({
"records": records,
"total": total,
"limit": limit,
"offset": offset,
}))
.into_response(),
));
return Ok(Some(build_admin_usage_records_response(
&records,
&users_by_id,
&provider_key_names,
total,
limit,
offset,
)));
}
_ => {}
}