mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-03 09:50:21 +08:00
refactor: 大规模模块拆分与代码精简，新增 ai-pipeline/data-contracts 独立 crate
- 新增 aether-ai-pipeline 和 aether-data-contracts crate，将 pipeline 逻辑与数据契约从 gateway 中解耦 - 重构 admin handlers：拆分单体模块为 auth/billing/endpoint/features/model/observability/provider/system 等独立子模块 - 合并 chat/cli 重复代码路径：精简 conversion、finalize、planner 中的 sync/chat/cli 分支 - 重构 scheduler/executor/data 层，引入 facade 模式降低模块间耦合 - 移除冗余的 intent 模块，将 plan_fallback/policy/stream_path/sync_path 迁移至 executor - 前端适配：调整 admin API 调用和 provider 模型测试对话框
This commit is contained in:
fawney19
@@ -0,0 +1,7 @@
|
||||
mod monitoring;
|
||||
pub(crate) mod stats;
|
||||
mod usage;
|
||||
|
||||
pub(crate) use self::monitoring::maybe_build_local_admin_monitoring_response;
|
||||
pub(crate) use self::stats::maybe_build_local_admin_stats_response;
|
||||
pub(crate) use self::usage::maybe_build_local_admin_usage_response;
|
||||
@@ -0,0 +1,335 @@
|
||||
use super::responses::admin_monitoring_bad_request_response;
|
||||
use super::route_filters::{
|
||||
admin_monitoring_escape_like_pattern, parse_admin_monitoring_days,
|
||||
parse_admin_monitoring_event_type_filter, parse_admin_monitoring_hours,
|
||||
parse_admin_monitoring_limit, parse_admin_monitoring_offset,
|
||||
parse_admin_monitoring_username_filter,
|
||||
};
|
||||
use super::usage_helpers::admin_monitoring_usage_is_error;
|
||||
use crate::constants::INTERNAL_GATEWAY_PATH_PREFIXES;
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use crate::query::monitoring as monitoring_query;
|
||||
use crate::{AppState, GatewayError};
|
||||
use aether_data_contracts::repository::usage::UsageAuditListQuery;
|
||||
use axum::{
|
||||
body::Body,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
fn build_admin_monitoring_audit_logs_payload(
|
||||
items: Vec<serde_json::Value>,
|
||||
total: usize,
|
||||
limit: usize,
|
||||
offset: usize,
|
||||
username: Option<String>,
|
||||
event_type: Option<String>,
|
||||
days: i64,
|
||||
) -> Response<Body> {
|
||||
let count = items.len();
|
||||
Json(json!({
|
||||
"items": items,
|
||||
"meta": {
|
||||
"total": total,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
"count": count,
|
||||
},
|
||||
"filters": {
|
||||
"username": username,
|
||||
"event_type": event_type,
|
||||
"days": days,
|
||||
},
|
||||
}))
|
||||
.into_response()
|
||||
}
|
||||
|
||||
fn build_admin_monitoring_suspicious_activities_payload(
|
||||
activities: Vec<serde_json::Value>,
|
||||
hours: i64,
|
||||
) -> Response<Body> {
|
||||
let count = activities.len();
|
||||
Json(json!({
|
||||
"activities": activities,
|
||||
"count": count,
|
||||
"time_range_hours": hours,
|
||||
}))
|
||||
.into_response()
|
||||
}
|
||||
|
||||
fn build_admin_monitoring_user_behavior_payload(
|
||||
user_id: String,
|
||||
days: i64,
|
||||
event_counts: std::collections::BTreeMap<String, u64>,
|
||||
failed_requests: u64,
|
||||
success_requests: u64,
|
||||
suspicious_activities: u64,
|
||||
) -> Response<Body> {
|
||||
let total_requests = success_requests.saturating_add(failed_requests);
|
||||
let success_rate = if total_requests == 0 {
|
||||
0.0
|
||||
} else {
|
||||
success_requests as f64 / total_requests as f64
|
||||
};
|
||||
|
||||
Json(json!({
|
||||
"user_id": user_id,
|
||||
"period_days": days,
|
||||
"event_counts": event_counts,
|
||||
"failed_requests": failed_requests,
|
||||
"success_requests": success_requests,
|
||||
"success_rate": success_rate,
|
||||
"suspicious_activities": suspicious_activities,
|
||||
"analysis_time": chrono::Utc::now().to_rfc3339(),
|
||||
}))
|
||||
.into_response()
|
||||
}
|
||||
|
||||
fn admin_monitoring_user_behavior_user_id_from_path(request_path: &str) -> Option<String> {
|
||||
let value = request_path
|
||||
.strip_prefix("/api/admin/monitoring/user-behavior/")?
|
||||
.trim()
|
||||
.trim_matches('/')
|
||||
.to_string();
|
||||
if value.is_empty() || value.contains('/') {
|
||||
None
|
||||
} else {
|
||||
Some(value)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_audit_logs_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let query = request_context.request_query_string.as_deref();
|
||||
let username = parse_admin_monitoring_username_filter(query);
|
||||
let event_type = parse_admin_monitoring_event_type_filter(query);
|
||||
let limit = match parse_admin_monitoring_limit(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(admin_monitoring_bad_request_response(detail)),
|
||||
};
|
||||
let offset = match parse_admin_monitoring_offset(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(admin_monitoring_bad_request_response(detail)),
|
||||
};
|
||||
let days = match parse_admin_monitoring_days(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(admin_monitoring_bad_request_response(detail)),
|
||||
};
|
||||
|
||||
let Some(pool) = state.postgres_pool() else {
|
||||
return Ok(build_admin_monitoring_audit_logs_payload(
|
||||
Vec::new(),
|
||||
0,
|
||||
limit,
|
||||
offset,
|
||||
username,
|
||||
event_type,
|
||||
days,
|
||||
));
|
||||
};
|
||||
|
||||
let cutoff_time = chrono::Utc::now() - chrono::Duration::days(days);
|
||||
let username_pattern = username
|
||||
.as_deref()
|
||||
.map(admin_monitoring_escape_like_pattern)
|
||||
.map(|value| format!("%{value}%"));
|
||||
|
||||
let (items, total) = monitoring_query::list_admin_audit_logs(
|
||||
&pool,
|
||||
cutoff_time,
|
||||
username_pattern.as_deref(),
|
||||
event_type.as_deref(),
|
||||
limit,
|
||||
offset,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(build_admin_monitoring_audit_logs_payload(
|
||||
items, total, limit, offset, username, event_type, days,
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_suspicious_activities_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let query = request_context.request_query_string.as_deref();
|
||||
let hours = match parse_admin_monitoring_hours(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(admin_monitoring_bad_request_response(detail)),
|
||||
};
|
||||
|
||||
let Some(pool) = state.postgres_pool() else {
|
||||
return Ok(build_admin_monitoring_suspicious_activities_payload(
|
||||
Vec::new(),
|
||||
hours,
|
||||
));
|
||||
};
|
||||
|
||||
let cutoff_time = chrono::Utc::now() - chrono::Duration::hours(hours);
|
||||
let activities = monitoring_query::list_admin_suspicious_activities(&pool, cutoff_time).await?;
|
||||
|
||||
Ok(build_admin_monitoring_suspicious_activities_payload(
|
||||
activities, hours,
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_user_behavior_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let Some(user_id) =
|
||||
admin_monitoring_user_behavior_user_id_from_path(&request_context.request_path)
|
||||
else {
|
||||
return Ok(admin_monitoring_bad_request_response("缺少 user_id"));
|
||||
};
|
||||
let days = match parse_admin_monitoring_days(request_context.request_query_string.as_deref()) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(admin_monitoring_bad_request_response(detail)),
|
||||
};
|
||||
|
||||
let Some(pool) = state.postgres_pool() else {
|
||||
return Ok(build_admin_monitoring_user_behavior_payload(
|
||||
user_id,
|
||||
days,
|
||||
std::collections::BTreeMap::new(),
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
));
|
||||
};
|
||||
|
||||
let cutoff_time = chrono::Utc::now() - chrono::Duration::days(days);
|
||||
|
||||
let event_counts =
|
||||
monitoring_query::read_admin_user_behavior_event_counts(&pool, &user_id, cutoff_time)
|
||||
.await?;
|
||||
|
||||
let failed_requests = event_counts
|
||||
.get("request_failed")
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
let success_requests = event_counts
|
||||
.get("request_success")
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
let suspicious_activities = event_counts
|
||||
.get("suspicious_activity")
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.saturating_add(
|
||||
event_counts
|
||||
.get("unauthorized_access")
|
||||
.copied()
|
||||
.unwrap_or_default(),
|
||||
);
|
||||
|
||||
Ok(build_admin_monitoring_user_behavior_payload(
|
||||
user_id,
|
||||
days,
|
||||
event_counts,
|
||||
failed_requests,
|
||||
success_requests,
|
||||
suspicious_activities,
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_system_status_response(
|
||||
state: &AppState,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let now = chrono::Utc::now();
|
||||
let today_start = now
|
||||
.date_naive()
|
||||
.and_hms_opt(0, 0, 0)
|
||||
.expect("midnight should be valid")
|
||||
.and_utc();
|
||||
let recent_error_from = now - chrono::Duration::hours(1);
|
||||
let now_unix_secs = now.timestamp().max(0) as u64;
|
||||
|
||||
let user_summary = state.summarize_export_users().await?;
|
||||
let total_users = user_summary.total;
|
||||
let active_users = user_summary.active;
|
||||
|
||||
let providers = state
|
||||
.data
|
||||
.list_provider_catalog_providers(false)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let total_providers = providers.len();
|
||||
let active_providers = providers.iter().filter(|item| item.is_active).count();
|
||||
|
||||
let user_api_key_summary = state
|
||||
.summarize_auth_api_key_export_non_standalone_records(now_unix_secs)
|
||||
.await?;
|
||||
let standalone_api_key_summary = state
|
||||
.summarize_auth_api_key_export_standalone_records(now_unix_secs)
|
||||
.await?;
|
||||
let total_api_keys = user_api_key_summary
|
||||
.total
|
||||
.saturating_add(standalone_api_key_summary.total);
|
||||
let active_api_keys = user_api_key_summary
|
||||
.active
|
||||
.saturating_add(standalone_api_key_summary.active);
|
||||
|
||||
let today_usage = state
|
||||
.list_usage_audits(&UsageAuditListQuery {
|
||||
created_from_unix_secs: Some(today_start.timestamp().max(0) as u64),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let today_requests = today_usage.len();
|
||||
let today_tokens = today_usage
|
||||
.iter()
|
||||
.map(|item| item.total_tokens)
|
||||
.sum::<u64>();
|
||||
let today_cost = today_usage
|
||||
.iter()
|
||||
.map(|item| item.total_cost_usd)
|
||||
.sum::<f64>();
|
||||
|
||||
let recent_errors = state
|
||||
.list_usage_audits(&UsageAuditListQuery {
|
||||
created_from_unix_secs: Some(recent_error_from.timestamp().max(0) as u64),
|
||||
..Default::default()
|
||||
})
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter(admin_monitoring_usage_is_error)
|
||||
.count();
|
||||
let tunnel = state.tunnel.stats();
|
||||
|
||||
Ok(Json(json!({
|
||||
"timestamp": now.to_rfc3339(),
|
||||
"users": {
|
||||
"total": total_users,
|
||||
"active": active_users,
|
||||
},
|
||||
"providers": {
|
||||
"total": total_providers,
|
||||
"active": active_providers,
|
||||
},
|
||||
"api_keys": {
|
||||
"total": total_api_keys,
|
||||
"active": active_api_keys,
|
||||
},
|
||||
"today_stats": {
|
||||
"requests": today_requests,
|
||||
"tokens": today_tokens,
|
||||
"cost_usd": format!("${today_cost:.4}"),
|
||||
},
|
||||
"tunnel": {
|
||||
"proxy_connections": tunnel.proxy_connections,
|
||||
"nodes": tunnel.nodes,
|
||||
"active_streams": tunnel.active_streams,
|
||||
},
|
||||
"internal_gateway": {
|
||||
"status": "rust_native_control_plane",
|
||||
"path_prefixes": INTERNAL_GATEWAY_PATH_PREFIXES,
|
||||
},
|
||||
"recent_errors": recent_errors,
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
use super::cache_config::{
|
||||
ADMIN_MONITORING_CACHE_AFFINITY_DEFAULT_TTL_SECS, ADMIN_MONITORING_CACHE_RESERVATION_RATIO,
|
||||
ADMIN_MONITORING_DYNAMIC_RESERVATION_HIGH_LOAD_THRESHOLD,
|
||||
ADMIN_MONITORING_DYNAMIC_RESERVATION_LOW_LOAD_THRESHOLD,
|
||||
ADMIN_MONITORING_DYNAMIC_RESERVATION_PROBE_PHASE_REQUESTS,
|
||||
ADMIN_MONITORING_DYNAMIC_RESERVATION_PROBE_RESERVATION,
|
||||
ADMIN_MONITORING_DYNAMIC_RESERVATION_STABLE_MAX_RESERVATION,
|
||||
ADMIN_MONITORING_DYNAMIC_RESERVATION_STABLE_MIN_RESERVATION,
|
||||
};
|
||||
use super::cache_store::build_admin_monitoring_cache_snapshot;
|
||||
use crate::{AppState, GatewayError};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
pub(super) async fn build_admin_monitoring_cache_stats_response(
|
||||
state: &AppState,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let snapshot = build_admin_monitoring_cache_snapshot(state).await?;
|
||||
|
||||
Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"data": {
|
||||
"scheduler": snapshot.scheduler_name,
|
||||
"total_affinities": snapshot.total_affinities,
|
||||
"cache_hit_rate": snapshot.cache_hit_rate,
|
||||
"provider_switches": snapshot.provider_switches,
|
||||
"key_switches": snapshot.key_switches,
|
||||
"cache_hits": snapshot.cache_hits,
|
||||
"cache_misses": snapshot.cache_misses,
|
||||
"scheduler_metrics": {
|
||||
"cache_hits": snapshot.cache_hits,
|
||||
"cache_misses": snapshot.cache_misses,
|
||||
"cache_hit_rate": snapshot.cache_hit_rate,
|
||||
"total_batches": 0,
|
||||
"last_batch_size": 0,
|
||||
"total_candidates": 0,
|
||||
"last_candidate_count": 0,
|
||||
"concurrency_denied": 0,
|
||||
"avg_candidates_per_batch": 0.0,
|
||||
"scheduling_mode": snapshot.scheduling_mode,
|
||||
"provider_priority_mode": snapshot.provider_priority_mode,
|
||||
},
|
||||
"affinity_stats": {
|
||||
"storage_type": snapshot.storage_type,
|
||||
"total_affinities": snapshot.total_affinities,
|
||||
"cache_hits": snapshot.cache_hits,
|
||||
"cache_misses": snapshot.cache_misses,
|
||||
"cache_hit_rate": snapshot.cache_hit_rate,
|
||||
"cache_invalidations": snapshot.cache_invalidations,
|
||||
"provider_switches": snapshot.provider_switches,
|
||||
"key_switches": snapshot.key_switches,
|
||||
"config": {
|
||||
"default_ttl": ADMIN_MONITORING_CACHE_AFFINITY_DEFAULT_TTL_SECS,
|
||||
}
|
||||
}
|
||||
}
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_cache_metrics_response(
|
||||
state: &AppState,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let snapshot = build_admin_monitoring_cache_snapshot(state).await?;
|
||||
let metrics = [
|
||||
(
|
||||
"cache_scheduler_total_batches",
|
||||
"Number of scheduling batches processed",
|
||||
0.0,
|
||||
),
|
||||
(
|
||||
"cache_scheduler_last_batch_size",
|
||||
"Size of the most recent scheduling batch",
|
||||
0.0,
|
||||
),
|
||||
(
|
||||
"cache_scheduler_total_candidates",
|
||||
"Total candidates seen during scheduling",
|
||||
0.0,
|
||||
),
|
||||
(
|
||||
"cache_scheduler_last_candidate_count",
|
||||
"Number of candidates in the most recent batch",
|
||||
0.0,
|
||||
),
|
||||
(
|
||||
"cache_scheduler_cache_hits",
|
||||
"Cache hits counted during scheduling",
|
||||
snapshot.cache_hits as f64,
|
||||
),
|
||||
(
|
||||
"cache_scheduler_cache_misses",
|
||||
"Cache misses counted during scheduling",
|
||||
snapshot.cache_misses as f64,
|
||||
),
|
||||
(
|
||||
"cache_scheduler_cache_hit_rate",
|
||||
"Cache hit rate during scheduling",
|
||||
snapshot.cache_hit_rate,
|
||||
),
|
||||
(
|
||||
"cache_scheduler_concurrency_denied",
|
||||
"Times candidate rejected due to concurrency limits",
|
||||
0.0,
|
||||
),
|
||||
(
|
||||
"cache_scheduler_avg_candidates_per_batch",
|
||||
"Average candidates per batch",
|
||||
0.0,
|
||||
),
|
||||
(
|
||||
"cache_affinity_total",
|
||||
"Total cache affinities stored",
|
||||
snapshot.total_affinities as f64,
|
||||
),
|
||||
(
|
||||
"cache_affinity_hits",
|
||||
"Affinity cache hits",
|
||||
snapshot.cache_hits as f64,
|
||||
),
|
||||
(
|
||||
"cache_affinity_misses",
|
||||
"Affinity cache misses",
|
||||
snapshot.cache_misses as f64,
|
||||
),
|
||||
(
|
||||
"cache_affinity_hit_rate",
|
||||
"Affinity cache hit rate",
|
||||
snapshot.cache_hit_rate,
|
||||
),
|
||||
(
|
||||
"cache_affinity_invalidations",
|
||||
"Affinity invalidations",
|
||||
snapshot.cache_invalidations as f64,
|
||||
),
|
||||
(
|
||||
"cache_affinity_provider_switches",
|
||||
"Affinity provider switches",
|
||||
snapshot.provider_switches as f64,
|
||||
),
|
||||
(
|
||||
"cache_affinity_key_switches",
|
||||
"Affinity key switches",
|
||||
snapshot.key_switches as f64,
|
||||
),
|
||||
];
|
||||
|
||||
let mut lines = Vec::with_capacity(metrics.len() * 3 + 1);
|
||||
for (name, help_text, value) in metrics {
|
||||
lines.push(format!("# HELP {name} {help_text}"));
|
||||
lines.push(format!("# TYPE {name} gauge"));
|
||||
lines.push(format!("{name} {value}"));
|
||||
}
|
||||
lines.push(format!(
|
||||
"cache_scheduler_info{{scheduler=\"{}\"}} 1",
|
||||
snapshot.scheduler_name
|
||||
));
|
||||
|
||||
Ok((
|
||||
[(
|
||||
http::header::CONTENT_TYPE,
|
||||
"text/plain; version=0.0.4; charset=utf-8",
|
||||
)],
|
||||
lines.join("\n") + "\n",
|
||||
)
|
||||
.into_response())
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_cache_config_response(
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"data": {
|
||||
"cache_ttl_seconds": ADMIN_MONITORING_CACHE_AFFINITY_DEFAULT_TTL_SECS,
|
||||
"cache_reservation_ratio": ADMIN_MONITORING_CACHE_RESERVATION_RATIO,
|
||||
"dynamic_reservation": {
|
||||
"enabled": true,
|
||||
"config": {
|
||||
"probe_phase_requests": ADMIN_MONITORING_DYNAMIC_RESERVATION_PROBE_PHASE_REQUESTS,
|
||||
"probe_reservation": ADMIN_MONITORING_DYNAMIC_RESERVATION_PROBE_RESERVATION,
|
||||
"stable_min_reservation": ADMIN_MONITORING_DYNAMIC_RESERVATION_STABLE_MIN_RESERVATION,
|
||||
"stable_max_reservation": ADMIN_MONITORING_DYNAMIC_RESERVATION_STABLE_MAX_RESERVATION,
|
||||
"low_load_threshold": ADMIN_MONITORING_DYNAMIC_RESERVATION_LOW_LOAD_THRESHOLD,
|
||||
"high_load_threshold": ADMIN_MONITORING_DYNAMIC_RESERVATION_HIGH_LOAD_THRESHOLD,
|
||||
},
|
||||
"description": {
|
||||
"probe_phase_requests": "探测阶段请求数阈值",
|
||||
"probe_reservation": "探测阶段预留比例",
|
||||
"stable_min_reservation": "稳定阶段最小预留比例",
|
||||
"stable_max_reservation": "稳定阶段最大预留比例",
|
||||
"low_load_threshold": "低负载阈值（低于此值使用最小预留）",
|
||||
"high_load_threshold": "高负载阈值（高于此值根据置信度使用较高预留）",
|
||||
},
|
||||
},
|
||||
"description": {
|
||||
"cache_ttl": "缓存亲和性有效期（秒）",
|
||||
"cache_reservation_ratio": "静态预留比例（已被动态预留替代）",
|
||||
"dynamic_reservation": "动态预留机制配置",
|
||||
},
|
||||
}
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
use super::cache_types::AdminMonitoringCacheAffinityRecord;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
fn parse_admin_monitoring_cache_affinity_key(raw_key: &str) -> Option<(String, String, String)> {
|
||||
let parts = raw_key.split(':').collect::<Vec<_>>();
|
||||
let start = parts
|
||||
.iter()
|
||||
.position(|segment| *segment == "cache_affinity")?;
|
||||
let affinity_key = parts.get(start + 1)?.trim();
|
||||
if affinity_key.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let api_format = parts
|
||||
.get(start + 2)
|
||||
.map(|value| value.trim())
|
||||
.filter(|value| !value.is_empty())
|
||||
.unwrap_or("unknown")
|
||||
.to_string();
|
||||
let model_name = parts
|
||||
.get(start + 3..)
|
||||
.filter(|segments| !segments.is_empty())
|
||||
.map(|segments| segments.join(":"))
|
||||
.unwrap_or_else(|| "unknown".to_string());
|
||||
Some((affinity_key.to_string(), api_format, model_name))
|
||||
}
|
||||
|
||||
pub(super) fn admin_monitoring_scheduler_affinity_cache_key(
|
||||
record: &AdminMonitoringCacheAffinityRecord,
|
||||
) -> Option<String> {
|
||||
let affinity_key = record.affinity_key.trim();
|
||||
let api_format = record.api_format.trim().to_ascii_lowercase();
|
||||
let model_name = record.model_name.trim();
|
||||
if affinity_key.is_empty() || api_format.is_empty() || model_name.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(format!(
|
||||
"scheduler_affinity:{affinity_key}:{api_format}:{model_name}"
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) fn admin_monitoring_cache_affinity_record(
|
||||
raw_key: &str,
|
||||
raw_value: &str,
|
||||
) -> Option<AdminMonitoringCacheAffinityRecord> {
|
||||
let payload = serde_json::from_str::<serde_json::Value>(raw_value).ok()?;
|
||||
let object = payload.as_object()?;
|
||||
let (affinity_key, parsed_api_format, parsed_model_name) =
|
||||
parse_admin_monitoring_cache_affinity_key(raw_key)?;
|
||||
let api_format = object
|
||||
.get("api_format")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.unwrap_or(parsed_api_format.as_str())
|
||||
.to_string();
|
||||
let model_name = object
|
||||
.get("model_name")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.unwrap_or(parsed_model_name.as_str())
|
||||
.to_string();
|
||||
let request_count = object
|
||||
.get("request_count")
|
||||
.and_then(|value| {
|
||||
value
|
||||
.as_u64()
|
||||
.or_else(|| value.as_i64().and_then(|number| u64::try_from(number).ok()))
|
||||
})
|
||||
.unwrap_or(0);
|
||||
Some(AdminMonitoringCacheAffinityRecord {
|
||||
raw_key: raw_key.to_string(),
|
||||
affinity_key,
|
||||
api_format,
|
||||
model_name,
|
||||
provider_id: object
|
||||
.get("provider_id")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(ToOwned::to_owned),
|
||||
endpoint_id: object
|
||||
.get("endpoint_id")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(ToOwned::to_owned),
|
||||
key_id: object
|
||||
.get("key_id")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(ToOwned::to_owned),
|
||||
created_at: object.get("created_at").cloned(),
|
||||
expire_at: object.get("expire_at").cloned(),
|
||||
request_count,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn clear_admin_monitoring_scheduler_affinity_entries(
|
||||
state: &AppState,
|
||||
records: &[AdminMonitoringCacheAffinityRecord],
|
||||
) {
|
||||
let scheduler_keys = records
|
||||
.iter()
|
||||
.filter_map(admin_monitoring_scheduler_affinity_cache_key)
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
for scheduler_key in scheduler_keys {
|
||||
let _ = state.remove_scheduler_affinity_cache_entry(&scheduler_key);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn delete_admin_monitoring_cache_affinity_entries_for_tests(
|
||||
state: &AppState,
|
||||
raw_keys: &[String],
|
||||
) -> usize {
|
||||
state.remove_admin_monitoring_cache_affinity_entries_for_tests(raw_keys)
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
pub(super) fn delete_admin_monitoring_cache_affinity_entries_for_tests(
|
||||
_state: &AppState,
|
||||
_raw_keys: &[String],
|
||||
) -> usize {
|
||||
0
|
||||
}
|
||||
|
||||
pub(super) async fn delete_admin_monitoring_cache_affinity_raw_keys(
|
||||
state: &AppState,
|
||||
raw_keys: &[String],
|
||||
) -> Result<usize, GatewayError> {
|
||||
if raw_keys.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
if let Some(runner) = state.redis_kv_runner() {
|
||||
let mut connection = runner
|
||||
.client()
|
||||
.get_multiplexed_async_connection()
|
||||
.await
|
||||
.map_err(|err| {
|
||||
GatewayError::Internal(format!("admin monitoring redis connect failed: {err}"))
|
||||
})?;
|
||||
let deleted = redis::cmd("DEL")
|
||||
.arg(raw_keys)
|
||||
.query_async::<i64>(&mut connection)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
GatewayError::Internal(format!("admin monitoring redis delete failed: {err}"))
|
||||
})?;
|
||||
return Ok(usize::try_from(deleted).unwrap_or(0));
|
||||
}
|
||||
|
||||
Ok(delete_admin_monitoring_cache_affinity_entries_for_tests(
|
||||
state, raw_keys,
|
||||
))
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
use super::cache_identity::{
|
||||
admin_monitoring_find_user_summary_by_id, admin_monitoring_list_export_api_key_records_by_ids,
|
||||
admin_monitoring_load_affinity_identity_maps,
|
||||
};
|
||||
use super::cache_payloads::{
|
||||
admin_monitoring_cache_affinity_sort_value, admin_monitoring_masked_provider_key_prefix,
|
||||
admin_monitoring_masked_user_api_key_prefix,
|
||||
};
|
||||
use super::cache_route_helpers::{
|
||||
admin_monitoring_cache_affinity_not_found_response,
|
||||
admin_monitoring_cache_affinity_user_identifier_from_path,
|
||||
parse_admin_monitoring_keyword_filter,
|
||||
};
|
||||
use super::cache_store::{
|
||||
list_admin_monitoring_cache_affinity_records,
|
||||
list_admin_monitoring_cache_affinity_records_by_affinity_keys,
|
||||
};
|
||||
use super::responses::admin_monitoring_bad_request_response;
|
||||
use super::route_filters::{parse_admin_monitoring_limit, parse_admin_monitoring_offset};
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use crate::{AppState, GatewayError};
|
||||
use axum::{
|
||||
body::Body,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
fn normalize_keyword<'a>(keyword: Option<&'a String>) -> Option<String> {
|
||||
keyword.map(|value| value.to_ascii_lowercase())
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_cache_affinities_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let limit = match parse_admin_monitoring_limit(request_context.request_query_string.as_deref())
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(admin_monitoring_bad_request_response(detail)),
|
||||
};
|
||||
let offset =
|
||||
match parse_admin_monitoring_offset(request_context.request_query_string.as_deref()) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(admin_monitoring_bad_request_response(detail)),
|
||||
};
|
||||
let keyword =
|
||||
parse_admin_monitoring_keyword_filter(request_context.request_query_string.as_deref());
|
||||
|
||||
let mut matched_user_id = None::<String>;
|
||||
let mut matched_api_key_id = None::<String>;
|
||||
let filtered_affinities = if let Some(keyword_value) = keyword.as_deref() {
|
||||
let direct_affinity_keys =
|
||||
std::iter::once(keyword_value.to_string()).collect::<std::collections::BTreeSet<_>>();
|
||||
let direct_affinities = list_admin_monitoring_cache_affinity_records_by_affinity_keys(
|
||||
state,
|
||||
&direct_affinity_keys,
|
||||
)
|
||||
.await?;
|
||||
if !direct_affinities.is_empty() {
|
||||
matched_api_key_id = Some(keyword_value.to_string());
|
||||
matched_user_id = admin_monitoring_list_export_api_key_records_by_ids(
|
||||
state,
|
||||
&[keyword_value.to_string()],
|
||||
)
|
||||
.await?
|
||||
.get(keyword_value)
|
||||
.map(|item| item.user_id.clone());
|
||||
direct_affinities
|
||||
} else if let Some(user) = state.find_user_auth_by_identifier(keyword_value).await? {
|
||||
matched_user_id = Some(user.id.clone());
|
||||
let user_api_key_ids = state
|
||||
.list_auth_api_key_export_records_by_user_ids(std::slice::from_ref(&user.id))
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|item| item.api_key_id)
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
list_admin_monitoring_cache_affinity_records_by_affinity_keys(state, &user_api_key_ids)
|
||||
.await?
|
||||
} else {
|
||||
list_admin_monitoring_cache_affinity_records(state).await?
|
||||
}
|
||||
} else {
|
||||
list_admin_monitoring_cache_affinity_records(state).await?
|
||||
};
|
||||
let (api_key_by_id, user_by_id) =
|
||||
admin_monitoring_load_affinity_identity_maps(state, &filtered_affinities).await?;
|
||||
|
||||
let provider_ids = filtered_affinities
|
||||
.iter()
|
||||
.filter_map(|item| item.provider_id.clone())
|
||||
.collect::<std::collections::BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>();
|
||||
let endpoint_ids = filtered_affinities
|
||||
.iter()
|
||||
.filter_map(|item| item.endpoint_id.clone())
|
||||
.collect::<std::collections::BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>();
|
||||
let key_ids = filtered_affinities
|
||||
.iter()
|
||||
.filter_map(|item| item.key_id.clone())
|
||||
.collect::<std::collections::BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let provider_by_id = state
|
||||
.data
|
||||
.list_provider_catalog_providers_by_ids(&provider_ids)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
.into_iter()
|
||||
.map(|item| (item.id.clone(), item))
|
||||
.collect::<std::collections::BTreeMap<_, _>>();
|
||||
let endpoint_by_id = state
|
||||
.data
|
||||
.list_provider_catalog_endpoints_by_ids(&endpoint_ids)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
.into_iter()
|
||||
.map(|item| (item.id.clone(), item))
|
||||
.collect::<std::collections::BTreeMap<_, _>>();
|
||||
let key_by_id = state
|
||||
.data
|
||||
.list_provider_catalog_keys_by_ids(&key_ids)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
.into_iter()
|
||||
.map(|item| (item.id.clone(), item))
|
||||
.collect::<std::collections::BTreeMap<_, _>>();
|
||||
|
||||
let keyword_lower = normalize_keyword(keyword.as_ref());
|
||||
let mut items = Vec::new();
|
||||
for affinity in filtered_affinities {
|
||||
let user_api_key = api_key_by_id.get(&affinity.affinity_key);
|
||||
let user_id = user_api_key.map(|item| item.user_id.clone());
|
||||
let user = user_id.as_ref().and_then(|id| user_by_id.get(id));
|
||||
let provider = affinity
|
||||
.provider_id
|
||||
.as_ref()
|
||||
.and_then(|id| provider_by_id.get(id));
|
||||
let endpoint = affinity
|
||||
.endpoint_id
|
||||
.as_ref()
|
||||
.and_then(|id| endpoint_by_id.get(id));
|
||||
let key = affinity.key_id.as_ref().and_then(|id| key_by_id.get(id));
|
||||
|
||||
let user_api_key_name = user_api_key.and_then(|item| item.name.clone());
|
||||
let user_api_key_prefix = user_api_key.and_then(|item| {
|
||||
admin_monitoring_masked_user_api_key_prefix(state, item.key_encrypted.as_deref())
|
||||
});
|
||||
let provider_name = provider.map(|item| item.name.clone());
|
||||
let endpoint_url = endpoint
|
||||
.map(|item| item.base_url.clone())
|
||||
.filter(|value| !value.trim().is_empty());
|
||||
let key_name = key.map(|item| item.name.clone());
|
||||
let key_prefix =
|
||||
key.and_then(|item| admin_monitoring_masked_provider_key_prefix(state, item));
|
||||
let user_id_text = user_id.clone();
|
||||
let username = user.map(|item| item.username.clone());
|
||||
let email = user.and_then(|item| item.email.clone());
|
||||
let provider_id = affinity.provider_id.clone();
|
||||
let key_id = affinity.key_id.clone();
|
||||
|
||||
if let Some(keyword_value) = keyword_lower.as_deref() {
|
||||
if matched_user_id.is_none() && matched_api_key_id.is_none() {
|
||||
let searchable = [
|
||||
Some(affinity.affinity_key.as_str()),
|
||||
user_api_key_name.as_deref(),
|
||||
user_id_text.as_deref(),
|
||||
username.as_deref(),
|
||||
email.as_deref(),
|
||||
provider_id.as_deref(),
|
||||
key_id.as_deref(),
|
||||
];
|
||||
if !searchable
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.any(|value| value.to_ascii_lowercase().contains(keyword_value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
items.push(json!({
|
||||
"affinity_key": affinity.affinity_key,
|
||||
"user_api_key_name": user_api_key_name,
|
||||
"user_api_key_prefix": user_api_key_prefix,
|
||||
"is_standalone": user_api_key.map(|item| item.is_standalone).unwrap_or(false),
|
||||
"user_id": user_id_text,
|
||||
"username": username,
|
||||
"email": email,
|
||||
"provider_id": provider_id,
|
||||
"provider_name": provider_name,
|
||||
"endpoint_id": affinity.endpoint_id,
|
||||
"endpoint_url": endpoint_url,
|
||||
"key_id": key_id,
|
||||
"key_name": key_name,
|
||||
"key_prefix": key_prefix,
|
||||
"rate_multipliers": key.and_then(|item| item.rate_multipliers.clone()),
|
||||
"global_model_id": affinity.model_name,
|
||||
"model_name": affinity.model_name,
|
||||
"model_display_name": serde_json::Value::Null,
|
||||
"api_format": affinity.api_format,
|
||||
"created_at": affinity.created_at,
|
||||
"expire_at": affinity.expire_at,
|
||||
"request_count": affinity.request_count,
|
||||
}));
|
||||
}
|
||||
|
||||
items.sort_by(|left, right| {
|
||||
admin_monitoring_cache_affinity_sort_value(right.get("expire_at"))
|
||||
.partial_cmp(&admin_monitoring_cache_affinity_sort_value(
|
||||
left.get("expire_at"),
|
||||
))
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
let total = items.len();
|
||||
let paged_items = items
|
||||
.into_iter()
|
||||
.skip(offset)
|
||||
.take(limit)
|
||||
.collect::<Vec<_>>();
|
||||
let paged_count = paged_items.len();
|
||||
|
||||
Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"data": {
|
||||
"items": paged_items,
|
||||
"meta": {
|
||||
"total": total,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
"count": paged_count,
|
||||
},
|
||||
"matched_user_id": matched_user_id,
|
||||
}
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_cache_affinity_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let Some(user_identifier) =
|
||||
admin_monitoring_cache_affinity_user_identifier_from_path(&request_context.request_path)
|
||||
else {
|
||||
return Ok(admin_monitoring_bad_request_response(
|
||||
"缺少 user_identifier",
|
||||
));
|
||||
};
|
||||
let direct_api_key_by_id =
|
||||
admin_monitoring_list_export_api_key_records_by_ids(state, &[user_identifier.clone()])
|
||||
.await?;
|
||||
let direct_affinity_keys =
|
||||
std::iter::once(user_identifier.clone()).collect::<std::collections::BTreeSet<_>>();
|
||||
let direct_affinities =
|
||||
list_admin_monitoring_cache_affinity_records_by_affinity_keys(state, &direct_affinity_keys)
|
||||
.await?;
|
||||
|
||||
let (resolved_user_id, username, email, filtered_affinities) = if !direct_affinities.is_empty()
|
||||
|| direct_api_key_by_id.contains_key(&user_identifier)
|
||||
{
|
||||
let user_id = direct_api_key_by_id
|
||||
.get(&user_identifier)
|
||||
.map(|item| item.user_id.clone());
|
||||
let user = match user_id.as_deref() {
|
||||
Some(user_id) => admin_monitoring_find_user_summary_by_id(state, user_id).await?,
|
||||
None => None,
|
||||
};
|
||||
(
|
||||
user_id,
|
||||
user.as_ref().map(|item| item.username.clone()),
|
||||
user.and_then(|item| item.email),
|
||||
direct_affinities,
|
||||
)
|
||||
} else if let Some(user) = state.find_user_auth_by_identifier(&user_identifier).await? {
|
||||
let user_api_key_ids = state
|
||||
.list_auth_api_key_export_records_by_user_ids(std::slice::from_ref(&user.id))
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|item| item.api_key_id)
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
let affinities =
|
||||
list_admin_monitoring_cache_affinity_records_by_affinity_keys(state, &user_api_key_ids)
|
||||
.await?;
|
||||
(Some(user.id), Some(user.username), user.email, affinities)
|
||||
} else {
|
||||
return Ok(admin_monitoring_cache_affinity_not_found_response(
|
||||
&user_identifier,
|
||||
));
|
||||
};
|
||||
|
||||
if filtered_affinities.is_empty() {
|
||||
let display_name = username.clone().unwrap_or_else(|| user_identifier.clone());
|
||||
return Ok(Json(json!({
|
||||
"status": "not_found",
|
||||
"message": format!(
|
||||
"用户 {} ({}) 没有缓存亲和性",
|
||||
display_name,
|
||||
email.clone().unwrap_or_else(|| "null".to_string()),
|
||||
),
|
||||
"user_info": {
|
||||
"user_id": resolved_user_id,
|
||||
"username": username,
|
||||
"email": email,
|
||||
},
|
||||
"affinities": [],
|
||||
}))
|
||||
.into_response());
|
||||
}
|
||||
|
||||
let mut affinities = filtered_affinities
|
||||
.into_iter()
|
||||
.map(|item| {
|
||||
json!({
|
||||
"provider_id": item.provider_id,
|
||||
"endpoint_id": item.endpoint_id,
|
||||
"key_id": item.key_id,
|
||||
"api_format": item.api_format,
|
||||
"model_name": item.model_name,
|
||||
"created_at": item.created_at,
|
||||
"expire_at": item.expire_at,
|
||||
"request_count": item.request_count,
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
affinities.sort_by(|left, right| {
|
||||
admin_monitoring_cache_affinity_sort_value(right.get("expire_at"))
|
||||
.partial_cmp(&admin_monitoring_cache_affinity_sort_value(
|
||||
left.get("expire_at"),
|
||||
))
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
let total_endpoints = affinities.len();
|
||||
|
||||
Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"user_info": {
|
||||
"user_id": resolved_user_id,
|
||||
"username": username,
|
||||
"email": email,
|
||||
},
|
||||
"affinities": affinities,
|
||||
"total_endpoints": total_endpoints,
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
pub(crate) const ADMIN_MONITORING_CACHE_AFFINITY_REDIS_REQUIRED_DETAIL: &str =
|
||||
"Redis未初始化，无法获取缓存亲和性";
|
||||
pub(crate) const ADMIN_MONITORING_REDIS_REQUIRED_DETAIL: &str = "Redis 未启用";
|
||||
pub(crate) const ADMIN_MONITORING_CACHE_AFFINITY_DEFAULT_TTL_SECS: u64 = 300;
|
||||
pub(crate) const ADMIN_MONITORING_CACHE_RESERVATION_RATIO: f64 = 0.1;
|
||||
pub(crate) const ADMIN_MONITORING_DYNAMIC_RESERVATION_PROBE_PHASE_REQUESTS: u64 = 100;
|
||||
pub(crate) const ADMIN_MONITORING_DYNAMIC_RESERVATION_PROBE_RESERVATION: f64 = 0.1;
|
||||
pub(crate) const ADMIN_MONITORING_DYNAMIC_RESERVATION_STABLE_MIN_RESERVATION: f64 = 0.1;
|
||||
pub(crate) const ADMIN_MONITORING_DYNAMIC_RESERVATION_STABLE_MAX_RESERVATION: f64 = 0.35;
|
||||
pub(crate) const ADMIN_MONITORING_DYNAMIC_RESERVATION_LOW_LOAD_THRESHOLD: f64 = 0.5;
|
||||
pub(crate) const ADMIN_MONITORING_DYNAMIC_RESERVATION_HIGH_LOAD_THRESHOLD: f64 = 0.8;
|
||||
pub(crate) const ADMIN_MONITORING_REDIS_CACHE_CATEGORIES: &[(&str, &str, &str, &str)] = &[
|
||||
(
|
||||
"upstream_models",
|
||||
"上游模型",
|
||||
"upstream_models:*",
|
||||
"Provider 上游获取的模型列表缓存",
|
||||
),
|
||||
("model_id", "模型 ID", "model:id:*", "Model 按 ID 缓存"),
|
||||
(
|
||||
"model_provider_global",
|
||||
"模型映射",
|
||||
"model:provider_global:*",
|
||||
"Provider-GlobalModel 模型映射缓存",
|
||||
),
|
||||
(
|
||||
"provider_mapping_preview",
|
||||
"映射预览",
|
||||
"admin:providers:mapping-preview:*",
|
||||
"Provider 详情页 mapping-preview 缓存",
|
||||
),
|
||||
(
|
||||
"global_model",
|
||||
"全局模型",
|
||||
"global_model:*",
|
||||
"GlobalModel 缓存（ID/名称/解析）",
|
||||
),
|
||||
(
|
||||
"models_list",
|
||||
"模型列表",
|
||||
"models:list:*",
|
||||
"/v1/models 端点模型列表缓存",
|
||||
),
|
||||
("user", "用户", "user:*", "用户信息缓存（ID/Email）"),
|
||||
(
|
||||
"apikey",
|
||||
"API Key",
|
||||
"apikey:*",
|
||||
"API Key 认证缓存（Hash/Auth）",
|
||||
),
|
||||
(
|
||||
"api_key_id",
|
||||
"API Key ID",
|
||||
"api_key:id:*",
|
||||
"API Key 按 ID 缓存",
|
||||
),
|
||||
(
|
||||
"cache_affinity",
|
||||
"缓存亲和性",
|
||||
"cache_affinity:*",
|
||||
"请求路由亲和性缓存",
|
||||
),
|
||||
(
|
||||
"provider_billing",
|
||||
"Provider 计费",
|
||||
"provider:billing_type:*",
|
||||
"Provider 计费类型缓存",
|
||||
),
|
||||
(
|
||||
"provider_rate",
|
||||
"Provider 费率",
|
||||
"provider_api_key:rate_multiplier:*",
|
||||
"ProviderAPIKey 费率倍数缓存",
|
||||
),
|
||||
(
|
||||
"provider_balance",
|
||||
"Provider 余额",
|
||||
"provider_ops:balance:*",
|
||||
"Provider 余额查询缓存",
|
||||
),
|
||||
("health", "健康检查", "health:*", "端点健康状态缓存"),
|
||||
(
|
||||
"endpoint_status",
|
||||
"端点状态",
|
||||
"endpoint_status:*",
|
||||
"用户端点状态缓存",
|
||||
),
|
||||
("dashboard", "仪表盘", "dashboard:*", "仪表盘统计缓存"),
|
||||
(
|
||||
"activity_heatmap",
|
||||
"活动热力图",
|
||||
"activity_heatmap:*",
|
||||
"用户活动热力图缓存",
|
||||
),
|
||||
(
|
||||
"gemini_files",
|
||||
"Gemini 文件映射",
|
||||
"gemini_files:*",
|
||||
"Gemini Files API 文件-Key 映射缓存",
|
||||
),
|
||||
(
|
||||
"provider_oauth",
|
||||
"OAuth 状态",
|
||||
"provider_oauth_state:*",
|
||||
"Provider OAuth 授权流程临时状态",
|
||||
),
|
||||
(
|
||||
"oauth_refresh_lock",
|
||||
"OAuth 刷新锁",
|
||||
"provider_oauth_refresh_lock:*",
|
||||
"OAuth Token 刷新分布式锁",
|
||||
),
|
||||
(
|
||||
"concurrency_lock",
|
||||
"并发锁",
|
||||
"concurrency:*",
|
||||
"请求并发控制锁",
|
||||
),
|
||||
];
|
||||
@@ -0,0 +1,83 @@
|
||||
use super::cache_types::AdminMonitoringCacheAffinityRecord;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
pub(super) async fn admin_monitoring_list_export_api_key_records_by_ids(
|
||||
state: &AppState,
|
||||
api_key_ids: &[String],
|
||||
) -> Result<
|
||||
std::collections::BTreeMap<String, aether_data::repository::auth::StoredAuthApiKeyExportRecord>,
|
||||
GatewayError,
|
||||
> {
|
||||
if api_key_ids.is_empty() {
|
||||
return Ok(std::collections::BTreeMap::new());
|
||||
}
|
||||
|
||||
Ok(state
|
||||
.list_auth_api_key_export_records_by_ids(api_key_ids)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|record| (record.api_key_id.clone(), record))
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn admin_monitoring_list_user_summaries_by_ids(
|
||||
state: &AppState,
|
||||
user_ids: &[String],
|
||||
) -> Result<
|
||||
std::collections::BTreeMap<String, aether_data::repository::users::StoredUserSummary>,
|
||||
GatewayError,
|
||||
> {
|
||||
if user_ids.is_empty() {
|
||||
return Ok(std::collections::BTreeMap::new());
|
||||
}
|
||||
|
||||
Ok(state
|
||||
.list_users_by_ids(user_ids)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|user| (user.id.clone(), user))
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub(super) async fn admin_monitoring_load_affinity_identity_maps(
|
||||
state: &AppState,
|
||||
affinities: &[AdminMonitoringCacheAffinityRecord],
|
||||
) -> Result<
|
||||
(
|
||||
std::collections::BTreeMap<
|
||||
String,
|
||||
aether_data::repository::auth::StoredAuthApiKeyExportRecord,
|
||||
>,
|
||||
std::collections::BTreeMap<String, aether_data::repository::users::StoredUserSummary>,
|
||||
),
|
||||
GatewayError,
|
||||
> {
|
||||
let api_key_ids = affinities
|
||||
.iter()
|
||||
.map(|item| item.affinity_key.clone())
|
||||
.collect::<std::collections::BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>();
|
||||
let api_key_by_id =
|
||||
admin_monitoring_list_export_api_key_records_by_ids(state, &api_key_ids).await?;
|
||||
let user_ids = api_key_by_id
|
||||
.values()
|
||||
.map(|record| record.user_id.clone())
|
||||
.collect::<std::collections::BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>();
|
||||
let user_by_id = admin_monitoring_list_user_summaries_by_ids(state, &user_ids).await?;
|
||||
Ok((api_key_by_id, user_by_id))
|
||||
}
|
||||
|
||||
pub(super) async fn admin_monitoring_find_user_summary_by_id(
|
||||
state: &AppState,
|
||||
user_id: &str,
|
||||
) -> Result<Option<aether_data::repository::users::StoredUserSummary>, GatewayError> {
|
||||
if user_id.trim().is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let user_ids = [user_id.to_string()];
|
||||
Ok(state.list_users_by_ids(&user_ids).await?.into_iter().next())
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
use super::cache_config::ADMIN_MONITORING_REDIS_CACHE_CATEGORIES;
|
||||
use super::cache_store::{
|
||||
admin_monitoring_has_test_redis_keys, list_admin_monitoring_namespaced_keys,
|
||||
};
|
||||
use crate::AppState;
|
||||
use crate::GatewayError;
|
||||
use axum::{
|
||||
body::Body,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
pub(super) async fn build_admin_monitoring_model_mapping_stats_response(
|
||||
state: &AppState,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
|
||||
return Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"data": {
|
||||
"available": false,
|
||||
"message": "Redis 未启用，模型映射缓存不可用",
|
||||
}
|
||||
}))
|
||||
.into_response());
|
||||
};
|
||||
|
||||
let model_id_keys = list_admin_monitoring_namespaced_keys(state, "model:id:*").await?;
|
||||
let global_model_id_keys =
|
||||
list_admin_monitoring_namespaced_keys(state, "global_model:id:*").await?;
|
||||
let global_model_name_keys =
|
||||
list_admin_monitoring_namespaced_keys(state, "global_model:name:*").await?;
|
||||
let global_model_resolve_keys =
|
||||
list_admin_monitoring_namespaced_keys(state, "global_model:resolve:*").await?;
|
||||
let provider_global_keys =
|
||||
list_admin_monitoring_namespaced_keys(state, "model:provider_global:*")
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter(|key| !key.starts_with("model:provider_global:hits:"))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let total_keys = model_id_keys.len()
|
||||
+ global_model_id_keys.len()
|
||||
+ global_model_name_keys.len()
|
||||
+ global_model_resolve_keys.len()
|
||||
+ provider_global_keys.len();
|
||||
|
||||
Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"data": {
|
||||
"available": true,
|
||||
"ttl_seconds": 300,
|
||||
"total_keys": total_keys,
|
||||
"breakdown": {
|
||||
"model_by_id": model_id_keys.len(),
|
||||
"model_by_provider_global": provider_global_keys.len(),
|
||||
"global_model_by_id": global_model_id_keys.len(),
|
||||
"global_model_by_name": global_model_name_keys.len(),
|
||||
"global_model_resolve": global_model_resolve_keys.len(),
|
||||
},
|
||||
"mappings": [],
|
||||
"provider_model_mappings": serde_json::Value::Null,
|
||||
"unmapped": serde_json::Value::Null,
|
||||
}
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_redis_cache_categories_response(
|
||||
state: &AppState,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
|
||||
return Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"data": {
|
||||
"available": false,
|
||||
"message": "Redis 未启用",
|
||||
}
|
||||
}))
|
||||
.into_response());
|
||||
};
|
||||
|
||||
let mut categories = Vec::with_capacity(ADMIN_MONITORING_REDIS_CACHE_CATEGORIES.len());
|
||||
let mut total_keys = 0usize;
|
||||
|
||||
for (key, name, pattern, description) in ADMIN_MONITORING_REDIS_CACHE_CATEGORIES {
|
||||
let count = list_admin_monitoring_namespaced_keys(state, pattern)
|
||||
.await?
|
||||
.len();
|
||||
total_keys += count;
|
||||
categories.push(json!({
|
||||
"key": key,
|
||||
"name": name,
|
||||
"pattern": pattern,
|
||||
"description": description,
|
||||
"count": count,
|
||||
}));
|
||||
}
|
||||
|
||||
Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"data": {
|
||||
"available": true,
|
||||
"categories": categories,
|
||||
"total_keys": total_keys,
|
||||
}
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
use super::cache_affinity::{
|
||||
clear_admin_monitoring_scheduler_affinity_entries,
|
||||
delete_admin_monitoring_cache_affinity_raw_keys,
|
||||
};
|
||||
use super::cache_config::ADMIN_MONITORING_REDIS_CACHE_CATEGORIES;
|
||||
use super::cache_identity::{
|
||||
admin_monitoring_find_user_summary_by_id, admin_monitoring_list_export_api_key_records_by_ids,
|
||||
};
|
||||
use super::cache_route_helpers::{
|
||||
admin_monitoring_cache_affinity_delete_params_from_path,
|
||||
admin_monitoring_cache_affinity_unavailable_response,
|
||||
admin_monitoring_cache_model_mapping_provider_params_from_path,
|
||||
admin_monitoring_cache_model_name_from_path, admin_monitoring_cache_provider_id_from_path,
|
||||
admin_monitoring_cache_redis_category_from_path,
|
||||
admin_monitoring_cache_users_not_found_response,
|
||||
admin_monitoring_cache_users_user_identifier_from_path,
|
||||
admin_monitoring_redis_unavailable_response,
|
||||
};
|
||||
use super::cache_store::{
|
||||
admin_monitoring_has_test_redis_keys, delete_admin_monitoring_namespaced_keys,
|
||||
list_admin_monitoring_cache_affinity_records,
|
||||
list_admin_monitoring_cache_affinity_records_by_affinity_keys,
|
||||
list_admin_monitoring_namespaced_keys, load_admin_monitoring_cache_affinity_entries_for_tests,
|
||||
};
|
||||
use super::responses::{
|
||||
admin_monitoring_bad_request_response, admin_monitoring_not_found_response,
|
||||
};
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use crate::{AppState, GatewayError};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
pub(super) async fn build_admin_monitoring_cache_users_delete_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let Some(user_identifier) =
|
||||
admin_monitoring_cache_users_user_identifier_from_path(&request_context.request_path)
|
||||
else {
|
||||
return Ok(admin_monitoring_bad_request_response(
|
||||
"缺少 user_identifier",
|
||||
));
|
||||
};
|
||||
|
||||
if state.redis_kv_runner().is_none()
|
||||
&& load_admin_monitoring_cache_affinity_entries_for_tests(state).is_empty()
|
||||
{
|
||||
return Ok(admin_monitoring_cache_affinity_unavailable_response());
|
||||
}
|
||||
|
||||
let direct_api_key_by_id =
|
||||
admin_monitoring_list_export_api_key_records_by_ids(state, &[user_identifier.clone()])
|
||||
.await?;
|
||||
|
||||
if let Some(api_key) = direct_api_key_by_id.get(&user_identifier) {
|
||||
let target_affinity_keys =
|
||||
std::iter::once(user_identifier.clone()).collect::<std::collections::BTreeSet<_>>();
|
||||
let target_affinities = list_admin_monitoring_cache_affinity_records_by_affinity_keys(
|
||||
state,
|
||||
&target_affinity_keys,
|
||||
)
|
||||
.await?;
|
||||
let raw_keys = target_affinities
|
||||
.iter()
|
||||
.map(|item| item.raw_key.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let _ = delete_admin_monitoring_cache_affinity_raw_keys(state, &raw_keys).await?;
|
||||
clear_admin_monitoring_scheduler_affinity_entries(state, &target_affinities);
|
||||
|
||||
let user = admin_monitoring_find_user_summary_by_id(state, &api_key.user_id).await?;
|
||||
let api_key_name = api_key
|
||||
.name
|
||||
.clone()
|
||||
.unwrap_or_else(|| user_identifier.clone());
|
||||
return Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"message": format!("已清除 API Key {api_key_name} 的缓存亲和性"),
|
||||
"user_info": {
|
||||
"user_id": Some(api_key.user_id.clone()),
|
||||
"username": user.as_ref().map(|item| item.username.clone()),
|
||||
"email": user.and_then(|item| item.email),
|
||||
"api_key_id": user_identifier,
|
||||
"api_key_name": api_key.name.clone(),
|
||||
},
|
||||
}))
|
||||
.into_response());
|
||||
}
|
||||
|
||||
let Some(user) = state.find_user_auth_by_identifier(&user_identifier).await? else {
|
||||
return Ok(admin_monitoring_cache_users_not_found_response(
|
||||
&user_identifier,
|
||||
));
|
||||
};
|
||||
|
||||
let user_api_key_ids = state
|
||||
.list_auth_api_key_export_records_by_user_ids(std::slice::from_ref(&user.id))
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|item| item.api_key_id.clone())
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
let target_affinities =
|
||||
list_admin_monitoring_cache_affinity_records_by_affinity_keys(state, &user_api_key_ids)
|
||||
.await?;
|
||||
let raw_keys = target_affinities
|
||||
.iter()
|
||||
.map(|item| item.raw_key.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let _ = delete_admin_monitoring_cache_affinity_raw_keys(state, &raw_keys).await?;
|
||||
clear_admin_monitoring_scheduler_affinity_entries(state, &target_affinities);
|
||||
|
||||
Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"message": format!("已清除用户 {} 的所有缓存亲和性", user.username),
|
||||
"user_info": {
|
||||
"user_id": user.id,
|
||||
"username": user.username,
|
||||
"email": user.email,
|
||||
},
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_cache_affinity_delete_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let Some((affinity_key, endpoint_id, model_id, api_format)) =
|
||||
admin_monitoring_cache_affinity_delete_params_from_path(&request_context.request_path)
|
||||
else {
|
||||
return Ok(admin_monitoring_bad_request_response(
|
||||
"缺少 affinity_key、endpoint_id、model_id 或 api_format",
|
||||
));
|
||||
};
|
||||
|
||||
if state.redis_kv_runner().is_none()
|
||||
&& load_admin_monitoring_cache_affinity_entries_for_tests(state).is_empty()
|
||||
{
|
||||
return Ok(admin_monitoring_cache_affinity_unavailable_response());
|
||||
}
|
||||
|
||||
let target_affinity_keys =
|
||||
std::iter::once(affinity_key.clone()).collect::<std::collections::BTreeSet<_>>();
|
||||
let target_affinity =
|
||||
list_admin_monitoring_cache_affinity_records_by_affinity_keys(state, &target_affinity_keys)
|
||||
.await?
|
||||
.into_iter()
|
||||
.find(|item| {
|
||||
item.affinity_key == affinity_key
|
||||
&& item.endpoint_id.as_deref() == Some(endpoint_id.as_str())
|
||||
&& item.model_name == model_id
|
||||
&& item.api_format.eq_ignore_ascii_case(&api_format)
|
||||
});
|
||||
let Some(target_affinity) = target_affinity else {
|
||||
return Ok(admin_monitoring_not_found_response(
|
||||
"未找到指定的缓存亲和性记录",
|
||||
));
|
||||
};
|
||||
|
||||
let _ = delete_admin_monitoring_cache_affinity_raw_keys(
|
||||
state,
|
||||
std::slice::from_ref(&target_affinity.raw_key),
|
||||
)
|
||||
.await?;
|
||||
clear_admin_monitoring_scheduler_affinity_entries(
|
||||
state,
|
||||
std::slice::from_ref(&target_affinity),
|
||||
);
|
||||
|
||||
let mut api_key_by_id = admin_monitoring_list_export_api_key_records_by_ids(
|
||||
state,
|
||||
std::slice::from_ref(&affinity_key),
|
||||
)
|
||||
.await?;
|
||||
let api_key_name = api_key_by_id
|
||||
.remove(&affinity_key)
|
||||
.and_then(|item| item.name)
|
||||
.unwrap_or_else(|| affinity_key.chars().take(8).collect::<String>());
|
||||
|
||||
Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"message": format!("已清除缓存亲和性: {api_key_name}"),
|
||||
"affinity_key": affinity_key,
|
||||
"endpoint_id": endpoint_id,
|
||||
"model_id": model_id,
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_cache_flush_response(
|
||||
state: &AppState,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let raw_affinities = list_admin_monitoring_cache_affinity_records(state).await?;
|
||||
if state.redis_kv_runner().is_none() && raw_affinities.is_empty() {
|
||||
return Ok(admin_monitoring_cache_affinity_unavailable_response());
|
||||
}
|
||||
|
||||
let raw_keys = raw_affinities
|
||||
.iter()
|
||||
.map(|item| item.raw_key.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let deleted = delete_admin_monitoring_cache_affinity_raw_keys(state, &raw_keys).await?;
|
||||
clear_admin_monitoring_scheduler_affinity_entries(state, &raw_affinities);
|
||||
|
||||
Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"message": "已清除全部缓存亲和性",
|
||||
"deleted_affinities": deleted,
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_cache_provider_delete_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let Some(provider_id) =
|
||||
admin_monitoring_cache_provider_id_from_path(&request_context.request_path)
|
||||
else {
|
||||
return Ok(admin_monitoring_bad_request_response("缺少 provider_id"));
|
||||
};
|
||||
|
||||
let raw_affinities = list_admin_monitoring_cache_affinity_records(state).await?;
|
||||
if state.redis_kv_runner().is_none() && raw_affinities.is_empty() {
|
||||
return Ok(admin_monitoring_cache_affinity_unavailable_response());
|
||||
}
|
||||
|
||||
let target_affinities = raw_affinities
|
||||
.into_iter()
|
||||
.filter(|item| item.provider_id.as_deref() == Some(provider_id.as_str()))
|
||||
.collect::<Vec<_>>();
|
||||
if target_affinities.is_empty() {
|
||||
return Ok((
|
||||
http::StatusCode::NOT_FOUND,
|
||||
Json(json!({
|
||||
"detail": format!("未找到 provider {provider_id} 的缓存亲和性记录")
|
||||
})),
|
||||
)
|
||||
.into_response());
|
||||
}
|
||||
|
||||
let raw_keys = target_affinities
|
||||
.iter()
|
||||
.map(|item| item.raw_key.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let deleted = delete_admin_monitoring_cache_affinity_raw_keys(state, &raw_keys).await?;
|
||||
clear_admin_monitoring_scheduler_affinity_entries(state, &target_affinities);
|
||||
|
||||
Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"message": format!("已清除 provider {provider_id} 的缓存亲和性"),
|
||||
"provider_id": provider_id,
|
||||
"deleted_affinities": deleted,
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_model_mapping_delete_response(
|
||||
state: &AppState,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
|
||||
return Ok(admin_monitoring_redis_unavailable_response());
|
||||
}
|
||||
|
||||
let mut raw_keys = list_admin_monitoring_namespaced_keys(state, "model:*").await?;
|
||||
raw_keys.extend(list_admin_monitoring_namespaced_keys(state, "global_model:*").await?);
|
||||
raw_keys.sort();
|
||||
raw_keys.dedup();
|
||||
let deleted_count = delete_admin_monitoring_namespaced_keys(state, &raw_keys).await?;
|
||||
|
||||
Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"message": "已清除所有模型映射缓存",
|
||||
"deleted_count": deleted_count,
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_model_mapping_delete_model_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let Some(model_name) =
|
||||
admin_monitoring_cache_model_name_from_path(&request_context.request_path)
|
||||
else {
|
||||
return Ok(admin_monitoring_bad_request_response("缺少 model_name"));
|
||||
};
|
||||
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
|
||||
return Ok(admin_monitoring_redis_unavailable_response());
|
||||
}
|
||||
|
||||
let candidate_keys = [
|
||||
format!("global_model:resolve:{model_name}"),
|
||||
format!("global_model:name:{model_name}"),
|
||||
];
|
||||
let mut existing_keys = Vec::new();
|
||||
for key in candidate_keys {
|
||||
let matches = list_admin_monitoring_namespaced_keys(state, key.as_str()).await?;
|
||||
existing_keys.extend(matches);
|
||||
}
|
||||
existing_keys.sort();
|
||||
existing_keys.dedup();
|
||||
|
||||
let deleted_count = delete_admin_monitoring_namespaced_keys(state, &existing_keys).await?;
|
||||
let deleted_keys = if deleted_count == 0 {
|
||||
Vec::new()
|
||||
} else {
|
||||
existing_keys
|
||||
};
|
||||
|
||||
Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"message": format!("已清除模型 {model_name} 的映射缓存"),
|
||||
"model_name": model_name,
|
||||
"deleted_keys": deleted_keys,
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_model_mapping_delete_provider_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let Some((provider_id, global_model_id)) =
|
||||
admin_monitoring_cache_model_mapping_provider_params_from_path(
|
||||
&request_context.request_path,
|
||||
)
|
||||
else {
|
||||
return Ok(admin_monitoring_bad_request_response(
|
||||
"缺少 provider_id 或 global_model_id",
|
||||
));
|
||||
};
|
||||
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
|
||||
return Ok(admin_monitoring_redis_unavailable_response());
|
||||
}
|
||||
|
||||
let candidate_keys = [
|
||||
format!("model:provider_global:{provider_id}:{global_model_id}"),
|
||||
format!("model:provider_global:hits:{provider_id}:{global_model_id}"),
|
||||
];
|
||||
let mut existing_keys = Vec::new();
|
||||
for key in candidate_keys {
|
||||
let matches = list_admin_monitoring_namespaced_keys(state, key.as_str()).await?;
|
||||
existing_keys.extend(matches);
|
||||
}
|
||||
existing_keys.sort();
|
||||
existing_keys.dedup();
|
||||
|
||||
let _ = delete_admin_monitoring_namespaced_keys(state, &existing_keys).await?;
|
||||
|
||||
Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"message": "已清除 Provider 模型映射缓存",
|
||||
"provider_id": provider_id,
|
||||
"global_model_id": global_model_id,
|
||||
"deleted_keys": existing_keys,
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_redis_keys_delete_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let Some(category) =
|
||||
admin_monitoring_cache_redis_category_from_path(&request_context.request_path)
|
||||
else {
|
||||
return Ok(admin_monitoring_bad_request_response("缺少 category"));
|
||||
};
|
||||
|
||||
let Some((cat_key, name, pattern, _description)) = ADMIN_MONITORING_REDIS_CACHE_CATEGORIES
|
||||
.iter()
|
||||
.find(|(cat_key, _, _, _)| *cat_key == category)
|
||||
else {
|
||||
return Ok((
|
||||
http::StatusCode::NOT_FOUND,
|
||||
Json(json!({ "detail": format!("未知的缓存分类: {category}") })),
|
||||
)
|
||||
.into_response());
|
||||
};
|
||||
|
||||
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
|
||||
return Ok(admin_monitoring_redis_unavailable_response());
|
||||
}
|
||||
|
||||
let raw_keys = list_admin_monitoring_namespaced_keys(state, pattern).await?;
|
||||
let deleted_count = delete_admin_monitoring_namespaced_keys(state, &raw_keys).await?;
|
||||
|
||||
Ok(Json(json!({
|
||||
"status": "ok",
|
||||
"message": format!("已清除 {name} 缓存"),
|
||||
"category": cat_key,
|
||||
"deleted_count": deleted_count,
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
use crate::AppState;
|
||||
use aether_crypto::decrypt_python_fernet_ciphertext;
|
||||
#[cfg(test)]
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
|
||||
|
||||
pub(super) fn admin_monitoring_masked_user_api_key_prefix(
|
||||
state: &AppState,
|
||||
ciphertext: Option<&str>,
|
||||
) -> Option<String> {
|
||||
let Some(ciphertext) = ciphertext.map(str::trim).filter(|value| !value.is_empty()) else {
|
||||
return None;
|
||||
};
|
||||
let full_key = admin_monitoring_try_decrypt_secret(state, ciphertext)?;
|
||||
let prefix_len = full_key.len().min(10);
|
||||
let prefix = &full_key[..prefix_len];
|
||||
let suffix = if full_key.len() >= 4 {
|
||||
&full_key[full_key.len().saturating_sub(4)..]
|
||||
} else {
|
||||
""
|
||||
};
|
||||
Some(format!("{prefix}...{suffix}"))
|
||||
}
|
||||
|
||||
pub(super) fn admin_monitoring_masked_provider_key_prefix(
|
||||
state: &AppState,
|
||||
key: &StoredProviderCatalogKey,
|
||||
) -> Option<String> {
|
||||
match key.auth_type.trim() {
|
||||
"service_account" | "vertex_ai" => Some("[Service Account]".to_string()),
|
||||
"oauth" => Some("[OAuth Token]".to_string()),
|
||||
_ => {
|
||||
let full_key = admin_monitoring_try_decrypt_secret(state, &key.encrypted_api_key)?;
|
||||
if full_key.len() <= 12 {
|
||||
Some(format!("{full_key}***"))
|
||||
} else {
|
||||
Some(format!(
|
||||
"{}***{}",
|
||||
&full_key[..8],
|
||||
&full_key[full_key.len().saturating_sub(4)..]
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn admin_monitoring_try_decrypt_secret(state: &AppState, ciphertext: &str) -> Option<String> {
|
||||
let ciphertext = ciphertext.trim();
|
||||
if ciphertext.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let encryption_key = state.encryption_key().map(str::trim).unwrap_or("");
|
||||
if !encryption_key.is_empty() {
|
||||
if let Ok(value) = decrypt_python_fernet_ciphertext(encryption_key, ciphertext) {
|
||||
return Some(value);
|
||||
}
|
||||
}
|
||||
for env_key in ["AETHER_GATEWAY_DATA_ENCRYPTION_KEY", "ENCRYPTION_KEY"] {
|
||||
let Ok(candidate) = std::env::var(env_key) else {
|
||||
continue;
|
||||
};
|
||||
let candidate = candidate.trim();
|
||||
if candidate.is_empty() || candidate == encryption_key {
|
||||
continue;
|
||||
}
|
||||
if let Ok(value) = decrypt_python_fernet_ciphertext(candidate, ciphertext) {
|
||||
return Some(value);
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
if encryption_key != DEVELOPMENT_ENCRYPTION_KEY {
|
||||
if let Ok(value) = decrypt_python_fernet_ciphertext(DEVELOPMENT_ENCRYPTION_KEY, ciphertext)
|
||||
{
|
||||
return Some(value);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub(super) fn admin_monitoring_cache_affinity_sort_value(value: Option<&serde_json::Value>) -> f64 {
|
||||
let Some(value) = value else {
|
||||
return 0.0;
|
||||
};
|
||||
if let Some(number) = value.as_f64() {
|
||||
return number;
|
||||
}
|
||||
if let Some(number) = value.as_i64() {
|
||||
return number as f64;
|
||||
}
|
||||
if let Some(number) = value.as_u64() {
|
||||
return number as f64;
|
||||
}
|
||||
if let Some(text) = value.as_str() {
|
||||
if let Ok(number) = text.parse::<f64>() {
|
||||
return number;
|
||||
}
|
||||
if let Ok(parsed) = chrono::DateTime::parse_from_rfc3339(text) {
|
||||
return parsed.timestamp() as f64;
|
||||
}
|
||||
}
|
||||
0.0
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
use super::cache_config::{
|
||||
ADMIN_MONITORING_CACHE_AFFINITY_REDIS_REQUIRED_DETAIL, ADMIN_MONITORING_REDIS_REQUIRED_DETAIL,
|
||||
};
|
||||
use crate::handlers::admin::shared::query_param_value;
|
||||
use axum::{
|
||||
body::Body,
|
||||
http,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
pub(super) fn parse_admin_monitoring_keyword_filter(query: Option<&str>) -> Option<String> {
|
||||
query_param_value(query, "keyword")
|
||||
.map(|value| value.trim().to_string())
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn admin_monitoring_cache_path_identifier_from_path(
|
||||
request_path: &str,
|
||||
prefix: &str,
|
||||
) -> Option<String> {
|
||||
let value = request_path
|
||||
.strip_prefix(prefix)?
|
||||
.trim()
|
||||
.trim_matches('/')
|
||||
.to_string();
|
||||
if value.is_empty() || value.contains('/') {
|
||||
None
|
||||
} else {
|
||||
Some(value)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn admin_monitoring_cache_affinity_user_identifier_from_path(
|
||||
request_path: &str,
|
||||
) -> Option<String> {
|
||||
admin_monitoring_cache_path_identifier_from_path(
|
||||
request_path,
|
||||
"/api/admin/monitoring/cache/affinity/",
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn admin_monitoring_cache_users_user_identifier_from_path(
|
||||
request_path: &str,
|
||||
) -> Option<String> {
|
||||
admin_monitoring_cache_path_identifier_from_path(
|
||||
request_path,
|
||||
"/api/admin/monitoring/cache/users/",
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn admin_monitoring_cache_provider_id_from_path(request_path: &str) -> Option<String> {
|
||||
admin_monitoring_cache_path_identifier_from_path(
|
||||
request_path,
|
||||
"/api/admin/monitoring/cache/providers/",
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn admin_monitoring_cache_model_name_from_path(request_path: &str) -> Option<String> {
|
||||
admin_monitoring_cache_path_identifier_from_path(
|
||||
request_path,
|
||||
"/api/admin/monitoring/cache/model-mapping/",
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn admin_monitoring_cache_redis_category_from_path(
|
||||
request_path: &str,
|
||||
) -> Option<String> {
|
||||
admin_monitoring_cache_path_identifier_from_path(
|
||||
request_path,
|
||||
"/api/admin/monitoring/cache/redis-keys/",
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn admin_monitoring_cache_model_mapping_provider_params_from_path(
|
||||
request_path: &str,
|
||||
) -> Option<(String, String)> {
|
||||
let suffix = request_path
|
||||
.strip_prefix("/api/admin/monitoring/cache/model-mapping/provider/")?
|
||||
.trim()
|
||||
.trim_matches('/');
|
||||
let segments = suffix.split('/').collect::<Vec<_>>();
|
||||
if segments.len() != 2 || segments.iter().any(|segment| segment.trim().is_empty()) {
|
||||
return None;
|
||||
}
|
||||
Some((
|
||||
segments[0].trim().to_string(),
|
||||
segments[1].trim().to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) fn admin_monitoring_cache_affinity_delete_params_from_path(
|
||||
request_path: &str,
|
||||
) -> Option<(String, String, String, String)> {
|
||||
let suffix = request_path
|
||||
.strip_prefix("/api/admin/monitoring/cache/affinity/")?
|
||||
.trim()
|
||||
.trim_matches('/');
|
||||
let segments = suffix.split('/').collect::<Vec<_>>();
|
||||
if segments.len() != 4 || segments.iter().any(|segment| segment.trim().is_empty()) {
|
||||
return None;
|
||||
}
|
||||
Some((
|
||||
segments[0].trim().to_string(),
|
||||
segments[1].trim().to_string(),
|
||||
segments[2].trim().to_string(),
|
||||
segments[3].trim().to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) fn admin_monitoring_cache_affinity_unavailable_response() -> Response<Body> {
|
||||
(
|
||||
http::StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({ "detail": ADMIN_MONITORING_CACHE_AFFINITY_REDIS_REQUIRED_DETAIL })),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
pub(super) fn admin_monitoring_redis_unavailable_response() -> Response<Body> {
|
||||
(
|
||||
http::StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({ "detail": ADMIN_MONITORING_REDIS_REQUIRED_DETAIL })),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
pub(super) fn admin_monitoring_cache_affinity_not_found_response(
|
||||
user_identifier: &str,
|
||||
) -> Response<Body> {
|
||||
(
|
||||
http::StatusCode::NOT_FOUND,
|
||||
Json(json!({
|
||||
"detail": format!(
|
||||
"无法识别的用户标识符: {user_identifier}。支持用户名、邮箱、User ID或API Key ID"
|
||||
)
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
pub(super) fn admin_monitoring_cache_users_not_found_response(
|
||||
user_identifier: &str,
|
||||
) -> Response<Body> {
|
||||
(
|
||||
http::StatusCode::NOT_FOUND,
|
||||
Json(json!({
|
||||
"detail": format!(
|
||||
"无法识别的标识符: {user_identifier}。支持用户名、邮箱、User ID或API Key ID"
|
||||
)
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
use super::cache_affinity::admin_monitoring_cache_affinity_record;
|
||||
use super::cache_types::{AdminMonitoringCacheAffinityRecord, AdminMonitoringCacheSnapshot};
|
||||
use crate::handlers::admin::observability::stats::round_to;
|
||||
use crate::{AppState, GatewayError};
|
||||
use aether_data_contracts::repository::usage::UsageAuditListQuery;
|
||||
|
||||
async fn count_admin_monitoring_cache_affinity_entries(state: &AppState) -> usize {
|
||||
let Some(runner) = state.redis_kv_runner() else {
|
||||
return 0;
|
||||
};
|
||||
let mut connection = match runner.client().get_multiplexed_async_connection().await {
|
||||
Ok(value) => value,
|
||||
Err(_) => return 0,
|
||||
};
|
||||
let pattern = runner.keyspace().key("cache_affinity:*");
|
||||
let mut cursor = 0u64;
|
||||
let mut total = 0usize;
|
||||
loop {
|
||||
let (next_cursor, keys) = match redis::cmd("SCAN")
|
||||
.arg(cursor)
|
||||
.arg("MATCH")
|
||||
.arg(&pattern)
|
||||
.arg("COUNT")
|
||||
.arg(200)
|
||||
.query_async::<(u64, Vec<String>)>(&mut connection)
|
||||
.await
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(_) => return total,
|
||||
};
|
||||
total += keys.len();
|
||||
if next_cursor == 0 {
|
||||
break;
|
||||
}
|
||||
cursor = next_cursor;
|
||||
}
|
||||
total
|
||||
}
|
||||
|
||||
async fn scan_admin_monitoring_namespaced_keys(
|
||||
runner: &aether_data::redis::RedisKvRunner,
|
||||
pattern: &str,
|
||||
) -> Result<Vec<String>, GatewayError> {
|
||||
let mut connection = runner
|
||||
.client()
|
||||
.get_multiplexed_async_connection()
|
||||
.await
|
||||
.map_err(|err| {
|
||||
GatewayError::Internal(format!("admin monitoring redis connect failed: {err}"))
|
||||
})?;
|
||||
let namespaced_pattern = runner.keyspace().key(pattern);
|
||||
let mut cursor = 0u64;
|
||||
let mut keys = Vec::new();
|
||||
loop {
|
||||
let (next_cursor, batch) = redis::cmd("SCAN")
|
||||
.arg(cursor)
|
||||
.arg("MATCH")
|
||||
.arg(&namespaced_pattern)
|
||||
.arg("COUNT")
|
||||
.arg(200)
|
||||
.query_async::<(u64, Vec<String>)>(&mut connection)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
GatewayError::Internal(format!("admin monitoring redis scan failed: {err}"))
|
||||
})?;
|
||||
keys.extend(batch);
|
||||
if next_cursor == 0 {
|
||||
break;
|
||||
}
|
||||
cursor = next_cursor;
|
||||
}
|
||||
Ok(keys)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn load_admin_monitoring_cache_affinity_entries_for_tests(
|
||||
state: &AppState,
|
||||
) -> Vec<(String, String)> {
|
||||
state.list_admin_monitoring_cache_affinity_entries_for_tests()
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
pub(super) fn load_admin_monitoring_cache_affinity_entries_for_tests(
|
||||
_state: &AppState,
|
||||
) -> Vec<(String, String)> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn load_admin_monitoring_redis_keys_for_tests(state: &AppState) -> Vec<String> {
|
||||
state.list_admin_monitoring_redis_keys_for_tests()
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn load_admin_monitoring_redis_keys_for_tests(_state: &AppState) -> Vec<String> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn delete_admin_monitoring_redis_keys_for_tests(state: &AppState, raw_keys: &[String]) -> usize {
|
||||
state.remove_admin_monitoring_redis_keys_for_tests(raw_keys)
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn delete_admin_monitoring_redis_keys_for_tests(_state: &AppState, _raw_keys: &[String]) -> usize {
|
||||
0
|
||||
}
|
||||
|
||||
fn admin_monitoring_test_key_matches_pattern(key: &str, pattern: &str) -> bool {
|
||||
match pattern.strip_suffix('*') {
|
||||
Some(prefix) => key.starts_with(prefix),
|
||||
None => key == pattern,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn admin_monitoring_has_test_redis_keys(state: &AppState) -> bool {
|
||||
!load_admin_monitoring_redis_keys_for_tests(state).is_empty()
|
||||
}
|
||||
|
||||
pub(super) async fn list_admin_monitoring_namespaced_keys(
|
||||
state: &AppState,
|
||||
pattern: &str,
|
||||
) -> Result<Vec<String>, GatewayError> {
|
||||
if let Some(runner) = state.redis_kv_runner() {
|
||||
return scan_admin_monitoring_namespaced_keys(&runner, pattern).await;
|
||||
}
|
||||
|
||||
let mut keys = load_admin_monitoring_redis_keys_for_tests(state)
|
||||
.into_iter()
|
||||
.filter(|key| admin_monitoring_test_key_matches_pattern(key, pattern))
|
||||
.collect::<Vec<_>>();
|
||||
keys.sort();
|
||||
Ok(keys)
|
||||
}
|
||||
|
||||
pub(super) async fn delete_admin_monitoring_namespaced_keys(
|
||||
state: &AppState,
|
||||
raw_keys: &[String],
|
||||
) -> Result<usize, GatewayError> {
|
||||
if raw_keys.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
if let Some(runner) = state.redis_kv_runner() {
|
||||
let mut connection = runner
|
||||
.client()
|
||||
.get_multiplexed_async_connection()
|
||||
.await
|
||||
.map_err(|err| {
|
||||
GatewayError::Internal(format!("admin monitoring redis connect failed: {err}"))
|
||||
})?;
|
||||
let deleted = redis::cmd("DEL")
|
||||
.arg(raw_keys)
|
||||
.query_async::<i64>(&mut connection)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
GatewayError::Internal(format!("admin monitoring redis delete failed: {err}"))
|
||||
})?;
|
||||
return Ok(usize::try_from(deleted).unwrap_or(0));
|
||||
}
|
||||
|
||||
Ok(delete_admin_monitoring_redis_keys_for_tests(
|
||||
state, raw_keys,
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) async fn list_admin_monitoring_cache_affinity_records(
|
||||
state: &AppState,
|
||||
) -> Result<Vec<AdminMonitoringCacheAffinityRecord>, GatewayError> {
|
||||
list_admin_monitoring_cache_affinity_records_matching(state, None).await
|
||||
}
|
||||
|
||||
pub(super) async fn list_admin_monitoring_cache_affinity_records_by_affinity_keys(
|
||||
state: &AppState,
|
||||
affinity_keys: &std::collections::BTreeSet<String>,
|
||||
) -> Result<Vec<AdminMonitoringCacheAffinityRecord>, GatewayError> {
|
||||
if affinity_keys.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
list_admin_monitoring_cache_affinity_records_matching(state, Some(affinity_keys)).await
|
||||
}
|
||||
|
||||
async fn list_admin_monitoring_cache_affinity_records_matching(
|
||||
state: &AppState,
|
||||
affinity_keys: Option<&std::collections::BTreeSet<String>>,
|
||||
) -> Result<Vec<AdminMonitoringCacheAffinityRecord>, GatewayError> {
|
||||
let mut records = Vec::new();
|
||||
let mut seen_raw_keys = std::collections::BTreeSet::new();
|
||||
|
||||
if let Some(runner) = state.redis_kv_runner() {
|
||||
let mut connection = runner
|
||||
.client()
|
||||
.get_multiplexed_async_connection()
|
||||
.await
|
||||
.map_err(|err| {
|
||||
GatewayError::Internal(format!("admin monitoring redis connect failed: {err}"))
|
||||
})?;
|
||||
let patterns = affinity_keys
|
||||
.map(|keys| {
|
||||
keys.iter()
|
||||
.map(|affinity_key| {
|
||||
runner
|
||||
.keyspace()
|
||||
.key(&format!("cache_affinity:{affinity_key}:*"))
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.unwrap_or_else(|| vec![runner.keyspace().key("cache_affinity:*")]);
|
||||
|
||||
for pattern in patterns {
|
||||
let mut cursor = 0u64;
|
||||
loop {
|
||||
let (next_cursor, keys) = redis::cmd("SCAN")
|
||||
.arg(cursor)
|
||||
.arg("MATCH")
|
||||
.arg(&pattern)
|
||||
.arg("COUNT")
|
||||
.arg(200)
|
||||
.query_async::<(u64, Vec<String>)>(&mut connection)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
GatewayError::Internal(format!("admin monitoring redis scan failed: {err}"))
|
||||
})?;
|
||||
if !keys.is_empty() {
|
||||
let values = redis::cmd("MGET")
|
||||
.arg(&keys)
|
||||
.query_async::<Vec<Option<String>>>(&mut connection)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
GatewayError::Internal(format!(
|
||||
"admin monitoring redis mget failed: {err}"
|
||||
))
|
||||
})?;
|
||||
for (key, raw_value) in keys.into_iter().zip(values.into_iter()) {
|
||||
let Some(raw_value) = raw_value else {
|
||||
continue;
|
||||
};
|
||||
let Some(record) = admin_monitoring_cache_affinity_record(&key, &raw_value)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if affinity_keys.is_some_and(|keys| !keys.contains(&record.affinity_key)) {
|
||||
continue;
|
||||
}
|
||||
if seen_raw_keys.insert(record.raw_key.clone()) {
|
||||
records.push(record);
|
||||
}
|
||||
}
|
||||
}
|
||||
if next_cursor == 0 {
|
||||
break;
|
||||
}
|
||||
cursor = next_cursor;
|
||||
}
|
||||
}
|
||||
return Ok(records);
|
||||
}
|
||||
|
||||
for (key, raw_value) in load_admin_monitoring_cache_affinity_entries_for_tests(state) {
|
||||
let Some(record) = admin_monitoring_cache_affinity_record(&key, &raw_value) else {
|
||||
continue;
|
||||
};
|
||||
if affinity_keys.is_some_and(|keys| !keys.contains(&record.affinity_key)) {
|
||||
continue;
|
||||
}
|
||||
if seen_raw_keys.insert(record.raw_key.clone()) {
|
||||
records.push(record);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(records)
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_cache_snapshot(
|
||||
state: &AppState,
|
||||
) -> Result<AdminMonitoringCacheSnapshot, GatewayError> {
|
||||
let scheduling_mode = state
|
||||
.read_system_config_json_value("scheduling_mode")
|
||||
.await?
|
||||
.and_then(|value| value.as_str().map(ToOwned::to_owned))
|
||||
.unwrap_or_else(|| "cache_affinity".to_string());
|
||||
let provider_priority_mode = state
|
||||
.read_system_config_json_value("provider_priority_mode")
|
||||
.await?
|
||||
.and_then(|value| value.as_str().map(ToOwned::to_owned))
|
||||
.unwrap_or_else(|| "provider".to_string());
|
||||
|
||||
let now = chrono::Utc::now();
|
||||
let usage = if state.has_usage_data_reader() {
|
||||
state
|
||||
.list_usage_audits(&UsageAuditListQuery {
|
||||
created_from_unix_secs: Some(
|
||||
(now - chrono::Duration::hours(24)).timestamp().max(0) as u64,
|
||||
),
|
||||
..Default::default()
|
||||
})
|
||||
.await?
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let cache_hits = usage
|
||||
.iter()
|
||||
.filter(|item| item.cache_read_input_tokens > 0)
|
||||
.count();
|
||||
let cache_misses = usage.len().saturating_sub(cache_hits);
|
||||
let cache_hit_rate = if usage.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
round_to(cache_hits as f64 / usage.len() as f64, 4)
|
||||
};
|
||||
let total_affinities = count_admin_monitoring_cache_affinity_entries(state).await;
|
||||
let storage_type = if state.redis_kv_runner().is_some() {
|
||||
"redis"
|
||||
} else {
|
||||
"memory"
|
||||
};
|
||||
let scheduler_name = if scheduling_mode == "cache_affinity" {
|
||||
"cache_aware".to_string()
|
||||
} else {
|
||||
"random".to_string()
|
||||
};
|
||||
|
||||
Ok(AdminMonitoringCacheSnapshot {
|
||||
scheduler_name,
|
||||
scheduling_mode,
|
||||
provider_priority_mode,
|
||||
storage_type,
|
||||
total_affinities,
|
||||
cache_hits,
|
||||
cache_misses,
|
||||
cache_hit_rate,
|
||||
provider_switches: 0,
|
||||
key_switches: 0,
|
||||
cache_invalidations: 0,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct AdminMonitoringCacheAffinityRecord {
|
||||
pub(super) raw_key: String,
|
||||
pub(super) affinity_key: String,
|
||||
pub(super) api_format: String,
|
||||
pub(super) model_name: String,
|
||||
pub(super) provider_id: Option<String>,
|
||||
pub(super) endpoint_id: Option<String>,
|
||||
pub(super) key_id: Option<String>,
|
||||
pub(super) created_at: Option<serde_json::Value>,
|
||||
pub(super) expire_at: Option<serde_json::Value>,
|
||||
pub(super) request_count: u64,
|
||||
}
|
||||
|
||||
pub(super) struct AdminMonitoringCacheSnapshot {
|
||||
pub(super) scheduler_name: String,
|
||||
pub(super) scheduling_mode: String,
|
||||
pub(super) provider_priority_mode: String,
|
||||
pub(super) storage_type: &'static str,
|
||||
pub(super) total_affinities: usize,
|
||||
pub(super) cache_hits: usize,
|
||||
pub(super) cache_misses: usize,
|
||||
pub(super) cache_hit_rate: f64,
|
||||
pub(super) provider_switches: usize,
|
||||
pub(super) key_switches: usize,
|
||||
pub(super) cache_invalidations: usize,
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use crate::{AppState, GatewayError};
|
||||
use axum::{body::Body, response::Response};
|
||||
|
||||
mod activity;
|
||||
mod cache;
|
||||
mod cache_affinity;
|
||||
mod cache_affinity_reads;
|
||||
mod cache_config;
|
||||
mod cache_identity;
|
||||
mod cache_model_mapping;
|
||||
mod cache_mutations;
|
||||
mod cache_payloads;
|
||||
mod cache_route_helpers;
|
||||
mod cache_store;
|
||||
mod cache_types;
|
||||
mod resilience;
|
||||
mod responses;
|
||||
mod route_filters;
|
||||
mod routes;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_support;
|
||||
mod trace;
|
||||
mod usage_helpers;
|
||||
|
||||
pub(crate) async fn maybe_build_local_admin_monitoring_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
routes::maybe_build_local_admin_monitoring_response(state, request_context).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,488 @@
|
||||
use super::responses::admin_monitoring_bad_request_response;
|
||||
use super::usage_helpers::admin_monitoring_usage_is_error;
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use crate::handlers::admin::shared::{
|
||||
provider_key_health_summary, query_param_value, unix_secs_to_rfc3339,
|
||||
};
|
||||
use crate::{AppState, GatewayError};
|
||||
use aether_data_contracts::repository::{
|
||||
provider_catalog::StoredProviderCatalogKey, usage::UsageAuditListQuery,
|
||||
};
|
||||
use axum::{
|
||||
body::Body,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
struct AdminMonitoringResilienceSnapshot {
|
||||
timestamp: chrono::DateTime<chrono::Utc>,
|
||||
health_score: i64,
|
||||
status: &'static str,
|
||||
error_statistics: serde_json::Value,
|
||||
recent_errors: Vec<serde_json::Value>,
|
||||
recommendations: Vec<String>,
|
||||
previous_stats: serde_json::Value,
|
||||
}
|
||||
|
||||
fn build_admin_monitoring_resilience_recommendations(
|
||||
total_errors: usize,
|
||||
health_score: i64,
|
||||
open_breaker_labels: &[String],
|
||||
) -> Vec<String> {
|
||||
let mut recommendations = Vec::new();
|
||||
if health_score < 50 {
|
||||
recommendations.push("系统健康状况严重，请立即检查错误日志".to_string());
|
||||
}
|
||||
if total_errors > 100 {
|
||||
recommendations.push("错误频率过高，建议检查系统配置和外部依赖".to_string());
|
||||
}
|
||||
if !open_breaker_labels.is_empty() {
|
||||
recommendations.push(format!(
|
||||
"以下服务熔断器已打开：{}",
|
||||
open_breaker_labels.join(", ")
|
||||
));
|
||||
}
|
||||
if health_score > 90 {
|
||||
recommendations.push("系统运行良好".to_string());
|
||||
}
|
||||
recommendations
|
||||
}
|
||||
|
||||
fn parse_admin_monitoring_circuit_history_limit(query: Option<&str>) -> Result<usize, String> {
|
||||
match query_param_value(query, "limit") {
|
||||
None => Ok(50),
|
||||
Some(value) => {
|
||||
let parsed = value
|
||||
.parse::<usize>()
|
||||
.map_err(|_| "limit must be an integer between 1 and 200".to_string())?;
|
||||
if (1..=200).contains(&parsed) {
|
||||
Ok(parsed)
|
||||
} else {
|
||||
Err("limit must be an integer between 1 and 200".to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_admin_monitoring_circuit_history_items(
|
||||
keys: &[StoredProviderCatalogKey],
|
||||
provider_name_by_id: &BTreeMap<String, String>,
|
||||
limit: usize,
|
||||
) -> Vec<serde_json::Value> {
|
||||
let mut items = Vec::new();
|
||||
|
||||
for key in keys {
|
||||
let health_by_format = key
|
||||
.health_by_format
|
||||
.as_ref()
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
let circuit_by_format = key
|
||||
.circuit_breaker_by_format
|
||||
.as_ref()
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
|
||||
for (api_format, circuit_value) in circuit_by_format {
|
||||
let Some(circuit) = circuit_value.as_object() else {
|
||||
continue;
|
||||
};
|
||||
let is_open = circuit
|
||||
.get("open")
|
||||
.and_then(serde_json::Value::as_bool)
|
||||
.unwrap_or(false);
|
||||
let is_half_open = circuit
|
||||
.get("half_open_until")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.is_some();
|
||||
|
||||
if !is_open && !is_half_open {
|
||||
continue;
|
||||
}
|
||||
|
||||
let health = health_by_format
|
||||
.get(&api_format)
|
||||
.and_then(serde_json::Value::as_object);
|
||||
let timestamp = circuit
|
||||
.get("open_at")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.or_else(|| {
|
||||
circuit
|
||||
.get("half_open_until")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
})
|
||||
.or_else(|| {
|
||||
health.and_then(|value| {
|
||||
value
|
||||
.get("last_failure_at")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
})
|
||||
})
|
||||
.map(ToOwned::to_owned);
|
||||
let event = if is_half_open { "half_open" } else { "opened" };
|
||||
let reason = circuit
|
||||
.get("reason")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(ToOwned::to_owned)
|
||||
.or_else(|| {
|
||||
health
|
||||
.and_then(|value| {
|
||||
value
|
||||
.get("consecutive_failures")
|
||||
.and_then(serde_json::Value::as_i64)
|
||||
})
|
||||
.filter(|value| *value > 0)
|
||||
.map(|value| format!("连续失败 {value} 次"))
|
||||
})
|
||||
.or_else(|| {
|
||||
Some(if is_half_open {
|
||||
"熔断器处于半开状态".to_string()
|
||||
} else {
|
||||
"熔断器处于打开状态".to_string()
|
||||
})
|
||||
});
|
||||
let recovery_seconds = circuit
|
||||
.get("recovery_seconds")
|
||||
.and_then(serde_json::Value::as_i64)
|
||||
.or_else(|| {
|
||||
let open_at = circuit
|
||||
.get("open_at")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.and_then(|value| chrono::DateTime::parse_from_rfc3339(value).ok());
|
||||
let next_probe_at = circuit
|
||||
.get("next_probe_at")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.and_then(|value| chrono::DateTime::parse_from_rfc3339(value).ok());
|
||||
match (open_at, next_probe_at) {
|
||||
(Some(open_at), Some(next_probe_at)) => {
|
||||
Some((next_probe_at - open_at).num_seconds().max(0))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
});
|
||||
|
||||
items.push(json!({
|
||||
"event": event,
|
||||
"key_id": key.id,
|
||||
"provider_id": key.provider_id,
|
||||
"provider_name": provider_name_by_id.get(&key.provider_id).cloned(),
|
||||
"key_name": key.name,
|
||||
"api_format": api_format,
|
||||
"reason": reason,
|
||||
"recovery_seconds": recovery_seconds,
|
||||
"timestamp": timestamp,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
items.sort_by(|left, right| {
|
||||
let left_ts = left
|
||||
.get("timestamp")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.unwrap_or_default();
|
||||
let right_ts = right
|
||||
.get("timestamp")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.unwrap_or_default();
|
||||
right_ts.cmp(left_ts)
|
||||
});
|
||||
items.truncate(limit);
|
||||
items
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_resilience_circuit_history_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let limit = match parse_admin_monitoring_circuit_history_limit(
|
||||
request_context.request_query_string.as_deref(),
|
||||
) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(admin_monitoring_bad_request_response(detail)),
|
||||
};
|
||||
|
||||
let providers = state.list_provider_catalog_providers(false).await?;
|
||||
let provider_ids = providers
|
||||
.iter()
|
||||
.map(|item| item.id.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let provider_name_by_id = providers
|
||||
.iter()
|
||||
.map(|item| (item.id.clone(), item.name.clone()))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let keys = if provider_ids.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
state
|
||||
.list_provider_catalog_keys_by_provider_ids(&provider_ids)
|
||||
.await?
|
||||
};
|
||||
|
||||
let items = build_admin_monitoring_circuit_history_items(&keys, &provider_name_by_id, limit);
|
||||
let count = items.len();
|
||||
Ok(Json(json!({
|
||||
"items": items,
|
||||
"count": count,
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_resilience_status_response(
|
||||
state: &AppState,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let snapshot = build_admin_monitoring_resilience_snapshot(state).await?;
|
||||
|
||||
Ok(Json(json!({
|
||||
"timestamp": snapshot.timestamp.to_rfc3339(),
|
||||
"health_score": snapshot.health_score,
|
||||
"status": snapshot.status,
|
||||
"error_statistics": snapshot.error_statistics,
|
||||
"recent_errors": snapshot.recent_errors,
|
||||
"recommendations": snapshot.recommendations,
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
|
||||
async fn build_admin_monitoring_resilience_snapshot(
|
||||
state: &AppState,
|
||||
) -> Result<AdminMonitoringResilienceSnapshot, GatewayError> {
|
||||
let now = chrono::Utc::now();
|
||||
let recent_error_from = std::cmp::max(
|
||||
now - chrono::Duration::hours(24),
|
||||
chrono::DateTime::<chrono::Utc>::from_timestamp(
|
||||
state
|
||||
.admin_monitoring_error_stats_reset_at()
|
||||
.unwrap_or_default() as i64,
|
||||
0,
|
||||
)
|
||||
.unwrap_or_else(|| {
|
||||
chrono::DateTime::<chrono::Utc>::from_timestamp(0, 0).expect("unix epoch should exist")
|
||||
}),
|
||||
);
|
||||
|
||||
let providers = state.list_provider_catalog_providers(false).await?;
|
||||
let provider_ids = providers
|
||||
.iter()
|
||||
.map(|item| item.id.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let provider_name_by_id = providers
|
||||
.iter()
|
||||
.map(|item| (item.id.clone(), item.name.clone()))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let keys = if provider_ids.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
state
|
||||
.list_provider_catalog_keys_by_provider_ids(&provider_ids)
|
||||
.await?
|
||||
};
|
||||
|
||||
let active_keys = keys.iter().filter(|item| item.is_active).count();
|
||||
let mut degraded_keys = 0usize;
|
||||
let mut unhealthy_keys = 0usize;
|
||||
let mut open_circuit_breakers = 0usize;
|
||||
let mut open_breaker_labels = Vec::new();
|
||||
let mut circuit_breakers = serde_json::Map::new();
|
||||
let mut previous_circuit_breakers = serde_json::Map::new();
|
||||
|
||||
for key in &keys {
|
||||
let (
|
||||
health_score,
|
||||
consecutive_failures,
|
||||
last_failure_at,
|
||||
circuit_breaker_open,
|
||||
circuit_by_format,
|
||||
) = provider_key_health_summary(key);
|
||||
if health_score < 0.8 {
|
||||
degraded_keys += 1;
|
||||
}
|
||||
if health_score < 0.5 {
|
||||
unhealthy_keys += 1;
|
||||
}
|
||||
|
||||
let open_formats = circuit_by_format
|
||||
.iter()
|
||||
.filter_map(|(api_format, value)| {
|
||||
value
|
||||
.get("open")
|
||||
.and_then(serde_json::Value::as_bool)
|
||||
.filter(|open| *open)
|
||||
.map(|_| api_format.clone())
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if circuit_breaker_open {
|
||||
open_circuit_breakers += 1;
|
||||
let provider_label = provider_name_by_id
|
||||
.get(&key.provider_id)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| key.provider_id.clone());
|
||||
open_breaker_labels.push(format!("{provider_label}/{}", key.name));
|
||||
}
|
||||
|
||||
if circuit_breaker_open || consecutive_failures > 0 || health_score < 1.0 {
|
||||
circuit_breakers.insert(
|
||||
key.id.clone(),
|
||||
json!({
|
||||
"state": if circuit_breaker_open { "open" } else { "closed" },
|
||||
"provider_id": key.provider_id,
|
||||
"provider_name": provider_name_by_id.get(&key.provider_id).cloned(),
|
||||
"key_name": key.name,
|
||||
"health_score": health_score,
|
||||
"consecutive_failures": consecutive_failures,
|
||||
"last_failure_at": last_failure_at,
|
||||
"open_formats": open_formats,
|
||||
}),
|
||||
);
|
||||
previous_circuit_breakers.insert(
|
||||
key.id.clone(),
|
||||
json!({
|
||||
"state": if circuit_breaker_open { "open" } else { "closed" },
|
||||
"failure_count": consecutive_failures,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mut recent_usage_errors = state
|
||||
.list_usage_audits(&UsageAuditListQuery {
|
||||
created_from_unix_secs: Some(recent_error_from.timestamp().max(0) as u64),
|
||||
..Default::default()
|
||||
})
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter(admin_monitoring_usage_is_error)
|
||||
.collect::<Vec<_>>();
|
||||
recent_usage_errors
|
||||
.sort_by(|left, right| right.created_at_unix_secs.cmp(&left.created_at_unix_secs));
|
||||
|
||||
let total_errors = recent_usage_errors.len();
|
||||
let mut error_breakdown = std::collections::BTreeMap::<String, usize>::new();
|
||||
for item in &recent_usage_errors {
|
||||
let error_type = item
|
||||
.error_category
|
||||
.clone()
|
||||
.unwrap_or_else(|| item.status.clone());
|
||||
let operation = format!(
|
||||
"{}:{}",
|
||||
item.provider_name,
|
||||
item.api_format
|
||||
.clone()
|
||||
.unwrap_or_else(|| item.model.clone())
|
||||
);
|
||||
*error_breakdown
|
||||
.entry(format!("{error_type}:{operation}"))
|
||||
.or_default() += 1;
|
||||
}
|
||||
let recent_errors = recent_usage_errors
|
||||
.iter()
|
||||
.take(10)
|
||||
.map(|item| {
|
||||
let error_type = item
|
||||
.error_category
|
||||
.clone()
|
||||
.unwrap_or_else(|| item.status.clone());
|
||||
let operation = format!(
|
||||
"{}:{}",
|
||||
item.provider_name,
|
||||
item.api_format
|
||||
.clone()
|
||||
.unwrap_or_else(|| item.model.clone())
|
||||
);
|
||||
json!({
|
||||
"error_id": item.id,
|
||||
"error_type": error_type,
|
||||
"operation": operation,
|
||||
"timestamp": unix_secs_to_rfc3339(item.created_at_unix_secs),
|
||||
"context": {
|
||||
"request_id": item.request_id,
|
||||
"provider_id": item.provider_id,
|
||||
"provider_name": item.provider_name,
|
||||
"model": item.model,
|
||||
"api_format": item.api_format,
|
||||
"status_code": item.status_code,
|
||||
"error_message": item.error_message,
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let health_score = (100_i64
|
||||
- i64::try_from(total_errors)
|
||||
.unwrap_or(i64::MAX)
|
||||
.saturating_mul(2)
|
||||
- i64::try_from(open_circuit_breakers)
|
||||
.unwrap_or(i64::MAX)
|
||||
.saturating_mul(20))
|
||||
.clamp(0, 100);
|
||||
let status = if health_score > 80 {
|
||||
"healthy"
|
||||
} else if health_score > 50 {
|
||||
"degraded"
|
||||
} else {
|
||||
"critical"
|
||||
};
|
||||
let recommendations = build_admin_monitoring_resilience_recommendations(
|
||||
total_errors,
|
||||
health_score,
|
||||
&open_breaker_labels,
|
||||
);
|
||||
|
||||
Ok(AdminMonitoringResilienceSnapshot {
|
||||
timestamp: now,
|
||||
health_score,
|
||||
status,
|
||||
error_statistics: json!({
|
||||
"total_errors": total_errors,
|
||||
"active_keys": active_keys,
|
||||
"degraded_keys": degraded_keys,
|
||||
"unhealthy_keys": unhealthy_keys,
|
||||
"open_circuit_breakers": open_circuit_breakers,
|
||||
"circuit_breakers": circuit_breakers,
|
||||
}),
|
||||
recent_errors,
|
||||
recommendations,
|
||||
previous_stats: json!({
|
||||
"total_errors": total_errors,
|
||||
"error_breakdown": error_breakdown,
|
||||
"recent_errors": total_errors,
|
||||
"circuit_breakers": previous_circuit_breakers,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_reset_error_stats_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let snapshot = build_admin_monitoring_resilience_snapshot(state).await?;
|
||||
let reset_at = chrono::Utc::now();
|
||||
state.mark_admin_monitoring_error_stats_reset(reset_at.timestamp().max(0) as u64);
|
||||
|
||||
let reset_by = if let Some(user_id) = request_context
|
||||
.control_decision
|
||||
.as_ref()
|
||||
.and_then(|decision| decision.admin_principal.as_ref())
|
||||
.map(|principal| principal.user_id.clone())
|
||||
{
|
||||
state
|
||||
.find_user_auth_by_id(&user_id)
|
||||
.await?
|
||||
.and_then(|user| user.email.or(Some(user.username)))
|
||||
.or(Some(user_id))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok(Json(json!({
|
||||
"message": "错误统计已重置",
|
||||
"previous_stats": snapshot.previous_stats,
|
||||
"reset_by": reset_by,
|
||||
"reset_at": reset_at.to_rfc3339(),
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
use axum::{
|
||||
body::Body,
|
||||
http,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
pub(super) fn admin_monitoring_bad_request_response(detail: impl Into<String>) -> Response<Body> {
|
||||
(
|
||||
http::StatusCode::BAD_REQUEST,
|
||||
Json(json!({ "detail": detail.into() })),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
pub(super) fn admin_monitoring_not_found_response(detail: &'static str) -> Response<Body> {
|
||||
(
|
||||
http::StatusCode::NOT_FOUND,
|
||||
Json(json!({ "detail": detail })),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
use crate::handlers::admin::shared::query_param_value;
|
||||
|
||||
pub(super) fn admin_monitoring_escape_like_pattern(value: &str) -> String {
|
||||
value
|
||||
.chars()
|
||||
.map(|ch| match ch {
|
||||
'%' | '_' => format!("\\{}", ch),
|
||||
_ => ch.to_string(),
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("")
|
||||
}
|
||||
|
||||
pub(super) fn parse_admin_monitoring_offset(query: Option<&str>) -> Result<usize, String> {
|
||||
match query_param_value(query, "offset") {
|
||||
Some(value) => value
|
||||
.parse::<usize>()
|
||||
.map_err(|_| "offset must be a non-negative integer".to_string()),
|
||||
None => Ok(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn parse_admin_monitoring_days(query: Option<&str>) -> Result<i64, String> {
|
||||
match query_param_value(query, "days") {
|
||||
Some(value) => {
|
||||
let parsed = value
|
||||
.parse::<i64>()
|
||||
.map_err(|_| "days must be an integer between 0 and 365".to_string())?;
|
||||
if (0..=365).contains(&parsed) {
|
||||
Ok(parsed)
|
||||
} else {
|
||||
Err("days must be an integer between 0 and 365".to_string())
|
||||
}
|
||||
}
|
||||
None => Ok(30),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn parse_admin_monitoring_hours(query: Option<&str>) -> Result<i64, String> {
|
||||
match query_param_value(query, "hours") {
|
||||
Some(value) => {
|
||||
let parsed = value
|
||||
.parse::<i64>()
|
||||
.map_err(|_| "hours must be an integer between 1 and 720".to_string())?;
|
||||
if (1..=720).contains(&parsed) {
|
||||
Ok(parsed)
|
||||
} else {
|
||||
Err("hours must be an integer between 1 and 720".to_string())
|
||||
}
|
||||
}
|
||||
None => Ok(24),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn parse_admin_monitoring_username_filter(query: Option<&str>) -> Option<String> {
|
||||
query_param_value(query, "username")
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_string)
|
||||
}
|
||||
|
||||
pub(super) fn parse_admin_monitoring_event_type_filter(query: Option<&str>) -> Option<String> {
|
||||
query_param_value(query, "event_type")
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_string)
|
||||
}
|
||||
|
||||
pub(super) fn parse_admin_monitoring_limit(query: Option<&str>) -> Result<usize, String> {
|
||||
match query_param_value(query, "limit") {
|
||||
Some(value) => {
|
||||
let parsed = value
|
||||
.parse::<usize>()
|
||||
.map_err(|_| "limit must be an integer between 1 and 1000".to_string())?;
|
||||
if (1..=1000).contains(&parsed) {
|
||||
Ok(parsed)
|
||||
} else {
|
||||
Err("limit must be an integer between 1 and 1000".to_string())
|
||||
}
|
||||
}
|
||||
None => Ok(100),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,304 @@
|
||||
use super::activity::{
|
||||
build_admin_monitoring_audit_logs_response,
|
||||
build_admin_monitoring_suspicious_activities_response,
|
||||
build_admin_monitoring_system_status_response, build_admin_monitoring_user_behavior_response,
|
||||
};
|
||||
use super::cache::{
|
||||
build_admin_monitoring_cache_config_response, build_admin_monitoring_cache_metrics_response,
|
||||
build_admin_monitoring_cache_stats_response,
|
||||
};
|
||||
use super::cache_affinity_reads::{
|
||||
build_admin_monitoring_cache_affinities_response,
|
||||
build_admin_monitoring_cache_affinity_response,
|
||||
};
|
||||
use super::cache_model_mapping::{
|
||||
build_admin_monitoring_model_mapping_stats_response,
|
||||
build_admin_monitoring_redis_cache_categories_response,
|
||||
};
|
||||
use super::cache_mutations::{
|
||||
build_admin_monitoring_cache_affinity_delete_response,
|
||||
build_admin_monitoring_cache_flush_response,
|
||||
build_admin_monitoring_cache_provider_delete_response,
|
||||
build_admin_monitoring_cache_users_delete_response,
|
||||
build_admin_monitoring_model_mapping_delete_model_response,
|
||||
build_admin_monitoring_model_mapping_delete_provider_response,
|
||||
build_admin_monitoring_model_mapping_delete_response,
|
||||
build_admin_monitoring_redis_keys_delete_response,
|
||||
};
|
||||
use super::resilience::{
|
||||
build_admin_monitoring_reset_error_stats_response,
|
||||
build_admin_monitoring_resilience_circuit_history_response,
|
||||
build_admin_monitoring_resilience_status_response,
|
||||
};
|
||||
use super::trace::{
|
||||
build_admin_monitoring_trace_provider_stats_response,
|
||||
build_admin_monitoring_trace_request_response,
|
||||
};
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use crate::handlers::admin::shared::attach_admin_audit_response;
|
||||
use crate::{AppState, GatewayError};
|
||||
use axum::{body::Body, http, response::Response};
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum AdminMonitoringRoute {
|
||||
AuditLogs,
|
||||
SystemStatus,
|
||||
SuspiciousActivities,
|
||||
UserBehavior,
|
||||
ResilienceStatus,
|
||||
ResilienceErrorStats,
|
||||
ResilienceCircuitHistory,
|
||||
TraceRequest,
|
||||
TraceProviderStats,
|
||||
CacheStats,
|
||||
CacheAffinity,
|
||||
CacheAffinities,
|
||||
CacheUsersDelete,
|
||||
CacheAffinityDelete,
|
||||
CacheFlush,
|
||||
CacheProviderDelete,
|
||||
CacheConfig,
|
||||
CacheMetrics,
|
||||
CacheModelMappingStats,
|
||||
CacheModelMappingDelete,
|
||||
CacheModelMappingDeleteModel,
|
||||
CacheModelMappingDeleteProvider,
|
||||
CacheRedisKeys,
|
||||
CacheRedisKeysDelete,
|
||||
}
|
||||
|
||||
pub(crate) async fn maybe_build_local_admin_monitoring_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
let Some(route) = match_admin_monitoring_route(
|
||||
&request_context.request_method,
|
||||
request_context.request_path.as_str(),
|
||||
) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
match route {
|
||||
AdminMonitoringRoute::AuditLogs => Ok(Some(attach_admin_audit_response(
|
||||
build_admin_monitoring_audit_logs_response(state, request_context).await?,
|
||||
"admin_monitoring_audit_logs_viewed",
|
||||
"view_admin_audit_logs",
|
||||
"audit_log",
|
||||
&admin_monitoring_audit_target_id(request_context),
|
||||
))),
|
||||
AdminMonitoringRoute::ResilienceStatus => Ok(Some(
|
||||
build_admin_monitoring_resilience_status_response(state).await?,
|
||||
)),
|
||||
AdminMonitoringRoute::ResilienceErrorStats => Ok(Some(
|
||||
build_admin_monitoring_reset_error_stats_response(state, request_context).await?,
|
||||
)),
|
||||
AdminMonitoringRoute::ResilienceCircuitHistory => Ok(Some(
|
||||
build_admin_monitoring_resilience_circuit_history_response(state, request_context)
|
||||
.await?,
|
||||
)),
|
||||
AdminMonitoringRoute::CacheStats => Ok(Some(
|
||||
build_admin_monitoring_cache_stats_response(state).await?,
|
||||
)),
|
||||
AdminMonitoringRoute::CacheAffinities => Ok(Some(attach_admin_audit_response(
|
||||
build_admin_monitoring_cache_affinities_response(state, request_context).await?,
|
||||
"admin_monitoring_cache_affinities_viewed",
|
||||
"view_cache_affinities",
|
||||
"cache_affinity",
|
||||
&admin_monitoring_audit_target_id(request_context),
|
||||
))),
|
||||
AdminMonitoringRoute::CacheAffinity => Ok(Some(attach_admin_audit_response(
|
||||
build_admin_monitoring_cache_affinity_response(state, request_context).await?,
|
||||
"admin_monitoring_cache_affinity_viewed",
|
||||
"view_cache_affinity",
|
||||
"cache_affinity",
|
||||
&admin_monitoring_audit_target_id(request_context),
|
||||
))),
|
||||
AdminMonitoringRoute::CacheUsersDelete => Ok(Some(
|
||||
build_admin_monitoring_cache_users_delete_response(state, request_context).await?,
|
||||
)),
|
||||
AdminMonitoringRoute::CacheAffinityDelete => Ok(Some(
|
||||
build_admin_monitoring_cache_affinity_delete_response(state, request_context).await?,
|
||||
)),
|
||||
AdminMonitoringRoute::CacheFlush => Ok(Some(
|
||||
build_admin_monitoring_cache_flush_response(state).await?,
|
||||
)),
|
||||
AdminMonitoringRoute::CacheProviderDelete => Ok(Some(
|
||||
build_admin_monitoring_cache_provider_delete_response(state, request_context).await?,
|
||||
)),
|
||||
AdminMonitoringRoute::CacheModelMappingDelete => Ok(Some(
|
||||
build_admin_monitoring_model_mapping_delete_response(state).await?,
|
||||
)),
|
||||
AdminMonitoringRoute::CacheModelMappingDeleteModel => Ok(Some(
|
||||
build_admin_monitoring_model_mapping_delete_model_response(state, request_context)
|
||||
.await?,
|
||||
)),
|
||||
AdminMonitoringRoute::CacheModelMappingDeleteProvider => Ok(Some(
|
||||
build_admin_monitoring_model_mapping_delete_provider_response(state, request_context)
|
||||
.await?,
|
||||
)),
|
||||
AdminMonitoringRoute::CacheRedisKeysDelete => Ok(Some(
|
||||
build_admin_monitoring_redis_keys_delete_response(state, request_context).await?,
|
||||
)),
|
||||
AdminMonitoringRoute::CacheMetrics => Ok(Some(
|
||||
build_admin_monitoring_cache_metrics_response(state).await?,
|
||||
)),
|
||||
AdminMonitoringRoute::CacheConfig => {
|
||||
Ok(Some(build_admin_monitoring_cache_config_response().await?))
|
||||
}
|
||||
AdminMonitoringRoute::CacheModelMappingStats => Ok(Some(
|
||||
build_admin_monitoring_model_mapping_stats_response(state).await?,
|
||||
)),
|
||||
AdminMonitoringRoute::CacheRedisKeys => Ok(Some(
|
||||
build_admin_monitoring_redis_cache_categories_response(state).await?,
|
||||
)),
|
||||
AdminMonitoringRoute::SystemStatus => Ok(Some(
|
||||
build_admin_monitoring_system_status_response(state).await?,
|
||||
)),
|
||||
AdminMonitoringRoute::SuspiciousActivities => Ok(Some(attach_admin_audit_response(
|
||||
build_admin_monitoring_suspicious_activities_response(state, request_context).await?,
|
||||
"admin_monitoring_suspicious_activities_viewed",
|
||||
"view_suspicious_activities",
|
||||
"suspicious_activity",
|
||||
&admin_monitoring_audit_target_id(request_context),
|
||||
))),
|
||||
AdminMonitoringRoute::UserBehavior => Ok(Some(attach_admin_audit_response(
|
||||
build_admin_monitoring_user_behavior_response(state, request_context).await?,
|
||||
"admin_monitoring_user_behavior_viewed",
|
||||
"view_user_behavior",
|
||||
"user",
|
||||
&admin_monitoring_audit_target_id(request_context),
|
||||
))),
|
||||
AdminMonitoringRoute::TraceRequest => Ok(Some(attach_admin_audit_response(
|
||||
build_admin_monitoring_trace_request_response(state, request_context).await?,
|
||||
"admin_monitoring_request_trace_viewed",
|
||||
"view_request_trace",
|
||||
"request_trace",
|
||||
&admin_monitoring_audit_target_id(request_context),
|
||||
))),
|
||||
AdminMonitoringRoute::TraceProviderStats => Ok(Some(attach_admin_audit_response(
|
||||
build_admin_monitoring_trace_provider_stats_response(state, request_context).await?,
|
||||
"admin_monitoring_provider_trace_stats_viewed",
|
||||
"view_provider_trace_stats",
|
||||
"provider",
|
||||
&admin_monitoring_audit_target_id(request_context),
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn admin_monitoring_audit_target_id(request_context: &GatewayPublicRequestContext) -> String {
|
||||
match request_context.request_query_string.as_deref() {
|
||||
Some(query) if !query.trim().is_empty() => {
|
||||
format!("{}?{query}", request_context.request_path)
|
||||
}
|
||||
_ => request_context.request_path.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn match_admin_monitoring_route(
|
||||
method: &http::Method,
|
||||
path: &str,
|
||||
) -> Option<AdminMonitoringRoute> {
|
||||
let path = normalize_admin_monitoring_path(path);
|
||||
|
||||
match *method {
|
||||
http::Method::GET => match path {
|
||||
"/api/admin/monitoring/audit-logs" => Some(AdminMonitoringRoute::AuditLogs),
|
||||
"/api/admin/monitoring/system-status" => Some(AdminMonitoringRoute::SystemStatus),
|
||||
"/api/admin/monitoring/suspicious-activities" => {
|
||||
Some(AdminMonitoringRoute::SuspiciousActivities)
|
||||
}
|
||||
"/api/admin/monitoring/resilience-status" => {
|
||||
Some(AdminMonitoringRoute::ResilienceStatus)
|
||||
}
|
||||
"/api/admin/monitoring/resilience/circuit-history" => {
|
||||
Some(AdminMonitoringRoute::ResilienceCircuitHistory)
|
||||
}
|
||||
"/api/admin/monitoring/cache/stats" => Some(AdminMonitoringRoute::CacheStats),
|
||||
"/api/admin/monitoring/cache/affinities" => Some(AdminMonitoringRoute::CacheAffinities),
|
||||
"/api/admin/monitoring/cache/config" => Some(AdminMonitoringRoute::CacheConfig),
|
||||
"/api/admin/monitoring/cache/metrics" => Some(AdminMonitoringRoute::CacheMetrics),
|
||||
"/api/admin/monitoring/cache/model-mapping/stats" => {
|
||||
Some(AdminMonitoringRoute::CacheModelMappingStats)
|
||||
}
|
||||
"/api/admin/monitoring/cache/redis-keys" => Some(AdminMonitoringRoute::CacheRedisKeys),
|
||||
_ if matches_dynamic_segments(path, "/api/admin/monitoring/user-behavior/", 1) => {
|
||||
Some(AdminMonitoringRoute::UserBehavior)
|
||||
}
|
||||
_ if matches_dynamic_segments(
|
||||
path,
|
||||
"/api/admin/monitoring/trace/stats/provider/",
|
||||
1,
|
||||
) =>
|
||||
{
|
||||
Some(AdminMonitoringRoute::TraceProviderStats)
|
||||
}
|
||||
_ if matches_dynamic_segments(path, "/api/admin/monitoring/trace/", 1) => {
|
||||
Some(AdminMonitoringRoute::TraceRequest)
|
||||
}
|
||||
_ if matches_dynamic_segments(path, "/api/admin/monitoring/cache/affinity/", 1) => {
|
||||
Some(AdminMonitoringRoute::CacheAffinity)
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
http::Method::DELETE => match path {
|
||||
"/api/admin/monitoring/resilience/error-stats" => {
|
||||
Some(AdminMonitoringRoute::ResilienceErrorStats)
|
||||
}
|
||||
"/api/admin/monitoring/cache" => Some(AdminMonitoringRoute::CacheFlush),
|
||||
"/api/admin/monitoring/cache/model-mapping" => {
|
||||
Some(AdminMonitoringRoute::CacheModelMappingDelete)
|
||||
}
|
||||
_ if matches_dynamic_segments(path, "/api/admin/monitoring/cache/users/", 1) => {
|
||||
Some(AdminMonitoringRoute::CacheUsersDelete)
|
||||
}
|
||||
_ if matches_dynamic_segments(path, "/api/admin/monitoring/cache/providers/", 1) => {
|
||||
Some(AdminMonitoringRoute::CacheProviderDelete)
|
||||
}
|
||||
_ if matches_dynamic_segments(path, "/api/admin/monitoring/cache/redis-keys/", 1) => {
|
||||
Some(AdminMonitoringRoute::CacheRedisKeysDelete)
|
||||
}
|
||||
_ if matches_dynamic_segments(
|
||||
path,
|
||||
"/api/admin/monitoring/cache/model-mapping/provider/",
|
||||
2,
|
||||
) =>
|
||||
{
|
||||
Some(AdminMonitoringRoute::CacheModelMappingDeleteProvider)
|
||||
}
|
||||
_ if matches_dynamic_segments(
|
||||
path,
|
||||
"/api/admin/monitoring/cache/model-mapping/",
|
||||
1,
|
||||
) =>
|
||||
{
|
||||
Some(AdminMonitoringRoute::CacheModelMappingDeleteModel)
|
||||
}
|
||||
_ if matches_dynamic_segments(path, "/api/admin/monitoring/cache/affinity/", 4) => {
|
||||
Some(AdminMonitoringRoute::CacheAffinityDelete)
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_admin_monitoring_path(path: &str) -> &str {
|
||||
let normalized = path.trim_end_matches('/');
|
||||
if normalized.is_empty() {
|
||||
"/"
|
||||
} else {
|
||||
normalized
|
||||
}
|
||||
}
|
||||
|
||||
fn matches_dynamic_segments(path: &str, prefix: &str, dynamic_segments: usize) -> bool {
|
||||
let Some(suffix) = path.strip_prefix(prefix) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
let segments = suffix
|
||||
.split('/')
|
||||
.filter(|segment| !segment.is_empty())
|
||||
.collect::<Vec<_>>();
|
||||
segments.len() == dynamic_segments
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
|
||||
use aether_data_contracts::repository::{
|
||||
candidates::{RequestCandidateStatus, StoredRequestCandidate},
|
||||
provider_catalog::{
|
||||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
|
||||
},
|
||||
usage::StoredRequestUsageAudit,
|
||||
};
|
||||
use axum::http::{self, Uri};
|
||||
use serde_json::json;
|
||||
|
||||
use aether_data::repository::auth::StoredAuthApiKeyExportRecord;
|
||||
use aether_data::repository::users::{StoredUserAuthRecord, StoredUserExportRow};
|
||||
|
||||
pub(super) fn request_context(method: http::Method, uri: &str) -> GatewayPublicRequestContext {
|
||||
GatewayPublicRequestContext::from_request_parts(
|
||||
"trace-123",
|
||||
&method,
|
||||
&uri.parse::<Uri>().expect("uri should parse"),
|
||||
&http::HeaderMap::new(),
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn sample_usage(
|
||||
request_id: &str,
|
||||
provider_id: &str,
|
||||
provider_name: &str,
|
||||
total_tokens: i32,
|
||||
total_cost_usd: f64,
|
||||
status: &str,
|
||||
status_code: Option<i32>,
|
||||
created_at_unix_secs: i64,
|
||||
) -> StoredRequestUsageAudit {
|
||||
let is_error = status_code.is_some_and(|value| value >= 400)
|
||||
|| status.trim().eq_ignore_ascii_case("failed")
|
||||
|| status.trim().eq_ignore_ascii_case("error");
|
||||
StoredRequestUsageAudit::new(
|
||||
format!("usage-{request_id}"),
|
||||
request_id.to_string(),
|
||||
Some("user-1".to_string()),
|
||||
Some("api-key-1".to_string()),
|
||||
Some("alice".to_string()),
|
||||
Some("monitoring-key".to_string()),
|
||||
provider_name.to_string(),
|
||||
"gpt-4.1".to_string(),
|
||||
None,
|
||||
Some(provider_id.to_string()),
|
||||
Some("endpoint-1".to_string()),
|
||||
Some("provider-key-1".to_string()),
|
||||
Some("chat".to_string()),
|
||||
Some("openai:chat".to_string()),
|
||||
Some("openai".to_string()),
|
||||
Some("chat".to_string()),
|
||||
Some("openai:chat".to_string()),
|
||||
Some("openai".to_string()),
|
||||
Some("chat".to_string()),
|
||||
false,
|
||||
false,
|
||||
total_tokens / 2,
|
||||
total_tokens / 2,
|
||||
total_tokens,
|
||||
total_cost_usd,
|
||||
total_cost_usd,
|
||||
status_code,
|
||||
is_error.then(|| "boom".to_string()),
|
||||
is_error.then(|| "upstream_error".to_string()),
|
||||
Some(120),
|
||||
Some(30),
|
||||
status.to_string(),
|
||||
"billed".to_string(),
|
||||
created_at_unix_secs,
|
||||
created_at_unix_secs,
|
||||
Some(created_at_unix_secs),
|
||||
)
|
||||
.expect("usage should build")
|
||||
}
|
||||
|
||||
pub(super) fn sample_candidate(
|
||||
id: &str,
|
||||
request_id: &str,
|
||||
candidate_index: i32,
|
||||
status: RequestCandidateStatus,
|
||||
started_at_unix_secs: Option<i64>,
|
||||
latency_ms: Option<i32>,
|
||||
status_code: Option<i32>,
|
||||
) -> StoredRequestCandidate {
|
||||
StoredRequestCandidate::new(
|
||||
id.to_string(),
|
||||
request_id.to_string(),
|
||||
Some("user-1".to_string()),
|
||||
Some("api-key-1".to_string()),
|
||||
Some("alice".to_string()),
|
||||
Some("default".to_string()),
|
||||
candidate_index,
|
||||
0,
|
||||
Some("provider-1".to_string()),
|
||||
Some("endpoint-1".to_string()),
|
||||
Some("provider-key-1".to_string()),
|
||||
status,
|
||||
None,
|
||||
false,
|
||||
status_code,
|
||||
None,
|
||||
None,
|
||||
latency_ms,
|
||||
Some(1),
|
||||
None,
|
||||
Some(json!({"cache_1h": true})),
|
||||
100 + i64::from(candidate_index),
|
||||
started_at_unix_secs,
|
||||
started_at_unix_secs.map(|value| value + 1),
|
||||
)
|
||||
.expect("candidate should build")
|
||||
}
|
||||
|
||||
pub(super) fn sample_provider() -> StoredProviderCatalogProvider {
|
||||
StoredProviderCatalogProvider::new(
|
||||
"provider-1".to_string(),
|
||||
"OpenAI".to_string(),
|
||||
Some("https://openai.com".to_string()),
|
||||
"custom".to_string(),
|
||||
)
|
||||
.expect("provider should build")
|
||||
}
|
||||
|
||||
pub(super) fn sample_inactive_provider() -> StoredProviderCatalogProvider {
|
||||
StoredProviderCatalogProvider::new(
|
||||
"provider-2".to_string(),
|
||||
"Anthropic".to_string(),
|
||||
Some("https://anthropic.com".to_string()),
|
||||
"custom".to_string(),
|
||||
)
|
||||
.expect("provider should build")
|
||||
.with_transport_fields(false, false, false, None, None, None, None, None, None)
|
||||
}
|
||||
|
||||
pub(super) fn sample_endpoint() -> StoredProviderCatalogEndpoint {
|
||||
StoredProviderCatalogEndpoint::new(
|
||||
"endpoint-1".to_string(),
|
||||
"provider-1".to_string(),
|
||||
"openai:chat".to_string(),
|
||||
Some("openai".to_string()),
|
||||
Some("chat".to_string()),
|
||||
true,
|
||||
)
|
||||
.expect("endpoint should build")
|
||||
}
|
||||
|
||||
pub(super) fn sample_key() -> StoredProviderCatalogKey {
|
||||
StoredProviderCatalogKey::new(
|
||||
"provider-key-1".to_string(),
|
||||
"provider-1".to_string(),
|
||||
"prod-key".to_string(),
|
||||
"api_key".to_string(),
|
||||
Some(json!({"cache_1h": true})),
|
||||
true,
|
||||
)
|
||||
.expect("key should build")
|
||||
}
|
||||
|
||||
pub(super) fn sample_monitoring_auth_user(user_id: &str) -> StoredUserAuthRecord {
|
||||
StoredUserAuthRecord::new(
|
||||
user_id.to_string(),
|
||||
Some("alice@example.com".to_string()),
|
||||
true,
|
||||
"alice".to_string(),
|
||||
None,
|
||||
"user".to_string(),
|
||||
"local".to_string(),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("auth user should build")
|
||||
}
|
||||
|
||||
pub(super) fn sample_monitoring_export_user(user_id: &str) -> StoredUserExportRow {
|
||||
StoredUserExportRow::new(
|
||||
user_id.to_string(),
|
||||
Some("alice@example.com".to_string()),
|
||||
true,
|
||||
"alice".to_string(),
|
||||
None,
|
||||
"user".to_string(),
|
||||
"local".to_string(),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.expect("export user should build")
|
||||
}
|
||||
|
||||
pub(super) fn sample_monitoring_export_api_key(
|
||||
user_id: &str,
|
||||
api_key_id: &str,
|
||||
) -> StoredAuthApiKeyExportRecord {
|
||||
StoredAuthApiKeyExportRecord::new(
|
||||
user_id.to_string(),
|
||||
api_key_id.to_string(),
|
||||
format!("hash-{api_key_id}"),
|
||||
Some(
|
||||
encrypt_python_fernet_plaintext(DEVELOPMENT_ENCRYPTION_KEY, "sk-user-monitoring-1234")
|
||||
.expect("user key should encrypt"),
|
||||
),
|
||||
Some("Alice Key".to_string()),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
None,
|
||||
false,
|
||||
0,
|
||||
0.0,
|
||||
false,
|
||||
)
|
||||
.expect("export api key should build")
|
||||
}
|
||||
|
||||
pub(super) fn sample_monitoring_catalog_endpoint() -> StoredProviderCatalogEndpoint {
|
||||
sample_endpoint()
|
||||
.with_transport_fields(
|
||||
"https://api.openai.example/v1".to_string(),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("endpoint transport fields should build")
|
||||
}
|
||||
|
||||
pub(super) fn sample_monitoring_catalog_key() -> StoredProviderCatalogKey {
|
||||
sample_key()
|
||||
.with_transport_fields(
|
||||
None,
|
||||
encrypt_python_fernet_plaintext(
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
"sk-upstream-monitoring-5678",
|
||||
)
|
||||
.expect("provider key should encrypt"),
|
||||
None,
|
||||
Some(json!({"cache": 1.0})),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("provider key transport fields should build")
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
use super::super::cache_config::ADMIN_MONITORING_REDIS_REQUIRED_DETAIL;
|
||||
use super::super::routes::{
|
||||
match_admin_monitoring_route, maybe_build_local_admin_monitoring_response, AdminMonitoringRoute,
|
||||
};
|
||||
use super::super::test_support::{request_context, sample_key, sample_provider, sample_usage};
|
||||
use crate::AppState;
|
||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
use aether_data::repository::usage::InMemoryUsageReadRepository;
|
||||
use axum::body::to_bytes;
|
||||
use serde_json::json;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[test]
|
||||
fn admin_monitoring_matches_typical_routes() {
|
||||
assert_eq!(
|
||||
match_admin_monitoring_route(&http::Method::GET, "/api/admin/monitoring/audit-logs"),
|
||||
Some(AdminMonitoringRoute::AuditLogs)
|
||||
);
|
||||
assert_eq!(
|
||||
match_admin_monitoring_route(&http::Method::GET, "/api/admin/monitoring/trace/request-1"),
|
||||
Some(AdminMonitoringRoute::TraceRequest)
|
||||
);
|
||||
assert_eq!(
|
||||
match_admin_monitoring_route(&http::Method::GET, "/api/admin/monitoring/cache/stats"),
|
||||
Some(AdminMonitoringRoute::CacheStats)
|
||||
);
|
||||
assert_eq!(
|
||||
match_admin_monitoring_route(
|
||||
&http::Method::GET,
|
||||
"/api/admin/monitoring/resilience-status"
|
||||
),
|
||||
Some(AdminMonitoringRoute::ResilienceStatus)
|
||||
);
|
||||
assert_eq!(
|
||||
match_admin_monitoring_route(
|
||||
&http::Method::GET,
|
||||
"/api/admin/monitoring/user-behavior/user-1"
|
||||
),
|
||||
Some(AdminMonitoringRoute::UserBehavior)
|
||||
);
|
||||
assert_eq!(
|
||||
match_admin_monitoring_route(
|
||||
&http::Method::GET,
|
||||
"/api/admin/monitoring/trace/stats/provider/provider-1"
|
||||
),
|
||||
Some(AdminMonitoringRoute::TraceProviderStats)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admin_monitoring_matches_cache_delete_shapes_and_trailing_slashes() {
|
||||
assert_eq!(
|
||||
match_admin_monitoring_route(&http::Method::DELETE, "/api/admin/monitoring/cache/"),
|
||||
Some(AdminMonitoringRoute::CacheFlush)
|
||||
);
|
||||
assert_eq!(
|
||||
match_admin_monitoring_route(
|
||||
&http::Method::DELETE,
|
||||
"/api/admin/monitoring/cache/model-mapping/provider/provider-1/model-1"
|
||||
),
|
||||
Some(AdminMonitoringRoute::CacheModelMappingDeleteProvider)
|
||||
);
|
||||
assert_eq!(
|
||||
match_admin_monitoring_route(
|
||||
&http::Method::DELETE,
|
||||
"/api/admin/monitoring/cache/affinity/a/b/c/d"
|
||||
),
|
||||
Some(AdminMonitoringRoute::CacheAffinityDelete)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_model_mapping_delete_requires_redis_without_runtime_or_test_entries() {
|
||||
let state = AppState::new().expect("state should build");
|
||||
let context = request_context(
|
||||
http::Method::DELETE,
|
||||
"/api/admin/monitoring/cache/model-mapping",
|
||||
);
|
||||
let response = maybe_build_local_admin_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("monitoring route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::SERVICE_UNAVAILABLE);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(
|
||||
payload,
|
||||
json!({ "detail": ADMIN_MONITORING_REDIS_REQUIRED_DETAIL })
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_user_behavior_returns_empty_local_payload_without_postgres() {
|
||||
let state = AppState::new().expect("state should build");
|
||||
let context = request_context(
|
||||
http::Method::GET,
|
||||
"/api/admin/monitoring/user-behavior/user-123?days=30",
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("user behavior route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["user_id"], json!("user-123"));
|
||||
assert_eq!(payload["period_days"], json!(30));
|
||||
assert_eq!(payload["event_counts"], json!({}));
|
||||
assert_eq!(payload["failed_requests"], json!(0));
|
||||
assert_eq!(payload["success_requests"], json!(0));
|
||||
assert_eq!(payload["success_rate"], json!(0.0));
|
||||
assert_eq!(payload["suspicious_activities"], json!(0));
|
||||
assert!(payload["analysis_time"].as_str().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_audit_logs_returns_empty_local_payload_without_postgres() {
|
||||
let state = AppState::new().expect("state should build");
|
||||
let context = request_context(
|
||||
http::Method::GET,
|
||||
"/api/admin/monitoring/audit-logs?username=alice&event_type=login_failed&days=14&limit=20&offset=5",
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("monitoring route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["items"], json!([]));
|
||||
assert_eq!(payload["meta"]["total"], json!(0));
|
||||
assert_eq!(payload["meta"]["limit"], json!(20));
|
||||
assert_eq!(payload["meta"]["offset"], json!(5));
|
||||
assert_eq!(payload["meta"]["count"], json!(0));
|
||||
assert_eq!(payload["filters"]["username"], json!("alice"));
|
||||
assert_eq!(payload["filters"]["event_type"], json!("login_failed"));
|
||||
assert_eq!(payload["filters"]["days"], json!(14));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_suspicious_activities_returns_empty_local_payload_without_postgres() {
|
||||
let state = AppState::new().expect("state should build");
|
||||
let context = request_context(
|
||||
http::Method::GET,
|
||||
"/api/admin/monitoring/suspicious-activities?hours=48",
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("monitoring route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["activities"], json!([]));
|
||||
assert_eq!(payload["count"], json!(0));
|
||||
assert_eq!(payload["time_range_hours"], json!(48));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_resilience_status_returns_local_payload() {
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||
vec![sample_provider()],
|
||||
vec![],
|
||||
vec![sample_key().with_health_fields(
|
||||
Some(json!({
|
||||
"openai:chat": {
|
||||
"health_score": 0.25,
|
||||
"consecutive_failures": 3,
|
||||
"last_failure_at": "2026-03-30T12:00:00+00:00"
|
||||
}
|
||||
})),
|
||||
Some(json!({
|
||||
"openai:chat": {
|
||||
"open": true
|
||||
}
|
||||
})),
|
||||
)],
|
||||
));
|
||||
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![
|
||||
sample_usage(
|
||||
"request-recent-failed",
|
||||
"provider-1",
|
||||
"OpenAI",
|
||||
10,
|
||||
0.10,
|
||||
"failed",
|
||||
Some(502),
|
||||
now - 120,
|
||||
),
|
||||
sample_usage(
|
||||
"request-old-failed",
|
||||
"provider-1",
|
||||
"OpenAI",
|
||||
12,
|
||||
0.15,
|
||||
"failed",
|
||||
Some(500),
|
||||
now - 172_800,
|
||||
),
|
||||
]));
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(
|
||||
crate::data::GatewayDataState::with_provider_catalog_and_usage_reader_for_tests(
|
||||
provider_catalog,
|
||||
usage_repository,
|
||||
),
|
||||
);
|
||||
let context = request_context(http::Method::GET, "/api/admin/monitoring/resilience-status");
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["health_score"], json!(78));
|
||||
assert_eq!(payload["status"], json!("degraded"));
|
||||
assert_eq!(payload["error_statistics"]["total_errors"], json!(1));
|
||||
assert_eq!(
|
||||
payload["error_statistics"]["open_circuit_breakers"],
|
||||
json!(1)
|
||||
);
|
||||
assert_eq!(
|
||||
payload["error_statistics"]["circuit_breakers"]["provider-key-1"]["state"],
|
||||
json!("open")
|
||||
);
|
||||
assert_eq!(payload["recent_errors"].as_array().map(Vec::len), Some(1));
|
||||
assert_eq!(
|
||||
payload["recent_errors"][0]["error_id"],
|
||||
json!("usage-request-recent-failed")
|
||||
);
|
||||
let recommendations = payload["recommendations"]
|
||||
.as_array()
|
||||
.expect("recommendations should be array");
|
||||
assert!(recommendations.iter().any(|item| item
|
||||
.as_str()
|
||||
.is_some_and(|value| value.contains("prod-key"))));
|
||||
assert!(payload["timestamp"].as_str().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_cache_stats_returns_local_payload() {
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![
|
||||
sample_usage(
|
||||
"request-cache-hit",
|
||||
"provider-1",
|
||||
"OpenAI",
|
||||
20,
|
||||
0.20,
|
||||
"success",
|
||||
Some(200),
|
||||
now - 60,
|
||||
)
|
||||
.with_cache_input_tokens(10, 5),
|
||||
sample_usage(
|
||||
"request-cache-miss",
|
||||
"provider-1",
|
||||
"OpenAI",
|
||||
15,
|
||||
0.10,
|
||||
"success",
|
||||
Some(200),
|
||||
now - 120,
|
||||
),
|
||||
]));
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(
|
||||
crate::data::GatewayDataState::with_usage_reader_for_tests(usage_repository)
|
||||
.with_system_config_values_for_tests([
|
||||
("scheduling_mode".to_string(), json!("cache_affinity")),
|
||||
("provider_priority_mode".to_string(), json!("provider")),
|
||||
]),
|
||||
);
|
||||
let context = request_context(http::Method::GET, "/api/admin/monitoring/cache/stats");
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["status"], json!("ok"));
|
||||
assert_eq!(payload["data"]["scheduler"], json!("cache_aware"));
|
||||
assert_eq!(payload["data"]["total_affinities"], json!(0));
|
||||
assert_eq!(payload["data"]["cache_hits"], json!(1));
|
||||
assert_eq!(payload["data"]["cache_misses"], json!(1));
|
||||
assert_eq!(payload["data"]["cache_hit_rate"], json!(0.5));
|
||||
assert_eq!(
|
||||
payload["data"]["scheduler_metrics"]["scheduling_mode"],
|
||||
json!("cache_affinity")
|
||||
);
|
||||
assert_eq!(
|
||||
payload["data"]["affinity_stats"]["storage_type"],
|
||||
json!("memory")
|
||||
);
|
||||
assert_eq!(
|
||||
payload["data"]["affinity_stats"]["config"]["default_ttl"],
|
||||
json!(300)
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,993 @@
|
||||
use super::test_support::*;
|
||||
use super::{maybe_build_local_admin_monitoring_response, AppState};
|
||||
use aether_data_contracts::repository::{
|
||||
candidates::{RequestCandidateStatus, StoredRequestCandidate},
|
||||
provider_catalog::{
|
||||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
|
||||
},
|
||||
usage::StoredRequestUsageAudit,
|
||||
};
|
||||
use axum::body::to_bytes;
|
||||
use serde_json::json;
|
||||
use std::sync::Arc;
|
||||
|
||||
use aether_data::repository::auth::{
|
||||
InMemoryAuthApiKeySnapshotRepository, StoredAuthApiKeyExportRecord,
|
||||
};
|
||||
use aether_data::repository::candidates::InMemoryRequestCandidateRepository;
|
||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
use aether_data::repository::usage::InMemoryUsageReadRepository;
|
||||
use aether_data::repository::users::{
|
||||
InMemoryUserReadRepository, StoredUserAuthRecord, StoredUserExportRow,
|
||||
};
|
||||
|
||||
mod basics;
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_cache_affinities_returns_empty_payload_without_runtime_or_test_entries() {
|
||||
let state = AppState::new().expect("state should build");
|
||||
let context = request_context(http::Method::GET, "/api/admin/monitoring/cache/affinities");
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["status"], json!("ok"));
|
||||
assert_eq!(payload["data"]["items"], json!([]));
|
||||
assert_eq!(payload["data"]["meta"]["total"], json!(0));
|
||||
assert_eq!(payload["data"]["meta"]["count"], json!(0));
|
||||
assert_eq!(payload["data"]["matched_user_id"], serde_json::Value::Null);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_cache_affinity_returns_not_found_without_runtime_or_test_entries() {
|
||||
let user_repository = Arc::new(
|
||||
InMemoryUserReadRepository::seed_auth_users(vec![sample_monitoring_auth_user("user-1")])
|
||||
.with_export_users(vec![sample_monitoring_export_user("user-1")]),
|
||||
);
|
||||
let auth_repository = Arc::new(
|
||||
InMemoryAuthApiKeySnapshotRepository::default().with_export_records(vec![
|
||||
sample_monitoring_export_api_key("user-1", "user-key-1"),
|
||||
]),
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(
|
||||
crate::data::GatewayDataState::with_user_reader_for_tests(user_repository)
|
||||
.with_auth_api_key_reader(auth_repository),
|
||||
);
|
||||
let context = request_context(
|
||||
http::Method::GET,
|
||||
"/api/admin/monitoring/cache/affinity/alice",
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["status"], json!("not_found"));
|
||||
assert_eq!(payload["user_info"]["user_id"], json!("user-1"));
|
||||
assert_eq!(payload["affinities"], json!([]));
|
||||
assert_eq!(
|
||||
payload["message"],
|
||||
json!("用户 alice (alice@example.com) 没有缓存亲和性")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_cache_affinities_and_affinity_return_local_payload_from_test_store() {
|
||||
let user_repository = Arc::new(
|
||||
InMemoryUserReadRepository::seed_auth_users(vec![sample_monitoring_auth_user("user-1")])
|
||||
.with_export_users(vec![sample_monitoring_export_user("user-1")]),
|
||||
);
|
||||
let auth_repository = Arc::new(
|
||||
InMemoryAuthApiKeySnapshotRepository::default().with_export_records(vec![
|
||||
sample_monitoring_export_api_key("user-1", "user-key-1"),
|
||||
]),
|
||||
);
|
||||
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||
vec![sample_provider()],
|
||||
vec![sample_monitoring_catalog_endpoint()],
|
||||
vec![sample_monitoring_catalog_key()],
|
||||
));
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(
|
||||
crate::data::GatewayDataState::with_provider_catalog_reader_for_tests(provider_catalog)
|
||||
.with_user_reader(user_repository)
|
||||
.with_auth_api_key_reader(auth_repository),
|
||||
)
|
||||
.with_admin_monitoring_cache_affinity_entry_for_tests(
|
||||
"cache_affinity:user-key-1:openai:model-alpha",
|
||||
json!({
|
||||
"provider_id": "provider-1",
|
||||
"endpoint_id": "endpoint-1",
|
||||
"key_id": "provider-key-1",
|
||||
"created_at": 1710000000,
|
||||
"expire_at": 1710000300,
|
||||
"request_count": 7,
|
||||
}),
|
||||
)
|
||||
.with_admin_monitoring_cache_affinity_entry_for_tests(
|
||||
"cache_affinity:user-key-2:openai:model-beta",
|
||||
json!({
|
||||
"provider_id": "provider-2",
|
||||
"endpoint_id": "endpoint-2",
|
||||
"key_id": "provider-key-2",
|
||||
"created_at": 1710000000,
|
||||
"expire_at": 1710000300,
|
||||
"request_count": 4,
|
||||
}),
|
||||
);
|
||||
|
||||
let list_context = request_context(
|
||||
http::Method::GET,
|
||||
"/api/admin/monitoring/cache/affinities?keyword=alice&limit=20&offset=0",
|
||||
);
|
||||
let list_response = maybe_build_local_admin_monitoring_response(&state, &list_context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
assert_eq!(list_response.status(), http::StatusCode::OK);
|
||||
let list_body = to_bytes(list_response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let list_payload: serde_json::Value =
|
||||
serde_json::from_slice(&list_body).expect("json body should parse");
|
||||
assert_eq!(list_payload["status"], json!("ok"));
|
||||
assert_eq!(list_payload["data"]["meta"]["total"], json!(1));
|
||||
assert_eq!(list_payload["data"]["matched_user_id"], json!("user-1"));
|
||||
assert_eq!(
|
||||
list_payload["data"]["items"][0]["affinity_key"],
|
||||
json!("user-key-1")
|
||||
);
|
||||
assert_eq!(list_payload["data"]["items"][0]["username"], json!("alice"));
|
||||
assert_eq!(
|
||||
list_payload["data"]["items"][0]["provider_name"],
|
||||
json!("OpenAI")
|
||||
);
|
||||
assert_eq!(
|
||||
list_payload["data"]["items"][0]["endpoint_url"],
|
||||
json!("https://api.openai.example/v1")
|
||||
);
|
||||
assert_eq!(
|
||||
list_payload["data"]["items"][0]["key_name"],
|
||||
json!("prod-key")
|
||||
);
|
||||
assert_eq!(list_payload["data"]["items"][0]["request_count"], json!(7));
|
||||
|
||||
let detail_context = request_context(
|
||||
http::Method::GET,
|
||||
"/api/admin/monitoring/cache/affinity/alice",
|
||||
);
|
||||
let detail_response = maybe_build_local_admin_monitoring_response(&state, &detail_context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
assert_eq!(detail_response.status(), http::StatusCode::OK);
|
||||
let detail_body = to_bytes(detail_response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let detail_payload: serde_json::Value =
|
||||
serde_json::from_slice(&detail_body).expect("json body should parse");
|
||||
assert_eq!(detail_payload["status"], json!("ok"));
|
||||
assert_eq!(detail_payload["user_info"]["user_id"], json!("user-1"));
|
||||
assert_eq!(
|
||||
detail_payload["affinities"].as_array().map(Vec::len),
|
||||
Some(1)
|
||||
);
|
||||
assert_eq!(
|
||||
detail_payload["affinities"][0]["api_format"],
|
||||
json!("openai")
|
||||
);
|
||||
assert_eq!(detail_payload["total_endpoints"], json!(1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_cache_users_delete_returns_local_payload_from_test_store() {
|
||||
let user_repository = Arc::new(
|
||||
InMemoryUserReadRepository::seed_auth_users(vec![sample_monitoring_auth_user("user-1")])
|
||||
.with_export_users(vec![sample_monitoring_export_user("user-1")]),
|
||||
);
|
||||
let auth_repository = Arc::new(
|
||||
InMemoryAuthApiKeySnapshotRepository::default().with_export_records(vec![
|
||||
sample_monitoring_export_api_key("user-1", "user-key-1"),
|
||||
]),
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(
|
||||
crate::data::GatewayDataState::with_user_reader_for_tests(user_repository)
|
||||
.with_auth_api_key_reader(auth_repository),
|
||||
)
|
||||
.with_admin_monitoring_cache_affinity_entry_for_tests(
|
||||
"cache_affinity:user-key-1:openai:model-alpha",
|
||||
json!({
|
||||
"provider_id": "provider-1",
|
||||
"endpoint_id": "endpoint-1",
|
||||
"key_id": "provider-key-1",
|
||||
"created_at": 1710000000,
|
||||
"expire_at": 1710000300,
|
||||
"request_count": 7,
|
||||
}),
|
||||
)
|
||||
.with_admin_monitoring_cache_affinity_entry_for_tests(
|
||||
"cache_affinity:user-key-2:openai:model-beta",
|
||||
json!({
|
||||
"provider_id": "provider-2",
|
||||
"endpoint_id": "endpoint-2",
|
||||
"key_id": "provider-key-2",
|
||||
"created_at": 1710000000,
|
||||
"expire_at": 1710000300,
|
||||
"request_count": 4,
|
||||
}),
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(
|
||||
&state,
|
||||
&request_context(
|
||||
http::Method::DELETE,
|
||||
"/api/admin/monitoring/cache/users/alice",
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["status"], json!("ok"));
|
||||
assert_eq!(
|
||||
payload["message"],
|
||||
json!("已清除用户 alice 的所有缓存亲和性")
|
||||
);
|
||||
assert_eq!(payload["user_info"]["user_id"], json!("user-1"));
|
||||
let remaining = state.list_admin_monitoring_cache_affinity_entries_for_tests();
|
||||
assert_eq!(remaining.len(), 1);
|
||||
assert!(remaining
|
||||
.iter()
|
||||
.any(|(key, _)| key == "cache_affinity:user-key-2:openai:model-beta"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_cache_users_delete_returns_not_found_for_unknown_identifier() {
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_admin_monitoring_cache_affinity_entry_for_tests(
|
||||
"cache_affinity:user-key-1:openai:model-alpha",
|
||||
json!({
|
||||
"provider_id": "provider-1",
|
||||
"endpoint_id": "endpoint-1",
|
||||
"key_id": "provider-key-1",
|
||||
"created_at": 1710000000,
|
||||
"expire_at": 1710000300,
|
||||
"request_count": 7,
|
||||
}),
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(
|
||||
&state,
|
||||
&request_context(
|
||||
http::Method::DELETE,
|
||||
"/api/admin/monitoring/cache/users/unknown",
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::NOT_FOUND);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(
|
||||
payload["detail"],
|
||||
json!("无法识别的标识符: unknown。支持用户名、邮箱、User ID或API Key ID")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_cache_flush_returns_local_payload_from_test_store() {
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_admin_monitoring_cache_affinity_entry_for_tests(
|
||||
"cache_affinity:user-key-1:openai:model-alpha",
|
||||
json!({
|
||||
"provider_id": "provider-1",
|
||||
"endpoint_id": "endpoint-1",
|
||||
"key_id": "provider-key-1",
|
||||
"created_at": 1710000000,
|
||||
"expire_at": 1710000300,
|
||||
"request_count": 7,
|
||||
}),
|
||||
)
|
||||
.with_admin_monitoring_cache_affinity_entry_for_tests(
|
||||
"cache_affinity:user-key-2:openai:model-beta",
|
||||
json!({
|
||||
"provider_id": "provider-2",
|
||||
"endpoint_id": "endpoint-2",
|
||||
"key_id": "provider-key-2",
|
||||
"created_at": 1710000000,
|
||||
"expire_at": 1710000300,
|
||||
"request_count": 4,
|
||||
}),
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(
|
||||
&state,
|
||||
&request_context(http::Method::DELETE, "/api/admin/monitoring/cache"),
|
||||
)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["status"], json!("ok"));
|
||||
assert_eq!(payload["message"], json!("已清除全部缓存亲和性"));
|
||||
assert_eq!(payload["deleted_affinities"], json!(2));
|
||||
assert!(state
|
||||
.list_admin_monitoring_cache_affinity_entries_for_tests()
|
||||
.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_cache_provider_delete_returns_local_payload_from_test_store() {
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_admin_monitoring_cache_affinity_entry_for_tests(
|
||||
"cache_affinity:user-key-1:openai:model-alpha",
|
||||
json!({
|
||||
"provider_id": "provider-1",
|
||||
"endpoint_id": "endpoint-1",
|
||||
"key_id": "provider-key-1",
|
||||
"created_at": 1710000000,
|
||||
"expire_at": 1710000300,
|
||||
"request_count": 7,
|
||||
}),
|
||||
)
|
||||
.with_admin_monitoring_cache_affinity_entry_for_tests(
|
||||
"cache_affinity:user-key-2:openai:model-beta",
|
||||
json!({
|
||||
"provider_id": "provider-2",
|
||||
"endpoint_id": "endpoint-2",
|
||||
"key_id": "provider-key-2",
|
||||
"created_at": 1710000000,
|
||||
"expire_at": 1710000300,
|
||||
"request_count": 4,
|
||||
}),
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(
|
||||
&state,
|
||||
&request_context(
|
||||
http::Method::DELETE,
|
||||
"/api/admin/monitoring/cache/providers/provider-1",
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["status"], json!("ok"));
|
||||
assert_eq!(
|
||||
payload["message"],
|
||||
json!("已清除 provider provider-1 的缓存亲和性")
|
||||
);
|
||||
assert_eq!(payload["provider_id"], json!("provider-1"));
|
||||
assert_eq!(payload["deleted_affinities"], json!(1));
|
||||
assert_eq!(
|
||||
state
|
||||
.list_admin_monitoring_cache_affinity_entries_for_tests()
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_model_mapping_delete_returns_local_payload_from_test_store() {
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_admin_monitoring_redis_key_for_tests("model:id:model-1", json!({"id": "model-1"}))
|
||||
.with_admin_monitoring_redis_key_for_tests(
|
||||
"model:provider_global:provider-1:model-alpha",
|
||||
json!({"provider_id": "provider-1", "global_model_id": "model-alpha"}),
|
||||
)
|
||||
.with_admin_monitoring_redis_key_for_tests(
|
||||
"global_model:name:model-alpha",
|
||||
json!({"name": "model-alpha"}),
|
||||
)
|
||||
.with_admin_monitoring_redis_key_for_tests(
|
||||
"global_model:resolve:model-alpha",
|
||||
json!({"id": "model-alpha"}),
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(
|
||||
&state,
|
||||
&request_context(
|
||||
http::Method::DELETE,
|
||||
"/api/admin/monitoring/cache/model-mapping",
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["status"], json!("ok"));
|
||||
assert_eq!(payload["message"], json!("已清除所有模型映射缓存"));
|
||||
assert_eq!(payload["deleted_count"], json!(4));
|
||||
assert!(state
|
||||
.list_admin_monitoring_redis_keys_for_tests()
|
||||
.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_model_mapping_delete_model_returns_local_payload_from_test_store() {
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_admin_monitoring_redis_key_for_tests(
|
||||
"global_model:name:model-alpha",
|
||||
json!({"name": "model-alpha"}),
|
||||
)
|
||||
.with_admin_monitoring_redis_key_for_tests(
|
||||
"global_model:resolve:model-alpha",
|
||||
json!({"id": "model-alpha"}),
|
||||
)
|
||||
.with_admin_monitoring_redis_key_for_tests(
|
||||
"global_model:name:model-beta",
|
||||
json!({"name": "model-beta"}),
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(
|
||||
&state,
|
||||
&request_context(
|
||||
http::Method::DELETE,
|
||||
"/api/admin/monitoring/cache/model-mapping/model-alpha",
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["status"], json!("ok"));
|
||||
assert_eq!(payload["model_name"], json!("model-alpha"));
|
||||
assert_eq!(
|
||||
payload["deleted_keys"],
|
||||
json!([
|
||||
"global_model:name:model-alpha",
|
||||
"global_model:resolve:model-alpha"
|
||||
])
|
||||
);
|
||||
assert_eq!(
|
||||
state.list_admin_monitoring_redis_keys_for_tests(),
|
||||
vec!["global_model:name:model-beta".to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_model_mapping_delete_provider_returns_local_payload_from_test_store() {
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_admin_monitoring_redis_key_for_tests(
|
||||
"model:provider_global:provider-1:model-alpha",
|
||||
json!({"provider_id": "provider-1"}),
|
||||
)
|
||||
.with_admin_monitoring_redis_key_for_tests(
|
||||
"model:provider_global:hits:provider-1:model-alpha",
|
||||
json!(12),
|
||||
)
|
||||
.with_admin_monitoring_redis_key_for_tests(
|
||||
"model:provider_global:provider-2:model-alpha",
|
||||
json!({"provider_id": "provider-2"}),
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(
|
||||
&state,
|
||||
&request_context(
|
||||
http::Method::DELETE,
|
||||
"/api/admin/monitoring/cache/model-mapping/provider/provider-1/model-alpha",
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["status"], json!("ok"));
|
||||
assert_eq!(payload["provider_id"], json!("provider-1"));
|
||||
assert_eq!(payload["global_model_id"], json!("model-alpha"));
|
||||
assert_eq!(
|
||||
payload["deleted_keys"],
|
||||
json!([
|
||||
"model:provider_global:hits:provider-1:model-alpha",
|
||||
"model:provider_global:provider-1:model-alpha"
|
||||
])
|
||||
);
|
||||
assert_eq!(
|
||||
state.list_admin_monitoring_redis_keys_for_tests(),
|
||||
vec!["model:provider_global:provider-2:model-alpha".to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_redis_keys_delete_returns_local_payload_from_test_store() {
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_admin_monitoring_redis_key_for_tests("dashboard:summary:user-1", json!({"ok": true}))
|
||||
.with_admin_monitoring_redis_key_for_tests("dashboard:stats:user-1", json!({"ok": true}))
|
||||
.with_admin_monitoring_redis_key_for_tests("user:user-1", json!({"ok": true}));
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(
|
||||
&state,
|
||||
&request_context(
|
||||
http::Method::DELETE,
|
||||
"/api/admin/monitoring/cache/redis-keys/dashboard",
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["status"], json!("ok"));
|
||||
assert_eq!(payload["category"], json!("dashboard"));
|
||||
assert_eq!(payload["deleted_count"], json!(2));
|
||||
assert_eq!(payload["message"], json!("已清除 仪表盘 缓存"));
|
||||
assert_eq!(
|
||||
state.list_admin_monitoring_redis_keys_for_tests(),
|
||||
vec!["user:user-1".to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_cache_affinity_delete_returns_local_payload_from_test_store() {
|
||||
let user_repository = Arc::new(
|
||||
InMemoryUserReadRepository::seed_auth_users(vec![sample_monitoring_auth_user("user-1")])
|
||||
.with_export_users(vec![sample_monitoring_export_user("user-1")]),
|
||||
);
|
||||
let auth_repository = Arc::new(
|
||||
InMemoryAuthApiKeySnapshotRepository::default().with_export_records(vec![
|
||||
sample_monitoring_export_api_key("user-1", "user-key-1"),
|
||||
]),
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(
|
||||
crate::data::GatewayDataState::with_user_reader_for_tests(user_repository)
|
||||
.with_auth_api_key_reader(auth_repository),
|
||||
)
|
||||
.with_admin_monitoring_cache_affinity_entry_for_tests(
|
||||
"cache_affinity:user-key-1:openai:model-alpha",
|
||||
json!({
|
||||
"provider_id": "provider-1",
|
||||
"endpoint_id": "endpoint-1",
|
||||
"key_id": "provider-key-1",
|
||||
"created_at": 1710000000,
|
||||
"expire_at": 1710000300,
|
||||
"request_count": 7,
|
||||
}),
|
||||
)
|
||||
.with_admin_monitoring_cache_affinity_entry_for_tests(
|
||||
"cache_affinity:user-key-2:openai:model-beta",
|
||||
json!({
|
||||
"provider_id": "provider-2",
|
||||
"endpoint_id": "endpoint-2",
|
||||
"key_id": "provider-key-2",
|
||||
"created_at": 1710000000,
|
||||
"expire_at": 1710000300,
|
||||
"request_count": 4,
|
||||
}),
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(
|
||||
&state,
|
||||
&request_context(
|
||||
http::Method::DELETE,
|
||||
"/api/admin/monitoring/cache/affinity/user-key-1/endpoint-1/model-alpha/openai",
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["status"], json!("ok"));
|
||||
assert_eq!(payload["message"], json!("已清除缓存亲和性: Alice Key"));
|
||||
assert_eq!(payload["affinity_key"], json!("user-key-1"));
|
||||
assert_eq!(payload["endpoint_id"], json!("endpoint-1"));
|
||||
assert_eq!(payload["model_id"], json!("model-alpha"));
|
||||
let remaining = state.list_admin_monitoring_cache_affinity_entries_for_tests();
|
||||
assert_eq!(remaining.len(), 1);
|
||||
assert!(remaining
|
||||
.iter()
|
||||
.any(|(key, _)| key == "cache_affinity:user-key-2:openai:model-beta"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_cache_affinity_delete_returns_not_found_for_mismatched_endpoint() {
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_admin_monitoring_cache_affinity_entry_for_tests(
|
||||
"cache_affinity:user-key-1:openai:model-alpha",
|
||||
json!({
|
||||
"provider_id": "provider-1",
|
||||
"endpoint_id": "endpoint-1",
|
||||
"key_id": "provider-key-1",
|
||||
"created_at": 1710000000,
|
||||
"expire_at": 1710000300,
|
||||
"request_count": 7,
|
||||
}),
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(
|
||||
&state,
|
||||
&request_context(
|
||||
http::Method::DELETE,
|
||||
"/api/admin/monitoring/cache/affinity/user-key-1/endpoint-2/model-alpha/openai",
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::NOT_FOUND);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["detail"], json!("未找到指定的缓存亲和性记录"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_cache_metrics_returns_local_payload() {
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![
|
||||
sample_usage(
|
||||
"request-cache-hit",
|
||||
"provider-1",
|
||||
"OpenAI",
|
||||
20,
|
||||
0.20,
|
||||
"success",
|
||||
Some(200),
|
||||
now - 60,
|
||||
)
|
||||
.with_cache_input_tokens(10, 5),
|
||||
sample_usage(
|
||||
"request-cache-miss",
|
||||
"provider-1",
|
||||
"OpenAI",
|
||||
15,
|
||||
0.10,
|
||||
"success",
|
||||
Some(200),
|
||||
now - 120,
|
||||
),
|
||||
]));
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(
|
||||
crate::data::GatewayDataState::with_usage_reader_for_tests(usage_repository)
|
||||
.with_system_config_values_for_tests([
|
||||
("scheduling_mode".to_string(), json!("cache_affinity")),
|
||||
("provider_priority_mode".to_string(), json!("provider")),
|
||||
]),
|
||||
);
|
||||
let context = request_context(http::Method::GET, "/api/admin/monitoring/cache/metrics");
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
assert_eq!(
|
||||
response.headers().get(http::header::CONTENT_TYPE),
|
||||
Some(&http::HeaderValue::from_static(
|
||||
"text/plain; version=0.0.4; charset=utf-8"
|
||||
))
|
||||
);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload = String::from_utf8(body.to_vec()).expect("body should be utf8");
|
||||
assert!(
|
||||
payload.contains("# HELP cache_scheduler_cache_hits Cache hits counted during scheduling")
|
||||
);
|
||||
assert!(payload.contains("cache_scheduler_cache_hits 1"));
|
||||
assert!(payload.contains("cache_scheduler_cache_misses 1"));
|
||||
assert!(payload.contains("cache_scheduler_cache_hit_rate 0.5"));
|
||||
assert!(payload.contains("cache_affinity_total 0"));
|
||||
assert!(payload.contains("cache_scheduler_info{scheduler=\"cache_aware\"} 1"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_cache_config_returns_local_payload() {
|
||||
let state = AppState::new().expect("state should build");
|
||||
let context = request_context(http::Method::GET, "/api/admin/monitoring/cache/config");
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["status"], json!("ok"));
|
||||
assert_eq!(payload["data"]["cache_ttl_seconds"], json!(300));
|
||||
assert_eq!(payload["data"]["cache_reservation_ratio"], json!(0.1));
|
||||
assert_eq!(
|
||||
payload["data"]["dynamic_reservation"]["enabled"],
|
||||
json!(true)
|
||||
);
|
||||
assert_eq!(
|
||||
payload["data"]["dynamic_reservation"]["config"]["probe_phase_requests"],
|
||||
json!(100)
|
||||
);
|
||||
assert_eq!(
|
||||
payload["data"]["dynamic_reservation"]["config"]["stable_max_reservation"],
|
||||
json!(0.35)
|
||||
);
|
||||
assert_eq!(
|
||||
payload["data"]["description"]["dynamic_reservation"],
|
||||
json!("动态预留机制配置")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_model_mapping_stats_returns_local_payload_without_redis() {
|
||||
let state = AppState::new().expect("state should build");
|
||||
let context = request_context(
|
||||
http::Method::GET,
|
||||
"/api/admin/monitoring/cache/model-mapping/stats",
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["status"], json!("ok"));
|
||||
assert_eq!(payload["data"]["available"], json!(false));
|
||||
assert_eq!(
|
||||
payload["data"]["message"],
|
||||
json!("Redis 未启用，模型映射缓存不可用")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_reset_error_stats_returns_local_payload_and_clears_future_snapshot() {
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||
vec![sample_provider()],
|
||||
vec![],
|
||||
vec![sample_key().with_health_fields(
|
||||
Some(json!({
|
||||
"openai:chat": {
|
||||
"health_score": 0.25,
|
||||
"consecutive_failures": 3,
|
||||
"last_failure_at": "2026-03-30T12:00:00+00:00"
|
||||
}
|
||||
})),
|
||||
Some(json!({
|
||||
"openai:chat": {
|
||||
"open": true
|
||||
}
|
||||
})),
|
||||
)],
|
||||
));
|
||||
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![sample_usage(
|
||||
"request-recent-failed",
|
||||
"provider-1",
|
||||
"OpenAI",
|
||||
10,
|
||||
0.10,
|
||||
"failed",
|
||||
Some(502),
|
||||
now - 120,
|
||||
)]));
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(
|
||||
crate::data::GatewayDataState::with_provider_catalog_and_usage_reader_for_tests(
|
||||
provider_catalog,
|
||||
usage_repository,
|
||||
),
|
||||
);
|
||||
|
||||
let reset_context = request_context(
|
||||
http::Method::DELETE,
|
||||
"/api/admin/monitoring/resilience/error-stats",
|
||||
);
|
||||
let reset_response = maybe_build_local_admin_monitoring_response(&state, &reset_context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(reset_response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(reset_response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["message"], json!("错误统计已重置"));
|
||||
assert_eq!(payload["previous_stats"]["total_errors"], json!(1));
|
||||
assert_eq!(payload["previous_stats"]["recent_errors"], json!(1));
|
||||
assert_eq!(
|
||||
payload["previous_stats"]["circuit_breakers"]["provider-key-1"]["state"],
|
||||
json!("open")
|
||||
);
|
||||
assert_eq!(payload["reset_by"], serde_json::Value::Null);
|
||||
assert!(payload["reset_at"].as_str().is_some());
|
||||
|
||||
let status_context =
|
||||
request_context(http::Method::GET, "/api/admin/monitoring/resilience-status");
|
||||
let status_response = maybe_build_local_admin_monitoring_response(&state, &status_context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(status_response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(status_response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["error_statistics"]["total_errors"], json!(0));
|
||||
assert_eq!(payload["recent_errors"], json!([]));
|
||||
assert_eq!(
|
||||
payload["error_statistics"]["open_circuit_breakers"],
|
||||
json!(1)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_redis_keys_returns_local_payload_without_redis() {
|
||||
let state = AppState::new().expect("state should build");
|
||||
let context = request_context(http::Method::GET, "/api/admin/monitoring/cache/redis-keys");
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["status"], json!("ok"));
|
||||
assert_eq!(payload["data"]["available"], json!(false));
|
||||
assert_eq!(payload["data"]["message"], json!("Redis 未启用"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_redis_keys_delete_returns_unavailable_without_redis() {
|
||||
let state = AppState::new().expect("state should build");
|
||||
let context = request_context(
|
||||
http::Method::DELETE,
|
||||
"/api/admin/monitoring/cache/redis-keys/upstream_models",
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::SERVICE_UNAVAILABLE);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["detail"], json!("Redis 未启用"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_circuit_history_returns_local_payload() {
|
||||
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||
vec![sample_provider()],
|
||||
vec![],
|
||||
vec![sample_key().with_health_fields(
|
||||
Some(json!({
|
||||
"openai:chat": {
|
||||
"health_score": 0.25,
|
||||
"consecutive_failures": 3,
|
||||
"last_failure_at": "2026-03-30T12:00:00+00:00"
|
||||
}
|
||||
})),
|
||||
Some(json!({
|
||||
"openai:chat": {
|
||||
"open": true,
|
||||
"open_at": "2026-03-30T12:00:00+00:00",
|
||||
"next_probe_at": "2026-03-30T12:05:00+00:00",
|
||||
"reason": "错误率过高"
|
||||
}
|
||||
})),
|
||||
)],
|
||||
));
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(
|
||||
crate::data::GatewayDataState::with_provider_catalog_reader_for_tests(provider_catalog),
|
||||
);
|
||||
let context = request_context(
|
||||
http::Method::GET,
|
||||
"/api/admin/monitoring/resilience/circuit-history?limit=10",
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["count"], json!(1));
|
||||
assert_eq!(payload["items"][0]["event"], json!("opened"));
|
||||
assert_eq!(payload["items"][0]["key_id"], json!("provider-key-1"));
|
||||
assert_eq!(payload["items"][0]["provider_name"], json!("OpenAI"));
|
||||
assert_eq!(payload["items"][0]["api_format"], json!("openai:chat"));
|
||||
assert_eq!(payload["items"][0]["reason"], json!("错误率过高"));
|
||||
assert_eq!(payload["items"][0]["recovery_seconds"], json!(300));
|
||||
assert_eq!(
|
||||
payload["items"][0]["timestamp"],
|
||||
json!("2026-03-30T12:00:00+00:00")
|
||||
);
|
||||
}
|
||||
|
||||
mod trace;
|
||||
@@ -0,0 +1,179 @@
|
||||
use super::super::routes::maybe_build_local_admin_monitoring_response;
|
||||
use super::super::test_support::{
|
||||
request_context, sample_candidate, sample_endpoint, sample_key, sample_provider,
|
||||
};
|
||||
use crate::AppState;
|
||||
use aether_data_contracts::repository::candidates::RequestCandidateStatus;
|
||||
use axum::body::to_bytes;
|
||||
use serde_json::json;
|
||||
use std::sync::Arc;
|
||||
|
||||
use aether_data::repository::candidates::InMemoryRequestCandidateRepository;
|
||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_trace_request_returns_local_payload() {
|
||||
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
|
||||
sample_candidate(
|
||||
"cand-unused",
|
||||
"request-1",
|
||||
0,
|
||||
RequestCandidateStatus::Pending,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
),
|
||||
sample_candidate(
|
||||
"cand-used",
|
||||
"request-1",
|
||||
1,
|
||||
RequestCandidateStatus::Failed,
|
||||
Some(101),
|
||||
Some(33),
|
||||
Some(502),
|
||||
),
|
||||
]));
|
||||
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||
vec![sample_provider()],
|
||||
vec![sample_endpoint()],
|
||||
vec![sample_key()],
|
||||
));
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_decision_trace_data_readers_for_tests(request_candidates, provider_catalog);
|
||||
let context = request_context(
|
||||
http::Method::GET,
|
||||
"/api/admin/monitoring/trace/request-1?attempted_only=true",
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["request_id"], json!("request-1"));
|
||||
assert_eq!(payload["total_candidates"], json!(1));
|
||||
assert_eq!(payload["final_status"], json!("failed"));
|
||||
assert_eq!(payload["candidates"][0]["id"], json!("cand-used"));
|
||||
assert_eq!(payload["candidates"][0]["provider_name"], json!("OpenAI"));
|
||||
assert_eq!(
|
||||
payload["candidates"][0]["provider_website"],
|
||||
json!("https://openai.com")
|
||||
);
|
||||
assert_eq!(
|
||||
payload["candidates"][0]["endpoint_name"],
|
||||
json!("openai:chat")
|
||||
);
|
||||
assert_eq!(payload["candidates"][0]["key_name"], json!("prod-key"));
|
||||
assert_eq!(payload["candidates"][0]["key_auth_type"], json!("api_key"));
|
||||
assert_eq!(payload["candidates"][0]["latency_ms"], json!(33));
|
||||
assert_eq!(payload["candidates"][0]["status_code"], json!(502));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_trace_provider_stats_returns_local_payload() {
|
||||
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
|
||||
sample_candidate(
|
||||
"cand-1",
|
||||
"req-a",
|
||||
0,
|
||||
RequestCandidateStatus::Success,
|
||||
Some(101),
|
||||
Some(20),
|
||||
Some(200),
|
||||
),
|
||||
sample_candidate(
|
||||
"cand-2",
|
||||
"req-b",
|
||||
0,
|
||||
RequestCandidateStatus::Failed,
|
||||
Some(201),
|
||||
Some(40),
|
||||
Some(502),
|
||||
),
|
||||
sample_candidate(
|
||||
"cand-3",
|
||||
"req-c",
|
||||
0,
|
||||
RequestCandidateStatus::Cancelled,
|
||||
Some(301),
|
||||
Some(60),
|
||||
Some(499),
|
||||
),
|
||||
sample_candidate(
|
||||
"cand-4",
|
||||
"req-d",
|
||||
0,
|
||||
RequestCandidateStatus::Available,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
),
|
||||
sample_candidate(
|
||||
"cand-5",
|
||||
"req-e",
|
||||
0,
|
||||
RequestCandidateStatus::Unused,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
),
|
||||
]));
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_request_candidate_data_reader_for_tests(request_candidates);
|
||||
let context = request_context(
|
||||
http::Method::GET,
|
||||
"/api/admin/monitoring/trace/stats/provider/provider-1?limit=10",
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["provider_id"], json!("provider-1"));
|
||||
assert_eq!(payload["total_attempts"], json!(5));
|
||||
assert_eq!(payload["success_count"], json!(1));
|
||||
assert_eq!(payload["failed_count"], json!(1));
|
||||
assert_eq!(payload["cancelled_count"], json!(1));
|
||||
assert_eq!(payload["skipped_count"], json!(0));
|
||||
assert_eq!(payload["pending_count"], json!(0));
|
||||
assert_eq!(payload["available_count"], json!(1));
|
||||
assert_eq!(payload["unused_count"], json!(1));
|
||||
assert_eq!(payload["failure_rate"], json!(50.0));
|
||||
assert_eq!(payload["avg_latency_ms"], json!(40.0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_trace_request_returns_contextual_not_found_payload() {
|
||||
let state = AppState::new().expect("state should build");
|
||||
let context = request_context(
|
||||
http::Method::GET,
|
||||
"/api/admin/monitoring/trace/provider-test-missing?attempted_only=false",
|
||||
);
|
||||
|
||||
let response = maybe_build_local_admin_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::NOT_FOUND);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["detail"], json!("Request trace not found"));
|
||||
assert_eq!(payload["request_id"], json!("provider-test-missing"));
|
||||
assert_eq!(payload["attempted_only"], json!(false));
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
use super::responses::admin_monitoring_bad_request_response;
|
||||
use super::route_filters::parse_admin_monitoring_limit;
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use crate::handlers::admin::shared::{query_param_value, unix_secs_to_rfc3339};
|
||||
use crate::log_ids::short_request_id;
|
||||
use crate::{AppState, GatewayError};
|
||||
use aether_data_contracts::repository::candidates::RequestCandidateStatus;
|
||||
use axum::{
|
||||
body::Body,
|
||||
http,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
use tracing::warn;
|
||||
|
||||
fn admin_monitoring_trace_request_id_from_path(request_path: &str) -> Option<String> {
|
||||
let value = request_path
|
||||
.strip_prefix("/api/admin/monitoring/trace/")?
|
||||
.trim()
|
||||
.trim_matches('/')
|
||||
.to_string();
|
||||
if value.is_empty() || value.contains('/') {
|
||||
None
|
||||
} else {
|
||||
Some(value)
|
||||
}
|
||||
}
|
||||
|
||||
fn admin_monitoring_trace_provider_id_from_path(request_path: &str) -> Option<String> {
|
||||
let value = request_path
|
||||
.strip_prefix("/api/admin/monitoring/trace/stats/provider/")?
|
||||
.trim()
|
||||
.trim_matches('/')
|
||||
.to_string();
|
||||
if value.is_empty() || value.contains('/') {
|
||||
None
|
||||
} else {
|
||||
Some(value)
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_admin_monitoring_attempted_only(query: Option<&str>) -> Result<bool, String> {
|
||||
match query_param_value(query, "attempted_only") {
|
||||
None => Ok(false),
|
||||
Some(value) => match value.trim().to_ascii_lowercase().as_str() {
|
||||
"true" | "1" | "yes" => Ok(true),
|
||||
"false" | "0" | "no" => Ok(false),
|
||||
_ => Err("attempted_only must be a boolean".to_string()),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn admin_monitoring_trace_not_found_response(
|
||||
request_id: &str,
|
||||
attempted_only: bool,
|
||||
) -> Response<Body> {
|
||||
(
|
||||
http::StatusCode::NOT_FOUND,
|
||||
Json(json!({
|
||||
"detail": "Request trace not found",
|
||||
"request_id": request_id,
|
||||
"attempted_only": attempted_only,
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_trace_request_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let Some(request_id) =
|
||||
admin_monitoring_trace_request_id_from_path(&request_context.request_path)
|
||||
else {
|
||||
return Ok(admin_monitoring_bad_request_response("缺少 request_id"));
|
||||
};
|
||||
let attempted_only = match parse_admin_monitoring_attempted_only(
|
||||
request_context.request_query_string.as_deref(),
|
||||
) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(admin_monitoring_bad_request_response(detail)),
|
||||
};
|
||||
|
||||
let Some(trace) = state
|
||||
.data
|
||||
.read_decision_trace(&request_id, attempted_only)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
else {
|
||||
warn!(
|
||||
event_name = "admin_monitoring_request_trace_not_found",
|
||||
log_type = "admin_monitoring",
|
||||
request_id = %short_request_id(request_id.as_str()),
|
||||
attempted_only,
|
||||
path = %request_context.request_path,
|
||||
"admin monitoring request trace not found"
|
||||
);
|
||||
return Ok(admin_monitoring_trace_not_found_response(
|
||||
&request_id,
|
||||
attempted_only,
|
||||
));
|
||||
};
|
||||
|
||||
let candidates = trace
|
||||
.candidates
|
||||
.iter()
|
||||
.map(|item| {
|
||||
let candidate = &item.candidate;
|
||||
json!({
|
||||
"id": candidate.id,
|
||||
"request_id": candidate.request_id,
|
||||
"candidate_index": candidate.candidate_index,
|
||||
"retry_index": candidate.retry_index,
|
||||
"provider_id": candidate.provider_id,
|
||||
"provider_name": item.provider_name,
|
||||
"provider_website": item.provider_website,
|
||||
"endpoint_id": candidate.endpoint_id,
|
||||
"endpoint_name": item.endpoint_api_format,
|
||||
"key_id": candidate.key_id,
|
||||
"key_name": item.provider_key_name,
|
||||
"key_account_label": serde_json::Value::Null,
|
||||
"key_preview": serde_json::Value::Null,
|
||||
"key_auth_type": item.provider_key_auth_type,
|
||||
"key_oauth_plan_type": serde_json::Value::Null,
|
||||
"key_capabilities": item.provider_key_capabilities,
|
||||
"required_capabilities": candidate.required_capabilities,
|
||||
"status": candidate.status,
|
||||
"skip_reason": candidate.skip_reason,
|
||||
"is_cached": candidate.is_cached,
|
||||
"status_code": candidate.status_code,
|
||||
"error_type": candidate.error_type,
|
||||
"error_message": candidate.error_message,
|
||||
"latency_ms": candidate.latency_ms,
|
||||
"concurrent_requests": candidate.concurrent_requests,
|
||||
"extra_data": candidate.extra_data,
|
||||
"created_at": unix_secs_to_rfc3339(candidate.created_at_unix_secs),
|
||||
"started_at": candidate.started_at_unix_secs.and_then(unix_secs_to_rfc3339),
|
||||
"finished_at": candidate.finished_at_unix_secs.and_then(unix_secs_to_rfc3339),
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
Ok(Json(json!({
|
||||
"request_id": trace.request_id,
|
||||
"total_candidates": trace.total_candidates,
|
||||
"final_status": trace.final_status,
|
||||
"total_latency_ms": trace.total_latency_ms,
|
||||
"candidates": candidates,
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_monitoring_trace_provider_stats_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let Some(provider_id) =
|
||||
admin_monitoring_trace_provider_id_from_path(&request_context.request_path)
|
||||
else {
|
||||
return Ok(admin_monitoring_bad_request_response("缺少 provider_id"));
|
||||
};
|
||||
let limit = match parse_admin_monitoring_limit(request_context.request_query_string.as_deref())
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(admin_monitoring_bad_request_response(detail)),
|
||||
};
|
||||
|
||||
let candidates = state
|
||||
.read_request_candidates_by_provider_id(&provider_id, limit)
|
||||
.await?;
|
||||
let total_attempts = candidates.len();
|
||||
let success_count = candidates
|
||||
.iter()
|
||||
.filter(|item| item.status == RequestCandidateStatus::Success)
|
||||
.count();
|
||||
let failed_count = candidates
|
||||
.iter()
|
||||
.filter(|item| item.status == RequestCandidateStatus::Failed)
|
||||
.count();
|
||||
let cancelled_count = candidates
|
||||
.iter()
|
||||
.filter(|item| item.status == RequestCandidateStatus::Cancelled)
|
||||
.count();
|
||||
let skipped_count = candidates
|
||||
.iter()
|
||||
.filter(|item| item.status == RequestCandidateStatus::Skipped)
|
||||
.count();
|
||||
let pending_count = candidates
|
||||
.iter()
|
||||
.filter(|item| item.status == RequestCandidateStatus::Pending)
|
||||
.count();
|
||||
let available_count = candidates
|
||||
.iter()
|
||||
.filter(|item| item.status == RequestCandidateStatus::Available)
|
||||
.count();
|
||||
let unused_count = candidates
|
||||
.iter()
|
||||
.filter(|item| item.status == RequestCandidateStatus::Unused)
|
||||
.count();
|
||||
let completed_count = success_count + failed_count;
|
||||
let failure_rate = if completed_count == 0 {
|
||||
0.0
|
||||
} else {
|
||||
((failed_count as f64 / completed_count as f64) * 10000.0).round() / 100.0
|
||||
};
|
||||
let latency_values = candidates
|
||||
.iter()
|
||||
.filter_map(|item| item.latency_ms.map(|value| value as f64))
|
||||
.collect::<Vec<_>>();
|
||||
let avg_latency_ms = if latency_values.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
let total = latency_values.iter().sum::<f64>();
|
||||
((total / latency_values.len() as f64) * 100.0).round() / 100.0
|
||||
};
|
||||
|
||||
Ok(Json(json!({
|
||||
"provider_id": provider_id,
|
||||
"total_attempts": total_attempts,
|
||||
"success_count": success_count,
|
||||
"failed_count": failed_count,
|
||||
"cancelled_count": cancelled_count,
|
||||
"skipped_count": skipped_count,
|
||||
"pending_count": pending_count,
|
||||
"available_count": available_count,
|
||||
"unused_count": unused_count,
|
||||
"failure_rate": failure_rate,
|
||||
"avg_latency_ms": avg_latency_ms,
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
use aether_data_contracts::repository::usage::StoredRequestUsageAudit;
|
||||
|
||||
pub(super) fn admin_monitoring_usage_is_error(item: &StoredRequestUsageAudit) -> bool {
|
||||
item.status_code.is_some_and(|value| value >= 400)
|
||||
|| item.status.trim().eq_ignore_ascii_case("failed")
|
||||
|| item.status.trim().eq_ignore_ascii_case("error")
|
||||
|| item.error_message.is_some()
|
||||
|| item.error_category.is_some()
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
use super::helpers::{
|
||||
round_to, AdminStatsComparisonType, AdminStatsGranularity, AdminStatsTimeRange,
|
||||
AdminStatsUsageFilter,
|
||||
};
|
||||
use super::range::{build_comparison_range, list_usage_for_range};
|
||||
use super::responses::{
|
||||
admin_stats_bad_request_response, admin_stats_comparison_empty_response,
|
||||
admin_stats_error_distribution_empty_response,
|
||||
admin_stats_performance_percentiles_empty_response, admin_stats_time_series_empty_response,
|
||||
};
|
||||
use super::timeseries::{
|
||||
aggregate_usage_stats, build_time_series_payload, pct_change_value, percentile_cont,
|
||||
};
|
||||
use crate::control::GatewayControlDecision;
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use crate::handlers::admin::shared::query_param_value;
|
||||
use crate::{AppState, GatewayError};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
pub(super) async fn maybe_build_local_admin_stats_analytics_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
decision: &GatewayControlDecision,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
if decision.route_kind.as_deref() == Some("comparison")
|
||||
&& request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/admin/stats/comparison" | "/api/admin/stats/comparison/"
|
||||
)
|
||||
{
|
||||
let current_range = match AdminStatsTimeRange::resolve_required(
|
||||
request_context.request_query_string.as_deref(),
|
||||
"current_start",
|
||||
"current_end",
|
||||
) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let comparison_type = match query_param_value(
|
||||
request_context.request_query_string.as_deref(),
|
||||
"comparison_type",
|
||||
)
|
||||
.as_deref()
|
||||
{
|
||||
None | Some("period") => AdminStatsComparisonType::Period,
|
||||
Some("year") => AdminStatsComparisonType::Year,
|
||||
Some(_) => {
|
||||
return Ok(Some(admin_stats_bad_request_response(
|
||||
"comparison_type must be 'period' or 'year'".to_string(),
|
||||
)));
|
||||
}
|
||||
};
|
||||
|
||||
let comparison_range = match build_comparison_range(¤t_range, comparison_type) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_stats_comparison_empty_response(
|
||||
¤t_range,
|
||||
&comparison_range,
|
||||
)));
|
||||
}
|
||||
let current_usage =
|
||||
list_usage_for_range(state, ¤t_range, &AdminStatsUsageFilter::default()).await?;
|
||||
let comparison_usage =
|
||||
list_usage_for_range(state, &comparison_range, &AdminStatsUsageFilter::default())
|
||||
.await?;
|
||||
let current = aggregate_usage_stats(¤t_usage);
|
||||
let comparison = aggregate_usage_stats(&comparison_usage);
|
||||
|
||||
return Ok(Some(
|
||||
Json(json!({
|
||||
"current": {
|
||||
"total_requests": current.total_requests,
|
||||
"total_tokens": current.total_tokens,
|
||||
"total_cost": round_to(current.total_cost, 6),
|
||||
"actual_total_cost": round_to(current.actual_total_cost, 6),
|
||||
"avg_response_time_ms": round_to(current.avg_response_time_ms(), 2),
|
||||
"error_requests": current.error_requests,
|
||||
},
|
||||
"comparison": {
|
||||
"total_requests": comparison.total_requests,
|
||||
"total_tokens": comparison.total_tokens,
|
||||
"total_cost": round_to(comparison.total_cost, 6),
|
||||
"actual_total_cost": round_to(comparison.actual_total_cost, 6),
|
||||
"avg_response_time_ms": round_to(comparison.avg_response_time_ms(), 2),
|
||||
"error_requests": comparison.error_requests,
|
||||
},
|
||||
"change_percent": {
|
||||
"total_requests": pct_change_value(current.total_requests as f64, comparison.total_requests as f64),
|
||||
"total_tokens": pct_change_value(current.total_tokens as f64, comparison.total_tokens as f64),
|
||||
"total_cost": pct_change_value(current.total_cost, comparison.total_cost),
|
||||
"actual_total_cost": pct_change_value(current.actual_total_cost, comparison.actual_total_cost),
|
||||
"avg_response_time_ms": pct_change_value(current.avg_response_time_ms(), comparison.avg_response_time_ms()),
|
||||
"error_requests": pct_change_value(current.error_requests as f64, comparison.error_requests as f64),
|
||||
},
|
||||
"current_start": current_range.start_date.to_string(),
|
||||
"current_end": current_range.end_date.to_string(),
|
||||
"comparison_start": comparison_range.start_date.to_string(),
|
||||
"comparison_end": comparison_range.end_date.to_string(),
|
||||
}))
|
||||
.into_response(),
|
||||
));
|
||||
}
|
||||
|
||||
if decision.route_kind.as_deref() == Some("error_distribution")
|
||||
&& request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/admin/stats/errors/distribution" | "/api/admin/stats/errors/distribution/"
|
||||
)
|
||||
{
|
||||
let Some(time_range) = (match AdminStatsTimeRange::resolve_optional(
|
||||
request_context.request_query_string.as_deref(),
|
||||
) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
}) else {
|
||||
return Ok(Some(
|
||||
Json(json!({
|
||||
"distribution": [],
|
||||
"trend": [],
|
||||
}))
|
||||
.into_response(),
|
||||
));
|
||||
};
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_stats_error_distribution_empty_response()));
|
||||
}
|
||||
|
||||
let usage =
|
||||
list_usage_for_range(state, &time_range, &AdminStatsUsageFilter::default()).await?;
|
||||
let mut distribution: std::collections::BTreeMap<String, u64> =
|
||||
std::collections::BTreeMap::new();
|
||||
let mut trend: std::collections::BTreeMap<String, std::collections::BTreeMap<String, u64>> =
|
||||
std::collections::BTreeMap::new();
|
||||
|
||||
for item in usage {
|
||||
let Some(category) = item
|
||||
.error_category
|
||||
.as_ref()
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let Some(local_day) =
|
||||
time_range.local_date_string_for_unix_secs(item.created_at_unix_secs)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
*distribution.entry(category.clone()).or_default() += 1;
|
||||
*trend
|
||||
.entry(local_day)
|
||||
.or_default()
|
||||
.entry(category.clone())
|
||||
.or_default() += 1;
|
||||
}
|
||||
|
||||
let mut distribution_items: Vec<_> = distribution
|
||||
.into_iter()
|
||||
.map(|(category, count)| json!({ "category": category, "count": count }))
|
||||
.collect();
|
||||
distribution_items.sort_by(|left, right| {
|
||||
let left_count = left
|
||||
.get("count")
|
||||
.and_then(serde_json::Value::as_u64)
|
||||
.unwrap_or(0);
|
||||
let right_count = right
|
||||
.get("count")
|
||||
.and_then(serde_json::Value::as_u64)
|
||||
.unwrap_or(0);
|
||||
right_count.cmp(&left_count).then_with(|| {
|
||||
left.get("category")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.cmp(&right.get("category").and_then(serde_json::Value::as_str))
|
||||
})
|
||||
});
|
||||
|
||||
let trend_items: Vec<_> = trend
|
||||
.into_iter()
|
||||
.map(|(date, categories)| {
|
||||
let total: u64 = categories.values().copied().sum();
|
||||
json!({
|
||||
"date": date,
|
||||
"total": total,
|
||||
"categories": categories,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
return Ok(Some(
|
||||
Json(json!({
|
||||
"distribution": distribution_items,
|
||||
"trend": trend_items,
|
||||
}))
|
||||
.into_response(),
|
||||
));
|
||||
}
|
||||
|
||||
if decision.route_kind.as_deref() == Some("performance_percentiles")
|
||||
&& request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/admin/stats/performance/percentiles"
|
||||
| "/api/admin/stats/performance/percentiles/"
|
||||
)
|
||||
{
|
||||
let Some(time_range) = (match AdminStatsTimeRange::resolve_optional(
|
||||
request_context.request_query_string.as_deref(),
|
||||
) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
}) else {
|
||||
return Ok(Some(Json(json!([])).into_response()));
|
||||
};
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_stats_performance_percentiles_empty_response()));
|
||||
}
|
||||
|
||||
let usage =
|
||||
list_usage_for_range(state, &time_range, &AdminStatsUsageFilter::default()).await?;
|
||||
let mut by_day: std::collections::BTreeMap<String, (Vec<u64>, Vec<u64>)> = time_range
|
||||
.local_date_strings()
|
||||
.into_iter()
|
||||
.map(|date| (date, (Vec::new(), Vec::new())))
|
||||
.collect();
|
||||
|
||||
for item in usage {
|
||||
if item.status != "completed" {
|
||||
continue;
|
||||
}
|
||||
let Some(local_day) =
|
||||
time_range.local_date_string_for_unix_secs(item.created_at_unix_secs)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let Some((response_times, first_byte_times)) = by_day.get_mut(&local_day) else {
|
||||
continue;
|
||||
};
|
||||
if let Some(response_time_ms) = item.response_time_ms {
|
||||
response_times.push(response_time_ms);
|
||||
}
|
||||
if let Some(first_byte_time_ms) = item.first_byte_time_ms {
|
||||
first_byte_times.push(first_byte_time_ms);
|
||||
}
|
||||
}
|
||||
|
||||
let payload: Vec<_> = by_day
|
||||
.into_iter()
|
||||
.map(|(date, (mut response_times, mut first_byte_times))| {
|
||||
json!({
|
||||
"date": date,
|
||||
"p50_response_time_ms": percentile_cont(&mut response_times, 0.5),
|
||||
"p90_response_time_ms": percentile_cont(&mut response_times, 0.9),
|
||||
"p99_response_time_ms": percentile_cont(&mut response_times, 0.99),
|
||||
"p50_first_byte_time_ms": percentile_cont(&mut first_byte_times, 0.5),
|
||||
"p90_first_byte_time_ms": percentile_cont(&mut first_byte_times, 0.9),
|
||||
"p99_first_byte_time_ms": percentile_cont(&mut first_byte_times, 0.99),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
return Ok(Some(
|
||||
Json(serde_json::Value::Array(payload)).into_response(),
|
||||
));
|
||||
}
|
||||
|
||||
if decision.route_kind.as_deref() == Some("time_series")
|
||||
&& request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/admin/stats/time-series" | "/api/admin/stats/time-series/"
|
||||
)
|
||||
{
|
||||
let granularity =
|
||||
match AdminStatsGranularity::parse(request_context.request_query_string.as_deref()) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let Some(time_range) = (match AdminStatsTimeRange::resolve_optional(
|
||||
request_context.request_query_string.as_deref(),
|
||||
) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
}) else {
|
||||
return Ok(Some(Json(json!([])).into_response()));
|
||||
};
|
||||
if let Err(detail) = time_range.validate_for_time_series(granularity) {
|
||||
return Ok(Some(admin_stats_bad_request_response(detail)));
|
||||
}
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_stats_time_series_empty_response()));
|
||||
}
|
||||
|
||||
let filters =
|
||||
AdminStatsUsageFilter::from_query(request_context.request_query_string.as_deref());
|
||||
let usage = list_usage_for_range(state, &time_range, &filters).await?;
|
||||
let payload = build_time_series_payload(&time_range, granularity, &usage);
|
||||
|
||||
return Ok(Some(
|
||||
Json(serde_json::Value::Array(payload)).into_response(),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
use super::helpers::{
|
||||
round_to, AdminStatsForecastPoint, AdminStatsGranularity, AdminStatsTimeRange,
|
||||
AdminStatsUsageFilter,
|
||||
};
|
||||
use super::range::{
|
||||
build_time_range_from_days, list_usage_for_range, parse_bounded_u32, parse_tz_offset_minutes,
|
||||
};
|
||||
use super::responses::{
|
||||
admin_stats_bad_request_response, admin_stats_cost_forecast_empty_response,
|
||||
admin_stats_cost_savings_empty_response,
|
||||
};
|
||||
use super::timeseries::{build_daily_time_series_buckets, linear_regression};
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use crate::handlers::admin::shared::query_param_value;
|
||||
use crate::{AppState, GatewayError};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
pub(super) async fn maybe_build_local_admin_stats_cost_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
let query = request_context.request_query_string.as_deref();
|
||||
|
||||
if request_context
|
||||
.control_decision
|
||||
.as_ref()
|
||||
.and_then(|decision| decision.route_kind.as_deref())
|
||||
== Some("cost_forecast")
|
||||
&& request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/admin/stats/cost/forecast" | "/api/admin/stats/cost/forecast/"
|
||||
)
|
||||
{
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_stats_cost_forecast_empty_response()));
|
||||
}
|
||||
|
||||
let forecast_days = match query_param_value(query, "forecast_days")
|
||||
.map(|value| parse_bounded_u32("forecast_days", &value, 1, 90))
|
||||
.transpose()
|
||||
{
|
||||
Ok(Some(value)) => value,
|
||||
Ok(None) => 7,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let tz_offset_minutes = match parse_tz_offset_minutes(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
|
||||
Ok(Some(value)) => value,
|
||||
Ok(None) => {
|
||||
let days = match query_param_value(query, "days")
|
||||
.map(|value| parse_bounded_u32("days", &value, 7, 365))
|
||||
.transpose()
|
||||
{
|
||||
Ok(Some(value)) => value,
|
||||
Ok(None) => 30,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
match build_time_range_from_days(days, tz_offset_minutes) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
}
|
||||
}
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
if let Err(detail) = time_range.validate_for_time_series(AdminStatsGranularity::Day) {
|
||||
return Ok(Some(admin_stats_bad_request_response(detail)));
|
||||
}
|
||||
|
||||
let usage =
|
||||
list_usage_for_range(state, &time_range, &AdminStatsUsageFilter::default()).await?;
|
||||
let daily = build_daily_time_series_buckets(&time_range, &usage);
|
||||
let history: Vec<AdminStatsForecastPoint> = daily
|
||||
.into_iter()
|
||||
.map(|(date, bucket)| AdminStatsForecastPoint {
|
||||
date,
|
||||
total_cost: bucket.total_cost,
|
||||
})
|
||||
.collect();
|
||||
let values: Vec<f64> = history.iter().map(|item| item.total_cost).collect();
|
||||
let (slope, intercept) = linear_regression(&values);
|
||||
let last_date = history
|
||||
.last()
|
||||
.map(|item| item.date)
|
||||
.unwrap_or(time_range.end_date);
|
||||
let forecast: Vec<_> = (0..forecast_days)
|
||||
.map(|index| {
|
||||
let idx = values.len() + index as usize;
|
||||
let predicted = (slope * idx as f64 + intercept).max(0.0);
|
||||
json!({
|
||||
"date": last_date
|
||||
.checked_add_signed(chrono::Duration::days(i64::from(index + 1)))
|
||||
.unwrap_or(last_date)
|
||||
.to_string(),
|
||||
"total_cost": round_to(predicted, 4),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
return Ok(Some(
|
||||
Json(json!({
|
||||
"history": history.into_iter().map(|item| json!({
|
||||
"date": item.date.to_string(),
|
||||
"total_cost": round_to(item.total_cost, 6),
|
||||
})).collect::<Vec<_>>(),
|
||||
"forecast": forecast,
|
||||
"slope": round_to(slope, 6),
|
||||
"intercept": round_to(intercept, 6),
|
||||
"start_date": time_range.start_date.to_string(),
|
||||
"end_date": time_range.end_date.to_string(),
|
||||
}))
|
||||
.into_response(),
|
||||
));
|
||||
}
|
||||
|
||||
if request_context
|
||||
.control_decision
|
||||
.as_ref()
|
||||
.and_then(|decision| decision.route_kind.as_deref())
|
||||
== Some("cost_savings")
|
||||
&& request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/admin/stats/cost/savings" | "/api/admin/stats/cost/savings/"
|
||||
)
|
||||
{
|
||||
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
if time_range.is_none() {
|
||||
return Ok(Some(admin_stats_cost_savings_empty_response()));
|
||||
}
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_stats_cost_savings_empty_response()));
|
||||
}
|
||||
|
||||
let filters = AdminStatsUsageFilter {
|
||||
user_id: None,
|
||||
provider_name: query_param_value(query, "provider_name"),
|
||||
model: query_param_value(query, "model"),
|
||||
};
|
||||
let usage = list_usage_for_range(
|
||||
state,
|
||||
time_range.as_ref().expect("time range exists"),
|
||||
&filters,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let cache_read_tokens: u64 = usage.iter().map(|item| item.cache_read_input_tokens).sum();
|
||||
let cache_read_cost: f64 = usage.iter().map(|item| item.cache_read_cost_usd).sum();
|
||||
let cache_creation_cost: f64 = usage.iter().map(|item| item.cache_creation_cost_usd).sum();
|
||||
let mut estimated_full_cost: f64 = usage
|
||||
.iter()
|
||||
.map(|item| {
|
||||
item.output_price_per_1m.unwrap_or(0.0) * item.cache_read_input_tokens as f64
|
||||
/ 1_000_000.0
|
||||
})
|
||||
.sum();
|
||||
if estimated_full_cost <= 0.0 && cache_read_cost > 0.0 {
|
||||
estimated_full_cost = cache_read_cost * 10.0;
|
||||
}
|
||||
let cache_savings = estimated_full_cost - cache_read_cost;
|
||||
|
||||
return Ok(Some(
|
||||
Json(json!({
|
||||
"cache_read_tokens": cache_read_tokens,
|
||||
"cache_read_cost": round_to(cache_read_cost, 6),
|
||||
"cache_creation_cost": round_to(cache_creation_cost, 6),
|
||||
"estimated_full_cost": round_to(estimated_full_cost, 6),
|
||||
"cache_savings": round_to(cache_savings, 6),
|
||||
}))
|
||||
.into_response(),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
@@ -0,0 +1,387 @@
|
||||
use super::range::{
|
||||
admin_usage_default_days, parse_naive_date, parse_tz_offset_minutes, resolve_preset_dates,
|
||||
user_today,
|
||||
};
|
||||
use crate::handlers::admin::shared::query_param_value;
|
||||
use aether_data_contracts::repository::usage::StoredRequestUsageAudit;
|
||||
use chrono::Utc;
|
||||
use serde_json::json;
|
||||
|
||||
pub(crate) const MIN_PERCENTILE_SAMPLES: usize = 10;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum AdminStatsComparisonType {
|
||||
Period,
|
||||
Year,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum AdminStatsGranularity {
|
||||
Hour,
|
||||
Day,
|
||||
Week,
|
||||
Month,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct AdminStatsTimeRange {
|
||||
pub(crate) start_date: chrono::NaiveDate,
|
||||
pub(crate) end_date: chrono::NaiveDate,
|
||||
pub(crate) tz_offset_minutes: i32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct AdminStatsUsageFilter {
|
||||
pub(crate) user_id: Option<String>,
|
||||
pub(crate) provider_name: Option<String>,
|
||||
pub(crate) model: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct AdminStatsAggregate {
|
||||
pub(crate) total_requests: u64,
|
||||
pub(crate) total_tokens: u64,
|
||||
pub(crate) total_cost: f64,
|
||||
pub(crate) actual_total_cost: f64,
|
||||
pub(crate) total_response_time_ms: f64,
|
||||
pub(crate) error_requests: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct AdminStatsForecastPoint {
|
||||
pub(crate) date: chrono::NaiveDate,
|
||||
pub(crate) total_cost: f64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum AdminStatsLeaderboardMetric {
|
||||
Requests,
|
||||
Tokens,
|
||||
Cost,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum AdminStatsSortOrder {
|
||||
Asc,
|
||||
Desc,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct AdminStatsLeaderboardItem {
|
||||
pub(crate) id: String,
|
||||
pub(crate) name: String,
|
||||
pub(crate) requests: u64,
|
||||
pub(crate) tokens: u64,
|
||||
pub(crate) cost: f64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct AdminStatsUserMetadata {
|
||||
pub(crate) name: String,
|
||||
pub(crate) role: String,
|
||||
pub(crate) is_active: bool,
|
||||
pub(crate) is_deleted: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct AdminStatsTimeSeriesBucket {
|
||||
pub(crate) total_requests: u64,
|
||||
pub(crate) input_tokens: u64,
|
||||
pub(crate) output_tokens: u64,
|
||||
pub(crate) cache_creation_tokens: u64,
|
||||
pub(crate) cache_read_tokens: u64,
|
||||
pub(crate) total_cost: f64,
|
||||
pub(crate) total_response_time_ms: f64,
|
||||
}
|
||||
|
||||
impl AdminStatsAggregate {
|
||||
pub(crate) fn avg_response_time_ms(&self) -> f64 {
|
||||
if self.total_requests == 0 {
|
||||
0.0
|
||||
} else {
|
||||
self.total_response_time_ms / self.total_requests as f64
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AdminStatsGranularity {
|
||||
pub(crate) fn parse(query: Option<&str>) -> Result<Self, String> {
|
||||
match query_param_value(query, "granularity").as_deref() {
|
||||
None | Some("day") => Ok(Self::Day),
|
||||
Some("hour") => Ok(Self::Hour),
|
||||
Some("week") => Ok(Self::Week),
|
||||
Some("month") => Ok(Self::Month),
|
||||
Some(_) => Err("granularity must be one of: hour, day, week, month".to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AdminStatsLeaderboardMetric {
|
||||
pub(crate) fn parse(query: Option<&str>) -> Result<Self, String> {
|
||||
match query_param_value(query, "metric").as_deref() {
|
||||
None | Some("requests") => Ok(Self::Requests),
|
||||
Some("tokens") => Ok(Self::Tokens),
|
||||
Some("cost") => Ok(Self::Cost),
|
||||
Some(_) => Err("metric must be one of: requests, tokens, cost".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Requests => "requests",
|
||||
Self::Tokens => "tokens",
|
||||
Self::Cost => "cost",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AdminStatsSortOrder {
|
||||
pub(crate) fn parse(query: Option<&str>) -> Result<Self, String> {
|
||||
match query_param_value(query, "order").as_deref() {
|
||||
None | Some("desc") => Ok(Self::Desc),
|
||||
Some("asc") => Ok(Self::Asc),
|
||||
Some(_) => Err("order must be one of: asc, desc".to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AdminStatsUsageFilter {
|
||||
pub(crate) fn from_query(query: Option<&str>) -> Self {
|
||||
Self {
|
||||
user_id: query_param_value(query, "user_id"),
|
||||
provider_name: query_param_value(query, "provider_name"),
|
||||
model: query_param_value(query, "model"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AdminStatsTimeSeriesBucket {
|
||||
pub(crate) fn add_usage(&mut self, item: &StoredRequestUsageAudit) {
|
||||
self.total_requests = self.total_requests.saturating_add(1);
|
||||
self.input_tokens = self.input_tokens.saturating_add(item.input_tokens);
|
||||
self.output_tokens = self.output_tokens.saturating_add(item.output_tokens);
|
||||
self.cache_creation_tokens = self
|
||||
.cache_creation_tokens
|
||||
.saturating_add(item.cache_creation_input_tokens);
|
||||
self.cache_read_tokens = self
|
||||
.cache_read_tokens
|
||||
.saturating_add(item.cache_read_input_tokens);
|
||||
self.total_cost += item.total_cost_usd;
|
||||
self.total_response_time_ms += item.response_time_ms.unwrap_or(0) as f64;
|
||||
}
|
||||
|
||||
pub(crate) fn merge(&mut self, other: &Self) {
|
||||
self.total_requests = self.total_requests.saturating_add(other.total_requests);
|
||||
self.input_tokens = self.input_tokens.saturating_add(other.input_tokens);
|
||||
self.output_tokens = self.output_tokens.saturating_add(other.output_tokens);
|
||||
self.cache_creation_tokens = self
|
||||
.cache_creation_tokens
|
||||
.saturating_add(other.cache_creation_tokens);
|
||||
self.cache_read_tokens = self
|
||||
.cache_read_tokens
|
||||
.saturating_add(other.cache_read_tokens);
|
||||
self.total_cost += other.total_cost;
|
||||
self.total_response_time_ms += other.total_response_time_ms;
|
||||
}
|
||||
|
||||
pub(crate) fn avg_response_time_ms(&self) -> f64 {
|
||||
if self.total_requests == 0 {
|
||||
0.0
|
||||
} else {
|
||||
self.total_response_time_ms / self.total_requests as f64
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn to_json_with_avg(&self, date: String) -> serde_json::Value {
|
||||
json!({
|
||||
"date": date,
|
||||
"total_requests": self.total_requests,
|
||||
"input_tokens": self.input_tokens,
|
||||
"output_tokens": self.output_tokens,
|
||||
"cache_creation_tokens": self.cache_creation_tokens,
|
||||
"cache_read_tokens": self.cache_read_tokens,
|
||||
"total_cost": round_to(self.total_cost, 6),
|
||||
"avg_response_time_ms": round_to(self.avg_response_time_ms(), 2),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn to_json_without_avg(&self, date: String) -> serde_json::Value {
|
||||
json!({
|
||||
"date": date,
|
||||
"total_requests": self.total_requests,
|
||||
"input_tokens": self.input_tokens,
|
||||
"output_tokens": self.output_tokens,
|
||||
"cache_creation_tokens": self.cache_creation_tokens,
|
||||
"cache_read_tokens": self.cache_read_tokens,
|
||||
"total_cost": round_to(self.total_cost, 6),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl AdminStatsTimeRange {
|
||||
pub(crate) fn resolve_optional(query: Option<&str>) -> Result<Option<Self>, String> {
|
||||
let tz_offset_minutes = parse_tz_offset_minutes(query)?;
|
||||
let start_date = query_param_value(query, "start_date")
|
||||
.map(|value| parse_naive_date("start_date", &value))
|
||||
.transpose()?;
|
||||
let end_date = query_param_value(query, "end_date")
|
||||
.map(|value| parse_naive_date("end_date", &value))
|
||||
.transpose()?;
|
||||
let preset = query_param_value(query, "preset");
|
||||
|
||||
if preset.is_none() && start_date.is_none() && end_date.is_none() {
|
||||
let default_days = admin_usage_default_days();
|
||||
if default_days == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
let end_date = user_today(tz_offset_minutes);
|
||||
let start_date = end_date
|
||||
.checked_sub_signed(chrono::Duration::days(
|
||||
i64::try_from(default_days.saturating_sub(1)).unwrap_or(0),
|
||||
))
|
||||
.unwrap_or(end_date);
|
||||
return Ok(Some(Self {
|
||||
start_date,
|
||||
end_date,
|
||||
tz_offset_minutes,
|
||||
}));
|
||||
}
|
||||
|
||||
let (start_date, end_date) = match (preset.as_deref(), start_date, end_date) {
|
||||
(Some(preset), None, None) => resolve_preset_dates(preset, tz_offset_minutes)?,
|
||||
(None, Some(start_date), Some(end_date)) => (start_date, end_date),
|
||||
(Some(_), Some(_), _) | (Some(_), _, Some(_)) => {
|
||||
return Err("preset cannot be combined with start_date or end_date".to_string());
|
||||
}
|
||||
_ => {
|
||||
return Err(
|
||||
"Either preset or both start_date and end_date must be provided".to_string(),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
if start_date > end_date {
|
||||
return Err("start_date must be <= end_date".to_string());
|
||||
}
|
||||
|
||||
let days = (end_date - start_date).num_days();
|
||||
if days > 365 {
|
||||
return Err("Query range cannot exceed 365 days".to_string());
|
||||
}
|
||||
|
||||
Ok(Some(Self {
|
||||
start_date,
|
||||
end_date,
|
||||
tz_offset_minutes,
|
||||
}))
|
||||
}
|
||||
|
||||
pub(crate) fn resolve_required(
|
||||
query: Option<&str>,
|
||||
start_key: &str,
|
||||
end_key: &str,
|
||||
) -> Result<Self, String> {
|
||||
let tz_offset_minutes = parse_tz_offset_minutes(query)?;
|
||||
let start_date = query_param_value(query, start_key)
|
||||
.ok_or_else(|| format!("{start_key} is required"))
|
||||
.and_then(|value| parse_naive_date(start_key, &value))?;
|
||||
let end_date = query_param_value(query, end_key)
|
||||
.ok_or_else(|| format!("{end_key} is required"))
|
||||
.and_then(|value| parse_naive_date(end_key, &value))?;
|
||||
|
||||
if start_date > end_date {
|
||||
return Err(format!("{start_key} must be <= {end_key}"));
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
start_date,
|
||||
end_date,
|
||||
tz_offset_minutes,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn to_unix_bounds(&self) -> Option<(u64, u64)> {
|
||||
let offset = chrono::Duration::minutes(i64::from(self.tz_offset_minutes));
|
||||
let start_local = self.start_date.and_hms_opt(0, 0, 0)?;
|
||||
let end_local = self
|
||||
.end_date
|
||||
.checked_add_signed(chrono::Duration::days(1))?
|
||||
.and_hms_opt(0, 0, 0)?;
|
||||
let start_utc =
|
||||
chrono::DateTime::<Utc>::from_naive_utc_and_offset(start_local - offset, Utc)
|
||||
.timestamp();
|
||||
let end_utc =
|
||||
chrono::DateTime::<Utc>::from_naive_utc_and_offset(end_local - offset, Utc).timestamp();
|
||||
if start_utc < 0 || end_utc <= 0 {
|
||||
return None;
|
||||
}
|
||||
Some((start_utc as u64, end_utc as u64))
|
||||
}
|
||||
|
||||
pub(crate) fn to_utc_datetime_bounds(
|
||||
&self,
|
||||
) -> Option<(chrono::DateTime<Utc>, chrono::DateTime<Utc>)> {
|
||||
let offset = chrono::Duration::minutes(i64::from(self.tz_offset_minutes));
|
||||
let start_local = self.start_date.and_hms_opt(0, 0, 0)?;
|
||||
let end_local = self
|
||||
.end_date
|
||||
.checked_add_signed(chrono::Duration::days(1))?
|
||||
.and_hms_opt(0, 0, 0)?;
|
||||
Some((
|
||||
chrono::DateTime::<Utc>::from_naive_utc_and_offset(start_local - offset, Utc),
|
||||
chrono::DateTime::<Utc>::from_naive_utc_and_offset(end_local - offset, Utc),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn validate_for_time_series(
|
||||
&self,
|
||||
granularity: AdminStatsGranularity,
|
||||
) -> Result<(), String> {
|
||||
if granularity == AdminStatsGranularity::Hour && self.start_date != self.end_date {
|
||||
return Err("Hour granularity only supports single day query".to_string());
|
||||
}
|
||||
let days_inclusive = (self.end_date - self.start_date).num_days() + 1;
|
||||
if days_inclusive > 90 {
|
||||
return Err(format!(
|
||||
"Time series query range cannot exceed 90 days (requested {days_inclusive} days). For longer ranges, use aggregated statistics instead."
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn local_dates(&self) -> Vec<chrono::NaiveDate> {
|
||||
let mut current = self.start_date;
|
||||
let mut dates = Vec::new();
|
||||
while current <= self.end_date {
|
||||
dates.push(current);
|
||||
let Some(next) = current.checked_add_signed(chrono::Duration::days(1)) else {
|
||||
break;
|
||||
};
|
||||
current = next;
|
||||
}
|
||||
dates
|
||||
}
|
||||
|
||||
pub(crate) fn local_date_strings(&self) -> Vec<String> {
|
||||
self.local_dates()
|
||||
.into_iter()
|
||||
.map(|date| date.to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn local_date_for_unix_secs(&self, unix_secs: u64) -> Option<chrono::NaiveDate> {
|
||||
let timestamp = chrono::DateTime::<Utc>::from_timestamp(i64::try_from(unix_secs).ok()?, 0)?;
|
||||
let local = timestamp
|
||||
.checked_add_signed(chrono::Duration::minutes(i64::from(self.tz_offset_minutes)))?;
|
||||
Some(local.date_naive())
|
||||
}
|
||||
|
||||
pub(crate) fn local_date_string_for_unix_secs(&self, unix_secs: u64) -> Option<String> {
|
||||
Some(self.local_date_for_unix_secs(unix_secs)?.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn round_to(value: f64, decimals: u32) -> f64 {
|
||||
let factor = 10_f64.powi(i32::try_from(decimals).unwrap_or(0));
|
||||
(value * factor).round() / factor
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
use super::helpers::{
|
||||
AdminStatsLeaderboardItem, AdminStatsLeaderboardMetric, AdminStatsSortOrder,
|
||||
AdminStatsUserMetadata,
|
||||
};
|
||||
use crate::{AppState, GatewayError};
|
||||
use aether_data_contracts::repository::usage::StoredRequestUsageAudit;
|
||||
|
||||
pub(super) fn build_model_leaderboard_items(
|
||||
items: &[StoredRequestUsageAudit],
|
||||
) -> Vec<AdminStatsLeaderboardItem> {
|
||||
let mut grouped: std::collections::BTreeMap<String, AdminStatsLeaderboardItem> =
|
||||
std::collections::BTreeMap::new();
|
||||
for item in items {
|
||||
if matches!(item.status.as_str(), "pending" | "streaming")
|
||||
|| matches!(item.provider_name.as_str(), "unknown" | "pending")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let entry =
|
||||
grouped
|
||||
.entry(item.model.clone())
|
||||
.or_insert_with(|| AdminStatsLeaderboardItem {
|
||||
id: item.model.clone(),
|
||||
name: item.model.clone(),
|
||||
requests: 0,
|
||||
tokens: 0,
|
||||
cost: 0.0,
|
||||
});
|
||||
entry.requests = entry.requests.saturating_add(1);
|
||||
entry.tokens = entry.tokens.saturating_add(
|
||||
item.input_tokens
|
||||
.saturating_add(item.output_tokens)
|
||||
.saturating_add(item.cache_creation_input_tokens)
|
||||
.saturating_add(item.cache_read_input_tokens),
|
||||
);
|
||||
entry.cost += item.total_cost_usd;
|
||||
}
|
||||
grouped.into_values().collect()
|
||||
}
|
||||
|
||||
pub(super) fn build_api_key_leaderboard_items(
|
||||
items: &[StoredRequestUsageAudit],
|
||||
snapshots: Option<&[aether_data::repository::auth::StoredAuthApiKeySnapshot]>,
|
||||
include_inactive: bool,
|
||||
exclude_admin: bool,
|
||||
) -> Vec<AdminStatsLeaderboardItem> {
|
||||
let snapshot_by_api_key_id: std::collections::BTreeMap<_, _> = snapshots
|
||||
.unwrap_or(&[])
|
||||
.iter()
|
||||
.map(|snapshot| (snapshot.api_key_id.as_str(), snapshot))
|
||||
.collect();
|
||||
let mut grouped: std::collections::BTreeMap<String, AdminStatsLeaderboardItem> =
|
||||
std::collections::BTreeMap::new();
|
||||
let snapshots_available = snapshots.is_some();
|
||||
|
||||
for item in items {
|
||||
if matches!(item.status.as_str(), "pending" | "streaming")
|
||||
|| matches!(item.provider_name.as_str(), "unknown" | "pending")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let Some(api_key_id) = item.api_key_id.as_deref() else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let entry_name = if let Some(snapshot) = snapshot_by_api_key_id.get(api_key_id) {
|
||||
if snapshot.user_is_deleted {
|
||||
continue;
|
||||
}
|
||||
if !include_inactive && !snapshot.api_key_is_active {
|
||||
continue;
|
||||
}
|
||||
if exclude_admin && snapshot.user_role.eq_ignore_ascii_case("admin") {
|
||||
continue;
|
||||
}
|
||||
snapshot
|
||||
.api_key_name
|
||||
.clone()
|
||||
.or_else(|| item.api_key_name.clone())
|
||||
.unwrap_or_else(|| api_key_id.to_string())
|
||||
} else {
|
||||
if snapshots_available {
|
||||
continue;
|
||||
}
|
||||
item.api_key_name
|
||||
.clone()
|
||||
.unwrap_or_else(|| api_key_id.to_string())
|
||||
};
|
||||
|
||||
let entry =
|
||||
grouped
|
||||
.entry(api_key_id.to_string())
|
||||
.or_insert_with(|| AdminStatsLeaderboardItem {
|
||||
id: api_key_id.to_string(),
|
||||
name: entry_name,
|
||||
requests: 0,
|
||||
tokens: 0,
|
||||
cost: 0.0,
|
||||
});
|
||||
entry.requests = entry.requests.saturating_add(1);
|
||||
entry.tokens = entry.tokens.saturating_add(
|
||||
item.input_tokens
|
||||
.saturating_add(item.output_tokens)
|
||||
.saturating_add(item.cache_creation_input_tokens)
|
||||
.saturating_add(item.cache_read_input_tokens),
|
||||
);
|
||||
entry.cost += item.total_cost_usd;
|
||||
}
|
||||
|
||||
grouped.into_values().collect()
|
||||
}
|
||||
|
||||
pub(super) fn build_user_leaderboard_items(
|
||||
items: &[StoredRequestUsageAudit],
|
||||
users: &std::collections::BTreeMap<String, AdminStatsUserMetadata>,
|
||||
include_inactive: bool,
|
||||
exclude_admin: bool,
|
||||
) -> Vec<AdminStatsLeaderboardItem> {
|
||||
let mut grouped: std::collections::BTreeMap<String, AdminStatsLeaderboardItem> =
|
||||
std::collections::BTreeMap::new();
|
||||
|
||||
for item in items {
|
||||
if matches!(item.status.as_str(), "pending" | "streaming")
|
||||
|| matches!(item.provider_name.as_str(), "unknown" | "pending")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let Some(user_id) = item.user_id.as_deref() else {
|
||||
continue;
|
||||
};
|
||||
let entry_name = if let Some(user) = users.get(user_id) {
|
||||
if user.is_deleted {
|
||||
continue;
|
||||
}
|
||||
if !include_inactive && !user.is_active {
|
||||
continue;
|
||||
}
|
||||
if exclude_admin && user.role.eq_ignore_ascii_case("admin") {
|
||||
continue;
|
||||
}
|
||||
user.name.clone()
|
||||
} else {
|
||||
if exclude_admin {
|
||||
continue;
|
||||
}
|
||||
item.username
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
.unwrap_or_else(|| user_id.to_string())
|
||||
};
|
||||
|
||||
let entry =
|
||||
grouped
|
||||
.entry(user_id.to_string())
|
||||
.or_insert_with(|| AdminStatsLeaderboardItem {
|
||||
id: user_id.to_string(),
|
||||
name: entry_name,
|
||||
requests: 0,
|
||||
tokens: 0,
|
||||
cost: 0.0,
|
||||
});
|
||||
entry.requests = entry.requests.saturating_add(1);
|
||||
entry.tokens = entry.tokens.saturating_add(
|
||||
item.input_tokens
|
||||
.saturating_add(item.output_tokens)
|
||||
.saturating_add(item.cache_creation_input_tokens)
|
||||
.saturating_add(item.cache_read_input_tokens),
|
||||
);
|
||||
entry.cost += item.total_cost_usd;
|
||||
}
|
||||
|
||||
grouped.into_values().collect()
|
||||
}
|
||||
|
||||
pub(super) async fn load_user_leaderboard_metadata(
|
||||
state: &AppState,
|
||||
user_ids: &[String],
|
||||
) -> Result<std::collections::BTreeMap<String, AdminStatsUserMetadata>, GatewayError> {
|
||||
let mut metadata = std::collections::BTreeMap::new();
|
||||
|
||||
if state.has_user_data_reader() {
|
||||
for user in state.list_users_by_ids(user_ids).await? {
|
||||
let name = if !user.username.trim().is_empty() {
|
||||
user.username
|
||||
} else {
|
||||
user.email.unwrap_or(user.id.clone())
|
||||
};
|
||||
metadata.insert(
|
||||
user.id,
|
||||
AdminStatsUserMetadata {
|
||||
name,
|
||||
role: user.role,
|
||||
is_active: user.is_active,
|
||||
is_deleted: user.is_deleted,
|
||||
},
|
||||
);
|
||||
}
|
||||
return Ok(metadata);
|
||||
}
|
||||
|
||||
for user_id in user_ids {
|
||||
let Some(user) = state.find_user_auth_by_id(user_id).await? else {
|
||||
continue;
|
||||
};
|
||||
let name = if !user.username.trim().is_empty() {
|
||||
user.username.clone()
|
||||
} else {
|
||||
user.email.clone().unwrap_or_else(|| user.id.clone())
|
||||
};
|
||||
metadata.insert(
|
||||
user.id.clone(),
|
||||
AdminStatsUserMetadata {
|
||||
name,
|
||||
role: user.role.clone(),
|
||||
is_active: user.is_active,
|
||||
is_deleted: user.is_deleted,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
pub(super) fn compare_leaderboard_items(
|
||||
metric: AdminStatsLeaderboardMetric,
|
||||
order: AdminStatsSortOrder,
|
||||
left: &AdminStatsLeaderboardItem,
|
||||
right: &AdminStatsLeaderboardItem,
|
||||
) -> std::cmp::Ordering {
|
||||
let metric_order = match metric {
|
||||
AdminStatsLeaderboardMetric::Requests => left.requests.cmp(&right.requests),
|
||||
AdminStatsLeaderboardMetric::Tokens => left.tokens.cmp(&right.tokens),
|
||||
AdminStatsLeaderboardMetric::Cost => left
|
||||
.cost
|
||||
.partial_cmp(&right.cost)
|
||||
.unwrap_or(std::cmp::Ordering::Equal),
|
||||
};
|
||||
let metric_order = match order {
|
||||
AdminStatsSortOrder::Asc => metric_order,
|
||||
AdminStatsSortOrder::Desc => metric_order.reverse(),
|
||||
};
|
||||
if metric_order == std::cmp::Ordering::Equal {
|
||||
left.id.cmp(&right.id)
|
||||
} else {
|
||||
metric_order
|
||||
}
|
||||
}
|
||||
|
||||
fn leaderboard_metric_equal(
|
||||
metric: AdminStatsLeaderboardMetric,
|
||||
left: &AdminStatsLeaderboardItem,
|
||||
right: &AdminStatsLeaderboardItem,
|
||||
) -> bool {
|
||||
match metric {
|
||||
AdminStatsLeaderboardMetric::Requests => left.requests == right.requests,
|
||||
AdminStatsLeaderboardMetric::Tokens => left.tokens == right.tokens,
|
||||
AdminStatsLeaderboardMetric::Cost => (left.cost - right.cost).abs() < 1e-9,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn compute_dense_rank(
|
||||
metric: AdminStatsLeaderboardMetric,
|
||||
items: &[AdminStatsLeaderboardItem],
|
||||
index: usize,
|
||||
) -> usize {
|
||||
if index == 0 {
|
||||
return 1;
|
||||
}
|
||||
let mut rank = 1usize;
|
||||
for current in 1..=index {
|
||||
if !leaderboard_metric_equal(metric, &items[current - 1], &items[current]) {
|
||||
rank = rank.saturating_add(1);
|
||||
}
|
||||
}
|
||||
rank
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
use super::helpers::{
|
||||
round_to, AdminStatsLeaderboardMetric, AdminStatsSortOrder, AdminStatsTimeRange,
|
||||
AdminStatsUsageFilter,
|
||||
};
|
||||
use super::leaderboard::{
|
||||
build_api_key_leaderboard_items, build_model_leaderboard_items, build_user_leaderboard_items,
|
||||
compare_leaderboard_items, compute_dense_rank, load_user_leaderboard_metadata,
|
||||
};
|
||||
use super::range::{list_usage_for_optional_range, parse_bounded_u32, parse_nonnegative_usize};
|
||||
use super::responses::{admin_stats_bad_request_response, admin_stats_leaderboard_empty_response};
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use crate::handlers::admin::shared::{query_param_bool, query_param_value};
|
||||
use crate::{AppState, GatewayError};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
let query = request_context.request_query_string.as_deref();
|
||||
|
||||
if request_context
|
||||
.control_decision
|
||||
.as_ref()
|
||||
.and_then(|decision| decision.route_kind.as_deref())
|
||||
== Some("leaderboard_models")
|
||||
&& request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/admin/stats/leaderboard/models" | "/api/admin/stats/leaderboard/models/"
|
||||
)
|
||||
{
|
||||
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let metric = match AdminStatsLeaderboardMetric::parse(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let order = match AdminStatsSortOrder::parse(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let limit = match query_param_value(query, "limit")
|
||||
.map(|value| parse_bounded_u32("limit", &value, 1, 100))
|
||||
.transpose()
|
||||
{
|
||||
Ok(Some(value)) => value as usize,
|
||||
Ok(None) => 10,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let offset = match query_param_value(query, "offset")
|
||||
.map(|value| parse_nonnegative_usize("offset", &value))
|
||||
.transpose()
|
||||
{
|
||||
Ok(Some(value)) => value,
|
||||
Ok(None) => 0,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_stats_leaderboard_empty_response(
|
||||
metric,
|
||||
time_range.as_ref(),
|
||||
)));
|
||||
}
|
||||
let filters = AdminStatsUsageFilter::from_query(query);
|
||||
let usage = list_usage_for_optional_range(state, time_range.as_ref(), &filters).await?;
|
||||
let mut leaderboard = build_model_leaderboard_items(&usage);
|
||||
leaderboard.sort_by(|left, right| compare_leaderboard_items(metric, order, left, right));
|
||||
|
||||
let total = leaderboard.len();
|
||||
let items: Vec<_> = leaderboard
|
||||
.iter()
|
||||
.enumerate()
|
||||
.skip(offset)
|
||||
.take(limit)
|
||||
.map(|(index, item)| {
|
||||
let rank = compute_dense_rank(metric, &leaderboard, index);
|
||||
let value = match metric {
|
||||
AdminStatsLeaderboardMetric::Requests => json!(item.requests),
|
||||
AdminStatsLeaderboardMetric::Tokens => json!(item.tokens),
|
||||
AdminStatsLeaderboardMetric::Cost => json!(round_to(item.cost, 6)),
|
||||
};
|
||||
json!({
|
||||
"rank": rank,
|
||||
"id": item.id,
|
||||
"name": item.id,
|
||||
"value": value,
|
||||
"requests": item.requests,
|
||||
"tokens": item.tokens,
|
||||
"cost": round_to(item.cost, 6),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
return Ok(Some(
|
||||
Json(json!({
|
||||
"items": items,
|
||||
"total": total,
|
||||
"metric": metric.as_str(),
|
||||
"start_date": time_range.as_ref().map(|value| value.start_date.to_string()),
|
||||
"end_date": time_range.as_ref().map(|value| value.end_date.to_string()),
|
||||
}))
|
||||
.into_response(),
|
||||
));
|
||||
}
|
||||
|
||||
if request_context
|
||||
.control_decision
|
||||
.as_ref()
|
||||
.and_then(|decision| decision.route_kind.as_deref())
|
||||
== Some("leaderboard_api_keys")
|
||||
&& request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/admin/stats/leaderboard/api-keys" | "/api/admin/stats/leaderboard/api-keys/"
|
||||
)
|
||||
{
|
||||
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let metric = match AdminStatsLeaderboardMetric::parse(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let order = match AdminStatsSortOrder::parse(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let limit = match query_param_value(query, "limit")
|
||||
.map(|value| parse_bounded_u32("limit", &value, 1, 100))
|
||||
.transpose()
|
||||
{
|
||||
Ok(Some(value)) => value as usize,
|
||||
Ok(None) => 10,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let offset = match query_param_value(query, "offset")
|
||||
.map(|value| parse_nonnegative_usize("offset", &value))
|
||||
.transpose()
|
||||
{
|
||||
Ok(Some(value)) => value,
|
||||
Ok(None) => 0,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_stats_leaderboard_empty_response(
|
||||
metric,
|
||||
time_range.as_ref(),
|
||||
)));
|
||||
}
|
||||
let include_inactive = query_param_bool(query, "include_inactive", false);
|
||||
let exclude_admin = query_param_bool(query, "exclude_admin", false);
|
||||
let filters = AdminStatsUsageFilter::from_query(query);
|
||||
let usage = list_usage_for_optional_range(state, time_range.as_ref(), &filters).await?;
|
||||
let snapshots = if state.has_auth_api_key_data_reader() {
|
||||
let api_key_ids: Vec<String> = usage
|
||||
.iter()
|
||||
.filter_map(|item| item.api_key_id.clone())
|
||||
.collect::<std::collections::BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect();
|
||||
Some(
|
||||
state
|
||||
.data
|
||||
.list_auth_api_key_snapshots_by_ids(&api_key_ids)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut leaderboard = build_api_key_leaderboard_items(
|
||||
&usage,
|
||||
snapshots.as_deref(),
|
||||
include_inactive,
|
||||
exclude_admin,
|
||||
);
|
||||
leaderboard.sort_by(|left, right| compare_leaderboard_items(metric, order, left, right));
|
||||
|
||||
let total = leaderboard.len();
|
||||
let items: Vec<_> = leaderboard
|
||||
.iter()
|
||||
.enumerate()
|
||||
.skip(offset)
|
||||
.take(limit)
|
||||
.map(|(index, item)| {
|
||||
let rank = compute_dense_rank(metric, &leaderboard, index);
|
||||
let value = match metric {
|
||||
AdminStatsLeaderboardMetric::Requests => json!(item.requests),
|
||||
AdminStatsLeaderboardMetric::Tokens => json!(item.tokens),
|
||||
AdminStatsLeaderboardMetric::Cost => json!(round_to(item.cost, 6)),
|
||||
};
|
||||
json!({
|
||||
"rank": rank,
|
||||
"id": item.id,
|
||||
"name": item.name,
|
||||
"value": value,
|
||||
"requests": item.requests,
|
||||
"tokens": item.tokens,
|
||||
"cost": round_to(item.cost, 6),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
return Ok(Some(
|
||||
Json(json!({
|
||||
"items": items,
|
||||
"total": total,
|
||||
"metric": metric.as_str(),
|
||||
"start_date": time_range.as_ref().map(|value| value.start_date.to_string()),
|
||||
"end_date": time_range.as_ref().map(|value| value.end_date.to_string()),
|
||||
}))
|
||||
.into_response(),
|
||||
));
|
||||
}
|
||||
|
||||
if request_context
|
||||
.control_decision
|
||||
.as_ref()
|
||||
.and_then(|decision| decision.route_kind.as_deref())
|
||||
== Some("leaderboard_users")
|
||||
&& request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/admin/stats/leaderboard/users" | "/api/admin/stats/leaderboard/users/"
|
||||
)
|
||||
{
|
||||
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let metric = match AdminStatsLeaderboardMetric::parse(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let order = match AdminStatsSortOrder::parse(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let limit = match query_param_value(query, "limit")
|
||||
.map(|value| parse_bounded_u32("limit", &value, 1, 100))
|
||||
.transpose()
|
||||
{
|
||||
Ok(Some(value)) => value as usize,
|
||||
Ok(None) => 10,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let offset = match query_param_value(query, "offset")
|
||||
.map(|value| parse_nonnegative_usize("offset", &value))
|
||||
.transpose()
|
||||
{
|
||||
Ok(Some(value)) => value,
|
||||
Ok(None) => 0,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_stats_leaderboard_empty_response(
|
||||
metric,
|
||||
time_range.as_ref(),
|
||||
)));
|
||||
}
|
||||
let include_inactive = query_param_bool(query, "include_inactive", false);
|
||||
let exclude_admin = query_param_bool(query, "exclude_admin", false);
|
||||
let filters = AdminStatsUsageFilter::from_query(query);
|
||||
let usage = list_usage_for_optional_range(state, time_range.as_ref(), &filters).await?;
|
||||
let user_ids: Vec<String> = usage
|
||||
.iter()
|
||||
.filter_map(|item| item.user_id.clone())
|
||||
.collect::<std::collections::BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect();
|
||||
let user_metadata = load_user_leaderboard_metadata(state, &user_ids).await?;
|
||||
let mut leaderboard =
|
||||
build_user_leaderboard_items(&usage, &user_metadata, include_inactive, exclude_admin);
|
||||
leaderboard.sort_by(|left, right| compare_leaderboard_items(metric, order, left, right));
|
||||
|
||||
let total = leaderboard.len();
|
||||
let items: Vec<_> = leaderboard
|
||||
.iter()
|
||||
.enumerate()
|
||||
.skip(offset)
|
||||
.take(limit)
|
||||
.map(|(index, item)| {
|
||||
let rank = compute_dense_rank(metric, &leaderboard, index);
|
||||
let value = match metric {
|
||||
AdminStatsLeaderboardMetric::Requests => json!(item.requests),
|
||||
AdminStatsLeaderboardMetric::Tokens => json!(item.tokens),
|
||||
AdminStatsLeaderboardMetric::Cost => json!(round_to(item.cost, 6)),
|
||||
};
|
||||
json!({
|
||||
"rank": rank,
|
||||
"id": item.id,
|
||||
"name": item.name,
|
||||
"value": value,
|
||||
"requests": item.requests,
|
||||
"tokens": item.tokens,
|
||||
"cost": round_to(item.cost, 6),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
return Ok(Some(
|
||||
Json(json!({
|
||||
"items": items,
|
||||
"total": total,
|
||||
"metric": metric.as_str(),
|
||||
"start_date": time_range.as_ref().map(|value| value.start_date.to_string()),
|
||||
"end_date": time_range.as_ref().map(|value| value.end_date.to_string()),
|
||||
}))
|
||||
.into_response(),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use crate::{AppState, GatewayError};
|
||||
use axum::{body::Body, response::Response};
|
||||
|
||||
mod analytics_routes;
|
||||
mod cost_routes;
|
||||
mod helpers;
|
||||
mod leaderboard;
|
||||
mod leaderboard_routes;
|
||||
mod provider_quota_routes;
|
||||
mod range;
|
||||
mod responses;
|
||||
mod timeseries;
|
||||
pub(crate) use self::helpers::{round_to, AdminStatsTimeRange, AdminStatsUsageFilter};
|
||||
pub(crate) use self::range::{list_usage_for_optional_range, parse_bounded_u32};
|
||||
pub(crate) use self::responses::admin_stats_bad_request_response;
|
||||
pub(crate) use self::timeseries::aggregate_usage_stats;
|
||||
|
||||
pub(crate) async fn maybe_build_local_admin_stats_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
let Some(decision) = request_context.control_decision.as_ref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
if decision.route_family.as_deref() != Some("stats_manage") {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if let Some(response) =
|
||||
provider_quota_routes::maybe_build_local_admin_stats_provider_quota_response(
|
||||
state,
|
||||
request_context,
|
||||
decision,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
return Ok(Some(response));
|
||||
}
|
||||
|
||||
if let Some(response) = analytics_routes::maybe_build_local_admin_stats_analytics_response(
|
||||
state,
|
||||
request_context,
|
||||
decision,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
return Ok(Some(response));
|
||||
}
|
||||
|
||||
if let Some(response) =
|
||||
cost_routes::maybe_build_local_admin_stats_cost_response(state, request_context).await?
|
||||
{
|
||||
return Ok(Some(response));
|
||||
}
|
||||
|
||||
if let Some(response) = leaderboard_routes::maybe_build_local_admin_stats_leaderboard_response(
|
||||
state,
|
||||
request_context,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
return Ok(Some(response));
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
use super::responses::admin_stats_provider_quota_usage_empty_response;
|
||||
use crate::control::GatewayControlDecision;
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use crate::handlers::admin::shared::unix_secs_to_rfc3339;
|
||||
use crate::{AppState, GatewayError};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use chrono::{Datelike, Utc};
|
||||
use serde_json::json;
|
||||
|
||||
pub(super) async fn maybe_build_local_admin_stats_provider_quota_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
decision: &GatewayControlDecision,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
if decision.route_kind.as_deref() != Some("provider_quota_usage")
|
||||
|| request_context.request_method != http::Method::GET
|
||||
|| !matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/admin/stats/providers/quota-usage" | "/api/admin/stats/providers/quota-usage/"
|
||||
)
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if !state.has_provider_catalog_data_reader() {
|
||||
return Ok(Some(admin_stats_provider_quota_usage_empty_response()));
|
||||
}
|
||||
|
||||
let now = Utc::now();
|
||||
let now_unix_secs = now.timestamp().max(0) as u64;
|
||||
let now_day = u64::from(now.day());
|
||||
let mut providers = state.list_provider_catalog_providers(false).await?;
|
||||
providers.retain(|provider| {
|
||||
provider.billing_type.as_deref() == Some("monthly_quota")
|
||||
|| provider.monthly_quota_usd.is_some()
|
||||
});
|
||||
|
||||
let mut payload: Vec<serde_json::Value> = providers
|
||||
.into_iter()
|
||||
.map(|provider| {
|
||||
let quota = provider.monthly_quota_usd.unwrap_or(0.0);
|
||||
let used = provider.monthly_used_usd.unwrap_or(0.0);
|
||||
let remaining = (quota - used).max(0.0);
|
||||
let usage_percent = if quota > 0.0 {
|
||||
((used / quota) * 10_000.0).round() / 100.0
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
let days_elapsed = provider
|
||||
.quota_last_reset_at_unix_secs
|
||||
.map(|reset_at| ((now_unix_secs.saturating_sub(reset_at)) / 86_400).max(1))
|
||||
.unwrap_or_else(|| now_day.saturating_sub(1).max(1));
|
||||
|
||||
let daily_rate = if used > 0.0 {
|
||||
used / days_elapsed as f64
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let estimated_exhaust_at_unix_secs = if daily_rate > 0.0 && remaining > 0.0 {
|
||||
let estimated = now_unix_secs
|
||||
.saturating_add(((remaining / daily_rate) * 86_400.0).max(0.0) as u64);
|
||||
Some(
|
||||
provider
|
||||
.quota_expires_at_unix_secs
|
||||
.map(|quota_expires_at| quota_expires_at.min(estimated))
|
||||
.unwrap_or(estimated),
|
||||
)
|
||||
} else {
|
||||
provider.quota_expires_at_unix_secs
|
||||
};
|
||||
|
||||
json!({
|
||||
"id": provider.id,
|
||||
"name": provider.name,
|
||||
"quota_usd": quota,
|
||||
"used_usd": used,
|
||||
"remaining_usd": remaining,
|
||||
"usage_percent": usage_percent,
|
||||
"quota_expires_at": provider.quota_expires_at_unix_secs.and_then(unix_secs_to_rfc3339),
|
||||
"estimated_exhaust_at": estimated_exhaust_at_unix_secs.and_then(unix_secs_to_rfc3339),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
payload.sort_by(|left, right| {
|
||||
let left_value = left
|
||||
.get("usage_percent")
|
||||
.and_then(serde_json::Value::as_f64)
|
||||
.unwrap_or(0.0);
|
||||
let right_value = right
|
||||
.get("usage_percent")
|
||||
.and_then(serde_json::Value::as_f64)
|
||||
.unwrap_or(0.0);
|
||||
right_value.total_cmp(&left_value)
|
||||
});
|
||||
|
||||
Ok(Some(Json(json!({ "providers": payload })).into_response()))
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
use super::helpers::{AdminStatsComparisonType, AdminStatsTimeRange, AdminStatsUsageFilter};
|
||||
use crate::handlers::admin::shared::query_param_value;
|
||||
use crate::{AppState, GatewayError};
|
||||
use aether_data_contracts::repository::usage::{StoredRequestUsageAudit, UsageAuditListQuery};
|
||||
use chrono::{Datelike, Utc};
|
||||
|
||||
pub(super) fn parse_tz_offset_minutes(query: Option<&str>) -> Result<i32, String> {
|
||||
query_param_value(query, "tz_offset_minutes")
|
||||
.map(|value| {
|
||||
value
|
||||
.parse::<i32>()
|
||||
.map_err(|_| "tz_offset_minutes must be a valid integer".to_string())
|
||||
})
|
||||
.transpose()
|
||||
.map(|value| value.unwrap_or(0))
|
||||
}
|
||||
|
||||
pub(super) fn parse_naive_date(field: &str, value: &str) -> Result<chrono::NaiveDate, String> {
|
||||
chrono::NaiveDate::parse_from_str(value, "%Y-%m-%d")
|
||||
.map_err(|_| format!("{field} must be a valid date in YYYY-MM-DD format"))
|
||||
}
|
||||
|
||||
pub(crate) fn parse_bounded_u32(
|
||||
field: &str,
|
||||
value: &str,
|
||||
min: u32,
|
||||
max: u32,
|
||||
) -> Result<u32, String> {
|
||||
let parsed = value
|
||||
.parse::<u32>()
|
||||
.map_err(|_| format!("{field} must be a valid integer"))?;
|
||||
if parsed < min || parsed > max {
|
||||
return Err(format!("{field} must be between {min} and {max}"));
|
||||
}
|
||||
Ok(parsed)
|
||||
}
|
||||
|
||||
pub(super) fn parse_nonnegative_usize(field: &str, value: &str) -> Result<usize, String> {
|
||||
value
|
||||
.parse::<usize>()
|
||||
.map_err(|_| format!("{field} must be a valid integer"))
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_default_days() -> usize {
|
||||
match std::env::var("ADMIN_USAGE_DEFAULT_DAYS") {
|
||||
Ok(value) => value.parse::<usize>().ok().unwrap_or(0),
|
||||
Err(_) => match std::env::var("ENVIRONMENT") {
|
||||
Ok(value) if !matches!(value.as_str(), "development" | "test" | "testing") => 30,
|
||||
_ => 0,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn user_today(tz_offset_minutes: i32) -> chrono::NaiveDate {
|
||||
(Utc::now() + chrono::Duration::minutes(i64::from(tz_offset_minutes))).date_naive()
|
||||
}
|
||||
|
||||
pub(super) fn resolve_preset_dates(
|
||||
preset: &str,
|
||||
tz_offset_minutes: i32,
|
||||
) -> Result<(chrono::NaiveDate, chrono::NaiveDate), String> {
|
||||
let user_today = user_today(tz_offset_minutes);
|
||||
match preset {
|
||||
"today" => Ok((user_today, user_today)),
|
||||
"yesterday" => {
|
||||
let value = user_today
|
||||
.checked_sub_signed(chrono::Duration::days(1))
|
||||
.unwrap_or(user_today);
|
||||
Ok((value, value))
|
||||
}
|
||||
"last7days" => Ok((
|
||||
user_today
|
||||
.checked_sub_signed(chrono::Duration::days(6))
|
||||
.unwrap_or(user_today),
|
||||
user_today,
|
||||
)),
|
||||
"last30days" => Ok((
|
||||
user_today
|
||||
.checked_sub_signed(chrono::Duration::days(29))
|
||||
.unwrap_or(user_today),
|
||||
user_today,
|
||||
)),
|
||||
"last90days" => Ok((
|
||||
user_today
|
||||
.checked_sub_signed(chrono::Duration::days(89))
|
||||
.unwrap_or(user_today),
|
||||
user_today,
|
||||
)),
|
||||
"this_week" => {
|
||||
let week_start = user_today
|
||||
.checked_sub_signed(chrono::Duration::days(i64::from(
|
||||
user_today.weekday().num_days_from_monday(),
|
||||
)))
|
||||
.unwrap_or(user_today);
|
||||
Ok((week_start, user_today))
|
||||
}
|
||||
"last_week" => {
|
||||
let this_week_start = user_today
|
||||
.checked_sub_signed(chrono::Duration::days(i64::from(
|
||||
user_today.weekday().num_days_from_monday(),
|
||||
)))
|
||||
.unwrap_or(user_today);
|
||||
let last_week_end = this_week_start
|
||||
.checked_sub_signed(chrono::Duration::days(1))
|
||||
.unwrap_or(this_week_start);
|
||||
let last_week_start = last_week_end
|
||||
.checked_sub_signed(chrono::Duration::days(6))
|
||||
.unwrap_or(last_week_end);
|
||||
Ok((last_week_start, last_week_end))
|
||||
}
|
||||
"this_month" => {
|
||||
let start = user_today.with_day(1).unwrap_or(user_today);
|
||||
Ok((start, user_today))
|
||||
}
|
||||
"last_month" => {
|
||||
let first_of_this_month = user_today.with_day(1).unwrap_or(user_today);
|
||||
let last_month_end = first_of_this_month
|
||||
.checked_sub_signed(chrono::Duration::days(1))
|
||||
.unwrap_or(first_of_this_month);
|
||||
let last_month_start = last_month_end.with_day(1).unwrap_or(last_month_end);
|
||||
Ok((last_month_start, last_month_end))
|
||||
}
|
||||
"this_year" => {
|
||||
let start =
|
||||
chrono::NaiveDate::from_ymd_opt(user_today.year(), 1, 1).unwrap_or(user_today);
|
||||
Ok((start, user_today))
|
||||
}
|
||||
_ => Err("Invalid preset".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn build_time_range_from_days(
|
||||
days: u32,
|
||||
tz_offset_minutes: i32,
|
||||
) -> Result<AdminStatsTimeRange, String> {
|
||||
let end_date = user_today(tz_offset_minutes);
|
||||
let start_date = end_date
|
||||
.checked_sub_signed(chrono::Duration::days(i64::from(days.saturating_sub(1))))
|
||||
.unwrap_or(end_date);
|
||||
Ok(AdminStatsTimeRange {
|
||||
start_date,
|
||||
end_date,
|
||||
tz_offset_minutes,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn build_comparison_range(
|
||||
current: &AdminStatsTimeRange,
|
||||
comparison_type: AdminStatsComparisonType,
|
||||
) -> Result<AdminStatsTimeRange, String> {
|
||||
let comparison = match comparison_type {
|
||||
AdminStatsComparisonType::Period => {
|
||||
let days = (current.end_date - current.start_date).num_days() + 1;
|
||||
let comparison_end = current
|
||||
.start_date
|
||||
.checked_sub_signed(chrono::Duration::days(1))
|
||||
.ok_or_else(|| "comparison range underflow".to_string())?;
|
||||
let comparison_start = comparison_end
|
||||
.checked_sub_signed(chrono::Duration::days(days - 1))
|
||||
.ok_or_else(|| "comparison range underflow".to_string())?;
|
||||
(comparison_start, comparison_end)
|
||||
}
|
||||
AdminStatsComparisonType::Year => (
|
||||
safe_year_shift(current.start_date),
|
||||
safe_year_shift(current.end_date),
|
||||
),
|
||||
};
|
||||
|
||||
Ok(AdminStatsTimeRange {
|
||||
start_date: comparison.0,
|
||||
end_date: comparison.1,
|
||||
tz_offset_minutes: current.tz_offset_minutes,
|
||||
})
|
||||
}
|
||||
|
||||
fn safe_year_shift(value: chrono::NaiveDate) -> chrono::NaiveDate {
|
||||
value
|
||||
.with_year(value.year() - 1)
|
||||
.or_else(|| chrono::NaiveDate::from_ymd_opt(value.year() - 1, value.month(), 28))
|
||||
.unwrap_or(value)
|
||||
}
|
||||
|
||||
pub(super) async fn list_usage_for_range(
|
||||
state: &AppState,
|
||||
time_range: &AdminStatsTimeRange,
|
||||
filters: &AdminStatsUsageFilter,
|
||||
) -> Result<Vec<StoredRequestUsageAudit>, GatewayError> {
|
||||
let Some((created_from_unix_secs, created_until_unix_secs)) = time_range.to_unix_bounds()
|
||||
else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
|
||||
state
|
||||
.list_usage_audits(&UsageAuditListQuery {
|
||||
created_from_unix_secs: Some(created_from_unix_secs),
|
||||
created_until_unix_secs: Some(created_until_unix_secs),
|
||||
user_id: filters.user_id.clone(),
|
||||
provider_name: filters.provider_name.clone(),
|
||||
model: filters.model.clone(),
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn list_usage_for_optional_range(
|
||||
state: &AppState,
|
||||
time_range: Option<&AdminStatsTimeRange>,
|
||||
filters: &AdminStatsUsageFilter,
|
||||
) -> Result<Vec<StoredRequestUsageAudit>, GatewayError> {
|
||||
match time_range {
|
||||
Some(time_range) => list_usage_for_range(state, time_range, filters).await,
|
||||
None => {
|
||||
state
|
||||
.list_usage_audits(&UsageAuditListQuery {
|
||||
created_from_unix_secs: None,
|
||||
created_until_unix_secs: None,
|
||||
user_id: filters.user_id.clone(),
|
||||
provider_name: filters.provider_name.clone(),
|
||||
model: filters.model.clone(),
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
use super::helpers::{AdminStatsLeaderboardMetric, AdminStatsTimeRange};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
pub(super) fn admin_stats_provider_quota_usage_empty_response() -> Response<Body> {
|
||||
Json(json!({
|
||||
"providers": [],
|
||||
"data_source_available": false,
|
||||
}))
|
||||
.into_response()
|
||||
}
|
||||
|
||||
pub(super) fn admin_stats_cost_forecast_empty_response() -> Response<Body> {
|
||||
Json(json!({
|
||||
"history": [],
|
||||
"forecast": [],
|
||||
"slope": 0.0,
|
||||
"intercept": 0.0,
|
||||
"data_source_available": false,
|
||||
}))
|
||||
.into_response()
|
||||
}
|
||||
|
||||
pub(super) fn admin_stats_comparison_empty_response(
|
||||
current_range: &AdminStatsTimeRange,
|
||||
comparison_range: &AdminStatsTimeRange,
|
||||
) -> Response<Body> {
|
||||
Json(json!({
|
||||
"current": {
|
||||
"total_requests": 0,
|
||||
"total_tokens": 0,
|
||||
"total_cost": 0.0,
|
||||
"actual_total_cost": 0.0,
|
||||
"avg_response_time_ms": 0.0,
|
||||
"error_requests": 0,
|
||||
},
|
||||
"comparison": {
|
||||
"total_requests": 0,
|
||||
"total_tokens": 0,
|
||||
"total_cost": 0.0,
|
||||
"actual_total_cost": 0.0,
|
||||
"avg_response_time_ms": 0.0,
|
||||
"error_requests": 0,
|
||||
},
|
||||
"change_percent": {
|
||||
"total_requests": serde_json::Value::Null,
|
||||
"total_tokens": serde_json::Value::Null,
|
||||
"total_cost": serde_json::Value::Null,
|
||||
"actual_total_cost": serde_json::Value::Null,
|
||||
"avg_response_time_ms": serde_json::Value::Null,
|
||||
"error_requests": serde_json::Value::Null,
|
||||
},
|
||||
"current_start": current_range.start_date.to_string(),
|
||||
"current_end": current_range.end_date.to_string(),
|
||||
"comparison_start": comparison_range.start_date.to_string(),
|
||||
"comparison_end": comparison_range.end_date.to_string(),
|
||||
}))
|
||||
.into_response()
|
||||
}
|
||||
|
||||
pub(super) fn admin_stats_error_distribution_empty_response() -> Response<Body> {
|
||||
Json(json!({
|
||||
"distribution": [],
|
||||
"trend": [],
|
||||
}))
|
||||
.into_response()
|
||||
}
|
||||
|
||||
pub(super) fn admin_stats_performance_percentiles_empty_response() -> Response<Body> {
|
||||
Json(json!([])).into_response()
|
||||
}
|
||||
|
||||
pub(super) fn admin_stats_cost_savings_empty_response() -> Response<Body> {
|
||||
Json(json!({
|
||||
"cache_read_tokens": 0,
|
||||
"cache_read_cost": 0.0,
|
||||
"cache_creation_cost": 0.0,
|
||||
"estimated_full_cost": 0.0,
|
||||
"cache_savings": 0.0,
|
||||
}))
|
||||
.into_response()
|
||||
}
|
||||
|
||||
pub(super) fn admin_stats_leaderboard_empty_response(
|
||||
metric: AdminStatsLeaderboardMetric,
|
||||
time_range: Option<&AdminStatsTimeRange>,
|
||||
) -> Response<Body> {
|
||||
Json(json!({
|
||||
"items": [],
|
||||
"total": 0,
|
||||
"metric": metric.as_str(),
|
||||
"start_date": time_range.map(|value| value.start_date.to_string()),
|
||||
"end_date": time_range.map(|value| value.end_date.to_string()),
|
||||
}))
|
||||
.into_response()
|
||||
}
|
||||
|
||||
pub(super) fn admin_stats_time_series_empty_response() -> Response<Body> {
|
||||
Json(json!([])).into_response()
|
||||
}
|
||||
|
||||
pub(crate) fn admin_stats_bad_request_response(detail: String) -> Response<Body> {
|
||||
(
|
||||
http::StatusCode::BAD_REQUEST,
|
||||
Json(json!({ "detail": detail })),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
use super::helpers::{
|
||||
round_to, AdminStatsAggregate, AdminStatsGranularity, AdminStatsTimeRange,
|
||||
AdminStatsTimeSeriesBucket, MIN_PERCENTILE_SAMPLES,
|
||||
};
|
||||
use aether_data_contracts::repository::usage::StoredRequestUsageAudit;
|
||||
use chrono::{Datelike, Utc};
|
||||
use serde_json::json;
|
||||
|
||||
pub(super) fn build_time_series_payload(
|
||||
time_range: &AdminStatsTimeRange,
|
||||
granularity: AdminStatsGranularity,
|
||||
items: &[StoredRequestUsageAudit],
|
||||
) -> Vec<serde_json::Value> {
|
||||
match granularity {
|
||||
AdminStatsGranularity::Hour => build_hourly_time_series_payload(time_range, items),
|
||||
AdminStatsGranularity::Day => build_daily_time_series_payload(time_range, items),
|
||||
AdminStatsGranularity::Week => build_weekly_time_series_payload(time_range, items),
|
||||
AdminStatsGranularity::Month => build_monthly_time_series_payload(time_range, items),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn build_daily_time_series_buckets(
|
||||
time_range: &AdminStatsTimeRange,
|
||||
items: &[StoredRequestUsageAudit],
|
||||
) -> std::collections::BTreeMap<chrono::NaiveDate, AdminStatsTimeSeriesBucket> {
|
||||
let mut buckets: std::collections::BTreeMap<chrono::NaiveDate, AdminStatsTimeSeriesBucket> =
|
||||
time_range
|
||||
.local_dates()
|
||||
.into_iter()
|
||||
.map(|date| (date, AdminStatsTimeSeriesBucket::default()))
|
||||
.collect();
|
||||
|
||||
for item in items {
|
||||
let Some(local_day) = time_range.local_date_for_unix_secs(item.created_at_unix_secs) else {
|
||||
continue;
|
||||
};
|
||||
let Some(bucket) = buckets.get_mut(&local_day) else {
|
||||
continue;
|
||||
};
|
||||
bucket.add_usage(item);
|
||||
}
|
||||
|
||||
buckets
|
||||
}
|
||||
|
||||
fn build_daily_time_series_payload(
|
||||
time_range: &AdminStatsTimeRange,
|
||||
items: &[StoredRequestUsageAudit],
|
||||
) -> Vec<serde_json::Value> {
|
||||
build_daily_time_series_buckets(time_range, items)
|
||||
.into_iter()
|
||||
.map(|(date, bucket)| bucket.to_json_with_avg(date.to_string()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn build_weekly_time_series_payload(
|
||||
time_range: &AdminStatsTimeRange,
|
||||
items: &[StoredRequestUsageAudit],
|
||||
) -> Vec<serde_json::Value> {
|
||||
let mut weekly: std::collections::BTreeMap<
|
||||
(i32, u32),
|
||||
(chrono::NaiveDate, AdminStatsTimeSeriesBucket),
|
||||
> = std::collections::BTreeMap::new();
|
||||
|
||||
for (date, bucket) in build_daily_time_series_buckets(time_range, items) {
|
||||
let iso = date.iso_week();
|
||||
let entry = weekly
|
||||
.entry((iso.year(), iso.week()))
|
||||
.or_insert_with(|| (date, AdminStatsTimeSeriesBucket::default()));
|
||||
entry.0 = entry.0.min(date);
|
||||
entry.1.merge(&bucket);
|
||||
}
|
||||
|
||||
weekly
|
||||
.into_values()
|
||||
.map(|(date, bucket)| bucket.to_json_with_avg(date.to_string()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn build_monthly_time_series_payload(
|
||||
time_range: &AdminStatsTimeRange,
|
||||
items: &[StoredRequestUsageAudit],
|
||||
) -> Vec<serde_json::Value> {
|
||||
let mut monthly: std::collections::BTreeMap<
|
||||
(i32, u32),
|
||||
(chrono::NaiveDate, AdminStatsTimeSeriesBucket),
|
||||
> = std::collections::BTreeMap::new();
|
||||
|
||||
for (date, bucket) in build_daily_time_series_buckets(time_range, items) {
|
||||
let Some(month_start) = chrono::NaiveDate::from_ymd_opt(date.year(), date.month(), 1)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let entry = monthly
|
||||
.entry((date.year(), date.month()))
|
||||
.or_insert_with(|| (month_start, AdminStatsTimeSeriesBucket::default()));
|
||||
entry.1.merge(&bucket);
|
||||
}
|
||||
|
||||
monthly
|
||||
.into_values()
|
||||
.map(|(date, bucket)| bucket.to_json_with_avg(date.to_string()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn build_hourly_time_series_payload(
|
||||
time_range: &AdminStatsTimeRange,
|
||||
items: &[StoredRequestUsageAudit],
|
||||
) -> Vec<serde_json::Value> {
|
||||
let Some((mut current, end)) = time_range.to_utc_datetime_bounds() else {
|
||||
return Vec::new();
|
||||
};
|
||||
let offset = chrono::Duration::minutes(i64::from(time_range.tz_offset_minutes));
|
||||
let mut buckets: std::collections::BTreeMap<String, AdminStatsTimeSeriesBucket> =
|
||||
std::collections::BTreeMap::new();
|
||||
|
||||
while current < end {
|
||||
let label = (current + offset)
|
||||
.format("%Y-%m-%dT%H:00:00+00:00")
|
||||
.to_string();
|
||||
buckets.insert(label, AdminStatsTimeSeriesBucket::default());
|
||||
let Some(next) = current.checked_add_signed(chrono::Duration::hours(1)) else {
|
||||
break;
|
||||
};
|
||||
current = next;
|
||||
}
|
||||
|
||||
for item in items {
|
||||
let Some(unix_secs) = i64::try_from(item.created_at_unix_secs).ok() else {
|
||||
continue;
|
||||
};
|
||||
let Some(timestamp) = chrono::DateTime::<Utc>::from_timestamp(unix_secs, 0) else {
|
||||
continue;
|
||||
};
|
||||
let Some(local) = timestamp.checked_add_signed(offset) else {
|
||||
continue;
|
||||
};
|
||||
let label = local.format("%Y-%m-%dT%H:00:00+00:00").to_string();
|
||||
let Some(bucket) = buckets.get_mut(&label) else {
|
||||
continue;
|
||||
};
|
||||
bucket.add_usage(item);
|
||||
}
|
||||
|
||||
buckets
|
||||
.into_iter()
|
||||
.map(|(date, bucket)| bucket.to_json_without_avg(date))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn aggregate_usage_stats(items: &[StoredRequestUsageAudit]) -> AdminStatsAggregate {
|
||||
let mut aggregate = AdminStatsAggregate::default();
|
||||
for item in items {
|
||||
aggregate.total_requests = aggregate.total_requests.saturating_add(1);
|
||||
aggregate.total_tokens = aggregate.total_tokens.saturating_add(item.total_tokens);
|
||||
aggregate.total_cost += item.total_cost_usd;
|
||||
aggregate.actual_total_cost += item.actual_total_cost_usd;
|
||||
aggregate.total_response_time_ms += item.response_time_ms.unwrap_or(0) as f64;
|
||||
if item.status_code.is_some_and(|value| value >= 400) || item.error_message.is_some() {
|
||||
aggregate.error_requests = aggregate.error_requests.saturating_add(1);
|
||||
}
|
||||
}
|
||||
aggregate
|
||||
}
|
||||
|
||||
pub(super) fn percentile_cont(values: &mut [u64], percentile: f64) -> Option<u64> {
|
||||
if values.len() < MIN_PERCENTILE_SAMPLES {
|
||||
return None;
|
||||
}
|
||||
values.sort_unstable();
|
||||
|
||||
let position = percentile * (values.len().saturating_sub(1)) as f64;
|
||||
let lower = position.floor() as usize;
|
||||
let upper = position.ceil() as usize;
|
||||
let lower_value = values[lower] as f64;
|
||||
let upper_value = values[upper] as f64;
|
||||
Some((lower_value + (upper_value - lower_value) * (position - lower as f64)).trunc() as u64)
|
||||
}
|
||||
|
||||
pub(super) fn pct_change_value(current: f64, previous: f64) -> serde_json::Value {
|
||||
if previous == 0.0 {
|
||||
if current == 0.0 {
|
||||
json!(0.0)
|
||||
} else {
|
||||
serde_json::Value::Null
|
||||
}
|
||||
} else {
|
||||
json!(round_to((current - previous) / previous * 100.0, 2))
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn linear_regression(values: &[f64]) -> (f64, f64) {
|
||||
let n = values.len();
|
||||
if n <= 1 {
|
||||
return (0.0, values.first().copied().unwrap_or(0.0));
|
||||
}
|
||||
let sum_x: f64 = (0..n).map(|value| value as f64).sum();
|
||||
let sum_y: f64 = values.iter().sum();
|
||||
let sum_x2: f64 = (0..n).map(|value| (value * value) as f64).sum();
|
||||
let sum_xy: f64 = values
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, value)| index as f64 * *value)
|
||||
.sum();
|
||||
let n = n as f64;
|
||||
let denom = n * sum_x2 - sum_x * sum_x;
|
||||
if denom == 0.0 {
|
||||
return (0.0, values.last().copied().unwrap_or(0.0));
|
||||
}
|
||||
let slope = (n * sum_xy - sum_x * sum_y) / denom;
|
||||
let intercept = (sum_y - slope * sum_x) / n;
|
||||
(slope, intercept)
|
||||
}
|
||||
@@ -0,0 +1,777 @@
|
||||
use super::super::stats::{parse_bounded_u32, round_to};
|
||||
use crate::handlers::admin::shared::{query_param_value, unix_secs_to_rfc3339};
|
||||
use crate::{AppState, GatewayError};
|
||||
use aether_data_contracts::repository::usage::{StoredRequestUsageAudit, UsageAuditListQuery};
|
||||
use serde_json::{json, Value};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
pub(super) fn admin_usage_total_tokens(item: &StoredRequestUsageAudit) -> u64 {
|
||||
item.input_tokens
|
||||
.saturating_add(item.output_tokens)
|
||||
.saturating_add(item.cache_creation_input_tokens)
|
||||
.saturating_add(item.cache_read_input_tokens)
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_parse_limit(query: Option<&str>) -> Result<usize, String> {
|
||||
match query_param_value(query, "limit") {
|
||||
None => Ok(100),
|
||||
Some(value) => {
|
||||
let parsed = value
|
||||
.parse::<usize>()
|
||||
.map_err(|_| "limit must be a positive integer".to_string())?;
|
||||
if parsed == 0 || parsed > 500 {
|
||||
return Err("limit must be between 1 and 500".to_string());
|
||||
}
|
||||
Ok(parsed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_parse_offset(query: Option<&str>) -> Result<usize, String> {
|
||||
match query_param_value(query, "offset") {
|
||||
None => Ok(0),
|
||||
Some(value) => value
|
||||
.parse::<usize>()
|
||||
.map_err(|_| "offset must be a non-negative integer".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_parse_ids(query: Option<&str>) -> Option<BTreeSet<String>> {
|
||||
let ids = query_param_value(query, "ids")?;
|
||||
let parsed: BTreeSet<String> = ids
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
.collect();
|
||||
Some(parsed)
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_parse_recent_hours(
|
||||
query: Option<&str>,
|
||||
default: u32,
|
||||
) -> Result<u32, String> {
|
||||
match query_param_value(query, "hours") {
|
||||
Some(value) => parse_bounded_u32("hours", &value, 1, 720),
|
||||
None => Ok(default),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_parse_timeline_limit(query: Option<&str>) -> Result<usize, String> {
|
||||
match query_param_value(query, "limit") {
|
||||
Some(value) => {
|
||||
let parsed = value
|
||||
.parse::<usize>()
|
||||
.map_err(|_| "limit must be an integer between 100 and 50000".to_string())?;
|
||||
if (100..=50_000).contains(&parsed) {
|
||||
Ok(parsed)
|
||||
} else {
|
||||
Err("limit must be an integer between 100 and 50000".to_string())
|
||||
}
|
||||
}
|
||||
None => Ok(3_000),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_parse_aggregation_limit(query: Option<&str>) -> Result<usize, String> {
|
||||
match query_param_value(query, "limit") {
|
||||
Some(value) => {
|
||||
let parsed = value
|
||||
.parse::<usize>()
|
||||
.map_err(|_| "limit must be an integer between 1 and 100".to_string())?;
|
||||
if (1..=100).contains(&parsed) {
|
||||
Ok(parsed)
|
||||
} else {
|
||||
Err("limit must be an integer between 1 and 100".to_string())
|
||||
}
|
||||
}
|
||||
None => Ok(20),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_token_cache_hit_rate(input_tokens: u64, cache_read_tokens: u64) -> f64 {
|
||||
let total_input_context = input_tokens.saturating_add(cache_read_tokens);
|
||||
if total_input_context == 0 {
|
||||
0.0
|
||||
} else {
|
||||
round_to(
|
||||
cache_read_tokens as f64 / total_input_context as f64 * 100.0,
|
||||
2,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_aggregation_by_model_json(
|
||||
usage: &[StoredRequestUsageAudit],
|
||||
limit: usize,
|
||||
) -> serde_json::Value {
|
||||
let mut grouped: BTreeMap<String, (u64, u64, u64, u64, f64, f64)> = BTreeMap::new();
|
||||
for item in usage {
|
||||
let key = item.model.clone();
|
||||
let entry = grouped.entry(key).or_insert((0, 0, 0, 0, 0.0, 0.0));
|
||||
entry.0 = entry.0.saturating_add(1);
|
||||
entry.1 = entry.1.saturating_add(item.total_tokens);
|
||||
entry.2 = entry.2.saturating_add(item.input_tokens);
|
||||
entry.3 = entry.3.saturating_add(item.cache_read_input_tokens);
|
||||
entry.4 += item.total_cost_usd;
|
||||
entry.5 += item.actual_total_cost_usd;
|
||||
}
|
||||
|
||||
let mut items: Vec<serde_json::Value> = grouped
|
||||
.into_iter()
|
||||
.map(
|
||||
|(model, (request_count, total_tokens, input_tokens, cache_read_tokens, total_cost, actual_cost))| {
|
||||
json!({
|
||||
"model": model,
|
||||
"request_count": request_count,
|
||||
"total_tokens": total_tokens,
|
||||
"total_input_context": input_tokens.saturating_add(cache_read_tokens),
|
||||
"output_tokens": total_tokens.saturating_sub(input_tokens),
|
||||
"total_cost": round_to(total_cost, 6),
|
||||
"actual_cost": round_to(actual_cost, 6),
|
||||
"cache_read_tokens": cache_read_tokens,
|
||||
"cache_creation_tokens": 0,
|
||||
"cache_hit_rate": admin_usage_token_cache_hit_rate(input_tokens, cache_read_tokens),
|
||||
})
|
||||
},
|
||||
)
|
||||
.collect();
|
||||
items.sort_by(|left, right| {
|
||||
right["request_count"]
|
||||
.as_u64()
|
||||
.unwrap_or_default()
|
||||
.cmp(&left["request_count"].as_u64().unwrap_or_default())
|
||||
.then_with(|| {
|
||||
left["model"]
|
||||
.as_str()
|
||||
.unwrap_or_default()
|
||||
.cmp(right["model"].as_str().unwrap_or_default())
|
||||
})
|
||||
});
|
||||
items.truncate(limit);
|
||||
json!(items)
|
||||
}
|
||||
|
||||
pub(super) async fn admin_usage_aggregation_by_user_json(
|
||||
state: &AppState,
|
||||
usage: &[StoredRequestUsageAudit],
|
||||
limit: usize,
|
||||
) -> Result<serde_json::Value, GatewayError> {
|
||||
let mut grouped: BTreeMap<String, (u64, u64, f64)> = BTreeMap::new();
|
||||
for item in usage {
|
||||
let Some(user_id) = item.user_id.as_ref() else {
|
||||
continue;
|
||||
};
|
||||
let entry = grouped.entry(user_id.clone()).or_insert((0, 0, 0.0));
|
||||
entry.0 = entry.0.saturating_add(1);
|
||||
entry.1 = entry.1.saturating_add(item.total_tokens);
|
||||
entry.2 += item.total_cost_usd;
|
||||
}
|
||||
|
||||
let usernames = if state.has_user_data_reader() && !grouped.is_empty() {
|
||||
state
|
||||
.list_users_by_ids(&grouped.keys().cloned().collect::<Vec<_>>())
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|user| (user.id, (user.email, user.username)))
|
||||
.collect::<BTreeMap<_, _>>()
|
||||
} else {
|
||||
BTreeMap::new()
|
||||
};
|
||||
|
||||
let mut items: Vec<serde_json::Value> = grouped
|
||||
.into_iter()
|
||||
.map(|(user_id, (request_count, total_tokens, total_cost))| {
|
||||
let (email, username) = usernames
|
||||
.get(&user_id)
|
||||
.cloned()
|
||||
.unwrap_or((None, String::new()));
|
||||
json!({
|
||||
"user_id": user_id,
|
||||
"email": email,
|
||||
"username": if username.is_empty() { serde_json::Value::Null } else { json!(username) },
|
||||
"request_count": request_count,
|
||||
"total_tokens": total_tokens,
|
||||
"total_cost": round_to(total_cost, 6),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
items.sort_by(|left, right| {
|
||||
right["request_count"]
|
||||
.as_u64()
|
||||
.unwrap_or_default()
|
||||
.cmp(&left["request_count"].as_u64().unwrap_or_default())
|
||||
.then_with(|| {
|
||||
left["user_id"]
|
||||
.as_str()
|
||||
.unwrap_or_default()
|
||||
.cmp(right["user_id"].as_str().unwrap_or_default())
|
||||
})
|
||||
});
|
||||
items.truncate(limit);
|
||||
Ok(json!(items))
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_aggregation_by_provider_json(
|
||||
usage: &[StoredRequestUsageAudit],
|
||||
limit: usize,
|
||||
) -> serde_json::Value {
|
||||
let mut grouped: BTreeMap<String, (u64, u64, u64, u64, f64, f64, u64, u64)> = BTreeMap::new();
|
||||
for item in usage {
|
||||
let key = item
|
||||
.provider_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| "unknown".to_string());
|
||||
let entry = grouped.entry(key).or_insert((0, 0, 0, 0, 0.0, 0.0, 0, 0));
|
||||
entry.0 = entry.0.saturating_add(1);
|
||||
entry.1 = entry.1.saturating_add(item.total_tokens);
|
||||
entry.2 = entry.2.saturating_add(item.input_tokens);
|
||||
entry.3 = entry.3.saturating_add(item.cache_read_input_tokens);
|
||||
entry.4 += item.total_cost_usd;
|
||||
entry.5 += item.actual_total_cost_usd;
|
||||
entry.6 = entry
|
||||
.6
|
||||
.saturating_add(item.response_time_ms.unwrap_or_default());
|
||||
entry.7 = entry
|
||||
.7
|
||||
.saturating_add(if admin_usage_is_success(item) { 1 } else { 0 });
|
||||
}
|
||||
|
||||
let mut items: Vec<serde_json::Value> = grouped
|
||||
.into_iter()
|
||||
.map(
|
||||
|(provider_id, (request_count, total_tokens, input_tokens, cache_read_tokens, total_cost, actual_cost, response_time_ms_sum, success_count))| {
|
||||
let avg_response_time_ms = if request_count == 0 {
|
||||
0.0
|
||||
} else {
|
||||
round_to(response_time_ms_sum as f64 / request_count as f64, 2)
|
||||
};
|
||||
let error_count = request_count.saturating_sub(success_count);
|
||||
let success_rate = if request_count == 0 {
|
||||
0.0
|
||||
} else {
|
||||
round_to(success_count as f64 / request_count as f64 * 100.0, 2)
|
||||
};
|
||||
json!({
|
||||
"provider_id": provider_id,
|
||||
"provider": serde_json::Value::Null,
|
||||
"request_count": request_count,
|
||||
"total_tokens": total_tokens,
|
||||
"total_input_context": input_tokens.saturating_add(cache_read_tokens),
|
||||
"output_tokens": total_tokens.saturating_sub(input_tokens),
|
||||
"total_cost": round_to(total_cost, 6),
|
||||
"actual_cost": round_to(actual_cost, 6),
|
||||
"avg_response_time_ms": avg_response_time_ms,
|
||||
"success_rate": success_rate,
|
||||
"error_count": error_count,
|
||||
"cache_read_tokens": cache_read_tokens,
|
||||
"cache_creation_tokens": 0,
|
||||
"cache_hit_rate": admin_usage_token_cache_hit_rate(input_tokens, cache_read_tokens),
|
||||
})
|
||||
},
|
||||
)
|
||||
.collect();
|
||||
items.sort_by(|left, right| {
|
||||
right["request_count"]
|
||||
.as_u64()
|
||||
.unwrap_or_default()
|
||||
.cmp(&left["request_count"].as_u64().unwrap_or_default())
|
||||
.then_with(|| {
|
||||
left["provider_id"]
|
||||
.as_str()
|
||||
.unwrap_or_default()
|
||||
.cmp(right["provider_id"].as_str().unwrap_or_default())
|
||||
})
|
||||
});
|
||||
items.truncate(limit);
|
||||
json!(items)
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_aggregation_by_api_format_json(
|
||||
usage: &[StoredRequestUsageAudit],
|
||||
limit: usize,
|
||||
) -> serde_json::Value {
|
||||
let mut grouped: BTreeMap<String, (u64, u64, u64, u64, f64, f64, u64)> = BTreeMap::new();
|
||||
for item in usage {
|
||||
let key = item
|
||||
.api_format
|
||||
.clone()
|
||||
.unwrap_or_else(|| "unknown".to_string());
|
||||
let entry = grouped.entry(key).or_insert((0, 0, 0, 0, 0.0, 0.0, 0));
|
||||
entry.0 = entry.0.saturating_add(1);
|
||||
entry.1 = entry.1.saturating_add(item.total_tokens);
|
||||
entry.2 = entry.2.saturating_add(item.input_tokens);
|
||||
entry.3 = entry.3.saturating_add(item.cache_read_input_tokens);
|
||||
entry.4 += item.total_cost_usd;
|
||||
entry.5 += item.actual_total_cost_usd;
|
||||
entry.6 = entry
|
||||
.6
|
||||
.saturating_add(item.response_time_ms.unwrap_or_default());
|
||||
}
|
||||
|
||||
let mut items: Vec<serde_json::Value> = grouped
|
||||
.into_iter()
|
||||
.map(
|
||||
|(api_format, (request_count, total_tokens, input_tokens, cache_read_tokens, total_cost, actual_cost, response_time_ms_sum))| {
|
||||
let avg_response_time_ms = if request_count == 0 {
|
||||
0.0
|
||||
} else {
|
||||
round_to(response_time_ms_sum as f64 / request_count as f64, 2)
|
||||
};
|
||||
json!({
|
||||
"api_format": api_format,
|
||||
"request_count": request_count,
|
||||
"total_tokens": total_tokens,
|
||||
"total_input_context": input_tokens.saturating_add(cache_read_tokens),
|
||||
"output_tokens": total_tokens.saturating_sub(input_tokens),
|
||||
"total_cost": round_to(total_cost, 6),
|
||||
"actual_cost": round_to(actual_cost, 6),
|
||||
"avg_response_time_ms": avg_response_time_ms,
|
||||
"cache_read_tokens": cache_read_tokens,
|
||||
"cache_creation_tokens": 0,
|
||||
"cache_hit_rate": admin_usage_token_cache_hit_rate(input_tokens, cache_read_tokens),
|
||||
})
|
||||
},
|
||||
)
|
||||
.collect();
|
||||
items.sort_by(|left, right| {
|
||||
right["request_count"]
|
||||
.as_u64()
|
||||
.unwrap_or_default()
|
||||
.cmp(&left["request_count"].as_u64().unwrap_or_default())
|
||||
.then_with(|| {
|
||||
left["api_format"]
|
||||
.as_str()
|
||||
.unwrap_or_default()
|
||||
.cmp(right["api_format"].as_str().unwrap_or_default())
|
||||
})
|
||||
});
|
||||
items.truncate(limit);
|
||||
json!(items)
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_heatmap_json(usage: &[StoredRequestUsageAudit]) -> serde_json::Value {
|
||||
let today = chrono::Utc::now().date_naive();
|
||||
let start_date = today
|
||||
.checked_sub_signed(chrono::Duration::days(364))
|
||||
.unwrap_or(today);
|
||||
let mut grouped: BTreeMap<chrono::NaiveDate, (u64, u64, f64, f64)> = BTreeMap::new();
|
||||
for item in usage {
|
||||
let Ok(created_at_unix_secs) = i64::try_from(item.created_at_unix_secs) else {
|
||||
continue;
|
||||
};
|
||||
let Some(created_at) =
|
||||
chrono::DateTime::<chrono::Utc>::from_timestamp(created_at_unix_secs, 0)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let date_key = created_at.date_naive();
|
||||
if date_key < start_date || date_key > today {
|
||||
continue;
|
||||
}
|
||||
let entry = grouped.entry(date_key).or_insert((0, 0, 0.0, 0.0));
|
||||
entry.0 = entry.0.saturating_add(1);
|
||||
entry.1 = entry.1.saturating_add(admin_usage_total_tokens(item));
|
||||
entry.2 += item.total_cost_usd;
|
||||
entry.3 += item.actual_total_cost_usd;
|
||||
}
|
||||
|
||||
let mut max_requests = 0_u64;
|
||||
let mut cursor = start_date;
|
||||
let mut days = Vec::new();
|
||||
while cursor <= today {
|
||||
let (requests, total_tokens, total_cost, actual_total_cost) =
|
||||
grouped.get(&cursor).copied().unwrap_or((0, 0, 0.0, 0.0));
|
||||
max_requests = max_requests.max(requests);
|
||||
days.push(json!({
|
||||
"date": cursor.to_string(),
|
||||
"requests": requests,
|
||||
"total_tokens": total_tokens,
|
||||
"total_cost": round_to(total_cost, 6),
|
||||
"actual_total_cost": round_to(actual_total_cost, 6),
|
||||
}));
|
||||
cursor = cursor
|
||||
.checked_add_signed(chrono::Duration::days(1))
|
||||
.unwrap_or(today + chrono::Duration::days(1));
|
||||
}
|
||||
|
||||
json!({
|
||||
"start_date": start_date.to_string(),
|
||||
"end_date": today.to_string(),
|
||||
"total_days": days.len(),
|
||||
"max_requests": max_requests,
|
||||
"days": days,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_is_success(item: &StoredRequestUsageAudit) -> bool {
|
||||
matches!(
|
||||
item.status.as_str(),
|
||||
"completed" | "success" | "ok" | "billed" | "settled"
|
||||
) && item.status_code.is_none_or(|code| code < 400)
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_matches_optional_id(value: Option<&str>, expected: Option<&str>) -> bool {
|
||||
let Some(expected) = expected.map(str::trim).filter(|value| !value.is_empty()) else {
|
||||
return true;
|
||||
};
|
||||
value.is_some_and(|candidate| candidate == expected)
|
||||
}
|
||||
|
||||
pub(super) async fn list_recent_completed_usage_for_cache_affinity(
|
||||
state: &AppState,
|
||||
hours: u32,
|
||||
user_id: Option<&str>,
|
||||
) -> Result<Vec<StoredRequestUsageAudit>, GatewayError> {
|
||||
let now_unix_secs = u64::try_from(chrono::Utc::now().timestamp()).unwrap_or_default();
|
||||
let created_from_unix_secs = now_unix_secs.saturating_sub(u64::from(hours) * 3600);
|
||||
let mut items = state
|
||||
.list_usage_audits(&UsageAuditListQuery {
|
||||
created_from_unix_secs: Some(created_from_unix_secs),
|
||||
created_until_unix_secs: None,
|
||||
user_id: user_id.map(ToOwned::to_owned),
|
||||
provider_name: None,
|
||||
model: None,
|
||||
})
|
||||
.await?;
|
||||
items.retain(|item| item.status == "completed");
|
||||
items.sort_by(|left, right| {
|
||||
left.created_at_unix_secs
|
||||
.cmp(&right.created_at_unix_secs)
|
||||
.then_with(|| left.id.cmp(&right.id))
|
||||
});
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_group_completed_by_user(
|
||||
items: &[StoredRequestUsageAudit],
|
||||
) -> BTreeMap<String, Vec<StoredRequestUsageAudit>> {
|
||||
let mut grouped = BTreeMap::new();
|
||||
for item in items.iter().filter(|item| item.user_id.is_some()) {
|
||||
grouped
|
||||
.entry(item.user_id.clone().unwrap_or_default())
|
||||
.or_insert_with(Vec::new)
|
||||
.push(item.clone());
|
||||
}
|
||||
grouped
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_group_completed_by_api_key(
|
||||
items: &[StoredRequestUsageAudit],
|
||||
api_key_id: Option<&str>,
|
||||
) -> BTreeMap<String, Vec<StoredRequestUsageAudit>> {
|
||||
let mut grouped = BTreeMap::new();
|
||||
for item in items.iter().filter(|item| item.api_key_id.is_some()) {
|
||||
if !admin_usage_matches_optional_id(item.api_key_id.as_deref(), api_key_id) {
|
||||
continue;
|
||||
}
|
||||
grouped
|
||||
.entry(item.api_key_id.clone().unwrap_or_default())
|
||||
.or_insert_with(Vec::new)
|
||||
.push(item.clone());
|
||||
}
|
||||
grouped
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_collect_request_intervals_minutes(
|
||||
items: &[StoredRequestUsageAudit],
|
||||
) -> Vec<f64> {
|
||||
let mut previous_created_at_unix_secs = None;
|
||||
let mut intervals = Vec::new();
|
||||
for item in items {
|
||||
if let Some(previous) = previous_created_at_unix_secs {
|
||||
intervals.push(item.created_at_unix_secs.saturating_sub(previous) as f64 / 60.0);
|
||||
}
|
||||
previous_created_at_unix_secs = Some(item.created_at_unix_secs);
|
||||
}
|
||||
intervals
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_percentile_cont(values: &[f64], percentile: f64) -> Option<f64> {
|
||||
if values.is_empty() {
|
||||
return None;
|
||||
}
|
||||
if values.len() == 1 {
|
||||
return Some(values[0]);
|
||||
}
|
||||
let position = percentile.clamp(0.0, 1.0) * (values.len() - 1) as f64;
|
||||
let lower_index = position.floor() as usize;
|
||||
let upper_index = position.ceil() as usize;
|
||||
let lower = values[lower_index];
|
||||
let upper = values[upper_index];
|
||||
Some(lower + (upper - lower) * (position - lower_index as f64))
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_calculate_recommended_ttl(
|
||||
p75_interval: Option<f64>,
|
||||
p90_interval: Option<f64>,
|
||||
) -> u64 {
|
||||
let Some(p75_interval) = p75_interval else {
|
||||
return 5;
|
||||
};
|
||||
let Some(p90_interval) = p90_interval else {
|
||||
return 5;
|
||||
};
|
||||
|
||||
if p90_interval <= 5.0 {
|
||||
5
|
||||
} else if p75_interval <= 15.0 {
|
||||
15
|
||||
} else if p75_interval <= 30.0 {
|
||||
30
|
||||
} else {
|
||||
60
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_ttl_recommendation_reason(
|
||||
ttl: u64,
|
||||
p75_interval: Option<f64>,
|
||||
p90_interval: Option<f64>,
|
||||
) -> String {
|
||||
let Some(p75_interval) = p75_interval else {
|
||||
return "数据不足，使用默认值".to_string();
|
||||
};
|
||||
let Some(p90_interval) = p90_interval else {
|
||||
return "数据不足，使用默认值".to_string();
|
||||
};
|
||||
|
||||
match ttl {
|
||||
5 => format!("高频用户：90% 的请求间隔在 {:.1} 分钟内", p90_interval),
|
||||
15 => format!("中高频用户：75% 的请求间隔在 {:.1} 分钟内", p75_interval),
|
||||
30 => format!("中频用户：75% 的请求间隔在 {:.1} 分钟内", p75_interval),
|
||||
_ => format!(
|
||||
"低频用户：75% 的请求间隔为 {:.1} 分钟，建议使用长 TTL",
|
||||
p75_interval
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_proportional_limits(
|
||||
grouped: &BTreeMap<String, Vec<serde_json::Value>>,
|
||||
limit: usize,
|
||||
total_points: usize,
|
||||
) -> BTreeMap<String, usize> {
|
||||
let mut limits = BTreeMap::new();
|
||||
for (group_id, items) in grouped {
|
||||
let computed = if total_points <= limit || total_points == 0 {
|
||||
items.len()
|
||||
} else {
|
||||
let scaled =
|
||||
((items.len() as f64 * limit as f64) / total_points as f64).ceil() as usize;
|
||||
std::cmp::max(scaled, 1)
|
||||
};
|
||||
limits.insert(group_id.clone(), computed);
|
||||
}
|
||||
limits
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_point_sort_key(
|
||||
left: &serde_json::Value,
|
||||
right: &serde_json::Value,
|
||||
) -> std::cmp::Ordering {
|
||||
left["x"]
|
||||
.as_str()
|
||||
.unwrap_or_default()
|
||||
.cmp(right["x"].as_str().unwrap_or_default())
|
||||
.then_with(|| {
|
||||
left["user_id"]
|
||||
.as_str()
|
||||
.unwrap_or_default()
|
||||
.cmp(right["user_id"].as_str().unwrap_or_default())
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_matches_search(
|
||||
item: &StoredRequestUsageAudit,
|
||||
search: Option<&str>,
|
||||
) -> bool {
|
||||
let Some(search) = search.map(str::trim).filter(|value| !value.is_empty()) else {
|
||||
return true;
|
||||
};
|
||||
let haystack = [
|
||||
item.username.as_deref(),
|
||||
item.api_key_name.as_deref(),
|
||||
Some(item.model.as_str()),
|
||||
Some(item.provider_name.as_str()),
|
||||
];
|
||||
search.split_whitespace().all(|keyword| {
|
||||
let keyword = keyword.to_ascii_lowercase();
|
||||
haystack
|
||||
.iter()
|
||||
.flatten()
|
||||
.any(|value| value.to_ascii_lowercase().contains(keyword.as_str()))
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_matches_username(
|
||||
item: &StoredRequestUsageAudit,
|
||||
username: Option<&str>,
|
||||
) -> bool {
|
||||
let Some(username) = username.map(str::trim).filter(|value| !value.is_empty()) else {
|
||||
return true;
|
||||
};
|
||||
item.username
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.to_ascii_lowercase()
|
||||
.contains(username.to_ascii_lowercase().as_str())
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_matches_eq(value: &str, query: Option<&str>) -> bool {
|
||||
let Some(query) = query
|
||||
.map(str::trim)
|
||||
.filter(|candidate| !candidate.is_empty())
|
||||
else {
|
||||
return true;
|
||||
};
|
||||
value.eq_ignore_ascii_case(query)
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_matches_api_format(
|
||||
item: &StoredRequestUsageAudit,
|
||||
api_format: Option<&str>,
|
||||
) -> bool {
|
||||
let Some(api_format) = api_format.map(str::trim).filter(|value| !value.is_empty()) else {
|
||||
return true;
|
||||
};
|
||||
item.api_format
|
||||
.as_deref()
|
||||
.is_some_and(|value| value.eq_ignore_ascii_case(api_format))
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_matches_status(
|
||||
item: &StoredRequestUsageAudit,
|
||||
status: Option<&str>,
|
||||
) -> bool {
|
||||
let Some(status) = status.map(str::trim).filter(|value| !value.is_empty()) else {
|
||||
return true;
|
||||
};
|
||||
match status {
|
||||
"stream" => item.is_stream,
|
||||
"standard" => !item.is_stream,
|
||||
"error" => {
|
||||
item.status_code.is_some_and(|value| value >= 400) || item.error_message.is_some()
|
||||
}
|
||||
"pending" | "streaming" | "completed" | "cancelled" => item.status == status,
|
||||
"failed" => {
|
||||
item.status == "failed"
|
||||
|| item.status_code.is_some_and(|value| value >= 400)
|
||||
|| item.error_message.is_some()
|
||||
}
|
||||
"active" => matches!(item.status.as_str(), "pending" | "streaming"),
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn admin_usage_provider_key_names(
|
||||
state: &AppState,
|
||||
usage: &[StoredRequestUsageAudit],
|
||||
) -> Result<BTreeMap<String, String>, GatewayError> {
|
||||
if !state.has_provider_catalog_data_reader() {
|
||||
return Ok(BTreeMap::new());
|
||||
}
|
||||
|
||||
let key_ids = usage
|
||||
.iter()
|
||||
.filter_map(|item| item.provider_api_key_id.clone())
|
||||
.collect::<BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>();
|
||||
if key_ids.is_empty() {
|
||||
return Ok(BTreeMap::new());
|
||||
}
|
||||
|
||||
Ok(state
|
||||
.list_provider_catalog_keys_by_ids(&key_ids)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|key| (key.id, key.name))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn admin_usage_request_metadata_string(
|
||||
item: &StoredRequestUsageAudit,
|
||||
key: &str,
|
||||
) -> Option<String> {
|
||||
item.request_metadata
|
||||
.as_ref()
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|metadata| metadata.get(key))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_provider_key_name(
|
||||
item: &StoredRequestUsageAudit,
|
||||
provider_key_names: &BTreeMap<String, String>,
|
||||
) -> Option<String> {
|
||||
item.provider_api_key_id
|
||||
.as_ref()
|
||||
.and_then(|key_id| provider_key_names.get(key_id))
|
||||
.cloned()
|
||||
.or_else(|| admin_usage_request_metadata_string(item, "key_name"))
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_record_json(
|
||||
item: &StoredRequestUsageAudit,
|
||||
users_by_id: &BTreeMap<String, aether_data::repository::users::StoredUserSummary>,
|
||||
provider_key_name: Option<&str>,
|
||||
) -> Value {
|
||||
let user = item
|
||||
.user_id
|
||||
.as_ref()
|
||||
.and_then(|user_id| users_by_id.get(user_id));
|
||||
let username = user
|
||||
.map(|value| value.username.clone())
|
||||
.or_else(|| item.username.clone())
|
||||
.unwrap_or_else(|| "已删除用户".to_string());
|
||||
let user_email = user
|
||||
.and_then(|value| value.email.clone())
|
||||
.unwrap_or_else(|| "已删除用户".to_string());
|
||||
|
||||
json!({
|
||||
"id": item.id,
|
||||
"user_id": item.user_id,
|
||||
"user_email": user_email,
|
||||
"username": username,
|
||||
"api_key": item.api_key_id.as_ref().map(|api_key_id| json!({
|
||||
"id": api_key_id,
|
||||
"name": item.api_key_name.clone(),
|
||||
"display": item.api_key_name.clone().unwrap_or_else(|| api_key_id.clone()),
|
||||
})),
|
||||
"provider": item.provider_name,
|
||||
"model": item.model,
|
||||
"target_model": item.target_model,
|
||||
"input_tokens": item.input_tokens,
|
||||
"output_tokens": item.output_tokens,
|
||||
"cache_creation_input_tokens": item.cache_creation_input_tokens,
|
||||
"cache_read_input_tokens": item.cache_read_input_tokens,
|
||||
"total_tokens": admin_usage_total_tokens(item),
|
||||
"cost": round_to(item.total_cost_usd, 6),
|
||||
"actual_cost": round_to(item.actual_total_cost_usd, 6),
|
||||
"rate_multiplier": Value::Null,
|
||||
"response_time_ms": item.response_time_ms,
|
||||
"first_byte_time_ms": item.first_byte_time_ms,
|
||||
"created_at": unix_secs_to_rfc3339(item.created_at_unix_secs),
|
||||
"is_stream": item.is_stream,
|
||||
"input_price_per_1m": Value::Null,
|
||||
"output_price_per_1m": item.output_price_per_1m,
|
||||
"cache_creation_price_per_1m": Value::Null,
|
||||
"cache_read_price_per_1m": Value::Null,
|
||||
"status_code": item.status_code,
|
||||
"error_message": item.error_message,
|
||||
"status": item.status,
|
||||
"has_fallback": false,
|
||||
"has_retry": false,
|
||||
"has_rectified": false,
|
||||
"api_format": item.api_format,
|
||||
"endpoint_api_format": item.endpoint_api_format,
|
||||
"has_format_conversion": item.has_format_conversion,
|
||||
"api_key_name": item.api_key_name,
|
||||
"provider_key_name": provider_key_name,
|
||||
"model_version": Value::Null,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,515 @@
|
||||
use super::super::stats::{
|
||||
list_usage_for_optional_range, round_to, AdminStatsTimeRange, AdminStatsUsageFilter,
|
||||
};
|
||||
use super::analytics::{
|
||||
admin_usage_aggregation_by_api_format_json, admin_usage_aggregation_by_model_json,
|
||||
admin_usage_aggregation_by_provider_json, admin_usage_aggregation_by_user_json,
|
||||
admin_usage_calculate_recommended_ttl, admin_usage_collect_request_intervals_minutes,
|
||||
admin_usage_group_completed_by_api_key, admin_usage_group_completed_by_user,
|
||||
admin_usage_heatmap_json, admin_usage_matches_optional_id, admin_usage_parse_aggregation_limit,
|
||||
admin_usage_parse_recent_hours, admin_usage_parse_timeline_limit, admin_usage_percentile_cont,
|
||||
admin_usage_point_sort_key, admin_usage_proportional_limits,
|
||||
admin_usage_ttl_recommendation_reason, list_recent_completed_usage_for_cache_affinity,
|
||||
};
|
||||
use super::helpers::{
|
||||
admin_usage_bad_request_response, admin_usage_data_unavailable_response,
|
||||
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
|
||||
};
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use crate::handlers::admin::shared::{query_param_bool, query_param_value, unix_secs_to_rfc3339};
|
||||
use crate::{AppState, GatewayError};
|
||||
use aether_data_contracts::repository::usage::UsageAuditListQuery;
|
||||
use axum::{
|
||||
body::Body,
|
||||
http,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
pub(super) async fn maybe_build_local_admin_usage_analytics_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
let route_kind = request_context
|
||||
.control_decision
|
||||
.as_ref()
|
||||
.and_then(|decision| decision.route_kind.as_deref());
|
||||
|
||||
match route_kind {
|
||||
Some("aggregation_stats")
|
||||
if request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/admin/usage/aggregation/stats" | "/api/admin/usage/aggregation/stats/"
|
||||
) =>
|
||||
{
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_usage_data_unavailable_response(
|
||||
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
|
||||
)));
|
||||
}
|
||||
|
||||
let query = request_context.request_query_string.as_deref();
|
||||
let group_by = query_param_value(query, "group_by")
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_ascii_lowercase();
|
||||
if !matches!(
|
||||
group_by.as_str(),
|
||||
"model" | "user" | "provider" | "api_format"
|
||||
) {
|
||||
return Ok(Some(admin_usage_bad_request_response(
|
||||
"Invalid group_by value: must be one of model, user, provider, api_format",
|
||||
)));
|
||||
}
|
||||
let limit = match admin_usage_parse_aggregation_limit(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
|
||||
};
|
||||
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
|
||||
};
|
||||
|
||||
let mut usage = list_usage_for_optional_range(
|
||||
state,
|
||||
time_range.as_ref(),
|
||||
&AdminStatsUsageFilter::default(),
|
||||
)
|
||||
.await?;
|
||||
usage.retain(|item| item.status != "pending" && item.status != "streaming");
|
||||
|
||||
let response = match group_by.as_str() {
|
||||
"model" => admin_usage_aggregation_by_model_json(&usage, limit),
|
||||
"user" => admin_usage_aggregation_by_user_json(state, &usage, limit).await?,
|
||||
"provider" => admin_usage_aggregation_by_provider_json(&usage, limit),
|
||||
"api_format" => admin_usage_aggregation_by_api_format_json(&usage, limit),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
return Ok(Some(Json(response).into_response()));
|
||||
}
|
||||
Some("heatmap")
|
||||
if request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/admin/usage/heatmap" | "/api/admin/usage/heatmap/"
|
||||
) =>
|
||||
{
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_usage_data_unavailable_response(
|
||||
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
|
||||
)));
|
||||
}
|
||||
let now_unix_secs = u64::try_from(chrono::Utc::now().timestamp()).unwrap_or_default();
|
||||
let created_from_unix_secs = now_unix_secs.saturating_sub(365 * 24 * 3600);
|
||||
let mut usage = state
|
||||
.list_usage_audits(&UsageAuditListQuery {
|
||||
created_from_unix_secs: Some(created_from_unix_secs),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
usage.retain(|item| item.status != "pending" && item.status != "streaming");
|
||||
return Ok(Some(Json(admin_usage_heatmap_json(&usage)).into_response()));
|
||||
}
|
||||
Some("cache_affinity_hit_analysis")
|
||||
if request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/admin/usage/cache-affinity/hit-analysis"
|
||||
| "/api/admin/usage/cache-affinity/hit-analysis/"
|
||||
) =>
|
||||
{
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_usage_data_unavailable_response(
|
||||
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
|
||||
)));
|
||||
}
|
||||
|
||||
let query = request_context.request_query_string.as_deref();
|
||||
let hours = match admin_usage_parse_recent_hours(query, 168) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
|
||||
};
|
||||
let user_id = query_param_value(query, "user_id");
|
||||
let api_key_id = query_param_value(query, "api_key_id");
|
||||
let usage =
|
||||
list_recent_completed_usage_for_cache_affinity(state, hours, user_id.as_deref())
|
||||
.await?;
|
||||
let filtered: Vec<_> = usage
|
||||
.into_iter()
|
||||
.filter(|item| {
|
||||
admin_usage_matches_optional_id(item.user_id.as_deref(), user_id.as_deref())
|
||||
&& admin_usage_matches_optional_id(
|
||||
item.api_key_id.as_deref(),
|
||||
api_key_id.as_deref(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let total_requests = filtered.len();
|
||||
let total_input_tokens: u64 = filtered.iter().map(|item| item.input_tokens).sum();
|
||||
let total_cache_read_tokens: u64 = filtered
|
||||
.iter()
|
||||
.map(|item| item.cache_read_input_tokens)
|
||||
.sum();
|
||||
let total_cache_creation_tokens: u64 = filtered
|
||||
.iter()
|
||||
.map(|item| item.cache_creation_input_tokens)
|
||||
.sum();
|
||||
let total_cache_read_cost: f64 =
|
||||
filtered.iter().map(|item| item.cache_read_cost_usd).sum();
|
||||
let total_cache_creation_cost: f64 = filtered
|
||||
.iter()
|
||||
.map(|item| item.cache_creation_cost_usd)
|
||||
.sum();
|
||||
let requests_with_cache_hit = filtered
|
||||
.iter()
|
||||
.filter(|item| item.cache_read_input_tokens > 0)
|
||||
.count();
|
||||
let total_context_tokens = total_input_tokens.saturating_add(total_cache_read_tokens);
|
||||
let token_cache_hit_rate = if total_context_tokens == 0 {
|
||||
0.0
|
||||
} else {
|
||||
round_to(
|
||||
total_cache_read_tokens as f64 / total_context_tokens as f64 * 100.0,
|
||||
2,
|
||||
)
|
||||
};
|
||||
let request_cache_hit_rate = if total_requests == 0 {
|
||||
0.0
|
||||
} else {
|
||||
round_to(
|
||||
requests_with_cache_hit as f64 / total_requests as f64 * 100.0,
|
||||
2,
|
||||
)
|
||||
};
|
||||
|
||||
return Ok(Some(
|
||||
Json(json!({
|
||||
"analysis_period_hours": hours,
|
||||
"total_requests": total_requests,
|
||||
"requests_with_cache_hit": requests_with_cache_hit,
|
||||
"request_cache_hit_rate": request_cache_hit_rate,
|
||||
"total_input_tokens": total_input_tokens,
|
||||
"total_cache_read_tokens": total_cache_read_tokens,
|
||||
"total_cache_creation_tokens": total_cache_creation_tokens,
|
||||
"token_cache_hit_rate": token_cache_hit_rate,
|
||||
"total_cache_read_cost_usd": round_to(total_cache_read_cost, 4),
|
||||
"total_cache_creation_cost_usd": round_to(total_cache_creation_cost, 4),
|
||||
"estimated_savings_usd": round_to(total_cache_read_cost * 9.0, 4),
|
||||
}))
|
||||
.into_response(),
|
||||
));
|
||||
}
|
||||
Some("cache_affinity_interval_timeline")
|
||||
if request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/admin/usage/cache-affinity/interval-timeline"
|
||||
| "/api/admin/usage/cache-affinity/interval-timeline/"
|
||||
) =>
|
||||
{
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_usage_data_unavailable_response(
|
||||
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
|
||||
)));
|
||||
}
|
||||
|
||||
let query = request_context.request_query_string.as_deref();
|
||||
let hours = match admin_usage_parse_recent_hours(query, 24) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
|
||||
};
|
||||
let limit = match admin_usage_parse_timeline_limit(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
|
||||
};
|
||||
let user_id = query_param_value(query, "user_id");
|
||||
let include_user_info = query_param_bool(query, "include_user_info", false);
|
||||
let usage =
|
||||
list_recent_completed_usage_for_cache_affinity(state, hours, user_id.as_deref())
|
||||
.await?;
|
||||
let mut grouped: BTreeMap<String, Vec<serde_json::Value>> = BTreeMap::new();
|
||||
let mut models = BTreeSet::new();
|
||||
let mut usernames_by_user_id = BTreeMap::new();
|
||||
|
||||
for (group_user_id, items) in admin_usage_group_completed_by_user(&usage) {
|
||||
if let Some(ref requested_user_id) = user_id {
|
||||
if &group_user_id != requested_user_id {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let mut previous_created_at_unix_secs = None;
|
||||
for item in items {
|
||||
if let Some(previous) = previous_created_at_unix_secs {
|
||||
let interval_minutes =
|
||||
item.created_at_unix_secs.saturating_sub(previous) as f64 / 60.0;
|
||||
if interval_minutes <= 120.0 {
|
||||
let mut point = json!({
|
||||
"x": unix_secs_to_rfc3339(item.created_at_unix_secs),
|
||||
"y": round_to(interval_minutes, 2),
|
||||
});
|
||||
if !item.model.trim().is_empty() {
|
||||
point["model"] = json!(item.model.clone());
|
||||
models.insert(item.model.clone());
|
||||
}
|
||||
if include_user_info && user_id.is_none() {
|
||||
point["user_id"] = json!(group_user_id.clone());
|
||||
if let Some(username) = item.username.clone() {
|
||||
usernames_by_user_id
|
||||
.entry(group_user_id.clone())
|
||||
.or_insert(username);
|
||||
}
|
||||
}
|
||||
grouped
|
||||
.entry(group_user_id.clone())
|
||||
.or_default()
|
||||
.push(point);
|
||||
}
|
||||
}
|
||||
previous_created_at_unix_secs = Some(item.created_at_unix_secs);
|
||||
}
|
||||
}
|
||||
|
||||
if include_user_info && user_id.is_none() && state.has_user_data_reader() {
|
||||
let user_ids: Vec<_> = grouped.keys().cloned().collect();
|
||||
let user_map: BTreeMap<_, _> = state
|
||||
.list_users_by_ids(&user_ids)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|user| (user.id, user.username))
|
||||
.collect();
|
||||
for (user_id, username) in user_map {
|
||||
usernames_by_user_id.insert(user_id, username);
|
||||
}
|
||||
}
|
||||
|
||||
let total_points_before_limit: usize = grouped.values().map(Vec::len).sum();
|
||||
let points: Vec<serde_json::Value> = if include_user_info && user_id.is_none() {
|
||||
let user_limits =
|
||||
admin_usage_proportional_limits(&grouped, limit, total_points_before_limit);
|
||||
let mut selected = Vec::new();
|
||||
for (group_user_id, mut items) in grouped {
|
||||
let take = user_limits
|
||||
.get(&group_user_id)
|
||||
.copied()
|
||||
.unwrap_or(items.len());
|
||||
selected.extend(items.drain(..std::cmp::min(take, items.len())));
|
||||
}
|
||||
selected.sort_by(admin_usage_point_sort_key);
|
||||
selected
|
||||
} else {
|
||||
let mut selected = grouped
|
||||
.into_values()
|
||||
.flatten()
|
||||
.collect::<Vec<serde_json::Value>>();
|
||||
selected.sort_by(admin_usage_point_sort_key);
|
||||
if selected.len() > limit {
|
||||
selected.truncate(limit);
|
||||
}
|
||||
selected
|
||||
};
|
||||
|
||||
let mut response = json!({
|
||||
"analysis_period_hours": hours,
|
||||
"total_points": points.len(),
|
||||
"points": points,
|
||||
});
|
||||
if include_user_info && user_id.is_none() {
|
||||
response["users"] = json!(usernames_by_user_id);
|
||||
}
|
||||
if !models.is_empty() {
|
||||
response["models"] = json!(models.into_iter().collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
return Ok(Some(Json(response).into_response()));
|
||||
}
|
||||
Some("cache_affinity_ttl_analysis")
|
||||
if request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/admin/usage/cache-affinity/ttl-analysis"
|
||||
| "/api/admin/usage/cache-affinity/ttl-analysis/"
|
||||
) =>
|
||||
{
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_usage_data_unavailable_response(
|
||||
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
|
||||
)));
|
||||
}
|
||||
|
||||
let query = request_context.request_query_string.as_deref();
|
||||
let hours = match admin_usage_parse_recent_hours(query, 168) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
|
||||
};
|
||||
let user_id = query_param_value(query, "user_id");
|
||||
let api_key_id = query_param_value(query, "api_key_id");
|
||||
let group_by_api_key = api_key_id.is_some();
|
||||
let usage =
|
||||
list_recent_completed_usage_for_cache_affinity(state, hours, user_id.as_deref())
|
||||
.await?;
|
||||
|
||||
let grouped = if group_by_api_key {
|
||||
admin_usage_group_completed_by_api_key(&usage, api_key_id.as_deref())
|
||||
} else {
|
||||
admin_usage_group_completed_by_user(&usage)
|
||||
.into_iter()
|
||||
.filter(|(group_user_id, _)| {
|
||||
admin_usage_matches_optional_id(
|
||||
Some(group_user_id.as_str()),
|
||||
user_id.as_deref(),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
|
||||
let user_map: BTreeMap<String, aether_data::repository::users::StoredUserSummary> =
|
||||
if !group_by_api_key && state.has_user_data_reader() {
|
||||
let user_ids = grouped.keys().cloned().collect::<Vec<_>>();
|
||||
state
|
||||
.list_users_by_ids(&user_ids)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|user| (user.id.clone(), user))
|
||||
.collect()
|
||||
} else {
|
||||
BTreeMap::new()
|
||||
};
|
||||
|
||||
let mut ttl_distribution = json!({
|
||||
"5min": 0_u64,
|
||||
"15min": 0_u64,
|
||||
"30min": 0_u64,
|
||||
"60min": 0_u64,
|
||||
});
|
||||
let mut users = Vec::new();
|
||||
|
||||
for (group_id, items) in grouped {
|
||||
let intervals = admin_usage_collect_request_intervals_minutes(&items);
|
||||
if intervals.len() < 2 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let within_5min = intervals.iter().filter(|value| **value <= 5.0).count() as u64;
|
||||
let within_15min = intervals
|
||||
.iter()
|
||||
.filter(|value| **value > 5.0 && **value <= 15.0)
|
||||
.count() as u64;
|
||||
let within_30min = intervals
|
||||
.iter()
|
||||
.filter(|value| **value > 15.0 && **value <= 30.0)
|
||||
.count() as u64;
|
||||
let within_60min = intervals
|
||||
.iter()
|
||||
.filter(|value| **value > 30.0 && **value <= 60.0)
|
||||
.count() as u64;
|
||||
let over_60min = intervals.iter().filter(|value| **value > 60.0).count() as u64;
|
||||
let request_count = intervals.len() as u64;
|
||||
let p50 = admin_usage_percentile_cont(&intervals, 0.5);
|
||||
let p75 = admin_usage_percentile_cont(&intervals, 0.75);
|
||||
let p90 = admin_usage_percentile_cont(&intervals, 0.90);
|
||||
let avg_interval = intervals.iter().copied().sum::<f64>() / intervals.len() as f64;
|
||||
let min_interval = intervals.iter().copied().reduce(f64::min);
|
||||
let max_interval = intervals.iter().copied().reduce(f64::max);
|
||||
let recommended_ttl = admin_usage_calculate_recommended_ttl(p75, p90);
|
||||
match recommended_ttl {
|
||||
0..=5 => {
|
||||
ttl_distribution["5min"] = json!(ttl_distribution["5min"]
|
||||
.as_u64()
|
||||
.unwrap_or(0)
|
||||
.saturating_add(1))
|
||||
}
|
||||
6..=15 => {
|
||||
ttl_distribution["15min"] = json!(ttl_distribution["15min"]
|
||||
.as_u64()
|
||||
.unwrap_or(0)
|
||||
.saturating_add(1))
|
||||
}
|
||||
16..=30 => {
|
||||
ttl_distribution["30min"] = json!(ttl_distribution["30min"]
|
||||
.as_u64()
|
||||
.unwrap_or(0)
|
||||
.saturating_add(1))
|
||||
}
|
||||
_ => {
|
||||
ttl_distribution["60min"] = json!(ttl_distribution["60min"]
|
||||
.as_u64()
|
||||
.unwrap_or(0)
|
||||
.saturating_add(1))
|
||||
}
|
||||
}
|
||||
|
||||
let (username, email) = if group_by_api_key {
|
||||
(Value::Null, Value::Null)
|
||||
} else if let Some(user) = user_map.get(&group_id) {
|
||||
(
|
||||
json!(user.username.clone()),
|
||||
json!(user.email.clone().unwrap_or_default()),
|
||||
)
|
||||
} else {
|
||||
(Value::Null, Value::Null)
|
||||
};
|
||||
|
||||
users.push(json!({
|
||||
"group_id": group_id,
|
||||
"username": username,
|
||||
"email": email,
|
||||
"request_count": request_count,
|
||||
"interval_distribution": {
|
||||
"within_5min": within_5min,
|
||||
"within_15min": within_15min,
|
||||
"within_30min": within_30min,
|
||||
"within_60min": within_60min,
|
||||
"over_60min": over_60min,
|
||||
},
|
||||
"interval_percentages": {
|
||||
"within_5min": round_to(within_5min as f64 / request_count as f64 * 100.0, 1),
|
||||
"within_15min": round_to(within_15min as f64 / request_count as f64 * 100.0, 1),
|
||||
"within_30min": round_to(within_30min as f64 / request_count as f64 * 100.0, 1),
|
||||
"within_60min": round_to(within_60min as f64 / request_count as f64 * 100.0, 1),
|
||||
"over_60min": round_to(over_60min as f64 / request_count as f64 * 100.0, 1),
|
||||
},
|
||||
"percentiles": {
|
||||
"p50": p50.map(|value| round_to(value, 2)),
|
||||
"p75": p75.map(|value| round_to(value, 2)),
|
||||
"p90": p90.map(|value| round_to(value, 2)),
|
||||
},
|
||||
"avg_interval_minutes": round_to(avg_interval, 2),
|
||||
"min_interval_minutes": min_interval.map(|value| round_to(value, 2)),
|
||||
"max_interval_minutes": max_interval.map(|value| round_to(value, 2)),
|
||||
"recommended_ttl_minutes": recommended_ttl,
|
||||
"recommendation_reason": admin_usage_ttl_recommendation_reason(recommended_ttl, p75, p90),
|
||||
}));
|
||||
}
|
||||
|
||||
users.sort_by(|left, right| {
|
||||
right["request_count"]
|
||||
.as_u64()
|
||||
.unwrap_or(0)
|
||||
.cmp(&left["request_count"].as_u64().unwrap_or(0))
|
||||
.then_with(|| {
|
||||
left["group_id"]
|
||||
.as_str()
|
||||
.unwrap_or_default()
|
||||
.cmp(right["group_id"].as_str().unwrap_or_default())
|
||||
})
|
||||
});
|
||||
|
||||
return Ok(Some(
|
||||
Json(json!({
|
||||
"analysis_period_hours": hours,
|
||||
"total_users_analyzed": users.len(),
|
||||
"ttl_distribution": ttl_distribution,
|
||||
"users": users,
|
||||
}))
|
||||
.into_response(),
|
||||
));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
use super::super::stats::round_to;
|
||||
use super::analytics::{
|
||||
admin_usage_provider_key_name, admin_usage_provider_key_names, admin_usage_record_json,
|
||||
};
|
||||
use super::helpers::{
|
||||
admin_usage_bad_request_response, admin_usage_data_unavailable_response,
|
||||
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
|
||||
};
|
||||
use super::replay::{
|
||||
admin_usage_build_curl_command, admin_usage_curl_headers, admin_usage_curl_url,
|
||||
admin_usage_headers_from_value, admin_usage_id_from_action_path,
|
||||
admin_usage_id_from_detail_path, admin_usage_resolve_request_preview_body,
|
||||
build_admin_usage_replay_response,
|
||||
};
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use crate::handlers::admin::shared::{attach_admin_audit_response, query_param_bool};
|
||||
use crate::{AppState, GatewayError};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
pub(super) async fn maybe_build_local_admin_usage_detail_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
request_body: Option<&axum::body::Bytes>,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
let route_kind = request_context
|
||||
.control_decision
|
||||
.as_ref()
|
||||
.and_then(|decision| decision.route_kind.as_deref());
|
||||
|
||||
match route_kind {
|
||||
Some("curl")
|
||||
if request_context.request_method == http::Method::GET
|
||||
&& request_context
|
||||
.request_path
|
||||
.starts_with("/api/admin/usage/")
|
||||
&& request_context.request_path.ends_with("/curl") =>
|
||||
{
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_usage_data_unavailable_response(
|
||||
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
|
||||
)));
|
||||
}
|
||||
|
||||
let Some(usage_id) =
|
||||
admin_usage_id_from_action_path(&request_context.request_path, "/curl")
|
||||
else {
|
||||
return Ok(Some(admin_usage_bad_request_response("usage_id 无效")));
|
||||
};
|
||||
|
||||
let Some(item) = state
|
||||
.data
|
||||
.find_request_usage_by_id(&usage_id)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
else {
|
||||
return Ok(Some(
|
||||
(
|
||||
http::StatusCode::NOT_FOUND,
|
||||
Json(json!({ "detail": "Usage record not found" })),
|
||||
)
|
||||
.into_response(),
|
||||
));
|
||||
};
|
||||
|
||||
let endpoint = if let Some(endpoint_id) = item.provider_endpoint_id.as_ref() {
|
||||
state
|
||||
.read_provider_catalog_endpoints_by_ids(std::slice::from_ref(endpoint_id))
|
||||
.await?
|
||||
.into_iter()
|
||||
.next()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let url = endpoint
|
||||
.as_ref()
|
||||
.map(|endpoint| admin_usage_curl_url(endpoint, &item));
|
||||
let headers_json = item
|
||||
.provider_request_headers
|
||||
.clone()
|
||||
.or_else(|| item.request_headers.clone());
|
||||
let headers = headers_json
|
||||
.as_ref()
|
||||
.and_then(admin_usage_headers_from_value)
|
||||
.filter(|headers| !headers.is_empty())
|
||||
.unwrap_or_else(admin_usage_curl_headers);
|
||||
let body = item
|
||||
.provider_request_body
|
||||
.clone()
|
||||
.or_else(|| item.request_body.clone())
|
||||
.unwrap_or_else(|| admin_usage_resolve_request_preview_body(&item, None));
|
||||
let curl = admin_usage_build_curl_command(url.as_deref(), &headers, Some(&body));
|
||||
|
||||
return Ok(Some(attach_admin_audit_response(
|
||||
Json(json!({
|
||||
"url": url,
|
||||
"method": "POST",
|
||||
"headers": headers_json.unwrap_or_else(|| json!(headers.clone())),
|
||||
"body": body,
|
||||
"curl": curl,
|
||||
"original_request_body_available": item.request_body.is_some() || item.provider_request_body.is_some(),
|
||||
}))
|
||||
.into_response(),
|
||||
"admin_usage_curl_viewed",
|
||||
"view_usage_curl_replay",
|
||||
"usage_record",
|
||||
&item.id,
|
||||
)));
|
||||
}
|
||||
Some("replay") => {
|
||||
let mut response =
|
||||
build_admin_usage_replay_response(state, request_context, request_body).await?;
|
||||
if response.status().is_success() {
|
||||
if let Some(usage_id) =
|
||||
admin_usage_id_from_action_path(&request_context.request_path, "/replay")
|
||||
{
|
||||
response = attach_admin_audit_response(
|
||||
response,
|
||||
"admin_usage_replay_preview_generated",
|
||||
"preview_usage_replay",
|
||||
"usage_record",
|
||||
&usage_id,
|
||||
);
|
||||
}
|
||||
}
|
||||
return Ok(Some(response));
|
||||
}
|
||||
Some("detail")
|
||||
if request_context.request_method == http::Method::GET
|
||||
&& request_context
|
||||
.request_path
|
||||
.starts_with("/api/admin/usage/") =>
|
||||
{
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_usage_data_unavailable_response(
|
||||
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
|
||||
)));
|
||||
}
|
||||
|
||||
let Some(usage_id) = admin_usage_id_from_detail_path(&request_context.request_path)
|
||||
else {
|
||||
return Ok(Some(admin_usage_bad_request_response("usage_id 无效")));
|
||||
};
|
||||
let include_bodies = query_param_bool(
|
||||
request_context.request_query_string.as_deref(),
|
||||
"include_bodies",
|
||||
true,
|
||||
);
|
||||
|
||||
let Some(item) = state
|
||||
.data
|
||||
.find_request_usage_by_id(&usage_id)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
else {
|
||||
return Ok(Some(
|
||||
(
|
||||
http::StatusCode::NOT_FOUND,
|
||||
Json(json!({ "detail": "Usage record not found" })),
|
||||
)
|
||||
.into_response(),
|
||||
));
|
||||
};
|
||||
|
||||
let users_by_id: BTreeMap<String, aether_data::repository::users::StoredUserSummary> =
|
||||
if state.has_user_data_reader() {
|
||||
if let Some(user_id) = item.user_id.as_ref() {
|
||||
state
|
||||
.list_users_by_ids(std::slice::from_ref(user_id))
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|user| (user.id.clone(), user))
|
||||
.collect()
|
||||
} else {
|
||||
BTreeMap::new()
|
||||
}
|
||||
} else {
|
||||
BTreeMap::new()
|
||||
};
|
||||
let provider_key_names =
|
||||
admin_usage_provider_key_names(state, std::slice::from_ref(&item)).await?;
|
||||
let provider_key_name = admin_usage_provider_key_name(&item, &provider_key_names);
|
||||
|
||||
let mut payload =
|
||||
admin_usage_record_json(&item, &users_by_id, provider_key_name.as_deref());
|
||||
let request_body = item
|
||||
.request_body
|
||||
.clone()
|
||||
.unwrap_or_else(|| admin_usage_resolve_request_preview_body(&item, None));
|
||||
let request_preview_source = if item.request_body.is_some() {
|
||||
"stored_original"
|
||||
} else {
|
||||
"local_reconstruction"
|
||||
};
|
||||
let mut metadata = match item.request_metadata.clone() {
|
||||
Some(serde_json::Value::Object(object)) => serde_json::Value::Object(object),
|
||||
Some(value) => json!({ "request_metadata": value }),
|
||||
None => json!({}),
|
||||
};
|
||||
if let Some(object) = metadata.as_object_mut() {
|
||||
object.insert(
|
||||
"request_preview_source".to_string(),
|
||||
json!(request_preview_source),
|
||||
);
|
||||
object.insert(
|
||||
"original_request_body_available".to_string(),
|
||||
json!(item.request_body.is_some()),
|
||||
);
|
||||
object.insert(
|
||||
"original_response_body_available".to_string(),
|
||||
json!(item.response_body.is_some() || item.client_response_body.is_some()),
|
||||
);
|
||||
}
|
||||
payload["user"] = match item.user_id.as_ref() {
|
||||
Some(user_id) => json!({
|
||||
"id": user_id,
|
||||
"email": payload["user_email"].clone(),
|
||||
"username": payload["username"].clone(),
|
||||
}),
|
||||
None => Value::Null,
|
||||
};
|
||||
payload["request_id"] = json!(item.request_id);
|
||||
payload["billing_status"] = json!(item.billing_status);
|
||||
payload["request_type"] = json!(item.request_type);
|
||||
payload["provider_id"] = json!(item.provider_id);
|
||||
payload["provider_endpoint_id"] = json!(item.provider_endpoint_id);
|
||||
payload["provider_api_key_id"] = json!(item.provider_api_key_id);
|
||||
payload["error_category"] = json!(item.error_category);
|
||||
payload["cache_creation_cost"] = json!(round_to(item.cache_creation_cost_usd, 6));
|
||||
payload["cache_read_cost"] = json!(round_to(item.cache_read_cost_usd, 6));
|
||||
payload["request_cost"] = json!(round_to(item.total_cost_usd, 6));
|
||||
payload["request_headers"] = item
|
||||
.request_headers
|
||||
.clone()
|
||||
.unwrap_or_else(|| json!(admin_usage_curl_headers()));
|
||||
payload["provider_request_headers"] = item
|
||||
.provider_request_headers
|
||||
.clone()
|
||||
.unwrap_or_else(|| json!(admin_usage_curl_headers()));
|
||||
payload["response_headers"] = item.response_headers.clone().unwrap_or(Value::Null);
|
||||
payload["client_response_headers"] =
|
||||
item.client_response_headers.clone().unwrap_or(Value::Null);
|
||||
payload["metadata"] = metadata;
|
||||
payload["has_request_body"] = json!(true);
|
||||
payload["has_provider_request_body"] = json!(item.provider_request_body.is_some());
|
||||
payload["has_response_body"] = json!(item.response_body.is_some());
|
||||
payload["has_client_response_body"] = json!(item.client_response_body.is_some());
|
||||
payload["tiered_pricing"] = serde_json::Value::Null;
|
||||
if include_bodies {
|
||||
payload["request_body"] = request_body;
|
||||
payload["provider_request_body"] =
|
||||
item.provider_request_body.clone().unwrap_or(Value::Null);
|
||||
payload["response_body"] = item.response_body.clone().unwrap_or(Value::Null);
|
||||
payload["client_response_body"] =
|
||||
item.client_response_body.clone().unwrap_or(Value::Null);
|
||||
} else {
|
||||
payload["request_body"] = serde_json::Value::Null;
|
||||
payload["provider_request_body"] = serde_json::Value::Null;
|
||||
payload["response_body"] = serde_json::Value::Null;
|
||||
payload["client_response_body"] = serde_json::Value::Null;
|
||||
}
|
||||
|
||||
return Ok(Some(attach_admin_audit_response(
|
||||
Json(payload).into_response(),
|
||||
"admin_usage_detail_viewed",
|
||||
"view_usage_detail",
|
||||
"usage_record",
|
||||
&item.id,
|
||||
)));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
use axum::{
|
||||
body::Body,
|
||||
http,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
pub(crate) const ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL: &str = "Admin usage data unavailable";
|
||||
|
||||
pub(crate) fn admin_usage_data_unavailable_response(detail: &'static str) -> Response<Body> {
|
||||
(
|
||||
http::StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({ "detail": detail })),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
pub(crate) fn admin_usage_bad_request_response(detail: impl Into<String>) -> Response<Body> {
|
||||
(
|
||||
http::StatusCode::BAD_REQUEST,
|
||||
Json(json!({ "detail": detail.into() })),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use crate::{AppState, GatewayError};
|
||||
use axum::{
|
||||
body::Body,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
|
||||
mod analytics;
|
||||
mod analytics_routes;
|
||||
mod detail_routes;
|
||||
mod helpers;
|
||||
mod replay;
|
||||
mod summary_routes;
|
||||
|
||||
pub(crate) async fn maybe_build_local_admin_usage_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
request_body: Option<&axum::body::Bytes>,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
let Some(decision) = request_context.control_decision.as_ref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
if decision.route_family.as_deref() != Some("usage_manage") {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if let Some(response) = detail_routes::maybe_build_local_admin_usage_detail_response(
|
||||
state,
|
||||
request_context,
|
||||
request_body,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
return Ok(Some(response));
|
||||
}
|
||||
|
||||
if let Some(response) =
|
||||
summary_routes::maybe_build_local_admin_usage_summary_response(state, request_context)
|
||||
.await?
|
||||
{
|
||||
return Ok(Some(response));
|
||||
}
|
||||
|
||||
if let Some(response) =
|
||||
analytics_routes::maybe_build_local_admin_usage_analytics_response(state, request_context)
|
||||
.await?
|
||||
{
|
||||
return Ok(Some(response));
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
@@ -0,0 +1,390 @@
|
||||
use super::helpers::{
|
||||
admin_usage_bad_request_response, admin_usage_data_unavailable_response,
|
||||
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
|
||||
};
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use crate::{AppState, GatewayError};
|
||||
use aether_data_contracts::repository::{
|
||||
provider_catalog::StoredProviderCatalogEndpoint, usage::StoredRequestUsageAudit,
|
||||
};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
fn admin_usage_id_from_path_suffix(request_path: &str, suffix: Option<&str>) -> Option<String> {
|
||||
let mut value = request_path
|
||||
.strip_prefix("/api/admin/usage/")?
|
||||
.trim()
|
||||
.trim_matches('/')
|
||||
.to_string();
|
||||
if let Some(suffix) = suffix {
|
||||
value = value.strip_suffix(suffix)?.trim_matches('/').to_string();
|
||||
}
|
||||
if value.is_empty() || value.contains('/') {
|
||||
None
|
||||
} else {
|
||||
Some(value)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_id_from_detail_path(request_path: &str) -> Option<String> {
|
||||
admin_usage_id_from_path_suffix(request_path, None)
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_id_from_action_path(request_path: &str, action: &str) -> Option<String> {
|
||||
admin_usage_id_from_path_suffix(request_path, Some(action))
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
struct AdminUsageReplayRequest {
|
||||
#[serde(default, alias = "target_provider_id")]
|
||||
provider_id: Option<String>,
|
||||
#[serde(default, alias = "target_endpoint_id")]
|
||||
endpoint_id: Option<String>,
|
||||
#[serde(default, alias = "target_api_key_id")]
|
||||
api_key_id: Option<String>,
|
||||
#[serde(default)]
|
||||
body_override: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
fn admin_usage_resolve_replay_mode(same_provider: bool, same_endpoint: bool) -> &'static str {
|
||||
if same_provider && same_endpoint {
|
||||
"same_endpoint_reuse"
|
||||
} else if same_provider {
|
||||
"same_provider_remap"
|
||||
} else {
|
||||
"cross_provider_remap"
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_resolve_request_preview_body(
|
||||
item: &StoredRequestUsageAudit,
|
||||
body_override: Option<serde_json::Value>,
|
||||
) -> serde_json::Value {
|
||||
let resolved_model = item.model.clone();
|
||||
let mut request_body = body_override
|
||||
.or_else(|| item.request_body.clone())
|
||||
.unwrap_or_else(|| {
|
||||
json!({
|
||||
"model": resolved_model,
|
||||
"stream": item.is_stream,
|
||||
})
|
||||
});
|
||||
if let Some(body) = request_body.as_object_mut() {
|
||||
body.entry("model".to_string())
|
||||
.or_insert_with(|| json!(resolved_model));
|
||||
if !body.contains_key("stream") {
|
||||
body.insert("stream".to_string(), json!(item.is_stream));
|
||||
}
|
||||
if let Some(target_model) = item
|
||||
.target_model
|
||||
.as_ref()
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
{
|
||||
body.entry("target_model".to_string())
|
||||
.or_insert_with(|| json!(target_model));
|
||||
}
|
||||
if let Some(request_type) = item
|
||||
.request_type
|
||||
.as_ref()
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
{
|
||||
body.entry("request_type".to_string())
|
||||
.or_insert_with(|| json!(request_type));
|
||||
}
|
||||
if let Some(api_format) = item
|
||||
.api_format
|
||||
.as_ref()
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
{
|
||||
body.entry("api_format".to_string())
|
||||
.or_insert_with(|| json!(api_format));
|
||||
}
|
||||
}
|
||||
request_body
|
||||
}
|
||||
|
||||
pub(super) async fn build_admin_usage_replay_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
request_body: Option<&axum::body::Bytes>,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
if !state.has_usage_data_reader() || !state.has_provider_catalog_data_reader() {
|
||||
return Ok(admin_usage_data_unavailable_response(
|
||||
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
|
||||
));
|
||||
}
|
||||
|
||||
let Some(usage_id) = admin_usage_id_from_action_path(&request_context.request_path, "/replay")
|
||||
else {
|
||||
return Ok(admin_usage_bad_request_response("usage_id 无效"));
|
||||
};
|
||||
|
||||
let payload = match request_body {
|
||||
Some(body) if !body.is_empty() => {
|
||||
serde_json::from_slice::<AdminUsageReplayRequest>(body).unwrap_or_default()
|
||||
}
|
||||
_ => AdminUsageReplayRequest::default(),
|
||||
};
|
||||
|
||||
let Some(item) = state
|
||||
.data
|
||||
.find_request_usage_by_id(&usage_id)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
else {
|
||||
return Ok((
|
||||
http::StatusCode::NOT_FOUND,
|
||||
Json(json!({ "detail": "Usage record not found" })),
|
||||
)
|
||||
.into_response());
|
||||
};
|
||||
|
||||
let target_provider_id = payload
|
||||
.provider_id
|
||||
.clone()
|
||||
.or_else(|| item.provider_id.clone())
|
||||
.map(|value| value.trim().to_string())
|
||||
.filter(|value| !value.is_empty());
|
||||
let Some(target_provider_id) = target_provider_id else {
|
||||
return Ok(admin_usage_bad_request_response(
|
||||
"Replay target provider is unavailable",
|
||||
));
|
||||
};
|
||||
let Some(target_provider) = state
|
||||
.read_provider_catalog_providers_by_ids(std::slice::from_ref(&target_provider_id))
|
||||
.await?
|
||||
.into_iter()
|
||||
.next()
|
||||
else {
|
||||
return Ok((
|
||||
http::StatusCode::NOT_FOUND,
|
||||
Json(json!({ "detail": format!("Provider {target_provider_id} 不存在") })),
|
||||
)
|
||||
.into_response());
|
||||
};
|
||||
|
||||
let requested_endpoint_id = payload
|
||||
.endpoint_id
|
||||
.clone()
|
||||
.map(|value| value.trim().to_string())
|
||||
.filter(|value| !value.is_empty());
|
||||
let target_endpoint = if let Some(endpoint_id) = requested_endpoint_id.clone() {
|
||||
let Some(endpoint) = state
|
||||
.read_provider_catalog_endpoints_by_ids(std::slice::from_ref(&endpoint_id))
|
||||
.await?
|
||||
.into_iter()
|
||||
.next()
|
||||
else {
|
||||
return Ok((
|
||||
http::StatusCode::NOT_FOUND,
|
||||
Json(json!({ "detail": format!("Endpoint {endpoint_id} 不存在") })),
|
||||
)
|
||||
.into_response());
|
||||
};
|
||||
if endpoint.provider_id != target_provider.id {
|
||||
return Ok(admin_usage_bad_request_response(
|
||||
"Target endpoint does not belong to the target provider",
|
||||
));
|
||||
}
|
||||
endpoint
|
||||
} else {
|
||||
let preferred_endpoint_id = item
|
||||
.provider_endpoint_id
|
||||
.clone()
|
||||
.filter(|_| item.provider_id.as_deref() == Some(target_provider.id.as_str()));
|
||||
if let Some(endpoint_id) = preferred_endpoint_id {
|
||||
if let Some(endpoint) = state
|
||||
.read_provider_catalog_endpoints_by_ids(std::slice::from_ref(&endpoint_id))
|
||||
.await?
|
||||
.into_iter()
|
||||
.find(|endpoint| endpoint.provider_id == target_provider.id)
|
||||
{
|
||||
endpoint
|
||||
} else {
|
||||
let mut endpoints = state
|
||||
.list_provider_catalog_endpoints_by_provider_ids(std::slice::from_ref(
|
||||
&target_provider.id,
|
||||
))
|
||||
.await?;
|
||||
let preferred_api_format = item
|
||||
.endpoint_api_format
|
||||
.as_deref()
|
||||
.or(item.api_format.as_deref())
|
||||
.unwrap_or_default();
|
||||
endpoints
|
||||
.iter()
|
||||
.find(|endpoint| {
|
||||
endpoint.is_active && endpoint.api_format == preferred_api_format
|
||||
})
|
||||
.cloned()
|
||||
.or_else(|| {
|
||||
endpoints
|
||||
.iter()
|
||||
.find(|endpoint| endpoint.is_active)
|
||||
.cloned()
|
||||
})
|
||||
.or_else(|| endpoints.into_iter().next())
|
||||
.ok_or_else(|| {
|
||||
GatewayError::Internal("target provider has no endpoints".to_string())
|
||||
})?
|
||||
}
|
||||
} else {
|
||||
let mut endpoints = state
|
||||
.list_provider_catalog_endpoints_by_provider_ids(std::slice::from_ref(
|
||||
&target_provider.id,
|
||||
))
|
||||
.await?;
|
||||
let preferred_api_format = item
|
||||
.endpoint_api_format
|
||||
.as_deref()
|
||||
.or(item.api_format.as_deref())
|
||||
.unwrap_or_default();
|
||||
endpoints
|
||||
.iter()
|
||||
.find(|endpoint| endpoint.is_active && endpoint.api_format == preferred_api_format)
|
||||
.cloned()
|
||||
.or_else(|| {
|
||||
endpoints
|
||||
.iter()
|
||||
.find(|endpoint| endpoint.is_active)
|
||||
.cloned()
|
||||
})
|
||||
.or_else(|| endpoints.into_iter().next())
|
||||
.ok_or_else(|| {
|
||||
GatewayError::Internal("target provider has no endpoints".to_string())
|
||||
})?
|
||||
}
|
||||
};
|
||||
|
||||
let same_provider = item.provider_id.as_deref() == Some(target_provider.id.as_str());
|
||||
let same_endpoint = item.provider_endpoint_id.as_deref() == Some(target_endpoint.id.as_str());
|
||||
let resolved_model = item.model.clone();
|
||||
let mapping_source = "none";
|
||||
let request_body = admin_usage_resolve_request_preview_body(&item, payload.body_override);
|
||||
|
||||
let url = admin_usage_curl_url(&target_endpoint, &item);
|
||||
let headers = admin_usage_curl_headers();
|
||||
let curl = admin_usage_build_curl_command(Some(&url), &headers, Some(&request_body));
|
||||
Ok(Json(json!({
|
||||
"dry_run": true,
|
||||
"usage_id": item.id,
|
||||
"request_id": item.request_id,
|
||||
"mode": admin_usage_resolve_replay_mode(same_provider, same_endpoint),
|
||||
"target_provider_id": target_provider.id,
|
||||
"target_provider_name": target_provider.name,
|
||||
"target_endpoint_id": target_endpoint.id,
|
||||
"target_api_key_id": payload.api_key_id.or(item.provider_api_key_id.clone()),
|
||||
"target_api_format": target_endpoint.api_format,
|
||||
"resolved_model": resolved_model,
|
||||
"mapping_source": mapping_source,
|
||||
"method": "POST",
|
||||
"url": url,
|
||||
"request_headers": headers,
|
||||
"request_body": request_body,
|
||||
"original_request_body_available": item.request_body.is_some(),
|
||||
"note": "Rust local replay currently exposes a dry-run plan and does not dispatch upstream",
|
||||
"curl": curl,
|
||||
}))
|
||||
.into_response())
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_headers_from_value(
|
||||
value: &serde_json::Value,
|
||||
) -> Option<BTreeMap<String, String>> {
|
||||
let object = value.as_object()?;
|
||||
Some(BTreeMap::from_iter(object.iter().filter_map(
|
||||
|(key, value)| {
|
||||
if value.is_null() {
|
||||
return None;
|
||||
}
|
||||
let value = value
|
||||
.as_str()
|
||||
.map(ToOwned::to_owned)
|
||||
.unwrap_or_else(|| value.to_string());
|
||||
Some((key.clone(), value))
|
||||
},
|
||||
)))
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_curl_headers() -> BTreeMap<String, String> {
|
||||
BTreeMap::from([("Content-Type".to_string(), "application/json".to_string())])
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_curl_url(
|
||||
endpoint: &StoredProviderCatalogEndpoint,
|
||||
item: &StoredRequestUsageAudit,
|
||||
) -> String {
|
||||
let api_format = item
|
||||
.endpoint_api_format
|
||||
.as_deref()
|
||||
.or(item.api_format.as_deref())
|
||||
.unwrap_or(endpoint.api_format.as_str());
|
||||
|
||||
if let Some(custom_path) = endpoint
|
||||
.custom_path
|
||||
.as_deref()
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
return crate::provider_transport::url::build_passthrough_path_url(
|
||||
&endpoint.base_url,
|
||||
custom_path,
|
||||
None,
|
||||
&[],
|
||||
)
|
||||
.unwrap_or_else(|| endpoint.base_url.clone());
|
||||
}
|
||||
|
||||
match api_format {
|
||||
value if value.starts_with("claude:") => {
|
||||
crate::provider_transport::url::build_claude_messages_url(&endpoint.base_url, None)
|
||||
}
|
||||
value if value.starts_with("gemini:") => {
|
||||
crate::provider_transport::url::build_gemini_content_url(
|
||||
&endpoint.base_url,
|
||||
item.target_model.as_deref().unwrap_or(item.model.as_str()),
|
||||
item.is_stream,
|
||||
None,
|
||||
)
|
||||
.unwrap_or_else(|| endpoint.base_url.clone())
|
||||
}
|
||||
value if value.starts_with("openai:") => {
|
||||
crate::provider_transport::url::build_openai_chat_url(&endpoint.base_url, None)
|
||||
}
|
||||
_ => endpoint.base_url.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn admin_usage_curl_shell_quote(value: &str) -> String {
|
||||
format!("'{}'", value.replace('\'', "'\"'\"'"))
|
||||
}
|
||||
|
||||
pub(super) fn admin_usage_build_curl_command(
|
||||
url: Option<&str>,
|
||||
headers: &BTreeMap<String, String>,
|
||||
body: Option<&serde_json::Value>,
|
||||
) -> String {
|
||||
let mut parts = vec!["curl".to_string()];
|
||||
if let Some(url) = url {
|
||||
parts.push(admin_usage_curl_shell_quote(url));
|
||||
}
|
||||
parts.push("-X POST".to_string());
|
||||
for (key, value) in headers {
|
||||
parts.push(format!(
|
||||
"-H {}",
|
||||
admin_usage_curl_shell_quote(&format!("{key}: {value}"))
|
||||
));
|
||||
}
|
||||
if let Some(body) = body {
|
||||
parts.push(format!(
|
||||
"-d {}",
|
||||
admin_usage_curl_shell_quote(&body.to_string())
|
||||
));
|
||||
}
|
||||
parts.join(" \\\n ")
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
use super::super::stats::{
|
||||
aggregate_usage_stats, list_usage_for_optional_range, round_to, AdminStatsTimeRange,
|
||||
AdminStatsUsageFilter,
|
||||
};
|
||||
use super::analytics::{
|
||||
admin_usage_matches_api_format, admin_usage_matches_eq, admin_usage_matches_search,
|
||||
admin_usage_matches_status, admin_usage_matches_username, admin_usage_parse_ids,
|
||||
admin_usage_parse_limit, admin_usage_parse_offset, admin_usage_provider_key_name,
|
||||
admin_usage_provider_key_names, admin_usage_record_json, admin_usage_total_tokens,
|
||||
};
|
||||
use super::helpers::{
|
||||
admin_usage_bad_request_response, admin_usage_data_unavailable_response,
|
||||
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
|
||||
};
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use crate::handlers::admin::shared::query_param_value;
|
||||
use crate::{AppState, GatewayError};
|
||||
use aether_data_contracts::repository::usage::UsageAuditListQuery;
|
||||
use axum::{
|
||||
body::Body,
|
||||
http,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
pub(super) async fn maybe_build_local_admin_usage_summary_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
let route_kind = request_context
|
||||
.control_decision
|
||||
.as_ref()
|
||||
.and_then(|decision| decision.route_kind.as_deref());
|
||||
|
||||
match route_kind {
|
||||
Some("stats")
|
||||
if request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/admin/usage/stats" | "/api/admin/usage/stats/"
|
||||
) =>
|
||||
{
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_usage_data_unavailable_response(
|
||||
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
|
||||
)));
|
||||
}
|
||||
|
||||
let query = request_context.request_query_string.as_deref();
|
||||
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
|
||||
};
|
||||
let usage = list_usage_for_optional_range(
|
||||
state,
|
||||
time_range.as_ref(),
|
||||
&AdminStatsUsageFilter::default(),
|
||||
)
|
||||
.await?;
|
||||
let aggregate = aggregate_usage_stats(&usage);
|
||||
let cache_creation_tokens: u64 = usage
|
||||
.iter()
|
||||
.map(|item| item.cache_creation_input_tokens)
|
||||
.sum();
|
||||
let cache_read_tokens: u64 =
|
||||
usage.iter().map(|item| item.cache_read_input_tokens).sum();
|
||||
let cache_creation_cost: f64 =
|
||||
usage.iter().map(|item| item.cache_creation_cost_usd).sum();
|
||||
let cache_read_cost: f64 = usage.iter().map(|item| item.cache_read_cost_usd).sum();
|
||||
let total_tokens: u64 = usage.iter().map(admin_usage_total_tokens).sum();
|
||||
let avg_response_time = round_to(aggregate.avg_response_time_ms() / 1000.0, 2);
|
||||
let error_rate = if aggregate.total_requests == 0 {
|
||||
0.0
|
||||
} else {
|
||||
round_to(
|
||||
(aggregate.error_requests as f64 / aggregate.total_requests as f64) * 100.0,
|
||||
2,
|
||||
)
|
||||
};
|
||||
|
||||
return Ok(Some(
|
||||
Json(json!({
|
||||
"total_requests": aggregate.total_requests,
|
||||
"total_tokens": total_tokens,
|
||||
"total_cost": round_to(aggregate.total_cost, 6),
|
||||
"total_actual_cost": round_to(aggregate.actual_total_cost, 6),
|
||||
"avg_response_time": avg_response_time,
|
||||
"error_count": aggregate.error_requests,
|
||||
"error_rate": error_rate,
|
||||
"cache_stats": {
|
||||
"cache_creation_tokens": cache_creation_tokens,
|
||||
"cache_read_tokens": cache_read_tokens,
|
||||
"cache_creation_cost": round_to(cache_creation_cost, 6),
|
||||
"cache_read_cost": round_to(cache_read_cost, 6),
|
||||
}
|
||||
}))
|
||||
.into_response(),
|
||||
));
|
||||
}
|
||||
Some("active")
|
||||
if request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/admin/usage/active" | "/api/admin/usage/active/"
|
||||
) =>
|
||||
{
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_usage_data_unavailable_response(
|
||||
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
|
||||
)));
|
||||
}
|
||||
|
||||
let query = request_context.request_query_string.as_deref();
|
||||
let requested_ids = admin_usage_parse_ids(query);
|
||||
let usage = state
|
||||
.list_usage_audits(&UsageAuditListQuery::default())
|
||||
.await?;
|
||||
let mut items: Vec<_> = usage
|
||||
.into_iter()
|
||||
.filter(|item| match requested_ids.as_ref() {
|
||||
Some(ids) => ids.contains(&item.id),
|
||||
None => matches!(item.status.as_str(), "pending" | "streaming"),
|
||||
})
|
||||
.collect();
|
||||
items.sort_by(|left, right| {
|
||||
right
|
||||
.created_at_unix_secs
|
||||
.cmp(&left.created_at_unix_secs)
|
||||
.then_with(|| left.id.cmp(&right.id))
|
||||
});
|
||||
if requested_ids.is_none() && items.len() > 50 {
|
||||
items.truncate(50);
|
||||
}
|
||||
let provider_key_names = admin_usage_provider_key_names(state, &items).await?;
|
||||
|
||||
let payload: Vec<_> = items
|
||||
.into_iter()
|
||||
.map(|item| {
|
||||
let provider_key_name =
|
||||
admin_usage_provider_key_name(&item, &provider_key_names);
|
||||
let mut value = json!({
|
||||
"id": item.id,
|
||||
"status": item.status,
|
||||
"input_tokens": item.input_tokens,
|
||||
"output_tokens": item.output_tokens,
|
||||
"cache_creation_input_tokens": item.cache_creation_input_tokens,
|
||||
"cache_read_input_tokens": item.cache_read_input_tokens,
|
||||
"cost": round_to(item.total_cost_usd, 6),
|
||||
"actual_cost": round_to(item.actual_total_cost_usd, 6),
|
||||
"response_time_ms": item.response_time_ms,
|
||||
"first_byte_time_ms": item.first_byte_time_ms,
|
||||
"provider": item.provider_name,
|
||||
"api_key_name": item.api_key_name,
|
||||
"provider_key_name": provider_key_name,
|
||||
});
|
||||
if let Some(api_format) = item.api_format {
|
||||
value["api_format"] = json!(api_format);
|
||||
}
|
||||
if let Some(endpoint_api_format) = item.endpoint_api_format {
|
||||
value["endpoint_api_format"] = json!(endpoint_api_format);
|
||||
}
|
||||
value["has_format_conversion"] = json!(item.has_format_conversion);
|
||||
if let Some(target_model) = item.target_model {
|
||||
value["target_model"] = json!(target_model);
|
||||
}
|
||||
value
|
||||
})
|
||||
.collect();
|
||||
|
||||
return Ok(Some(Json(json!({ "requests": payload })).into_response()));
|
||||
}
|
||||
Some("records")
|
||||
if request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/admin/usage/records" | "/api/admin/usage/records/"
|
||||
) =>
|
||||
{
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_usage_data_unavailable_response(
|
||||
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
|
||||
)));
|
||||
}
|
||||
|
||||
let query = request_context.request_query_string.as_deref();
|
||||
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
|
||||
};
|
||||
let filters = AdminStatsUsageFilter {
|
||||
user_id: query_param_value(query, "user_id"),
|
||||
provider_name: None,
|
||||
model: None,
|
||||
};
|
||||
let mut usage =
|
||||
list_usage_for_optional_range(state, time_range.as_ref(), &filters).await?;
|
||||
|
||||
let search = query_param_value(query, "search");
|
||||
let username_filter = query_param_value(query, "username");
|
||||
let model_filter = query_param_value(query, "model");
|
||||
let provider_filter = query_param_value(query, "provider");
|
||||
let api_format_filter = query_param_value(query, "api_format");
|
||||
let status_filter = query_param_value(query, "status");
|
||||
let limit = match admin_usage_parse_limit(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
|
||||
};
|
||||
let offset = match admin_usage_parse_offset(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
|
||||
};
|
||||
|
||||
usage.retain(|item| {
|
||||
admin_usage_matches_search(item, search.as_deref())
|
||||
&& admin_usage_matches_username(item, username_filter.as_deref())
|
||||
&& admin_usage_matches_eq(item.model.as_str(), model_filter.as_deref())
|
||||
&& admin_usage_matches_eq(
|
||||
item.provider_name.as_str(),
|
||||
provider_filter.as_deref(),
|
||||
)
|
||||
&& admin_usage_matches_api_format(item, api_format_filter.as_deref())
|
||||
&& admin_usage_matches_status(item, status_filter.as_deref())
|
||||
});
|
||||
usage.sort_by(|left, right| {
|
||||
right
|
||||
.created_at_unix_secs
|
||||
.cmp(&left.created_at_unix_secs)
|
||||
.then_with(|| left.id.cmp(&right.id))
|
||||
});
|
||||
let total = usage.len();
|
||||
|
||||
let user_ids: Vec<String> = usage
|
||||
.iter()
|
||||
.filter_map(|item| item.user_id.clone())
|
||||
.collect::<BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect();
|
||||
let users_by_id: BTreeMap<String, aether_data::repository::users::StoredUserSummary> =
|
||||
if state.has_user_data_reader() && !user_ids.is_empty() {
|
||||
state
|
||||
.list_users_by_ids(&user_ids)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|user| (user.id.clone(), user))
|
||||
.collect()
|
||||
} else {
|
||||
BTreeMap::new()
|
||||
};
|
||||
|
||||
let provider_key_names = admin_usage_provider_key_names(state, &usage).await?;
|
||||
|
||||
let records: Vec<_> = usage
|
||||
.into_iter()
|
||||
.skip(offset)
|
||||
.take(limit)
|
||||
.map(|item| {
|
||||
let provider_key_name =
|
||||
admin_usage_provider_key_name(&item, &provider_key_names);
|
||||
admin_usage_record_json(&item, &users_by_id, provider_key_name.as_deref())
|
||||
})
|
||||
.collect();
|
||||
|
||||
return Ok(Some(
|
||||
Json(json!({
|
||||
"records": records,
|
||||
"total": total,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
}))
|
||||
.into_response(),
|
||||
));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
Reference in New Issue
Block a user