fix(admin): 修复 usage 观测全表扫描并下推聚合查询

- 收紧 admin usage/stats 默认时间范围和 active 轮询查询
- 将 summary/records/aggregation/time-series/leaderboard 下推到 SQL 侧
- 统一前端时间参数并补齐 admin usage/stats 回归测试
This commit is contained in:
fawney19
2026-04-16 17:41:32 +08:00
parent 10605d9fb0
commit 65cd9dc3e5
28 changed files with 3085 additions and 381 deletions

View File

@@ -1,4 +1,5 @@
use super::range::build_comparison_range;
use super::resolve_admin_usage_time_range;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::query_param_value;
use crate::GatewayError;
@@ -6,17 +7,29 @@ use aether_admin::observability::stats::{
admin_stats_bad_request_response, admin_stats_comparison_empty_response,
admin_stats_error_distribution_empty_response,
admin_stats_performance_percentiles_empty_response, admin_stats_time_series_empty_response,
build_admin_stats_comparison_response, build_admin_stats_error_distribution_response,
build_admin_stats_performance_percentiles_response, build_admin_stats_time_series_response,
build_admin_stats_comparison_response_from_aggregates,
build_admin_stats_error_distribution_response,
build_admin_stats_performance_percentiles_response,
build_admin_stats_time_series_response_from_summaries, AdminStatsAggregate,
AdminStatsComparisonType, AdminStatsGranularity, AdminStatsTimeRange, AdminStatsUsageFilter,
};
use axum::{
body::Body,
http,
response::{IntoResponse, Response},
Json,
use aether_data_contracts::repository::usage::{
UsageAuditSummaryQuery, UsageTimeSeriesGranularity, UsageTimeSeriesQuery,
};
use serde_json::json;
use axum::{body::Body, http, response::Response};
fn usage_summary_to_admin_stats_aggregate(
summary: &aether_data_contracts::repository::usage::StoredUsageAuditSummary,
) -> AdminStatsAggregate {
AdminStatsAggregate {
total_requests: summary.total_requests,
total_tokens: summary.recorded_total_tokens,
total_cost: summary.total_cost_usd,
actual_total_cost: summary.actual_total_cost_usd,
total_response_time_ms: summary.total_response_time_ms,
error_requests: summary.error_requests,
}
}
pub(super) async fn maybe_build_local_admin_stats_analytics_response(
state: &AdminAppState<'_>,
@@ -58,15 +71,39 @@ pub(super) async fn maybe_build_local_admin_stats_analytics_response(
&comparison_range,
)));
}
let current_usage = state
.list_admin_usage_for_range(&current_range, &AdminStatsUsageFilter::default())
let Some((current_from_unix_secs, current_until_unix_secs)) =
current_range.to_unix_bounds()
else {
return Ok(Some(admin_stats_comparison_empty_response(
&current_range,
&comparison_range,
)));
};
let Some((comparison_from_unix_secs, comparison_until_unix_secs)) =
comparison_range.to_unix_bounds()
else {
return Ok(Some(admin_stats_comparison_empty_response(
&current_range,
&comparison_range,
)));
};
let current_summary = state
.summarize_usage_audits(&UsageAuditSummaryQuery {
created_from_unix_secs: current_from_unix_secs,
created_until_unix_secs: current_until_unix_secs,
..Default::default()
})
.await?;
let comparison_usage = state
.list_admin_usage_for_range(&comparison_range, &AdminStatsUsageFilter::default())
let comparison_summary = state
.summarize_usage_audits(&UsageAuditSummaryQuery {
created_from_unix_secs: comparison_from_unix_secs,
created_until_unix_secs: comparison_until_unix_secs,
..Default::default()
})
.await?;
return Ok(Some(build_admin_stats_comparison_response(
&current_usage,
&comparison_usage,
return Ok(Some(build_admin_stats_comparison_response_from_aggregates(
&usage_summary_to_admin_stats_aggregate(&current_summary),
&usage_summary_to_admin_stats_aggregate(&comparison_summary),
&current_range,
&comparison_range,
)));
@@ -79,19 +116,9 @@ pub(super) async fn maybe_build_local_admin_stats_analytics_response(
"/api/admin/stats/errors/distribution" | "/api/admin/stats/errors/distribution/"
)
{
let Some(time_range) =
(match AdminStatsTimeRange::resolve_optional(request_context.query_string()) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
})
else {
return Ok(Some(
Json(json!({
"distribution": [],
"trend": [],
}))
.into_response(),
));
let time_range = match resolve_admin_usage_time_range(request_context.query_string()) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
if !state.has_usage_data_reader() {
return Ok(Some(admin_stats_error_distribution_empty_response()));
@@ -114,13 +141,9 @@ pub(super) async fn maybe_build_local_admin_stats_analytics_response(
| "/api/admin/stats/performance/percentiles/"
)
{
let Some(time_range) =
(match AdminStatsTimeRange::resolve_optional(request_context.query_string()) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
})
else {
return Ok(Some(Json(json!([])).into_response()));
let time_range = match resolve_admin_usage_time_range(request_context.query_string()) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
if !state.has_usage_data_reader() {
return Ok(Some(admin_stats_performance_percentiles_empty_response()));
@@ -146,13 +169,9 @@ pub(super) async fn maybe_build_local_admin_stats_analytics_response(
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
let Some(time_range) =
(match AdminStatsTimeRange::resolve_optional(request_context.query_string()) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
})
else {
return Ok(Some(Json(json!([])).into_response()));
let time_range = match resolve_admin_usage_time_range(request_context.query_string()) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
if let Err(detail) = time_range.validate_for_time_series(granularity) {
return Ok(Some(admin_stats_bad_request_response(detail)));
@@ -162,13 +181,31 @@ pub(super) async fn maybe_build_local_admin_stats_analytics_response(
}
let filters = AdminStatsUsageFilter::from_query(request_context.query_string());
let usage = state
.list_admin_usage_for_range(&time_range, &filters)
let query_granularity = match granularity {
AdminStatsGranularity::Hour => UsageTimeSeriesGranularity::Hour,
AdminStatsGranularity::Day
| AdminStatsGranularity::Week
| AdminStatsGranularity::Month => UsageTimeSeriesGranularity::Day,
};
let Some((created_from_unix_secs, created_until_unix_secs)) = time_range.to_unix_bounds()
else {
return Ok(Some(admin_stats_time_series_empty_response()));
};
let buckets = state
.summarize_usage_time_series(&UsageTimeSeriesQuery {
created_from_unix_secs,
created_until_unix_secs,
granularity: query_granularity,
tz_offset_minutes: time_range.tz_offset_minutes,
user_id: filters.user_id,
provider_name: filters.provider_name,
model: filters.model,
})
.await?;
return Ok(Some(build_admin_stats_time_series_response(
return Ok(Some(build_admin_stats_time_series_response_from_summaries(
&time_range,
granularity,
&usage,
&buckets,
)));
}

View File

@@ -1,15 +1,31 @@
use super::range::{build_time_range_from_days, parse_bounded_u32, parse_tz_offset_minutes};
use super::resolve_admin_usage_time_range;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::query_param_value;
use crate::GatewayError;
use aether_admin::observability::stats::{
admin_stats_bad_request_response, admin_stats_cost_forecast_empty_response,
admin_stats_cost_savings_empty_response, build_admin_stats_cost_forecast_response,
build_admin_stats_cost_savings_response, AdminStatsGranularity, AdminStatsTimeRange,
AdminStatsUsageFilter,
build_admin_stats_cost_savings_response, AdminStatsGranularity, AdminStatsUsageFilter,
};
use axum::{body::Body, http, response::Response};
fn resolve_cost_forecast_time_range(
query: Option<&str>,
) -> Result<super::AdminStatsTimeRange, String> {
match super::AdminStatsTimeRange::resolve_optional(query)? {
Some(value) => Ok(value),
None => {
let tz_offset_minutes = parse_tz_offset_minutes(query)?;
let days = query_param_value(query, "days")
.map(|value| parse_bounded_u32("days", &value, 7, 365))
.transpose()?
.unwrap_or(30);
build_time_range_from_days(days, tz_offset_minutes)
}
}
}
pub(super) async fn maybe_build_local_admin_stats_cost_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
@@ -34,32 +50,13 @@ pub(super) async fn maybe_build_local_admin_stats_cost_response(
.map(|value| parse_bounded_u32("forecast_days", &value, 1, 90))
.transpose()
{
Ok(Some(value)) => value,
Ok(None) => 7,
Ok(value) => value.unwrap_or(7),
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
let tz_offset_minutes = match parse_tz_offset_minutes(query) {
let time_range = match resolve_cost_forecast_time_range(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
Ok(Some(value)) => value,
Ok(None) => {
let days = match query_param_value(query, "days")
.map(|value| parse_bounded_u32("days", &value, 7, 365))
.transpose()
{
Ok(Some(value)) => value,
Ok(None) => 30,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
match build_time_range_from_days(days, tz_offset_minutes) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
}
}
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
if let Err(detail) = time_range.validate_for_time_series(AdminStatsGranularity::Day) {
return Ok(Some(admin_stats_bad_request_response(detail)));
}
@@ -84,13 +81,10 @@ pub(super) async fn maybe_build_local_admin_stats_cost_response(
"/api/admin/stats/cost/savings" | "/api/admin/stats/cost/savings/"
)
{
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
let time_range = match resolve_admin_usage_time_range(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
if time_range.is_none() {
return Ok(Some(admin_stats_cost_savings_empty_response()));
}
if !state.has_usage_data_reader() {
return Ok(Some(admin_stats_cost_savings_empty_response()));
}
@@ -101,7 +95,7 @@ pub(super) async fn maybe_build_local_admin_stats_cost_response(
model: query_param_value(query, "model"),
};
let usage = state
.list_admin_usage_for_range(time_range.as_ref().expect("time range exists"), &filters)
.list_admin_usage_for_range(&time_range, &filters)
.await?;
return Ok(Some(build_admin_stats_cost_savings_response(&usage)));

View File

@@ -4,9 +4,11 @@ use aether_data_contracts::repository::usage::StoredRequestUsageAudit;
pub(super) use aether_admin::observability::stats::{
build_admin_stats_leaderboard_response, build_api_key_leaderboard_items,
build_model_leaderboard_items, build_user_leaderboard_items, compare_leaderboard_items,
compute_dense_rank, AdminStatsLeaderboardItem, AdminStatsLeaderboardMetric,
AdminStatsLeaderboardNameMode, AdminStatsSortOrder, AdminStatsUserMetadata,
build_api_key_leaderboard_items_from_summaries, build_model_leaderboard_items,
build_model_leaderboard_items_from_summaries, build_user_leaderboard_items,
build_user_leaderboard_items_from_summaries, compare_leaderboard_items, compute_dense_rank,
AdminStatsLeaderboardItem, AdminStatsLeaderboardMetric, AdminStatsLeaderboardNameMode,
AdminStatsSortOrder, AdminStatsUserMetadata,
};
pub(super) async fn load_user_leaderboard_metadata(

View File

@@ -1,16 +1,18 @@
use super::leaderboard::{
build_admin_stats_leaderboard_response, build_api_key_leaderboard_items,
build_model_leaderboard_items, build_user_leaderboard_items, compare_leaderboard_items,
load_user_leaderboard_metadata, AdminStatsLeaderboardNameMode,
build_admin_stats_leaderboard_response, build_api_key_leaderboard_items_from_summaries,
build_model_leaderboard_items_from_summaries, build_user_leaderboard_items_from_summaries,
compare_leaderboard_items, load_user_leaderboard_metadata, AdminStatsLeaderboardNameMode,
};
use super::range::{parse_bounded_u32, parse_nonnegative_usize};
use super::resolve_admin_usage_time_range;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::{query_param_bool, query_param_value};
use crate::GatewayError;
use aether_admin::observability::stats::{
admin_stats_bad_request_response, admin_stats_leaderboard_empty_response,
AdminStatsLeaderboardMetric, AdminStatsSortOrder, AdminStatsTimeRange, AdminStatsUsageFilter,
AdminStatsLeaderboardMetric, AdminStatsSortOrder, AdminStatsUsageFilter,
};
use aether_data_contracts::repository::usage::{UsageLeaderboardGroupBy, UsageLeaderboardQuery};
use axum::{body::Body, http, response::Response};
pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
@@ -29,7 +31,7 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
"/api/admin/stats/leaderboard/models" | "/api/admin/stats/leaderboard/models/"
)
{
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
let time_range = match resolve_admin_usage_time_range(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
@@ -60,19 +62,33 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
if !state.has_usage_data_reader() {
return Ok(Some(admin_stats_leaderboard_empty_response(
metric,
time_range.as_ref(),
Some(&time_range),
)));
}
let filters = AdminStatsUsageFilter::from_query(query);
let usage = state
.list_admin_usage_for_optional_range(time_range.as_ref(), &filters)
let Some((created_from_unix_secs, created_until_unix_secs)) = time_range.to_unix_bounds()
else {
return Ok(Some(admin_stats_leaderboard_empty_response(
metric,
Some(&time_range),
)));
};
let summaries = state
.summarize_usage_leaderboard(&UsageLeaderboardQuery {
created_from_unix_secs,
created_until_unix_secs,
group_by: UsageLeaderboardGroupBy::Model,
user_id: filters.user_id,
provider_name: filters.provider_name,
model: filters.model,
})
.await?;
let mut leaderboard = build_model_leaderboard_items(&usage);
let mut leaderboard = build_model_leaderboard_items_from_summaries(&summaries);
leaderboard.sort_by(|left, right| compare_leaderboard_items(metric, order, left, right));
return Ok(Some(build_admin_stats_leaderboard_response(
metric,
time_range.as_ref(),
Some(&time_range),
&leaderboard,
offset,
limit,
@@ -90,7 +106,7 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
"/api/admin/stats/leaderboard/api-keys" | "/api/admin/stats/leaderboard/api-keys/"
)
{
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
let time_range = match resolve_admin_usage_time_range(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
@@ -121,20 +137,32 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
if !state.has_usage_data_reader() {
return Ok(Some(admin_stats_leaderboard_empty_response(
metric,
time_range.as_ref(),
Some(&time_range),
)));
}
let include_inactive = query_param_bool(query, "include_inactive", false);
let exclude_admin = query_param_bool(query, "exclude_admin", false);
let filters = AdminStatsUsageFilter::from_query(query);
let usage = state
.list_admin_usage_for_optional_range(time_range.as_ref(), &filters)
let Some((created_from_unix_secs, created_until_unix_secs)) = time_range.to_unix_bounds()
else {
return Ok(Some(admin_stats_leaderboard_empty_response(
metric,
Some(&time_range),
)));
};
let summaries = state
.summarize_usage_leaderboard(&UsageLeaderboardQuery {
created_from_unix_secs,
created_until_unix_secs,
group_by: UsageLeaderboardGroupBy::ApiKey,
user_id: filters.user_id,
provider_name: filters.provider_name,
model: filters.model,
})
.await?;
let api_key_ids: Vec<String> = usage
let api_key_ids: Vec<String> = summaries
.iter()
.filter_map(|item| item.api_key_id.clone())
.collect::<std::collections::BTreeSet<_>>()
.into_iter()
.map(|item| item.group_key.clone())
.collect();
let snapshots = if state.has_auth_api_key_data_reader() {
Some(
@@ -152,8 +180,8 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
} else {
std::collections::BTreeMap::new()
};
let mut leaderboard = build_api_key_leaderboard_items(
&usage,
let mut leaderboard = build_api_key_leaderboard_items_from_summaries(
&summaries,
snapshots.as_deref(),
&api_key_names,
include_inactive,
@@ -163,7 +191,7 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
return Ok(Some(build_admin_stats_leaderboard_response(
metric,
time_range.as_ref(),
Some(&time_range),
&leaderboard,
offset,
limit,
@@ -181,7 +209,7 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
"/api/admin/stats/leaderboard/users" | "/api/admin/stats/leaderboard/users/"
)
{
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
let time_range = match resolve_admin_usage_time_range(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
@@ -212,26 +240,39 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
if !state.has_usage_data_reader() {
return Ok(Some(admin_stats_leaderboard_empty_response(
metric,
time_range.as_ref(),
Some(&time_range),
)));
}
let include_inactive = query_param_bool(query, "include_inactive", false);
let exclude_admin = query_param_bool(query, "exclude_admin", false);
let filters = AdminStatsUsageFilter::from_query(query);
let usage = state
.list_admin_usage_for_optional_range(time_range.as_ref(), &filters)
let Some((created_from_unix_secs, created_until_unix_secs)) = time_range.to_unix_bounds()
else {
return Ok(Some(admin_stats_leaderboard_empty_response(
metric,
Some(&time_range),
)));
};
let summaries = state
.summarize_usage_leaderboard(&UsageLeaderboardQuery {
created_from_unix_secs,
created_until_unix_secs,
group_by: UsageLeaderboardGroupBy::User,
user_id: filters.user_id,
provider_name: filters.provider_name,
model: filters.model,
})
.await?;
let user_ids: Vec<String> = usage
let user_ids: Vec<String> = summaries
.iter()
.filter_map(|item| item.user_id.clone())
.collect::<std::collections::BTreeSet<_>>()
.into_iter()
.map(|item| item.group_key.clone())
.collect();
let user_metadata = load_user_leaderboard_metadata(state, &user_ids).await?;
let mut leaderboard = build_user_leaderboard_items(
&usage,
let mut leaderboard = build_user_leaderboard_items_from_summaries(
&summaries,
&user_metadata,
state.has_auth_user_data_reader(),
state.has_user_data_reader(),
include_inactive,
exclude_admin,
);
@@ -239,7 +280,7 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
return Ok(Some(build_admin_stats_leaderboard_response(
metric,
time_range.as_ref(),
Some(&time_range),
&leaderboard,
offset,
limit,

View File

@@ -10,6 +10,7 @@ mod provider_quota_routes;
mod range;
pub(crate) use self::range::{
list_usage_for_optional_range, list_usage_for_range, parse_bounded_u32,
resolve_admin_usage_time_range,
};
pub(crate) use aether_admin::observability::stats::{
admin_stats_bad_request_response, aggregate_usage_stats, round_to, AdminStatsTimeRange,

View File

@@ -8,6 +8,22 @@ pub(super) use aether_admin::observability::stats::{
use aether_admin::observability::stats::{AdminStatsTimeRange, AdminStatsUsageFilter};
use aether_data_contracts::repository::usage::{StoredRequestUsageAudit, UsageAuditListQuery};
pub(crate) fn resolve_admin_usage_time_range(
query: Option<&str>,
) -> Result<AdminStatsTimeRange, String> {
match AdminStatsTimeRange::resolve_optional(query)? {
Some(time_range) => Ok(time_range),
None => {
let tz_offset_minutes = parse_tz_offset_minutes(query)?;
let default_days = u32::try_from(admin_usage_default_days())
.ok()
.filter(|value| *value > 0)
.unwrap_or(1);
build_time_range_from_days(default_days, tz_offset_minutes)
}
}
}
pub(crate) async fn list_usage_for_range(
state: &AdminAppState<'_>,
time_range: &AdminStatsTimeRange,
@@ -25,8 +41,13 @@ pub(crate) async fn list_usage_for_range(
user_id: filters.user_id.clone(),
provider_name: filters.provider_name.clone(),
model: filters.model.clone(),
api_format: None,
statuses: None,
is_stream: None,
error_only: false,
limit: None,
offset: None,
newest_first: false,
})
.await
}
@@ -39,17 +60,9 @@ pub(crate) async fn list_usage_for_optional_range(
match time_range {
Some(time_range) => list_usage_for_range(state, time_range, filters).await,
None => {
state
.list_usage_audits(&UsageAuditListQuery {
created_from_unix_secs: None,
created_until_unix_secs: None,
user_id: filters.user_id.clone(),
provider_name: filters.provider_name.clone(),
model: filters.model.clone(),
statuses: None,
limit: None,
})
.await
let default_time_range = build_time_range_from_days(1, 0)
.map_err(|detail| GatewayError::Internal(detail.to_string()))?;
list_usage_for_range(state, &default_time_range, filters).await
}
}
}

View File

@@ -1,29 +1,21 @@
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
use aether_admin::observability::stats::round_to;
use aether_data_contracts::repository::usage::{StoredRequestUsageAudit, UsageAuditListQuery};
use aether_data_contracts::repository::usage::StoredUsageAuditAggregation;
use serde_json::json;
use std::collections::BTreeMap;
pub(in super::super) async fn admin_usage_aggregation_by_user_json(
state: &AdminAppState<'_>,
usage: &[StoredRequestUsageAudit],
limit: usize,
rows: &[StoredUsageAuditAggregation],
) -> Result<serde_json::Value, GatewayError> {
let mut grouped: BTreeMap<String, (u64, u64, f64)> = BTreeMap::new();
for item in usage {
let Some(user_id) = item.user_id.as_ref() else {
continue;
};
let entry = grouped.entry(user_id.clone()).or_insert((0, 0, 0.0));
entry.0 = entry.0.saturating_add(1);
entry.1 = entry.1.saturating_add(item.total_tokens);
entry.2 += item.total_cost_usd;
}
let usernames = if state.has_user_data_reader() && !grouped.is_empty() {
let user_ids = rows
.iter()
.map(|row| row.group_key.clone())
.collect::<Vec<_>>();
let usernames = if state.has_user_data_reader() && !user_ids.is_empty() {
state
.list_users_by_ids(&grouped.keys().cloned().collect::<Vec<_>>())
.list_users_by_ids(&user_ids)
.await?
.into_iter()
.map(|user| (user.id, (user.email, user.username)))
@@ -32,35 +24,25 @@ pub(in super::super) async fn admin_usage_aggregation_by_user_json(
BTreeMap::new()
};
let mut items: Vec<serde_json::Value> = grouped
.into_iter()
.map(|(user_id, (request_count, total_tokens, total_cost))| {
Ok(json!(rows
.iter()
.map(|row| {
let (email, username) = usernames
.get(&user_id)
.get(&row.group_key)
.cloned()
.unwrap_or((None, String::new()));
json!({
"user_id": user_id,
"user_id": row.group_key,
"email": email,
"username": if username.is_empty() { serde_json::Value::Null } else { json!(username) },
"request_count": request_count,
"total_tokens": total_tokens,
"total_cost": round_to(total_cost, 6),
"username": if username.is_empty() {
serde_json::Value::Null
} else {
json!(username)
},
"request_count": row.request_count,
"total_tokens": row.total_tokens,
"total_cost": round_to(row.total_cost_usd, 6),
})
})
.collect();
items.sort_by(|left, right| {
right["request_count"]
.as_u64()
.unwrap_or_default()
.cmp(&left["request_count"].as_u64().unwrap_or_default())
.then_with(|| {
left["user_id"]
.as_str()
.unwrap_or_default()
.cmp(right["user_id"].as_str().unwrap_or_default())
})
});
items.truncate(limit);
Ok(json!(items))
.collect::<Vec<_>>()))
}

View File

@@ -16,8 +16,13 @@ pub(in super::super) async fn list_recent_completed_usage_for_cache_affinity(
user_id: user_id.map(ToOwned::to_owned),
provider_name: None,
model: None,
api_format: None,
statuses: None,
is_stream: None,
error_only: false,
limit: None,
offset: None,
newest_first: false,
})
.await?;
items.retain(|item| item.status == "completed");

View File

@@ -1,14 +1,17 @@
use super::super::super::stats::{AdminStatsTimeRange, AdminStatsUsageFilter};
use super::super::super::stats::resolve_admin_usage_time_range;
use super::super::analytics::admin_usage_aggregation_by_user_json;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::query_param_value;
use crate::GatewayError;
use aether_admin::observability::stats::round_to;
use aether_admin::observability::usage::{
admin_usage_aggregation_by_api_format_json, admin_usage_aggregation_by_model_json,
admin_usage_aggregation_by_provider_json, admin_usage_bad_request_response,
admin_usage_data_unavailable_response, admin_usage_parse_aggregation_limit,
admin_usage_bad_request_response, admin_usage_data_unavailable_response,
admin_usage_parse_aggregation_limit, admin_usage_token_cache_hit_rate,
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
};
use aether_data_contracts::repository::usage::{
StoredUsageAuditAggregation, UsageAuditAggregationGroupBy, UsageAuditAggregationQuery,
};
use axum::{
body::Body,
response::{IntoResponse, Response},
@@ -16,6 +19,102 @@ use axum::{
};
use serde_json::json;
fn admin_usage_aggregation_by_model_json(
rows: &[StoredUsageAuditAggregation],
) -> serde_json::Value {
json!(rows
.iter()
.map(|row| {
json!({
"model": row.group_key,
"request_count": row.request_count,
"total_tokens": row.total_tokens,
"effective_input_tokens": row.effective_input_tokens,
"total_input_context": row.total_input_context,
"output_tokens": row.output_tokens,
"total_cost": round_to(row.total_cost_usd, 6),
"actual_cost": round_to(row.actual_total_cost_usd, 6),
"cache_creation_tokens": row.cache_creation_tokens,
"cache_creation_ephemeral_5m_tokens": row.cache_creation_ephemeral_5m_tokens,
"cache_creation_ephemeral_1h_tokens": row.cache_creation_ephemeral_1h_tokens,
"cache_read_tokens": row.cache_read_tokens,
"cache_hit_rate": admin_usage_token_cache_hit_rate(
row.total_input_context,
row.cache_read_tokens,
),
})
})
.collect::<Vec<_>>())
}
fn admin_usage_aggregation_by_provider_json(
rows: &[StoredUsageAuditAggregation],
) -> serde_json::Value {
json!(rows
.iter()
.map(|row| {
let success_count = row.success_count.unwrap_or_default();
let error_count = row.request_count.saturating_sub(success_count);
let success_rate = if row.request_count == 0 {
0.0
} else {
round_to(success_count as f64 / row.request_count as f64 * 100.0, 2)
};
json!({
"provider_id": row.group_key,
"provider": row.display_name.clone().unwrap_or_else(|| "Unknown".to_string()),
"request_count": row.request_count,
"total_tokens": row.total_tokens,
"effective_input_tokens": row.effective_input_tokens,
"total_input_context": row.total_input_context,
"output_tokens": row.output_tokens,
"total_cost": round_to(row.total_cost_usd, 6),
"actual_cost": round_to(row.actual_total_cost_usd, 6),
"avg_response_time_ms": round_to(row.avg_response_time_ms.unwrap_or(0.0), 2),
"success_rate": success_rate,
"error_count": error_count,
"cache_creation_tokens": row.cache_creation_tokens,
"cache_creation_ephemeral_5m_tokens": row.cache_creation_ephemeral_5m_tokens,
"cache_creation_ephemeral_1h_tokens": row.cache_creation_ephemeral_1h_tokens,
"cache_read_tokens": row.cache_read_tokens,
"cache_hit_rate": admin_usage_token_cache_hit_rate(
row.total_input_context,
row.cache_read_tokens,
),
})
})
.collect::<Vec<_>>())
}
fn admin_usage_aggregation_by_api_format_json(
rows: &[StoredUsageAuditAggregation],
) -> serde_json::Value {
json!(rows
.iter()
.map(|row| {
json!({
"api_format": row.group_key,
"request_count": row.request_count,
"total_tokens": row.total_tokens,
"effective_input_tokens": row.effective_input_tokens,
"total_input_context": row.total_input_context,
"output_tokens": row.output_tokens,
"total_cost": round_to(row.total_cost_usd, 6),
"actual_cost": round_to(row.actual_total_cost_usd, 6),
"avg_response_time_ms": round_to(row.avg_response_time_ms.unwrap_or(0.0), 2),
"cache_creation_tokens": row.cache_creation_tokens,
"cache_creation_ephemeral_5m_tokens": row.cache_creation_ephemeral_5m_tokens,
"cache_creation_ephemeral_1h_tokens": row.cache_creation_ephemeral_1h_tokens,
"cache_read_tokens": row.cache_read_tokens,
"cache_hit_rate": admin_usage_token_cache_hit_rate(
row.total_input_context,
row.cache_read_tokens,
),
})
})
.collect::<Vec<_>>())
}
pub(super) async fn build_admin_usage_aggregation_stats_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
@@ -43,21 +142,35 @@ pub(super) async fn build_admin_usage_aggregation_stats_response(
Ok(value) => value,
Err(detail) => return Ok(admin_usage_bad_request_response(detail)),
};
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
let time_range = match resolve_admin_usage_time_range(query) {
Ok(value) => value,
Err(detail) => return Ok(admin_usage_bad_request_response(detail)),
};
let mut usage = state
.list_admin_usage_for_optional_range(time_range.as_ref(), &AdminStatsUsageFilter::default())
let Some((created_from_unix_secs, created_until_unix_secs)) = time_range.to_unix_bounds()
else {
return Ok(Json(json!([])).into_response());
};
let group_by_query = match group_by.as_str() {
"model" => UsageAuditAggregationGroupBy::Model,
"user" => UsageAuditAggregationGroupBy::User,
"provider" => UsageAuditAggregationGroupBy::Provider,
"api_format" => UsageAuditAggregationGroupBy::ApiFormat,
_ => unreachable!(),
};
let usage = state
.aggregate_usage_audits(&UsageAuditAggregationQuery {
created_from_unix_secs,
created_until_unix_secs,
group_by: group_by_query,
limit,
})
.await?;
usage.retain(|item| item.status != "pending" && item.status != "streaming");
let response = match group_by.as_str() {
"model" => admin_usage_aggregation_by_model_json(&usage, limit),
"user" => admin_usage_aggregation_by_user_json(state, &usage, limit).await?,
"provider" => admin_usage_aggregation_by_provider_json(&usage, limit),
"api_format" => admin_usage_aggregation_by_api_format_json(&usage, limit),
"model" => admin_usage_aggregation_by_model_json(&usage),
"user" => admin_usage_aggregation_by_user_json(state, &usage).await?,
"provider" => admin_usage_aggregation_by_provider_json(&usage),
"api_format" => admin_usage_aggregation_by_api_format_json(&usage),
_ => unreachable!(),
};
Ok(Json(response).into_response())

View File

@@ -1,4 +1,4 @@
use super::super::stats::{AdminStatsTimeRange, AdminStatsUsageFilter};
use super::super::stats::resolve_admin_usage_time_range;
use super::analytics::admin_usage_api_key_names;
use super::analytics::admin_usage_provider_key_names;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
@@ -6,16 +6,89 @@ use crate::handlers::admin::shared::query_param_value;
use crate::GatewayError;
use aether_admin::observability::usage::{
admin_usage_bad_request_response, admin_usage_data_unavailable_response,
admin_usage_matches_api_format, admin_usage_matches_eq, admin_usage_matches_search,
admin_usage_matches_status, admin_usage_matches_username, admin_usage_parse_ids,
admin_usage_matches_search, admin_usage_matches_username, admin_usage_parse_ids,
admin_usage_parse_limit, admin_usage_parse_offset, build_admin_usage_active_requests_response,
build_admin_usage_records_response, build_admin_usage_summary_stats_response,
build_admin_usage_records_response, build_admin_usage_summary_stats_response_from_summary,
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
};
use aether_data_contracts::repository::usage::UsageAuditListQuery;
use aether_data_contracts::repository::usage::{
StoredRequestUsageAudit, UsageAuditListQuery, UsageAuditSummaryQuery,
};
use axum::{body::Body, http, response::Response};
use std::collections::{BTreeMap, BTreeSet};
const ADMIN_USAGE_ACTIVE_LIMIT: usize = 50;
async fn load_admin_usage_by_ids(
state: &AdminAppState<'_>,
requested_ids: &BTreeSet<String>,
) -> Result<Vec<StoredRequestUsageAudit>, GatewayError> {
let mut items = Vec::with_capacity(requested_ids.len());
for usage_id in requested_ids {
if let Some(item) = state.find_request_usage_by_id(usage_id).await? {
items.push(item);
}
}
Ok(items)
}
fn sort_usage_newest_first(items: &mut [StoredRequestUsageAudit]) {
items.sort_by(|left, right| {
right
.created_at_unix_ms
.cmp(&left.created_at_unix_ms)
.then_with(|| left.id.cmp(&right.id))
});
}
fn apply_admin_usage_status_filter(query: &mut UsageAuditListQuery, status: Option<&str>) {
let Some(status) = status
.map(str::trim)
.filter(|candidate| !candidate.is_empty())
else {
return;
};
match status {
"stream" => query.is_stream = Some(true),
"standard" => query.is_stream = Some(false),
"error" | "failed" => query.error_only = true,
"active" => {
query.statuses = Some(vec!["pending".to_string(), "streaming".to_string()]);
}
"pending" | "streaming" | "completed" | "cancelled" => {
query.statuses = Some(vec![status.to_string()]);
}
_ => {}
}
}
fn build_admin_usage_records_query(
created_from_unix_secs: u64,
created_until_unix_secs: u64,
query: Option<&str>,
limit: Option<usize>,
offset: Option<usize>,
) -> UsageAuditListQuery {
let mut list_query = UsageAuditListQuery {
created_from_unix_secs: Some(created_from_unix_secs),
created_until_unix_secs: Some(created_until_unix_secs),
user_id: query_param_value(query, "user_id"),
provider_name: query_param_value(query, "provider"),
model: query_param_value(query, "model"),
api_format: query_param_value(query, "api_format"),
limit,
offset,
newest_first: true,
..Default::default()
};
apply_admin_usage_status_filter(
&mut list_query,
query_param_value(query, "status").as_deref(),
);
list_query
}
pub(super) async fn maybe_build_local_admin_usage_summary_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
@@ -40,17 +113,27 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
}
let query = request_context.request_query_string.as_deref();
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
let time_range = match resolve_admin_usage_time_range(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
let usage = state
.list_admin_usage_for_optional_range(
time_range.as_ref(),
&AdminStatsUsageFilter::default(),
)
let Some((created_from_unix_secs, created_until_unix_secs)) =
time_range.to_unix_bounds()
else {
return Ok(Some(build_admin_usage_summary_stats_response_from_summary(
&Default::default(),
)));
};
let summary = state
.summarize_usage_audits(&UsageAuditSummaryQuery {
created_from_unix_secs,
created_until_unix_secs,
..Default::default()
})
.await?;
return Ok(Some(build_admin_usage_summary_stats_response(&usage)));
return Ok(Some(build_admin_usage_summary_stats_response_from_summary(
&summary,
)));
}
Some("active")
if request_context.request_method == http::Method::GET
@@ -67,30 +150,37 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
let query = request_context.request_query_string.as_deref();
let requested_ids = admin_usage_parse_ids(query);
let list_query = if requested_ids.is_some() {
UsageAuditListQuery::default()
let mut items = if let Some(requested_ids) = requested_ids.as_ref() {
load_admin_usage_by_ids(state, requested_ids).await?
} else {
UsageAuditListQuery {
statuses: Some(vec!["pending".to_string(), "streaming".to_string()]),
..Default::default()
}
let time_range = match resolve_admin_usage_time_range(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
let Some((created_from_unix_secs, created_until_unix_secs)) =
time_range.to_unix_bounds()
else {
return Ok(Some(build_admin_usage_active_requests_response(
&[],
&BTreeMap::new(),
state.has_auth_api_key_data_reader(),
&BTreeMap::new(),
)));
};
state
.list_usage_audits(&UsageAuditListQuery {
created_from_unix_secs: Some(created_from_unix_secs),
created_until_unix_secs: Some(created_until_unix_secs),
statuses: Some(vec!["pending".to_string(), "streaming".to_string()]),
limit: Some(ADMIN_USAGE_ACTIVE_LIMIT),
newest_first: true,
..Default::default()
})
.await?
};
let usage = state.list_usage_audits(&list_query).await?;
let mut items: Vec<_> = usage
.into_iter()
.filter(|item| match requested_ids.as_ref() {
Some(ids) => ids.contains(&item.id),
None => true,
})
.collect();
items.sort_by(|left, right| {
right
.created_at_unix_ms
.cmp(&left.created_at_unix_ms)
.then_with(|| left.id.cmp(&right.id))
});
if requested_ids.is_none() && items.len() > 50 {
items.truncate(50);
sort_usage_newest_first(&mut items);
if requested_ids.is_none() && items.len() > ADMIN_USAGE_ACTIVE_LIMIT {
items.truncate(ADMIN_USAGE_ACTIVE_LIMIT);
}
let api_key_names = admin_usage_api_key_names(state, &items).await?;
let provider_key_names = admin_usage_provider_key_names(state, &items).await?;
@@ -116,25 +206,12 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
}
let query = request_context.request_query_string.as_deref();
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
let time_range = match resolve_admin_usage_time_range(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
let filters = AdminStatsUsageFilter {
user_id: query_param_value(query, "user_id"),
provider_name: None,
model: None,
};
let mut usage = state
.list_admin_usage_for_optional_range(time_range.as_ref(), &filters)
.await?;
let search = query_param_value(query, "search");
let username_filter = query_param_value(query, "username");
let model_filter = query_param_value(query, "model");
let provider_filter = query_param_value(query, "provider");
let api_format_filter = query_param_value(query, "api_format");
let status_filter = query_param_value(query, "status");
let limit = match admin_usage_parse_limit(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
@@ -143,6 +220,74 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
let Some((created_from_unix_secs, created_until_unix_secs)) =
time_range.to_unix_bounds()
else {
return Ok(Some(build_admin_usage_records_response(
&[],
&BTreeMap::new(),
&BTreeMap::new(),
state.has_auth_user_data_reader(),
state.has_auth_api_key_data_reader(),
&BTreeMap::new(),
0,
limit,
offset,
)));
};
let base_query = build_admin_usage_records_query(
created_from_unix_secs,
created_until_unix_secs,
query,
None,
None,
);
let use_metadata_fallback = search.is_some() || username_filter.is_some();
let (usage, total) = if use_metadata_fallback {
let mut usage = state.list_usage_audits(&base_query).await?;
let user_ids: Vec<String> = usage
.iter()
.filter_map(|item| item.user_id.clone())
.collect::<BTreeSet<_>>()
.into_iter()
.collect();
let users_by_id: BTreeMap<
String,
aether_data::repository::users::StoredUserSummary,
> = state.resolve_auth_user_summaries_by_ids(&user_ids).await?;
let api_key_names = admin_usage_api_key_names(state, &usage).await?;
usage.retain(|item| {
admin_usage_matches_search(
item,
search.as_deref(),
&users_by_id,
&api_key_names,
state.has_auth_user_data_reader(),
state.has_auth_api_key_data_reader(),
) && admin_usage_matches_username(
item,
username_filter.as_deref(),
&users_by_id,
state.has_auth_user_data_reader(),
)
});
sort_usage_newest_first(&mut usage);
let total = usage.len();
let records = usage
.into_iter()
.skip(offset)
.take(limit)
.collect::<Vec<_>>();
(records, total)
} else {
let total = usize::try_from(state.count_usage_audits(&base_query).await?)
.unwrap_or(usize::MAX);
let mut paged_query = base_query.clone();
paged_query.limit = Some(limit);
paged_query.offset = Some(offset);
(state.list_usage_audits(&paged_query).await?, total)
};
let user_ids: Vec<String> = usage
.iter()
@@ -153,46 +298,10 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
let users_by_id: BTreeMap<String, aether_data::repository::users::StoredUserSummary> =
state.resolve_auth_user_summaries_by_ids(&user_ids).await?;
let api_key_names = admin_usage_api_key_names(state, &usage).await?;
usage.retain(|item| {
admin_usage_matches_search(
item,
search.as_deref(),
&users_by_id,
&api_key_names,
state.has_auth_user_data_reader(),
state.has_auth_api_key_data_reader(),
) && admin_usage_matches_username(
item,
username_filter.as_deref(),
&users_by_id,
state.has_auth_user_data_reader(),
) && admin_usage_matches_eq(item.model.as_str(), model_filter.as_deref())
&& admin_usage_matches_eq(
item.provider_name.as_str(),
provider_filter.as_deref(),
)
&& admin_usage_matches_api_format(item, api_format_filter.as_deref())
&& admin_usage_matches_status(item, status_filter.as_deref())
});
usage.sort_by(|left, right| {
right
.created_at_unix_ms
.cmp(&left.created_at_unix_ms)
.then_with(|| left.id.cmp(&right.id))
});
let total = usage.len();
let provider_key_names = admin_usage_provider_key_names(state, &usage).await?;
let records = usage
.into_iter()
.skip(offset)
.take(limit)
.collect::<Vec<_>>();
return Ok(Some(build_admin_usage_records_response(
&records,
&usage,
&users_by_id,
&api_key_names,
state.has_auth_user_data_reader(),

View File

@@ -40,6 +40,51 @@ impl<'a> AdminAppState<'a> {
self.app.list_usage_audits(query).await
}
pub(crate) async fn count_usage_audits(
&self,
query: &aether_data_contracts::repository::usage::UsageAuditListQuery,
) -> Result<u64, GatewayError> {
self.app.count_usage_audits(query).await
}
pub(crate) async fn aggregate_usage_audits(
&self,
query: &aether_data_contracts::repository::usage::UsageAuditAggregationQuery,
) -> Result<
Vec<aether_data_contracts::repository::usage::StoredUsageAuditAggregation>,
GatewayError,
> {
self.app.aggregate_usage_audits(query).await
}
pub(crate) async fn summarize_usage_audits(
&self,
query: &aether_data_contracts::repository::usage::UsageAuditSummaryQuery,
) -> Result<aether_data_contracts::repository::usage::StoredUsageAuditSummary, GatewayError>
{
self.app.summarize_usage_audits(query).await
}
pub(crate) async fn summarize_usage_time_series(
&self,
query: &aether_data_contracts::repository::usage::UsageTimeSeriesQuery,
) -> Result<
Vec<aether_data_contracts::repository::usage::StoredUsageTimeSeriesBucket>,
GatewayError,
> {
self.app.summarize_usage_time_series(query).await
}
pub(crate) async fn summarize_usage_leaderboard(
&self,
query: &aether_data_contracts::repository::usage::UsageLeaderboardQuery,
) -> Result<
Vec<aether_data_contracts::repository::usage::StoredUsageLeaderboardSummary>,
GatewayError,
> {
self.app.summarize_usage_leaderboard(query).await
}
pub(crate) async fn summarize_usage_daily_heatmap(
&self,
query: &aether_data_contracts::repository::usage::UsageDailyHeatmapQuery,

View File

@@ -485,8 +485,13 @@ async fn dashboard_list_usage_for_range(
user_id: user_id.map(ToOwned::to_owned),
provider_name: None,
model: None,
api_format: None,
statuses: None,
is_stream: None,
error_only: false,
limit: None,
offset: None,
newest_first: false,
})
.await
{
@@ -1215,8 +1220,13 @@ pub(super) async fn handle_dashboard_provider_status_get(
user_id: None,
provider_name: None,
model: None,
api_format: None,
statuses: None,
is_stream: None,
error_only: false,
limit: None,
offset: None,
newest_first: false,
})
.await
{

View File

@@ -899,8 +899,13 @@ pub(super) async fn handle_users_me_usage_active_get(
user_id: Some(auth.user.id.clone()),
provider_name: None,
model: None,
api_format: None,
statuses: None,
is_stream: None,
error_only: false,
limit: None,
offset: None,
newest_first: false,
})
.await
{
@@ -972,8 +977,13 @@ pub(super) async fn handle_users_me_usage_interval_timeline_get(
user_id: Some(auth.user.id.clone()),
provider_name: None,
model: None,
api_format: None,
statuses: None,
is_stream: None,
error_only: false,
limit: None,
offset: None,
newest_first: false,
})
.await
{

View File

@@ -217,8 +217,13 @@ pub(super) async fn handle_wallet_today_cost(
user_id: Some(auth.user.id.clone()),
provider_name: None,
model: None,
api_format: None,
statuses: None,
is_stream: None,
error_only: false,
limit: None,
offset: None,
newest_first: false,
})
.await
{