i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-01 17:00:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(admin): 修复 usage 观测全表扫描并下推聚合查询

Browse Source 
- 收紧 admin usage/stats 默认时间范围和 active 轮询查询
- 将 summary/records/aggregation/time-series/leaderboard 下推到 SQL 侧
- 统一前端时间参数并补齐 admin usage/stats 回归测试
This commit is contained in:
fawney19
2026-04-16 17:41:32 +08:00
parent 10605d9fb0
commit 65cd9dc3e5
28 changed files with 3085 additions and 381 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
apps/aether-gateway/src/data/state/runtime.rs
View File

@@ -666,6 +666,68 @@ impl GatewayDataState {
}
}
pub(crate) async fn count_usage_audits(
&self,
query: &UsageAuditListQuery,
) -> Result<u64, DataLayerError> {
match &self.usage_reader {
Some(repository) => repository.count_usage_audits(query).await,
None => Ok(0),
}
}
pub(crate) async fn aggregate_usage_audits(
&self,
query: &aether_data_contracts::repository::usage::UsageAuditAggregationQuery,
) -> Result<
Vec<aether_data_contracts::repository::usage::StoredUsageAuditAggregation>,
DataLayerError,
> {
match &self.usage_reader {
Some(repository) => repository.aggregate_usage_audits(query).await,
None => Ok(Vec::new()),
}
}
pub(crate) async fn summarize_usage_audits(
&self,
query: &aether_data_contracts::repository::usage::UsageAuditSummaryQuery,
) -> Result<aether_data_contracts::repository::usage::StoredUsageAuditSummary, DataLayerError>
{
match &self.usage_reader {
Some(repository) => repository.summarize_usage_audits(query).await,
None => {
Ok(aether_data_contracts::repository::usage::StoredUsageAuditSummary::default())
}
}
}
pub(crate) async fn summarize_usage_time_series(
&self,
query: &aether_data_contracts::repository::usage::UsageTimeSeriesQuery,
) -> Result<
Vec<aether_data_contracts::repository::usage::StoredUsageTimeSeriesBucket>,
DataLayerError,
> {
match &self.usage_reader {
Some(repository) => repository.summarize_usage_time_series(query).await,
None => Ok(Vec::new()),
}
}
pub(crate) async fn summarize_usage_leaderboard(
&self,
query: &aether_data_contracts::repository::usage::UsageLeaderboardQuery,
) -> Result<
Vec<aether_data_contracts::repository::usage::StoredUsageLeaderboardSummary>,
DataLayerError,
> {
match &self.usage_reader {
Some(repository) => repository.summarize_usage_leaderboard(query).await,
None => Ok(Vec::new()),
}
}
pub(crate) async fn summarize_usage_daily_heatmap(
&self,
query: &UsageDailyHeatmapQuery,

129
apps/aether-gateway/src/handlers/admin/observability/stats/analytics_routes.rs
View File

@@ -1,4 +1,5 @@
use super::range::build_comparison_range;
use super::resolve_admin_usage_time_range;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::query_param_value;
use crate::GatewayError;
@@ -6,17 +7,29 @@ use aether_admin::observability::stats::{
admin_stats_bad_request_response, admin_stats_comparison_empty_response,
admin_stats_error_distribution_empty_response,
admin_stats_performance_percentiles_empty_response, admin_stats_time_series_empty_response,
build_admin_stats_comparison_response, build_admin_stats_error_distribution_response,
build_admin_stats_performance_percentiles_response, build_admin_stats_time_series_response,
build_admin_stats_comparison_response_from_aggregates,
build_admin_stats_error_distribution_response,
build_admin_stats_performance_percentiles_response,
build_admin_stats_time_series_response_from_summaries, AdminStatsAggregate,
AdminStatsComparisonType, AdminStatsGranularity, AdminStatsTimeRange, AdminStatsUsageFilter,
};
use axum::{
body::Body,
http,
response::{IntoResponse, Response},
Json,
use aether_data_contracts::repository::usage::{
UsageAuditSummaryQuery, UsageTimeSeriesGranularity, UsageTimeSeriesQuery,
};
use serde_json::json;
use axum::{body::Body, http, response::Response};
fn usage_summary_to_admin_stats_aggregate(
summary: &aether_data_contracts::repository::usage::StoredUsageAuditSummary,
) -> AdminStatsAggregate {
AdminStatsAggregate {
total_requests: summary.total_requests,
total_tokens: summary.recorded_total_tokens,
total_cost: summary.total_cost_usd,
actual_total_cost: summary.actual_total_cost_usd,
total_response_time_ms: summary.total_response_time_ms,
error_requests: summary.error_requests,
}
}
pub(super) async fn maybe_build_local_admin_stats_analytics_response(
state: &AdminAppState<'_>,
@@ -58,15 +71,39 @@ pub(super) async fn maybe_build_local_admin_stats_analytics_response(
&comparison_range,
)));
}
let current_usage = state
.list_admin_usage_for_range(&current_range, &AdminStatsUsageFilter::default())
let Some((current_from_unix_secs, current_until_unix_secs)) =
current_range.to_unix_bounds()
else {
return Ok(Some(admin_stats_comparison_empty_response(
&current_range,
&comparison_range,
)));
};
let Some((comparison_from_unix_secs, comparison_until_unix_secs)) =
comparison_range.to_unix_bounds()
else {
return Ok(Some(admin_stats_comparison_empty_response(
&current_range,
&comparison_range,
)));
};
let current_summary = state
.summarize_usage_audits(&UsageAuditSummaryQuery {
created_from_unix_secs: current_from_unix_secs,
created_until_unix_secs: current_until_unix_secs,
..Default::default()
})
.await?;
let comparison_usage = state
.list_admin_usage_for_range(&comparison_range, &AdminStatsUsageFilter::default())
let comparison_summary = state
.summarize_usage_audits(&UsageAuditSummaryQuery {
created_from_unix_secs: comparison_from_unix_secs,
created_until_unix_secs: comparison_until_unix_secs,
..Default::default()
})
.await?;
return Ok(Some(build_admin_stats_comparison_response(
&current_usage,
&comparison_usage,
return Ok(Some(build_admin_stats_comparison_response_from_aggregates(
&usage_summary_to_admin_stats_aggregate(&current_summary),
&usage_summary_to_admin_stats_aggregate(&comparison_summary),
&current_range,
&comparison_range,
)));
@@ -79,19 +116,9 @@ pub(super) async fn maybe_build_local_admin_stats_analytics_response(
"/api/admin/stats/errors/distribution" | "/api/admin/stats/errors/distribution/"
)
{
let Some(time_range) =
(match AdminStatsTimeRange::resolve_optional(request_context.query_string()) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
})
else {
return Ok(Some(
Json(json!({
"distribution": [],
"trend": [],
}))
.into_response(),
));
let time_range = match resolve_admin_usage_time_range(request_context.query_string()) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
if !state.has_usage_data_reader() {
return Ok(Some(admin_stats_error_distribution_empty_response()));
@@ -114,13 +141,9 @@ pub(super) async fn maybe_build_local_admin_stats_analytics_response(
| "/api/admin/stats/performance/percentiles/"
)
{
let Some(time_range) =
(match AdminStatsTimeRange::resolve_optional(request_context.query_string()) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
})
else {
return Ok(Some(Json(json!([])).into_response()));
let time_range = match resolve_admin_usage_time_range(request_context.query_string()) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
if !state.has_usage_data_reader() {
return Ok(Some(admin_stats_performance_percentiles_empty_response()));
@@ -146,13 +169,9 @@ pub(super) async fn maybe_build_local_admin_stats_analytics_response(
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
let Some(time_range) =
(match AdminStatsTimeRange::resolve_optional(request_context.query_string()) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
})
else {
return Ok(Some(Json(json!([])).into_response()));
let time_range = match resolve_admin_usage_time_range(request_context.query_string()) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
if let Err(detail) = time_range.validate_for_time_series(granularity) {
return Ok(Some(admin_stats_bad_request_response(detail)));
@@ -162,13 +181,31 @@ pub(super) async fn maybe_build_local_admin_stats_analytics_response(
}
let filters = AdminStatsUsageFilter::from_query(request_context.query_string());
let usage = state
.list_admin_usage_for_range(&time_range, &filters)
let query_granularity = match granularity {
AdminStatsGranularity::Hour => UsageTimeSeriesGranularity::Hour,
AdminStatsGranularity::Day
| AdminStatsGranularity::Week
| AdminStatsGranularity::Month => UsageTimeSeriesGranularity::Day,
};
let Some((created_from_unix_secs, created_until_unix_secs)) = time_range.to_unix_bounds()
else {
return Ok(Some(admin_stats_time_series_empty_response()));
};
let buckets = state
.summarize_usage_time_series(&UsageTimeSeriesQuery {
created_from_unix_secs,
created_until_unix_secs,
granularity: query_granularity,
tz_offset_minutes: time_range.tz_offset_minutes,
user_id: filters.user_id,
provider_name: filters.provider_name,
model: filters.model,
})
.await?;
return Ok(Some(build_admin_stats_time_series_response(
return Ok(Some(build_admin_stats_time_series_response_from_summaries(
&time_range,
granularity,
&usage,
&buckets,
)));
}

50
apps/aether-gateway/src/handlers/admin/observability/stats/cost_routes.rs
View File

@@ -1,15 +1,31 @@
use super::range::{build_time_range_from_days, parse_bounded_u32, parse_tz_offset_minutes};
use super::resolve_admin_usage_time_range;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::query_param_value;
use crate::GatewayError;
use aether_admin::observability::stats::{
admin_stats_bad_request_response, admin_stats_cost_forecast_empty_response,
admin_stats_cost_savings_empty_response, build_admin_stats_cost_forecast_response,
build_admin_stats_cost_savings_response, AdminStatsGranularity, AdminStatsTimeRange,
AdminStatsUsageFilter,
build_admin_stats_cost_savings_response, AdminStatsGranularity, AdminStatsUsageFilter,
};
use axum::{body::Body, http, response::Response};
fn resolve_cost_forecast_time_range(
query: Option<&str>,
) -> Result<super::AdminStatsTimeRange, String> {
match super::AdminStatsTimeRange::resolve_optional(query)? {
Some(value) => Ok(value),
None => {
let tz_offset_minutes = parse_tz_offset_minutes(query)?;
let days = query_param_value(query, "days")
.map(|value| parse_bounded_u32("days", &value, 7, 365))
.transpose()?
.unwrap_or(30);
build_time_range_from_days(days, tz_offset_minutes)
}
}
}
pub(super) async fn maybe_build_local_admin_stats_cost_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
@@ -34,32 +50,13 @@ pub(super) async fn maybe_build_local_admin_stats_cost_response(
.map(|value| parse_bounded_u32("forecast_days", &value, 1, 90))
.transpose()
{
Ok(Some(value)) => value,
Ok(None) => 7,
Ok(value) => value.unwrap_or(7),
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
let tz_offset_minutes = match parse_tz_offset_minutes(query) {
let time_range = match resolve_cost_forecast_time_range(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
Ok(Some(value)) => value,
Ok(None) => {
let days = match query_param_value(query, "days")
.map(|value| parse_bounded_u32("days", &value, 7, 365))
.transpose()
{
Ok(Some(value)) => value,
Ok(None) => 30,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
match build_time_range_from_days(days, tz_offset_minutes) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
}
}
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
if let Err(detail) = time_range.validate_for_time_series(AdminStatsGranularity::Day) {
return Ok(Some(admin_stats_bad_request_response(detail)));
}
@@ -84,13 +81,10 @@ pub(super) async fn maybe_build_local_admin_stats_cost_response(
"/api/admin/stats/cost/savings" | "/api/admin/stats/cost/savings/"
)
{
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
let time_range = match resolve_admin_usage_time_range(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
if time_range.is_none() {
return Ok(Some(admin_stats_cost_savings_empty_response()));
}
if !state.has_usage_data_reader() {
return Ok(Some(admin_stats_cost_savings_empty_response()));
}
@@ -101,7 +95,7 @@ pub(super) async fn maybe_build_local_admin_stats_cost_response(
model: query_param_value(query, "model"),
};
let usage = state
.list_admin_usage_for_range(time_range.as_ref().expect("time range exists"), &filters)
.list_admin_usage_for_range(&time_range, &filters)
.await?;
return Ok(Some(build_admin_stats_cost_savings_response(&usage)));

8
apps/aether-gateway/src/handlers/admin/observability/stats/leaderboard.rs
View File

@@ -4,9 +4,11 @@ use aether_data_contracts::repository::usage::StoredRequestUsageAudit;
pub(super) use aether_admin::observability::stats::{
build_admin_stats_leaderboard_response, build_api_key_leaderboard_items,
build_model_leaderboard_items, build_user_leaderboard_items, compare_leaderboard_items,
compute_dense_rank, AdminStatsLeaderboardItem, AdminStatsLeaderboardMetric,
AdminStatsLeaderboardNameMode, AdminStatsSortOrder, AdminStatsUserMetadata,
build_api_key_leaderboard_items_from_summaries, build_model_leaderboard_items,
build_model_leaderboard_items_from_summaries, build_user_leaderboard_items,
build_user_leaderboard_items_from_summaries, compare_leaderboard_items, compute_dense_rank,
AdminStatsLeaderboardItem, AdminStatsLeaderboardMetric, AdminStatsLeaderboardNameMode,
AdminStatsSortOrder, AdminStatsUserMetadata,
};
pub(super) async fn load_user_leaderboard_metadata(

105
apps/aether-gateway/src/handlers/admin/observability/stats/leaderboard_routes.rs
View File

@@ -1,16 +1,18 @@
use super::leaderboard::{
build_admin_stats_leaderboard_response, build_api_key_leaderboard_items,
build_model_leaderboard_items, build_user_leaderboard_items, compare_leaderboard_items,
load_user_leaderboard_metadata, AdminStatsLeaderboardNameMode,
build_admin_stats_leaderboard_response, build_api_key_leaderboard_items_from_summaries,
build_model_leaderboard_items_from_summaries, build_user_leaderboard_items_from_summaries,
compare_leaderboard_items, load_user_leaderboard_metadata, AdminStatsLeaderboardNameMode,
};
use super::range::{parse_bounded_u32, parse_nonnegative_usize};
use super::resolve_admin_usage_time_range;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::{query_param_bool, query_param_value};
use crate::GatewayError;
use aether_admin::observability::stats::{
admin_stats_bad_request_response, admin_stats_leaderboard_empty_response,
AdminStatsLeaderboardMetric, AdminStatsSortOrder, AdminStatsTimeRange, AdminStatsUsageFilter,
AdminStatsLeaderboardMetric, AdminStatsSortOrder, AdminStatsUsageFilter,
};
use aether_data_contracts::repository::usage::{UsageLeaderboardGroupBy, UsageLeaderboardQuery};
use axum::{body::Body, http, response::Response};
pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
@@ -29,7 +31,7 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
"/api/admin/stats/leaderboard/models" | "/api/admin/stats/leaderboard/models/"
)
{
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
let time_range = match resolve_admin_usage_time_range(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
@@ -60,19 +62,33 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
if !state.has_usage_data_reader() {
return Ok(Some(admin_stats_leaderboard_empty_response(
metric,
time_range.as_ref(),
Some(&time_range),
)));
}
let filters = AdminStatsUsageFilter::from_query(query);
let usage = state
.list_admin_usage_for_optional_range(time_range.as_ref(), &filters)
let Some((created_from_unix_secs, created_until_unix_secs)) = time_range.to_unix_bounds()
else {
return Ok(Some(admin_stats_leaderboard_empty_response(
metric,
Some(&time_range),
)));
};
let summaries = state
.summarize_usage_leaderboard(&UsageLeaderboardQuery {
created_from_unix_secs,
created_until_unix_secs,
group_by: UsageLeaderboardGroupBy::Model,
user_id: filters.user_id,
provider_name: filters.provider_name,
model: filters.model,
})
.await?;
let mut leaderboard = build_model_leaderboard_items(&usage);
let mut leaderboard = build_model_leaderboard_items_from_summaries(&summaries);
leaderboard.sort_by(|left, right| compare_leaderboard_items(metric, order, left, right));
return Ok(Some(build_admin_stats_leaderboard_response(
metric,
time_range.as_ref(),
Some(&time_range),
&leaderboard,
offset,
limit,
@@ -90,7 +106,7 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
"/api/admin/stats/leaderboard/api-keys" | "/api/admin/stats/leaderboard/api-keys/"
)
{
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
let time_range = match resolve_admin_usage_time_range(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
@@ -121,20 +137,32 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
if !state.has_usage_data_reader() {
return Ok(Some(admin_stats_leaderboard_empty_response(
metric,
time_range.as_ref(),
Some(&time_range),
)));
}
let include_inactive = query_param_bool(query, "include_inactive", false);
let exclude_admin = query_param_bool(query, "exclude_admin", false);
let filters = AdminStatsUsageFilter::from_query(query);
let usage = state
.list_admin_usage_for_optional_range(time_range.as_ref(), &filters)
let Some((created_from_unix_secs, created_until_unix_secs)) = time_range.to_unix_bounds()
else {
return Ok(Some(admin_stats_leaderboard_empty_response(
metric,
Some(&time_range),
)));
};
let summaries = state
.summarize_usage_leaderboard(&UsageLeaderboardQuery {
created_from_unix_secs,
created_until_unix_secs,
group_by: UsageLeaderboardGroupBy::ApiKey,
user_id: filters.user_id,
provider_name: filters.provider_name,
model: filters.model,
})
.await?;
let api_key_ids: Vec<String> = usage
let api_key_ids: Vec<String> = summaries
.iter()
.filter_map(|item| item.api_key_id.clone())
.collect::<std::collections::BTreeSet<_>>()
.into_iter()
.map(|item| item.group_key.clone())
.collect();
let snapshots = if state.has_auth_api_key_data_reader() {
Some(
@@ -152,8 +180,8 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
} else {
std::collections::BTreeMap::new()
};
let mut leaderboard = build_api_key_leaderboard_items(
&usage,
let mut leaderboard = build_api_key_leaderboard_items_from_summaries(
&summaries,
snapshots.as_deref(),
&api_key_names,
include_inactive,
@@ -163,7 +191,7 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
return Ok(Some(build_admin_stats_leaderboard_response(
metric,
time_range.as_ref(),
Some(&time_range),
&leaderboard,
offset,
limit,
@@ -181,7 +209,7 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
"/api/admin/stats/leaderboard/users" | "/api/admin/stats/leaderboard/users/"
)
{
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
let time_range = match resolve_admin_usage_time_range(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
@@ -212,26 +240,39 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
if !state.has_usage_data_reader() {
return Ok(Some(admin_stats_leaderboard_empty_response(
metric,
time_range.as_ref(),
Some(&time_range),
)));
}
let include_inactive = query_param_bool(query, "include_inactive", false);
let exclude_admin = query_param_bool(query, "exclude_admin", false);
let filters = AdminStatsUsageFilter::from_query(query);
let usage = state
.list_admin_usage_for_optional_range(time_range.as_ref(), &filters)
let Some((created_from_unix_secs, created_until_unix_secs)) = time_range.to_unix_bounds()
else {
return Ok(Some(admin_stats_leaderboard_empty_response(
metric,
Some(&time_range),
)));
};
let summaries = state
.summarize_usage_leaderboard(&UsageLeaderboardQuery {
created_from_unix_secs,
created_until_unix_secs,
group_by: UsageLeaderboardGroupBy::User,
user_id: filters.user_id,
provider_name: filters.provider_name,
model: filters.model,
})
.await?;
let user_ids: Vec<String> = usage
let user_ids: Vec<String> = summaries
.iter()
.filter_map(|item| item.user_id.clone())
.collect::<std::collections::BTreeSet<_>>()
.into_iter()
.map(|item| item.group_key.clone())
.collect();
let user_metadata = load_user_leaderboard_metadata(state, &user_ids).await?;
let mut leaderboard = build_user_leaderboard_items(
&usage,
let mut leaderboard = build_user_leaderboard_items_from_summaries(
&summaries,
&user_metadata,
state.has_auth_user_data_reader(),
state.has_user_data_reader(),
include_inactive,
exclude_admin,
);
@@ -239,7 +280,7 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
return Ok(Some(build_admin_stats_leaderboard_response(
metric,
time_range.as_ref(),
Some(&time_range),
&leaderboard,
offset,
limit,

1
apps/aether-gateway/src/handlers/admin/observability/stats/mod.rs
View File

@@ -10,6 +10,7 @@ mod provider_quota_routes;
mod range;
pub(crate) use self::range::{
list_usage_for_optional_range, list_usage_for_range, parse_bounded_u32,
resolve_admin_usage_time_range,
};
pub(crate) use aether_admin::observability::stats::{
admin_stats_bad_request_response, aggregate_usage_stats, round_to, AdminStatsTimeRange,

35
apps/aether-gateway/src/handlers/admin/observability/stats/range.rs
View File

@@ -8,6 +8,22 @@ pub(super) use aether_admin::observability::stats::{
use aether_admin::observability::stats::{AdminStatsTimeRange, AdminStatsUsageFilter};
use aether_data_contracts::repository::usage::{StoredRequestUsageAudit, UsageAuditListQuery};
pub(crate) fn resolve_admin_usage_time_range(
query: Option<&str>,
) -> Result<AdminStatsTimeRange, String> {
match AdminStatsTimeRange::resolve_optional(query)? {
Some(time_range) => Ok(time_range),
None => {
let tz_offset_minutes = parse_tz_offset_minutes(query)?;
let default_days = u32::try_from(admin_usage_default_days())
.ok()
.filter(|value| *value > 0)
.unwrap_or(1);
build_time_range_from_days(default_days, tz_offset_minutes)
}
}
}
pub(crate) async fn list_usage_for_range(
state: &AdminAppState<'_>,
time_range: &AdminStatsTimeRange,
@@ -25,8 +41,13 @@ pub(crate) async fn list_usage_for_range(
user_id: filters.user_id.clone(),
provider_name: filters.provider_name.clone(),
model: filters.model.clone(),
api_format: None,
statuses: None,
is_stream: None,
error_only: false,
limit: None,
offset: None,
newest_first: false,
})
.await
}
@@ -39,17 +60,9 @@ pub(crate) async fn list_usage_for_optional_range(
match time_range {
Some(time_range) => list_usage_for_range(state, time_range, filters).await,
None => {
state
.list_usage_audits(&UsageAuditListQuery {
created_from_unix_secs: None,
created_until_unix_secs: None,
user_id: filters.user_id.clone(),
provider_name: filters.provider_name.clone(),
model: filters.model.clone(),
statuses: None,
limit: None,
})
.await
let default_time_range = build_time_range_from_days(1, 0)
.map_err(|detail| GatewayError::Internal(detail.to_string()))?;
list_usage_for_range(state, &default_time_range, filters).await
}
}
}

62
apps/aether-gateway/src/handlers/admin/observability/usage/analytics/aggregations.rs
View File

@@ -1,29 +1,21 @@
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
use aether_admin::observability::stats::round_to;
use aether_data_contracts::repository::usage::{StoredRequestUsageAudit, UsageAuditListQuery};
use aether_data_contracts::repository::usage::StoredUsageAuditAggregation;
use serde_json::json;
use std::collections::BTreeMap;
pub(in super::super) async fn admin_usage_aggregation_by_user_json(
state: &AdminAppState<'_>,
usage: &[StoredRequestUsageAudit],
limit: usize,
rows: &[StoredUsageAuditAggregation],
) -> Result<serde_json::Value, GatewayError> {
let mut grouped: BTreeMap<String, (u64, u64, f64)> = BTreeMap::new();
for item in usage {
let Some(user_id) = item.user_id.as_ref() else {
continue;
};
let entry = grouped.entry(user_id.clone()).or_insert((0, 0, 0.0));
entry.0 = entry.0.saturating_add(1);
entry.1 = entry.1.saturating_add(item.total_tokens);
entry.2 += item.total_cost_usd;
}
let usernames = if state.has_user_data_reader() && !grouped.is_empty() {
let user_ids = rows
.iter()
.map(|row| row.group_key.clone())
.collect::<Vec<_>>();
let usernames = if state.has_user_data_reader() && !user_ids.is_empty() {
state
.list_users_by_ids(&grouped.keys().cloned().collect::<Vec<_>>())
.list_users_by_ids(&user_ids)
.await?
.into_iter()
.map(|user| (user.id, (user.email, user.username)))
@@ -32,35 +24,25 @@ pub(in super::super) async fn admin_usage_aggregation_by_user_json(
BTreeMap::new()
};
let mut items: Vec<serde_json::Value> = grouped
.into_iter()
.map(|(user_id, (request_count, total_tokens, total_cost))| {
Ok(json!(rows
.iter()
.map(|row| {
let (email, username) = usernames
.get(&user_id)
.get(&row.group_key)
.cloned()
.unwrap_or((None, String::new()));
json!({
"user_id": user_id,
"user_id": row.group_key,
"email": email,
"username": if username.is_empty() { serde_json::Value::Null } else { json!(username) },
"request_count": request_count,
"total_tokens": total_tokens,
"total_cost": round_to(total_cost, 6),
"username": if username.is_empty() {
serde_json::Value::Null
} else {
json!(username)
},
"request_count": row.request_count,
"total_tokens": row.total_tokens,
"total_cost": round_to(row.total_cost_usd, 6),
})
})
.collect();
items.sort_by(|left, right| {
right["request_count"]
.as_u64()
.unwrap_or_default()
.cmp(&left["request_count"].as_u64().unwrap_or_default())
.then_with(|| {
left["user_id"]
.as_str()
.unwrap_or_default()
.cmp(right["user_id"].as_str().unwrap_or_default())
})
});
items.truncate(limit);
Ok(json!(items))
.collect::<Vec<_>>()))
}

5
apps/aether-gateway/src/handlers/admin/observability/usage/analytics/cache_affinity.rs
View File

@@ -16,8 +16,13 @@ pub(in super::super) async fn list_recent_completed_usage_for_cache_affinity(
user_id: user_id.map(ToOwned::to_owned),
provider_name: None,
model: None,
api_format: None,
statuses: None,
is_stream: None,
error_only: false,
limit: None,
offset: None,
newest_first: false,
})
.await?;
items.retain(|item| item.status == "completed");

139
apps/aether-gateway/src/handlers/admin/observability/usage/analytics_routes/aggregation.rs
View File

@@ -1,14 +1,17 @@
use super::super::super::stats::{AdminStatsTimeRange, AdminStatsUsageFilter};
use super::super::super::stats::resolve_admin_usage_time_range;
use super::super::analytics::admin_usage_aggregation_by_user_json;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::query_param_value;
use crate::GatewayError;
use aether_admin::observability::stats::round_to;
use aether_admin::observability::usage::{
admin_usage_aggregation_by_api_format_json, admin_usage_aggregation_by_model_json,
admin_usage_aggregation_by_provider_json, admin_usage_bad_request_response,
admin_usage_data_unavailable_response, admin_usage_parse_aggregation_limit,
admin_usage_bad_request_response, admin_usage_data_unavailable_response,
admin_usage_parse_aggregation_limit, admin_usage_token_cache_hit_rate,
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
};
use aether_data_contracts::repository::usage::{
StoredUsageAuditAggregation, UsageAuditAggregationGroupBy, UsageAuditAggregationQuery,
};
use axum::{
body::Body,
response::{IntoResponse, Response},
@@ -16,6 +19,102 @@ use axum::{
};
use serde_json::json;
fn admin_usage_aggregation_by_model_json(
rows: &[StoredUsageAuditAggregation],
) -> serde_json::Value {
json!(rows
.iter()
.map(|row| {
json!({
"model": row.group_key,
"request_count": row.request_count,
"total_tokens": row.total_tokens,
"effective_input_tokens": row.effective_input_tokens,
"total_input_context": row.total_input_context,
"output_tokens": row.output_tokens,
"total_cost": round_to(row.total_cost_usd, 6),
"actual_cost": round_to(row.actual_total_cost_usd, 6),
"cache_creation_tokens": row.cache_creation_tokens,
"cache_creation_ephemeral_5m_tokens": row.cache_creation_ephemeral_5m_tokens,
"cache_creation_ephemeral_1h_tokens": row.cache_creation_ephemeral_1h_tokens,
"cache_read_tokens": row.cache_read_tokens,
"cache_hit_rate": admin_usage_token_cache_hit_rate(
row.total_input_context,
row.cache_read_tokens,
),
})
})
.collect::<Vec<_>>())
}
fn admin_usage_aggregation_by_provider_json(
rows: &[StoredUsageAuditAggregation],
) -> serde_json::Value {
json!(rows
.iter()
.map(|row| {
let success_count = row.success_count.unwrap_or_default();
let error_count = row.request_count.saturating_sub(success_count);
let success_rate = if row.request_count == 0 {
0.0
} else {
round_to(success_count as f64 / row.request_count as f64 * 100.0, 2)
};
json!({
"provider_id": row.group_key,
"provider": row.display_name.clone().unwrap_or_else(|| "Unknown".to_string()),
"request_count": row.request_count,
"total_tokens": row.total_tokens,
"effective_input_tokens": row.effective_input_tokens,
"total_input_context": row.total_input_context,
"output_tokens": row.output_tokens,
"total_cost": round_to(row.total_cost_usd, 6),
"actual_cost": round_to(row.actual_total_cost_usd, 6),
"avg_response_time_ms": round_to(row.avg_response_time_ms.unwrap_or(0.0), 2),
"success_rate": success_rate,
"error_count": error_count,
"cache_creation_tokens": row.cache_creation_tokens,
"cache_creation_ephemeral_5m_tokens": row.cache_creation_ephemeral_5m_tokens,
"cache_creation_ephemeral_1h_tokens": row.cache_creation_ephemeral_1h_tokens,
"cache_read_tokens": row.cache_read_tokens,
"cache_hit_rate": admin_usage_token_cache_hit_rate(
row.total_input_context,
row.cache_read_tokens,
),
})
})
.collect::<Vec<_>>())
}
fn admin_usage_aggregation_by_api_format_json(
rows: &[StoredUsageAuditAggregation],
) -> serde_json::Value {
json!(rows
.iter()
.map(|row| {
json!({
"api_format": row.group_key,
"request_count": row.request_count,
"total_tokens": row.total_tokens,
"effective_input_tokens": row.effective_input_tokens,
"total_input_context": row.total_input_context,
"output_tokens": row.output_tokens,
"total_cost": round_to(row.total_cost_usd, 6),
"actual_cost": round_to(row.actual_total_cost_usd, 6),
"avg_response_time_ms": round_to(row.avg_response_time_ms.unwrap_or(0.0), 2),
"cache_creation_tokens": row.cache_creation_tokens,
"cache_creation_ephemeral_5m_tokens": row.cache_creation_ephemeral_5m_tokens,
"cache_creation_ephemeral_1h_tokens": row.cache_creation_ephemeral_1h_tokens,
"cache_read_tokens": row.cache_read_tokens,
"cache_hit_rate": admin_usage_token_cache_hit_rate(
row.total_input_context,
row.cache_read_tokens,
),
})
})
.collect::<Vec<_>>())
}
pub(super) async fn build_admin_usage_aggregation_stats_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
@@ -43,21 +142,35 @@ pub(super) async fn build_admin_usage_aggregation_stats_response(
Ok(value) => value,
Err(detail) => return Ok(admin_usage_bad_request_response(detail)),
};
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
let time_range = match resolve_admin_usage_time_range(query) {
Ok(value) => value,
Err(detail) => return Ok(admin_usage_bad_request_response(detail)),
};
let mut usage = state
.list_admin_usage_for_optional_range(time_range.as_ref(), &AdminStatsUsageFilter::default())
let Some((created_from_unix_secs, created_until_unix_secs)) = time_range.to_unix_bounds()
else {
return Ok(Json(json!([])).into_response());
};
let group_by_query = match group_by.as_str() {
"model" => UsageAuditAggregationGroupBy::Model,
"user" => UsageAuditAggregationGroupBy::User,
"provider" => UsageAuditAggregationGroupBy::Provider,
"api_format" => UsageAuditAggregationGroupBy::ApiFormat,
_ => unreachable!(),
};
let usage = state
.aggregate_usage_audits(&UsageAuditAggregationQuery {
created_from_unix_secs,
created_until_unix_secs,
group_by: group_by_query,
limit,
})
.await?;
usage.retain(|item| item.status != "pending" && item.status != "streaming");
let response = match group_by.as_str() {
"model" => admin_usage_aggregation_by_model_json(&usage, limit),
"user" => admin_usage_aggregation_by_user_json(state, &usage, limit).await?,
"provider" => admin_usage_aggregation_by_provider_json(&usage, limit),
"api_format" => admin_usage_aggregation_by_api_format_json(&usage, limit),
"model" => admin_usage_aggregation_by_model_json(&usage),
"user" => admin_usage_aggregation_by_user_json(state, &usage).await?,
"provider" => admin_usage_aggregation_by_provider_json(&usage),
"api_format" => admin_usage_aggregation_by_api_format_json(&usage),
_ => unreachable!(),
};
Ok(Json(response).into_response())

279
apps/aether-gateway/src/handlers/admin/observability/usage/summary_routes.rs
View File

@@ -1,4 +1,4 @@
use super::super::stats::{AdminStatsTimeRange, AdminStatsUsageFilter};
use super::super::stats::resolve_admin_usage_time_range;
use super::analytics::admin_usage_api_key_names;
use super::analytics::admin_usage_provider_key_names;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
@@ -6,16 +6,89 @@ use crate::handlers::admin::shared::query_param_value;
use crate::GatewayError;
use aether_admin::observability::usage::{
admin_usage_bad_request_response, admin_usage_data_unavailable_response,
admin_usage_matches_api_format, admin_usage_matches_eq, admin_usage_matches_search,
admin_usage_matches_status, admin_usage_matches_username, admin_usage_parse_ids,
admin_usage_matches_search, admin_usage_matches_username, admin_usage_parse_ids,
admin_usage_parse_limit, admin_usage_parse_offset, build_admin_usage_active_requests_response,
build_admin_usage_records_response, build_admin_usage_summary_stats_response,
build_admin_usage_records_response, build_admin_usage_summary_stats_response_from_summary,
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
};
use aether_data_contracts::repository::usage::UsageAuditListQuery;
use aether_data_contracts::repository::usage::{
StoredRequestUsageAudit, UsageAuditListQuery, UsageAuditSummaryQuery,
};
use axum::{body::Body, http, response::Response};
use std::collections::{BTreeMap, BTreeSet};
const ADMIN_USAGE_ACTIVE_LIMIT: usize = 50;
async fn load_admin_usage_by_ids(
state: &AdminAppState<'_>,
requested_ids: &BTreeSet<String>,
) -> Result<Vec<StoredRequestUsageAudit>, GatewayError> {
let mut items = Vec::with_capacity(requested_ids.len());
for usage_id in requested_ids {
if let Some(item) = state.find_request_usage_by_id(usage_id).await? {
items.push(item);
}
}
Ok(items)
}
fn sort_usage_newest_first(items: &mut [StoredRequestUsageAudit]) {
items.sort_by(|left, right| {
right
.created_at_unix_ms
.cmp(&left.created_at_unix_ms)
.then_with(|| left.id.cmp(&right.id))
});
}
fn apply_admin_usage_status_filter(query: &mut UsageAuditListQuery, status: Option<&str>) {
let Some(status) = status
.map(str::trim)
.filter(|candidate| !candidate.is_empty())
else {
return;
};
match status {
"stream" => query.is_stream = Some(true),
"standard" => query.is_stream = Some(false),
"error" | "failed" => query.error_only = true,
"active" => {
query.statuses = Some(vec!["pending".to_string(), "streaming".to_string()]);
}
"pending" | "streaming" | "completed" | "cancelled" => {
query.statuses = Some(vec![status.to_string()]);
}
_ => {}
}
}
fn build_admin_usage_records_query(
created_from_unix_secs: u64,
created_until_unix_secs: u64,
query: Option<&str>,
limit: Option<usize>,
offset: Option<usize>,
) -> UsageAuditListQuery {
let mut list_query = UsageAuditListQuery {
created_from_unix_secs: Some(created_from_unix_secs),
created_until_unix_secs: Some(created_until_unix_secs),
user_id: query_param_value(query, "user_id"),
provider_name: query_param_value(query, "provider"),
model: query_param_value(query, "model"),
api_format: query_param_value(query, "api_format"),
limit,
offset,
newest_first: true,
..Default::default()
};
apply_admin_usage_status_filter(
&mut list_query,
query_param_value(query, "status").as_deref(),
);
list_query
}
pub(super) async fn maybe_build_local_admin_usage_summary_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
@@ -40,17 +113,27 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
}
let query = request_context.request_query_string.as_deref();
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
let time_range = match resolve_admin_usage_time_range(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
let usage = state
.list_admin_usage_for_optional_range(
time_range.as_ref(),
&AdminStatsUsageFilter::default(),
)
let Some((created_from_unix_secs, created_until_unix_secs)) =
time_range.to_unix_bounds()
else {
return Ok(Some(build_admin_usage_summary_stats_response_from_summary(
&Default::default(),
)));
};
let summary = state
.summarize_usage_audits(&UsageAuditSummaryQuery {
created_from_unix_secs,
created_until_unix_secs,
..Default::default()
})
.await?;
return Ok(Some(build_admin_usage_summary_stats_response(&usage)));
return Ok(Some(build_admin_usage_summary_stats_response_from_summary(
&summary,
)));
}
Some("active")
if request_context.request_method == http::Method::GET
@@ -67,30 +150,37 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
let query = request_context.request_query_string.as_deref();
let requested_ids = admin_usage_parse_ids(query);
let list_query = if requested_ids.is_some() {
UsageAuditListQuery::default()
let mut items = if let Some(requested_ids) = requested_ids.as_ref() {
load_admin_usage_by_ids(state, requested_ids).await?
} else {
UsageAuditListQuery {
statuses: Some(vec!["pending".to_string(), "streaming".to_string()]),
..Default::default()
}
let time_range = match resolve_admin_usage_time_range(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
let Some((created_from_unix_secs, created_until_unix_secs)) =
time_range.to_unix_bounds()
else {
return Ok(Some(build_admin_usage_active_requests_response(
&[],
&BTreeMap::new(),
state.has_auth_api_key_data_reader(),
&BTreeMap::new(),
)));
};
state
.list_usage_audits(&UsageAuditListQuery {
created_from_unix_secs: Some(created_from_unix_secs),
created_until_unix_secs: Some(created_until_unix_secs),
statuses: Some(vec!["pending".to_string(), "streaming".to_string()]),
limit: Some(ADMIN_USAGE_ACTIVE_LIMIT),
newest_first: true,
..Default::default()
})
.await?
};
let usage = state.list_usage_audits(&list_query).await?;
let mut items: Vec<_> = usage
.into_iter()
.filter(|item| match requested_ids.as_ref() {
Some(ids) => ids.contains(&item.id),
None => true,
})
.collect();
items.sort_by(|left, right| {
right
.created_at_unix_ms
.cmp(&left.created_at_unix_ms)
.then_with(|| left.id.cmp(&right.id))
});
if requested_ids.is_none() && items.len() > 50 {
items.truncate(50);
sort_usage_newest_first(&mut items);
if requested_ids.is_none() && items.len() > ADMIN_USAGE_ACTIVE_LIMIT {
items.truncate(ADMIN_USAGE_ACTIVE_LIMIT);
}
let api_key_names = admin_usage_api_key_names(state, &items).await?;
let provider_key_names = admin_usage_provider_key_names(state, &items).await?;
@@ -116,25 +206,12 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
}
let query = request_context.request_query_string.as_deref();
let time_range = match AdminStatsTimeRange::resolve_optional(query) {
let time_range = match resolve_admin_usage_time_range(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
let filters = AdminStatsUsageFilter {
user_id: query_param_value(query, "user_id"),
provider_name: None,
model: None,
};
let mut usage = state
.list_admin_usage_for_optional_range(time_range.as_ref(), &filters)
.await?;
let search = query_param_value(query, "search");
let username_filter = query_param_value(query, "username");
let model_filter = query_param_value(query, "model");
let provider_filter = query_param_value(query, "provider");
let api_format_filter = query_param_value(query, "api_format");
let status_filter = query_param_value(query, "status");
let limit = match admin_usage_parse_limit(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
@@ -143,6 +220,74 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
let Some((created_from_unix_secs, created_until_unix_secs)) =
time_range.to_unix_bounds()
else {
return Ok(Some(build_admin_usage_records_response(
&[],
&BTreeMap::new(),
&BTreeMap::new(),
state.has_auth_user_data_reader(),
state.has_auth_api_key_data_reader(),
&BTreeMap::new(),
0,
limit,
offset,
)));
};
let base_query = build_admin_usage_records_query(
created_from_unix_secs,
created_until_unix_secs,
query,
None,
None,
);
let use_metadata_fallback = search.is_some() || username_filter.is_some();
let (usage, total) = if use_metadata_fallback {
let mut usage = state.list_usage_audits(&base_query).await?;
let user_ids: Vec<String> = usage
.iter()
.filter_map(|item| item.user_id.clone())
.collect::<BTreeSet<_>>()
.into_iter()
.collect();
let users_by_id: BTreeMap<
String,
aether_data::repository::users::StoredUserSummary,
> = state.resolve_auth_user_summaries_by_ids(&user_ids).await?;
let api_key_names = admin_usage_api_key_names(state, &usage).await?;
usage.retain(|item| {
admin_usage_matches_search(
item,
search.as_deref(),
&users_by_id,
&api_key_names,
state.has_auth_user_data_reader(),
state.has_auth_api_key_data_reader(),
) && admin_usage_matches_username(
item,
username_filter.as_deref(),
&users_by_id,
state.has_auth_user_data_reader(),
)
});
sort_usage_newest_first(&mut usage);
let total = usage.len();
let records = usage
.into_iter()
.skip(offset)
.take(limit)
.collect::<Vec<_>>();
(records, total)
} else {
let total = usize::try_from(state.count_usage_audits(&base_query).await?)
.unwrap_or(usize::MAX);
let mut paged_query = base_query.clone();
paged_query.limit = Some(limit);
paged_query.offset = Some(offset);
(state.list_usage_audits(&paged_query).await?, total)
};
let user_ids: Vec<String> = usage
.iter()
@@ -153,46 +298,10 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
let users_by_id: BTreeMap<String, aether_data::repository::users::StoredUserSummary> =
state.resolve_auth_user_summaries_by_ids(&user_ids).await?;
let api_key_names = admin_usage_api_key_names(state, &usage).await?;
usage.retain(|item| {
admin_usage_matches_search(
item,
search.as_deref(),
&users_by_id,
&api_key_names,
state.has_auth_user_data_reader(),
state.has_auth_api_key_data_reader(),
) && admin_usage_matches_username(
item,
username_filter.as_deref(),
&users_by_id,
state.has_auth_user_data_reader(),
) && admin_usage_matches_eq(item.model.as_str(), model_filter.as_deref())
&& admin_usage_matches_eq(
item.provider_name.as_str(),
provider_filter.as_deref(),
)
&& admin_usage_matches_api_format(item, api_format_filter.as_deref())
&& admin_usage_matches_status(item, status_filter.as_deref())
});
usage.sort_by(|left, right| {
right
.created_at_unix_ms
.cmp(&left.created_at_unix_ms)
.then_with(|| left.id.cmp(&right.id))
});
let total = usage.len();
let provider_key_names = admin_usage_provider_key_names(state, &usage).await?;
let records = usage
.into_iter()
.skip(offset)
.take(limit)
.collect::<Vec<_>>();
return Ok(Some(build_admin_usage_records_response(
&records,
&usage,
&users_by_id,
&api_key_names,
state.has_auth_user_data_reader(),

45
apps/aether-gateway/src/handlers/admin/request/observability.rs
View File

@@ -40,6 +40,51 @@ impl<'a> AdminAppState<'a> {
self.app.list_usage_audits(query).await
}
pub(crate) async fn count_usage_audits(
&self,
query: &aether_data_contracts::repository::usage::UsageAuditListQuery,
) -> Result<u64, GatewayError> {
self.app.count_usage_audits(query).await
}
pub(crate) async fn aggregate_usage_audits(
&self,
query: &aether_data_contracts::repository::usage::UsageAuditAggregationQuery,
) -> Result<
Vec<aether_data_contracts::repository::usage::StoredUsageAuditAggregation>,
GatewayError,
> {
self.app.aggregate_usage_audits(query).await
}
pub(crate) async fn summarize_usage_audits(
&self,
query: &aether_data_contracts::repository::usage::UsageAuditSummaryQuery,
) -> Result<aether_data_contracts::repository::usage::StoredUsageAuditSummary, GatewayError>
{
self.app.summarize_usage_audits(query).await
}
pub(crate) async fn summarize_usage_time_series(
&self,
query: &aether_data_contracts::repository::usage::UsageTimeSeriesQuery,
) -> Result<
Vec<aether_data_contracts::repository::usage::StoredUsageTimeSeriesBucket>,
GatewayError,
> {
self.app.summarize_usage_time_series(query).await
}
pub(crate) async fn summarize_usage_leaderboard(
&self,
query: &aether_data_contracts::repository::usage::UsageLeaderboardQuery,
) -> Result<
Vec<aether_data_contracts::repository::usage::StoredUsageLeaderboardSummary>,
GatewayError,
> {
self.app.summarize_usage_leaderboard(query).await
}
pub(crate) async fn summarize_usage_daily_heatmap(
&self,
query: &aether_data_contracts::repository::usage::UsageDailyHeatmapQuery,

10
apps/aether-gateway/src/handlers/public/support/dashboard_filters.rs
View File

@@ -485,8 +485,13 @@ async fn dashboard_list_usage_for_range(
user_id: user_id.map(ToOwned::to_owned),
provider_name: None,
model: None,
api_format: None,
statuses: None,
is_stream: None,
error_only: false,
limit: None,
offset: None,
newest_first: false,
})
.await
{
@@ -1215,8 +1220,13 @@ pub(super) async fn handle_dashboard_provider_status_get(
user_id: None,
provider_name: None,
model: None,
api_format: None,
statuses: None,
is_stream: None,
error_only: false,
limit: None,
offset: None,
newest_first: false,
})
.await
{

10
apps/aether-gateway/src/handlers/public/support/user_me_usage.rs
View File

@@ -899,8 +899,13 @@ pub(super) async fn handle_users_me_usage_active_get(
user_id: Some(auth.user.id.clone()),
provider_name: None,
model: None,
api_format: None,
statuses: None,
is_stream: None,
error_only: false,
limit: None,
offset: None,
newest_first: false,
})
.await
{
@@ -972,8 +977,13 @@ pub(super) async fn handle_users_me_usage_interval_timeline_get(
user_id: Some(auth.user.id.clone()),
provider_name: None,
model: None,
api_format: None,
statuses: None,
is_stream: None,
error_only: false,
limit: None,
offset: None,
newest_first: false,
})
.await
{

5
apps/aether-gateway/src/handlers/public/support/wallet/reads.rs
View File

@@ -217,8 +217,13 @@ pub(super) async fn handle_wallet_today_cost(
user_id: Some(auth.user.id.clone()),
provider_name: None,
model: None,
api_format: None,
statuses: None,
is_stream: None,
error_only: false,
limit: None,
offset: None,
newest_first: false,
})
.await
{

50
apps/aether-gateway/src/state/runtime/usage_queries.rs
View File

@@ -45,6 +45,56 @@ impl AppState {
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn count_usage_audits(
&self,
query: &usage::UsageAuditListQuery,
) -> Result<u64, GatewayError> {
self.data
.count_usage_audits(query)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn aggregate_usage_audits(
&self,
query: &usage::UsageAuditAggregationQuery,
) -> Result<Vec<usage::StoredUsageAuditAggregation>, GatewayError> {
self.data
.aggregate_usage_audits(query)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn summarize_usage_audits(
&self,
query: &usage::UsageAuditSummaryQuery,
) -> Result<usage::StoredUsageAuditSummary, GatewayError> {
self.data
.summarize_usage_audits(query)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn summarize_usage_time_series(
&self,
query: &usage::UsageTimeSeriesQuery,
) -> Result<Vec<usage::StoredUsageTimeSeriesBucket>, GatewayError> {
self.data
.summarize_usage_time_series(query)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn summarize_usage_leaderboard(
&self,
query: &usage::UsageLeaderboardQuery,
) -> Result<Vec<usage::StoredUsageLeaderboardSummary>, GatewayError> {
self.data
.summarize_usage_leaderboard(query)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn summarize_usage_daily_heatmap(
&self,
query: &UsageDailyHeatmapQuery,

221
apps/aether-gateway/src/tests/control/admin/stats.rs
View File

@@ -156,6 +156,11 @@ fn sample_api_key_snapshot(
snapshot
}
fn recent_unix_secs(minutes_ago: u64) -> i64 {
let now = chrono::Utc::now().timestamp();
now.saturating_sub((minutes_ago * 60) as i64)
}
#[derive(Debug)]
struct PartialListAuthApiKeyRepository {
lookup: InMemoryAuthApiKeySnapshotRepository,
@@ -673,6 +678,83 @@ async fn gateway_handles_admin_stats_error_distribution_locally_without_usage_re
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_defaults_admin_stats_error_distribution_to_bounded_recent_window_when_query_missing(
) {
let (_upstream_url, upstream_hits, upstream_handle) =
start_stats_upstream("/api/admin/stats/errors/distribution").await;
let mut recent_error = sample_usage_row(
"usage-error-recent",
"req-error-recent",
Some("user-1"),
Some("key-1"),
Some("primary"),
"OpenAI",
"gpt-5",
20,
10,
0.02,
0.02,
recent_unix_secs(5),
);
recent_error.status_code = Some(429);
recent_error.error_category = Some("rate_limit".to_string());
recent_error.error_message = Some("rate limited".to_string());
let mut stale_error = sample_usage_row(
"usage-error-stale",
"req-error-stale",
Some("user-1"),
Some("key-1"),
Some("primary"),
"OpenAI",
"gpt-5",
20,
10,
0.02,
0.02,
recent_unix_secs(60 * 48),
);
stale_error.status_code = Some(401);
stale_error.error_category = Some("auth".to_string());
stale_error.error_message = Some("bad key".to_string());
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![
recent_error,
stale_error,
]));
let gateway = build_router_with_state(
AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(GatewayDataState::with_usage_reader_for_tests(
usage_repository,
)),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = admin_request(
reqwest::Client::new().get(format!("{gateway_url}/api/admin/stats/errors/distribution")),
)
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = response.json().await.expect("json body should parse");
assert_eq!(
payload["distribution"].as_array().map(|items| items.len()),
Some(1)
);
assert_eq!(payload["distribution"][0]["category"], "rate_limit");
assert_eq!(payload["distribution"][0]["count"], 1);
assert_eq!(payload["trend"][0]["total"], 1);
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_handles_admin_stats_performance_percentiles_locally_with_trusted_admin_principal()
{
@@ -839,6 +921,80 @@ async fn gateway_handles_admin_stats_time_series_locally_with_trusted_admin_prin
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_handles_admin_stats_time_series_hourly_locally_with_model_filter() {
let (_upstream_url, upstream_hits, upstream_handle) =
start_stats_upstream("/api/admin/stats/time-series").await;
let mut matching_row = sample_usage_row(
"usage-ts-hour-match",
"req-ts-hour-match",
Some("user-1"),
Some("key-1"),
Some("primary"),
"OpenAI",
"gpt-5",
80,
20,
0.12,
0.12,
1_710_997_800,
);
matching_row.cache_creation_input_tokens = 4;
matching_row.cache_read_input_tokens = 6;
let other_model_row = sample_usage_row(
"usage-ts-hour-other",
"req-ts-hour-other",
Some("user-1"),
Some("key-1"),
Some("primary"),
"OpenAI",
"gpt-4o-mini",
200,
40,
0.22,
0.22,
1_710_998_400,
);
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![
matching_row,
other_model_row,
]));
let gateway = build_router_with_state(
AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(GatewayDataState::with_usage_reader_for_tests(
usage_repository,
)),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = admin_request(
reqwest::Client::new().get(format!(
"{gateway_url}/api/admin/stats/time-series?start_date=2024-03-21&end_date=2024-03-21&granularity=hour&model=gpt-5&tz_offset_minutes=0"
)),
)
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = response.json().await.expect("json body should parse");
assert_eq!(payload.as_array().map(|items| items.len()), Some(24));
assert_eq!(payload[5]["date"], "2024-03-21T05:00:00+00:00");
assert_eq!(payload[5]["total_requests"], 1);
assert_eq!(payload[5]["input_tokens"], 80);
assert_eq!(payload[5]["cache_creation_tokens"], 4);
assert_eq!(payload[5]["cache_read_tokens"], 6);
assert_eq!(payload[6]["total_requests"], 0);
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_handles_admin_stats_time_series_locally_without_usage_reader() {
let (upstream_url, upstream_hits, upstream_handle) =
@@ -1055,6 +1211,71 @@ async fn gateway_handles_admin_stats_leaderboard_models_locally_without_usage_re
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_defaults_admin_stats_leaderboard_models_to_bounded_recent_window_when_query_missing(
) {
let (_upstream_url, upstream_hits, upstream_handle) =
start_stats_upstream("/api/admin/stats/leaderboard/models").await;
let recent_row = sample_usage_row(
"usage-model-recent",
"req-model-recent",
Some("user-1"),
Some("key-1"),
Some("primary"),
"OpenAI",
"gpt-5",
100,
50,
0.4,
0.4,
recent_unix_secs(10),
);
let stale_row = sample_usage_row(
"usage-model-stale",
"req-model-stale",
Some("user-1"),
Some("key-1"),
Some("primary"),
"Anthropic",
"claude-3-5-sonnet",
60,
20,
0.2,
0.2,
recent_unix_secs(60 * 48),
);
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![
recent_row, stale_row,
]));
let gateway = build_router_with_state(
AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(GatewayDataState::with_usage_reader_for_tests(
usage_repository,
)),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = admin_request(reqwest::Client::new().get(format!(
"{gateway_url}/api/admin/stats/leaderboard/models?metric=tokens&order=desc"
)))
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = response.json().await.expect("json body should parse");
assert_eq!(payload["total"], 1);
assert_eq!(payload["items"][0]["id"], "gpt-5");
assert_eq!(payload["items"][0]["value"], 150);
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_handles_admin_stats_leaderboard_api_keys_locally_with_trusted_admin_principal() {
let (upstream_url, upstream_hits, upstream_handle) =

218
apps/aether-gateway/src/tests/control/admin/usage.rs
View File

@@ -346,6 +346,72 @@ async fn gateway_handles_admin_usage_stats_locally_with_trusted_admin_principal(
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_defaults_admin_usage_stats_to_bounded_recent_window_when_query_missing() {
let (upstream_url, upstream_hits, upstream_handle) =
start_usage_upstream("/api/admin/usage/stats").await;
let recent_created_at = recent_unix_secs(5);
let stale_created_at = recent_unix_secs(60 * 48);
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![
sample_usage_row(
"usage-recent",
"req-recent",
Some("user-1"),
Some("key-1"),
Some("primary"),
"OpenAI",
"gpt-5",
"completed",
120,
30,
0.3,
0.36,
recent_created_at,
),
sample_usage_row(
"usage-stale",
"req-stale",
Some("user-2"),
Some("key-2"),
Some("secondary"),
"Anthropic",
"claude-3-7",
"failed",
40,
10,
0.1,
0.12,
stale_created_at,
),
]));
let gateway = build_router_with_state(
AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(GatewayDataState::with_usage_reader_for_tests(
usage_repository,
)),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response =
admin_request(reqwest::Client::new().get(format!("{gateway_url}/api/admin/usage/stats")))
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = response.json().await.expect("json body should parse");
assert_eq!(payload["total_requests"], 1);
assert_eq!(payload["total_tokens"], 170);
assert_eq!(payload["error_count"], 0);
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_returns_service_unavailable_for_admin_usage_stats_without_usage_reader() {
assert_admin_usage_route_returns_local_503(
@@ -435,7 +501,7 @@ async fn gateway_handles_admin_usage_aggregation_stats_locally_with_trusted_admi
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = admin_request(reqwest::Client::new().get(format!(
"{gateway_url}/api/admin/usage/aggregation/stats?group_by=model&limit=10"
"{gateway_url}/api/admin/usage/aggregation/stats?group_by=model&limit=10&start_date=2024-03-21&end_date=2024-03-22&tz_offset_minutes=0"
)))
.send()
.await
@@ -458,7 +524,7 @@ async fn gateway_handles_admin_usage_aggregation_stats_locally_with_trusted_admi
assert_eq!(items[1]["output_tokens"], 20);
let provider_response = admin_request(reqwest::Client::new().get(format!(
"{gateway_url}/api/admin/usage/aggregation/stats?group_by=provider&limit=10"
"{gateway_url}/api/admin/usage/aggregation/stats?group_by=provider&limit=10&start_date=2024-03-21&end_date=2024-03-22&tz_offset_minutes=0"
)))
.send()
.await
@@ -477,7 +543,7 @@ async fn gateway_handles_admin_usage_aggregation_stats_locally_with_trusted_admi
assert_eq!(provider_items[1]["output_tokens"], 20);
let api_format_response = admin_request(reqwest::Client::new().get(format!(
"{gateway_url}/api/admin/usage/aggregation/stats?group_by=api_format&limit=10"
"{gateway_url}/api/admin/usage/aggregation/stats?group_by=api_format&limit=10&start_date=2024-03-21&end_date=2024-03-22&tz_offset_minutes=0"
)))
.send()
.await
@@ -578,7 +644,7 @@ async fn gateway_handles_admin_usage_aggregation_stats_locally_with_bearer_admin
let response = reqwest::Client::new()
.get(format!(
"{gateway_url}/api/admin/usage/aggregation/stats?group_by=user"
"{gateway_url}/api/admin/usage/aggregation/stats?group_by=user&start_date=2024-03-21&end_date=2024-03-22&tz_offset_minutes=0"
))
.header("authorization", format!("Bearer {access_token}"))
.header("x-client-device-id", "device-admin-usage")
@@ -739,10 +805,12 @@ async fn gateway_handles_admin_usage_active_locally_with_trusted_admin_principal
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response =
admin_request(reqwest::Client::new().get(format!("{gateway_url}/api/admin/usage/active")))
.send()
.await
.expect("request should succeed");
admin_request(reqwest::Client::new().get(format!(
"{gateway_url}/api/admin/usage/active?start_date=2024-03-21&end_date=2024-03-22&tz_offset_minutes=0"
)))
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = response.json().await.expect("json body should parse");
@@ -761,6 +829,136 @@ async fn gateway_handles_admin_usage_active_locally_with_trusted_admin_principal
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_defaults_admin_usage_active_to_recent_window_when_query_missing() {
let (upstream_url, upstream_hits, upstream_handle) =
start_usage_upstream("/api/admin/usage/active").await;
let recent_created_at = recent_unix_secs(3);
let stale_created_at = recent_unix_secs(60 * 48);
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![
sample_usage_row(
"usage-recent-pending",
"req-recent-pending",
Some("user-1"),
Some("key-1"),
Some("primary"),
"OpenAI",
"gpt-5",
"pending",
10,
0,
0.0,
0.0,
recent_created_at,
),
sample_usage_row(
"usage-stale-pending",
"req-stale-pending",
Some("user-2"),
Some("key-2"),
Some("secondary"),
"Anthropic",
"claude-3-7",
"pending",
40,
10,
0.1,
0.12,
stale_created_at,
),
sample_usage_row(
"usage-recent-completed",
"req-recent-completed",
Some("user-3"),
Some("key-3"),
Some("tertiary"),
"Google",
"gemini-2.5-pro",
"completed",
20,
5,
0.2,
0.22,
recent_created_at + 1,
),
]));
let gateway = build_router_with_state(
AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(GatewayDataState::with_usage_reader_for_tests(
usage_repository,
)),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response =
admin_request(reqwest::Client::new().get(format!("{gateway_url}/api/admin/usage/active")))
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = response.json().await.expect("json body should parse");
let requests = payload["requests"].as_array().expect("array");
assert_eq!(requests.len(), 1);
assert_eq!(requests[0]["id"], "usage-recent-pending");
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_handles_admin_usage_active_ids_for_terminal_updates() {
let (upstream_url, upstream_hits, upstream_handle) =
start_usage_upstream("/api/admin/usage/active").await;
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![sample_usage_row(
"usage-completed",
"req-completed",
Some("user-1"),
Some("key-1"),
Some("primary"),
"OpenAI",
"gpt-5",
"completed",
20,
5,
0.2,
0.24,
recent_unix_secs(2),
)]));
let gateway = build_router_with_state(
AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(GatewayDataState::with_usage_reader_for_tests(
usage_repository,
)),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = admin_request(reqwest::Client::new().get(format!(
"{gateway_url}/api/admin/usage/active?ids=usage-completed"
)))
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = response.json().await.expect("json body should parse");
let requests = payload["requests"].as_array().expect("array");
assert_eq!(requests.len(), 1);
assert_eq!(requests[0]["id"], "usage-completed");
assert_eq!(requests[0]["status"], "completed");
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_handles_admin_usage_records_locally_with_trusted_admin_principal() {
let (upstream_url, upstream_hits, upstream_handle) =
@@ -825,7 +1023,7 @@ async fn gateway_handles_admin_usage_records_locally_with_trusted_admin_principa
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = admin_request(reqwest::Client::new().get(format!(
"{gateway_url}/api/admin/usage/records?status=failed&provider=Anthropic&limit=10&offset=0"
"{gateway_url}/api/admin/usage/records?start_date=2024-03-21&end_date=2024-03-22&tz_offset_minutes=0&status=failed&provider=Anthropic&limit=10&offset=0"
)))
.send()
.await
@@ -949,7 +1147,7 @@ async fn gateway_handles_admin_usage_records_with_snapshot_first_user_and_api_ke
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = admin_request(reqwest::Client::new().get(format!(
"{gateway_url}/api/admin/usage/records?search=fresh-secondary&username=fresh-bob&limit=10&offset=0"
"{gateway_url}/api/admin/usage/records?start_date=2024-03-21&end_date=2024-03-22&tz_offset_minutes=0&search=fresh-secondary&username=fresh-bob&limit=10&offset=0"
)))
.send()
.await