feat: revamp analytics dashboards and harden database migrations

Add dashboard and overview analytics, health monitoring, provider expense tracking, and announcement updates across the gateway and frontend.

Keep schema migrations free of historical backfills while preserving automatic backfill execution. Bound migration deadlines, run schema preparation before Compose replacement, and anonymize deleted dashboard users.

Include the current documentation cleanup and regression coverage.
This commit is contained in:
elky
2026-10-01 11:48:17 +08:00
parent 60b89cc840
commit 066ea87d72
327 changed files with 31728 additions and 20645 deletions
@@ -1,4 +1,5 @@
mod monitoring;
mod overview;
mod routes;
mod stats;
mod usage;
@@ -22,6 +22,8 @@ pub(crate) mod test_support;
mod trace;
mod usage_helpers;
pub(super) use resilience::overview_resilience_payload;
pub(crate) async fn maybe_build_local_admin_monitoring_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
@@ -6,3 +6,23 @@ mod status;
pub(super) use history::build_admin_monitoring_resilience_circuit_history_response;
pub(super) use reset::build_admin_monitoring_reset_error_stats_response;
pub(super) use status::build_admin_monitoring_resilience_status_response;
pub(in super::super) async fn overview_resilience_payload(
state: &crate::handlers::admin::request::AdminAppState<'_>,
) -> Result<serde_json::Value, crate::GatewayError> {
let snapshot = snapshot::build_admin_monitoring_resilience_snapshot(state).await?;
let from = (snapshot.timestamp - chrono::Duration::hours(24))
.timestamp()
.max(
state
.admin_monitoring_error_stats_reset_at()
.unwrap_or_default() as i64,
);
Ok(serde_json::json!({
"scope": {"kind": "installation"},
"error_range": {"from": chrono::DateTime::from_timestamp(from, 0), "to": snapshot.timestamp},
"timestamp": snapshot.timestamp, "health_score": snapshot.health_score,
"status": snapshot.status, "error_statistics": snapshot.error_statistics,
"recent_errors": snapshot.recent_errors, "recommendations": snapshot.recommendations,
}))
}
@@ -0,0 +1,134 @@
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use aether_admin::observability::analytics::{dashboard_value, parse_dashboard_query};
use axum::{
body::Body,
http::{self, StatusCode},
response::{IntoResponse, Response},
Json,
};
pub(super) async fn response(
state: &AdminAppState<'_>,
context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
let query = match parse_dashboard_query(context.query_string()) {
Ok(query) => query,
Err(detail) => return Ok(super::error(StatusCode::BAD_REQUEST, &detail)),
};
if !state.as_ref().has_usage_data_reader() {
return Ok(super::error(
StatusCode::SERVICE_UNAVAILABLE,
"usage analytics is unavailable",
));
}
let snapshot = match tokio::time::timeout(
std::time::Duration::from_secs(15),
state.as_ref().query_dashboard_analytics(&query),
)
.await
{
Ok(result) => result?,
Err(_) => {
return Ok(super::error(
StatusCode::GATEWAY_TIMEOUT,
"dashboard query exceeded its time budget",
))
}
};
let data = dashboard_value(&query, &snapshot).map_err(GatewayError::Internal)?;
Ok((
[(http::header::CACHE_CONTROL, "private, no-store")],
Json(data),
)
.into_response())
}
pub(super) async fn total_response(
state: &AdminAppState<'_>,
context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
use crate::cache::OverviewTotalRead;
use aether_data_contracts::repository::usage::UsageDashboardAnalyticsQuery;
use serde_json::json;
use std::sync::Arc;
use std::time::{Duration, Instant};
let query = match parse_dashboard_query(context.query_string()) {
Ok(query) => query,
Err(detail) => return Ok(super::error(StatusCode::BAD_REQUEST, &detail)),
};
if !state.as_ref().has_usage_data_reader() {
return Ok(super::error(
StatusCode::SERVICE_UNAVAILABLE,
"usage analytics is unavailable",
));
}
let (cached, refresh) = state.as_ref().overview_total_cache.read(Instant::now());
if let Some(refresh) = refresh {
let app = state.as_ref();
let data = if app.background_data.has_usage_reader() {
Arc::clone(&app.background_data)
} else {
Arc::clone(&app.data)
};
// Lifetime boundaries do not depend on the viewer's timezone. Every
// administrator shares one refresh, including after a page reload.
tokio::spawn(async move {
let query = UsageDashboardAnalyticsQuery {
timezone: "UTC".into(),
};
let result = tokio::time::timeout(
Duration::from_secs(185),
data.query_dashboard_analytics(&query),
)
.await;
let snapshot = match result {
Ok(Ok(snapshot)) => Some(snapshot),
Ok(Err(error)) => {
tracing::warn!(%error, "dashboard lifetime refresh failed");
None
}
Err(_) => {
tracing::warn!("dashboard lifetime refresh exceeded its time budget");
None
}
};
refresh.finish(snapshot, Instant::now());
});
}
let (status, body, retry_after) = match cached {
OverviewTotalRead::Pending => {
(StatusCode::ACCEPTED, json!({"status":"pending"}), Some("3"))
}
OverviewTotalRead::Failed => (
StatusCode::SERVICE_UNAVAILABLE,
json!({"status":"failed", "detail":"cumulative dashboard totals are temporarily unavailable; retry shortly"}),
Some("10"),
),
OverviewTotalRead::Ready { snapshot, stale } => {
let mut value = dashboard_value(&query, &snapshot).map_err(GatewayError::Internal)?;
(
StatusCode::OK,
json!({
"status":"ready", "total": value["total"].take(),
"history_complete": snapshot.history_complete, "stale": stale,
}),
None,
)
}
};
let mut response = (
status,
[(http::header::CACHE_CONTROL, "private, no-store")],
Json(body),
)
.into_response();
if let Some(retry_after) = retry_after {
response.headers_mut().insert(
http::header::RETRY_AFTER,
http::HeaderValue::from_static(retry_after),
);
}
Ok(response)
}
@@ -0,0 +1,46 @@
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use aether_admin::observability::analytics::{dashboard_summary_value, parse_dashboard_query};
use axum::{
body::Body,
http::{header, StatusCode},
response::{IntoResponse, Response},
Json,
};
pub(super) async fn response(
state: &AdminAppState<'_>,
context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
let query = match parse_dashboard_query(context.query_string()) {
Ok(query) => query,
Err(detail) => return Ok(super::error(StatusCode::BAD_REQUEST, &detail)),
};
if !state.as_ref().has_usage_data_reader() {
return Ok(super::error(
StatusCode::SERVICE_UNAVAILABLE,
"dashboard statistics are unavailable",
));
}
let snapshot = match tokio::time::timeout(
std::time::Duration::from_secs(5),
state.as_ref().data.query_dashboard_summary(&query),
)
.await
{
Ok(Ok(snapshot)) => snapshot,
Ok(Err(error)) => return Err(GatewayError::Internal(error.to_string())),
Err(_) => {
return Ok(super::error(
StatusCode::GATEWAY_TIMEOUT,
"dashboard statistics exceeded their time budget",
))
}
};
let mut value = dashboard_summary_value(&snapshot);
value["concurrency"] = state
.as_ref()
.today_concurrency(&query.timezone)
.map_err(GatewayError::Internal)?;
Ok(([(header::CACHE_CONTROL, "private, no-store")], Json(value)).into_response())
}
@@ -0,0 +1,144 @@
use super::error;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use aether_admin::observability::analytics::{envelope, metrics_value, OverviewRequest};
use aether_data_contracts::repository::usage::{UsageAnalyticsQuery, USAGE_ANALYTICS_VERSION};
use axum::{
body::Body,
http::{header, HeaderValue, StatusCode},
response::{IntoResponse, Response},
Json,
};
use serde_json::json;
pub(super) async fn response(
state: &AdminAppState<'_>,
context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> {
if context
.query_string()
.is_some_and(|query| !query.is_empty())
{
return Ok(error(
StatusCode::BAD_REQUEST,
"live diagnostics do not accept historical filters",
));
}
let app = state.as_ref();
let _ = app.metric_samples().await;
let snapshot = app.metric_snapshot.read().await.clone();
let captured = snapshot.as_ref().map(|(captured, _)| *captured);
let now = chrono::Utc::now();
let observed_at = captured
.and_then(|captured| chrono::Duration::from_std(captured.elapsed()).ok())
.map(|age| now - age);
let mut unavailable = Vec::new();
let (resilience_result, recent_result) = tokio::join!(
tokio::time::timeout(
std::time::Duration::from_secs(3),
super::super::monitoring::overview_resilience_payload(state)
),
tokio::time::timeout(
std::time::Duration::from_secs(3),
recent_activity(state, now)
),
);
let resilience = match resilience_result {
Ok(Ok(value)) => Some(value),
_ => {
tracing::warn!("overview resilience snapshot unavailable");
unavailable.push("resilience");
None
}
};
let recent_activity = match recent_result {
Ok(Ok(value)) => Some(value),
_ => {
unavailable.push("recent_activity");
None
}
};
if captured.is_none() {
unavailable.push("metrics");
}
let mut response = Json(json!({
"meta": {
"schema_version": 1, "metric_version": USAGE_ANALYTICS_VERSION, "scope": {"kind": "node"},
"generated_at": now, "data_through": observed_at, "read_revision": observed_at.map(|value| value.timestamp_millis().to_string()),
"coverage": {"status": if unavailable.is_empty() {"complete"} else {"partial"}},
},
"data": {
"observed_at": observed_at, "window_seconds": null, "node_id": null,
"scope": {"kind": "node", "node_ids": []},
"metrics_text": snapshot.map(|(_, samples)| aether_runtime::metrics::render_prometheus_text(&samples)),
"resilience": resilience, "recent_activity": recent_activity,
"execution_activity": app.execution_activity.snapshot(),
"unavailable_sections": unavailable,
},
})).into_response();
response.headers_mut().insert(
header::CACHE_CONTROL,
HeaderValue::from_static("private, no-store"),
);
Ok(response)
}
async fn recent_activity(
state: &AdminAppState<'_>,
now: chrono::DateTime<chrono::Utc>,
) -> Result<serde_json::Value, GatewayError> {
let to = now.timestamp_millis().max(60_000) as u64;
let request = OverviewRequest {
query: UsageAnalyticsQuery {
from_unix_ms: to - 60_000,
to_unix_ms: to,
timezone: "UTC".into(),
limit: 1,
..Default::default()
},
amount_basis: "billable".into(),
csv: false,
};
let snapshot = state.as_ref().query_usage_analytics(&request.query).await?;
let data = recent_activity_data(&snapshot);
Ok(envelope(&request, &snapshot, data))
}
fn recent_activity_data(
snapshot: &aether_data_contracts::repository::usage::StoredUsageAnalytics,
) -> serde_json::Value {
let mut data = metrics_value(&snapshot.summary);
data["requests_per_second"] = json!(snapshot.summary.request_count as f64 / 60.0);
data["requests_per_minute"] = json!(snapshot.summary.request_count);
data["tokens_per_minute"] = data["total_tokens"].clone();
data["window_seconds"] = json!(60);
data
}
#[cfg(test)]
mod tests {
use super::*;
use aether_data_contracts::repository::usage::{StoredUsageAnalytics, UsageAnalyticsMetrics};
#[test]
fn recent_activity_reports_one_minute_rates_without_inventing_missing_tokens() {
let mut snapshot = StoredUsageAnalytics {
summary: UsageAnalyticsMetrics {
request_count: 120,
usage_available_count: 120,
total_tokens: 4200,
..Default::default()
},
..Default::default()
};
let value = recent_activity_data(&snapshot);
assert_eq!(value["window_seconds"], 60);
assert_eq!(value["requests_per_second"], 2.0);
assert_eq!(value["requests_per_minute"], 120);
assert_eq!(value["tokens_per_minute"], 4200);
snapshot.summary.usage_available_count = 0;
assert!(recent_activity_data(&snapshot)["tokens_per_minute"].is_null());
snapshot.summary = UsageAnalyticsMetrics::default();
assert_eq!(recent_activity_data(&snapshot)["tokens_per_minute"], 0);
}
}
@@ -0,0 +1,184 @@
mod dashboard;
mod dashboard_summary;
mod live;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use aether_admin::observability::analytics::{
costs_value, dashboard_charts_value, envelope, export_csv, metrics_value, page_value,
parse_dashboard_charts_query, parse_overview_query, performance_value, user_finance_value,
user_payments_value,
};
use aether_data_contracts::repository::usage::{UsageAnalyticsGranularity, UsageAnalyticsView};
use axum::{
body::Body,
http::{self, StatusCode},
response::{IntoResponse, Response},
Json,
};
use serde_json::json;
pub(crate) async fn maybe_build_overview_response(
state: &AdminAppState<'_>,
context: &AdminRequestContext<'_>,
) -> Result<Option<Response<Body>>, GatewayError> {
if context.route_family() != Some("overview_manage") || context.method() != http::Method::GET {
return Ok(None);
}
let kind = context.route_kind().unwrap_or_default();
if kind == "dashboard_summary" {
return dashboard_summary::response(state, context).await.map(Some);
}
if kind == "dashboard_total" {
return dashboard::total_response(state, context).await.map(Some);
}
if kind == "dashboard" {
return dashboard::response(state, context).await.map(Some);
}
if matches!(kind, "operations_live" | "operations_resources") {
return live::response(state, context).await.map(Some);
}
let view = match kind {
"dashboard_charts" => UsageAnalyticsView::DashboardCharts,
"summary" => UsageAnalyticsView::Summary,
"timeseries" | "costs" => UsageAnalyticsView::Timeseries,
"operations_performance" => UsageAnalyticsView::Performance,
"users" | "user_detail" => UsageAnalyticsView::Users,
"breakdown" => UsageAnalyticsView::Breakdown,
"consumption" => UsageAnalyticsView::Consumption,
_ => return Ok(None),
};
let parsed = if kind == "dashboard_charts" {
parse_dashboard_charts_query(context.query_string())
} else {
parse_overview_query(context.query_string(), view)
};
let mut request = match parsed {
Ok(value) => value,
Err(detail) => return Ok(Some(error(StatusCode::BAD_REQUEST, &detail))),
};
if kind == "user_detail" {
let encoded = context
.path()
.trim_end_matches('/')
.rsplit('/')
.next()
.unwrap_or_default();
let Ok(id) = percent_encoding::percent_decode_str(encoded).decode_utf8() else {
return Ok(Some(error(
StatusCode::BAD_REQUEST,
"invalid user identifier",
)));
};
let id = id.as_ref();
if id.is_empty() || id.len() > 512 || id.contains('/') || id.chars().any(char::is_control) {
return Ok(Some(error(
StatusCode::BAD_REQUEST,
"invalid user identifier",
)));
}
if request
.query
.actor_user_id
.as_deref()
.is_some_and(|value| value != id)
|| request
.query
.credential_owner_id
.as_deref()
.is_some_and(|value| value != id)
{
return Ok(Some(error(
StatusCode::BAD_REQUEST,
"user filter conflicts with the requested employee",
)));
}
if request.query.attribution_kind.as_deref() == Some("employee") {
request.query.actor_user_id = Some(id.into());
} else {
request.query.credential_owner_id = Some(id.into());
}
request.query.limit = 1;
request.query.offset = 0;
}
if matches!(
view,
UsageAnalyticsView::Timeseries | UsageAnalyticsView::Performance
) {
request.query.limit = 10_000;
request.query.offset = 0;
}
if kind == "costs" {
request.query.granularity = UsageAnalyticsGranularity::Day;
}
if !state.as_ref().has_usage_data_reader() {
return Ok(Some(error(
StatusCode::SERVICE_UNAVAILABLE,
"usage analytics is unavailable",
)));
}
let snapshot = match tokio::time::timeout(
std::time::Duration::from_secs(if request.csv { 30 } else { 15 }),
state.as_ref().query_usage_analytics(&request.query),
)
.await
{
Ok(result) => result?,
Err(_) => {
return Ok(Some(error(
StatusCode::GATEWAY_TIMEOUT,
"report query exceeded its time budget; narrow the range or filters",
)))
}
};
if request.csv {
return Ok(Some(match export_csv(&request, &snapshot) {
Ok(csv) => (
[
(http::header::CONTENT_TYPE, "text/csv; charset=utf-8"),
(
http::header::CONTENT_DISPOSITION,
"attachment; filename=overview.csv",
),
(http::header::CACHE_CONTROL, "private, no-store"),
],
csv,
)
.into_response(),
Err(detail) => error(StatusCode::UNPROCESSABLE_ENTITY, &detail),
}));
}
let data = match kind {
"dashboard_charts" => dashboard_charts_value(&snapshot),
"summary" => metrics_value(&snapshot.summary),
"user_detail" => {
let Some(user) = snapshot.users.first() else {
return Ok(Some(error(StatusCode::NOT_FOUND, "employee not found")));
};
json!({
"user": { "id": user.user_id, "username": user.username, "email": user.email, "is_active": user.is_active },
"summary": metrics_value(&user.metrics),
"finance": user_finance_value(user.finance.as_ref()),
"payments": user_payments_value(snapshot.user_payments.as_ref()),
})
}
"costs" => costs_value(&request, &snapshot),
"timeseries" => {
let mut page = page_value(&request, &snapshot);
page["granularity"] = json!(request.query.granularity);
page
}
"operations_performance" => performance_value(&request, &snapshot),
_ => page_value(&request, &snapshot),
};
let mut response = Json(envelope(&request, &snapshot, data)).into_response();
response.headers_mut().insert(
http::header::CACHE_CONTROL,
http::HeaderValue::from_static("private, no-store"),
);
Ok(Some(response))
}
fn error(status: StatusCode, detail: &str) -> Response<Body> {
(status, Json(json!({"detail": detail}))).into_response()
}
@@ -1,9 +1,15 @@
use super::{monitoring, stats, usage};
use super::{monitoring, overview, stats, usage};
use crate::handlers::admin::request::{AdminRouteRequest, AdminRouteResult};
pub(crate) async fn maybe_build_local_admin_observability_response(
request: AdminRouteRequest<'_>,
) -> AdminRouteResult {
if let Some(response) =
overview::maybe_build_overview_response(&request.state(), &request.request_context())
.await?
{
return Ok(Some(response));
}
if let Some(response) =
stats::maybe_build_local_admin_stats_response(&request.state(), &request.request_context())
.await?
@@ -1,5 +1,5 @@
use super::super::super::stats::resolve_admin_usage_time_range;
use super::super::analytics::admin_usage_aggregation_by_user_json;
use super::super::summary_routes::resolve_record_time_bounds;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::query_param_value;
use crate::GatewayError;
@@ -159,12 +159,11 @@ pub(super) async fn build_admin_usage_aggregation_stats_response(
Ok(value) => value,
Err(detail) => return Ok(admin_usage_bad_request_response(detail)),
};
let time_range = match resolve_admin_usage_time_range(query) {
let time_bounds = match resolve_record_time_bounds(query) {
Ok(value) => value,
Err(detail) => return Ok(admin_usage_bad_request_response(detail)),
};
let Some((created_from_unix_secs, created_until_unix_secs)) = time_range.to_unix_bounds()
else {
let Some((created_from_unix_secs, created_until_unix_secs)) = time_bounds else {
return Ok(Json(json!([])).into_response());
};
let group_by_query = match group_by.as_str() {
@@ -33,6 +33,50 @@ use std::collections::{BTreeMap, BTreeSet};
const ADMIN_USAGE_ACTIVE_LIMIT: usize = 50;
pub(super) fn resolve_record_time_bounds(
query: Option<&str>,
) -> Result<Option<(u64, u64)>, String> {
let entries =
url::form_urlencoded::parse(query.unwrap_or_default().as_bytes()).collect::<Vec<_>>();
let from = entries
.iter()
.filter(|(key, _)| key == "from")
.collect::<Vec<_>>();
let to = entries
.iter()
.filter(|(key, _)| key == "to")
.collect::<Vec<_>>();
if from.is_empty() && to.is_empty() {
return resolve_admin_usage_time_range(query).map(|range| range.to_unix_bounds());
}
if from.len() != 1 || to.len() != 1 {
return Err("from and to must each be provided once".into());
}
if entries
.iter()
.any(|(key, _)| matches!(key.as_ref(), "start_date" | "end_date" | "preset" | "days"))
{
return Err("precise from/to cannot be combined with date presets".into());
}
if let Some(zone) = query_param_value(query, "timezone") {
zone.parse::<chrono_tz::Tz>()
.map_err(|_| "invalid timezone".to_string())?;
}
let parse = |value: &str| -> Result<u64, String> {
let value = chrono::DateTime::parse_from_rfc3339(value)
.map_err(|_| "from/to must be RFC 3339 timestamps".to_string())?;
if value.timestamp_subsec_nanos() != 0 {
return Err("request records support second-aligned ranges".into());
}
u64::try_from(value.timestamp()).map_err(|_| "from/to must not precede Unix epoch".into())
};
let bounds = (parse(&from[0].1)?, parse(&to[0].1)?);
if bounds.0 >= bounds.1 || bounds.1 - bounds.0 > 366 * 86_400 {
return Err("from/to must define a nonempty range of at most 366 days".into());
}
Ok(Some(bounds))
}
async fn load_admin_usage_by_ids(
state: &AdminAppState<'_>,
requested_ids: &BTreeSet<String>,
@@ -70,6 +114,7 @@ fn apply_admin_usage_status_filter(query: &mut UsageAuditListQuery, status: Opti
}
"websocket" | "ws" => query.is_websocket = Some(true),
"error" | "failed" => query.error_only = true,
"success" => query.statuses = Some(vec!["completed".to_string()]),
"active" => {
query.statuses = Some(vec!["pending".to_string(), "streaming".to_string()]);
}
@@ -523,12 +568,37 @@ fn build_admin_usage_records_query(
query: Option<&str>,
limit: Option<usize>,
offset: Option<usize>,
) -> UsageAuditListQuery {
) -> Result<UsageAuditListQuery, String> {
let boolean = |key| match query_param_value(query, key).as_deref() {
None => Ok(None),
Some("true" | "1") => Ok(Some(true)),
Some("false" | "0") => Ok(Some(false)),
Some(_) => Err(format!("invalid {key}: expected true or false")),
};
let slow_threshold_ms = query_param_value(query, "slow_threshold_ms")
.map(|value| {
value
.parse::<u64>()
.ok()
.filter(|value| (1..=86_400_000).contains(value))
.ok_or_else(|| "slow_threshold_ms must be between 1 and 86400000".to_string())
})
.transpose()?;
let mut list_query = UsageAuditListQuery {
created_from_unix_secs: Some(created_from_unix_secs),
created_until_unix_secs: Some(created_until_unix_secs),
user_id: query_param_value(query, "user_id"),
provider_name: query_param_value(query, "provider"),
provider_id: query_param_value(query, "provider_id"),
api_key_id: query_param_value(query, "api_key_id"),
request_id: query_param_value(query, "request_id"),
attribution_kind: query_param_value(query, "attribution_kind"),
actor_user_id: query_param_value(query, "actor_user_id"),
slow_threshold_ms,
endpoint_kind: query_param_value(query, "endpoint_kind"),
request_type: query_param_value(query, "request_type"),
has_format_conversion: boolean("has_format_conversion")?,
is_stream: boolean("is_stream")?,
model: query_param_value(query, "model"),
api_format: query_param_value(query, "api_format"),
limit,
@@ -536,11 +606,23 @@ fn build_admin_usage_records_query(
newest_first: true,
..Default::default()
};
if list_query
.attribution_kind
.as_deref()
.is_some_and(|kind| !matches!(kind, "employee" | "standalone" | "unknown"))
{
return Err("invalid attribution_kind".into());
}
if list_query.attribution_kind.as_deref() == Some("employee")
&& list_query.actor_user_id.is_none()
{
list_query.actor_user_id = list_query.user_id.take();
}
apply_admin_usage_status_filter(
&mut list_query,
query_param_value(query, "status").as_deref(),
);
list_query
Ok(list_query)
}
fn parse_admin_usage_search_keywords(search: &str) -> Vec<String> {
@@ -653,6 +735,15 @@ fn build_admin_usage_keyword_search_query(
created_until_unix_secs: base_query.created_until_unix_secs,
user_id: base_query.user_id.clone(),
provider_name: base_query.provider_name.clone(),
provider_id: base_query.provider_id.clone(),
api_key_id: base_query.api_key_id.clone(),
request_id: base_query.request_id.clone(),
attribution_kind: base_query.attribution_kind.clone(),
actor_user_id: base_query.actor_user_id.clone(),
slow_threshold_ms: base_query.slow_threshold_ms,
endpoint_kind: base_query.endpoint_kind.clone(),
request_type: base_query.request_type.clone(),
has_format_conversion: base_query.has_format_conversion,
model: base_query.model.clone(),
api_format: base_query.api_format.clone(),
client_family: base_query.client_family.clone(),
@@ -699,13 +790,11 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
}
let query = request_context.request_query_string.as_deref();
let time_range = match resolve_admin_usage_time_range(query) {
let time_bounds = match resolve_record_time_bounds(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
let Some((created_from_unix_secs, created_until_unix_secs)) =
time_range.to_unix_bounds()
else {
let Some((created_from_unix_secs, created_until_unix_secs)) = time_bounds else {
return Ok(Some(build_admin_usage_summary_stats_response_from_summary(
&Default::default(),
)));
@@ -743,13 +832,11 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
sort_usage_newest_first(&mut items);
items
} else {
let time_range = match resolve_admin_usage_time_range(query) {
let time_bounds = match resolve_record_time_bounds(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
let Some((created_from_unix_secs, created_until_unix_secs)) =
time_range.to_unix_bounds()
else {
let Some((created_from_unix_secs, created_until_unix_secs)) = time_bounds else {
return Ok(Some(build_admin_usage_active_requests_response(
&[],
&BTreeMap::new(),
@@ -806,7 +893,7 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
}
let query = request_context.request_query_string.as_deref();
let time_range = match resolve_admin_usage_time_range(query) {
let time_bounds = match resolve_record_time_bounds(query) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
@@ -827,9 +914,7 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
let Some((created_from_unix_secs, created_until_unix_secs)) =
time_range.to_unix_bounds()
else {
let Some((created_from_unix_secs, created_until_unix_secs)) = time_bounds else {
return Ok(Some(build_admin_usage_records_response(
&[],
&BTreeMap::new(),
@@ -849,13 +934,16 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
let active_client_family_filter = client_family_filter
.as_deref()
.filter(|value| !value.trim().is_empty());
let mut base_query = build_admin_usage_records_query(
let mut base_query = match build_admin_usage_records_query(
created_from_unix_secs,
created_until_unix_secs,
query,
None,
None,
);
) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
};
base_query.client_family = active_client_family_filter.map(str::to_owned);
base_query.exclude_unknown_model_or_provider = hide_unknown_records;
let (usage, total, total_is_estimated) = if attempt_status_filter.is_some() {
@@ -1028,6 +1116,89 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
#[cfg(test)]
mod tests {
#[test]
fn precise_record_ranges_preserve_minutes_and_reject_mixed_presets() {
let range = "from=2026-09-01T23:45:00Z&to=2026-09-02T00:15:00Z&timezone=Asia%2FShanghai";
let (from, to) = super::resolve_record_time_bounds(Some(range))
.unwrap()
.unwrap();
assert_eq!(to - from, 30 * 60);
assert!(super::resolve_record_time_bounds(Some(&format!("{range}&preset=today"))).is_err());
assert!(super::resolve_record_time_bounds(Some("from=2026-09-01T00:00:00Z")).is_err());
}
#[test]
fn overview_record_drilldown_preserves_actor_and_performance_filters() {
let raw = "user_id=employee-1&attribution_kind=employee&provider_id=provider-1&api_key_id=key-1&request_id=request-1&endpoint_kind=chat&request_type=chat&is_stream=true&has_format_conversion=false&slow_threshold_ms=12000&status=success";
let query =
super::build_admin_usage_records_query(100, 200, Some(raw), None, None).unwrap();
assert_eq!(query.user_id, None);
assert_eq!(query.actor_user_id.as_deref(), Some("employee-1"));
assert_eq!(query.provider_id.as_deref(), Some("provider-1"));
assert_eq!(query.api_key_id.as_deref(), Some("key-1"));
assert_eq!(query.request_id.as_deref(), Some("request-1"));
assert_eq!(query.slow_threshold_ms, Some(12_000));
assert_eq!(query.is_stream, Some(true));
assert_eq!(query.has_format_conversion, Some(false));
assert_eq!(query.statuses, Some(vec!["completed".into()]));
let keyword = super::build_admin_usage_keyword_search_query(
&query,
vec!["example".into()],
None,
Default::default(),
false,
false,
None,
None,
);
assert_eq!(keyword.actor_user_id, query.actor_user_id);
assert_eq!(keyword.slow_threshold_ms, query.slow_threshold_ms);
assert_eq!(keyword.has_format_conversion, query.has_format_conversion);
for invalid in [
"is_stream=maybe",
"slow_threshold_ms=0",
"attribution_kind=owner",
] {
assert!(
super::build_admin_usage_records_query(100, 200, Some(invalid), None, None)
.is_err()
);
}
}
#[test]
fn overview_record_drilldown_preserves_standalone_key_ownership() {
let raw = "user_id=owner-1&attribution_kind=standalone&api_key_id=standalone-key";
let query =
super::build_admin_usage_records_query(100, 200, Some(raw), None, None).unwrap();
assert_eq!(query.attribution_kind.as_deref(), Some("standalone"));
assert_eq!(query.user_id.as_deref(), Some("owner-1"));
assert_eq!(query.actor_user_id, None);
assert_eq!(query.api_key_id.as_deref(), Some("standalone-key"));
let keyword = super::build_admin_usage_keyword_search_query(
&query,
vec!["example".into()],
None,
Default::default(),
false,
false,
None,
None,
);
assert_eq!(keyword.attribution_kind, query.attribution_kind);
assert_eq!(keyword.user_id, query.user_id);
assert_eq!(keyword.actor_user_id, query.actor_user_id);
assert_eq!(keyword.api_key_id, query.api_key_id);
for retired_kind in ["service", "shared"] {
let raw = format!("attribution_kind={retired_kind}");
assert!(
super::build_admin_usage_records_query(100, 200, Some(&raw), None, None).is_err(),
"{retired_kind}"
);
}
}
use aether_data_contracts::repository::candidates::{
RequestCandidateStatus, StoredRequestCandidate,
};
@@ -1169,7 +1340,7 @@ mod tests {
for status in ["websocket", "ws", "WS"] {
let raw_query = format!("status={status}");
let list_query =
build_admin_usage_records_query(100, 200, Some(&raw_query), None, None);
build_admin_usage_records_query(100, 200, Some(&raw_query), None, None).unwrap();
assert_eq!(list_query.is_websocket, Some(true));
assert_eq!(list_query.is_stream, None);
@@ -1190,7 +1361,7 @@ mod tests {
for (status, expected_stream) in [("stream", true), ("standard", false)] {
let raw_query = format!("status={status}");
let list_query =
build_admin_usage_records_query(100, 200, Some(&raw_query), None, None);
build_admin_usage_records_query(100, 200, Some(&raw_query), None, None).unwrap();
assert_eq!(list_query.is_stream, Some(expected_stream));
assert_eq!(list_query.is_websocket, Some(false));