feat: revamp analytics dashboards and harden database migrations

Add dashboard and overview analytics, health monitoring, provider expense tracking, and announcement updates across the gateway and frontend.

Keep schema migrations free of historical backfills while preserving automatic backfill execution. Bound migration deadlines, run schema preparation before Compose replacement, and anonymize deleted dashboard users.

Include the current documentation cleanup and regression coverage.
This commit is contained in:
elky
2026-10-01 11:48:17 +08:00
parent 60b89cc840
commit 066ea87d72
327 changed files with 31728 additions and 20645 deletions
@@ -16,6 +16,8 @@ mod collectors;
mod payments;
mod plans;
mod presets;
mod provider_accounts;
mod provider_expenses;
mod routes;
mod rules;
mod wallets;
@@ -207,6 +209,15 @@ pub(crate) async fn maybe_build_local_admin_billing_response(
return Ok(None);
}
if let Some(response) = provider_accounts::response(state, request_context).await? {
return Ok(Some(response));
}
if let Some(response) =
provider_expenses::response(state, request_context, request_body).await?
{
return Ok(Some(response));
}
let path = request_context.path();
let is_billing_route = (request_context.method() == http::Method::GET
&& matches!(
@@ -0,0 +1,159 @@
//! Current provider finance snapshots. This endpoint never calls upstream services.
use super::build_admin_billing_data_unavailable_response;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use axum::{
body::Body,
http,
response::{IntoResponse, Response},
Json,
};
use serde_json::{json, Value};
fn finite(value: Option<&Value>) -> Option<f64> {
value
.and_then(|v| {
v.as_f64()
.or_else(|| v.as_str().and_then(|v| v.parse::<f64>().ok()))
})
.filter(|v| v.is_finite())
}
fn text(value: Option<&Value>) -> Option<&str> {
value
.and_then(Value::as_str)
.map(str::trim)
.filter(|v| !v.is_empty() && v.len() <= 256 && !v.chars().any(char::is_control))
}
fn timestamp(value: Option<&Value>) -> Option<String> {
let value = value?;
if let Some(raw) = value.as_str() {
if let Ok(date) = chrono::DateTime::parse_from_rfc3339(raw) {
return Some(date.to_rfc3339_opts(chrono::SecondsFormat::Millis, true));
}
}
let secs = finite(Some(value))?;
if !(0.0..=253_402_300_799.0).contains(&secs) {
return None;
}
chrono::DateTime::from_timestamp(secs as i64, 0)
.map(|v| v.to_rfc3339_opts(chrono::SecondsFormat::Millis, true))
}
fn subscription(value: &Value) -> Value {
json!({
"group_name": text(value.get("group_name")),
"status": text(value.get("status")),
"daily_used_usd": finite(value.get("daily_used_usd")),
"daily_limit_usd": finite(value.get("daily_limit_usd")),
"weekly_used_usd": finite(value.get("weekly_used_usd")),
"weekly_limit_usd": finite(value.get("weekly_limit_usd")),
"monthly_used_usd": finite(value.get("monthly_used_usd")),
"monthly_limit_usd": finite(value.get("monthly_limit_usd")),
"expires_at": timestamp(value.get("expires_at")),
})
}
fn balance(value: &Value) -> Option<Value> {
if value.get("action_type").and_then(Value::as_str) != Some("query_balance") {
return None;
}
let status = text(value.get("status"))?;
if !matches!(status, "success" | "auth_expired" | "auth_failed") {
return None;
}
let data = value
.get("data")
.filter(|_| matches!(status, "success" | "auth_expired"));
let extra = data.and_then(|d| d.get("extra"));
let subscriptions = extra
.and_then(|e| e.get("subscriptions"))
.and_then(Value::as_array)
.map(|items| {
items
.iter()
.filter(|v| v.is_object())
.take(128)
.map(subscription)
.collect::<Vec<_>>()
})
.unwrap_or_default();
Some(json!({
"status": status,
"observed_at": timestamp(value.get("executed_at")),
"currency": data.and_then(|d| text(d.get("currency"))),
"available": data.and_then(|d| finite(d.get("total_available"))),
"used": data.and_then(|d| finite(d.get("total_used"))),
"granted": data.and_then(|d| finite(d.get("total_granted"))),
"plan_name": extra.and_then(|e| text(e.get("plan_name"))),
"subscriptions": subscriptions,
}))
}
pub(super) async fn response(
state: &AdminAppState<'_>,
context: &AdminRequestContext<'_>,
) -> Result<Option<Response<Body>>, GatewayError> {
if context.method() != http::Method::GET
|| context.path().trim_end_matches('/') != "/api/admin/billing/provider-accounts"
|| context.route_family() != Some("billing_manage")
{
return Ok(None);
}
if !state.has_provider_catalog_data_reader() {
return Ok(Some(build_admin_billing_data_unavailable_response()));
}
let mut providers = state.list_provider_catalog_providers(false).await?;
providers.sort_by(|a, b| a.name.cmp(&b.name).then_with(|| a.id.cmp(&b.id)));
let keys = providers
.iter()
.map(|p| format!("provider_ops:balance:{}", p.id))
.collect::<Vec<_>>();
let (cached, unavailable) = if keys.is_empty() {
(Vec::new(), false)
} else {
match state.runtime_state().kv_get_many(&keys).await {
Ok(v) => (v, false),
Err(_) => (vec![None; keys.len()], true),
}
};
let items = providers.iter().enumerate().map(|(index, p)| {
let limit = p.monthly_quota_usd.filter(|v| v.is_finite() && *v >= 0.0);
let used = p.monthly_used_usd.filter(|v| v.is_finite() && *v >= 0.0);
let quota = if p.billing_type.as_deref() == Some("monthly_quota") || limit.is_some() {
json!({
"limit": limit, "used": used,
"remaining": limit.zip(used).map(|(l,u)| (l-u).max(0.0)),
"currency": "USD",
"period_start": p.quota_last_reset_at_unix_secs.and_then(|v| timestamp(Some(&json!(v)))),
"expires_at": p.quota_expires_at_unix_secs.and_then(|v| timestamp(Some(&json!(v)))),
})
} else { Value::Null };
let balance = cached.get(index).and_then(|v| v.as_deref())
.and_then(|v| serde_json::from_str::<Value>(v).ok()).and_then(|v| balance(&v));
json!({
"provider_id": p.id, "provider_name": p.name, "is_active": p.is_active,
"billing_type": p.billing_type, "quota": quota, "balance": balance,
})
}).collect::<Vec<_>>();
Ok(Some((
[(http::header::CACHE_CONTROL, "private, no-store")],
Json(json!({
"observed_at": chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis,true),
"items": items, "balance_snapshot_unavailable": unavailable,
})),
).into_response()))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn provider_accounts_only_expose_finance_allowlist_and_preserve_unknown() {
let snapshot=balance(&json!({"status":"success","action_type":"query_balance","executed_at":"2026-09-20T00:00:00Z","data":{"currency":"USD","total_available":null,"extra":{"access_token":"secret","plan_name":"Pro","subscriptions":[{"group_name":"Team","monthly_used_usd":"12.25","expires_at":1800000000,"private_token":"secret"}]}}})).unwrap();
assert!(snapshot["available"].is_null());
assert_eq!(
snapshot["subscriptions"][0]["monthly_used_usd"],
json!(12.25)
);
assert!(!snapshot.to_string().contains("secret"));
assert!(!snapshot.to_string().contains("access_token"));
let failed=balance(&json!({"status":"auth_failed","action_type":"query_balance","data":{"total_available":999}})).unwrap();
assert!(failed["available"].is_null());
}
}
@@ -0,0 +1,334 @@
use super::{
build_admin_billing_bad_request_response as bad_request,
build_admin_billing_conflict_response as conflict,
build_admin_billing_data_unavailable_response as unavailable,
build_admin_billing_not_found_response as not_found,
};
use crate::handlers::admin::{
request::{AdminAppState, AdminRequestContext},
shared::{attach_admin_audit_response, query_param_value},
};
use crate::handlers::shared::normalize_payment_currency;
use crate::GatewayError;
use aether_data_contracts::repository::billing::*;
use axum::{
body::{Body, Bytes},
http::{self, StatusCode},
response::{IntoResponse, Response},
Json,
};
use serde::Deserialize;
use serde_json::{json, Value};
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct ExpenseRequest {
client_request_id: String,
provider_id: String,
kind: String,
amount: String,
currency: String,
paid_at: String,
period_start: Option<String>,
period_end: Option<String>,
note: Option<String>,
external_reference: Option<String>,
}
fn datetime(value: u64) -> String {
chrono::DateTime::from_timestamp_millis(value as i64)
.expect("valid stored timestamp")
.to_rfc3339_opts(chrono::SecondsFormat::Millis, true)
}
fn parse_date(value: &str) -> Result<u64, String> {
chrono::DateTime::parse_from_rfc3339(value)
.ok()
.and_then(|v| u64::try_from(v.timestamp_millis()).ok())
.filter(|v| *v <= 253_402_300_799_000)
.ok_or_else(|| "timestamps must be RFC3339 dates on or after 1970".into())
}
fn optional_text(value: Option<String>) -> Option<String> {
value.map(|v| v.trim().to_owned()).filter(|v| !v.is_empty())
}
fn expense_json(record: &ProviderExpenseRecord) -> Value {
let e = &record.entry;
json!({
"id": record.id, "client_request_id": e.client_request_id,
"provider_id": e.provider_id, "provider_name": e.provider_name,
"kind": e.kind, "amount": e.amount, "currency": e.currency,
"paid_at": datetime(e.paid_at_unix_ms),
"period_start": e.period_start_unix_ms.map(datetime),
"period_end": e.period_end_unix_ms.map(datetime),
"note": e.note, "external_reference": e.external_reference,
"created_by": e.created_by, "created_at": datetime(record.created_at_unix_ms),
"status": if record.voided_at_unix_ms.is_some() { "void" } else { "recorded" },
"voided_at": record.voided_at_unix_ms.map(datetime), "voided_by": record.voided_by,
})
}
fn csv_cell(value: &str) -> String {
let value = if value.trim_start().starts_with(['=', '+', '-', '@'])
|| value.starts_with(['\t', '\r', '\n'])
{
format!("'{value}")
} else {
value.to_string()
};
format!("\"{}\"", value.replace('"', "\"\""))
}
fn csv_report(items: &[ProviderExpenseRecord]) -> String {
let mut result=String::from("\u{feff}id,provider_id,provider_name,kind,amount,currency,paid_at,period_start,period_end,note,external_reference,created_by,created_at\r\n");
for r in items {
let e = &r.entry;
let fields = [
r.id.clone(),
e.provider_id.clone(),
e.provider_name.clone(),
e.kind.clone(),
e.amount.clone(),
e.currency.clone(),
datetime(e.paid_at_unix_ms),
e.period_start_unix_ms.map(datetime).unwrap_or_default(),
e.period_end_unix_ms.map(datetime).unwrap_or_default(),
e.note.clone().unwrap_or_default(),
e.external_reference.clone().unwrap_or_default(),
e.created_by.clone().unwrap_or_default(),
datetime(r.created_at_unix_ms),
];
result.push_str(
&fields
.iter()
.map(|s| csv_cell(s))
.collect::<Vec<_>>()
.join(","),
);
result.push_str("\r\n");
}
result
}
fn query(context: &AdminRequestContext<'_>, csv: bool) -> Result<ProviderExpenseQuery, String> {
let q = context.query_string();
let now = chrono::Utc::now().timestamp_millis().max(0) as u64;
let from = query_param_value(q, "from")
.map(|v| parse_date(&v))
.transpose()?
.unwrap_or(now.saturating_sub(30 * 86_400_000));
let to = query_param_value(q, "to")
.map(|v| parse_date(&v))
.transpose()?
.unwrap_or(now);
let limit = if csv {
10_001
} else {
query_param_value(q, "limit")
.map(|v| v.parse::<u32>().map_err(|_| "invalid limit".to_string()))
.transpose()?
.unwrap_or(25)
};
let offset = if csv {
0
} else {
query_param_value(q, "offset")
.map(|v| v.parse::<u64>().map_err(|_| "invalid offset".to_string()))
.transpose()?
.unwrap_or(0)
};
if !csv && limit > 200 {
return Err("limit must be at most 200".into());
}
let q = ProviderExpenseQuery {
from_unix_ms: from,
to_unix_ms: to,
limit,
offset,
};
q.validate().map_err(|e| e.to_string())?;
Ok(q)
}
pub(super) async fn response(
state: &AdminAppState<'_>,
context: &AdminRequestContext<'_>,
body: Option<&Bytes>,
) -> Result<Option<Response<Body>>, GatewayError> {
let path = context.path().trim_end_matches('/');
if context.route_family() != Some("billing_manage")
|| !path.starts_with("/api/admin/billing/provider-expenses")
{
return Ok(None);
}
let operator = context
.decision()
.and_then(|d| d.admin_principal.as_ref())
.map(|p| p.user_id.clone());
if path == "/api/admin/billing/provider-expenses" && context.method() == http::Method::GET {
let csv = query_param_value(context.query_string(), "format").as_deref() == Some("csv");
let q = match query(context, csv) {
Ok(v) => v,
Err(e) => return Ok(Some(bad_request(e))),
};
let Some(page) = state
.app()
.data
.list_provider_expenses(&q)
.await
.map_err(|e| GatewayError::Internal(e.to_string()))?
else {
return Ok(Some(unavailable()));
};
if csv {
if page.total > 10_000 {
return Ok(Some(
(
StatusCode::UNPROCESSABLE_ENTITY,
Json(json!({"detail":"导出超过 10000 条,请缩小时间范围"})),
)
.into_response(),
));
}
return Ok(Some(
(
[
(http::header::CONTENT_TYPE, "text/csv; charset=utf-8"),
(
http::header::CONTENT_DISPOSITION,
"attachment; filename=provider-expenses.csv",
),
(http::header::CACHE_CONTROL, "private, no-store"),
],
csv_report(&page.items),
)
.into_response(),
));
}
return Ok(Some(
(
[(http::header::CACHE_CONTROL, "private, no-store")],
Json(json!({
"items": page.items.iter().map(expense_json).collect::<Vec<_>>(),
"total": page.total, "totals": page.totals, "providers": page.providers,
"limit": q.limit, "offset": q.offset,
"from": datetime(q.from_unix_ms), "to": datetime(q.to_unix_ms),
"time_basis": "paid_at", "source": "manual_ledger",
})),
)
.into_response(),
));
}
if path == "/api/admin/billing/provider-expenses" && context.method() == http::Method::POST {
let Some(body) = body else {
return Ok(Some(bad_request("缺少请求体")));
};
let payload = match serde_json::from_slice::<ExpenseRequest>(body) {
Ok(v) => v,
Err(_) => return Ok(Some(bad_request("输入验证失败"))),
};
let input = (|| -> Result<ProviderExpenseInput, String> {
let units = provider_expense_amount_units(&payload.amount)
.ok_or("amount must be a positive decimal string with at most 8 decimal places")?;
let input = ProviderExpenseInput {
client_request_id: uuid::Uuid::parse_str(&payload.client_request_id)
.map_err(|_| "client_request_id must be a UUID")?
.to_string(),
provider_id: payload.provider_id.trim().into(),
provider_name: "pending".into(),
kind: payload.kind,
amount: format_provider_expense_amount(units),
currency: normalize_payment_currency(&payload.currency, "currency")?,
paid_at_unix_ms: parse_date(&payload.paid_at)?,
period_start_unix_ms: payload
.period_start
.as_deref()
.map(parse_date)
.transpose()?,
period_end_unix_ms: payload.period_end.as_deref().map(parse_date).transpose()?,
note: optional_text(payload.note),
external_reference: optional_text(payload.external_reference),
created_by: operator.clone(),
};
input.validate()?;
Ok(input)
})();
let mut input = match input {
Ok(v) => v,
Err(e) => return Ok(Some(bad_request(e))),
};
let providers = state
.read_provider_catalog_providers_by_ids(&[input.provider_id.clone()])
.await?;
let Some(provider) = providers.first() else {
return Ok(Some(not_found("Provider not found")));
};
input.provider_name = provider.name.clone();
let result = state
.app()
.data
.create_provider_expense(&input)
.await
.map_err(|e| GatewayError::Internal(e.to_string()))?;
return Ok(Some(mutation_response(
result,
"admin_provider_expense_recorded",
"record_provider_expense",
)));
}
if context.method() == http::Method::POST {
if let Some(id) = path
.strip_prefix("/api/admin/billing/provider-expenses/")
.and_then(|v| v.strip_suffix("/void"))
.filter(|v| !v.is_empty() && !v.contains('/'))
{
if uuid::Uuid::parse_str(id).is_err() {
return Ok(Some(bad_request("invalid expense id")));
}
let result = state
.app()
.data
.void_provider_expense(id, operator.as_deref())
.await
.map_err(|e| GatewayError::Internal(e.to_string()))?;
return Ok(Some(mutation_response(
result,
"admin_provider_expense_voided",
"void_provider_expense",
)));
}
}
Ok(None)
}
fn mutation_response(
outcome: AdminBillingMutationOutcome<ProviderExpenseRecord>,
event: &'static str,
action: &'static str,
) -> Response<Body> {
match outcome {
AdminBillingMutationOutcome::Applied(record) => attach_admin_audit_response(
Json(json!({"item":expense_json(&record)})).into_response(),
event,
action,
"provider_expense",
&record.id,
),
AdminBillingMutationOutcome::Invalid(e) => conflict(e),
AdminBillingMutationOutcome::NotFound => not_found("Provider expense not found"),
AdminBillingMutationOutcome::Unavailable => unavailable(),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn provider_expense_csv_neutralizes_formulas_and_quotes_fields() {
assert_eq!(csv_cell("=cmd()"), "\"'=cmd()\"");
assert_eq!(csv_cell(" @cmd"), "\"' @cmd\"");
assert_eq!(csv_cell("\tcmd"), "\"'\tcmd\"");
assert_eq!(csv_cell("a,\"b\"\nc"), "\"a,\"\"b\"\"\nc\"");
assert_eq!(csv_cell("12.34"), "\"12.34\"");
}
#[test]
fn provider_expense_dates_require_explicit_timezone_and_nonnegative_epoch() {
assert_eq!(
parse_date("2026-09-20T08:00:00+08:00"),
parse_date("2026-09-20T00:00:00Z")
);
assert!(parse_date("2026-09-20").is_err());
assert!(parse_date("1969-01-01T00:00:00Z").is_err());
}
}
@@ -27,11 +27,18 @@ pub(in super::super) async fn build_admin_wallet_list_response(
Ok(value) => value,
Err(detail) => return Ok(build_admin_wallets_bad_request_response(detail)),
};
let user_id = query_param_value(query, "user_id");
let status = query_param_value(query, "status");
let owner_type = parse_admin_wallets_owner_type_filter(query);
let (wallets, total) = state
.list_admin_wallets(status.as_deref(), owner_type.as_deref(), limit, offset)
.list_admin_wallets(
user_id.as_deref(),
status.as_deref(),
owner_type.as_deref(),
limit,
offset,
)
.await?;
let mut items = Vec::with_capacity(wallets.len());
for wallet in wallets {