mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-09 10:57:03 +08:00
feat: revamp analytics dashboards and harden database migrations
Add dashboard and overview analytics, health monitoring, provider expense tracking, and announcement updates across the gateway and frontend. Keep schema migrations free of historical backfills while preserving automatic backfill execution. Bound migration deadlines, run schema preparation before Compose replacement, and anonymize deleted dashboard users. Include the current documentation cleanup and regression coverage.
This commit is contained in:
@@ -16,6 +16,8 @@ mod collectors;
|
||||
mod payments;
|
||||
mod plans;
|
||||
mod presets;
|
||||
mod provider_accounts;
|
||||
mod provider_expenses;
|
||||
mod routes;
|
||||
mod rules;
|
||||
mod wallets;
|
||||
@@ -207,6 +209,15 @@ pub(crate) async fn maybe_build_local_admin_billing_response(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if let Some(response) = provider_accounts::response(state, request_context).await? {
|
||||
return Ok(Some(response));
|
||||
}
|
||||
if let Some(response) =
|
||||
provider_expenses::response(state, request_context, request_body).await?
|
||||
{
|
||||
return Ok(Some(response));
|
||||
}
|
||||
|
||||
let path = request_context.path();
|
||||
let is_billing_route = (request_context.method() == http::Method::GET
|
||||
&& matches!(
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
//! Current provider finance snapshots. This endpoint never calls upstream services.
|
||||
use super::build_admin_billing_data_unavailable_response;
|
||||
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
|
||||
use crate::GatewayError;
|
||||
use axum::{
|
||||
body::Body,
|
||||
http,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
fn finite(value: Option<&Value>) -> Option<f64> {
|
||||
value
|
||||
.and_then(|v| {
|
||||
v.as_f64()
|
||||
.or_else(|| v.as_str().and_then(|v| v.parse::<f64>().ok()))
|
||||
})
|
||||
.filter(|v| v.is_finite())
|
||||
}
|
||||
fn text(value: Option<&Value>) -> Option<&str> {
|
||||
value
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|v| !v.is_empty() && v.len() <= 256 && !v.chars().any(char::is_control))
|
||||
}
|
||||
fn timestamp(value: Option<&Value>) -> Option<String> {
|
||||
let value = value?;
|
||||
if let Some(raw) = value.as_str() {
|
||||
if let Ok(date) = chrono::DateTime::parse_from_rfc3339(raw) {
|
||||
return Some(date.to_rfc3339_opts(chrono::SecondsFormat::Millis, true));
|
||||
}
|
||||
}
|
||||
let secs = finite(Some(value))?;
|
||||
if !(0.0..=253_402_300_799.0).contains(&secs) {
|
||||
return None;
|
||||
}
|
||||
chrono::DateTime::from_timestamp(secs as i64, 0)
|
||||
.map(|v| v.to_rfc3339_opts(chrono::SecondsFormat::Millis, true))
|
||||
}
|
||||
fn subscription(value: &Value) -> Value {
|
||||
json!({
|
||||
"group_name": text(value.get("group_name")),
|
||||
"status": text(value.get("status")),
|
||||
"daily_used_usd": finite(value.get("daily_used_usd")),
|
||||
"daily_limit_usd": finite(value.get("daily_limit_usd")),
|
||||
"weekly_used_usd": finite(value.get("weekly_used_usd")),
|
||||
"weekly_limit_usd": finite(value.get("weekly_limit_usd")),
|
||||
"monthly_used_usd": finite(value.get("monthly_used_usd")),
|
||||
"monthly_limit_usd": finite(value.get("monthly_limit_usd")),
|
||||
"expires_at": timestamp(value.get("expires_at")),
|
||||
})
|
||||
}
|
||||
fn balance(value: &Value) -> Option<Value> {
|
||||
if value.get("action_type").and_then(Value::as_str) != Some("query_balance") {
|
||||
return None;
|
||||
}
|
||||
let status = text(value.get("status"))?;
|
||||
if !matches!(status, "success" | "auth_expired" | "auth_failed") {
|
||||
return None;
|
||||
}
|
||||
let data = value
|
||||
.get("data")
|
||||
.filter(|_| matches!(status, "success" | "auth_expired"));
|
||||
let extra = data.and_then(|d| d.get("extra"));
|
||||
let subscriptions = extra
|
||||
.and_then(|e| e.get("subscriptions"))
|
||||
.and_then(Value::as_array)
|
||||
.map(|items| {
|
||||
items
|
||||
.iter()
|
||||
.filter(|v| v.is_object())
|
||||
.take(128)
|
||||
.map(subscription)
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
Some(json!({
|
||||
"status": status,
|
||||
"observed_at": timestamp(value.get("executed_at")),
|
||||
"currency": data.and_then(|d| text(d.get("currency"))),
|
||||
"available": data.and_then(|d| finite(d.get("total_available"))),
|
||||
"used": data.and_then(|d| finite(d.get("total_used"))),
|
||||
"granted": data.and_then(|d| finite(d.get("total_granted"))),
|
||||
"plan_name": extra.and_then(|e| text(e.get("plan_name"))),
|
||||
"subscriptions": subscriptions,
|
||||
}))
|
||||
}
|
||||
pub(super) async fn response(
|
||||
state: &AdminAppState<'_>,
|
||||
context: &AdminRequestContext<'_>,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
if context.method() != http::Method::GET
|
||||
|| context.path().trim_end_matches('/') != "/api/admin/billing/provider-accounts"
|
||||
|| context.route_family() != Some("billing_manage")
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
if !state.has_provider_catalog_data_reader() {
|
||||
return Ok(Some(build_admin_billing_data_unavailable_response()));
|
||||
}
|
||||
let mut providers = state.list_provider_catalog_providers(false).await?;
|
||||
providers.sort_by(|a, b| a.name.cmp(&b.name).then_with(|| a.id.cmp(&b.id)));
|
||||
let keys = providers
|
||||
.iter()
|
||||
.map(|p| format!("provider_ops:balance:{}", p.id))
|
||||
.collect::<Vec<_>>();
|
||||
let (cached, unavailable) = if keys.is_empty() {
|
||||
(Vec::new(), false)
|
||||
} else {
|
||||
match state.runtime_state().kv_get_many(&keys).await {
|
||||
Ok(v) => (v, false),
|
||||
Err(_) => (vec![None; keys.len()], true),
|
||||
}
|
||||
};
|
||||
let items = providers.iter().enumerate().map(|(index, p)| {
|
||||
let limit = p.monthly_quota_usd.filter(|v| v.is_finite() && *v >= 0.0);
|
||||
let used = p.monthly_used_usd.filter(|v| v.is_finite() && *v >= 0.0);
|
||||
let quota = if p.billing_type.as_deref() == Some("monthly_quota") || limit.is_some() {
|
||||
json!({
|
||||
"limit": limit, "used": used,
|
||||
"remaining": limit.zip(used).map(|(l,u)| (l-u).max(0.0)),
|
||||
"currency": "USD",
|
||||
"period_start": p.quota_last_reset_at_unix_secs.and_then(|v| timestamp(Some(&json!(v)))),
|
||||
"expires_at": p.quota_expires_at_unix_secs.and_then(|v| timestamp(Some(&json!(v)))),
|
||||
})
|
||||
} else { Value::Null };
|
||||
let balance = cached.get(index).and_then(|v| v.as_deref())
|
||||
.and_then(|v| serde_json::from_str::<Value>(v).ok()).and_then(|v| balance(&v));
|
||||
json!({
|
||||
"provider_id": p.id, "provider_name": p.name, "is_active": p.is_active,
|
||||
"billing_type": p.billing_type, "quota": quota, "balance": balance,
|
||||
})
|
||||
}).collect::<Vec<_>>();
|
||||
Ok(Some((
|
||||
[(http::header::CACHE_CONTROL, "private, no-store")],
|
||||
Json(json!({
|
||||
"observed_at": chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis,true),
|
||||
"items": items, "balance_snapshot_unavailable": unavailable,
|
||||
})),
|
||||
).into_response()))
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn provider_accounts_only_expose_finance_allowlist_and_preserve_unknown() {
|
||||
let snapshot=balance(&json!({"status":"success","action_type":"query_balance","executed_at":"2026-09-20T00:00:00Z","data":{"currency":"USD","total_available":null,"extra":{"access_token":"secret","plan_name":"Pro","subscriptions":[{"group_name":"Team","monthly_used_usd":"12.25","expires_at":1800000000,"private_token":"secret"}]}}})).unwrap();
|
||||
assert!(snapshot["available"].is_null());
|
||||
assert_eq!(
|
||||
snapshot["subscriptions"][0]["monthly_used_usd"],
|
||||
json!(12.25)
|
||||
);
|
||||
assert!(!snapshot.to_string().contains("secret"));
|
||||
assert!(!snapshot.to_string().contains("access_token"));
|
||||
let failed=balance(&json!({"status":"auth_failed","action_type":"query_balance","data":{"total_available":999}})).unwrap();
|
||||
assert!(failed["available"].is_null());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
use super::{
|
||||
build_admin_billing_bad_request_response as bad_request,
|
||||
build_admin_billing_conflict_response as conflict,
|
||||
build_admin_billing_data_unavailable_response as unavailable,
|
||||
build_admin_billing_not_found_response as not_found,
|
||||
};
|
||||
use crate::handlers::admin::{
|
||||
request::{AdminAppState, AdminRequestContext},
|
||||
shared::{attach_admin_audit_response, query_param_value},
|
||||
};
|
||||
use crate::handlers::shared::normalize_payment_currency;
|
||||
use crate::GatewayError;
|
||||
use aether_data_contracts::repository::billing::*;
|
||||
use axum::{
|
||||
body::{Body, Bytes},
|
||||
http::{self, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct ExpenseRequest {
|
||||
client_request_id: String,
|
||||
provider_id: String,
|
||||
kind: String,
|
||||
amount: String,
|
||||
currency: String,
|
||||
paid_at: String,
|
||||
period_start: Option<String>,
|
||||
period_end: Option<String>,
|
||||
note: Option<String>,
|
||||
external_reference: Option<String>,
|
||||
}
|
||||
fn datetime(value: u64) -> String {
|
||||
chrono::DateTime::from_timestamp_millis(value as i64)
|
||||
.expect("valid stored timestamp")
|
||||
.to_rfc3339_opts(chrono::SecondsFormat::Millis, true)
|
||||
}
|
||||
fn parse_date(value: &str) -> Result<u64, String> {
|
||||
chrono::DateTime::parse_from_rfc3339(value)
|
||||
.ok()
|
||||
.and_then(|v| u64::try_from(v.timestamp_millis()).ok())
|
||||
.filter(|v| *v <= 253_402_300_799_000)
|
||||
.ok_or_else(|| "timestamps must be RFC3339 dates on or after 1970".into())
|
||||
}
|
||||
fn optional_text(value: Option<String>) -> Option<String> {
|
||||
value.map(|v| v.trim().to_owned()).filter(|v| !v.is_empty())
|
||||
}
|
||||
fn expense_json(record: &ProviderExpenseRecord) -> Value {
|
||||
let e = &record.entry;
|
||||
json!({
|
||||
"id": record.id, "client_request_id": e.client_request_id,
|
||||
"provider_id": e.provider_id, "provider_name": e.provider_name,
|
||||
"kind": e.kind, "amount": e.amount, "currency": e.currency,
|
||||
"paid_at": datetime(e.paid_at_unix_ms),
|
||||
"period_start": e.period_start_unix_ms.map(datetime),
|
||||
"period_end": e.period_end_unix_ms.map(datetime),
|
||||
"note": e.note, "external_reference": e.external_reference,
|
||||
"created_by": e.created_by, "created_at": datetime(record.created_at_unix_ms),
|
||||
"status": if record.voided_at_unix_ms.is_some() { "void" } else { "recorded" },
|
||||
"voided_at": record.voided_at_unix_ms.map(datetime), "voided_by": record.voided_by,
|
||||
})
|
||||
}
|
||||
fn csv_cell(value: &str) -> String {
|
||||
let value = if value.trim_start().starts_with(['=', '+', '-', '@'])
|
||||
|| value.starts_with(['\t', '\r', '\n'])
|
||||
{
|
||||
format!("'{value}")
|
||||
} else {
|
||||
value.to_string()
|
||||
};
|
||||
format!("\"{}\"", value.replace('"', "\"\""))
|
||||
}
|
||||
fn csv_report(items: &[ProviderExpenseRecord]) -> String {
|
||||
let mut result=String::from("\u{feff}id,provider_id,provider_name,kind,amount,currency,paid_at,period_start,period_end,note,external_reference,created_by,created_at\r\n");
|
||||
for r in items {
|
||||
let e = &r.entry;
|
||||
let fields = [
|
||||
r.id.clone(),
|
||||
e.provider_id.clone(),
|
||||
e.provider_name.clone(),
|
||||
e.kind.clone(),
|
||||
e.amount.clone(),
|
||||
e.currency.clone(),
|
||||
datetime(e.paid_at_unix_ms),
|
||||
e.period_start_unix_ms.map(datetime).unwrap_or_default(),
|
||||
e.period_end_unix_ms.map(datetime).unwrap_or_default(),
|
||||
e.note.clone().unwrap_or_default(),
|
||||
e.external_reference.clone().unwrap_or_default(),
|
||||
e.created_by.clone().unwrap_or_default(),
|
||||
datetime(r.created_at_unix_ms),
|
||||
];
|
||||
result.push_str(
|
||||
&fields
|
||||
.iter()
|
||||
.map(|s| csv_cell(s))
|
||||
.collect::<Vec<_>>()
|
||||
.join(","),
|
||||
);
|
||||
result.push_str("\r\n");
|
||||
}
|
||||
result
|
||||
}
|
||||
fn query(context: &AdminRequestContext<'_>, csv: bool) -> Result<ProviderExpenseQuery, String> {
|
||||
let q = context.query_string();
|
||||
let now = chrono::Utc::now().timestamp_millis().max(0) as u64;
|
||||
let from = query_param_value(q, "from")
|
||||
.map(|v| parse_date(&v))
|
||||
.transpose()?
|
||||
.unwrap_or(now.saturating_sub(30 * 86_400_000));
|
||||
let to = query_param_value(q, "to")
|
||||
.map(|v| parse_date(&v))
|
||||
.transpose()?
|
||||
.unwrap_or(now);
|
||||
let limit = if csv {
|
||||
10_001
|
||||
} else {
|
||||
query_param_value(q, "limit")
|
||||
.map(|v| v.parse::<u32>().map_err(|_| "invalid limit".to_string()))
|
||||
.transpose()?
|
||||
.unwrap_or(25)
|
||||
};
|
||||
let offset = if csv {
|
||||
0
|
||||
} else {
|
||||
query_param_value(q, "offset")
|
||||
.map(|v| v.parse::<u64>().map_err(|_| "invalid offset".to_string()))
|
||||
.transpose()?
|
||||
.unwrap_or(0)
|
||||
};
|
||||
if !csv && limit > 200 {
|
||||
return Err("limit must be at most 200".into());
|
||||
}
|
||||
let q = ProviderExpenseQuery {
|
||||
from_unix_ms: from,
|
||||
to_unix_ms: to,
|
||||
limit,
|
||||
offset,
|
||||
};
|
||||
q.validate().map_err(|e| e.to_string())?;
|
||||
Ok(q)
|
||||
}
|
||||
pub(super) async fn response(
|
||||
state: &AdminAppState<'_>,
|
||||
context: &AdminRequestContext<'_>,
|
||||
body: Option<&Bytes>,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
let path = context.path().trim_end_matches('/');
|
||||
if context.route_family() != Some("billing_manage")
|
||||
|| !path.starts_with("/api/admin/billing/provider-expenses")
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let operator = context
|
||||
.decision()
|
||||
.and_then(|d| d.admin_principal.as_ref())
|
||||
.map(|p| p.user_id.clone());
|
||||
if path == "/api/admin/billing/provider-expenses" && context.method() == http::Method::GET {
|
||||
let csv = query_param_value(context.query_string(), "format").as_deref() == Some("csv");
|
||||
let q = match query(context, csv) {
|
||||
Ok(v) => v,
|
||||
Err(e) => return Ok(Some(bad_request(e))),
|
||||
};
|
||||
let Some(page) = state
|
||||
.app()
|
||||
.data
|
||||
.list_provider_expenses(&q)
|
||||
.await
|
||||
.map_err(|e| GatewayError::Internal(e.to_string()))?
|
||||
else {
|
||||
return Ok(Some(unavailable()));
|
||||
};
|
||||
if csv {
|
||||
if page.total > 10_000 {
|
||||
return Ok(Some(
|
||||
(
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
Json(json!({"detail":"导出超过 10000 条,请缩小时间范围"})),
|
||||
)
|
||||
.into_response(),
|
||||
));
|
||||
}
|
||||
return Ok(Some(
|
||||
(
|
||||
[
|
||||
(http::header::CONTENT_TYPE, "text/csv; charset=utf-8"),
|
||||
(
|
||||
http::header::CONTENT_DISPOSITION,
|
||||
"attachment; filename=provider-expenses.csv",
|
||||
),
|
||||
(http::header::CACHE_CONTROL, "private, no-store"),
|
||||
],
|
||||
csv_report(&page.items),
|
||||
)
|
||||
.into_response(),
|
||||
));
|
||||
}
|
||||
return Ok(Some(
|
||||
(
|
||||
[(http::header::CACHE_CONTROL, "private, no-store")],
|
||||
Json(json!({
|
||||
"items": page.items.iter().map(expense_json).collect::<Vec<_>>(),
|
||||
"total": page.total, "totals": page.totals, "providers": page.providers,
|
||||
"limit": q.limit, "offset": q.offset,
|
||||
"from": datetime(q.from_unix_ms), "to": datetime(q.to_unix_ms),
|
||||
"time_basis": "paid_at", "source": "manual_ledger",
|
||||
})),
|
||||
)
|
||||
.into_response(),
|
||||
));
|
||||
}
|
||||
if path == "/api/admin/billing/provider-expenses" && context.method() == http::Method::POST {
|
||||
let Some(body) = body else {
|
||||
return Ok(Some(bad_request("缺少请求体")));
|
||||
};
|
||||
let payload = match serde_json::from_slice::<ExpenseRequest>(body) {
|
||||
Ok(v) => v,
|
||||
Err(_) => return Ok(Some(bad_request("输入验证失败"))),
|
||||
};
|
||||
let input = (|| -> Result<ProviderExpenseInput, String> {
|
||||
let units = provider_expense_amount_units(&payload.amount)
|
||||
.ok_or("amount must be a positive decimal string with at most 8 decimal places")?;
|
||||
let input = ProviderExpenseInput {
|
||||
client_request_id: uuid::Uuid::parse_str(&payload.client_request_id)
|
||||
.map_err(|_| "client_request_id must be a UUID")?
|
||||
.to_string(),
|
||||
provider_id: payload.provider_id.trim().into(),
|
||||
provider_name: "pending".into(),
|
||||
kind: payload.kind,
|
||||
amount: format_provider_expense_amount(units),
|
||||
currency: normalize_payment_currency(&payload.currency, "currency")?,
|
||||
paid_at_unix_ms: parse_date(&payload.paid_at)?,
|
||||
period_start_unix_ms: payload
|
||||
.period_start
|
||||
.as_deref()
|
||||
.map(parse_date)
|
||||
.transpose()?,
|
||||
period_end_unix_ms: payload.period_end.as_deref().map(parse_date).transpose()?,
|
||||
note: optional_text(payload.note),
|
||||
external_reference: optional_text(payload.external_reference),
|
||||
created_by: operator.clone(),
|
||||
};
|
||||
input.validate()?;
|
||||
Ok(input)
|
||||
})();
|
||||
let mut input = match input {
|
||||
Ok(v) => v,
|
||||
Err(e) => return Ok(Some(bad_request(e))),
|
||||
};
|
||||
let providers = state
|
||||
.read_provider_catalog_providers_by_ids(&[input.provider_id.clone()])
|
||||
.await?;
|
||||
let Some(provider) = providers.first() else {
|
||||
return Ok(Some(not_found("Provider not found")));
|
||||
};
|
||||
input.provider_name = provider.name.clone();
|
||||
let result = state
|
||||
.app()
|
||||
.data
|
||||
.create_provider_expense(&input)
|
||||
.await
|
||||
.map_err(|e| GatewayError::Internal(e.to_string()))?;
|
||||
return Ok(Some(mutation_response(
|
||||
result,
|
||||
"admin_provider_expense_recorded",
|
||||
"record_provider_expense",
|
||||
)));
|
||||
}
|
||||
if context.method() == http::Method::POST {
|
||||
if let Some(id) = path
|
||||
.strip_prefix("/api/admin/billing/provider-expenses/")
|
||||
.and_then(|v| v.strip_suffix("/void"))
|
||||
.filter(|v| !v.is_empty() && !v.contains('/'))
|
||||
{
|
||||
if uuid::Uuid::parse_str(id).is_err() {
|
||||
return Ok(Some(bad_request("invalid expense id")));
|
||||
}
|
||||
let result = state
|
||||
.app()
|
||||
.data
|
||||
.void_provider_expense(id, operator.as_deref())
|
||||
.await
|
||||
.map_err(|e| GatewayError::Internal(e.to_string()))?;
|
||||
return Ok(Some(mutation_response(
|
||||
result,
|
||||
"admin_provider_expense_voided",
|
||||
"void_provider_expense",
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
fn mutation_response(
|
||||
outcome: AdminBillingMutationOutcome<ProviderExpenseRecord>,
|
||||
event: &'static str,
|
||||
action: &'static str,
|
||||
) -> Response<Body> {
|
||||
match outcome {
|
||||
AdminBillingMutationOutcome::Applied(record) => attach_admin_audit_response(
|
||||
Json(json!({"item":expense_json(&record)})).into_response(),
|
||||
event,
|
||||
action,
|
||||
"provider_expense",
|
||||
&record.id,
|
||||
),
|
||||
AdminBillingMutationOutcome::Invalid(e) => conflict(e),
|
||||
AdminBillingMutationOutcome::NotFound => not_found("Provider expense not found"),
|
||||
AdminBillingMutationOutcome::Unavailable => unavailable(),
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn provider_expense_csv_neutralizes_formulas_and_quotes_fields() {
|
||||
assert_eq!(csv_cell("=cmd()"), "\"'=cmd()\"");
|
||||
assert_eq!(csv_cell(" @cmd"), "\"' @cmd\"");
|
||||
assert_eq!(csv_cell("\tcmd"), "\"'\tcmd\"");
|
||||
assert_eq!(csv_cell("a,\"b\"\nc"), "\"a,\"\"b\"\"\nc\"");
|
||||
assert_eq!(csv_cell("12.34"), "\"12.34\"");
|
||||
}
|
||||
#[test]
|
||||
fn provider_expense_dates_require_explicit_timezone_and_nonnegative_epoch() {
|
||||
assert_eq!(
|
||||
parse_date("2026-09-20T08:00:00+08:00"),
|
||||
parse_date("2026-09-20T00:00:00Z")
|
||||
);
|
||||
assert!(parse_date("2026-09-20").is_err());
|
||||
assert!(parse_date("1969-01-01T00:00:00Z").is_err());
|
||||
}
|
||||
}
|
||||
@@ -27,11 +27,18 @@ pub(in super::super) async fn build_admin_wallet_list_response(
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(build_admin_wallets_bad_request_response(detail)),
|
||||
};
|
||||
let user_id = query_param_value(query, "user_id");
|
||||
let status = query_param_value(query, "status");
|
||||
let owner_type = parse_admin_wallets_owner_type_filter(query);
|
||||
|
||||
let (wallets, total) = state
|
||||
.list_admin_wallets(status.as_deref(), owner_type.as_deref(), limit, offset)
|
||||
.list_admin_wallets(
|
||||
user_id.as_deref(),
|
||||
status.as_deref(),
|
||||
owner_type.as_deref(),
|
||||
limit,
|
||||
offset,
|
||||
)
|
||||
.await?;
|
||||
let mut items = Vec::with_capacity(wallets.len());
|
||||
for wallet in wallets {
|
||||
|
||||
@@ -35,11 +35,39 @@ fn build_admin_endpoint_health_bad_request_response(detail: &str) -> Response<Bo
|
||||
pub(super) async fn maybe_build_local_admin_endpoints_health_response(
|
||||
state: &AdminAppState<'_>,
|
||||
request_context: &AdminRequestContext<'_>,
|
||||
request_body: Option<&axum::body::Bytes>,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
let Some(decision) = request_context.decision() else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
if decision.route_family.as_deref() == Some("endpoints_health") {
|
||||
if decision.route_kind.as_deref() == Some("health_v2") {
|
||||
return Ok(Some(
|
||||
crate::handlers::shared::health_monitor::build_health_v2_response(
|
||||
state.app(),
|
||||
request_context.path(),
|
||||
request_context.query_string(),
|
||||
crate::handlers::shared::health_monitor::HealthAudience::Admin,
|
||||
)
|
||||
.await,
|
||||
));
|
||||
}
|
||||
if decision.route_kind.as_deref() == Some("health_v2_publication") {
|
||||
return Ok(Some(
|
||||
crate::handlers::shared::health_monitor::build_publication_response(
|
||||
state.app(),
|
||||
if request_context.method() == http::Method::PUT {
|
||||
Some(request_body.map_or(&[][..], |body| body.as_ref()))
|
||||
} else {
|
||||
None
|
||||
},
|
||||
)
|
||||
.await,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
if decision.route_family.as_deref() == Some("endpoints_health")
|
||||
&& decision.route_kind.as_deref() == Some("health_summary")
|
||||
&& request_context.path() == "/api/admin/endpoints/health/summary"
|
||||
|
||||
@@ -8,6 +8,7 @@ pub(crate) async fn maybe_build_local_admin_endpoints_response(
|
||||
if let Some(response) = health::maybe_build_local_admin_endpoints_health_response(
|
||||
&request.state(),
|
||||
&request.request_context(),
|
||||
request.request_body(),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
mod monitoring;
|
||||
mod overview;
|
||||
mod routes;
|
||||
mod stats;
|
||||
mod usage;
|
||||
|
||||
@@ -22,6 +22,8 @@ pub(crate) mod test_support;
|
||||
mod trace;
|
||||
mod usage_helpers;
|
||||
|
||||
pub(super) use resilience::overview_resilience_payload;
|
||||
|
||||
pub(crate) async fn maybe_build_local_admin_monitoring_response(
|
||||
state: &AdminAppState<'_>,
|
||||
request_context: &AdminRequestContext<'_>,
|
||||
|
||||
@@ -6,3 +6,23 @@ mod status;
|
||||
pub(super) use history::build_admin_monitoring_resilience_circuit_history_response;
|
||||
pub(super) use reset::build_admin_monitoring_reset_error_stats_response;
|
||||
pub(super) use status::build_admin_monitoring_resilience_status_response;
|
||||
|
||||
pub(in super::super) async fn overview_resilience_payload(
|
||||
state: &crate::handlers::admin::request::AdminAppState<'_>,
|
||||
) -> Result<serde_json::Value, crate::GatewayError> {
|
||||
let snapshot = snapshot::build_admin_monitoring_resilience_snapshot(state).await?;
|
||||
let from = (snapshot.timestamp - chrono::Duration::hours(24))
|
||||
.timestamp()
|
||||
.max(
|
||||
state
|
||||
.admin_monitoring_error_stats_reset_at()
|
||||
.unwrap_or_default() as i64,
|
||||
);
|
||||
Ok(serde_json::json!({
|
||||
"scope": {"kind": "installation"},
|
||||
"error_range": {"from": chrono::DateTime::from_timestamp(from, 0), "to": snapshot.timestamp},
|
||||
"timestamp": snapshot.timestamp, "health_score": snapshot.health_score,
|
||||
"status": snapshot.status, "error_statistics": snapshot.error_statistics,
|
||||
"recent_errors": snapshot.recent_errors, "recommendations": snapshot.recommendations,
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
|
||||
use crate::GatewayError;
|
||||
use aether_admin::observability::analytics::{dashboard_value, parse_dashboard_query};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{self, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
|
||||
pub(super) async fn response(
|
||||
state: &AdminAppState<'_>,
|
||||
context: &AdminRequestContext<'_>,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let query = match parse_dashboard_query(context.query_string()) {
|
||||
Ok(query) => query,
|
||||
Err(detail) => return Ok(super::error(StatusCode::BAD_REQUEST, &detail)),
|
||||
};
|
||||
if !state.as_ref().has_usage_data_reader() {
|
||||
return Ok(super::error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"usage analytics is unavailable",
|
||||
));
|
||||
}
|
||||
let snapshot = match tokio::time::timeout(
|
||||
std::time::Duration::from_secs(15),
|
||||
state.as_ref().query_dashboard_analytics(&query),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => result?,
|
||||
Err(_) => {
|
||||
return Ok(super::error(
|
||||
StatusCode::GATEWAY_TIMEOUT,
|
||||
"dashboard query exceeded its time budget",
|
||||
))
|
||||
}
|
||||
};
|
||||
let data = dashboard_value(&query, &snapshot).map_err(GatewayError::Internal)?;
|
||||
Ok((
|
||||
[(http::header::CACHE_CONTROL, "private, no-store")],
|
||||
Json(data),
|
||||
)
|
||||
.into_response())
|
||||
}
|
||||
|
||||
pub(super) async fn total_response(
|
||||
state: &AdminAppState<'_>,
|
||||
context: &AdminRequestContext<'_>,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
use crate::cache::OverviewTotalRead;
|
||||
use aether_data_contracts::repository::usage::UsageDashboardAnalyticsQuery;
|
||||
use serde_json::json;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
let query = match parse_dashboard_query(context.query_string()) {
|
||||
Ok(query) => query,
|
||||
Err(detail) => return Ok(super::error(StatusCode::BAD_REQUEST, &detail)),
|
||||
};
|
||||
if !state.as_ref().has_usage_data_reader() {
|
||||
return Ok(super::error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"usage analytics is unavailable",
|
||||
));
|
||||
}
|
||||
let (cached, refresh) = state.as_ref().overview_total_cache.read(Instant::now());
|
||||
if let Some(refresh) = refresh {
|
||||
let app = state.as_ref();
|
||||
let data = if app.background_data.has_usage_reader() {
|
||||
Arc::clone(&app.background_data)
|
||||
} else {
|
||||
Arc::clone(&app.data)
|
||||
};
|
||||
// Lifetime boundaries do not depend on the viewer's timezone. Every
|
||||
// administrator shares one refresh, including after a page reload.
|
||||
tokio::spawn(async move {
|
||||
let query = UsageDashboardAnalyticsQuery {
|
||||
timezone: "UTC".into(),
|
||||
};
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(185),
|
||||
data.query_dashboard_analytics(&query),
|
||||
)
|
||||
.await;
|
||||
let snapshot = match result {
|
||||
Ok(Ok(snapshot)) => Some(snapshot),
|
||||
Ok(Err(error)) => {
|
||||
tracing::warn!(%error, "dashboard lifetime refresh failed");
|
||||
None
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::warn!("dashboard lifetime refresh exceeded its time budget");
|
||||
None
|
||||
}
|
||||
};
|
||||
refresh.finish(snapshot, Instant::now());
|
||||
});
|
||||
}
|
||||
let (status, body, retry_after) = match cached {
|
||||
OverviewTotalRead::Pending => {
|
||||
(StatusCode::ACCEPTED, json!({"status":"pending"}), Some("3"))
|
||||
}
|
||||
OverviewTotalRead::Failed => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
json!({"status":"failed", "detail":"cumulative dashboard totals are temporarily unavailable; retry shortly"}),
|
||||
Some("10"),
|
||||
),
|
||||
OverviewTotalRead::Ready { snapshot, stale } => {
|
||||
let mut value = dashboard_value(&query, &snapshot).map_err(GatewayError::Internal)?;
|
||||
(
|
||||
StatusCode::OK,
|
||||
json!({
|
||||
"status":"ready", "total": value["total"].take(),
|
||||
"history_complete": snapshot.history_complete, "stale": stale,
|
||||
}),
|
||||
None,
|
||||
)
|
||||
}
|
||||
};
|
||||
let mut response = (
|
||||
status,
|
||||
[(http::header::CACHE_CONTROL, "private, no-store")],
|
||||
Json(body),
|
||||
)
|
||||
.into_response();
|
||||
if let Some(retry_after) = retry_after {
|
||||
response.headers_mut().insert(
|
||||
http::header::RETRY_AFTER,
|
||||
http::HeaderValue::from_static(retry_after),
|
||||
);
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
|
||||
use crate::GatewayError;
|
||||
use aether_admin::observability::analytics::{dashboard_summary_value, parse_dashboard_query};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{header, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
|
||||
pub(super) async fn response(
|
||||
state: &AdminAppState<'_>,
|
||||
context: &AdminRequestContext<'_>,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let query = match parse_dashboard_query(context.query_string()) {
|
||||
Ok(query) => query,
|
||||
Err(detail) => return Ok(super::error(StatusCode::BAD_REQUEST, &detail)),
|
||||
};
|
||||
if !state.as_ref().has_usage_data_reader() {
|
||||
return Ok(super::error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"dashboard statistics are unavailable",
|
||||
));
|
||||
}
|
||||
let snapshot = match tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
state.as_ref().data.query_dashboard_summary(&query),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(snapshot)) => snapshot,
|
||||
Ok(Err(error)) => return Err(GatewayError::Internal(error.to_string())),
|
||||
Err(_) => {
|
||||
return Ok(super::error(
|
||||
StatusCode::GATEWAY_TIMEOUT,
|
||||
"dashboard statistics exceeded their time budget",
|
||||
))
|
||||
}
|
||||
};
|
||||
let mut value = dashboard_summary_value(&snapshot);
|
||||
value["concurrency"] = state
|
||||
.as_ref()
|
||||
.today_concurrency(&query.timezone)
|
||||
.map_err(GatewayError::Internal)?;
|
||||
Ok(([(header::CACHE_CONTROL, "private, no-store")], Json(value)).into_response())
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
use super::error;
|
||||
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
|
||||
use crate::GatewayError;
|
||||
use aether_admin::observability::analytics::{envelope, metrics_value, OverviewRequest};
|
||||
use aether_data_contracts::repository::usage::{UsageAnalyticsQuery, USAGE_ANALYTICS_VERSION};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{header, HeaderValue, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
pub(super) async fn response(
|
||||
state: &AdminAppState<'_>,
|
||||
context: &AdminRequestContext<'_>,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
if context
|
||||
.query_string()
|
||||
.is_some_and(|query| !query.is_empty())
|
||||
{
|
||||
return Ok(error(
|
||||
StatusCode::BAD_REQUEST,
|
||||
"live diagnostics do not accept historical filters",
|
||||
));
|
||||
}
|
||||
let app = state.as_ref();
|
||||
let _ = app.metric_samples().await;
|
||||
let snapshot = app.metric_snapshot.read().await.clone();
|
||||
let captured = snapshot.as_ref().map(|(captured, _)| *captured);
|
||||
let now = chrono::Utc::now();
|
||||
let observed_at = captured
|
||||
.and_then(|captured| chrono::Duration::from_std(captured.elapsed()).ok())
|
||||
.map(|age| now - age);
|
||||
let mut unavailable = Vec::new();
|
||||
let (resilience_result, recent_result) = tokio::join!(
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(3),
|
||||
super::super::monitoring::overview_resilience_payload(state)
|
||||
),
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(3),
|
||||
recent_activity(state, now)
|
||||
),
|
||||
);
|
||||
let resilience = match resilience_result {
|
||||
Ok(Ok(value)) => Some(value),
|
||||
_ => {
|
||||
tracing::warn!("overview resilience snapshot unavailable");
|
||||
unavailable.push("resilience");
|
||||
None
|
||||
}
|
||||
};
|
||||
let recent_activity = match recent_result {
|
||||
Ok(Ok(value)) => Some(value),
|
||||
_ => {
|
||||
unavailable.push("recent_activity");
|
||||
None
|
||||
}
|
||||
};
|
||||
if captured.is_none() {
|
||||
unavailable.push("metrics");
|
||||
}
|
||||
let mut response = Json(json!({
|
||||
"meta": {
|
||||
"schema_version": 1, "metric_version": USAGE_ANALYTICS_VERSION, "scope": {"kind": "node"},
|
||||
"generated_at": now, "data_through": observed_at, "read_revision": observed_at.map(|value| value.timestamp_millis().to_string()),
|
||||
"coverage": {"status": if unavailable.is_empty() {"complete"} else {"partial"}},
|
||||
},
|
||||
"data": {
|
||||
"observed_at": observed_at, "window_seconds": null, "node_id": null,
|
||||
"scope": {"kind": "node", "node_ids": []},
|
||||
"metrics_text": snapshot.map(|(_, samples)| aether_runtime::metrics::render_prometheus_text(&samples)),
|
||||
"resilience": resilience, "recent_activity": recent_activity,
|
||||
"execution_activity": app.execution_activity.snapshot(),
|
||||
"unavailable_sections": unavailable,
|
||||
},
|
||||
})).into_response();
|
||||
response.headers_mut().insert(
|
||||
header::CACHE_CONTROL,
|
||||
HeaderValue::from_static("private, no-store"),
|
||||
);
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
async fn recent_activity(
|
||||
state: &AdminAppState<'_>,
|
||||
now: chrono::DateTime<chrono::Utc>,
|
||||
) -> Result<serde_json::Value, GatewayError> {
|
||||
let to = now.timestamp_millis().max(60_000) as u64;
|
||||
let request = OverviewRequest {
|
||||
query: UsageAnalyticsQuery {
|
||||
from_unix_ms: to - 60_000,
|
||||
to_unix_ms: to,
|
||||
timezone: "UTC".into(),
|
||||
limit: 1,
|
||||
..Default::default()
|
||||
},
|
||||
amount_basis: "billable".into(),
|
||||
csv: false,
|
||||
};
|
||||
let snapshot = state.as_ref().query_usage_analytics(&request.query).await?;
|
||||
let data = recent_activity_data(&snapshot);
|
||||
Ok(envelope(&request, &snapshot, data))
|
||||
}
|
||||
|
||||
fn recent_activity_data(
|
||||
snapshot: &aether_data_contracts::repository::usage::StoredUsageAnalytics,
|
||||
) -> serde_json::Value {
|
||||
let mut data = metrics_value(&snapshot.summary);
|
||||
data["requests_per_second"] = json!(snapshot.summary.request_count as f64 / 60.0);
|
||||
data["requests_per_minute"] = json!(snapshot.summary.request_count);
|
||||
data["tokens_per_minute"] = data["total_tokens"].clone();
|
||||
data["window_seconds"] = json!(60);
|
||||
data
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aether_data_contracts::repository::usage::{StoredUsageAnalytics, UsageAnalyticsMetrics};
|
||||
|
||||
#[test]
|
||||
fn recent_activity_reports_one_minute_rates_without_inventing_missing_tokens() {
|
||||
let mut snapshot = StoredUsageAnalytics {
|
||||
summary: UsageAnalyticsMetrics {
|
||||
request_count: 120,
|
||||
usage_available_count: 120,
|
||||
total_tokens: 4200,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let value = recent_activity_data(&snapshot);
|
||||
assert_eq!(value["window_seconds"], 60);
|
||||
assert_eq!(value["requests_per_second"], 2.0);
|
||||
assert_eq!(value["requests_per_minute"], 120);
|
||||
assert_eq!(value["tokens_per_minute"], 4200);
|
||||
snapshot.summary.usage_available_count = 0;
|
||||
assert!(recent_activity_data(&snapshot)["tokens_per_minute"].is_null());
|
||||
snapshot.summary = UsageAnalyticsMetrics::default();
|
||||
assert_eq!(recent_activity_data(&snapshot)["tokens_per_minute"], 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
mod dashboard;
|
||||
mod dashboard_summary;
|
||||
mod live;
|
||||
|
||||
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
|
||||
use crate::GatewayError;
|
||||
use aether_admin::observability::analytics::{
|
||||
costs_value, dashboard_charts_value, envelope, export_csv, metrics_value, page_value,
|
||||
parse_dashboard_charts_query, parse_overview_query, performance_value, user_finance_value,
|
||||
user_payments_value,
|
||||
};
|
||||
use aether_data_contracts::repository::usage::{UsageAnalyticsGranularity, UsageAnalyticsView};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{self, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
pub(crate) async fn maybe_build_overview_response(
|
||||
state: &AdminAppState<'_>,
|
||||
context: &AdminRequestContext<'_>,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
if context.route_family() != Some("overview_manage") || context.method() != http::Method::GET {
|
||||
return Ok(None);
|
||||
}
|
||||
let kind = context.route_kind().unwrap_or_default();
|
||||
if kind == "dashboard_summary" {
|
||||
return dashboard_summary::response(state, context).await.map(Some);
|
||||
}
|
||||
if kind == "dashboard_total" {
|
||||
return dashboard::total_response(state, context).await.map(Some);
|
||||
}
|
||||
if kind == "dashboard" {
|
||||
return dashboard::response(state, context).await.map(Some);
|
||||
}
|
||||
if matches!(kind, "operations_live" | "operations_resources") {
|
||||
return live::response(state, context).await.map(Some);
|
||||
}
|
||||
let view = match kind {
|
||||
"dashboard_charts" => UsageAnalyticsView::DashboardCharts,
|
||||
"summary" => UsageAnalyticsView::Summary,
|
||||
"timeseries" | "costs" => UsageAnalyticsView::Timeseries,
|
||||
"operations_performance" => UsageAnalyticsView::Performance,
|
||||
"users" | "user_detail" => UsageAnalyticsView::Users,
|
||||
"breakdown" => UsageAnalyticsView::Breakdown,
|
||||
"consumption" => UsageAnalyticsView::Consumption,
|
||||
_ => return Ok(None),
|
||||
};
|
||||
let parsed = if kind == "dashboard_charts" {
|
||||
parse_dashboard_charts_query(context.query_string())
|
||||
} else {
|
||||
parse_overview_query(context.query_string(), view)
|
||||
};
|
||||
let mut request = match parsed {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(error(StatusCode::BAD_REQUEST, &detail))),
|
||||
};
|
||||
if kind == "user_detail" {
|
||||
let encoded = context
|
||||
.path()
|
||||
.trim_end_matches('/')
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.unwrap_or_default();
|
||||
let Ok(id) = percent_encoding::percent_decode_str(encoded).decode_utf8() else {
|
||||
return Ok(Some(error(
|
||||
StatusCode::BAD_REQUEST,
|
||||
"invalid user identifier",
|
||||
)));
|
||||
};
|
||||
let id = id.as_ref();
|
||||
if id.is_empty() || id.len() > 512 || id.contains('/') || id.chars().any(char::is_control) {
|
||||
return Ok(Some(error(
|
||||
StatusCode::BAD_REQUEST,
|
||||
"invalid user identifier",
|
||||
)));
|
||||
}
|
||||
if request
|
||||
.query
|
||||
.actor_user_id
|
||||
.as_deref()
|
||||
.is_some_and(|value| value != id)
|
||||
|| request
|
||||
.query
|
||||
.credential_owner_id
|
||||
.as_deref()
|
||||
.is_some_and(|value| value != id)
|
||||
{
|
||||
return Ok(Some(error(
|
||||
StatusCode::BAD_REQUEST,
|
||||
"user filter conflicts with the requested employee",
|
||||
)));
|
||||
}
|
||||
if request.query.attribution_kind.as_deref() == Some("employee") {
|
||||
request.query.actor_user_id = Some(id.into());
|
||||
} else {
|
||||
request.query.credential_owner_id = Some(id.into());
|
||||
}
|
||||
request.query.limit = 1;
|
||||
request.query.offset = 0;
|
||||
}
|
||||
if matches!(
|
||||
view,
|
||||
UsageAnalyticsView::Timeseries | UsageAnalyticsView::Performance
|
||||
) {
|
||||
request.query.limit = 10_000;
|
||||
request.query.offset = 0;
|
||||
}
|
||||
if kind == "costs" {
|
||||
request.query.granularity = UsageAnalyticsGranularity::Day;
|
||||
}
|
||||
if !state.as_ref().has_usage_data_reader() {
|
||||
return Ok(Some(error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"usage analytics is unavailable",
|
||||
)));
|
||||
}
|
||||
let snapshot = match tokio::time::timeout(
|
||||
std::time::Duration::from_secs(if request.csv { 30 } else { 15 }),
|
||||
state.as_ref().query_usage_analytics(&request.query),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => result?,
|
||||
Err(_) => {
|
||||
return Ok(Some(error(
|
||||
StatusCode::GATEWAY_TIMEOUT,
|
||||
"report query exceeded its time budget; narrow the range or filters",
|
||||
)))
|
||||
}
|
||||
};
|
||||
if request.csv {
|
||||
return Ok(Some(match export_csv(&request, &snapshot) {
|
||||
Ok(csv) => (
|
||||
[
|
||||
(http::header::CONTENT_TYPE, "text/csv; charset=utf-8"),
|
||||
(
|
||||
http::header::CONTENT_DISPOSITION,
|
||||
"attachment; filename=overview.csv",
|
||||
),
|
||||
(http::header::CACHE_CONTROL, "private, no-store"),
|
||||
],
|
||||
csv,
|
||||
)
|
||||
.into_response(),
|
||||
Err(detail) => error(StatusCode::UNPROCESSABLE_ENTITY, &detail),
|
||||
}));
|
||||
}
|
||||
let data = match kind {
|
||||
"dashboard_charts" => dashboard_charts_value(&snapshot),
|
||||
"summary" => metrics_value(&snapshot.summary),
|
||||
"user_detail" => {
|
||||
let Some(user) = snapshot.users.first() else {
|
||||
return Ok(Some(error(StatusCode::NOT_FOUND, "employee not found")));
|
||||
};
|
||||
json!({
|
||||
"user": { "id": user.user_id, "username": user.username, "email": user.email, "is_active": user.is_active },
|
||||
"summary": metrics_value(&user.metrics),
|
||||
"finance": user_finance_value(user.finance.as_ref()),
|
||||
"payments": user_payments_value(snapshot.user_payments.as_ref()),
|
||||
})
|
||||
}
|
||||
"costs" => costs_value(&request, &snapshot),
|
||||
"timeseries" => {
|
||||
let mut page = page_value(&request, &snapshot);
|
||||
page["granularity"] = json!(request.query.granularity);
|
||||
page
|
||||
}
|
||||
"operations_performance" => performance_value(&request, &snapshot),
|
||||
_ => page_value(&request, &snapshot),
|
||||
};
|
||||
let mut response = Json(envelope(&request, &snapshot, data)).into_response();
|
||||
response.headers_mut().insert(
|
||||
http::header::CACHE_CONTROL,
|
||||
http::HeaderValue::from_static("private, no-store"),
|
||||
);
|
||||
Ok(Some(response))
|
||||
}
|
||||
|
||||
fn error(status: StatusCode, detail: &str) -> Response<Body> {
|
||||
(status, Json(json!({"detail": detail}))).into_response()
|
||||
}
|
||||
@@ -1,9 +1,15 @@
|
||||
use super::{monitoring, stats, usage};
|
||||
use super::{monitoring, overview, stats, usage};
|
||||
use crate::handlers::admin::request::{AdminRouteRequest, AdminRouteResult};
|
||||
|
||||
pub(crate) async fn maybe_build_local_admin_observability_response(
|
||||
request: AdminRouteRequest<'_>,
|
||||
) -> AdminRouteResult {
|
||||
if let Some(response) =
|
||||
overview::maybe_build_overview_response(&request.state(), &request.request_context())
|
||||
.await?
|
||||
{
|
||||
return Ok(Some(response));
|
||||
}
|
||||
if let Some(response) =
|
||||
stats::maybe_build_local_admin_stats_response(&request.state(), &request.request_context())
|
||||
.await?
|
||||
|
||||
+3
-4
@@ -1,5 +1,5 @@
|
||||
use super::super::super::stats::resolve_admin_usage_time_range;
|
||||
use super::super::analytics::admin_usage_aggregation_by_user_json;
|
||||
use super::super::summary_routes::resolve_record_time_bounds;
|
||||
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
|
||||
use crate::handlers::admin::shared::query_param_value;
|
||||
use crate::GatewayError;
|
||||
@@ -159,12 +159,11 @@ pub(super) async fn build_admin_usage_aggregation_stats_response(
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(admin_usage_bad_request_response(detail)),
|
||||
};
|
||||
let time_range = match resolve_admin_usage_time_range(query) {
|
||||
let time_bounds = match resolve_record_time_bounds(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(admin_usage_bad_request_response(detail)),
|
||||
};
|
||||
let Some((created_from_unix_secs, created_until_unix_secs)) = time_range.to_unix_bounds()
|
||||
else {
|
||||
let Some((created_from_unix_secs, created_until_unix_secs)) = time_bounds else {
|
||||
return Ok(Json(json!([])).into_response());
|
||||
};
|
||||
let group_by_query = match group_by.as_str() {
|
||||
|
||||
@@ -33,6 +33,50 @@ use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
const ADMIN_USAGE_ACTIVE_LIMIT: usize = 50;
|
||||
|
||||
pub(super) fn resolve_record_time_bounds(
|
||||
query: Option<&str>,
|
||||
) -> Result<Option<(u64, u64)>, String> {
|
||||
let entries =
|
||||
url::form_urlencoded::parse(query.unwrap_or_default().as_bytes()).collect::<Vec<_>>();
|
||||
let from = entries
|
||||
.iter()
|
||||
.filter(|(key, _)| key == "from")
|
||||
.collect::<Vec<_>>();
|
||||
let to = entries
|
||||
.iter()
|
||||
.filter(|(key, _)| key == "to")
|
||||
.collect::<Vec<_>>();
|
||||
if from.is_empty() && to.is_empty() {
|
||||
return resolve_admin_usage_time_range(query).map(|range| range.to_unix_bounds());
|
||||
}
|
||||
if from.len() != 1 || to.len() != 1 {
|
||||
return Err("from and to must each be provided once".into());
|
||||
}
|
||||
if entries
|
||||
.iter()
|
||||
.any(|(key, _)| matches!(key.as_ref(), "start_date" | "end_date" | "preset" | "days"))
|
||||
{
|
||||
return Err("precise from/to cannot be combined with date presets".into());
|
||||
}
|
||||
if let Some(zone) = query_param_value(query, "timezone") {
|
||||
zone.parse::<chrono_tz::Tz>()
|
||||
.map_err(|_| "invalid timezone".to_string())?;
|
||||
}
|
||||
let parse = |value: &str| -> Result<u64, String> {
|
||||
let value = chrono::DateTime::parse_from_rfc3339(value)
|
||||
.map_err(|_| "from/to must be RFC 3339 timestamps".to_string())?;
|
||||
if value.timestamp_subsec_nanos() != 0 {
|
||||
return Err("request records support second-aligned ranges".into());
|
||||
}
|
||||
u64::try_from(value.timestamp()).map_err(|_| "from/to must not precede Unix epoch".into())
|
||||
};
|
||||
let bounds = (parse(&from[0].1)?, parse(&to[0].1)?);
|
||||
if bounds.0 >= bounds.1 || bounds.1 - bounds.0 > 366 * 86_400 {
|
||||
return Err("from/to must define a nonempty range of at most 366 days".into());
|
||||
}
|
||||
Ok(Some(bounds))
|
||||
}
|
||||
|
||||
async fn load_admin_usage_by_ids(
|
||||
state: &AdminAppState<'_>,
|
||||
requested_ids: &BTreeSet<String>,
|
||||
@@ -70,6 +114,7 @@ fn apply_admin_usage_status_filter(query: &mut UsageAuditListQuery, status: Opti
|
||||
}
|
||||
"websocket" | "ws" => query.is_websocket = Some(true),
|
||||
"error" | "failed" => query.error_only = true,
|
||||
"success" => query.statuses = Some(vec!["completed".to_string()]),
|
||||
"active" => {
|
||||
query.statuses = Some(vec!["pending".to_string(), "streaming".to_string()]);
|
||||
}
|
||||
@@ -523,12 +568,37 @@ fn build_admin_usage_records_query(
|
||||
query: Option<&str>,
|
||||
limit: Option<usize>,
|
||||
offset: Option<usize>,
|
||||
) -> UsageAuditListQuery {
|
||||
) -> Result<UsageAuditListQuery, String> {
|
||||
let boolean = |key| match query_param_value(query, key).as_deref() {
|
||||
None => Ok(None),
|
||||
Some("true" | "1") => Ok(Some(true)),
|
||||
Some("false" | "0") => Ok(Some(false)),
|
||||
Some(_) => Err(format!("invalid {key}: expected true or false")),
|
||||
};
|
||||
let slow_threshold_ms = query_param_value(query, "slow_threshold_ms")
|
||||
.map(|value| {
|
||||
value
|
||||
.parse::<u64>()
|
||||
.ok()
|
||||
.filter(|value| (1..=86_400_000).contains(value))
|
||||
.ok_or_else(|| "slow_threshold_ms must be between 1 and 86400000".to_string())
|
||||
})
|
||||
.transpose()?;
|
||||
let mut list_query = UsageAuditListQuery {
|
||||
created_from_unix_secs: Some(created_from_unix_secs),
|
||||
created_until_unix_secs: Some(created_until_unix_secs),
|
||||
user_id: query_param_value(query, "user_id"),
|
||||
provider_name: query_param_value(query, "provider"),
|
||||
provider_id: query_param_value(query, "provider_id"),
|
||||
api_key_id: query_param_value(query, "api_key_id"),
|
||||
request_id: query_param_value(query, "request_id"),
|
||||
attribution_kind: query_param_value(query, "attribution_kind"),
|
||||
actor_user_id: query_param_value(query, "actor_user_id"),
|
||||
slow_threshold_ms,
|
||||
endpoint_kind: query_param_value(query, "endpoint_kind"),
|
||||
request_type: query_param_value(query, "request_type"),
|
||||
has_format_conversion: boolean("has_format_conversion")?,
|
||||
is_stream: boolean("is_stream")?,
|
||||
model: query_param_value(query, "model"),
|
||||
api_format: query_param_value(query, "api_format"),
|
||||
limit,
|
||||
@@ -536,11 +606,23 @@ fn build_admin_usage_records_query(
|
||||
newest_first: true,
|
||||
..Default::default()
|
||||
};
|
||||
if list_query
|
||||
.attribution_kind
|
||||
.as_deref()
|
||||
.is_some_and(|kind| !matches!(kind, "employee" | "standalone" | "unknown"))
|
||||
{
|
||||
return Err("invalid attribution_kind".into());
|
||||
}
|
||||
if list_query.attribution_kind.as_deref() == Some("employee")
|
||||
&& list_query.actor_user_id.is_none()
|
||||
{
|
||||
list_query.actor_user_id = list_query.user_id.take();
|
||||
}
|
||||
apply_admin_usage_status_filter(
|
||||
&mut list_query,
|
||||
query_param_value(query, "status").as_deref(),
|
||||
);
|
||||
list_query
|
||||
Ok(list_query)
|
||||
}
|
||||
|
||||
fn parse_admin_usage_search_keywords(search: &str) -> Vec<String> {
|
||||
@@ -653,6 +735,15 @@ fn build_admin_usage_keyword_search_query(
|
||||
created_until_unix_secs: base_query.created_until_unix_secs,
|
||||
user_id: base_query.user_id.clone(),
|
||||
provider_name: base_query.provider_name.clone(),
|
||||
provider_id: base_query.provider_id.clone(),
|
||||
api_key_id: base_query.api_key_id.clone(),
|
||||
request_id: base_query.request_id.clone(),
|
||||
attribution_kind: base_query.attribution_kind.clone(),
|
||||
actor_user_id: base_query.actor_user_id.clone(),
|
||||
slow_threshold_ms: base_query.slow_threshold_ms,
|
||||
endpoint_kind: base_query.endpoint_kind.clone(),
|
||||
request_type: base_query.request_type.clone(),
|
||||
has_format_conversion: base_query.has_format_conversion,
|
||||
model: base_query.model.clone(),
|
||||
api_format: base_query.api_format.clone(),
|
||||
client_family: base_query.client_family.clone(),
|
||||
@@ -699,13 +790,11 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
|
||||
}
|
||||
|
||||
let query = request_context.request_query_string.as_deref();
|
||||
let time_range = match resolve_admin_usage_time_range(query) {
|
||||
let time_bounds = match resolve_record_time_bounds(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
|
||||
};
|
||||
let Some((created_from_unix_secs, created_until_unix_secs)) =
|
||||
time_range.to_unix_bounds()
|
||||
else {
|
||||
let Some((created_from_unix_secs, created_until_unix_secs)) = time_bounds else {
|
||||
return Ok(Some(build_admin_usage_summary_stats_response_from_summary(
|
||||
&Default::default(),
|
||||
)));
|
||||
@@ -743,13 +832,11 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
|
||||
sort_usage_newest_first(&mut items);
|
||||
items
|
||||
} else {
|
||||
let time_range = match resolve_admin_usage_time_range(query) {
|
||||
let time_bounds = match resolve_record_time_bounds(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
|
||||
};
|
||||
let Some((created_from_unix_secs, created_until_unix_secs)) =
|
||||
time_range.to_unix_bounds()
|
||||
else {
|
||||
let Some((created_from_unix_secs, created_until_unix_secs)) = time_bounds else {
|
||||
return Ok(Some(build_admin_usage_active_requests_response(
|
||||
&[],
|
||||
&BTreeMap::new(),
|
||||
@@ -806,7 +893,7 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
|
||||
}
|
||||
|
||||
let query = request_context.request_query_string.as_deref();
|
||||
let time_range = match resolve_admin_usage_time_range(query) {
|
||||
let time_bounds = match resolve_record_time_bounds(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
|
||||
};
|
||||
@@ -827,9 +914,7 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
|
||||
};
|
||||
let Some((created_from_unix_secs, created_until_unix_secs)) =
|
||||
time_range.to_unix_bounds()
|
||||
else {
|
||||
let Some((created_from_unix_secs, created_until_unix_secs)) = time_bounds else {
|
||||
return Ok(Some(build_admin_usage_records_response(
|
||||
&[],
|
||||
&BTreeMap::new(),
|
||||
@@ -849,13 +934,16 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
|
||||
let active_client_family_filter = client_family_filter
|
||||
.as_deref()
|
||||
.filter(|value| !value.trim().is_empty());
|
||||
let mut base_query = build_admin_usage_records_query(
|
||||
let mut base_query = match build_admin_usage_records_query(
|
||||
created_from_unix_secs,
|
||||
created_until_unix_secs,
|
||||
query,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_usage_bad_request_response(detail))),
|
||||
};
|
||||
base_query.client_family = active_client_family_filter.map(str::to_owned);
|
||||
base_query.exclude_unknown_model_or_provider = hide_unknown_records;
|
||||
let (usage, total, total_is_estimated) = if attempt_status_filter.is_some() {
|
||||
@@ -1028,6 +1116,89 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn precise_record_ranges_preserve_minutes_and_reject_mixed_presets() {
|
||||
let range = "from=2026-09-01T23:45:00Z&to=2026-09-02T00:15:00Z&timezone=Asia%2FShanghai";
|
||||
let (from, to) = super::resolve_record_time_bounds(Some(range))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(to - from, 30 * 60);
|
||||
assert!(super::resolve_record_time_bounds(Some(&format!("{range}&preset=today"))).is_err());
|
||||
assert!(super::resolve_record_time_bounds(Some("from=2026-09-01T00:00:00Z")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overview_record_drilldown_preserves_actor_and_performance_filters() {
|
||||
let raw = "user_id=employee-1&attribution_kind=employee&provider_id=provider-1&api_key_id=key-1&request_id=request-1&endpoint_kind=chat&request_type=chat&is_stream=true&has_format_conversion=false&slow_threshold_ms=12000&status=success";
|
||||
let query =
|
||||
super::build_admin_usage_records_query(100, 200, Some(raw), None, None).unwrap();
|
||||
assert_eq!(query.user_id, None);
|
||||
assert_eq!(query.actor_user_id.as_deref(), Some("employee-1"));
|
||||
assert_eq!(query.provider_id.as_deref(), Some("provider-1"));
|
||||
assert_eq!(query.api_key_id.as_deref(), Some("key-1"));
|
||||
assert_eq!(query.request_id.as_deref(), Some("request-1"));
|
||||
assert_eq!(query.slow_threshold_ms, Some(12_000));
|
||||
assert_eq!(query.is_stream, Some(true));
|
||||
assert_eq!(query.has_format_conversion, Some(false));
|
||||
assert_eq!(query.statuses, Some(vec!["completed".into()]));
|
||||
let keyword = super::build_admin_usage_keyword_search_query(
|
||||
&query,
|
||||
vec!["example".into()],
|
||||
None,
|
||||
Default::default(),
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
assert_eq!(keyword.actor_user_id, query.actor_user_id);
|
||||
assert_eq!(keyword.slow_threshold_ms, query.slow_threshold_ms);
|
||||
assert_eq!(keyword.has_format_conversion, query.has_format_conversion);
|
||||
for invalid in [
|
||||
"is_stream=maybe",
|
||||
"slow_threshold_ms=0",
|
||||
"attribution_kind=owner",
|
||||
] {
|
||||
assert!(
|
||||
super::build_admin_usage_records_query(100, 200, Some(invalid), None, None)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overview_record_drilldown_preserves_standalone_key_ownership() {
|
||||
let raw = "user_id=owner-1&attribution_kind=standalone&api_key_id=standalone-key";
|
||||
let query =
|
||||
super::build_admin_usage_records_query(100, 200, Some(raw), None, None).unwrap();
|
||||
assert_eq!(query.attribution_kind.as_deref(), Some("standalone"));
|
||||
assert_eq!(query.user_id.as_deref(), Some("owner-1"));
|
||||
assert_eq!(query.actor_user_id, None);
|
||||
assert_eq!(query.api_key_id.as_deref(), Some("standalone-key"));
|
||||
let keyword = super::build_admin_usage_keyword_search_query(
|
||||
&query,
|
||||
vec!["example".into()],
|
||||
None,
|
||||
Default::default(),
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
assert_eq!(keyword.attribution_kind, query.attribution_kind);
|
||||
assert_eq!(keyword.user_id, query.user_id);
|
||||
assert_eq!(keyword.actor_user_id, query.actor_user_id);
|
||||
assert_eq!(keyword.api_key_id, query.api_key_id);
|
||||
|
||||
for retired_kind in ["service", "shared"] {
|
||||
let raw = format!("attribution_kind={retired_kind}");
|
||||
assert!(
|
||||
super::build_admin_usage_records_query(100, 200, Some(&raw), None, None).is_err(),
|
||||
"{retired_kind}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
use aether_data_contracts::repository::candidates::{
|
||||
RequestCandidateStatus, StoredRequestCandidate,
|
||||
};
|
||||
@@ -1169,7 +1340,7 @@ mod tests {
|
||||
for status in ["websocket", "ws", "WS"] {
|
||||
let raw_query = format!("status={status}");
|
||||
let list_query =
|
||||
build_admin_usage_records_query(100, 200, Some(&raw_query), None, None);
|
||||
build_admin_usage_records_query(100, 200, Some(&raw_query), None, None).unwrap();
|
||||
|
||||
assert_eq!(list_query.is_websocket, Some(true));
|
||||
assert_eq!(list_query.is_stream, None);
|
||||
@@ -1190,7 +1361,7 @@ mod tests {
|
||||
for (status, expected_stream) in [("stream", true), ("standard", false)] {
|
||||
let raw_query = format!("status={status}");
|
||||
let list_query =
|
||||
build_admin_usage_records_query(100, 200, Some(&raw_query), None, None);
|
||||
build_admin_usage_records_query(100, 200, Some(&raw_query), None, None).unwrap();
|
||||
assert_eq!(list_query.is_stream, Some(expected_stream));
|
||||
assert_eq!(list_query.is_websocket, Some(false));
|
||||
|
||||
|
||||
@@ -121,6 +121,7 @@ impl<'a> AdminAppState<'a> {
|
||||
|
||||
pub(crate) async fn list_admin_wallets(
|
||||
&self,
|
||||
user_id: Option<&str>,
|
||||
status: Option<&str>,
|
||||
owner_type: Option<&str>,
|
||||
limit: usize,
|
||||
@@ -133,7 +134,7 @@ impl<'a> AdminAppState<'a> {
|
||||
GatewayError,
|
||||
> {
|
||||
self.app
|
||||
.list_admin_wallets(status, owner_type, limit, offset)
|
||||
.list_admin_wallets(user_id, status, owner_type, limit, offset)
|
||||
.await
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
use super::{build_admin_users_bad_request_response, build_admin_users_data_unavailable_response};
|
||||
use crate::handlers::admin::billing::admin_payment_gateway_response_projection;
|
||||
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
|
||||
use crate::handlers::admin::shared::{attach_admin_audit_response, unix_secs_to_rfc3339};
|
||||
use crate::handlers::admin::shared::{
|
||||
attach_admin_audit_response, query_param_value, unix_secs_to_rfc3339,
|
||||
};
|
||||
use crate::handlers::shared::unix_ms_to_rfc3339;
|
||||
use crate::GatewayError;
|
||||
use aether_data_contracts::repository::billing::{BillingPlanRecord, UserPlanEntitlementRecord};
|
||||
@@ -177,8 +179,13 @@ fn entitlement_payload(
|
||||
async fn load_admin_user_entitlements_payload(
|
||||
state: &AdminAppState<'_>,
|
||||
user_id: &str,
|
||||
include_inactive: bool,
|
||||
) -> Result<Option<serde_json::Value>, GatewayError> {
|
||||
let entitlements = match state.app().list_user_plan_entitlements(user_id).await? {
|
||||
let entitlements = match state
|
||||
.app()
|
||||
.list_user_plan_entitlements_with_history(user_id, include_inactive)
|
||||
.await?
|
||||
{
|
||||
Some(value) => value,
|
||||
None => return Ok(None),
|
||||
};
|
||||
@@ -214,7 +221,17 @@ pub(in super::super) async fn build_admin_list_user_billing_entitlements_respons
|
||||
)
|
||||
.into_response());
|
||||
}
|
||||
match load_admin_user_entitlements_payload(state, &user_id).await? {
|
||||
let include_inactive =
|
||||
match query_param_value(request_context.query_string(), "include_inactive").as_deref() {
|
||||
None | Some("false" | "0") => false,
|
||||
Some("true" | "1") => true,
|
||||
_ => {
|
||||
return Ok(build_admin_users_bad_request_response(
|
||||
"include_inactive 必须为布尔值",
|
||||
))
|
||||
}
|
||||
};
|
||||
match load_admin_user_entitlements_payload(state, &user_id, include_inactive).await? {
|
||||
Some(payload) => Ok(Json(payload).into_response()),
|
||||
None => Ok(build_admin_users_data_unavailable_response()),
|
||||
}
|
||||
@@ -256,7 +273,7 @@ pub(in super::super) async fn build_admin_revoke_user_billing_entitlement_respon
|
||||
return Ok(build_admin_users_data_unavailable_response());
|
||||
}
|
||||
}
|
||||
let entitlements = match load_admin_user_entitlements_payload(state, &user_id).await? {
|
||||
let entitlements = match load_admin_user_entitlements_payload(state, &user_id, false).await? {
|
||||
Some(value) => value,
|
||||
None => return Ok(build_admin_users_data_unavailable_response()),
|
||||
};
|
||||
@@ -401,7 +418,7 @@ pub(in super::super) async fn build_admin_grant_user_billing_plan_response(
|
||||
return Ok(build_admin_users_data_unavailable_response());
|
||||
}
|
||||
};
|
||||
let entitlements = match load_admin_user_entitlements_payload(state, &user_id).await? {
|
||||
let entitlements = match load_admin_user_entitlements_payload(state, &user_id, false).await? {
|
||||
Some(value) => value,
|
||||
None => return Ok(build_admin_users_data_unavailable_response()),
|
||||
};
|
||||
|
||||
@@ -1996,9 +1996,22 @@ async fn proxy_request_inner(
|
||||
request_permit = aether_runtime::AdmissionPermit::combine(
|
||||
request_permit.into_iter().chain(plan_usage_permit),
|
||||
);
|
||||
if let Some(request_permit) = request_permit.as_ref() {
|
||||
// The affinity-forwarding node already returned above. Observe only local
|
||||
// AI execution, retaining the guard in both the body and detached execution.
|
||||
let activity_permit = control_decision
|
||||
.is_some_and(|decision| {
|
||||
decision.route_class.as_deref() == Some("ai_public")
|
||||
&& decision.execution_runtime_candidate
|
||||
})
|
||||
.then(|| state.request_activity.begin().into_admission_permit());
|
||||
if let Some(activity) = activity_permit.as_ref() {
|
||||
crate::request_lifecycle::track_request_activity(activity.clone());
|
||||
}
|
||||
if let Some(background_permit) = aether_runtime::AdmissionPermit::combine(
|
||||
request_permit.clone().into_iter().chain(activity_permit),
|
||||
) {
|
||||
parts.extensions.insert(
|
||||
crate::executor::candidate_loop::BackgroundAdmissionPermit::new(request_permit.clone()),
|
||||
crate::executor::candidate_loop::BackgroundAdmissionPermit::new(background_permit),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -376,6 +376,12 @@ async fn handle_live_http(
|
||||
state,
|
||||
&attempt.plan,
|
||||
request_context.trace_id.as_str(),
|
||||
Some(
|
||||
attempt
|
||||
.report_context
|
||||
.as_ref()
|
||||
.unwrap_or(&serde_json::Value::Null),
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -1140,8 +1140,13 @@ async fn acquire_live_relay_admission(
|
||||
});
|
||||
}
|
||||
}
|
||||
match ResponsesWebSocketTurnAdmission::acquire(state, &attempt.plan, context.trace_id.as_str())
|
||||
.await
|
||||
match ResponsesWebSocketTurnAdmission::acquire(
|
||||
state,
|
||||
&attempt.plan,
|
||||
context.trace_id.as_str(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(capacity) => Ok(LiveRelayAdmission { capacity, audit }),
|
||||
Err(error) => Err(LiveRelayAdmissionFailure {
|
||||
|
||||
@@ -136,6 +136,7 @@ pub(super) async fn prepare_realtime_websocket(
|
||||
state,
|
||||
&candidate.admission_plan,
|
||||
context.trace_id.as_str(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -11,7 +11,8 @@ use aether_contracts::ExecutionPlan;
|
||||
|
||||
use crate::execution_runtime::acquire_upstream_execution_gate;
|
||||
use crate::provider_pool_demand::{
|
||||
acquire_provider_pool_execution_guard, ProviderPoolInFlightAdmission, ProviderPoolInFlightGuard,
|
||||
acquire_provider_pool_execution_guard, acquire_provider_pool_execution_guard_unobserved,
|
||||
ProviderPoolInFlightAdmission, ProviderPoolInFlightGuard,
|
||||
};
|
||||
use crate::upstream_admission::UpstreamTargetAdmissionPermit;
|
||||
use crate::{AppState, GatewayError};
|
||||
@@ -28,6 +29,7 @@ impl ResponsesWebSocketTurnAdmission {
|
||||
state: &AppState,
|
||||
plan: &ExecutionPlan,
|
||||
trace_id: &str,
|
||||
observation_context: Option<&serde_json::Value>,
|
||||
) -> Result<Self, GatewayError> {
|
||||
let upstream_execution = acquire_upstream_execution_gate(state, trace_id).await?;
|
||||
let upstream_target = match state
|
||||
@@ -41,7 +43,13 @@ impl ResponsesWebSocketTurnAdmission {
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let provider_pool = match acquire_provider_pool_execution_guard(state, plan).await? {
|
||||
let provider_admission = match observation_context {
|
||||
Some(context) => {
|
||||
acquire_provider_pool_execution_guard(state, plan, Some(context)).await?
|
||||
}
|
||||
None => acquire_provider_pool_execution_guard_unobserved(state, plan).await?,
|
||||
};
|
||||
let provider_pool = match provider_admission {
|
||||
ProviderPoolInFlightAdmission::Acquired(guard) => guard,
|
||||
ProviderPoolInFlightAdmission::Saturated { limit } => {
|
||||
drop(upstream_target);
|
||||
|
||||
@@ -32,6 +32,8 @@ pub(super) struct ResponsesWebSocketTurnControl {
|
||||
pub(super) decision: GatewayControlDecision,
|
||||
pub(super) auth_snapshot: Option<GatewayAuthApiKeySnapshot>,
|
||||
pub(super) rpm_bypassed: bool,
|
||||
// Shared across transparent retries and owned by LogicalTurn, not the socket.
|
||||
pub(super) activity: std::sync::Arc<crate::request_activity::RequestActivityGuard>,
|
||||
}
|
||||
|
||||
pub(super) async fn resolve_responses_websocket_turn_control(
|
||||
@@ -124,6 +126,7 @@ pub(super) async fn resolve_responses_websocket_turn_control(
|
||||
decision,
|
||||
auth_snapshot,
|
||||
rpm_bypassed,
|
||||
activity: std::sync::Arc::new(state.request_activity.begin()),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -606,6 +606,7 @@ pub(super) async fn begin_unowned_responses_websocket_turn(
|
||||
state,
|
||||
&plan,
|
||||
plan.request_id.as_str(),
|
||||
Some(report_context.as_ref().unwrap_or(&Value::Null)),
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -260,6 +260,52 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrency_activity_survives_retry_and_ends_with_the_logical_turn() {
|
||||
let activity = std::sync::Arc::new(crate::request_activity::RequestActivity::default());
|
||||
let first = logical().with_turn_control(super::ResponsesWebSocketTurnControl {
|
||||
decision: crate::control::GatewayControlDecision::synthetic(
|
||||
"/v1/responses",
|
||||
Some("ai_public".into()),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
),
|
||||
auth_snapshot: None,
|
||||
rpm_bypassed: false,
|
||||
activity: std::sync::Arc::new(activity.begin()),
|
||||
});
|
||||
let mut state = ResponsesTurnState::Idle;
|
||||
state.begin(first, FakeAttempt(1));
|
||||
assert_eq!(activity.active(), 1);
|
||||
assert_eq!(state.detach_attempt(), Some(FakeAttempt(1)));
|
||||
assert_eq!(
|
||||
activity.active(),
|
||||
1,
|
||||
"transparent retry retains one logical request"
|
||||
);
|
||||
state.resume(FakeAttempt(2)).unwrap();
|
||||
assert_eq!(activity.active(), 1);
|
||||
assert_eq!(state.end(), Some(FakeAttempt(2)));
|
||||
assert_eq!(activity.active(), 0);
|
||||
|
||||
let logical = logical().with_turn_control(super::ResponsesWebSocketTurnControl {
|
||||
decision: crate::control::GatewayControlDecision::synthetic(
|
||||
"/v1/responses",
|
||||
Some("ai_public".into()),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
),
|
||||
auth_snapshot: None,
|
||||
rpm_bypassed: false,
|
||||
activity: std::sync::Arc::new(activity.begin()),
|
||||
});
|
||||
state.begin(logical, FakeAttempt(3));
|
||||
drop(state);
|
||||
assert_eq!(activity.active(), 0, "disconnect drops the logical request");
|
||||
}
|
||||
|
||||
/// 透明重试失败之后:旧 attempt 已经被 detach 并结算过,logical turn 仍停在
|
||||
/// `Replanning`。此时 `end()` 不能再交出 attempt,否则同一个 attempt 会被
|
||||
/// 结算两次(两条 usage terminal、两次 pool lease 释放)。
|
||||
|
||||
@@ -166,6 +166,23 @@ async fn build_local_public_support_response(
|
||||
return None;
|
||||
}
|
||||
|
||||
if decision.route_family.as_deref() == Some("health_user") {
|
||||
if let Err(response) =
|
||||
resolve_authenticated_local_user(state, request_context, headers).await
|
||||
{
|
||||
return Some(response);
|
||||
}
|
||||
return Some(
|
||||
crate::handlers::shared::health_monitor::build_health_v2_response(
|
||||
state,
|
||||
&request_context.request_path,
|
||||
request_context.request_query_string.as_deref(),
|
||||
crate::handlers::shared::health_monitor::HealthAudience::Authenticated,
|
||||
)
|
||||
.await,
|
||||
);
|
||||
}
|
||||
|
||||
if decision.route_family.as_deref() == Some("auth") {
|
||||
return maybe_build_local_auth_response(
|
||||
state,
|
||||
@@ -306,6 +323,17 @@ async fn build_local_public_support_response(
|
||||
}
|
||||
|
||||
if decision.route_family.as_deref() == Some("public_catalog") {
|
||||
if decision.route_kind.as_deref() == Some("health_v2") {
|
||||
return Some(
|
||||
crate::handlers::shared::health_monitor::build_health_v2_response(
|
||||
state,
|
||||
&request_context.request_path,
|
||||
request_context.request_query_string.as_deref(),
|
||||
crate::handlers::shared::health_monitor::HealthAudience::Public,
|
||||
)
|
||||
.await,
|
||||
);
|
||||
}
|
||||
if decision.route_kind.as_deref() == Some("site_info")
|
||||
&& request_context.request_path == "/api/public/site-info"
|
||||
{
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use aether_data::repository::announcements::UserAnnouncementListQuery;
|
||||
use axum::{
|
||||
body::Body,
|
||||
http,
|
||||
@@ -33,6 +34,40 @@ fn parse_announcement_read_status_request(
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_user_announcements_query(
|
||||
raw: Option<&str>,
|
||||
now_unix_secs: u64,
|
||||
) -> Result<UserAnnouncementListQuery, String> {
|
||||
let mut query = UserAnnouncementListQuery {
|
||||
unread_only: false,
|
||||
offset: 0,
|
||||
limit: 20,
|
||||
now_unix_secs,
|
||||
};
|
||||
let mut seen = std::collections::BTreeSet::new();
|
||||
for (key, value) in url::form_urlencoded::parse(raw.unwrap_or_default().as_bytes()) {
|
||||
if !seen.insert(key.clone()) {
|
||||
return Err(format!("duplicate announcement query parameter: {key}"));
|
||||
}
|
||||
match key.as_ref() {
|
||||
"limit" => {
|
||||
query.limit = value.parse().map_err(|_| "invalid announcement limit")?;
|
||||
}
|
||||
"offset" => {
|
||||
query.offset = value.parse().map_err(|_| "invalid announcement offset")?;
|
||||
}
|
||||
"unread_only" => {
|
||||
query.unread_only = value
|
||||
.parse()
|
||||
.map_err(|_| "unread_only must be true or false")?;
|
||||
}
|
||||
_ => return Err(format!("unsupported announcement query parameter: {key}")),
|
||||
}
|
||||
}
|
||||
query.validate().map_err(|err| err.to_string())?;
|
||||
Ok(query)
|
||||
}
|
||||
|
||||
pub(crate) async fn maybe_build_local_announcement_user_response(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
@@ -54,6 +89,48 @@ pub(crate) async fn maybe_build_local_announcement_user_response(
|
||||
let now_unix_secs = chrono::Utc::now().timestamp().max(0) as u64;
|
||||
|
||||
match decision.route_kind.as_deref() {
|
||||
Some("list")
|
||||
if request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.request_path.as_str(),
|
||||
"/api/announcements/users/me" | "/api/announcements/users/me/"
|
||||
) =>
|
||||
{
|
||||
let query = match parse_user_announcements_query(
|
||||
request_context.request_query_string.as_deref(),
|
||||
now_unix_secs,
|
||||
) {
|
||||
Ok(query) => query,
|
||||
Err(detail) => return Some(announcements_bad_request_response(detail)),
|
||||
};
|
||||
let page = match state.list_user_announcements(&auth.user.id, &query).await {
|
||||
Ok(page) => page,
|
||||
Err(err) => {
|
||||
return Some(announcements_internal_error_response(
|
||||
announcements_internal_detail(err),
|
||||
))
|
||||
}
|
||||
};
|
||||
let items = page
|
||||
.items
|
||||
.iter()
|
||||
.map(|item| {
|
||||
let mut value = build_public_announcement_payload(&item.announcement);
|
||||
value["is_read"] = json!(item.is_read);
|
||||
value
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
Some(
|
||||
Json(json!({
|
||||
"items": items,
|
||||
"total": page.total,
|
||||
"unread_count": page.unread_count,
|
||||
"limit": query.limit,
|
||||
"offset": query.offset,
|
||||
}))
|
||||
.into_response(),
|
||||
)
|
||||
}
|
||||
Some("unread_count")
|
||||
if request_context.request_method == http::Method::GET
|
||||
&& matches!(
|
||||
|
||||
@@ -1209,6 +1209,7 @@ pub(super) async fn handle_users_me_usage_get(
|
||||
limit: None,
|
||||
offset: None,
|
||||
newest_first: true,
|
||||
..Default::default()
|
||||
};
|
||||
total_record_count = match state
|
||||
.count_usage_audits_by_keyword_search(&keyword_query)
|
||||
@@ -1259,6 +1260,7 @@ pub(super) async fn handle_users_me_usage_get(
|
||||
limit: None,
|
||||
offset: None,
|
||||
newest_first: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
{
|
||||
@@ -1289,6 +1291,7 @@ pub(super) async fn handle_users_me_usage_get(
|
||||
limit: Some(limit),
|
||||
offset: Some(offset),
|
||||
newest_first: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
{
|
||||
@@ -1434,6 +1437,7 @@ pub(super) async fn handle_users_me_usage_active_get(
|
||||
limit: Some(50),
|
||||
offset: None,
|
||||
newest_first: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -0,0 +1,511 @@
|
||||
use super::policy::{overall_status, HealthPolicy, HealthRatio, HealthStatus};
|
||||
use super::publication::{HealthPublication, PUBLICATION_KEY};
|
||||
use crate::handlers::shared::unix_ms_to_rfc3339;
|
||||
use crate::{AppState, GatewayError};
|
||||
use aether_data_contracts::repository::global_models::AdminGlobalModelListQuery;
|
||||
use aether_data_contracts::repository::usage::{
|
||||
HealthObservationMetrics, HealthObservationObjectKind, HealthObservationQuery,
|
||||
};
|
||||
use axum::{body::Body, http::StatusCode, response::IntoResponse, response::Response, Json};
|
||||
use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine};
|
||||
use chrono::Utc;
|
||||
use serde::Serialize;
|
||||
use serde_json::json;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum HealthAudience {
|
||||
Admin,
|
||||
Authenticated,
|
||||
Public,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(super) struct HealthRequest {
|
||||
pub kind: HealthObservationObjectKind,
|
||||
pub from_unix_ms: u64,
|
||||
pub to_unix_ms: u64,
|
||||
pub limit: usize,
|
||||
pub offset: usize,
|
||||
}
|
||||
|
||||
impl HealthRequest {
|
||||
pub fn parse(query: Option<&str>, public: bool, now_ms: u64) -> Result<Self, &'static str> {
|
||||
let mut params = BTreeMap::new();
|
||||
for (key, value) in url::form_urlencoded::parse(query.unwrap_or_default().as_bytes()) {
|
||||
if !matches!(key.as_ref(), "kind" | "window" | "limit" | "offset") {
|
||||
return Err("Unsupported health query parameter");
|
||||
}
|
||||
if params
|
||||
.insert(key.into_owned(), value.into_owned())
|
||||
.is_some()
|
||||
{
|
||||
return Err("Duplicate health query parameter");
|
||||
}
|
||||
}
|
||||
let kind = match params
|
||||
.get("kind")
|
||||
.map(String::as_str)
|
||||
.unwrap_or("api_format")
|
||||
{
|
||||
"api_format" => HealthObservationObjectKind::ApiFormat,
|
||||
"model" => HealthObservationObjectKind::Model,
|
||||
"provider" if !public => HealthObservationObjectKind::Provider,
|
||||
_ => return Err("Unsupported health object kind"),
|
||||
};
|
||||
let hours = match params.get("window").map(String::as_str).unwrap_or("6h") {
|
||||
"1h" => 1,
|
||||
"6h" => 6,
|
||||
"24h" => 24,
|
||||
"72h" => 72,
|
||||
_ => return Err("Health window must be 1h, 6h, 24h or 72h"),
|
||||
};
|
||||
let parse_size = |name: &str, default: usize| -> Result<usize, &'static str> {
|
||||
params
|
||||
.get(name)
|
||||
.map(|value| value.parse().map_err(|_| "Invalid pagination value"))
|
||||
.unwrap_or(Ok(default))
|
||||
};
|
||||
let limit = parse_size("limit", 25)?;
|
||||
let offset = parse_size("offset", 0)?;
|
||||
if !(1..=100).contains(&limit) || offset > 10_000 {
|
||||
return Err("Health pagination exceeds the supported range");
|
||||
}
|
||||
Ok(Self {
|
||||
kind,
|
||||
from_unix_ms: now_ms.saturating_sub(hours * 3_600_000),
|
||||
to_unix_ms: now_ms,
|
||||
limit,
|
||||
offset,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(super) struct HealthCoverage {
|
||||
pub status: &'static str,
|
||||
pub sample_status: &'static str,
|
||||
pub classified_count: u64,
|
||||
pub unknown_failure_count: u64,
|
||||
pub excluded_count: u64,
|
||||
pub exclusion_policy: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(super) struct PublicHealthObject {
|
||||
pub id: String,
|
||||
pub kind: HealthObservationObjectKind,
|
||||
pub name: String,
|
||||
pub status: HealthStatus,
|
||||
pub request_count: u64,
|
||||
pub request_success: HealthRatio,
|
||||
pub service_availability: HealthRatio,
|
||||
pub coverage: HealthCoverage,
|
||||
pub average_latency_ms: Option<f64>,
|
||||
pub latency_sample_count: u64,
|
||||
pub last_request_at: Option<String>,
|
||||
pub timeline: Vec<PublicHealthBucket>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(super) struct PublicHealthBucket {
|
||||
pub from: String,
|
||||
pub to: String,
|
||||
pub status: HealthStatus,
|
||||
pub service_availability: HealthRatio,
|
||||
pub unknown_failure_count: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(super) struct AdminHealthObject {
|
||||
#[serde(flatten)]
|
||||
pub service: PublicHealthObject,
|
||||
pub source_value: String,
|
||||
pub attempts: AdminAttemptMetrics,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(super) struct AdminAttemptMetrics {
|
||||
pub succeeded_count: u64,
|
||||
pub failed_count: u64,
|
||||
pub in_progress_count: u64,
|
||||
pub cancelled_count: u64,
|
||||
pub success: HealthRatio,
|
||||
}
|
||||
|
||||
pub(super) fn public_projection(
|
||||
id: String,
|
||||
kind: HealthObservationObjectKind,
|
||||
name: String,
|
||||
metrics: &HealthObservationMetrics,
|
||||
policy: &HealthPolicy,
|
||||
) -> PublicHealthObject {
|
||||
let classified = metrics
|
||||
.service_succeeded_count
|
||||
.saturating_add(metrics.service_failed_count);
|
||||
PublicHealthObject {
|
||||
id,
|
||||
kind,
|
||||
name,
|
||||
status: policy.status(
|
||||
metrics.service_succeeded_count,
|
||||
metrics.service_failed_count,
|
||||
metrics.unknown_failure_count,
|
||||
),
|
||||
request_count: metrics.request_count,
|
||||
request_success: HealthRatio::new(
|
||||
metrics.succeeded_count,
|
||||
metrics
|
||||
.succeeded_count
|
||||
.saturating_add(metrics.failed_count)
|
||||
.saturating_add(metrics.cancelled_count),
|
||||
),
|
||||
service_availability: HealthRatio::new(metrics.service_succeeded_count, classified),
|
||||
coverage: HealthCoverage {
|
||||
status: if metrics.unknown_failure_count > 0 {
|
||||
"partial"
|
||||
} else {
|
||||
"complete"
|
||||
},
|
||||
sample_status: if classified == 0 {
|
||||
"empty"
|
||||
} else if classified < policy.minimum_samples {
|
||||
"insufficient"
|
||||
} else {
|
||||
"sufficient"
|
||||
},
|
||||
classified_count: classified,
|
||||
unknown_failure_count: metrics.unknown_failure_count,
|
||||
excluded_count: metrics.excluded_count,
|
||||
exclusion_policy: "client_cancelled_invalid_input_identity_or_quota_policy",
|
||||
},
|
||||
average_latency_ms: (metrics.latency_sample_count > 0)
|
||||
.then(|| metrics.latency_sum_ms as f64 / metrics.latency_sample_count as f64),
|
||||
latency_sample_count: metrics.latency_sample_count,
|
||||
last_request_at: metrics.last_request_at_unix_ms.and_then(unix_ms_to_rfc3339),
|
||||
timeline: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn response_error(status: StatusCode, detail: &str) -> Response<Body> {
|
||||
(status, Json(json!({ "detail": detail }))).into_response()
|
||||
}
|
||||
|
||||
async fn read_publication(state: &AppState) -> Result<HealthPublication, GatewayError> {
|
||||
let config = match state
|
||||
.read_system_config_json_value_strong(PUBLICATION_KEY)
|
||||
.await?
|
||||
{
|
||||
Some(value) => serde_json::from_value(value).map_err(|error| {
|
||||
GatewayError::Internal(format!("Invalid health publication configuration: {error}"))
|
||||
})?,
|
||||
None => HealthPublication::default(),
|
||||
};
|
||||
config
|
||||
.validate()
|
||||
.map_err(|error| GatewayError::Internal(error.to_string()))?;
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
pub(crate) async fn build_publication_response(
|
||||
state: &AppState,
|
||||
body: Option<&[u8]>,
|
||||
) -> Response<Body> {
|
||||
if let Some(body) = body {
|
||||
let config: HealthPublication = match serde_json::from_slice(body) {
|
||||
Ok(config) => config,
|
||||
Err(_) => {
|
||||
return response_error(
|
||||
StatusCode::BAD_REQUEST,
|
||||
"Invalid health publication configuration",
|
||||
)
|
||||
}
|
||||
};
|
||||
if let Err(error) = config.validate() {
|
||||
return response_error(StatusCode::BAD_REQUEST, error);
|
||||
}
|
||||
let value = match serde_json::to_value(&config) {
|
||||
Ok(value) => value,
|
||||
Err(_) => {
|
||||
return response_error(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"Could not encode health publication",
|
||||
)
|
||||
}
|
||||
};
|
||||
return match state
|
||||
.upsert_system_config_json_value(
|
||||
PUBLICATION_KEY,
|
||||
&value,
|
||||
Some("Public status object allowlist"),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => Json(config).into_response(),
|
||||
Err(_) => response_error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"Health publication is unavailable",
|
||||
),
|
||||
};
|
||||
}
|
||||
match read_publication(state).await {
|
||||
Ok(config) => Json(config).into_response(),
|
||||
Err(_) => response_error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"Health publication is unavailable",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn build_health_v2_response(
|
||||
state: &AppState,
|
||||
path: &str,
|
||||
query: Option<&str>,
|
||||
audience: HealthAudience,
|
||||
) -> Response<Body> {
|
||||
let now_ms = Utc::now().timestamp_millis().max(0) as u64;
|
||||
let request = match HealthRequest::parse(query, audience != HealthAudience::Admin, now_ms) {
|
||||
Ok(request) => request,
|
||||
Err(detail) => return response_error(StatusCode::BAD_REQUEST, detail),
|
||||
};
|
||||
let prefix = match audience {
|
||||
HealthAudience::Public => "/api/public/health/v2/",
|
||||
HealthAudience::Authenticated => "/api/users/me/health/v2/",
|
||||
HealthAudience::Admin => "/api/admin/endpoints/health/v2/",
|
||||
};
|
||||
let tail = path.strip_prefix(prefix).unwrap_or_default();
|
||||
if tail != "summary"
|
||||
&& tail != "objects"
|
||||
&& !tail
|
||||
.strip_prefix("objects/")
|
||||
.is_some_and(|id| !id.is_empty() && !id.contains('/'))
|
||||
{
|
||||
return response_error(StatusCode::NOT_FOUND, "Health resource not found");
|
||||
}
|
||||
match build_health_payload(state, &request, tail, audience, now_ms).await {
|
||||
Ok(Some(payload)) => Json(payload).into_response(),
|
||||
Ok(None) => response_error(
|
||||
StatusCode::NOT_FOUND,
|
||||
"Health object is not published or does not exist",
|
||||
),
|
||||
Err(error) => {
|
||||
tracing::warn!(error = %crate::error::redact_error_detail(&format!("{error:?}")), "health observation query failed");
|
||||
response_error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"Health observations are temporarily unavailable",
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn admin_objects(
|
||||
state: &AppState,
|
||||
kind: HealthObservationObjectKind,
|
||||
) -> Result<BTreeMap<String, String>, GatewayError> {
|
||||
let mut objects = BTreeMap::new();
|
||||
match kind {
|
||||
HealthObservationObjectKind::ApiFormat => {
|
||||
let providers = state.list_provider_catalog_providers(true).await?;
|
||||
let ids: Vec<_> = providers
|
||||
.iter()
|
||||
.map(|provider| provider.id.clone())
|
||||
.collect();
|
||||
for endpoint in state
|
||||
.list_provider_catalog_endpoints_by_provider_ids(&ids)
|
||||
.await?
|
||||
{
|
||||
if endpoint.is_active {
|
||||
objects.insert(endpoint.api_format.clone(), endpoint.api_format);
|
||||
}
|
||||
}
|
||||
}
|
||||
HealthObservationObjectKind::Provider => {
|
||||
for provider in state.list_provider_catalog_providers(false).await? {
|
||||
objects.insert(provider.id, provider.name);
|
||||
}
|
||||
}
|
||||
HealthObservationObjectKind::Model => {
|
||||
let mut offset = 0;
|
||||
loop {
|
||||
let page = state
|
||||
.list_admin_global_models(&AdminGlobalModelListQuery {
|
||||
offset,
|
||||
limit: 500,
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
for model in &page.items {
|
||||
objects.insert(model.name.clone(), model.display_name.clone());
|
||||
}
|
||||
offset += page.items.len();
|
||||
if offset >= page.total || page.items.is_empty() {
|
||||
break;
|
||||
}
|
||||
if offset >= 10_000 {
|
||||
return Err(GatewayError::Internal(
|
||||
"Health model catalog exceeds query budget".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(objects)
|
||||
}
|
||||
|
||||
async fn build_health_payload(
|
||||
state: &AppState,
|
||||
request: &HealthRequest,
|
||||
tail: &str,
|
||||
audience: HealthAudience,
|
||||
now_ms: u64,
|
||||
) -> Result<Option<serde_json::Value>, GatewayError> {
|
||||
let policy = HealthPolicy::default();
|
||||
let public = audience == HealthAudience::Public;
|
||||
let redact_internal = audience != HealthAudience::Admin;
|
||||
let publication = if public {
|
||||
Some(read_publication(state).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if publication.as_ref().is_some_and(|config| !config.enabled) {
|
||||
return Ok(None);
|
||||
}
|
||||
let published: BTreeMap<_, _> = publication
|
||||
.as_ref()
|
||||
.map(|config| {
|
||||
config
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|object| object.kind == request.kind)
|
||||
.map(|object| (object.value.clone(), object))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let mut names = if public {
|
||||
published
|
||||
.iter()
|
||||
.map(|(value, object)| (value.clone(), object.display_name.clone()))
|
||||
.collect()
|
||||
} else {
|
||||
admin_objects(state, request.kind).await?
|
||||
};
|
||||
let observation = if public && published.is_empty() {
|
||||
Default::default()
|
||||
} else {
|
||||
state
|
||||
.data
|
||||
.summarize_health_observations(&HealthObservationQuery {
|
||||
from_unix_ms: request.from_unix_ms,
|
||||
to_unix_ms: request.to_unix_ms,
|
||||
object_kind: request.kind,
|
||||
object_values: public.then(|| published.keys().cloned().collect()),
|
||||
segments: 24,
|
||||
})
|
||||
.await
|
||||
.map_err(|error| GatewayError::Internal(error.to_string()))?
|
||||
};
|
||||
let mut metrics_by_value = BTreeMap::new();
|
||||
let mut timeline_by_value = BTreeMap::new();
|
||||
for object in observation.objects {
|
||||
if !public || published.contains_key(&object.object_value) {
|
||||
names
|
||||
.entry(object.object_value.clone())
|
||||
.or_insert_with(|| object.object_value.clone());
|
||||
timeline_by_value.insert(object.object_value.clone(), object.timeline);
|
||||
metrics_by_value.insert(object.object_value, object.metrics);
|
||||
}
|
||||
}
|
||||
let mut objects = Vec::new();
|
||||
for (value, name) in names {
|
||||
let metrics = metrics_by_value.remove(&value).unwrap_or_default();
|
||||
let id = if public {
|
||||
published[&value].public_id.clone()
|
||||
} else {
|
||||
URL_SAFE_NO_PAD.encode(value.as_bytes())
|
||||
};
|
||||
let mut service = public_projection(id, request.kind, name, &metrics, &policy);
|
||||
service.timeline = timeline_by_value
|
||||
.remove(&value)
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|bucket| PublicHealthBucket {
|
||||
from: unix_ms_to_rfc3339(bucket.from_unix_ms).unwrap_or_default(),
|
||||
to: unix_ms_to_rfc3339(bucket.to_unix_ms).unwrap_or_default(),
|
||||
status: policy.status(
|
||||
bucket.metrics.service_succeeded_count,
|
||||
bucket.metrics.service_failed_count,
|
||||
bucket.metrics.unknown_failure_count,
|
||||
),
|
||||
service_availability: HealthRatio::new(
|
||||
bucket.metrics.service_succeeded_count,
|
||||
bucket
|
||||
.metrics
|
||||
.service_succeeded_count
|
||||
.saturating_add(bucket.metrics.service_failed_count),
|
||||
),
|
||||
unknown_failure_count: bucket.metrics.unknown_failure_count,
|
||||
})
|
||||
.collect();
|
||||
objects.push(AdminHealthObject {
|
||||
service,
|
||||
source_value: value,
|
||||
attempts: AdminAttemptMetrics {
|
||||
succeeded_count: metrics.attempt_succeeded_count,
|
||||
failed_count: metrics.attempt_failed_count,
|
||||
in_progress_count: metrics.attempt_in_progress_count,
|
||||
cancelled_count: metrics.attempt_cancelled_count,
|
||||
success: HealthRatio::new(
|
||||
metrics.attempt_succeeded_count,
|
||||
metrics
|
||||
.attempt_succeeded_count
|
||||
.saturating_add(metrics.attempt_failed_count),
|
||||
),
|
||||
},
|
||||
});
|
||||
}
|
||||
let status = overall_status(objects.iter().map(|object| object.service.status));
|
||||
let meta = json!({
|
||||
"schema_version": 2, "metric_version": policy.version,
|
||||
"scope": { "kind": match audience { HealthAudience::Public => "published", HealthAudience::Authenticated => "authenticated", HealthAudience::Admin => "installation" }, "object_kind": request.kind },
|
||||
"range": { "from": unix_ms_to_rfc3339(request.from_unix_ms), "to": unix_ms_to_rfc3339(request.to_unix_ms), "timezone": "UTC", "time_basis": "request_started_at" },
|
||||
"generated_at": unix_ms_to_rfc3339(now_ms),
|
||||
"data_through": observation.data_through_unix_ms.and_then(unix_ms_to_rfc3339),
|
||||
"freshness": if observation.data_through_unix_ms.is_some_and(|time| now_ms.saturating_sub(time) > 120_000) { "stale" } else if observation.data_through_unix_ms.is_some() { "current" } else { "unknown" },
|
||||
"policy": policy,
|
||||
});
|
||||
let data = if tail == "summary" {
|
||||
json!({ "status": status, "object_count": objects.len(),
|
||||
"healthy_count": objects.iter().filter(|object| object.service.status == HealthStatus::Healthy).count(),
|
||||
"degraded_count": objects.iter().filter(|object| object.service.status == HealthStatus::Degraded).count(),
|
||||
"unavailable_count": objects.iter().filter(|object| object.service.status == HealthStatus::Unavailable).count(),
|
||||
"unknown_count": objects.iter().filter(|object| object.service.status == HealthStatus::Unknown).count(),
|
||||
"requests": public_projection(String::new(), request.kind, String::new(), &observation.overall, &policy),
|
||||
})
|
||||
} else if let Some(id) = tail.strip_prefix("objects/") {
|
||||
let Some(object) = objects.iter().find(|object| object.service.id == id) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if redact_internal {
|
||||
serde_json::to_value(&object.service)
|
||||
} else {
|
||||
serde_json::to_value(object)
|
||||
}
|
||||
.map_err(|error| GatewayError::Internal(error.to_string()))?
|
||||
} else {
|
||||
let items: Vec<serde_json::Value> = objects
|
||||
.iter()
|
||||
.skip(request.offset)
|
||||
.take(request.limit)
|
||||
.map(|object| {
|
||||
if redact_internal {
|
||||
serde_json::to_value(&object.service)
|
||||
} else {
|
||||
serde_json::to_value(object)
|
||||
}
|
||||
})
|
||||
.collect::<Result<_, _>>()
|
||||
.map_err(|error| GatewayError::Internal(error.to_string()))?;
|
||||
json!({ "items": items, "total": objects.len(), "limit": request.limit, "offset": request.offset })
|
||||
};
|
||||
Ok(Some(json!({ "meta": meta, "data": data })))
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
mod api;
|
||||
mod policy;
|
||||
mod publication;
|
||||
|
||||
pub(crate) use api::{build_health_v2_response, build_publication_response, HealthAudience};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,95 @@
|
||||
use serde::Serialize;
|
||||
|
||||
pub(super) const POLICY_VERSION: &str = "service-health-v1";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(super) enum HealthStatus {
|
||||
Healthy,
|
||||
Degraded,
|
||||
Unavailable,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(super) struct HealthPolicy {
|
||||
pub version: &'static str,
|
||||
pub minimum_samples: u64,
|
||||
pub healthy_threshold: f64,
|
||||
pub degraded_threshold: f64,
|
||||
}
|
||||
|
||||
impl Default for HealthPolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
version: POLICY_VERSION,
|
||||
minimum_samples: 20,
|
||||
healthy_threshold: 0.99,
|
||||
degraded_threshold: 0.95,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(super) struct HealthRatio {
|
||||
pub numerator: u64,
|
||||
pub denominator: u64,
|
||||
pub value: Option<f64>,
|
||||
}
|
||||
|
||||
impl HealthRatio {
|
||||
pub fn new(numerator: u64, denominator: u64) -> Self {
|
||||
Self {
|
||||
numerator,
|
||||
denominator,
|
||||
value: (denominator > 0).then(|| numerator as f64 / denominator as f64),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl HealthPolicy {
|
||||
pub fn status(&self, successes: u64, failures: u64, unknown: u64) -> HealthStatus {
|
||||
let samples = successes.saturating_add(failures);
|
||||
if samples < self.minimum_samples {
|
||||
return HealthStatus::Unknown;
|
||||
}
|
||||
if unknown > 0 {
|
||||
let best_possible_rate =
|
||||
successes.saturating_add(unknown) as f64 / samples.saturating_add(unknown) as f64;
|
||||
return if best_possible_rate < self.degraded_threshold {
|
||||
HealthStatus::Unavailable
|
||||
} else {
|
||||
HealthStatus::Unknown
|
||||
};
|
||||
}
|
||||
let ratio = successes as f64 / samples as f64;
|
||||
if ratio >= self.healthy_threshold {
|
||||
HealthStatus::Healthy
|
||||
} else if ratio >= self.degraded_threshold {
|
||||
HealthStatus::Degraded
|
||||
} else {
|
||||
HealthStatus::Unavailable
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn overall_status(statuses: impl Iterator<Item = HealthStatus>) -> HealthStatus {
|
||||
let mut result = HealthStatus::Healthy;
|
||||
let mut count = 0;
|
||||
for status in statuses {
|
||||
count += 1;
|
||||
match status {
|
||||
HealthStatus::Unavailable => return HealthStatus::Unavailable,
|
||||
HealthStatus::Degraded => result = HealthStatus::Degraded,
|
||||
HealthStatus::Unknown if result == HealthStatus::Healthy => {
|
||||
result = HealthStatus::Unknown
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if count == 0 {
|
||||
HealthStatus::Unknown
|
||||
} else {
|
||||
result
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
use aether_data_contracts::repository::usage::HealthObservationObjectKind;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
pub(super) const PUBLICATION_KEY: &str = "health_publication_v1";
|
||||
|
||||
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(super) struct HealthPublication {
|
||||
pub enabled: bool,
|
||||
pub objects: Vec<PublishedHealthObject>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(super) struct PublishedHealthObject {
|
||||
pub public_id: String,
|
||||
pub kind: HealthObservationObjectKind,
|
||||
pub value: String,
|
||||
pub display_name: String,
|
||||
}
|
||||
|
||||
impl HealthPublication {
|
||||
pub fn validate(&self) -> Result<(), &'static str> {
|
||||
if self.objects.len() > 200 {
|
||||
return Err("At most 200 public health objects may be published");
|
||||
}
|
||||
let mut ids = BTreeSet::new();
|
||||
let mut values = BTreeSet::new();
|
||||
for object in &self.objects {
|
||||
if object.public_id.is_empty()
|
||||
|| object.public_id.len() > 80
|
||||
|| !object
|
||||
.public_id
|
||||
.bytes()
|
||||
.all(|c| c.is_ascii_alphanumeric() || c == b'-' || c == b'_')
|
||||
{
|
||||
return Err(
|
||||
"Public IDs must contain 1 to 80 letters, digits, hyphens or underscores",
|
||||
);
|
||||
}
|
||||
if object.kind == HealthObservationObjectKind::Provider {
|
||||
return Err("Internal providers cannot be published as public health objects");
|
||||
}
|
||||
if object.value.trim().is_empty()
|
||||
|| object.value.len() > 256
|
||||
|| object.display_name.trim().is_empty()
|
||||
|| object.display_name.len() > 120
|
||||
{
|
||||
return Err("Every public object requires a bounded source value and display name");
|
||||
}
|
||||
if !ids.insert(object.public_id.clone())
|
||||
|| !values.insert(format!("{:?}:{}", object.kind, object.value))
|
||||
{
|
||||
return Err("Public IDs and health source objects must be unique");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
use super::api::{public_projection, AdminAttemptMetrics, AdminHealthObject, HealthRequest};
|
||||
use super::policy::{overall_status, HealthPolicy, HealthRatio, HealthStatus};
|
||||
use super::publication::HealthPublication;
|
||||
use aether_data_contracts::repository::usage::{
|
||||
HealthObservationMetrics, HealthObservationObjectKind,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn health_policy_distinguishes_unknown_and_insufficient_samples() {
|
||||
let policy = HealthPolicy::default();
|
||||
assert_eq!(policy.status(0, 0, 0), HealthStatus::Unknown);
|
||||
assert_eq!(policy.status(19, 0, 0), HealthStatus::Unknown);
|
||||
assert_eq!(policy.status(100, 0, 1), HealthStatus::Unknown);
|
||||
assert_eq!(policy.status(0, 100, 1), HealthStatus::Unavailable);
|
||||
assert_eq!(policy.status(99, 1, 0), HealthStatus::Healthy);
|
||||
assert_eq!(policy.status(96, 4, 0), HealthStatus::Degraded);
|
||||
assert_eq!(policy.status(90, 10, 0), HealthStatus::Unavailable);
|
||||
assert_eq!(HealthRatio::new(0, 0).value, None);
|
||||
assert_eq!(
|
||||
overall_status([HealthStatus::Healthy, HealthStatus::Unknown].into_iter()),
|
||||
HealthStatus::Unknown
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn health_query_rejects_ambiguous_and_internal_public_filters() {
|
||||
assert!(HealthRequest::parse(Some("provider_id=secret"), true, 30_000_000).is_err());
|
||||
assert!(HealthRequest::parse(Some("kind=provider"), true, 30_000_000).is_err());
|
||||
assert!(HealthRequest::parse(Some("window=6h&window=24h"), false, 30_000_000).is_err());
|
||||
assert!(HealthRequest::parse(Some("limit=101"), false, 30_000_000).is_err());
|
||||
let query = HealthRequest::parse(
|
||||
Some("kind=model&window=1h&limit=50&offset=25"),
|
||||
true,
|
||||
30_000_000,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(query.from_unix_ms, 26_400_000);
|
||||
assert_eq!(query.to_unix_ms, 30_000_000);
|
||||
assert_eq!(query.offset, 25);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_publication_is_explicit_and_rejects_provider_and_duplicate_sources() {
|
||||
let config: HealthPublication = serde_json::from_value(json!({"enabled": true, "objects": [
|
||||
{"public_id": "chat", "kind": "api_format", "value": "openai:chat", "display_name": "Chat"}
|
||||
]}))
|
||||
.unwrap();
|
||||
assert!(config.validate().is_ok());
|
||||
let mut duplicated = config.clone();
|
||||
duplicated.objects.push(duplicated.objects[0].clone());
|
||||
assert!(duplicated.validate().is_err());
|
||||
let mut provider = config;
|
||||
provider.objects[0].kind = HealthObservationObjectKind::Provider;
|
||||
assert!(provider.validate().is_err());
|
||||
assert!(serde_json::from_value::<HealthPublication>(
|
||||
json!({"enabled": true, "objects": [], "publish_all": true})
|
||||
)
|
||||
.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_dto_cannot_serialize_internal_source_or_attempts() {
|
||||
let metrics = HealthObservationMetrics {
|
||||
request_count: 105,
|
||||
succeeded_count: 100,
|
||||
failed_count: 4,
|
||||
cancelled_count: 1,
|
||||
service_succeeded_count: 100,
|
||||
service_failed_count: 2,
|
||||
excluded_count: 2,
|
||||
unknown_failure_count: 1,
|
||||
..Default::default()
|
||||
};
|
||||
let public = public_projection(
|
||||
"chat".into(),
|
||||
HealthObservationObjectKind::ApiFormat,
|
||||
"Chat".into(),
|
||||
&metrics,
|
||||
&HealthPolicy::default(),
|
||||
);
|
||||
assert_eq!(public.request_success.denominator, 105);
|
||||
assert_eq!(public.service_availability.denominator, 102);
|
||||
let admin = AdminHealthObject {
|
||||
service: public.clone(),
|
||||
source_value: "internal-provider-id".into(),
|
||||
attempts: AdminAttemptMetrics {
|
||||
succeeded_count: 100,
|
||||
failed_count: 20,
|
||||
in_progress_count: 1,
|
||||
cancelled_count: 1,
|
||||
success: HealthRatio::new(100, 120),
|
||||
},
|
||||
};
|
||||
let public_json = serde_json::to_value(public).unwrap();
|
||||
assert!(public_json.get("source_value").is_none());
|
||||
assert!(public_json.get("attempts").is_none());
|
||||
assert!(!public_json.to_string().contains("internal-provider-id"));
|
||||
assert!(serde_json::to_value(admin)
|
||||
.unwrap()
|
||||
.get("attempts")
|
||||
.is_some());
|
||||
}
|
||||
@@ -4,6 +4,7 @@ mod auth_api_key_secret;
|
||||
mod catalog;
|
||||
mod email_templates;
|
||||
mod external_models;
|
||||
pub(crate) mod health_monitor;
|
||||
mod identity_oauth_provider_secret;
|
||||
mod multipart;
|
||||
mod normalize;
|
||||
|
||||
@@ -232,6 +232,7 @@ pub(crate) fn admin_proxy_local_requires_buffered_body(
|
||||
decision.route_kind.as_deref(),
|
||||
) {
|
||||
(Some("endpoints_manage"), http::Method::POST, Some("create_provider_key"))
|
||||
| (Some("endpoints_health"), http::Method::PUT, Some("health_v2_publication"))
|
||||
| (Some("endpoints_manage"), http::Method::POST, Some("create_endpoint"))
|
||||
| (Some("endpoints_manage"), http::Method::POST, Some("batch_delete_keys"))
|
||||
| (Some("endpoints_manage"), http::Method::POST, Some("refresh_quota"))
|
||||
@@ -341,6 +342,7 @@ pub(crate) fn admin_proxy_local_requires_buffered_body(
|
||||
| (Some("billing_manage"), http::Method::PUT, Some("update_rule"))
|
||||
| (Some("billing_manage"), http::Method::POST, Some("create_collector"))
|
||||
| (Some("billing_manage"), http::Method::PUT, Some("update_collector"))
|
||||
| (Some("billing_manage"), http::Method::POST, Some("create_provider_expense"))
|
||||
| (Some("billing_manage"), http::Method::POST, Some("create_plan"))
|
||||
| (Some("billing_manage"), http::Method::PUT, Some("update_plan"))
|
||||
| (Some("billing_manage"), http::Method::PATCH, Some("set_plan_status"))
|
||||
|
||||
Reference in New Issue
Block a user