feat(stats): 添加 Provider 性能统计分析

This commit is contained in:
Entropy.Xu
2026-05-05 11:24:16 +08:00
parent 627eb4f335
commit ff1ed8b57d
17 changed files with 1683 additions and 22 deletions

View File

@@ -495,6 +495,19 @@ pub(super) fn classify_admin_observability_family_route(
"admin:stats", "admin:stats",
false, false,
)) ))
} else if method == http::Method::GET
&& matches!(
normalized_path,
"/api/admin/stats/performance/providers" | "/api/admin/stats/performance/providers/"
)
{
Some(classified(
"admin_proxy",
"stats_manage",
"provider_performance",
"admin:stats",
false,
))
} else if method == http::Method::GET } else if method == http::Method::GET
&& matches!( && matches!(
normalized_path, normalized_path,

View File

@@ -81,6 +81,25 @@ fn classifies_admin_stats_performance_percentiles_as_admin_proxy_route() {
assert!(!decision.is_execution_runtime_candidate()); assert!(!decision.is_execution_runtime_candidate());
} }
#[test]
fn classifies_admin_stats_provider_performance_as_admin_proxy_route() {
let headers = headers(&[]);
let uri: Uri = "/api/admin/stats/performance/providers"
.parse()
.expect("uri should parse");
let decision =
classify_control_route(&http::Method::GET, &uri, &headers).expect("route should classify");
assert_eq!(decision.route_class.as_deref(), Some("admin_proxy"));
assert_eq!(decision.route_family.as_deref(), Some("stats_manage"));
assert_eq!(decision.route_kind.as_deref(), Some("provider_performance"));
assert_eq!(
decision.auth_endpoint_signature.as_deref(),
Some("admin:stats")
);
assert!(!decision.is_execution_runtime_candidate());
}
#[test] #[test]
fn classifies_admin_stats_cost_forecast_as_admin_proxy_route() { fn classifies_admin_stats_cost_forecast_as_admin_proxy_route() {
let headers = headers(&[]); let headers = headers(&[]);

View File

@@ -1029,6 +1029,21 @@ impl GatewayDataState {
} }
} }
pub(crate) async fn summarize_usage_provider_performance(
&self,
query: &aether_data_contracts::repository::usage::UsageProviderPerformanceQuery,
) -> Result<
aether_data_contracts::repository::usage::StoredUsageProviderPerformance,
DataLayerError,
> {
match &self.usage_reader {
Some(repository) => repository.summarize_usage_provider_performance(query).await,
None => Ok(
aether_data_contracts::repository::usage::StoredUsageProviderPerformance::default(),
),
}
}
pub(crate) async fn summarize_usage_cost_savings( pub(crate) async fn summarize_usage_cost_savings(
&self, &self,
query: &aether_data_contracts::repository::usage::UsageCostSavingsSummaryQuery, query: &aether_data_contracts::repository::usage::UsageCostSavingsSummaryQuery,

View File

@@ -1,4 +1,4 @@
use super::range::build_comparison_range; use super::range::{build_comparison_range, parse_bounded_u32};
use super::resolve_admin_usage_time_range; use super::resolve_admin_usage_time_range;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext}; use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::query_param_value; use crate::handlers::admin::shared::query_param_value;
@@ -6,16 +6,18 @@ use crate::GatewayError;
use aether_admin::observability::stats::{ use aether_admin::observability::stats::{
admin_stats_bad_request_response, admin_stats_comparison_empty_response, admin_stats_bad_request_response, admin_stats_comparison_empty_response,
admin_stats_error_distribution_empty_response, admin_stats_error_distribution_empty_response,
admin_stats_performance_percentiles_empty_response, admin_stats_time_series_empty_response, admin_stats_performance_percentiles_empty_response,
admin_stats_provider_performance_empty_response, admin_stats_time_series_empty_response,
build_admin_stats_comparison_response_from_aggregates, build_admin_stats_comparison_response_from_aggregates,
build_admin_stats_error_distribution_response_from_summaries, build_admin_stats_error_distribution_response_from_summaries,
build_admin_stats_performance_percentiles_response_from_summaries, build_admin_stats_performance_percentiles_response_from_summaries,
build_admin_stats_provider_performance_response,
build_admin_stats_time_series_response_from_summaries, AdminStatsAggregate, build_admin_stats_time_series_response_from_summaries, AdminStatsAggregate,
AdminStatsComparisonType, AdminStatsGranularity, AdminStatsTimeRange, AdminStatsUsageFilter, AdminStatsComparisonType, AdminStatsGranularity, AdminStatsTimeRange, AdminStatsUsageFilter,
}; };
use aether_data_contracts::repository::usage::{ use aether_data_contracts::repository::usage::{
UsageAuditSummaryQuery, UsageErrorDistributionQuery, UsagePerformancePercentilesQuery, UsageAuditSummaryQuery, UsageErrorDistributionQuery, UsagePerformancePercentilesQuery,
UsageTimeSeriesGranularity, UsageTimeSeriesQuery, UsageProviderPerformanceQuery, UsageTimeSeriesGranularity, UsageTimeSeriesQuery,
}; };
use axum::{body::Body, http, response::Response}; use axum::{body::Body, http, response::Response};
@@ -173,6 +175,56 @@ pub(super) async fn maybe_build_local_admin_stats_analytics_response(
)); ));
} }
if request_context.route_kind() == Some("provider_performance")
&& request_context.method() == http::Method::GET
&& matches!(
request_context.path(),
"/api/admin/stats/performance/providers" | "/api/admin/stats/performance/providers/"
)
{
let time_range = match resolve_admin_usage_time_range(request_context.query_string()) {
Ok(value) => value,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
let granularity =
match query_param_value(request_context.query_string(), "granularity").as_deref() {
None | Some("day") => UsageTimeSeriesGranularity::Day,
Some("hour") => UsageTimeSeriesGranularity::Hour,
Some(_) => {
return Ok(Some(admin_stats_bad_request_response(
"granularity must be one of: day, hour".to_string(),
)));
}
};
let limit = match query_param_value(request_context.query_string(), "limit")
.map(|value| parse_bounded_u32("limit", &value, 1, 20))
.transpose()
{
Ok(value) => value.unwrap_or(8) as usize,
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
};
if !state.has_usage_data_reader() {
return Ok(Some(admin_stats_provider_performance_empty_response()));
}
let Some((created_from_unix_secs, created_until_unix_secs)) = time_range.to_unix_bounds()
else {
return Ok(Some(admin_stats_provider_performance_empty_response()));
};
let performance = state
.summarize_usage_provider_performance(&UsageProviderPerformanceQuery {
created_from_unix_secs,
created_until_unix_secs,
granularity,
tz_offset_minutes: time_range.tz_offset_minutes,
limit,
})
.await?;
return Ok(Some(build_admin_stats_provider_performance_response(
&performance,
)));
}
if request_context.route_kind() == Some("time_series") if request_context.route_kind() == Some("time_series")
&& request_context.method() == http::Method::GET && request_context.method() == http::Method::GET
&& matches!( && matches!(

View File

@@ -168,6 +168,16 @@ impl<'a> AdminAppState<'a> {
.await .await
} }
pub(crate) async fn summarize_usage_provider_performance(
&self,
query: &aether_data_contracts::repository::usage::UsageProviderPerformanceQuery,
) -> Result<
aether_data_contracts::repository::usage::StoredUsageProviderPerformance,
GatewayError,
> {
self.app.summarize_usage_provider_performance(query).await
}
pub(crate) async fn summarize_usage_cost_savings( pub(crate) async fn summarize_usage_cost_savings(
&self, &self,
query: &aether_data_contracts::repository::usage::UsageCostSavingsSummaryQuery, query: &aether_data_contracts::repository::usage::UsageCostSavingsSummaryQuery,

View File

@@ -251,6 +251,16 @@ impl AppState {
.map_err(|err| GatewayError::Internal(err.to_string())) .map_err(|err| GatewayError::Internal(err.to_string()))
} }
pub(crate) async fn summarize_usage_provider_performance(
&self,
query: &usage::UsageProviderPerformanceQuery,
) -> Result<usage::StoredUsageProviderPerformance, GatewayError> {
self.data
.summarize_usage_provider_performance(query)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn summarize_usage_cost_savings( pub(crate) async fn summarize_usage_cost_savings(
&self, &self,
query: &usage::UsageCostSavingsSummaryQuery, query: &usage::UsageCostSavingsSummaryQuery,

View File

@@ -853,6 +853,187 @@ async fn gateway_handles_admin_stats_performance_percentiles_locally_without_usa
upstream_handle.abort(); upstream_handle.abort();
} }
#[tokio::test]
async fn gateway_handles_admin_stats_provider_performance_locally_with_trusted_admin_principal() {
let (_upstream_url, upstream_hits, upstream_handle) =
start_stats_upstream("/api/admin/stats/performance/providers").await;
let mut usage = (1..=10)
.map(|index| {
let mut row = sample_usage_row(
&format!("usage-provider-perf-a-{index}"),
&format!("req-provider-perf-a-{index}"),
Some("user-1"),
Some("key-1"),
Some("primary"),
"OpenAI",
"gpt-5",
10,
10,
0.01,
0.01,
DAY_1_UNIX_SECS + i64::from(index),
);
row.response_time_ms = Some((index * 100) as u64);
row.first_byte_time_ms = Some((index * 10) as u64);
row
})
.collect::<Vec<_>>();
let mut failed = sample_usage_row(
"usage-provider-perf-failed",
"req-provider-perf-failed",
Some("user-1"),
Some("key-1"),
Some("primary"),
"OpenAI",
"gpt-5",
10,
99,
0.01,
0.01,
DAY_1_UNIX_SECS + 20,
);
failed.status = "failed".to_string();
failed.status_code = Some(500);
failed.error_message = Some("upstream failed".to_string());
usage.push(failed);
let mut provider_b = sample_usage_row(
"usage-provider-perf-b",
"req-provider-perf-b",
Some("user-1"),
Some("key-1"),
Some("primary"),
"Anthropic",
"claude-sonnet",
10,
20,
0.01,
0.01,
DAY_1_UNIX_SECS + 30,
);
provider_b.provider_id = Some("provider-2".to_string());
provider_b.response_time_ms = Some(1000);
provider_b.first_byte_time_ms = None;
usage.push(provider_b);
let mut unknown_provider = sample_usage_row(
"usage-provider-perf-unknown",
"req-provider-perf-unknown",
Some("user-1"),
Some("key-1"),
Some("primary"),
"unknown",
"gpt-5",
10,
999,
0.01,
0.01,
DAY_1_UNIX_SECS + 40,
);
unknown_provider.provider_id = None;
usage.push(unknown_provider);
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(usage));
let gateway = build_router_with_state(
AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(GatewayDataState::with_usage_reader_for_tests(
usage_repository,
)),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = admin_request(reqwest::Client::new().get(format!(
"{gateway_url}/api/admin/stats/performance/providers?start_date=2024-03-21&end_date=2024-03-21&granularity=hour&limit=2&tz_offset_minutes=0"
)))
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = response.json().await.expect("json body should parse");
assert_eq!(payload["summary"]["request_count"], 12);
assert_eq!(payload["summary"]["success_rate"], 91.67);
assert_eq!(payload["summary"]["avg_output_tps"], 18.46);
assert_eq!(payload["summary"]["avg_first_byte_time_ms"], 55.0);
assert_eq!(payload["summary"]["avg_response_time_ms"], 590.91);
assert_eq!(payload["providers"].as_array().map(Vec::len), Some(2));
assert_eq!(payload["providers"][0]["provider_id"], "provider-1");
assert_eq!(payload["providers"][0]["provider"], "OpenAI");
assert_eq!(payload["providers"][0]["request_count"], 11);
assert_eq!(payload["providers"][0]["success_count"], 10);
assert_eq!(payload["providers"][0]["error_count"], 1);
assert_eq!(payload["providers"][0]["success_rate"], 90.91);
assert_eq!(payload["providers"][0]["output_tokens"], 199);
assert_eq!(payload["providers"][0]["avg_output_tps"], 18.18);
assert_eq!(payload["providers"][0]["avg_first_byte_time_ms"], 55.0);
assert_eq!(payload["providers"][0]["avg_response_time_ms"], 550.0);
assert_eq!(payload["providers"][0]["p90_response_time_ms"], 910);
assert_eq!(payload["providers"][0]["p90_first_byte_time_ms"], 91);
assert_eq!(payload["providers"][0]["tps_sample_count"], 10);
assert_eq!(payload["providers"][0]["first_byte_sample_count"], 10);
assert_eq!(payload["providers"][1]["provider_id"], "provider-2");
assert_eq!(payload["providers"][1]["provider"], "Anthropic");
assert_eq!(payload["providers"][1]["avg_output_tps"], 20.0);
assert_eq!(
payload["providers"][1]["avg_first_byte_time_ms"],
serde_json::Value::Null
);
assert_eq!(
payload["providers"][1]["p90_response_time_ms"],
serde_json::Value::Null
);
assert_eq!(payload["timeline"].as_array().map(Vec::len), Some(2));
assert_eq!(payload["timeline"][0]["date"], "2024-03-21T05:00:00+00:00");
assert_eq!(payload["timeline"][0]["provider_id"], "provider-1");
assert_eq!(payload["timeline"][0]["avg_output_tps"], 18.18);
assert_eq!(payload["timeline"][0]["success_rate"], 90.91);
assert_eq!(payload["timeline"][1]["provider_id"], "provider-2");
assert_eq!(
payload["timeline"][1]["avg_first_byte_time_ms"],
serde_json::Value::Null
);
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_returns_empty_admin_stats_provider_performance_without_usage_reader() {
let (_upstream_url, upstream_hits, upstream_handle) =
start_stats_upstream("/api/admin/stats/performance/providers").await;
let gateway = build_router_with_state(AppState::new().expect("gateway should build"));
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = admin_request(reqwest::Client::new().get(format!(
"{gateway_url}/api/admin/stats/performance/providers?start_date=2024-03-21&end_date=2024-03-21&limit=2"
)))
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = response.json().await.expect("json body should parse");
assert_eq!(payload["summary"]["request_count"], 0);
assert_eq!(payload["summary"]["success_rate"], 0.0);
assert_eq!(
payload["summary"]["avg_output_tps"],
serde_json::Value::Null
);
assert_eq!(payload["providers"].as_array().map(Vec::len), Some(0));
assert_eq!(payload["timeline"].as_array().map(Vec::len), Some(0));
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test] #[tokio::test]
async fn gateway_handles_admin_stats_time_series_locally_with_trusted_admin_principal() { async fn gateway_handles_admin_stats_time_series_locally_with_trusted_admin_principal() {
let (upstream_url, upstream_hits, upstream_handle) = let (upstream_url, upstream_hits, upstream_handle) =

View File

@@ -4,7 +4,7 @@ use aether_data_contracts::repository::{
usage::{ usage::{
StoredRequestUsageAudit, StoredUsageCostSavingsSummary, StoredUsageErrorDistributionRow, StoredRequestUsageAudit, StoredUsageCostSavingsSummary, StoredUsageErrorDistributionRow,
StoredUsageLeaderboardSummary, StoredUsagePerformancePercentilesRow, StoredUsageLeaderboardSummary, StoredUsagePerformancePercentilesRow,
StoredUsageTimeSeriesBucket, StoredUsageProviderPerformance, StoredUsageTimeSeriesBucket,
}, },
}; };
use axum::{ use axum::{
@@ -584,6 +584,20 @@ pub fn round_to(value: f64, decimals: u32) -> f64 {
(value * factor).round() / factor (value * factor).round() / factor
} }
fn rounded_option(value: Option<f64>, decimals: u32) -> serde_json::Value {
value
.map(|value| json!(round_to(value, decimals)))
.unwrap_or(serde_json::Value::Null)
}
fn success_rate(request_count: u64, success_count: u64) -> f64 {
if request_count == 0 {
0.0
} else {
round_to(success_count as f64 / request_count as f64 * 100.0, 2)
}
}
pub fn admin_stats_provider_quota_usage_empty_response() -> Response<Body> { pub fn admin_stats_provider_quota_usage_empty_response() -> Response<Body> {
Json(json!({ Json(json!({
"providers": [], "providers": [],
@@ -652,6 +666,21 @@ pub fn admin_stats_performance_percentiles_empty_response() -> Response<Body> {
Json(json!([])).into_response() Json(json!([])).into_response()
} }
pub fn admin_stats_provider_performance_empty_response() -> Response<Body> {
Json(json!({
"summary": {
"request_count": 0,
"success_rate": 0.0,
"avg_output_tps": serde_json::Value::Null,
"avg_first_byte_time_ms": serde_json::Value::Null,
"avg_response_time_ms": serde_json::Value::Null,
},
"providers": [],
"timeline": [],
}))
.into_response()
}
pub fn admin_stats_cost_savings_empty_response() -> Response<Body> { pub fn admin_stats_cost_savings_empty_response() -> Response<Body> {
Json(json!({ Json(json!({
"cache_read_tokens": 0, "cache_read_tokens": 0,
@@ -1061,6 +1090,64 @@ pub fn build_admin_stats_performance_percentiles_response_from_summaries(
Json(serde_json::Value::Array(payload)).into_response() Json(serde_json::Value::Array(payload)).into_response()
} }
pub fn build_admin_stats_provider_performance_response(
performance: &StoredUsageProviderPerformance,
) -> Response<Body> {
let summary = &performance.summary;
let providers = performance
.providers
.iter()
.map(|row| {
json!({
"provider_id": row.provider_id.as_str(),
"provider": row.provider.as_str(),
"request_count": row.request_count,
"success_count": row.success_count,
"error_count": row.request_count.saturating_sub(row.success_count),
"success_rate": success_rate(row.request_count, row.success_count),
"output_tokens": row.output_tokens,
"avg_output_tps": rounded_option(row.avg_output_tps, 2),
"avg_first_byte_time_ms": rounded_option(row.avg_first_byte_time_ms, 2),
"avg_response_time_ms": rounded_option(row.avg_response_time_ms, 2),
"p90_response_time_ms": row.p90_response_time_ms,
"p90_first_byte_time_ms": row.p90_first_byte_time_ms,
"tps_sample_count": row.tps_sample_count,
"first_byte_sample_count": row.first_byte_sample_count,
})
})
.collect::<Vec<_>>();
let timeline = performance
.timeline
.iter()
.map(|row| {
json!({
"date": row.date.as_str(),
"provider_id": row.provider_id.as_str(),
"provider": row.provider.as_str(),
"request_count": row.request_count,
"output_tokens": row.output_tokens,
"avg_output_tps": rounded_option(row.avg_output_tps, 2),
"avg_first_byte_time_ms": rounded_option(row.avg_first_byte_time_ms, 2),
"avg_response_time_ms": rounded_option(row.avg_response_time_ms, 2),
"success_rate": success_rate(row.request_count, row.success_count),
})
})
.collect::<Vec<_>>();
Json(json!({
"summary": {
"request_count": summary.request_count,
"success_rate": success_rate(summary.request_count, summary.success_count),
"avg_output_tps": rounded_option(summary.avg_output_tps, 2),
"avg_first_byte_time_ms": rounded_option(summary.avg_first_byte_time_ms, 2),
"avg_response_time_ms": rounded_option(summary.avg_response_time_ms, 2),
},
"providers": providers,
"timeline": timeline,
}))
.into_response()
}
pub fn build_admin_stats_time_series_response( pub fn build_admin_stats_time_series_response(
time_range: &AdminStatsTimeRange, time_range: &AdminStatsTimeRange,
granularity: AdminStatsGranularity, granularity: AdminStatsGranularity,

View File

@@ -8,7 +8,9 @@ pub use types::{
StoredUsageCacheHitSummary, StoredUsageCostSavingsSummary, StoredUsageDailySummary, StoredUsageCacheHitSummary, StoredUsageCostSavingsSummary, StoredUsageDailySummary,
StoredUsageDashboardDailyBreakdownRow, StoredUsageDashboardProviderCount, StoredUsageDashboardDailyBreakdownRow, StoredUsageDashboardProviderCount,
StoredUsageDashboardSummary, StoredUsageErrorDistributionRow, StoredUsageLeaderboardSummary, StoredUsageDashboardSummary, StoredUsageErrorDistributionRow, StoredUsageLeaderboardSummary,
StoredUsagePerformancePercentilesRow, StoredUsageSettledCostSummary, StoredUsagePerformancePercentilesRow, StoredUsageProviderPerformance,
StoredUsageProviderPerformanceProviderRow, StoredUsageProviderPerformanceSummary,
StoredUsageProviderPerformanceTimelineRow, StoredUsageSettledCostSummary,
StoredUsageTimeSeriesBucket, StoredUsageUserTotals, UpsertUsageRecord, StoredUsageTimeSeriesBucket, StoredUsageUserTotals, UpsertUsageRecord,
UsageAuditAggregationGroupBy, UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery, UsageAuditAggregationGroupBy, UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery,
UsageAuditListQuery, UsageAuditSummaryQuery, UsageBodyCaptureResult, UsageBodyCaptureState, UsageAuditListQuery, UsageAuditSummaryQuery, UsageBodyCaptureResult, UsageBodyCaptureState,
@@ -18,7 +20,7 @@ pub use types::{
UsageDailyHeatmapQuery, UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery, UsageDailyHeatmapQuery, UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy, UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy,
UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery, UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery,
UsagePerformancePercentilesQuery, UsageReadRepository, UsageRepository, UsagePerformancePercentilesQuery, UsageProviderPerformanceQuery, UsageReadRepository,
UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity, UsageTimeSeriesQuery, UsageRepository, UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity,
UsageWriteRepository, UsageTimeSeriesQuery, UsageWriteRepository,
}; };

View File

@@ -949,6 +949,60 @@ pub struct StoredUsagePerformancePercentilesRow {
pub p99_first_byte_time_ms: Option<u64>, pub p99_first_byte_time_ms: Option<u64>,
} }
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct UsageProviderPerformanceQuery {
pub created_from_unix_secs: u64,
pub created_until_unix_secs: u64,
pub granularity: UsageTimeSeriesGranularity,
pub tz_offset_minutes: i32,
pub limit: usize,
}
#[derive(Debug, Clone, PartialEq, Default, serde::Serialize, serde::Deserialize)]
pub struct StoredUsageProviderPerformanceSummary {
pub request_count: u64,
pub success_count: u64,
pub avg_output_tps: Option<f64>,
pub avg_first_byte_time_ms: Option<f64>,
pub avg_response_time_ms: Option<f64>,
}
#[derive(Debug, Clone, PartialEq, Default, serde::Serialize, serde::Deserialize)]
pub struct StoredUsageProviderPerformanceProviderRow {
pub provider_id: String,
pub provider: String,
pub request_count: u64,
pub success_count: u64,
pub output_tokens: u64,
pub avg_output_tps: Option<f64>,
pub avg_first_byte_time_ms: Option<f64>,
pub avg_response_time_ms: Option<f64>,
pub p90_response_time_ms: Option<u64>,
pub p90_first_byte_time_ms: Option<u64>,
pub tps_sample_count: u64,
pub first_byte_sample_count: u64,
}
#[derive(Debug, Clone, PartialEq, Default, serde::Serialize, serde::Deserialize)]
pub struct StoredUsageProviderPerformanceTimelineRow {
pub date: String,
pub provider_id: String,
pub provider: String,
pub request_count: u64,
pub success_count: u64,
pub output_tokens: u64,
pub avg_output_tps: Option<f64>,
pub avg_first_byte_time_ms: Option<f64>,
pub avg_response_time_ms: Option<f64>,
}
#[derive(Debug, Clone, PartialEq, Default, serde::Serialize, serde::Deserialize)]
pub struct StoredUsageProviderPerformance {
pub summary: StoredUsageProviderPerformanceSummary,
pub providers: Vec<StoredUsageProviderPerformanceProviderRow>,
pub timeline: Vec<StoredUsageProviderPerformanceTimelineRow>,
}
#[derive(Debug, Clone, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)] #[derive(Debug, Clone, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
pub struct UsageCostSavingsSummaryQuery { pub struct UsageCostSavingsSummaryQuery {
pub created_from_unix_secs: u64, pub created_from_unix_secs: u64,
@@ -1364,6 +1418,11 @@ pub trait UsageReadRepository: Send + Sync {
query: &UsagePerformancePercentilesQuery, query: &UsagePerformancePercentilesQuery,
) -> Result<Vec<StoredUsagePerformancePercentilesRow>, crate::DataLayerError>; ) -> Result<Vec<StoredUsagePerformancePercentilesRow>, crate::DataLayerError>;
async fn summarize_usage_provider_performance(
&self,
query: &UsageProviderPerformanceQuery,
) -> Result<StoredUsageProviderPerformance, crate::DataLayerError>;
async fn summarize_usage_cost_savings( async fn summarize_usage_cost_savings(
&self, &self,
query: &UsageCostSavingsSummaryQuery, query: &UsageCostSavingsSummaryQuery,

View File

@@ -8,7 +8,9 @@ use aether_data_contracts::repository::usage::{
StoredUsageCacheAffinityIntervalRow, StoredUsageCacheHitSummary, StoredUsageCostSavingsSummary, StoredUsageCacheAffinityIntervalRow, StoredUsageCacheHitSummary, StoredUsageCostSavingsSummary,
StoredUsageDashboardDailyBreakdownRow, StoredUsageDashboardProviderCount, StoredUsageDashboardDailyBreakdownRow, StoredUsageDashboardProviderCount,
StoredUsageDashboardSummary, StoredUsageErrorDistributionRow, StoredUsageLeaderboardSummary, StoredUsageDashboardSummary, StoredUsageErrorDistributionRow, StoredUsageLeaderboardSummary,
StoredUsagePerformancePercentilesRow, StoredUsageSettledCostSummary, StoredUsagePerformancePercentilesRow, StoredUsageProviderPerformance,
StoredUsageProviderPerformanceProviderRow, StoredUsageProviderPerformanceSummary,
StoredUsageProviderPerformanceTimelineRow, StoredUsageSettledCostSummary,
StoredUsageTimeSeriesBucket, StoredUsageUserTotals, UsageAuditAggregationGroupBy, StoredUsageTimeSeriesBucket, StoredUsageUserTotals, UsageAuditAggregationGroupBy,
UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery, UsageAuditSummaryQuery, UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery, UsageAuditSummaryQuery,
UsageBodyField, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery, UsageBodyField, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery,
@@ -17,8 +19,8 @@ use aether_data_contracts::repository::usage::{
UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery, UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy, UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy,
UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery, UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery,
UsagePerformancePercentilesQuery, UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity, UsagePerformancePercentilesQuery, UsageProviderPerformanceQuery, UsageSettledCostSummaryQuery,
UsageTimeSeriesQuery, UsageTimeSeriesGranularity, UsageTimeSeriesQuery,
}; };
use async_trait::async_trait; use async_trait::async_trait;
use chrono::Utc; use chrono::Utc;
@@ -587,6 +589,38 @@ fn usage_matches_performance_percentiles_query(
&& item.status == "completed" && item.status == "completed"
} }
fn usage_provider_performance_identity(item: &StoredRequestUsageAudit) -> Option<(String, String)> {
let provider_id = item.provider_id.as_deref()?.trim();
let provider_id_status = provider_id.to_ascii_lowercase();
if provider_id.is_empty() || matches!(provider_id_status.as_str(), "unknown" | "pending") {
return None;
}
let provider_name = item.provider_name.trim();
let provider_name_status = provider_name.to_ascii_lowercase();
if matches!(provider_name_status.as_str(), "unknown" | "pending") {
return None;
}
let display_name = if provider_name.is_empty() {
provider_id
} else {
provider_name
};
Some((provider_id.to_string(), display_name.to_string()))
}
fn usage_matches_provider_performance_query(
item: &StoredRequestUsageAudit,
query: &UsageProviderPerformanceQuery,
) -> Option<(String, String)> {
if item.created_at_unix_ms < query.created_from_unix_secs
|| item.created_at_unix_ms >= query.created_until_unix_secs
|| matches!(item.status.as_str(), "pending" | "streaming")
{
return None;
}
usage_provider_performance_identity(item)
}
fn usage_matches_cost_savings_query( fn usage_matches_cost_savings_query(
item: &StoredRequestUsageAudit, item: &StoredRequestUsageAudit,
query: &UsageCostSavingsSummaryQuery, query: &UsageCostSavingsSummaryQuery,
@@ -1650,6 +1684,218 @@ impl UsageReadRepository for InMemoryUsageReadRepository {
.collect()) .collect())
} }
async fn summarize_usage_provider_performance(
&self,
query: &UsageProviderPerformanceQuery,
) -> Result<StoredUsageProviderPerformance, DataLayerError> {
#[derive(Default)]
struct ProviderPerformanceBucket {
provider: String,
request_count: u64,
success_count: u64,
output_tokens: u64,
tps_output_tokens: u64,
tps_response_time_ms_sum: u64,
tps_sample_count: u64,
first_byte_time_ms_sum: u64,
first_byte_sample_count: u64,
response_time_ms_sum: u64,
response_time_sample_count: u64,
response_times: Vec<u64>,
first_byte_times: Vec<u64>,
}
impl ProviderPerformanceBucket {
fn add(&mut self, item: &StoredRequestUsageAudit) {
self.request_count = self.request_count.saturating_add(1);
self.output_tokens = self.output_tokens.saturating_add(item.output_tokens);
if !usage_is_success(item) {
return;
}
self.success_count = self.success_count.saturating_add(1);
if let Some(response_time_ms) = item.response_time_ms {
self.response_time_ms_sum =
self.response_time_ms_sum.saturating_add(response_time_ms);
self.response_time_sample_count =
self.response_time_sample_count.saturating_add(1);
self.response_times.push(response_time_ms);
if response_time_ms > 0 && item.output_tokens > 0 {
self.tps_output_tokens =
self.tps_output_tokens.saturating_add(item.output_tokens);
self.tps_response_time_ms_sum = self
.tps_response_time_ms_sum
.saturating_add(response_time_ms);
self.tps_sample_count = self.tps_sample_count.saturating_add(1);
}
}
if let Some(first_byte_time_ms) = item.first_byte_time_ms {
self.first_byte_time_ms_sum = self
.first_byte_time_ms_sum
.saturating_add(first_byte_time_ms);
self.first_byte_sample_count = self.first_byte_sample_count.saturating_add(1);
self.first_byte_times.push(first_byte_time_ms);
}
}
}
fn avg(sum: u64, samples: u64) -> Option<f64> {
if samples == 0 {
None
} else {
Some(sum as f64 / samples as f64)
}
}
fn avg_tps(tokens: u64, response_time_ms_sum: u64) -> Option<f64> {
if response_time_ms_sum == 0 {
None
} else {
Some(tokens as f64 * 1000.0 / response_time_ms_sum as f64)
}
}
let usage = self
.by_request_id
.read()
.expect("usage repository lock")
.values()
.cloned()
.collect::<Vec<_>>();
let mut grouped = BTreeMap::<String, ProviderPerformanceBucket>::new();
let mut summary_bucket = ProviderPerformanceBucket::default();
for item in &usage {
let Some((provider_id, provider)) =
usage_matches_provider_performance_query(item, query)
else {
continue;
};
summary_bucket.add(item);
let bucket = grouped.entry(provider_id).or_default();
if bucket.provider.is_empty() {
bucket.provider = provider;
}
bucket.add(item);
}
let summary = StoredUsageProviderPerformanceSummary {
request_count: summary_bucket.request_count,
success_count: summary_bucket.success_count,
avg_output_tps: avg_tps(
summary_bucket.tps_output_tokens,
summary_bucket.tps_response_time_ms_sum,
),
avg_first_byte_time_ms: avg(
summary_bucket.first_byte_time_ms_sum,
summary_bucket.first_byte_sample_count,
),
avg_response_time_ms: avg(
summary_bucket.response_time_ms_sum,
summary_bucket.response_time_sample_count,
),
};
let mut providers = grouped
.into_iter()
.map(|(provider_id, mut bucket)| {
let p90_response_time_ms = usage_percentile_cont(&mut bucket.response_times, 0.9);
let p90_first_byte_time_ms =
usage_percentile_cont(&mut bucket.first_byte_times, 0.9);
StoredUsageProviderPerformanceProviderRow {
provider_id,
provider: bucket.provider,
request_count: bucket.request_count,
success_count: bucket.success_count,
output_tokens: bucket.output_tokens,
avg_output_tps: avg_tps(
bucket.tps_output_tokens,
bucket.tps_response_time_ms_sum,
),
avg_first_byte_time_ms: avg(
bucket.first_byte_time_ms_sum,
bucket.first_byte_sample_count,
),
avg_response_time_ms: avg(
bucket.response_time_ms_sum,
bucket.response_time_sample_count,
),
p90_response_time_ms,
p90_first_byte_time_ms,
tps_sample_count: bucket.tps_sample_count,
first_byte_sample_count: bucket.first_byte_sample_count,
}
})
.collect::<Vec<_>>();
providers.sort_by(|left, right| {
right
.request_count
.cmp(&left.request_count)
.then_with(|| left.provider_id.cmp(&right.provider_id))
});
providers.truncate(query.limit.max(1));
let top_provider_ids = providers
.iter()
.map(|row| row.provider_id.clone())
.collect::<Vec<_>>();
let mut timeline_grouped = BTreeMap::<(String, String), ProviderPerformanceBucket>::new();
for item in &usage {
let Some((provider_id, provider)) =
usage_matches_provider_performance_query(item, query)
else {
continue;
};
if !top_provider_ids.iter().any(|value| value == &provider_id) {
continue;
}
let Some(bucket_key) =
usage_time_series_bucket_key(item, query.granularity, query.tz_offset_minutes)
else {
continue;
};
let bucket = timeline_grouped
.entry((bucket_key, provider_id))
.or_default();
if bucket.provider.is_empty() {
bucket.provider = provider;
}
bucket.add(item);
}
let timeline = timeline_grouped
.into_iter()
.map(
|((date, provider_id), bucket)| StoredUsageProviderPerformanceTimelineRow {
date,
provider_id,
provider: bucket.provider,
request_count: bucket.request_count,
success_count: bucket.success_count,
output_tokens: bucket.output_tokens,
avg_output_tps: avg_tps(
bucket.tps_output_tokens,
bucket.tps_response_time_ms_sum,
),
avg_first_byte_time_ms: avg(
bucket.first_byte_time_ms_sum,
bucket.first_byte_sample_count,
),
avg_response_time_ms: avg(
bucket.response_time_ms_sum,
bucket.response_time_sample_count,
),
},
)
.collect();
Ok(StoredUsageProviderPerformance {
summary,
providers,
timeline,
})
}
async fn summarize_usage_cost_savings( async fn summarize_usage_cost_savings(
&self, &self,
query: &UsageCostSavingsSummaryQuery, query: &UsageCostSavingsSummaryQuery,
@@ -2579,7 +2825,9 @@ mod tests {
StoredProviderUsageWindow, StoredRequestUsageAudit, UpsertUsageRecord, UsageReadRepository, StoredProviderUsageWindow, StoredRequestUsageAudit, UpsertUsageRecord, UsageReadRepository,
UsageWriteRepository, UsageWriteRepository,
}; };
use aether_data_contracts::repository::usage::{usage_body_ref, UsageBodyField}; use aether_data_contracts::repository::usage::{
usage_body_ref, UsageBodyField, UsageProviderPerformanceQuery, UsageTimeSeriesGranularity,
};
use serde_json::json; use serde_json::json;
fn sample_usage(request_id: &str, created_at_unix_ms: i64) -> StoredRequestUsageAudit { fn sample_usage(request_id: &str, created_at_unix_ms: i64) -> StoredRequestUsageAudit {
@@ -4562,4 +4810,76 @@ mod tests {
assert_eq!(key.total_tokens, 300); assert_eq!(key.total_tokens, 300);
assert_eq!(key.total_cost_usd, 0.24); assert_eq!(key.total_cost_usd, 0.24);
} }
#[tokio::test]
async fn summarize_usage_provider_performance_computes_tps_and_top_provider_timeline() {
let mut first = sample_usage("req-provider-perf-1", 1_711_000_000);
first.output_tokens = 60;
first.response_time_ms = Some(3000);
first.first_byte_time_ms = Some(100);
let mut second = sample_usage("req-provider-perf-2", 1_711_000_300);
second.output_tokens = 40;
second.response_time_ms = Some(1000);
second.first_byte_time_ms = Some(200);
let mut failed = sample_usage("req-provider-perf-failed", 1_711_000_400);
failed.output_tokens = 999;
failed.response_time_ms = Some(10);
failed.first_byte_time_ms = Some(1);
failed.status = "failed".to_string();
failed.status_code = Some(500);
let mut other_provider = sample_usage("req-provider-perf-other", 1_711_003_600);
other_provider.provider_id = Some("provider-2".to_string());
other_provider.provider_name = "Anthropic".to_string();
other_provider.output_tokens = 30;
other_provider.response_time_ms = Some(3000);
other_provider.first_byte_time_ms = None;
let repository =
InMemoryUsageReadRepository::seed(vec![first, second, failed, other_provider]);
let summary = repository
.summarize_usage_provider_performance(&UsageProviderPerformanceQuery {
created_from_unix_secs: 1_711_000_000,
created_until_unix_secs: 1_711_010_000,
granularity: UsageTimeSeriesGranularity::Hour,
tz_offset_minutes: 0,
limit: 1,
})
.await
.expect("provider performance should summarize");
assert_eq!(summary.summary.request_count, 4);
assert_eq!(summary.summary.success_count, 3);
assert!((summary.summary.avg_output_tps.expect("summary tps") - 18.571_428).abs() < 0.001);
assert_eq!(summary.summary.avg_first_byte_time_ms, Some(150.0));
assert!(
(summary
.summary
.avg_response_time_ms
.expect("summary response")
- 2333.333)
.abs()
< 0.001
);
assert_eq!(summary.providers.len(), 1);
let provider = &summary.providers[0];
assert_eq!(provider.provider_id, "provider-1");
assert_eq!(provider.request_count, 3);
assert_eq!(provider.success_count, 2);
assert_eq!(provider.output_tokens, 1099);
assert_eq!(provider.avg_output_tps, Some(25.0));
assert_eq!(provider.avg_first_byte_time_ms, Some(150.0));
assert_eq!(provider.avg_response_time_ms, Some(2000.0));
assert_eq!(provider.p90_response_time_ms, None);
assert_eq!(provider.tps_sample_count, 2);
assert_eq!(provider.first_byte_sample_count, 2);
assert_eq!(summary.timeline.len(), 1);
assert_eq!(summary.timeline[0].date, "2024-03-21T05:00:00+00:00");
assert_eq!(summary.timeline[0].provider_id, "provider-1");
assert_eq!(summary.timeline[0].avg_output_tps, Some(25.0));
}
} }

View File

@@ -10,7 +10,9 @@ pub(crate) use aether_data_contracts::repository::usage::{
StoredUsageDailySummary, StoredUsageDashboardDailyBreakdownRow, StoredUsageDailySummary, StoredUsageDashboardDailyBreakdownRow,
StoredUsageDashboardProviderCount, StoredUsageDashboardSummary, StoredUsageDashboardProviderCount, StoredUsageDashboardSummary,
StoredUsageErrorDistributionRow, StoredUsageLeaderboardSummary, StoredUsageErrorDistributionRow, StoredUsageLeaderboardSummary,
StoredUsagePerformancePercentilesRow, StoredUsageSettledCostSummary, StoredUsagePerformancePercentilesRow, StoredUsageProviderPerformance,
StoredUsageProviderPerformanceProviderRow, StoredUsageProviderPerformanceSummary,
StoredUsageProviderPerformanceTimelineRow, StoredUsageSettledCostSummary,
StoredUsageTimeSeriesBucket, StoredUsageUserTotals, UpsertUsageRecord, StoredUsageTimeSeriesBucket, StoredUsageUserTotals, UpsertUsageRecord,
UsageAuditAggregationGroupBy, UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery, UsageAuditAggregationGroupBy, UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery,
UsageAuditListQuery, UsageAuditSummaryQuery, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery, UsageAuditListQuery, UsageAuditSummaryQuery, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery,
@@ -19,9 +21,9 @@ pub(crate) use aether_data_contracts::repository::usage::{
UsageDailyHeatmapQuery, UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery, UsageDailyHeatmapQuery, UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy, UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy,
UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery, UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery,
UsagePerformancePercentilesQuery, UsageReadRepository, UsageRepository, UsagePerformancePercentilesQuery, UsageProviderPerformanceQuery, UsageReadRepository,
UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity, UsageTimeSeriesQuery, UsageRepository, UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity,
UsageWriteRepository, UsageTimeSeriesQuery, UsageWriteRepository,
}; };
pub use memory::InMemoryUsageReadRepository; pub use memory::InMemoryUsageReadRepository;
pub use sql::SqlxUsageReadRepository; pub use sql::SqlxUsageReadRepository;

View File

@@ -4,7 +4,9 @@ use aether_data_contracts::repository::usage::{
StoredUsageCacheAffinityIntervalRow, StoredUsageCacheHitSummary, StoredUsageCostSavingsSummary, StoredUsageCacheAffinityIntervalRow, StoredUsageCacheHitSummary, StoredUsageCostSavingsSummary,
StoredUsageDashboardDailyBreakdownRow, StoredUsageDashboardProviderCount, StoredUsageDashboardDailyBreakdownRow, StoredUsageDashboardProviderCount,
StoredUsageDashboardSummary, StoredUsageErrorDistributionRow, StoredUsageLeaderboardSummary, StoredUsageDashboardSummary, StoredUsageErrorDistributionRow, StoredUsageLeaderboardSummary,
StoredUsagePerformancePercentilesRow, StoredUsageSettledCostSummary, StoredUsagePerformancePercentilesRow, StoredUsageProviderPerformance,
StoredUsageProviderPerformanceProviderRow, StoredUsageProviderPerformanceSummary,
StoredUsageProviderPerformanceTimelineRow, StoredUsageSettledCostSummary,
StoredUsageTimeSeriesBucket, StoredUsageUserTotals, UsageAuditAggregationGroupBy, StoredUsageTimeSeriesBucket, StoredUsageUserTotals, UsageAuditAggregationGroupBy,
UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery, UsageAuditSummaryQuery, UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery, UsageAuditSummaryQuery,
UsageBodyCaptureState, UsageBodyField, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery, UsageBodyCaptureState, UsageBodyField, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery,
@@ -13,8 +15,8 @@ use aether_data_contracts::repository::usage::{
UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery, UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy, UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy,
UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery, UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery,
UsagePerformancePercentilesQuery, UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity, UsagePerformancePercentilesQuery, UsageProviderPerformanceQuery, UsageSettledCostSummaryQuery,
UsageTimeSeriesQuery, UsageTimeSeriesGranularity, UsageTimeSeriesQuery,
}; };
use async_trait::async_trait; use async_trait::async_trait;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
@@ -803,6 +805,107 @@ fn decode_usage_performance_percentiles_row(
}) })
} }
fn decode_usage_provider_performance_summary(
row: &PgRow,
) -> Result<StoredUsageProviderPerformanceSummary, DataLayerError> {
Ok(StoredUsageProviderPerformanceSummary {
request_count: row
.try_get::<i64, _>("request_count")
.map_postgres_err()?
.max(0) as u64,
success_count: row
.try_get::<i64, _>("success_count")
.map_postgres_err()?
.max(0) as u64,
avg_output_tps: row
.try_get::<Option<f64>, _>("avg_output_tps")
.map_postgres_err()?,
avg_first_byte_time_ms: row
.try_get::<Option<f64>, _>("avg_first_byte_time_ms")
.map_postgres_err()?,
avg_response_time_ms: row
.try_get::<Option<f64>, _>("avg_response_time_ms")
.map_postgres_err()?,
})
}
fn decode_usage_provider_performance_provider_row(
row: &PgRow,
) -> Result<StoredUsageProviderPerformanceProviderRow, DataLayerError> {
Ok(StoredUsageProviderPerformanceProviderRow {
provider_id: row.try_get::<String, _>("provider_id").map_postgres_err()?,
provider: row.try_get::<String, _>("provider").map_postgres_err()?,
request_count: row
.try_get::<i64, _>("request_count")
.map_postgres_err()?
.max(0) as u64,
success_count: row
.try_get::<i64, _>("success_count")
.map_postgres_err()?
.max(0) as u64,
output_tokens: row
.try_get::<i64, _>("output_tokens")
.map_postgres_err()?
.max(0) as u64,
avg_output_tps: row
.try_get::<Option<f64>, _>("avg_output_tps")
.map_postgres_err()?,
avg_first_byte_time_ms: row
.try_get::<Option<f64>, _>("avg_first_byte_time_ms")
.map_postgres_err()?,
avg_response_time_ms: row
.try_get::<Option<f64>, _>("avg_response_time_ms")
.map_postgres_err()?,
p90_response_time_ms: row
.try_get::<Option<i64>, _>("p90_response_time_ms")
.map_postgres_err()?
.map(|value| value.max(0) as u64),
p90_first_byte_time_ms: row
.try_get::<Option<i64>, _>("p90_first_byte_time_ms")
.map_postgres_err()?
.map(|value| value.max(0) as u64),
tps_sample_count: row
.try_get::<i64, _>("tps_sample_count")
.map_postgres_err()?
.max(0) as u64,
first_byte_sample_count: row
.try_get::<i64, _>("first_byte_sample_count")
.map_postgres_err()?
.max(0) as u64,
})
}
fn decode_usage_provider_performance_timeline_row(
row: &PgRow,
) -> Result<StoredUsageProviderPerformanceTimelineRow, DataLayerError> {
Ok(StoredUsageProviderPerformanceTimelineRow {
date: row.try_get::<String, _>("date").map_postgres_err()?,
provider_id: row.try_get::<String, _>("provider_id").map_postgres_err()?,
provider: row.try_get::<String, _>("provider").map_postgres_err()?,
request_count: row
.try_get::<i64, _>("request_count")
.map_postgres_err()?
.max(0) as u64,
success_count: row
.try_get::<i64, _>("success_count")
.map_postgres_err()?
.max(0) as u64,
output_tokens: row
.try_get::<i64, _>("output_tokens")
.map_postgres_err()?
.max(0) as u64,
avg_output_tps: row
.try_get::<Option<f64>, _>("avg_output_tps")
.map_postgres_err()?,
avg_first_byte_time_ms: row
.try_get::<Option<f64>, _>("avg_first_byte_time_ms")
.map_postgres_err()?,
avg_response_time_ms: row
.try_get::<Option<f64>, _>("avg_response_time_ms")
.map_postgres_err()?,
})
}
fn decode_usage_time_series_bucket_row( fn decode_usage_time_series_bucket_row(
row: &PgRow, row: &PgRow,
) -> Result<StoredUsageTimeSeriesBucket, DataLayerError> { ) -> Result<StoredUsageTimeSeriesBucket, DataLayerError> {
@@ -4371,6 +4474,306 @@ ORDER BY date ASC
Ok(items) Ok(items)
} }
async fn summarize_usage_provider_performance_summary(
&self,
query: &UsageProviderPerformanceQuery,
) -> Result<StoredUsageProviderPerformanceSummary, DataLayerError> {
let mut builder = QueryBuilder::<Postgres>::new(
r#"
WITH filtered_usage AS (
SELECT
GREATEST(COALESCE("usage".output_tokens, 0), 0) AS output_tokens,
GREATEST(COALESCE("usage".response_time_ms, 0), 0) AS response_time_ms,
GREATEST(COALESCE("usage".first_byte_time_ms, 0), 0) AS first_byte_time_ms,
"usage".response_time_ms IS NOT NULL AS has_response_time,
"usage".first_byte_time_ms IS NOT NULL AS has_first_byte_time,
CASE
WHEN lower(COALESCE("usage".status, '')) IN ('completed', 'success', 'ok', 'billed', 'settled')
AND ("usage".status_code IS NULL OR "usage".status_code < 400)
THEN 1
ELSE 0
END AS success_flag
FROM usage_billing_facts AS "usage"
WHERE "usage".created_at >= TO_TIMESTAMP("#,
);
builder.push_bind(query.created_from_unix_secs as f64);
builder.push(
r#"::double precision)
AND "usage".created_at < TO_TIMESTAMP("#,
);
builder.push_bind(query.created_until_unix_secs as f64);
builder.push(
r#"::double precision)
AND COALESCE("usage".status, '') NOT IN ('pending', 'streaming')
AND NULLIF(BTRIM(COALESCE("usage".provider_id, '')), '') IS NOT NULL
AND lower(BTRIM(COALESCE("usage".provider_id, ''))) NOT IN ('unknown', 'pending')
AND lower(BTRIM(COALESCE("usage".provider_name, ''))) NOT IN ('unknown', 'pending')
)
SELECT
COUNT(*)::BIGINT AS request_count,
COALESCE(SUM(success_flag), 0)::BIGINT AS success_count,
CASE
WHEN COALESCE(SUM(CASE
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
THEN response_time_ms
ELSE 0
END), 0) > 0
THEN COALESCE(SUM(CASE
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
THEN output_tokens
ELSE 0
END), 0)::DOUBLE PRECISION * 1000.0 / COALESCE(SUM(CASE
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
THEN response_time_ms
ELSE 0
END), 0)::DOUBLE PRECISION
ELSE NULL
END AS avg_output_tps,
AVG(first_byte_time_ms::DOUBLE PRECISION)
FILTER (WHERE success_flag = 1 AND has_first_byte_time) AS avg_first_byte_time_ms,
AVG(response_time_ms::DOUBLE PRECISION)
FILTER (WHERE success_flag = 1 AND has_response_time) AS avg_response_time_ms
FROM filtered_usage
"#,
);
let row = builder
.build()
.fetch_one(&self.pool)
.await
.map_postgres_err()?;
decode_usage_provider_performance_summary(&row)
}
async fn summarize_usage_provider_performance_providers(
&self,
query: &UsageProviderPerformanceQuery,
) -> Result<Vec<StoredUsageProviderPerformanceProviderRow>, DataLayerError> {
let mut builder = QueryBuilder::<Postgres>::new(
r#"
WITH filtered_usage AS (
SELECT
COALESCE("usage".provider_id, '') AS provider_id,
COALESCE(NULLIF(BTRIM("usage".provider_name), ''), COALESCE("usage".provider_id, '')) AS provider,
GREATEST(COALESCE("usage".output_tokens, 0), 0) AS output_tokens,
GREATEST(COALESCE("usage".response_time_ms, 0), 0) AS response_time_ms,
GREATEST(COALESCE("usage".first_byte_time_ms, 0), 0) AS first_byte_time_ms,
"usage".response_time_ms IS NOT NULL AS has_response_time,
"usage".first_byte_time_ms IS NOT NULL AS has_first_byte_time,
CASE
WHEN lower(COALESCE("usage".status, '')) IN ('completed', 'success', 'ok', 'billed', 'settled')
AND ("usage".status_code IS NULL OR "usage".status_code < 400)
THEN 1
ELSE 0
END AS success_flag
FROM usage_billing_facts AS "usage"
WHERE "usage".created_at >= TO_TIMESTAMP("#,
);
builder.push_bind(query.created_from_unix_secs as f64);
builder.push(
r#"::double precision)
AND "usage".created_at < TO_TIMESTAMP("#,
);
builder.push_bind(query.created_until_unix_secs as f64);
builder.push(
r#"::double precision)
AND COALESCE("usage".status, '') NOT IN ('pending', 'streaming')
AND NULLIF(BTRIM(COALESCE("usage".provider_id, '')), '') IS NOT NULL
AND lower(BTRIM(COALESCE("usage".provider_id, ''))) NOT IN ('unknown', 'pending')
AND lower(BTRIM(COALESCE("usage".provider_name, ''))) NOT IN ('unknown', 'pending')
)
SELECT
provider_id,
COALESCE(MAX(NULLIF(provider, '')), provider_id) AS provider,
COUNT(*)::BIGINT AS request_count,
COALESCE(SUM(success_flag), 0)::BIGINT AS success_count,
COALESCE(SUM(output_tokens), 0)::BIGINT AS output_tokens,
CASE
WHEN COALESCE(SUM(CASE
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
THEN response_time_ms
ELSE 0
END), 0) > 0
THEN COALESCE(SUM(CASE
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
THEN output_tokens
ELSE 0
END), 0)::DOUBLE PRECISION * 1000.0 / COALESCE(SUM(CASE
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
THEN response_time_ms
ELSE 0
END), 0)::DOUBLE PRECISION
ELSE NULL
END AS avg_output_tps,
AVG(first_byte_time_ms::DOUBLE PRECISION)
FILTER (WHERE success_flag = 1 AND has_first_byte_time) AS avg_first_byte_time_ms,
AVG(response_time_ms::DOUBLE PRECISION)
FILTER (WHERE success_flag = 1 AND has_response_time) AS avg_response_time_ms,
CASE
WHEN COUNT(response_time_ms) FILTER (WHERE success_flag = 1 AND has_response_time) >= 10
THEN FLOOR(PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY response_time_ms)
FILTER (WHERE success_flag = 1 AND has_response_time))::BIGINT
ELSE NULL
END AS p90_response_time_ms,
CASE
WHEN COUNT(first_byte_time_ms) FILTER (WHERE success_flag = 1 AND has_first_byte_time) >= 10
THEN FLOOR(PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY first_byte_time_ms)
FILTER (WHERE success_flag = 1 AND has_first_byte_time))::BIGINT
ELSE NULL
END AS p90_first_byte_time_ms,
COALESCE(SUM(CASE
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
THEN 1
ELSE 0
END), 0)::BIGINT AS tps_sample_count,
(COUNT(first_byte_time_ms) FILTER (WHERE success_flag = 1 AND has_first_byte_time))::BIGINT
AS first_byte_sample_count
FROM filtered_usage
GROUP BY provider_id
ORDER BY request_count DESC, provider_id ASC
"#,
);
let mut rows = builder.build().fetch(&self.pool);
let mut items = Vec::new();
while let Some(row) = rows.try_next().await.map_postgres_err()? {
items.push(decode_usage_provider_performance_provider_row(&row)?);
}
Ok(items)
}
async fn summarize_usage_provider_performance_timeline(
&self,
query: &UsageProviderPerformanceQuery,
provider_ids: &[String],
) -> Result<Vec<StoredUsageProviderPerformanceTimelineRow>, DataLayerError> {
if provider_ids.is_empty() {
return Ok(Vec::new());
}
let mut builder = QueryBuilder::<Postgres>::new("WITH filtered_usage AS ( SELECT ");
match query.granularity {
UsageTimeSeriesGranularity::Day => {
builder
.push("TO_CHAR(date_trunc('day', \"usage\".created_at + (")
.push_bind(query.tz_offset_minutes)
.push("::integer * INTERVAL '1 minute')), 'YYYY-MM-DD') AS date");
}
UsageTimeSeriesGranularity::Hour => {
builder
.push("TO_CHAR(date_trunc('hour', \"usage\".created_at + (")
.push_bind(query.tz_offset_minutes)
.push("::integer * INTERVAL '1 minute')), 'YYYY-MM-DD\"T\"HH24:00:00+00:00') AS date");
}
}
builder.push(
r#",
COALESCE("usage".provider_id, '') AS provider_id,
COALESCE(NULLIF(BTRIM("usage".provider_name), ''), COALESCE("usage".provider_id, '')) AS provider,
GREATEST(COALESCE("usage".output_tokens, 0), 0) AS output_tokens,
GREATEST(COALESCE("usage".response_time_ms, 0), 0) AS response_time_ms,
GREATEST(COALESCE("usage".first_byte_time_ms, 0), 0) AS first_byte_time_ms,
"usage".response_time_ms IS NOT NULL AS has_response_time,
"usage".first_byte_time_ms IS NOT NULL AS has_first_byte_time,
CASE
WHEN lower(COALESCE("usage".status, '')) IN ('completed', 'success', 'ok', 'billed', 'settled')
AND ("usage".status_code IS NULL OR "usage".status_code < 400)
THEN 1
ELSE 0
END AS success_flag
FROM usage_billing_facts AS "usage"
WHERE "usage".created_at >= TO_TIMESTAMP("#,
);
builder.push_bind(query.created_from_unix_secs as f64);
builder.push(
r#"::double precision)
AND "usage".created_at < TO_TIMESTAMP("#,
);
builder.push_bind(query.created_until_unix_secs as f64);
builder.push(
r#"::double precision)
AND COALESCE("usage".status, '') NOT IN ('pending', 'streaming')
AND NULLIF(BTRIM(COALESCE("usage".provider_id, '')), '') IS NOT NULL
AND lower(BTRIM(COALESCE("usage".provider_id, ''))) NOT IN ('unknown', 'pending')
AND lower(BTRIM(COALESCE("usage".provider_name, ''))) NOT IN ('unknown', 'pending')
AND "usage".provider_id = ANY("#,
);
builder.push_bind(provider_ids.to_vec());
builder.push(
r#")
)
SELECT
date,
provider_id,
COALESCE(MAX(NULLIF(provider, '')), provider_id) AS provider,
COUNT(*)::BIGINT AS request_count,
COALESCE(SUM(success_flag), 0)::BIGINT AS success_count,
COALESCE(SUM(output_tokens), 0)::BIGINT AS output_tokens,
CASE
WHEN COALESCE(SUM(CASE
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
THEN response_time_ms
ELSE 0
END), 0) > 0
THEN COALESCE(SUM(CASE
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
THEN output_tokens
ELSE 0
END), 0)::DOUBLE PRECISION * 1000.0 / COALESCE(SUM(CASE
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
THEN response_time_ms
ELSE 0
END), 0)::DOUBLE PRECISION
ELSE NULL
END AS avg_output_tps,
AVG(first_byte_time_ms::DOUBLE PRECISION)
FILTER (WHERE success_flag = 1 AND has_first_byte_time) AS avg_first_byte_time_ms,
AVG(response_time_ms::DOUBLE PRECISION)
FILTER (WHERE success_flag = 1 AND has_response_time) AS avg_response_time_ms
FROM filtered_usage
GROUP BY date, provider_id
ORDER BY date ASC, provider_id ASC
"#,
);
let mut rows = builder.build().fetch(&self.pool);
let mut items = Vec::new();
while let Some(row) = rows.try_next().await.map_postgres_err()? {
items.push(decode_usage_provider_performance_timeline_row(&row)?);
}
Ok(items)
}
pub async fn summarize_usage_provider_performance(
&self,
query: &UsageProviderPerformanceQuery,
) -> Result<StoredUsageProviderPerformance, DataLayerError> {
if query.created_from_unix_secs >= query.created_until_unix_secs {
return Ok(StoredUsageProviderPerformance::default());
}
let summary = self
.summarize_usage_provider_performance_summary(query)
.await?;
let mut providers = self
.summarize_usage_provider_performance_providers(query)
.await?;
providers.truncate(query.limit.max(1));
let provider_ids = providers
.iter()
.map(|row| row.provider_id.clone())
.collect::<Vec<_>>();
let timeline = self
.summarize_usage_provider_performance_timeline(query, &provider_ids)
.await?;
Ok(StoredUsageProviderPerformance {
summary,
providers,
timeline,
})
}
async fn summarize_usage_cost_savings_raw_from_range( async fn summarize_usage_cost_savings_raw_from_range(
&self, &self,
start_utc: DateTime<Utc>, start_utc: DateTime<Utc>,
@@ -6775,6 +7178,13 @@ impl UsageReadRepository for SqlxUsageReadRepository {
Self::summarize_usage_performance_percentiles(self, query).await Self::summarize_usage_performance_percentiles(self, query).await
} }
async fn summarize_usage_provider_performance(
&self,
query: &UsageProviderPerformanceQuery,
) -> Result<StoredUsageProviderPerformance, DataLayerError> {
Self::summarize_usage_provider_performance(self, query).await
}
async fn summarize_usage_cost_savings( async fn summarize_usage_cost_savings(
&self, &self,
query: &UsageCostSavingsSummaryQuery, query: &UsageCostSavingsSummaryQuery,

View File

@@ -447,6 +447,49 @@ export interface PercentileItem {
p99_first_byte_time_ms?: number | null p99_first_byte_time_ms?: number | null
} }
export interface ProviderPerformanceSummary {
request_count: number
success_rate: number
avg_output_tps: number | null
avg_first_byte_time_ms: number | null
avg_response_time_ms: number | null
}
export interface ProviderPerformanceItem {
provider_id: string
provider: string
request_count: number
success_count: number
error_count: number
success_rate: number
output_tokens: number
avg_output_tps: number | null
avg_first_byte_time_ms: number | null
avg_response_time_ms: number | null
p90_response_time_ms: number | null
p90_first_byte_time_ms: number | null
tps_sample_count: number
first_byte_sample_count: number
}
export interface ProviderPerformanceTimelineItem {
date: string
provider_id: string
provider: string
request_count: number
output_tokens: number
avg_output_tps: number | null
avg_first_byte_time_ms: number | null
avg_response_time_ms: number | null
success_rate: number
}
export interface ProviderPerformanceResponse {
summary: ProviderPerformanceSummary
providers: ProviderPerformanceItem[]
timeline: ProviderPerformanceTimelineItem[]
}
export interface ErrorDistributionItem { export interface ErrorDistributionItem {
category: string category: string
count: number count: number
@@ -932,6 +975,28 @@ export const adminApi = {
) )
}, },
async getProviderPerformance(params?: {
start_date?: string
end_date?: string
preset?: string
timezone?: string
tz_offset_minutes?: number
granularity?: 'day' | 'hour'
limit?: number
}): Promise<ProviderPerformanceResponse> {
const cacheKey = buildCacheKey('admin:stats:performance:providers', params)
return cachedRequest(
cacheKey,
async () => {
const response = await apiClient.get<ProviderPerformanceResponse>('/api/admin/stats/performance/providers', {
params
})
return response.data
},
20 * 1000
)
},
async getErrorDistribution(params?: { async getErrorDistribution(params?: {
start_date?: string start_date?: string
end_date?: string end_date?: string

View File

@@ -491,6 +491,183 @@
</Card> </Card>
</div> </div>
<Card class="space-y-4 p-4">
<div class="flex flex-col gap-2 sm:flex-row sm:items-center sm:justify-between">
<div>
<h3 class="text-sm font-semibold">
Provider 性能
</h3>
<p class="text-xs text-muted-foreground">
{{ providerPerformanceSubtitle }}
</p>
</div>
<Badge variant="outline">
Top {{ providerPerformanceRows.length || 0 }}
</Badge>
</div>
<div
v-if="providerPerformanceLoading"
class="p-6"
>
<LoadingState />
</div>
<div
v-else
class="space-y-4"
>
<div class="grid grid-cols-1 gap-3 sm:grid-cols-2 xl:grid-cols-4">
<div
v-for="card in providerPerformanceSummaryCards"
:key="card.title"
class="rounded-xl border border-border/70 bg-card/70 px-4 py-3"
>
<div class="flex items-center justify-between gap-3">
<span class="text-xs text-muted-foreground">{{ card.title }}</span>
<component
:is="card.icon"
class="h-4 w-4"
:class="card.iconClass"
/>
</div>
<div class="mt-3 text-2xl font-semibold tracking-tight">
{{ card.value }}
</div>
<div class="mt-2 text-xs text-muted-foreground">
{{ card.hint }}
</div>
</div>
</div>
<div
v-if="providerPerformanceRows.length"
class="overflow-x-auto rounded-lg border border-border/70"
>
<table class="min-w-full divide-y divide-border/70 text-sm">
<thead class="bg-muted/30 text-xs text-muted-foreground">
<tr>
<th class="px-3 py-2 text-left font-medium">
Provider
</th>
<th class="px-3 py-2 text-right font-medium">
请求
</th>
<th class="px-3 py-2 text-right font-medium">
成功率
</th>
<th class="px-3 py-2 text-right font-medium">
输出 TPS
</th>
<th class="px-3 py-2 text-right font-medium">
平均首字
</th>
<th class="px-3 py-2 text-right font-medium">
平均响应
</th>
<th class="px-3 py-2 text-right font-medium">
P90 响应 / 首字
</th>
<th class="px-3 py-2 text-right font-medium">
样本
</th>
</tr>
</thead>
<tbody class="divide-y divide-border/60">
<tr
v-for="provider in providerPerformanceRows"
:key="provider.provider_id"
class="bg-background/40"
>
<td class="max-w-[220px] px-3 py-2">
<div class="truncate font-medium">
{{ provider.provider }}
</div>
<div class="truncate text-xs text-muted-foreground">
{{ provider.provider_id }}
</div>
</td>
<td class="px-3 py-2 text-right">
{{ formatMetricNumber(provider.request_count) }}
</td>
<td class="px-3 py-2 text-right">
{{ formatProviderPerformanceMetric(provider.success_rate, '%') }}
</td>
<td class="px-3 py-2 text-right">
{{ formatProviderPerformanceMetric(provider.avg_output_tps, '/s') }}
</td>
<td class="px-3 py-2 text-right">
{{ formatProviderPerformanceMetric(provider.avg_first_byte_time_ms, 'ms') }}
</td>
<td class="px-3 py-2 text-right">
{{ formatProviderPerformanceMetric(provider.avg_response_time_ms, 'ms') }}
</td>
<td class="px-3 py-2 text-right">
{{ formatProviderPerformanceMetric(provider.p90_response_time_ms, 'ms', 0) }}
/
{{ formatProviderPerformanceMetric(provider.p90_first_byte_time_ms, 'ms', 0) }}
</td>
<td class="px-3 py-2 text-right text-xs text-muted-foreground">
{{ formatMetricNumber(provider.tps_sample_count) }} /
{{ formatMetricNumber(provider.first_byte_sample_count) }}
</td>
</tr>
</tbody>
</table>
</div>
<div
v-else
class="rounded-lg border border-dashed border-border/70 px-3 py-4 text-sm text-muted-foreground"
>
当前没有 Provider 性能数据
</div>
</div>
</Card>
<div class="grid grid-cols-1 gap-4 lg:grid-cols-2">
<Card class="space-y-3 p-4">
<h3 class="text-sm font-semibold">
输出 TPS 趋势
</h3>
<div
v-if="providerPerformanceLoading"
class="p-6"
>
<LoadingState />
</div>
<div
v-else
class="h-[260px]"
>
<LineChart
:data="providerTpsChartData"
:options="providerTpsChartOptions"
/>
</div>
</Card>
<Card class="space-y-3 p-4">
<h3 class="text-sm font-semibold">
平均首字趋势
</h3>
<div
v-if="providerPerformanceLoading"
class="p-6"
>
<LoadingState />
</div>
<div
v-else
class="h-[260px]"
>
<LineChart
:data="providerFirstByteChartData"
:options="providerLatencyChartOptions"
/>
</div>
</Card>
</div>
<div class="grid grid-cols-1 gap-4 lg:grid-cols-2"> <div class="grid grid-cols-1 gap-4 lg:grid-cols-2">
<Card class="p-4"> <Card class="p-4">
<ErrorDistributionChart <ErrorDistributionChart
@@ -562,11 +739,20 @@ import {
Activity, Activity,
AlertTriangle, AlertTriangle,
Cable, Cable,
CheckCircle2,
GitBranch, GitBranch,
Gauge,
ShieldCheck, ShieldCheck,
Timer,
Workflow, Workflow,
Zap,
} from 'lucide-vue-next' } from 'lucide-vue-next'
import { adminApi, type ErrorDistributionResponse, type PercentileItem } from '@/api/admin' import {
adminApi,
type ErrorDistributionResponse,
type PercentileItem,
type ProviderPerformanceResponse,
} from '@/api/admin'
import { dashboardApi, type ProviderStatus } from '@/api/dashboard' import { dashboardApi, type ProviderStatus } from '@/api/dashboard'
import { import {
monitoringApi, monitoringApi,
@@ -586,6 +772,10 @@ import { getDateRangeFromPeriod } from '@/features/usage/composables'
import type { DateRangeParams } from '@/features/usage/types' import type { DateRangeParams } from '@/features/usage/types'
import { formatDate, formatNumber, formatTokens } from '@/utils/format' import { formatDate, formatNumber, formatTokens } from '@/utils/format'
import { log } from '@/utils/logger' import { log } from '@/utils/logger'
import {
buildProviderPerformanceChartData,
formatProviderPerformanceMetric,
} from './performanceAnalysisHelpers'
const LIVE_REFRESH_INTERVAL_MS = 10_000 const LIVE_REFRESH_INTERVAL_MS = 10_000
@@ -601,6 +791,8 @@ const errorLoading = ref(false)
const providerStatus = ref<ProviderStatus[]>([]) const providerStatus = ref<ProviderStatus[]>([])
const providerLoading = ref(false) const providerLoading = ref(false)
const providerPerformance = ref<ProviderPerformanceResponse | null>(null)
const providerPerformanceLoading = ref(false)
const systemStatus = ref<AdminMonitoringSystemStatus | null>(null) const systemStatus = ref<AdminMonitoringSystemStatus | null>(null)
const resilienceStatus = ref<AdminMonitoringResilienceStatus | null>(null) const resilienceStatus = ref<AdminMonitoringResilienceStatus | null>(null)
@@ -615,6 +807,7 @@ const liveLastUpdatedAt = ref<string | null>(null)
let percentilesRequestId = 0 let percentilesRequestId = 0
let errorsRequestId = 0 let errorsRequestId = 0
let providersRequestId = 0 let providersRequestId = 0
let providerPerformanceRequestId = 0
let liveRequestId = 0 let liveRequestId = 0
let loadAllPromise: Promise<void> | null = null let loadAllPromise: Promise<void> | null = null
let hasPendingLoadAll = false let hasPendingLoadAll = false
@@ -699,6 +892,28 @@ async function loadProviders() {
} }
} }
async function loadProviderPerformance() {
const requestId = ++providerPerformanceRequestId
providerPerformanceLoading.value = true
try {
const data = await adminApi.getProviderPerformance({
...buildTimeRangeParams(),
granularity: 'day',
limit: 8,
})
if (requestId !== providerPerformanceRequestId) return
providerPerformance.value = data
} catch (error) {
if (requestId !== providerPerformanceRequestId) return
providerPerformance.value = null
log.error('加载 Provider 性能统计失败', error)
} finally {
if (requestId === providerPerformanceRequestId) {
providerPerformanceLoading.value = false
}
}
}
async function loadLiveData(options: { silent?: boolean } = {}) { async function loadLiveData(options: { silent?: boolean } = {}) {
const requestId = ++liveRequestId const requestId = ++liveRequestId
const initialLoad = !liveReady.value const initialLoad = !liveReady.value
@@ -866,6 +1081,83 @@ const fallbackRows = computed(() => {
})) }))
}) })
const providerPerformanceRows = computed(() => providerPerformance.value?.providers ?? [])
const providerPerformanceSubtitle = computed(() => {
const requests = providerPerformance.value?.summary.request_count ?? 0
return `完成窗口内 ${formatMetricNumber(requests)} 个 Provider 请求样本`
})
const providerPerformanceSummaryCards = computed(() => {
const summary = providerPerformance.value?.summary
return [
{
title: '输出 TPS',
value: formatProviderPerformanceMetric(summary?.avg_output_tps, '/s'),
hint: `请求 ${formatMetricNumber(summary?.request_count)}`,
icon: Zap,
iconClass: 'text-amber-500',
},
{
title: '平均首字',
value: formatProviderPerformanceMetric(summary?.avg_first_byte_time_ms, 'ms'),
hint: '成功请求首字样本',
icon: Timer,
iconClass: 'text-sky-500',
},
{
title: '平均响应',
value: formatProviderPerformanceMetric(summary?.avg_response_time_ms, 'ms'),
hint: '成功请求响应耗时',
icon: Gauge,
iconClass: 'text-violet-500',
},
{
title: '成功率',
value: formatProviderPerformanceMetric(summary?.success_rate, '%'),
hint: `${formatMetricNumber(providerPerformanceRows.value.length)} 个 Provider`,
icon: CheckCircle2,
iconClass: 'text-emerald-500',
},
]
})
const providerTpsChartData = computed(() => (
buildProviderPerformanceChartData(
providerPerformance.value?.timeline ?? [],
'avg_output_tps',
providerPerformanceRows.value,
)
))
const providerFirstByteChartData = computed(() => (
buildProviderPerformanceChartData(
providerPerformance.value?.timeline ?? [],
'avg_first_byte_time_ms',
providerPerformanceRows.value,
)
))
const providerTpsChartOptions = computed(() => ({
scales: {
y: {
ticks: {
callback: (value: string | number) => `${value}/s`,
},
},
},
}))
const providerLatencyChartOptions = computed(() => ({
scales: {
y: {
ticks: {
callback: (value: string | number) => `${value}ms`,
},
},
},
}))
const liveSummaryCards = computed(() => [ const liveSummaryCards = computed(() => [
{ {
title: '系统健康', title: '系统健康',
@@ -920,7 +1212,8 @@ const isRefreshing = computed(() => (
liveRefreshing.value || liveRefreshing.value ||
percentileLoading.value || percentileLoading.value ||
errorLoading.value || errorLoading.value ||
providerLoading.value providerLoading.value ||
providerPerformanceLoading.value
)) ))
async function loadAll() { async function loadAll() {
@@ -929,7 +1222,12 @@ async function loadAll() {
return loadAllPromise return loadAllPromise
} }
loadAllPromise = Promise.all([loadPercentiles(), loadErrors(), loadProviders()]) loadAllPromise = Promise.all([
loadPercentiles(),
loadErrors(),
loadProviders(),
loadProviderPerformance(),
])
.then(() => undefined) .then(() => undefined)
.finally(() => { .finally(() => {
loadAllPromise = null loadAllPromise = null
@@ -983,6 +1281,7 @@ onUnmounted(() => {
percentilesRequestId += 1 percentilesRequestId += 1
errorsRequestId += 1 errorsRequestId += 1
providersRequestId += 1 providersRequestId += 1
providerPerformanceRequestId += 1
liveRequestId += 1 liveRequestId += 1
}) })
</script> </script>

View File

@@ -0,0 +1,53 @@
import { describe, expect, it } from 'vitest'
import {
buildProviderPerformanceChartData,
formatProviderPerformanceMetric,
} from '../performanceAnalysisHelpers'
import type { ProviderPerformanceItem, ProviderPerformanceTimelineItem } from '@/api/admin'
describe('performanceAnalysisHelpers', () => {
it('formats null provider metrics as placeholders', () => {
expect(formatProviderPerformanceMetric(null, 'ms')).toBe('-')
expect(formatProviderPerformanceMetric(undefined, '/s')).toBe('-')
expect(formatProviderPerformanceMetric(Number.NaN)).toBe('-')
expect(formatProviderPerformanceMetric(18.456, '/s')).toBe('18.46/s')
})
it('builds stable provider trend datasets with null gaps', () => {
const providers: Pick<ProviderPerformanceItem, 'provider_id' | 'provider'>[] = [
{ provider_id: 'provider-a', provider: 'OpenAI' },
{ provider_id: 'provider-b', provider: 'Anthropic' },
]
const timeline: ProviderPerformanceTimelineItem[] = [
{
date: '2024-03-21',
provider_id: 'provider-a',
provider: 'OpenAI',
request_count: 2,
output_tokens: 100,
avg_output_tps: 25,
avg_first_byte_time_ms: 120,
avg_response_time_ms: 1000,
success_rate: 100,
},
{
date: '2024-03-22',
provider_id: 'provider-b',
provider: 'Anthropic',
request_count: 1,
output_tokens: 40,
avg_output_tps: null,
avg_first_byte_time_ms: 220,
avg_response_time_ms: 1500,
success_rate: 100,
},
]
const chart = buildProviderPerformanceChartData(timeline, 'avg_output_tps', providers)
expect(chart.labels).toEqual(['2024-03-21', '2024-03-22'])
expect(chart.datasets.map(dataset => dataset.label)).toEqual(['OpenAI', 'Anthropic'])
expect(chart.datasets[0].data).toEqual([25, null])
expect(chart.datasets[1].data).toEqual([null, null])
})
})

View File

@@ -0,0 +1,64 @@
import type { ChartData } from 'chart.js'
import type {
ProviderPerformanceItem,
ProviderPerformanceTimelineItem,
} from '@/api/admin'
export type ProviderPerformanceMetricKey = 'avg_output_tps' | 'avg_first_byte_time_ms'
const PROVIDER_CHART_COLORS = [
'rgb(59, 130, 246)',
'rgb(16, 185, 129)',
'rgb(234, 179, 8)',
'rgb(239, 68, 68)',
'rgb(139, 92, 246)',
'rgb(14, 165, 233)',
'rgb(249, 115, 22)',
'rgb(20, 184, 166)',
]
export function formatProviderPerformanceMetric(
value: number | null | undefined,
suffix = '',
decimals = 2
): string {
if (value == null || Number.isNaN(value)) {
return '-'
}
return `${value.toFixed(decimals)}${suffix}`
}
export function buildProviderPerformanceChartData(
timeline: ProviderPerformanceTimelineItem[],
metric: ProviderPerformanceMetricKey,
providers: Pick<ProviderPerformanceItem, 'provider_id' | 'provider'>[] = []
): ChartData<'line'> {
const labels = Array.from(new Set(timeline.map(item => item.date)))
const providerMap = new Map<string, string>()
for (const provider of providers) {
providerMap.set(provider.provider_id, provider.provider)
}
for (const item of timeline) {
if (!providerMap.has(item.provider_id)) {
providerMap.set(item.provider_id, item.provider)
}
}
return {
labels,
datasets: Array.from(providerMap.entries()).map(([providerId, provider], index) => {
const byDate = new Map(
timeline
.filter(item => item.provider_id === providerId)
.map(item => [item.date, item[metric]] as const)
)
return {
label: provider,
data: labels.map(label => byDate.get(label) ?? null),
borderColor: PROVIDER_CHART_COLORS[index % PROVIDER_CHART_COLORS.length],
tension: 0.25,
pointRadius: 2,
}
}),
}
}