mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 01:10:23 +08:00
feat(stats): 添加 Provider 性能统计分析
This commit is contained in:
@@ -495,6 +495,19 @@ pub(super) fn classify_admin_observability_family_route(
|
|||||||
"admin:stats",
|
"admin:stats",
|
||||||
false,
|
false,
|
||||||
))
|
))
|
||||||
|
} else if method == http::Method::GET
|
||||||
|
&& matches!(
|
||||||
|
normalized_path,
|
||||||
|
"/api/admin/stats/performance/providers" | "/api/admin/stats/performance/providers/"
|
||||||
|
)
|
||||||
|
{
|
||||||
|
Some(classified(
|
||||||
|
"admin_proxy",
|
||||||
|
"stats_manage",
|
||||||
|
"provider_performance",
|
||||||
|
"admin:stats",
|
||||||
|
false,
|
||||||
|
))
|
||||||
} else if method == http::Method::GET
|
} else if method == http::Method::GET
|
||||||
&& matches!(
|
&& matches!(
|
||||||
normalized_path,
|
normalized_path,
|
||||||
|
|||||||
@@ -81,6 +81,25 @@ fn classifies_admin_stats_performance_percentiles_as_admin_proxy_route() {
|
|||||||
assert!(!decision.is_execution_runtime_candidate());
|
assert!(!decision.is_execution_runtime_candidate());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn classifies_admin_stats_provider_performance_as_admin_proxy_route() {
|
||||||
|
let headers = headers(&[]);
|
||||||
|
let uri: Uri = "/api/admin/stats/performance/providers"
|
||||||
|
.parse()
|
||||||
|
.expect("uri should parse");
|
||||||
|
let decision =
|
||||||
|
classify_control_route(&http::Method::GET, &uri, &headers).expect("route should classify");
|
||||||
|
|
||||||
|
assert_eq!(decision.route_class.as_deref(), Some("admin_proxy"));
|
||||||
|
assert_eq!(decision.route_family.as_deref(), Some("stats_manage"));
|
||||||
|
assert_eq!(decision.route_kind.as_deref(), Some("provider_performance"));
|
||||||
|
assert_eq!(
|
||||||
|
decision.auth_endpoint_signature.as_deref(),
|
||||||
|
Some("admin:stats")
|
||||||
|
);
|
||||||
|
assert!(!decision.is_execution_runtime_candidate());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn classifies_admin_stats_cost_forecast_as_admin_proxy_route() {
|
fn classifies_admin_stats_cost_forecast_as_admin_proxy_route() {
|
||||||
let headers = headers(&[]);
|
let headers = headers(&[]);
|
||||||
|
|||||||
@@ -1029,6 +1029,21 @@ impl GatewayDataState {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn summarize_usage_provider_performance(
|
||||||
|
&self,
|
||||||
|
query: &aether_data_contracts::repository::usage::UsageProviderPerformanceQuery,
|
||||||
|
) -> Result<
|
||||||
|
aether_data_contracts::repository::usage::StoredUsageProviderPerformance,
|
||||||
|
DataLayerError,
|
||||||
|
> {
|
||||||
|
match &self.usage_reader {
|
||||||
|
Some(repository) => repository.summarize_usage_provider_performance(query).await,
|
||||||
|
None => Ok(
|
||||||
|
aether_data_contracts::repository::usage::StoredUsageProviderPerformance::default(),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn summarize_usage_cost_savings(
|
pub(crate) async fn summarize_usage_cost_savings(
|
||||||
&self,
|
&self,
|
||||||
query: &aether_data_contracts::repository::usage::UsageCostSavingsSummaryQuery,
|
query: &aether_data_contracts::repository::usage::UsageCostSavingsSummaryQuery,
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
use super::range::build_comparison_range;
|
use super::range::{build_comparison_range, parse_bounded_u32};
|
||||||
use super::resolve_admin_usage_time_range;
|
use super::resolve_admin_usage_time_range;
|
||||||
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
|
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
|
||||||
use crate::handlers::admin::shared::query_param_value;
|
use crate::handlers::admin::shared::query_param_value;
|
||||||
@@ -6,16 +6,18 @@ use crate::GatewayError;
|
|||||||
use aether_admin::observability::stats::{
|
use aether_admin::observability::stats::{
|
||||||
admin_stats_bad_request_response, admin_stats_comparison_empty_response,
|
admin_stats_bad_request_response, admin_stats_comparison_empty_response,
|
||||||
admin_stats_error_distribution_empty_response,
|
admin_stats_error_distribution_empty_response,
|
||||||
admin_stats_performance_percentiles_empty_response, admin_stats_time_series_empty_response,
|
admin_stats_performance_percentiles_empty_response,
|
||||||
|
admin_stats_provider_performance_empty_response, admin_stats_time_series_empty_response,
|
||||||
build_admin_stats_comparison_response_from_aggregates,
|
build_admin_stats_comparison_response_from_aggregates,
|
||||||
build_admin_stats_error_distribution_response_from_summaries,
|
build_admin_stats_error_distribution_response_from_summaries,
|
||||||
build_admin_stats_performance_percentiles_response_from_summaries,
|
build_admin_stats_performance_percentiles_response_from_summaries,
|
||||||
|
build_admin_stats_provider_performance_response,
|
||||||
build_admin_stats_time_series_response_from_summaries, AdminStatsAggregate,
|
build_admin_stats_time_series_response_from_summaries, AdminStatsAggregate,
|
||||||
AdminStatsComparisonType, AdminStatsGranularity, AdminStatsTimeRange, AdminStatsUsageFilter,
|
AdminStatsComparisonType, AdminStatsGranularity, AdminStatsTimeRange, AdminStatsUsageFilter,
|
||||||
};
|
};
|
||||||
use aether_data_contracts::repository::usage::{
|
use aether_data_contracts::repository::usage::{
|
||||||
UsageAuditSummaryQuery, UsageErrorDistributionQuery, UsagePerformancePercentilesQuery,
|
UsageAuditSummaryQuery, UsageErrorDistributionQuery, UsagePerformancePercentilesQuery,
|
||||||
UsageTimeSeriesGranularity, UsageTimeSeriesQuery,
|
UsageProviderPerformanceQuery, UsageTimeSeriesGranularity, UsageTimeSeriesQuery,
|
||||||
};
|
};
|
||||||
use axum::{body::Body, http, response::Response};
|
use axum::{body::Body, http, response::Response};
|
||||||
|
|
||||||
@@ -173,6 +175,56 @@ pub(super) async fn maybe_build_local_admin_stats_analytics_response(
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if request_context.route_kind() == Some("provider_performance")
|
||||||
|
&& request_context.method() == http::Method::GET
|
||||||
|
&& matches!(
|
||||||
|
request_context.path(),
|
||||||
|
"/api/admin/stats/performance/providers" | "/api/admin/stats/performance/providers/"
|
||||||
|
)
|
||||||
|
{
|
||||||
|
let time_range = match resolve_admin_usage_time_range(request_context.query_string()) {
|
||||||
|
Ok(value) => value,
|
||||||
|
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||||
|
};
|
||||||
|
let granularity =
|
||||||
|
match query_param_value(request_context.query_string(), "granularity").as_deref() {
|
||||||
|
None | Some("day") => UsageTimeSeriesGranularity::Day,
|
||||||
|
Some("hour") => UsageTimeSeriesGranularity::Hour,
|
||||||
|
Some(_) => {
|
||||||
|
return Ok(Some(admin_stats_bad_request_response(
|
||||||
|
"granularity must be one of: day, hour".to_string(),
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let limit = match query_param_value(request_context.query_string(), "limit")
|
||||||
|
.map(|value| parse_bounded_u32("limit", &value, 1, 20))
|
||||||
|
.transpose()
|
||||||
|
{
|
||||||
|
Ok(value) => value.unwrap_or(8) as usize,
|
||||||
|
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||||
|
};
|
||||||
|
if !state.has_usage_data_reader() {
|
||||||
|
return Ok(Some(admin_stats_provider_performance_empty_response()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some((created_from_unix_secs, created_until_unix_secs)) = time_range.to_unix_bounds()
|
||||||
|
else {
|
||||||
|
return Ok(Some(admin_stats_provider_performance_empty_response()));
|
||||||
|
};
|
||||||
|
let performance = state
|
||||||
|
.summarize_usage_provider_performance(&UsageProviderPerformanceQuery {
|
||||||
|
created_from_unix_secs,
|
||||||
|
created_until_unix_secs,
|
||||||
|
granularity,
|
||||||
|
tz_offset_minutes: time_range.tz_offset_minutes,
|
||||||
|
limit,
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
return Ok(Some(build_admin_stats_provider_performance_response(
|
||||||
|
&performance,
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
if request_context.route_kind() == Some("time_series")
|
if request_context.route_kind() == Some("time_series")
|
||||||
&& request_context.method() == http::Method::GET
|
&& request_context.method() == http::Method::GET
|
||||||
&& matches!(
|
&& matches!(
|
||||||
|
|||||||
@@ -168,6 +168,16 @@ impl<'a> AdminAppState<'a> {
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn summarize_usage_provider_performance(
|
||||||
|
&self,
|
||||||
|
query: &aether_data_contracts::repository::usage::UsageProviderPerformanceQuery,
|
||||||
|
) -> Result<
|
||||||
|
aether_data_contracts::repository::usage::StoredUsageProviderPerformance,
|
||||||
|
GatewayError,
|
||||||
|
> {
|
||||||
|
self.app.summarize_usage_provider_performance(query).await
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn summarize_usage_cost_savings(
|
pub(crate) async fn summarize_usage_cost_savings(
|
||||||
&self,
|
&self,
|
||||||
query: &aether_data_contracts::repository::usage::UsageCostSavingsSummaryQuery,
|
query: &aether_data_contracts::repository::usage::UsageCostSavingsSummaryQuery,
|
||||||
|
|||||||
@@ -251,6 +251,16 @@ impl AppState {
|
|||||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn summarize_usage_provider_performance(
|
||||||
|
&self,
|
||||||
|
query: &usage::UsageProviderPerformanceQuery,
|
||||||
|
) -> Result<usage::StoredUsageProviderPerformance, GatewayError> {
|
||||||
|
self.data
|
||||||
|
.summarize_usage_provider_performance(query)
|
||||||
|
.await
|
||||||
|
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn summarize_usage_cost_savings(
|
pub(crate) async fn summarize_usage_cost_savings(
|
||||||
&self,
|
&self,
|
||||||
query: &usage::UsageCostSavingsSummaryQuery,
|
query: &usage::UsageCostSavingsSummaryQuery,
|
||||||
|
|||||||
@@ -853,6 +853,187 @@ async fn gateway_handles_admin_stats_performance_percentiles_locally_without_usa
|
|||||||
upstream_handle.abort();
|
upstream_handle.abort();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn gateway_handles_admin_stats_provider_performance_locally_with_trusted_admin_principal() {
|
||||||
|
let (_upstream_url, upstream_hits, upstream_handle) =
|
||||||
|
start_stats_upstream("/api/admin/stats/performance/providers").await;
|
||||||
|
|
||||||
|
let mut usage = (1..=10)
|
||||||
|
.map(|index| {
|
||||||
|
let mut row = sample_usage_row(
|
||||||
|
&format!("usage-provider-perf-a-{index}"),
|
||||||
|
&format!("req-provider-perf-a-{index}"),
|
||||||
|
Some("user-1"),
|
||||||
|
Some("key-1"),
|
||||||
|
Some("primary"),
|
||||||
|
"OpenAI",
|
||||||
|
"gpt-5",
|
||||||
|
10,
|
||||||
|
10,
|
||||||
|
0.01,
|
||||||
|
0.01,
|
||||||
|
DAY_1_UNIX_SECS + i64::from(index),
|
||||||
|
);
|
||||||
|
row.response_time_ms = Some((index * 100) as u64);
|
||||||
|
row.first_byte_time_ms = Some((index * 10) as u64);
|
||||||
|
row
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let mut failed = sample_usage_row(
|
||||||
|
"usage-provider-perf-failed",
|
||||||
|
"req-provider-perf-failed",
|
||||||
|
Some("user-1"),
|
||||||
|
Some("key-1"),
|
||||||
|
Some("primary"),
|
||||||
|
"OpenAI",
|
||||||
|
"gpt-5",
|
||||||
|
10,
|
||||||
|
99,
|
||||||
|
0.01,
|
||||||
|
0.01,
|
||||||
|
DAY_1_UNIX_SECS + 20,
|
||||||
|
);
|
||||||
|
failed.status = "failed".to_string();
|
||||||
|
failed.status_code = Some(500);
|
||||||
|
failed.error_message = Some("upstream failed".to_string());
|
||||||
|
usage.push(failed);
|
||||||
|
|
||||||
|
let mut provider_b = sample_usage_row(
|
||||||
|
"usage-provider-perf-b",
|
||||||
|
"req-provider-perf-b",
|
||||||
|
Some("user-1"),
|
||||||
|
Some("key-1"),
|
||||||
|
Some("primary"),
|
||||||
|
"Anthropic",
|
||||||
|
"claude-sonnet",
|
||||||
|
10,
|
||||||
|
20,
|
||||||
|
0.01,
|
||||||
|
0.01,
|
||||||
|
DAY_1_UNIX_SECS + 30,
|
||||||
|
);
|
||||||
|
provider_b.provider_id = Some("provider-2".to_string());
|
||||||
|
provider_b.response_time_ms = Some(1000);
|
||||||
|
provider_b.first_byte_time_ms = None;
|
||||||
|
usage.push(provider_b);
|
||||||
|
|
||||||
|
let mut unknown_provider = sample_usage_row(
|
||||||
|
"usage-provider-perf-unknown",
|
||||||
|
"req-provider-perf-unknown",
|
||||||
|
Some("user-1"),
|
||||||
|
Some("key-1"),
|
||||||
|
Some("primary"),
|
||||||
|
"unknown",
|
||||||
|
"gpt-5",
|
||||||
|
10,
|
||||||
|
999,
|
||||||
|
0.01,
|
||||||
|
0.01,
|
||||||
|
DAY_1_UNIX_SECS + 40,
|
||||||
|
);
|
||||||
|
unknown_provider.provider_id = None;
|
||||||
|
usage.push(unknown_provider);
|
||||||
|
|
||||||
|
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(usage));
|
||||||
|
let gateway = build_router_with_state(
|
||||||
|
AppState::new()
|
||||||
|
.expect("gateway should build")
|
||||||
|
.with_data_state_for_tests(GatewayDataState::with_usage_reader_for_tests(
|
||||||
|
usage_repository,
|
||||||
|
)),
|
||||||
|
);
|
||||||
|
let (gateway_url, gateway_handle) = start_server(gateway).await;
|
||||||
|
|
||||||
|
let response = admin_request(reqwest::Client::new().get(format!(
|
||||||
|
"{gateway_url}/api/admin/stats/performance/providers?start_date=2024-03-21&end_date=2024-03-21&granularity=hour&limit=2&tz_offset_minutes=0"
|
||||||
|
)))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.expect("request should succeed");
|
||||||
|
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
let payload: serde_json::Value = response.json().await.expect("json body should parse");
|
||||||
|
assert_eq!(payload["summary"]["request_count"], 12);
|
||||||
|
assert_eq!(payload["summary"]["success_rate"], 91.67);
|
||||||
|
assert_eq!(payload["summary"]["avg_output_tps"], 18.46);
|
||||||
|
assert_eq!(payload["summary"]["avg_first_byte_time_ms"], 55.0);
|
||||||
|
assert_eq!(payload["summary"]["avg_response_time_ms"], 590.91);
|
||||||
|
|
||||||
|
assert_eq!(payload["providers"].as_array().map(Vec::len), Some(2));
|
||||||
|
assert_eq!(payload["providers"][0]["provider_id"], "provider-1");
|
||||||
|
assert_eq!(payload["providers"][0]["provider"], "OpenAI");
|
||||||
|
assert_eq!(payload["providers"][0]["request_count"], 11);
|
||||||
|
assert_eq!(payload["providers"][0]["success_count"], 10);
|
||||||
|
assert_eq!(payload["providers"][0]["error_count"], 1);
|
||||||
|
assert_eq!(payload["providers"][0]["success_rate"], 90.91);
|
||||||
|
assert_eq!(payload["providers"][0]["output_tokens"], 199);
|
||||||
|
assert_eq!(payload["providers"][0]["avg_output_tps"], 18.18);
|
||||||
|
assert_eq!(payload["providers"][0]["avg_first_byte_time_ms"], 55.0);
|
||||||
|
assert_eq!(payload["providers"][0]["avg_response_time_ms"], 550.0);
|
||||||
|
assert_eq!(payload["providers"][0]["p90_response_time_ms"], 910);
|
||||||
|
assert_eq!(payload["providers"][0]["p90_first_byte_time_ms"], 91);
|
||||||
|
assert_eq!(payload["providers"][0]["tps_sample_count"], 10);
|
||||||
|
assert_eq!(payload["providers"][0]["first_byte_sample_count"], 10);
|
||||||
|
|
||||||
|
assert_eq!(payload["providers"][1]["provider_id"], "provider-2");
|
||||||
|
assert_eq!(payload["providers"][1]["provider"], "Anthropic");
|
||||||
|
assert_eq!(payload["providers"][1]["avg_output_tps"], 20.0);
|
||||||
|
assert_eq!(
|
||||||
|
payload["providers"][1]["avg_first_byte_time_ms"],
|
||||||
|
serde_json::Value::Null
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
payload["providers"][1]["p90_response_time_ms"],
|
||||||
|
serde_json::Value::Null
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(payload["timeline"].as_array().map(Vec::len), Some(2));
|
||||||
|
assert_eq!(payload["timeline"][0]["date"], "2024-03-21T05:00:00+00:00");
|
||||||
|
assert_eq!(payload["timeline"][0]["provider_id"], "provider-1");
|
||||||
|
assert_eq!(payload["timeline"][0]["avg_output_tps"], 18.18);
|
||||||
|
assert_eq!(payload["timeline"][0]["success_rate"], 90.91);
|
||||||
|
assert_eq!(payload["timeline"][1]["provider_id"], "provider-2");
|
||||||
|
assert_eq!(
|
||||||
|
payload["timeline"][1]["avg_first_byte_time_ms"],
|
||||||
|
serde_json::Value::Null
|
||||||
|
);
|
||||||
|
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
|
||||||
|
|
||||||
|
gateway_handle.abort();
|
||||||
|
upstream_handle.abort();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn gateway_returns_empty_admin_stats_provider_performance_without_usage_reader() {
|
||||||
|
let (_upstream_url, upstream_hits, upstream_handle) =
|
||||||
|
start_stats_upstream("/api/admin/stats/performance/providers").await;
|
||||||
|
|
||||||
|
let gateway = build_router_with_state(AppState::new().expect("gateway should build"));
|
||||||
|
let (gateway_url, gateway_handle) = start_server(gateway).await;
|
||||||
|
|
||||||
|
let response = admin_request(reqwest::Client::new().get(format!(
|
||||||
|
"{gateway_url}/api/admin/stats/performance/providers?start_date=2024-03-21&end_date=2024-03-21&limit=2"
|
||||||
|
)))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.expect("request should succeed");
|
||||||
|
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
let payload: serde_json::Value = response.json().await.expect("json body should parse");
|
||||||
|
assert_eq!(payload["summary"]["request_count"], 0);
|
||||||
|
assert_eq!(payload["summary"]["success_rate"], 0.0);
|
||||||
|
assert_eq!(
|
||||||
|
payload["summary"]["avg_output_tps"],
|
||||||
|
serde_json::Value::Null
|
||||||
|
);
|
||||||
|
assert_eq!(payload["providers"].as_array().map(Vec::len), Some(0));
|
||||||
|
assert_eq!(payload["timeline"].as_array().map(Vec::len), Some(0));
|
||||||
|
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
|
||||||
|
|
||||||
|
gateway_handle.abort();
|
||||||
|
upstream_handle.abort();
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn gateway_handles_admin_stats_time_series_locally_with_trusted_admin_principal() {
|
async fn gateway_handles_admin_stats_time_series_locally_with_trusted_admin_principal() {
|
||||||
let (upstream_url, upstream_hits, upstream_handle) =
|
let (upstream_url, upstream_hits, upstream_handle) =
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ use aether_data_contracts::repository::{
|
|||||||
usage::{
|
usage::{
|
||||||
StoredRequestUsageAudit, StoredUsageCostSavingsSummary, StoredUsageErrorDistributionRow,
|
StoredRequestUsageAudit, StoredUsageCostSavingsSummary, StoredUsageErrorDistributionRow,
|
||||||
StoredUsageLeaderboardSummary, StoredUsagePerformancePercentilesRow,
|
StoredUsageLeaderboardSummary, StoredUsagePerformancePercentilesRow,
|
||||||
StoredUsageTimeSeriesBucket,
|
StoredUsageProviderPerformance, StoredUsageTimeSeriesBucket,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
use axum::{
|
use axum::{
|
||||||
@@ -584,6 +584,20 @@ pub fn round_to(value: f64, decimals: u32) -> f64 {
|
|||||||
(value * factor).round() / factor
|
(value * factor).round() / factor
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn rounded_option(value: Option<f64>, decimals: u32) -> serde_json::Value {
|
||||||
|
value
|
||||||
|
.map(|value| json!(round_to(value, decimals)))
|
||||||
|
.unwrap_or(serde_json::Value::Null)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn success_rate(request_count: u64, success_count: u64) -> f64 {
|
||||||
|
if request_count == 0 {
|
||||||
|
0.0
|
||||||
|
} else {
|
||||||
|
round_to(success_count as f64 / request_count as f64 * 100.0, 2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn admin_stats_provider_quota_usage_empty_response() -> Response<Body> {
|
pub fn admin_stats_provider_quota_usage_empty_response() -> Response<Body> {
|
||||||
Json(json!({
|
Json(json!({
|
||||||
"providers": [],
|
"providers": [],
|
||||||
@@ -652,6 +666,21 @@ pub fn admin_stats_performance_percentiles_empty_response() -> Response<Body> {
|
|||||||
Json(json!([])).into_response()
|
Json(json!([])).into_response()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn admin_stats_provider_performance_empty_response() -> Response<Body> {
|
||||||
|
Json(json!({
|
||||||
|
"summary": {
|
||||||
|
"request_count": 0,
|
||||||
|
"success_rate": 0.0,
|
||||||
|
"avg_output_tps": serde_json::Value::Null,
|
||||||
|
"avg_first_byte_time_ms": serde_json::Value::Null,
|
||||||
|
"avg_response_time_ms": serde_json::Value::Null,
|
||||||
|
},
|
||||||
|
"providers": [],
|
||||||
|
"timeline": [],
|
||||||
|
}))
|
||||||
|
.into_response()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn admin_stats_cost_savings_empty_response() -> Response<Body> {
|
pub fn admin_stats_cost_savings_empty_response() -> Response<Body> {
|
||||||
Json(json!({
|
Json(json!({
|
||||||
"cache_read_tokens": 0,
|
"cache_read_tokens": 0,
|
||||||
@@ -1061,6 +1090,64 @@ pub fn build_admin_stats_performance_percentiles_response_from_summaries(
|
|||||||
Json(serde_json::Value::Array(payload)).into_response()
|
Json(serde_json::Value::Array(payload)).into_response()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn build_admin_stats_provider_performance_response(
|
||||||
|
performance: &StoredUsageProviderPerformance,
|
||||||
|
) -> Response<Body> {
|
||||||
|
let summary = &performance.summary;
|
||||||
|
let providers = performance
|
||||||
|
.providers
|
||||||
|
.iter()
|
||||||
|
.map(|row| {
|
||||||
|
json!({
|
||||||
|
"provider_id": row.provider_id.as_str(),
|
||||||
|
"provider": row.provider.as_str(),
|
||||||
|
"request_count": row.request_count,
|
||||||
|
"success_count": row.success_count,
|
||||||
|
"error_count": row.request_count.saturating_sub(row.success_count),
|
||||||
|
"success_rate": success_rate(row.request_count, row.success_count),
|
||||||
|
"output_tokens": row.output_tokens,
|
||||||
|
"avg_output_tps": rounded_option(row.avg_output_tps, 2),
|
||||||
|
"avg_first_byte_time_ms": rounded_option(row.avg_first_byte_time_ms, 2),
|
||||||
|
"avg_response_time_ms": rounded_option(row.avg_response_time_ms, 2),
|
||||||
|
"p90_response_time_ms": row.p90_response_time_ms,
|
||||||
|
"p90_first_byte_time_ms": row.p90_first_byte_time_ms,
|
||||||
|
"tps_sample_count": row.tps_sample_count,
|
||||||
|
"first_byte_sample_count": row.first_byte_sample_count,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let timeline = performance
|
||||||
|
.timeline
|
||||||
|
.iter()
|
||||||
|
.map(|row| {
|
||||||
|
json!({
|
||||||
|
"date": row.date.as_str(),
|
||||||
|
"provider_id": row.provider_id.as_str(),
|
||||||
|
"provider": row.provider.as_str(),
|
||||||
|
"request_count": row.request_count,
|
||||||
|
"output_tokens": row.output_tokens,
|
||||||
|
"avg_output_tps": rounded_option(row.avg_output_tps, 2),
|
||||||
|
"avg_first_byte_time_ms": rounded_option(row.avg_first_byte_time_ms, 2),
|
||||||
|
"avg_response_time_ms": rounded_option(row.avg_response_time_ms, 2),
|
||||||
|
"success_rate": success_rate(row.request_count, row.success_count),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
Json(json!({
|
||||||
|
"summary": {
|
||||||
|
"request_count": summary.request_count,
|
||||||
|
"success_rate": success_rate(summary.request_count, summary.success_count),
|
||||||
|
"avg_output_tps": rounded_option(summary.avg_output_tps, 2),
|
||||||
|
"avg_first_byte_time_ms": rounded_option(summary.avg_first_byte_time_ms, 2),
|
||||||
|
"avg_response_time_ms": rounded_option(summary.avg_response_time_ms, 2),
|
||||||
|
},
|
||||||
|
"providers": providers,
|
||||||
|
"timeline": timeline,
|
||||||
|
}))
|
||||||
|
.into_response()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn build_admin_stats_time_series_response(
|
pub fn build_admin_stats_time_series_response(
|
||||||
time_range: &AdminStatsTimeRange,
|
time_range: &AdminStatsTimeRange,
|
||||||
granularity: AdminStatsGranularity,
|
granularity: AdminStatsGranularity,
|
||||||
|
|||||||
@@ -8,7 +8,9 @@ pub use types::{
|
|||||||
StoredUsageCacheHitSummary, StoredUsageCostSavingsSummary, StoredUsageDailySummary,
|
StoredUsageCacheHitSummary, StoredUsageCostSavingsSummary, StoredUsageDailySummary,
|
||||||
StoredUsageDashboardDailyBreakdownRow, StoredUsageDashboardProviderCount,
|
StoredUsageDashboardDailyBreakdownRow, StoredUsageDashboardProviderCount,
|
||||||
StoredUsageDashboardSummary, StoredUsageErrorDistributionRow, StoredUsageLeaderboardSummary,
|
StoredUsageDashboardSummary, StoredUsageErrorDistributionRow, StoredUsageLeaderboardSummary,
|
||||||
StoredUsagePerformancePercentilesRow, StoredUsageSettledCostSummary,
|
StoredUsagePerformancePercentilesRow, StoredUsageProviderPerformance,
|
||||||
|
StoredUsageProviderPerformanceProviderRow, StoredUsageProviderPerformanceSummary,
|
||||||
|
StoredUsageProviderPerformanceTimelineRow, StoredUsageSettledCostSummary,
|
||||||
StoredUsageTimeSeriesBucket, StoredUsageUserTotals, UpsertUsageRecord,
|
StoredUsageTimeSeriesBucket, StoredUsageUserTotals, UpsertUsageRecord,
|
||||||
UsageAuditAggregationGroupBy, UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery,
|
UsageAuditAggregationGroupBy, UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery,
|
||||||
UsageAuditListQuery, UsageAuditSummaryQuery, UsageBodyCaptureResult, UsageBodyCaptureState,
|
UsageAuditListQuery, UsageAuditSummaryQuery, UsageBodyCaptureResult, UsageBodyCaptureState,
|
||||||
@@ -18,7 +20,7 @@ pub use types::{
|
|||||||
UsageDailyHeatmapQuery, UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
|
UsageDailyHeatmapQuery, UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
|
||||||
UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy,
|
UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy,
|
||||||
UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery,
|
UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery,
|
||||||
UsagePerformancePercentilesQuery, UsageReadRepository, UsageRepository,
|
UsagePerformancePercentilesQuery, UsageProviderPerformanceQuery, UsageReadRepository,
|
||||||
UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity, UsageTimeSeriesQuery,
|
UsageRepository, UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity,
|
||||||
UsageWriteRepository,
|
UsageTimeSeriesQuery, UsageWriteRepository,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -949,6 +949,60 @@ pub struct StoredUsagePerformancePercentilesRow {
|
|||||||
pub p99_first_byte_time_ms: Option<u64>,
|
pub p99_first_byte_time_ms: Option<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct UsageProviderPerformanceQuery {
|
||||||
|
pub created_from_unix_secs: u64,
|
||||||
|
pub created_until_unix_secs: u64,
|
||||||
|
pub granularity: UsageTimeSeriesGranularity,
|
||||||
|
pub tz_offset_minutes: i32,
|
||||||
|
pub limit: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Default, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct StoredUsageProviderPerformanceSummary {
|
||||||
|
pub request_count: u64,
|
||||||
|
pub success_count: u64,
|
||||||
|
pub avg_output_tps: Option<f64>,
|
||||||
|
pub avg_first_byte_time_ms: Option<f64>,
|
||||||
|
pub avg_response_time_ms: Option<f64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Default, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct StoredUsageProviderPerformanceProviderRow {
|
||||||
|
pub provider_id: String,
|
||||||
|
pub provider: String,
|
||||||
|
pub request_count: u64,
|
||||||
|
pub success_count: u64,
|
||||||
|
pub output_tokens: u64,
|
||||||
|
pub avg_output_tps: Option<f64>,
|
||||||
|
pub avg_first_byte_time_ms: Option<f64>,
|
||||||
|
pub avg_response_time_ms: Option<f64>,
|
||||||
|
pub p90_response_time_ms: Option<u64>,
|
||||||
|
pub p90_first_byte_time_ms: Option<u64>,
|
||||||
|
pub tps_sample_count: u64,
|
||||||
|
pub first_byte_sample_count: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Default, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct StoredUsageProviderPerformanceTimelineRow {
|
||||||
|
pub date: String,
|
||||||
|
pub provider_id: String,
|
||||||
|
pub provider: String,
|
||||||
|
pub request_count: u64,
|
||||||
|
pub success_count: u64,
|
||||||
|
pub output_tokens: u64,
|
||||||
|
pub avg_output_tps: Option<f64>,
|
||||||
|
pub avg_first_byte_time_ms: Option<f64>,
|
||||||
|
pub avg_response_time_ms: Option<f64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Default, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct StoredUsageProviderPerformance {
|
||||||
|
pub summary: StoredUsageProviderPerformanceSummary,
|
||||||
|
pub providers: Vec<StoredUsageProviderPerformanceProviderRow>,
|
||||||
|
pub timeline: Vec<StoredUsageProviderPerformanceTimelineRow>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
|
||||||
pub struct UsageCostSavingsSummaryQuery {
|
pub struct UsageCostSavingsSummaryQuery {
|
||||||
pub created_from_unix_secs: u64,
|
pub created_from_unix_secs: u64,
|
||||||
@@ -1364,6 +1418,11 @@ pub trait UsageReadRepository: Send + Sync {
|
|||||||
query: &UsagePerformancePercentilesQuery,
|
query: &UsagePerformancePercentilesQuery,
|
||||||
) -> Result<Vec<StoredUsagePerformancePercentilesRow>, crate::DataLayerError>;
|
) -> Result<Vec<StoredUsagePerformancePercentilesRow>, crate::DataLayerError>;
|
||||||
|
|
||||||
|
async fn summarize_usage_provider_performance(
|
||||||
|
&self,
|
||||||
|
query: &UsageProviderPerformanceQuery,
|
||||||
|
) -> Result<StoredUsageProviderPerformance, crate::DataLayerError>;
|
||||||
|
|
||||||
async fn summarize_usage_cost_savings(
|
async fn summarize_usage_cost_savings(
|
||||||
&self,
|
&self,
|
||||||
query: &UsageCostSavingsSummaryQuery,
|
query: &UsageCostSavingsSummaryQuery,
|
||||||
|
|||||||
@@ -8,7 +8,9 @@ use aether_data_contracts::repository::usage::{
|
|||||||
StoredUsageCacheAffinityIntervalRow, StoredUsageCacheHitSummary, StoredUsageCostSavingsSummary,
|
StoredUsageCacheAffinityIntervalRow, StoredUsageCacheHitSummary, StoredUsageCostSavingsSummary,
|
||||||
StoredUsageDashboardDailyBreakdownRow, StoredUsageDashboardProviderCount,
|
StoredUsageDashboardDailyBreakdownRow, StoredUsageDashboardProviderCount,
|
||||||
StoredUsageDashboardSummary, StoredUsageErrorDistributionRow, StoredUsageLeaderboardSummary,
|
StoredUsageDashboardSummary, StoredUsageErrorDistributionRow, StoredUsageLeaderboardSummary,
|
||||||
StoredUsagePerformancePercentilesRow, StoredUsageSettledCostSummary,
|
StoredUsagePerformancePercentilesRow, StoredUsageProviderPerformance,
|
||||||
|
StoredUsageProviderPerformanceProviderRow, StoredUsageProviderPerformanceSummary,
|
||||||
|
StoredUsageProviderPerformanceTimelineRow, StoredUsageSettledCostSummary,
|
||||||
StoredUsageTimeSeriesBucket, StoredUsageUserTotals, UsageAuditAggregationGroupBy,
|
StoredUsageTimeSeriesBucket, StoredUsageUserTotals, UsageAuditAggregationGroupBy,
|
||||||
UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery, UsageAuditSummaryQuery,
|
UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery, UsageAuditSummaryQuery,
|
||||||
UsageBodyField, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery,
|
UsageBodyField, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery,
|
||||||
@@ -17,8 +19,8 @@ use aether_data_contracts::repository::usage::{
|
|||||||
UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
|
UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
|
||||||
UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy,
|
UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy,
|
||||||
UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery,
|
UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery,
|
||||||
UsagePerformancePercentilesQuery, UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity,
|
UsagePerformancePercentilesQuery, UsageProviderPerformanceQuery, UsageSettledCostSummaryQuery,
|
||||||
UsageTimeSeriesQuery,
|
UsageTimeSeriesGranularity, UsageTimeSeriesQuery,
|
||||||
};
|
};
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
@@ -587,6 +589,38 @@ fn usage_matches_performance_percentiles_query(
|
|||||||
&& item.status == "completed"
|
&& item.status == "completed"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn usage_provider_performance_identity(item: &StoredRequestUsageAudit) -> Option<(String, String)> {
|
||||||
|
let provider_id = item.provider_id.as_deref()?.trim();
|
||||||
|
let provider_id_status = provider_id.to_ascii_lowercase();
|
||||||
|
if provider_id.is_empty() || matches!(provider_id_status.as_str(), "unknown" | "pending") {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let provider_name = item.provider_name.trim();
|
||||||
|
let provider_name_status = provider_name.to_ascii_lowercase();
|
||||||
|
if matches!(provider_name_status.as_str(), "unknown" | "pending") {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let display_name = if provider_name.is_empty() {
|
||||||
|
provider_id
|
||||||
|
} else {
|
||||||
|
provider_name
|
||||||
|
};
|
||||||
|
Some((provider_id.to_string(), display_name.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn usage_matches_provider_performance_query(
|
||||||
|
item: &StoredRequestUsageAudit,
|
||||||
|
query: &UsageProviderPerformanceQuery,
|
||||||
|
) -> Option<(String, String)> {
|
||||||
|
if item.created_at_unix_ms < query.created_from_unix_secs
|
||||||
|
|| item.created_at_unix_ms >= query.created_until_unix_secs
|
||||||
|
|| matches!(item.status.as_str(), "pending" | "streaming")
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
usage_provider_performance_identity(item)
|
||||||
|
}
|
||||||
|
|
||||||
fn usage_matches_cost_savings_query(
|
fn usage_matches_cost_savings_query(
|
||||||
item: &StoredRequestUsageAudit,
|
item: &StoredRequestUsageAudit,
|
||||||
query: &UsageCostSavingsSummaryQuery,
|
query: &UsageCostSavingsSummaryQuery,
|
||||||
@@ -1650,6 +1684,218 @@ impl UsageReadRepository for InMemoryUsageReadRepository {
|
|||||||
.collect())
|
.collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn summarize_usage_provider_performance(
|
||||||
|
&self,
|
||||||
|
query: &UsageProviderPerformanceQuery,
|
||||||
|
) -> Result<StoredUsageProviderPerformance, DataLayerError> {
|
||||||
|
#[derive(Default)]
|
||||||
|
struct ProviderPerformanceBucket {
|
||||||
|
provider: String,
|
||||||
|
request_count: u64,
|
||||||
|
success_count: u64,
|
||||||
|
output_tokens: u64,
|
||||||
|
tps_output_tokens: u64,
|
||||||
|
tps_response_time_ms_sum: u64,
|
||||||
|
tps_sample_count: u64,
|
||||||
|
first_byte_time_ms_sum: u64,
|
||||||
|
first_byte_sample_count: u64,
|
||||||
|
response_time_ms_sum: u64,
|
||||||
|
response_time_sample_count: u64,
|
||||||
|
response_times: Vec<u64>,
|
||||||
|
first_byte_times: Vec<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ProviderPerformanceBucket {
|
||||||
|
fn add(&mut self, item: &StoredRequestUsageAudit) {
|
||||||
|
self.request_count = self.request_count.saturating_add(1);
|
||||||
|
self.output_tokens = self.output_tokens.saturating_add(item.output_tokens);
|
||||||
|
if !usage_is_success(item) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.success_count = self.success_count.saturating_add(1);
|
||||||
|
if let Some(response_time_ms) = item.response_time_ms {
|
||||||
|
self.response_time_ms_sum =
|
||||||
|
self.response_time_ms_sum.saturating_add(response_time_ms);
|
||||||
|
self.response_time_sample_count =
|
||||||
|
self.response_time_sample_count.saturating_add(1);
|
||||||
|
self.response_times.push(response_time_ms);
|
||||||
|
if response_time_ms > 0 && item.output_tokens > 0 {
|
||||||
|
self.tps_output_tokens =
|
||||||
|
self.tps_output_tokens.saturating_add(item.output_tokens);
|
||||||
|
self.tps_response_time_ms_sum = self
|
||||||
|
.tps_response_time_ms_sum
|
||||||
|
.saturating_add(response_time_ms);
|
||||||
|
self.tps_sample_count = self.tps_sample_count.saturating_add(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(first_byte_time_ms) = item.first_byte_time_ms {
|
||||||
|
self.first_byte_time_ms_sum = self
|
||||||
|
.first_byte_time_ms_sum
|
||||||
|
.saturating_add(first_byte_time_ms);
|
||||||
|
self.first_byte_sample_count = self.first_byte_sample_count.saturating_add(1);
|
||||||
|
self.first_byte_times.push(first_byte_time_ms);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn avg(sum: u64, samples: u64) -> Option<f64> {
|
||||||
|
if samples == 0 {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(sum as f64 / samples as f64)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn avg_tps(tokens: u64, response_time_ms_sum: u64) -> Option<f64> {
|
||||||
|
if response_time_ms_sum == 0 {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(tokens as f64 * 1000.0 / response_time_ms_sum as f64)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let usage = self
|
||||||
|
.by_request_id
|
||||||
|
.read()
|
||||||
|
.expect("usage repository lock")
|
||||||
|
.values()
|
||||||
|
.cloned()
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
let mut grouped = BTreeMap::<String, ProviderPerformanceBucket>::new();
|
||||||
|
let mut summary_bucket = ProviderPerformanceBucket::default();
|
||||||
|
for item in &usage {
|
||||||
|
let Some((provider_id, provider)) =
|
||||||
|
usage_matches_provider_performance_query(item, query)
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
summary_bucket.add(item);
|
||||||
|
let bucket = grouped.entry(provider_id).or_default();
|
||||||
|
if bucket.provider.is_empty() {
|
||||||
|
bucket.provider = provider;
|
||||||
|
}
|
||||||
|
bucket.add(item);
|
||||||
|
}
|
||||||
|
|
||||||
|
let summary = StoredUsageProviderPerformanceSummary {
|
||||||
|
request_count: summary_bucket.request_count,
|
||||||
|
success_count: summary_bucket.success_count,
|
||||||
|
avg_output_tps: avg_tps(
|
||||||
|
summary_bucket.tps_output_tokens,
|
||||||
|
summary_bucket.tps_response_time_ms_sum,
|
||||||
|
),
|
||||||
|
avg_first_byte_time_ms: avg(
|
||||||
|
summary_bucket.first_byte_time_ms_sum,
|
||||||
|
summary_bucket.first_byte_sample_count,
|
||||||
|
),
|
||||||
|
avg_response_time_ms: avg(
|
||||||
|
summary_bucket.response_time_ms_sum,
|
||||||
|
summary_bucket.response_time_sample_count,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut providers = grouped
|
||||||
|
.into_iter()
|
||||||
|
.map(|(provider_id, mut bucket)| {
|
||||||
|
let p90_response_time_ms = usage_percentile_cont(&mut bucket.response_times, 0.9);
|
||||||
|
let p90_first_byte_time_ms =
|
||||||
|
usage_percentile_cont(&mut bucket.first_byte_times, 0.9);
|
||||||
|
StoredUsageProviderPerformanceProviderRow {
|
||||||
|
provider_id,
|
||||||
|
provider: bucket.provider,
|
||||||
|
request_count: bucket.request_count,
|
||||||
|
success_count: bucket.success_count,
|
||||||
|
output_tokens: bucket.output_tokens,
|
||||||
|
avg_output_tps: avg_tps(
|
||||||
|
bucket.tps_output_tokens,
|
||||||
|
bucket.tps_response_time_ms_sum,
|
||||||
|
),
|
||||||
|
avg_first_byte_time_ms: avg(
|
||||||
|
bucket.first_byte_time_ms_sum,
|
||||||
|
bucket.first_byte_sample_count,
|
||||||
|
),
|
||||||
|
avg_response_time_ms: avg(
|
||||||
|
bucket.response_time_ms_sum,
|
||||||
|
bucket.response_time_sample_count,
|
||||||
|
),
|
||||||
|
p90_response_time_ms,
|
||||||
|
p90_first_byte_time_ms,
|
||||||
|
tps_sample_count: bucket.tps_sample_count,
|
||||||
|
first_byte_sample_count: bucket.first_byte_sample_count,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
providers.sort_by(|left, right| {
|
||||||
|
right
|
||||||
|
.request_count
|
||||||
|
.cmp(&left.request_count)
|
||||||
|
.then_with(|| left.provider_id.cmp(&right.provider_id))
|
||||||
|
});
|
||||||
|
providers.truncate(query.limit.max(1));
|
||||||
|
|
||||||
|
let top_provider_ids = providers
|
||||||
|
.iter()
|
||||||
|
.map(|row| row.provider_id.clone())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let mut timeline_grouped = BTreeMap::<(String, String), ProviderPerformanceBucket>::new();
|
||||||
|
for item in &usage {
|
||||||
|
let Some((provider_id, provider)) =
|
||||||
|
usage_matches_provider_performance_query(item, query)
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if !top_provider_ids.iter().any(|value| value == &provider_id) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(bucket_key) =
|
||||||
|
usage_time_series_bucket_key(item, query.granularity, query.tz_offset_minutes)
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let bucket = timeline_grouped
|
||||||
|
.entry((bucket_key, provider_id))
|
||||||
|
.or_default();
|
||||||
|
if bucket.provider.is_empty() {
|
||||||
|
bucket.provider = provider;
|
||||||
|
}
|
||||||
|
bucket.add(item);
|
||||||
|
}
|
||||||
|
|
||||||
|
let timeline = timeline_grouped
|
||||||
|
.into_iter()
|
||||||
|
.map(
|
||||||
|
|((date, provider_id), bucket)| StoredUsageProviderPerformanceTimelineRow {
|
||||||
|
date,
|
||||||
|
provider_id,
|
||||||
|
provider: bucket.provider,
|
||||||
|
request_count: bucket.request_count,
|
||||||
|
success_count: bucket.success_count,
|
||||||
|
output_tokens: bucket.output_tokens,
|
||||||
|
avg_output_tps: avg_tps(
|
||||||
|
bucket.tps_output_tokens,
|
||||||
|
bucket.tps_response_time_ms_sum,
|
||||||
|
),
|
||||||
|
avg_first_byte_time_ms: avg(
|
||||||
|
bucket.first_byte_time_ms_sum,
|
||||||
|
bucket.first_byte_sample_count,
|
||||||
|
),
|
||||||
|
avg_response_time_ms: avg(
|
||||||
|
bucket.response_time_ms_sum,
|
||||||
|
bucket.response_time_sample_count,
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
Ok(StoredUsageProviderPerformance {
|
||||||
|
summary,
|
||||||
|
providers,
|
||||||
|
timeline,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async fn summarize_usage_cost_savings(
|
async fn summarize_usage_cost_savings(
|
||||||
&self,
|
&self,
|
||||||
query: &UsageCostSavingsSummaryQuery,
|
query: &UsageCostSavingsSummaryQuery,
|
||||||
@@ -2579,7 +2825,9 @@ mod tests {
|
|||||||
StoredProviderUsageWindow, StoredRequestUsageAudit, UpsertUsageRecord, UsageReadRepository,
|
StoredProviderUsageWindow, StoredRequestUsageAudit, UpsertUsageRecord, UsageReadRepository,
|
||||||
UsageWriteRepository,
|
UsageWriteRepository,
|
||||||
};
|
};
|
||||||
use aether_data_contracts::repository::usage::{usage_body_ref, UsageBodyField};
|
use aether_data_contracts::repository::usage::{
|
||||||
|
usage_body_ref, UsageBodyField, UsageProviderPerformanceQuery, UsageTimeSeriesGranularity,
|
||||||
|
};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
fn sample_usage(request_id: &str, created_at_unix_ms: i64) -> StoredRequestUsageAudit {
|
fn sample_usage(request_id: &str, created_at_unix_ms: i64) -> StoredRequestUsageAudit {
|
||||||
@@ -4562,4 +4810,76 @@ mod tests {
|
|||||||
assert_eq!(key.total_tokens, 300);
|
assert_eq!(key.total_tokens, 300);
|
||||||
assert_eq!(key.total_cost_usd, 0.24);
|
assert_eq!(key.total_cost_usd, 0.24);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn summarize_usage_provider_performance_computes_tps_and_top_provider_timeline() {
|
||||||
|
let mut first = sample_usage("req-provider-perf-1", 1_711_000_000);
|
||||||
|
first.output_tokens = 60;
|
||||||
|
first.response_time_ms = Some(3000);
|
||||||
|
first.first_byte_time_ms = Some(100);
|
||||||
|
|
||||||
|
let mut second = sample_usage("req-provider-perf-2", 1_711_000_300);
|
||||||
|
second.output_tokens = 40;
|
||||||
|
second.response_time_ms = Some(1000);
|
||||||
|
second.first_byte_time_ms = Some(200);
|
||||||
|
|
||||||
|
let mut failed = sample_usage("req-provider-perf-failed", 1_711_000_400);
|
||||||
|
failed.output_tokens = 999;
|
||||||
|
failed.response_time_ms = Some(10);
|
||||||
|
failed.first_byte_time_ms = Some(1);
|
||||||
|
failed.status = "failed".to_string();
|
||||||
|
failed.status_code = Some(500);
|
||||||
|
|
||||||
|
let mut other_provider = sample_usage("req-provider-perf-other", 1_711_003_600);
|
||||||
|
other_provider.provider_id = Some("provider-2".to_string());
|
||||||
|
other_provider.provider_name = "Anthropic".to_string();
|
||||||
|
other_provider.output_tokens = 30;
|
||||||
|
other_provider.response_time_ms = Some(3000);
|
||||||
|
other_provider.first_byte_time_ms = None;
|
||||||
|
|
||||||
|
let repository =
|
||||||
|
InMemoryUsageReadRepository::seed(vec![first, second, failed, other_provider]);
|
||||||
|
let summary = repository
|
||||||
|
.summarize_usage_provider_performance(&UsageProviderPerformanceQuery {
|
||||||
|
created_from_unix_secs: 1_711_000_000,
|
||||||
|
created_until_unix_secs: 1_711_010_000,
|
||||||
|
granularity: UsageTimeSeriesGranularity::Hour,
|
||||||
|
tz_offset_minutes: 0,
|
||||||
|
limit: 1,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("provider performance should summarize");
|
||||||
|
|
||||||
|
assert_eq!(summary.summary.request_count, 4);
|
||||||
|
assert_eq!(summary.summary.success_count, 3);
|
||||||
|
assert!((summary.summary.avg_output_tps.expect("summary tps") - 18.571_428).abs() < 0.001);
|
||||||
|
assert_eq!(summary.summary.avg_first_byte_time_ms, Some(150.0));
|
||||||
|
assert!(
|
||||||
|
(summary
|
||||||
|
.summary
|
||||||
|
.avg_response_time_ms
|
||||||
|
.expect("summary response")
|
||||||
|
- 2333.333)
|
||||||
|
.abs()
|
||||||
|
< 0.001
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(summary.providers.len(), 1);
|
||||||
|
let provider = &summary.providers[0];
|
||||||
|
assert_eq!(provider.provider_id, "provider-1");
|
||||||
|
assert_eq!(provider.request_count, 3);
|
||||||
|
assert_eq!(provider.success_count, 2);
|
||||||
|
assert_eq!(provider.output_tokens, 1099);
|
||||||
|
assert_eq!(provider.avg_output_tps, Some(25.0));
|
||||||
|
assert_eq!(provider.avg_first_byte_time_ms, Some(150.0));
|
||||||
|
assert_eq!(provider.avg_response_time_ms, Some(2000.0));
|
||||||
|
assert_eq!(provider.p90_response_time_ms, None);
|
||||||
|
assert_eq!(provider.tps_sample_count, 2);
|
||||||
|
assert_eq!(provider.first_byte_sample_count, 2);
|
||||||
|
|
||||||
|
assert_eq!(summary.timeline.len(), 1);
|
||||||
|
assert_eq!(summary.timeline[0].date, "2024-03-21T05:00:00+00:00");
|
||||||
|
assert_eq!(summary.timeline[0].provider_id, "provider-1");
|
||||||
|
assert_eq!(summary.timeline[0].avg_output_tps, Some(25.0));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,7 +10,9 @@ pub(crate) use aether_data_contracts::repository::usage::{
|
|||||||
StoredUsageDailySummary, StoredUsageDashboardDailyBreakdownRow,
|
StoredUsageDailySummary, StoredUsageDashboardDailyBreakdownRow,
|
||||||
StoredUsageDashboardProviderCount, StoredUsageDashboardSummary,
|
StoredUsageDashboardProviderCount, StoredUsageDashboardSummary,
|
||||||
StoredUsageErrorDistributionRow, StoredUsageLeaderboardSummary,
|
StoredUsageErrorDistributionRow, StoredUsageLeaderboardSummary,
|
||||||
StoredUsagePerformancePercentilesRow, StoredUsageSettledCostSummary,
|
StoredUsagePerformancePercentilesRow, StoredUsageProviderPerformance,
|
||||||
|
StoredUsageProviderPerformanceProviderRow, StoredUsageProviderPerformanceSummary,
|
||||||
|
StoredUsageProviderPerformanceTimelineRow, StoredUsageSettledCostSummary,
|
||||||
StoredUsageTimeSeriesBucket, StoredUsageUserTotals, UpsertUsageRecord,
|
StoredUsageTimeSeriesBucket, StoredUsageUserTotals, UpsertUsageRecord,
|
||||||
UsageAuditAggregationGroupBy, UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery,
|
UsageAuditAggregationGroupBy, UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery,
|
||||||
UsageAuditListQuery, UsageAuditSummaryQuery, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery,
|
UsageAuditListQuery, UsageAuditSummaryQuery, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery,
|
||||||
@@ -19,9 +21,9 @@ pub(crate) use aether_data_contracts::repository::usage::{
|
|||||||
UsageDailyHeatmapQuery, UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
|
UsageDailyHeatmapQuery, UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
|
||||||
UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy,
|
UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy,
|
||||||
UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery,
|
UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery,
|
||||||
UsagePerformancePercentilesQuery, UsageReadRepository, UsageRepository,
|
UsagePerformancePercentilesQuery, UsageProviderPerformanceQuery, UsageReadRepository,
|
||||||
UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity, UsageTimeSeriesQuery,
|
UsageRepository, UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity,
|
||||||
UsageWriteRepository,
|
UsageTimeSeriesQuery, UsageWriteRepository,
|
||||||
};
|
};
|
||||||
pub use memory::InMemoryUsageReadRepository;
|
pub use memory::InMemoryUsageReadRepository;
|
||||||
pub use sql::SqlxUsageReadRepository;
|
pub use sql::SqlxUsageReadRepository;
|
||||||
|
|||||||
@@ -4,7 +4,9 @@ use aether_data_contracts::repository::usage::{
|
|||||||
StoredUsageCacheAffinityIntervalRow, StoredUsageCacheHitSummary, StoredUsageCostSavingsSummary,
|
StoredUsageCacheAffinityIntervalRow, StoredUsageCacheHitSummary, StoredUsageCostSavingsSummary,
|
||||||
StoredUsageDashboardDailyBreakdownRow, StoredUsageDashboardProviderCount,
|
StoredUsageDashboardDailyBreakdownRow, StoredUsageDashboardProviderCount,
|
||||||
StoredUsageDashboardSummary, StoredUsageErrorDistributionRow, StoredUsageLeaderboardSummary,
|
StoredUsageDashboardSummary, StoredUsageErrorDistributionRow, StoredUsageLeaderboardSummary,
|
||||||
StoredUsagePerformancePercentilesRow, StoredUsageSettledCostSummary,
|
StoredUsagePerformancePercentilesRow, StoredUsageProviderPerformance,
|
||||||
|
StoredUsageProviderPerformanceProviderRow, StoredUsageProviderPerformanceSummary,
|
||||||
|
StoredUsageProviderPerformanceTimelineRow, StoredUsageSettledCostSummary,
|
||||||
StoredUsageTimeSeriesBucket, StoredUsageUserTotals, UsageAuditAggregationGroupBy,
|
StoredUsageTimeSeriesBucket, StoredUsageUserTotals, UsageAuditAggregationGroupBy,
|
||||||
UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery, UsageAuditSummaryQuery,
|
UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery, UsageAuditSummaryQuery,
|
||||||
UsageBodyCaptureState, UsageBodyField, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery,
|
UsageBodyCaptureState, UsageBodyField, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery,
|
||||||
@@ -13,8 +15,8 @@ use aether_data_contracts::repository::usage::{
|
|||||||
UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
|
UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
|
||||||
UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy,
|
UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy,
|
||||||
UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery,
|
UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery,
|
||||||
UsagePerformancePercentilesQuery, UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity,
|
UsagePerformancePercentilesQuery, UsageProviderPerformanceQuery, UsageSettledCostSummaryQuery,
|
||||||
UsageTimeSeriesQuery,
|
UsageTimeSeriesGranularity, UsageTimeSeriesQuery,
|
||||||
};
|
};
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, Utc};
|
||||||
@@ -803,6 +805,107 @@ fn decode_usage_performance_percentiles_row(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn decode_usage_provider_performance_summary(
|
||||||
|
row: &PgRow,
|
||||||
|
) -> Result<StoredUsageProviderPerformanceSummary, DataLayerError> {
|
||||||
|
Ok(StoredUsageProviderPerformanceSummary {
|
||||||
|
request_count: row
|
||||||
|
.try_get::<i64, _>("request_count")
|
||||||
|
.map_postgres_err()?
|
||||||
|
.max(0) as u64,
|
||||||
|
success_count: row
|
||||||
|
.try_get::<i64, _>("success_count")
|
||||||
|
.map_postgres_err()?
|
||||||
|
.max(0) as u64,
|
||||||
|
avg_output_tps: row
|
||||||
|
.try_get::<Option<f64>, _>("avg_output_tps")
|
||||||
|
.map_postgres_err()?,
|
||||||
|
avg_first_byte_time_ms: row
|
||||||
|
.try_get::<Option<f64>, _>("avg_first_byte_time_ms")
|
||||||
|
.map_postgres_err()?,
|
||||||
|
avg_response_time_ms: row
|
||||||
|
.try_get::<Option<f64>, _>("avg_response_time_ms")
|
||||||
|
.map_postgres_err()?,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_usage_provider_performance_provider_row(
|
||||||
|
row: &PgRow,
|
||||||
|
) -> Result<StoredUsageProviderPerformanceProviderRow, DataLayerError> {
|
||||||
|
Ok(StoredUsageProviderPerformanceProviderRow {
|
||||||
|
provider_id: row.try_get::<String, _>("provider_id").map_postgres_err()?,
|
||||||
|
provider: row.try_get::<String, _>("provider").map_postgres_err()?,
|
||||||
|
request_count: row
|
||||||
|
.try_get::<i64, _>("request_count")
|
||||||
|
.map_postgres_err()?
|
||||||
|
.max(0) as u64,
|
||||||
|
success_count: row
|
||||||
|
.try_get::<i64, _>("success_count")
|
||||||
|
.map_postgres_err()?
|
||||||
|
.max(0) as u64,
|
||||||
|
output_tokens: row
|
||||||
|
.try_get::<i64, _>("output_tokens")
|
||||||
|
.map_postgres_err()?
|
||||||
|
.max(0) as u64,
|
||||||
|
avg_output_tps: row
|
||||||
|
.try_get::<Option<f64>, _>("avg_output_tps")
|
||||||
|
.map_postgres_err()?,
|
||||||
|
avg_first_byte_time_ms: row
|
||||||
|
.try_get::<Option<f64>, _>("avg_first_byte_time_ms")
|
||||||
|
.map_postgres_err()?,
|
||||||
|
avg_response_time_ms: row
|
||||||
|
.try_get::<Option<f64>, _>("avg_response_time_ms")
|
||||||
|
.map_postgres_err()?,
|
||||||
|
p90_response_time_ms: row
|
||||||
|
.try_get::<Option<i64>, _>("p90_response_time_ms")
|
||||||
|
.map_postgres_err()?
|
||||||
|
.map(|value| value.max(0) as u64),
|
||||||
|
p90_first_byte_time_ms: row
|
||||||
|
.try_get::<Option<i64>, _>("p90_first_byte_time_ms")
|
||||||
|
.map_postgres_err()?
|
||||||
|
.map(|value| value.max(0) as u64),
|
||||||
|
tps_sample_count: row
|
||||||
|
.try_get::<i64, _>("tps_sample_count")
|
||||||
|
.map_postgres_err()?
|
||||||
|
.max(0) as u64,
|
||||||
|
first_byte_sample_count: row
|
||||||
|
.try_get::<i64, _>("first_byte_sample_count")
|
||||||
|
.map_postgres_err()?
|
||||||
|
.max(0) as u64,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_usage_provider_performance_timeline_row(
|
||||||
|
row: &PgRow,
|
||||||
|
) -> Result<StoredUsageProviderPerformanceTimelineRow, DataLayerError> {
|
||||||
|
Ok(StoredUsageProviderPerformanceTimelineRow {
|
||||||
|
date: row.try_get::<String, _>("date").map_postgres_err()?,
|
||||||
|
provider_id: row.try_get::<String, _>("provider_id").map_postgres_err()?,
|
||||||
|
provider: row.try_get::<String, _>("provider").map_postgres_err()?,
|
||||||
|
request_count: row
|
||||||
|
.try_get::<i64, _>("request_count")
|
||||||
|
.map_postgres_err()?
|
||||||
|
.max(0) as u64,
|
||||||
|
success_count: row
|
||||||
|
.try_get::<i64, _>("success_count")
|
||||||
|
.map_postgres_err()?
|
||||||
|
.max(0) as u64,
|
||||||
|
output_tokens: row
|
||||||
|
.try_get::<i64, _>("output_tokens")
|
||||||
|
.map_postgres_err()?
|
||||||
|
.max(0) as u64,
|
||||||
|
avg_output_tps: row
|
||||||
|
.try_get::<Option<f64>, _>("avg_output_tps")
|
||||||
|
.map_postgres_err()?,
|
||||||
|
avg_first_byte_time_ms: row
|
||||||
|
.try_get::<Option<f64>, _>("avg_first_byte_time_ms")
|
||||||
|
.map_postgres_err()?,
|
||||||
|
avg_response_time_ms: row
|
||||||
|
.try_get::<Option<f64>, _>("avg_response_time_ms")
|
||||||
|
.map_postgres_err()?,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn decode_usage_time_series_bucket_row(
|
fn decode_usage_time_series_bucket_row(
|
||||||
row: &PgRow,
|
row: &PgRow,
|
||||||
) -> Result<StoredUsageTimeSeriesBucket, DataLayerError> {
|
) -> Result<StoredUsageTimeSeriesBucket, DataLayerError> {
|
||||||
@@ -4371,6 +4474,306 @@ ORDER BY date ASC
|
|||||||
Ok(items)
|
Ok(items)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn summarize_usage_provider_performance_summary(
|
||||||
|
&self,
|
||||||
|
query: &UsageProviderPerformanceQuery,
|
||||||
|
) -> Result<StoredUsageProviderPerformanceSummary, DataLayerError> {
|
||||||
|
let mut builder = QueryBuilder::<Postgres>::new(
|
||||||
|
r#"
|
||||||
|
WITH filtered_usage AS (
|
||||||
|
SELECT
|
||||||
|
GREATEST(COALESCE("usage".output_tokens, 0), 0) AS output_tokens,
|
||||||
|
GREATEST(COALESCE("usage".response_time_ms, 0), 0) AS response_time_ms,
|
||||||
|
GREATEST(COALESCE("usage".first_byte_time_ms, 0), 0) AS first_byte_time_ms,
|
||||||
|
"usage".response_time_ms IS NOT NULL AS has_response_time,
|
||||||
|
"usage".first_byte_time_ms IS NOT NULL AS has_first_byte_time,
|
||||||
|
CASE
|
||||||
|
WHEN lower(COALESCE("usage".status, '')) IN ('completed', 'success', 'ok', 'billed', 'settled')
|
||||||
|
AND ("usage".status_code IS NULL OR "usage".status_code < 400)
|
||||||
|
THEN 1
|
||||||
|
ELSE 0
|
||||||
|
END AS success_flag
|
||||||
|
FROM usage_billing_facts AS "usage"
|
||||||
|
WHERE "usage".created_at >= TO_TIMESTAMP("#,
|
||||||
|
);
|
||||||
|
builder.push_bind(query.created_from_unix_secs as f64);
|
||||||
|
builder.push(
|
||||||
|
r#"::double precision)
|
||||||
|
AND "usage".created_at < TO_TIMESTAMP("#,
|
||||||
|
);
|
||||||
|
builder.push_bind(query.created_until_unix_secs as f64);
|
||||||
|
builder.push(
|
||||||
|
r#"::double precision)
|
||||||
|
AND COALESCE("usage".status, '') NOT IN ('pending', 'streaming')
|
||||||
|
AND NULLIF(BTRIM(COALESCE("usage".provider_id, '')), '') IS NOT NULL
|
||||||
|
AND lower(BTRIM(COALESCE("usage".provider_id, ''))) NOT IN ('unknown', 'pending')
|
||||||
|
AND lower(BTRIM(COALESCE("usage".provider_name, ''))) NOT IN ('unknown', 'pending')
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
COUNT(*)::BIGINT AS request_count,
|
||||||
|
COALESCE(SUM(success_flag), 0)::BIGINT AS success_count,
|
||||||
|
CASE
|
||||||
|
WHEN COALESCE(SUM(CASE
|
||||||
|
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
|
||||||
|
THEN response_time_ms
|
||||||
|
ELSE 0
|
||||||
|
END), 0) > 0
|
||||||
|
THEN COALESCE(SUM(CASE
|
||||||
|
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
|
||||||
|
THEN output_tokens
|
||||||
|
ELSE 0
|
||||||
|
END), 0)::DOUBLE PRECISION * 1000.0 / COALESCE(SUM(CASE
|
||||||
|
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
|
||||||
|
THEN response_time_ms
|
||||||
|
ELSE 0
|
||||||
|
END), 0)::DOUBLE PRECISION
|
||||||
|
ELSE NULL
|
||||||
|
END AS avg_output_tps,
|
||||||
|
AVG(first_byte_time_ms::DOUBLE PRECISION)
|
||||||
|
FILTER (WHERE success_flag = 1 AND has_first_byte_time) AS avg_first_byte_time_ms,
|
||||||
|
AVG(response_time_ms::DOUBLE PRECISION)
|
||||||
|
FILTER (WHERE success_flag = 1 AND has_response_time) AS avg_response_time_ms
|
||||||
|
FROM filtered_usage
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
|
||||||
|
let row = builder
|
||||||
|
.build()
|
||||||
|
.fetch_one(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_postgres_err()?;
|
||||||
|
decode_usage_provider_performance_summary(&row)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn summarize_usage_provider_performance_providers(
|
||||||
|
&self,
|
||||||
|
query: &UsageProviderPerformanceQuery,
|
||||||
|
) -> Result<Vec<StoredUsageProviderPerformanceProviderRow>, DataLayerError> {
|
||||||
|
let mut builder = QueryBuilder::<Postgres>::new(
|
||||||
|
r#"
|
||||||
|
WITH filtered_usage AS (
|
||||||
|
SELECT
|
||||||
|
COALESCE("usage".provider_id, '') AS provider_id,
|
||||||
|
COALESCE(NULLIF(BTRIM("usage".provider_name), ''), COALESCE("usage".provider_id, '')) AS provider,
|
||||||
|
GREATEST(COALESCE("usage".output_tokens, 0), 0) AS output_tokens,
|
||||||
|
GREATEST(COALESCE("usage".response_time_ms, 0), 0) AS response_time_ms,
|
||||||
|
GREATEST(COALESCE("usage".first_byte_time_ms, 0), 0) AS first_byte_time_ms,
|
||||||
|
"usage".response_time_ms IS NOT NULL AS has_response_time,
|
||||||
|
"usage".first_byte_time_ms IS NOT NULL AS has_first_byte_time,
|
||||||
|
CASE
|
||||||
|
WHEN lower(COALESCE("usage".status, '')) IN ('completed', 'success', 'ok', 'billed', 'settled')
|
||||||
|
AND ("usage".status_code IS NULL OR "usage".status_code < 400)
|
||||||
|
THEN 1
|
||||||
|
ELSE 0
|
||||||
|
END AS success_flag
|
||||||
|
FROM usage_billing_facts AS "usage"
|
||||||
|
WHERE "usage".created_at >= TO_TIMESTAMP("#,
|
||||||
|
);
|
||||||
|
builder.push_bind(query.created_from_unix_secs as f64);
|
||||||
|
builder.push(
|
||||||
|
r#"::double precision)
|
||||||
|
AND "usage".created_at < TO_TIMESTAMP("#,
|
||||||
|
);
|
||||||
|
builder.push_bind(query.created_until_unix_secs as f64);
|
||||||
|
builder.push(
|
||||||
|
r#"::double precision)
|
||||||
|
AND COALESCE("usage".status, '') NOT IN ('pending', 'streaming')
|
||||||
|
AND NULLIF(BTRIM(COALESCE("usage".provider_id, '')), '') IS NOT NULL
|
||||||
|
AND lower(BTRIM(COALESCE("usage".provider_id, ''))) NOT IN ('unknown', 'pending')
|
||||||
|
AND lower(BTRIM(COALESCE("usage".provider_name, ''))) NOT IN ('unknown', 'pending')
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
provider_id,
|
||||||
|
COALESCE(MAX(NULLIF(provider, '')), provider_id) AS provider,
|
||||||
|
COUNT(*)::BIGINT AS request_count,
|
||||||
|
COALESCE(SUM(success_flag), 0)::BIGINT AS success_count,
|
||||||
|
COALESCE(SUM(output_tokens), 0)::BIGINT AS output_tokens,
|
||||||
|
CASE
|
||||||
|
WHEN COALESCE(SUM(CASE
|
||||||
|
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
|
||||||
|
THEN response_time_ms
|
||||||
|
ELSE 0
|
||||||
|
END), 0) > 0
|
||||||
|
THEN COALESCE(SUM(CASE
|
||||||
|
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
|
||||||
|
THEN output_tokens
|
||||||
|
ELSE 0
|
||||||
|
END), 0)::DOUBLE PRECISION * 1000.0 / COALESCE(SUM(CASE
|
||||||
|
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
|
||||||
|
THEN response_time_ms
|
||||||
|
ELSE 0
|
||||||
|
END), 0)::DOUBLE PRECISION
|
||||||
|
ELSE NULL
|
||||||
|
END AS avg_output_tps,
|
||||||
|
AVG(first_byte_time_ms::DOUBLE PRECISION)
|
||||||
|
FILTER (WHERE success_flag = 1 AND has_first_byte_time) AS avg_first_byte_time_ms,
|
||||||
|
AVG(response_time_ms::DOUBLE PRECISION)
|
||||||
|
FILTER (WHERE success_flag = 1 AND has_response_time) AS avg_response_time_ms,
|
||||||
|
CASE
|
||||||
|
WHEN COUNT(response_time_ms) FILTER (WHERE success_flag = 1 AND has_response_time) >= 10
|
||||||
|
THEN FLOOR(PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY response_time_ms)
|
||||||
|
FILTER (WHERE success_flag = 1 AND has_response_time))::BIGINT
|
||||||
|
ELSE NULL
|
||||||
|
END AS p90_response_time_ms,
|
||||||
|
CASE
|
||||||
|
WHEN COUNT(first_byte_time_ms) FILTER (WHERE success_flag = 1 AND has_first_byte_time) >= 10
|
||||||
|
THEN FLOOR(PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY first_byte_time_ms)
|
||||||
|
FILTER (WHERE success_flag = 1 AND has_first_byte_time))::BIGINT
|
||||||
|
ELSE NULL
|
||||||
|
END AS p90_first_byte_time_ms,
|
||||||
|
COALESCE(SUM(CASE
|
||||||
|
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
|
||||||
|
THEN 1
|
||||||
|
ELSE 0
|
||||||
|
END), 0)::BIGINT AS tps_sample_count,
|
||||||
|
(COUNT(first_byte_time_ms) FILTER (WHERE success_flag = 1 AND has_first_byte_time))::BIGINT
|
||||||
|
AS first_byte_sample_count
|
||||||
|
FROM filtered_usage
|
||||||
|
GROUP BY provider_id
|
||||||
|
ORDER BY request_count DESC, provider_id ASC
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut rows = builder.build().fetch(&self.pool);
|
||||||
|
let mut items = Vec::new();
|
||||||
|
while let Some(row) = rows.try_next().await.map_postgres_err()? {
|
||||||
|
items.push(decode_usage_provider_performance_provider_row(&row)?);
|
||||||
|
}
|
||||||
|
Ok(items)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn summarize_usage_provider_performance_timeline(
|
||||||
|
&self,
|
||||||
|
query: &UsageProviderPerformanceQuery,
|
||||||
|
provider_ids: &[String],
|
||||||
|
) -> Result<Vec<StoredUsageProviderPerformanceTimelineRow>, DataLayerError> {
|
||||||
|
if provider_ids.is_empty() {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut builder = QueryBuilder::<Postgres>::new("WITH filtered_usage AS ( SELECT ");
|
||||||
|
match query.granularity {
|
||||||
|
UsageTimeSeriesGranularity::Day => {
|
||||||
|
builder
|
||||||
|
.push("TO_CHAR(date_trunc('day', \"usage\".created_at + (")
|
||||||
|
.push_bind(query.tz_offset_minutes)
|
||||||
|
.push("::integer * INTERVAL '1 minute')), 'YYYY-MM-DD') AS date");
|
||||||
|
}
|
||||||
|
UsageTimeSeriesGranularity::Hour => {
|
||||||
|
builder
|
||||||
|
.push("TO_CHAR(date_trunc('hour', \"usage\".created_at + (")
|
||||||
|
.push_bind(query.tz_offset_minutes)
|
||||||
|
.push("::integer * INTERVAL '1 minute')), 'YYYY-MM-DD\"T\"HH24:00:00+00:00') AS date");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
builder.push(
|
||||||
|
r#",
|
||||||
|
COALESCE("usage".provider_id, '') AS provider_id,
|
||||||
|
COALESCE(NULLIF(BTRIM("usage".provider_name), ''), COALESCE("usage".provider_id, '')) AS provider,
|
||||||
|
GREATEST(COALESCE("usage".output_tokens, 0), 0) AS output_tokens,
|
||||||
|
GREATEST(COALESCE("usage".response_time_ms, 0), 0) AS response_time_ms,
|
||||||
|
GREATEST(COALESCE("usage".first_byte_time_ms, 0), 0) AS first_byte_time_ms,
|
||||||
|
"usage".response_time_ms IS NOT NULL AS has_response_time,
|
||||||
|
"usage".first_byte_time_ms IS NOT NULL AS has_first_byte_time,
|
||||||
|
CASE
|
||||||
|
WHEN lower(COALESCE("usage".status, '')) IN ('completed', 'success', 'ok', 'billed', 'settled')
|
||||||
|
AND ("usage".status_code IS NULL OR "usage".status_code < 400)
|
||||||
|
THEN 1
|
||||||
|
ELSE 0
|
||||||
|
END AS success_flag
|
||||||
|
FROM usage_billing_facts AS "usage"
|
||||||
|
WHERE "usage".created_at >= TO_TIMESTAMP("#,
|
||||||
|
);
|
||||||
|
builder.push_bind(query.created_from_unix_secs as f64);
|
||||||
|
builder.push(
|
||||||
|
r#"::double precision)
|
||||||
|
AND "usage".created_at < TO_TIMESTAMP("#,
|
||||||
|
);
|
||||||
|
builder.push_bind(query.created_until_unix_secs as f64);
|
||||||
|
builder.push(
|
||||||
|
r#"::double precision)
|
||||||
|
AND COALESCE("usage".status, '') NOT IN ('pending', 'streaming')
|
||||||
|
AND NULLIF(BTRIM(COALESCE("usage".provider_id, '')), '') IS NOT NULL
|
||||||
|
AND lower(BTRIM(COALESCE("usage".provider_id, ''))) NOT IN ('unknown', 'pending')
|
||||||
|
AND lower(BTRIM(COALESCE("usage".provider_name, ''))) NOT IN ('unknown', 'pending')
|
||||||
|
AND "usage".provider_id = ANY("#,
|
||||||
|
);
|
||||||
|
builder.push_bind(provider_ids.to_vec());
|
||||||
|
builder.push(
|
||||||
|
r#")
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
date,
|
||||||
|
provider_id,
|
||||||
|
COALESCE(MAX(NULLIF(provider, '')), provider_id) AS provider,
|
||||||
|
COUNT(*)::BIGINT AS request_count,
|
||||||
|
COALESCE(SUM(success_flag), 0)::BIGINT AS success_count,
|
||||||
|
COALESCE(SUM(output_tokens), 0)::BIGINT AS output_tokens,
|
||||||
|
CASE
|
||||||
|
WHEN COALESCE(SUM(CASE
|
||||||
|
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
|
||||||
|
THEN response_time_ms
|
||||||
|
ELSE 0
|
||||||
|
END), 0) > 0
|
||||||
|
THEN COALESCE(SUM(CASE
|
||||||
|
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
|
||||||
|
THEN output_tokens
|
||||||
|
ELSE 0
|
||||||
|
END), 0)::DOUBLE PRECISION * 1000.0 / COALESCE(SUM(CASE
|
||||||
|
WHEN success_flag = 1 AND response_time_ms > 0 AND output_tokens > 0
|
||||||
|
THEN response_time_ms
|
||||||
|
ELSE 0
|
||||||
|
END), 0)::DOUBLE PRECISION
|
||||||
|
ELSE NULL
|
||||||
|
END AS avg_output_tps,
|
||||||
|
AVG(first_byte_time_ms::DOUBLE PRECISION)
|
||||||
|
FILTER (WHERE success_flag = 1 AND has_first_byte_time) AS avg_first_byte_time_ms,
|
||||||
|
AVG(response_time_ms::DOUBLE PRECISION)
|
||||||
|
FILTER (WHERE success_flag = 1 AND has_response_time) AS avg_response_time_ms
|
||||||
|
FROM filtered_usage
|
||||||
|
GROUP BY date, provider_id
|
||||||
|
ORDER BY date ASC, provider_id ASC
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut rows = builder.build().fetch(&self.pool);
|
||||||
|
let mut items = Vec::new();
|
||||||
|
while let Some(row) = rows.try_next().await.map_postgres_err()? {
|
||||||
|
items.push(decode_usage_provider_performance_timeline_row(&row)?);
|
||||||
|
}
|
||||||
|
Ok(items)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn summarize_usage_provider_performance(
|
||||||
|
&self,
|
||||||
|
query: &UsageProviderPerformanceQuery,
|
||||||
|
) -> Result<StoredUsageProviderPerformance, DataLayerError> {
|
||||||
|
if query.created_from_unix_secs >= query.created_until_unix_secs {
|
||||||
|
return Ok(StoredUsageProviderPerformance::default());
|
||||||
|
}
|
||||||
|
|
||||||
|
let summary = self
|
||||||
|
.summarize_usage_provider_performance_summary(query)
|
||||||
|
.await?;
|
||||||
|
let mut providers = self
|
||||||
|
.summarize_usage_provider_performance_providers(query)
|
||||||
|
.await?;
|
||||||
|
providers.truncate(query.limit.max(1));
|
||||||
|
let provider_ids = providers
|
||||||
|
.iter()
|
||||||
|
.map(|row| row.provider_id.clone())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let timeline = self
|
||||||
|
.summarize_usage_provider_performance_timeline(query, &provider_ids)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok(StoredUsageProviderPerformance {
|
||||||
|
summary,
|
||||||
|
providers,
|
||||||
|
timeline,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async fn summarize_usage_cost_savings_raw_from_range(
|
async fn summarize_usage_cost_savings_raw_from_range(
|
||||||
&self,
|
&self,
|
||||||
start_utc: DateTime<Utc>,
|
start_utc: DateTime<Utc>,
|
||||||
@@ -6775,6 +7178,13 @@ impl UsageReadRepository for SqlxUsageReadRepository {
|
|||||||
Self::summarize_usage_performance_percentiles(self, query).await
|
Self::summarize_usage_performance_percentiles(self, query).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn summarize_usage_provider_performance(
|
||||||
|
&self,
|
||||||
|
query: &UsageProviderPerformanceQuery,
|
||||||
|
) -> Result<StoredUsageProviderPerformance, DataLayerError> {
|
||||||
|
Self::summarize_usage_provider_performance(self, query).await
|
||||||
|
}
|
||||||
|
|
||||||
async fn summarize_usage_cost_savings(
|
async fn summarize_usage_cost_savings(
|
||||||
&self,
|
&self,
|
||||||
query: &UsageCostSavingsSummaryQuery,
|
query: &UsageCostSavingsSummaryQuery,
|
||||||
|
|||||||
@@ -447,6 +447,49 @@ export interface PercentileItem {
|
|||||||
p99_first_byte_time_ms?: number | null
|
p99_first_byte_time_ms?: number | null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ProviderPerformanceSummary {
|
||||||
|
request_count: number
|
||||||
|
success_rate: number
|
||||||
|
avg_output_tps: number | null
|
||||||
|
avg_first_byte_time_ms: number | null
|
||||||
|
avg_response_time_ms: number | null
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ProviderPerformanceItem {
|
||||||
|
provider_id: string
|
||||||
|
provider: string
|
||||||
|
request_count: number
|
||||||
|
success_count: number
|
||||||
|
error_count: number
|
||||||
|
success_rate: number
|
||||||
|
output_tokens: number
|
||||||
|
avg_output_tps: number | null
|
||||||
|
avg_first_byte_time_ms: number | null
|
||||||
|
avg_response_time_ms: number | null
|
||||||
|
p90_response_time_ms: number | null
|
||||||
|
p90_first_byte_time_ms: number | null
|
||||||
|
tps_sample_count: number
|
||||||
|
first_byte_sample_count: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ProviderPerformanceTimelineItem {
|
||||||
|
date: string
|
||||||
|
provider_id: string
|
||||||
|
provider: string
|
||||||
|
request_count: number
|
||||||
|
output_tokens: number
|
||||||
|
avg_output_tps: number | null
|
||||||
|
avg_first_byte_time_ms: number | null
|
||||||
|
avg_response_time_ms: number | null
|
||||||
|
success_rate: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ProviderPerformanceResponse {
|
||||||
|
summary: ProviderPerformanceSummary
|
||||||
|
providers: ProviderPerformanceItem[]
|
||||||
|
timeline: ProviderPerformanceTimelineItem[]
|
||||||
|
}
|
||||||
|
|
||||||
export interface ErrorDistributionItem {
|
export interface ErrorDistributionItem {
|
||||||
category: string
|
category: string
|
||||||
count: number
|
count: number
|
||||||
@@ -932,6 +975,28 @@ export const adminApi = {
|
|||||||
)
|
)
|
||||||
},
|
},
|
||||||
|
|
||||||
|
async getProviderPerformance(params?: {
|
||||||
|
start_date?: string
|
||||||
|
end_date?: string
|
||||||
|
preset?: string
|
||||||
|
timezone?: string
|
||||||
|
tz_offset_minutes?: number
|
||||||
|
granularity?: 'day' | 'hour'
|
||||||
|
limit?: number
|
||||||
|
}): Promise<ProviderPerformanceResponse> {
|
||||||
|
const cacheKey = buildCacheKey('admin:stats:performance:providers', params)
|
||||||
|
return cachedRequest(
|
||||||
|
cacheKey,
|
||||||
|
async () => {
|
||||||
|
const response = await apiClient.get<ProviderPerformanceResponse>('/api/admin/stats/performance/providers', {
|
||||||
|
params
|
||||||
|
})
|
||||||
|
return response.data
|
||||||
|
},
|
||||||
|
20 * 1000
|
||||||
|
)
|
||||||
|
},
|
||||||
|
|
||||||
async getErrorDistribution(params?: {
|
async getErrorDistribution(params?: {
|
||||||
start_date?: string
|
start_date?: string
|
||||||
end_date?: string
|
end_date?: string
|
||||||
|
|||||||
@@ -491,6 +491,183 @@
|
|||||||
</Card>
|
</Card>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<Card class="space-y-4 p-4">
|
||||||
|
<div class="flex flex-col gap-2 sm:flex-row sm:items-center sm:justify-between">
|
||||||
|
<div>
|
||||||
|
<h3 class="text-sm font-semibold">
|
||||||
|
Provider 性能
|
||||||
|
</h3>
|
||||||
|
<p class="text-xs text-muted-foreground">
|
||||||
|
{{ providerPerformanceSubtitle }}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<Badge variant="outline">
|
||||||
|
Top {{ providerPerformanceRows.length || 0 }}
|
||||||
|
</Badge>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div
|
||||||
|
v-if="providerPerformanceLoading"
|
||||||
|
class="p-6"
|
||||||
|
>
|
||||||
|
<LoadingState />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div
|
||||||
|
v-else
|
||||||
|
class="space-y-4"
|
||||||
|
>
|
||||||
|
<div class="grid grid-cols-1 gap-3 sm:grid-cols-2 xl:grid-cols-4">
|
||||||
|
<div
|
||||||
|
v-for="card in providerPerformanceSummaryCards"
|
||||||
|
:key="card.title"
|
||||||
|
class="rounded-xl border border-border/70 bg-card/70 px-4 py-3"
|
||||||
|
>
|
||||||
|
<div class="flex items-center justify-between gap-3">
|
||||||
|
<span class="text-xs text-muted-foreground">{{ card.title }}</span>
|
||||||
|
<component
|
||||||
|
:is="card.icon"
|
||||||
|
class="h-4 w-4"
|
||||||
|
:class="card.iconClass"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div class="mt-3 text-2xl font-semibold tracking-tight">
|
||||||
|
{{ card.value }}
|
||||||
|
</div>
|
||||||
|
<div class="mt-2 text-xs text-muted-foreground">
|
||||||
|
{{ card.hint }}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div
|
||||||
|
v-if="providerPerformanceRows.length"
|
||||||
|
class="overflow-x-auto rounded-lg border border-border/70"
|
||||||
|
>
|
||||||
|
<table class="min-w-full divide-y divide-border/70 text-sm">
|
||||||
|
<thead class="bg-muted/30 text-xs text-muted-foreground">
|
||||||
|
<tr>
|
||||||
|
<th class="px-3 py-2 text-left font-medium">
|
||||||
|
Provider
|
||||||
|
</th>
|
||||||
|
<th class="px-3 py-2 text-right font-medium">
|
||||||
|
请求
|
||||||
|
</th>
|
||||||
|
<th class="px-3 py-2 text-right font-medium">
|
||||||
|
成功率
|
||||||
|
</th>
|
||||||
|
<th class="px-3 py-2 text-right font-medium">
|
||||||
|
输出 TPS
|
||||||
|
</th>
|
||||||
|
<th class="px-3 py-2 text-right font-medium">
|
||||||
|
平均首字
|
||||||
|
</th>
|
||||||
|
<th class="px-3 py-2 text-right font-medium">
|
||||||
|
平均响应
|
||||||
|
</th>
|
||||||
|
<th class="px-3 py-2 text-right font-medium">
|
||||||
|
P90 响应 / 首字
|
||||||
|
</th>
|
||||||
|
<th class="px-3 py-2 text-right font-medium">
|
||||||
|
样本
|
||||||
|
</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody class="divide-y divide-border/60">
|
||||||
|
<tr
|
||||||
|
v-for="provider in providerPerformanceRows"
|
||||||
|
:key="provider.provider_id"
|
||||||
|
class="bg-background/40"
|
||||||
|
>
|
||||||
|
<td class="max-w-[220px] px-3 py-2">
|
||||||
|
<div class="truncate font-medium">
|
||||||
|
{{ provider.provider }}
|
||||||
|
</div>
|
||||||
|
<div class="truncate text-xs text-muted-foreground">
|
||||||
|
{{ provider.provider_id }}
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
<td class="px-3 py-2 text-right">
|
||||||
|
{{ formatMetricNumber(provider.request_count) }}
|
||||||
|
</td>
|
||||||
|
<td class="px-3 py-2 text-right">
|
||||||
|
{{ formatProviderPerformanceMetric(provider.success_rate, '%') }}
|
||||||
|
</td>
|
||||||
|
<td class="px-3 py-2 text-right">
|
||||||
|
{{ formatProviderPerformanceMetric(provider.avg_output_tps, '/s') }}
|
||||||
|
</td>
|
||||||
|
<td class="px-3 py-2 text-right">
|
||||||
|
{{ formatProviderPerformanceMetric(provider.avg_first_byte_time_ms, 'ms') }}
|
||||||
|
</td>
|
||||||
|
<td class="px-3 py-2 text-right">
|
||||||
|
{{ formatProviderPerformanceMetric(provider.avg_response_time_ms, 'ms') }}
|
||||||
|
</td>
|
||||||
|
<td class="px-3 py-2 text-right">
|
||||||
|
{{ formatProviderPerformanceMetric(provider.p90_response_time_ms, 'ms', 0) }}
|
||||||
|
/
|
||||||
|
{{ formatProviderPerformanceMetric(provider.p90_first_byte_time_ms, 'ms', 0) }}
|
||||||
|
</td>
|
||||||
|
<td class="px-3 py-2 text-right text-xs text-muted-foreground">
|
||||||
|
{{ formatMetricNumber(provider.tps_sample_count) }} /
|
||||||
|
{{ formatMetricNumber(provider.first_byte_sample_count) }}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div
|
||||||
|
v-else
|
||||||
|
class="rounded-lg border border-dashed border-border/70 px-3 py-4 text-sm text-muted-foreground"
|
||||||
|
>
|
||||||
|
当前没有 Provider 性能数据。
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
<div class="grid grid-cols-1 gap-4 lg:grid-cols-2">
|
||||||
|
<Card class="space-y-3 p-4">
|
||||||
|
<h3 class="text-sm font-semibold">
|
||||||
|
输出 TPS 趋势
|
||||||
|
</h3>
|
||||||
|
<div
|
||||||
|
v-if="providerPerformanceLoading"
|
||||||
|
class="p-6"
|
||||||
|
>
|
||||||
|
<LoadingState />
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
v-else
|
||||||
|
class="h-[260px]"
|
||||||
|
>
|
||||||
|
<LineChart
|
||||||
|
:data="providerTpsChartData"
|
||||||
|
:options="providerTpsChartOptions"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</Card>
|
||||||
|
<Card class="space-y-3 p-4">
|
||||||
|
<h3 class="text-sm font-semibold">
|
||||||
|
平均首字趋势
|
||||||
|
</h3>
|
||||||
|
<div
|
||||||
|
v-if="providerPerformanceLoading"
|
||||||
|
class="p-6"
|
||||||
|
>
|
||||||
|
<LoadingState />
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
v-else
|
||||||
|
class="h-[260px]"
|
||||||
|
>
|
||||||
|
<LineChart
|
||||||
|
:data="providerFirstByteChartData"
|
||||||
|
:options="providerLatencyChartOptions"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</Card>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="grid grid-cols-1 gap-4 lg:grid-cols-2">
|
<div class="grid grid-cols-1 gap-4 lg:grid-cols-2">
|
||||||
<Card class="p-4">
|
<Card class="p-4">
|
||||||
<ErrorDistributionChart
|
<ErrorDistributionChart
|
||||||
@@ -562,11 +739,20 @@ import {
|
|||||||
Activity,
|
Activity,
|
||||||
AlertTriangle,
|
AlertTriangle,
|
||||||
Cable,
|
Cable,
|
||||||
|
CheckCircle2,
|
||||||
GitBranch,
|
GitBranch,
|
||||||
|
Gauge,
|
||||||
ShieldCheck,
|
ShieldCheck,
|
||||||
|
Timer,
|
||||||
Workflow,
|
Workflow,
|
||||||
|
Zap,
|
||||||
} from 'lucide-vue-next'
|
} from 'lucide-vue-next'
|
||||||
import { adminApi, type ErrorDistributionResponse, type PercentileItem } from '@/api/admin'
|
import {
|
||||||
|
adminApi,
|
||||||
|
type ErrorDistributionResponse,
|
||||||
|
type PercentileItem,
|
||||||
|
type ProviderPerformanceResponse,
|
||||||
|
} from '@/api/admin'
|
||||||
import { dashboardApi, type ProviderStatus } from '@/api/dashboard'
|
import { dashboardApi, type ProviderStatus } from '@/api/dashboard'
|
||||||
import {
|
import {
|
||||||
monitoringApi,
|
monitoringApi,
|
||||||
@@ -586,6 +772,10 @@ import { getDateRangeFromPeriod } from '@/features/usage/composables'
|
|||||||
import type { DateRangeParams } from '@/features/usage/types'
|
import type { DateRangeParams } from '@/features/usage/types'
|
||||||
import { formatDate, formatNumber, formatTokens } from '@/utils/format'
|
import { formatDate, formatNumber, formatTokens } from '@/utils/format'
|
||||||
import { log } from '@/utils/logger'
|
import { log } from '@/utils/logger'
|
||||||
|
import {
|
||||||
|
buildProviderPerformanceChartData,
|
||||||
|
formatProviderPerformanceMetric,
|
||||||
|
} from './performanceAnalysisHelpers'
|
||||||
|
|
||||||
const LIVE_REFRESH_INTERVAL_MS = 10_000
|
const LIVE_REFRESH_INTERVAL_MS = 10_000
|
||||||
|
|
||||||
@@ -601,6 +791,8 @@ const errorLoading = ref(false)
|
|||||||
|
|
||||||
const providerStatus = ref<ProviderStatus[]>([])
|
const providerStatus = ref<ProviderStatus[]>([])
|
||||||
const providerLoading = ref(false)
|
const providerLoading = ref(false)
|
||||||
|
const providerPerformance = ref<ProviderPerformanceResponse | null>(null)
|
||||||
|
const providerPerformanceLoading = ref(false)
|
||||||
|
|
||||||
const systemStatus = ref<AdminMonitoringSystemStatus | null>(null)
|
const systemStatus = ref<AdminMonitoringSystemStatus | null>(null)
|
||||||
const resilienceStatus = ref<AdminMonitoringResilienceStatus | null>(null)
|
const resilienceStatus = ref<AdminMonitoringResilienceStatus | null>(null)
|
||||||
@@ -615,6 +807,7 @@ const liveLastUpdatedAt = ref<string | null>(null)
|
|||||||
let percentilesRequestId = 0
|
let percentilesRequestId = 0
|
||||||
let errorsRequestId = 0
|
let errorsRequestId = 0
|
||||||
let providersRequestId = 0
|
let providersRequestId = 0
|
||||||
|
let providerPerformanceRequestId = 0
|
||||||
let liveRequestId = 0
|
let liveRequestId = 0
|
||||||
let loadAllPromise: Promise<void> | null = null
|
let loadAllPromise: Promise<void> | null = null
|
||||||
let hasPendingLoadAll = false
|
let hasPendingLoadAll = false
|
||||||
@@ -699,6 +892,28 @@ async function loadProviders() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function loadProviderPerformance() {
|
||||||
|
const requestId = ++providerPerformanceRequestId
|
||||||
|
providerPerformanceLoading.value = true
|
||||||
|
try {
|
||||||
|
const data = await adminApi.getProviderPerformance({
|
||||||
|
...buildTimeRangeParams(),
|
||||||
|
granularity: 'day',
|
||||||
|
limit: 8,
|
||||||
|
})
|
||||||
|
if (requestId !== providerPerformanceRequestId) return
|
||||||
|
providerPerformance.value = data
|
||||||
|
} catch (error) {
|
||||||
|
if (requestId !== providerPerformanceRequestId) return
|
||||||
|
providerPerformance.value = null
|
||||||
|
log.error('加载 Provider 性能统计失败', error)
|
||||||
|
} finally {
|
||||||
|
if (requestId === providerPerformanceRequestId) {
|
||||||
|
providerPerformanceLoading.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function loadLiveData(options: { silent?: boolean } = {}) {
|
async function loadLiveData(options: { silent?: boolean } = {}) {
|
||||||
const requestId = ++liveRequestId
|
const requestId = ++liveRequestId
|
||||||
const initialLoad = !liveReady.value
|
const initialLoad = !liveReady.value
|
||||||
@@ -866,6 +1081,83 @@ const fallbackRows = computed(() => {
|
|||||||
}))
|
}))
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const providerPerformanceRows = computed(() => providerPerformance.value?.providers ?? [])
|
||||||
|
|
||||||
|
const providerPerformanceSubtitle = computed(() => {
|
||||||
|
const requests = providerPerformance.value?.summary.request_count ?? 0
|
||||||
|
return `完成窗口内 ${formatMetricNumber(requests)} 个 Provider 请求样本`
|
||||||
|
})
|
||||||
|
|
||||||
|
const providerPerformanceSummaryCards = computed(() => {
|
||||||
|
const summary = providerPerformance.value?.summary
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
title: '输出 TPS',
|
||||||
|
value: formatProviderPerformanceMetric(summary?.avg_output_tps, '/s'),
|
||||||
|
hint: `请求 ${formatMetricNumber(summary?.request_count)}`,
|
||||||
|
icon: Zap,
|
||||||
|
iconClass: 'text-amber-500',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: '平均首字',
|
||||||
|
value: formatProviderPerformanceMetric(summary?.avg_first_byte_time_ms, 'ms'),
|
||||||
|
hint: '成功请求首字样本',
|
||||||
|
icon: Timer,
|
||||||
|
iconClass: 'text-sky-500',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: '平均响应',
|
||||||
|
value: formatProviderPerformanceMetric(summary?.avg_response_time_ms, 'ms'),
|
||||||
|
hint: '成功请求响应耗时',
|
||||||
|
icon: Gauge,
|
||||||
|
iconClass: 'text-violet-500',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: '成功率',
|
||||||
|
value: formatProviderPerformanceMetric(summary?.success_rate, '%'),
|
||||||
|
hint: `${formatMetricNumber(providerPerformanceRows.value.length)} 个 Provider`,
|
||||||
|
icon: CheckCircle2,
|
||||||
|
iconClass: 'text-emerald-500',
|
||||||
|
},
|
||||||
|
]
|
||||||
|
})
|
||||||
|
|
||||||
|
const providerTpsChartData = computed(() => (
|
||||||
|
buildProviderPerformanceChartData(
|
||||||
|
providerPerformance.value?.timeline ?? [],
|
||||||
|
'avg_output_tps',
|
||||||
|
providerPerformanceRows.value,
|
||||||
|
)
|
||||||
|
))
|
||||||
|
|
||||||
|
const providerFirstByteChartData = computed(() => (
|
||||||
|
buildProviderPerformanceChartData(
|
||||||
|
providerPerformance.value?.timeline ?? [],
|
||||||
|
'avg_first_byte_time_ms',
|
||||||
|
providerPerformanceRows.value,
|
||||||
|
)
|
||||||
|
))
|
||||||
|
|
||||||
|
const providerTpsChartOptions = computed(() => ({
|
||||||
|
scales: {
|
||||||
|
y: {
|
||||||
|
ticks: {
|
||||||
|
callback: (value: string | number) => `${value}/s`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
|
||||||
|
const providerLatencyChartOptions = computed(() => ({
|
||||||
|
scales: {
|
||||||
|
y: {
|
||||||
|
ticks: {
|
||||||
|
callback: (value: string | number) => `${value}ms`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
|
||||||
const liveSummaryCards = computed(() => [
|
const liveSummaryCards = computed(() => [
|
||||||
{
|
{
|
||||||
title: '系统健康',
|
title: '系统健康',
|
||||||
@@ -920,7 +1212,8 @@ const isRefreshing = computed(() => (
|
|||||||
liveRefreshing.value ||
|
liveRefreshing.value ||
|
||||||
percentileLoading.value ||
|
percentileLoading.value ||
|
||||||
errorLoading.value ||
|
errorLoading.value ||
|
||||||
providerLoading.value
|
providerLoading.value ||
|
||||||
|
providerPerformanceLoading.value
|
||||||
))
|
))
|
||||||
|
|
||||||
async function loadAll() {
|
async function loadAll() {
|
||||||
@@ -929,7 +1222,12 @@ async function loadAll() {
|
|||||||
return loadAllPromise
|
return loadAllPromise
|
||||||
}
|
}
|
||||||
|
|
||||||
loadAllPromise = Promise.all([loadPercentiles(), loadErrors(), loadProviders()])
|
loadAllPromise = Promise.all([
|
||||||
|
loadPercentiles(),
|
||||||
|
loadErrors(),
|
||||||
|
loadProviders(),
|
||||||
|
loadProviderPerformance(),
|
||||||
|
])
|
||||||
.then(() => undefined)
|
.then(() => undefined)
|
||||||
.finally(() => {
|
.finally(() => {
|
||||||
loadAllPromise = null
|
loadAllPromise = null
|
||||||
@@ -983,6 +1281,7 @@ onUnmounted(() => {
|
|||||||
percentilesRequestId += 1
|
percentilesRequestId += 1
|
||||||
errorsRequestId += 1
|
errorsRequestId += 1
|
||||||
providersRequestId += 1
|
providersRequestId += 1
|
||||||
|
providerPerformanceRequestId += 1
|
||||||
liveRequestId += 1
|
liveRequestId += 1
|
||||||
})
|
})
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -0,0 +1,53 @@
|
|||||||
|
import { describe, expect, it } from 'vitest'
|
||||||
|
import {
|
||||||
|
buildProviderPerformanceChartData,
|
||||||
|
formatProviderPerformanceMetric,
|
||||||
|
} from '../performanceAnalysisHelpers'
|
||||||
|
import type { ProviderPerformanceItem, ProviderPerformanceTimelineItem } from '@/api/admin'
|
||||||
|
|
||||||
|
describe('performanceAnalysisHelpers', () => {
|
||||||
|
it('formats null provider metrics as placeholders', () => {
|
||||||
|
expect(formatProviderPerformanceMetric(null, 'ms')).toBe('-')
|
||||||
|
expect(formatProviderPerformanceMetric(undefined, '/s')).toBe('-')
|
||||||
|
expect(formatProviderPerformanceMetric(Number.NaN)).toBe('-')
|
||||||
|
expect(formatProviderPerformanceMetric(18.456, '/s')).toBe('18.46/s')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('builds stable provider trend datasets with null gaps', () => {
|
||||||
|
const providers: Pick<ProviderPerformanceItem, 'provider_id' | 'provider'>[] = [
|
||||||
|
{ provider_id: 'provider-a', provider: 'OpenAI' },
|
||||||
|
{ provider_id: 'provider-b', provider: 'Anthropic' },
|
||||||
|
]
|
||||||
|
const timeline: ProviderPerformanceTimelineItem[] = [
|
||||||
|
{
|
||||||
|
date: '2024-03-21',
|
||||||
|
provider_id: 'provider-a',
|
||||||
|
provider: 'OpenAI',
|
||||||
|
request_count: 2,
|
||||||
|
output_tokens: 100,
|
||||||
|
avg_output_tps: 25,
|
||||||
|
avg_first_byte_time_ms: 120,
|
||||||
|
avg_response_time_ms: 1000,
|
||||||
|
success_rate: 100,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
date: '2024-03-22',
|
||||||
|
provider_id: 'provider-b',
|
||||||
|
provider: 'Anthropic',
|
||||||
|
request_count: 1,
|
||||||
|
output_tokens: 40,
|
||||||
|
avg_output_tps: null,
|
||||||
|
avg_first_byte_time_ms: 220,
|
||||||
|
avg_response_time_ms: 1500,
|
||||||
|
success_rate: 100,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
const chart = buildProviderPerformanceChartData(timeline, 'avg_output_tps', providers)
|
||||||
|
|
||||||
|
expect(chart.labels).toEqual(['2024-03-21', '2024-03-22'])
|
||||||
|
expect(chart.datasets.map(dataset => dataset.label)).toEqual(['OpenAI', 'Anthropic'])
|
||||||
|
expect(chart.datasets[0].data).toEqual([25, null])
|
||||||
|
expect(chart.datasets[1].data).toEqual([null, null])
|
||||||
|
})
|
||||||
|
})
|
||||||
64
frontend/src/views/admin/performanceAnalysisHelpers.ts
Normal file
64
frontend/src/views/admin/performanceAnalysisHelpers.ts
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
import type { ChartData } from 'chart.js'
|
||||||
|
import type {
|
||||||
|
ProviderPerformanceItem,
|
||||||
|
ProviderPerformanceTimelineItem,
|
||||||
|
} from '@/api/admin'
|
||||||
|
|
||||||
|
export type ProviderPerformanceMetricKey = 'avg_output_tps' | 'avg_first_byte_time_ms'
|
||||||
|
|
||||||
|
const PROVIDER_CHART_COLORS = [
|
||||||
|
'rgb(59, 130, 246)',
|
||||||
|
'rgb(16, 185, 129)',
|
||||||
|
'rgb(234, 179, 8)',
|
||||||
|
'rgb(239, 68, 68)',
|
||||||
|
'rgb(139, 92, 246)',
|
||||||
|
'rgb(14, 165, 233)',
|
||||||
|
'rgb(249, 115, 22)',
|
||||||
|
'rgb(20, 184, 166)',
|
||||||
|
]
|
||||||
|
|
||||||
|
export function formatProviderPerformanceMetric(
|
||||||
|
value: number | null | undefined,
|
||||||
|
suffix = '',
|
||||||
|
decimals = 2
|
||||||
|
): string {
|
||||||
|
if (value == null || Number.isNaN(value)) {
|
||||||
|
return '-'
|
||||||
|
}
|
||||||
|
return `${value.toFixed(decimals)}${suffix}`
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildProviderPerformanceChartData(
|
||||||
|
timeline: ProviderPerformanceTimelineItem[],
|
||||||
|
metric: ProviderPerformanceMetricKey,
|
||||||
|
providers: Pick<ProviderPerformanceItem, 'provider_id' | 'provider'>[] = []
|
||||||
|
): ChartData<'line'> {
|
||||||
|
const labels = Array.from(new Set(timeline.map(item => item.date)))
|
||||||
|
const providerMap = new Map<string, string>()
|
||||||
|
for (const provider of providers) {
|
||||||
|
providerMap.set(provider.provider_id, provider.provider)
|
||||||
|
}
|
||||||
|
for (const item of timeline) {
|
||||||
|
if (!providerMap.has(item.provider_id)) {
|
||||||
|
providerMap.set(item.provider_id, item.provider)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
labels,
|
||||||
|
datasets: Array.from(providerMap.entries()).map(([providerId, provider], index) => {
|
||||||
|
const byDate = new Map(
|
||||||
|
timeline
|
||||||
|
.filter(item => item.provider_id === providerId)
|
||||||
|
.map(item => [item.date, item[metric]] as const)
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
label: provider,
|
||||||
|
data: labels.map(label => byDate.get(label) ?? null),
|
||||||
|
borderColor: PROVIDER_CHART_COLORS[index % PROVIDER_CHART_COLORS.length],
|
||||||
|
tension: 0.25,
|
||||||
|
pointRadius: 2,
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user