fix monitoring error totals and counter health

This commit is contained in:
elky
2026-06-26 10:48:45 +08:00
parent 063834e95b
commit 6c5e70ccb1
4 changed files with 92 additions and 12 deletions
@@ -3,7 +3,8 @@ use crate::handlers::admin::request::AdminAppState;
use crate::handlers::admin::shared::{provider_key_health_summary_at, unix_secs_to_rfc3339};
use crate::GatewayError;
use aether_data_contracts::repository::{
provider_catalog::StoredProviderCatalogKey, usage::UsageMonitoringErrorListQuery,
provider_catalog::StoredProviderCatalogKey,
usage::{UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery},
};
use serde_json::json;
use std::collections::BTreeMap;
@@ -152,11 +153,20 @@ pub(super) async fn build_admin_monitoring_resilience_snapshot(
}
}
let error_window_start = recent_error_from.timestamp().max(0) as u64;
let error_window_end = (now.timestamp().max(0) as u64).saturating_add(1);
let total_errors = state
.count_monitoring_usage_errors(&UsageMonitoringErrorCountQuery {
created_from_unix_secs: error_window_start,
created_until_unix_secs: error_window_end,
})
.await? as usize;
let mut recent_usage_errors = state
.list_monitoring_usage_errors(&UsageMonitoringErrorListQuery {
created_from_unix_secs: recent_error_from.timestamp().max(0) as u64,
created_until_unix_secs: (now.timestamp().max(0) as u64).saturating_add(1),
limit: None,
created_from_unix_secs: error_window_start,
created_until_unix_secs: error_window_end,
limit: Some(10),
})
.await?
.into_iter()
@@ -164,7 +174,6 @@ pub(super) async fn build_admin_monitoring_resilience_snapshot(
.collect::<Vec<_>>();
recent_usage_errors.sort_by_key(|item| std::cmp::Reverse(item.created_at_unix_ms));
let total_errors = recent_usage_errors.len();
let mut error_breakdown = BTreeMap::<String, usize>::new();
for item in &recent_usage_errors {
let error_type = item
@@ -256,6 +256,57 @@ async fn admin_monitoring_resilience_status_returns_local_payload() {
assert!(payload["timestamp"].as_str().is_some());
}
#[tokio::test]
async fn admin_monitoring_resilience_status_limits_recent_error_rows() {
let now = chrono::Utc::now().timestamp();
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider()],
vec![],
vec![],
));
let usage_rows = (0..12)
.map(|index| {
sample_usage(
&format!("request-recent-failed-{index}"),
"provider-1",
"OpenAI",
10,
0.10,
"failed",
Some(502),
now - i64::from(index),
)
})
.collect::<Vec<_>>();
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(usage_rows));
let state = AppState::new()
.expect("state should build")
.with_data_state_for_tests(
crate::data::GatewayDataState::with_provider_catalog_and_usage_reader_for_tests(
provider_catalog,
usage_repository,
),
);
let context = request_context(http::Method::GET, "/api/admin/monitoring/resilience-status");
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
assert_eq!(response.status(), http::StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
assert_eq!(payload["error_statistics"]["total_errors"], json!(12));
assert_eq!(payload["recent_errors"].as_array().map(Vec::len), Some(10));
assert_eq!(
payload["recent_errors"][0]["error_id"],
json!("usage-request-recent-failed-0")
);
}
#[tokio::test]
async fn admin_monitoring_cache_stats_count_runtime_scheduler_affinities() {
let state = AppState::new().expect("state should build");
@@ -70,6 +70,13 @@ impl<'a> AdminAppState<'a> {
self.app.list_monitoring_usage_errors(query).await
}
pub(crate) async fn count_monitoring_usage_errors(
&self,
query: &aether_data_contracts::repository::usage::UsageMonitoringErrorCountQuery,
) -> Result<u64, GatewayError> {
self.app.count_monitoring_usage_errors(query).await
}
pub(crate) async fn aggregate_usage_audits(
&self,
query: &aether_data_contracts::repository::usage::UsageAuditAggregationQuery,
@@ -1444,13 +1444,26 @@ ORDER BY delta.created_at ASC, delta.id ASC
const READ_USAGE_COUNTER_HEALTH_SQL: &str = r#"
SELECT
COUNT(*) FILTER (WHERE processed_at IS NULL)::BIGINT AS pending_rows,
COUNT(*) FILTER (WHERE processed_at IS NOT NULL)::BIGINT AS processed_rows,
CAST(EXTRACT(EPOCH FROM MIN(created_at) FILTER (WHERE processed_at IS NULL)) AS BIGINT)
AS oldest_pending_created_at_unix_secs,
CAST(EXTRACT(EPOCH FROM MAX(processed_at)) AS BIGINT)
AS latest_processed_at_unix_secs
FROM usage_counter_deltas
(
SELECT COUNT(*)::BIGINT
FROM usage_counter_deltas
WHERE processed_at IS NULL
) AS pending_rows,
(
SELECT COUNT(*)::BIGINT
FROM usage_counter_deltas
WHERE processed_at IS NOT NULL
) AS processed_rows,
(
SELECT CAST(EXTRACT(EPOCH FROM MIN(created_at)) AS BIGINT)
FROM usage_counter_deltas
WHERE processed_at IS NULL
) AS oldest_pending_created_at_unix_secs,
(
SELECT CAST(EXTRACT(EPOCH FROM MAX(processed_at)) AS BIGINT)
FROM usage_counter_deltas
WHERE processed_at IS NOT NULL
) AS latest_processed_at_unix_secs
"#;
const READ_PENDING_USAGE_COUNTER_DELTAS_BY_KIND_SQL: &str = r#"