mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-10 05:00:19 +08:00
fix monitoring error totals and counter health
This commit is contained in:
+14
-5
@@ -3,7 +3,8 @@ use crate::handlers::admin::request::AdminAppState;
|
||||
use crate::handlers::admin::shared::{provider_key_health_summary_at, unix_secs_to_rfc3339};
|
||||
use crate::GatewayError;
|
||||
use aether_data_contracts::repository::{
|
||||
provider_catalog::StoredProviderCatalogKey, usage::UsageMonitoringErrorListQuery,
|
||||
provider_catalog::StoredProviderCatalogKey,
|
||||
usage::{UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery},
|
||||
};
|
||||
use serde_json::json;
|
||||
use std::collections::BTreeMap;
|
||||
@@ -152,11 +153,20 @@ pub(super) async fn build_admin_monitoring_resilience_snapshot(
|
||||
}
|
||||
}
|
||||
|
||||
let error_window_start = recent_error_from.timestamp().max(0) as u64;
|
||||
let error_window_end = (now.timestamp().max(0) as u64).saturating_add(1);
|
||||
let total_errors = state
|
||||
.count_monitoring_usage_errors(&UsageMonitoringErrorCountQuery {
|
||||
created_from_unix_secs: error_window_start,
|
||||
created_until_unix_secs: error_window_end,
|
||||
})
|
||||
.await? as usize;
|
||||
|
||||
let mut recent_usage_errors = state
|
||||
.list_monitoring_usage_errors(&UsageMonitoringErrorListQuery {
|
||||
created_from_unix_secs: recent_error_from.timestamp().max(0) as u64,
|
||||
created_until_unix_secs: (now.timestamp().max(0) as u64).saturating_add(1),
|
||||
limit: None,
|
||||
created_from_unix_secs: error_window_start,
|
||||
created_until_unix_secs: error_window_end,
|
||||
limit: Some(10),
|
||||
})
|
||||
.await?
|
||||
.into_iter()
|
||||
@@ -164,7 +174,6 @@ pub(super) async fn build_admin_monitoring_resilience_snapshot(
|
||||
.collect::<Vec<_>>();
|
||||
recent_usage_errors.sort_by_key(|item| std::cmp::Reverse(item.created_at_unix_ms));
|
||||
|
||||
let total_errors = recent_usage_errors.len();
|
||||
let mut error_breakdown = BTreeMap::<String, usize>::new();
|
||||
for item in &recent_usage_errors {
|
||||
let error_type = item
|
||||
|
||||
@@ -256,6 +256,57 @@ async fn admin_monitoring_resilience_status_returns_local_payload() {
|
||||
assert!(payload["timestamp"].as_str().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_resilience_status_limits_recent_error_rows() {
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||
vec![sample_provider()],
|
||||
vec![],
|
||||
vec![],
|
||||
));
|
||||
let usage_rows = (0..12)
|
||||
.map(|index| {
|
||||
sample_usage(
|
||||
&format!("request-recent-failed-{index}"),
|
||||
"provider-1",
|
||||
"OpenAI",
|
||||
10,
|
||||
0.10,
|
||||
"failed",
|
||||
Some(502),
|
||||
now - i64::from(index),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(usage_rows));
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(
|
||||
crate::data::GatewayDataState::with_provider_catalog_and_usage_reader_for_tests(
|
||||
provider_catalog,
|
||||
usage_repository,
|
||||
),
|
||||
);
|
||||
let context = request_context(http::Method::GET, "/api/admin/monitoring/resilience-status");
|
||||
|
||||
let response = local_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["error_statistics"]["total_errors"], json!(12));
|
||||
assert_eq!(payload["recent_errors"].as_array().map(Vec::len), Some(10));
|
||||
assert_eq!(
|
||||
payload["recent_errors"][0]["error_id"],
|
||||
json!("usage-request-recent-failed-0")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_cache_stats_count_runtime_scheduler_affinities() {
|
||||
let state = AppState::new().expect("state should build");
|
||||
|
||||
@@ -70,6 +70,13 @@ impl<'a> AdminAppState<'a> {
|
||||
self.app.list_monitoring_usage_errors(query).await
|
||||
}
|
||||
|
||||
pub(crate) async fn count_monitoring_usage_errors(
|
||||
&self,
|
||||
query: &aether_data_contracts::repository::usage::UsageMonitoringErrorCountQuery,
|
||||
) -> Result<u64, GatewayError> {
|
||||
self.app.count_monitoring_usage_errors(query).await
|
||||
}
|
||||
|
||||
pub(crate) async fn aggregate_usage_audits(
|
||||
&self,
|
||||
query: &aether_data_contracts::repository::usage::UsageAuditAggregationQuery,
|
||||
|
||||
@@ -1444,13 +1444,26 @@ ORDER BY delta.created_at ASC, delta.id ASC
|
||||
|
||||
const READ_USAGE_COUNTER_HEALTH_SQL: &str = r#"
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE processed_at IS NULL)::BIGINT AS pending_rows,
|
||||
COUNT(*) FILTER (WHERE processed_at IS NOT NULL)::BIGINT AS processed_rows,
|
||||
CAST(EXTRACT(EPOCH FROM MIN(created_at) FILTER (WHERE processed_at IS NULL)) AS BIGINT)
|
||||
AS oldest_pending_created_at_unix_secs,
|
||||
CAST(EXTRACT(EPOCH FROM MAX(processed_at)) AS BIGINT)
|
||||
AS latest_processed_at_unix_secs
|
||||
FROM usage_counter_deltas
|
||||
(
|
||||
SELECT COUNT(*)::BIGINT
|
||||
FROM usage_counter_deltas
|
||||
WHERE processed_at IS NULL
|
||||
) AS pending_rows,
|
||||
(
|
||||
SELECT COUNT(*)::BIGINT
|
||||
FROM usage_counter_deltas
|
||||
WHERE processed_at IS NOT NULL
|
||||
) AS processed_rows,
|
||||
(
|
||||
SELECT CAST(EXTRACT(EPOCH FROM MIN(created_at)) AS BIGINT)
|
||||
FROM usage_counter_deltas
|
||||
WHERE processed_at IS NULL
|
||||
) AS oldest_pending_created_at_unix_secs,
|
||||
(
|
||||
SELECT CAST(EXTRACT(EPOCH FROM MAX(processed_at)) AS BIGINT)
|
||||
FROM usage_counter_deltas
|
||||
WHERE processed_at IS NOT NULL
|
||||
) AS latest_processed_at_unix_secs
|
||||
"#;
|
||||
|
||||
const READ_PENDING_USAGE_COUNTER_DELTAS_BY_KIND_SQL: &str = r#"
|
||||
|
||||
Reference in New Issue
Block a user