fix(gateway): align admin key health circuit summary

This commit is contained in:
fawney19
2026-05-25 01:18:07 +08:00
parent cc3ce8b5d7
commit fe7fb17ff5
2 changed files with 86 additions and 11 deletions
@@ -114,15 +114,16 @@ pub(crate) async fn build_admin_key_health_payload(
.unwrap_or(0));
} else {
let mut formats_payload = serde_json::Map::new();
let mut any_circuit_open = false;
for format_name in
provider_key_effective_api_formats(&key, &provider.provider_type, &endpoints)
{
let health_data = health_by_format.and_then(|formats| formats.get(&format_name));
let circuit_data = circuit_by_format.and_then(|formats| formats.get(&format_name));
let is_open = circuit_data
.and_then(|value| value.get("open"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false);
let active_open = circuit_data.is_some_and(|value| {
provider_key_circuit_payload_is_active_open_at(value, now_unix_secs)
});
any_circuit_open |= active_open;
formats_payload.insert(
format_name.clone(),
json!({
@@ -141,8 +142,8 @@ pub(crate) async fn build_admin_key_health_payload(
.cloned()
.unwrap_or(serde_json::Value::Null),
"circuit_breaker": {
"state": if is_open { "open" } else { "closed" },
"open": is_open,
"state": if active_open { "open" } else { "closed" },
"open": active_open,
"open_at": circuit_data
.and_then(|value| value.get("open_at"))
.cloned()
@@ -174,11 +175,6 @@ pub(crate) async fn build_admin_key_health_payload(
.filter_map(serde_json::Value::as_f64)
.reduce(f64::min)
.unwrap_or(1.0);
let any_circuit_open = formats_payload.values().any(|value| {
value.get("circuit_breaker").is_some_and(|circuit| {
provider_key_circuit_payload_is_active_open_at(circuit, now_unix_secs)
})
});
payload["key_health_score"] = json!(key_health_score);
payload["any_circuit_open"] = json!(any_circuit_open);
@@ -408,6 +408,85 @@ async fn gateway_handles_admin_key_health_locally_with_trusted_admin_principal()
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_admin_key_health_summary_treats_expired_unix_circuit_as_closed() {
let upstream_hits = Arc::new(Mutex::new(0usize));
let upstream_hits_clone = Arc::clone(&upstream_hits);
let upstream = Router::new().route(
"/api/admin/endpoints/health/key/key-openai",
any(move |_request: Request| {
let upstream_hits_inner = Arc::clone(&upstream_hits_clone);
async move {
*upstream_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::OK, Body::from("unexpected upstream hit"))
}
}),
);
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider("provider-openai", "openai", 10)],
vec![sample_endpoint(
"endpoint-openai",
"provider-openai",
"openai:chat",
"https://api.openai.example",
)],
vec![
sample_key("key-openai", "provider-openai", "openai:chat", "sk-test")
.with_health_fields(
Some(json!({"openai:chat": {
"health_score": 0.7,
"consecutive_failures": 2
}})),
Some(json!({"openai:chat": {
"open": true,
"open_at": "2026-03-26T12:01:00+00:00",
"next_probe_at_unix_secs": 1u64
}})),
),
],
));
let (_upstream_url, upstream_handle) = start_server(upstream).await;
let gateway = build_router_with_state(
AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(
GatewayDataState::with_provider_catalog_reader_for_tests(
provider_catalog_repository,
)
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.get(format!(
"{gateway_url}/api/admin/endpoints/health/key/key-openai"
))
.header(crate::constants::GATEWAY_HEADER, "rust-phase3b")
.header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123")
.header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin")
.header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123")
.send()
.await
.expect("request should succeed");
let status = response.status();
let body = response.text().await.expect("body should read");
assert_eq!(status, StatusCode::OK, "body={body}");
let payload: serde_json::Value = serde_json::from_str(&body).expect("json body should parse");
let circuit = &payload["health_by_format"]["openai:chat"]["circuit_breaker"];
assert_eq!(payload["any_circuit_open"], false);
assert_eq!(circuit["open"], false);
assert_eq!(circuit["state"], "closed");
assert_eq!(payload["key_health_score"], 0.7);
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_recovers_admin_key_health_locally_with_trusted_admin_principal() {
let upstream_hits = Arc::new(Mutex::new(0usize));