Disable key circuit breaker for pool providers

This commit is contained in:
fawney19
2026-05-29 15:16:06 +08:00
parent 9fb600df1b
commit 1658925f52
12 changed files with 252 additions and 129 deletions
@@ -16,6 +16,6 @@ pub(crate) use self::reads::{
};
pub(crate) use self::status::build_admin_provider_pool_status_payload;
pub(crate) use self::writes::{
admin_provider_pool_key_circuit_breaker_reason, record_admin_provider_pool_error,
admin_provider_pool_key_terminal_error_reason, record_admin_provider_pool_error,
record_admin_provider_pool_stream_timeout, record_admin_provider_pool_success,
};
@@ -222,7 +222,7 @@ fn first_error_text(
})
}
pub(crate) fn admin_provider_pool_key_circuit_breaker_reason(
pub(crate) fn admin_provider_pool_key_terminal_error_reason(
status_code: u16,
error_body: Option<&str>,
) -> Option<String> {
@@ -475,7 +475,7 @@ pub(crate) async fn record_admin_provider_pool_error(
if status_code == 400 {
// Bad Request is usually attributable to the caller payload, not key health.
// Account-level 400s are handled by the orchestration circuit-breaker path.
// Account-level 400s are handled by orchestration pool-score feedback.
return;
}
@@ -588,7 +588,7 @@ pub(crate) async fn record_admin_provider_pool_stream_timeout(
#[cfg(test)]
mod tests {
use super::{
admin_provider_pool_key_circuit_breaker_reason, parse_google_quota_cooldown_seconds_at,
admin_provider_pool_key_terminal_error_reason, parse_google_quota_cooldown_seconds_at,
record_admin_provider_pool_error, record_admin_provider_pool_stream_timeout,
record_admin_provider_pool_success,
};
@@ -747,9 +747,9 @@ mod tests {
}
#[test]
fn circuit_reason_detects_workspace_deactivated_errors() {
fn terminal_error_reason_detects_workspace_deactivated_errors() {
assert_eq!(
admin_provider_pool_key_circuit_breaker_reason(
admin_provider_pool_key_terminal_error_reason(
402,
Some(r#"{"error":{"message":"workspace has been deactivated"}}"#),
)
@@ -757,7 +757,7 @@ mod tests {
Some("workspace_deactivated_402:workspace has been deactivated")
);
assert_eq!(
admin_provider_pool_key_circuit_breaker_reason(
admin_provider_pool_key_terminal_error_reason(
400,
Some(r#"{"error":{"message":"deactivated_workspace"}}"#),
)
@@ -767,9 +767,9 @@ mod tests {
}
#[test]
fn circuit_reason_detects_account_ban_errors() {
fn terminal_error_reason_detects_account_ban_errors() {
assert_eq!(
admin_provider_pool_key_circuit_breaker_reason(
admin_provider_pool_key_terminal_error_reason(
403,
Some(r#"{"error":{"message":"AccountSuspendedException: account suspended"}}"#),
)
@@ -777,7 +777,7 @@ mod tests {
Some("forbidden_403")
);
assert_eq!(
admin_provider_pool_key_circuit_breaker_reason(
admin_provider_pool_key_terminal_error_reason(
423,
Some(r#"{"error":{"message":"account access denied"}}"#),
)
@@ -1099,7 +1099,7 @@ mod tests {
}
#[tokio::test]
async fn severe_account_errors_use_circuit_breaker_instead_of_pool_cooldown() {
async fn severe_account_errors_do_not_use_pool_cooldown() {
let Some(redis) = start_managed_redis_or_skip().await else {
return;
};
@@ -1109,7 +1109,7 @@ mod tests {
let key_ids = vec!["key-account-disabled".to_string()];
assert_eq!(
admin_provider_pool_key_circuit_breaker_reason(
admin_provider_pool_key_terminal_error_reason(
401,
Some(r#"{"error":{"message":"account has been deactivated"}}"#),
)
@@ -13,7 +13,6 @@ use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
};
use aether_data_contracts::repository::usage::StoredProviderApiKeyWindowUsageSummary;
use aether_scheduler_core::provider_key_circuit_payload_is_active_open_at;
use serde_json::json;
use std::collections::BTreeMap;
use std::time::{SystemTime, UNIX_EPOCH};
@@ -934,18 +933,6 @@ fn admin_pool_health_score(key: &StoredProviderCatalogKey) -> f64 {
}
}
fn admin_pool_circuit_breaker_open(key: &StoredProviderCatalogKey, now_unix_secs: u64) -> bool {
key.circuit_breaker_by_format
.as_ref()
.and_then(serde_json::Value::as_object)
.map(|formats| {
formats
.values()
.any(|item| provider_key_circuit_payload_is_active_open_at(item, now_unix_secs))
})
.unwrap_or(false)
}
fn admin_pool_scheduling_payload(
key: &StoredProviderCatalogKey,
cooldown_reason: Option<&str>,
@@ -1040,7 +1027,7 @@ pub(super) fn build_admin_pool_key_payload(
.as_ref()
.and_then(|_| runtime.cooldown_ttl_by_key.get(&key.id).copied());
let health_score = admin_pool_health_score(key);
let circuit_breaker_open = admin_pool_circuit_breaker_open(key, now_unix_secs);
let circuit_breaker_open = false;
let auth_semantics = provider_key_auth_semantics(key, provider_type);
let account_quota_exhausted = pool_config
.as_ref()
@@ -2,7 +2,7 @@ pub(crate) use super::super::admin::provider::pool::config::{
admin_provider_pool_cache_affinity_enabled, admin_provider_pool_config_from_config_value,
};
pub(crate) use super::super::admin::provider::pool::runtime::{
admin_provider_pool_key_circuit_breaker_reason, read_admin_provider_pool_key_cooldown_reason,
admin_provider_pool_key_terminal_error_reason, read_admin_provider_pool_key_cooldown_reason,
read_admin_provider_pool_runtime_state, record_admin_provider_pool_error,
record_admin_provider_pool_stream_timeout, record_admin_provider_pool_success,
release_admin_provider_pool_key_lease,