mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-04 16:37:46 +08:00
Disable key circuit breaker for pool providers
This commit is contained in:
@@ -16,6 +16,6 @@ pub(crate) use self::reads::{
|
||||
};
|
||||
pub(crate) use self::status::build_admin_provider_pool_status_payload;
|
||||
pub(crate) use self::writes::{
|
||||
admin_provider_pool_key_circuit_breaker_reason, record_admin_provider_pool_error,
|
||||
admin_provider_pool_key_terminal_error_reason, record_admin_provider_pool_error,
|
||||
record_admin_provider_pool_stream_timeout, record_admin_provider_pool_success,
|
||||
};
|
||||
|
||||
@@ -222,7 +222,7 @@ fn first_error_text(
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn admin_provider_pool_key_circuit_breaker_reason(
|
||||
pub(crate) fn admin_provider_pool_key_terminal_error_reason(
|
||||
status_code: u16,
|
||||
error_body: Option<&str>,
|
||||
) -> Option<String> {
|
||||
@@ -475,7 +475,7 @@ pub(crate) async fn record_admin_provider_pool_error(
|
||||
|
||||
if status_code == 400 {
|
||||
// Bad Request is usually attributable to the caller payload, not key health.
|
||||
// Account-level 400s are handled by the orchestration circuit-breaker path.
|
||||
// Account-level 400s are handled by orchestration pool-score feedback.
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -588,7 +588,7 @@ pub(crate) async fn record_admin_provider_pool_stream_timeout(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
admin_provider_pool_key_circuit_breaker_reason, parse_google_quota_cooldown_seconds_at,
|
||||
admin_provider_pool_key_terminal_error_reason, parse_google_quota_cooldown_seconds_at,
|
||||
record_admin_provider_pool_error, record_admin_provider_pool_stream_timeout,
|
||||
record_admin_provider_pool_success,
|
||||
};
|
||||
@@ -747,9 +747,9 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn circuit_reason_detects_workspace_deactivated_errors() {
|
||||
fn terminal_error_reason_detects_workspace_deactivated_errors() {
|
||||
assert_eq!(
|
||||
admin_provider_pool_key_circuit_breaker_reason(
|
||||
admin_provider_pool_key_terminal_error_reason(
|
||||
402,
|
||||
Some(r#"{"error":{"message":"workspace has been deactivated"}}"#),
|
||||
)
|
||||
@@ -757,7 +757,7 @@ mod tests {
|
||||
Some("workspace_deactivated_402:workspace has been deactivated")
|
||||
);
|
||||
assert_eq!(
|
||||
admin_provider_pool_key_circuit_breaker_reason(
|
||||
admin_provider_pool_key_terminal_error_reason(
|
||||
400,
|
||||
Some(r#"{"error":{"message":"deactivated_workspace"}}"#),
|
||||
)
|
||||
@@ -767,9 +767,9 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn circuit_reason_detects_account_ban_errors() {
|
||||
fn terminal_error_reason_detects_account_ban_errors() {
|
||||
assert_eq!(
|
||||
admin_provider_pool_key_circuit_breaker_reason(
|
||||
admin_provider_pool_key_terminal_error_reason(
|
||||
403,
|
||||
Some(r#"{"error":{"message":"AccountSuspendedException: account suspended"}}"#),
|
||||
)
|
||||
@@ -777,7 +777,7 @@ mod tests {
|
||||
Some("forbidden_403")
|
||||
);
|
||||
assert_eq!(
|
||||
admin_provider_pool_key_circuit_breaker_reason(
|
||||
admin_provider_pool_key_terminal_error_reason(
|
||||
423,
|
||||
Some(r#"{"error":{"message":"account access denied"}}"#),
|
||||
)
|
||||
@@ -1099,7 +1099,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn severe_account_errors_use_circuit_breaker_instead_of_pool_cooldown() {
|
||||
async fn severe_account_errors_do_not_use_pool_cooldown() {
|
||||
let Some(redis) = start_managed_redis_or_skip().await else {
|
||||
return;
|
||||
};
|
||||
@@ -1109,7 +1109,7 @@ mod tests {
|
||||
let key_ids = vec!["key-account-disabled".to_string()];
|
||||
|
||||
assert_eq!(
|
||||
admin_provider_pool_key_circuit_breaker_reason(
|
||||
admin_provider_pool_key_terminal_error_reason(
|
||||
401,
|
||||
Some(r#"{"error":{"message":"account has been deactivated"}}"#),
|
||||
)
|
||||
|
||||
@@ -13,7 +13,6 @@ use aether_data_contracts::repository::provider_catalog::{
|
||||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
|
||||
};
|
||||
use aether_data_contracts::repository::usage::StoredProviderApiKeyWindowUsageSummary;
|
||||
use aether_scheduler_core::provider_key_circuit_payload_is_active_open_at;
|
||||
use serde_json::json;
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
@@ -934,18 +933,6 @@ fn admin_pool_health_score(key: &StoredProviderCatalogKey) -> f64 {
|
||||
}
|
||||
}
|
||||
|
||||
fn admin_pool_circuit_breaker_open(key: &StoredProviderCatalogKey, now_unix_secs: u64) -> bool {
|
||||
key.circuit_breaker_by_format
|
||||
.as_ref()
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.map(|formats| {
|
||||
formats
|
||||
.values()
|
||||
.any(|item| provider_key_circuit_payload_is_active_open_at(item, now_unix_secs))
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn admin_pool_scheduling_payload(
|
||||
key: &StoredProviderCatalogKey,
|
||||
cooldown_reason: Option<&str>,
|
||||
@@ -1040,7 +1027,7 @@ pub(super) fn build_admin_pool_key_payload(
|
||||
.as_ref()
|
||||
.and_then(|_| runtime.cooldown_ttl_by_key.get(&key.id).copied());
|
||||
let health_score = admin_pool_health_score(key);
|
||||
let circuit_breaker_open = admin_pool_circuit_breaker_open(key, now_unix_secs);
|
||||
let circuit_breaker_open = false;
|
||||
let auth_semantics = provider_key_auth_semantics(key, provider_type);
|
||||
let account_quota_exhausted = pool_config
|
||||
.as_ref()
|
||||
|
||||
@@ -2,7 +2,7 @@ pub(crate) use super::super::admin::provider::pool::config::{
|
||||
admin_provider_pool_cache_affinity_enabled, admin_provider_pool_config_from_config_value,
|
||||
};
|
||||
pub(crate) use super::super::admin::provider::pool::runtime::{
|
||||
admin_provider_pool_key_circuit_breaker_reason, read_admin_provider_pool_key_cooldown_reason,
|
||||
admin_provider_pool_key_terminal_error_reason, read_admin_provider_pool_key_cooldown_reason,
|
||||
read_admin_provider_pool_runtime_state, record_admin_provider_pool_error,
|
||||
record_admin_provider_pool_stream_timeout, record_admin_provider_pool_success,
|
||||
release_admin_provider_pool_key_lease,
|
||||
|
||||
Reference in New Issue
Block a user