Disable key circuit breaker for pool providers

apps/aether-gateway/src/handlers/admin/provider/pool/runtime/mod.rs

@@ -16,6 +16,6 @@ pub(crate) use self::reads::{
 };
 pub(crate) use self::status::build_admin_provider_pool_status_payload;
 pub(crate) use self::writes::{
-    admin_provider_pool_key_circuit_breaker_reason, record_admin_provider_pool_error,
+    admin_provider_pool_key_terminal_error_reason, record_admin_provider_pool_error,
     record_admin_provider_pool_stream_timeout, record_admin_provider_pool_success,
 };

apps/aether-gateway/src/handlers/admin/provider/pool/runtime/writes.rs

@@ -222,7 +222,7 @@ fn first_error_text(
     })
 }
 
-pub(crate) fn admin_provider_pool_key_circuit_breaker_reason(
+pub(crate) fn admin_provider_pool_key_terminal_error_reason(
     status_code: u16,
     error_body: Option<&str>,
 ) -> Option<String> {
@@ -475,7 +475,7 @@ pub(crate) async fn record_admin_provider_pool_error(
 
     if status_code == 400 {
         // Bad Request is usually attributable to the caller payload, not key health.
-        // Account-level 400s are handled by the orchestration circuit-breaker path.
+        // Account-level 400s are handled by orchestration pool-score feedback.
         return;
     }
 
@@ -588,7 +588,7 @@ pub(crate) async fn record_admin_provider_pool_stream_timeout(
 #[cfg(test)]
 mod tests {
     use super::{
-        admin_provider_pool_key_circuit_breaker_reason, parse_google_quota_cooldown_seconds_at,
+        admin_provider_pool_key_terminal_error_reason, parse_google_quota_cooldown_seconds_at,
         record_admin_provider_pool_error, record_admin_provider_pool_stream_timeout,
         record_admin_provider_pool_success,
     };
@@ -747,9 +747,9 @@ mod tests {
     }
 
     #[test]
-    fn circuit_reason_detects_workspace_deactivated_errors() {
+    fn terminal_error_reason_detects_workspace_deactivated_errors() {
         assert_eq!(
-            admin_provider_pool_key_circuit_breaker_reason(
+            admin_provider_pool_key_terminal_error_reason(
                 402,
                 Some(r#"{"error":{"message":"workspace has been deactivated"}}"#),
             )
@@ -757,7 +757,7 @@ mod tests {
             Some("workspace_deactivated_402:workspace has been deactivated")
         );
         assert_eq!(
-            admin_provider_pool_key_circuit_breaker_reason(
+            admin_provider_pool_key_terminal_error_reason(
                 400,
                 Some(r#"{"error":{"message":"deactivated_workspace"}}"#),
             )
@@ -767,9 +767,9 @@ mod tests {
     }
 
     #[test]
-    fn circuit_reason_detects_account_ban_errors() {
+    fn terminal_error_reason_detects_account_ban_errors() {
         assert_eq!(
-            admin_provider_pool_key_circuit_breaker_reason(
+            admin_provider_pool_key_terminal_error_reason(
                 403,
                 Some(r#"{"error":{"message":"AccountSuspendedException: account suspended"}}"#),
             )
@@ -777,7 +777,7 @@ mod tests {
             Some("forbidden_403")
         );
         assert_eq!(
-            admin_provider_pool_key_circuit_breaker_reason(
+            admin_provider_pool_key_terminal_error_reason(
                 423,
                 Some(r#"{"error":{"message":"account access denied"}}"#),
             )
@@ -1099,7 +1099,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn severe_account_errors_use_circuit_breaker_instead_of_pool_cooldown() {
+    async fn severe_account_errors_do_not_use_pool_cooldown() {
         let Some(redis) = start_managed_redis_or_skip().await else {
             return;
         };
@@ -1109,7 +1109,7 @@ mod tests {
         let key_ids = vec!["key-account-disabled".to_string()];
 
         assert_eq!(
-            admin_provider_pool_key_circuit_breaker_reason(
+            admin_provider_pool_key_terminal_error_reason(
                 401,
                 Some(r#"{"error":{"message":"account has been deactivated"}}"#),
             )

apps/aether-gateway/src/handlers/admin/provider/pool_admin/payloads.rs

@@ -13,7 +13,6 @@ use aether_data_contracts::repository::provider_catalog::{
     StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
 };
 use aether_data_contracts::repository::usage::StoredProviderApiKeyWindowUsageSummary;
-use aether_scheduler_core::provider_key_circuit_payload_is_active_open_at;
 use serde_json::json;
 use std::collections::BTreeMap;
 use std::time::{SystemTime, UNIX_EPOCH};
@@ -934,18 +933,6 @@ fn admin_pool_health_score(key: &StoredProviderCatalogKey) -> f64 {
     }
 }
 
-fn admin_pool_circuit_breaker_open(key: &StoredProviderCatalogKey, now_unix_secs: u64) -> bool {
-    key.circuit_breaker_by_format
-        .as_ref()
-        .and_then(serde_json::Value::as_object)
-        .map(|formats| {
-            formats
-                .values()
-                .any(|item| provider_key_circuit_payload_is_active_open_at(item, now_unix_secs))
-        })
-        .unwrap_or(false)
-}
-
 fn admin_pool_scheduling_payload(
     key: &StoredProviderCatalogKey,
     cooldown_reason: Option<&str>,
@@ -1040,7 +1027,7 @@ pub(super) fn build_admin_pool_key_payload(
         .as_ref()
         .and_then(|_| runtime.cooldown_ttl_by_key.get(&key.id).copied());
     let health_score = admin_pool_health_score(key);
-    let circuit_breaker_open = admin_pool_circuit_breaker_open(key, now_unix_secs);
+    let circuit_breaker_open = false;
     let auth_semantics = provider_key_auth_semantics(key, provider_type);
     let account_quota_exhausted = pool_config
         .as_ref()

apps/aether-gateway/src/handlers/shared/provider_pool.rs

@@ -2,7 +2,7 @@ pub(crate) use super::super::admin::provider::pool::config::{
     admin_provider_pool_cache_affinity_enabled, admin_provider_pool_config_from_config_value,
 };
 pub(crate) use super::super::admin::provider::pool::runtime::{
-    admin_provider_pool_key_circuit_breaker_reason, read_admin_provider_pool_key_cooldown_reason,
+    admin_provider_pool_key_terminal_error_reason, read_admin_provider_pool_key_cooldown_reason,
     read_admin_provider_pool_runtime_state, record_admin_provider_pool_error,
     record_admin_provider_pool_stream_timeout, record_admin_provider_pool_success,
     release_admin_provider_pool_key_lease,

apps/aether-gateway/src/orchestration/effects.rs

@@ -19,8 +19,7 @@ use tracing::warn;
 use super::{
     local_failover_error_message, project_local_adaptive_rate_limit,
     project_local_adaptive_success, project_local_failure_health, project_local_key_circuit_closed,
-    project_local_key_circuit_failure, project_local_key_circuit_open,
-    project_local_success_health, LocalFailoverClassification,
+    project_local_key_circuit_failure, project_local_success_health, LocalFailoverClassification,
 };
 use crate::ai_serving::extract_pool_sticky_session_token;
 use crate::client_session_affinity::{
@@ -29,7 +28,7 @@ use crate::client_session_affinity::{
 use crate::clock::current_unix_secs;
 use crate::handlers::shared::provider_pool::admin_provider_pool_config_from_config_value;
 use crate::handlers::shared::provider_pool::{
-    admin_provider_pool_key_circuit_breaker_reason, record_admin_provider_pool_error,
+    admin_provider_pool_key_terminal_error_reason, record_admin_provider_pool_error,
     record_admin_provider_pool_stream_timeout, record_admin_provider_pool_success,
     release_admin_provider_pool_key_lease, AdminProviderPoolConfig,
 };
@@ -574,6 +573,7 @@ async fn record_health_failure_effect(
     else {
         return;
     };
+    let is_pool_provider = local_execution_plan_uses_pool(state, context.plan).await;
     let observed_at_unix_secs = current_unix_secs();
     let Some(health_by_format) = project_local_failure_health(
         current_key.health_by_format.as_ref(),
@@ -589,16 +589,24 @@ async fn record_health_failure_effect(
         .and_then(|value| value.get("consecutive_failures"))
         .and_then(Value::as_u64)
         .unwrap_or(0);
-    let circuit_breaker_by_format = project_local_key_circuit_failure(
-        current_key.circuit_breaker_by_format.as_ref(),
-        api_format,
-        observed_at_unix_secs,
-        consecutive_failures,
-        current_key.max_probe_interval_minutes,
-    );
-    let circuit_breaker_update = circuit_breaker_by_format
-        .as_ref()
-        .or(current_key.circuit_breaker_by_format.as_ref());
+    let circuit_breaker_update_owned = if is_pool_provider {
+        None
+    } else {
+        project_local_key_circuit_failure(
+            current_key.circuit_breaker_by_format.as_ref(),
+            api_format,
+            observed_at_unix_secs,
+            consecutive_failures,
+            current_key.max_probe_interval_minutes,
+        )
+    };
+    let circuit_breaker_update = if is_pool_provider {
+        None
+    } else {
+        circuit_breaker_update_owned
+            .as_ref()
+            .or(current_key.circuit_breaker_by_format.as_ref())
+    };
 
     if let Err(err) = state
         .update_provider_catalog_key_health_state(
@@ -636,23 +644,35 @@ async fn record_health_success_effect(
     else {
         return;
     };
+    let is_pool_provider = local_execution_plan_uses_pool(state, context.plan).await;
     let Some(health_by_format) =
         project_local_success_health(current_key.health_by_format.as_ref(), api_format)
     else {
         return;
     };
-    let circuit_breaker_by_format = current_key
-        .circuit_breaker_by_format
-        .as_ref()
-        .and_then(|current| project_local_key_circuit_closed(Some(current), api_format));
+    let circuit_breaker_update_owned = if is_pool_provider {
+        None
+    } else {
+        current_key
+            .circuit_breaker_by_format
+            .as_ref()
+            .and_then(|current| project_local_key_circuit_closed(Some(current), api_format))
+    };
     if current_key.health_by_format.as_ref() == Some(&health_by_format)
-        && circuit_breaker_by_format.as_ref() == current_key.circuit_breaker_by_format.as_ref()
+        && ((is_pool_provider && current_key.circuit_breaker_by_format.is_none())
+            || (!is_pool_provider
+                && circuit_breaker_update_owned.as_ref()
+                    == current_key.circuit_breaker_by_format.as_ref()))
     {
         return;
     }
-    let circuit_breaker_update = circuit_breaker_by_format
-        .as_ref()
-        .or(current_key.circuit_breaker_by_format.as_ref());
+    let circuit_breaker_update = if is_pool_provider {
+        None
+    } else {
+        circuit_breaker_update_owned
+            .as_ref()
+            .or(current_key.circuit_breaker_by_format.as_ref())
+    };
 
     if let Err(err) = state
         .update_provider_catalog_key_health_state(
@@ -711,9 +731,9 @@ async fn record_pool_error_effect(
     context: LocalExecutionEffectContext<'_>,
     effect: LocalPoolErrorEffect<'_>,
 ) {
-    let circuit_reason =
-        admin_provider_pool_key_circuit_breaker_reason(effect.status_code, effect.error_body);
-    if circuit_reason.is_none()
+    let terminal_error_reason =
+        admin_provider_pool_key_terminal_error_reason(effect.status_code, effect.error_body);
+    if terminal_error_reason.is_none()
         && !local_candidate_failure_should_record_pool_error(
             effect.classification,
             effect.status_code,
@@ -726,10 +746,7 @@ async fn record_pool_error_effect(
         return;
     };
 
-    if let Some(reason) = circuit_reason {
-        open_pool_key_circuit_breaker(state, context, &reason).await;
-    }
-
+    clear_pool_key_circuit_breaker(state, context).await;
     record_admin_provider_pool_error(
         state.runtime_state.as_ref(),
         &context.plan.provider_id,
@@ -757,16 +774,10 @@ async fn record_pool_error_effect(
     .await;
 }
 
-async fn open_pool_key_circuit_breaker(
+async fn clear_pool_key_circuit_breaker(
     state: &AppState,
     context: LocalExecutionEffectContext<'_>,
-    reason: &str,
 ) {
-    let api_format = context.plan.provider_api_format.trim();
-    if api_format.is_empty() {
-        return;
-    }
-
     let Some(current_key) = state
         .read_provider_catalog_keys_by_ids(std::slice::from_ref(&context.plan.key_id))
         .await
@@ -775,27 +786,21 @@ async fn open_pool_key_circuit_breaker(
     else {
         return;
     };
-    let Some(circuit_breaker_by_format) = project_local_key_circuit_open(
-        current_key.circuit_breaker_by_format.as_ref(),
-        api_format,
-        reason,
-        current_unix_secs(),
-        current_key.max_probe_interval_minutes,
-    ) else {
+    if current_key.circuit_breaker_by_format.is_none() {
         return;
-    };
+    }
 
     if let Err(err) = state
         .update_provider_catalog_key_health_state(
             &context.plan.key_id,
             current_key.is_active,
             current_key.health_by_format.as_ref(),
-            Some(&circuit_breaker_by_format),
+            None,
         )
         .await
     {
         warn!(
-            "gateway orchestration effects: failed to open pool key circuit for provider {} endpoint {} key {}: {:?}",
+            "gateway orchestration effects: failed to clear pool key circuit for provider {} endpoint {} key {}: {:?}",
             context.plan.provider_id, context.plan.endpoint_id, context.plan.key_id, err
         );
     }
@@ -984,6 +989,10 @@ fn pool_score_hard_state_for_status(
     status_code: u16,
     error_body: Option<&str>,
 ) -> Option<PoolMemberHardState> {
+    if let Some(reason) = admin_provider_pool_key_terminal_error_reason(status_code, error_body) {
+        return Some(pool_score_hard_state_for_terminal_error_reason(&reason));
+    }
+
     match status_code {
         401 | 403 => Some(PoolMemberHardState::AuthInvalid),
         402 => Some(PoolMemberHardState::QuotaExhausted),
@@ -1006,6 +1015,16 @@ fn pool_score_hard_state_for_status(
     }
 }
 
+fn pool_score_hard_state_for_terminal_error_reason(reason: &str) -> PoolMemberHardState {
+    if reason.starts_with("payment_required_") {
+        PoolMemberHardState::QuotaExhausted
+    } else if reason.starts_with("forbidden_") {
+        PoolMemberHardState::AuthInvalid
+    } else {
+        PoolMemberHardState::Banned
+    }
+}
+
 fn pool_score_delta_for_status(status_code: u16) -> i32 {
     match status_code {
         401 | 403 => -2_000,
@@ -1028,6 +1047,7 @@ mod tests {
     use aether_data_contracts::repository::candidates::{
         RequestCandidateStatus, StoredRequestCandidate,
     };
+    use aether_data_contracts::repository::pool_scores::PoolMemberHardState;
     use aether_data_contracts::repository::provider_catalog::{
         StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
     };
@@ -1036,9 +1056,10 @@ mod tests {
 
     use super::{
         apply_local_execution_effect, local_candidate_failure_should_record_pool_error,
-        LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect, LocalAttemptFailureEffect,
-        LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
-        LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
+        pool_score_hard_state_for_status, LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect,
+        LocalAttemptFailureEffect, LocalExecutionEffect, LocalExecutionEffectContext,
+        LocalHealthFailureEffect, LocalHealthSuccessEffect, LocalOAuthInvalidationEffect,
+        LocalPoolErrorEffect,
     };
     use crate::data::{GatewayDataConfig, GatewayDataState};
     use crate::orchestration::LocalFailoverClassification;
@@ -1943,13 +1964,46 @@ mod tests {
         ));
     }
 
+    #[test]
+    fn terminal_pool_account_errors_project_pool_hard_state() {
+        assert_eq!(
+            pool_score_hard_state_for_status(
+                400,
+                Some(r#"{"error":{"message":"deactivated_workspace"}}"#),
+            ),
+            Some(PoolMemberHardState::Banned)
+        );
+        assert_eq!(
+            pool_score_hard_state_for_status(
+                402,
+                Some(r#"{"error":{"message":"payment required"}}"#),
+            ),
+            Some(PoolMemberHardState::QuotaExhausted)
+        );
+    }
+
     #[tokio::test]
-    async fn pool_account_error_opens_key_circuit() {
+    async fn pool_account_error_does_not_open_key_circuit() {
         let Some(redis) = start_managed_redis_or_skip().await else {
             return;
         };
         let state = codex_state_with_redis(redis.redis_url(), "orchestration_pool_circuit");
         let plan = sample_codex_plan();
+        let legacy_circuit = json!({
+            "openai:responses": {
+                "open": true,
+                "reason": "legacy"
+            }
+        });
+        state
+            .update_provider_catalog_key_health_state(
+                &plan.key_id,
+                true,
+                None,
+                Some(&legacy_circuit),
+            )
+            .await
+            .expect("legacy circuit should seed");
 
         apply_local_execution_effect(
             &state,
@@ -1973,15 +2027,7 @@ mod tests {
             .into_iter()
             .next()
             .expect("stored key should exist");
-        let circuit = stored_key
-            .circuit_breaker_by_format
-            .as_ref()
-            .and_then(|value| value.get("openai:responses"))
-            .expect("format circuit should be stored");
-        assert_eq!(circuit["open"], json!(true));
-        assert_eq!(circuit["reason"], json!("account_deactivated_401"));
-        assert!(circuit["next_probe_at"].is_string());
-        assert!(circuit["next_probe_at_unix_secs"].as_u64().is_some());
+        assert_eq!(stored_key.circuit_breaker_by_format, None);
     }
 
     #[tokio::test]
@@ -2144,6 +2190,45 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn pool_health_failure_does_not_open_key_circuit_after_eight_consecutive_failures() {
+        let state = pool_health_state();
+        let plan = sample_plan();
+
+        for _ in 0..8 {
+            apply_local_execution_effect(
+                &state,
+                LocalExecutionEffectContext {
+                    plan: &plan,
+                    report_context: None,
+                },
+                LocalExecutionEffect::HealthFailure(LocalHealthFailureEffect {
+                    status_code: 503,
+                    classification: LocalFailoverClassification::RetryUpstreamFailure,
+                }),
+            )
+            .await;
+        }
+
+        let stored_key = state
+            .read_provider_catalog_keys_by_ids(std::slice::from_ref(&plan.key_id))
+            .await
+            .expect("provider catalog keys should load")
+            .into_iter()
+            .next()
+            .expect("stored key should exist");
+        assert_eq!(stored_key.circuit_breaker_by_format, None);
+        assert_eq!(
+            stored_key
+                .health_by_format
+                .as_ref()
+                .and_then(|value| value.get("openai:chat"))
+                .and_then(|value| value.get("consecutive_failures"))
+                .and_then(Value::as_u64),
+            Some(8)
+        );
+    }
+
     #[tokio::test]
     async fn health_success_projection_resets_key_health_for_format() {
         let state = health_state();

apps/aether-gateway/src/orchestration/mod.rs

@@ -35,8 +35,7 @@ pub(crate) use self::effects::{
 };
 pub(crate) use self::health::{
     project_local_failure_health, project_local_key_circuit_closed,
-    project_local_key_circuit_failure, project_local_key_circuit_open,
-    project_local_success_health,
+    project_local_key_circuit_failure, project_local_success_health,
 };
 pub(crate) use self::policy::{
     append_local_failover_policy_to_value, local_failover_policy_from_report_context,

apps/aether-gateway/src/scheduler/candidate/runtime.rs

@@ -128,6 +128,7 @@ pub(super) fn is_candidate_selectable(
                 .get(candidate.key_id.as_str())
                 .copied()
                 .unwrap_or(false),
+        enforce_key_circuit_breaker: !pool_group,
         rpm_reset_at: (!pool_group)
             .then(|| {
                 snapshot
@@ -182,6 +183,7 @@ pub(super) fn current_candidate_runtime_skip_reason(
                 .get(candidate.key_id.as_str())
                 .copied()
                 .unwrap_or(false),
+        enforce_key_circuit_breaker: !pool_group,
         rpm_reset_at,
     })
 }

apps/aether-gateway/src/scheduler/candidate/tests/selection.rs

@@ -1553,6 +1553,71 @@ async fn selects_next_candidate_when_first_provider_key_circuit_is_open() {
     assert_eq!(selected.key_id, "key-b");
 }
 
+#[tokio::test]
+async fn pool_provider_ignores_key_circuit_open_runtime_skip() {
+    let mut first = sample_row();
+    first.provider_id = "provider-a".to_string();
+    first.provider_name = "openai-a".to_string();
+    first.endpoint_id = "endpoint-a".to_string();
+    first.key_id = "key-a".to_string();
+    first.key_name = "alpha".to_string();
+    first.key_global_priority_by_format = Some(serde_json::json!({"openai:chat": 1}));
+
+    let mut second = sample_row();
+    second.provider_id = "provider-b".to_string();
+    second.provider_name = "openai-b".to_string();
+    second.endpoint_id = "endpoint-b".to_string();
+    second.key_id = "key-b".to_string();
+    second.key_name = "beta".to_string();
+    second.key_global_priority_by_format = Some(serde_json::json!({"openai:chat": 2}));
+
+    let candidates = Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
+        first, second,
+    ]));
+    let mut pool_provider = sample_provider("provider-a", None);
+    pool_provider.config = Some(serde_json::json!({"pool_advanced": {"enabled": true}}));
+    let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
+        vec![pool_provider, sample_provider("provider-b", None)],
+        Vec::new(),
+        vec![
+            sample_key("key-a", "provider-a", Some(10)).with_health_fields(
+                Some(serde_json::json!({"openai:chat": {"health_score": 0.2}})),
+                Some(serde_json::json!({"openai:chat": {"open": true}})),
+            ),
+            sample_key("key-b", "provider-b", Some(10)),
+        ],
+    ));
+    let quotas = Arc::new(InMemoryProviderQuotaRepository::seed(vec![]));
+    let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![]));
+    let state = AppState::new()
+        .expect("state should build")
+        .with_data_state_for_tests(
+            GatewayDataState::with_candidate_selection_provider_catalog_quota_and_request_candidates_for_tests(
+                candidates,
+                provider_catalog,
+                quotas,
+                request_candidates,
+            ),
+        );
+
+    let (selected, skipped) = collect_selectable_candidates_with_skip_reasons(
+        state.data.as_ref(),
+        &state,
+        "openai:chat",
+        "gpt-4.1",
+        false,
+        None,
+        100,
+    )
+    .await
+    .expect("selection should succeed");
+
+    assert!(selected
+        .iter()
+        .any(|candidate| candidate.provider_id == "provider-a"));
+    assert!(skipped.is_empty());
+}
+
 #[tokio::test]
 async fn exposes_runtime_skipped_candidates_with_skip_reasons() {
     let mut first = sample_row();

apps/aether-gateway/src/tests/control/admin/pool.rs

@@ -1595,7 +1595,7 @@ async fn gateway_ignores_health_signals_in_pool_scheduling_status() {
 
     assert_eq!(keys.len(), 2);
     assert_eq!(keys[0]["key_name"], json!("circuit-open"));
-    assert_eq!(keys[0]["circuit_breaker_open"], json!(true));
+    assert_eq!(keys[0]["circuit_breaker_open"], json!(false));
     assert_eq!(keys[0]["scheduling_status"], json!("available"));
     assert_eq!(keys[0]["scheduling_reason"], json!("available"));
     assert_eq!(keys[0]["scheduling_label"], json!("可用"));