fix: restore provider key circuit breaker backoff

apps/aether-gateway/src/handlers/admin/provider/write/keys/create.rs

@@ -2,7 +2,7 @@ use crate::handlers::admin::provider::shared::payloads::AdminProviderKeyCreateRe
 use crate::handlers::admin::provider::write::normalize::{
     normalize_allow_auth_channel_mismatch_formats, normalize_api_format_json_object_keys,
     normalize_api_format_list, normalize_auth_type, normalize_auth_type_by_format,
-    validate_vertex_api_formats,
+    normalize_max_probe_interval_minutes, validate_vertex_api_formats,
 };
 use crate::handlers::admin::request::AdminAppState;
 use crate::handlers::admin::shared::{
@@ -181,7 +181,8 @@ pub(crate) async fn build_admin_create_provider_key_record(
     key.rpm_limit = payload.rpm_limit;
     key.concurrent_limit = normalize_optional_api_key_concurrent_limit(payload.concurrent_limit)?;
     key.cache_ttl_minutes = payload.cache_ttl_minutes.unwrap_or(5);
-    key.max_probe_interval_minutes = payload.max_probe_interval_minutes.unwrap_or(32);
+    key.max_probe_interval_minutes =
+        normalize_max_probe_interval_minutes(payload.max_probe_interval_minutes.unwrap_or(32))?;
     key.request_count = Some(0);
     key.success_count = Some(0);
     key.error_count = Some(0);

apps/aether-gateway/src/handlers/admin/provider/write/keys/payload.rs

@@ -41,8 +41,9 @@ async fn build_admin_provider_key_items_payload(
         .ok()
         .map(|duration| duration.as_secs())
         .unwrap_or(0);
-    let items = key_page
+    let keys = key_page.items;
-        .items
+
+    let items = keys
         .into_iter()
         .map(|key| {
             let api_formats =

apps/aether-gateway/src/handlers/admin/provider/write/keys/update.rs

@@ -2,7 +2,7 @@ use crate::handlers::admin::provider::shared::payloads::AdminProviderKeyUpdatePa
 use crate::handlers::admin::provider::write::normalize::{
     normalize_allow_auth_channel_mismatch_formats, normalize_api_format_json_object_keys,
     normalize_api_format_list, normalize_auth_type, normalize_auth_type_by_format,
-    validate_vertex_api_formats,
+    normalize_max_probe_interval_minutes, validate_vertex_api_formats,
 };
 use crate::handlers::admin::request::AdminAppState;
 use crate::handlers::admin::shared::{
@@ -293,7 +293,8 @@ pub(crate) async fn build_admin_update_provider_key_record(
         updated.cache_ttl_minutes = cache_ttl_minutes;
     }
     if let Some(max_probe_interval_minutes) = payload.max_probe_interval_minutes {
-        updated.max_probe_interval_minutes = max_probe_interval_minutes;
+        updated.max_probe_interval_minutes =
+            normalize_max_probe_interval_minutes(max_probe_interval_minutes)?;
     }
     if let Some(is_active) = payload.is_active {
         updated.is_active = is_active;

apps/aether-gateway/src/handlers/admin/provider/write/normalize.rs

@@ -115,6 +115,14 @@ pub(crate) fn normalize_auth_type(value: Option<&str>) -> Result<String, String>
     }
 }
 
+pub(crate) fn normalize_max_probe_interval_minutes(value: i32) -> Result<i32, String> {
+    if (0..=32).contains(&value) {
+        Ok(value)
+    } else {
+        Err("max_probe_interval_minutes 必须在 0 到 32 之间".to_string())
+    }
+}
+
 pub(crate) fn normalize_pool_advanced_config(
     value: Option<serde_json::Value>,
 ) -> Result<Option<serde_json::Value>, String> {

apps/aether-gateway/src/orchestration/effects.rs

@@ -19,7 +19,8 @@ use tracing::warn;
 use super::{
     local_failover_error_message, project_local_adaptive_rate_limit,
     project_local_adaptive_success, project_local_failure_health, project_local_key_circuit_closed,
-    project_local_key_circuit_open, project_local_success_health, LocalFailoverClassification,
+    project_local_key_circuit_failure, project_local_key_circuit_open,
+    project_local_success_health, LocalFailoverClassification,
 };
 use crate::ai_serving::extract_pool_sticky_session_token;
 use crate::client_session_affinity::{
@@ -521,21 +522,38 @@ async fn record_health_failure_effect(
     else {
         return;
     };
+    let observed_at_unix_secs = current_unix_secs();
     let Some(health_by_format) = project_local_failure_health(
         current_key.health_by_format.as_ref(),
         api_format,
         effect.classification,
         effect.status_code,
-        current_unix_secs(),
+        observed_at_unix_secs,
     ) else {
         return;
     };
+    let consecutive_failures = health_by_format
+        .get(api_format)
+        .and_then(|value| value.get("consecutive_failures"))
+        .and_then(Value::as_u64)
+        .unwrap_or(0);
+    let circuit_breaker_by_format = project_local_key_circuit_failure(
+        current_key.circuit_breaker_by_format.as_ref(),
+        api_format,
+        observed_at_unix_secs,
+        consecutive_failures,
+        current_key.max_probe_interval_minutes,
+    );
+    let circuit_breaker_update = circuit_breaker_by_format
+        .as_ref()
+        .or(current_key.circuit_breaker_by_format.as_ref());
 
     if let Err(err) = state
-        .update_provider_catalog_key_format_health(
+        .update_provider_catalog_key_health_state(
             &context.plan.key_id,
-            api_format,
+            current_key.is_active,
-            &health_by_format,
+            Some(&health_by_format),
+            circuit_breaker_update,
         )
         .await
     {
@@ -705,6 +723,7 @@ async fn open_pool_key_circuit_breaker(
         api_format,
         reason,
         current_unix_secs(),
+        current_key.max_probe_interval_minutes,
     ) else {
         return;
     };
@@ -1831,6 +1850,51 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn health_failure_opens_circuit_after_eight_consecutive_failures() {
+        let state = health_state();
+        let plan = sample_plan();
+
+        for _ in 0..8 {
+            apply_local_execution_effect(
+                &state,
+                LocalExecutionEffectContext {
+                    plan: &plan,
+                    report_context: None,
+                },
+                LocalExecutionEffect::HealthFailure(LocalHealthFailureEffect {
+                    status_code: 503,
+                    classification: LocalFailoverClassification::RetryUpstreamFailure,
+                }),
+            )
+            .await;
+        }
+
+        let stored_key = state
+            .read_provider_catalog_keys_by_ids(std::slice::from_ref(&plan.key_id))
+            .await
+            .expect("provider catalog keys should load")
+            .into_iter()
+            .next()
+            .expect("stored key should exist");
+        let circuit = stored_key
+            .circuit_breaker_by_format
+            .as_ref()
+            .and_then(|value| value.get("openai:chat"))
+            .expect("format circuit should be stored");
+        assert_eq!(circuit["open"], json!(true));
+        assert_eq!(circuit["reason"], json!("consecutive_failures_8"));
+        assert_eq!(circuit["probe_interval_minutes"], json!(1));
+        assert!(circuit["next_probe_at_unix_secs"].as_u64().is_some());
+        assert_eq!(
+            circuit["request_results_window"]
+                .as_array()
+                .map(Vec::len)
+                .unwrap_or_default(),
+            8
+        );
+    }
+
     #[tokio::test]
     async fn health_success_projection_resets_key_health_for_format() {
         let state = health_state();

apps/aether-gateway/src/orchestration/health.rs

@@ -4,7 +4,8 @@ use super::LocalFailoverClassification;
 use crate::handlers::shared::unix_secs_to_rfc3339;
 
 const LOCAL_HEALTH_SCORE_FLOOR: f64 = 0.2;
-const LOCAL_KEY_CIRCUIT_PROBE_DELAY_SECS: u64 = 32 * 60;
+pub(crate) const LOCAL_KEY_CIRCUIT_FAILURE_THRESHOLD: u64 = 8;
+const LOCAL_KEY_CIRCUIT_MAX_PROBE_INTERVAL_MINUTES: u64 = 32;
 
 pub(crate) fn project_local_failure_health(
     current_health_by_format: Option<&Value>,
@@ -79,6 +80,7 @@ pub(crate) fn project_local_key_circuit_open(
     api_format: &str,
     reason: &str,
     observed_at_unix_secs: u64,
+    max_probe_interval_minutes: i32,
 ) -> Option<Value> {
     let api_format = api_format.trim();
     let reason = reason.trim();
@@ -86,23 +88,143 @@ pub(crate) fn project_local_key_circuit_open(
         return None;
     }
 
-    let next_probe_at_unix_secs =
-        observed_at_unix_secs.saturating_add(LOCAL_KEY_CIRCUIT_PROBE_DELAY_SECS);
     let mut circuit_by_format = current_circuit_by_format
         .and_then(Value::as_object)
         .cloned()
         .unwrap_or_default();
+    let current = circuit_by_format
+        .get(api_format)
+        .and_then(Value::as_object)
+        .cloned()
+        .unwrap_or_default();
+    let max_probe_interval_minutes =
+        normalize_max_probe_interval_minutes(max_probe_interval_minutes);
+    let probe_interval_minutes =
+        next_circuit_probe_interval_minutes(&current, max_probe_interval_minutes);
+    let next_probe_at_unix_secs =
+        next_probe_at_unix_secs(observed_at_unix_secs, probe_interval_minutes);
+    let open_at = current
+        .get("open_at")
+        .filter(|_| current_bool(&current, "open"))
+        .cloned()
+        .unwrap_or_else(|| json!(unix_secs_to_rfc3339(observed_at_unix_secs)));
+    let half_open_failures = current
+        .get("half_open_failures")
+        .and_then(Value::as_u64)
+        .unwrap_or(0)
+        .saturating_add(u64::from(current_bool(&current, "open")));
+    let request_results_window =
+        append_request_result_window(&current, observed_at_unix_secs, false);
     circuit_by_format.insert(
         api_format.to_string(),
         json!({
             "open": true,
-            "open_at": unix_secs_to_rfc3339(observed_at_unix_secs),
+            "open_at": open_at,
             "reason": reason,
             "next_probe_at": unix_secs_to_rfc3339(next_probe_at_unix_secs),
             "next_probe_at_unix_secs": next_probe_at_unix_secs,
+            "probe_interval_minutes": probe_interval_minutes,
+            "max_probe_interval_minutes": max_probe_interval_minutes,
+            "last_failure_at": unix_secs_to_rfc3339(observed_at_unix_secs),
+            "last_probe_failure_at": if half_open_failures > 0 {
+                json!(unix_secs_to_rfc3339(observed_at_unix_secs))
+            } else {
+                Value::Null
+            },
             "half_open_until": Value::Null,
             "half_open_successes": 0,
-            "half_open_failures": 0,
+            "half_open_failures": half_open_failures,
+            "request_results_window": request_results_window,
+        }),
+    );
+    Some(Value::Object(circuit_by_format))
+}
+
+pub(crate) fn project_local_key_circuit_failure(
+    current_circuit_by_format: Option<&Value>,
+    api_format: &str,
+    observed_at_unix_secs: u64,
+    consecutive_failures: u64,
+    max_probe_interval_minutes: i32,
+) -> Option<Value> {
+    let api_format = api_format.trim();
+    if api_format.is_empty() {
+        return None;
+    }
+
+    let mut circuit_by_format = current_circuit_by_format
+        .and_then(Value::as_object)
+        .cloned()
+        .unwrap_or_default();
+    let current = circuit_by_format
+        .get(api_format)
+        .and_then(Value::as_object)
+        .cloned()
+        .unwrap_or_default();
+    let request_results_window =
+        append_request_result_window(&current, observed_at_unix_secs, false);
+    let already_open = current_bool(&current, "open");
+    if !already_open && consecutive_failures < LOCAL_KEY_CIRCUIT_FAILURE_THRESHOLD {
+        circuit_by_format.insert(
+            api_format.to_string(),
+            json!({
+                "open": false,
+                "open_at": Value::Null,
+                "reason": Value::Null,
+                "next_probe_at": Value::Null,
+                "next_probe_at_unix_secs": Value::Null,
+                "probe_interval_minutes": 0,
+                "max_probe_interval_minutes": normalize_max_probe_interval_minutes(max_probe_interval_minutes),
+                "failure_count": consecutive_failures,
+                "last_failure_at": unix_secs_to_rfc3339(observed_at_unix_secs),
+                "last_probe_failure_at": Value::Null,
+                "half_open_until": Value::Null,
+                "half_open_successes": 0,
+                "half_open_failures": 0,
+                "request_results_window": request_results_window,
+            }),
+        );
+        return Some(Value::Object(circuit_by_format));
+    }
+
+    let max_probe_interval_minutes =
+        normalize_max_probe_interval_minutes(max_probe_interval_minutes);
+    let probe_interval_minutes =
+        next_circuit_probe_interval_minutes(&current, max_probe_interval_minutes);
+    let next_probe_at_unix_secs =
+        next_probe_at_unix_secs(observed_at_unix_secs, probe_interval_minutes);
+    let open_at = current
+        .get("open_at")
+        .filter(|_| already_open)
+        .cloned()
+        .unwrap_or_else(|| json!(unix_secs_to_rfc3339(observed_at_unix_secs)));
+    let half_open_failures = current
+        .get("half_open_failures")
+        .and_then(Value::as_u64)
+        .unwrap_or(0)
+        .saturating_add(u64::from(already_open));
+
+    circuit_by_format.insert(
+        api_format.to_string(),
+        json!({
+            "open": true,
+            "open_at": open_at,
+            "reason": format!("consecutive_failures_{LOCAL_KEY_CIRCUIT_FAILURE_THRESHOLD}"),
+            "next_probe_at": unix_secs_to_rfc3339(next_probe_at_unix_secs),
+            "next_probe_at_unix_secs": next_probe_at_unix_secs,
+            "probe_interval_minutes": probe_interval_minutes,
+            "max_probe_interval_minutes": max_probe_interval_minutes,
+            "failure_count": consecutive_failures,
+            "last_failure_at": unix_secs_to_rfc3339(observed_at_unix_secs),
+            "last_probe_failure_at": if already_open {
+                json!(unix_secs_to_rfc3339(observed_at_unix_secs))
+            } else {
+                Value::Null
+            },
+            "half_open_until": Value::Null,
+            "half_open_successes": 0,
+            "half_open_failures": half_open_failures,
+            "request_results_window": request_results_window,
         }),
     );
     Some(Value::Object(circuit_by_format))
@@ -137,6 +259,60 @@ pub(crate) fn project_local_key_circuit_closed(
     Some(Value::Object(circuit_by_format))
 }
 
+fn current_bool(current: &serde_json::Map<String, Value>, field: &str) -> bool {
+    current.get(field).and_then(Value::as_bool).unwrap_or(false)
+}
+
+fn normalize_max_probe_interval_minutes(value: i32) -> u64 {
+    value.clamp(0, LOCAL_KEY_CIRCUIT_MAX_PROBE_INTERVAL_MINUTES as i32) as u64
+}
+
+fn next_circuit_probe_interval_minutes(
+    current: &serde_json::Map<String, Value>,
+    max_probe_interval_minutes: u64,
+) -> u64 {
+    if max_probe_interval_minutes == 0 {
+        return 0;
+    }
+    if !current_bool(current, "open") {
+        return 1.min(max_probe_interval_minutes);
+    }
+    current
+        .get("probe_interval_minutes")
+        .and_then(Value::as_u64)
+        .unwrap_or(1)
+        .max(1)
+        .saturating_mul(2)
+        .min(max_probe_interval_minutes)
+}
+
+fn next_probe_at_unix_secs(observed_at_unix_secs: u64, interval_minutes: u64) -> u64 {
+    observed_at_unix_secs.saturating_add(interval_minutes.saturating_mul(60))
+}
+
+fn append_request_result_window(
+    current: &serde_json::Map<String, Value>,
+    observed_at_unix_secs: u64,
+    ok: bool,
+) -> Value {
+    let mut window = current
+        .get("request_results_window")
+        .and_then(Value::as_array)
+        .cloned()
+        .unwrap_or_default();
+    window.push(json!({
+        "ts": observed_at_unix_secs,
+        "ok": ok,
+    }));
+    let keep = usize::try_from(LOCAL_KEY_CIRCUIT_FAILURE_THRESHOLD)
+        .unwrap_or(8)
+        .max(1);
+    if window.len() > keep {
+        window = window.split_off(window.len() - keep);
+    }
+    Value::Array(window)
+}
+
 fn local_candidate_failure_should_project_health(
     classification: LocalFailoverClassification,
     status_code: u16,
@@ -182,7 +358,8 @@ mod tests {
 
     use super::{
         project_local_failure_health, project_local_key_circuit_closed,
-        project_local_key_circuit_open, project_local_success_health,
+        project_local_key_circuit_failure, project_local_key_circuit_open,
+        project_local_success_health,
     };
     use crate::orchestration::LocalFailoverClassification;
 
@@ -269,6 +446,7 @@ mod tests {
             "openai:chat",
             "account_deactivated_401",
             1_760_000_000,
+            32,
         )
         .expect("projection should exist");
 
@@ -279,7 +457,47 @@ mod tests {
         );
         assert_eq!(
             projected["openai:chat"]["next_probe_at_unix_secs"],
-            json!(1_760_001_920u64)
+            json!(1_760_000_060u64)
+        );
+        assert_eq!(projected["openai:chat"]["probe_interval_minutes"], json!(1));
+    }
+
+    #[test]
+    fn consecutive_failure_circuit_opens_after_threshold_and_backs_off() {
+        let before_threshold =
+            project_local_key_circuit_failure(None, "openai:chat", 1_760_000_000, 7, 32)
+                .expect("projection should exist");
+        assert_eq!(before_threshold["openai:chat"]["open"], json!(false));
+
+        let opened = project_local_key_circuit_failure(
+            Some(&before_threshold),
+            "openai:chat",
+            1_760_000_060,
+            8,
+            32,
+        )
+        .expect("projection should exist");
+        assert_eq!(opened["openai:chat"]["open"], json!(true));
+        assert_eq!(
+            opened["openai:chat"]["reason"],
+            json!("consecutive_failures_8")
+        );
+        assert_eq!(opened["openai:chat"]["probe_interval_minutes"], json!(1));
+        assert_eq!(
+            opened["openai:chat"]["next_probe_at_unix_secs"],
+            json!(1_760_000_120u64)
+        );
+
+        let backed_off =
+            project_local_key_circuit_failure(Some(&opened), "openai:chat", 1_760_000_120, 9, 32)
+                .expect("projection should exist");
+        assert_eq!(
+            backed_off["openai:chat"]["probe_interval_minutes"],
+            json!(2)
+        );
+        assert_eq!(
+            backed_off["openai:chat"]["next_probe_at_unix_secs"],
+            json!(1_760_000_240u64)
         );
     }
 

apps/aether-gateway/src/orchestration/mod.rs

@@ -34,7 +34,8 @@ pub(crate) use self::effects::{
     LocalPoolErrorEffect,
 };
 pub(crate) use self::health::{
-    project_local_failure_health, project_local_key_circuit_closed, project_local_key_circuit_open,
+    project_local_failure_health, project_local_key_circuit_closed,
+    project_local_key_circuit_failure, project_local_key_circuit_open,
     project_local_success_health,
 };
 pub(crate) use self::policy::{

apps/aether-gateway/src/tests/architecture/runtime_and_security.rs

@@ -406,7 +406,6 @@ fn scheduler_candidate_runtime_paths_depend_on_scheduler_core_and_state_trait()
         "fn candidate_provider_key_health_bucket(",
         "fn candidate_provider_key_health_score(",
         "count_recent_active_requests_for_provider(",
-        "is_candidate_in_recent_failure_cooldown(",
         "provider_key_health_score(",
         "provider_key_rpm_allows_request_since(",
         "read_recent_request_candidates(128)",

apps/aether-gateway/src/tests/control/admin/endpoints/keys.rs

@@ -118,6 +118,107 @@ async fn gateway_handles_admin_provider_keys_locally_with_trusted_admin_principa
     upstream_handle.abort();
 }
 
+#[tokio::test]
+async fn gateway_provider_keys_expose_circuit_breaker_and_recover_clears_it() {
+    let key = sample_key("key-1", "provider-1", "openai:chat", "sk-test-a").with_health_fields(
+        Some(json!({"openai:chat": {
+            "health_score": 0.2,
+            "consecutive_failures": 8,
+            "last_failure_at": "2026-03-26T12:00:00+00:00"
+        }})),
+        Some(json!({"openai:chat": {
+            "open": true,
+            "open_at": "2026-03-26T12:00:00+00:00",
+            "reason": "consecutive_failures_8",
+            "next_probe_at": "2026-03-26T12:01:00+00:00",
+            "next_probe_at_unix_secs": 1774526460u64,
+            "probe_interval_minutes": 1,
+            "max_probe_interval_minutes": 32,
+            "half_open_until": null,
+            "half_open_successes": 0,
+            "half_open_failures": 0
+        }})),
+    );
+    let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
+        vec![sample_provider("provider-1", "openai", 10)],
+        vec![sample_endpoint(
+            "endpoint-1",
+            "provider-1",
+            "openai:chat",
+            "https://example.com/v1",
+        )],
+        vec![key],
+    ));
+    let gateway_state = AppState::new()
+        .expect("gateway should build")
+        .with_data_state_for_tests(
+            GatewayDataState::with_provider_catalog_repository_for_tests(Arc::clone(
+                &provider_catalog_repository,
+            )),
+        );
+    let gateway = build_router_with_state(gateway_state);
+    let (gateway_url, gateway_handle) = start_server(gateway).await;
+
+    let client = reqwest::Client::new();
+    let response = client
+        .get(format!(
+            "{gateway_url}/api/admin/endpoints/providers/provider-1/keys?skip=0&limit=50"
+        ))
+        .header(GATEWAY_HEADER, "rust-phase3b")
+        .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123")
+        .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin")
+        .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123")
+        .send()
+        .await
+        .expect("request should succeed");
+    assert_eq!(response.status(), StatusCode::OK);
+    let payload: serde_json::Value = response.json().await.expect("json body should parse");
+    assert_eq!(payload[0]["circuit_breaker_open"], true);
+    assert_eq!(
+        payload[0]["circuit_breaker_by_format"]["openai:chat"]["reason"],
+        "consecutive_failures_8"
+    );
+    assert_eq!(
+        payload[0]["circuit_breaker_by_format"]["openai:chat"]["probe_interval_minutes"],
+        1
+    );
+    assert!(
+        payload[0]["circuit_breaker_by_format"]["openai:chat"]["next_probe_at_unix_secs"]
+            .as_u64()
+            .is_some()
+    );
+
+    let recover_response = client
+        .patch(format!(
+            "{gateway_url}/api/admin/endpoints/health/keys/key-1"
+        ))
+        .header(GATEWAY_HEADER, "rust-phase3b")
+        .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123")
+        .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin")
+        .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123")
+        .send()
+        .await
+        .expect("recover request should succeed");
+    assert_eq!(recover_response.status(), StatusCode::OK);
+
+    let response = client
+        .get(format!(
+            "{gateway_url}/api/admin/endpoints/providers/provider-1/keys?skip=0&limit=50"
+        ))
+        .header(GATEWAY_HEADER, "rust-phase3b")
+        .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123")
+        .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin")
+        .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123")
+        .send()
+        .await
+        .expect("request should succeed");
+    assert_eq!(response.status(), StatusCode::OK);
+    let payload: serde_json::Value = response.json().await.expect("json body should parse");
+    assert_eq!(payload[0]["circuit_breaker_open"], false);
+
+    gateway_handle.abort();
+}
+
 #[tokio::test]
 async fn gateway_handles_admin_provider_keys_page_locally_with_total() {
     let upstream_hits = Arc::new(Mutex::new(0usize));

crates/aether-scheduler-core/src/candidate/mod.rs

@@ -441,6 +441,29 @@ mod tests {
         );
     }
 
+    #[test]
+    fn recent_failures_do_not_skip_without_persisted_circuit() {
+        let recent_candidates = vec![
+            stored_candidate("failed", RequestCandidateStatus::Failed, 95_000),
+            stored_candidate("cancelled", RequestCandidateStatus::Cancelled, 99_000),
+        ];
+
+        assert_eq!(
+            candidate_runtime_skip_reason_with_state(CandidateRuntimeSelectabilityInput {
+                candidate: &sample_candidate("1", None),
+                recent_candidates: &recent_candidates,
+                provider_concurrent_limits: &BTreeMap::new(),
+                provider_key_rpm_states: &BTreeMap::new(),
+                now_unix_secs: 100,
+                provider_quota_blocks_requests: false,
+                account_quota_exhausted: false,
+                oauth_invalid: false,
+                rpm_reset_at: None,
+            }),
+            None
+        );
+    }
+
     #[test]
     fn provider_key_concurrency_limit_preserves_key_circuit_and_rpm_checks() {
         let mut circuit_open_key = sample_key_with_concurrent_limit("1", Some(2));

crates/aether-scheduler-core/src/candidate/selectability.rs

@@ -62,15 +62,6 @@ pub fn candidate_runtime_skip_reason_with_state(
     if oauth_invalid {
         return Some("oauth_invalid");
     }
-    if crate::is_candidate_in_recent_failure_cooldown(
-        recent_candidates,
-        candidate.provider_id.as_str(),
-        candidate.endpoint_id.as_str(),
-        candidate.key_id.as_str(),
-        now_unix_secs,
-    ) {
-        return Some("recent_failure_cooldown");
-    }
     if provider_concurrent_limits
         .get(&candidate.provider_id)
         .is_some_and(|limit| {

crates/aether-scheduler-core/src/health.rs

@@ -4,7 +4,7 @@ use aether_data_contracts::repository::candidates::{
 use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
 
 const FAILURE_COOLDOWN_WINDOW_SECS: u64 = 60;
-const FAILURE_COOLDOWN_THRESHOLD: usize = 2;
+const FAILURE_COOLDOWN_THRESHOLD: usize = 8;
 const ACTIVE_REQUEST_WINDOW_SECS: u64 = 300;
 pub const PROVIDER_KEY_RPM_WINDOW_SECS: u64 = 60;
 const PROBE_PHASE_REQUESTS: u32 = 100;
@@ -694,11 +694,16 @@ mod tests {
     }
 
     #[test]
-    fn cooldown_triggers_after_two_recent_failures() {
+    fn cooldown_triggers_after_eight_recent_failures() {
-        let recent_candidates = vec![
+        let recent_candidates = (0..8)
-            stored_candidate("one", RequestCandidateStatus::Failed, 95),
+            .map(|index| {
-            stored_candidate("two", RequestCandidateStatus::Cancelled, 99),
+                stored_candidate(
-        ];
+                    &format!("failed-{index}"),
+                    RequestCandidateStatus::Failed,
+                    92 + index,
+                )
+            })
+            .collect::<Vec<_>>();
 
         assert!(is_candidate_in_recent_failure_cooldown(
             &recent_candidates,

frontend/src/api/endpoints/types/provider.ts

@@ -423,11 +423,20 @@ export interface FormatHealthData {
 // 按格式的熔断器数据
 export interface FormatCircuitBreakerData {
   open: boolean
+  reason?: string | null
   open_at?: string | null
   next_probe_at?: string | null
+  next_probe_at_unix_secs?: number | null
+  probe_interval_minutes?: number | null
+  max_probe_interval_minutes?: number | null
+  failure_count?: number | null
+  consecutive_failures?: number | null
+  last_failure_at?: string | null
+  last_probe_failure_at?: string | null
   half_open_until?: string | null
   half_open_successes: number
   half_open_failures: number
+  request_results_window?: Array<{ ts: number; ok: boolean }>
 }
 
 export interface EndpointAPIKeyUpdate {

frontend/src/features/providers/components/ProviderDetailDrawer.vue

@@ -425,8 +425,9 @@
                           v-if="key.circuit_breaker_open"
                           variant="destructive"
                           class="text-[10px] px-1.5 py-0 shrink-0"
+                          :title="getKeyCircuitBreakerTitle(key)"
                         >
-                          熔断
+                          熔断{{ getKeyCircuitProbeCountdown(key) }}
                         </Badge>
                         <!-- 健康度 -->
                         <div
@@ -448,11 +449,11 @@
                           </span>
                         </div>
                         <Button
-                          v-if="key.circuit_breaker_open || (key.health_score !== undefined && key.health_score < 0.5)"
+                          v-if="isKeyRecoverable(key)"
                           variant="ghost"
                           size="icon"
                           class="h-7 w-7 text-green-600"
-                          title="刷新健康状态"
+                          :title="getRecoverKeyTitle(key)"
                           @click="handleRecoverKey(key)"
                         >
                           <RefreshCw class="w-3.5 h-3.5" />
@@ -3505,6 +3506,65 @@ function getHealthScoreBarColor(score: number): string {
   return 'bg-red-500 dark:bg-red-400'
 }
 
+function isKeyRecoverable(key: EndpointAPIKey): boolean {
+  return Boolean(
+    key.circuit_breaker_open
+    || (key.health_score !== undefined && key.health_score < 0.5)
+  )
+}
+
+function getOpenCircuitEntries(key: EndpointAPIKey): Array<[string, NonNullable<EndpointAPIKey['circuit_breaker_by_format']>[string]]> {
+  return Object.entries(key.circuit_breaker_by_format || {})
+    .filter(([, value]) => value?.open === true)
+}
+
+function getKeyCircuitProbeCountdown(key: EndpointAPIKey): string {
+  void countdownTick.value
+  const nextProbe = getOpenCircuitEntries(key)
+    .map(([, value]) => {
+      if (typeof value.next_probe_at_unix_secs === 'number' && Number.isFinite(value.next_probe_at_unix_secs)) {
+        return value.next_probe_at_unix_secs * 1000
+      }
+      if (value.next_probe_at) {
+        const ms = new Date(value.next_probe_at).getTime()
+        return Number.isFinite(ms) ? ms : null
+      }
+      return null
+    })
+    .filter((value): value is number => value !== null)
+    .sort((a, b) => a - b)[0]
+  if (!nextProbe) {
+    return ''
+  }
+  const diffMs = nextProbe - Date.now()
+  return diffMs > 0 ? ` ${formatCountdown(diffMs)}` : ' 探测中'
+}
+
+function getKeyCircuitBreakerTitle(key: EndpointAPIKey): string {
+  const entries = getOpenCircuitEntries(key)
+  if (entries.length === 0) return '熔断器已打开'
+  const parts = entries.map(([format, value]) => {
+    const label = formatApiFormatShort(format)
+    const reason = value.reason ? `原因: ${value.reason}` : '原因: 连续失败'
+    const interval = typeof value.probe_interval_minutes === 'number'
+      ? `探测间隔: ${value.probe_interval_minutes} 分钟`
+      : ''
+    const countdown = getFormatProbeCountdown(key, format).trim()
+    return [label, reason, interval, countdown ? `状态: ${countdown}` : '']
+      .filter(Boolean)
+      .join(' / ')
+  })
+  parts.push('点击恢复按钮可重置熔断器')
+  return parts.join('\n')
+}
+
+function getRecoverKeyTitle(key: EndpointAPIKey): string {
+  if (key.circuit_breaker_open) {
+    return '重置熔断器并恢复健康状态'
+  }
+  return '刷新健康状态'
+}
+
 // 获取自动获取模型状态的 title 提示
 function getAutoFetchStatusTitle(key: EndpointAPIKey): string {
   const parts: string[] = ['自动获取模型已启用']
@@ -3546,10 +3606,11 @@ function getFormatProbeCountdown(key: EndpointAPIKey, format: string): string {
     }
   }
   // 等待探测
-  if (formatData.next_probe_at) {
+  if (formatData.next_probe_at_unix_secs || formatData.next_probe_at) {
-    const nextProbe = new Date(formatData.next_probe_at)
+    const nextProbeMs = typeof formatData.next_probe_at_unix_secs === 'number'
-    const now = new Date()
+      ? formatData.next_probe_at_unix_secs * 1000
-    const diffMs = nextProbe.getTime() - now.getTime()
+      : new Date(formatData.next_probe_at || '').getTime()
+    const diffMs = nextProbeMs - Date.now()
     if (diffMs > 0) {
       return ` ${formatCountdown(diffMs)}`
     } else {