Cap Codex pool cooldowns and add key circuit breaker

apps/aether-gateway/src/orchestration/health.rs

@@ -4,6 +4,7 @@ use super::LocalFailoverClassification;
 use crate::handlers::shared::unix_secs_to_rfc3339;
 
 const LOCAL_HEALTH_SCORE_FLOOR: f64 = 0.2;
+const LOCAL_KEY_CIRCUIT_PROBE_DELAY_SECS: u64 = 32 * 60;
 
 pub(crate) fn project_local_failure_health(
     current_health_by_format: Option<&Value>,
@@ -73,6 +74,69 @@ pub(crate) fn project_local_success_health(
     Some(Value::Object(health_by_format))
 }
 
+pub(crate) fn project_local_key_circuit_open(
+    current_circuit_by_format: Option<&Value>,
+    api_format: &str,
+    reason: &str,
+    observed_at_unix_secs: u64,
+) -> Option<Value> {
+    let api_format = api_format.trim();
+    let reason = reason.trim();
+    if api_format.is_empty() || reason.is_empty() {
+        return None;
+    }
+
+    let next_probe_at_unix_secs =
+        observed_at_unix_secs.saturating_add(LOCAL_KEY_CIRCUIT_PROBE_DELAY_SECS);
+    let mut circuit_by_format = current_circuit_by_format
+        .and_then(Value::as_object)
+        .cloned()
+        .unwrap_or_default();
+    circuit_by_format.insert(
+        api_format.to_string(),
+        json!({
+            "open": true,
+            "open_at": unix_secs_to_rfc3339(observed_at_unix_secs),
+            "reason": reason,
+            "next_probe_at": unix_secs_to_rfc3339(next_probe_at_unix_secs),
+            "next_probe_at_unix_secs": next_probe_at_unix_secs,
+            "half_open_until": Value::Null,
+            "half_open_successes": 0,
+            "half_open_failures": 0,
+        }),
+    );
+    Some(Value::Object(circuit_by_format))
+}
+
+pub(crate) fn project_local_key_circuit_closed(
+    current_circuit_by_format: Option<&Value>,
+    api_format: &str,
+) -> Option<Value> {
+    let api_format = api_format.trim();
+    if api_format.is_empty() {
+        return None;
+    }
+
+    let mut circuit_by_format = current_circuit_by_format
+        .and_then(Value::as_object)
+        .cloned()
+        .unwrap_or_default();
+    circuit_by_format.insert(
+        api_format.to_string(),
+        json!({
+            "open": false,
+            "open_at": Value::Null,
+            "reason": Value::Null,
+            "next_probe_at": Value::Null,
+            "next_probe_at_unix_secs": Value::Null,
+            "half_open_until": Value::Null,
+            "half_open_successes": 0,
+            "half_open_failures": 0,
+        }),
+    );
+    Some(Value::Object(circuit_by_format))
+}
+
 fn local_candidate_failure_should_project_health(
     classification: LocalFailoverClassification,
     status_code: u16,
@@ -112,7 +176,10 @@ fn projected_failure_health_score(
 mod tests {
     use serde_json::{json, Value};
 
-    use super::{project_local_failure_health, project_local_success_health};
+    use super::{
+        project_local_failure_health, project_local_key_circuit_closed,
+        project_local_key_circuit_open, project_local_success_health,
+    };
     use crate::orchestration::LocalFailoverClassification;
 
     #[test]
@@ -178,4 +245,47 @@ mod tests {
         );
         assert_eq!(projected["openai:responses"]["health_score"], json!(0.8));
     }
+
+    #[test]
+    fn circuit_open_projection_sets_probe_deadline() {
+        let projected = project_local_key_circuit_open(
+            None,
+            "openai:chat",
+            "account_deactivated_401",
+            1_760_000_000,
+        )
+        .expect("projection should exist");
+
+        assert_eq!(projected["openai:chat"]["open"], json!(true));
+        assert_eq!(
+            projected["openai:chat"]["reason"],
+            json!("account_deactivated_401")
+        );
+        assert_eq!(
+            projected["openai:chat"]["next_probe_at_unix_secs"],
+            json!(1_760_001_920u64)
+        );
+    }
+
+    #[test]
+    fn circuit_closed_projection_resets_format_circuit() {
+        let projected = project_local_key_circuit_closed(
+            Some(&json!({
+                "openai:chat": {
+                    "open": true,
+                    "reason": "account_deactivated_401",
+                    "next_probe_at_unix_secs": 1_760_001_920u64
+                }
+            })),
+            "openai:chat",
+        )
+        .expect("projection should exist");
+
+        assert_eq!(projected["openai:chat"]["open"], json!(false));
+        assert_eq!(projected["openai:chat"]["reason"], Value::Null);
+        assert_eq!(
+            projected["openai:chat"]["next_probe_at_unix_secs"],
+            Value::Null
+        );
+    }
 }