Merge pull request #690 from zhefox/main

fix(provider): classify deleted Codex agent runtime as invalid

apps/aether-gateway/src/handlers/admin/provider/pool_admin/read_routes/keys.rs

@@ -806,6 +806,26 @@ mod tests {
         );
     }
 
+    #[test]
+    fn oauth_status_filter_matches_deleted_agent_runtime_as_invalid() {
+        let mut key = sample_key("oauth");
+        key.oauth_invalid_at_unix_secs = Some(1_784_728_663);
+        key.oauth_invalid_reason =
+            Some("[REQUEST_FAILED] Agent runtime has been deleted.".to_string());
+        let status_snapshot = provider_key_status_snapshot_payload(&key, "codex");
+
+        assert_eq!(
+            admin_pool_oauth_status_filter(
+                &key,
+                "codex",
+                None,
+                status_snapshot.get("oauth").and_then(Value::as_object),
+                1_784_729_000,
+            ),
+            Some("invalid")
+        );
+    }
+
     #[test]
     fn codex_cycle_usage_request_uses_actual_monthly_window_boundaries() {
         let key = sample_key("oauth");

apps/aether-gateway/src/handlers/shared/catalog.rs

@@ -319,6 +319,18 @@ fn build_provider_key_oauth_status_snapshot(key: &StoredProviderCatalogKey) -> V
     if let Some(reason) =
         tagged_oauth_invalid_reason(invalid_reason.as_deref(), OAUTH_REQUEST_FAILED_PREFIX)
     {
+        if admin_provider_quota_pure::codex_looks_like_token_invalidated(Some(&reason)) {
+            return json!({
+                "code": "invalid",
+                "label": "已失效",
+                "reason": reason,
+                "expires_at": expires_at_unix_secs,
+                "invalid_at": invalid_at_unix_secs,
+                "source": "oauth_invalid",
+                "requires_reauth": true,
+                "expiring_soon": false,
+            });
+        }
         return json!({
             "code": "check_failed",
             "label": "检查失败",
@@ -4097,6 +4109,26 @@ mod tests {
         assert_eq!(account.get("source"), Some(&json!("oauth_invalid")));
     }
 
+    #[test]
+    fn provider_key_status_snapshot_payload_upgrades_deleted_agent_runtime_to_invalid() {
+        let mut key = sample_catalog_key();
+        key.auth_type = "oauth".to_string();
+        key.oauth_invalid_at_unix_secs = Some(1_784_728_663);
+        key.oauth_invalid_reason =
+            Some("[REQUEST_FAILED] Agent runtime has been deleted.".to_string());
+
+        let payload = provider_key_status_snapshot_payload(&key, "codex");
+        let oauth = payload
+            .get("oauth")
+            .and_then(Value::as_object)
+            .expect("oauth snapshot should be object");
+
+        assert_eq!(oauth.get("code"), Some(&json!("invalid")));
+        assert_eq!(oauth.get("label"), Some(&json!("已失效")));
+        assert_eq!(oauth.get("invalid_at"), Some(&json!(1_784_728_663u64)));
+        assert_eq!(oauth.get("requires_reauth"), Some(&json!(true)));
+    }
+
     #[test]
     fn provider_key_status_snapshot_payload_backfills_workspace_deactivated_from_metadata() {
         let mut key = sample_catalog_key();

apps/aether-gateway/src/state/oauth.rs

@@ -530,6 +530,18 @@ fn build_oauth_status_snapshot_value(key: &StoredProviderCatalogKey) -> Value {
         });
     }
     if let Some(reason) = tagged_reason(invalid_reason.as_deref(), OAUTH_REQUEST_FAILED_PREFIX) {
+        if admin_provider_quota_pure::codex_looks_like_token_invalidated(Some(&reason)) {
+            return json!({
+                "code": "invalid",
+                "label": "已失效",
+                "reason": reason,
+                "expires_at": expires_at_unix_secs,
+                "invalid_at": invalid_at_unix_secs,
+                "source": "oauth_invalid",
+                "requires_reauth": true,
+                "expiring_soon": false,
+            });
+        }
         return json!({
             "code": "check_failed",
             "label": "检查失败",

crates/aether-admin/src/provider/quota.rs

@@ -1311,6 +1311,7 @@ pub fn codex_looks_like_token_invalidated(message: Option<&str>) -> bool {
         || lowered.contains("authentication token has been invalidated")
         || lowered.contains("token has been invalidated")
         || lowered.contains("token invalidated")
+        || lowered.contains("agent runtime has been deleted")
         || lowered.contains("personal access token owner is inactive")
         || lowered.contains("biscuit_baker_service_auth_credential_error_status")
         || lowered.contains("auth_credential")
@@ -2223,6 +2224,16 @@ mod tests {
         );
     }
 
+    #[test]
+    fn codex_runtime_invalid_reason_marks_deleted_agent_runtime_as_invalid() {
+        assert_eq!(
+            codex_runtime_invalid_reason(403, Some("Agent runtime has been deleted.")),
+            Some(format!(
+                "{OAUTH_EXPIRED_PREFIX}Agent runtime has been deleted."
+            ))
+        );
+    }
+
     #[test]
     fn codex_runtime_invalid_reason_marks_402_as_account_blocked() {
         assert_eq!(

crates/aether-admin/src/provider/status.rs

@@ -451,6 +451,16 @@ fn resolve_from_oauth_invalid_reason(reason: Option<&str>) -> Option<PoolAccount
         } else {
             cleaned
         };
+        if oauth_request_failure_indicates_invalid_identity(&reason) {
+            return Some(PoolAccountState {
+                blocked: true,
+                code: Some("oauth_token_invalid".to_string()),
+                label: Some("Token 失效".to_string()),
+                reason: Some(reason),
+                source: Some("oauth_invalid".to_string()),
+                recoverable: false,
+            });
+        }
         return Some(PoolAccountState {
             blocked: false,
             code: Some("oauth_request_failed".to_string()),
@@ -485,6 +495,13 @@ fn resolve_from_oauth_invalid_reason(reason: Option<&str>) -> Option<PoolAccount
     None
 }
 
+fn oauth_request_failure_indicates_invalid_identity(reason: &str) -> bool {
+    reason
+        .trim()
+        .to_ascii_lowercase()
+        .contains("agent runtime has been deleted")
+}
+
 pub fn resolve_pool_account_state(
     provider_type: Option<&str>,
     upstream_metadata: Option<&Value>,
@@ -551,6 +568,24 @@ pub fn resolve_account_status_snapshot(
         };
     }
 
+    if let Some(cleaned) = tagged_reason(&text, "REQUEST_FAILED") {
+        let reason = if cleaned.is_empty() {
+            "账号状态检查失败".to_string()
+        } else {
+            cleaned
+        };
+        if oauth_request_failure_indicates_invalid_identity(&reason) {
+            return AccountStatusSnapshot {
+                code: "oauth_token_invalid".to_string(),
+                label: Some("Token 失效".to_string()),
+                reason: Some(reason),
+                blocked: true,
+                source: Some("oauth_invalid".to_string()),
+                recoverable: false,
+            };
+        }
+    }
+
     if tagged_reason(&text, "REFRESH_FAILED").is_some() {
         return AccountStatusSnapshot::default();
     }
@@ -739,6 +774,34 @@ mod tests {
         assert!(!snapshot.recoverable);
     }
 
+    #[test]
+    fn deleted_agent_runtime_request_failure_is_hard_invalid() {
+        let reason = "[REQUEST_FAILED] Agent runtime has been deleted.";
+        let state = resolve_pool_account_state(Some("codex"), None, Some(reason));
+        let snapshot = resolve_account_status_snapshot(Some("codex"), None, Some(reason));
+
+        assert!(state.blocked);
+        assert_eq!(state.code.as_deref(), Some("oauth_token_invalid"));
+        assert!(!state.recoverable);
+        assert_eq!(snapshot.code, "oauth_token_invalid");
+        assert_eq!(snapshot.label.as_deref(), Some("Token 失效"));
+        assert!(snapshot.blocked);
+        assert!(!snapshot.recoverable);
+    }
+
+    #[test]
+    fn transient_request_failure_remains_recoverable() {
+        let state = resolve_pool_account_state(
+            Some("codex"),
+            None,
+            Some("[REQUEST_FAILED] upstream request timed out"),
+        );
+
+        assert!(!state.blocked);
+        assert_eq!(state.code.as_deref(), Some("oauth_request_failed"));
+        assert!(state.recoverable);
+    }
+
     #[test]
     fn account_snapshot_marks_oauth_expired_as_token_expired() {
         let snapshot = resolve_account_status_snapshot(