feat(oauth): 完善账号异常识别并在调度/展示层拦截失效 OAuth 密钥

- 新增 aether-admin provider status 模块,统一解析账号状态(禁用/工作区停用等)
- 调度器 runtime 增加 oauth_invalid 判定,跳过刷新失败或已撤销的 OAuth 密钥(REQUEST_FAILED 保留可选)
- gateway state 在 local oauth 刷新返回 4xx 时持久化失败原因并同步状态快照
- admin pool 列表/详情回填 account 状态与 scheduling 阻塞原因(account_blocked)
- 共享 catalog 的 status_snapshot payload 附加 account 字段
This commit is contained in:
fawney19
2026-04-19 20:50:31 +08:00
parent d719a1329c
commit 77aac74590
12 changed files with 1409 additions and 60 deletions

View File

@@ -6,3 +6,4 @@ pub mod ops;
pub mod pool;
pub mod quota;
pub mod state;
pub mod status;

View File

@@ -6,6 +6,8 @@ use chrono::{TimeZone, Utc};
use serde_json::{json, Value};
use std::collections::{BTreeMap, BTreeSet};
use super::status as provider_status;
#[derive(Debug, Default, Clone, serde::Deserialize)]
pub struct AdminPoolResolveSelectionRequest {
#[serde(default)]
@@ -461,40 +463,17 @@ pub fn admin_pool_matches_search(
}
pub fn admin_pool_key_is_known_banned(key: &StoredProviderCatalogKey) -> bool {
if key
.oauth_invalid_reason
.as_deref()
.is_some_and(admin_pool_reason_indicates_ban)
{
let state = provider_status::resolve_pool_account_state(
None,
key.upstream_metadata.as_ref(),
key.oauth_invalid_reason.as_deref(),
);
if provider_status::account_state_indicates_known_ban(&state) {
return true;
}
let Some(account) = key
.status_snapshot
.as_ref()
.and_then(Value::as_object)
.and_then(|snapshot| snapshot.get("account"))
.and_then(Value::as_object)
else {
return false;
};
if !account
.get("blocked")
.and_then(Value::as_bool)
.unwrap_or(false)
{
return false;
}
account
.get("code")
.and_then(Value::as_str)
key.oauth_invalid_reason
.as_deref()
.is_some_and(admin_pool_reason_indicates_ban)
|| account
.get("reason")
.and_then(Value::as_str)
.is_some_and(admin_pool_reason_indicates_ban)
}
pub fn admin_pool_sort_keys(keys: &mut [StoredProviderCatalogKey]) {
@@ -665,7 +644,7 @@ pub fn build_admin_pool_selection_payload(keys: &[StoredProviderCatalogKey]) ->
#[cfg(test)]
#[allow(clippy::items_after_test_module)]
mod tests {
use super::admin_pool_key_account_quota_exhausted;
use super::{admin_pool_key_account_quota_exhausted, admin_pool_key_is_known_banned};
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
use serde_json::json;
@@ -790,6 +769,18 @@ mod tests {
"kiro",
));
}
#[test]
fn known_banned_detects_provider_bucket_account_blocks_without_provider_type() {
let key = sample_key(Some(json!({
"codex": {
"account_disabled": true,
"reason": "deactivated_workspace"
}
})));
assert!(admin_pool_key_is_known_banned(&key));
}
}
pub fn build_admin_pool_key_payload(

View File

@@ -3,6 +3,8 @@ use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKe
use serde_json::json;
use std::collections::BTreeMap;
use super::status as provider_status;
const OAUTH_ACCOUNT_BLOCK_PREFIX: &str = "[ACCOUNT_BLOCK] ";
const OAUTH_REFRESH_FAILED_PREFIX: &str = "[REFRESH_FAILED] ";
const OAUTH_EXPIRED_PREFIX: &str = "[OAUTH_EXPIRED] ";
@@ -18,9 +20,9 @@ pub fn provider_auto_remove_banned_keys(config: Option<&serde_json::Value>) -> b
}
pub fn should_auto_remove_structured_reason(reason: Option<&str>) -> bool {
reason
.map(str::trim)
.is_some_and(|value| value.starts_with(OAUTH_ACCOUNT_BLOCK_PREFIX))
provider_status::should_auto_remove_account_state(&provider_status::resolve_pool_account_state(
None, None, reason,
))
}
pub fn normalize_string_id_list(values: Option<Vec<String>>) -> Option<Vec<String>> {

View File

@@ -0,0 +1,600 @@
use serde_json::Value;
use std::collections::BTreeSet;
const OAUTH_ACCOUNT_BLOCK_PREFIX: &str = "[ACCOUNT_BLOCK] ";
const OAUTH_REFRESH_FAILED_PREFIX: &str = "[REFRESH_FAILED] ";
const OAUTH_EXPIRED_PREFIX: &str = "[OAUTH_EXPIRED] ";
const OAUTH_REQUEST_FAILED_PREFIX: &str = "[REQUEST_FAILED] ";
const ACCOUNT_BLOCK_REASON_KEYWORDS: &[&str] = &[
"suspended",
"banned",
"account_block",
"account blocked",
"account_forbidden",
"forbidden",
"封禁",
"封号",
"被封",
"账户已封禁",
"账号异常",
"account has been disabled",
"account disabled",
"account has been deactivated",
"account_deactivated",
"account deactivated",
"organization has been disabled",
"organization_disabled",
"deactivated_workspace",
"deactivated",
"访问被禁止",
"账户访问被禁止",
"访问受限",
"账户访问受限",
"authentication token has been invalidated",
"token has been invalidated",
"codex token 无效或已过期",
"validation_required",
"verify your account",
"需要验证",
"验证账号",
"验证身份",
];
const AUTO_REMOVABLE_ACCOUNT_STATE_CODES: &[&str] = &[
"account_banned",
"account_suspended",
"account_disabled",
"workspace_deactivated",
"account_forbidden",
];
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct PoolAccountState {
pub blocked: bool,
pub code: Option<String>,
pub label: Option<String>,
pub reason: Option<String>,
pub source: Option<String>,
pub recoverable: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountStatusSnapshot {
pub code: String,
pub label: Option<String>,
pub reason: Option<String>,
pub blocked: bool,
pub source: Option<String>,
pub recoverable: bool,
}
impl Default for AccountStatusSnapshot {
fn default() -> Self {
Self {
code: "ok".to_string(),
label: None,
reason: None,
blocked: false,
source: None,
recoverable: false,
}
}
}
fn clean_text(value: Option<&str>) -> Option<String> {
value
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
fn json_bool(value: Option<&Value>) -> bool {
match value {
Some(Value::Bool(value)) => *value,
Some(Value::Number(value)) => value.as_i64().is_some_and(|value| value != 0),
Some(Value::String(value)) => matches!(
value.trim().to_ascii_lowercase().as_str(),
"1" | "true" | "yes" | "y"
),
_ => false,
}
}
fn extract_reason(source: &serde_json::Map<String, Value>, fields: &[&str]) -> Option<String> {
fields
.iter()
.find_map(|field| source.get(*field).and_then(Value::as_str))
.and_then(|value| clean_text(Some(value)))
}
fn looks_like_workspace_deactivated(reason: Option<&str>) -> bool {
clean_text(reason)
.is_some_and(|value| value.to_ascii_lowercase().contains("deactivated_workspace"))
}
fn looks_like_account_verification(reason: &str) -> bool {
let lowered = reason.to_ascii_lowercase();
[
"validation_required",
"verify your account",
"需要验证",
"验证账号",
"验证身份",
]
.iter()
.any(|keyword| lowered.contains(keyword))
}
fn classify_block_reason(reason: &str) -> (&'static str, &'static str) {
let lowered = reason.to_ascii_lowercase();
if [
"authentication token has been invalidated",
"token has been invalidated",
"codex token 无效或已过期",
]
.iter()
.any(|keyword| lowered.contains(keyword))
{
return ("oauth_expired", "Token 失效");
}
if looks_like_account_verification(reason) {
return ("account_verification", "需要验证");
}
if lowered.contains("deactivated_workspace") {
return ("workspace_deactivated", "工作区停用");
}
if [
"account has been disabled",
"account disabled",
"account has been deactivated",
"account_deactivated",
"account deactivated",
"organization has been disabled",
"organization_disabled",
"访问被禁止",
"账户访问被禁止",
]
.iter()
.any(|keyword| lowered.contains(keyword))
{
return ("account_disabled", "账号停用");
}
if ["account_forbidden", "forbidden", "访问受限", "账户访问受限"]
.iter()
.any(|keyword| lowered.contains(keyword))
{
return ("account_forbidden", "访问受限");
}
if [
"suspended",
"banned",
"account_block",
"account blocked",
"封禁",
"封号",
"被封",
"账户已封禁",
"账号异常",
]
.iter()
.any(|keyword| lowered.contains(keyword))
{
return ("account_suspended", "账号封禁");
}
("account_blocked", "账号异常")
}
fn parse_tagged_reason_line(line: &str) -> Option<(String, String)> {
let trimmed = line.trim();
if !trimmed.starts_with('[') {
return None;
}
let end = trimmed.find(']')?;
let tag = trimmed.get(1..end)?.trim();
if tag.is_empty() || !tag.chars().all(|ch| ch.is_ascii_uppercase() || ch == '_') {
return None;
}
let detail = trimmed
.get(end + 1..)
.unwrap_or_default()
.trim()
.to_string();
Some((tag.to_string(), detail))
}
fn extract_tagged_reason_sections(reason: &str) -> Vec<(String, String)> {
let mut sections = Vec::<(String, String)>::new();
let mut current_tag = None::<String>;
for line in reason.lines() {
if let Some((tag, detail)) = parse_tagged_reason_line(line) {
current_tag = Some(tag.clone());
if sections.iter().all(|(existing, _)| existing != &tag) {
sections.push((tag, detail));
}
continue;
}
let continuation = line.trim();
if continuation.is_empty() {
continue;
}
let Some(tag) = current_tag.as_ref() else {
continue;
};
let Some((_, detail)) = sections.iter_mut().find(|(existing, _)| existing == tag) else {
continue;
};
if !detail.is_empty() {
detail.push('\n');
}
detail.push_str(continuation);
}
sections
}
fn tagged_reason(reason: &str, tag: &str) -> Option<String> {
extract_tagged_reason_sections(reason)
.into_iter()
.find_map(|(candidate, detail)| (candidate == tag).then_some(detail))
.and_then(|detail| clean_text(Some(detail.as_str())).or(Some(detail)))
}
fn metadata_sources<'a>(
provider_type: Option<&str>,
upstream_metadata: Option<&'a Value>,
) -> Vec<&'a serde_json::Map<String, Value>> {
let mut sources = Vec::new();
let Some(root) = upstream_metadata.and_then(Value::as_object) else {
return sources;
};
let normalized_provider_type = provider_type
.map(str::trim)
.filter(|value| !value.is_empty())
.map(|value| value.to_ascii_lowercase());
if let Some(provider_type) = normalized_provider_type.as_deref() {
if let Some(bucket) = root.get(provider_type).and_then(Value::as_object) {
sources.push(bucket);
}
sources.push(root);
return sources;
}
let mut seen = BTreeSet::new();
for value in root.values() {
let Some(object) = value.as_object() else {
continue;
};
let pointer = object as *const serde_json::Map<String, Value> as usize;
if seen.insert(pointer) {
sources.push(object);
}
}
sources.push(root);
sources
}
fn resolve_from_metadata(
provider_type: Option<&str>,
upstream_metadata: Option<&Value>,
) -> Option<PoolAccountState> {
for source in metadata_sources(provider_type, upstream_metadata) {
if json_bool(source.get("is_banned")) {
let reason = extract_reason(
source,
&["ban_reason", "forbidden_reason", "reason", "message"],
)
.unwrap_or_else(|| "账号已封禁".to_string());
return Some(PoolAccountState {
blocked: true,
code: Some("account_banned".to_string()),
label: Some("账号封禁".to_string()),
reason: Some(reason),
source: Some("metadata".to_string()),
recoverable: false,
});
}
if json_bool(source.get("is_forbidden")) || json_bool(source.get("account_disabled")) {
let reason = extract_reason(
source,
&["forbidden_reason", "ban_reason", "reason", "message"],
);
if looks_like_workspace_deactivated(reason.as_deref()) {
return Some(PoolAccountState {
blocked: true,
code: Some("workspace_deactivated".to_string()),
label: Some("工作区停用".to_string()),
reason: Some(reason.unwrap_or_else(|| "工作区已停用".to_string())),
source: Some("metadata".to_string()),
recoverable: false,
});
}
return Some(PoolAccountState {
blocked: true,
code: Some("account_forbidden".to_string()),
label: Some("访问受限".to_string()),
reason: Some(reason.unwrap_or_else(|| "账号访问受限".to_string())),
source: Some("metadata".to_string()),
recoverable: false,
});
}
}
None
}
fn resolve_from_oauth_invalid_reason(reason: Option<&str>) -> Option<PoolAccountState> {
let text = clean_text(reason)?;
if text.starts_with(OAUTH_ACCOUNT_BLOCK_PREFIX) {
let cleaned = clean_text(Some(text.trim_start_matches(OAUTH_ACCOUNT_BLOCK_PREFIX)))
.unwrap_or_else(|| "账号异常".to_string());
let (code, label) = classify_block_reason(&cleaned);
return Some(PoolAccountState {
blocked: true,
code: Some(code.to_string()),
label: Some(label.to_string()),
reason: Some(cleaned),
source: Some("oauth_invalid".to_string()),
recoverable: false,
});
}
if text.starts_with(OAUTH_EXPIRED_PREFIX) {
let cleaned = clean_text(Some(text.trim_start_matches(OAUTH_EXPIRED_PREFIX)))
.unwrap_or_else(|| "OAuth Token 已过期且无法续期".to_string());
return Some(PoolAccountState {
blocked: true,
code: Some("oauth_expired".to_string()),
label: Some("Token 失效".to_string()),
reason: Some(cleaned),
source: Some("oauth_invalid".to_string()),
recoverable: true,
});
}
if text.starts_with(OAUTH_REFRESH_FAILED_PREFIX) {
let cleaned = clean_text(Some(text.trim_start_matches(OAUTH_REFRESH_FAILED_PREFIX)))
.unwrap_or_else(|| "OAuth Token 续期失败".to_string());
return Some(PoolAccountState {
blocked: false,
code: Some("oauth_refresh_failed".to_string()),
label: Some("续期失败".to_string()),
reason: Some(cleaned),
source: Some("oauth_refresh".to_string()),
recoverable: true,
});
}
if text.starts_with(OAUTH_REQUEST_FAILED_PREFIX) {
let cleaned = clean_text(Some(text.trim_start_matches(OAUTH_REQUEST_FAILED_PREFIX)))
.unwrap_or_else(|| "账号状态检查失败".to_string());
return Some(PoolAccountState {
blocked: false,
code: Some("oauth_request_failed".to_string()),
label: Some("请求失败".to_string()),
reason: Some(cleaned),
source: Some("oauth_request".to_string()),
recoverable: true,
});
}
if text.starts_with('[') {
return None;
}
let lowered = text.to_ascii_lowercase();
if ACCOUNT_BLOCK_REASON_KEYWORDS
.iter()
.any(|keyword| lowered.contains(keyword))
{
let (code, label) = classify_block_reason(&text);
return Some(PoolAccountState {
blocked: true,
code: Some(code.to_string()),
label: Some(label.to_string()),
reason: Some(text),
source: Some("oauth_invalid".to_string()),
recoverable: false,
});
}
None
}
pub fn resolve_pool_account_state(
provider_type: Option<&str>,
upstream_metadata: Option<&Value>,
oauth_invalid_reason: Option<&str>,
) -> PoolAccountState {
resolve_from_metadata(provider_type, upstream_metadata)
.or_else(|| resolve_from_oauth_invalid_reason(oauth_invalid_reason))
.unwrap_or_default()
}
pub fn resolve_account_status_snapshot(
provider_type: Option<&str>,
upstream_metadata: Option<&Value>,
oauth_invalid_reason: Option<&str>,
) -> AccountStatusSnapshot {
if let Some(metadata_state) = resolve_from_metadata(provider_type, upstream_metadata) {
return AccountStatusSnapshot {
code: metadata_state
.code
.unwrap_or_else(|| "account_blocked".to_string()),
label: metadata_state.label,
reason: metadata_state.reason,
blocked: metadata_state.blocked,
source: metadata_state.source,
recoverable: metadata_state.recoverable,
};
}
let Some(text) = clean_text(oauth_invalid_reason) else {
return AccountStatusSnapshot::default();
};
if let Some(cleaned) = tagged_reason(&text, "ACCOUNT_BLOCK") {
let reason = if cleaned.is_empty() {
"账号异常".to_string()
} else {
cleaned
};
let (code, label) = classify_block_reason(&reason);
return AccountStatusSnapshot {
code: code.to_string(),
label: Some(label.to_string()),
reason: Some(reason),
blocked: true,
source: Some("oauth_invalid".to_string()),
recoverable: false,
};
}
if text.starts_with('[') {
return AccountStatusSnapshot::default();
}
let lowered = text.to_ascii_lowercase();
if ACCOUNT_BLOCK_REASON_KEYWORDS
.iter()
.any(|keyword| lowered.contains(keyword))
{
let (code, label) = classify_block_reason(&text);
return AccountStatusSnapshot {
code: code.to_string(),
label: Some(label.to_string()),
reason: Some(text),
blocked: true,
source: Some("oauth_invalid".to_string()),
recoverable: false,
};
}
AccountStatusSnapshot::default()
}
pub fn should_auto_remove_account_state(state: &PoolAccountState) -> bool {
state.blocked
&& !state.recoverable
&& state.code.as_deref().is_some_and(|code| {
AUTO_REMOVABLE_ACCOUNT_STATE_CODES
.iter()
.any(|candidate| code.eq_ignore_ascii_case(candidate))
})
}
pub fn account_state_indicates_known_ban(state: &PoolAccountState) -> bool {
if !state.blocked {
return false;
}
if should_auto_remove_account_state(state) {
return true;
}
if state
.code
.as_deref()
.is_some_and(|code| matches!(code, "account_verification" | "account_blocked"))
{
return true;
}
state.code.as_deref().is_some_and(reason_indicates_ban)
|| state.reason.as_deref().is_some_and(reason_indicates_ban)
}
fn reason_indicates_ban(reason: &str) -> bool {
let normalized = reason.trim().to_ascii_lowercase();
!normalized.is_empty()
&& [
"banned",
"forbidden",
"blocked",
"suspend",
"deactivated",
"disabled",
"verification",
"workspace",
"受限",
"",
"",
]
.iter()
.any(|hint| normalized.contains(hint))
}
#[cfg(test)]
mod tests {
use super::{
account_state_indicates_known_ban, resolve_account_status_snapshot,
resolve_pool_account_state, should_auto_remove_account_state,
};
use serde_json::json;
#[test]
fn resolves_workspace_deactivated_from_metadata() {
let state = resolve_pool_account_state(
Some("codex"),
Some(&json!({
"codex": {
"account_disabled": true,
"reason": "deactivated_workspace"
}
})),
None,
);
assert!(state.blocked);
assert_eq!(state.code.as_deref(), Some("workspace_deactivated"));
assert_eq!(state.label.as_deref(), Some("工作区停用"));
assert!(should_auto_remove_account_state(&state));
assert!(account_state_indicates_known_ban(&state));
}
#[test]
fn resolves_refresh_failed_as_recoverable_pool_state() {
let state = resolve_pool_account_state(
Some("codex"),
None,
Some("[REFRESH_FAILED] Token 续期失败 (401): refresh_token 已失效"),
);
assert!(!state.blocked);
assert!(state.recoverable);
assert_eq!(state.code.as_deref(), Some("oauth_refresh_failed"));
assert!(!should_auto_remove_account_state(&state));
}
#[test]
fn account_snapshot_ignores_refresh_failed_reason() {
let snapshot = resolve_account_status_snapshot(
Some("codex"),
None,
Some("[REFRESH_FAILED] Token 续期失败 (401): refresh_token 已失效"),
);
assert_eq!(snapshot.code, "ok");
assert!(!snapshot.blocked);
}
#[test]
fn account_snapshot_detects_account_block_and_verification() {
let snapshot = resolve_account_status_snapshot(
Some("codex"),
None,
Some("[ACCOUNT_BLOCK] verify your account before continuing"),
);
assert_eq!(snapshot.code, "account_verification");
assert_eq!(snapshot.label.as_deref(), Some("需要验证"));
assert!(snapshot.blocked);
}
#[test]
fn verification_state_is_not_auto_removed() {
let state = resolve_pool_account_state(
Some("codex"),
None,
Some("[ACCOUNT_BLOCK] verify your account before continuing"),
);
assert!(state.blocked);
assert_eq!(state.code.as_deref(), Some("account_verification"));
assert!(!should_auto_remove_account_state(&state));
assert!(account_state_indicates_known_ban(&state));
}
}

View File

@@ -373,6 +373,7 @@ pub struct CandidateRuntimeSelectabilityInput<'a> {
pub cached_affinity_target: Option<&'a crate::SchedulerAffinityTarget>,
pub provider_quota_blocks_requests: bool,
pub account_quota_exhausted: bool,
pub oauth_invalid: bool,
pub rpm_reset_at: Option<u64>,
}
@@ -394,6 +395,7 @@ pub fn candidate_runtime_skip_reason_with_state(
cached_affinity_target,
provider_quota_blocks_requests,
account_quota_exhausted,
oauth_invalid,
rpm_reset_at,
} = input;
@@ -403,6 +405,9 @@ pub fn candidate_runtime_skip_reason_with_state(
if account_quota_exhausted {
return Some("account_quota_exhausted");
}
if oauth_invalid {
return Some("oauth_invalid");
}
if crate::is_candidate_in_recent_failure_cooldown(
recent_candidates,
candidate.provider_id.as_str(),
@@ -816,6 +821,7 @@ mod tests {
cached_affinity_target: None,
provider_quota_blocks_requests: false,
account_quota_exhausted: false,
oauth_invalid: false,
rpm_reset_at: None,
},
));
@@ -835,6 +841,7 @@ mod tests {
cached_affinity_target: None,
provider_quota_blocks_requests: false,
account_quota_exhausted: false,
oauth_invalid: false,
rpm_reset_at: None,
},
));
@@ -848,6 +855,7 @@ mod tests {
cached_affinity_target: None,
provider_quota_blocks_requests: true,
account_quota_exhausted: false,
oauth_invalid: false,
rpm_reset_at: None,
},
));
@@ -865,6 +873,25 @@ mod tests {
cached_affinity_target: None,
provider_quota_blocks_requests: false,
account_quota_exhausted: true,
oauth_invalid: false,
rpm_reset_at: None,
},
));
}
#[test]
fn candidate_selectability_rejects_oauth_invalid_keys() {
assert!(!candidate_is_selectable_with_runtime_state(
CandidateRuntimeSelectabilityInput {
candidate: &sample_candidate("1", None),
recent_candidates: &[],
provider_concurrent_limits: &BTreeMap::new(),
provider_key_rpm_states: &BTreeMap::new(),
now_unix_secs: 100,
cached_affinity_target: None,
provider_quota_blocks_requests: false,
account_quota_exhausted: false,
oauth_invalid: true,
rpm_reset_at: None,
},
));