mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-03 01:40:21 +08:00
feat(oauth): 完善账号异常识别并在调度/展示层拦截失效 OAuth 密钥
- 新增 aether-admin provider status 模块,统一解析账号状态(禁用/工作区停用等) - 调度器 runtime 增加 oauth_invalid 判定,跳过刷新失败或已撤销的 OAuth 密钥(REQUEST_FAILED 保留可选) - gateway state 在 local oauth 刷新返回 4xx 时持久化失败原因并同步状态快照 - admin pool 列表/详情回填 account 状态与 scheduling 阻塞原因(account_blocked) - 共享 catalog 的 status_snapshot payload 附加 account 字段
This commit is contained in:
@@ -674,6 +674,11 @@ fn admin_pool_scheduling_payload(
|
|||||||
cooldown_ttl_seconds: Option<u64>,
|
cooldown_ttl_seconds: Option<u64>,
|
||||||
health_score: f64,
|
health_score: f64,
|
||||||
circuit_breaker_open: bool,
|
circuit_breaker_open: bool,
|
||||||
|
account_blocked: bool,
|
||||||
|
account_status_code: Option<&str>,
|
||||||
|
account_status_label: Option<&str>,
|
||||||
|
account_status_reason: Option<&str>,
|
||||||
|
account_status_source: Option<&str>,
|
||||||
account_quota_exhausted: bool,
|
account_quota_exhausted: bool,
|
||||||
) -> (String, String, String, Vec<serde_json::Value>) {
|
) -> (String, String, String, Vec<serde_json::Value>) {
|
||||||
if !key.is_active {
|
if !key.is_active {
|
||||||
@@ -691,6 +696,21 @@ fn admin_pool_scheduling_payload(
|
|||||||
})],
|
})],
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if account_blocked {
|
||||||
|
return (
|
||||||
|
"blocked".to_string(),
|
||||||
|
"account_blocked".to_string(),
|
||||||
|
account_status_label.unwrap_or("账号异常").to_string(),
|
||||||
|
vec![json!({
|
||||||
|
"code": account_status_code.unwrap_or("account_blocked"),
|
||||||
|
"label": account_status_label.unwrap_or("账号异常"),
|
||||||
|
"blocking": true,
|
||||||
|
"source": account_status_source,
|
||||||
|
"ttl_seconds": serde_json::Value::Null,
|
||||||
|
"detail": account_status_reason,
|
||||||
|
})],
|
||||||
|
);
|
||||||
|
}
|
||||||
if account_quota_exhausted {
|
if account_quota_exhausted {
|
||||||
return (
|
return (
|
||||||
"blocked".to_string(),
|
"blocked".to_string(),
|
||||||
@@ -776,15 +796,6 @@ pub(super) fn build_admin_pool_key_payload(
|
|||||||
.as_ref()
|
.as_ref()
|
||||||
.is_some_and(|config| config.skip_exhausted_accounts)
|
.is_some_and(|config| config.skip_exhausted_accounts)
|
||||||
&& admin_provider_pool_pure::admin_pool_key_account_quota_exhausted(key, provider_type);
|
&& admin_provider_pool_pure::admin_pool_key_account_quota_exhausted(key, provider_type);
|
||||||
let (scheduling_status, scheduling_reason, scheduling_label, scheduling_reasons) =
|
|
||||||
admin_pool_scheduling_payload(
|
|
||||||
key,
|
|
||||||
cooldown_reason.as_deref(),
|
|
||||||
cooldown_ttl_seconds,
|
|
||||||
health_score,
|
|
||||||
circuit_breaker_open,
|
|
||||||
account_quota_exhausted,
|
|
||||||
);
|
|
||||||
let auth_config = state.parse_catalog_auth_config_json(key);
|
let auth_config = state.parse_catalog_auth_config_json(key);
|
||||||
let oauth_expires_at =
|
let oauth_expires_at =
|
||||||
admin_pool_derive_oauth_expires_at(provider_type, key, auth_config.as_ref());
|
admin_pool_derive_oauth_expires_at(provider_type, key, auth_config.as_ref());
|
||||||
@@ -841,6 +852,20 @@ pub(super) fn build_admin_pool_key_payload(
|
|||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
let account_status_source =
|
let account_status_source =
|
||||||
admin_pool_trimmed_string(account_snapshot.and_then(|item| item.get("source")));
|
admin_pool_trimmed_string(account_snapshot.and_then(|item| item.get("source")));
|
||||||
|
let (scheduling_status, scheduling_reason, scheduling_label, scheduling_reasons) =
|
||||||
|
admin_pool_scheduling_payload(
|
||||||
|
key,
|
||||||
|
cooldown_reason.as_deref(),
|
||||||
|
cooldown_ttl_seconds,
|
||||||
|
health_score,
|
||||||
|
circuit_breaker_open,
|
||||||
|
account_status_blocked,
|
||||||
|
account_status_code.as_deref(),
|
||||||
|
account_status_label.as_deref(),
|
||||||
|
account_status_reason.as_deref(),
|
||||||
|
account_status_source.as_deref(),
|
||||||
|
account_quota_exhausted,
|
||||||
|
);
|
||||||
|
|
||||||
let mut payload = serde_json::Map::new();
|
let mut payload = serde_json::Map::new();
|
||||||
payload.insert("key_id".to_string(), json!(key.id));
|
payload.insert("key_id".to_string(), json!(key.id));
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ use crate::handlers::shared::{json_string_list, unix_secs_to_rfc3339};
|
|||||||
use crate::provider_key_auth::provider_key_auth_semantics;
|
use crate::provider_key_auth::provider_key_auth_semantics;
|
||||||
use crate::AppState;
|
use crate::AppState;
|
||||||
use aether_admin::provider::quota as admin_provider_quota_pure;
|
use aether_admin::provider::quota as admin_provider_quota_pure;
|
||||||
|
use aether_admin::provider::status as admin_provider_status_pure;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||||
use aether_crypto::{decrypt_python_fernet_ciphertext, encrypt_python_fernet_plaintext};
|
use aether_crypto::{decrypt_python_fernet_ciphertext, encrypt_python_fernet_plaintext};
|
||||||
@@ -154,6 +155,25 @@ pub(crate) fn default_provider_key_status_snapshot() -> serde_json::Value {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn build_provider_key_account_status_snapshot(
|
||||||
|
key: &StoredProviderCatalogKey,
|
||||||
|
provider_type: &str,
|
||||||
|
) -> Value {
|
||||||
|
let snapshot = admin_provider_status_pure::resolve_account_status_snapshot(
|
||||||
|
Some(provider_type),
|
||||||
|
key.upstream_metadata.as_ref(),
|
||||||
|
key.oauth_invalid_reason.as_deref(),
|
||||||
|
);
|
||||||
|
json!({
|
||||||
|
"code": snapshot.code,
|
||||||
|
"label": snapshot.label,
|
||||||
|
"reason": snapshot.reason,
|
||||||
|
"blocked": snapshot.blocked,
|
||||||
|
"source": snapshot.source,
|
||||||
|
"recoverable": snapshot.recoverable,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn provider_key_status_snapshot_object(
|
fn provider_key_status_snapshot_object(
|
||||||
status_snapshot: Option<&Value>,
|
status_snapshot: Option<&Value>,
|
||||||
) -> Option<Map<String, Value>> {
|
) -> Option<Map<String, Value>> {
|
||||||
@@ -817,20 +837,29 @@ pub(crate) fn provider_key_status_snapshot_payload(
|
|||||||
.and_then(|snapshot| snapshot.get("quota"))
|
.and_then(|snapshot| snapshot.get("quota"))
|
||||||
.and_then(Value::as_object);
|
.and_then(Value::as_object);
|
||||||
|
|
||||||
if quota_snapshot_has_materialized_data(quota_snapshot, provider_type) {
|
let payload = if quota_snapshot_has_materialized_data(quota_snapshot, provider_type) {
|
||||||
return status_snapshot
|
status_snapshot
|
||||||
.cloned()
|
.cloned()
|
||||||
.unwrap_or_else(default_provider_key_status_snapshot);
|
.unwrap_or_else(default_provider_key_status_snapshot)
|
||||||
}
|
} else {
|
||||||
|
sync_provider_key_quota_status_snapshot(
|
||||||
|
status_snapshot,
|
||||||
|
provider_type,
|
||||||
|
key.upstream_metadata.as_ref(),
|
||||||
|
"catalog_fallback",
|
||||||
|
)
|
||||||
|
.or_else(|| status_snapshot.cloned())
|
||||||
|
.unwrap_or_else(default_provider_key_status_snapshot)
|
||||||
|
};
|
||||||
|
|
||||||
sync_provider_key_quota_status_snapshot(
|
let mut snapshot = provider_key_status_snapshot_object(Some(&payload))
|
||||||
status_snapshot,
|
.or_else(|| default_provider_key_status_snapshot().as_object().cloned())
|
||||||
provider_type,
|
.unwrap_or_default();
|
||||||
key.upstream_metadata.as_ref(),
|
snapshot.insert(
|
||||||
"catalog_fallback",
|
"account".to_string(),
|
||||||
)
|
build_provider_key_account_status_snapshot(key, provider_type),
|
||||||
.or_else(|| status_snapshot.cloned())
|
);
|
||||||
.unwrap_or_else(default_provider_key_status_snapshot)
|
Value::Object(snapshot)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn provider_key_health_summary(
|
pub(crate) fn provider_key_health_summary(
|
||||||
@@ -1540,4 +1569,47 @@ mod tests {
|
|||||||
Some(2usize)
|
Some(2usize)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn provider_key_status_snapshot_payload_backfills_account_block_from_oauth_invalid_reason() {
|
||||||
|
let mut key = sample_catalog_key();
|
||||||
|
key.oauth_invalid_reason = Some("[ACCOUNT_BLOCK] account has been deactivated".to_string());
|
||||||
|
|
||||||
|
let payload = provider_key_status_snapshot_payload(&key, "codex");
|
||||||
|
let account = payload
|
||||||
|
.get("account")
|
||||||
|
.and_then(Value::as_object)
|
||||||
|
.expect("account snapshot should be object");
|
||||||
|
|
||||||
|
assert_eq!(account.get("code"), Some(&json!("account_disabled")));
|
||||||
|
assert_eq!(account.get("label"), Some(&json!("账号停用")));
|
||||||
|
assert_eq!(
|
||||||
|
account.get("reason"),
|
||||||
|
Some(&json!("account has been deactivated"))
|
||||||
|
);
|
||||||
|
assert_eq!(account.get("blocked"), Some(&json!(true)));
|
||||||
|
assert_eq!(account.get("source"), Some(&json!("oauth_invalid")));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn provider_key_status_snapshot_payload_backfills_workspace_deactivated_from_metadata() {
|
||||||
|
let mut key = sample_catalog_key();
|
||||||
|
key.upstream_metadata = Some(json!({
|
||||||
|
"codex": {
|
||||||
|
"account_disabled": true,
|
||||||
|
"reason": "deactivated_workspace"
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
|
||||||
|
let payload = provider_key_status_snapshot_payload(&key, "codex");
|
||||||
|
let account = payload
|
||||||
|
.get("account")
|
||||||
|
.and_then(Value::as_object)
|
||||||
|
.expect("account snapshot should be object");
|
||||||
|
|
||||||
|
assert_eq!(account.get("code"), Some(&json!("workspace_deactivated")));
|
||||||
|
assert_eq!(account.get("label"), Some(&json!("工作区停用")));
|
||||||
|
assert_eq!(account.get("blocked"), Some(&json!(true)));
|
||||||
|
assert_eq!(account.get("source"), Some(&json!("metadata")));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ pub(super) struct CandidateRuntimeSelectionSnapshot {
|
|||||||
pub(super) provider_key_rpm_states: BTreeMap<String, StoredProviderCatalogKey>,
|
pub(super) provider_key_rpm_states: BTreeMap<String, StoredProviderCatalogKey>,
|
||||||
provider_quota_blocks_requests: BTreeMap<String, bool>,
|
provider_quota_blocks_requests: BTreeMap<String, bool>,
|
||||||
key_account_quota_exhausted: BTreeMap<String, bool>,
|
key_account_quota_exhausted: BTreeMap<String, bool>,
|
||||||
|
key_oauth_invalid: BTreeMap<String, bool>,
|
||||||
provider_key_rpm_reset_ats: BTreeMap<String, Option<u64>>,
|
provider_key_rpm_reset_ats: BTreeMap<String, Option<u64>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -40,6 +41,8 @@ pub(super) async fn read_candidate_runtime_selection_snapshot(
|
|||||||
&provider_key_rpm_states,
|
&provider_key_rpm_states,
|
||||||
&provider_skip_exhausted_accounts,
|
&provider_skip_exhausted_accounts,
|
||||||
);
|
);
|
||||||
|
let key_oauth_invalid =
|
||||||
|
read_key_oauth_invalid_map(candidates, &provider_key_rpm_states, now_unix_secs);
|
||||||
let provider_quota_blocks_requests =
|
let provider_quota_blocks_requests =
|
||||||
read_provider_quota_block_map(state, candidates, now_unix_secs).await?;
|
read_provider_quota_block_map(state, candidates, now_unix_secs).await?;
|
||||||
let provider_key_rpm_reset_ats =
|
let provider_key_rpm_reset_ats =
|
||||||
@@ -51,6 +54,7 @@ pub(super) async fn read_candidate_runtime_selection_snapshot(
|
|||||||
provider_key_rpm_states,
|
provider_key_rpm_states,
|
||||||
provider_quota_blocks_requests,
|
provider_quota_blocks_requests,
|
||||||
key_account_quota_exhausted,
|
key_account_quota_exhausted,
|
||||||
|
key_oauth_invalid,
|
||||||
provider_key_rpm_reset_ats,
|
provider_key_rpm_reset_ats,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -104,6 +108,11 @@ pub(super) fn is_candidate_selectable(
|
|||||||
.get(candidate.key_id.as_str())
|
.get(candidate.key_id.as_str())
|
||||||
.copied()
|
.copied()
|
||||||
.unwrap_or(false),
|
.unwrap_or(false),
|
||||||
|
oauth_invalid: snapshot
|
||||||
|
.key_oauth_invalid
|
||||||
|
.get(candidate.key_id.as_str())
|
||||||
|
.copied()
|
||||||
|
.unwrap_or(false),
|
||||||
rpm_reset_at: snapshot
|
rpm_reset_at: snapshot
|
||||||
.provider_key_rpm_reset_ats
|
.provider_key_rpm_reset_ats
|
||||||
.get(candidate.key_id.as_str())
|
.get(candidate.key_id.as_str())
|
||||||
@@ -142,6 +151,11 @@ pub(super) fn current_candidate_runtime_skip_reason(
|
|||||||
.get(candidate.key_id.as_str())
|
.get(candidate.key_id.as_str())
|
||||||
.copied()
|
.copied()
|
||||||
.unwrap_or(false),
|
.unwrap_or(false),
|
||||||
|
oauth_invalid: snapshot
|
||||||
|
.key_oauth_invalid
|
||||||
|
.get(candidate.key_id.as_str())
|
||||||
|
.copied()
|
||||||
|
.unwrap_or(false),
|
||||||
rpm_reset_at,
|
rpm_reset_at,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -270,6 +284,40 @@ fn read_key_account_quota_exhaustion_map(
|
|||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn read_key_oauth_invalid_map(
|
||||||
|
candidates: &[SchedulerMinimalCandidateSelectionCandidate],
|
||||||
|
provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
|
||||||
|
now_unix_secs: u64,
|
||||||
|
) -> BTreeMap<String, bool> {
|
||||||
|
candidates
|
||||||
|
.iter()
|
||||||
|
.map(|candidate| {
|
||||||
|
let oauth_invalid = provider_key_rpm_states
|
||||||
|
.get(candidate.key_id.as_str())
|
||||||
|
.is_some_and(|key| key_requires_oauth_reauth(key, now_unix_secs));
|
||||||
|
(candidate.key_id.clone(), oauth_invalid)
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn key_requires_oauth_reauth(key: &StoredProviderCatalogKey, now_unix_secs: u64) -> bool {
|
||||||
|
if !key.auth_type.trim().eq_ignore_ascii_case("oauth") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
let invalid_reason = key
|
||||||
|
.oauth_invalid_reason
|
||||||
|
.as_deref()
|
||||||
|
.map(str::trim)
|
||||||
|
.unwrap_or_default();
|
||||||
|
if !invalid_reason.is_empty() {
|
||||||
|
return !invalid_reason.starts_with("[REQUEST_FAILED]");
|
||||||
|
}
|
||||||
|
|
||||||
|
key.expires_at_unix_secs
|
||||||
|
.is_some_and(|value| value > 0 && value <= now_unix_secs)
|
||||||
|
}
|
||||||
|
|
||||||
fn read_provider_key_rpm_reset_at_map(
|
fn read_provider_key_rpm_reset_at_map(
|
||||||
state: &(impl SchedulerRuntimeState + ?Sized),
|
state: &(impl SchedulerRuntimeState + ?Sized),
|
||||||
candidates: &[SchedulerMinimalCandidateSelectionCandidate],
|
candidates: &[SchedulerMinimalCandidateSelectionCandidate],
|
||||||
|
|||||||
@@ -1241,6 +1241,143 @@ async fn skips_codex_candidate_when_account_quota_is_exhausted_and_pool_flag_ena
|
|||||||
assert_eq!(skipped[0].skip_reason, "account_quota_exhausted");
|
assert_eq!(skipped[0].skip_reason, "account_quota_exhausted");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn skips_oauth_invalid_candidate_before_local_auth_resolution() {
|
||||||
|
let mut first = sample_row();
|
||||||
|
first.provider_id = "provider-codex".to_string();
|
||||||
|
first.provider_name = "codex".to_string();
|
||||||
|
first.provider_type = "codex".to_string();
|
||||||
|
first.endpoint_id = "endpoint-codex".to_string();
|
||||||
|
first.endpoint_api_format = "openai:cli".to_string();
|
||||||
|
first.key_id = "key-codex".to_string();
|
||||||
|
first.key_name = "codex-invalid".to_string();
|
||||||
|
first.key_auth_type = "oauth".to_string();
|
||||||
|
first.key_api_formats = Some(vec!["openai:cli".to_string()]);
|
||||||
|
first.key_global_priority_by_format = Some(serde_json::json!({"openai:cli": 1}));
|
||||||
|
|
||||||
|
let mut second = sample_row();
|
||||||
|
second.provider_id = "provider-openai".to_string();
|
||||||
|
second.provider_name = "openai".to_string();
|
||||||
|
second.endpoint_id = "endpoint-openai".to_string();
|
||||||
|
second.endpoint_api_format = "openai:cli".to_string();
|
||||||
|
second.key_id = "key-openai".to_string();
|
||||||
|
second.key_name = "fallback".to_string();
|
||||||
|
second.key_api_formats = Some(vec!["openai:cli".to_string()]);
|
||||||
|
second.key_global_priority_by_format = Some(serde_json::json!({"openai:cli": 2}));
|
||||||
|
|
||||||
|
let candidates = Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
|
||||||
|
first, second,
|
||||||
|
]));
|
||||||
|
let mut codex_provider = sample_provider("provider-codex", None);
|
||||||
|
codex_provider.provider_type = "codex".to_string();
|
||||||
|
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||||
|
vec![codex_provider, sample_provider("provider-openai", None)],
|
||||||
|
Vec::new(),
|
||||||
|
vec![
|
||||||
|
{
|
||||||
|
let mut key = sample_key("key-codex", "provider-codex", Some(10));
|
||||||
|
key.auth_type = "oauth".to_string();
|
||||||
|
key.oauth_invalid_at_unix_secs = Some(1_710_000_000);
|
||||||
|
key.oauth_invalid_reason = Some(
|
||||||
|
"[REFRESH_FAILED] Token 续期失败 (401): refresh_token 已被使用并轮换,请重新登录授权"
|
||||||
|
.to_string(),
|
||||||
|
);
|
||||||
|
key
|
||||||
|
},
|
||||||
|
sample_key("key-openai", "provider-openai", Some(10)),
|
||||||
|
],
|
||||||
|
));
|
||||||
|
let quotas = Arc::new(InMemoryProviderQuotaRepository::seed(vec![]));
|
||||||
|
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![]));
|
||||||
|
let state = AppState::new()
|
||||||
|
.expect("state should build")
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
GatewayDataState::with_candidate_selection_provider_catalog_quota_and_request_candidates_for_tests(
|
||||||
|
candidates,
|
||||||
|
provider_catalog,
|
||||||
|
quotas,
|
||||||
|
request_candidates,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
let (selected, skipped) = collect_selectable_candidates_with_skip_reasons(
|
||||||
|
state.data.as_ref(),
|
||||||
|
&state,
|
||||||
|
"openai:cli",
|
||||||
|
"gpt-4.1",
|
||||||
|
false,
|
||||||
|
None,
|
||||||
|
1_710_000_100,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("selection should succeed");
|
||||||
|
|
||||||
|
assert_eq!(selected.len(), 1);
|
||||||
|
assert_eq!(selected[0].provider_id, "provider-openai");
|
||||||
|
assert_eq!(skipped.len(), 1);
|
||||||
|
assert_eq!(skipped[0].candidate.provider_id, "provider-codex");
|
||||||
|
assert_eq!(skipped[0].skip_reason, "oauth_invalid");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn keeps_request_failed_oauth_candidate_selectable() {
|
||||||
|
let mut row = sample_row();
|
||||||
|
row.provider_id = "provider-codex".to_string();
|
||||||
|
row.provider_name = "codex".to_string();
|
||||||
|
row.provider_type = "codex".to_string();
|
||||||
|
row.endpoint_id = "endpoint-codex".to_string();
|
||||||
|
row.endpoint_api_format = "openai:cli".to_string();
|
||||||
|
row.key_id = "key-codex".to_string();
|
||||||
|
row.key_name = "codex-check-failed".to_string();
|
||||||
|
row.key_auth_type = "oauth".to_string();
|
||||||
|
row.key_api_formats = Some(vec!["openai:cli".to_string()]);
|
||||||
|
|
||||||
|
let candidates = Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
|
||||||
|
row,
|
||||||
|
]));
|
||||||
|
let mut provider = sample_provider("provider-codex", None);
|
||||||
|
provider.provider_type = "codex".to_string();
|
||||||
|
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||||
|
vec![provider],
|
||||||
|
Vec::new(),
|
||||||
|
vec![{
|
||||||
|
let mut key = sample_key("key-codex", "provider-codex", Some(10));
|
||||||
|
key.auth_type = "oauth".to_string();
|
||||||
|
key.oauth_invalid_at_unix_secs = Some(1_710_000_000);
|
||||||
|
key.oauth_invalid_reason = Some("[REQUEST_FAILED] 账号状态检查失败".to_string());
|
||||||
|
key
|
||||||
|
}],
|
||||||
|
));
|
||||||
|
let quotas = Arc::new(InMemoryProviderQuotaRepository::seed(vec![]));
|
||||||
|
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![]));
|
||||||
|
let state = AppState::new()
|
||||||
|
.expect("state should build")
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
GatewayDataState::with_candidate_selection_provider_catalog_quota_and_request_candidates_for_tests(
|
||||||
|
candidates,
|
||||||
|
provider_catalog,
|
||||||
|
quotas,
|
||||||
|
request_candidates,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
let (selected, skipped) = collect_selectable_candidates_with_skip_reasons(
|
||||||
|
state.data.as_ref(),
|
||||||
|
&state,
|
||||||
|
"openai:cli",
|
||||||
|
"gpt-4.1",
|
||||||
|
false,
|
||||||
|
None,
|
||||||
|
1_710_000_100,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("selection should succeed");
|
||||||
|
|
||||||
|
assert_eq!(selected.len(), 1);
|
||||||
|
assert_eq!(selected[0].provider_id, "provider-codex");
|
||||||
|
assert!(skipped.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn keeps_codex_candidate_selectable_when_exhausted_account_flag_is_disabled() {
|
async fn keeps_codex_candidate_selectable_when_exhausted_account_flag_is_disabled() {
|
||||||
let mut first = sample_row();
|
let mut first = sample_row();
|
||||||
|
|||||||
@@ -49,6 +49,157 @@ fn tagged_reason(reason: Option<&str>, prefix: &str) -> Option<String> {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn oauth_invalid_reason_is_account_block(reason: Option<&str>) -> bool {
|
||||||
|
reason
|
||||||
|
.map(str::trim)
|
||||||
|
.is_some_and(|value| value.starts_with(OAUTH_ACCOUNT_BLOCK_PREFIX))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalize_local_oauth_refresh_error_message(
|
||||||
|
status_code: Option<u16>,
|
||||||
|
body_excerpt: Option<&str>,
|
||||||
|
) -> String {
|
||||||
|
let mut message = None::<String>;
|
||||||
|
let mut error_code = None::<String>;
|
||||||
|
let mut error_type = None::<String>;
|
||||||
|
|
||||||
|
if let Some(body_excerpt) = body_excerpt {
|
||||||
|
if let Ok(value) = serde_json::from_str::<serde_json::Value>(body_excerpt) {
|
||||||
|
if let Some(object) = value.as_object() {
|
||||||
|
if let Some(error_object) =
|
||||||
|
object.get("error").and_then(serde_json::Value::as_object)
|
||||||
|
{
|
||||||
|
message = error_object
|
||||||
|
.get("message")
|
||||||
|
.or_else(|| error_object.get("error_description"))
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(ToOwned::to_owned);
|
||||||
|
error_code = error_object
|
||||||
|
.get("code")
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(|value| value.to_ascii_lowercase());
|
||||||
|
error_type = error_object
|
||||||
|
.get("type")
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(|value| value.to_ascii_lowercase());
|
||||||
|
}
|
||||||
|
if message.is_none() {
|
||||||
|
message = object
|
||||||
|
.get("message")
|
||||||
|
.or_else(|| object.get("error_description"))
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(ToOwned::to_owned);
|
||||||
|
}
|
||||||
|
if error_code.is_none() {
|
||||||
|
error_code = object
|
||||||
|
.get("code")
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(|value| value.to_ascii_lowercase());
|
||||||
|
}
|
||||||
|
if error_type.is_none() {
|
||||||
|
error_type = object
|
||||||
|
.get("type")
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(|value| value.to_ascii_lowercase());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let message = message
|
||||||
|
.or_else(|| {
|
||||||
|
body_excerpt
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(|value| value.chars().take(300).collect::<String>())
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
let lowered = message.to_ascii_lowercase();
|
||||||
|
let error_code = error_code.unwrap_or_default();
|
||||||
|
let error_type = error_type.unwrap_or_default();
|
||||||
|
|
||||||
|
if error_code == "refresh_token_reused"
|
||||||
|
|| lowered.contains("already been used to generate a new access token")
|
||||||
|
{
|
||||||
|
return "refresh_token 已被使用并轮换,请重新登录授权".to_string();
|
||||||
|
}
|
||||||
|
if error_code == "invalid_grant"
|
||||||
|
|| error_code == "invalid_refresh_token"
|
||||||
|
|| (lowered.contains("refresh token")
|
||||||
|
&& ["expired", "revoked", "invalid"]
|
||||||
|
.iter()
|
||||||
|
.any(|keyword| lowered.contains(keyword)))
|
||||||
|
{
|
||||||
|
return "refresh_token 无效、已过期或已撤销,请重新登录授权".to_string();
|
||||||
|
}
|
||||||
|
if error_type == "invalid_request_error" && !message.is_empty() {
|
||||||
|
return message;
|
||||||
|
}
|
||||||
|
if !message.is_empty() {
|
||||||
|
return message;
|
||||||
|
}
|
||||||
|
status_code
|
||||||
|
.map(|status_code| format!("HTTP {status_code}"))
|
||||||
|
.unwrap_or_else(|| "未知错误".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn merge_local_oauth_refresh_failure_reason(
|
||||||
|
current_reason: Option<&str>,
|
||||||
|
refresh_reason: &str,
|
||||||
|
) -> Option<String> {
|
||||||
|
let current_reason = current_reason.map(str::trim).unwrap_or_default();
|
||||||
|
let refresh_reason = refresh_reason.trim();
|
||||||
|
if refresh_reason.is_empty() {
|
||||||
|
return (!current_reason.is_empty()).then(|| current_reason.to_string());
|
||||||
|
}
|
||||||
|
if current_reason.is_empty() {
|
||||||
|
return Some(refresh_reason.to_string());
|
||||||
|
}
|
||||||
|
if current_reason.starts_with(OAUTH_EXPIRED_PREFIX) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if current_reason.starts_with(OAUTH_ACCOUNT_BLOCK_PREFIX) {
|
||||||
|
if let Some((head, _)) = current_reason.split_once("[REFRESH_FAILED]") {
|
||||||
|
return Some(
|
||||||
|
format!("{}\n{}", head.trim_end(), refresh_reason)
|
||||||
|
.trim()
|
||||||
|
.to_string(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return Some(format!("{current_reason}\n{refresh_reason}"));
|
||||||
|
}
|
||||||
|
Some(refresh_reason.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn local_oauth_refresh_success_invalid_state(
|
||||||
|
key: &StoredProviderCatalogKey,
|
||||||
|
) -> (Option<u64>, Option<String>) {
|
||||||
|
let current_reason = key
|
||||||
|
.oauth_invalid_reason
|
||||||
|
.as_deref()
|
||||||
|
.map(str::trim)
|
||||||
|
.unwrap_or_default();
|
||||||
|
if oauth_invalid_reason_is_account_block(Some(current_reason)) {
|
||||||
|
return (
|
||||||
|
key.oauth_invalid_at_unix_secs,
|
||||||
|
Some(current_reason.to_string()),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
(None, None)
|
||||||
|
}
|
||||||
|
|
||||||
fn default_oauth_status_snapshot_value() -> Value {
|
fn default_oauth_status_snapshot_value() -> Value {
|
||||||
default_provider_key_status_snapshot()
|
default_provider_key_status_snapshot()
|
||||||
.get("oauth")
|
.get("oauth")
|
||||||
@@ -557,7 +708,7 @@ impl AppState {
|
|||||||
let executor = GatewayLocalOAuthHttpExecutor { state: self };
|
let executor = GatewayLocalOAuthHttpExecutor { state: self };
|
||||||
|
|
||||||
for _ in 0..2 {
|
for _ in 0..2 {
|
||||||
let resolution = self
|
let resolution = match self
|
||||||
.oauth_refresh
|
.oauth_refresh
|
||||||
.resolve_with_result(
|
.resolve_with_result(
|
||||||
&executor,
|
&executor,
|
||||||
@@ -566,7 +717,32 @@ impl AppState {
|
|||||||
Some(lock_owner.as_str()),
|
Some(lock_owner.as_str()),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
{
|
||||||
|
Ok(resolution) => resolution,
|
||||||
|
Err(provider_transport::LocalOAuthRefreshError::HttpStatus {
|
||||||
|
status_code,
|
||||||
|
body_excerpt,
|
||||||
|
..
|
||||||
|
}) if matches!(status_code, 400 | 401 | 403) => {
|
||||||
|
if let Err(err) = self
|
||||||
|
.persist_local_oauth_refresh_failure_state(
|
||||||
|
¤t_transport,
|
||||||
|
status_code,
|
||||||
|
body_excerpt.as_str(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
tracing::warn!(
|
||||||
|
key_id = %current_transport.key.id,
|
||||||
|
provider_type = %current_transport.provider.provider_type,
|
||||||
|
error = ?err,
|
||||||
|
"gateway local oauth refresh failure persistence failed"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
Err(err) => return Err(GatewayError::Internal(err.to_string())),
|
||||||
|
};
|
||||||
|
|
||||||
if resolution
|
if resolution
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -790,8 +966,10 @@ impl AppState {
|
|||||||
latest_key.encrypted_auth_config = encrypted_auth_config;
|
latest_key.encrypted_auth_config = encrypted_auth_config;
|
||||||
latest_key.is_active = true;
|
latest_key.is_active = true;
|
||||||
latest_key.expires_at_unix_secs = entry.expires_at_unix_secs;
|
latest_key.expires_at_unix_secs = entry.expires_at_unix_secs;
|
||||||
latest_key.oauth_invalid_at_unix_secs = None;
|
let (oauth_invalid_at_unix_secs, oauth_invalid_reason) =
|
||||||
latest_key.oauth_invalid_reason = None;
|
local_oauth_refresh_success_invalid_state(&latest_key);
|
||||||
|
latest_key.oauth_invalid_at_unix_secs = oauth_invalid_at_unix_secs;
|
||||||
|
latest_key.oauth_invalid_reason = oauth_invalid_reason;
|
||||||
latest_key.updated_at_unix_secs = Some(
|
latest_key.updated_at_unix_secs = Some(
|
||||||
SystemTime::now()
|
SystemTime::now()
|
||||||
.duration_since(UNIX_EPOCH)
|
.duration_since(UNIX_EPOCH)
|
||||||
@@ -806,6 +984,72 @@ impl AppState {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn persist_local_oauth_refresh_failure_state(
|
||||||
|
&self,
|
||||||
|
transport: &provider_transport::GatewayProviderTransportSnapshot,
|
||||||
|
status_code: u16,
|
||||||
|
body_excerpt: &str,
|
||||||
|
) -> Result<bool, GatewayError> {
|
||||||
|
let key_id = transport.key.id.trim();
|
||||||
|
if key_id.is_empty() {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some(mut latest_key) = self
|
||||||
|
.data
|
||||||
|
.list_provider_catalog_keys_by_ids(&[key_id.to_string()])
|
||||||
|
.await
|
||||||
|
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||||
|
.into_iter()
|
||||||
|
.next()
|
||||||
|
else {
|
||||||
|
return Ok(false);
|
||||||
|
};
|
||||||
|
|
||||||
|
if !provider_key_is_oauth_managed(&latest_key, transport.provider.provider_type.as_str()) {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
let refresh_reason = format!(
|
||||||
|
"{OAUTH_REFRESH_FAILED_PREFIX}Token 续期失败 ({status_code}): {}",
|
||||||
|
normalize_local_oauth_refresh_error_message(Some(status_code), Some(body_excerpt))
|
||||||
|
);
|
||||||
|
let Some(merged_reason) = merge_local_oauth_refresh_failure_reason(
|
||||||
|
latest_key.oauth_invalid_reason.as_deref(),
|
||||||
|
&refresh_reason,
|
||||||
|
) else {
|
||||||
|
return Ok(false);
|
||||||
|
};
|
||||||
|
if latest_key.oauth_invalid_reason.as_deref() == Some(merged_reason.as_str())
|
||||||
|
&& latest_key.oauth_invalid_at_unix_secs.is_some()
|
||||||
|
{
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
let now_unix_secs = SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.ok()
|
||||||
|
.map(|duration| duration.as_secs())
|
||||||
|
.unwrap_or(0);
|
||||||
|
latest_key.oauth_invalid_at_unix_secs = latest_key
|
||||||
|
.oauth_invalid_at_unix_secs
|
||||||
|
.or(Some(now_unix_secs));
|
||||||
|
latest_key.oauth_invalid_reason = Some(merged_reason);
|
||||||
|
latest_key.updated_at_unix_secs = Some(now_unix_secs);
|
||||||
|
let current_status_snapshot = latest_key.status_snapshot.take();
|
||||||
|
latest_key.status_snapshot =
|
||||||
|
sync_provider_key_oauth_status_snapshot(current_status_snapshot, &latest_key);
|
||||||
|
|
||||||
|
let updated = self
|
||||||
|
.update_provider_catalog_key(&latest_key)
|
||||||
|
.await?
|
||||||
|
.is_some();
|
||||||
|
if updated {
|
||||||
|
let _ = self.invalidate_local_oauth_refresh_entry(key_id).await;
|
||||||
|
}
|
||||||
|
Ok(updated)
|
||||||
|
}
|
||||||
|
|
||||||
async fn execute_local_oauth_http_request(
|
async fn execute_local_oauth_http_request(
|
||||||
&self,
|
&self,
|
||||||
provider_type: &'static str,
|
provider_type: &'static str,
|
||||||
|
|||||||
@@ -2737,6 +2737,138 @@ async fn gateway_starts_admin_provider_oauth_kiro_batch_import_task_locally_with
|
|||||||
upstream_handle.abort();
|
upstream_handle.abort();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn gateway_marks_lazy_codex_oauth_refresh_failures_as_invalid() {
|
||||||
|
let token_hits = Arc::new(Mutex::new(0usize));
|
||||||
|
let token_hits_clone = Arc::clone(&token_hits);
|
||||||
|
let token_server = Router::new().route(
|
||||||
|
"/oauth/token",
|
||||||
|
post(move |_request: Request| {
|
||||||
|
let token_hits_inner = Arc::clone(&token_hits_clone);
|
||||||
|
async move {
|
||||||
|
*token_hits_inner.lock().expect("mutex should lock") += 1;
|
||||||
|
(
|
||||||
|
StatusCode::UNAUTHORIZED,
|
||||||
|
Json(json!({
|
||||||
|
"error": {
|
||||||
|
"message": "Your refresh token has already been used to generate a new access token. Please try signing in again.",
|
||||||
|
"type": "invalid_request_error",
|
||||||
|
"param": serde_json::Value::Null,
|
||||||
|
"code": "refresh_token_reused"
|
||||||
|
}
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut provider = sample_provider("provider-codex", "codex", 10);
|
||||||
|
provider.provider_type = "codex".to_string();
|
||||||
|
let endpoint = sample_endpoint(
|
||||||
|
"endpoint-codex-cli",
|
||||||
|
"provider-codex",
|
||||||
|
"openai:cli",
|
||||||
|
"https://chatgpt.com/backend-api/codex",
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut key = sample_key(
|
||||||
|
"key-codex-oauth-lazy",
|
||||||
|
"provider-codex",
|
||||||
|
"openai:cli",
|
||||||
|
"stale-codex-access-token",
|
||||||
|
);
|
||||||
|
key.auth_type = "oauth".to_string();
|
||||||
|
key.expires_at_unix_secs = Some(1);
|
||||||
|
key.encrypted_auth_config = Some(
|
||||||
|
encrypt_python_fernet_plaintext(
|
||||||
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
|
r#"{"provider_type":"codex","refresh_token":"used-refresh-token","email":"alice@example.com","account_id":"acct-codex-123","plan_type":"plus","expires_at":1}"#,
|
||||||
|
)
|
||||||
|
.expect("auth config ciphertext should build"),
|
||||||
|
);
|
||||||
|
|
||||||
|
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||||
|
vec![provider],
|
||||||
|
vec![endpoint],
|
||||||
|
vec![key],
|
||||||
|
));
|
||||||
|
|
||||||
|
let (token_url, token_handle) = start_server(token_server).await;
|
||||||
|
let oauth_refresh =
|
||||||
|
crate::provider_transport::LocalOAuthRefreshCoordinator::with_adapters_for_tests(vec![
|
||||||
|
Arc::new(
|
||||||
|
crate::provider_transport::oauth_refresh::GenericOAuthRefreshAdapter::default()
|
||||||
|
.with_token_url_for_tests("codex", format!("{token_url}/oauth/token")),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
let state = AppState::new()
|
||||||
|
.expect("state should build")
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
GatewayDataState::with_provider_catalog_repository_for_tests(
|
||||||
|
provider_catalog_repository.clone(),
|
||||||
|
)
|
||||||
|
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||||
|
)
|
||||||
|
.with_oauth_refresh_coordinator_for_tests(oauth_refresh);
|
||||||
|
|
||||||
|
let transport = state
|
||||||
|
.read_provider_transport_snapshot(
|
||||||
|
"provider-codex",
|
||||||
|
"endpoint-codex-cli",
|
||||||
|
"key-codex-oauth-lazy",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("transport snapshot should load")
|
||||||
|
.expect("transport snapshot should exist");
|
||||||
|
let resolved = state
|
||||||
|
.resolve_local_oauth_request_auth(&transport)
|
||||||
|
.await
|
||||||
|
.expect("refresh-token reuse should degrade into oauth-unavailable");
|
||||||
|
|
||||||
|
assert_eq!(resolved, None);
|
||||||
|
assert_eq!(*token_hits.lock().expect("mutex should lock"), 1);
|
||||||
|
|
||||||
|
let stored_key = provider_catalog_repository
|
||||||
|
.list_keys_by_ids(&["key-codex-oauth-lazy".to_string()])
|
||||||
|
.await
|
||||||
|
.expect("keys should list")
|
||||||
|
.into_iter()
|
||||||
|
.next()
|
||||||
|
.expect("oauth key should exist");
|
||||||
|
assert!(stored_key.oauth_invalid_at_unix_secs.is_some());
|
||||||
|
assert_eq!(
|
||||||
|
stored_key.oauth_invalid_reason.as_deref(),
|
||||||
|
Some("[REFRESH_FAILED] Token 续期失败 (401): refresh_token 已被使用并轮换,请重新登录授权")
|
||||||
|
);
|
||||||
|
let oauth_snapshot = stored_key
|
||||||
|
.status_snapshot
|
||||||
|
.as_ref()
|
||||||
|
.and_then(serde_json::Value::as_object)
|
||||||
|
.and_then(|snapshot| snapshot.get("oauth"))
|
||||||
|
.and_then(serde_json::Value::as_object)
|
||||||
|
.expect("oauth status snapshot should exist");
|
||||||
|
assert_eq!(
|
||||||
|
oauth_snapshot
|
||||||
|
.get("code")
|
||||||
|
.and_then(serde_json::Value::as_str),
|
||||||
|
Some("invalid")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
oauth_snapshot
|
||||||
|
.get("source")
|
||||||
|
.and_then(serde_json::Value::as_str),
|
||||||
|
Some("oauth_refresh")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
oauth_snapshot
|
||||||
|
.get("requires_reauth")
|
||||||
|
.and_then(serde_json::Value::as_bool),
|
||||||
|
Some(true)
|
||||||
|
);
|
||||||
|
|
||||||
|
token_handle.abort();
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn gateway_refreshes_admin_provider_oauth_key_locally_with_trusted_admin_principal() {
|
async fn gateway_refreshes_admin_provider_oauth_key_locally_with_trusted_admin_principal() {
|
||||||
let upstream_hits = Arc::new(Mutex::new(0usize));
|
let upstream_hits = Arc::new(Mutex::new(0usize));
|
||||||
|
|||||||
@@ -691,6 +691,76 @@ async fn gateway_handles_admin_pool_list_keys_locally_with_trusted_admin_princip
|
|||||||
upstream_handle.abort();
|
upstream_handle.abort();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn gateway_marks_account_blocked_pool_key_in_list_keys_response() {
|
||||||
|
let mut provider = sample_provider("provider-codex", "codex", 10).with_transport_fields(
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
Some(json!({
|
||||||
|
"pool_advanced": {
|
||||||
|
"enabled": true
|
||||||
|
}
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
provider.provider_type = "codex".to_string();
|
||||||
|
|
||||||
|
let mut key = sample_key(
|
||||||
|
"key-codex-blocked",
|
||||||
|
"provider-codex",
|
||||||
|
"openai:cli",
|
||||||
|
"oauth-placeholder",
|
||||||
|
);
|
||||||
|
key.name = "blocked-codex".to_string();
|
||||||
|
key.auth_type = "oauth".to_string();
|
||||||
|
key.oauth_invalid_reason = Some("[ACCOUNT_BLOCK] account has been deactivated".to_string());
|
||||||
|
key.oauth_invalid_at_unix_secs = Some(1_700_000_000);
|
||||||
|
|
||||||
|
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||||
|
vec![provider],
|
||||||
|
Vec::new(),
|
||||||
|
vec![key],
|
||||||
|
));
|
||||||
|
let state = AppState::new()
|
||||||
|
.expect("gateway should build")
|
||||||
|
.with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests(
|
||||||
|
provider_catalog_repository,
|
||||||
|
));
|
||||||
|
|
||||||
|
let response = local_admin_pool_response(
|
||||||
|
&state,
|
||||||
|
http::Method::GET,
|
||||||
|
"/api/admin/pool/provider-codex/keys?page=1&page_size=50&status=all",
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
let payload: serde_json::Value = serde_json::from_slice(
|
||||||
|
&to_bytes(response.into_body(), usize::MAX)
|
||||||
|
.await
|
||||||
|
.expect("body should read"),
|
||||||
|
)
|
||||||
|
.expect("json body should parse");
|
||||||
|
let keys = payload["keys"].as_array().expect("keys should be array");
|
||||||
|
|
||||||
|
assert_eq!(keys.len(), 1);
|
||||||
|
assert_eq!(keys[0]["account_status_code"], json!("account_disabled"));
|
||||||
|
assert_eq!(keys[0]["account_status_blocked"], json!(true));
|
||||||
|
assert_eq!(keys[0]["scheduling_status"], json!("blocked"));
|
||||||
|
assert_eq!(keys[0]["scheduling_reason"], json!("account_blocked"));
|
||||||
|
assert_eq!(keys[0]["scheduling_label"], json!("账号停用"));
|
||||||
|
assert_eq!(
|
||||||
|
keys[0]["status_snapshot"]["account"]["source"],
|
||||||
|
json!("oauth_invalid")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn gateway_handles_admin_pool_list_keys_with_quota_compatibility_fields() {
|
async fn gateway_handles_admin_pool_list_keys_with_quota_compatibility_fields() {
|
||||||
let upstream_hits = Arc::new(Mutex::new(0usize));
|
let upstream_hits = Arc::new(Mutex::new(0usize));
|
||||||
|
|||||||
@@ -6,3 +6,4 @@ pub mod ops;
|
|||||||
pub mod pool;
|
pub mod pool;
|
||||||
pub mod quota;
|
pub mod quota;
|
||||||
pub mod state;
|
pub mod state;
|
||||||
|
pub mod status;
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ use chrono::{TimeZone, Utc};
|
|||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
use std::collections::{BTreeMap, BTreeSet};
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
|
|
||||||
|
use super::status as provider_status;
|
||||||
|
|
||||||
#[derive(Debug, Default, Clone, serde::Deserialize)]
|
#[derive(Debug, Default, Clone, serde::Deserialize)]
|
||||||
pub struct AdminPoolResolveSelectionRequest {
|
pub struct AdminPoolResolveSelectionRequest {
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
@@ -461,40 +463,17 @@ pub fn admin_pool_matches_search(
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn admin_pool_key_is_known_banned(key: &StoredProviderCatalogKey) -> bool {
|
pub fn admin_pool_key_is_known_banned(key: &StoredProviderCatalogKey) -> bool {
|
||||||
if key
|
let state = provider_status::resolve_pool_account_state(
|
||||||
.oauth_invalid_reason
|
None,
|
||||||
.as_deref()
|
key.upstream_metadata.as_ref(),
|
||||||
.is_some_and(admin_pool_reason_indicates_ban)
|
key.oauth_invalid_reason.as_deref(),
|
||||||
{
|
);
|
||||||
|
if provider_status::account_state_indicates_known_ban(&state) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
key.oauth_invalid_reason
|
||||||
let Some(account) = key
|
.as_deref()
|
||||||
.status_snapshot
|
|
||||||
.as_ref()
|
|
||||||
.and_then(Value::as_object)
|
|
||||||
.and_then(|snapshot| snapshot.get("account"))
|
|
||||||
.and_then(Value::as_object)
|
|
||||||
else {
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
|
|
||||||
if !account
|
|
||||||
.get("blocked")
|
|
||||||
.and_then(Value::as_bool)
|
|
||||||
.unwrap_or(false)
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
account
|
|
||||||
.get("code")
|
|
||||||
.and_then(Value::as_str)
|
|
||||||
.is_some_and(admin_pool_reason_indicates_ban)
|
.is_some_and(admin_pool_reason_indicates_ban)
|
||||||
|| account
|
|
||||||
.get("reason")
|
|
||||||
.and_then(Value::as_str)
|
|
||||||
.is_some_and(admin_pool_reason_indicates_ban)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn admin_pool_sort_keys(keys: &mut [StoredProviderCatalogKey]) {
|
pub fn admin_pool_sort_keys(keys: &mut [StoredProviderCatalogKey]) {
|
||||||
@@ -665,7 +644,7 @@ pub fn build_admin_pool_selection_payload(keys: &[StoredProviderCatalogKey]) ->
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
#[allow(clippy::items_after_test_module)]
|
#[allow(clippy::items_after_test_module)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::admin_pool_key_account_quota_exhausted;
|
use super::{admin_pool_key_account_quota_exhausted, admin_pool_key_is_known_banned};
|
||||||
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
|
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
@@ -790,6 +769,18 @@ mod tests {
|
|||||||
"kiro",
|
"kiro",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn known_banned_detects_provider_bucket_account_blocks_without_provider_type() {
|
||||||
|
let key = sample_key(Some(json!({
|
||||||
|
"codex": {
|
||||||
|
"account_disabled": true,
|
||||||
|
"reason": "deactivated_workspace"
|
||||||
|
}
|
||||||
|
})));
|
||||||
|
|
||||||
|
assert!(admin_pool_key_is_known_banned(&key));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn build_admin_pool_key_payload(
|
pub fn build_admin_pool_key_payload(
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKe
|
|||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
|
use super::status as provider_status;
|
||||||
|
|
||||||
const OAUTH_ACCOUNT_BLOCK_PREFIX: &str = "[ACCOUNT_BLOCK] ";
|
const OAUTH_ACCOUNT_BLOCK_PREFIX: &str = "[ACCOUNT_BLOCK] ";
|
||||||
const OAUTH_REFRESH_FAILED_PREFIX: &str = "[REFRESH_FAILED] ";
|
const OAUTH_REFRESH_FAILED_PREFIX: &str = "[REFRESH_FAILED] ";
|
||||||
const OAUTH_EXPIRED_PREFIX: &str = "[OAUTH_EXPIRED] ";
|
const OAUTH_EXPIRED_PREFIX: &str = "[OAUTH_EXPIRED] ";
|
||||||
@@ -18,9 +20,9 @@ pub fn provider_auto_remove_banned_keys(config: Option<&serde_json::Value>) -> b
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn should_auto_remove_structured_reason(reason: Option<&str>) -> bool {
|
pub fn should_auto_remove_structured_reason(reason: Option<&str>) -> bool {
|
||||||
reason
|
provider_status::should_auto_remove_account_state(&provider_status::resolve_pool_account_state(
|
||||||
.map(str::trim)
|
None, None, reason,
|
||||||
.is_some_and(|value| value.starts_with(OAUTH_ACCOUNT_BLOCK_PREFIX))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn normalize_string_id_list(values: Option<Vec<String>>) -> Option<Vec<String>> {
|
pub fn normalize_string_id_list(values: Option<Vec<String>>) -> Option<Vec<String>> {
|
||||||
|
|||||||
600
crates/aether-admin/src/provider/status.rs
Normal file
600
crates/aether-admin/src/provider/status.rs
Normal file
@@ -0,0 +1,600 @@
|
|||||||
|
use serde_json::Value;
|
||||||
|
use std::collections::BTreeSet;
|
||||||
|
|
||||||
|
const OAUTH_ACCOUNT_BLOCK_PREFIX: &str = "[ACCOUNT_BLOCK] ";
|
||||||
|
const OAUTH_REFRESH_FAILED_PREFIX: &str = "[REFRESH_FAILED] ";
|
||||||
|
const OAUTH_EXPIRED_PREFIX: &str = "[OAUTH_EXPIRED] ";
|
||||||
|
const OAUTH_REQUEST_FAILED_PREFIX: &str = "[REQUEST_FAILED] ";
|
||||||
|
|
||||||
|
const ACCOUNT_BLOCK_REASON_KEYWORDS: &[&str] = &[
|
||||||
|
"suspended",
|
||||||
|
"banned",
|
||||||
|
"account_block",
|
||||||
|
"account blocked",
|
||||||
|
"account_forbidden",
|
||||||
|
"forbidden",
|
||||||
|
"封禁",
|
||||||
|
"封号",
|
||||||
|
"被封",
|
||||||
|
"账户已封禁",
|
||||||
|
"账号异常",
|
||||||
|
"account has been disabled",
|
||||||
|
"account disabled",
|
||||||
|
"account has been deactivated",
|
||||||
|
"account_deactivated",
|
||||||
|
"account deactivated",
|
||||||
|
"organization has been disabled",
|
||||||
|
"organization_disabled",
|
||||||
|
"deactivated_workspace",
|
||||||
|
"deactivated",
|
||||||
|
"访问被禁止",
|
||||||
|
"账户访问被禁止",
|
||||||
|
"访问受限",
|
||||||
|
"账户访问受限",
|
||||||
|
"authentication token has been invalidated",
|
||||||
|
"token has been invalidated",
|
||||||
|
"codex token 无效或已过期",
|
||||||
|
"validation_required",
|
||||||
|
"verify your account",
|
||||||
|
"需要验证",
|
||||||
|
"验证账号",
|
||||||
|
"验证身份",
|
||||||
|
];
|
||||||
|
|
||||||
|
const AUTO_REMOVABLE_ACCOUNT_STATE_CODES: &[&str] = &[
|
||||||
|
"account_banned",
|
||||||
|
"account_suspended",
|
||||||
|
"account_disabled",
|
||||||
|
"workspace_deactivated",
|
||||||
|
"account_forbidden",
|
||||||
|
];
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||||
|
pub struct PoolAccountState {
|
||||||
|
pub blocked: bool,
|
||||||
|
pub code: Option<String>,
|
||||||
|
pub label: Option<String>,
|
||||||
|
pub reason: Option<String>,
|
||||||
|
pub source: Option<String>,
|
||||||
|
pub recoverable: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct AccountStatusSnapshot {
|
||||||
|
pub code: String,
|
||||||
|
pub label: Option<String>,
|
||||||
|
pub reason: Option<String>,
|
||||||
|
pub blocked: bool,
|
||||||
|
pub source: Option<String>,
|
||||||
|
pub recoverable: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for AccountStatusSnapshot {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
code: "ok".to_string(),
|
||||||
|
label: None,
|
||||||
|
reason: None,
|
||||||
|
blocked: false,
|
||||||
|
source: None,
|
||||||
|
recoverable: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn clean_text(value: Option<&str>) -> Option<String> {
|
||||||
|
value
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn json_bool(value: Option<&Value>) -> bool {
|
||||||
|
match value {
|
||||||
|
Some(Value::Bool(value)) => *value,
|
||||||
|
Some(Value::Number(value)) => value.as_i64().is_some_and(|value| value != 0),
|
||||||
|
Some(Value::String(value)) => matches!(
|
||||||
|
value.trim().to_ascii_lowercase().as_str(),
|
||||||
|
"1" | "true" | "yes" | "y"
|
||||||
|
),
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn extract_reason(source: &serde_json::Map<String, Value>, fields: &[&str]) -> Option<String> {
|
||||||
|
fields
|
||||||
|
.iter()
|
||||||
|
.find_map(|field| source.get(*field).and_then(Value::as_str))
|
||||||
|
.and_then(|value| clean_text(Some(value)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn looks_like_workspace_deactivated(reason: Option<&str>) -> bool {
|
||||||
|
clean_text(reason)
|
||||||
|
.is_some_and(|value| value.to_ascii_lowercase().contains("deactivated_workspace"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn looks_like_account_verification(reason: &str) -> bool {
|
||||||
|
let lowered = reason.to_ascii_lowercase();
|
||||||
|
[
|
||||||
|
"validation_required",
|
||||||
|
"verify your account",
|
||||||
|
"需要验证",
|
||||||
|
"验证账号",
|
||||||
|
"验证身份",
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.any(|keyword| lowered.contains(keyword))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn classify_block_reason(reason: &str) -> (&'static str, &'static str) {
|
||||||
|
let lowered = reason.to_ascii_lowercase();
|
||||||
|
if [
|
||||||
|
"authentication token has been invalidated",
|
||||||
|
"token has been invalidated",
|
||||||
|
"codex token 无效或已过期",
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.any(|keyword| lowered.contains(keyword))
|
||||||
|
{
|
||||||
|
return ("oauth_expired", "Token 失效");
|
||||||
|
}
|
||||||
|
if looks_like_account_verification(reason) {
|
||||||
|
return ("account_verification", "需要验证");
|
||||||
|
}
|
||||||
|
if lowered.contains("deactivated_workspace") {
|
||||||
|
return ("workspace_deactivated", "工作区停用");
|
||||||
|
}
|
||||||
|
if [
|
||||||
|
"account has been disabled",
|
||||||
|
"account disabled",
|
||||||
|
"account has been deactivated",
|
||||||
|
"account_deactivated",
|
||||||
|
"account deactivated",
|
||||||
|
"organization has been disabled",
|
||||||
|
"organization_disabled",
|
||||||
|
"访问被禁止",
|
||||||
|
"账户访问被禁止",
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.any(|keyword| lowered.contains(keyword))
|
||||||
|
{
|
||||||
|
return ("account_disabled", "账号停用");
|
||||||
|
}
|
||||||
|
if ["account_forbidden", "forbidden", "访问受限", "账户访问受限"]
|
||||||
|
.iter()
|
||||||
|
.any(|keyword| lowered.contains(keyword))
|
||||||
|
{
|
||||||
|
return ("account_forbidden", "访问受限");
|
||||||
|
}
|
||||||
|
if [
|
||||||
|
"suspended",
|
||||||
|
"banned",
|
||||||
|
"account_block",
|
||||||
|
"account blocked",
|
||||||
|
"封禁",
|
||||||
|
"封号",
|
||||||
|
"被封",
|
||||||
|
"账户已封禁",
|
||||||
|
"账号异常",
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.any(|keyword| lowered.contains(keyword))
|
||||||
|
{
|
||||||
|
return ("account_suspended", "账号封禁");
|
||||||
|
}
|
||||||
|
("account_blocked", "账号异常")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_tagged_reason_line(line: &str) -> Option<(String, String)> {
|
||||||
|
let trimmed = line.trim();
|
||||||
|
if !trimmed.starts_with('[') {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let end = trimmed.find(']')?;
|
||||||
|
let tag = trimmed.get(1..end)?.trim();
|
||||||
|
if tag.is_empty() || !tag.chars().all(|ch| ch.is_ascii_uppercase() || ch == '_') {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let detail = trimmed
|
||||||
|
.get(end + 1..)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.trim()
|
||||||
|
.to_string();
|
||||||
|
Some((tag.to_string(), detail))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn extract_tagged_reason_sections(reason: &str) -> Vec<(String, String)> {
|
||||||
|
let mut sections = Vec::<(String, String)>::new();
|
||||||
|
let mut current_tag = None::<String>;
|
||||||
|
for line in reason.lines() {
|
||||||
|
if let Some((tag, detail)) = parse_tagged_reason_line(line) {
|
||||||
|
current_tag = Some(tag.clone());
|
||||||
|
if sections.iter().all(|(existing, _)| existing != &tag) {
|
||||||
|
sections.push((tag, detail));
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let continuation = line.trim();
|
||||||
|
if continuation.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(tag) = current_tag.as_ref() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let Some((_, detail)) = sections.iter_mut().find(|(existing, _)| existing == tag) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if !detail.is_empty() {
|
||||||
|
detail.push('\n');
|
||||||
|
}
|
||||||
|
detail.push_str(continuation);
|
||||||
|
}
|
||||||
|
sections
|
||||||
|
}
|
||||||
|
|
||||||
|
fn tagged_reason(reason: &str, tag: &str) -> Option<String> {
|
||||||
|
extract_tagged_reason_sections(reason)
|
||||||
|
.into_iter()
|
||||||
|
.find_map(|(candidate, detail)| (candidate == tag).then_some(detail))
|
||||||
|
.and_then(|detail| clean_text(Some(detail.as_str())).or(Some(detail)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn metadata_sources<'a>(
|
||||||
|
provider_type: Option<&str>,
|
||||||
|
upstream_metadata: Option<&'a Value>,
|
||||||
|
) -> Vec<&'a serde_json::Map<String, Value>> {
|
||||||
|
let mut sources = Vec::new();
|
||||||
|
let Some(root) = upstream_metadata.and_then(Value::as_object) else {
|
||||||
|
return sources;
|
||||||
|
};
|
||||||
|
|
||||||
|
let normalized_provider_type = provider_type
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(|value| value.to_ascii_lowercase());
|
||||||
|
|
||||||
|
if let Some(provider_type) = normalized_provider_type.as_deref() {
|
||||||
|
if let Some(bucket) = root.get(provider_type).and_then(Value::as_object) {
|
||||||
|
sources.push(bucket);
|
||||||
|
}
|
||||||
|
sources.push(root);
|
||||||
|
return sources;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut seen = BTreeSet::new();
|
||||||
|
for value in root.values() {
|
||||||
|
let Some(object) = value.as_object() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let pointer = object as *const serde_json::Map<String, Value> as usize;
|
||||||
|
if seen.insert(pointer) {
|
||||||
|
sources.push(object);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sources.push(root);
|
||||||
|
sources
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolve_from_metadata(
|
||||||
|
provider_type: Option<&str>,
|
||||||
|
upstream_metadata: Option<&Value>,
|
||||||
|
) -> Option<PoolAccountState> {
|
||||||
|
for source in metadata_sources(provider_type, upstream_metadata) {
|
||||||
|
if json_bool(source.get("is_banned")) {
|
||||||
|
let reason = extract_reason(
|
||||||
|
source,
|
||||||
|
&["ban_reason", "forbidden_reason", "reason", "message"],
|
||||||
|
)
|
||||||
|
.unwrap_or_else(|| "账号已封禁".to_string());
|
||||||
|
return Some(PoolAccountState {
|
||||||
|
blocked: true,
|
||||||
|
code: Some("account_banned".to_string()),
|
||||||
|
label: Some("账号封禁".to_string()),
|
||||||
|
reason: Some(reason),
|
||||||
|
source: Some("metadata".to_string()),
|
||||||
|
recoverable: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if json_bool(source.get("is_forbidden")) || json_bool(source.get("account_disabled")) {
|
||||||
|
let reason = extract_reason(
|
||||||
|
source,
|
||||||
|
&["forbidden_reason", "ban_reason", "reason", "message"],
|
||||||
|
);
|
||||||
|
if looks_like_workspace_deactivated(reason.as_deref()) {
|
||||||
|
return Some(PoolAccountState {
|
||||||
|
blocked: true,
|
||||||
|
code: Some("workspace_deactivated".to_string()),
|
||||||
|
label: Some("工作区停用".to_string()),
|
||||||
|
reason: Some(reason.unwrap_or_else(|| "工作区已停用".to_string())),
|
||||||
|
source: Some("metadata".to_string()),
|
||||||
|
recoverable: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return Some(PoolAccountState {
|
||||||
|
blocked: true,
|
||||||
|
code: Some("account_forbidden".to_string()),
|
||||||
|
label: Some("访问受限".to_string()),
|
||||||
|
reason: Some(reason.unwrap_or_else(|| "账号访问受限".to_string())),
|
||||||
|
source: Some("metadata".to_string()),
|
||||||
|
recoverable: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolve_from_oauth_invalid_reason(reason: Option<&str>) -> Option<PoolAccountState> {
|
||||||
|
let text = clean_text(reason)?;
|
||||||
|
|
||||||
|
if text.starts_with(OAUTH_ACCOUNT_BLOCK_PREFIX) {
|
||||||
|
let cleaned = clean_text(Some(text.trim_start_matches(OAUTH_ACCOUNT_BLOCK_PREFIX)))
|
||||||
|
.unwrap_or_else(|| "账号异常".to_string());
|
||||||
|
let (code, label) = classify_block_reason(&cleaned);
|
||||||
|
return Some(PoolAccountState {
|
||||||
|
blocked: true,
|
||||||
|
code: Some(code.to_string()),
|
||||||
|
label: Some(label.to_string()),
|
||||||
|
reason: Some(cleaned),
|
||||||
|
source: Some("oauth_invalid".to_string()),
|
||||||
|
recoverable: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if text.starts_with(OAUTH_EXPIRED_PREFIX) {
|
||||||
|
let cleaned = clean_text(Some(text.trim_start_matches(OAUTH_EXPIRED_PREFIX)))
|
||||||
|
.unwrap_or_else(|| "OAuth Token 已过期且无法续期".to_string());
|
||||||
|
return Some(PoolAccountState {
|
||||||
|
blocked: true,
|
||||||
|
code: Some("oauth_expired".to_string()),
|
||||||
|
label: Some("Token 失效".to_string()),
|
||||||
|
reason: Some(cleaned),
|
||||||
|
source: Some("oauth_invalid".to_string()),
|
||||||
|
recoverable: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if text.starts_with(OAUTH_REFRESH_FAILED_PREFIX) {
|
||||||
|
let cleaned = clean_text(Some(text.trim_start_matches(OAUTH_REFRESH_FAILED_PREFIX)))
|
||||||
|
.unwrap_or_else(|| "OAuth Token 续期失败".to_string());
|
||||||
|
return Some(PoolAccountState {
|
||||||
|
blocked: false,
|
||||||
|
code: Some("oauth_refresh_failed".to_string()),
|
||||||
|
label: Some("续期失败".to_string()),
|
||||||
|
reason: Some(cleaned),
|
||||||
|
source: Some("oauth_refresh".to_string()),
|
||||||
|
recoverable: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if text.starts_with(OAUTH_REQUEST_FAILED_PREFIX) {
|
||||||
|
let cleaned = clean_text(Some(text.trim_start_matches(OAUTH_REQUEST_FAILED_PREFIX)))
|
||||||
|
.unwrap_or_else(|| "账号状态检查失败".to_string());
|
||||||
|
return Some(PoolAccountState {
|
||||||
|
blocked: false,
|
||||||
|
code: Some("oauth_request_failed".to_string()),
|
||||||
|
label: Some("请求失败".to_string()),
|
||||||
|
reason: Some(cleaned),
|
||||||
|
source: Some("oauth_request".to_string()),
|
||||||
|
recoverable: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if text.starts_with('[') {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let lowered = text.to_ascii_lowercase();
|
||||||
|
if ACCOUNT_BLOCK_REASON_KEYWORDS
|
||||||
|
.iter()
|
||||||
|
.any(|keyword| lowered.contains(keyword))
|
||||||
|
{
|
||||||
|
let (code, label) = classify_block_reason(&text);
|
||||||
|
return Some(PoolAccountState {
|
||||||
|
blocked: true,
|
||||||
|
code: Some(code.to_string()),
|
||||||
|
label: Some(label.to_string()),
|
||||||
|
reason: Some(text),
|
||||||
|
source: Some("oauth_invalid".to_string()),
|
||||||
|
recoverable: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn resolve_pool_account_state(
|
||||||
|
provider_type: Option<&str>,
|
||||||
|
upstream_metadata: Option<&Value>,
|
||||||
|
oauth_invalid_reason: Option<&str>,
|
||||||
|
) -> PoolAccountState {
|
||||||
|
resolve_from_metadata(provider_type, upstream_metadata)
|
||||||
|
.or_else(|| resolve_from_oauth_invalid_reason(oauth_invalid_reason))
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn resolve_account_status_snapshot(
|
||||||
|
provider_type: Option<&str>,
|
||||||
|
upstream_metadata: Option<&Value>,
|
||||||
|
oauth_invalid_reason: Option<&str>,
|
||||||
|
) -> AccountStatusSnapshot {
|
||||||
|
if let Some(metadata_state) = resolve_from_metadata(provider_type, upstream_metadata) {
|
||||||
|
return AccountStatusSnapshot {
|
||||||
|
code: metadata_state
|
||||||
|
.code
|
||||||
|
.unwrap_or_else(|| "account_blocked".to_string()),
|
||||||
|
label: metadata_state.label,
|
||||||
|
reason: metadata_state.reason,
|
||||||
|
blocked: metadata_state.blocked,
|
||||||
|
source: metadata_state.source,
|
||||||
|
recoverable: metadata_state.recoverable,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some(text) = clean_text(oauth_invalid_reason) else {
|
||||||
|
return AccountStatusSnapshot::default();
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Some(cleaned) = tagged_reason(&text, "ACCOUNT_BLOCK") {
|
||||||
|
let reason = if cleaned.is_empty() {
|
||||||
|
"账号异常".to_string()
|
||||||
|
} else {
|
||||||
|
cleaned
|
||||||
|
};
|
||||||
|
let (code, label) = classify_block_reason(&reason);
|
||||||
|
return AccountStatusSnapshot {
|
||||||
|
code: code.to_string(),
|
||||||
|
label: Some(label.to_string()),
|
||||||
|
reason: Some(reason),
|
||||||
|
blocked: true,
|
||||||
|
source: Some("oauth_invalid".to_string()),
|
||||||
|
recoverable: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if text.starts_with('[') {
|
||||||
|
return AccountStatusSnapshot::default();
|
||||||
|
}
|
||||||
|
|
||||||
|
let lowered = text.to_ascii_lowercase();
|
||||||
|
if ACCOUNT_BLOCK_REASON_KEYWORDS
|
||||||
|
.iter()
|
||||||
|
.any(|keyword| lowered.contains(keyword))
|
||||||
|
{
|
||||||
|
let (code, label) = classify_block_reason(&text);
|
||||||
|
return AccountStatusSnapshot {
|
||||||
|
code: code.to_string(),
|
||||||
|
label: Some(label.to_string()),
|
||||||
|
reason: Some(text),
|
||||||
|
blocked: true,
|
||||||
|
source: Some("oauth_invalid".to_string()),
|
||||||
|
recoverable: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
AccountStatusSnapshot::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn should_auto_remove_account_state(state: &PoolAccountState) -> bool {
|
||||||
|
state.blocked
|
||||||
|
&& !state.recoverable
|
||||||
|
&& state.code.as_deref().is_some_and(|code| {
|
||||||
|
AUTO_REMOVABLE_ACCOUNT_STATE_CODES
|
||||||
|
.iter()
|
||||||
|
.any(|candidate| code.eq_ignore_ascii_case(candidate))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn account_state_indicates_known_ban(state: &PoolAccountState) -> bool {
|
||||||
|
if !state.blocked {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if should_auto_remove_account_state(state) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if state
|
||||||
|
.code
|
||||||
|
.as_deref()
|
||||||
|
.is_some_and(|code| matches!(code, "account_verification" | "account_blocked"))
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
state.code.as_deref().is_some_and(reason_indicates_ban)
|
||||||
|
|| state.reason.as_deref().is_some_and(reason_indicates_ban)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reason_indicates_ban(reason: &str) -> bool {
|
||||||
|
let normalized = reason.trim().to_ascii_lowercase();
|
||||||
|
!normalized.is_empty()
|
||||||
|
&& [
|
||||||
|
"banned",
|
||||||
|
"forbidden",
|
||||||
|
"blocked",
|
||||||
|
"suspend",
|
||||||
|
"deactivated",
|
||||||
|
"disabled",
|
||||||
|
"verification",
|
||||||
|
"workspace",
|
||||||
|
"受限",
|
||||||
|
"封",
|
||||||
|
"禁",
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.any(|hint| normalized.contains(hint))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::{
|
||||||
|
account_state_indicates_known_ban, resolve_account_status_snapshot,
|
||||||
|
resolve_pool_account_state, should_auto_remove_account_state,
|
||||||
|
};
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolves_workspace_deactivated_from_metadata() {
|
||||||
|
let state = resolve_pool_account_state(
|
||||||
|
Some("codex"),
|
||||||
|
Some(&json!({
|
||||||
|
"codex": {
|
||||||
|
"account_disabled": true,
|
||||||
|
"reason": "deactivated_workspace"
|
||||||
|
}
|
||||||
|
})),
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(state.blocked);
|
||||||
|
assert_eq!(state.code.as_deref(), Some("workspace_deactivated"));
|
||||||
|
assert_eq!(state.label.as_deref(), Some("工作区停用"));
|
||||||
|
assert!(should_auto_remove_account_state(&state));
|
||||||
|
assert!(account_state_indicates_known_ban(&state));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolves_refresh_failed_as_recoverable_pool_state() {
|
||||||
|
let state = resolve_pool_account_state(
|
||||||
|
Some("codex"),
|
||||||
|
None,
|
||||||
|
Some("[REFRESH_FAILED] Token 续期失败 (401): refresh_token 已失效"),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(!state.blocked);
|
||||||
|
assert!(state.recoverable);
|
||||||
|
assert_eq!(state.code.as_deref(), Some("oauth_refresh_failed"));
|
||||||
|
assert!(!should_auto_remove_account_state(&state));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn account_snapshot_ignores_refresh_failed_reason() {
|
||||||
|
let snapshot = resolve_account_status_snapshot(
|
||||||
|
Some("codex"),
|
||||||
|
None,
|
||||||
|
Some("[REFRESH_FAILED] Token 续期失败 (401): refresh_token 已失效"),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(snapshot.code, "ok");
|
||||||
|
assert!(!snapshot.blocked);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn account_snapshot_detects_account_block_and_verification() {
|
||||||
|
let snapshot = resolve_account_status_snapshot(
|
||||||
|
Some("codex"),
|
||||||
|
None,
|
||||||
|
Some("[ACCOUNT_BLOCK] verify your account before continuing"),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(snapshot.code, "account_verification");
|
||||||
|
assert_eq!(snapshot.label.as_deref(), Some("需要验证"));
|
||||||
|
assert!(snapshot.blocked);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn verification_state_is_not_auto_removed() {
|
||||||
|
let state = resolve_pool_account_state(
|
||||||
|
Some("codex"),
|
||||||
|
None,
|
||||||
|
Some("[ACCOUNT_BLOCK] verify your account before continuing"),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(state.blocked);
|
||||||
|
assert_eq!(state.code.as_deref(), Some("account_verification"));
|
||||||
|
assert!(!should_auto_remove_account_state(&state));
|
||||||
|
assert!(account_state_indicates_known_ban(&state));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -373,6 +373,7 @@ pub struct CandidateRuntimeSelectabilityInput<'a> {
|
|||||||
pub cached_affinity_target: Option<&'a crate::SchedulerAffinityTarget>,
|
pub cached_affinity_target: Option<&'a crate::SchedulerAffinityTarget>,
|
||||||
pub provider_quota_blocks_requests: bool,
|
pub provider_quota_blocks_requests: bool,
|
||||||
pub account_quota_exhausted: bool,
|
pub account_quota_exhausted: bool,
|
||||||
|
pub oauth_invalid: bool,
|
||||||
pub rpm_reset_at: Option<u64>,
|
pub rpm_reset_at: Option<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -394,6 +395,7 @@ pub fn candidate_runtime_skip_reason_with_state(
|
|||||||
cached_affinity_target,
|
cached_affinity_target,
|
||||||
provider_quota_blocks_requests,
|
provider_quota_blocks_requests,
|
||||||
account_quota_exhausted,
|
account_quota_exhausted,
|
||||||
|
oauth_invalid,
|
||||||
rpm_reset_at,
|
rpm_reset_at,
|
||||||
} = input;
|
} = input;
|
||||||
|
|
||||||
@@ -403,6 +405,9 @@ pub fn candidate_runtime_skip_reason_with_state(
|
|||||||
if account_quota_exhausted {
|
if account_quota_exhausted {
|
||||||
return Some("account_quota_exhausted");
|
return Some("account_quota_exhausted");
|
||||||
}
|
}
|
||||||
|
if oauth_invalid {
|
||||||
|
return Some("oauth_invalid");
|
||||||
|
}
|
||||||
if crate::is_candidate_in_recent_failure_cooldown(
|
if crate::is_candidate_in_recent_failure_cooldown(
|
||||||
recent_candidates,
|
recent_candidates,
|
||||||
candidate.provider_id.as_str(),
|
candidate.provider_id.as_str(),
|
||||||
@@ -816,6 +821,7 @@ mod tests {
|
|||||||
cached_affinity_target: None,
|
cached_affinity_target: None,
|
||||||
provider_quota_blocks_requests: false,
|
provider_quota_blocks_requests: false,
|
||||||
account_quota_exhausted: false,
|
account_quota_exhausted: false,
|
||||||
|
oauth_invalid: false,
|
||||||
rpm_reset_at: None,
|
rpm_reset_at: None,
|
||||||
},
|
},
|
||||||
));
|
));
|
||||||
@@ -835,6 +841,7 @@ mod tests {
|
|||||||
cached_affinity_target: None,
|
cached_affinity_target: None,
|
||||||
provider_quota_blocks_requests: false,
|
provider_quota_blocks_requests: false,
|
||||||
account_quota_exhausted: false,
|
account_quota_exhausted: false,
|
||||||
|
oauth_invalid: false,
|
||||||
rpm_reset_at: None,
|
rpm_reset_at: None,
|
||||||
},
|
},
|
||||||
));
|
));
|
||||||
@@ -848,6 +855,7 @@ mod tests {
|
|||||||
cached_affinity_target: None,
|
cached_affinity_target: None,
|
||||||
provider_quota_blocks_requests: true,
|
provider_quota_blocks_requests: true,
|
||||||
account_quota_exhausted: false,
|
account_quota_exhausted: false,
|
||||||
|
oauth_invalid: false,
|
||||||
rpm_reset_at: None,
|
rpm_reset_at: None,
|
||||||
},
|
},
|
||||||
));
|
));
|
||||||
@@ -865,6 +873,25 @@ mod tests {
|
|||||||
cached_affinity_target: None,
|
cached_affinity_target: None,
|
||||||
provider_quota_blocks_requests: false,
|
provider_quota_blocks_requests: false,
|
||||||
account_quota_exhausted: true,
|
account_quota_exhausted: true,
|
||||||
|
oauth_invalid: false,
|
||||||
|
rpm_reset_at: None,
|
||||||
|
},
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn candidate_selectability_rejects_oauth_invalid_keys() {
|
||||||
|
assert!(!candidate_is_selectable_with_runtime_state(
|
||||||
|
CandidateRuntimeSelectabilityInput {
|
||||||
|
candidate: &sample_candidate("1", None),
|
||||||
|
recent_candidates: &[],
|
||||||
|
provider_concurrent_limits: &BTreeMap::new(),
|
||||||
|
provider_key_rpm_states: &BTreeMap::new(),
|
||||||
|
now_unix_secs: 100,
|
||||||
|
cached_affinity_target: None,
|
||||||
|
provider_quota_blocks_requests: false,
|
||||||
|
account_quota_exhausted: false,
|
||||||
|
oauth_invalid: true,
|
||||||
rpm_reset_at: None,
|
rpm_reset_at: None,
|
||||||
},
|
},
|
||||||
));
|
));
|
||||||
|
|||||||
Reference in New Issue
Block a user