fix(kiro,pool,model): 对齐 Kiro 管理链路并修复全局模型删除行为 (#305)

* feat(pool): 号池支持跳过额度耗尽账号

- 新增 pool_advanced.skip_exhausted_accounts 开关及高级设置 UI, 默认关闭并兼容旧配置
- 为 Codex/Kiro 增加额度耗尽判定, 接入请求侧候选跳过并新增 account_quota_exhausted skip reason
- 号池列表将额度耗尽账号标记为 blocked/额度耗尽, 并补充前后端相关测试

* fix(kiro): 对齐账号管理与 provider-query 的 Rust 行为

- 修复 Kiro 单条导入误走 import-refresh-token 的前端分流, 并为误用路径返回明确错误提示
- 为 Kiro 导入与本地请求链补齐 bearer 兼容, 同步放开账号启停等 Key 更新操作的 auth_type 校验
- 实现 Kiro provider-query 本地模型测试与 failover 执行链, 并修复结果弹窗在无 trace 时无法展示 attempts/响应体的问题

* fix(model): 删除全局模型时级联清理关联提供商模型

- 对齐 Python 版本删除逻辑, GlobalModel 删除前先在事务内清理关联的 Provider Model 记录
- 修复已绑定 Provider 的模型在 Rust SQL 仓库下会被外键约束拦住、无法正常删除的问题
- 增加管理端回归测试, 覆盖绑定 Provider Model 的 GlobalModel 删除场景

* fix(kiro,ci): 恢复 Kiro OAuth 持久化并修复 Rust CI

* Fix oauth-managed provider key semantics

---------

Co-authored-by: fawney19 <elky0401@gmail.com>
This commit is contained in:
Entropy.Xu
2026-04-17 12:57:06 +08:00
committed by GitHub
parent 96a25d058b
commit ac1a126756
43 changed files with 2909 additions and 168 deletions

View File

@@ -10,6 +10,7 @@ use super::shared::{
}; };
use crate::handlers::admin::provider::shared::paths::admin_provider_oauth_complete_key_id; use crate::handlers::admin::provider::shared::paths::admin_provider_oauth_complete_key_id;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext}; use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::provider_key_auth::provider_key_is_oauth_managed;
use crate::GatewayError; use crate::GatewayError;
use axum::{ use axum::{
body::{Body, Bytes}, body::{Body, Bytes},
@@ -76,12 +77,6 @@ pub(super) async fn handle_admin_provider_oauth_complete_key(
"Key 不存在", "Key 不存在",
)); ));
}; };
if !key.auth_type.eq_ignore_ascii_case("oauth") {
return Ok(build_internal_control_error_response(
http::StatusCode::BAD_REQUEST,
"该 Key 不是 oauth 认证类型",
));
}
if !state_data.provider_id.trim().is_empty() && state_data.provider_id != key.provider_id { if !state_data.provider_id.trim().is_empty() && state_data.provider_id != key.provider_id {
return Ok(build_internal_control_error_response( return Ok(build_internal_control_error_response(
http::StatusCode::BAD_REQUEST, http::StatusCode::BAD_REQUEST,
@@ -102,6 +97,12 @@ pub(super) async fn handle_admin_provider_oauth_complete_key(
)); ));
}; };
let provider_type = provider.provider_type.trim().to_ascii_lowercase(); let provider_type = provider.provider_type.trim().to_ascii_lowercase();
if !provider_key_is_oauth_managed(&key, provider_type.as_str()) {
return Ok(build_internal_control_error_response(
http::StatusCode::BAD_REQUEST,
"该 Key 不是 OAuth 管理账号",
));
}
if !is_fixed_provider_type_for_provider_oauth(&provider_type) { if !is_fixed_provider_type_for_provider_oauth(&provider_type) {
return Ok(build_internal_control_error_response( return Ok(build_internal_control_error_response(
http::StatusCode::BAD_REQUEST, http::StatusCode::BAD_REQUEST,

View File

@@ -94,6 +94,12 @@ pub(super) async fn handle_admin_provider_oauth_import_refresh_token(
"该 Provider 不是固定类型,无法使用 provider-oauth", "该 Provider 不是固定类型,无法使用 provider-oauth",
)); ));
} }
if provider_type == "kiro" {
return Ok(build_internal_control_error_response(
http::StatusCode::BAD_REQUEST,
"Kiro 不支持单条 Refresh Token 导入,请使用批量导入或设备授权。",
));
}
let Some(template) = admin_provider_oauth_template(&provider_type) else { let Some(template) = admin_provider_oauth_template(&provider_type) else {
return Ok(build_admin_provider_oauth_backend_unavailable_response()); return Ok(build_admin_provider_oauth_backend_unavailable_response());
}; };

View File

@@ -4,6 +4,7 @@ use super::helpers::{self, RefreshDispatch, RefreshRequestContext};
use super::response; use super::response;
use crate::handlers::admin::provider::shared::paths::admin_provider_oauth_refresh_key_id; use crate::handlers::admin::provider::shared::paths::admin_provider_oauth_refresh_key_id;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext}; use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::provider_key_auth::provider_key_is_oauth_managed;
use crate::GatewayError; use crate::GatewayError;
use axum::http; use axum::http;
@@ -28,13 +29,6 @@ pub(super) async fn parse_admin_provider_oauth_refresh_request(
"Key 不存在", "Key 不存在",
))); )));
}; };
if !key.auth_type.eq_ignore_ascii_case("oauth") {
return Ok(RefreshDispatch::Respond(response::control_error_response(
http::StatusCode::BAD_REQUEST,
"该 Key 不是 oauth 认证类型",
)));
}
let Some(encrypted_auth_config) = key.encrypted_auth_config.as_deref() else { let Some(encrypted_auth_config) = key.encrypted_auth_config.as_deref() else {
return Ok(RefreshDispatch::Respond(response::control_error_response( return Ok(RefreshDispatch::Respond(response::control_error_response(
http::StatusCode::BAD_REQUEST, http::StatusCode::BAD_REQUEST,
@@ -69,6 +63,12 @@ pub(super) async fn parse_admin_provider_oauth_refresh_request(
))); )));
}; };
let provider_type = provider.provider_type.trim().to_ascii_lowercase(); let provider_type = provider.provider_type.trim().to_ascii_lowercase();
if !provider_key_is_oauth_managed(&key, provider_type.as_str()) {
return Ok(RefreshDispatch::Respond(response::control_error_response(
http::StatusCode::BAD_REQUEST,
"该 Key 不是 OAuth 管理账号",
)));
}
if !is_fixed_provider_type_for_provider_oauth(&provider_type) { if !is_fixed_provider_type_for_provider_oauth(&provider_type) {
return Ok(RefreshDispatch::Respond(response::control_error_response( return Ok(RefreshDispatch::Respond(response::control_error_response(
http::StatusCode::BAD_REQUEST, http::StatusCode::BAD_REQUEST,

View File

@@ -8,6 +8,7 @@ use crate::handlers::admin::provider::shared::paths::{
admin_provider_oauth_start_key_id, admin_provider_oauth_start_provider_id, admin_provider_oauth_start_key_id, admin_provider_oauth_start_provider_id,
}; };
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext}; use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::provider_key_auth::provider_key_is_oauth_managed;
use crate::GatewayError; use crate::GatewayError;
use axum::{ use axum::{
body::Body, body::Body,
@@ -37,13 +38,6 @@ pub(super) async fn handle_admin_provider_oauth_start_key(
"Key 不存在", "Key 不存在",
)); ));
}; };
if !key.auth_type.eq_ignore_ascii_case("oauth") {
return Ok(build_internal_control_error_response(
http::StatusCode::BAD_REQUEST,
"该 Key 不是 oauth 认证类型",
));
}
let provider_id = key.provider_id.clone(); let provider_id = key.provider_id.clone();
let provider = state let provider = state
.read_provider_catalog_providers_by_ids(std::slice::from_ref(&provider_id)) .read_provider_catalog_providers_by_ids(std::slice::from_ref(&provider_id))
@@ -57,6 +51,12 @@ pub(super) async fn handle_admin_provider_oauth_start_key(
)); ));
}; };
let provider_type = provider.provider_type.trim().to_ascii_lowercase(); let provider_type = provider.provider_type.trim().to_ascii_lowercase();
if !provider_key_is_oauth_managed(&key, provider_type.as_str()) {
return Ok(build_internal_control_error_response(
http::StatusCode::BAD_REQUEST,
"该 Key 不是 OAuth 管理账号",
));
}
if !is_fixed_provider_type_for_provider_oauth(&provider_type) { if !is_fixed_provider_type_for_provider_oauth(&provider_type) {
return Ok(build_internal_control_error_response( return Ok(build_internal_control_error_response(
http::StatusCode::BAD_REQUEST, http::StatusCode::BAD_REQUEST,

View File

@@ -1,4 +1,5 @@
use crate::handlers::admin::request::AdminAppState; use crate::handlers::admin::request::AdminAppState;
use crate::provider_key_auth::provider_key_is_oauth_managed;
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey; use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
fn normalize_codex_plan_group_for_provider_oauth( fn normalize_codex_plan_group_for_provider_oauth(
@@ -141,8 +142,13 @@ pub(crate) async fn find_duplicate_provider_oauth_key(
.await .await
.map_err(|err| format!("{err:?}"))?; .map_err(|err| format!("{err:?}"))?;
let provider_type = auth_config
.get("provider_type")
.and_then(serde_json::Value::as_str)
.unwrap_or_default()
.to_string();
for existing_key in existing_keys.into_iter().filter(|key| { for existing_key in existing_keys.into_iter().filter(|key| {
key.auth_type.trim().eq_ignore_ascii_case("oauth") provider_key_is_oauth_managed(key, provider_type.as_str())
&& exclude_key_id.is_none_or(|exclude| key.id != exclude) && exclude_key_id.is_none_or(|exclude| key.id != exclude)
}) { }) {
let Some(existing_auth_config) = state.parse_catalog_auth_config_json(&existing_key) else { let Some(existing_auth_config) = state.parse_catalog_auth_config_json(&existing_key) else {

View File

@@ -18,6 +18,7 @@ use super::shared::{
should_auto_remove_structured_reason, ProviderQuotaExecutionOutcome, should_auto_remove_structured_reason, ProviderQuotaExecutionOutcome,
}; };
use crate::handlers::admin::request::AdminAppState; use crate::handlers::admin::request::AdminAppState;
use crate::provider_key_auth::provider_key_is_oauth_managed;
use crate::GatewayError; use crate::GatewayError;
use aether_data_contracts::repository::provider_catalog::{ use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider, StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
@@ -55,11 +56,12 @@ pub(crate) async fn refresh_codex_provider_quota_locally(
} }
}; };
let resolved_oauth_auth = if key.auth_type.trim().eq_ignore_ascii_case("oauth") { let resolved_oauth_auth =
state.resolve_local_oauth_header_auth(&transport).await? if provider_key_is_oauth_managed(&key, provider.provider_type.as_str()) {
} else { state.resolve_local_oauth_header_auth(&transport).await?
None } else {
}; None
};
let headers = match build_codex_refresh_headers(&transport, resolved_oauth_auth) { let headers = match build_codex_refresh_headers(&transport, resolved_oauth_auth) {
Ok(headers) => headers, Ok(headers) => headers,

View File

@@ -2,6 +2,7 @@ use super::quota::antigravity::refresh_antigravity_provider_quota_locally;
use super::quota::codex::refresh_codex_provider_quota_locally; use super::quota::codex::refresh_codex_provider_quota_locally;
use super::quota::kiro::refresh_kiro_provider_quota_locally; use super::quota::kiro::refresh_kiro_provider_quota_locally;
use crate::handlers::admin::request::AdminAppState; use crate::handlers::admin::request::AdminAppState;
use crate::provider_key_auth::provider_key_is_oauth_managed;
use crate::GatewayError; use crate::GatewayError;
use aether_data_contracts::repository::provider_catalog::{ use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogProvider, StoredProviderCatalogEndpoint, StoredProviderCatalogProvider,
@@ -85,6 +86,9 @@ pub(crate) async fn refresh_provider_oauth_account_state_after_update(
else { else {
return Ok((false, None)); return Ok((false, None));
}; };
if provider_type == "kiro" && !provider_key_is_oauth_managed(&key, provider_type.as_str()) {
return Ok((false, None));
}
let payload = match provider_type.as_str() { let payload = match provider_type.as_str() {
"codex" => { "codex" => {

View File

@@ -53,6 +53,7 @@ pub(crate) fn admin_provider_pool_config(
let Some(pool_advanced) = raw_pool_advanced.as_object() else { let Some(pool_advanced) = raw_pool_advanced.as_object() else {
return Some(AdminProviderPoolConfig { return Some(AdminProviderPoolConfig {
lru_enabled: false, lru_enabled: false,
skip_exhausted_accounts: false,
cost_window_seconds: 18_000, cost_window_seconds: 18_000,
cost_limit_per_key_tokens: None, cost_limit_per_key_tokens: None,
}); });
@@ -60,6 +61,10 @@ pub(crate) fn admin_provider_pool_config(
Some(AdminProviderPoolConfig { Some(AdminProviderPoolConfig {
lru_enabled: admin_provider_pool_lru_enabled(pool_advanced), lru_enabled: admin_provider_pool_lru_enabled(pool_advanced),
skip_exhausted_accounts: pool_advanced
.get("skip_exhausted_accounts")
.and_then(Value::as_bool)
.unwrap_or(false),
cost_window_seconds: pool_advanced cost_window_seconds: pool_advanced
.get("cost_window_seconds") .get("cost_window_seconds")
.and_then(json_u64) .and_then(json_u64)
@@ -70,3 +75,57 @@ pub(crate) fn admin_provider_pool_config(
.and_then(json_u64), .and_then(json_u64),
}) })
} }
#[cfg(test)]
mod tests {
use super::admin_provider_pool_config;
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogProvider;
use serde_json::json;
fn sample_provider(config: serde_json::Value) -> StoredProviderCatalogProvider {
StoredProviderCatalogProvider::new(
"provider-1".to_string(),
"provider-1".to_string(),
Some("https://example.com".to_string()),
"codex".to_string(),
)
.expect("provider should build")
.with_transport_fields(
true,
false,
false,
None,
None,
None,
None,
None,
Some(config),
)
}
#[test]
fn defaults_skip_exhausted_accounts_to_false() {
let provider = sample_provider(json!({ "pool_advanced": {} }));
let config = admin_provider_pool_config(&provider).expect("pool config should exist");
assert!(!config.skip_exhausted_accounts);
}
#[test]
fn parses_skip_exhausted_accounts_from_pool_advanced() {
let provider = sample_provider(json!({
"pool_advanced": {
"skip_exhausted_accounts": true,
"lru_enabled": true,
"cost_window_seconds": 7200,
"cost_limit_per_key_tokens": 12000
}
}));
let config = admin_provider_pool_config(&provider).expect("pool config should exist");
assert!(config.skip_exhausted_accounts);
assert!(config.lru_enabled);
assert_eq!(config.cost_window_seconds, 7200);
assert_eq!(config.cost_limit_per_key_tokens, Some(12_000));
}
}

View File

@@ -3,6 +3,8 @@ use crate::handlers::admin::provider::shared::support::{
}; };
use crate::handlers::admin::request::AdminAppState; use crate::handlers::admin::request::AdminAppState;
use crate::handlers::admin::shared::{provider_key_status_snapshot_payload, unix_secs_to_rfc3339}; use crate::handlers::admin::shared::{provider_key_status_snapshot_payload, unix_secs_to_rfc3339};
use crate::provider_key_auth::provider_key_auth_semantics;
use aether_admin::provider::pool as admin_provider_pool_pure;
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey; use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
use serde_json::json; use serde_json::json;
@@ -122,10 +124,11 @@ fn admin_pool_normalize_oauth_plan_type(value: &str, provider_type: &str) -> Opt
} }
fn admin_pool_derive_oauth_expires_at( fn admin_pool_derive_oauth_expires_at(
provider_type: &str,
key: &StoredProviderCatalogKey, key: &StoredProviderCatalogKey,
auth_config: Option<&serde_json::Map<String, serde_json::Value>>, auth_config: Option<&serde_json::Map<String, serde_json::Value>>,
) -> Option<u64> { ) -> Option<u64> {
if !key.auth_type.trim().eq_ignore_ascii_case("oauth") { if !provider_key_auth_semantics(key, provider_type).oauth_managed() {
return None; return None;
} }
@@ -144,7 +147,7 @@ fn admin_pool_derive_oauth_plan_type(
provider_type: &str, provider_type: &str,
auth_config: Option<&serde_json::Map<String, serde_json::Value>>, auth_config: Option<&serde_json::Map<String, serde_json::Value>>,
) -> Option<String> { ) -> Option<String> {
if !key.auth_type.trim().eq_ignore_ascii_case("oauth") { if !provider_key_auth_semantics(key, provider_type).oauth_managed() {
return None; return None;
} }
@@ -551,6 +554,7 @@ fn admin_pool_scheduling_payload(
cooldown_ttl_seconds: Option<u64>, cooldown_ttl_seconds: Option<u64>,
health_score: f64, health_score: f64,
circuit_breaker_open: bool, circuit_breaker_open: bool,
account_quota_exhausted: bool,
) -> (String, String, String, Vec<serde_json::Value>) { ) -> (String, String, String, Vec<serde_json::Value>) {
if !key.is_active { if !key.is_active {
return ( return (
@@ -567,6 +571,21 @@ fn admin_pool_scheduling_payload(
})], })],
); );
} }
if account_quota_exhausted {
return (
"blocked".to_string(),
"account_quota_exhausted".to_string(),
"额度耗尽".to_string(),
vec![json!({
"code": "account_quota_exhausted",
"label": "额度耗尽",
"blocking": true,
"source": "quota",
"ttl_seconds": serde_json::Value::Null,
"detail": serde_json::Value::Null,
})],
);
}
if let Some(reason) = cooldown_reason { if let Some(reason) = cooldown_reason {
return ( return (
"degraded".to_string(), "degraded".to_string(),
@@ -632,6 +651,9 @@ pub(super) fn build_admin_pool_key_payload(
.and_then(|_| runtime.cooldown_ttl_by_key.get(&key.id).copied()); .and_then(|_| runtime.cooldown_ttl_by_key.get(&key.id).copied());
let health_score = admin_pool_health_score(key); let health_score = admin_pool_health_score(key);
let circuit_breaker_open = admin_pool_circuit_breaker_open(key); let circuit_breaker_open = admin_pool_circuit_breaker_open(key);
let auth_semantics = provider_key_auth_semantics(key, provider_type);
let account_quota_exhausted = pool_config.is_some_and(|config| config.skip_exhausted_accounts)
&& admin_provider_pool_pure::admin_pool_key_account_quota_exhausted(key, provider_type);
let (scheduling_status, scheduling_reason, scheduling_label, scheduling_reasons) = let (scheduling_status, scheduling_reason, scheduling_label, scheduling_reasons) =
admin_pool_scheduling_payload( admin_pool_scheduling_payload(
key, key,
@@ -639,9 +661,11 @@ pub(super) fn build_admin_pool_key_payload(
cooldown_ttl_seconds, cooldown_ttl_seconds,
health_score, health_score,
circuit_breaker_open, circuit_breaker_open,
account_quota_exhausted,
); );
let auth_config = state.parse_catalog_auth_config_json(key); let auth_config = state.parse_catalog_auth_config_json(key);
let oauth_expires_at = admin_pool_derive_oauth_expires_at(key, auth_config.as_ref()); let oauth_expires_at =
admin_pool_derive_oauth_expires_at(provider_type, key, auth_config.as_ref());
let oauth_plan_type = let oauth_plan_type =
admin_pool_derive_oauth_plan_type(key, provider_type, auth_config.as_ref()); admin_pool_derive_oauth_plan_type(key, provider_type, auth_config.as_ref());
let status_snapshot = provider_key_status_snapshot_payload(key); let status_snapshot = provider_key_status_snapshot_payload(key);
@@ -656,15 +680,29 @@ pub(super) fn build_admin_pool_key_payload(
.and_then(serde_json::Value::as_object); .and_then(serde_json::Value::as_object);
let quota_updated_at = let quota_updated_at =
admin_pool_json_to_u64(quota_snapshot.and_then(|item| item.get("updated_at"))); admin_pool_json_to_u64(quota_snapshot.and_then(|item| item.get("updated_at")));
let oauth_invalid_at = let oauth_invalid_at = if auth_semantics.can_show_oauth_metadata() {
admin_pool_json_to_u64(oauth_snapshot.and_then(|item| item.get("invalid_at"))) admin_pool_json_to_u64(oauth_snapshot.and_then(|item| item.get("invalid_at")))
.or(key.oauth_invalid_at_unix_secs); .or(key.oauth_invalid_at_unix_secs)
let oauth_account_id = admin_pool_trimmed_string_from_map(auth_config.as_ref(), "account_id"); } else {
let oauth_account_name = None
admin_pool_trimmed_string_from_map(auth_config.as_ref(), "account_name"); };
let oauth_account_user_id = let oauth_account_id = auth_semantics
admin_pool_trimmed_string_from_map(auth_config.as_ref(), "account_user_id"); .can_show_oauth_metadata()
let oauth_organizations = admin_pool_oauth_organizations(auth_config.as_ref()); .then(|| admin_pool_trimmed_string_from_map(auth_config.as_ref(), "account_id"))
.flatten();
let oauth_account_name = auth_semantics
.can_show_oauth_metadata()
.then(|| admin_pool_trimmed_string_from_map(auth_config.as_ref(), "account_name"))
.flatten();
let oauth_account_user_id = auth_semantics
.can_show_oauth_metadata()
.then(|| admin_pool_trimmed_string_from_map(auth_config.as_ref(), "account_user_id"))
.flatten();
let oauth_organizations = if auth_semantics.can_show_oauth_metadata() {
admin_pool_oauth_organizations(auth_config.as_ref())
} else {
Vec::new()
};
let account_status_code = admin_pool_trimmed_string_from_map(account_snapshot, "code"); let account_status_code = admin_pool_trimmed_string_from_map(account_snapshot, "code");
let account_status_label = let account_status_label =
admin_pool_trimmed_string(account_snapshot.and_then(|item| item.get("label"))); admin_pool_trimmed_string(account_snapshot.and_then(|item| item.get("label")));
@@ -686,11 +724,38 @@ pub(super) fn build_admin_pool_key_payload(
payload.insert("key_name".to_string(), json!(key.name)); payload.insert("key_name".to_string(), json!(key.name));
payload.insert("is_active".to_string(), json!(key.is_active)); payload.insert("is_active".to_string(), json!(key.is_active));
payload.insert("auth_type".to_string(), json!(key.auth_type)); payload.insert("auth_type".to_string(), json!(key.auth_type));
payload.insert(
"credential_kind".to_string(),
json!(auth_semantics.credential_kind().as_str()),
);
payload.insert(
"runtime_auth_kind".to_string(),
json!(auth_semantics.runtime_auth_kind().as_str()),
);
payload.insert(
"oauth_managed".to_string(),
json!(auth_semantics.oauth_managed()),
);
payload.insert(
"can_refresh_oauth".to_string(),
json!(auth_semantics.can_refresh_oauth()),
);
payload.insert(
"can_export_oauth".to_string(),
json!(auth_semantics.can_export_oauth()),
);
payload.insert(
"can_edit_oauth".to_string(),
json!(auth_semantics.can_edit_oauth()),
);
payload.insert("oauth_expires_at".to_string(), json!(oauth_expires_at)); payload.insert("oauth_expires_at".to_string(), json!(oauth_expires_at));
payload.insert("oauth_invalid_at".to_string(), json!(oauth_invalid_at)); payload.insert("oauth_invalid_at".to_string(), json!(oauth_invalid_at));
payload.insert( payload.insert(
"oauth_invalid_reason".to_string(), "oauth_invalid_reason".to_string(),
json!(key.oauth_invalid_reason), json!(auth_semantics
.can_show_oauth_metadata()
.then_some(key.oauth_invalid_reason.clone())
.flatten()),
); );
payload.insert("oauth_plan_type".to_string(), json!(oauth_plan_type)); payload.insert("oauth_plan_type".to_string(), json!(oauth_plan_type));
payload.insert("oauth_account_id".to_string(), json!(oauth_account_id)); payload.insert("oauth_account_id".to_string(), json!(oauth_account_id));

View File

@@ -3,6 +3,7 @@ use super::{
AdminPoolResolveSelectionRequest, ADMIN_POOL_PROVIDER_CATALOG_READER_UNAVAILABLE_DETAIL, AdminPoolResolveSelectionRequest, ADMIN_POOL_PROVIDER_CATALOG_READER_UNAVAILABLE_DETAIL,
}; };
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext}; use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::provider_key_auth::provider_key_auth_semantics;
use crate::GatewayError; use crate::GatewayError;
use aether_admin::provider::pool as admin_provider_pool_pure; use aether_admin::provider::pool as admin_provider_pool_pure;
use axum::{ use axum::{
@@ -11,6 +12,7 @@ use axum::{
response::{IntoResponse, Response}, response::{IntoResponse, Response},
Json, Json,
}; };
use serde_json::json;
pub(super) async fn build_admin_pool_resolve_selection_response( pub(super) async fn build_admin_pool_resolve_selection_response(
state: &AdminAppState<'_>, state: &AdminAppState<'_>,
@@ -88,5 +90,27 @@ pub(super) async fn build_admin_pool_resolve_selection_response(
.then_with(|| left.name.cmp(&right.name)) .then_with(|| left.name.cmp(&right.name))
}); });
Ok(Json(admin_provider_pool_pure::build_admin_pool_selection_payload(&keys)).into_response()) let items = keys
.iter()
.map(|key| {
let auth_semantics = provider_key_auth_semantics(key, &provider_type);
json!({
"key_id": key.id,
"key_name": key.name,
"auth_type": key.auth_type,
"credential_kind": auth_semantics.credential_kind().as_str(),
"runtime_auth_kind": auth_semantics.runtime_auth_kind().as_str(),
"oauth_managed": auth_semantics.oauth_managed(),
"can_refresh_oauth": auth_semantics.can_refresh_oauth(),
"can_export_oauth": auth_semantics.can_export_oauth(),
"can_edit_oauth": auth_semantics.can_edit_oauth(),
})
})
.collect::<Vec<_>>();
Ok(Json(json!({
"total": items.len(),
"items": items,
}))
.into_response())
} }

View File

@@ -1,4 +1,5 @@
use crate::handlers::admin::request::AdminAppState; use crate::handlers::admin::request::AdminAppState;
use crate::provider_key_auth::provider_key_is_oauth_managed;
use aether_admin::provider::pool as admin_provider_pool_pure; use aether_admin::provider::pool as admin_provider_pool_pure;
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey; use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
@@ -44,7 +45,7 @@ fn admin_pool_derive_oauth_plan_type(
} }
}; };
if key.auth_type.trim() != "oauth" { if !provider_key_is_oauth_managed(key, provider_type) {
return None; return None;
} }

View File

@@ -1,17 +1,29 @@
use super::payload::{ use super::payload::{
provider_query_extract_api_key_id, provider_query_extract_force_refresh, provider_query_extract_api_key_id, provider_query_extract_force_refresh,
provider_query_extract_provider_id, provider_query_extract_model, provider_query_extract_provider_id,
provider_query_extract_request_id,
}; };
use super::response::{ use super::response::{
build_admin_provider_query_bad_request_response, build_admin_provider_query_not_found_response, build_admin_provider_query_bad_request_response, build_admin_provider_query_not_found_response,
ADMIN_PROVIDER_QUERY_API_KEY_NOT_FOUND_DETAIL, ADMIN_PROVIDER_QUERY_NO_ACTIVE_API_KEY_DETAIL, ADMIN_PROVIDER_QUERY_API_KEY_NOT_FOUND_DETAIL, ADMIN_PROVIDER_QUERY_MODEL_REQUIRED_DETAIL,
ADMIN_PROVIDER_QUERY_NO_ACTIVE_API_KEY_DETAIL, ADMIN_PROVIDER_QUERY_NO_LOCAL_MODELS_DETAIL,
ADMIN_PROVIDER_QUERY_PROVIDER_ID_REQUIRED_DETAIL, ADMIN_PROVIDER_QUERY_PROVIDER_ID_REQUIRED_DETAIL,
ADMIN_PROVIDER_QUERY_PROVIDER_NOT_FOUND_DETAIL, ADMIN_PROVIDER_QUERY_PROVIDER_NOT_FOUND_DETAIL,
}; };
use crate::ai_pipeline::{maybe_build_sync_finalize_outcome, GatewayControlDecision};
use crate::execution_runtime; use crate::execution_runtime;
use crate::handlers::admin::request::AdminAppState; use crate::handlers::admin::request::AdminAppState;
use crate::model_fetch::ModelFetchRuntimeState; use crate::model_fetch::ModelFetchRuntimeState;
use crate::provider_transport::kiro::{
build_kiro_generate_assistant_response_url, build_kiro_provider_headers,
build_kiro_provider_request_body, supports_local_kiro_request_transport_with_network,
KiroProviderHeadersInput, KIRO_ENVELOPE_NAME,
};
use crate::usage::GatewaySyncReportRequest;
use crate::{AppState, GatewayError}; use crate::{AppState, GatewayError};
use aether_admin::provider::pool as admin_provider_pool_pure;
use aether_contracts::{ExecutionPlan, RequestBody};
use aether_data_contracts::repository::global_models::AdminProviderModelListQuery;
use aether_data_contracts::repository::provider_catalog::{ use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider, StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
}; };
@@ -19,9 +31,16 @@ use aether_model_fetch::{
aggregate_models_for_cache, fetch_models_from_transports, json_string_list, aggregate_models_for_cache, fetch_models_from_transports, json_string_list,
preset_models_for_provider, selected_models_fetch_endpoints, preset_models_for_provider, selected_models_fetch_endpoints,
}; };
use axum::{body::Body, http::Response, response::IntoResponse, Json}; use axum::{
body::{to_bytes, Body},
http::{HeaderMap, HeaderName, HeaderValue},
response::{IntoResponse, Response},
Json,
};
use base64::Engine as _;
use serde_json::{json, Value}; use serde_json::{json, Value};
use std::collections::BTreeSet; use std::collections::{BTreeMap, BTreeSet};
use uuid::Uuid;
pub(crate) const ADMIN_PROVIDER_QUERY_LOCAL_TEST_MODEL_MESSAGE: &str = pub(crate) const ADMIN_PROVIDER_QUERY_LOCAL_TEST_MODEL_MESSAGE: &str =
"Rust local provider-query model test is not configured"; "Rust local provider-query model test is not configured";
@@ -32,7 +51,10 @@ const ADMIN_PROVIDER_QUERY_NO_ACTIVE_ENDPOINT_DETAIL: &str =
const ADMIN_PROVIDER_QUERY_NO_MODELS_FROM_ENDPOINT_DETAIL: &str = const ADMIN_PROVIDER_QUERY_NO_MODELS_FROM_ENDPOINT_DETAIL: &str =
"No models returned from any endpoint"; "No models returned from any endpoint";
const ADMIN_PROVIDER_QUERY_NO_MODELS_FROM_KEY_DETAIL: &str = "No models returned from any key"; const ADMIN_PROVIDER_QUERY_NO_MODELS_FROM_KEY_DETAIL: &str = "No models returned from any key";
const ADMIN_PROVIDER_QUERY_NO_ACTIVE_TEST_CANDIDATE_DETAIL: &str =
"No active endpoint or API key found";
const ANTIGRAVITY_PROVIDER_CACHE_KEY_PREFIX: &str = "upstream_models_provider:"; const ANTIGRAVITY_PROVIDER_CACHE_KEY_PREFIX: &str = "upstream_models_provider:";
const DEFAULT_PROVIDER_QUERY_TEST_MESSAGE: &str = "Hello! This is a test message.";
#[derive(Debug)] #[derive(Debug)]
struct ProviderQueryKeyFetchResult { struct ProviderQueryKeyFetchResult {
@@ -42,6 +64,47 @@ struct ProviderQueryKeyFetchResult {
has_success: bool, has_success: bool,
} }
#[derive(Debug, Clone)]
struct ProviderQueryTestCandidate {
endpoint: StoredProviderCatalogEndpoint,
key: StoredProviderCatalogKey,
effective_model: String,
}
#[derive(Debug, Clone)]
struct ProviderQueryTestAttempt {
candidate_index: usize,
endpoint_api_format: String,
endpoint_base_url: String,
key_name: String,
key_id: String,
auth_type: String,
effective_model: String,
status: &'static str,
skip_reason: Option<String>,
error_message: Option<String>,
status_code: Option<u16>,
latency_ms: Option<u64>,
request_url: Option<String>,
request_headers: Option<BTreeMap<String, String>>,
request_body: Option<Value>,
response_headers: Option<BTreeMap<String, String>>,
response_body: Option<Value>,
}
#[derive(Debug, Clone)]
struct ProviderQueryExecutionOutcome {
status: &'static str,
error_message: Option<String>,
status_code: Option<u16>,
latency_ms: Option<u64>,
request_url: String,
request_headers: BTreeMap<String, String>,
request_body: Value,
response_headers: BTreeMap<String, String>,
response_body: Option<Value>,
}
fn provider_query_provider_payload(provider: &StoredProviderCatalogProvider) -> Value { fn provider_query_provider_payload(provider: &StoredProviderCatalogProvider) -> Value {
json!({ json!({
"id": provider.id.clone(), "id": provider.id.clone(),
@@ -50,6 +113,753 @@ fn provider_query_provider_payload(provider: &StoredProviderCatalogProvider) ->
}) })
} }
fn provider_query_test_mode(payload: &Value) -> &str {
payload
.get("mode")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or("global")
}
fn provider_query_extract_endpoint_id(payload: &Value) -> Option<String> {
payload
.get("endpoint_id")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
fn provider_query_extract_api_format(payload: &Value) -> Option<String> {
payload
.get("api_format")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
fn provider_query_extract_message(payload: &Value) -> Option<String> {
payload
.get("message")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
fn provider_query_extract_request_body(payload: &Value) -> Option<Value> {
payload
.get("request_body")
.filter(|value| value.is_object())
.cloned()
}
fn provider_query_extract_request_headers(payload: &Value) -> HeaderMap {
let mut headers = HeaderMap::new();
let Some(values) = payload.get("request_headers").and_then(Value::as_object) else {
return headers;
};
for (key, value) in values {
let key = key.trim();
if key.is_empty() {
continue;
}
let Some(value) = (match value {
Value::String(value) => Some(value.trim().to_string()),
Value::Bool(value) => Some(value.to_string()),
Value::Number(value) => Some(value.to_string()),
other => serde_json::to_string(other).ok(),
}) else {
continue;
};
if value.is_empty() {
continue;
}
let Ok(name) = HeaderName::from_bytes(key.as_bytes()) else {
continue;
};
let Ok(value) = HeaderValue::from_str(&value) else {
continue;
};
headers.insert(name, value);
}
headers
}
fn provider_query_build_test_request_body(payload: &Value, model: &str) -> Value {
if let Some(mut body) = provider_query_extract_request_body(payload) {
if let Some(object) = body.as_object_mut() {
object.insert("model".to_string(), Value::String(model.to_string()));
}
return body;
}
json!({
"model": model,
"messages": [{
"role": "user",
"content": provider_query_extract_message(payload)
.unwrap_or_else(|| DEFAULT_PROVIDER_QUERY_TEST_MESSAGE.to_string())
}],
"max_tokens": 30,
"temperature": 0.7,
"stream": true,
})
}
fn provider_query_select_kiro_endpoint<'a>(
endpoints: &'a [StoredProviderCatalogEndpoint],
endpoint_id: Option<&str>,
api_format: Option<&str>,
) -> Result<Option<&'a StoredProviderCatalogEndpoint>, &'static str> {
if let Some(endpoint_id) = endpoint_id {
let endpoint = endpoints.iter().find(|endpoint| endpoint.id == endpoint_id);
return endpoint
.ok_or("Endpoint not found")
.map(|endpoint| Some(endpoint));
}
if let Some(api_format) = api_format {
let endpoint = endpoints.iter().find(|endpoint| {
endpoint.is_active && endpoint.api_format.trim().eq_ignore_ascii_case(api_format)
});
return Ok(endpoint);
}
Ok(endpoints.iter().find(|endpoint| endpoint.is_active))
}
fn provider_query_key_supports_endpoint(
key: &StoredProviderCatalogKey,
endpoint_api_format: &str,
) -> bool {
let formats = json_string_list(key.api_formats.as_ref());
formats.is_empty()
|| formats
.iter()
.any(|value| value.eq_ignore_ascii_case(endpoint_api_format))
}
fn provider_query_test_key_sort_key(
provider_type: &str,
key: &StoredProviderCatalogKey,
endpoint_api_format: &str,
) -> (u8, u8, i32, u64, i32) {
let quota_exhausted =
admin_provider_pool_pure::admin_pool_key_account_quota_exhausted(key, provider_type);
let circuit_open = key
.circuit_breaker_by_format
.as_ref()
.and_then(Value::as_object)
.and_then(|value| value.get(endpoint_api_format))
.and_then(Value::as_object)
.and_then(|value| value.get("open"))
.and_then(Value::as_bool)
.unwrap_or(false);
let health_score = key
.health_by_format
.as_ref()
.and_then(Value::as_object)
.and_then(|value| value.get(endpoint_api_format))
.and_then(Value::as_object)
.and_then(|value| value.get("health_score"))
.and_then(Value::as_f64)
.unwrap_or(1.0);
let consecutive_failures = key
.health_by_format
.as_ref()
.and_then(Value::as_object)
.and_then(|value| value.get(endpoint_api_format))
.and_then(Value::as_object)
.and_then(|value| value.get("consecutive_failures"))
.and_then(Value::as_u64)
.unwrap_or(0);
let normalized_health = (health_score.clamp(0.0, 1.0) * 1000.0).round() as i32;
(
if quota_exhausted { 1 } else { 0 },
if circuit_open { 1 } else { 0 },
-normalized_health,
consecutive_failures,
key.internal_priority,
)
}
async fn provider_query_resolve_global_effective_model(
state: &AdminAppState<'_>,
provider_id: &str,
requested_model: &str,
) -> Result<String, GatewayError> {
let models = state
.list_admin_provider_models(&AdminProviderModelListQuery {
provider_id: provider_id.to_string(),
is_active: Some(true),
offset: 0,
limit: 1024,
})
.await
.unwrap_or_default();
Ok(models
.into_iter()
.find(|model| {
model.is_available
&& model
.global_model_name
.as_deref()
.is_some_and(|value| value.eq_ignore_ascii_case(requested_model))
})
.map(|model| model.provider_model_name)
.unwrap_or_else(|| requested_model.to_string()))
}
async fn provider_query_build_kiro_test_candidates(
state: &AdminAppState<'_>,
provider: &StoredProviderCatalogProvider,
payload: &Value,
) -> Result<Vec<ProviderQueryTestCandidate>, Response<Body>> {
let provider_ids = vec![provider.id.clone()];
let endpoints = state
.app()
.list_provider_catalog_endpoints_by_provider_ids(&provider_ids)
.await
.map_err(|_| {
build_admin_provider_query_bad_request_response(
ADMIN_PROVIDER_QUERY_NO_ACTIVE_API_KEY_DETAIL,
)
})?;
let endpoint = match provider_query_select_kiro_endpoint(
&endpoints,
provider_query_extract_endpoint_id(payload).as_deref(),
provider_query_extract_api_format(payload).as_deref(),
) {
Ok(Some(endpoint)) => endpoint.clone(),
Ok(None) => {
return Err(build_admin_provider_query_not_found_response(
ADMIN_PROVIDER_QUERY_NO_ACTIVE_API_KEY_DETAIL,
));
}
Err("Endpoint not found") => {
return Err(build_admin_provider_query_not_found_response(
ADMIN_PROVIDER_QUERY_API_KEY_NOT_FOUND_DETAIL,
));
}
Err(_) => {
return Err(build_admin_provider_query_not_found_response(
ADMIN_PROVIDER_QUERY_NO_ACTIVE_API_KEY_DETAIL,
));
}
};
let all_keys = state
.app()
.list_provider_catalog_keys_by_provider_ids(&provider_ids)
.await
.map_err(|_| {
build_admin_provider_query_bad_request_response(
ADMIN_PROVIDER_QUERY_NO_ACTIVE_API_KEY_DETAIL,
)
})?;
let selected_key_id = provider_query_extract_api_key_id(payload);
if let Some(api_key_id) = selected_key_id.as_deref() {
let Some(key) = all_keys.iter().find(|key| key.id == api_key_id) else {
return Err(build_admin_provider_query_not_found_response(
ADMIN_PROVIDER_QUERY_API_KEY_NOT_FOUND_DETAIL,
));
};
if !key.is_active || !provider_query_key_supports_endpoint(key, &endpoint.api_format) {
return Err(build_admin_provider_query_not_found_response(
ADMIN_PROVIDER_QUERY_NO_ACTIVE_TEST_CANDIDATE_DETAIL,
));
}
}
let requested_model = provider_query_extract_model(payload).ok_or_else(|| {
build_admin_provider_query_bad_request_response(ADMIN_PROVIDER_QUERY_MODEL_REQUIRED_DETAIL)
})?;
let effective_model = if provider_query_test_mode(payload).eq_ignore_ascii_case("direct") {
requested_model.clone()
} else {
provider_query_resolve_global_effective_model(state, &provider.id, &requested_model)
.await
.unwrap_or(requested_model.clone())
};
let mut keys = all_keys
.into_iter()
.filter(|key| key.is_active)
.filter(|key| {
selected_key_id
.as_deref()
.is_none_or(|value| value == key.id.as_str())
})
.filter(|key| provider_query_key_supports_endpoint(key, &endpoint.api_format))
.collect::<Vec<_>>();
keys.sort_by_key(|key| {
provider_query_test_key_sort_key(provider.provider_type.as_str(), key, &endpoint.api_format)
});
let candidates = keys
.into_iter()
.map(|key| ProviderQueryTestCandidate {
endpoint: endpoint.clone(),
key,
effective_model: effective_model.clone(),
})
.collect::<Vec<_>>();
if candidates.is_empty() {
return Err(build_admin_provider_query_not_found_response(
ADMIN_PROVIDER_QUERY_NO_ACTIVE_TEST_CANDIDATE_DETAIL,
));
}
Ok(candidates)
}
fn provider_query_decode_execution_body(
result: &aether_contracts::ExecutionResult,
) -> Option<Vec<u8>> {
result
.body
.as_ref()
.and_then(|body| body.body_bytes_b64.as_deref())
.and_then(|value| base64::engine::general_purpose::STANDARD.decode(value).ok())
}
fn provider_query_extract_error_message(
result: &aether_contracts::ExecutionResult,
) -> Option<String> {
result
.body
.as_ref()
.and_then(|body| body.json_body.as_ref())
.and_then(Value::as_object)
.and_then(|value| {
value
.get("error")
.and_then(Value::as_object)
.and_then(|error| error.get("message"))
.and_then(Value::as_str)
.or_else(|| value.get("message").and_then(Value::as_str))
})
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.or_else(|| {
provider_query_decode_execution_body(result)
.and_then(|bytes| String::from_utf8(bytes).ok())
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
})
.or_else(|| {
result
.error
.as_ref()
.map(|error| error.message.trim().to_string())
.filter(|value| !value.is_empty())
})
}
async fn provider_query_finalize_kiro_result(
route_path: &str,
trace_id: &str,
requested_model: &str,
endpoint_api_format: &str,
effective_model: &str,
original_request_body: &Value,
result: &aether_contracts::ExecutionResult,
) -> Result<Option<Value>, GatewayError> {
let decision = GatewayControlDecision::synthetic(
route_path,
Some("admin_proxy".to_string()),
Some("provider_query_manage".to_string()),
Some("test_model_failover".to_string()),
Some(endpoint_api_format.to_string()),
);
let payload = GatewaySyncReportRequest {
trace_id: trace_id.to_string(),
report_kind: "claude_cli_sync_finalize".to_string(),
report_context: Some(json!({
"client_api_format": endpoint_api_format,
"provider_api_format": endpoint_api_format,
"model": requested_model,
"mapped_model": effective_model,
"needs_conversion": false,
"has_envelope": true,
"envelope_name": KIRO_ENVELOPE_NAME,
"original_request_body": original_request_body,
})),
status_code: result.status_code,
headers: result.headers.clone(),
body_json: result.body.as_ref().and_then(|body| body.json_body.clone()),
client_body_json: None,
body_base64: result
.body
.as_ref()
.and_then(|body| body.body_bytes_b64.clone()),
telemetry: result.telemetry.clone(),
};
let Some(outcome) = maybe_build_sync_finalize_outcome(trace_id, &decision, &payload)? else {
return Ok(None);
};
let bytes = to_bytes(outcome.response.into_body(), usize::MAX)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
serde_json::from_slice::<Value>(&bytes)
.map(Some)
.map_err(|err| GatewayError::Internal(err.to_string()))
}
async fn provider_query_execute_kiro_test_candidate(
state: &AdminAppState<'_>,
provider: &StoredProviderCatalogProvider,
candidate: &ProviderQueryTestCandidate,
payload: &Value,
route_path: &str,
trace_id: &str,
requested_model: &str,
) -> Result<ProviderQueryExecutionOutcome, GatewayError> {
let Some(transport) = state
.read_provider_transport_snapshot(&provider.id, &candidate.endpoint.id, &candidate.key.id)
.await?
else {
return Ok(ProviderQueryExecutionOutcome {
status: "skipped",
error_message: None,
status_code: None,
latency_ms: None,
request_url: String::new(),
request_headers: BTreeMap::new(),
request_body: Value::Null,
response_headers: BTreeMap::new(),
response_body: None,
});
};
if !supports_local_kiro_request_transport_with_network(&transport) {
return Ok(ProviderQueryExecutionOutcome {
status: "skipped",
error_message: None,
status_code: None,
latency_ms: None,
request_url: String::new(),
request_headers: BTreeMap::new(),
request_body: Value::Null,
response_headers: BTreeMap::new(),
response_body: None,
});
}
let Some(kiro_auth) = state
.resolve_local_oauth_kiro_request_auth(&transport)
.await?
else {
return Ok(ProviderQueryExecutionOutcome {
status: "failed",
error_message: Some("oauth auth failed".to_string()),
status_code: None,
latency_ms: None,
request_url: String::new(),
request_headers: BTreeMap::new(),
request_body: Value::Null,
response_headers: BTreeMap::new(),
response_body: None,
});
};
let request_body = provider_query_build_test_request_body(payload, &candidate.effective_model);
let provider_request_body = match build_kiro_provider_request_body(
&request_body,
&candidate.effective_model,
&kiro_auth.auth_config,
transport.endpoint.body_rules.as_ref(),
) {
Some(body) => body,
None => {
return Ok(ProviderQueryExecutionOutcome {
status: "failed",
error_message: Some("provider request body build failed".to_string()),
status_code: None,
latency_ms: None,
request_url: String::new(),
request_headers: BTreeMap::new(),
request_body,
response_headers: BTreeMap::new(),
response_body: None,
});
}
};
let mut synthetic_request = http::Request::builder()
.uri(route_path)
.body(())
.map_err(|err| GatewayError::Internal(err.to_string()))?;
*synthetic_request.headers_mut() = provider_query_extract_request_headers(payload);
let (parts, _) = synthetic_request.into_parts();
let request_url = build_kiro_generate_assistant_response_url(
&transport.endpoint.base_url,
parts.uri.query(),
Some(kiro_auth.auth_config.effective_api_region()),
)
.ok_or_else(|| GatewayError::Internal("kiro request url is unavailable".to_string()))?;
let request_headers = build_kiro_provider_headers(KiroProviderHeadersInput {
headers: &parts.headers,
provider_request_body: &provider_request_body,
original_request_body: &request_body,
header_rules: transport.endpoint.header_rules.as_ref(),
auth_header: kiro_auth.name,
auth_value: &kiro_auth.value,
auth_config: &kiro_auth.auth_config,
machine_id: kiro_auth.machine_id.as_str(),
})
.ok_or_else(|| GatewayError::Internal("kiro request headers are unavailable".to_string()))?;
let plan = ExecutionPlan {
request_id: trace_id.to_string(),
candidate_id: Some(format!("provider-query-{}", candidate.key.id)),
provider_name: Some(provider.name.clone()),
provider_id: provider.id.clone(),
endpoint_id: candidate.endpoint.id.clone(),
key_id: candidate.key.id.clone(),
method: "POST".to_string(),
url: request_url.clone(),
headers: request_headers.clone(),
content_type: Some("application/json".to_string()),
content_encoding: None,
body: RequestBody::from_json(provider_request_body.clone()),
stream: true,
client_api_format: candidate.endpoint.api_format.clone(),
provider_api_format: candidate.endpoint.api_format.clone(),
model_name: Some(candidate.effective_model.clone()),
proxy: state
.resolve_transport_proxy_snapshot_with_tunnel_affinity(&transport)
.await,
tls_profile: state.resolve_transport_tls_profile(&transport),
timeouts: state.resolve_transport_execution_timeouts(&transport),
};
let result = state
.execute_execution_runtime_sync_plan(Some(trace_id), &plan)
.await?;
let response_body = if result.status_code < 400 {
provider_query_finalize_kiro_result(
route_path,
trace_id,
requested_model,
candidate.endpoint.api_format.as_str(),
candidate.effective_model.as_str(),
&request_body,
&result,
)
.await?
} else {
result.body.as_ref().and_then(|body| body.json_body.clone())
};
let error_message = if result.status_code >= 400 {
provider_query_extract_error_message(&result)
} else if response_body.is_none()
&& provider_query_decode_execution_body(&result)
.is_some_and(|body| crate::ai_pipeline::stream_body_contains_error_event(&body))
{
Some("Kiro upstream returned embedded stream error".to_string())
} else {
None
};
Ok(ProviderQueryExecutionOutcome {
status: if error_message.is_some() {
"failed"
} else {
"success"
},
error_message,
status_code: Some(result.status_code),
latency_ms: result.telemetry.as_ref().and_then(|value| value.elapsed_ms),
request_url,
request_headers,
request_body: provider_request_body,
response_headers: result.headers,
response_body,
})
}
fn provider_query_test_attempt_payload(
candidate_index: usize,
candidate: &ProviderQueryTestCandidate,
execution: &ProviderQueryExecutionOutcome,
) -> Value {
json!({
"candidate_index": candidate_index,
"retry_index": 0,
"endpoint_api_format": candidate.endpoint.api_format,
"endpoint_base_url": candidate.endpoint.base_url,
"key_name": provider_query_key_display_name(&candidate.key),
"key_id": candidate.key.id,
"auth_type": candidate.key.auth_type,
"effective_model": candidate.effective_model,
"status": execution.status,
"skip_reason": Value::Null,
"error_message": execution.error_message,
"status_code": execution.status_code,
"latency_ms": execution.latency_ms,
"request_url": execution.request_url,
"request_headers": execution.request_headers,
"request_body": execution.request_body,
"response_headers": execution.response_headers,
"response_body": execution.response_body,
})
}
async fn build_admin_provider_query_kiro_failover_response(
state: &AdminAppState<'_>,
payload: &Value,
route_path: &str,
) -> Result<Response<Body>, GatewayError> {
let Some(provider_id) = provider_query_extract_provider_id(payload) else {
return Ok(build_admin_provider_query_bad_request_response(
ADMIN_PROVIDER_QUERY_PROVIDER_ID_REQUIRED_DETAIL,
));
};
let requested_model = match provider_query_extract_model(payload) {
Some(model) => model,
None => {
return Ok(build_admin_provider_query_bad_request_response(
ADMIN_PROVIDER_QUERY_MODEL_REQUIRED_DETAIL,
));
}
};
let Some(provider) = state
.app()
.read_provider_catalog_providers_by_ids(std::slice::from_ref(&provider_id))
.await?
.into_iter()
.find(|item| item.id == provider_id)
else {
return Ok(build_admin_provider_query_not_found_response(
ADMIN_PROVIDER_QUERY_PROVIDER_NOT_FOUND_DETAIL,
));
};
if !provider.provider_type.trim().eq_ignore_ascii_case("kiro") {
return Ok(build_admin_provider_query_test_model_failover_response(
provider_id,
super::payload::provider_query_extract_failover_models(payload),
));
}
let candidates =
match provider_query_build_kiro_test_candidates(state, &provider, payload).await {
Ok(candidates) => candidates,
Err(response) => return Ok(response),
};
let trace_id = provider_query_extract_request_id(payload)
.unwrap_or_else(|| format!("provider-query-test-{}", Uuid::new_v4().simple()));
let mut attempts = Vec::new();
let mut total_attempts = 0usize;
let mut success_body = None;
for (candidate_index, candidate) in candidates.iter().enumerate() {
let execution = provider_query_execute_kiro_test_candidate(
state,
&provider,
candidate,
payload,
route_path,
&trace_id,
&requested_model,
)
.await?;
if execution.status != "skipped" {
total_attempts += 1;
}
let is_success = execution.status == "success";
let response_body = execution.response_body.clone();
attempts.push(provider_query_test_attempt_payload(
candidate_index,
candidate,
&execution,
));
if is_success {
success_body = response_body;
break;
}
}
let success = success_body.is_some();
let error = if success {
Value::Null
} else {
attempts
.iter()
.rev()
.find_map(|attempt| {
attempt
.get("error_message")
.cloned()
.filter(|value| !value.is_null())
})
.unwrap_or_else(|| json!(ADMIN_PROVIDER_QUERY_NO_LOCAL_MODELS_DETAIL))
};
Ok(Json(json!({
"success": success,
"model": requested_model,
"provider": provider_query_provider_payload(&provider),
"attempts": attempts,
"total_candidates": candidates.len(),
"total_attempts": total_attempts,
"data": success_body.as_ref().map(|body| json!({
"stream": true,
"response": body,
})),
"error": error,
}))
.into_response())
}
pub(crate) async fn build_admin_provider_query_test_model_local_response(
state: &AdminAppState<'_>,
payload: &Value,
) -> Result<Response<Body>, GatewayError> {
let response = build_admin_provider_query_kiro_failover_response(
state,
payload,
"/api/admin/provider-query/test-model",
)
.await?;
let body = to_bytes(response.into_body(), usize::MAX)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
let parsed: Value =
serde_json::from_slice(&body).map_err(|err| GatewayError::Internal(err.to_string()))?;
Ok(Json(json!({
"success": parsed.get("success").cloned().unwrap_or(Value::Bool(false)),
"error": parsed.get("error").cloned().unwrap_or(Value::Null),
"data": parsed.get("data").cloned().unwrap_or(Value::Null),
"provider": parsed.get("provider").cloned().unwrap_or(Value::Null),
"model": parsed.get("model").cloned().unwrap_or(Value::Null),
}))
.into_response())
}
pub(crate) async fn build_admin_provider_query_test_model_failover_local_response(
state: &AdminAppState<'_>,
payload: &Value,
) -> Result<Response<Body>, GatewayError> {
build_admin_provider_query_kiro_failover_response(
state,
payload,
"/api/admin/provider-query/test-model-failover",
)
.await
}
fn provider_query_key_display_name(key: &StoredProviderCatalogKey) -> String { fn provider_query_key_display_name(key: &StoredProviderCatalogKey) -> String {
let trimmed = key.name.trim(); let trimmed = key.name.trim();
if trimmed.is_empty() { if trimmed.is_empty() {

View File

@@ -13,6 +13,7 @@ pub(crate) const ADMIN_PROVIDER_OAUTH_DATA_UNAVAILABLE_DETAIL: &str =
#[derive(Debug, Clone, Copy)] #[derive(Debug, Clone, Copy)]
pub(crate) struct AdminProviderPoolConfig { pub(crate) struct AdminProviderPoolConfig {
pub(crate) lru_enabled: bool, pub(crate) lru_enabled: bool,
pub(crate) skip_exhausted_accounts: bool,
pub(crate) cost_window_seconds: u64, pub(crate) cost_window_seconds: u64,
pub(crate) cost_limit_per_key_tokens: Option<u64>, pub(crate) cost_limit_per_key_tokens: Option<u64>,
} }

View File

@@ -13,8 +13,8 @@ pub(crate) fn normalize_provider_type_input(value: &str) -> Result<String, Strin
pub(crate) fn normalize_auth_type(value: Option<&str>) -> Result<String, String> { pub(crate) fn normalize_auth_type(value: Option<&str>) -> Result<String, String> {
let auth_type = value.unwrap_or("api_key").trim().to_ascii_lowercase(); let auth_type = value.unwrap_or("api_key").trim().to_ascii_lowercase();
match auth_type.as_str() { match auth_type.as_str() {
"api_key" | "service_account" | "oauth" => Ok(auth_type), "api_key" | "service_account" | "oauth" | "bearer" => Ok(auth_type),
_ => Err("auth_type 仅支持 api_key / service_account / oauth".to_string()), _ => Err("auth_type 仅支持 api_key / service_account / oauth / bearer".to_string()),
} }
} }
@@ -63,7 +63,7 @@ pub(crate) fn validate_vertex_api_formats(
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::normalize_pool_advanced_config; use super::{normalize_auth_type, normalize_pool_advanced_config};
use serde_json::json; use serde_json::json;
#[test] #[test]
@@ -85,4 +85,12 @@ mod tests {
"pool_advanced 必须是 JSON 对象" "pool_advanced 必须是 JSON 对象"
); );
} }
#[test]
fn normalize_auth_type_supports_bearer() {
assert_eq!(
normalize_auth_type(Some("bearer")).expect("bearer should normalize"),
"bearer"
);
}
} }

View File

@@ -8,17 +8,37 @@ fn normalize_reveal_auth_type(value: &str) -> &str {
use crate::handlers::admin::request::AdminAppState; use crate::handlers::admin::request::AdminAppState;
use crate::handlers::admin::shared::parse_catalog_auth_config_json; use crate::handlers::admin::shared::parse_catalog_auth_config_json;
use crate::provider_key_auth::provider_key_auth_semantics;
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey; use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
use chrono::{SecondsFormat, Utc}; use chrono::{SecondsFormat, Utc};
use serde_json::json; use serde_json::json;
fn reveal_provider_type_from_auth_config(
auth_config: Option<&serde_json::Map<String, serde_json::Value>>,
) -> String {
auth_config
.and_then(|value| value.get("provider_type"))
.and_then(serde_json::Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or_default()
.to_string()
}
pub(crate) fn build_admin_reveal_key_payload( pub(crate) fn build_admin_reveal_key_payload(
state: &AdminAppState<'_>, state: &AdminAppState<'_>,
key: &StoredProviderCatalogKey, key: &StoredProviderCatalogKey,
) -> Result<serde_json::Value, String> { ) -> Result<serde_json::Value, String> {
let auth_type = normalize_reveal_auth_type(&key.auth_type); let parsed_auth_config = state.parse_catalog_auth_config_json(key);
let provider_type = reveal_provider_type_from_auth_config(parsed_auth_config.as_ref());
let auth_semantics = provider_key_auth_semantics(key, provider_type.as_str());
let auth_type = if auth_semantics.oauth_managed() {
"oauth"
} else {
normalize_reveal_auth_type(&key.auth_type)
};
if matches!(auth_type, "service_account") { if matches!(auth_type, "service_account") {
if let Some(auth_config) = state.parse_catalog_auth_config_json(key) { if let Some(auth_config) = parsed_auth_config {
return Ok(json!({ return Ok(json!({
"auth_type": auth_type, "auth_type": auth_type,
"auth_config": auth_config, "auth_config": auth_config,
@@ -92,11 +112,6 @@ pub(crate) async fn build_admin_export_key_payload(
state: &AdminAppState<'_>, state: &AdminAppState<'_>,
key: &StoredProviderCatalogKey, key: &StoredProviderCatalogKey,
) -> Result<serde_json::Value, String> { ) -> Result<serde_json::Value, String> {
let auth_type = normalize_reveal_auth_type(&key.auth_type);
if auth_type != "oauth" {
return Err("仅 OAuth 类型的 Key 支持导出".to_string());
}
let ciphertext = key let ciphertext = key
.encrypted_auth_config .encrypted_auth_config
.as_deref() .as_deref()
@@ -136,6 +151,9 @@ pub(crate) async fn build_admin_export_key_payload(
.map(|provider| provider.provider_type) .map(|provider| provider.provider_type)
.unwrap_or_default() .unwrap_or_default()
}; };
if !provider_key_auth_semantics(key, provider_type.as_str()).can_export_oauth() {
return Err("仅 OAuth 管理账号支持导出".to_string());
}
let mut payload = let mut payload =
provider_oauth_export_payload(&provider_type, &auth_config, key.upstream_metadata.as_ref()); provider_oauth_export_payload(&provider_type, &auth_config, key.upstream_metadata.as_ref());

View File

@@ -1,7 +1,9 @@
use crate::handlers::admin::provider::query::{ use crate::handlers::admin::provider::query::{
models::{ models::{
build_admin_provider_query_models_response, build_admin_provider_query_models_response,
build_admin_provider_query_test_model_failover_local_response,
build_admin_provider_query_test_model_failover_response, build_admin_provider_query_test_model_failover_response,
build_admin_provider_query_test_model_local_response,
build_admin_provider_query_test_model_response, build_admin_provider_query_test_model_response,
}, },
payload::{ payload::{
@@ -78,10 +80,25 @@ impl<'a> AdminAppState<'a> {
ADMIN_PROVIDER_QUERY_MODEL_REQUIRED_DETAIL, ADMIN_PROVIDER_QUERY_MODEL_REQUIRED_DETAIL,
))); )));
}; };
Ok(Some(build_admin_provider_query_test_model_response( let provider_type = self
provider_id, .app()
model, .read_provider_catalog_providers_by_ids(std::slice::from_ref(&provider_id))
))) .await?
.into_iter()
.find(|provider| provider.id == provider_id)
.map(|provider| provider.provider_type)
.unwrap_or_default();
if provider_type.trim().eq_ignore_ascii_case("kiro") {
Ok(Some(
build_admin_provider_query_test_model_local_response(self, &payload)
.await?,
))
} else {
Ok(Some(build_admin_provider_query_test_model_response(
provider_id,
model,
)))
}
} }
"test_model_failover" => { "test_model_failover" => {
let Some(provider_id) = provider_query_extract_provider_id(&payload) else { let Some(provider_id) = provider_query_extract_provider_id(&payload) else {
@@ -107,12 +124,29 @@ impl<'a> AdminAppState<'a> {
ADMIN_PROVIDER_QUERY_FAILOVER_MODELS_REQUIRED_DETAIL, ADMIN_PROVIDER_QUERY_FAILOVER_MODELS_REQUIRED_DETAIL,
))); )));
} }
Ok(Some( let provider_type = self
build_admin_provider_query_test_model_failover_response( .app()
provider_id, .read_provider_catalog_providers_by_ids(std::slice::from_ref(&provider_id))
failover_models, .await?
), .into_iter()
)) .find(|provider| provider.id == provider_id)
.map(|provider| provider.provider_type)
.unwrap_or_default();
if provider_type.trim().eq_ignore_ascii_case("kiro") {
Ok(Some(
build_admin_provider_query_test_model_failover_local_response(
self, &payload,
)
.await?,
))
} else {
Ok(Some(
build_admin_provider_query_test_model_failover_response(
provider_id,
failover_models,
),
))
}
} }
_ => Ok(Some( _ => Ok(Some(
build_admin_provider_query_models_response(self, &payload).await?, build_admin_provider_query_models_response(self, &payload).await?,

View File

@@ -1,4 +1,5 @@
use crate::handlers::shared::{json_string_list, unix_secs_to_rfc3339}; use crate::handlers::shared::{json_string_list, unix_secs_to_rfc3339};
use crate::provider_key_auth::provider_key_auth_semantics;
use crate::AppState; use crate::AppState;
#[cfg(test)] #[cfg(test)]
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY; use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
@@ -273,7 +274,7 @@ fn derive_catalog_oauth_plan_type(
provider_type: &str, provider_type: &str,
auth_config: Option<&serde_json::Map<String, serde_json::Value>>, auth_config: Option<&serde_json::Map<String, serde_json::Value>>,
) -> Option<String> { ) -> Option<String> {
if !key.auth_type.trim().eq_ignore_ascii_case("oauth") { if !provider_key_auth_semantics(key, provider_type).oauth_managed() {
return None; return None;
} }
@@ -339,13 +340,18 @@ pub(crate) fn build_admin_provider_key_response(
} else { } else {
0.0 0.0
}; };
let auth_semantics = provider_key_auth_semantics(key, provider_type);
let auth_config = parse_catalog_auth_config_json(state, key); let auth_config = parse_catalog_auth_config_json(state, key);
let oauth_organizations = auth_config let oauth_organizations = if auth_semantics.can_show_oauth_metadata() {
.as_ref() auth_config
.and_then(|config| config.get("organizations")) .as_ref()
.and_then(serde_json::Value::as_array) .and_then(|config| config.get("organizations"))
.cloned() .and_then(serde_json::Value::as_array)
.unwrap_or_default(); .cloned()
.unwrap_or_default()
} else {
Vec::new()
};
let oauth_plan_type = derive_catalog_oauth_plan_type(key, provider_type, auth_config.as_ref()); let oauth_plan_type = derive_catalog_oauth_plan_type(key, provider_type, auth_config.as_ref());
let ( let (
health_score, health_score,
@@ -387,6 +393,30 @@ pub(crate) fn build_admin_provider_key_response(
); );
payload.insert("api_key_plain".to_string(), serde_json::Value::Null); payload.insert("api_key_plain".to_string(), serde_json::Value::Null);
payload.insert("auth_type".to_string(), json!(key.auth_type)); payload.insert("auth_type".to_string(), json!(key.auth_type));
payload.insert(
"credential_kind".to_string(),
json!(auth_semantics.credential_kind().as_str()),
);
payload.insert(
"runtime_auth_kind".to_string(),
json!(auth_semantics.runtime_auth_kind().as_str()),
);
payload.insert(
"oauth_managed".to_string(),
json!(auth_semantics.oauth_managed()),
);
payload.insert(
"can_refresh_oauth".to_string(),
json!(auth_semantics.can_refresh_oauth()),
);
payload.insert(
"can_export_oauth".to_string(),
json!(auth_semantics.can_export_oauth()),
);
payload.insert(
"can_edit_oauth".to_string(),
json!(auth_semantics.can_edit_oauth()),
);
payload.insert("name".to_string(), json!(key.name)); payload.insert("name".to_string(), json!(key.name));
payload.insert("rate_multipliers".to_string(), json!(key.rate_multipliers)); payload.insert("rate_multipliers".to_string(), json!(key.rate_multipliers));
payload.insert( payload.insert(
@@ -410,40 +440,59 @@ pub(crate) fn build_admin_provider_key_response(
payload.insert("capabilities".to_string(), json!(key.capabilities)); payload.insert("capabilities".to_string(), json!(key.capabilities));
payload.insert( payload.insert(
"oauth_expires_at".to_string(), "oauth_expires_at".to_string(),
json!(key.expires_at_unix_secs), json!(auth_semantics
.can_show_oauth_metadata()
.then_some(key.expires_at_unix_secs)
.flatten()),
); );
payload.insert( payload.insert(
"oauth_email".to_string(), "oauth_email".to_string(),
auth_config if auth_semantics.can_show_oauth_metadata() {
.as_ref() auth_config
.and_then(|config| config.get("email")) .as_ref()
.cloned() .and_then(|config| config.get("email"))
.unwrap_or(serde_json::Value::Null), .cloned()
.unwrap_or(serde_json::Value::Null)
} else {
serde_json::Value::Null
},
); );
payload.insert("oauth_plan_type".to_string(), json!(oauth_plan_type)); payload.insert("oauth_plan_type".to_string(), json!(oauth_plan_type));
payload.insert( payload.insert(
"oauth_account_id".to_string(), "oauth_account_id".to_string(),
auth_config if auth_semantics.can_show_oauth_metadata() {
.as_ref() auth_config
.and_then(|config| config.get("account_id")) .as_ref()
.cloned() .and_then(|config| config.get("account_id"))
.unwrap_or(serde_json::Value::Null), .cloned()
.unwrap_or(serde_json::Value::Null)
} else {
serde_json::Value::Null
},
); );
payload.insert( payload.insert(
"oauth_account_name".to_string(), "oauth_account_name".to_string(),
auth_config if auth_semantics.can_show_oauth_metadata() {
.as_ref() auth_config
.and_then(|config| config.get("account_name")) .as_ref()
.cloned() .and_then(|config| config.get("account_name"))
.unwrap_or(serde_json::Value::Null), .cloned()
.unwrap_or(serde_json::Value::Null)
} else {
serde_json::Value::Null
},
); );
payload.insert( payload.insert(
"oauth_account_user_id".to_string(), "oauth_account_user_id".to_string(),
auth_config if auth_semantics.can_show_oauth_metadata() {
.as_ref() auth_config
.and_then(|config| config.get("account_user_id")) .as_ref()
.cloned() .and_then(|config| config.get("account_user_id"))
.unwrap_or(serde_json::Value::Null), .cloned()
.unwrap_or(serde_json::Value::Null)
} else {
serde_json::Value::Null
},
); );
payload.insert( payload.insert(
"oauth_organizations".to_string(), "oauth_organizations".to_string(),
@@ -451,11 +500,17 @@ pub(crate) fn build_admin_provider_key_response(
); );
payload.insert( payload.insert(
"oauth_invalid_at".to_string(), "oauth_invalid_at".to_string(),
json!(key.oauth_invalid_at_unix_secs), json!(auth_semantics
.can_show_oauth_metadata()
.then_some(key.oauth_invalid_at_unix_secs)
.flatten()),
); );
payload.insert( payload.insert(
"oauth_invalid_reason".to_string(), "oauth_invalid_reason".to_string(),
json!(key.oauth_invalid_reason), json!(auth_semantics
.can_show_oauth_metadata()
.then_some(key.oauth_invalid_reason.clone())
.flatten()),
); );
payload.insert( payload.insert(
"status_snapshot".to_string(), "status_snapshot".to_string(),

View File

@@ -48,6 +48,7 @@ mod log_ids;
mod maintenance; mod maintenance;
pub(crate) mod middleware; pub(crate) mod middleware;
mod model_fetch; mod model_fetch;
mod provider_key_auth;
pub(crate) use aether_provider_transport as provider_transport; pub(crate) use aether_provider_transport as provider_transport;
mod query; mod query;
mod rate_limit; mod rate_limit;

View File

@@ -0,0 +1,230 @@
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ProviderKeyCredentialKind {
RawSecret,
OAuthSession,
ServiceAccount,
}
impl ProviderKeyCredentialKind {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::RawSecret => "raw_secret",
Self::OAuthSession => "oauth_session",
Self::ServiceAccount => "service_account",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ProviderKeyRuntimeAuthKind {
ApiKey,
Bearer,
ServiceAccount,
Unknown,
}
impl ProviderKeyRuntimeAuthKind {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::ApiKey => "api_key",
Self::Bearer => "bearer",
Self::ServiceAccount => "service_account",
Self::Unknown => "unknown",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct ProviderKeyAuthSemantics {
credential_kind: ProviderKeyCredentialKind,
runtime_auth_kind: ProviderKeyRuntimeAuthKind,
oauth_managed: bool,
}
impl ProviderKeyAuthSemantics {
pub(crate) const fn credential_kind(self) -> ProviderKeyCredentialKind {
self.credential_kind
}
pub(crate) const fn runtime_auth_kind(self) -> ProviderKeyRuntimeAuthKind {
self.runtime_auth_kind
}
pub(crate) const fn oauth_managed(self) -> bool {
self.oauth_managed
}
pub(crate) const fn can_refresh_oauth(self) -> bool {
self.oauth_managed
}
pub(crate) const fn can_export_oauth(self) -> bool {
self.oauth_managed
}
pub(crate) const fn can_edit_oauth(self) -> bool {
self.oauth_managed
}
pub(crate) const fn can_show_oauth_metadata(self) -> bool {
self.oauth_managed
}
}
fn normalized_auth_type(key: &StoredProviderCatalogKey) -> String {
key.auth_type.trim().to_ascii_lowercase()
}
fn key_has_auth_config(key: &StoredProviderCatalogKey) -> bool {
key.encrypted_auth_config
.as_deref()
.map(str::trim)
.is_some_and(|value| !value.is_empty())
}
fn provider_uses_bearer_oauth_runtime(provider_type: &str) -> bool {
matches!(
provider_type.trim().to_ascii_lowercase().as_str(),
"claude_code" | "codex" | "gemini_cli" | "antigravity" | "kiro"
)
}
fn provider_key_is_legacy_kiro_oauth_session(
key: &StoredProviderCatalogKey,
provider_type: &str,
auth_type: &str,
) -> bool {
provider_type.trim().eq_ignore_ascii_case("kiro")
&& auth_type.eq_ignore_ascii_case("bearer")
&& key_has_auth_config(key)
}
pub(crate) fn provider_key_auth_semantics(
key: &StoredProviderCatalogKey,
provider_type: &str,
) -> ProviderKeyAuthSemantics {
let auth_type = normalized_auth_type(key);
let oauth_managed = auth_type == "oauth"
|| provider_key_is_legacy_kiro_oauth_session(key, provider_type, &auth_type);
let credential_kind = if oauth_managed {
ProviderKeyCredentialKind::OAuthSession
} else if matches!(auth_type.as_str(), "service_account" | "vertex_ai") {
ProviderKeyCredentialKind::ServiceAccount
} else {
ProviderKeyCredentialKind::RawSecret
};
let runtime_auth_kind = match credential_kind {
ProviderKeyCredentialKind::OAuthSession => {
if provider_uses_bearer_oauth_runtime(provider_type) {
ProviderKeyRuntimeAuthKind::Bearer
} else {
ProviderKeyRuntimeAuthKind::Unknown
}
}
ProviderKeyCredentialKind::ServiceAccount => ProviderKeyRuntimeAuthKind::ServiceAccount,
ProviderKeyCredentialKind::RawSecret => match auth_type.as_str() {
"bearer" => ProviderKeyRuntimeAuthKind::Bearer,
"api_key" => ProviderKeyRuntimeAuthKind::ApiKey,
_ => ProviderKeyRuntimeAuthKind::Unknown,
},
};
ProviderKeyAuthSemantics {
credential_kind,
runtime_auth_kind,
oauth_managed,
}
}
pub(crate) fn provider_key_is_oauth_managed(
key: &StoredProviderCatalogKey,
provider_type: &str,
) -> bool {
provider_key_auth_semantics(key, provider_type).oauth_managed()
}
#[cfg(test)]
mod tests {
use super::{
provider_key_auth_semantics, ProviderKeyCredentialKind, ProviderKeyRuntimeAuthKind,
};
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
fn sample_key(auth_type: &str) -> StoredProviderCatalogKey {
StoredProviderCatalogKey::new(
"key-1".to_string(),
"provider-1".to_string(),
"key-1".to_string(),
auth_type.to_string(),
None,
true,
)
.expect("key should build")
}
#[test]
fn recognizes_oauth_managed_key() {
let semantics = provider_key_auth_semantics(&sample_key("oauth"), "codex");
assert!(semantics.oauth_managed());
assert_eq!(
semantics.credential_kind(),
ProviderKeyCredentialKind::OAuthSession
);
assert_eq!(
semantics.runtime_auth_kind(),
ProviderKeyRuntimeAuthKind::Bearer
);
}
#[test]
fn recognizes_legacy_kiro_bearer_key_with_auth_config_as_oauth_managed() {
let mut key = sample_key("bearer");
key.encrypted_auth_config = Some("ciphertext".to_string());
let semantics = provider_key_auth_semantics(&key, "kiro");
assert!(semantics.oauth_managed());
assert_eq!(
semantics.credential_kind(),
ProviderKeyCredentialKind::OAuthSession
);
assert_eq!(
semantics.runtime_auth_kind(),
ProviderKeyRuntimeAuthKind::Bearer
);
}
#[test]
fn keeps_plain_bearer_key_as_raw_secret() {
let semantics = provider_key_auth_semantics(&sample_key("bearer"), "kiro");
assert!(!semantics.oauth_managed());
assert_eq!(
semantics.credential_kind(),
ProviderKeyCredentialKind::RawSecret
);
assert_eq!(
semantics.runtime_auth_kind(),
ProviderKeyRuntimeAuthKind::Bearer
);
}
#[test]
fn recognizes_service_account_key() {
let semantics = provider_key_auth_semantics(&sample_key("service_account"), "vertex_ai");
assert!(!semantics.oauth_managed());
assert_eq!(
semantics.credential_kind(),
ProviderKeyCredentialKind::ServiceAccount
);
assert_eq!(
semantics.runtime_auth_kind(),
ProviderKeyRuntimeAuthKind::ServiceAccount
);
}
}

View File

@@ -1,5 +1,6 @@
use std::collections::{BTreeMap, BTreeSet}; use std::collections::{BTreeMap, BTreeSet};
use aether_admin::provider::pool as admin_provider_pool_pure;
use aether_data_contracts::repository::candidates::StoredRequestCandidate; use aether_data_contracts::repository::candidates::StoredRequestCandidate;
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey; use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
use aether_scheduler_core::{ use aether_scheduler_core::{
@@ -20,6 +21,7 @@ pub(super) struct CandidateRuntimeSelectionSnapshot {
pub(super) provider_concurrent_limits: BTreeMap<String, usize>, pub(super) provider_concurrent_limits: BTreeMap<String, usize>,
pub(super) provider_key_rpm_states: BTreeMap<String, StoredProviderCatalogKey>, pub(super) provider_key_rpm_states: BTreeMap<String, StoredProviderCatalogKey>,
provider_quota_blocks_requests: BTreeMap<String, bool>, provider_quota_blocks_requests: BTreeMap<String, bool>,
key_account_quota_exhausted: BTreeMap<String, bool>,
provider_key_rpm_reset_ats: BTreeMap<String, Option<u64>>, provider_key_rpm_reset_ats: BTreeMap<String, Option<u64>>,
} }
@@ -30,7 +32,14 @@ pub(super) async fn read_candidate_runtime_selection_snapshot(
) -> Result<CandidateRuntimeSelectionSnapshot, GatewayError> { ) -> Result<CandidateRuntimeSelectionSnapshot, GatewayError> {
let recent_candidates = state.read_recent_request_candidates(128).await?; let recent_candidates = state.read_recent_request_candidates(128).await?;
let provider_concurrent_limits = read_provider_concurrent_limits(state, candidates).await?; let provider_concurrent_limits = read_provider_concurrent_limits(state, candidates).await?;
let provider_skip_exhausted_accounts =
read_provider_skip_exhausted_account_map(state, candidates).await?;
let provider_key_rpm_states = read_provider_key_rpm_states(state, candidates).await?; let provider_key_rpm_states = read_provider_key_rpm_states(state, candidates).await?;
let key_account_quota_exhausted = read_key_account_quota_exhaustion_map(
candidates,
&provider_key_rpm_states,
&provider_skip_exhausted_accounts,
);
let provider_quota_blocks_requests = let provider_quota_blocks_requests =
read_provider_quota_block_map(state, candidates, now_unix_secs).await?; read_provider_quota_block_map(state, candidates, now_unix_secs).await?;
let provider_key_rpm_reset_ats = let provider_key_rpm_reset_ats =
@@ -41,6 +50,7 @@ pub(super) async fn read_candidate_runtime_selection_snapshot(
provider_concurrent_limits, provider_concurrent_limits,
provider_key_rpm_states, provider_key_rpm_states,
provider_quota_blocks_requests, provider_quota_blocks_requests,
key_account_quota_exhausted,
provider_key_rpm_reset_ats, provider_key_rpm_reset_ats,
}) })
} }
@@ -89,6 +99,11 @@ pub(super) fn is_candidate_selectable(
.get(candidate.provider_id.as_str()) .get(candidate.provider_id.as_str())
.copied() .copied()
.unwrap_or(false), .unwrap_or(false),
account_quota_exhausted: snapshot
.key_account_quota_exhausted
.get(candidate.key_id.as_str())
.copied()
.unwrap_or(false),
rpm_reset_at: snapshot rpm_reset_at: snapshot
.provider_key_rpm_reset_ats .provider_key_rpm_reset_ats
.get(candidate.key_id.as_str()) .get(candidate.key_id.as_str())
@@ -122,6 +137,11 @@ pub(super) fn current_candidate_runtime_skip_reason(
now_unix_secs, now_unix_secs,
cached_affinity_target, cached_affinity_target,
provider_quota_blocks_requests, provider_quota_blocks_requests,
account_quota_exhausted: snapshot
.key_account_quota_exhausted
.get(candidate.key_id.as_str())
.copied()
.unwrap_or(false),
rpm_reset_at, rpm_reset_at,
}) })
} }
@@ -192,6 +212,64 @@ async fn read_provider_quota_block_map(
Ok(quota_blocks) Ok(quota_blocks)
} }
async fn read_provider_skip_exhausted_account_map(
state: &(impl SchedulerRuntimeState + ?Sized),
candidates: &[SchedulerMinimalCandidateSelectionCandidate],
) -> Result<BTreeMap<String, bool>, GatewayError> {
let provider_ids = candidates
.iter()
.map(|candidate| candidate.provider_id.clone())
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
if provider_ids.is_empty() {
return Ok(BTreeMap::new());
}
let providers = state
.read_provider_catalog_providers_by_ids(&provider_ids)
.await?;
Ok(providers
.into_iter()
.map(|provider| {
let skip_exhausted_accounts = provider
.config
.as_ref()
.and_then(|value| value.get("pool_advanced"))
.and_then(serde_json::Value::as_object)
.and_then(|value| value.get("skip_exhausted_accounts"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false);
(provider.id, skip_exhausted_accounts)
})
.collect())
}
fn read_key_account_quota_exhaustion_map(
candidates: &[SchedulerMinimalCandidateSelectionCandidate],
provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
provider_skip_exhausted_accounts: &BTreeMap<String, bool>,
) -> BTreeMap<String, bool> {
candidates
.iter()
.map(|candidate| {
let exhausted = provider_skip_exhausted_accounts
.get(candidate.provider_id.as_str())
.copied()
.unwrap_or(false)
&& provider_key_rpm_states
.get(candidate.key_id.as_str())
.is_some_and(|key| {
admin_provider_pool_pure::admin_pool_key_account_quota_exhausted(
key,
candidate.provider_type.as_str(),
)
});
(candidate.key_id.clone(), exhausted)
})
.collect()
}
fn read_provider_key_rpm_reset_at_map( fn read_provider_key_rpm_reset_at_map(
state: &(impl SchedulerRuntimeState + ?Sized), state: &(impl SchedulerRuntimeState + ?Sized),
candidates: &[SchedulerMinimalCandidateSelectionCandidate], candidates: &[SchedulerMinimalCandidateSelectionCandidate],

View File

@@ -1140,6 +1140,253 @@ async fn exposes_runtime_skipped_candidates_with_skip_reasons() {
assert_eq!(skipped[0].skip_reason, "key_circuit_open"); assert_eq!(skipped[0].skip_reason, "key_circuit_open");
} }
#[tokio::test]
async fn skips_codex_candidate_when_account_quota_is_exhausted_and_pool_flag_enabled() {
let mut first = sample_row();
first.provider_id = "provider-codex".to_string();
first.provider_name = "codex".to_string();
first.provider_type = "codex".to_string();
first.endpoint_id = "endpoint-codex".to_string();
first.endpoint_api_format = "openai:cli".to_string();
first.key_id = "key-codex".to_string();
first.key_name = "codex-exhausted".to_string();
first.key_auth_type = "oauth".to_string();
first.key_api_formats = Some(vec!["openai:cli".to_string()]);
first.key_global_priority_by_format = Some(serde_json::json!({"openai:cli": 1}));
let mut second = sample_row();
second.provider_id = "provider-openai".to_string();
second.provider_name = "openai".to_string();
second.endpoint_id = "endpoint-openai".to_string();
second.endpoint_api_format = "openai:cli".to_string();
second.key_id = "key-openai".to_string();
second.key_name = "fallback".to_string();
second.key_api_formats = Some(vec!["openai:cli".to_string()]);
second.key_global_priority_by_format = Some(serde_json::json!({"openai:cli": 2}));
let candidates = Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
first, second,
]));
let mut codex_provider = sample_provider("provider-codex", None);
codex_provider.provider_type = "codex".to_string();
codex_provider.config = Some(serde_json::json!({
"pool_advanced": {
"skip_exhausted_accounts": true
}
}));
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![codex_provider, sample_provider("provider-openai", None)],
Vec::new(),
vec![
{
let mut key = sample_key("key-codex", "provider-codex", Some(10));
key.auth_type = "oauth".to_string();
key.upstream_metadata = Some(serde_json::json!({
"codex": {
"secondary_used_percent": 100.0
}
}));
key
},
sample_key("key-openai", "provider-openai", Some(10)),
],
));
let quotas = Arc::new(InMemoryProviderQuotaRepository::seed(vec![]));
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![]));
let state = AppState::new()
.expect("state should build")
.with_data_state_for_tests(
GatewayDataState::with_candidate_selection_provider_catalog_quota_and_request_candidates_for_tests(
candidates,
provider_catalog,
quotas,
request_candidates,
),
);
let (selected, skipped) = collect_selectable_candidates_with_skip_reasons(
state.data.as_ref(),
&state,
"openai:cli",
"gpt-4.1",
false,
None,
100,
)
.await
.expect("selection should succeed");
assert_eq!(selected.len(), 1);
assert_eq!(selected[0].provider_id, "provider-openai");
assert_eq!(skipped.len(), 1);
assert_eq!(skipped[0].candidate.provider_id, "provider-codex");
assert_eq!(skipped[0].skip_reason, "account_quota_exhausted");
}
#[tokio::test]
async fn keeps_codex_candidate_selectable_when_exhausted_account_flag_is_disabled() {
let mut first = sample_row();
first.provider_id = "provider-codex".to_string();
first.provider_name = "codex".to_string();
first.provider_type = "codex".to_string();
first.endpoint_id = "endpoint-codex".to_string();
first.endpoint_api_format = "openai:cli".to_string();
first.key_id = "key-codex".to_string();
first.key_name = "codex-exhausted".to_string();
first.key_auth_type = "oauth".to_string();
first.key_api_formats = Some(vec!["openai:cli".to_string()]);
first.key_global_priority_by_format = Some(serde_json::json!({"openai:cli": 1}));
let mut second = sample_row();
second.provider_id = "provider-openai".to_string();
second.provider_name = "openai".to_string();
second.endpoint_id = "endpoint-openai".to_string();
second.endpoint_api_format = "openai:cli".to_string();
second.key_id = "key-openai".to_string();
second.key_name = "fallback".to_string();
second.key_api_formats = Some(vec!["openai:cli".to_string()]);
second.key_global_priority_by_format = Some(serde_json::json!({"openai:cli": 2}));
let candidates = Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
first, second,
]));
let mut codex_provider = sample_provider("provider-codex", None);
codex_provider.provider_type = "codex".to_string();
codex_provider.config = Some(serde_json::json!({
"pool_advanced": {}
}));
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![codex_provider, sample_provider("provider-openai", None)],
Vec::new(),
vec![
{
let mut key = sample_key("key-codex", "provider-codex", Some(10));
key.auth_type = "oauth".to_string();
key.upstream_metadata = Some(serde_json::json!({
"codex": {
"secondary_used_percent": 100.0
}
}));
key
},
sample_key("key-openai", "provider-openai", Some(10)),
],
));
let quotas = Arc::new(InMemoryProviderQuotaRepository::seed(vec![]));
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![]));
let state = AppState::new()
.expect("state should build")
.with_data_state_for_tests(
GatewayDataState::with_candidate_selection_provider_catalog_quota_and_request_candidates_for_tests(
candidates,
provider_catalog,
quotas,
request_candidates,
),
);
let (selected, skipped) = collect_selectable_candidates_with_skip_reasons(
state.data.as_ref(),
&state,
"openai:cli",
"gpt-4.1",
false,
None,
100,
)
.await
.expect("selection should succeed");
assert_eq!(selected.len(), 2);
assert!(selected
.iter()
.any(|candidate| candidate.provider_id == "provider-codex"));
assert!(skipped.is_empty());
}
#[tokio::test]
async fn skips_kiro_candidate_when_account_quota_is_exhausted_and_pool_flag_enabled() {
let mut first = sample_row();
first.provider_id = "provider-kiro".to_string();
first.provider_name = "kiro".to_string();
first.provider_type = "kiro".to_string();
first.endpoint_id = "endpoint-kiro".to_string();
first.endpoint_api_format = "claude:cli".to_string();
first.key_id = "key-kiro".to_string();
first.key_name = "kiro-exhausted".to_string();
first.key_auth_type = "oauth".to_string();
first.key_api_formats = Some(vec!["claude:cli".to_string()]);
first.key_global_priority_by_format = Some(serde_json::json!({"claude:cli": 1}));
let mut second = sample_row();
second.provider_id = "provider-openai".to_string();
second.provider_name = "openai".to_string();
second.endpoint_id = "endpoint-openai".to_string();
second.endpoint_api_format = "claude:cli".to_string();
second.key_id = "key-openai".to_string();
second.key_name = "fallback".to_string();
second.key_api_formats = Some(vec!["claude:cli".to_string()]);
second.key_global_priority_by_format = Some(serde_json::json!({"claude:cli": 2}));
let candidates = Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
first, second,
]));
let mut kiro_provider = sample_provider("provider-kiro", None);
kiro_provider.provider_type = "kiro".to_string();
kiro_provider.config = Some(serde_json::json!({
"pool_advanced": {
"skip_exhausted_accounts": true
}
}));
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![kiro_provider, sample_provider("provider-openai", None)],
Vec::new(),
vec![
{
let mut key = sample_key("key-kiro", "provider-kiro", Some(10));
key.auth_type = "oauth".to_string();
key.upstream_metadata = Some(serde_json::json!({
"kiro": {
"remaining": 0
}
}));
key
},
sample_key("key-openai", "provider-openai", Some(10)),
],
));
let quotas = Arc::new(InMemoryProviderQuotaRepository::seed(vec![]));
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![]));
let state = AppState::new()
.expect("state should build")
.with_data_state_for_tests(
GatewayDataState::with_candidate_selection_provider_catalog_quota_and_request_candidates_for_tests(
candidates,
provider_catalog,
quotas,
request_candidates,
),
);
let (selected, skipped) = collect_selectable_candidates_with_skip_reasons(
state.data.as_ref(),
&state,
"claude:cli",
"gpt-4.1",
false,
None,
100,
)
.await
.expect("selection should succeed");
assert_eq!(selected.len(), 1);
assert_eq!(selected[0].provider_id, "provider-openai");
assert_eq!(skipped.len(), 1);
assert_eq!(skipped[0].candidate.provider_id, "provider-kiro");
assert_eq!(skipped[0].skip_reason, "account_quota_exhausted");
}
#[tokio::test] #[tokio::test]
async fn same_priority_candidates_prefer_healthier_provider_key_before_id_order() { async fn same_priority_candidates_prefer_healthier_provider_key_before_id_order() {
let mut first = sample_row(); let mut first = sample_row();

View File

@@ -1027,6 +1027,82 @@ async fn gateway_batch_deletes_admin_global_models_locally_with_trusted_admin_pr
upstream_handle.abort(); upstream_handle.abort();
} }
#[tokio::test]
async fn gateway_deletes_admin_global_model_with_bound_provider_models_locally() {
let upstream_hits = Arc::new(Mutex::new(0usize));
let upstream_hits_clone = Arc::clone(&upstream_hits);
let upstream = Router::new().route(
"/api/admin/models/global/global-gpt-5",
any(move |_request: Request| {
let upstream_hits_inner = Arc::clone(&upstream_hits_clone);
async move {
*upstream_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::OK, Body::from("unexpected upstream hit"))
}
}),
);
let global_model_repository = Arc::new(
InMemoryGlobalModelReadRepository::seed(Vec::new())
.with_admin_global_models(vec![sample_admin_global_model(
"global-gpt-5",
"gpt-5",
"GPT 5",
)])
.with_admin_provider_models(vec![sample_admin_provider_model(
"model-openai-gpt5",
"provider-openai",
"global-gpt-5",
"gpt-5-upstream",
)]),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let gateway = build_router_with_state(
AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(
GatewayDataState::disabled()
.with_global_model_repository_for_tests(global_model_repository.clone()),
),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.delete(format!(
"{gateway_url}/api/admin/models/global/global-gpt-5"
))
.header(crate::constants::GATEWAY_HEADER, "rust-phase3b")
.header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123")
.header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin")
.header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::NO_CONTENT);
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
let deleted = global_model_repository
.get_admin_global_model_by_id("global-gpt-5")
.await
.expect("model lookup should succeed");
assert!(deleted.is_none());
let provider_models = global_model_repository
.list_admin_provider_models(&AdminProviderModelListQuery {
provider_id: "provider-openai".to_string(),
is_active: None,
offset: 0,
limit: 20,
})
.await
.expect("provider models should read");
assert!(provider_models.is_empty());
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test] #[tokio::test]
async fn gateway_assigns_admin_global_model_to_providers_locally_with_trusted_admin_principal() { async fn gateway_assigns_admin_global_model_to_providers_locally_with_trusted_admin_principal() {
let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits = Arc::new(Mutex::new(0usize));

View File

@@ -1792,6 +1792,51 @@ async fn gateway_imports_admin_provider_oauth_refresh_token_locally_with_trusted
upstream_handle.abort(); upstream_handle.abort();
} }
#[tokio::test]
async fn gateway_rejects_kiro_single_refresh_token_import_with_clear_error() {
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![{
let mut provider = sample_provider("provider-kiro", "kiro", 10);
provider.provider_type = "kiro".to_string();
provider
}],
vec![sample_endpoint(
"endpoint-kiro-chat",
"provider-kiro",
"kiro:generateAssistantResponse",
"https://service.kiro.dev",
)],
Vec::new(),
));
let state = AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests(
provider_catalog_repository,
));
let response = local_admin_provider_oauth_response(
&state,
http::Method::POST,
"/api/admin/provider-oauth/providers/provider-kiro/import-refresh-token",
Some(json!({
"refresh_token": "kiro-refresh-token"
})),
)
.await;
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let payload: serde_json::Value = serde_json::from_slice(
&to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read"),
)
.expect("json body should parse");
assert_eq!(
payload["detail"],
json!("Kiro 不支持单条 Refresh Token 导入,请使用批量导入或设备授权。")
);
}
#[tokio::test] #[tokio::test]
async fn gateway_imports_admin_provider_oauth_refresh_token_via_execution_runtime_proxy_node() { async fn gateway_imports_admin_provider_oauth_refresh_token_via_execution_runtime_proxy_node() {
let execution_plans = Arc::new(Mutex::new(Vec::<ExecutionPlan>::new())); let execution_plans = Arc::new(Mutex::new(Vec::<ExecutionPlan>::new()));
@@ -2382,20 +2427,37 @@ async fn gateway_batch_imports_admin_provider_oauth_kiro_via_execution_runtime_p
.lock() .lock()
.expect("mutex should lock") .expect("mutex should lock")
.push(plan.clone()); .push(plan.clone());
assert_eq!(plan.request_id, "kiro_batch_refresh:social");
assert_eq!(plan.url, "https://oauth.example/refreshToken");
assert_eq!( assert_eq!(
plan.proxy plan.proxy
.as_ref() .as_ref()
.and_then(|proxy| proxy.node_id.as_deref()), .and_then(|proxy| proxy.node_id.as_deref()),
Some("proxy-node-kiro-batch-runtime") Some("proxy-node-kiro-batch-runtime")
); );
assert_eq!( if plan.request_id == "kiro_batch_refresh:social" {
plan.headers assert_eq!(plan.url, "https://oauth.example/refreshToken");
.get(EXECUTION_REQUEST_FOLLOW_REDIRECTS_HEADER) assert_eq!(
.map(String::as_str), plan.headers
Some("true") .get(EXECUTION_REQUEST_FOLLOW_REDIRECTS_HEADER)
); .map(String::as_str),
Some("true")
);
return Json(json!({
"request_id": plan.request_id,
"status_code": 200,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {
"accessToken": sample_kiro_device_access_token("kiro-runtime@example.com"),
"refreshToken": "kiro-runtime-refresh-token-new",
"expiresIn": 1800,
}
}
}));
}
assert_eq!(plan.request_id, "kiro-quota:key-kiro-batch-runtime");
Json(json!({ Json(json!({
"request_id": plan.request_id, "request_id": plan.request_id,
"status_code": 200, "status_code": 200,
@@ -2404,9 +2466,17 @@ async fn gateway_batch_imports_admin_provider_oauth_kiro_via_execution_runtime_p
}, },
"body": { "body": {
"json_body": { "json_body": {
"accessToken": sample_kiro_device_access_token("kiro-runtime@example.com"), "subscriptionInfo": {
"refreshToken": "kiro-runtime-refresh-token-new", "subscriptionTitle": "KIRO PRO+"
"expiresIn": 1800, },
"usageBreakdownList": [{
"currentUsageWithPrecision": 5.0,
"usageLimitWithPrecision": 20.0,
"nextDateReset": 1_900_000_000u64
}],
"desktopUserInfo": {
"email": "kiro-runtime@example.com"
}
} }
} }
})) }))
@@ -2496,7 +2566,7 @@ async fn gateway_batch_imports_admin_provider_oauth_kiro_via_execution_runtime_p
{ {
let plans = execution_plans.lock().expect("mutex should lock"); let plans = execution_plans.lock().expect("mutex should lock");
assert_eq!(plans.len(), 1); assert_eq!(plans.len(), 2);
} }
let stored_key = provider_catalog_repository let stored_key = provider_catalog_repository

View File

@@ -1109,6 +1109,162 @@ async fn gateway_formats_codex_quota_countdown_from_reset_after_seconds() {
); );
} }
#[tokio::test]
async fn gateway_marks_exhausted_codex_pool_key_as_blocked_when_flag_enabled() {
let mut provider = sample_provider("provider-codex", "codex", 10).with_transport_fields(
true,
false,
true,
None,
None,
None,
None,
None,
Some(json!({
"pool_advanced": {
"enabled": true,
"skip_exhausted_accounts": true
}
})),
);
provider.provider_type = "codex".to_string();
let mut key = sample_key(
"key-codex-exhausted",
"provider-codex",
"openai:cli",
"oauth-placeholder",
);
key.name = "codex exhausted".to_string();
key.auth_type = "oauth".to_string();
key.upstream_metadata = Some(json!({
"codex": {
"secondary_used_percent": 100.0,
"plan_type": "plus"
}
}));
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![provider],
Vec::new(),
vec![key],
));
let state = AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests(
provider_catalog_repository,
));
let response = local_admin_pool_response(
&state,
http::Method::GET,
"/api/admin/pool/provider-codex/keys?page=1&page_size=50&status=all",
None,
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = serde_json::from_slice(
&to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read"),
)
.expect("json body should parse");
let keys = payload["keys"].as_array().expect("keys should be array");
assert_eq!(keys[0]["scheduling_status"], json!("blocked"));
assert_eq!(
keys[0]["scheduling_reason"],
json!("account_quota_exhausted")
);
assert_eq!(keys[0]["scheduling_label"], json!("额度耗尽"));
assert_eq!(
keys[0]["scheduling_reasons"][0],
json!({
"code": "account_quota_exhausted",
"label": "额度耗尽",
"blocking": true,
"source": "quota",
"ttl_seconds": serde_json::Value::Null,
"detail": serde_json::Value::Null,
})
);
assert_eq!(keys[0]["account_quota"], json!("5H剩余 0.0%"));
}
#[tokio::test]
async fn gateway_marks_exhausted_kiro_pool_key_as_blocked_when_flag_enabled() {
let mut provider = sample_provider("provider-kiro", "kiro", 10).with_transport_fields(
true,
false,
true,
None,
None,
None,
None,
None,
Some(json!({
"pool_advanced": {
"enabled": true,
"skip_exhausted_accounts": true
}
})),
);
provider.provider_type = "kiro".to_string();
let mut key = sample_key(
"key-kiro-exhausted",
"provider-kiro",
"claude:cli",
"oauth-placeholder",
);
key.name = "kiro exhausted".to_string();
key.auth_type = "oauth".to_string();
key.upstream_metadata = Some(json!({
"kiro": {
"remaining": 0.0,
"usage_limit": 100.0,
"current_usage": 100.0
}
}));
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![provider],
Vec::new(),
vec![key],
));
let state = AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests(
provider_catalog_repository,
));
let response = local_admin_pool_response(
&state,
http::Method::GET,
"/api/admin/pool/provider-kiro/keys?page=1&page_size=50&status=all",
None,
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = serde_json::from_slice(
&to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read"),
)
.expect("json body should parse");
let keys = payload["keys"].as_array().expect("keys should be array");
assert_eq!(keys[0]["scheduling_status"], json!("blocked"));
assert_eq!(
keys[0]["scheduling_reason"],
json!("account_quota_exhausted")
);
assert_eq!(keys[0]["scheduling_label"], json!("额度耗尽"));
assert_eq!(keys[0]["account_quota"], json!("剩余 0/100"));
}
#[tokio::test] #[tokio::test]
async fn gateway_codex_quota_resets_to_full_after_countdown_elapsed() { async fn gateway_codex_quota_resets_to_full_after_countdown_elapsed() {
let mut provider = sample_provider("provider-codex", "codex", 10).with_transport_fields( let mut provider = sample_provider("provider-codex", "codex", 10).with_transport_fields(
@@ -1427,6 +1583,65 @@ async fn gateway_handles_admin_pool_resolve_selection_locally_with_trusted_admin
upstream_handle.abort(); upstream_handle.abort();
} }
#[tokio::test]
async fn gateway_resolve_selection_marks_legacy_kiro_bearer_keys_as_oauth_managed() {
let mut provider = sample_provider("provider-kiro", "kiro", 10);
provider.provider_type = "kiro".to_string();
let mut key = sample_key(
"key-kiro-legacy",
"provider-kiro",
"kiro:generateAssistantResponse",
"kiro-access-token",
);
key.auth_type = "bearer".to_string();
key.encrypted_auth_config = Some(
encrypt_python_fernet_plaintext(
DEVELOPMENT_ENCRYPTION_KEY,
r#"{"provider_type":"kiro","email":"legacy-kiro@example.com","refresh_token":"legacy-kiro-refresh-token"}"#,
)
.expect("auth config ciphertext should build"),
);
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![provider],
Vec::new(),
vec![key],
));
let state = AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(
GatewayDataState::with_provider_catalog_reader_for_tests(provider_catalog_repository)
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
);
let response = local_admin_pool_response(
&state,
http::Method::POST,
"/api/admin/pool/provider-kiro/keys/resolve-selection",
Some(json!({})),
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = serde_json::from_slice(
&to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read"),
)
.expect("json body should parse");
assert_eq!(payload["total"], json!(1));
let items = payload["items"].as_array().expect("items should be array");
assert_eq!(items.len(), 1);
assert_eq!(items[0]["key_id"], json!("key-kiro-legacy"));
assert_eq!(items[0]["auth_type"], json!("bearer"));
assert_eq!(items[0]["credential_kind"], json!("oauth_session"));
assert_eq!(items[0]["runtime_auth_kind"], json!("bearer"));
assert_eq!(items[0]["oauth_managed"], json!(true));
assert_eq!(items[0]["can_refresh_oauth"], json!(true));
assert_eq!(items[0]["can_export_oauth"], json!(true));
assert_eq!(items[0]["can_edit_oauth"], json!(true));
}
#[tokio::test] #[tokio::test]
async fn gateway_handles_admin_pool_batch_action_locally_with_trusted_admin_principal() { async fn gateway_handles_admin_pool_batch_action_locally_with_trusted_admin_principal() {
let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits = Arc::new(Mutex::new(0usize));

View File

@@ -7,6 +7,7 @@ use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogEn
use axum::body::Body; use axum::body::Body;
use axum::routing::any; use axum::routing::any;
use axum::{extract::Request, Json, Router}; use axum::{extract::Request, Json, Router};
use base64::Engine as _;
use http::StatusCode; use http::StatusCode;
use serde_json::json; use serde_json::json;
@@ -20,6 +21,56 @@ use crate::constants::{
}; };
use crate::data::GatewayDataState; use crate::data::GatewayDataState;
fn crc32(data: &[u8]) -> u32 {
let mut crc = 0xffff_ffffu32;
for &byte in data {
crc ^= byte as u32;
for _ in 0..8 {
let mask = if crc & 1 == 1 { 0xedb8_8320 } else { 0 };
crc = (crc >> 1) ^ mask;
}
}
!crc
}
fn encode_string_header(name: &str, value: &str) -> Vec<u8> {
let mut out = Vec::new();
out.push(name.len() as u8);
out.extend_from_slice(name.as_bytes());
out.push(7);
out.extend_from_slice(&(value.len() as u16).to_be_bytes());
out.extend_from_slice(value.as_bytes());
out
}
fn encode_frame(headers: Vec<u8>, payload: Vec<u8>) -> Vec<u8> {
let total_len = 12 + headers.len() + payload.len() + 4;
let header_len = headers.len();
let mut out = Vec::with_capacity(total_len);
out.extend_from_slice(&(total_len as u32).to_be_bytes());
out.extend_from_slice(&(header_len as u32).to_be_bytes());
let prelude_crc = crc32(&out[..8]);
out.extend_from_slice(&prelude_crc.to_be_bytes());
out.extend_from_slice(&headers);
out.extend_from_slice(&payload);
let message_crc = crc32(&out);
out.extend_from_slice(&message_crc.to_be_bytes());
out
}
fn encode_kiro_event_frame(event_type: &str, payload: serde_json::Value) -> Vec<u8> {
let mut headers = encode_string_header(":message-type", "event");
headers.extend_from_slice(&encode_string_header(":event-type", event_type));
let payload = serde_json::to_vec(&payload).expect("payload should encode");
encode_frame(headers, payload)
}
fn encode_kiro_exception_frame(exception_type: &str) -> Vec<u8> {
let mut headers = encode_string_header(":message-type", "exception");
headers.extend_from_slice(&encode_string_header(":exception-type", exception_type));
encode_frame(headers, Vec::new())
}
async fn assert_admin_provider_query_route( async fn assert_admin_provider_query_route(
path: &str, path: &str,
request_payload: serde_json::Value, request_payload: serde_json::Value,
@@ -676,6 +727,277 @@ async fn gateway_handles_admin_provider_query_test_model_failover_locally_with_t
.await; .await;
} }
#[tokio::test]
async fn gateway_handles_admin_provider_query_test_model_for_kiro_locally() {
let execution_runtime = Router::new().route(
"/v1/execute/sync",
any(move |Json(plan): Json<ExecutionPlan>| async move {
assert_eq!(plan.provider_id, "provider-kiro");
assert_eq!(plan.endpoint_id, "endpoint-kiro-cli");
assert_eq!(plan.key_id, "key-kiro-primary");
assert_eq!(plan.provider_api_format, "claude:cli");
assert_eq!(plan.model_name.as_deref(), Some("claude-sonnet-4-upstream"));
Json(json!({
"request_id": plan.request_id,
"candidate_id": plan.candidate_id,
"status_code": 200,
"headers": {
"content-type": "application/vnd.amazon.eventstream"
},
"body": {
"body_bytes_b64": base64::engine::general_purpose::STANDARD.encode(
[
encode_kiro_event_frame("assistantResponseEvent", json!({"content": "Hello from Kiro"})),
encode_kiro_exception_frame("ContentLengthExceededException"),
]
.concat()
)
},
"telemetry": {
"elapsed_ms": 42
}
}))
}),
);
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let mut provider = sample_provider("provider-kiro", "Kiro", 10);
provider.provider_type = "kiro".to_string();
let mut key = sample_key(
"key-kiro-primary",
"provider-kiro",
"claude:cli",
"__placeholder__",
);
key.auth_type = "oauth".to_string();
key.encrypted_auth_config = Some(
aether_crypto::encrypt_python_fernet_plaintext(
DEVELOPMENT_ENCRYPTION_KEY,
r#"{
"provider_type":"kiro",
"auth_method":"idc",
"access_token":"cached-kiro-token",
"refresh_token":"rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr",
"machine_id":"123e4567-e89b-12d3-a456-426614174000",
"api_region":"us-east-1",
"client_id":"client-id",
"client_secret":"client-secret"
}"#,
)
.expect("auth config should encrypt"),
);
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![provider],
vec![StoredProviderCatalogEndpoint::new(
"endpoint-kiro-cli".to_string(),
"provider-kiro".to_string(),
"claude:cli".to_string(),
Some("claude".to_string()),
Some("cli".to_string()),
true,
)
.expect("endpoint should build")
.with_transport_fields(
"https://q.{region}.amazonaws.com".to_string(),
None,
None,
None,
None,
None,
None,
None,
)
.expect("endpoint transport should build")],
vec![key],
));
let gateway = build_router_with_state(
build_state_with_execution_runtime_override(execution_runtime_url)
.with_data_state_for_tests(GatewayDataState::with_provider_transport_reader_for_tests(
provider_catalog_repository,
DEVELOPMENT_ENCRYPTION_KEY.to_string(),
)),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/api/admin/provider-query/test-model"))
.header(GATEWAY_HEADER, "rust-phase3b")
.header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123")
.header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin")
.header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123")
.json(&json!({
"provider_id": "provider-kiro",
"model_name": "claude-sonnet-4-upstream",
"api_format": "claude:cli"
}))
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = response.json().await.expect("json body should parse");
assert_eq!(payload["success"], json!(true));
assert_eq!(payload["provider"]["id"], json!("provider-kiro"));
assert_eq!(payload["model"], json!("claude-sonnet-4-upstream"));
assert_eq!(
payload["data"]["response"]["content"][0]["text"],
json!("Hello from Kiro")
);
gateway_handle.abort();
execution_runtime_handle.abort();
}
#[tokio::test]
async fn gateway_handles_admin_provider_query_test_model_failover_for_kiro_locally() {
let execution_runtime = Router::new().route(
"/v1/execute/sync",
any(move |Json(plan): Json<ExecutionPlan>| async move {
let payload = if plan.key_id == "key-kiro-first" {
json!({
"request_id": plan.request_id,
"candidate_id": plan.candidate_id,
"status_code": 429,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {
"message": "too many requests"
}
},
"telemetry": {
"elapsed_ms": 11
}
})
} else {
json!({
"request_id": plan.request_id,
"candidate_id": plan.candidate_id,
"status_code": 200,
"headers": {
"content-type": "application/vnd.amazon.eventstream"
},
"body": {
"body_bytes_b64": base64::engine::general_purpose::STANDARD.encode(
[
encode_kiro_event_frame("assistantResponseEvent", json!({"content": "Recovered from failover"})),
encode_kiro_exception_frame("ContentLengthExceededException"),
]
.concat()
)
},
"telemetry": {
"elapsed_ms": 27
}
})
};
Json(payload)
}),
);
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let mut provider = sample_provider("provider-kiro", "Kiro", 10);
provider.provider_type = "kiro".to_string();
let build_key = |id: &str| {
let mut key = sample_key(id, "provider-kiro", "claude:cli", "__placeholder__");
key.auth_type = "oauth".to_string();
key.encrypted_auth_config = Some(
aether_crypto::encrypt_python_fernet_plaintext(
DEVELOPMENT_ENCRYPTION_KEY,
r#"{
"provider_type":"kiro",
"auth_method":"idc",
"access_token":"cached-kiro-token",
"refresh_token":"rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr",
"machine_id":"123e4567-e89b-12d3-a456-426614174000",
"api_region":"us-east-1",
"client_id":"client-id",
"client_secret":"client-secret"
}"#,
)
.expect("auth config should encrypt"),
);
key
};
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![provider],
vec![StoredProviderCatalogEndpoint::new(
"endpoint-kiro-cli".to_string(),
"provider-kiro".to_string(),
"claude:cli".to_string(),
Some("claude".to_string()),
Some("cli".to_string()),
true,
)
.expect("endpoint should build")
.with_transport_fields(
"https://q.{region}.amazonaws.com".to_string(),
None,
None,
None,
None,
None,
None,
None,
)
.expect("endpoint transport should build")],
vec![build_key("key-kiro-first"), build_key("key-kiro-second")],
));
let gateway = build_router_with_state(
build_state_with_execution_runtime_override(execution_runtime_url)
.with_data_state_for_tests(GatewayDataState::with_provider_transport_reader_for_tests(
provider_catalog_repository,
DEVELOPMENT_ENCRYPTION_KEY.to_string(),
)),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!(
"{gateway_url}/api/admin/provider-query/test-model-failover"
))
.header(GATEWAY_HEADER, "rust-phase3b")
.header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123")
.header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin")
.header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123")
.json(&json!({
"provider_id": "provider-kiro",
"mode": "direct",
"model_name": "claude-sonnet-4-upstream",
"failover_models": ["claude-sonnet-4-upstream"],
"api_format": "claude:cli",
"request_id": "provider-test-kiro"
}))
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = response.json().await.expect("json body should parse");
assert_eq!(payload["success"], json!(true));
assert_eq!(payload["total_candidates"], json!(2));
assert_eq!(payload["total_attempts"], json!(2));
let attempts = payload["attempts"]
.as_array()
.expect("attempts should be an array");
assert_eq!(attempts.len(), 2);
assert_eq!(attempts[0]["status"], json!("failed"));
assert_eq!(attempts[0]["status_code"], json!(429));
assert_eq!(attempts[1]["status"], json!("success"));
assert_eq!(
payload["data"]["response"]["content"][0]["text"],
json!("Recovered from failover")
);
gateway_handle.abort();
execution_runtime_handle.abort();
}
#[tokio::test] #[tokio::test]
async fn gateway_handles_admin_provider_query_test_model_failover_with_single_model_name_alias() { async fn gateway_handles_admin_provider_query_test_model_failover_with_single_model_name_alias() {
assert_admin_provider_query_route( assert_admin_provider_query_route(

View File

@@ -90,6 +90,81 @@ fn admin_pool_reason_indicates_ban(reason: &str) -> bool {
.any(|hint| normalized.contains(hint)) .any(|hint| normalized.contains(hint))
} }
fn admin_pool_metadata_bucket<'a>(
upstream_metadata: Option<&'a Value>,
provider_type: &str,
) -> Option<&'a serde_json::Map<String, Value>> {
upstream_metadata
.and_then(Value::as_object)
.and_then(|metadata| metadata.get(&provider_type.trim().to_ascii_lowercase()))
.and_then(Value::as_object)
}
fn admin_pool_json_bool(value: Option<&Value>) -> Option<bool> {
match value {
Some(Value::Bool(value)) => Some(*value),
Some(Value::String(value)) => match value.trim().to_ascii_lowercase().as_str() {
"true" | "1" => Some(true),
"false" | "0" => Some(false),
_ => None,
},
_ => None,
}
}
fn admin_pool_json_f64(value: Option<&Value>) -> Option<f64> {
match value {
Some(Value::Number(number)) => number.as_f64(),
Some(Value::String(value)) => value.trim().parse::<f64>().ok(),
_ => None,
}
.filter(|value| value.is_finite())
}
pub fn admin_pool_key_account_quota_exhausted(
key: &StoredProviderCatalogKey,
provider_type: &str,
) -> bool {
let provider_type = provider_type.trim().to_ascii_lowercase();
let Some(bucket) = admin_pool_metadata_bucket(key.upstream_metadata.as_ref(), &provider_type)
else {
return false;
};
match provider_type.as_str() {
"codex" => {
if admin_pool_json_bool(bucket.get("credits_unlimited")) == Some(true) {
return false;
}
if admin_pool_json_bool(bucket.get("has_credits")) == Some(false) {
return true;
}
admin_pool_json_f64(bucket.get("primary_used_percent"))
.is_some_and(|value| value >= 100.0)
|| admin_pool_json_f64(bucket.get("secondary_used_percent"))
.is_some_and(|value| value >= 100.0)
}
"kiro" => {
if admin_pool_json_f64(bucket.get("remaining")).is_some_and(|value| value <= 0.0) {
return true;
}
if admin_pool_json_f64(bucket.get("usage_percentage"))
.is_some_and(|value| value >= 100.0)
{
return true;
}
match (
admin_pool_json_f64(bucket.get("usage_limit")),
admin_pool_json_f64(bucket.get("current_usage")),
) {
(Some(limit), Some(current)) if limit > 0.0 => current >= limit,
_ => false,
}
}
_ => false,
}
}
fn admin_pool_has_proxy(key: &StoredProviderCatalogKey) -> bool { fn admin_pool_has_proxy(key: &StoredProviderCatalogKey) -> bool {
match key.proxy.as_ref() { match key.proxy.as_ref() {
Some(Value::Object(values)) => !values.is_empty(), Some(Value::Object(values)) => !values.is_empty(),
@@ -525,6 +600,103 @@ pub fn build_admin_pool_selection_payload(keys: &[StoredProviderCatalogKey]) ->
}) })
} }
#[cfg(test)]
#[allow(clippy::items_after_test_module)]
mod tests {
use super::admin_pool_key_account_quota_exhausted;
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
use serde_json::json;
fn sample_key(upstream_metadata: Option<serde_json::Value>) -> StoredProviderCatalogKey {
let mut key = StoredProviderCatalogKey::new(
"key-1".to_string(),
"provider-1".to_string(),
"key-1".to_string(),
"oauth".to_string(),
None,
true,
)
.expect("key should build");
key.upstream_metadata = upstream_metadata;
key
}
#[test]
fn detects_codex_exhaustion_from_metadata() {
assert!(admin_pool_key_account_quota_exhausted(
&sample_key(Some(json!({
"codex": {
"has_credits": false,
"credits_unlimited": false
}
}))),
"codex",
));
assert!(admin_pool_key_account_quota_exhausted(
&sample_key(Some(json!({
"codex": {
"primary_used_percent": 100.0
}
}))),
"codex",
));
assert!(admin_pool_key_account_quota_exhausted(
&sample_key(Some(json!({
"codex": {
"secondary_used_percent": 100.0
}
}))),
"codex",
));
assert!(!admin_pool_key_account_quota_exhausted(
&sample_key(Some(json!({
"codex": {
"has_credits": false,
"credits_unlimited": true
}
}))),
"codex",
));
assert!(!admin_pool_key_account_quota_exhausted(
&sample_key(None),
"codex",
));
}
#[test]
fn detects_kiro_exhaustion_from_metadata() {
assert!(admin_pool_key_account_quota_exhausted(
&sample_key(Some(json!({
"kiro": {
"remaining": 0
}
}))),
"kiro",
));
assert!(admin_pool_key_account_quota_exhausted(
&sample_key(Some(json!({
"kiro": {
"usage_percentage": 100.0
}
}))),
"kiro",
));
assert!(admin_pool_key_account_quota_exhausted(
&sample_key(Some(json!({
"kiro": {
"usage_limit": 100.0,
"current_usage": 100.0
}
}))),
"kiro",
));
assert!(!admin_pool_key_account_quota_exhausted(
&sample_key(None),
"kiro",
));
}
}
pub fn build_admin_pool_key_payload( pub fn build_admin_pool_key_payload(
key: &StoredProviderCatalogKey, key: &StoredProviderCatalogKey,
context: &AdminPoolKeyPayloadContext, context: &AdminPoolKeyPayloadContext,

View File

@@ -719,6 +719,19 @@ RETURNING id
&self, &self,
global_model_id: &str, global_model_id: &str,
) -> Result<bool, DataLayerError> { ) -> Result<bool, DataLayerError> {
let mut tx = self.pool.begin().await.map_postgres_err()?;
sqlx::query(
r#"
DELETE FROM models
WHERE global_model_id = $1
"#,
)
.bind(global_model_id)
.execute(&mut *tx)
.await
.map_postgres_err()?;
let deleted = sqlx::query( let deleted = sqlx::query(
r#" r#"
DELETE FROM global_models DELETE FROM global_models
@@ -727,10 +740,12 @@ RETURNING id
"#, "#,
) )
.bind(global_model_id) .bind(global_model_id)
.fetch_optional(&self.pool) .fetch_optional(&mut *tx)
.await .await
.map_postgres_err()?; .map_postgres_err()?;
tx.commit().await.map_postgres_err()?;
Ok(deleted.is_some()) Ok(deleted.is_some())
} }
} }

View File

@@ -53,12 +53,7 @@ pub fn resolve_local_kiro_bearer_auth(
if transport.key.decrypted_auth_config.is_some() { if transport.key.decrypted_auth_config.is_some() {
return None; return None;
} }
if !transport if !kiro_auth_type_supported(transport.key.auth_type.as_str()) {
.key
.auth_type
.trim()
.eq_ignore_ascii_case("bearer")
{
return None; return None;
} }
@@ -90,12 +85,7 @@ pub fn resolve_local_kiro_request_auth(
{ {
return None; return None;
} }
if !transport if !kiro_auth_type_supported(transport.key.auth_type.as_str()) {
.key
.auth_type
.trim()
.eq_ignore_ascii_case("bearer")
{
return None; return None;
} }
@@ -137,15 +127,18 @@ pub fn supports_local_kiro_request_auth_resolution(
.provider_type .provider_type
.trim() .trim()
.eq_ignore_ascii_case(PROVIDER_TYPE) .eq_ignore_ascii_case(PROVIDER_TYPE)
&& transport && kiro_auth_type_supported(transport.key.auth_type.as_str())
.key
.auth_type
.trim()
.eq_ignore_ascii_case("bearer")
&& auth_config.can_refresh_access_token() && auth_config.can_refresh_access_token()
}) })
} }
fn kiro_auth_type_supported(auth_type: &str) -> bool {
matches!(
auth_type.trim().to_ascii_lowercase().as_str(),
"bearer" | "oauth"
)
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::super::super::snapshot::{ use super::super::super::snapshot::{
@@ -235,6 +228,27 @@ mod tests {
assert!(resolve_local_kiro_bearer_auth(&transport).is_none()); assert!(resolve_local_kiro_bearer_auth(&transport).is_none());
} }
#[test]
fn resolves_request_auth_when_legacy_oauth_auth_type_is_used() {
let mut transport = sample_transport();
transport.key.auth_type = "oauth".to_string();
transport.key.decrypted_api_key = "__placeholder__".to_string();
transport.key.decrypted_auth_config = Some(
r#"{
"access_token":"cached-token",
"refresh_token":"rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr",
"machine_id":"123e4567-e89b-12d3-a456-426614174000",
"api_region":"us-west-2"
}"#
.to_string(),
);
let auth = resolve_local_kiro_request_auth(&transport)
.expect("request auth should resolve from legacy oauth auth_type");
assert_eq!(auth.value, "Bearer cached-token");
assert!(supports_local_kiro_request_auth_resolution(&transport));
}
#[test] #[test]
fn resolves_request_auth_from_cached_access_token() { fn resolves_request_auth_from_cached_access_token() {
let mut transport = sample_transport(); let mut transport = sample_transport();

View File

@@ -370,6 +370,7 @@ pub struct CandidateRuntimeSelectabilityInput<'a> {
pub now_unix_secs: u64, pub now_unix_secs: u64,
pub cached_affinity_target: Option<&'a crate::SchedulerAffinityTarget>, pub cached_affinity_target: Option<&'a crate::SchedulerAffinityTarget>,
pub provider_quota_blocks_requests: bool, pub provider_quota_blocks_requests: bool,
pub account_quota_exhausted: bool,
pub rpm_reset_at: Option<u64>, pub rpm_reset_at: Option<u64>,
} }
@@ -390,12 +391,16 @@ pub fn candidate_runtime_skip_reason_with_state(
now_unix_secs, now_unix_secs,
cached_affinity_target, cached_affinity_target,
provider_quota_blocks_requests, provider_quota_blocks_requests,
account_quota_exhausted,
rpm_reset_at, rpm_reset_at,
} = input; } = input;
if provider_quota_blocks_requests { if provider_quota_blocks_requests {
return Some("provider_quota_blocked"); return Some("provider_quota_blocked");
} }
if account_quota_exhausted {
return Some("account_quota_exhausted");
}
if crate::is_candidate_in_recent_failure_cooldown( if crate::is_candidate_in_recent_failure_cooldown(
recent_candidates, recent_candidates,
candidate.provider_id.as_str(), candidate.provider_id.as_str(),
@@ -808,6 +813,7 @@ mod tests {
now_unix_secs: 100, now_unix_secs: 100,
cached_affinity_target: None, cached_affinity_target: None,
provider_quota_blocks_requests: false, provider_quota_blocks_requests: false,
account_quota_exhausted: false,
rpm_reset_at: None, rpm_reset_at: None,
}, },
)); ));
@@ -826,6 +832,7 @@ mod tests {
now_unix_secs: 100, now_unix_secs: 100,
cached_affinity_target: None, cached_affinity_target: None,
provider_quota_blocks_requests: false, provider_quota_blocks_requests: false,
account_quota_exhausted: false,
rpm_reset_at: None, rpm_reset_at: None,
}, },
)); ));
@@ -838,6 +845,24 @@ mod tests {
now_unix_secs: 100, now_unix_secs: 100,
cached_affinity_target: None, cached_affinity_target: None,
provider_quota_blocks_requests: true, provider_quota_blocks_requests: true,
account_quota_exhausted: false,
rpm_reset_at: None,
},
));
}
#[test]
fn candidate_selectability_rejects_exhausted_account_quota() {
assert!(!candidate_is_selectable_with_runtime_state(
CandidateRuntimeSelectabilityInput {
candidate: &sample_candidate("1", None),
recent_candidates: &[],
provider_concurrent_limits: &BTreeMap::new(),
provider_key_rpm_states: &BTreeMap::new(),
now_unix_secs: 100,
cached_affinity_target: None,
provider_quota_blocks_requests: false,
account_quota_exhausted: true,
rpm_reset_at: None, rpm_reset_at: None,
}, },
)); ));

View File

@@ -60,7 +60,7 @@ export async function getModelCapabilities(modelName: string): Promise<ModelCapa
* 获取完整的 API Key用于查看和复制 * 获取完整的 API Key用于查看和复制
*/ */
export interface RevealKeyResult { export interface RevealKeyResult {
auth_type: 'api_key' | 'service_account' | 'oauth' auth_type: 'api_key' | 'service_account' | 'oauth' | 'bearer'
api_key?: string api_key?: string
refresh_token?: string refresh_token?: string
auth_config?: string | Record<string, unknown> auth_config?: string | Record<string, unknown>
@@ -165,7 +165,7 @@ export async function updateProviderKey(
data: Partial<{ data: Partial<{
api_formats: string[] // 支持的 API 格式列表 api_formats: string[] // 支持的 API 格式列表
api_key: string api_key: string
auth_type: 'api_key' | 'service_account' | 'oauth' // 认证类型 auth_type: 'api_key' | 'service_account' | 'oauth' | 'bearer' // 认证类型
auth_config: Record<string, unknown> // 认证配置Vertex AI Service Account JSON auth_config: Record<string, unknown> // 认证配置Vertex AI Service Account JSON
name: string name: string
rate_multipliers: Record<string, number> | null // 按 API 格式的成本倍率 rate_multipliers: Record<string, number> | null // 按 API 格式的成本倍率

View File

@@ -104,6 +104,12 @@ export interface PoolKeyDetail {
key_name: string key_name: string
is_active: boolean is_active: boolean
auth_type: string auth_type: string
credential_kind?: 'raw_secret' | 'oauth_session' | 'service_account' | string | null
runtime_auth_kind?: 'api_key' | 'bearer' | 'service_account' | 'unknown' | string | null
oauth_managed?: boolean
can_refresh_oauth?: boolean
can_export_oauth?: boolean
can_edit_oauth?: boolean
oauth_expires_at?: number | null oauth_expires_at?: number | null
oauth_invalid_at?: number | null // 兼容字段;优先使用 status_snapshot.oauth oauth_invalid_at?: number | null // 兼容字段;优先使用 status_snapshot.oauth
oauth_invalid_reason?: string | null // 兼容字段;优先使用 status_snapshot.oauth oauth_invalid_reason?: string | null // 兼容字段;优先使用 status_snapshot.oauth
@@ -200,6 +206,12 @@ export interface PoolKeySelectionItem {
key_id: string key_id: string
key_name: string key_name: string
auth_type: string auth_type: string
credential_kind?: 'raw_secret' | 'oauth_session' | 'service_account' | string | null
runtime_auth_kind?: 'api_key' | 'bearer' | 'service_account' | 'unknown' | string | null
oauth_managed?: boolean
can_refresh_oauth?: boolean
can_export_oauth?: boolean
can_edit_oauth?: boolean
} }
export interface PoolKeySelectionResponse { export interface PoolKeySelectionResponse {

View File

@@ -229,7 +229,13 @@ export interface EndpointAPIKey {
api_formats: string[] // 支持的 endpoint signature 列表(如 "openai:chat" api_formats: string[] // 支持的 endpoint signature 列表(如 "openai:chat"
api_key_masked: string api_key_masked: string
api_key_plain?: string | null api_key_plain?: string | null
auth_type: 'api_key' | 'service_account' | 'oauth' // 认证类型(必返回) auth_type: 'api_key' | 'service_account' | 'oauth' | 'bearer' // 认证类型(必返回)
credential_kind?: 'raw_secret' | 'oauth_session' | 'service_account' | string | null
runtime_auth_kind?: 'api_key' | 'bearer' | 'service_account' | 'unknown' | string | null
oauth_managed?: boolean
can_refresh_oauth?: boolean
can_export_oauth?: boolean
can_edit_oauth?: boolean
name: string // 密钥名称(必填,用于识别) name: string // 密钥名称(必填,用于识别)
rate_multipliers?: Record<string, number> | null // 按 endpoint signature 的成本倍率 rate_multipliers?: Record<string, number> | null // 按 endpoint signature 的成本倍率
internal_priority: number // Key 内部优先级 internal_priority: number // Key 内部优先级
@@ -492,6 +498,7 @@ export interface PoolAdvancedConfig {
global_priority?: number | null global_priority?: number | null
sticky_session_ttl_seconds?: number | null sticky_session_ttl_seconds?: number | null
load_threshold_percent?: number | null load_threshold_percent?: number | null
skip_exhausted_accounts?: boolean | null
// 旧字段(兼容读取) // 旧字段(兼容读取)
lru_enabled?: boolean lru_enabled?: boolean
scheduling_mode?: 'lru' | 'multi_score' | null scheduling_mode?: 'lru' | 'multi_score' | null

View File

@@ -127,7 +127,7 @@
<Badge <Badge
variant="outline" variant="outline"
class="text-[10px] px-1 py-0 h-4 shrink-0" class="text-[10px] px-1 py-0 h-4 shrink-0"
>{{ normalizeAuthTypeLabel(key.auth_type) }}</Badge> >{{ normalizeAuthTypeLabel(key) }}</Badge>
<Badge <Badge
v-if="getStatusBadgeLabel(key)" v-if="getStatusBadgeLabel(key)"
variant="destructive" variant="destructive"
@@ -289,6 +289,11 @@ import { exportKey, refreshProviderQuota } from '@/api/endpoints/keys'
import { refreshProviderOAuth } from '@/api/endpoints/provider_oauth' import { refreshProviderOAuth } from '@/api/endpoints/provider_oauth'
import { useProxyNodesStore } from '@/stores/proxy-nodes' import { useProxyNodesStore } from '@/stores/proxy-nodes'
import { getOAuthOrgBadge } from '@/utils/oauthIdentity' import { getOAuthOrgBadge } from '@/utils/oauthIdentity'
import {
canExportOAuthCredential,
canRefreshOAuthCredential,
getProviderAuthLabel,
} from '@/utils/providerKeyAuth'
import { import {
getAccountStatusDisplay, getAccountStatusDisplay,
getAccountStatusTitle, getAccountStatusTitle,
@@ -416,10 +421,6 @@ const isCurrentPageFullySelected = computed(() => {
const canClearSelection = computed(() => selectAllFiltered.value || selectedKeyIds.value.length > 0) const canClearSelection = computed(() => selectAllFiltered.value || selectedKeyIds.value.length > 0)
const activeQuickSelectorSet = computed(() => new Set(activeQuickSelectors.value)) const activeQuickSelectorSet = computed(() => new Set(activeQuickSelectors.value))
function normalizeText(value: unknown): string {
return String(value || '').trim().toLowerCase()
}
function sanitizeFileNamePart(value: unknown, fallback: string): string { function sanitizeFileNamePart(value: unknown, fallback: string): string {
const sanitized = String(value || '') const sanitized = String(value || '')
.trim() .trim()
@@ -452,11 +453,8 @@ function downloadJsonFile(data: unknown, filename: string): void {
URL.revokeObjectURL(url) URL.revokeObjectURL(url)
} }
function normalizeAuthTypeLabel(authType: string): string { function normalizeAuthTypeLabel(key: PoolKeyDetail | PoolKeySelectionItem): string {
const text = normalizeText(authType) return getProviderAuthLabel(key)
if (text === 'oauth') return 'OAuth'
if (text === 'service_account') return 'Service'
return 'API Key'
} }
function getStatusBadgeLabel(key: PoolKeyDetail): string | null { function getStatusBadgeLabel(key: PoolKeyDetail): string | null {
@@ -741,6 +739,12 @@ async function resolveSelectedItems(): Promise<PoolKeySelectionItem[]> {
key_id: keyId, key_id: keyId,
key_name: key?.key_name || '', key_name: key?.key_name || '',
auth_type: key?.auth_type || 'api_key', auth_type: key?.auth_type || 'api_key',
credential_kind: key?.credential_kind,
runtime_auth_kind: key?.runtime_auth_kind,
oauth_managed: key?.oauth_managed,
can_refresh_oauth: key?.can_refresh_oauth,
can_export_oauth: key?.can_export_oauth,
can_edit_oauth: key?.can_edit_oauth,
} }
}) })
} }
@@ -810,7 +814,7 @@ async function executeAction(actionOverride?: BatchActionValue): Promise<void> {
progressDone.value = Math.min(i + BATCH_SIZE, targetIds.length) progressDone.value = Math.min(i + BATCH_SIZE, targetIds.length)
} }
} else if (selectedAction.value === 'export') { } else if (selectedAction.value === 'export') {
const exportableKeys = selectedKeys.filter((key) => normalizeText(key.auth_type) === 'oauth') const exportableKeys = selectedKeys.filter((key) => canExportOAuthCredential(key))
const exportedEntries: Array<Record<string, unknown> | null> = Array.from({ length: exportableKeys.length }, () => null) const exportedEntries: Array<Record<string, unknown> | null> = Array.from({ length: exportableKeys.length }, () => null)
skippedCount += selectedKeys.length - exportableKeys.length skippedCount += selectedKeys.length - exportableKeys.length
@@ -920,7 +924,7 @@ async function executeAction(actionOverride?: BatchActionValue): Promise<void> {
const CONCURRENCY = props.batchConcurrency || 8 const CONCURRENCY = props.batchConcurrency || 8
const tasks: Array<() => Promise<'success' | 'skip'>> = [] const tasks: Array<() => Promise<'success' | 'skip'>> = []
for (const key of selectedKeys) { for (const key of selectedKeys) {
if (selectedAction.value === 'refresh_oauth' && normalizeText(key.auth_type) !== 'oauth') { if (selectedAction.value === 'refresh_oauth' && !canRefreshOAuthCredential(key)) {
skippedCount += 1 skippedCount += 1
progressDone.value += 1 progressDone.value += 1
continue continue

View File

@@ -467,6 +467,7 @@ const form = ref({
probing_enabled: false, probing_enabled: false,
probing_interval_minutes: null as number | null | undefined, probing_interval_minutes: null as number | null | undefined,
auto_remove_banned_keys: false, auto_remove_banned_keys: false,
skip_exhausted_accounts: false,
}) })
interface ClaudeFormState { interface ClaudeFormState {
@@ -503,6 +504,8 @@ function getHealthToggleValue(key: PoolHealthToggleKey): boolean {
return form.value.probing_enabled return form.value.probing_enabled
case 'auto_remove_banned_keys': case 'auto_remove_banned_keys':
return form.value.auto_remove_banned_keys return form.value.auto_remove_banned_keys
case 'skip_exhausted_accounts':
return form.value.skip_exhausted_accounts
} }
} }
@@ -516,6 +519,9 @@ function updateHealthToggleValue(key: PoolHealthToggleKey, value: boolean): void
return return
case 'auto_remove_banned_keys': case 'auto_remove_banned_keys':
form.value.auto_remove_banned_keys = value form.value.auto_remove_banned_keys = value
return
case 'skip_exhausted_accounts':
form.value.skip_exhausted_accounts = value
} }
} }
@@ -536,6 +542,7 @@ watch(() => props.modelValue, (open) => {
probing_enabled: cfg?.probing_enabled ?? false, probing_enabled: cfg?.probing_enabled ?? false,
probing_interval_minutes: cfg?.probing_interval_minutes ?? null, probing_interval_minutes: cfg?.probing_interval_minutes ?? null,
auto_remove_banned_keys: cfg?.auto_remove_banned_keys ?? false, auto_remove_banned_keys: cfg?.auto_remove_banned_keys ?? false,
skip_exhausted_accounts: cfg?.skip_exhausted_accounts ?? false,
} }
const cc = props.currentClaudeConfig const cc = props.currentClaudeConfig
@@ -570,6 +577,7 @@ async function handleSave() {
? (form.value.probing_interval_minutes ?? undefined) ? (form.value.probing_interval_minutes ?? undefined)
: undefined, : undefined,
auto_remove_banned_keys: form.value.auto_remove_banned_keys, auto_remove_banned_keys: form.value.auto_remove_banned_keys,
skip_exhausted_accounts: form.value.skip_exhausted_accounts,
} }
const payload: Parameters<typeof updateProvider>[1] = { const payload: Parameters<typeof updateProvider>[1] = {

View File

@@ -13,6 +13,7 @@ describe('poolAdvancedDialog', () => {
'health_policy_enabled', 'health_policy_enabled',
'probing_enabled', 'probing_enabled',
'auto_remove_banned_keys', 'auto_remove_banned_keys',
'skip_exhausted_accounts',
]) ])
}) })
@@ -33,6 +34,11 @@ describe('poolAdvancedDialog', () => {
label: '异常自动清除', label: '异常自动清除',
description: '仅在检测到不可恢复的账号异常时自动从号池移除,不处理纯 Token 失效。', description: '仅在检测到不可恢复的账号异常时自动从号池移除,不处理纯 Token 失效。',
}, },
{
key: 'skip_exhausted_accounts',
label: '跳过额度耗尽账号',
description: '当 Codex / Kiro 账号额度已耗尽时,直接标记为不可调度并在请求侧跳过。',
},
]) ])
}) })

View File

@@ -2,6 +2,7 @@ export type PoolHealthToggleKey =
| 'health_policy_enabled' | 'health_policy_enabled'
| 'probing_enabled' | 'probing_enabled'
| 'auto_remove_banned_keys' | 'auto_remove_banned_keys'
| 'skip_exhausted_accounts'
export interface PoolHealthToggleCard { export interface PoolHealthToggleCard {
key: PoolHealthToggleKey key: PoolHealthToggleKey
@@ -40,6 +41,11 @@ export function buildPoolHealthToggleCards(): PoolHealthToggleCard[] {
label: '异常自动清除', label: '异常自动清除',
description: '仅在检测到不可恢复的账号异常时自动从号池移除,不处理纯 Token 失效。', description: '仅在检测到不可恢复的账号异常时自动从号池移除,不处理纯 Token 失效。',
}, },
{
key: 'skip_exhausted_accounts',
label: '跳过额度耗尽账号',
description: '当 Codex / Kiro 账号额度已耗尽时,直接标记为不可调度并在请求侧跳过。',
},
] ]
} }

View File

@@ -939,8 +939,8 @@ async function handleImport() {
let keepImporting = false let keepImporting = false
try { try {
const proxyNodeId = selectedProxyNodeId.value || undefined const proxyNodeId = selectedProxyNodeId.value || undefined
// 检测是否为批量导入 // Kiro 的单条 JSON 凭据也必须走 batch-import 路径,后端需要完整 auth_config。
if (isBatchImport(inputText)) { if (isKiroProvider.value || isBatchImport(inputText)) {
const task = await startBatchImportOAuthTask(props.providerId, inputText, proxyNodeId) const task = await startBatchImportOAuthTask(props.providerId, inputText, proxyNodeId)
importTask.value = { importTask.value = {
task_id: task.task_id, task_id: task.task_id,

View File

@@ -325,10 +325,10 @@
</div> </div>
<div class="flex items-center gap-1"> <div class="flex items-center gap-1">
<span class="text-[11px] font-mono text-muted-foreground"> <span class="text-[11px] font-mono text-muted-foreground">
{{ key.auth_type === 'oauth' ? '[Refresh Token]' : (key.auth_type === 'service_account' ? '[Service Account]' : key.api_key_masked) }} {{ isOAuthManagedCredential(key) ? '[Refresh Token]' : (isServiceAccountCredential(key) ? '[Service Account]' : key.api_key_masked) }}
</span> </span>
<Button <Button
v-if="key.auth_type === 'oauth'" v-if="canExportOAuthCredential(key)"
variant="ghost" variant="ghost"
size="icon" size="icon"
class="h-4 w-4 shrink-0" class="h-4 w-4 shrink-0"
@@ -403,7 +403,7 @@
</template> </template>
<!-- Antigravity 账号未激活提示 --> <!-- Antigravity 账号未激活提示 -->
<span <span
v-if="provider.provider_type === 'antigravity' && key.is_active && key.auth_type === 'oauth' && (!key.upstream_metadata || !hasAntigravityQuotaData(key.upstream_metadata))" v-if="provider.provider_type === 'antigravity' && key.is_active && isOAuthManagedCredential(key) && (!key.upstream_metadata || !hasAntigravityQuotaData(key.upstream_metadata))"
class="text-[10px] text-orange-500 dark:text-orange-400" class="text-[10px] text-orange-500 dark:text-orange-400"
title="该账号尚未完成 Gemini Code Assist 激活,无法获取配额和使用模型" title="该账号尚未完成 Gemini Code Assist 激活,无法获取配额和使用模型"
> >
@@ -1157,6 +1157,12 @@ import { formatApiFormat } from '@/api/endpoints/types/api-format'
import { isOAuthAccountProviderType, isKeyManagedProviderType } from '../utils/providerTypeUtils' import { isOAuthAccountProviderType, isKeyManagedProviderType } from '../utils/providerTypeUtils'
import { getOAuthOrgBadge } from '@/utils/oauthIdentity' import { getOAuthOrgBadge } from '@/utils/oauthIdentity'
import { getOAuthRefreshFeedback } from '@/utils/oauthRefreshFeedback' import { getOAuthRefreshFeedback } from '@/utils/oauthRefreshFeedback'
import {
canEditOAuthCredential,
canExportOAuthCredential,
isOAuthManagedCredential,
isServiceAccountCredential,
} from '@/utils/providerKeyAuth'
import { import {
getAccountStatusDisplay, getAccountStatusDisplay,
getAccountStatusTitle, getAccountStatusTitle,
@@ -1579,7 +1585,7 @@ function handleEditKey(endpoint: ProviderEndpoint | undefined, key: EndpointAPIK
currentEndpoint.value = endpoint || null currentEndpoint.value = endpoint || null
editingKey.value = key editingKey.value = key
// OAuth 密钥使用专门的编辑对话框 // OAuth 密钥使用专门的编辑对话框
if (key.auth_type === 'oauth') { if (canEditOAuthCredential(key)) {
oauthKeyEditDialogOpen.value = true oauthKeyEditDialogOpen.value = true
} else { } else {
keyFormDialogOpen.value = true keyFormDialogOpen.value = true

View File

@@ -715,7 +715,7 @@ const requestBodyDraft = computed(() => props.requestBodyDraft ?? '')
const traceCandidates = computed(() => props.trace?.candidates ?? []) const traceCandidates = computed(() => props.trace?.candidates ?? [])
const showSetup = computed(() => props.open && !props.testing && !props.result) const showSetup = computed(() => props.open && !props.testing && !props.result)
const showResult = computed(() => !!props.result) const showResult = computed(() => !!props.result)
const showTraceTimeline = computed(() => Boolean(props.requestId)) const showTraceTimeline = computed(() => Boolean(props.requestId) && traceCandidates.value.length > 0)
const isDark = computed(() => typeof document !== 'undefined' && document.documentElement.classList.contains('dark')) const isDark = computed(() => typeof document !== 'undefined' && document.documentElement.classList.contains('dark'))
const { copyToClipboard } = useClipboard() const { copyToClipboard } = useClipboard()

View File

@@ -104,4 +104,24 @@ describe('providerKeyStatus', () => {
}, },
}, 0)).toContain('Token 剩余有效期:') }, 0)).toContain('Token 剩余有效期:')
}) })
it('treats oauth_managed bearer credentials as oauth for legacy countdown fallback', () => {
const future = Math.floor(Date.now() / 1000) + 2 * 24 * 3600
const status = getOAuthStatusDisplay(
{
auth_type: 'bearer',
oauth_managed: true,
oauth_expires_at: future,
},
0,
)
expect(status).not.toBeNull()
expect(status?.isExpired).toBe(false)
expect(getOAuthStatusTitle({
auth_type: 'bearer',
oauth_managed: true,
oauth_expires_at: future,
}, 0)).toContain('Token 剩余有效期:')
})
}) })

View File

@@ -0,0 +1,100 @@
export interface ProviderKeyAuthCarrier {
auth_type?: string | null
credential_kind?: string | null
runtime_auth_kind?: string | null
oauth_managed?: boolean | null
can_refresh_oauth?: boolean | null
can_export_oauth?: boolean | null
can_edit_oauth?: boolean | null
}
function normalizeText(value: unknown): string | null {
if (typeof value !== 'string') return null
const text = value.trim().toLowerCase()
return text || null
}
export function getProviderCredentialKind(
input: ProviderKeyAuthCarrier,
): 'raw_secret' | 'oauth_session' | 'service_account' {
const credentialKind = normalizeText(input.credential_kind)
if (
credentialKind === 'raw_secret'
|| credentialKind === 'oauth_session'
|| credentialKind === 'service_account'
) {
return credentialKind
}
if (typeof input.oauth_managed === 'boolean') {
return input.oauth_managed ? 'oauth_session' : 'raw_secret'
}
const authType = normalizeText(input.auth_type)
if (authType === 'oauth') return 'oauth_session'
if (authType === 'service_account' || authType === 'vertex_ai') return 'service_account'
return 'raw_secret'
}
export function getProviderRuntimeAuthKind(
input: ProviderKeyAuthCarrier,
): 'api_key' | 'bearer' | 'service_account' | 'unknown' {
const runtimeAuthKind = normalizeText(input.runtime_auth_kind)
if (
runtimeAuthKind === 'api_key'
|| runtimeAuthKind === 'bearer'
|| runtimeAuthKind === 'service_account'
) {
return runtimeAuthKind
}
const authType = normalizeText(input.auth_type)
if (authType === 'service_account' || authType === 'vertex_ai') return 'service_account'
if (authType === 'bearer') return 'bearer'
if (authType === 'api_key') return 'api_key'
return 'unknown'
}
export function isOAuthManagedCredential(input: ProviderKeyAuthCarrier): boolean {
if (typeof input.oauth_managed === 'boolean') {
return input.oauth_managed
}
return getProviderCredentialKind(input) === 'oauth_session'
}
export function isServiceAccountCredential(input: ProviderKeyAuthCarrier): boolean {
return getProviderCredentialKind(input) === 'service_account'
}
export function canRefreshOAuthCredential(input: ProviderKeyAuthCarrier): boolean {
if (typeof input.can_refresh_oauth === 'boolean') {
return input.can_refresh_oauth
}
return isOAuthManagedCredential(input)
}
export function canExportOAuthCredential(input: ProviderKeyAuthCarrier): boolean {
if (typeof input.can_export_oauth === 'boolean') {
return input.can_export_oauth
}
return isOAuthManagedCredential(input)
}
export function canEditOAuthCredential(input: ProviderKeyAuthCarrier): boolean {
if (typeof input.can_edit_oauth === 'boolean') {
return input.can_edit_oauth
}
return isOAuthManagedCredential(input)
}
export function getProviderAuthLabel(input: ProviderKeyAuthCarrier): string {
if (isOAuthManagedCredential(input)) return 'OAuth'
if (isServiceAccountCredential(input)) return '服务账号'
return getProviderRuntimeAuthKind(input) === 'bearer' ? 'Bearer' : 'API Key'
}
export function getProviderMaskedSecretLabel(input: ProviderKeyAuthCarrier): string {
if (isOAuthManagedCredential(input)) return '[OAuth Token]'
if (isServiceAccountCredential(input)) return '[Service Account]'
return getProviderRuntimeAuthKind(input) === 'bearer' ? '[Bearer Token]' : '[Key]'
}

View File

@@ -6,9 +6,12 @@ import {
isAccountLevelBlockReason, isAccountLevelBlockReason,
isRefreshFailedReason, isRefreshFailedReason,
} from './accountBlock' } from './accountBlock'
import {
isOAuthManagedCredential,
type ProviderKeyAuthCarrier,
} from './providerKeyAuth'
export interface ProviderKeyStatusCarrier { export interface ProviderKeyStatusCarrier extends ProviderKeyAuthCarrier {
auth_type?: string | null
oauth_expires_at?: number | null oauth_expires_at?: number | null
oauth_invalid_at?: number | null // compatibility only oauth_invalid_at?: number | null // compatibility only
oauth_invalid_reason?: string | null // compatibility only oauth_invalid_reason?: string | null // compatibility only
@@ -107,7 +110,7 @@ function getLegacyOAuthState(
input: ProviderKeyStatusCarrier, input: ProviderKeyStatusCarrier,
tick: number, tick: number,
): OAuthStatusInfo | null { ): OAuthStatusInfo | null {
if (normalizeText(input.auth_type) !== 'oauth') return null if (!isOAuthManagedCredential(input)) return null
if (!input.oauth_expires_at && !input.oauth_invalid_at && !input.oauth_invalid_reason) return null if (!input.oauth_expires_at && !input.oauth_invalid_at && !input.oauth_invalid_reason) return null
const rawReason = normalizeText(input.oauth_invalid_reason) const rawReason = normalizeText(input.oauth_invalid_reason)

View File

@@ -441,7 +441,7 @@
P{{ key.internal_priority ?? 50 }} P{{ key.internal_priority ?? 50 }}
</button> </button>
<Button <Button
v-if="key.auth_type === 'oauth'" v-if="canExportOAuthCredential(key)"
variant="ghost" variant="ghost"
size="icon" size="icon"
class="h-4 w-4 shrink-0" class="h-4 w-4 shrink-0"
@@ -461,9 +461,9 @@
<Copy class="w-2.5 h-2.5" /> <Copy class="w-2.5 h-2.5" />
</Button> </Button>
<span class="font-mono"> <span class="font-mono">
{{ key.auth_type === 'oauth' ? '[OAuth Token]' : (key.auth_type === 'service_account' ? '[Service Account]' : '[Key]') }} {{ getProviderMaskedSecretLabel(key) }}
</span> </span>
<template v-if="key.auth_type === 'oauth'"> <template v-if="canRefreshOAuthCredential(key)">
<Button <Button
variant="ghost" variant="ghost"
size="icon" size="icon"
@@ -853,7 +853,7 @@
class="min-w-0 flex-1 flex justify-center" class="min-w-0 flex-1 flex justify-center"
> >
<Button <Button
v-if="actionId === 'copy_or_download' && key.auth_type === 'oauth'" v-if="actionId === 'copy_or_download' && canExportOAuthCredential(key)"
variant="ghost" variant="ghost"
size="icon" size="icon"
class="h-7 w-7 shrink-0" class="h-7 w-7 shrink-0"
@@ -1207,6 +1207,15 @@ import {
} from '@/features/pool/utils/poolManagementState' } from '@/features/pool/utils/poolManagementState'
import { getOAuthOrgBadge } from '@/utils/oauthIdentity' import { getOAuthOrgBadge } from '@/utils/oauthIdentity'
import { getOAuthRefreshFeedback } from '@/utils/oauthRefreshFeedback' import { getOAuthRefreshFeedback } from '@/utils/oauthRefreshFeedback'
import {
canEditOAuthCredential,
canExportOAuthCredential,
canRefreshOAuthCredential,
getProviderAuthLabel,
getProviderMaskedSecretLabel,
isOAuthManagedCredential,
isServiceAccountCredential,
} from '@/utils/providerKeyAuth'
import { import {
getAccountStatusDisplay, getAccountStatusDisplay,
getAccountStatusTitle, getAccountStatusTitle,
@@ -1836,8 +1845,9 @@ watch(searchQuery, () => {
}, 300) }, 300)
}) })
function normalizeAuthTypeForEdit(authType: string): EndpointAPIKey['auth_type'] { function normalizeAuthTypeForEdit(key: PoolKeyDetail): EndpointAPIKey['auth_type'] {
if (authType === 'oauth' || authType === 'service_account') return authType if (isOAuthManagedCredential(key)) return 'oauth'
if (isServiceAccountCredential(key)) return 'service_account'
return 'api_key' return 'api_key'
} }
@@ -1847,12 +1857,14 @@ function toEndpointApiKey(key: PoolKeyDetail): EndpointAPIKey {
id: key.key_id, id: key.key_id,
provider_id: selectedProviderId.value || '', provider_id: selectedProviderId.value || '',
api_formats: key.api_formats || [], api_formats: key.api_formats || [],
api_key_masked: key.auth_type === 'oauth' api_key_masked: getProviderMaskedSecretLabel(key),
? '[OAuth Token]' auth_type: normalizeAuthTypeForEdit(key),
: key.auth_type === 'service_account' credential_kind: key.credential_kind ?? null,
? '[Service Account]' runtime_auth_kind: key.runtime_auth_kind ?? null,
: '[Key]', oauth_managed: key.oauth_managed ?? undefined,
auth_type: normalizeAuthTypeForEdit(key.auth_type), can_refresh_oauth: key.can_refresh_oauth ?? undefined,
can_export_oauth: key.can_export_oauth ?? undefined,
can_edit_oauth: key.can_edit_oauth ?? undefined,
name: key.key_name || '未命名', name: key.key_name || '未命名',
rate_multipliers: key.rate_multipliers ?? null, rate_multipliers: key.rate_multipliers ?? null,
internal_priority: key.internal_priority ?? 50, internal_priority: key.internal_priority ?? 50,
@@ -1959,7 +1971,7 @@ async function finishEditInternalPriority(
function handleEditKey(key: PoolKeyDetail) { function handleEditKey(key: PoolKeyDetail) {
editingKeyDetail.value = key editingKeyDetail.value = key
if (key.auth_type === 'oauth') { if (canEditOAuthCredential(key)) {
oauthKeyEditDialogOpen.value = true oauthKeyEditDialogOpen.value = true
} else { } else {
keyFormDialogOpen.value = true keyFormDialogOpen.value = true
@@ -2457,10 +2469,8 @@ function getRowClass(key: PoolKeyDetail): string {
return '' return ''
} }
function getAuthTypeChipLabel(authType: string): string { function getAuthTypeChipLabel(key: PoolKeyDetail): string {
if (authType === 'oauth') return 'OAuth' return getProviderAuthLabel(key)
if (authType === 'service_account') return '服务账号'
return 'API Key'
} }
function getMobileOAuthTone(key: PoolKeyDetail): PoolMobileTagTone | null { function getMobileOAuthTone(key: PoolKeyDetail): PoolMobileTagTone | null {
@@ -2482,7 +2492,7 @@ function getMobileTagItems(key: PoolKeyDetail): PoolMobileTagItem[] {
oauthStatusLabel: oauthState?.text ?? null, oauthStatusLabel: oauthState?.text ?? null,
oauthStatusTone: getMobileOAuthTone(key), oauthStatusTone: getMobileOAuthTone(key),
priorityLabel: `P${key.internal_priority ?? 50}`, priorityLabel: `P${key.internal_priority ?? 50}`,
authLabel: getAuthTypeChipLabel(key.auth_type), authLabel: getAuthTypeChipLabel(key),
planLabel: key.oauth_plan_type ? formatOAuthPlanType(key.oauth_plan_type) : null, planLabel: key.oauth_plan_type ? formatOAuthPlanType(key.oauth_plan_type) : null,
orgLabel: orgBadge?.label ?? null, orgLabel: orgBadge?.label ?? null,
proxyLabel: key.proxy?.node_id ? '独立代理' : null, proxyLabel: key.proxy?.node_id ? '独立代理' : null,
@@ -2492,7 +2502,7 @@ function getMobileTagItems(key: PoolKeyDetail): PoolMobileTagItem[] {
function getMobileActionIds(key: PoolKeyDetail): PoolMobileActionId[] { function getMobileActionIds(key: PoolKeyDetail): PoolMobileActionId[] {
return splitPoolMobileActions({ return splitPoolMobileActions({
canDownloadOrCopy: true, canDownloadOrCopy: true,
canRefreshToken: key.auth_type === 'oauth', canRefreshToken: canRefreshOAuthCredential(key),
canClearCooldown: Boolean(key.cooldown_reason), canClearCooldown: Boolean(key.cooldown_reason),
canRecoverHealth: key.circuit_breaker_open || (key.health_score ?? 1) < 0.5, canRecoverHealth: key.circuit_breaker_open || (key.health_score ?? 1) < 0.5,
hasProxy: true, hasProxy: true,