feat(adaptive): 完善自适应 RPM 学习并将 Pool 调度状态与健康分解耦

- orchestration 新增 AdaptiveSuccess 效果,在成功回报路径上根据利用率窗口扩张 learned_rpm_limit
- 429 路径改用 429_observation/adjustment 记录以及基于历史的置信度评估,新增 last_rpm_peak 边界字段
- Pool 调度状态不再因 health_score 低或熔断而降级/拦截,前端同步移除相关按钮与文案兜底
- 新增前端 poolTrace 工具(附测试)承接原 HorizontalRequestTimeline 内的候选合并逻辑
This commit is contained in:
fawney19
2026-04-20 00:27:48 +08:00
parent 1f74e660de
commit c1b9d94c84
16 changed files with 1776 additions and 405 deletions

View File

@@ -67,9 +67,10 @@ use crate::execution_runtime::{
use crate::execution_runtime::{MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES}; use crate::execution_runtime::{MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES};
use crate::log_ids::short_request_id; use crate::log_ids::short_request_id;
use crate::orchestration::{ use crate::orchestration::{
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAttemptFailureEffect, apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect,
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect, LocalAttemptFailureEffect, LocalExecutionEffect, LocalExecutionEffectContext,
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect, LocalHealthFailureEffect, LocalHealthSuccessEffect, LocalOAuthInvalidationEffect,
LocalPoolErrorEffect,
}; };
use crate::request_candidate_runtime::{ use crate::request_candidate_runtime::{
ensure_execution_request_candidate_slot, record_local_request_candidate_status, ensure_execution_request_candidate_slot, record_local_request_candidate_status,
@@ -1668,6 +1669,15 @@ async fn execute_stream_from_frame_stream(
LocalExecutionEffect::HealthSuccess(LocalHealthSuccessEffect), LocalExecutionEffect::HealthSuccess(LocalHealthSuccessEffect),
) )
.await; .await;
apply_local_execution_effect(
&state_for_report,
LocalExecutionEffectContext {
plan: &plan_for_report,
report_context: report_context_owned.as_ref(),
},
LocalExecutionEffect::AdaptiveSuccess(LocalAdaptiveSuccessEffect),
)
.await;
apply_local_execution_effect( apply_local_execution_effect(
&state_for_report, &state_for_report,
LocalExecutionEffectContext { LocalExecutionEffectContext {

View File

@@ -36,9 +36,10 @@ use crate::execution_runtime::{
}; };
use crate::log_ids::short_request_id; use crate::log_ids::short_request_id;
use crate::orchestration::{ use crate::orchestration::{
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAttemptFailureEffect, apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect,
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect, LocalAttemptFailureEffect, LocalExecutionEffect, LocalExecutionEffectContext,
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect, LocalHealthFailureEffect, LocalHealthSuccessEffect, LocalOAuthInvalidationEffect,
LocalPoolErrorEffect,
}; };
use crate::request_candidate_runtime::{ use crate::request_candidate_runtime::{
ensure_execution_request_candidate_slot, record_local_request_candidate_status, ensure_execution_request_candidate_slot, record_local_request_candidate_status,
@@ -467,6 +468,15 @@ pub(crate) async fn execute_execution_runtime_sync(
LocalExecutionEffect::HealthSuccess(LocalHealthSuccessEffect), LocalExecutionEffect::HealthSuccess(LocalHealthSuccessEffect),
) )
.await; .await;
apply_local_execution_effect(
state,
LocalExecutionEffectContext {
plan: &plan,
report_context: report_context.as_ref(),
},
LocalExecutionEffect::AdaptiveSuccess(LocalAdaptiveSuccessEffect),
)
.await;
apply_local_execution_effect( apply_local_execution_effect(
state, state,
LocalExecutionEffectContext { LocalExecutionEffectContext {

View File

@@ -672,8 +672,6 @@ fn admin_pool_scheduling_payload(
key: &StoredProviderCatalogKey, key: &StoredProviderCatalogKey,
cooldown_reason: Option<&str>, cooldown_reason: Option<&str>,
cooldown_ttl_seconds: Option<u64>, cooldown_ttl_seconds: Option<u64>,
health_score: f64,
circuit_breaker_open: bool,
account_blocked: bool, account_blocked: bool,
account_status_code: Option<&str>, account_status_code: Option<&str>,
account_status_label: Option<&str>, account_status_label: Option<&str>,
@@ -741,36 +739,6 @@ fn admin_pool_scheduling_payload(
})], })],
); );
} }
if circuit_breaker_open {
return (
"degraded".to_string(),
"circuit_breaker".to_string(),
"熔断中".to_string(),
vec![json!({
"code": "circuit_breaker",
"label": "熔断中",
"blocking": true,
"source": "health",
"ttl_seconds": serde_json::Value::Null,
"detail": serde_json::Value::Null,
})],
);
}
if health_score < 0.5 {
return (
"degraded".to_string(),
"health_low".to_string(),
"健康度较低".to_string(),
vec![json!({
"code": "health_low",
"label": "健康度较低",
"blocking": false,
"source": "health",
"ttl_seconds": serde_json::Value::Null,
"detail": serde_json::Value::Null,
})],
);
}
( (
"available".to_string(), "available".to_string(),
"available".to_string(), "available".to_string(),
@@ -857,8 +825,6 @@ pub(super) fn build_admin_pool_key_payload(
key, key,
cooldown_reason.as_deref(), cooldown_reason.as_deref(),
cooldown_ttl_seconds, cooldown_ttl_seconds,
health_score,
circuit_breaker_open,
account_status_blocked, account_status_blocked,
account_status_code.as_deref(), account_status_code.as_deref(),
account_status_label.as_deref(), account_status_label.as_deref(),

View File

@@ -139,6 +139,9 @@ impl<'a> AdminAppState<'a> {
key.last_429_at_unix_secs = None; key.last_429_at_unix_secs = None;
key.last_429_type = None; key.last_429_type = None;
key.adjustment_history = None; key.adjustment_history = None;
key.utilization_samples = None;
key.last_probe_increase_at_unix_secs = None;
key.last_rpm_peak = None;
let Some(updated) = self.update_provider_catalog_key(&key).await? else { let Some(updated) = self.update_provider_catalog_key(&key).await? else {
return Ok(admin_adaptive_key_not_found_response(key_id)); return Ok(admin_adaptive_key_not_found_response(key_id));
}; };

File diff suppressed because it is too large Load Diff

View File

@@ -2,7 +2,9 @@ use std::collections::BTreeMap;
use aether_admin::provider::quota as admin_provider_quota_pure; use aether_admin::provider::quota as admin_provider_quota_pure;
use aether_contracts::{ExecutionPlan, ExecutionTelemetry}; use aether_contracts::{ExecutionPlan, ExecutionTelemetry};
use aether_scheduler_core::build_scheduler_affinity_cache_key_for_api_key_id; use aether_scheduler_core::{
build_scheduler_affinity_cache_key_for_api_key_id, count_recent_rpm_requests_for_provider_key,
};
use aether_usage_runtime::{ use aether_usage_runtime::{
build_stream_terminal_usage_outcome, build_sync_terminal_usage_outcome, build_stream_terminal_usage_outcome, build_sync_terminal_usage_outcome,
GatewayStreamReportRequest, GatewaySyncReportRequest, TerminalUsageOutcome, GatewayStreamReportRequest, GatewaySyncReportRequest, TerminalUsageOutcome,
@@ -11,8 +13,9 @@ use serde_json::Value;
use tracing::warn; use tracing::warn;
use super::{ use super::{
local_failover_error_message, project_local_adaptive_rate_limit, project_local_failure_health, local_failover_error_message, project_local_adaptive_rate_limit,
project_local_success_health, LocalFailoverClassification, project_local_adaptive_success, project_local_failure_health, project_local_success_health,
LocalFailoverClassification,
}; };
use crate::ai_pipeline::extract_pool_sticky_session_token; use crate::ai_pipeline::extract_pool_sticky_session_token;
use crate::clock::current_unix_secs; use crate::clock::current_unix_secs;
@@ -59,6 +62,9 @@ pub(crate) struct LocalHealthFailureEffect {
#[derive(Debug, Clone, Copy)] #[derive(Debug, Clone, Copy)]
pub(crate) struct LocalHealthSuccessEffect; pub(crate) struct LocalHealthSuccessEffect;
#[derive(Debug, Clone, Copy)]
pub(crate) struct LocalAdaptiveSuccessEffect;
#[derive(Debug, Clone, Copy)] #[derive(Debug, Clone, Copy)]
pub(crate) struct LocalOAuthInvalidationEffect<'a> { pub(crate) struct LocalOAuthInvalidationEffect<'a> {
pub(crate) status_code: u16, pub(crate) status_code: u16,
@@ -71,6 +77,7 @@ pub(crate) enum LocalExecutionEffect<'a> {
AdaptiveRateLimit(LocalAdaptiveRateLimitEffect<'a>), AdaptiveRateLimit(LocalAdaptiveRateLimitEffect<'a>),
HealthFailure(LocalHealthFailureEffect), HealthFailure(LocalHealthFailureEffect),
HealthSuccess(LocalHealthSuccessEffect), HealthSuccess(LocalHealthSuccessEffect),
AdaptiveSuccess(LocalAdaptiveSuccessEffect),
OauthInvalidation(LocalOAuthInvalidationEffect<'a>), OauthInvalidation(LocalOAuthInvalidationEffect<'a>),
PoolSuccessSync { PoolSuccessSync {
payload: &'a GatewaySyncReportRequest, payload: &'a GatewaySyncReportRequest,
@@ -88,6 +95,8 @@ struct PoolFeedbackContext {
sticky_session_token: Option<String>, sticky_session_token: Option<String>,
} }
const ADAPTIVE_RPM_RECENT_CANDIDATE_LIMIT: usize = 512;
pub(crate) async fn apply_local_execution_effect( pub(crate) async fn apply_local_execution_effect(
state: &AppState, state: &AppState,
context: LocalExecutionEffectContext<'_>, context: LocalExecutionEffectContext<'_>,
@@ -106,6 +115,9 @@ pub(crate) async fn apply_local_execution_effect(
LocalExecutionEffect::HealthSuccess(effect) => { LocalExecutionEffect::HealthSuccess(effect) => {
record_health_success_effect(state, context, effect).await; record_health_success_effect(state, context, effect).await;
} }
LocalExecutionEffect::AdaptiveSuccess(effect) => {
record_adaptive_success_effect(state, context, effect).await;
}
LocalExecutionEffect::OauthInvalidation(effect) => { LocalExecutionEffect::OauthInvalidation(effect) => {
record_oauth_invalidation_effect(state, context, effect).await; record_oauth_invalidation_effect(state, context, effect).await;
} }
@@ -256,6 +268,7 @@ async fn record_adaptive_rate_limit_effect(
context: LocalExecutionEffectContext<'_>, context: LocalExecutionEffectContext<'_>,
effect: LocalAdaptiveRateLimitEffect<'_>, effect: LocalAdaptiveRateLimitEffect<'_>,
) { ) {
let observed_at_unix_secs = current_unix_secs();
let Some(current_key) = state let Some(current_key) = state
.read_provider_catalog_keys_by_ids(std::slice::from_ref(&context.plan.key_id)) .read_provider_catalog_keys_by_ids(std::slice::from_ref(&context.plan.key_id))
.await .await
@@ -264,22 +277,39 @@ async fn record_adaptive_rate_limit_effect(
else { else {
return; return;
}; };
let current_rpm = state
.read_recent_request_candidates(ADAPTIVE_RPM_RECENT_CANDIDATE_LIMIT)
.await
.ok()
.map(|recent_candidates| {
count_recent_rpm_requests_for_provider_key(
&recent_candidates,
&context.plan.key_id,
observed_at_unix_secs,
) as u32
});
let Some(projection) = project_local_adaptive_rate_limit( let Some(projection) = project_local_adaptive_rate_limit(
&current_key, &current_key,
effect.classification, effect.classification,
effect.status_code, effect.status_code,
current_rpm,
effect.headers, effect.headers,
current_unix_secs(), observed_at_unix_secs,
) else { ) else {
return; return;
}; };
let mut updated_key = current_key.clone(); let mut updated_key = current_key.clone();
updated_key.rpm_429_count = Some(projection.rpm_429_count); updated_key.rpm_429_count = Some(projection.rpm_429_count).filter(|value| *value > 0);
updated_key.learned_rpm_limit = projection.learned_rpm_limit;
updated_key.last_429_at_unix_secs = Some(projection.last_429_at_unix_secs); updated_key.last_429_at_unix_secs = Some(projection.last_429_at_unix_secs);
updated_key.last_429_type = Some(projection.last_429_type); updated_key.last_429_type = Some(projection.last_429_type);
updated_key.adjustment_history = projection.adjustment_history;
updated_key.utilization_samples = projection.utilization_samples;
updated_key.last_probe_increase_at_unix_secs = projection.last_probe_increase_at_unix_secs;
updated_key.last_rpm_peak = projection.last_rpm_peak;
updated_key.status_snapshot = Some(projection.status_snapshot); updated_key.status_snapshot = Some(projection.status_snapshot);
updated_key.updated_at_unix_secs = Some(current_unix_secs()); updated_key.updated_at_unix_secs = Some(observed_at_unix_secs);
if let Err(err) = state.update_provider_catalog_key(&updated_key).await { if let Err(err) = state.update_provider_catalog_key(&updated_key).await {
warn!( warn!(
@@ -289,6 +319,54 @@ async fn record_adaptive_rate_limit_effect(
} }
} }
async fn record_adaptive_success_effect(
state: &AppState,
context: LocalExecutionEffectContext<'_>,
_effect: LocalAdaptiveSuccessEffect,
) {
let observed_at_unix_secs = current_unix_secs();
let Some(current_key) = state
.read_provider_catalog_keys_by_ids(std::slice::from_ref(&context.plan.key_id))
.await
.ok()
.and_then(|mut keys| keys.drain(..).next())
else {
return;
};
let Some(recent_candidates) = state
.read_recent_request_candidates(ADAPTIVE_RPM_RECENT_CANDIDATE_LIMIT)
.await
.ok()
else {
return;
};
let current_rpm = count_recent_rpm_requests_for_provider_key(
&recent_candidates,
&context.plan.key_id,
observed_at_unix_secs,
) as u32;
let Some(projection) =
project_local_adaptive_success(&current_key, current_rpm, observed_at_unix_secs)
else {
return;
};
let mut updated_key = current_key.clone();
updated_key.learned_rpm_limit = projection.learned_rpm_limit;
updated_key.adjustment_history = projection.adjustment_history;
updated_key.utilization_samples = projection.utilization_samples;
updated_key.last_probe_increase_at_unix_secs = projection.last_probe_increase_at_unix_secs;
updated_key.status_snapshot = Some(projection.status_snapshot);
updated_key.updated_at_unix_secs = Some(observed_at_unix_secs);
if let Err(err) = state.update_provider_catalog_key(&updated_key).await {
warn!(
"gateway orchestration effects: failed to persist adaptive success projection for provider {} endpoint {} key {}: {:?}",
context.plan.provider_id, context.plan.endpoint_id, context.plan.key_id, err
);
}
}
async fn record_health_failure_effect( async fn record_health_failure_effect(
state: &AppState, state: &AppState,
context: LocalExecutionEffectContext<'_>, context: LocalExecutionEffectContext<'_>,
@@ -540,7 +618,11 @@ mod tests {
use aether_contracts::{ExecutionPlan, RequestBody}; use aether_contracts::{ExecutionPlan, RequestBody};
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY}; use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
use aether_data::repository::candidates::InMemoryRequestCandidateRepository;
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository; use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
use aether_data_contracts::repository::candidates::{
RequestCandidateStatus, StoredRequestCandidate,
};
use aether_data_contracts::repository::provider_catalog::{ use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider, StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
}; };
@@ -548,9 +630,9 @@ mod tests {
use super::{ use super::{
apply_local_execution_effect, local_candidate_failure_should_record_pool_error, apply_local_execution_effect, local_candidate_failure_should_record_pool_error,
LocalAdaptiveRateLimitEffect, LocalAttemptFailureEffect, LocalExecutionEffect, LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect, LocalAttemptFailureEffect,
LocalExecutionEffectContext, LocalHealthFailureEffect, LocalHealthSuccessEffect, LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
LocalOAuthInvalidationEffect, LocalHealthSuccessEffect, LocalOAuthInvalidationEffect,
}; };
use crate::data::GatewayDataState; use crate::data::GatewayDataState;
use crate::orchestration::LocalFailoverClassification; use crate::orchestration::LocalFailoverClassification;
@@ -791,6 +873,26 @@ mod tests {
) )
} }
fn adaptive_state_with_request_candidates(
key: StoredProviderCatalogKey,
request_candidates: Vec<StoredRequestCandidate>,
) -> AppState {
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_health_provider()],
vec![sample_health_endpoint()],
vec![key],
));
let request_candidates =
Arc::new(InMemoryRequestCandidateRepository::seed(request_candidates));
AppState::new()
.expect("gateway state should build")
.with_data_state_for_tests(
GatewayDataState::with_provider_catalog_repository_for_tests(provider_catalog)
.with_request_candidate_reader(request_candidates)
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
)
}
fn fixed_limit_state() -> AppState { fn fixed_limit_state() -> AppState {
let mut key = sample_health_key(); let mut key = sample_health_key();
key.rpm_limit = Some(24); key.rpm_limit = Some(24);
@@ -1168,7 +1270,7 @@ mod tests {
.status_snapshot .status_snapshot
.as_ref() .as_ref()
.and_then(|value| value.get("learning_confidence")), .and_then(|value| value.get("learning_confidence")),
Some(&json!(0.283)) Some(&json!(0.3))
); );
assert_eq!( assert_eq!(
stored_key stored_key
@@ -1212,4 +1314,97 @@ mod tests {
assert_eq!(stored_key.last_429_at_unix_secs, None); assert_eq!(stored_key.last_429_at_unix_secs, None);
assert_eq!(stored_key.last_429_type, None); assert_eq!(stored_key.last_429_type, None);
} }
#[tokio::test]
async fn adaptive_success_effect_expands_limit_from_recent_rpm_usage() {
let now_unix_secs = chrono::Utc::now().timestamp().max(0) as u64;
let mut key = sample_adaptive_key();
key.learned_rpm_limit = Some(20);
key.last_rpm_peak = Some(25);
key.last_429_at_unix_secs = Some(now_unix_secs.saturating_sub(600));
key.adjustment_history = Some(json!([
{
"timestamp": "2026-04-19T00:00:00Z",
"old_limit": 0,
"new_limit": 20,
"reason": "rpm_429",
"confidence": 0.8
}
]));
key.utilization_samples = Some(json!([
{"ts": now_unix_secs.saturating_sub(40), "util": 0.90},
{"ts": now_unix_secs.saturating_sub(30), "util": 0.95},
{"ts": now_unix_secs.saturating_sub(20), "util": 0.85},
{"ts": now_unix_secs.saturating_sub(10), "util": 0.80}
]));
let state = adaptive_state_with_request_candidates(
key,
vec![StoredRequestCandidate::new(
"candidate-1".to_string(),
"req-1".to_string(),
None,
None,
None,
None,
0,
0,
Some("prov-1".to_string()),
Some("ep-1".to_string()),
Some("key-1".to_string()),
RequestCandidateStatus::Success,
None,
false,
Some(200),
None,
None,
Some(10),
Some(19),
None,
None,
i64::try_from(now_unix_secs.saturating_sub(30) * 1000)
.expect("candidate created_at should fit i64"),
Some(
i64::try_from(now_unix_secs.saturating_sub(30) * 1000)
.expect("candidate started_at should fit i64"),
),
Some(
i64::try_from(now_unix_secs.saturating_sub(29) * 1000)
.expect("candidate finished_at should fit i64"),
),
)
.expect("request candidate should build")],
);
let plan = sample_plan();
apply_local_execution_effect(
&state,
LocalExecutionEffectContext {
plan: &plan,
report_context: None,
},
LocalExecutionEffect::AdaptiveSuccess(LocalAdaptiveSuccessEffect),
)
.await;
let stored_key = state
.read_provider_catalog_keys_by_ids(std::slice::from_ref(&plan.key_id))
.await
.expect("provider catalog keys should load")
.into_iter()
.next()
.expect("stored key should exist");
assert_eq!(stored_key.learned_rpm_limit, Some(25));
assert_eq!(stored_key.utilization_samples, Some(json!([])));
assert_eq!(
stored_key
.adjustment_history
.as_ref()
.and_then(Value::as_array)
.and_then(|items| items.last())
.and_then(Value::as_object)
.and_then(|record| record.get("reason"))
.and_then(Value::as_str),
Some("high_utilization")
);
}
} }

View File

@@ -12,7 +12,8 @@ mod recovery;
mod report_effects; mod report_effects;
pub(crate) use self::adaptive::{ pub(crate) use self::adaptive::{
project_local_adaptive_rate_limit, LocalAdaptiveRateLimitProjection, project_local_adaptive_rate_limit, project_local_adaptive_success,
LocalAdaptiveRateLimitProjection, LocalAdaptiveSuccessProjection,
}; };
pub(crate) use self::attempt::{ pub(crate) use self::attempt::{
attempt_identity_from_report_context, build_local_attempt_identities, attempt_identity_from_report_context, build_local_attempt_identities,
@@ -24,9 +25,10 @@ pub(crate) use self::classifier::{
LocalFailoverInput, LocalFailoverInput,
}; };
pub(crate) use self::effects::{ pub(crate) use self::effects::{
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAttemptFailureEffect, apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect,
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect, LocalAttemptFailureEffect, LocalExecutionEffect, LocalExecutionEffectContext,
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect, LocalHealthFailureEffect, LocalHealthSuccessEffect, LocalOAuthInvalidationEffect,
LocalPoolErrorEffect,
}; };
pub(crate) use self::health::{project_local_failure_health, project_local_success_health}; pub(crate) use self::health::{project_local_failure_health, project_local_success_health};
pub(crate) use self::policy::{ pub(crate) use self::policy::{

View File

@@ -761,6 +761,93 @@ async fn gateway_marks_account_blocked_pool_key_in_list_keys_response() {
); );
} }
#[tokio::test]
async fn gateway_ignores_health_signals_in_pool_scheduling_status() {
let mut provider = sample_provider("provider-openai", "openai", 10).with_transport_fields(
true,
false,
true,
None,
None,
None,
None,
None,
Some(json!({
"pool_advanced": {
"enabled": true
}
})),
);
provider.provider_type = "openai".to_string();
let mut circuit_key = sample_key(
"key-openai-circuit",
"provider-openai",
"openai:chat",
"sk-circuit",
);
circuit_key.name = "circuit-open".to_string();
circuit_key.circuit_breaker_by_format = Some(json!({
"openai:chat": {
"open": true
}
}));
let mut low_health_key = sample_key(
"key-openai-health",
"provider-openai",
"openai:chat",
"sk-health",
);
low_health_key.name = "low-health".to_string();
low_health_key.health_by_format = Some(json!({
"openai:chat": {
"health_score": 0.2
}
}));
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![provider],
Vec::new(),
vec![circuit_key, low_health_key],
));
let state = AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests(
provider_catalog_repository,
));
let response = local_admin_pool_response(
&state,
http::Method::GET,
"/api/admin/pool/provider-openai/keys?page=1&page_size=50&status=all",
None,
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = serde_json::from_slice(
&to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read"),
)
.expect("json body should parse");
let keys = payload["keys"].as_array().expect("keys should be array");
assert_eq!(keys.len(), 2);
assert_eq!(keys[0]["key_name"], json!("circuit-open"));
assert_eq!(keys[0]["circuit_breaker_open"], json!(true));
assert_eq!(keys[0]["scheduling_status"], json!("available"));
assert_eq!(keys[0]["scheduling_reason"], json!("available"));
assert_eq!(keys[0]["scheduling_label"], json!("可用"));
assert_eq!(keys[1]["key_name"], json!("low-health"));
assert_eq!(keys[1]["health_score"], json!(0.2));
assert_eq!(keys[1]["scheduling_status"], json!("available"));
assert_eq!(keys[1]["scheduling_reason"], json!("available"));
assert_eq!(keys[1]["scheduling_label"], json!("可用"));
}
#[tokio::test] #[tokio::test]
async fn gateway_handles_admin_pool_list_keys_with_quota_compatibility_fields() { async fn gateway_handles_admin_pool_list_keys_with_quota_compatibility_fields() {
let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits = Arc::new(Mutex::new(0usize));

View File

@@ -309,8 +309,6 @@ fn admin_pool_scheduling_payload(
key: &StoredProviderCatalogKey, key: &StoredProviderCatalogKey,
cooldown_reason: Option<&str>, cooldown_reason: Option<&str>,
cooldown_ttl_seconds: Option<u64>, cooldown_ttl_seconds: Option<u64>,
health_score: f64,
circuit_breaker_open: bool,
) -> (String, String, String, Vec<Value>) { ) -> (String, String, String, Vec<Value>) {
if !key.is_active { if !key.is_active {
return ( return (
@@ -342,36 +340,6 @@ fn admin_pool_scheduling_payload(
})], })],
); );
} }
if circuit_breaker_open {
return (
"degraded".to_string(),
"circuit_breaker".to_string(),
"熔断中".to_string(),
vec![json!({
"code": "circuit_breaker",
"label": "熔断中",
"blocking": true,
"source": "health",
"ttl_seconds": Value::Null,
"detail": Value::Null,
})],
);
}
if health_score < 0.5 {
return (
"degraded".to_string(),
"health_low".to_string(),
"健康度较低".to_string(),
vec![json!({
"code": "health_low",
"label": "健康度较低",
"blocking": false,
"source": "health",
"ttl_seconds": Value::Null,
"detail": Value::Null,
})],
);
}
( (
"available".to_string(), "available".to_string(),
"available".to_string(), "available".to_string(),
@@ -644,7 +612,10 @@ pub fn build_admin_pool_selection_payload(keys: &[StoredProviderCatalogKey]) ->
#[cfg(test)] #[cfg(test)]
#[allow(clippy::items_after_test_module)] #[allow(clippy::items_after_test_module)]
mod tests { mod tests {
use super::{admin_pool_key_account_quota_exhausted, admin_pool_key_is_known_banned}; use super::{
admin_pool_key_account_quota_exhausted, admin_pool_key_is_known_banned,
build_admin_pool_key_payload, AdminPoolKeyPayloadContext,
};
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey; use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
use serde_json::json; use serde_json::json;
@@ -781,6 +752,29 @@ mod tests {
assert!(admin_pool_key_is_known_banned(&key)); assert!(admin_pool_key_is_known_banned(&key));
} }
#[test]
fn build_admin_pool_key_payload_ignores_health_for_scheduling() {
let mut key = sample_key(None);
key.health_by_format = Some(json!({
"openai:chat": {
"health_score": 0.2
}
}));
key.circuit_breaker_by_format = Some(json!({
"openai:chat": {
"open": true
}
}));
let payload = build_admin_pool_key_payload(&key, &AdminPoolKeyPayloadContext::default());
assert_eq!(payload["health_score"], json!(0.2));
assert_eq!(payload["circuit_breaker_open"], json!(true));
assert_eq!(payload["scheduling_status"], json!("available"));
assert_eq!(payload["scheduling_reason"], json!("available"));
assert_eq!(payload["scheduling_label"], json!("可用"));
}
} }
pub fn build_admin_pool_key_payload( pub fn build_admin_pool_key_payload(
@@ -794,8 +788,6 @@ pub fn build_admin_pool_key_payload(
key, key,
context.cooldown_reason.as_deref(), context.cooldown_reason.as_deref(),
context.cooldown_ttl_seconds, context.cooldown_ttl_seconds,
health_score,
circuit_breaker_open,
); );
json!({ json!({

View File

@@ -269,6 +269,7 @@ pub struct StoredProviderCatalogKey {
pub adjustment_history: Option<serde_json::Value>, pub adjustment_history: Option<serde_json::Value>,
pub utilization_samples: Option<serde_json::Value>, pub utilization_samples: Option<serde_json::Value>,
pub last_probe_increase_at_unix_secs: Option<u64>, pub last_probe_increase_at_unix_secs: Option<u64>,
pub last_rpm_peak: Option<u32>,
pub request_count: Option<u32>, pub request_count: Option<u32>,
pub total_tokens: u64, pub total_tokens: u64,
pub total_cost_usd: f64, pub total_cost_usd: f64,
@@ -341,6 +342,7 @@ impl StoredProviderCatalogKey {
adjustment_history: None, adjustment_history: None,
utilization_samples: None, utilization_samples: None,
last_probe_increase_at_unix_secs: None, last_probe_increase_at_unix_secs: None,
last_rpm_peak: None,
request_count: None, request_count: None,
total_tokens: 0, total_tokens: 0,
total_cost_usd: 0.0, total_cost_usd: 0.0,

View File

@@ -161,6 +161,7 @@ SELECT
adjustment_history, adjustment_history,
utilization_samples, utilization_samples,
EXTRACT(EPOCH FROM last_probe_increase_at)::bigint AS last_probe_increase_at_unix_secs, EXTRACT(EPOCH FROM last_probe_increase_at)::bigint AS last_probe_increase_at_unix_secs,
last_rpm_peak,
request_count, request_count,
total_tokens, total_tokens,
CAST(total_cost_usd AS DOUBLE PRECISION) AS total_cost_usd, CAST(total_cost_usd AS DOUBLE PRECISION) AS total_cost_usd,
@@ -216,6 +217,7 @@ SELECT
adjustment_history, adjustment_history,
utilization_samples, utilization_samples,
EXTRACT(EPOCH FROM last_probe_increase_at)::bigint AS last_probe_increase_at_unix_secs, EXTRACT(EPOCH FROM last_probe_increase_at)::bigint AS last_probe_increase_at_unix_secs,
last_rpm_peak,
request_count, request_count,
total_tokens, total_tokens,
CAST(total_cost_usd AS DOUBLE PRECISION) AS total_cost_usd, CAST(total_cost_usd AS DOUBLE PRECISION) AS total_cost_usd,
@@ -271,6 +273,7 @@ SELECT
NULL::jsonb AS adjustment_history, NULL::jsonb AS adjustment_history,
NULL::jsonb AS utilization_samples, NULL::jsonb AS utilization_samples,
NULL::bigint AS last_probe_increase_at_unix_secs, NULL::bigint AS last_probe_increase_at_unix_secs,
NULL::integer AS last_rpm_peak,
request_count, request_count,
0::bigint AS total_tokens, 0::bigint AS total_tokens,
0::double precision AS total_cost_usd, 0::double precision AS total_cost_usd,
@@ -607,6 +610,7 @@ SELECT
adjustment_history, adjustment_history,
utilization_samples, utilization_samples,
EXTRACT(EPOCH FROM last_probe_increase_at)::bigint AS last_probe_increase_at_unix_secs, EXTRACT(EPOCH FROM last_probe_increase_at)::bigint AS last_probe_increase_at_unix_secs,
last_rpm_peak,
request_count, request_count,
total_tokens, total_tokens,
CAST(total_cost_usd AS DOUBLE PRECISION) AS total_cost_usd, CAST(total_cost_usd AS DOUBLE PRECISION) AS total_cost_usd,
@@ -1208,6 +1212,7 @@ INSERT INTO provider_api_keys (
adjustment_history, adjustment_history,
utilization_samples, utilization_samples,
last_probe_increase_at, last_probe_increase_at,
last_rpm_peak,
request_count, request_count,
total_tokens, total_tokens,
total_cost_usd, total_cost_usd,
@@ -1270,31 +1275,32 @@ INSERT INTO provider_api_keys (
WHEN $35::double precision IS NULL THEN NULL WHEN $35::double precision IS NULL THEN NULL
ELSE TO_TIMESTAMP($35::double precision) ELSE TO_TIMESTAMP($35::double precision)
END, END,
COALESCE($36, 0), $36,
COALESCE($37, 0), COALESCE($37, 0),
COALESCE($38, 0), COALESCE($38, 0),
COALESCE($39, 0), COALESCE($39, 0),
COALESCE($40, 0), COALESCE($40, 0),
COALESCE($41, 0), COALESCE($41, 0),
CASE COALESCE($42, 0),
WHEN $42::double precision IS NULL THEN NULL
ELSE TO_TIMESTAMP($42::double precision)
END,
CASE CASE
WHEN $43::double precision IS NULL THEN NULL WHEN $43::double precision IS NULL THEN NULL
ELSE TO_TIMESTAMP($43::double precision) ELSE TO_TIMESTAMP($43::double precision)
END, END,
$44, CASE
WHEN $44::double precision IS NULL THEN NULL
ELSE TO_TIMESTAMP($44::double precision)
END,
$45, $45,
$46, $46,
$47, $47,
CASE $48,
WHEN $48::double precision IS NULL THEN NOW()
ELSE TO_TIMESTAMP($48::double precision)
END,
CASE CASE
WHEN $49::double precision IS NULL THEN NOW() WHEN $49::double precision IS NULL THEN NOW()
ELSE TO_TIMESTAMP($49::double precision) ELSE TO_TIMESTAMP($49::double precision)
END,
CASE
WHEN $50::double precision IS NULL THEN NOW()
ELSE TO_TIMESTAMP($50::double precision)
END END
) )
"#, "#,
@@ -1337,6 +1343,7 @@ INSERT INTO provider_api_keys (
key.last_probe_increase_at_unix_secs key.last_probe_increase_at_unix_secs
.map(|value| value as f64), .map(|value| value as f64),
) )
.bind(key.last_rpm_peak.map(|value| value as i32))
.bind(key.request_count.map(|value| value as i32)) .bind(key.request_count.map(|value| value as i32))
.bind(Some(i64::try_from(key.total_tokens).map_err(|_| { .bind(Some(i64::try_from(key.total_tokens).map_err(|_| {
DataLayerError::InvalidInput(format!( DataLayerError::InvalidInput(format!(
@@ -1731,10 +1738,24 @@ SET
END, END,
oauth_invalid_reason = $26, oauth_invalid_reason = $26,
status_snapshot = $27, status_snapshot = $27,
is_active = $28, concurrent_429_count = $28,
rpm_429_count = $29,
last_429_at = CASE
WHEN $30::double precision IS NULL THEN NULL
ELSE TO_TIMESTAMP($30::double precision)
END,
last_429_type = $31,
adjustment_history = $32,
utilization_samples = $33,
last_probe_increase_at = CASE
WHEN $34::double precision IS NULL THEN NULL
ELSE TO_TIMESTAMP($34::double precision)
END,
last_rpm_peak = $35,
is_active = $36,
updated_at = CASE updated_at = CASE
WHEN $29::double precision IS NULL THEN NOW() WHEN $37::double precision IS NULL THEN NOW()
ELSE TO_TIMESTAMP($29::double precision) ELSE TO_TIMESTAMP($37::double precision)
END END
WHERE id = $1 WHERE id = $1
"#, "#,
@@ -1766,6 +1787,17 @@ WHERE id = $1
.bind(key.oauth_invalid_at_unix_secs.map(|value| value as f64)) .bind(key.oauth_invalid_at_unix_secs.map(|value| value as f64))
.bind(&key.oauth_invalid_reason) .bind(&key.oauth_invalid_reason)
.bind(&key.status_snapshot) .bind(&key.status_snapshot)
.bind(key.concurrent_429_count.map(|value| value as i32))
.bind(key.rpm_429_count.map(|value| value as i32))
.bind(key.last_429_at_unix_secs.map(|value| value as f64))
.bind(&key.last_429_type)
.bind(&key.adjustment_history)
.bind(&key.utilization_samples)
.bind(
key.last_probe_increase_at_unix_secs
.map(|value| value as f64),
)
.bind(key.last_rpm_peak.map(|value| value as i32))
.bind(key.is_active) .bind(key.is_active)
.bind(key.updated_at_unix_secs.map(|value| value as f64)) .bind(key.updated_at_unix_secs.map(|value| value as f64))
.execute(&self.pool) .execute(&self.pool)
@@ -2330,6 +2362,15 @@ fn map_key_row(row: &PgRow) -> Result<StoredProviderCatalogKey, DataLayerError>
}) })
}) })
.transpose()?; .transpose()?;
let last_rpm_peak = row_get::<Option<i32>>(row, "last_rpm_peak")?
.map(|value| {
u32::try_from(value).map_err(|_| {
DataLayerError::UnexpectedValue(format!(
"invalid provider_api_keys.last_rpm_peak: {value}"
))
})
})
.transpose()?;
let last_models_fetch_at_unix_secs = let last_models_fetch_at_unix_secs =
row_get::<Option<i64>>(row, "last_models_fetch_at_unix_secs")? row_get::<Option<i64>>(row, "last_models_fetch_at_unix_secs")?
.map(|value| { .map(|value| {
@@ -2425,6 +2466,7 @@ fn map_key_row(row: &PgRow) -> Result<StoredProviderCatalogKey, DataLayerError>
key.last_429_type = row.try_get("last_429_type").ok(); key.last_429_type = row.try_get("last_429_type").ok();
key.utilization_samples = row.try_get("utilization_samples").ok(); key.utilization_samples = row.try_get("utilization_samples").ok();
key.last_probe_increase_at_unix_secs = last_probe_increase_at_unix_secs; key.last_probe_increase_at_unix_secs = last_probe_increase_at_unix_secs;
key.last_rpm_peak = last_rpm_peak;
key.last_used_at_unix_secs = last_used_at_unix_secs; key.last_used_at_unix_secs = last_used_at_unix_secs;
key.auto_fetch_models = row.try_get("auto_fetch_models").unwrap_or(false); key.auto_fetch_models = row.try_get("auto_fetch_models").unwrap_or(false);
key.last_models_fetch_at_unix_secs = last_models_fetch_at_unix_secs; key.last_models_fetch_at_unix_secs = last_models_fetch_at_unix_secs;

View File

@@ -16,6 +16,10 @@ const COOLDOWN_HOURS_FOR_FULL_CONFIDENCE: f64 = 24.0;
const LOW_LOAD_THRESHOLD: f64 = 0.5; const LOW_LOAD_THRESHOLD: f64 = 0.5;
const HIGH_LOAD_THRESHOLD: f64 = 0.8; const HIGH_LOAD_THRESHOLD: f64 = 0.8;
const ENFORCEMENT_CONFIDENCE_THRESHOLD: f64 = 0.6; const ENFORCEMENT_CONFIDENCE_THRESHOLD: f64 = 0.6;
const CONFIDENCE_DECAY_PER_MINUTE: f64 = 0.005;
const MIN_CONSISTENT_OBSERVATIONS: usize = 3;
const MIN_HEADER_CONFIRMATIONS: usize = 2;
const OBSERVATION_CONSISTENCY_THRESHOLD: f64 = 0.3;
const HEALTH_DEGRADED_THRESHOLD: f64 = 0.8; const HEALTH_DEGRADED_THRESHOLD: f64 = 0.8;
const HEALTH_LOW_THRESHOLD: f64 = 0.5; const HEALTH_LOW_THRESHOLD: f64 = 0.5;
@@ -120,7 +124,9 @@ pub fn effective_provider_key_rpm_limit(
.learned_rpm_limit .learned_rpm_limit
.filter(|limit| *limit > 0) .filter(|limit| *limit > 0)
.and_then(|limit| usize::try_from(limit).ok())?; .and_then(|limit| usize::try_from(limit).ok())?;
if provider_key_reservation_confidence(key, now_unix_secs) < ENFORCEMENT_CONFIDENCE_THRESHOLD { if provider_key_adaptive_learning_confidence(key, now_unix_secs)
< ENFORCEMENT_CONFIDENCE_THRESHOLD
{
return None; return None;
} }
@@ -309,6 +315,145 @@ fn provider_key_dynamic_reservation_ratio(
+ confidence * (STABLE_MAX_RESERVATION_RATIO - STABLE_MIN_RESERVATION_RATIO) + confidence * (STABLE_MAX_RESERVATION_RATIO - STABLE_MIN_RESERVATION_RATIO)
} }
fn provider_key_adaptive_learning_confidence(
key: &StoredProviderCatalogKey,
now_unix_secs: u64,
) -> f64 {
if key.learned_rpm_limit.is_none() {
return 0.0;
}
let base_confidence = provider_key_adaptive_base_confidence(key);
if base_confidence <= 0.0 {
return 0.0;
}
let time_decay = match key.last_429_at_unix_secs {
Some(last_429_at_unix_secs) => {
now_unix_secs.saturating_sub(last_429_at_unix_secs) as f64 / 60.0
* CONFIDENCE_DECAY_PER_MINUTE
}
None => 1.0,
};
(base_confidence - time_decay).clamp(0.0, 1.0)
}
fn provider_key_adaptive_base_confidence(key: &StoredProviderCatalogKey) -> f64 {
let history = provider_key_adjustment_history(key);
for record in history.iter().rev() {
if record.get("type").and_then(serde_json::Value::as_str) == Some("429_observation") {
continue;
}
if let Some(confidence) = record.get("confidence").and_then(json_value_as_f64) {
return confidence.clamp(0.0, 1.0);
}
}
let (_, confidence) = evaluate_provider_key_observations(&history);
if confidence > 0.0 {
return confidence;
}
if key.learned_rpm_limit.is_some() {
return 0.3;
}
0.0
}
fn evaluate_provider_key_observations(
history: &[serde_json::Map<String, serde_json::Value>],
) -> (Option<u32>, f64) {
let observations = history
.iter()
.filter(|record| {
record.get("type").and_then(serde_json::Value::as_str) == Some("429_observation")
})
.collect::<Vec<_>>();
if observations.is_empty() {
return (None, 0.0);
}
let header_values = observations
.iter()
.filter_map(|record| provider_key_observation_u32(record, "upstream_limit"))
.collect::<Vec<_>>();
if header_values.len() >= MIN_HEADER_CONFIRMATIONS {
let recent = provider_key_recent_tail(&header_values, MIN_HEADER_CONFIRMATIONS * 2);
let last_n = provider_key_recent_tail(recent, MIN_HEADER_CONFIRMATIONS);
if provider_key_observations_consistent(last_n) {
return (None, 0.8);
}
}
let local_values = observations
.iter()
.filter_map(|record| provider_key_observation_u32(record, "current_rpm"))
.collect::<Vec<_>>();
if local_values.len() >= MIN_CONSISTENT_OBSERVATIONS {
let recent = provider_key_recent_tail(&local_values, MIN_CONSISTENT_OBSERVATIONS * 2);
let last_n = provider_key_recent_tail(recent, MIN_CONSISTENT_OBSERVATIONS);
if provider_key_observations_consistent(last_n) {
return (None, 0.6);
}
}
(None, 0.0)
}
fn provider_key_adjustment_history(
key: &StoredProviderCatalogKey,
) -> Vec<serde_json::Map<String, serde_json::Value>> {
key.adjustment_history
.as_ref()
.and_then(serde_json::Value::as_array)
.into_iter()
.flatten()
.filter_map(serde_json::Value::as_object)
.cloned()
.collect()
}
fn provider_key_observation_u32(
record: &serde_json::Map<String, serde_json::Value>,
field: &str,
) -> Option<u32> {
record
.get(field)
.and_then(serde_json::Value::as_u64)
.and_then(|value| u32::try_from(value).ok())
.filter(|value| *value > 0)
}
fn provider_key_observations_consistent(values: &[u32]) -> bool {
let median = provider_key_median(values);
median > 0.0
&& values.iter().all(|value| {
(*value as f64 - median).abs() / median <= OBSERVATION_CONSISTENCY_THRESHOLD
})
}
fn provider_key_median(values: &[u32]) -> f64 {
if values.is_empty() {
return 0.0;
}
let mut sorted = values.iter().map(|value| *value as f64).collect::<Vec<_>>();
sorted.sort_by(|left, right| left.partial_cmp(right).unwrap_or(std::cmp::Ordering::Equal));
let midpoint = sorted.len() / 2;
if sorted.len() % 2 == 0 {
(sorted[midpoint - 1] + sorted[midpoint]) / 2.0
} else {
sorted[midpoint]
}
}
fn provider_key_recent_tail<T>(values: &[T], limit: usize) -> &[T] {
let keep_from = values.len().saturating_sub(limit);
&values[keep_from..]
}
fn is_recently_active(candidate: &StoredRequestCandidate, now_unix_secs: u64) -> bool { fn is_recently_active(candidate: &StoredRequestCandidate, now_unix_secs: u64) -> bool {
if candidate.finished_at_unix_ms.is_some() { if candidate.finished_at_unix_ms.is_some() {
return false; return false;
@@ -667,26 +812,60 @@ mod tests {
); );
assert_eq!(effective_provider_key_rpm_limit(&low_confidence, 100), None); assert_eq!(effective_provider_key_rpm_limit(&low_confidence, 100), None);
let high_confidence = provider_catalog_key("key-a").with_rate_limit_fields( let mut high_confidence = provider_catalog_key("key-a").with_rate_limit_fields(
None, None,
Some(80), Some(80),
Some(0), Some(0),
Some(0), Some(0),
None, Some(99),
Some(serde_json::json!([ Some(serde_json::json!([
{"new_limit": 80}, {
{"new_limit": 81}, "timestamp": "2026-04-19T00:00:00Z",
{"new_limit": 80}, "old_limit": 0,
"new_limit": 80,
"reason": "rpm_429",
"confidence": 0.8
},
])), ])),
Some(120), Some(120),
Some(118), Some(118),
); );
high_confidence.last_429_type = Some("rpm".to_string());
assert_eq!( assert_eq!(
effective_provider_key_rpm_limit(&high_confidence, 100), effective_provider_key_rpm_limit(&high_confidence, 100),
Some(80) Some(80)
); );
} }
#[test]
fn learned_provider_key_rpm_limit_uses_confirmed_observations_as_fallback_confidence() {
let key = provider_catalog_key("key-a").with_rate_limit_fields(
None,
Some(80),
Some(0),
Some(2),
Some(99),
Some(serde_json::json!([
{
"type": "429_observation",
"timestamp": "2026-04-19T00:00:00Z",
"current_rpm": 90,
"upstream_limit": 84
},
{
"type": "429_observation",
"timestamp": "2026-04-19T00:01:00Z",
"current_rpm": 88,
"upstream_limit": 85
}
])),
Some(20),
Some(18),
);
assert_eq!(effective_provider_key_rpm_limit(&key, 100), Some(80));
}
#[test] #[test]
fn counts_recent_provider_key_rpm_from_snapshot_or_recent_attempts() { fn counts_recent_provider_key_rpm_from_snapshot_or_recent_attempts() {
let recent_candidates = vec![ let recent_candidates = vec![

View File

@@ -498,6 +498,13 @@ import { log } from '@/utils/logger'
import { parseApiError } from '@/utils/errorParser' import { parseApiError } from '@/utils/errorParser'
import { formatApiFormat } from '@/api/endpoints/types/api-format' import { formatApiFormat } from '@/api/endpoints/types/api-format'
import { resolveTimelineFinalStatus } from '../utils/status' import { resolveTimelineFinalStatus } from '../utils/status'
import {
buildPoolAttemptCandidatesFromAudit,
extractPoolGroupId,
isPoolAttemptedCandidate,
makeAttemptKey,
TIMELINE_STATUS,
} from '../utils/poolTrace'
// 节点组类型 // 节点组类型
interface NodeGroup { interface NodeGroup {
@@ -691,18 +698,6 @@ const proxyTimingBreakdown = (proxy: Record<string, unknown>): string => {
return parts.join(' / ') return parts.join(' / ')
} }
const TIMELINE_STATUS: CandidateRecord['status'][] = [
'success',
'failed',
'skipped',
'cancelled',
'pending',
'streaming',
'available',
'unused',
'stream_interrupted',
]
const STATUS_PRIORITY: Record<string, number> = { const STATUS_PRIORITY: Record<string, number> = {
available: 0, available: 0,
unused: 0, unused: 0,
@@ -715,47 +710,6 @@ const STATUS_PRIORITY: Record<string, number> = {
success: 4, success: 4,
} }
const toInt = (value: unknown, defaultValue = 0): number => {
const num = Number(value)
return Number.isFinite(num) ? Math.trunc(num) : defaultValue
}
const makeAttemptKey = (candidateIndex: number, retryIndex: number): string => {
return `${candidateIndex}:${retryIndex}`
}
const POOL_UNATTEMPTED_STATUS = new Set<CandidateRecord['status']>([
'available',
'unused',
'skipped',
])
const isPoolAttemptedCandidate = (candidate: CandidateRecord): boolean => {
if (POOL_UNATTEMPTED_STATUS.has(candidate.status)) return false
// pending 只有开始执行后才算真正进入号池内部尝试
if (candidate.status === 'pending' && !candidate.started_at) return false
return true
}
const normalizeTimelineStatus = (value: unknown): CandidateRecord['status'] => {
if (typeof value !== 'string') return 'failed'
const normalized = value.trim().toLowerCase()
if ((TIMELINE_STATUS as string[]).includes(normalized)) {
return normalized as CandidateRecord['status']
}
// 兜底:内部调度轨迹里非标准状态统一按失败展示
return 'failed'
}
const extractPoolGroupId = (candidate: CandidateRecord): string | null => {
const extra = candidate.extra_data
if (!extra || typeof extra !== 'object' || Array.isArray(extra)) return null
const value = (extra as Record<string, unknown>).pool_group_id
if (typeof value !== 'string') return null
const text = value.trim()
return text || null
}
// 候选时间线(按实际执行顺序排序) // 候选时间线(按实际执行顺序排序)
const rawTimeline = computed<CandidateRecord[]>(() => { const rawTimeline = computed<CandidateRecord[]>(() => {
if (!trace.value) return [] if (!trace.value) return []
@@ -794,97 +748,11 @@ const poolAttemptCandidates = computed<CandidateRecord[]>(() => {
// 兼容旧链路:回退到 request_metadata.scheduling_audit.attempts。 // 兼容旧链路:回退到 request_metadata.scheduling_audit.attempts。
const audit = schedulingAudit.value const audit = schedulingAudit.value
if (!audit) return [] if (!audit) return []
const attempts = audit.attempts return buildPoolAttemptCandidatesFromAudit(
if (!Array.isArray(attempts) || attempts.length === 0) return [] rawTimeline.value,
audit.attempts,
const providerNameById = new Map<string, string>() props.requestId,
for (const candidate of rawTimeline.value) { )
const providerId = String(candidate.provider_id || '').trim()
const providerName = String(candidate.provider_name || '').trim()
if (!providerId || !providerName) continue
if (!providerNameById.has(providerId)) {
providerNameById.set(providerId, providerName)
}
}
const providerTypeLikeNames = new Set<string>([
'codex',
'kiro',
'antigravity',
'claude_code',
'claude code',
'gemini_cli',
'gemini cli',
'oauth',
'api_key',
'api key',
])
const traceMap = new Map<string, CandidateRecord>()
for (const candidate of rawTimeline.value) {
traceMap.set(makeAttemptKey(candidate.candidate_index, candidate.retry_index), candidate)
}
return attempts
.map((item, index) => {
if (!item || typeof item !== 'object' || Array.isArray(item)) return null
const raw = item as Record<string, unknown>
const candidateIndex = toInt(raw.candidate_index, index)
const retryIndex = toInt(raw.retry_index, 0)
const key = makeAttemptKey(candidateIndex, retryIndex)
const fromTrace = traceMap.get(key)
const merged: CandidateRecord = fromTrace
? { ...fromTrace }
: {
id: `pool-${props.requestId}-${candidateIndex}-${retryIndex}-${index}`,
request_id: props.requestId,
candidate_index: candidateIndex,
retry_index: retryIndex,
provider_id: undefined,
provider_name: undefined,
endpoint_id: undefined,
key_id: undefined,
key_name: undefined,
status: 'failed',
is_cached: false,
created_at: new Date(0).toISOString(),
}
merged.status = normalizeTimelineStatus(raw.status ?? merged.status)
if (typeof raw.provider_id === 'string') merged.provider_id = raw.provider_id
if (typeof raw.provider_name === 'string') merged.provider_name = raw.provider_name
if (typeof raw.endpoint_id === 'string') merged.endpoint_id = raw.endpoint_id
if (typeof raw.key_id === 'string') merged.key_id = raw.key_id
if (typeof raw.key_name === 'string') merged.key_name = raw.key_name
if (typeof raw.status_code === 'number') merged.status_code = raw.status_code
if (typeof raw.error_type === 'string') merged.error_type = raw.error_type
const rawPoolGroupId = typeof raw.pool_group_id === 'string' ? raw.pool_group_id.trim() : ''
const fallbackPoolGroupId = typeof raw.provider_id === 'string' ? raw.provider_id.trim() : ''
const finalPoolGroupId = rawPoolGroupId || fallbackPoolGroupId
if (finalPoolGroupId) {
merged.extra_data = {
...(merged.extra_data || {}),
pool_group_id: finalPoolGroupId,
}
}
const mergedProviderId = String(merged.provider_id || '').trim()
if (mergedProviderId) {
const inferredProviderName = providerNameById.get(mergedProviderId)
const currentProviderName = String(merged.provider_name || '').trim()
if (
inferredProviderName
&& (
!currentProviderName
|| providerTypeLikeNames.has(currentProviderName.toLowerCase())
)
) {
merged.provider_name = inferredProviderName
}
}
return merged
})
.filter((item): item is CandidateRecord => item !== null)
}) })
const poolAttemptsByGroup = computed<Map<string, CandidateRecord[]>>(() => { const poolAttemptsByGroup = computed<Map<string, CandidateRecord[]>>(() => {

View File

@@ -0,0 +1,114 @@
import { describe, expect, it } from 'vitest'
import type { CandidateRecord } from '@/api/requestTrace'
import { buildPoolAttemptCandidatesFromAudit } from '@/features/usage/utils/poolTrace'
function buildCandidate(
overrides: Partial<CandidateRecord> = {},
): CandidateRecord {
return {
id: 'cand-1',
request_id: 'req-1',
candidate_index: 0,
retry_index: 0,
status: 'failed',
is_cached: false,
created_at: '1970-01-01T00:00:00.000Z',
...overrides,
}
}
describe('poolTrace', () => {
it('keeps only actually attempted pool nodes from scheduling audit fallback', () => {
const attempts = buildPoolAttemptCandidatesFromAudit([], [
{
candidate_index: 0,
retry_index: 0,
provider_id: 'provider-1',
provider_name: 'Codex反代',
key_id: 'key-success',
key_name: 'Success Key',
status: 'success',
pool_group_id: 'provider-1',
},
{
candidate_index: 1,
retry_index: 0,
provider_id: 'provider-1',
provider_name: 'Codex反代',
key_id: 'key-skipped',
key_name: 'Skipped Key',
status: 'skipped',
pool_group_id: 'provider-1',
},
{
candidate_index: 2,
retry_index: 0,
provider_id: 'provider-1',
provider_name: 'Codex反代',
key_id: 'key-available',
key_name: 'Available Key',
status: 'available',
pool_group_id: 'provider-1',
},
{
candidate_index: 3,
retry_index: 0,
provider_id: 'provider-1',
provider_name: 'Codex反代',
key_id: 'key-unknown',
key_name: 'Unknown Key',
status: 'selected',
pool_group_id: 'provider-1',
},
], 'req-1')
expect(attempts).toHaveLength(1)
expect(attempts[0].key_id).toBe('key-success')
expect(attempts[0].status).toBe('success')
})
it('preserves real trace attempts even when audit status is non-standard', () => {
const rawTimeline = [
buildCandidate({
id: 'cand-trace-1',
candidate_index: 4,
retry_index: 0,
provider_id: 'provider-1',
provider_name: 'Codex反代',
key_id: 'key-trace',
key_name: 'Trace Key',
status: 'failed',
started_at: '2026-04-19T12:00:00.000Z',
}),
]
const attempts = buildPoolAttemptCandidatesFromAudit(rawTimeline, [
{
candidate_index: 4,
retry_index: 0,
provider_id: 'provider-1',
provider_name: 'oauth',
key_id: 'key-trace',
key_name: 'Trace Key',
status: 'selected',
pool_group_id: 'provider-1',
},
{
candidate_index: 5,
retry_index: 0,
provider_id: 'provider-1',
provider_name: 'oauth',
key_id: 'key-ghost',
key_name: 'Ghost Key',
status: 'selected',
pool_group_id: 'provider-1',
},
], 'req-1')
expect(attempts).toHaveLength(1)
expect(attempts[0].id).toBe('cand-trace-1')
expect(attempts[0].status).toBe('failed')
expect(attempts[0].provider_name).toBe('Codex反代')
})
})

View File

@@ -0,0 +1,160 @@
import type { CandidateRecord } from '@/api/requestTrace'
export const TIMELINE_STATUS: CandidateRecord['status'][] = [
'success',
'failed',
'skipped',
'cancelled',
'pending',
'streaming',
'available',
'unused',
'stream_interrupted',
]
const POOL_UNATTEMPTED_STATUS = new Set<CandidateRecord['status']>([
'available',
'unused',
'skipped',
])
const PROVIDER_TYPE_LIKE_NAMES = new Set<string>([
'codex',
'kiro',
'antigravity',
'claude_code',
'claude code',
'gemini_cli',
'gemini cli',
'oauth',
'api_key',
'api key',
])
const toInt = (value: unknown, defaultValue = 0): number => {
const num = Number(value)
return Number.isFinite(num) ? Math.trunc(num) : defaultValue
}
export const makeAttemptKey = (candidateIndex: number, retryIndex: number): string => {
return `${candidateIndex}:${retryIndex}`
}
export const isPoolAttemptedCandidate = (candidate: CandidateRecord): boolean => {
if (POOL_UNATTEMPTED_STATUS.has(candidate.status)) return false
if (candidate.status === 'pending' && !candidate.started_at) return false
return true
}
export const parseTimelineStatus = (value: unknown): CandidateRecord['status'] | null => {
if (typeof value !== 'string') return null
const normalized = value.trim().toLowerCase()
if ((TIMELINE_STATUS as string[]).includes(normalized)) {
return normalized as CandidateRecord['status']
}
return null
}
export const extractPoolGroupId = (
candidate: Pick<CandidateRecord, 'extra_data'>,
): string | null => {
const extra = candidate.extra_data
if (!extra || typeof extra !== 'object' || Array.isArray(extra)) return null
const value = (extra as Record<string, unknown>).pool_group_id
if (typeof value !== 'string') return null
const text = value.trim()
return text || null
}
export function buildPoolAttemptCandidatesFromAudit(
rawTimeline: CandidateRecord[],
attempts: unknown,
requestId?: string | null,
): CandidateRecord[] {
if (!Array.isArray(attempts) || attempts.length === 0) return []
const providerNameById = new Map<string, string>()
for (const candidate of rawTimeline) {
const providerId = String(candidate.provider_id || '').trim()
const providerName = String(candidate.provider_name || '').trim()
if (!providerId || !providerName) continue
if (!providerNameById.has(providerId)) {
providerNameById.set(providerId, providerName)
}
}
const traceMap = new Map<string, CandidateRecord>()
for (const candidate of rawTimeline) {
traceMap.set(makeAttemptKey(candidate.candidate_index, candidate.retry_index), candidate)
}
return attempts
.map((item, index) => {
if (!item || typeof item !== 'object' || Array.isArray(item)) return null
const raw = item as Record<string, unknown>
const candidateIndex = toInt(raw.candidate_index, index)
const retryIndex = toInt(raw.retry_index, 0)
const key = makeAttemptKey(candidateIndex, retryIndex)
const fromTrace = traceMap.get(key)
const parsedStatus = parseTimelineStatus(raw.status)
if (!fromTrace && parsedStatus === null) {
return null
}
const merged: CandidateRecord = fromTrace
? { ...fromTrace }
: {
id: `pool-${requestId || 'unknown'}-${candidateIndex}-${retryIndex}-${index}`,
request_id: requestId || '',
candidate_index: candidateIndex,
retry_index: retryIndex,
provider_id: undefined,
provider_name: undefined,
endpoint_id: undefined,
key_id: undefined,
key_name: undefined,
status: 'failed',
is_cached: false,
created_at: new Date(0).toISOString(),
}
if (parsedStatus !== null) {
merged.status = parsedStatus
}
if (typeof raw.provider_id === 'string') merged.provider_id = raw.provider_id
if (typeof raw.provider_name === 'string') merged.provider_name = raw.provider_name
if (typeof raw.endpoint_id === 'string') merged.endpoint_id = raw.endpoint_id
if (typeof raw.key_id === 'string') merged.key_id = raw.key_id
if (typeof raw.key_name === 'string') merged.key_name = raw.key_name
if (typeof raw.status_code === 'number') merged.status_code = raw.status_code
if (typeof raw.error_type === 'string') merged.error_type = raw.error_type
const rawPoolGroupId = typeof raw.pool_group_id === 'string' ? raw.pool_group_id.trim() : ''
const fallbackPoolGroupId = typeof raw.provider_id === 'string' ? raw.provider_id.trim() : ''
const finalPoolGroupId = rawPoolGroupId || fallbackPoolGroupId
if (finalPoolGroupId) {
merged.extra_data = {
...(merged.extra_data || {}),
pool_group_id: finalPoolGroupId,
}
}
const mergedProviderId = String(merged.provider_id || '').trim()
if (mergedProviderId) {
const inferredProviderName = providerNameById.get(mergedProviderId)
const currentProviderName = String(merged.provider_name || '').trim()
if (
inferredProviderName
&& (
!currentProviderName
|| PROVIDER_TYPE_LIKE_NAMES.has(currentProviderName.toLowerCase())
)
) {
merged.provider_name = inferredProviderName
}
}
return isPoolAttemptedCandidate(merged) ? merged : null
})
.filter((item): item is CandidateRecord => item !== null)
}

View File

@@ -604,20 +604,6 @@
> >
<RefreshCw class="w-3.5 h-3.5" /> <RefreshCw class="w-3.5 h-3.5" />
</Button> </Button>
<Button
v-if="key.circuit_breaker_open || (key.health_score ?? 1) < 0.5"
variant="ghost"
size="icon"
class="h-7 w-7 text-green-600"
:disabled="recoveringHealthKeyId === key.key_id"
title="刷新健康状态"
@click="handleRecoverKey(key)"
>
<RefreshCw
class="w-3.5 h-3.5"
:class="{ 'animate-spin': recoveringHealthKeyId === key.key_id }"
/>
</Button>
<Button <Button
variant="ghost" variant="ghost"
size="icon" size="icon"
@@ -896,20 +882,6 @@
> >
<RefreshCw class="w-3.5 h-3.5" /> <RefreshCw class="w-3.5 h-3.5" />
</Button> </Button>
<Button
v-else-if="actionId === 'recover_health'"
variant="ghost"
size="icon"
class="h-7 w-7 shrink-0 text-green-600"
:disabled="recoveringHealthKeyId === key.key_id"
title="刷新健康状态"
@click="handleRecoverKey(key)"
>
<RefreshCw
class="w-3.5 h-3.5"
:class="{ 'animate-spin': recoveringHealthKeyId === key.key_id }"
/>
</Button>
<Button <Button
v-else-if="actionId === 'permissions'" v-else-if="actionId === 'permissions'"
variant="ghost" variant="ghost"
@@ -1167,7 +1139,6 @@ import {
refreshProviderQuota, refreshProviderQuota,
} from '@/api/endpoints/keys' } from '@/api/endpoints/keys'
import { refreshProviderOAuth } from '@/api/endpoints/provider_oauth' import { refreshProviderOAuth } from '@/api/endpoints/provider_oauth'
import { recoverKeyHealth } from '@/api/endpoints/health'
import type { import type {
PoolOverviewItem, PoolOverviewItem,
PoolKeyDetail, PoolKeyDetail,
@@ -1565,7 +1536,6 @@ const currentPage = ref(restoredViewState.page)
const pageSize = ref(restoredViewState.pageSize) const pageSize = ref(restoredViewState.pageSize)
const MANUAL_QUOTA_REFRESH_COOLDOWN_SECONDS = 5 * 60 const MANUAL_QUOTA_REFRESH_COOLDOWN_SECONDS = 5 * 60
const refreshingOAuthKeyId = ref<string | null>(null) const refreshingOAuthKeyId = ref<string | null>(null)
const recoveringHealthKeyId = ref<string | null>(null)
const savingProxyKeyId = ref<string | null>(null) const savingProxyKeyId = ref<string | null>(null)
const proxyDesktopPopoverOpenKeyId = ref<string | null>(null) const proxyDesktopPopoverOpenKeyId = ref<string | null>(null)
const proxyMobilePopoverOpenKeyId = ref<string | null>(null) const proxyMobilePopoverOpenKeyId = ref<string | null>(null)
@@ -2075,20 +2045,6 @@ async function clearKeyProxy(key: PoolKeyDetail) {
} }
} }
async function handleRecoverKey(key: PoolKeyDetail) {
if (recoveringHealthKeyId.value) return
recoveringHealthKeyId.value = key.key_id
try {
const result = await recoverKeyHealth(key.key_id)
success(result.message || 'Key 已恢复')
await loadKeys()
} catch (err) {
showError(parseApiError(err, 'Key恢复失败'))
} finally {
recoveringHealthKeyId.value = null
}
}
async function handleDeleteKey(key: PoolKeyDetail) { async function handleDeleteKey(key: PoolKeyDetail) {
const confirmed = await confirm({ const confirmed = await confirm({
title: '删除账号', title: '删除账号',
@@ -2399,19 +2355,54 @@ function formatCooldownReason(reason: string): string {
type PoolStatusVariant = 'default' | 'secondary' | 'destructive' | 'outline' | 'success' | 'warning' | 'dark' type PoolStatusVariant = 'default' | 'secondary' | 'destructive' | 'outline' | 'success' | 'warning' | 'dark'
function isHealthDerivedSchedulingReason(reason: string | null | undefined): boolean {
const normalized = String(reason || '').trim().toLowerCase()
return normalized === 'health_low'
|| normalized === 'health_degraded'
|| normalized === 'health'
|| normalized === 'circuit_open'
|| normalized === 'circuit_breaker'
}
function isHealthDerivedSchedulingLabel(label: string | null | undefined): boolean {
const normalized = String(label || '').trim()
return normalized === '健康低'
|| normalized === '健康度较低'
|| normalized === '降级'
|| normalized === '熔断'
|| normalized === '熔断中'
}
function getVisibleSchedulingReason(key: PoolKeyDetail): string | null {
const reason = String(key.scheduling_reason || '').trim()
if (!reason || isHealthDerivedSchedulingReason(reason)) return null
return reason
}
function getVisibleSchedulingReasons(key: PoolKeyDetail) {
return (key.scheduling_reasons ?? []).filter((item) => {
const source = String(item.source || '').trim().toLowerCase()
return source !== 'health'
&& !isHealthDerivedSchedulingReason(item.code)
&& !isHealthDerivedSchedulingLabel(item.label)
})
}
function getSchedulingStatus(key: PoolKeyDetail): 'available' | 'degraded' | 'blocked' { function getSchedulingStatus(key: PoolKeyDetail): 'available' | 'degraded' | 'blocked' {
if (getAccountAlertLabel(key)) return 'blocked' if (getAccountAlertLabel(key)) return 'blocked'
const status = key.scheduling_status const status = key.scheduling_status
if (status === 'available' || status === 'degraded' || status === 'blocked') { if (
(status === 'available' || status === 'degraded' || status === 'blocked')
&& !isHealthDerivedSchedulingReason(key.scheduling_reason)
&& !isHealthDerivedSchedulingLabel(key.scheduling_label)
) {
return status return status
} }
if (!key.is_active) return 'blocked' if (!key.is_active) return 'blocked'
if (key.cooldown_reason) return 'blocked' if (key.cooldown_reason) return 'degraded'
if (key.circuit_breaker_open) return 'blocked'
if (key.cost_limit != null && key.cost_limit > 0 && key.cost_window_usage >= key.cost_limit) return 'blocked' if (key.cost_limit != null && key.cost_limit > 0 && key.cost_window_usage >= key.cost_limit) return 'blocked'
if ((key.health_score ?? 1) < 0.8) return 'degraded'
return 'available' return 'available'
} }
@@ -2420,29 +2411,31 @@ function getSchedulingBadgeLabel(key: PoolKeyDetail): string {
if (accountAlert) return accountAlert if (accountAlert) return accountAlert
const rawLabel = String(key.scheduling_label || '').trim() const rawLabel = String(key.scheduling_label || '').trim()
if (rawLabel) { if (
rawLabel
&& !isHealthDerivedSchedulingReason(key.scheduling_reason)
&& !isHealthDerivedSchedulingLabel(rawLabel)
) {
if (rawLabel === '禁用' || rawLabel === '停用') return '禁用' if (rawLabel === '禁用' || rawLabel === '停用') return '禁用'
return rawLabel return rawLabel
} }
if (!key.is_active) return '禁用' if (!key.is_active) return '禁用'
if (key.cooldown_reason) return '冷却' if (key.cooldown_reason) return '冷却'
if (key.circuit_breaker_open) return '熔断'
if (key.cost_limit != null && key.cost_limit > 0 && key.cost_window_usage >= key.cost_limit) return '超限' if (key.cost_limit != null && key.cost_limit > 0 && key.cost_window_usage >= key.cost_limit) return '超限'
if ((key.health_score ?? 1) < 0.5) return '健康低'
if ((key.health_score ?? 1) < 0.8) return '降级'
return '可用' return '可用'
} }
function getSchedulingBadgeVariant(key: PoolKeyDetail): PoolStatusVariant { function getSchedulingBadgeVariant(key: PoolKeyDetail): PoolStatusVariant {
if (getAccountAlertLabel(key)) return 'destructive' if (getAccountAlertLabel(key)) return 'destructive'
const reason = key.scheduling_reason const reason = getVisibleSchedulingReason(key)
if (reason === 'manual_disabled') return 'secondary' if (reason === 'manual_disabled' || reason === 'inactive') return 'secondary'
if (reason === 'cooldown' || reason === 'circuit_open' || reason === 'cost_exhausted') return 'destructive' if (reason === 'account_blocked' || reason === 'account_quota_exhausted' || reason === 'cost_exhausted') return 'destructive'
if (reason === 'cooldown') return 'warning'
if (reason === 'cost_soft' || reason === 'cost') return 'warning' if (reason === 'cost_soft' || reason === 'cost') return 'warning'
if (reason === 'health_low' || reason === 'health_degraded' || reason === 'health') return 'warning'
if (reason === 'available') return 'default' if (reason === 'available') return 'default'
if (!reason && !key.is_active) return 'secondary'
const status = getSchedulingStatus(key) const status = getSchedulingStatus(key)
if (status === 'blocked') return 'destructive' if (status === 'blocked') return 'destructive'
@@ -2454,7 +2447,7 @@ function getSchedulingTitle(key: PoolKeyDetail): string {
const accountAlertTitle = getAccountAlertTitle(key) const accountAlertTitle = getAccountAlertTitle(key)
if (accountAlertTitle) return accountAlertTitle if (accountAlertTitle) return accountAlertTitle
const reasons = key.scheduling_reasons ?? [] const reasons = getVisibleSchedulingReasons(key)
if (reasons.length > 0) { if (reasons.length > 0) {
return reasons.map((item) => { return reasons.map((item) => {
const ttl = item.ttl_seconds && item.ttl_seconds > 0 ? ` (${formatTTL(item.ttl_seconds)})` : '' const ttl = item.ttl_seconds && item.ttl_seconds > 0 ? ` (${formatTTL(item.ttl_seconds)})` : ''
@@ -2518,7 +2511,6 @@ function getMobileActionIds(key: PoolKeyDetail): PoolMobileActionId[] {
canDownloadOrCopy: true, canDownloadOrCopy: true,
canRefreshToken: canRefreshOAuthCredential(key), canRefreshToken: canRefreshOAuthCredential(key),
canClearCooldown: Boolean(key.cooldown_reason), canClearCooldown: Boolean(key.cooldown_reason),
canRecoverHealth: key.circuit_breaker_open || (key.health_score ?? 1) < 0.5,
hasProxy: true, hasProxy: true,
}).primary }).primary
} }
@@ -2669,7 +2661,7 @@ function getQuotaProgressCountdown(item: QuotaProgressItem) {
function getQuotaProgressCountdownText(item: QuotaProgressItem): string { function getQuotaProgressCountdownText(item: QuotaProgressItem): string {
const status = getQuotaProgressCountdown(item) const status = getQuotaProgressCountdown(item)
if (!status) return '' if (!status) return ''
return status.isExpired ? status.text : `${status.text} 后重置` return status.isExpired ? '' : `${status.text} 后重置`
} }
function formatCompactQuotaCountdownText(text: string): string { function formatCompactQuotaCountdownText(text: string): string {
@@ -2681,10 +2673,15 @@ function formatCompactQuotaCountdownText(text: string): string {
return normalized.replace(/\s+后重置$/, '') return normalized.replace(/\s+后重置$/, '')
} }
function shouldHideQuotaProgressDetailText(text: string | null | undefined): boolean {
return (text ?? '').trim().includes('已重置')
}
function getQuotaProgressDisplayText(item: QuotaProgressItem): string { function getQuotaProgressDisplayText(item: QuotaProgressItem): string {
const countdownText = getQuotaProgressCountdownText(item) const countdownText = getQuotaProgressCountdownText(item)
if (countdownText) return formatCompactQuotaCountdownText(countdownText) if (countdownText) return formatCompactQuotaCountdownText(countdownText)
return item.detail?.trim() || '' const detail = item.detail?.trim() || ''
return shouldHideQuotaProgressDetailText(detail) ? '' : detail
} }
function getQuotaFallbackText(key: PoolKeyDetail): string | null { function getQuotaFallbackText(key: PoolKeyDetail): string | null {