mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 09:20:22 +08:00
feat(adaptive): 完善自适应 RPM 学习并将 Pool 调度状态与健康分解耦
- orchestration 新增 AdaptiveSuccess 效果,在成功回报路径上根据利用率窗口扩张 learned_rpm_limit - 429 路径改用 429_observation/adjustment 记录以及基于历史的置信度评估,新增 last_rpm_peak 边界字段 - Pool 调度状态不再因 health_score 低或熔断而降级/拦截,前端同步移除相关按钮与文案兜底 - 新增前端 poolTrace 工具(附测试)承接原 HorizontalRequestTimeline 内的候选合并逻辑
This commit is contained in:
@@ -67,9 +67,10 @@ use crate::execution_runtime::{
|
|||||||
use crate::execution_runtime::{MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES};
|
use crate::execution_runtime::{MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES};
|
||||||
use crate::log_ids::short_request_id;
|
use crate::log_ids::short_request_id;
|
||||||
use crate::orchestration::{
|
use crate::orchestration::{
|
||||||
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAttemptFailureEffect,
|
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect,
|
||||||
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
|
LocalAttemptFailureEffect, LocalExecutionEffect, LocalExecutionEffectContext,
|
||||||
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
|
LocalHealthFailureEffect, LocalHealthSuccessEffect, LocalOAuthInvalidationEffect,
|
||||||
|
LocalPoolErrorEffect,
|
||||||
};
|
};
|
||||||
use crate::request_candidate_runtime::{
|
use crate::request_candidate_runtime::{
|
||||||
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
|
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
|
||||||
@@ -1668,6 +1669,15 @@ async fn execute_stream_from_frame_stream(
|
|||||||
LocalExecutionEffect::HealthSuccess(LocalHealthSuccessEffect),
|
LocalExecutionEffect::HealthSuccess(LocalHealthSuccessEffect),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
apply_local_execution_effect(
|
||||||
|
&state_for_report,
|
||||||
|
LocalExecutionEffectContext {
|
||||||
|
plan: &plan_for_report,
|
||||||
|
report_context: report_context_owned.as_ref(),
|
||||||
|
},
|
||||||
|
LocalExecutionEffect::AdaptiveSuccess(LocalAdaptiveSuccessEffect),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
apply_local_execution_effect(
|
apply_local_execution_effect(
|
||||||
&state_for_report,
|
&state_for_report,
|
||||||
LocalExecutionEffectContext {
|
LocalExecutionEffectContext {
|
||||||
|
|||||||
@@ -36,9 +36,10 @@ use crate::execution_runtime::{
|
|||||||
};
|
};
|
||||||
use crate::log_ids::short_request_id;
|
use crate::log_ids::short_request_id;
|
||||||
use crate::orchestration::{
|
use crate::orchestration::{
|
||||||
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAttemptFailureEffect,
|
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect,
|
||||||
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
|
LocalAttemptFailureEffect, LocalExecutionEffect, LocalExecutionEffectContext,
|
||||||
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
|
LocalHealthFailureEffect, LocalHealthSuccessEffect, LocalOAuthInvalidationEffect,
|
||||||
|
LocalPoolErrorEffect,
|
||||||
};
|
};
|
||||||
use crate::request_candidate_runtime::{
|
use crate::request_candidate_runtime::{
|
||||||
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
|
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
|
||||||
@@ -467,6 +468,15 @@ pub(crate) async fn execute_execution_runtime_sync(
|
|||||||
LocalExecutionEffect::HealthSuccess(LocalHealthSuccessEffect),
|
LocalExecutionEffect::HealthSuccess(LocalHealthSuccessEffect),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
apply_local_execution_effect(
|
||||||
|
state,
|
||||||
|
LocalExecutionEffectContext {
|
||||||
|
plan: &plan,
|
||||||
|
report_context: report_context.as_ref(),
|
||||||
|
},
|
||||||
|
LocalExecutionEffect::AdaptiveSuccess(LocalAdaptiveSuccessEffect),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
apply_local_execution_effect(
|
apply_local_execution_effect(
|
||||||
state,
|
state,
|
||||||
LocalExecutionEffectContext {
|
LocalExecutionEffectContext {
|
||||||
|
|||||||
@@ -672,8 +672,6 @@ fn admin_pool_scheduling_payload(
|
|||||||
key: &StoredProviderCatalogKey,
|
key: &StoredProviderCatalogKey,
|
||||||
cooldown_reason: Option<&str>,
|
cooldown_reason: Option<&str>,
|
||||||
cooldown_ttl_seconds: Option<u64>,
|
cooldown_ttl_seconds: Option<u64>,
|
||||||
health_score: f64,
|
|
||||||
circuit_breaker_open: bool,
|
|
||||||
account_blocked: bool,
|
account_blocked: bool,
|
||||||
account_status_code: Option<&str>,
|
account_status_code: Option<&str>,
|
||||||
account_status_label: Option<&str>,
|
account_status_label: Option<&str>,
|
||||||
@@ -741,36 +739,6 @@ fn admin_pool_scheduling_payload(
|
|||||||
})],
|
})],
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if circuit_breaker_open {
|
|
||||||
return (
|
|
||||||
"degraded".to_string(),
|
|
||||||
"circuit_breaker".to_string(),
|
|
||||||
"熔断中".to_string(),
|
|
||||||
vec![json!({
|
|
||||||
"code": "circuit_breaker",
|
|
||||||
"label": "熔断中",
|
|
||||||
"blocking": true,
|
|
||||||
"source": "health",
|
|
||||||
"ttl_seconds": serde_json::Value::Null,
|
|
||||||
"detail": serde_json::Value::Null,
|
|
||||||
})],
|
|
||||||
);
|
|
||||||
}
|
|
||||||
if health_score < 0.5 {
|
|
||||||
return (
|
|
||||||
"degraded".to_string(),
|
|
||||||
"health_low".to_string(),
|
|
||||||
"健康度较低".to_string(),
|
|
||||||
vec![json!({
|
|
||||||
"code": "health_low",
|
|
||||||
"label": "健康度较低",
|
|
||||||
"blocking": false,
|
|
||||||
"source": "health",
|
|
||||||
"ttl_seconds": serde_json::Value::Null,
|
|
||||||
"detail": serde_json::Value::Null,
|
|
||||||
})],
|
|
||||||
);
|
|
||||||
}
|
|
||||||
(
|
(
|
||||||
"available".to_string(),
|
"available".to_string(),
|
||||||
"available".to_string(),
|
"available".to_string(),
|
||||||
@@ -857,8 +825,6 @@ pub(super) fn build_admin_pool_key_payload(
|
|||||||
key,
|
key,
|
||||||
cooldown_reason.as_deref(),
|
cooldown_reason.as_deref(),
|
||||||
cooldown_ttl_seconds,
|
cooldown_ttl_seconds,
|
||||||
health_score,
|
|
||||||
circuit_breaker_open,
|
|
||||||
account_status_blocked,
|
account_status_blocked,
|
||||||
account_status_code.as_deref(),
|
account_status_code.as_deref(),
|
||||||
account_status_label.as_deref(),
|
account_status_label.as_deref(),
|
||||||
|
|||||||
@@ -139,6 +139,9 @@ impl<'a> AdminAppState<'a> {
|
|||||||
key.last_429_at_unix_secs = None;
|
key.last_429_at_unix_secs = None;
|
||||||
key.last_429_type = None;
|
key.last_429_type = None;
|
||||||
key.adjustment_history = None;
|
key.adjustment_history = None;
|
||||||
|
key.utilization_samples = None;
|
||||||
|
key.last_probe_increase_at_unix_secs = None;
|
||||||
|
key.last_rpm_peak = None;
|
||||||
let Some(updated) = self.update_provider_catalog_key(&key).await? else {
|
let Some(updated) = self.update_provider_catalog_key(&key).await? else {
|
||||||
return Ok(admin_adaptive_key_not_found_response(key_id));
|
return Ok(admin_adaptive_key_not_found_response(key_id));
|
||||||
};
|
};
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -2,7 +2,9 @@ use std::collections::BTreeMap;
|
|||||||
|
|
||||||
use aether_admin::provider::quota as admin_provider_quota_pure;
|
use aether_admin::provider::quota as admin_provider_quota_pure;
|
||||||
use aether_contracts::{ExecutionPlan, ExecutionTelemetry};
|
use aether_contracts::{ExecutionPlan, ExecutionTelemetry};
|
||||||
use aether_scheduler_core::build_scheduler_affinity_cache_key_for_api_key_id;
|
use aether_scheduler_core::{
|
||||||
|
build_scheduler_affinity_cache_key_for_api_key_id, count_recent_rpm_requests_for_provider_key,
|
||||||
|
};
|
||||||
use aether_usage_runtime::{
|
use aether_usage_runtime::{
|
||||||
build_stream_terminal_usage_outcome, build_sync_terminal_usage_outcome,
|
build_stream_terminal_usage_outcome, build_sync_terminal_usage_outcome,
|
||||||
GatewayStreamReportRequest, GatewaySyncReportRequest, TerminalUsageOutcome,
|
GatewayStreamReportRequest, GatewaySyncReportRequest, TerminalUsageOutcome,
|
||||||
@@ -11,8 +13,9 @@ use serde_json::Value;
|
|||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
local_failover_error_message, project_local_adaptive_rate_limit, project_local_failure_health,
|
local_failover_error_message, project_local_adaptive_rate_limit,
|
||||||
project_local_success_health, LocalFailoverClassification,
|
project_local_adaptive_success, project_local_failure_health, project_local_success_health,
|
||||||
|
LocalFailoverClassification,
|
||||||
};
|
};
|
||||||
use crate::ai_pipeline::extract_pool_sticky_session_token;
|
use crate::ai_pipeline::extract_pool_sticky_session_token;
|
||||||
use crate::clock::current_unix_secs;
|
use crate::clock::current_unix_secs;
|
||||||
@@ -59,6 +62,9 @@ pub(crate) struct LocalHealthFailureEffect {
|
|||||||
#[derive(Debug, Clone, Copy)]
|
#[derive(Debug, Clone, Copy)]
|
||||||
pub(crate) struct LocalHealthSuccessEffect;
|
pub(crate) struct LocalHealthSuccessEffect;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
pub(crate) struct LocalAdaptiveSuccessEffect;
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy)]
|
#[derive(Debug, Clone, Copy)]
|
||||||
pub(crate) struct LocalOAuthInvalidationEffect<'a> {
|
pub(crate) struct LocalOAuthInvalidationEffect<'a> {
|
||||||
pub(crate) status_code: u16,
|
pub(crate) status_code: u16,
|
||||||
@@ -71,6 +77,7 @@ pub(crate) enum LocalExecutionEffect<'a> {
|
|||||||
AdaptiveRateLimit(LocalAdaptiveRateLimitEffect<'a>),
|
AdaptiveRateLimit(LocalAdaptiveRateLimitEffect<'a>),
|
||||||
HealthFailure(LocalHealthFailureEffect),
|
HealthFailure(LocalHealthFailureEffect),
|
||||||
HealthSuccess(LocalHealthSuccessEffect),
|
HealthSuccess(LocalHealthSuccessEffect),
|
||||||
|
AdaptiveSuccess(LocalAdaptiveSuccessEffect),
|
||||||
OauthInvalidation(LocalOAuthInvalidationEffect<'a>),
|
OauthInvalidation(LocalOAuthInvalidationEffect<'a>),
|
||||||
PoolSuccessSync {
|
PoolSuccessSync {
|
||||||
payload: &'a GatewaySyncReportRequest,
|
payload: &'a GatewaySyncReportRequest,
|
||||||
@@ -88,6 +95,8 @@ struct PoolFeedbackContext {
|
|||||||
sticky_session_token: Option<String>,
|
sticky_session_token: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const ADAPTIVE_RPM_RECENT_CANDIDATE_LIMIT: usize = 512;
|
||||||
|
|
||||||
pub(crate) async fn apply_local_execution_effect(
|
pub(crate) async fn apply_local_execution_effect(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
context: LocalExecutionEffectContext<'_>,
|
context: LocalExecutionEffectContext<'_>,
|
||||||
@@ -106,6 +115,9 @@ pub(crate) async fn apply_local_execution_effect(
|
|||||||
LocalExecutionEffect::HealthSuccess(effect) => {
|
LocalExecutionEffect::HealthSuccess(effect) => {
|
||||||
record_health_success_effect(state, context, effect).await;
|
record_health_success_effect(state, context, effect).await;
|
||||||
}
|
}
|
||||||
|
LocalExecutionEffect::AdaptiveSuccess(effect) => {
|
||||||
|
record_adaptive_success_effect(state, context, effect).await;
|
||||||
|
}
|
||||||
LocalExecutionEffect::OauthInvalidation(effect) => {
|
LocalExecutionEffect::OauthInvalidation(effect) => {
|
||||||
record_oauth_invalidation_effect(state, context, effect).await;
|
record_oauth_invalidation_effect(state, context, effect).await;
|
||||||
}
|
}
|
||||||
@@ -256,6 +268,7 @@ async fn record_adaptive_rate_limit_effect(
|
|||||||
context: LocalExecutionEffectContext<'_>,
|
context: LocalExecutionEffectContext<'_>,
|
||||||
effect: LocalAdaptiveRateLimitEffect<'_>,
|
effect: LocalAdaptiveRateLimitEffect<'_>,
|
||||||
) {
|
) {
|
||||||
|
let observed_at_unix_secs = current_unix_secs();
|
||||||
let Some(current_key) = state
|
let Some(current_key) = state
|
||||||
.read_provider_catalog_keys_by_ids(std::slice::from_ref(&context.plan.key_id))
|
.read_provider_catalog_keys_by_ids(std::slice::from_ref(&context.plan.key_id))
|
||||||
.await
|
.await
|
||||||
@@ -264,22 +277,39 @@ async fn record_adaptive_rate_limit_effect(
|
|||||||
else {
|
else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
let current_rpm = state
|
||||||
|
.read_recent_request_candidates(ADAPTIVE_RPM_RECENT_CANDIDATE_LIMIT)
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
|
.map(|recent_candidates| {
|
||||||
|
count_recent_rpm_requests_for_provider_key(
|
||||||
|
&recent_candidates,
|
||||||
|
&context.plan.key_id,
|
||||||
|
observed_at_unix_secs,
|
||||||
|
) as u32
|
||||||
|
});
|
||||||
let Some(projection) = project_local_adaptive_rate_limit(
|
let Some(projection) = project_local_adaptive_rate_limit(
|
||||||
¤t_key,
|
¤t_key,
|
||||||
effect.classification,
|
effect.classification,
|
||||||
effect.status_code,
|
effect.status_code,
|
||||||
|
current_rpm,
|
||||||
effect.headers,
|
effect.headers,
|
||||||
current_unix_secs(),
|
observed_at_unix_secs,
|
||||||
) else {
|
) else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut updated_key = current_key.clone();
|
let mut updated_key = current_key.clone();
|
||||||
updated_key.rpm_429_count = Some(projection.rpm_429_count);
|
updated_key.rpm_429_count = Some(projection.rpm_429_count).filter(|value| *value > 0);
|
||||||
|
updated_key.learned_rpm_limit = projection.learned_rpm_limit;
|
||||||
updated_key.last_429_at_unix_secs = Some(projection.last_429_at_unix_secs);
|
updated_key.last_429_at_unix_secs = Some(projection.last_429_at_unix_secs);
|
||||||
updated_key.last_429_type = Some(projection.last_429_type);
|
updated_key.last_429_type = Some(projection.last_429_type);
|
||||||
|
updated_key.adjustment_history = projection.adjustment_history;
|
||||||
|
updated_key.utilization_samples = projection.utilization_samples;
|
||||||
|
updated_key.last_probe_increase_at_unix_secs = projection.last_probe_increase_at_unix_secs;
|
||||||
|
updated_key.last_rpm_peak = projection.last_rpm_peak;
|
||||||
updated_key.status_snapshot = Some(projection.status_snapshot);
|
updated_key.status_snapshot = Some(projection.status_snapshot);
|
||||||
updated_key.updated_at_unix_secs = Some(current_unix_secs());
|
updated_key.updated_at_unix_secs = Some(observed_at_unix_secs);
|
||||||
|
|
||||||
if let Err(err) = state.update_provider_catalog_key(&updated_key).await {
|
if let Err(err) = state.update_provider_catalog_key(&updated_key).await {
|
||||||
warn!(
|
warn!(
|
||||||
@@ -289,6 +319,54 @@ async fn record_adaptive_rate_limit_effect(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn record_adaptive_success_effect(
|
||||||
|
state: &AppState,
|
||||||
|
context: LocalExecutionEffectContext<'_>,
|
||||||
|
_effect: LocalAdaptiveSuccessEffect,
|
||||||
|
) {
|
||||||
|
let observed_at_unix_secs = current_unix_secs();
|
||||||
|
let Some(current_key) = state
|
||||||
|
.read_provider_catalog_keys_by_ids(std::slice::from_ref(&context.plan.key_id))
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
|
.and_then(|mut keys| keys.drain(..).next())
|
||||||
|
else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let Some(recent_candidates) = state
|
||||||
|
.read_recent_request_candidates(ADAPTIVE_RPM_RECENT_CANDIDATE_LIMIT)
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
|
else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let current_rpm = count_recent_rpm_requests_for_provider_key(
|
||||||
|
&recent_candidates,
|
||||||
|
&context.plan.key_id,
|
||||||
|
observed_at_unix_secs,
|
||||||
|
) as u32;
|
||||||
|
let Some(projection) =
|
||||||
|
project_local_adaptive_success(¤t_key, current_rpm, observed_at_unix_secs)
|
||||||
|
else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut updated_key = current_key.clone();
|
||||||
|
updated_key.learned_rpm_limit = projection.learned_rpm_limit;
|
||||||
|
updated_key.adjustment_history = projection.adjustment_history;
|
||||||
|
updated_key.utilization_samples = projection.utilization_samples;
|
||||||
|
updated_key.last_probe_increase_at_unix_secs = projection.last_probe_increase_at_unix_secs;
|
||||||
|
updated_key.status_snapshot = Some(projection.status_snapshot);
|
||||||
|
updated_key.updated_at_unix_secs = Some(observed_at_unix_secs);
|
||||||
|
|
||||||
|
if let Err(err) = state.update_provider_catalog_key(&updated_key).await {
|
||||||
|
warn!(
|
||||||
|
"gateway orchestration effects: failed to persist adaptive success projection for provider {} endpoint {} key {}: {:?}",
|
||||||
|
context.plan.provider_id, context.plan.endpoint_id, context.plan.key_id, err
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn record_health_failure_effect(
|
async fn record_health_failure_effect(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
context: LocalExecutionEffectContext<'_>,
|
context: LocalExecutionEffectContext<'_>,
|
||||||
@@ -540,7 +618,11 @@ mod tests {
|
|||||||
|
|
||||||
use aether_contracts::{ExecutionPlan, RequestBody};
|
use aether_contracts::{ExecutionPlan, RequestBody};
|
||||||
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
|
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
|
||||||
|
use aether_data::repository::candidates::InMemoryRequestCandidateRepository;
|
||||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||||
|
use aether_data_contracts::repository::candidates::{
|
||||||
|
RequestCandidateStatus, StoredRequestCandidate,
|
||||||
|
};
|
||||||
use aether_data_contracts::repository::provider_catalog::{
|
use aether_data_contracts::repository::provider_catalog::{
|
||||||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
|
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
|
||||||
};
|
};
|
||||||
@@ -548,9 +630,9 @@ mod tests {
|
|||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
apply_local_execution_effect, local_candidate_failure_should_record_pool_error,
|
apply_local_execution_effect, local_candidate_failure_should_record_pool_error,
|
||||||
LocalAdaptiveRateLimitEffect, LocalAttemptFailureEffect, LocalExecutionEffect,
|
LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect, LocalAttemptFailureEffect,
|
||||||
LocalExecutionEffectContext, LocalHealthFailureEffect, LocalHealthSuccessEffect,
|
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
|
||||||
LocalOAuthInvalidationEffect,
|
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect,
|
||||||
};
|
};
|
||||||
use crate::data::GatewayDataState;
|
use crate::data::GatewayDataState;
|
||||||
use crate::orchestration::LocalFailoverClassification;
|
use crate::orchestration::LocalFailoverClassification;
|
||||||
@@ -791,6 +873,26 @@ mod tests {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn adaptive_state_with_request_candidates(
|
||||||
|
key: StoredProviderCatalogKey,
|
||||||
|
request_candidates: Vec<StoredRequestCandidate>,
|
||||||
|
) -> AppState {
|
||||||
|
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||||
|
vec![sample_health_provider()],
|
||||||
|
vec![sample_health_endpoint()],
|
||||||
|
vec![key],
|
||||||
|
));
|
||||||
|
let request_candidates =
|
||||||
|
Arc::new(InMemoryRequestCandidateRepository::seed(request_candidates));
|
||||||
|
AppState::new()
|
||||||
|
.expect("gateway state should build")
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
GatewayDataState::with_provider_catalog_repository_for_tests(provider_catalog)
|
||||||
|
.with_request_candidate_reader(request_candidates)
|
||||||
|
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
fn fixed_limit_state() -> AppState {
|
fn fixed_limit_state() -> AppState {
|
||||||
let mut key = sample_health_key();
|
let mut key = sample_health_key();
|
||||||
key.rpm_limit = Some(24);
|
key.rpm_limit = Some(24);
|
||||||
@@ -1168,7 +1270,7 @@ mod tests {
|
|||||||
.status_snapshot
|
.status_snapshot
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|value| value.get("learning_confidence")),
|
.and_then(|value| value.get("learning_confidence")),
|
||||||
Some(&json!(0.283))
|
Some(&json!(0.3))
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
stored_key
|
stored_key
|
||||||
@@ -1212,4 +1314,97 @@ mod tests {
|
|||||||
assert_eq!(stored_key.last_429_at_unix_secs, None);
|
assert_eq!(stored_key.last_429_at_unix_secs, None);
|
||||||
assert_eq!(stored_key.last_429_type, None);
|
assert_eq!(stored_key.last_429_type, None);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn adaptive_success_effect_expands_limit_from_recent_rpm_usage() {
|
||||||
|
let now_unix_secs = chrono::Utc::now().timestamp().max(0) as u64;
|
||||||
|
let mut key = sample_adaptive_key();
|
||||||
|
key.learned_rpm_limit = Some(20);
|
||||||
|
key.last_rpm_peak = Some(25);
|
||||||
|
key.last_429_at_unix_secs = Some(now_unix_secs.saturating_sub(600));
|
||||||
|
key.adjustment_history = Some(json!([
|
||||||
|
{
|
||||||
|
"timestamp": "2026-04-19T00:00:00Z",
|
||||||
|
"old_limit": 0,
|
||||||
|
"new_limit": 20,
|
||||||
|
"reason": "rpm_429",
|
||||||
|
"confidence": 0.8
|
||||||
|
}
|
||||||
|
]));
|
||||||
|
key.utilization_samples = Some(json!([
|
||||||
|
{"ts": now_unix_secs.saturating_sub(40), "util": 0.90},
|
||||||
|
{"ts": now_unix_secs.saturating_sub(30), "util": 0.95},
|
||||||
|
{"ts": now_unix_secs.saturating_sub(20), "util": 0.85},
|
||||||
|
{"ts": now_unix_secs.saturating_sub(10), "util": 0.80}
|
||||||
|
]));
|
||||||
|
let state = adaptive_state_with_request_candidates(
|
||||||
|
key,
|
||||||
|
vec![StoredRequestCandidate::new(
|
||||||
|
"candidate-1".to_string(),
|
||||||
|
"req-1".to_string(),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
Some("prov-1".to_string()),
|
||||||
|
Some("ep-1".to_string()),
|
||||||
|
Some("key-1".to_string()),
|
||||||
|
RequestCandidateStatus::Success,
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
Some(200),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
Some(10),
|
||||||
|
Some(19),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
i64::try_from(now_unix_secs.saturating_sub(30) * 1000)
|
||||||
|
.expect("candidate created_at should fit i64"),
|
||||||
|
Some(
|
||||||
|
i64::try_from(now_unix_secs.saturating_sub(30) * 1000)
|
||||||
|
.expect("candidate started_at should fit i64"),
|
||||||
|
),
|
||||||
|
Some(
|
||||||
|
i64::try_from(now_unix_secs.saturating_sub(29) * 1000)
|
||||||
|
.expect("candidate finished_at should fit i64"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.expect("request candidate should build")],
|
||||||
|
);
|
||||||
|
let plan = sample_plan();
|
||||||
|
|
||||||
|
apply_local_execution_effect(
|
||||||
|
&state,
|
||||||
|
LocalExecutionEffectContext {
|
||||||
|
plan: &plan,
|
||||||
|
report_context: None,
|
||||||
|
},
|
||||||
|
LocalExecutionEffect::AdaptiveSuccess(LocalAdaptiveSuccessEffect),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let stored_key = state
|
||||||
|
.read_provider_catalog_keys_by_ids(std::slice::from_ref(&plan.key_id))
|
||||||
|
.await
|
||||||
|
.expect("provider catalog keys should load")
|
||||||
|
.into_iter()
|
||||||
|
.next()
|
||||||
|
.expect("stored key should exist");
|
||||||
|
assert_eq!(stored_key.learned_rpm_limit, Some(25));
|
||||||
|
assert_eq!(stored_key.utilization_samples, Some(json!([])));
|
||||||
|
assert_eq!(
|
||||||
|
stored_key
|
||||||
|
.adjustment_history
|
||||||
|
.as_ref()
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.and_then(|items| items.last())
|
||||||
|
.and_then(Value::as_object)
|
||||||
|
.and_then(|record| record.get("reason"))
|
||||||
|
.and_then(Value::as_str),
|
||||||
|
Some("high_utilization")
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,7 +12,8 @@ mod recovery;
|
|||||||
mod report_effects;
|
mod report_effects;
|
||||||
|
|
||||||
pub(crate) use self::adaptive::{
|
pub(crate) use self::adaptive::{
|
||||||
project_local_adaptive_rate_limit, LocalAdaptiveRateLimitProjection,
|
project_local_adaptive_rate_limit, project_local_adaptive_success,
|
||||||
|
LocalAdaptiveRateLimitProjection, LocalAdaptiveSuccessProjection,
|
||||||
};
|
};
|
||||||
pub(crate) use self::attempt::{
|
pub(crate) use self::attempt::{
|
||||||
attempt_identity_from_report_context, build_local_attempt_identities,
|
attempt_identity_from_report_context, build_local_attempt_identities,
|
||||||
@@ -24,9 +25,10 @@ pub(crate) use self::classifier::{
|
|||||||
LocalFailoverInput,
|
LocalFailoverInput,
|
||||||
};
|
};
|
||||||
pub(crate) use self::effects::{
|
pub(crate) use self::effects::{
|
||||||
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAttemptFailureEffect,
|
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect,
|
||||||
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
|
LocalAttemptFailureEffect, LocalExecutionEffect, LocalExecutionEffectContext,
|
||||||
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
|
LocalHealthFailureEffect, LocalHealthSuccessEffect, LocalOAuthInvalidationEffect,
|
||||||
|
LocalPoolErrorEffect,
|
||||||
};
|
};
|
||||||
pub(crate) use self::health::{project_local_failure_health, project_local_success_health};
|
pub(crate) use self::health::{project_local_failure_health, project_local_success_health};
|
||||||
pub(crate) use self::policy::{
|
pub(crate) use self::policy::{
|
||||||
|
|||||||
@@ -761,6 +761,93 @@ async fn gateway_marks_account_blocked_pool_key_in_list_keys_response() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn gateway_ignores_health_signals_in_pool_scheduling_status() {
|
||||||
|
let mut provider = sample_provider("provider-openai", "openai", 10).with_transport_fields(
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
Some(json!({
|
||||||
|
"pool_advanced": {
|
||||||
|
"enabled": true
|
||||||
|
}
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
provider.provider_type = "openai".to_string();
|
||||||
|
|
||||||
|
let mut circuit_key = sample_key(
|
||||||
|
"key-openai-circuit",
|
||||||
|
"provider-openai",
|
||||||
|
"openai:chat",
|
||||||
|
"sk-circuit",
|
||||||
|
);
|
||||||
|
circuit_key.name = "circuit-open".to_string();
|
||||||
|
circuit_key.circuit_breaker_by_format = Some(json!({
|
||||||
|
"openai:chat": {
|
||||||
|
"open": true
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
|
||||||
|
let mut low_health_key = sample_key(
|
||||||
|
"key-openai-health",
|
||||||
|
"provider-openai",
|
||||||
|
"openai:chat",
|
||||||
|
"sk-health",
|
||||||
|
);
|
||||||
|
low_health_key.name = "low-health".to_string();
|
||||||
|
low_health_key.health_by_format = Some(json!({
|
||||||
|
"openai:chat": {
|
||||||
|
"health_score": 0.2
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
|
||||||
|
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||||
|
vec![provider],
|
||||||
|
Vec::new(),
|
||||||
|
vec![circuit_key, low_health_key],
|
||||||
|
));
|
||||||
|
let state = AppState::new()
|
||||||
|
.expect("gateway should build")
|
||||||
|
.with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests(
|
||||||
|
provider_catalog_repository,
|
||||||
|
));
|
||||||
|
|
||||||
|
let response = local_admin_pool_response(
|
||||||
|
&state,
|
||||||
|
http::Method::GET,
|
||||||
|
"/api/admin/pool/provider-openai/keys?page=1&page_size=50&status=all",
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
let payload: serde_json::Value = serde_json::from_slice(
|
||||||
|
&to_bytes(response.into_body(), usize::MAX)
|
||||||
|
.await
|
||||||
|
.expect("body should read"),
|
||||||
|
)
|
||||||
|
.expect("json body should parse");
|
||||||
|
let keys = payload["keys"].as_array().expect("keys should be array");
|
||||||
|
|
||||||
|
assert_eq!(keys.len(), 2);
|
||||||
|
assert_eq!(keys[0]["key_name"], json!("circuit-open"));
|
||||||
|
assert_eq!(keys[0]["circuit_breaker_open"], json!(true));
|
||||||
|
assert_eq!(keys[0]["scheduling_status"], json!("available"));
|
||||||
|
assert_eq!(keys[0]["scheduling_reason"], json!("available"));
|
||||||
|
assert_eq!(keys[0]["scheduling_label"], json!("可用"));
|
||||||
|
|
||||||
|
assert_eq!(keys[1]["key_name"], json!("low-health"));
|
||||||
|
assert_eq!(keys[1]["health_score"], json!(0.2));
|
||||||
|
assert_eq!(keys[1]["scheduling_status"], json!("available"));
|
||||||
|
assert_eq!(keys[1]["scheduling_reason"], json!("available"));
|
||||||
|
assert_eq!(keys[1]["scheduling_label"], json!("可用"));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn gateway_handles_admin_pool_list_keys_with_quota_compatibility_fields() {
|
async fn gateway_handles_admin_pool_list_keys_with_quota_compatibility_fields() {
|
||||||
let upstream_hits = Arc::new(Mutex::new(0usize));
|
let upstream_hits = Arc::new(Mutex::new(0usize));
|
||||||
|
|||||||
@@ -309,8 +309,6 @@ fn admin_pool_scheduling_payload(
|
|||||||
key: &StoredProviderCatalogKey,
|
key: &StoredProviderCatalogKey,
|
||||||
cooldown_reason: Option<&str>,
|
cooldown_reason: Option<&str>,
|
||||||
cooldown_ttl_seconds: Option<u64>,
|
cooldown_ttl_seconds: Option<u64>,
|
||||||
health_score: f64,
|
|
||||||
circuit_breaker_open: bool,
|
|
||||||
) -> (String, String, String, Vec<Value>) {
|
) -> (String, String, String, Vec<Value>) {
|
||||||
if !key.is_active {
|
if !key.is_active {
|
||||||
return (
|
return (
|
||||||
@@ -342,36 +340,6 @@ fn admin_pool_scheduling_payload(
|
|||||||
})],
|
})],
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if circuit_breaker_open {
|
|
||||||
return (
|
|
||||||
"degraded".to_string(),
|
|
||||||
"circuit_breaker".to_string(),
|
|
||||||
"熔断中".to_string(),
|
|
||||||
vec![json!({
|
|
||||||
"code": "circuit_breaker",
|
|
||||||
"label": "熔断中",
|
|
||||||
"blocking": true,
|
|
||||||
"source": "health",
|
|
||||||
"ttl_seconds": Value::Null,
|
|
||||||
"detail": Value::Null,
|
|
||||||
})],
|
|
||||||
);
|
|
||||||
}
|
|
||||||
if health_score < 0.5 {
|
|
||||||
return (
|
|
||||||
"degraded".to_string(),
|
|
||||||
"health_low".to_string(),
|
|
||||||
"健康度较低".to_string(),
|
|
||||||
vec![json!({
|
|
||||||
"code": "health_low",
|
|
||||||
"label": "健康度较低",
|
|
||||||
"blocking": false,
|
|
||||||
"source": "health",
|
|
||||||
"ttl_seconds": Value::Null,
|
|
||||||
"detail": Value::Null,
|
|
||||||
})],
|
|
||||||
);
|
|
||||||
}
|
|
||||||
(
|
(
|
||||||
"available".to_string(),
|
"available".to_string(),
|
||||||
"available".to_string(),
|
"available".to_string(),
|
||||||
@@ -644,7 +612,10 @@ pub fn build_admin_pool_selection_payload(keys: &[StoredProviderCatalogKey]) ->
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
#[allow(clippy::items_after_test_module)]
|
#[allow(clippy::items_after_test_module)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{admin_pool_key_account_quota_exhausted, admin_pool_key_is_known_banned};
|
use super::{
|
||||||
|
admin_pool_key_account_quota_exhausted, admin_pool_key_is_known_banned,
|
||||||
|
build_admin_pool_key_payload, AdminPoolKeyPayloadContext,
|
||||||
|
};
|
||||||
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
|
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
@@ -781,6 +752,29 @@ mod tests {
|
|||||||
|
|
||||||
assert!(admin_pool_key_is_known_banned(&key));
|
assert!(admin_pool_key_is_known_banned(&key));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_admin_pool_key_payload_ignores_health_for_scheduling() {
|
||||||
|
let mut key = sample_key(None);
|
||||||
|
key.health_by_format = Some(json!({
|
||||||
|
"openai:chat": {
|
||||||
|
"health_score": 0.2
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
key.circuit_breaker_by_format = Some(json!({
|
||||||
|
"openai:chat": {
|
||||||
|
"open": true
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
|
||||||
|
let payload = build_admin_pool_key_payload(&key, &AdminPoolKeyPayloadContext::default());
|
||||||
|
|
||||||
|
assert_eq!(payload["health_score"], json!(0.2));
|
||||||
|
assert_eq!(payload["circuit_breaker_open"], json!(true));
|
||||||
|
assert_eq!(payload["scheduling_status"], json!("available"));
|
||||||
|
assert_eq!(payload["scheduling_reason"], json!("available"));
|
||||||
|
assert_eq!(payload["scheduling_label"], json!("可用"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn build_admin_pool_key_payload(
|
pub fn build_admin_pool_key_payload(
|
||||||
@@ -794,8 +788,6 @@ pub fn build_admin_pool_key_payload(
|
|||||||
key,
|
key,
|
||||||
context.cooldown_reason.as_deref(),
|
context.cooldown_reason.as_deref(),
|
||||||
context.cooldown_ttl_seconds,
|
context.cooldown_ttl_seconds,
|
||||||
health_score,
|
|
||||||
circuit_breaker_open,
|
|
||||||
);
|
);
|
||||||
|
|
||||||
json!({
|
json!({
|
||||||
|
|||||||
@@ -269,6 +269,7 @@ pub struct StoredProviderCatalogKey {
|
|||||||
pub adjustment_history: Option<serde_json::Value>,
|
pub adjustment_history: Option<serde_json::Value>,
|
||||||
pub utilization_samples: Option<serde_json::Value>,
|
pub utilization_samples: Option<serde_json::Value>,
|
||||||
pub last_probe_increase_at_unix_secs: Option<u64>,
|
pub last_probe_increase_at_unix_secs: Option<u64>,
|
||||||
|
pub last_rpm_peak: Option<u32>,
|
||||||
pub request_count: Option<u32>,
|
pub request_count: Option<u32>,
|
||||||
pub total_tokens: u64,
|
pub total_tokens: u64,
|
||||||
pub total_cost_usd: f64,
|
pub total_cost_usd: f64,
|
||||||
@@ -341,6 +342,7 @@ impl StoredProviderCatalogKey {
|
|||||||
adjustment_history: None,
|
adjustment_history: None,
|
||||||
utilization_samples: None,
|
utilization_samples: None,
|
||||||
last_probe_increase_at_unix_secs: None,
|
last_probe_increase_at_unix_secs: None,
|
||||||
|
last_rpm_peak: None,
|
||||||
request_count: None,
|
request_count: None,
|
||||||
total_tokens: 0,
|
total_tokens: 0,
|
||||||
total_cost_usd: 0.0,
|
total_cost_usd: 0.0,
|
||||||
|
|||||||
@@ -161,6 +161,7 @@ SELECT
|
|||||||
adjustment_history,
|
adjustment_history,
|
||||||
utilization_samples,
|
utilization_samples,
|
||||||
EXTRACT(EPOCH FROM last_probe_increase_at)::bigint AS last_probe_increase_at_unix_secs,
|
EXTRACT(EPOCH FROM last_probe_increase_at)::bigint AS last_probe_increase_at_unix_secs,
|
||||||
|
last_rpm_peak,
|
||||||
request_count,
|
request_count,
|
||||||
total_tokens,
|
total_tokens,
|
||||||
CAST(total_cost_usd AS DOUBLE PRECISION) AS total_cost_usd,
|
CAST(total_cost_usd AS DOUBLE PRECISION) AS total_cost_usd,
|
||||||
@@ -216,6 +217,7 @@ SELECT
|
|||||||
adjustment_history,
|
adjustment_history,
|
||||||
utilization_samples,
|
utilization_samples,
|
||||||
EXTRACT(EPOCH FROM last_probe_increase_at)::bigint AS last_probe_increase_at_unix_secs,
|
EXTRACT(EPOCH FROM last_probe_increase_at)::bigint AS last_probe_increase_at_unix_secs,
|
||||||
|
last_rpm_peak,
|
||||||
request_count,
|
request_count,
|
||||||
total_tokens,
|
total_tokens,
|
||||||
CAST(total_cost_usd AS DOUBLE PRECISION) AS total_cost_usd,
|
CAST(total_cost_usd AS DOUBLE PRECISION) AS total_cost_usd,
|
||||||
@@ -271,6 +273,7 @@ SELECT
|
|||||||
NULL::jsonb AS adjustment_history,
|
NULL::jsonb AS adjustment_history,
|
||||||
NULL::jsonb AS utilization_samples,
|
NULL::jsonb AS utilization_samples,
|
||||||
NULL::bigint AS last_probe_increase_at_unix_secs,
|
NULL::bigint AS last_probe_increase_at_unix_secs,
|
||||||
|
NULL::integer AS last_rpm_peak,
|
||||||
request_count,
|
request_count,
|
||||||
0::bigint AS total_tokens,
|
0::bigint AS total_tokens,
|
||||||
0::double precision AS total_cost_usd,
|
0::double precision AS total_cost_usd,
|
||||||
@@ -607,6 +610,7 @@ SELECT
|
|||||||
adjustment_history,
|
adjustment_history,
|
||||||
utilization_samples,
|
utilization_samples,
|
||||||
EXTRACT(EPOCH FROM last_probe_increase_at)::bigint AS last_probe_increase_at_unix_secs,
|
EXTRACT(EPOCH FROM last_probe_increase_at)::bigint AS last_probe_increase_at_unix_secs,
|
||||||
|
last_rpm_peak,
|
||||||
request_count,
|
request_count,
|
||||||
total_tokens,
|
total_tokens,
|
||||||
CAST(total_cost_usd AS DOUBLE PRECISION) AS total_cost_usd,
|
CAST(total_cost_usd AS DOUBLE PRECISION) AS total_cost_usd,
|
||||||
@@ -1208,6 +1212,7 @@ INSERT INTO provider_api_keys (
|
|||||||
adjustment_history,
|
adjustment_history,
|
||||||
utilization_samples,
|
utilization_samples,
|
||||||
last_probe_increase_at,
|
last_probe_increase_at,
|
||||||
|
last_rpm_peak,
|
||||||
request_count,
|
request_count,
|
||||||
total_tokens,
|
total_tokens,
|
||||||
total_cost_usd,
|
total_cost_usd,
|
||||||
@@ -1270,31 +1275,32 @@ INSERT INTO provider_api_keys (
|
|||||||
WHEN $35::double precision IS NULL THEN NULL
|
WHEN $35::double precision IS NULL THEN NULL
|
||||||
ELSE TO_TIMESTAMP($35::double precision)
|
ELSE TO_TIMESTAMP($35::double precision)
|
||||||
END,
|
END,
|
||||||
COALESCE($36, 0),
|
$36,
|
||||||
COALESCE($37, 0),
|
COALESCE($37, 0),
|
||||||
COALESCE($38, 0),
|
COALESCE($38, 0),
|
||||||
COALESCE($39, 0),
|
COALESCE($39, 0),
|
||||||
COALESCE($40, 0),
|
COALESCE($40, 0),
|
||||||
COALESCE($41, 0),
|
COALESCE($41, 0),
|
||||||
CASE
|
COALESCE($42, 0),
|
||||||
WHEN $42::double precision IS NULL THEN NULL
|
|
||||||
ELSE TO_TIMESTAMP($42::double precision)
|
|
||||||
END,
|
|
||||||
CASE
|
CASE
|
||||||
WHEN $43::double precision IS NULL THEN NULL
|
WHEN $43::double precision IS NULL THEN NULL
|
||||||
ELSE TO_TIMESTAMP($43::double precision)
|
ELSE TO_TIMESTAMP($43::double precision)
|
||||||
END,
|
END,
|
||||||
$44,
|
CASE
|
||||||
|
WHEN $44::double precision IS NULL THEN NULL
|
||||||
|
ELSE TO_TIMESTAMP($44::double precision)
|
||||||
|
END,
|
||||||
$45,
|
$45,
|
||||||
$46,
|
$46,
|
||||||
$47,
|
$47,
|
||||||
CASE
|
$48,
|
||||||
WHEN $48::double precision IS NULL THEN NOW()
|
|
||||||
ELSE TO_TIMESTAMP($48::double precision)
|
|
||||||
END,
|
|
||||||
CASE
|
CASE
|
||||||
WHEN $49::double precision IS NULL THEN NOW()
|
WHEN $49::double precision IS NULL THEN NOW()
|
||||||
ELSE TO_TIMESTAMP($49::double precision)
|
ELSE TO_TIMESTAMP($49::double precision)
|
||||||
|
END,
|
||||||
|
CASE
|
||||||
|
WHEN $50::double precision IS NULL THEN NOW()
|
||||||
|
ELSE TO_TIMESTAMP($50::double precision)
|
||||||
END
|
END
|
||||||
)
|
)
|
||||||
"#,
|
"#,
|
||||||
@@ -1337,6 +1343,7 @@ INSERT INTO provider_api_keys (
|
|||||||
key.last_probe_increase_at_unix_secs
|
key.last_probe_increase_at_unix_secs
|
||||||
.map(|value| value as f64),
|
.map(|value| value as f64),
|
||||||
)
|
)
|
||||||
|
.bind(key.last_rpm_peak.map(|value| value as i32))
|
||||||
.bind(key.request_count.map(|value| value as i32))
|
.bind(key.request_count.map(|value| value as i32))
|
||||||
.bind(Some(i64::try_from(key.total_tokens).map_err(|_| {
|
.bind(Some(i64::try_from(key.total_tokens).map_err(|_| {
|
||||||
DataLayerError::InvalidInput(format!(
|
DataLayerError::InvalidInput(format!(
|
||||||
@@ -1731,10 +1738,24 @@ SET
|
|||||||
END,
|
END,
|
||||||
oauth_invalid_reason = $26,
|
oauth_invalid_reason = $26,
|
||||||
status_snapshot = $27,
|
status_snapshot = $27,
|
||||||
is_active = $28,
|
concurrent_429_count = $28,
|
||||||
|
rpm_429_count = $29,
|
||||||
|
last_429_at = CASE
|
||||||
|
WHEN $30::double precision IS NULL THEN NULL
|
||||||
|
ELSE TO_TIMESTAMP($30::double precision)
|
||||||
|
END,
|
||||||
|
last_429_type = $31,
|
||||||
|
adjustment_history = $32,
|
||||||
|
utilization_samples = $33,
|
||||||
|
last_probe_increase_at = CASE
|
||||||
|
WHEN $34::double precision IS NULL THEN NULL
|
||||||
|
ELSE TO_TIMESTAMP($34::double precision)
|
||||||
|
END,
|
||||||
|
last_rpm_peak = $35,
|
||||||
|
is_active = $36,
|
||||||
updated_at = CASE
|
updated_at = CASE
|
||||||
WHEN $29::double precision IS NULL THEN NOW()
|
WHEN $37::double precision IS NULL THEN NOW()
|
||||||
ELSE TO_TIMESTAMP($29::double precision)
|
ELSE TO_TIMESTAMP($37::double precision)
|
||||||
END
|
END
|
||||||
WHERE id = $1
|
WHERE id = $1
|
||||||
"#,
|
"#,
|
||||||
@@ -1766,6 +1787,17 @@ WHERE id = $1
|
|||||||
.bind(key.oauth_invalid_at_unix_secs.map(|value| value as f64))
|
.bind(key.oauth_invalid_at_unix_secs.map(|value| value as f64))
|
||||||
.bind(&key.oauth_invalid_reason)
|
.bind(&key.oauth_invalid_reason)
|
||||||
.bind(&key.status_snapshot)
|
.bind(&key.status_snapshot)
|
||||||
|
.bind(key.concurrent_429_count.map(|value| value as i32))
|
||||||
|
.bind(key.rpm_429_count.map(|value| value as i32))
|
||||||
|
.bind(key.last_429_at_unix_secs.map(|value| value as f64))
|
||||||
|
.bind(&key.last_429_type)
|
||||||
|
.bind(&key.adjustment_history)
|
||||||
|
.bind(&key.utilization_samples)
|
||||||
|
.bind(
|
||||||
|
key.last_probe_increase_at_unix_secs
|
||||||
|
.map(|value| value as f64),
|
||||||
|
)
|
||||||
|
.bind(key.last_rpm_peak.map(|value| value as i32))
|
||||||
.bind(key.is_active)
|
.bind(key.is_active)
|
||||||
.bind(key.updated_at_unix_secs.map(|value| value as f64))
|
.bind(key.updated_at_unix_secs.map(|value| value as f64))
|
||||||
.execute(&self.pool)
|
.execute(&self.pool)
|
||||||
@@ -2330,6 +2362,15 @@ fn map_key_row(row: &PgRow) -> Result<StoredProviderCatalogKey, DataLayerError>
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
.transpose()?;
|
.transpose()?;
|
||||||
|
let last_rpm_peak = row_get::<Option<i32>>(row, "last_rpm_peak")?
|
||||||
|
.map(|value| {
|
||||||
|
u32::try_from(value).map_err(|_| {
|
||||||
|
DataLayerError::UnexpectedValue(format!(
|
||||||
|
"invalid provider_api_keys.last_rpm_peak: {value}"
|
||||||
|
))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.transpose()?;
|
||||||
let last_models_fetch_at_unix_secs =
|
let last_models_fetch_at_unix_secs =
|
||||||
row_get::<Option<i64>>(row, "last_models_fetch_at_unix_secs")?
|
row_get::<Option<i64>>(row, "last_models_fetch_at_unix_secs")?
|
||||||
.map(|value| {
|
.map(|value| {
|
||||||
@@ -2425,6 +2466,7 @@ fn map_key_row(row: &PgRow) -> Result<StoredProviderCatalogKey, DataLayerError>
|
|||||||
key.last_429_type = row.try_get("last_429_type").ok();
|
key.last_429_type = row.try_get("last_429_type").ok();
|
||||||
key.utilization_samples = row.try_get("utilization_samples").ok();
|
key.utilization_samples = row.try_get("utilization_samples").ok();
|
||||||
key.last_probe_increase_at_unix_secs = last_probe_increase_at_unix_secs;
|
key.last_probe_increase_at_unix_secs = last_probe_increase_at_unix_secs;
|
||||||
|
key.last_rpm_peak = last_rpm_peak;
|
||||||
key.last_used_at_unix_secs = last_used_at_unix_secs;
|
key.last_used_at_unix_secs = last_used_at_unix_secs;
|
||||||
key.auto_fetch_models = row.try_get("auto_fetch_models").unwrap_or(false);
|
key.auto_fetch_models = row.try_get("auto_fetch_models").unwrap_or(false);
|
||||||
key.last_models_fetch_at_unix_secs = last_models_fetch_at_unix_secs;
|
key.last_models_fetch_at_unix_secs = last_models_fetch_at_unix_secs;
|
||||||
|
|||||||
@@ -16,6 +16,10 @@ const COOLDOWN_HOURS_FOR_FULL_CONFIDENCE: f64 = 24.0;
|
|||||||
const LOW_LOAD_THRESHOLD: f64 = 0.5;
|
const LOW_LOAD_THRESHOLD: f64 = 0.5;
|
||||||
const HIGH_LOAD_THRESHOLD: f64 = 0.8;
|
const HIGH_LOAD_THRESHOLD: f64 = 0.8;
|
||||||
const ENFORCEMENT_CONFIDENCE_THRESHOLD: f64 = 0.6;
|
const ENFORCEMENT_CONFIDENCE_THRESHOLD: f64 = 0.6;
|
||||||
|
const CONFIDENCE_DECAY_PER_MINUTE: f64 = 0.005;
|
||||||
|
const MIN_CONSISTENT_OBSERVATIONS: usize = 3;
|
||||||
|
const MIN_HEADER_CONFIRMATIONS: usize = 2;
|
||||||
|
const OBSERVATION_CONSISTENCY_THRESHOLD: f64 = 0.3;
|
||||||
const HEALTH_DEGRADED_THRESHOLD: f64 = 0.8;
|
const HEALTH_DEGRADED_THRESHOLD: f64 = 0.8;
|
||||||
const HEALTH_LOW_THRESHOLD: f64 = 0.5;
|
const HEALTH_LOW_THRESHOLD: f64 = 0.5;
|
||||||
|
|
||||||
@@ -120,7 +124,9 @@ pub fn effective_provider_key_rpm_limit(
|
|||||||
.learned_rpm_limit
|
.learned_rpm_limit
|
||||||
.filter(|limit| *limit > 0)
|
.filter(|limit| *limit > 0)
|
||||||
.and_then(|limit| usize::try_from(limit).ok())?;
|
.and_then(|limit| usize::try_from(limit).ok())?;
|
||||||
if provider_key_reservation_confidence(key, now_unix_secs) < ENFORCEMENT_CONFIDENCE_THRESHOLD {
|
if provider_key_adaptive_learning_confidence(key, now_unix_secs)
|
||||||
|
< ENFORCEMENT_CONFIDENCE_THRESHOLD
|
||||||
|
{
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -309,6 +315,145 @@ fn provider_key_dynamic_reservation_ratio(
|
|||||||
+ confidence * (STABLE_MAX_RESERVATION_RATIO - STABLE_MIN_RESERVATION_RATIO)
|
+ confidence * (STABLE_MAX_RESERVATION_RATIO - STABLE_MIN_RESERVATION_RATIO)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn provider_key_adaptive_learning_confidence(
|
||||||
|
key: &StoredProviderCatalogKey,
|
||||||
|
now_unix_secs: u64,
|
||||||
|
) -> f64 {
|
||||||
|
if key.learned_rpm_limit.is_none() {
|
||||||
|
return 0.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
let base_confidence = provider_key_adaptive_base_confidence(key);
|
||||||
|
if base_confidence <= 0.0 {
|
||||||
|
return 0.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
let time_decay = match key.last_429_at_unix_secs {
|
||||||
|
Some(last_429_at_unix_secs) => {
|
||||||
|
now_unix_secs.saturating_sub(last_429_at_unix_secs) as f64 / 60.0
|
||||||
|
* CONFIDENCE_DECAY_PER_MINUTE
|
||||||
|
}
|
||||||
|
None => 1.0,
|
||||||
|
};
|
||||||
|
|
||||||
|
(base_confidence - time_decay).clamp(0.0, 1.0)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn provider_key_adaptive_base_confidence(key: &StoredProviderCatalogKey) -> f64 {
|
||||||
|
let history = provider_key_adjustment_history(key);
|
||||||
|
for record in history.iter().rev() {
|
||||||
|
if record.get("type").and_then(serde_json::Value::as_str) == Some("429_observation") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Some(confidence) = record.get("confidence").and_then(json_value_as_f64) {
|
||||||
|
return confidence.clamp(0.0, 1.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let (_, confidence) = evaluate_provider_key_observations(&history);
|
||||||
|
if confidence > 0.0 {
|
||||||
|
return confidence;
|
||||||
|
}
|
||||||
|
|
||||||
|
if key.learned_rpm_limit.is_some() {
|
||||||
|
return 0.3;
|
||||||
|
}
|
||||||
|
|
||||||
|
0.0
|
||||||
|
}
|
||||||
|
|
||||||
|
fn evaluate_provider_key_observations(
|
||||||
|
history: &[serde_json::Map<String, serde_json::Value>],
|
||||||
|
) -> (Option<u32>, f64) {
|
||||||
|
let observations = history
|
||||||
|
.iter()
|
||||||
|
.filter(|record| {
|
||||||
|
record.get("type").and_then(serde_json::Value::as_str) == Some("429_observation")
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if observations.is_empty() {
|
||||||
|
return (None, 0.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
let header_values = observations
|
||||||
|
.iter()
|
||||||
|
.filter_map(|record| provider_key_observation_u32(record, "upstream_limit"))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if header_values.len() >= MIN_HEADER_CONFIRMATIONS {
|
||||||
|
let recent = provider_key_recent_tail(&header_values, MIN_HEADER_CONFIRMATIONS * 2);
|
||||||
|
let last_n = provider_key_recent_tail(recent, MIN_HEADER_CONFIRMATIONS);
|
||||||
|
if provider_key_observations_consistent(last_n) {
|
||||||
|
return (None, 0.8);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let local_values = observations
|
||||||
|
.iter()
|
||||||
|
.filter_map(|record| provider_key_observation_u32(record, "current_rpm"))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if local_values.len() >= MIN_CONSISTENT_OBSERVATIONS {
|
||||||
|
let recent = provider_key_recent_tail(&local_values, MIN_CONSISTENT_OBSERVATIONS * 2);
|
||||||
|
let last_n = provider_key_recent_tail(recent, MIN_CONSISTENT_OBSERVATIONS);
|
||||||
|
if provider_key_observations_consistent(last_n) {
|
||||||
|
return (None, 0.6);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
(None, 0.0)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn provider_key_adjustment_history(
|
||||||
|
key: &StoredProviderCatalogKey,
|
||||||
|
) -> Vec<serde_json::Map<String, serde_json::Value>> {
|
||||||
|
key.adjustment_history
|
||||||
|
.as_ref()
|
||||||
|
.and_then(serde_json::Value::as_array)
|
||||||
|
.into_iter()
|
||||||
|
.flatten()
|
||||||
|
.filter_map(serde_json::Value::as_object)
|
||||||
|
.cloned()
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn provider_key_observation_u32(
|
||||||
|
record: &serde_json::Map<String, serde_json::Value>,
|
||||||
|
field: &str,
|
||||||
|
) -> Option<u32> {
|
||||||
|
record
|
||||||
|
.get(field)
|
||||||
|
.and_then(serde_json::Value::as_u64)
|
||||||
|
.and_then(|value| u32::try_from(value).ok())
|
||||||
|
.filter(|value| *value > 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn provider_key_observations_consistent(values: &[u32]) -> bool {
|
||||||
|
let median = provider_key_median(values);
|
||||||
|
median > 0.0
|
||||||
|
&& values.iter().all(|value| {
|
||||||
|
(*value as f64 - median).abs() / median <= OBSERVATION_CONSISTENCY_THRESHOLD
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn provider_key_median(values: &[u32]) -> f64 {
|
||||||
|
if values.is_empty() {
|
||||||
|
return 0.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut sorted = values.iter().map(|value| *value as f64).collect::<Vec<_>>();
|
||||||
|
sorted.sort_by(|left, right| left.partial_cmp(right).unwrap_or(std::cmp::Ordering::Equal));
|
||||||
|
let midpoint = sorted.len() / 2;
|
||||||
|
if sorted.len() % 2 == 0 {
|
||||||
|
(sorted[midpoint - 1] + sorted[midpoint]) / 2.0
|
||||||
|
} else {
|
||||||
|
sorted[midpoint]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn provider_key_recent_tail<T>(values: &[T], limit: usize) -> &[T] {
|
||||||
|
let keep_from = values.len().saturating_sub(limit);
|
||||||
|
&values[keep_from..]
|
||||||
|
}
|
||||||
|
|
||||||
fn is_recently_active(candidate: &StoredRequestCandidate, now_unix_secs: u64) -> bool {
|
fn is_recently_active(candidate: &StoredRequestCandidate, now_unix_secs: u64) -> bool {
|
||||||
if candidate.finished_at_unix_ms.is_some() {
|
if candidate.finished_at_unix_ms.is_some() {
|
||||||
return false;
|
return false;
|
||||||
@@ -667,26 +812,60 @@ mod tests {
|
|||||||
);
|
);
|
||||||
assert_eq!(effective_provider_key_rpm_limit(&low_confidence, 100), None);
|
assert_eq!(effective_provider_key_rpm_limit(&low_confidence, 100), None);
|
||||||
|
|
||||||
let high_confidence = provider_catalog_key("key-a").with_rate_limit_fields(
|
let mut high_confidence = provider_catalog_key("key-a").with_rate_limit_fields(
|
||||||
None,
|
None,
|
||||||
Some(80),
|
Some(80),
|
||||||
Some(0),
|
Some(0),
|
||||||
Some(0),
|
Some(0),
|
||||||
None,
|
Some(99),
|
||||||
Some(serde_json::json!([
|
Some(serde_json::json!([
|
||||||
{"new_limit": 80},
|
{
|
||||||
{"new_limit": 81},
|
"timestamp": "2026-04-19T00:00:00Z",
|
||||||
{"new_limit": 80},
|
"old_limit": 0,
|
||||||
|
"new_limit": 80,
|
||||||
|
"reason": "rpm_429",
|
||||||
|
"confidence": 0.8
|
||||||
|
},
|
||||||
])),
|
])),
|
||||||
Some(120),
|
Some(120),
|
||||||
Some(118),
|
Some(118),
|
||||||
);
|
);
|
||||||
|
high_confidence.last_429_type = Some("rpm".to_string());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
effective_provider_key_rpm_limit(&high_confidence, 100),
|
effective_provider_key_rpm_limit(&high_confidence, 100),
|
||||||
Some(80)
|
Some(80)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn learned_provider_key_rpm_limit_uses_confirmed_observations_as_fallback_confidence() {
|
||||||
|
let key = provider_catalog_key("key-a").with_rate_limit_fields(
|
||||||
|
None,
|
||||||
|
Some(80),
|
||||||
|
Some(0),
|
||||||
|
Some(2),
|
||||||
|
Some(99),
|
||||||
|
Some(serde_json::json!([
|
||||||
|
{
|
||||||
|
"type": "429_observation",
|
||||||
|
"timestamp": "2026-04-19T00:00:00Z",
|
||||||
|
"current_rpm": 90,
|
||||||
|
"upstream_limit": 84
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "429_observation",
|
||||||
|
"timestamp": "2026-04-19T00:01:00Z",
|
||||||
|
"current_rpm": 88,
|
||||||
|
"upstream_limit": 85
|
||||||
|
}
|
||||||
|
])),
|
||||||
|
Some(20),
|
||||||
|
Some(18),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(effective_provider_key_rpm_limit(&key, 100), Some(80));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn counts_recent_provider_key_rpm_from_snapshot_or_recent_attempts() {
|
fn counts_recent_provider_key_rpm_from_snapshot_or_recent_attempts() {
|
||||||
let recent_candidates = vec![
|
let recent_candidates = vec![
|
||||||
|
|||||||
@@ -498,6 +498,13 @@ import { log } from '@/utils/logger'
|
|||||||
import { parseApiError } from '@/utils/errorParser'
|
import { parseApiError } from '@/utils/errorParser'
|
||||||
import { formatApiFormat } from '@/api/endpoints/types/api-format'
|
import { formatApiFormat } from '@/api/endpoints/types/api-format'
|
||||||
import { resolveTimelineFinalStatus } from '../utils/status'
|
import { resolveTimelineFinalStatus } from '../utils/status'
|
||||||
|
import {
|
||||||
|
buildPoolAttemptCandidatesFromAudit,
|
||||||
|
extractPoolGroupId,
|
||||||
|
isPoolAttemptedCandidate,
|
||||||
|
makeAttemptKey,
|
||||||
|
TIMELINE_STATUS,
|
||||||
|
} from '../utils/poolTrace'
|
||||||
|
|
||||||
// 节点组类型
|
// 节点组类型
|
||||||
interface NodeGroup {
|
interface NodeGroup {
|
||||||
@@ -691,18 +698,6 @@ const proxyTimingBreakdown = (proxy: Record<string, unknown>): string => {
|
|||||||
return parts.join(' / ')
|
return parts.join(' / ')
|
||||||
}
|
}
|
||||||
|
|
||||||
const TIMELINE_STATUS: CandidateRecord['status'][] = [
|
|
||||||
'success',
|
|
||||||
'failed',
|
|
||||||
'skipped',
|
|
||||||
'cancelled',
|
|
||||||
'pending',
|
|
||||||
'streaming',
|
|
||||||
'available',
|
|
||||||
'unused',
|
|
||||||
'stream_interrupted',
|
|
||||||
]
|
|
||||||
|
|
||||||
const STATUS_PRIORITY: Record<string, number> = {
|
const STATUS_PRIORITY: Record<string, number> = {
|
||||||
available: 0,
|
available: 0,
|
||||||
unused: 0,
|
unused: 0,
|
||||||
@@ -715,47 +710,6 @@ const STATUS_PRIORITY: Record<string, number> = {
|
|||||||
success: 4,
|
success: 4,
|
||||||
}
|
}
|
||||||
|
|
||||||
const toInt = (value: unknown, defaultValue = 0): number => {
|
|
||||||
const num = Number(value)
|
|
||||||
return Number.isFinite(num) ? Math.trunc(num) : defaultValue
|
|
||||||
}
|
|
||||||
|
|
||||||
const makeAttemptKey = (candidateIndex: number, retryIndex: number): string => {
|
|
||||||
return `${candidateIndex}:${retryIndex}`
|
|
||||||
}
|
|
||||||
|
|
||||||
const POOL_UNATTEMPTED_STATUS = new Set<CandidateRecord['status']>([
|
|
||||||
'available',
|
|
||||||
'unused',
|
|
||||||
'skipped',
|
|
||||||
])
|
|
||||||
|
|
||||||
const isPoolAttemptedCandidate = (candidate: CandidateRecord): boolean => {
|
|
||||||
if (POOL_UNATTEMPTED_STATUS.has(candidate.status)) return false
|
|
||||||
// pending 只有开始执行后才算真正进入号池内部尝试
|
|
||||||
if (candidate.status === 'pending' && !candidate.started_at) return false
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
const normalizeTimelineStatus = (value: unknown): CandidateRecord['status'] => {
|
|
||||||
if (typeof value !== 'string') return 'failed'
|
|
||||||
const normalized = value.trim().toLowerCase()
|
|
||||||
if ((TIMELINE_STATUS as string[]).includes(normalized)) {
|
|
||||||
return normalized as CandidateRecord['status']
|
|
||||||
}
|
|
||||||
// 兜底:内部调度轨迹里非标准状态统一按失败展示
|
|
||||||
return 'failed'
|
|
||||||
}
|
|
||||||
|
|
||||||
const extractPoolGroupId = (candidate: CandidateRecord): string | null => {
|
|
||||||
const extra = candidate.extra_data
|
|
||||||
if (!extra || typeof extra !== 'object' || Array.isArray(extra)) return null
|
|
||||||
const value = (extra as Record<string, unknown>).pool_group_id
|
|
||||||
if (typeof value !== 'string') return null
|
|
||||||
const text = value.trim()
|
|
||||||
return text || null
|
|
||||||
}
|
|
||||||
|
|
||||||
// 候选时间线(按实际执行顺序排序)
|
// 候选时间线(按实际执行顺序排序)
|
||||||
const rawTimeline = computed<CandidateRecord[]>(() => {
|
const rawTimeline = computed<CandidateRecord[]>(() => {
|
||||||
if (!trace.value) return []
|
if (!trace.value) return []
|
||||||
@@ -794,97 +748,11 @@ const poolAttemptCandidates = computed<CandidateRecord[]>(() => {
|
|||||||
// 兼容旧链路:回退到 request_metadata.scheduling_audit.attempts。
|
// 兼容旧链路:回退到 request_metadata.scheduling_audit.attempts。
|
||||||
const audit = schedulingAudit.value
|
const audit = schedulingAudit.value
|
||||||
if (!audit) return []
|
if (!audit) return []
|
||||||
const attempts = audit.attempts
|
return buildPoolAttemptCandidatesFromAudit(
|
||||||
if (!Array.isArray(attempts) || attempts.length === 0) return []
|
rawTimeline.value,
|
||||||
|
audit.attempts,
|
||||||
const providerNameById = new Map<string, string>()
|
props.requestId,
|
||||||
for (const candidate of rawTimeline.value) {
|
)
|
||||||
const providerId = String(candidate.provider_id || '').trim()
|
|
||||||
const providerName = String(candidate.provider_name || '').trim()
|
|
||||||
if (!providerId || !providerName) continue
|
|
||||||
if (!providerNameById.has(providerId)) {
|
|
||||||
providerNameById.set(providerId, providerName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const providerTypeLikeNames = new Set<string>([
|
|
||||||
'codex',
|
|
||||||
'kiro',
|
|
||||||
'antigravity',
|
|
||||||
'claude_code',
|
|
||||||
'claude code',
|
|
||||||
'gemini_cli',
|
|
||||||
'gemini cli',
|
|
||||||
'oauth',
|
|
||||||
'api_key',
|
|
||||||
'api key',
|
|
||||||
])
|
|
||||||
|
|
||||||
const traceMap = new Map<string, CandidateRecord>()
|
|
||||||
for (const candidate of rawTimeline.value) {
|
|
||||||
traceMap.set(makeAttemptKey(candidate.candidate_index, candidate.retry_index), candidate)
|
|
||||||
}
|
|
||||||
|
|
||||||
return attempts
|
|
||||||
.map((item, index) => {
|
|
||||||
if (!item || typeof item !== 'object' || Array.isArray(item)) return null
|
|
||||||
const raw = item as Record<string, unknown>
|
|
||||||
const candidateIndex = toInt(raw.candidate_index, index)
|
|
||||||
const retryIndex = toInt(raw.retry_index, 0)
|
|
||||||
const key = makeAttemptKey(candidateIndex, retryIndex)
|
|
||||||
const fromTrace = traceMap.get(key)
|
|
||||||
|
|
||||||
const merged: CandidateRecord = fromTrace
|
|
||||||
? { ...fromTrace }
|
|
||||||
: {
|
|
||||||
id: `pool-${props.requestId}-${candidateIndex}-${retryIndex}-${index}`,
|
|
||||||
request_id: props.requestId,
|
|
||||||
candidate_index: candidateIndex,
|
|
||||||
retry_index: retryIndex,
|
|
||||||
provider_id: undefined,
|
|
||||||
provider_name: undefined,
|
|
||||||
endpoint_id: undefined,
|
|
||||||
key_id: undefined,
|
|
||||||
key_name: undefined,
|
|
||||||
status: 'failed',
|
|
||||||
is_cached: false,
|
|
||||||
created_at: new Date(0).toISOString(),
|
|
||||||
}
|
|
||||||
|
|
||||||
merged.status = normalizeTimelineStatus(raw.status ?? merged.status)
|
|
||||||
if (typeof raw.provider_id === 'string') merged.provider_id = raw.provider_id
|
|
||||||
if (typeof raw.provider_name === 'string') merged.provider_name = raw.provider_name
|
|
||||||
if (typeof raw.endpoint_id === 'string') merged.endpoint_id = raw.endpoint_id
|
|
||||||
if (typeof raw.key_id === 'string') merged.key_id = raw.key_id
|
|
||||||
if (typeof raw.key_name === 'string') merged.key_name = raw.key_name
|
|
||||||
if (typeof raw.status_code === 'number') merged.status_code = raw.status_code
|
|
||||||
if (typeof raw.error_type === 'string') merged.error_type = raw.error_type
|
|
||||||
const rawPoolGroupId = typeof raw.pool_group_id === 'string' ? raw.pool_group_id.trim() : ''
|
|
||||||
const fallbackPoolGroupId = typeof raw.provider_id === 'string' ? raw.provider_id.trim() : ''
|
|
||||||
const finalPoolGroupId = rawPoolGroupId || fallbackPoolGroupId
|
|
||||||
if (finalPoolGroupId) {
|
|
||||||
merged.extra_data = {
|
|
||||||
...(merged.extra_data || {}),
|
|
||||||
pool_group_id: finalPoolGroupId,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const mergedProviderId = String(merged.provider_id || '').trim()
|
|
||||||
if (mergedProviderId) {
|
|
||||||
const inferredProviderName = providerNameById.get(mergedProviderId)
|
|
||||||
const currentProviderName = String(merged.provider_name || '').trim()
|
|
||||||
if (
|
|
||||||
inferredProviderName
|
|
||||||
&& (
|
|
||||||
!currentProviderName
|
|
||||||
|| providerTypeLikeNames.has(currentProviderName.toLowerCase())
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
merged.provider_name = inferredProviderName
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return merged
|
|
||||||
})
|
|
||||||
.filter((item): item is CandidateRecord => item !== null)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
const poolAttemptsByGroup = computed<Map<string, CandidateRecord[]>>(() => {
|
const poolAttemptsByGroup = computed<Map<string, CandidateRecord[]>>(() => {
|
||||||
|
|||||||
114
frontend/src/features/usage/utils/__tests__/poolTrace.spec.ts
Normal file
114
frontend/src/features/usage/utils/__tests__/poolTrace.spec.ts
Normal file
@@ -0,0 +1,114 @@
|
|||||||
|
import { describe, expect, it } from 'vitest'
|
||||||
|
|
||||||
|
import type { CandidateRecord } from '@/api/requestTrace'
|
||||||
|
import { buildPoolAttemptCandidatesFromAudit } from '@/features/usage/utils/poolTrace'
|
||||||
|
|
||||||
|
function buildCandidate(
|
||||||
|
overrides: Partial<CandidateRecord> = {},
|
||||||
|
): CandidateRecord {
|
||||||
|
return {
|
||||||
|
id: 'cand-1',
|
||||||
|
request_id: 'req-1',
|
||||||
|
candidate_index: 0,
|
||||||
|
retry_index: 0,
|
||||||
|
status: 'failed',
|
||||||
|
is_cached: false,
|
||||||
|
created_at: '1970-01-01T00:00:00.000Z',
|
||||||
|
...overrides,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('poolTrace', () => {
|
||||||
|
it('keeps only actually attempted pool nodes from scheduling audit fallback', () => {
|
||||||
|
const attempts = buildPoolAttemptCandidatesFromAudit([], [
|
||||||
|
{
|
||||||
|
candidate_index: 0,
|
||||||
|
retry_index: 0,
|
||||||
|
provider_id: 'provider-1',
|
||||||
|
provider_name: 'Codex反代',
|
||||||
|
key_id: 'key-success',
|
||||||
|
key_name: 'Success Key',
|
||||||
|
status: 'success',
|
||||||
|
pool_group_id: 'provider-1',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
candidate_index: 1,
|
||||||
|
retry_index: 0,
|
||||||
|
provider_id: 'provider-1',
|
||||||
|
provider_name: 'Codex反代',
|
||||||
|
key_id: 'key-skipped',
|
||||||
|
key_name: 'Skipped Key',
|
||||||
|
status: 'skipped',
|
||||||
|
pool_group_id: 'provider-1',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
candidate_index: 2,
|
||||||
|
retry_index: 0,
|
||||||
|
provider_id: 'provider-1',
|
||||||
|
provider_name: 'Codex反代',
|
||||||
|
key_id: 'key-available',
|
||||||
|
key_name: 'Available Key',
|
||||||
|
status: 'available',
|
||||||
|
pool_group_id: 'provider-1',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
candidate_index: 3,
|
||||||
|
retry_index: 0,
|
||||||
|
provider_id: 'provider-1',
|
||||||
|
provider_name: 'Codex反代',
|
||||||
|
key_id: 'key-unknown',
|
||||||
|
key_name: 'Unknown Key',
|
||||||
|
status: 'selected',
|
||||||
|
pool_group_id: 'provider-1',
|
||||||
|
},
|
||||||
|
], 'req-1')
|
||||||
|
|
||||||
|
expect(attempts).toHaveLength(1)
|
||||||
|
expect(attempts[0].key_id).toBe('key-success')
|
||||||
|
expect(attempts[0].status).toBe('success')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('preserves real trace attempts even when audit status is non-standard', () => {
|
||||||
|
const rawTimeline = [
|
||||||
|
buildCandidate({
|
||||||
|
id: 'cand-trace-1',
|
||||||
|
candidate_index: 4,
|
||||||
|
retry_index: 0,
|
||||||
|
provider_id: 'provider-1',
|
||||||
|
provider_name: 'Codex反代',
|
||||||
|
key_id: 'key-trace',
|
||||||
|
key_name: 'Trace Key',
|
||||||
|
status: 'failed',
|
||||||
|
started_at: '2026-04-19T12:00:00.000Z',
|
||||||
|
}),
|
||||||
|
]
|
||||||
|
|
||||||
|
const attempts = buildPoolAttemptCandidatesFromAudit(rawTimeline, [
|
||||||
|
{
|
||||||
|
candidate_index: 4,
|
||||||
|
retry_index: 0,
|
||||||
|
provider_id: 'provider-1',
|
||||||
|
provider_name: 'oauth',
|
||||||
|
key_id: 'key-trace',
|
||||||
|
key_name: 'Trace Key',
|
||||||
|
status: 'selected',
|
||||||
|
pool_group_id: 'provider-1',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
candidate_index: 5,
|
||||||
|
retry_index: 0,
|
||||||
|
provider_id: 'provider-1',
|
||||||
|
provider_name: 'oauth',
|
||||||
|
key_id: 'key-ghost',
|
||||||
|
key_name: 'Ghost Key',
|
||||||
|
status: 'selected',
|
||||||
|
pool_group_id: 'provider-1',
|
||||||
|
},
|
||||||
|
], 'req-1')
|
||||||
|
|
||||||
|
expect(attempts).toHaveLength(1)
|
||||||
|
expect(attempts[0].id).toBe('cand-trace-1')
|
||||||
|
expect(attempts[0].status).toBe('failed')
|
||||||
|
expect(attempts[0].provider_name).toBe('Codex反代')
|
||||||
|
})
|
||||||
|
})
|
||||||
160
frontend/src/features/usage/utils/poolTrace.ts
Normal file
160
frontend/src/features/usage/utils/poolTrace.ts
Normal file
@@ -0,0 +1,160 @@
|
|||||||
|
import type { CandidateRecord } from '@/api/requestTrace'
|
||||||
|
|
||||||
|
export const TIMELINE_STATUS: CandidateRecord['status'][] = [
|
||||||
|
'success',
|
||||||
|
'failed',
|
||||||
|
'skipped',
|
||||||
|
'cancelled',
|
||||||
|
'pending',
|
||||||
|
'streaming',
|
||||||
|
'available',
|
||||||
|
'unused',
|
||||||
|
'stream_interrupted',
|
||||||
|
]
|
||||||
|
|
||||||
|
const POOL_UNATTEMPTED_STATUS = new Set<CandidateRecord['status']>([
|
||||||
|
'available',
|
||||||
|
'unused',
|
||||||
|
'skipped',
|
||||||
|
])
|
||||||
|
|
||||||
|
const PROVIDER_TYPE_LIKE_NAMES = new Set<string>([
|
||||||
|
'codex',
|
||||||
|
'kiro',
|
||||||
|
'antigravity',
|
||||||
|
'claude_code',
|
||||||
|
'claude code',
|
||||||
|
'gemini_cli',
|
||||||
|
'gemini cli',
|
||||||
|
'oauth',
|
||||||
|
'api_key',
|
||||||
|
'api key',
|
||||||
|
])
|
||||||
|
|
||||||
|
const toInt = (value: unknown, defaultValue = 0): number => {
|
||||||
|
const num = Number(value)
|
||||||
|
return Number.isFinite(num) ? Math.trunc(num) : defaultValue
|
||||||
|
}
|
||||||
|
|
||||||
|
export const makeAttemptKey = (candidateIndex: number, retryIndex: number): string => {
|
||||||
|
return `${candidateIndex}:${retryIndex}`
|
||||||
|
}
|
||||||
|
|
||||||
|
export const isPoolAttemptedCandidate = (candidate: CandidateRecord): boolean => {
|
||||||
|
if (POOL_UNATTEMPTED_STATUS.has(candidate.status)) return false
|
||||||
|
if (candidate.status === 'pending' && !candidate.started_at) return false
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
export const parseTimelineStatus = (value: unknown): CandidateRecord['status'] | null => {
|
||||||
|
if (typeof value !== 'string') return null
|
||||||
|
const normalized = value.trim().toLowerCase()
|
||||||
|
if ((TIMELINE_STATUS as string[]).includes(normalized)) {
|
||||||
|
return normalized as CandidateRecord['status']
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
export const extractPoolGroupId = (
|
||||||
|
candidate: Pick<CandidateRecord, 'extra_data'>,
|
||||||
|
): string | null => {
|
||||||
|
const extra = candidate.extra_data
|
||||||
|
if (!extra || typeof extra !== 'object' || Array.isArray(extra)) return null
|
||||||
|
const value = (extra as Record<string, unknown>).pool_group_id
|
||||||
|
if (typeof value !== 'string') return null
|
||||||
|
const text = value.trim()
|
||||||
|
return text || null
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildPoolAttemptCandidatesFromAudit(
|
||||||
|
rawTimeline: CandidateRecord[],
|
||||||
|
attempts: unknown,
|
||||||
|
requestId?: string | null,
|
||||||
|
): CandidateRecord[] {
|
||||||
|
if (!Array.isArray(attempts) || attempts.length === 0) return []
|
||||||
|
|
||||||
|
const providerNameById = new Map<string, string>()
|
||||||
|
for (const candidate of rawTimeline) {
|
||||||
|
const providerId = String(candidate.provider_id || '').trim()
|
||||||
|
const providerName = String(candidate.provider_name || '').trim()
|
||||||
|
if (!providerId || !providerName) continue
|
||||||
|
if (!providerNameById.has(providerId)) {
|
||||||
|
providerNameById.set(providerId, providerName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const traceMap = new Map<string, CandidateRecord>()
|
||||||
|
for (const candidate of rawTimeline) {
|
||||||
|
traceMap.set(makeAttemptKey(candidate.candidate_index, candidate.retry_index), candidate)
|
||||||
|
}
|
||||||
|
|
||||||
|
return attempts
|
||||||
|
.map((item, index) => {
|
||||||
|
if (!item || typeof item !== 'object' || Array.isArray(item)) return null
|
||||||
|
const raw = item as Record<string, unknown>
|
||||||
|
const candidateIndex = toInt(raw.candidate_index, index)
|
||||||
|
const retryIndex = toInt(raw.retry_index, 0)
|
||||||
|
const key = makeAttemptKey(candidateIndex, retryIndex)
|
||||||
|
const fromTrace = traceMap.get(key)
|
||||||
|
const parsedStatus = parseTimelineStatus(raw.status)
|
||||||
|
|
||||||
|
if (!fromTrace && parsedStatus === null) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
const merged: CandidateRecord = fromTrace
|
||||||
|
? { ...fromTrace }
|
||||||
|
: {
|
||||||
|
id: `pool-${requestId || 'unknown'}-${candidateIndex}-${retryIndex}-${index}`,
|
||||||
|
request_id: requestId || '',
|
||||||
|
candidate_index: candidateIndex,
|
||||||
|
retry_index: retryIndex,
|
||||||
|
provider_id: undefined,
|
||||||
|
provider_name: undefined,
|
||||||
|
endpoint_id: undefined,
|
||||||
|
key_id: undefined,
|
||||||
|
key_name: undefined,
|
||||||
|
status: 'failed',
|
||||||
|
is_cached: false,
|
||||||
|
created_at: new Date(0).toISOString(),
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parsedStatus !== null) {
|
||||||
|
merged.status = parsedStatus
|
||||||
|
}
|
||||||
|
if (typeof raw.provider_id === 'string') merged.provider_id = raw.provider_id
|
||||||
|
if (typeof raw.provider_name === 'string') merged.provider_name = raw.provider_name
|
||||||
|
if (typeof raw.endpoint_id === 'string') merged.endpoint_id = raw.endpoint_id
|
||||||
|
if (typeof raw.key_id === 'string') merged.key_id = raw.key_id
|
||||||
|
if (typeof raw.key_name === 'string') merged.key_name = raw.key_name
|
||||||
|
if (typeof raw.status_code === 'number') merged.status_code = raw.status_code
|
||||||
|
if (typeof raw.error_type === 'string') merged.error_type = raw.error_type
|
||||||
|
const rawPoolGroupId = typeof raw.pool_group_id === 'string' ? raw.pool_group_id.trim() : ''
|
||||||
|
const fallbackPoolGroupId = typeof raw.provider_id === 'string' ? raw.provider_id.trim() : ''
|
||||||
|
const finalPoolGroupId = rawPoolGroupId || fallbackPoolGroupId
|
||||||
|
if (finalPoolGroupId) {
|
||||||
|
merged.extra_data = {
|
||||||
|
...(merged.extra_data || {}),
|
||||||
|
pool_group_id: finalPoolGroupId,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const mergedProviderId = String(merged.provider_id || '').trim()
|
||||||
|
if (mergedProviderId) {
|
||||||
|
const inferredProviderName = providerNameById.get(mergedProviderId)
|
||||||
|
const currentProviderName = String(merged.provider_name || '').trim()
|
||||||
|
if (
|
||||||
|
inferredProviderName
|
||||||
|
&& (
|
||||||
|
!currentProviderName
|
||||||
|
|| PROVIDER_TYPE_LIKE_NAMES.has(currentProviderName.toLowerCase())
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
merged.provider_name = inferredProviderName
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return isPoolAttemptedCandidate(merged) ? merged : null
|
||||||
|
})
|
||||||
|
.filter((item): item is CandidateRecord => item !== null)
|
||||||
|
}
|
||||||
@@ -604,20 +604,6 @@
|
|||||||
>
|
>
|
||||||
<RefreshCw class="w-3.5 h-3.5" />
|
<RefreshCw class="w-3.5 h-3.5" />
|
||||||
</Button>
|
</Button>
|
||||||
<Button
|
|
||||||
v-if="key.circuit_breaker_open || (key.health_score ?? 1) < 0.5"
|
|
||||||
variant="ghost"
|
|
||||||
size="icon"
|
|
||||||
class="h-7 w-7 text-green-600"
|
|
||||||
:disabled="recoveringHealthKeyId === key.key_id"
|
|
||||||
title="刷新健康状态"
|
|
||||||
@click="handleRecoverKey(key)"
|
|
||||||
>
|
|
||||||
<RefreshCw
|
|
||||||
class="w-3.5 h-3.5"
|
|
||||||
:class="{ 'animate-spin': recoveringHealthKeyId === key.key_id }"
|
|
||||||
/>
|
|
||||||
</Button>
|
|
||||||
<Button
|
<Button
|
||||||
variant="ghost"
|
variant="ghost"
|
||||||
size="icon"
|
size="icon"
|
||||||
@@ -896,20 +882,6 @@
|
|||||||
>
|
>
|
||||||
<RefreshCw class="w-3.5 h-3.5" />
|
<RefreshCw class="w-3.5 h-3.5" />
|
||||||
</Button>
|
</Button>
|
||||||
<Button
|
|
||||||
v-else-if="actionId === 'recover_health'"
|
|
||||||
variant="ghost"
|
|
||||||
size="icon"
|
|
||||||
class="h-7 w-7 shrink-0 text-green-600"
|
|
||||||
:disabled="recoveringHealthKeyId === key.key_id"
|
|
||||||
title="刷新健康状态"
|
|
||||||
@click="handleRecoverKey(key)"
|
|
||||||
>
|
|
||||||
<RefreshCw
|
|
||||||
class="w-3.5 h-3.5"
|
|
||||||
:class="{ 'animate-spin': recoveringHealthKeyId === key.key_id }"
|
|
||||||
/>
|
|
||||||
</Button>
|
|
||||||
<Button
|
<Button
|
||||||
v-else-if="actionId === 'permissions'"
|
v-else-if="actionId === 'permissions'"
|
||||||
variant="ghost"
|
variant="ghost"
|
||||||
@@ -1167,7 +1139,6 @@ import {
|
|||||||
refreshProviderQuota,
|
refreshProviderQuota,
|
||||||
} from '@/api/endpoints/keys'
|
} from '@/api/endpoints/keys'
|
||||||
import { refreshProviderOAuth } from '@/api/endpoints/provider_oauth'
|
import { refreshProviderOAuth } from '@/api/endpoints/provider_oauth'
|
||||||
import { recoverKeyHealth } from '@/api/endpoints/health'
|
|
||||||
import type {
|
import type {
|
||||||
PoolOverviewItem,
|
PoolOverviewItem,
|
||||||
PoolKeyDetail,
|
PoolKeyDetail,
|
||||||
@@ -1565,7 +1536,6 @@ const currentPage = ref(restoredViewState.page)
|
|||||||
const pageSize = ref(restoredViewState.pageSize)
|
const pageSize = ref(restoredViewState.pageSize)
|
||||||
const MANUAL_QUOTA_REFRESH_COOLDOWN_SECONDS = 5 * 60
|
const MANUAL_QUOTA_REFRESH_COOLDOWN_SECONDS = 5 * 60
|
||||||
const refreshingOAuthKeyId = ref<string | null>(null)
|
const refreshingOAuthKeyId = ref<string | null>(null)
|
||||||
const recoveringHealthKeyId = ref<string | null>(null)
|
|
||||||
const savingProxyKeyId = ref<string | null>(null)
|
const savingProxyKeyId = ref<string | null>(null)
|
||||||
const proxyDesktopPopoverOpenKeyId = ref<string | null>(null)
|
const proxyDesktopPopoverOpenKeyId = ref<string | null>(null)
|
||||||
const proxyMobilePopoverOpenKeyId = ref<string | null>(null)
|
const proxyMobilePopoverOpenKeyId = ref<string | null>(null)
|
||||||
@@ -2075,20 +2045,6 @@ async function clearKeyProxy(key: PoolKeyDetail) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function handleRecoverKey(key: PoolKeyDetail) {
|
|
||||||
if (recoveringHealthKeyId.value) return
|
|
||||||
recoveringHealthKeyId.value = key.key_id
|
|
||||||
try {
|
|
||||||
const result = await recoverKeyHealth(key.key_id)
|
|
||||||
success(result.message || 'Key 已恢复')
|
|
||||||
await loadKeys()
|
|
||||||
} catch (err) {
|
|
||||||
showError(parseApiError(err, 'Key恢复失败'))
|
|
||||||
} finally {
|
|
||||||
recoveringHealthKeyId.value = null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function handleDeleteKey(key: PoolKeyDetail) {
|
async function handleDeleteKey(key: PoolKeyDetail) {
|
||||||
const confirmed = await confirm({
|
const confirmed = await confirm({
|
||||||
title: '删除账号',
|
title: '删除账号',
|
||||||
@@ -2399,19 +2355,54 @@ function formatCooldownReason(reason: string): string {
|
|||||||
|
|
||||||
type PoolStatusVariant = 'default' | 'secondary' | 'destructive' | 'outline' | 'success' | 'warning' | 'dark'
|
type PoolStatusVariant = 'default' | 'secondary' | 'destructive' | 'outline' | 'success' | 'warning' | 'dark'
|
||||||
|
|
||||||
|
function isHealthDerivedSchedulingReason(reason: string | null | undefined): boolean {
|
||||||
|
const normalized = String(reason || '').trim().toLowerCase()
|
||||||
|
return normalized === 'health_low'
|
||||||
|
|| normalized === 'health_degraded'
|
||||||
|
|| normalized === 'health'
|
||||||
|
|| normalized === 'circuit_open'
|
||||||
|
|| normalized === 'circuit_breaker'
|
||||||
|
}
|
||||||
|
|
||||||
|
function isHealthDerivedSchedulingLabel(label: string | null | undefined): boolean {
|
||||||
|
const normalized = String(label || '').trim()
|
||||||
|
return normalized === '健康低'
|
||||||
|
|| normalized === '健康度较低'
|
||||||
|
|| normalized === '降级'
|
||||||
|
|| normalized === '熔断'
|
||||||
|
|| normalized === '熔断中'
|
||||||
|
}
|
||||||
|
|
||||||
|
function getVisibleSchedulingReason(key: PoolKeyDetail): string | null {
|
||||||
|
const reason = String(key.scheduling_reason || '').trim()
|
||||||
|
if (!reason || isHealthDerivedSchedulingReason(reason)) return null
|
||||||
|
return reason
|
||||||
|
}
|
||||||
|
|
||||||
|
function getVisibleSchedulingReasons(key: PoolKeyDetail) {
|
||||||
|
return (key.scheduling_reasons ?? []).filter((item) => {
|
||||||
|
const source = String(item.source || '').trim().toLowerCase()
|
||||||
|
return source !== 'health'
|
||||||
|
&& !isHealthDerivedSchedulingReason(item.code)
|
||||||
|
&& !isHealthDerivedSchedulingLabel(item.label)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
function getSchedulingStatus(key: PoolKeyDetail): 'available' | 'degraded' | 'blocked' {
|
function getSchedulingStatus(key: PoolKeyDetail): 'available' | 'degraded' | 'blocked' {
|
||||||
if (getAccountAlertLabel(key)) return 'blocked'
|
if (getAccountAlertLabel(key)) return 'blocked'
|
||||||
|
|
||||||
const status = key.scheduling_status
|
const status = key.scheduling_status
|
||||||
if (status === 'available' || status === 'degraded' || status === 'blocked') {
|
if (
|
||||||
|
(status === 'available' || status === 'degraded' || status === 'blocked')
|
||||||
|
&& !isHealthDerivedSchedulingReason(key.scheduling_reason)
|
||||||
|
&& !isHealthDerivedSchedulingLabel(key.scheduling_label)
|
||||||
|
) {
|
||||||
return status
|
return status
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!key.is_active) return 'blocked'
|
if (!key.is_active) return 'blocked'
|
||||||
if (key.cooldown_reason) return 'blocked'
|
if (key.cooldown_reason) return 'degraded'
|
||||||
if (key.circuit_breaker_open) return 'blocked'
|
|
||||||
if (key.cost_limit != null && key.cost_limit > 0 && key.cost_window_usage >= key.cost_limit) return 'blocked'
|
if (key.cost_limit != null && key.cost_limit > 0 && key.cost_window_usage >= key.cost_limit) return 'blocked'
|
||||||
if ((key.health_score ?? 1) < 0.8) return 'degraded'
|
|
||||||
return 'available'
|
return 'available'
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2420,29 +2411,31 @@ function getSchedulingBadgeLabel(key: PoolKeyDetail): string {
|
|||||||
if (accountAlert) return accountAlert
|
if (accountAlert) return accountAlert
|
||||||
|
|
||||||
const rawLabel = String(key.scheduling_label || '').trim()
|
const rawLabel = String(key.scheduling_label || '').trim()
|
||||||
if (rawLabel) {
|
if (
|
||||||
|
rawLabel
|
||||||
|
&& !isHealthDerivedSchedulingReason(key.scheduling_reason)
|
||||||
|
&& !isHealthDerivedSchedulingLabel(rawLabel)
|
||||||
|
) {
|
||||||
if (rawLabel === '禁用' || rawLabel === '停用') return '禁用'
|
if (rawLabel === '禁用' || rawLabel === '停用') return '禁用'
|
||||||
return rawLabel
|
return rawLabel
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!key.is_active) return '禁用'
|
if (!key.is_active) return '禁用'
|
||||||
if (key.cooldown_reason) return '冷却'
|
if (key.cooldown_reason) return '冷却中'
|
||||||
if (key.circuit_breaker_open) return '熔断'
|
|
||||||
if (key.cost_limit != null && key.cost_limit > 0 && key.cost_window_usage >= key.cost_limit) return '超限'
|
if (key.cost_limit != null && key.cost_limit > 0 && key.cost_window_usage >= key.cost_limit) return '超限'
|
||||||
if ((key.health_score ?? 1) < 0.5) return '健康低'
|
|
||||||
if ((key.health_score ?? 1) < 0.8) return '降级'
|
|
||||||
return '可用'
|
return '可用'
|
||||||
}
|
}
|
||||||
|
|
||||||
function getSchedulingBadgeVariant(key: PoolKeyDetail): PoolStatusVariant {
|
function getSchedulingBadgeVariant(key: PoolKeyDetail): PoolStatusVariant {
|
||||||
if (getAccountAlertLabel(key)) return 'destructive'
|
if (getAccountAlertLabel(key)) return 'destructive'
|
||||||
|
|
||||||
const reason = key.scheduling_reason
|
const reason = getVisibleSchedulingReason(key)
|
||||||
if (reason === 'manual_disabled') return 'secondary'
|
if (reason === 'manual_disabled' || reason === 'inactive') return 'secondary'
|
||||||
if (reason === 'cooldown' || reason === 'circuit_open' || reason === 'cost_exhausted') return 'destructive'
|
if (reason === 'account_blocked' || reason === 'account_quota_exhausted' || reason === 'cost_exhausted') return 'destructive'
|
||||||
|
if (reason === 'cooldown') return 'warning'
|
||||||
if (reason === 'cost_soft' || reason === 'cost') return 'warning'
|
if (reason === 'cost_soft' || reason === 'cost') return 'warning'
|
||||||
if (reason === 'health_low' || reason === 'health_degraded' || reason === 'health') return 'warning'
|
|
||||||
if (reason === 'available') return 'default'
|
if (reason === 'available') return 'default'
|
||||||
|
if (!reason && !key.is_active) return 'secondary'
|
||||||
|
|
||||||
const status = getSchedulingStatus(key)
|
const status = getSchedulingStatus(key)
|
||||||
if (status === 'blocked') return 'destructive'
|
if (status === 'blocked') return 'destructive'
|
||||||
@@ -2454,7 +2447,7 @@ function getSchedulingTitle(key: PoolKeyDetail): string {
|
|||||||
const accountAlertTitle = getAccountAlertTitle(key)
|
const accountAlertTitle = getAccountAlertTitle(key)
|
||||||
if (accountAlertTitle) return accountAlertTitle
|
if (accountAlertTitle) return accountAlertTitle
|
||||||
|
|
||||||
const reasons = key.scheduling_reasons ?? []
|
const reasons = getVisibleSchedulingReasons(key)
|
||||||
if (reasons.length > 0) {
|
if (reasons.length > 0) {
|
||||||
return reasons.map((item) => {
|
return reasons.map((item) => {
|
||||||
const ttl = item.ttl_seconds && item.ttl_seconds > 0 ? ` (${formatTTL(item.ttl_seconds)})` : ''
|
const ttl = item.ttl_seconds && item.ttl_seconds > 0 ? ` (${formatTTL(item.ttl_seconds)})` : ''
|
||||||
@@ -2518,7 +2511,6 @@ function getMobileActionIds(key: PoolKeyDetail): PoolMobileActionId[] {
|
|||||||
canDownloadOrCopy: true,
|
canDownloadOrCopy: true,
|
||||||
canRefreshToken: canRefreshOAuthCredential(key),
|
canRefreshToken: canRefreshOAuthCredential(key),
|
||||||
canClearCooldown: Boolean(key.cooldown_reason),
|
canClearCooldown: Boolean(key.cooldown_reason),
|
||||||
canRecoverHealth: key.circuit_breaker_open || (key.health_score ?? 1) < 0.5,
|
|
||||||
hasProxy: true,
|
hasProxy: true,
|
||||||
}).primary
|
}).primary
|
||||||
}
|
}
|
||||||
@@ -2669,7 +2661,7 @@ function getQuotaProgressCountdown(item: QuotaProgressItem) {
|
|||||||
function getQuotaProgressCountdownText(item: QuotaProgressItem): string {
|
function getQuotaProgressCountdownText(item: QuotaProgressItem): string {
|
||||||
const status = getQuotaProgressCountdown(item)
|
const status = getQuotaProgressCountdown(item)
|
||||||
if (!status) return ''
|
if (!status) return ''
|
||||||
return status.isExpired ? status.text : `${status.text} 后重置`
|
return status.isExpired ? '' : `${status.text} 后重置`
|
||||||
}
|
}
|
||||||
|
|
||||||
function formatCompactQuotaCountdownText(text: string): string {
|
function formatCompactQuotaCountdownText(text: string): string {
|
||||||
@@ -2681,10 +2673,15 @@ function formatCompactQuotaCountdownText(text: string): string {
|
|||||||
return normalized.replace(/\s+后重置$/, '')
|
return normalized.replace(/\s+后重置$/, '')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function shouldHideQuotaProgressDetailText(text: string | null | undefined): boolean {
|
||||||
|
return (text ?? '').trim().includes('已重置')
|
||||||
|
}
|
||||||
|
|
||||||
function getQuotaProgressDisplayText(item: QuotaProgressItem): string {
|
function getQuotaProgressDisplayText(item: QuotaProgressItem): string {
|
||||||
const countdownText = getQuotaProgressCountdownText(item)
|
const countdownText = getQuotaProgressCountdownText(item)
|
||||||
if (countdownText) return formatCompactQuotaCountdownText(countdownText)
|
if (countdownText) return formatCompactQuotaCountdownText(countdownText)
|
||||||
return item.detail?.trim() || ''
|
const detail = item.detail?.trim() || ''
|
||||||
|
return shouldHideQuotaProgressDetailText(detail) ? '' : detail
|
||||||
}
|
}
|
||||||
|
|
||||||
function getQuotaFallbackText(key: PoolKeyDetail): string | null {
|
function getQuotaFallbackText(key: PoolKeyDetail): string | null {
|
||||||
|
|||||||
Reference in New Issue
Block a user