Unify pool score display as account health

This commit is contained in:
fawney19
2026-05-12 15:27:21 +08:00
parent fa73655134
commit 9c05b5f4e0
14 changed files with 401 additions and 518 deletions

View File

@@ -49,7 +49,7 @@ pub(crate) use self::planner::{
build_standard_sync_plan_from_decision, candidate_auth_channel_skip_reason, build_standard_sync_plan_from_decision, candidate_auth_channel_skip_reason,
extract_pool_sticky_session_token, maybe_build_stream_decision_payload, extract_pool_sticky_session_token, maybe_build_stream_decision_payload,
maybe_build_stream_plan_payload, maybe_build_sync_decision_payload, maybe_build_stream_plan_payload, maybe_build_sync_decision_payload,
maybe_build_sync_plan_payload, planner_is_matching_stream_request, maybe_build_sync_plan_payload, planner_is_matching_stream_request, provider_key_pool_score_id,
provider_key_pool_score_scope, read_candidate_transport_snapshot, provider_key_pool_score_scope, read_candidate_transport_snapshot,
record_local_runtime_candidate_skip_reason, record_local_runtime_candidate_skip_reason,
set_local_openai_chat_execution_exhausted_diagnostic, set_local_openai_chat_execution_exhausted_diagnostic,

View File

@@ -41,8 +41,9 @@ pub(crate) use self::plan_builders::{
build_standard_stream_plan_from_decision, build_standard_sync_plan_from_decision, build_standard_stream_plan_from_decision, build_standard_sync_plan_from_decision,
AiStreamAttempt, AiSyncAttempt, AiStreamAttempt, AiSyncAttempt,
}; };
pub(crate) use self::pool_scores::build_provider_key_pool_score_upsert; pub(crate) use self::pool_scores::{
pub(crate) use self::pool_scores::provider_key_pool_score_scope; build_provider_key_pool_score_upsert, provider_key_pool_score_id, provider_key_pool_score_scope,
};
pub(crate) use self::route::is_matching_stream_request as planner_is_matching_stream_request; pub(crate) use self::route::is_matching_stream_request as planner_is_matching_stream_request;
pub(crate) use self::runtime_miss::{ pub(crate) use self::runtime_miss::{
apply_local_runtime_candidate_terminal_reason, record_local_runtime_candidate_skip_reason, apply_local_runtime_candidate_terminal_reason, record_local_runtime_candidate_skip_reason,

View File

@@ -3,7 +3,7 @@ use aether_ai_serving::{
}; };
use aether_data_contracts::repository::pool_scores::{ use aether_data_contracts::repository::pool_scores::{
PoolMemberIdentity, PoolMemberProbeStatus, PoolScoreScope, UpsertPoolMemberScore, PoolMemberIdentity, PoolMemberProbeStatus, PoolScoreScope, UpsertPoolMemberScore,
POOL_SCORE_SCOPE_KIND_MODEL, POOL_SCORE_CAPABILITY_ACCOUNT, POOL_SCORE_SCOPE_KIND_ACCOUNT,
}; };
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey; use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
use serde_json::Value; use serde_json::Value;
@@ -13,14 +13,12 @@ use crate::handlers::shared::{provider_key_health_summary, provider_key_status_s
pub(crate) fn build_provider_key_pool_score_upsert( pub(crate) fn build_provider_key_pool_score_upsert(
key: &StoredProviderCatalogKey, key: &StoredProviderCatalogKey,
provider_type: &str, provider_type: &str,
api_format: &str,
model_id: Option<&str>,
existing: Option<&aether_data_contracts::repository::pool_scores::StoredPoolMemberScore>, existing: Option<&aether_data_contracts::repository::pool_scores::StoredPoolMemberScore>,
now_unix_secs: u64, now_unix_secs: u64,
score_rules: PoolMemberScoreRules, score_rules: PoolMemberScoreRules,
) -> UpsertPoolMemberScore { ) -> UpsertPoolMemberScore {
let identity = PoolMemberIdentity::provider_api_key(key.provider_id.clone(), key.id.clone()); let identity = PoolMemberIdentity::provider_api_key(key.provider_id.clone(), key.id.clone());
let scope = provider_key_pool_score_scope(api_format, model_id); let scope = provider_key_pool_score_scope();
let input = provider_key_score_input( let input = provider_key_score_input(
key, key,
provider_type, provider_type,
@@ -54,17 +52,11 @@ pub(crate) fn build_provider_key_pool_score_upsert(
} }
} }
pub(crate) fn provider_key_pool_score_scope( pub(crate) fn provider_key_pool_score_scope() -> PoolScoreScope {
api_format: &str,
model_id: Option<&str>,
) -> PoolScoreScope {
PoolScoreScope { PoolScoreScope {
capability: api_format.trim().to_ascii_lowercase(), capability: POOL_SCORE_CAPABILITY_ACCOUNT.to_string(),
scope_kind: POOL_SCORE_SCOPE_KIND_MODEL.to_string(), scope_kind: POOL_SCORE_SCOPE_KIND_ACCOUNT.to_string(),
scope_id: model_id scope_id: None,
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned),
} }
} }
@@ -133,8 +125,7 @@ fn provider_key_score_input(
.and_then(Value::as_bool) .and_then(Value::as_bool)
.unwrap_or(false), .unwrap_or(false),
oauth_invalid_reason: key.oauth_invalid_reason.clone(), oauth_invalid_reason: key.oauth_invalid_reason.clone(),
circuit_open: any_circuit_open circuit_open: any_circuit_open,
|| circuit_open_for_scope(key.circuit_breaker_by_format.as_ref(), &scope),
success_count: key.success_count.unwrap_or(0).into(), success_count: key.success_count.unwrap_or(0).into(),
error_count: key.error_count.unwrap_or(0).into(), error_count: key.error_count.unwrap_or(0).into(),
total_response_time_ms: key.total_response_time_ms.unwrap_or(0).into(), total_response_time_ms: key.total_response_time_ms.unwrap_or(0).into(),
@@ -150,21 +141,6 @@ fn provider_key_score_input(
} }
} }
fn circuit_open_for_scope(circuit_by_format: Option<&Value>, scope: &PoolScoreScope) -> bool {
let Some(formats) = circuit_by_format.and_then(Value::as_object) else {
return false;
};
let keys = api_format_lookup_keys(&scope.capability);
keys.iter().any(|key| {
formats
.get(key)
.and_then(Value::as_object)
.and_then(|value| value.get("open"))
.and_then(Value::as_bool)
.unwrap_or(false)
})
}
fn json_f64(value: &Value) -> Option<f64> { fn json_f64(value: &Value) -> Option<f64> {
value.as_f64().or_else(|| { value.as_f64().or_else(|| {
value value
@@ -175,17 +151,6 @@ fn json_f64(value: &Value) -> Option<f64> {
}) })
} }
fn api_format_lookup_keys(api_format: &str) -> Vec<String> {
let normalized = crate::ai_serving::normalize_api_format_alias(api_format);
let mut keys = crate::ai_serving::api_format_storage_aliases(&normalized);
if !keys.iter().any(|value| value == &normalized) {
keys.push(normalized);
}
keys.sort();
keys.dedup();
keys
}
fn stable_hash(bytes: &[u8]) -> u64 { fn stable_hash(bytes: &[u8]) -> u64 {
let mut hash = 0xcbf29ce484222325u64; let mut hash = 0xcbf29ce484222325u64;
for byte in bytes { for byte in bytes {

View File

@@ -387,10 +387,7 @@ impl<'a> PoolKeyCursor<'a> {
} }
async fn next_score_candidates(&mut self) -> Option<Vec<EligibleLocalExecutionCandidate>> { async fn next_score_candidates(&mut self) -> Option<Vec<EligibleLocalExecutionCandidate>> {
let scope = provider_key_pool_score_scope( let scope = provider_key_pool_score_scope();
self.group.candidate.endpoint_api_format.as_str(),
Some(self.group.candidate.model_id.as_str()),
);
let query = ListRankedPoolMembersQuery { let query = ListRankedPoolMembersQuery {
pool_kind: POOL_KIND_PROVIDER_KEY_POOL.to_string(), pool_kind: POOL_KIND_PROVIDER_KEY_POOL.to_string(),
pool_id: self.group.candidate.provider_id.clone(), pool_id: self.group.candidate.provider_id.clone(),

View File

@@ -6,6 +6,7 @@ use crate::handlers::admin::shared::{provider_key_status_snapshot_payload, unix_
use crate::provider_key_auth::{provider_key_auth_semantics, provider_key_effective_api_formats}; use crate::provider_key_auth::{provider_key_auth_semantics, provider_key_effective_api_formats};
use aether_admin::provider::pool as admin_provider_pool_pure; use aether_admin::provider::pool as admin_provider_pool_pure;
use aether_admin::provider::quota as admin_provider_quota_pure; use aether_admin::provider::quota as admin_provider_quota_pure;
use aether_data_contracts::repository::pool_scores::StoredPoolMemberScore;
use aether_data_contracts::repository::provider_catalog::{ use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
}; };
@@ -906,6 +907,7 @@ pub(super) fn build_admin_pool_key_payload(
key: &StoredProviderCatalogKey, key: &StoredProviderCatalogKey,
runtime: &AdminProviderPoolRuntimeState, runtime: &AdminProviderPoolRuntimeState,
pool_config: Option<AdminProviderPoolConfig>, pool_config: Option<AdminProviderPoolConfig>,
pool_score: Option<&StoredPoolMemberScore>,
codex_cycle_usage_by_code: Option<&BTreeMap<String, StoredProviderApiKeyWindowUsageSummary>>, codex_cycle_usage_by_code: Option<&BTreeMap<String, StoredProviderApiKeyWindowUsageSummary>>,
now_unix_secs: u64, now_unix_secs: u64,
) -> serde_json::Value { ) -> serde_json::Value {
@@ -1087,6 +1089,34 @@ pub(super) fn build_admin_pool_key_payload(
payload.insert("status_snapshot".to_string(), status_snapshot); payload.insert("status_snapshot".to_string(), status_snapshot);
payload.insert("quota_updated_at".to_string(), json!(quota_updated_at)); payload.insert("quota_updated_at".to_string(), json!(quota_updated_at));
payload.insert("health_score".to_string(), json!(health_score)); payload.insert("health_score".to_string(), json!(health_score));
payload.insert(
"pool_score".to_string(),
pool_score
.map(|score| {
json!({
"id": score.id.clone(),
"capability": score.capability.clone(),
"scope_kind": score.scope_kind.clone(),
"scope_id": score.scope_id.clone(),
"score": score.score,
"hard_state": score.hard_state.as_database(),
"score_version": score.score_version,
"score_reason": score.score_reason.clone(),
"last_ranked_at": score.last_ranked_at,
"last_scheduled_at": score.last_scheduled_at,
"last_success_at": score.last_success_at,
"last_failure_at": score.last_failure_at,
"failure_count": score.failure_count,
"last_probe_attempt_at": score.last_probe_attempt_at,
"last_probe_success_at": score.last_probe_success_at,
"last_probe_failure_at": score.last_probe_failure_at,
"probe_failure_count": score.probe_failure_count,
"probe_status": score.probe_status.as_database(),
"updated_at": score.updated_at,
})
})
.unwrap_or(serde_json::Value::Null),
);
payload.insert( payload.insert(
"circuit_breaker_open".to_string(), "circuit_breaker_open".to_string(),
json!(circuit_breaker_open), json!(circuit_breaker_open),

View File

@@ -7,9 +7,13 @@ use super::{
AdminPoolKeySortField, AdminProviderPoolRuntimeState, ProviderCatalogKeyListOrder, AdminPoolKeySortField, AdminProviderPoolRuntimeState, ProviderCatalogKeyListOrder,
ProviderCatalogKeyListQuery, ADMIN_POOL_PROVIDER_CATALOG_READER_UNAVAILABLE_DETAIL, ProviderCatalogKeyListQuery, ADMIN_POOL_PROVIDER_CATALOG_READER_UNAVAILABLE_DETAIL,
}; };
use crate::ai_serving::{provider_key_pool_score_id, provider_key_pool_score_scope};
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext}; use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError; use crate::GatewayError;
use aether_admin::provider::pool as admin_provider_pool_pure; use aether_admin::provider::pool as admin_provider_pool_pure;
use aether_data_contracts::repository::pool_scores::{
GetPoolMemberScoresByIdsQuery, PoolMemberIdentity, StoredPoolMemberScore,
};
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey; use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
use aether_data_contracts::repository::usage::{ use aether_data_contracts::repository::usage::{
ProviderApiKeyWindowUsageRequest, StoredProviderApiKeyWindowUsageSummary, ProviderApiKeyWindowUsageRequest, StoredProviderApiKeyWindowUsageSummary,
@@ -56,6 +60,36 @@ fn admin_pool_current_unix_secs() -> u64 {
.unwrap_or(0) .unwrap_or(0)
} }
async fn read_admin_pool_scores_by_key_id(
state: &AdminAppState<'_>,
provider_id: &str,
key_ids: &[String],
) -> Result<BTreeMap<String, StoredPoolMemberScore>, GatewayError> {
if key_ids.is_empty() {
return Ok(BTreeMap::new());
}
let score_scope = provider_key_pool_score_scope();
let score_ids = key_ids
.iter()
.map(|key_id| {
let identity =
PoolMemberIdentity::provider_api_key(provider_id.to_string(), key_id.clone());
provider_key_pool_score_id(&identity, &score_scope)
})
.collect::<Vec<_>>();
let scores = state
.app()
.data
.get_pool_member_scores_by_ids(&GetPoolMemberScoresByIdsQuery { ids: score_ids })
.await
.map_err(|err| GatewayError::Internal(format!("{err:?}")))?;
Ok(scores
.into_iter()
.map(|score| (score.member_id.clone(), score))
.collect::<BTreeMap<_, _>>())
}
fn admin_pool_codex_cycle_usage_request( fn admin_pool_codex_cycle_usage_request(
key: &StoredProviderCatalogKey, key: &StoredProviderCatalogKey,
window: &serde_json::Map<String, serde_json::Value>, window: &serde_json::Map<String, serde_json::Value>,
@@ -379,6 +413,9 @@ pub(super) async fn build_admin_pool_list_keys_response(
}; };
let key_ids = keys.iter().map(|key| key.id.clone()).collect::<Vec<_>>(); let key_ids = keys.iter().map(|key| key.id.clone()).collect::<Vec<_>>();
let pool_scores_by_key_id = read_admin_pool_scores_by_key_id(state, &provider.id, &key_ids)
.await
.unwrap_or_default();
let endpoints = state let endpoints = state
.list_provider_catalog_endpoints_by_provider_ids(std::slice::from_ref(&provider.id)) .list_provider_catalog_endpoints_by_provider_ids(std::slice::from_ref(&provider.id))
.await?; .await?;
@@ -414,6 +451,7 @@ pub(super) async fn build_admin_pool_list_keys_response(
&key, &key,
&runtime, &runtime,
pool_config.clone(), pool_config.clone(),
pool_scores_by_key_id.get(&key.id),
codex_cycle_usage_by_key.get(&key.id), codex_cycle_usage_by_key.get(&key.id),
now_unix_secs, now_unix_secs,
) )

View File

@@ -7,7 +7,7 @@ use crate::handlers::shared::query_param_value;
use crate::GatewayError; use crate::GatewayError;
use aether_data_contracts::repository::pool_scores::{ use aether_data_contracts::repository::pool_scores::{
ListPoolMemberScoresQuery, PoolMemberHardState, PoolMemberProbeStatus, ListPoolMemberScoresQuery, PoolMemberHardState, PoolMemberProbeStatus,
POOL_KIND_PROVIDER_KEY_POOL, POOL_SCORE_SCOPE_KIND_MODEL, POOL_KIND_PROVIDER_KEY_POOL, POOL_SCORE_CAPABILITY_ACCOUNT, POOL_SCORE_SCOPE_KIND_ACCOUNT,
}; };
use axum::{ use axum::{
body::Body, body::Body,
@@ -49,9 +49,6 @@ pub(super) async fn build_admin_pool_scores_response(
} }
}; };
let offset = page.saturating_sub(1).saturating_mul(page_size); let offset = page.saturating_sub(1).saturating_mul(page_size);
let api_format = query_param_value(query, "api_format")
.map(|value| crate::ai_serving::normalize_api_format_alias(value.as_str()));
let model_id = query_param_value(query, "model_id");
let hard_states = match parse_hard_state_filter(query) { let hard_states = match parse_hard_state_filter(query) {
Ok(value) => value, Ok(value) => value,
Err(message) => { Err(message) => {
@@ -77,9 +74,9 @@ pub(super) async fn build_admin_pool_scores_response(
.list_pool_member_scores(&ListPoolMemberScoresQuery { .list_pool_member_scores(&ListPoolMemberScoresQuery {
pool_kind: POOL_KIND_PROVIDER_KEY_POOL.to_string(), pool_kind: POOL_KIND_PROVIDER_KEY_POOL.to_string(),
pool_id: provider_id.clone(), pool_id: provider_id.clone(),
capability: api_format.clone(), capability: Some(POOL_SCORE_CAPABILITY_ACCOUNT.to_string()),
scope_kind: Some(POOL_SCORE_SCOPE_KIND_MODEL.to_string()), scope_kind: Some(POOL_SCORE_SCOPE_KIND_ACCOUNT.to_string()),
scope_id: model_id.clone(), scope_id: None,
hard_states, hard_states,
probe_statuses, probe_statuses,
offset, offset,
@@ -146,8 +143,8 @@ pub(super) async fn build_admin_pool_scores_response(
"page": page, "page": page,
"page_size": page_size, "page_size": page_size,
"filters": { "filters": {
"api_format": api_format, "api_format": serde_json::Value::Null,
"model_id": model_id, "model_id": serde_json::Value::Null,
"hard_state": query_param_value(query, "hard_state"), "hard_state": query_param_value(query, "hard_state"),
"probe_status": query_param_value(query, "probe_status") "probe_status": query_param_value(query, "probe_status")
}, },

View File

@@ -1,7 +1,6 @@
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::time::{Duration, SystemTime, UNIX_EPOCH}; use std::time::{Duration, SystemTime, UNIX_EPOCH};
use aether_data_contracts::repository::global_models::AdminProviderModelListQuery;
use aether_data_contracts::repository::pool_scores::GetPoolMemberScoresByIdsQuery; use aether_data_contracts::repository::pool_scores::GetPoolMemberScoresByIdsQuery;
use aether_data_contracts::repository::provider_catalog::{ use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider, StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
@@ -113,32 +112,11 @@ fn provider_offset_cursor_key(provider_id: &str) -> String {
format!("{POOL_SCORE_REBUILD_PROVIDER_OFFSET_PREFIX}:{provider_id}") format!("{POOL_SCORE_REBUILD_PROVIDER_OFFSET_PREFIX}:{provider_id}")
} }
fn score_combo_indices(
flat_index: usize,
model_count: usize,
key_count: usize,
) -> (usize, usize, usize) {
let endpoint_stride = model_count.saturating_mul(key_count).max(1);
let endpoint_index = flat_index / endpoint_stride;
let remainder = flat_index % endpoint_stride;
let model_index = remainder / key_count.max(1);
let key_index = remainder % key_count.max(1);
(endpoint_index, model_index, key_index)
}
#[derive(Debug, Clone)]
struct ProviderScoreBuildItem {
endpoint_index: usize,
model_index: usize,
key_index: usize,
score_id: String,
}
pub(crate) async fn ensure_provider_key_pool_scores_for_keys( pub(crate) async fn ensure_provider_key_pool_scores_for_keys(
state: &AppState, state: &AppState,
provider: &StoredProviderCatalogProvider, provider: &StoredProviderCatalogProvider,
pool_config: &AdminProviderPoolConfig, pool_config: &AdminProviderPoolConfig,
endpoints: &[StoredProviderCatalogEndpoint], _endpoints: &[StoredProviderCatalogEndpoint],
keys: &[StoredProviderCatalogKey], keys: &[StoredProviderCatalogKey],
now_unix_secs: u64, now_unix_secs: u64,
max_upserts: usize, max_upserts: usize,
@@ -151,63 +129,28 @@ pub(crate) async fn ensure_provider_key_pool_scores_for_keys(
return Ok(0); return Ok(0);
} }
let endpoints = endpoints
.iter()
.filter(|endpoint| endpoint.is_active && !endpoint.api_format.trim().is_empty())
.collect::<Vec<_>>();
let keys = keys let keys = keys
.iter() .iter()
.filter(|key| key.is_active && key.provider_id == provider.id) .filter(|key| key.is_active && key.provider_id == provider.id)
.collect::<Vec<_>>(); .collect::<Vec<_>>();
if endpoints.is_empty() || keys.is_empty() { if keys.is_empty() {
return Ok(0); return Ok(0);
} }
let models = state let build_items = keys
.list_admin_provider_models(&AdminProviderModelListQuery {
provider_id: provider.id.clone(),
is_active: Some(true),
offset: 0,
limit: 10_000,
})
.await?
.into_iter() .into_iter()
.filter(|model| model.is_available) .take(max_upserts)
.collect::<Vec<_>>(); .map(|key| {
if models.is_empty() {
return Ok(0);
}
let max_items = endpoints
.len()
.saturating_mul(models.len())
.saturating_mul(keys.len())
.min(max_upserts);
let mut build_items = Vec::with_capacity(max_items);
'outer: for (endpoint_index, endpoint) in endpoints.iter().enumerate() {
for (model_index, model) in models.iter().enumerate() {
for (key_index, key) in keys.iter().enumerate() {
let draft = build_provider_key_pool_score_upsert( let draft = build_provider_key_pool_score_upsert(
key, key,
provider.provider_type.as_str(), provider.provider_type.as_str(),
endpoint.api_format.trim(),
Some(model.id.as_str()),
None, None,
now_unix_secs, now_unix_secs,
pool_config.score_rules, pool_config.score_rules,
); );
build_items.push(ProviderScoreBuildItem { (key, draft.id)
endpoint_index, })
model_index, .collect::<Vec<_>>();
key_index,
score_id: draft.id,
});
if build_items.len() >= max_upserts {
break 'outer;
}
}
}
}
if build_items.is_empty() { if build_items.is_empty() {
return Ok(0); return Ok(0);
} }
@@ -217,7 +160,7 @@ pub(crate) async fn ensure_provider_key_pool_scores_for_keys(
.get_pool_member_scores_by_ids(&GetPoolMemberScoresByIdsQuery { .get_pool_member_scores_by_ids(&GetPoolMemberScoresByIdsQuery {
ids: build_items ids: build_items
.iter() .iter()
.map(|item| item.score_id.clone()) .map(|(_, score_id)| score_id.clone())
.collect(), .collect(),
}) })
.await .await
@@ -234,18 +177,13 @@ pub(crate) async fn ensure_provider_key_pool_scores_for_keys(
.collect::<std::collections::BTreeSet<_>>(); .collect::<std::collections::BTreeSet<_>>();
let mut upserted = 0usize; let mut upserted = 0usize;
for item in &build_items { for (key, score_id) in &build_items {
if existing_score_ids.contains(&item.score_id) { if existing_score_ids.contains(score_id) {
continue; continue;
} }
let endpoint = endpoints[item.endpoint_index];
let model = &models[item.model_index];
let key = keys[item.key_index];
let upsert = build_provider_key_pool_score_upsert( let upsert = build_provider_key_pool_score_upsert(
key, key,
provider.provider_type.as_str(), provider.provider_type.as_str(),
endpoint.api_format.trim(),
Some(model.id.as_str()),
None, None,
now_unix_secs, now_unix_secs,
pool_config.score_rules, pool_config.score_rules,
@@ -292,19 +230,6 @@ pub(crate) async fn perform_pool_score_rebuild_once_with_config(
.iter() .iter()
.map(|(provider, _)| provider.id.clone()) .map(|(provider, _)| provider.id.clone())
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let mut endpoints_by_provider = BTreeMap::new();
for endpoint in state
.list_provider_catalog_endpoints_by_provider_ids(&provider_ids)
.await?
.into_iter()
.filter(|endpoint| endpoint.is_active)
{
endpoints_by_provider
.entry(endpoint.provider_id.clone())
.or_insert_with(Vec::new)
.push(endpoint);
}
let mut keys_by_provider = BTreeMap::new(); let mut keys_by_provider = BTreeMap::new();
for key in state for key in state
.list_provider_catalog_keys_by_provider_ids(&provider_ids) .list_provider_catalog_keys_by_provider_ids(&provider_ids)
@@ -315,6 +240,9 @@ pub(crate) async fn perform_pool_score_rebuild_once_with_config(
.or_insert_with(Vec::new) .or_insert_with(Vec::new)
.push(key); .push(key);
} }
for keys in keys_by_provider.values_mut() {
keys.sort_by(|left, right| left.id.cmp(&right.id));
}
let now = now_unix_secs(); let now = now_unix_secs();
let mut summary = PoolScoreRebuildRunSummary { let mut summary = PoolScoreRebuildRunSummary {
@@ -333,75 +261,39 @@ pub(crate) async fn perform_pool_score_rebuild_once_with_config(
} }
last_provider_index = Some(provider_index); last_provider_index = Some(provider_index);
let (provider, pool_config) = providers[provider_index].clone(); let (provider, pool_config) = providers[provider_index].clone();
let endpoints = endpoints_by_provider
.remove(&provider.id)
.unwrap_or_default();
let keys = keys_by_provider.remove(&provider.id).unwrap_or_default(); let keys = keys_by_provider.remove(&provider.id).unwrap_or_default();
if endpoints.is_empty() || keys.is_empty() { let keys = keys
continue;
}
let models = state
.list_admin_provider_models(&AdminProviderModelListQuery {
provider_id: provider.id.clone(),
is_active: Some(true),
offset: 0,
limit: 10_000,
})
.await?
.into_iter() .into_iter()
.filter(|model| model.is_available) .filter(|key| key.is_active)
.collect::<Vec<_>>(); .collect::<Vec<_>>();
if models.is_empty() { if keys.is_empty() {
continue;
}
let total_combinations = endpoints
.len()
.saturating_mul(models.len())
.saturating_mul(keys.len());
if total_combinations == 0 {
continue; continue;
} }
let total_keys = keys.len();
let provider_cursor_key = provider_offset_cursor_key(&provider.id); let provider_cursor_key = provider_offset_cursor_key(&provider.id);
let provider_cursor = let provider_cursor = load_runtime_usize(state, &provider_cursor_key).await % total_keys;
load_runtime_usize(state, &provider_cursor_key).await % total_combinations.max(1);
let remaining_budget = config let remaining_budget = config
.max_upserts_per_tick .max_upserts_per_tick
.saturating_sub(summary.scores_upserted); .saturating_sub(summary.scores_upserted);
let provider_budget = remaining_budget.min(total_combinations); let provider_budget = remaining_budget.min(total_keys);
let mut build_items = Vec::with_capacity(provider_budget); let mut build_items = Vec::with_capacity(provider_budget);
for offset in 0..provider_budget { for offset in 0..provider_budget {
let flat_index = (provider_cursor + offset) % total_combinations; let key_index = (provider_cursor + offset) % total_keys;
let (endpoint_index, model_index, key_index) =
score_combo_indices(flat_index, models.len(), keys.len());
let endpoint = &endpoints[endpoint_index];
let api_format = endpoint.api_format.trim();
if api_format.is_empty() {
continue;
}
let model = &models[model_index];
let key = &keys[key_index]; let key = &keys[key_index];
let draft = build_provider_key_pool_score_upsert( let draft = build_provider_key_pool_score_upsert(
key, key,
provider.provider_type.as_str(), provider.provider_type.as_str(),
api_format,
Some(model.id.as_str()),
None, None,
now, now,
pool_config.score_rules, pool_config.score_rules,
); );
build_items.push(ProviderScoreBuildItem { build_items.push((key_index, draft.id));
endpoint_index,
model_index,
key_index,
score_id: draft.id,
});
} }
if build_items.is_empty() { if build_items.is_empty() {
store_runtime_usize( store_runtime_usize(
state, state,
&provider_cursor_key, &provider_cursor_key,
(provider_cursor + provider_budget) % total_combinations, (provider_cursor + provider_budget) % total_keys,
) )
.await; .await;
continue; continue;
@@ -411,7 +303,7 @@ pub(crate) async fn perform_pool_score_rebuild_once_with_config(
.get_pool_member_scores_by_ids(&GetPoolMemberScoresByIdsQuery { .get_pool_member_scores_by_ids(&GetPoolMemberScoresByIdsQuery {
ids: build_items ids: build_items
.iter() .iter()
.map(|item| item.score_id.clone()) .map(|(_, score_id)| score_id.clone())
.collect(), .collect(),
}) })
.await .await
@@ -428,19 +320,15 @@ pub(crate) async fn perform_pool_score_rebuild_once_with_config(
.collect::<BTreeMap<_, _>>(); .collect::<BTreeMap<_, _>>();
let mut provider_upserts = 0usize; let mut provider_upserts = 0usize;
summary.keys_seen = summary.keys_seen.saturating_add(keys.len()); summary.keys_seen = summary.keys_seen.saturating_add(keys.len());
for item in &build_items { for (key_index, score_id) in &build_items {
if summary.scores_upserted >= config.max_upserts_per_tick { if summary.scores_upserted >= config.max_upserts_per_tick {
break; break;
} }
let endpoint = &endpoints[item.endpoint_index]; let key = &keys[*key_index];
let model = &models[item.model_index]; let existing = existing_scores.get(score_id);
let key = &keys[item.key_index];
let existing = existing_scores.get(&item.score_id);
let upsert = build_provider_key_pool_score_upsert( let upsert = build_provider_key_pool_score_upsert(
key, key,
provider.provider_type.as_str(), provider.provider_type.as_str(),
endpoint.api_format.trim(),
Some(model.id.as_str()),
existing, existing,
now, now,
pool_config.score_rules, pool_config.score_rules,
@@ -459,7 +347,7 @@ pub(crate) async fn perform_pool_score_rebuild_once_with_config(
store_runtime_usize( store_runtime_usize(
state, state,
&provider_cursor_key, &provider_cursor_key,
(provider_cursor + provider_budget) % total_combinations, (provider_cursor + provider_budget) % total_keys,
) )
.await; .await;
if provider_upserts > 0 { if provider_upserts > 0 {
@@ -528,16 +416,3 @@ pub(crate) fn spawn_pool_score_rebuild_worker(
} }
})) }))
} }
#[cfg(test)]
mod tests {
use super::score_combo_indices;
#[test]
fn score_combo_indices_walks_endpoint_model_key_order() {
assert_eq!(score_combo_indices(0, 2, 3), (0, 0, 0));
assert_eq!(score_combo_indices(2, 2, 3), (0, 0, 2));
assert_eq!(score_combo_indices(3, 2, 3), (0, 1, 0));
assert_eq!(score_combo_indices(6, 2, 3), (1, 0, 0));
}
}

View File

@@ -5,8 +5,7 @@ use aether_data::repository::pool_scores::InMemoryPoolMemberScoreRepository;
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository; use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
use aether_data::repository::usage::InMemoryUsageReadRepository; use aether_data::repository::usage::InMemoryUsageReadRepository;
use aether_data_contracts::repository::pool_scores::{ use aether_data_contracts::repository::pool_scores::{
PoolMemberHardState, PoolMemberProbeStatus, StoredPoolMemberScore, POOL_KIND_PROVIDER_KEY_POOL, PoolMemberHardState, PoolMemberIdentity, PoolMemberProbeStatus, StoredPoolMemberScore,
POOL_MEMBER_KIND_PROVIDER_API_KEY, POOL_SCORE_SCOPE_KIND_MODEL,
}; };
use aether_data_contracts::repository::provider_catalog::ProviderCatalogReadRepository; use aether_data_contracts::repository::provider_catalog::ProviderCatalogReadRepository;
use aether_data_contracts::repository::usage::StoredRequestUsageAudit; use aether_data_contracts::repository::usage::StoredRequestUsageAudit;
@@ -20,6 +19,7 @@ use super::super::{
build_router_with_state, sample_endpoint, sample_key, sample_provider, start_server, AppState, build_router_with_state, sample_endpoint, sample_key, sample_provider, start_server, AppState,
}; };
use crate::admin_api::{maybe_build_local_admin_pool_response, AdminAppState, AdminRequestContext}; use crate::admin_api::{maybe_build_local_admin_pool_response, AdminAppState, AdminRequestContext};
use crate::ai_serving::{provider_key_pool_score_id, provider_key_pool_score_scope};
use crate::audit::AdminAuditEvent; use crate::audit::AdminAuditEvent;
use crate::constants::{ use crate::constants::{
GATEWAY_HEADER, TRUSTED_ADMIN_MANAGEMENT_TOKEN_ID_HEADER, TRUSTED_ADMIN_SESSION_ID_HEADER, GATEWAY_HEADER, TRUSTED_ADMIN_MANAGEMENT_TOKEN_ID_HEADER, TRUSTED_ADMIN_SESSION_ID_HEADER,
@@ -372,16 +372,19 @@ async fn gateway_handles_admin_pool_scores_locally_with_trusted_admin_principal(
Vec::new(), Vec::new(),
vec![key.clone()], vec![key.clone()],
)); ));
let score_scope = provider_key_pool_score_scope();
let score_identity = PoolMemberIdentity::provider_api_key("provider-openai", "key-openai-a");
let score_id = provider_key_pool_score_id(&score_identity, &score_scope);
let pool_score_repository = Arc::new(InMemoryPoolMemberScoreRepository::seed(vec![ let pool_score_repository = Arc::new(InMemoryPoolMemberScoreRepository::seed(vec![
StoredPoolMemberScore { StoredPoolMemberScore {
id: "pms-provider-openai-key-openai-a-openai-chat-model-1".to_string(), id: score_id,
pool_kind: POOL_KIND_PROVIDER_KEY_POOL.to_string(), pool_kind: score_identity.pool_kind.clone(),
pool_id: "provider-openai".to_string(), pool_id: score_identity.pool_id.clone(),
member_kind: POOL_MEMBER_KIND_PROVIDER_API_KEY.to_string(), member_kind: score_identity.member_kind.clone(),
member_id: "key-openai-a".to_string(), member_id: score_identity.member_id.clone(),
capability: "openai:chat".to_string(), capability: score_scope.capability.clone(),
scope_kind: POOL_SCORE_SCOPE_KIND_MODEL.to_string(), scope_kind: score_scope.scope_kind.clone(),
scope_id: Some("model-1".to_string()), scope_id: score_scope.scope_id.clone(),
score: 0.875, score: 0.875,
hard_state: PoolMemberHardState::Available, hard_state: PoolMemberHardState::Available,
score_version: 1, score_version: 1,
@@ -414,7 +417,7 @@ async fn gateway_handles_admin_pool_scores_locally_with_trusted_admin_principal(
let response = reqwest::Client::new() let response = reqwest::Client::new()
.get(format!( .get(format!(
"{gateway_url}/api/admin/pool/provider-openai/scores?api_format=openai:chat&model_id=model-1" "{gateway_url}/api/admin/pool/provider-openai/scores"
)) ))
.header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(crate::constants::GATEWAY_HEADER, "rust-phase3b")
.header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123")
@@ -430,6 +433,9 @@ async fn gateway_handles_admin_pool_scores_locally_with_trusted_admin_principal(
Some(1) Some(1)
); );
assert_eq!(payload["items"][0]["member_id"], json!("key-openai-a")); assert_eq!(payload["items"][0]["member_id"], json!("key-openai-a"));
assert_eq!(payload["items"][0]["capability"], json!("account"));
assert_eq!(payload["items"][0]["scope_kind"], json!("account"));
assert_eq!(payload["items"][0]["scope_id"], serde_json::Value::Null);
assert_eq!(payload["items"][0]["key"]["name"], json!("score key")); assert_eq!(payload["items"][0]["key"]["name"], json!("score key"));
assert_eq!(payload["items"][0]["probe_status"], json!("ok")); assert_eq!(payload["items"][0]["probe_status"], json!("ok"));
@@ -734,14 +740,46 @@ async fn gateway_handles_admin_pool_list_keys_locally_with_trusted_admin_princip
Vec::new(), Vec::new(),
vec![primary_key, cooldown_key, inactive_key], vec![primary_key, cooldown_key, inactive_key],
)); ));
let score_scope = provider_key_pool_score_scope();
let score_identity = PoolMemberIdentity::provider_api_key("provider-openai", "key-openai-a");
let pool_score_repository = Arc::new(InMemoryPoolMemberScoreRepository::seed(vec![
StoredPoolMemberScore {
id: provider_key_pool_score_id(&score_identity, &score_scope),
pool_kind: score_identity.pool_kind.clone(),
pool_id: score_identity.pool_id.clone(),
member_kind: score_identity.member_kind.clone(),
member_id: score_identity.member_id.clone(),
capability: score_scope.capability.clone(),
scope_kind: score_scope.scope_kind.clone(),
scope_id: score_scope.scope_id.clone(),
score: 0.875,
hard_state: PoolMemberHardState::Available,
score_version: 1,
score_reason: json!({ "weights": { "manual_priority": 0.3 } }),
last_ranked_at: Some(1_700_000_000),
last_scheduled_at: Some(1_700_000_010),
last_success_at: Some(1_700_000_020),
last_failure_at: None,
failure_count: 0,
last_probe_attempt_at: Some(1_700_000_030),
last_probe_success_at: Some(1_700_000_040),
last_probe_failure_at: None,
probe_failure_count: 0,
probe_status: PoolMemberProbeStatus::Ok,
updated_at: 1_700_000_050,
},
]));
let (upstream_url, upstream_handle) = start_server(upstream).await; let (upstream_url, upstream_handle) = start_server(upstream).await;
let gateway = build_router_with_state( let gateway = build_router_with_state(
AppState::new() AppState::new()
.expect("gateway should build") .expect("gateway should build")
.with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests( .with_data_state_for_tests(
GatewayDataState::with_provider_catalog_reader_for_tests(
provider_catalog_repository, provider_catalog_repository,
)), )
.with_pool_score_repository_for_tests(pool_score_repository),
),
); );
let (gateway_url, gateway_handle) = start_server(gateway).await; let (gateway_url, gateway_handle) = start_server(gateway).await;
@@ -765,6 +803,9 @@ async fn gateway_handles_admin_pool_list_keys_locally_with_trusted_admin_princip
let keys = payload["keys"].as_array().expect("keys should be array"); let keys = payload["keys"].as_array().expect("keys should be array");
assert_eq!(keys.len(), 2); assert_eq!(keys.len(), 2);
assert_eq!(keys[0]["key_name"], json!("alpha")); assert_eq!(keys[0]["key_name"], json!("alpha"));
assert_eq!(keys[0]["pool_score"]["score"], json!(0.875));
assert_eq!(keys[0]["pool_score"]["scope_kind"], json!("account"));
assert_eq!(keys[0]["pool_score"]["scope_id"], serde_json::Value::Null);
assert_eq!(keys[0]["scheduling_status"], json!("available")); assert_eq!(keys[0]["scheduling_status"], json!("available"));
assert_eq!(keys[1]["key_name"], json!("beta")); assert_eq!(keys[1]["key_name"], json!("beta"));
assert_eq!(keys[1]["scheduling_reason"], json!("available")); assert_eq!(keys[1]["scheduling_reason"], json!("available"));

View File

@@ -6,6 +6,6 @@ pub use types::{
PoolMemberProbeResult, PoolMemberProbeStatus, PoolMemberScheduleFeedback, PoolMemberProbeResult, PoolMemberProbeStatus, PoolMemberScheduleFeedback,
PoolMemberScoreRepository, PoolMemberScoreWriteRepository, PoolScoreReadRepository, PoolMemberScoreRepository, PoolMemberScoreWriteRepository, PoolScoreReadRepository,
PoolScoreScope, StoredPoolMemberScore, UpsertPoolMemberScore, POOL_KIND_PROVIDER_KEY_POOL, PoolScoreScope, StoredPoolMemberScore, UpsertPoolMemberScore, POOL_KIND_PROVIDER_KEY_POOL,
POOL_MEMBER_KIND_PROVIDER_API_KEY, POOL_SCORE_CAPABILITY_API_FORMAT, POOL_MEMBER_KIND_PROVIDER_API_KEY, POOL_SCORE_CAPABILITY_ACCOUNT,
POOL_SCORE_SCOPE_KIND_MODEL, POOL_SCORE_CAPABILITY_API_FORMAT, POOL_SCORE_SCOPE_KIND_ACCOUNT, POOL_SCORE_SCOPE_KIND_MODEL,
}; };

View File

@@ -2,8 +2,10 @@ use async_trait::async_trait;
pub const POOL_KIND_PROVIDER_KEY_POOL: &str = "provider_key_pool"; pub const POOL_KIND_PROVIDER_KEY_POOL: &str = "provider_key_pool";
pub const POOL_MEMBER_KIND_PROVIDER_API_KEY: &str = "provider_api_key"; pub const POOL_MEMBER_KIND_PROVIDER_API_KEY: &str = "provider_api_key";
pub const POOL_SCORE_CAPABILITY_API_FORMAT: &str = "api_format"; pub const POOL_SCORE_CAPABILITY_ACCOUNT: &str = "account";
pub const POOL_SCORE_SCOPE_KIND_MODEL: &str = "model"; pub const POOL_SCORE_SCOPE_KIND_ACCOUNT: &str = "account";
pub const POOL_SCORE_CAPABILITY_API_FORMAT: &str = POOL_SCORE_CAPABILITY_ACCOUNT;
pub const POOL_SCORE_SCOPE_KIND_MODEL: &str = POOL_SCORE_SCOPE_KIND_ACCOUNT;
#[derive( #[derive(
Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize,

View File

@@ -132,6 +132,7 @@ export interface PoolKeyDetail {
quota_updated_at?: number | null quota_updated_at?: number | null
health_score?: number health_score?: number
circuit_breaker_open?: boolean circuit_breaker_open?: boolean
pool_score?: PoolKeyScoreDetail | null
api_formats?: string[] api_formats?: string[]
rate_multipliers?: Record<string, number> | null rate_multipliers?: Record<string, number> | null
internal_priority?: number internal_priority?: number
@@ -192,6 +193,28 @@ export interface PoolKeysPageResponse {
keys: PoolKeyDetail[] keys: PoolKeyDetail[]
} }
export interface PoolKeyScoreDetail {
id: string
capability: string
scope_kind: string
scope_id: string | null
score: number
hard_state: PoolScoreHardState
score_version: number
score_reason: Record<string, unknown> | null
last_ranked_at: number | null
last_scheduled_at: number | null
last_success_at: number | null
last_failure_at: number | null
failure_count: number
last_probe_attempt_at: number | null
last_probe_success_at: number | null
last_probe_failure_at: number | null
probe_failure_count: number
probe_status: PoolScoreProbeStatus
updated_at: number
}
export type PoolScoreHardState = export type PoolScoreHardState =
| 'available' | 'available'
| 'unknown' | 'unknown'
@@ -212,30 +235,11 @@ export interface PoolScoreKeySummary {
last_used_at: number | null last_used_at: number | null
} }
export interface PoolMemberScoreItem { export interface PoolMemberScoreItem extends PoolKeyScoreDetail {
id: string
pool_kind: string pool_kind: string
pool_id: string pool_id: string
member_kind: string member_kind: string
member_id: string member_id: string
capability: string
scope_kind: string
scope_id: string | null
score: number
hard_state: PoolScoreHardState
score_version: number
score_reason: Record<string, unknown> | null
last_ranked_at: number | null
last_scheduled_at: number | null
last_success_at: number | null
last_failure_at: number | null
failure_count: number
last_probe_attempt_at: number | null
last_probe_success_at: number | null
last_probe_failure_at: number | null
probe_failure_count: number
probe_status: PoolScoreProbeStatus
updated_at: number
key?: PoolScoreKeySummary | null key?: PoolScoreKeySummary | null
} }

View File

@@ -405,6 +405,12 @@
> >
最后使用 最后使用
</SortableTableHead> </SortableTableHead>
<TableHead
class="font-semibold text-center whitespace-nowrap"
:style="{ width: desktopColumnWidths.score }"
>
分数
</TableHead>
<SortableTableHead <SortableTableHead
class="font-semibold text-center whitespace-nowrap" class="font-semibold text-center whitespace-nowrap"
column-key="status" column-key="status"
@@ -665,6 +671,64 @@
{{ keyUiStateMap[key.key_id]?.lastUsedRelative || '-' }} {{ keyUiStateMap[key.key_id]?.lastUsedRelative || '-' }}
</span> </span>
</TableCell> </TableCell>
<TableCell class="py-3 text-center align-middle">
<div class="inline-flex items-center justify-center gap-1">
<span class="font-mono text-xs tabular-nums text-foreground/90">
{{ formatPoolScore(key.pool_score?.score) }}
</span>
<Popover
v-if="key.pool_score"
:open="scorePopoverOpenKeyId === key.key_id"
@update:open="(open: boolean) => handleScorePopoverToggle(key.key_id, open)"
>
<PopoverTrigger as-child>
<Button
variant="ghost"
size="icon"
class="h-5 w-5 rounded-full text-muted-foreground hover:text-foreground"
title="查看评分计算结果"
aria-label="查看评分计算结果"
@click.stop
>
<CircleHelp class="h-3.5 w-3.5" />
</Button>
</PopoverTrigger>
<PopoverContent
v-if="scorePopoverOpenKeyId === key.key_id"
class="w-96 max-w-[calc(100vw-2rem)] p-3"
side="bottom"
align="center"
>
<div class="space-y-2 text-left">
<div class="flex items-center justify-between gap-3">
<span class="text-xs font-medium">评分计算结果</span>
<span class="font-mono text-xs tabular-nums">
{{ formatPoolScore(key.pool_score?.score) }}
</span>
</div>
<div class="flex flex-wrap items-center gap-1.5">
<Badge
variant="outline"
class="text-[10px]"
>
{{ getPoolScoreHardStateLabel(key.pool_score?.hard_state) }}
</Badge>
<Badge
variant="secondary"
class="text-[10px]"
>
{{ getPoolScoreProbeStatusLabel(key.pool_score?.probe_status) }}
</Badge>
<span class="text-[10px] text-muted-foreground">
更新 {{ formatUnixSeconds(key.pool_score?.updated_at) }}
</span>
</div>
<pre class="max-h-64 overflow-auto rounded-lg bg-muted/40 p-3 text-[11px] leading-5 text-foreground whitespace-pre-wrap break-words">{{ formatPoolScoreReason(key.pool_score?.score_reason) }}</pre>
</div>
</PopoverContent>
</Popover>
</div>
</TableCell>
<TableCell class="py-3 text-center"> <TableCell class="py-3 text-center">
<Badge <Badge
:variant="keyUiStateMap[key.key_id]?.schedulingBadgeVariant || 'default'" :variant="keyUiStateMap[key.key_id]?.schedulingBadgeVariant || 'default'"
@@ -928,6 +992,65 @@
<span class="text-muted-foreground">最后使用</span> <span class="text-muted-foreground">最后使用</span>
<span class="font-medium text-foreground/90">{{ keyUiStateMap[key.key_id]?.lastUsedRelative || '-' }}</span> <span class="font-medium text-foreground/90">{{ keyUiStateMap[key.key_id]?.lastUsedRelative || '-' }}</span>
</div> </div>
<div class="flex items-center justify-between gap-2">
<span class="text-muted-foreground">分数</span>
<div class="flex items-center gap-1">
<span class="font-mono font-medium text-foreground/90 tabular-nums">
{{ formatPoolScore(key.pool_score?.score) }}
</span>
<Popover
v-if="key.pool_score"
:open="scorePopoverOpenKeyId === key.key_id"
@update:open="(open: boolean) => handleScorePopoverToggle(key.key_id, open)"
>
<PopoverTrigger as-child>
<Button
variant="ghost"
size="icon"
class="h-5 w-5 rounded-full text-muted-foreground hover:text-foreground"
title="查看评分计算结果"
aria-label="查看评分计算结果"
@click.stop
>
<CircleHelp class="h-3.5 w-3.5" />
</Button>
</PopoverTrigger>
<PopoverContent
v-if="scorePopoverOpenKeyId === key.key_id"
class="w-96 max-w-[calc(100vw-2rem)] p-3"
side="bottom"
align="center"
>
<div class="space-y-2 text-left">
<div class="flex items-center justify-between gap-3">
<span class="text-xs font-medium">评分计算结果</span>
<span class="font-mono text-xs tabular-nums">
{{ formatPoolScore(key.pool_score?.score) }}
</span>
</div>
<div class="flex flex-wrap items-center gap-1.5">
<Badge
variant="outline"
class="text-[10px]"
>
{{ getPoolScoreHardStateLabel(key.pool_score?.hard_state) }}
</Badge>
<Badge
variant="secondary"
class="text-[10px]"
>
{{ getPoolScoreProbeStatusLabel(key.pool_score?.probe_status) }}
</Badge>
<span class="text-[10px] text-muted-foreground">
更新 {{ formatUnixSeconds(key.pool_score?.updated_at) }}
</span>
</div>
<pre class="max-h-64 overflow-auto rounded-lg bg-muted/40 p-3 text-[11px] leading-5 text-foreground whitespace-pre-wrap break-words">{{ formatPoolScoreReason(key.pool_score?.score_reason) }}</pre>
</div>
</PopoverContent>
</Popover>
</div>
</div>
</div> </div>
</div> </div>
@@ -1189,170 +1312,6 @@
</template> </template>
</Card> </Card>
<Card
v-if="selectedProviderId"
variant="default"
class="overflow-hidden"
>
<div class="flex flex-col gap-3 border-b border-border/60 px-4 py-3 sm:flex-row sm:items-center sm:justify-between sm:px-6 sm:py-3.5">
<div class="min-w-0">
<h3 class="text-base font-semibold">
评分排序
</h3>
<p class="mt-1 text-xs text-muted-foreground">
Top {{ poolScorePage.items.length }} / page {{ poolScorePage.page || 1 }}
</p>
</div>
<RefreshButton
:loading="scoresLoading"
title="刷新评分"
@click="() => loadScores()"
/>
</div>
<div class="grid gap-2 border-b border-border/60 bg-muted/20 px-4 py-3 sm:grid-cols-4 sm:px-6">
<Input
v-model="scoreApiFormatFilter"
class="h-8 text-xs"
placeholder="api_format"
/>
<Input
v-model="scoreModelIdFilter"
class="h-8 text-xs"
placeholder="model_id"
/>
<Select v-model="scoreHardStateFilter">
<SelectTrigger class="h-8 text-xs border-border/60">
<SelectValue placeholder="硬状态" />
</SelectTrigger>
<SelectContent>
<SelectItem
v-for="item in poolScoreHardStateOptions"
:key="item.value"
:value="item.value"
>
{{ item.label }}
</SelectItem>
</SelectContent>
</Select>
<Select v-model="scoreProbeStatusFilter">
<SelectTrigger class="h-8 text-xs border-border/60">
<SelectValue placeholder="探测" />
</SelectTrigger>
<SelectContent>
<SelectItem
v-for="item in poolScoreProbeStatusOptions"
:key="item.value"
:value="item.value"
>
{{ item.label }}
</SelectItem>
</SelectContent>
</Select>
</div>
<div
v-if="scoresLoading && poolScorePage.items.length === 0"
class="flex items-center justify-center py-10"
>
<div class="animate-spin rounded-full h-7 w-7 border-b-2 border-primary" />
</div>
<div
v-else-if="poolScorePage.items.length === 0"
class="flex flex-col items-center justify-center py-10 text-center"
>
<div class="mx-auto flex h-12 w-12 items-center justify-center rounded-full bg-muted">
<SlidersHorizontal class="h-6 w-6 text-muted-foreground" />
</div>
<p class="mt-3 text-sm text-muted-foreground">
{{ hasPoolScoreFilters ? '未找到匹配评分' : '暂无评分数据' }}
</p>
</div>
<div
v-else
class="overflow-x-auto"
>
<Table class="w-full min-w-[760px] table-fixed">
<TableHeader>
<TableRow class="border-b border-border/60 hover:bg-transparent">
<TableHead class="w-[24%] font-semibold">
账号
</TableHead>
<TableHead class="w-[24%] font-semibold">
范围
</TableHead>
<TableHead class="w-[12%] text-center font-semibold">
分数
</TableHead>
<TableHead class="w-[12%] text-center font-semibold">
状态
</TableHead>
<TableHead class="w-[12%] text-center font-semibold">
探测
</TableHead>
<TableHead class="w-[16%] text-center font-semibold">
更新时间
</TableHead>
</TableRow>
</TableHeader>
<TableBody>
<TableRow
v-for="score in poolScorePage.items"
:key="score.id"
class="border-b border-border/40 last:border-b-0 hover:bg-muted/30"
>
<TableCell class="py-3">
<div class="min-w-0">
<div class="truncate text-sm">
{{ score.key?.name || score.member_id }}
</div>
<div class="truncate font-mono text-[10px] text-muted-foreground">
{{ score.member_id }}
</div>
</div>
</TableCell>
<TableCell class="py-3">
<div class="min-w-0 text-xs">
<div class="truncate text-foreground/90">
{{ score.capability || '-' }}
</div>
<div class="truncate text-[10px] text-muted-foreground">
{{ score.scope_kind }}: {{ score.scope_id || '*' }}
</div>
</div>
</TableCell>
<TableCell class="py-3 text-center">
<span class="font-mono text-xs tabular-nums">
{{ formatPoolScore(score.score) }}
</span>
</TableCell>
<TableCell class="py-3 text-center">
<Badge
variant="outline"
class="text-[10px]"
>
{{ getPoolScoreHardStateLabel(score.hard_state) }}
</Badge>
</TableCell>
<TableCell class="py-3 text-center">
<Badge
variant="secondary"
class="text-[10px]"
>
{{ getPoolScoreProbeStatusLabel(score.probe_status) }}
</Badge>
</TableCell>
<TableCell class="py-3 text-center">
<span class="text-[10px] text-muted-foreground">
{{ formatUnixSeconds(score.updated_at) }}
</span>
</TableCell>
</TableRow>
</TableBody>
</Table>
</div>
</Card>
<!-- Dialogs --> <!-- Dialogs -->
<OAuthAccountDialog <OAuthAccountDialog
v-if="selectedProviderId" v-if="selectedProviderId"
@@ -1437,6 +1396,7 @@ import {
Users, Users,
Settings2, Settings2,
SlidersHorizontal, SlidersHorizontal,
CircleHelp,
} from 'lucide-vue-next' } from 'lucide-vue-next'
import { import {
@@ -1473,7 +1433,6 @@ import {
getPoolOverview, getPoolOverview,
getPoolSchedulingPresets, getPoolSchedulingPresets,
listPoolKeys, listPoolKeys,
listPoolScores,
clearPoolCooldown, clearPoolCooldown,
} from '@/api/endpoints/pool' } from '@/api/endpoints/pool'
import { import {
@@ -1489,9 +1448,7 @@ import type {
PoolOverviewItem, PoolOverviewItem,
PoolKeyDetail, PoolKeyDetail,
PoolKeysPageResponse, PoolKeysPageResponse,
PoolMemberScoreItem,
PoolPresetMeta, PoolPresetMeta,
PoolScoresResponse,
} from '@/api/endpoints/pool' } from '@/api/endpoints/pool'
import type { import type {
ClaudeCodeAdvancedConfig, ClaudeCodeAdvancedConfig,
@@ -1558,6 +1515,8 @@ import {
getQuotaDisplayText, getQuotaDisplayText,
} from '@/utils/providerKeyQuota' } from '@/utils/providerKeyQuota'
type PoolKeyScore = NonNullable<PoolKeyDetail['pool_score']>
const { success, error: showError, warning: showWarning } = useToast() const { success, error: showError, warning: showWarning } = useToast()
const { confirm } = useConfirm() const { confirm } = useConfirm()
const { copyToClipboard } = useClipboard() const { copyToClipboard } = useClipboard()
@@ -1587,14 +1546,11 @@ let overviewRequestId = 0
let selectProviderRequestId = 0 let selectProviderRequestId = 0
let providerDataRequestId = 0 let providerDataRequestId = 0
let keysRequestId = 0 let keysRequestId = 0
let scoresRequestId = 0
let keysSearchDebounceTimer: number | null = null let keysSearchDebounceTimer: number | null = null
let scoresFilterDebounceTimer: number | null = null
let suppressFiltersWatch = false let suppressFiltersWatch = false
let hasHydratedInitialProviderSelection = false let hasHydratedInitialProviderSelection = false
const POOL_OVERVIEW_CACHE_TTL_MS = 10 * 1000 const POOL_OVERVIEW_CACHE_TTL_MS = 10 * 1000
const POOL_KEYS_CACHE_TTL_MS = 10 * 1000 const POOL_KEYS_CACHE_TTL_MS = 10 * 1000
const POOL_SCORES_CACHE_TTL_MS = 10 * 1000
const POOL_SCHEDULING_PRESETS_CACHE_TTL_MS = 5 * 60 * 1000 const POOL_SCHEDULING_PRESETS_CACHE_TTL_MS = 5 * 60 * 1000
const poolKeyStatusFilterOptions: Array<{ value: PoolManagementViewState['status'], label: string }> = [ const poolKeyStatusFilterOptions: Array<{ value: PoolManagementViewState['status'], label: string }> = [
{ value: 'all', label: '全部状态' }, { value: 'all', label: '全部状态' },
@@ -1676,7 +1632,6 @@ async function loadOverview(options: { cacheTtlMs?: number } = {}) {
showAccountBatchDialog.value = false showAccountBatchDialog.value = false
closeProviderProxyPopovers() closeProviderProxyPopovers()
resetKeyPage() resetKeyPage()
resetScorePage()
} }
} catch (err) { } catch (err) {
if (requestId !== overviewRequestId) return if (requestId !== overviewRequestId) return
@@ -1851,23 +1806,25 @@ const showAccountQuotaColumn = computed(() => {
const desktopColumnWidths = computed(() => { const desktopColumnWidths = computed(() => {
if (showAccountQuotaColumn.value) { if (showAccountQuotaColumn.value) {
return { return {
name: '22%', name: '21%',
quota: '21%', quota: '18%',
stats: '15%', stats: '13%',
imported: '10%', imported: '10%',
lastUsed: '9%', lastUsed: '8%',
score: '9%',
status: '7%', status: '7%',
actions: '16%', actions: '14%',
} }
} }
return { return {
name: '34%', name: '31%',
quota: '0%', quota: '0%',
stats: '16%', stats: '15%',
imported: '12%', imported: '11%',
lastUsed: '12%', lastUsed: '11%',
status: '9%', score: '9%',
actions: '17%', status: '8%',
actions: '15%',
} }
}) })
@@ -1892,6 +1849,7 @@ async function selectProvider(
closeProviderProxyPopovers() closeProviderProxyPopovers()
proxyDesktopPopoverOpenKeyId.value = null proxyDesktopPopoverOpenKeyId.value = null
proxyMobilePopoverOpenKeyId.value = null proxyMobilePopoverOpenKeyId.value = null
scorePopoverOpenKeyId.value = null
suppressFiltersWatch = true suppressFiltersWatch = true
if (!options.preservePagination) { if (!options.preservePagination) {
currentPage.value = 1 currentPage.value = 1
@@ -1909,11 +1867,9 @@ async function selectProvider(
} }
keysLoadedOnce.value = false keysLoadedOnce.value = false
resetKeyPage(currentPage.value, pageSize.value) resetKeyPage(currentPage.value, pageSize.value)
resetScorePage()
const keysTask = loadKeys({ cacheTtlMs: options.cacheTtlMs ?? 0 }) const keysTask = loadKeys({ cacheTtlMs: options.cacheTtlMs ?? 0 })
// Provider summary is non-blocking for key list rendering. // Provider summary is non-blocking for key list rendering.
void loadProviderData(id) void loadProviderData(id)
void loadScores({ cacheTtlMs: options.cacheTtlMs ? POOL_SCORES_CACHE_TTL_MS : 0 })
await keysTask await keysTask
if (requestId !== selectProviderRequestId) return if (requestId !== selectProviderRequestId) return
} }
@@ -1939,46 +1895,25 @@ function createEmptyKeyPage(page = 1, pageSizeValue = 50): PoolKeysPageResponse
return { total: 0, page, page_size: pageSizeValue, keys: [] } return { total: 0, page, page_size: pageSizeValue, keys: [] }
} }
function createEmptyScorePage(): PoolScoresResponse {
return {
provider_id: selectedProviderId.value || '',
page: 1,
page_size: 20,
filters: {},
items: [],
}
}
const keyPage = ref<PoolKeysPageResponse>(createEmptyKeyPage()) const keyPage = ref<PoolKeysPageResponse>(createEmptyKeyPage())
const poolScorePage = ref<PoolScoresResponse>(createEmptyScorePage())
const keysLoading = ref(false) const keysLoading = ref(false)
const scoresLoading = ref(false)
const keysLoadedOnce = ref(false) const keysLoadedOnce = ref(false)
const refreshingCurrentPageQuota = ref(false) const refreshingCurrentPageQuota = ref(false)
const searchQuery = ref(restoredViewState.search) const searchQuery = ref(restoredViewState.search)
const statusFilter = ref(restoredViewState.status) const statusFilter = ref(restoredViewState.status)
const scoreApiFormatFilter = ref('')
const scoreModelIdFilter = ref('')
const scoreHardStateFilter = ref('all')
const scoreProbeStatusFilter = ref('all')
const currentPage = ref(restoredViewState.page) const currentPage = ref(restoredViewState.page)
const pageSize = ref(restoredViewState.pageSize) const pageSize = ref(restoredViewState.pageSize)
const sortBy = ref<PoolManagementSortBy | null>(restoredViewState.sortBy) const sortBy = ref<PoolManagementSortBy | null>(restoredViewState.sortBy)
const sortOrder = ref<PoolManagementSortOrder>(restoredViewState.sortOrder) const sortOrder = ref<PoolManagementSortOrder>(restoredViewState.sortOrder)
const poolStatsMode = ref<PoolManagementStatsMode>(restoredViewState.statsMode) const poolStatsMode = ref<PoolManagementStatsMode>(restoredViewState.statsMode)
const hasPoolKeyFilters = computed(() => searchQuery.value.trim().length > 0 || statusFilter.value !== 'all') const hasPoolKeyFilters = computed(() => searchQuery.value.trim().length > 0 || statusFilter.value !== 'all')
const hasPoolScoreFilters = computed(() => {
return scoreApiFormatFilter.value.trim().length > 0
|| scoreModelIdFilter.value.trim().length > 0
|| scoreHardStateFilter.value !== 'all'
|| scoreProbeStatusFilter.value !== 'all'
})
const MANUAL_QUOTA_REFRESH_COOLDOWN_SECONDS = 5 * 60 const MANUAL_QUOTA_REFRESH_COOLDOWN_SECONDS = 5 * 60
const refreshingOAuthKeyId = ref<string | null>(null) const refreshingOAuthKeyId = ref<string | null>(null)
const resettingCycleKeyId = ref<string | null>(null) const resettingCycleKeyId = ref<string | null>(null)
const savingProxyKeyId = ref<string | null>(null) const savingProxyKeyId = ref<string | null>(null)
const proxyDesktopPopoverOpenKeyId = ref<string | null>(null) const proxyDesktopPopoverOpenKeyId = ref<string | null>(null)
const proxyMobilePopoverOpenKeyId = ref<string | null>(null) const proxyMobilePopoverOpenKeyId = ref<string | null>(null)
const scorePopoverOpenKeyId = ref<string | null>(null)
const deletingKeyId = ref<string | null>(null) const deletingKeyId = ref<string | null>(null)
const togglingKeyId = ref<string | null>(null) const togglingKeyId = ref<string | null>(null)
const editingPriorityKeyId = ref<string | null>(null) const editingPriorityKeyId = ref<string | null>(null)
@@ -2273,10 +2208,6 @@ function resetKeyPage(page = currentPage.value, pageSizeValue = pageSize.value):
keyPage.value = createEmptyKeyPage(page, pageSizeValue) keyPage.value = createEmptyKeyPage(page, pageSizeValue)
} }
function resetScorePage(): void {
poolScorePage.value = createEmptyScorePage()
}
function refreshOverviewInBackground(): void { function refreshOverviewInBackground(): void {
void loadOverview() void loadOverview()
} }
@@ -2416,36 +2347,6 @@ async function refreshCurrentPage() {
if (!quotaDidReload) { if (!quotaDidReload) {
await refresh() await refresh()
} }
void loadScores()
}
async function loadScores(options: { cacheTtlMs?: number } = {}) {
if (!selectedProviderId.value) return
const requestId = ++scoresRequestId
const providerId = selectedProviderId.value
scoresLoading.value = true
try {
const nextPage = await listPoolScores(providerId, {
page: 1,
page_size: 20,
api_format: scoreApiFormatFilter.value.trim() || undefined,
model_id: scoreModelIdFilter.value.trim() || undefined,
hard_state: scoreHardStateFilter.value === 'all' ? undefined : scoreHardStateFilter.value,
probe_status: scoreProbeStatusFilter.value === 'all' ? undefined : scoreProbeStatusFilter.value,
}, {
cacheTtlMs: options.cacheTtlMs ?? 0,
})
if (requestId !== scoresRequestId || selectedProviderId.value !== providerId) return
poolScorePage.value = nextPage
} catch (err) {
if (requestId !== scoresRequestId || selectedProviderId.value !== providerId) return
resetScorePage()
showError(parseApiError(err, '读取评分失败'))
} finally {
if (requestId === scoresRequestId) {
scoresLoading.value = false
}
}
} }
async function loadKeys(options: { cacheTtlMs?: number } = {}) { async function loadKeys(options: { cacheTtlMs?: number } = {}) {
@@ -2523,20 +2424,6 @@ watch(searchQuery, () => {
}, 300) }, 300)
}) })
watch([scoreApiFormatFilter, scoreModelIdFilter], () => {
if (scoresFilterDebounceTimer !== null) {
clearTimeout(scoresFilterDebounceTimer)
}
scoresFilterDebounceTimer = window.setTimeout(() => {
scoresFilterDebounceTimer = null
void loadScores({ cacheTtlMs: POOL_SCORES_CACHE_TTL_MS })
}, 300)
})
watch([scoreHardStateFilter, scoreProbeStatusFilter], () => {
void loadScores({ cacheTtlMs: POOL_SCORES_CACHE_TTL_MS })
})
function normalizeAuthTypeForEdit(key: PoolKeyDetail): EndpointAPIKey['auth_type'] { function normalizeAuthTypeForEdit(key: PoolKeyDetail): EndpointAPIKey['auth_type'] {
if (isOAuthManagedCredential(key)) return 'oauth' if (isOAuthManagedCredential(key)) return 'oauth'
if (isServiceAccountCredential(key)) return 'service_account' if (isServiceAccountCredential(key)) return 'service_account'
@@ -2710,6 +2597,10 @@ function getKeyProxyNodeName(key: PoolKeyDetail): string | null {
return node ? node.name : `${key.proxy.node_id.slice(0, 8)}...` return node ? node.name : `${key.proxy.node_id.slice(0, 8)}...`
} }
function handleScorePopoverToggle(keyId: string, open: boolean) {
scorePopoverOpenKeyId.value = open ? keyId : null
}
function handleProxyDesktopPopoverToggle(keyId: string, open: boolean) { function handleProxyDesktopPopoverToggle(keyId: string, open: boolean) {
proxyDesktopPopoverOpenKeyId.value = open ? keyId : null proxyDesktopPopoverOpenKeyId.value = open ? keyId : null
if (open) { if (open) {
@@ -3845,17 +3736,28 @@ function formatStatUsd(value: number | string | null | undefined): string {
return `$${n.toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })}` return `$${n.toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`
} }
function formatPoolScore(value: number): string { function formatPoolScore(value: number | null | undefined): string {
const n = Number(value) const n = Number(value)
if (!Number.isFinite(n)) return '0.000' if (!Number.isFinite(n)) return '-'
return n.toFixed(3) return n.toFixed(3)
} }
function getPoolScoreHardStateLabel(value: PoolMemberScoreItem['hard_state']): string { function formatPoolScoreReason(value: PoolKeyScore['score_reason'] | null | undefined): string {
if (!value) return '暂无计算结果'
try {
return JSON.stringify(value, null, 2)
} catch {
return String(value)
}
}
function getPoolScoreHardStateLabel(value: PoolKeyScore['hard_state'] | null | undefined): string {
if (!value) return '-'
return poolScoreHardStateOptions.find(item => item.value === value)?.label || value return poolScoreHardStateOptions.find(item => item.value === value)?.label || value
} }
function getPoolScoreProbeStatusLabel(value: PoolMemberScoreItem['probe_status']): string { function getPoolScoreProbeStatusLabel(value: PoolKeyScore['probe_status'] | null | undefined): string {
if (!value) return '-'
return poolScoreProbeStatusOptions.find(item => item.value === value)?.label || value return poolScoreProbeStatusOptions.find(item => item.value === value)?.label || value
} }
@@ -3892,14 +3794,9 @@ onBeforeUnmount(() => {
clearTimeout(keysSearchDebounceTimer) clearTimeout(keysSearchDebounceTimer)
keysSearchDebounceTimer = null keysSearchDebounceTimer = null
} }
if (scoresFilterDebounceTimer !== null) {
clearTimeout(scoresFilterDebounceTimer)
scoresFilterDebounceTimer = null
}
overviewRequestId += 1 overviewRequestId += 1
selectProviderRequestId += 1 selectProviderRequestId += 1
providerDataRequestId += 1 providerDataRequestId += 1
keysRequestId += 1 keysRequestId += 1
scoresRequestId += 1
}) })
</script> </script>

View File

@@ -139,6 +139,7 @@ vi.mock('lucide-vue-next', async () => {
Users: Icon, Users: Icon,
Settings2: Icon, Settings2: Icon,
SlidersHorizontal: Icon, SlidersHorizontal: Icon,
CircleHelp: Icon,
} }
}) })
@@ -523,6 +524,41 @@ describe('PoolManagement Codex cycle stats mode', () => {
expect(root.textContent).not.toContain('总计') expect(root.textContent).not.toContain('总计')
}) })
it('renders unified pool score in the key list with a calculation entry point', async () => {
const scoredKey = createPoolKey('codex', {
pool_score: {
id: 'pms-account-score',
capability: 'account',
scope_kind: 'account',
scope_id: null,
score: 0.875,
hard_state: 'available',
score_version: 1,
score_reason: { weights: { manual_priority: 0.3 } },
last_ranked_at: 1_700_000_000,
last_scheduled_at: 1_700_000_010,
last_success_at: 1_700_000_020,
last_failure_at: null,
failure_count: 0,
last_probe_attempt_at: 1_700_000_030,
last_probe_success_at: 1_700_000_040,
last_probe_failure_at: null,
probe_failure_count: 0,
probe_status: 'ok',
updated_at: 1_700_000_050,
},
})
endpointMocks.getPoolOverview.mockResolvedValue({ items: [createOverview('codex')] })
endpointMocks.listPoolKeys.mockResolvedValue(createKeyPage(scoredKey))
endpointMocks.getProvider.mockResolvedValue(createProvider('codex'))
const root = mountPoolManagement()
await settle()
expect(root.textContent).toContain('0.875')
expect(root.querySelectorAll('button[title="查看评分计算结果"]').length).toBeGreaterThan(0)
})
it('refreshes quota only for keys on the current page', async () => { it('refreshes quota only for keys on the current page', async () => {
const pageKeys = [ const pageKeys = [
createPoolKey('codex', { key_id: 'codex-page-key-1', quota_updated_at: null }), createPoolKey('codex', { key_id: 'codex-page-key-1', quota_updated_at: null }),