mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-07 18:07:47 +08:00
feat: improve routing policy diagnostics
This commit is contained in:
fawney19
@@ -688,8 +688,18 @@ impl<'a> PoolKeyCursor<'a> {
|
||||
return None;
|
||||
}
|
||||
};
|
||||
self.scanned_keys = self.scanned_keys.saturating_add(scores.len() as u32);
|
||||
self.budget_scanned_keys = self.budget_scanned_keys.saturating_add(scores.len() as u32);
|
||||
let materialized_row_count = rows.len() as u32;
|
||||
let missing_score_count = scores.len().saturating_sub(rows.len());
|
||||
if missing_score_count > 0 {
|
||||
*self
|
||||
.skip_reason_counts
|
||||
.entry("pool_score_member_missing")
|
||||
.or_insert(0) += u32::try_from(missing_score_count).unwrap_or(u32::MAX);
|
||||
}
|
||||
self.scanned_keys = self.scanned_keys.saturating_add(materialized_row_count);
|
||||
self.budget_scanned_keys = self
|
||||
.budget_scanned_keys
|
||||
.saturating_add(materialized_row_count);
|
||||
Some(self.build_page_eligible_candidates(rows).await)
|
||||
}
|
||||
|
||||
@@ -1044,7 +1054,10 @@ impl<'a> PoolKeyCursor<'a> {
|
||||
}
|
||||
|
||||
fn pool_skip_reason_releases_scan_budget(skip_reason: &str) -> bool {
|
||||
skip_reason == POOL_ACCOUNT_EXHAUSTED_SKIP_REASON
|
||||
matches!(
|
||||
skip_reason,
|
||||
POOL_ACCOUNT_EXHAUSTED_SKIP_REASON | POOL_ACCOUNT_BLOCKED_SKIP_REASON
|
||||
)
|
||||
}
|
||||
|
||||
fn pool_candidate_transport_policy_facts(
|
||||
@@ -1587,7 +1600,8 @@ mod tests {
|
||||
ROUTING_PROFILE_DISALLOWED_KEY_SKIP_REASON,
|
||||
};
|
||||
use crate::ai_serving::{
|
||||
apply_local_runtime_candidate_terminal_reason, EligibleLocalExecutionCandidate,
|
||||
apply_local_runtime_candidate_terminal_reason, provider_key_pool_score_id,
|
||||
provider_key_pool_score_scope, EligibleLocalExecutionCandidate,
|
||||
LocalExecutionCandidateKind, PlannerAppState,
|
||||
};
|
||||
use crate::data::GatewayDataState;
|
||||
@@ -1597,10 +1611,14 @@ mod tests {
|
||||
use crate::orchestration::LocalExecutionCandidateMetadata;
|
||||
use crate::{AppState, LocalExecutionRuntimeMissDiagnostic};
|
||||
use aether_data::repository::candidate_selection::InMemoryMinimalCandidateSelectionReadRepository;
|
||||
use aether_data::repository::pool_scores::InMemoryPoolMemberScoreRepository;
|
||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
use aether_data_contracts::repository::candidate_selection::{
|
||||
StoredMinimalCandidateSelectionRow, StoredPoolKeyCandidateOrder,
|
||||
};
|
||||
use aether_data_contracts::repository::pool_scores::{
|
||||
PoolMemberHardState, PoolMemberIdentity, PoolMemberProbeStatus, StoredPoolMemberScore,
|
||||
};
|
||||
use aether_data_contracts::repository::provider_catalog::{
|
||||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
|
||||
};
|
||||
@@ -3082,6 +3100,131 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_key_cursor_does_not_spend_effective_scan_budget_on_blocked_accounts() {
|
||||
const BLOCKED_COUNT: usize = 1_600;
|
||||
let provider_config = Some(json!({ "pool_advanced": {} }));
|
||||
let (provider, endpoint, mut keys, rows) =
|
||||
large_pool_fixture(BLOCKED_COUNT + 100, provider_config.clone());
|
||||
for key in keys.iter_mut().take(BLOCKED_COUNT) {
|
||||
key.oauth_invalid_reason = Some("blocked account".to_string());
|
||||
}
|
||||
|
||||
let data_state =
|
||||
GatewayDataState::with_provider_catalog_and_minimal_candidate_selection_for_tests(
|
||||
Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||
vec![provider],
|
||||
vec![endpoint],
|
||||
keys,
|
||||
)),
|
||||
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(rows)),
|
||||
)
|
||||
.with_encryption_key_for_tests(aether_crypto::DEVELOPMENT_ENCRYPTION_KEY);
|
||||
let app = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(data_state);
|
||||
let group = sample_eligible_candidate(
|
||||
"provider-pool",
|
||||
"endpoint-1",
|
||||
"pool-group",
|
||||
10,
|
||||
provider_config,
|
||||
);
|
||||
|
||||
let mut cursor = PoolKeyCursor::new(PlannerAppState::new(&app), group, None, None, None);
|
||||
assert_eq!(
|
||||
cursor.max_scanned_keys,
|
||||
aether_dispatch_core::DEFAULT_POOL_MAX_SCAN
|
||||
);
|
||||
assert!(
|
||||
cursor.absolute_max_scanned_keys >= u32::try_from(BLOCKED_COUNT + 1).unwrap(),
|
||||
"default absolute scan cap should allow scanning past a large blocked prefix"
|
||||
);
|
||||
|
||||
let candidate = cursor
|
||||
.next_key()
|
||||
.await
|
||||
.expect("cursor should scan past blocked accounts within the absolute cap");
|
||||
let key_index = candidate
|
||||
.candidate
|
||||
.key_id
|
||||
.strip_prefix("key-")
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.expect("fixture key id should contain a numeric suffix");
|
||||
assert!(
|
||||
key_index >= BLOCKED_COUNT,
|
||||
"cursor should not return one of the blocked leading keys"
|
||||
);
|
||||
assert_eq!(candidate.orchestration.pool_key_index, Some(0));
|
||||
assert!(
|
||||
cursor.budget_scanned_keys <= aether_dispatch_core::DEFAULT_POOL_PAGE_SIZE,
|
||||
"blocked accounts should not consume effective scan budget"
|
||||
);
|
||||
assert_eq!(
|
||||
cursor
|
||||
.skip_reason_counts
|
||||
.get(aether_pool_core::POOL_ACCOUNT_BLOCKED_SKIP_REASON),
|
||||
Some(&(BLOCKED_COUNT as u32))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_key_cursor_does_not_spend_scan_budget_on_missing_score_rows() {
|
||||
let provider_config = Some(json!({
|
||||
"pool_advanced": {
|
||||
"score_top_n": 128
|
||||
}
|
||||
}));
|
||||
let (provider, endpoint, keys, rows) = large_pool_fixture(1, provider_config.clone());
|
||||
let scores = (0..128)
|
||||
.map(|index| {
|
||||
sample_provider_key_pool_score(
|
||||
"provider-pool",
|
||||
&format!("missing-key-{index:03}"),
|
||||
1_000.0 - index as f64,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let data_state =
|
||||
GatewayDataState::with_provider_catalog_and_minimal_candidate_selection_for_tests(
|
||||
Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||
vec![provider],
|
||||
vec![endpoint],
|
||||
keys,
|
||||
)),
|
||||
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(rows)),
|
||||
)
|
||||
.with_pool_score_repository_for_tests(Arc::new(
|
||||
InMemoryPoolMemberScoreRepository::seed(scores),
|
||||
))
|
||||
.with_encryption_key_for_tests(aether_crypto::DEVELOPMENT_ENCRYPTION_KEY);
|
||||
let app = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(data_state);
|
||||
let group = sample_eligible_candidate(
|
||||
"provider-pool",
|
||||
"endpoint-1",
|
||||
"pool-group",
|
||||
10,
|
||||
provider_config,
|
||||
);
|
||||
|
||||
let mut cursor = PoolKeyCursor::new(PlannerAppState::new(&app), group, None, None, None);
|
||||
let candidate = cursor
|
||||
.next_key()
|
||||
.await
|
||||
.expect("cursor should fall back to catalog rows after stale scores");
|
||||
|
||||
assert_eq!(candidate.candidate.key_id, "key-00000");
|
||||
assert_eq!(cursor.scanned_keys, 1);
|
||||
assert_eq!(cursor.budget_scanned_keys, 1);
|
||||
assert_eq!(
|
||||
cursor.skip_reason_counts.get("pool_score_member_missing"),
|
||||
Some(&128)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_scheduler_skips_invalid_and_exhausted_high_priority_hot_pool_before_fallback_provider(
|
||||
) {
|
||||
@@ -3766,6 +3909,40 @@ mod tests {
|
||||
(provider, endpoint, keys, rows)
|
||||
}
|
||||
|
||||
fn sample_provider_key_pool_score(
|
||||
provider_id: &str,
|
||||
key_id: &str,
|
||||
score: f64,
|
||||
) -> StoredPoolMemberScore {
|
||||
let identity = PoolMemberIdentity::provider_api_key(provider_id, key_id);
|
||||
let scope = provider_key_pool_score_scope();
|
||||
StoredPoolMemberScore {
|
||||
id: provider_key_pool_score_id(&identity, &scope),
|
||||
pool_kind: identity.pool_kind,
|
||||
pool_id: identity.pool_id,
|
||||
member_kind: identity.member_kind,
|
||||
member_id: identity.member_id,
|
||||
capability: scope.capability,
|
||||
scope_kind: scope.scope_kind,
|
||||
scope_id: scope.scope_id,
|
||||
score,
|
||||
hard_state: PoolMemberHardState::Available,
|
||||
score_version: 1,
|
||||
score_reason: json!({}),
|
||||
last_ranked_at: Some(1_000),
|
||||
last_scheduled_at: None,
|
||||
last_success_at: None,
|
||||
last_failure_at: None,
|
||||
failure_count: 0,
|
||||
last_probe_attempt_at: None,
|
||||
last_probe_success_at: None,
|
||||
last_probe_failure_at: None,
|
||||
probe_failure_count: 0,
|
||||
probe_status: PoolMemberProbeStatus::Ok,
|
||||
updated_at: 1_000,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_codex_pool_provider(
|
||||
provider_id: &str,
|
||||
provider_priority: i32,
|
||||
|
||||
@@ -208,6 +208,79 @@ async fn admin_monitoring_trace_request_resolves_usage_request_id_to_metadata_tr
|
||||
assert_eq!(payload["candidates"][0]["id"], json!("cand-used"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_trace_request_falls_back_to_usage_routing_snapshot() {
|
||||
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||
vec![sample_provider()],
|
||||
vec![sample_endpoint()],
|
||||
vec![sample_key()],
|
||||
));
|
||||
let mut usage = sample_usage(
|
||||
"request-usage-snapshot",
|
||||
"provider-1",
|
||||
"OpenAI",
|
||||
0,
|
||||
0.0,
|
||||
"failed",
|
||||
Some(503),
|
||||
100,
|
||||
);
|
||||
usage.candidate_id = Some("routing-cand-1".to_string());
|
||||
usage.candidate_index = Some(0);
|
||||
usage.planner_kind = Some("openai_responses_stream".to_string());
|
||||
usage.execution_path = Some("local_execution_runtime_miss".to_string());
|
||||
usage.local_execution_runtime_miss_reason = Some("no_local_stream_plans".to_string());
|
||||
usage.api_format = Some("openai:responses".to_string());
|
||||
usage.endpoint_api_format = Some("openai:responses".to_string());
|
||||
usage.provider_api_key_id = Some("provider-key-1".to_string());
|
||||
usage.error_message = Some("no local stream plans".to_string());
|
||||
usage.response_time_ms = Some(45);
|
||||
|
||||
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![usage]));
|
||||
let data_state =
|
||||
crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
|
||||
request_candidates,
|
||||
usage_repository,
|
||||
)
|
||||
.with_provider_catalog_reader(provider_catalog);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(data_state);
|
||||
let context = request_context(
|
||||
http::Method::GET,
|
||||
"/api/admin/monitoring/trace/request-usage-snapshot?attempted_only=true",
|
||||
);
|
||||
|
||||
let response = local_monitoring_response(&state, &context)
|
||||
.await
|
||||
.expect("handler should not error")
|
||||
.expect("route should be handled locally");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||
assert_eq!(payload["request_id"], json!("request-usage-snapshot"));
|
||||
assert_eq!(payload["total_candidates"], json!(1));
|
||||
assert_eq!(payload["final_status"], json!("failed"));
|
||||
assert_eq!(payload["candidates"][0]["id"], json!("routing-cand-1"));
|
||||
assert_eq!(payload["candidates"][0]["status"], json!("failed"));
|
||||
assert_eq!(
|
||||
payload["candidates"][0]["error_type"],
|
||||
json!("no_local_stream_plans")
|
||||
);
|
||||
assert_eq!(
|
||||
payload["candidates"][0]["extra_data"]["source"],
|
||||
json!("usage_routing_snapshot")
|
||||
);
|
||||
assert_eq!(
|
||||
payload["candidates"][0]["extra_data"]["execution_path"],
|
||||
json!("local_execution_runtime_miss")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_monitoring_trace_request_returns_oauth_account_label_from_auth_config() {
|
||||
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
|
||||
|
||||
@@ -10,7 +10,10 @@ use aether_admin::observability::monitoring::{
|
||||
parse_admin_monitoring_attempted_only, AdminMonitoringKeyAccountDisplay,
|
||||
};
|
||||
use aether_data_contracts::repository::{
|
||||
candidates::{DecisionTrace, RequestCandidateStatus},
|
||||
candidates::{
|
||||
DecisionTrace, DecisionTraceCandidate, RequestCandidateFinalStatus, RequestCandidateStatus,
|
||||
StoredRequestCandidate,
|
||||
},
|
||||
provider_catalog::StoredProviderCatalogKey,
|
||||
usage::StoredRequestUsageAudit,
|
||||
};
|
||||
@@ -18,7 +21,7 @@ use axum::{
|
||||
body::Body,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use serde_json::{Map, Value};
|
||||
use serde_json::{json, Map, Value};
|
||||
use std::collections::BTreeMap;
|
||||
use tracing::debug;
|
||||
|
||||
@@ -107,6 +110,7 @@ async fn resolve_admin_monitoring_trace(
|
||||
}
|
||||
}
|
||||
|
||||
let mut usage_snapshot_fallback = None;
|
||||
for usage in usage_candidates {
|
||||
for trace_request_id in admin_monitoring_usage_trace_request_ids(&usage) {
|
||||
if trace_request_id == request_id {
|
||||
@@ -124,9 +128,211 @@ async fn resolve_admin_monitoring_trace(
|
||||
}));
|
||||
}
|
||||
}
|
||||
if usage_snapshot_fallback.is_none() {
|
||||
if let Some(trace) = build_admin_monitoring_usage_routing_snapshot_trace(&usage) {
|
||||
usage_snapshot_fallback = Some(ResolvedAdminMonitoringTrace {
|
||||
trace,
|
||||
usage: Some(usage.clone()),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
Ok(usage_snapshot_fallback)
|
||||
}
|
||||
|
||||
fn build_admin_monitoring_usage_routing_snapshot_trace(
|
||||
usage: &StoredRequestUsageAudit,
|
||||
) -> Option<DecisionTrace> {
|
||||
if !admin_monitoring_usage_has_routing_snapshot_trace_data(usage) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let status = admin_monitoring_usage_candidate_status(usage);
|
||||
let final_status = match status {
|
||||
RequestCandidateStatus::Success => RequestCandidateFinalStatus::Success,
|
||||
RequestCandidateStatus::Cancelled => RequestCandidateFinalStatus::Cancelled,
|
||||
RequestCandidateStatus::Streaming => RequestCandidateFinalStatus::Streaming,
|
||||
RequestCandidateStatus::Pending => RequestCandidateFinalStatus::Pending,
|
||||
RequestCandidateStatus::Available
|
||||
| RequestCandidateStatus::Unused
|
||||
| RequestCandidateStatus::Failed
|
||||
| RequestCandidateStatus::Skipped => RequestCandidateFinalStatus::Failed,
|
||||
};
|
||||
let latency_ms = usage.response_time_ms.unwrap_or_default();
|
||||
let candidate = StoredRequestCandidate {
|
||||
id: usage
|
||||
.routing_candidate_id()
|
||||
.map(ToOwned::to_owned)
|
||||
.unwrap_or_else(|| format!("usage-routing-snapshot:{}", usage.id)),
|
||||
request_id: admin_monitoring_usage_primary_trace_request_id(usage),
|
||||
user_id: usage.user_id.clone(),
|
||||
api_key_id: usage.api_key_id.clone(),
|
||||
username: usage.username.clone(),
|
||||
api_key_name: usage.api_key_name.clone(),
|
||||
candidate_index: usage
|
||||
.routing_candidate_index()
|
||||
.and_then(|value| u32::try_from(value).ok())
|
||||
.unwrap_or(0),
|
||||
retry_index: 0,
|
||||
provider_id: usage.provider_id.clone(),
|
||||
endpoint_id: usage.provider_endpoint_id.clone(),
|
||||
key_id: usage.provider_api_key_id.clone(),
|
||||
status,
|
||||
skip_reason: None,
|
||||
is_cached: false,
|
||||
status_code: usage.status_code,
|
||||
error_type: usage
|
||||
.routing_local_execution_runtime_miss_reason()
|
||||
.or(usage.error_category.as_deref())
|
||||
.map(ToOwned::to_owned),
|
||||
error_message: usage.error_message.clone(),
|
||||
latency_ms: usage.response_time_ms,
|
||||
concurrent_requests: None,
|
||||
extra_data: build_admin_monitoring_usage_routing_snapshot_extra_data(usage),
|
||||
required_capabilities: None,
|
||||
created_at_unix_ms: usage.created_at_unix_ms,
|
||||
started_at_unix_ms: Some(usage.created_at_unix_ms),
|
||||
finished_at_unix_ms: admin_monitoring_usage_finished_at_unix_ms(usage),
|
||||
};
|
||||
|
||||
Some(DecisionTrace {
|
||||
request_id: candidate.request_id.clone(),
|
||||
total_candidates: 1,
|
||||
final_status,
|
||||
total_latency_ms: latency_ms,
|
||||
candidates: vec![DecisionTraceCandidate {
|
||||
candidate,
|
||||
provider_name: non_empty_string(usage.provider_name.as_str()),
|
||||
provider_website: None,
|
||||
provider_type: None,
|
||||
provider_priority: None,
|
||||
provider_keep_priority_on_conversion: None,
|
||||
provider_enable_format_conversion: None,
|
||||
endpoint_api_format: usage
|
||||
.endpoint_api_format
|
||||
.clone()
|
||||
.or_else(|| usage.api_format.clone()),
|
||||
endpoint_api_family: usage
|
||||
.provider_api_family
|
||||
.clone()
|
||||
.or_else(|| usage.api_family.clone()),
|
||||
endpoint_kind: usage
|
||||
.provider_endpoint_kind
|
||||
.clone()
|
||||
.or_else(|| usage.endpoint_kind.clone()),
|
||||
endpoint_format_acceptance_config: None,
|
||||
provider_key_name: usage.routing_key_name().map(ToOwned::to_owned),
|
||||
provider_key_auth_type: None,
|
||||
provider_key_api_formats: None,
|
||||
provider_key_internal_priority: None,
|
||||
provider_key_global_priority_by_format: None,
|
||||
provider_key_capabilities: None,
|
||||
provider_key_is_active: None,
|
||||
}],
|
||||
})
|
||||
}
|
||||
|
||||
fn admin_monitoring_usage_has_routing_snapshot_trace_data(usage: &StoredRequestUsageAudit) -> bool {
|
||||
usage.routing_candidate_id().is_some()
|
||||
|| usage.routing_candidate_index().is_some()
|
||||
|| usage.routing_execution_path().is_some()
|
||||
|| usage
|
||||
.routing_local_execution_runtime_miss_reason()
|
||||
.is_some()
|
||||
}
|
||||
|
||||
fn admin_monitoring_usage_candidate_status(
|
||||
usage: &StoredRequestUsageAudit,
|
||||
) -> RequestCandidateStatus {
|
||||
if usage.status.trim().eq_ignore_ascii_case("cancelled")
|
||||
|| usage.status.trim().eq_ignore_ascii_case("canceled")
|
||||
{
|
||||
return RequestCandidateStatus::Cancelled;
|
||||
}
|
||||
|
||||
match usage.status_code {
|
||||
Some(status_code) if (200..300).contains(&status_code) => RequestCandidateStatus::Success,
|
||||
Some(_) => RequestCandidateStatus::Failed,
|
||||
None if usage.status.trim().eq_ignore_ascii_case("completed")
|
||||
|| usage.status.trim().eq_ignore_ascii_case("success") =>
|
||||
{
|
||||
RequestCandidateStatus::Success
|
||||
}
|
||||
None => RequestCandidateStatus::Failed,
|
||||
}
|
||||
}
|
||||
|
||||
fn admin_monitoring_usage_primary_trace_request_id(usage: &StoredRequestUsageAudit) -> String {
|
||||
if let Some(trace_id) = usage.trace_id() {
|
||||
return trace_id.to_string();
|
||||
}
|
||||
if let Some(trace_id) = usage_trace_id_from_headers(usage.request_headers.as_ref()) {
|
||||
return trace_id;
|
||||
}
|
||||
if let Some(trace_id) = usage_trace_id_from_headers(usage.provider_request_headers.as_ref()) {
|
||||
return trace_id;
|
||||
}
|
||||
usage.request_id.clone()
|
||||
}
|
||||
|
||||
fn admin_monitoring_usage_finished_at_unix_ms(usage: &StoredRequestUsageAudit) -> Option<u64> {
|
||||
usage
|
||||
.finalized_at_unix_secs
|
||||
.map(|value| value.saturating_mul(1_000))
|
||||
.or_else(|| {
|
||||
usage
|
||||
.response_time_ms
|
||||
.map(|latency_ms| usage.created_at_unix_ms.saturating_add(latency_ms))
|
||||
})
|
||||
}
|
||||
|
||||
fn build_admin_monitoring_usage_routing_snapshot_extra_data(
|
||||
usage: &StoredRequestUsageAudit,
|
||||
) -> Option<Value> {
|
||||
let mut object = Map::new();
|
||||
object.insert("source".to_string(), json!("usage_routing_snapshot"));
|
||||
insert_optional_string(&mut object, "planner_kind", usage.routing_planner_kind());
|
||||
insert_optional_string(&mut object, "route_family", usage.routing_route_family());
|
||||
insert_optional_string(&mut object, "route_kind", usage.routing_route_kind());
|
||||
insert_optional_string(
|
||||
&mut object,
|
||||
"execution_path",
|
||||
usage.routing_execution_path(),
|
||||
);
|
||||
insert_optional_string(
|
||||
&mut object,
|
||||
"local_execution_runtime_miss_reason",
|
||||
usage.routing_local_execution_runtime_miss_reason(),
|
||||
);
|
||||
insert_optional_string(&mut object, "key_name", usage.routing_key_name());
|
||||
insert_optional_string(&mut object, "model", Some(usage.model.as_str()));
|
||||
insert_optional_string(&mut object, "target_model", usage.target_model.as_deref());
|
||||
insert_optional_string(
|
||||
&mut object,
|
||||
"client_api_format",
|
||||
usage.api_format.as_deref(),
|
||||
);
|
||||
insert_optional_string(
|
||||
&mut object,
|
||||
"provider_api_format",
|
||||
usage.endpoint_api_format.as_deref(),
|
||||
);
|
||||
insert_optional_string(
|
||||
&mut object,
|
||||
"provider_api_family",
|
||||
usage.provider_api_family.as_deref(),
|
||||
);
|
||||
insert_optional_string(
|
||||
&mut object,
|
||||
"provider_endpoint_kind",
|
||||
usage.provider_endpoint_kind.as_deref(),
|
||||
);
|
||||
insert_optional_string(&mut object, "candidate_id", usage.routing_candidate_id());
|
||||
if let Some(candidate_index) = usage.routing_candidate_index() {
|
||||
object.insert("candidate_index".to_string(), json!(candidate_index));
|
||||
}
|
||||
Some(Value::Object(object))
|
||||
}
|
||||
|
||||
fn admin_monitoring_usage_trace_request_ids(usage: &StoredRequestUsageAudit) -> Vec<String> {
|
||||
@@ -167,6 +373,18 @@ fn push_non_empty_unique(values: &mut Vec<String>, value: &str) {
|
||||
values.push(value.to_string());
|
||||
}
|
||||
|
||||
fn non_empty_string(value: &str) -> Option<String> {
|
||||
let value = value.trim();
|
||||
(!value.is_empty()).then(|| value.to_string())
|
||||
}
|
||||
|
||||
fn insert_optional_string(object: &mut Map<String, Value>, key: &str, value: Option<&str>) {
|
||||
let Some(value) = value.and_then(non_empty_string) else {
|
||||
return;
|
||||
};
|
||||
object.insert(key.to_string(), Value::String(value));
|
||||
}
|
||||
|
||||
async fn build_admin_monitoring_key_account_display_map(
|
||||
state: &AdminAppState<'_>,
|
||||
trace: &DecisionTrace,
|
||||
|
||||
@@ -414,7 +414,7 @@ pub(crate) fn admin_provider_pool_config_from_config_value(
|
||||
account_self_check_interval_minutes: 60,
|
||||
account_self_check_concurrency: 4,
|
||||
score_top_n: 128,
|
||||
score_fallback_scan_limit: 1024,
|
||||
score_fallback_scan_limit: 4096,
|
||||
score_rules: PoolMemberScoreRules::default(),
|
||||
stream_timeout_threshold: 3,
|
||||
stream_timeout_window_seconds: 1800,
|
||||
@@ -512,7 +512,7 @@ pub(crate) fn admin_provider_pool_config_from_config_value(
|
||||
.and_then(json_u64)
|
||||
.filter(|value| *value > 0)
|
||||
.map(|value| value.min(50_000))
|
||||
.unwrap_or(1024),
|
||||
.unwrap_or(4096),
|
||||
score_rules,
|
||||
stream_timeout_threshold: pool_advanced
|
||||
.get("stream_timeout_threshold")
|
||||
|
||||
@@ -646,7 +646,7 @@ mod tests {
|
||||
account_self_check_interval_minutes: 60,
|
||||
account_self_check_concurrency: 4,
|
||||
score_top_n: 128,
|
||||
score_fallback_scan_limit: 1024,
|
||||
score_fallback_scan_limit: 4096,
|
||||
score_rules: aether_pool_core::PoolMemberScoreRules::default(),
|
||||
stream_timeout_threshold: 3,
|
||||
stream_timeout_window_seconds: 1800,
|
||||
|
||||
@@ -190,16 +190,26 @@ pub(crate) fn snapshot_local_request_candidate_status(
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())?;
|
||||
let metadata = parse_request_candidate_report_context(report_context)?;
|
||||
let candidate_index = metadata.candidate_index.unwrap_or(0);
|
||||
let metadata = parse_request_candidate_report_context(report_context);
|
||||
let candidate_index = metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| metadata.candidate_index)
|
||||
.unwrap_or(0);
|
||||
|
||||
Some(LocalRequestCandidateStatusSnapshot {
|
||||
candidate_id: candidate_id.to_string(),
|
||||
request_id: plan.request_id.clone(),
|
||||
user_id: metadata.user_id,
|
||||
api_key_id: metadata.api_key_id,
|
||||
user_id: metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| metadata.user_id.clone()),
|
||||
api_key_id: metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| metadata.api_key_id.clone()),
|
||||
candidate_index,
|
||||
retry_index: metadata.retry_index,
|
||||
retry_index: metadata
|
||||
.as_ref()
|
||||
.map(|metadata| metadata.retry_index)
|
||||
.unwrap_or(0),
|
||||
provider_id: plan.provider_id.clone(),
|
||||
endpoint_id: plan.endpoint_id.clone(),
|
||||
key_id: plan.key_id.clone(),
|
||||
@@ -438,11 +448,16 @@ pub(crate) async fn ensure_execution_request_candidate_slot(
|
||||
);
|
||||
return;
|
||||
}
|
||||
if plan
|
||||
let existing_candidate_id = plan
|
||||
.candidate_id
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.is_some_and(|value| !value.is_empty())
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned);
|
||||
let report_candidate_id = parse_request_candidate_report_context(report_context.as_ref())
|
||||
.and_then(|metadata| metadata.candidate_id);
|
||||
if existing_candidate_id.as_deref().is_some()
|
||||
&& report_candidate_id.as_deref() == existing_candidate_id.as_deref()
|
||||
{
|
||||
return;
|
||||
}
|
||||
@@ -451,7 +466,7 @@ pub(crate) async fn ensure_execution_request_candidate_slot(
|
||||
plan,
|
||||
report_context.as_ref(),
|
||||
current_unix_ms(),
|
||||
Uuid::new_v4().to_string(),
|
||||
existing_candidate_id.unwrap_or_else(|| Uuid::new_v4().to_string()),
|
||||
);
|
||||
let generated_candidate_id = seed.upsert_record.id.clone();
|
||||
let request_id = short_request_id(plan.request_id.as_str());
|
||||
@@ -879,13 +894,15 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn does_not_reseed_execution_request_candidate_slot_when_plan_already_has_candidate_id() {
|
||||
async fn does_not_reseed_execution_request_candidate_slot_when_report_context_matches_plan_candidate_id(
|
||||
) {
|
||||
let repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||
let state = build_test_state(Arc::clone(&repository));
|
||||
let mut plan = sample_plan();
|
||||
plan.candidate_id = Some("cand-existing-123".to_string());
|
||||
let mut report_context = Some(json!({
|
||||
"request_id": "req-request-candidate-seed-123"
|
||||
"request_id": "req-request-candidate-seed-123",
|
||||
"candidate_id": "cand-existing-123"
|
||||
}));
|
||||
|
||||
ensure_execution_request_candidate_slot(&state, &mut plan, &mut report_context).await;
|
||||
@@ -901,10 +918,37 @@ mod tests {
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("candidate_id"))
|
||||
.and_then(|value| value.as_str()),
|
||||
None
|
||||
Some("cand-existing-123")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn seeds_execution_request_candidate_slot_when_plan_candidate_id_lacks_report_context() {
|
||||
let repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||
let state = build_test_state(Arc::clone(&repository));
|
||||
let mut plan = sample_plan();
|
||||
plan.candidate_id = Some("cand-existing-123".to_string());
|
||||
let mut report_context = None;
|
||||
|
||||
ensure_execution_request_candidate_slot(&state, &mut plan, &mut report_context).await;
|
||||
|
||||
assert_eq!(plan.candidate_id.as_deref(), Some("cand-existing-123"));
|
||||
let report_context = report_context.expect("report context should be populated");
|
||||
assert_eq!(
|
||||
report_context
|
||||
.get("candidate_id")
|
||||
.and_then(|value| value.as_str()),
|
||||
Some("cand-existing-123")
|
||||
);
|
||||
let stored = repository
|
||||
.list_by_request_id("req-request-candidate-seed-123")
|
||||
.await
|
||||
.expect("request candidates should read");
|
||||
assert_eq!(stored.len(), 1);
|
||||
assert_eq!(stored[0].id, "cand-existing-123");
|
||||
assert_eq!(stored[0].status, RequestCandidateStatus::Pending);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn records_report_request_candidate_status_for_existing_slot() {
|
||||
let repository = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
|
||||
|
||||
Reference in New Issue
Block a user