Unify candidate ranking pipeline

This commit is contained in:
fawney19
2026-04-27 17:24:31 +08:00
parent 9b866a6d17
commit 3b542434a2
46 changed files with 3635 additions and 1914 deletions
File diff suppressed because it is too large Load Diff
@@ -1,393 +1 @@
use std::sync::Arc;
use aether_provider_transport::provider_types::provider_type_is_fixed;
use tracing::warn;
use aether_scheduler_core::SchedulerMinimalCandidateSelectionCandidate;
use crate::ai_pipeline::{GatewayProviderTransportSnapshot, PlannerAppState};
use crate::orchestration::LocalExecutionCandidateMetadata;
use super::candidate_affinity::rank_eligible_local_execution_candidates;
use super::pool_scheduler::apply_local_execution_pool_scheduler;
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct EligibleLocalExecutionCandidate {
pub(crate) candidate: SchedulerMinimalCandidateSelectionCandidate,
pub(crate) transport: Arc<GatewayProviderTransportSnapshot>,
pub(crate) provider_api_format: String,
pub(crate) orchestration: LocalExecutionCandidateMetadata,
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct SkippedLocalExecutionCandidate {
pub(crate) candidate: SchedulerMinimalCandidateSelectionCandidate,
pub(crate) skip_reason: &'static str,
pub(crate) transport: Option<Arc<GatewayProviderTransportSnapshot>>,
pub(crate) extra_data: Option<serde_json::Value>,
}
impl SkippedLocalExecutionCandidate {
pub(crate) fn transport_ref(&self) -> Option<&GatewayProviderTransportSnapshot> {
self.transport.as_deref()
}
}
pub(crate) async fn filter_and_rank_local_execution_candidates(
state: PlannerAppState<'_>,
candidates: Vec<SchedulerMinimalCandidateSelectionCandidate>,
client_api_format: &str,
requested_model: &str,
required_capabilities: Option<&serde_json::Value>,
sticky_session_token: Option<&str>,
) -> (
Vec<EligibleLocalExecutionCandidate>,
Vec<SkippedLocalExecutionCandidate>,
) {
let requested_model = requested_model.trim();
filter_and_rank_local_execution_candidates_with_gate(
state,
candidates,
client_api_format,
required_capabilities,
sticky_session_token,
|candidate, transport, normalized_client_api_format| {
current_local_execution_candidate_skip_reason_with_transport(
candidate,
transport,
normalized_client_api_format,
requested_model,
)
},
)
.await
}
pub(crate) async fn filter_and_rank_local_execution_candidates_without_transport_pair_gate(
state: PlannerAppState<'_>,
candidates: Vec<SchedulerMinimalCandidateSelectionCandidate>,
client_api_format: &str,
requested_model: Option<&str>,
required_capabilities: Option<&serde_json::Value>,
sticky_session_token: Option<&str>,
) -> (
Vec<EligibleLocalExecutionCandidate>,
Vec<SkippedLocalExecutionCandidate>,
) {
let requested_model = requested_model.map(str::trim);
filter_and_rank_local_execution_candidates_with_gate(
state,
candidates,
client_api_format,
required_capabilities,
sticky_session_token,
|candidate, transport, _normalized_client_api_format| {
current_local_execution_candidate_common_skip_reason_with_transport(
candidate,
transport,
requested_model,
)
},
)
.await
}
async fn filter_and_rank_local_execution_candidates_with_gate<F>(
state: PlannerAppState<'_>,
candidates: Vec<SchedulerMinimalCandidateSelectionCandidate>,
client_api_format: &str,
required_capabilities: Option<&serde_json::Value>,
sticky_session_token: Option<&str>,
runtime_skip_reason: F,
) -> (
Vec<EligibleLocalExecutionCandidate>,
Vec<SkippedLocalExecutionCandidate>,
)
where
F: Fn(
&SchedulerMinimalCandidateSelectionCandidate,
&GatewayProviderTransportSnapshot,
&str,
) -> Option<&'static str>,
{
let normalized_client_api_format = client_api_format.trim().to_ascii_lowercase();
let mut selectable = Vec::with_capacity(candidates.len());
let mut skipped = Vec::with_capacity(candidates.len());
for candidate in candidates {
let Some(transport) = read_candidate_transport_snapshot(state, &candidate).await else {
skipped.push(SkippedLocalExecutionCandidate {
candidate,
skip_reason: "transport_snapshot_missing",
transport: None,
extra_data: None,
});
continue;
};
let transport = Arc::new(transport);
if candidate_is_ineligible_due_to_disabled_format_conversion(
transport.as_ref(),
normalized_client_api_format.as_str(),
) {
continue;
}
match runtime_skip_reason(
&candidate,
transport.as_ref(),
normalized_client_api_format.as_str(),
) {
Some(skip_reason) => skipped.push(SkippedLocalExecutionCandidate {
candidate,
skip_reason,
transport: Some(transport),
extra_data: None,
}),
None => selectable.push(EligibleLocalExecutionCandidate {
provider_api_format: transport.endpoint.api_format.trim().to_ascii_lowercase(),
candidate,
transport,
orchestration: LocalExecutionCandidateMetadata::default(),
}),
}
}
let ranked = rank_eligible_local_execution_candidates(
state,
selectable,
normalized_client_api_format.as_str(),
required_capabilities,
)
.await;
let (ranked, pool_skipped) =
apply_local_execution_pool_scheduler(state, ranked, sticky_session_token).await;
skipped.extend(pool_skipped);
(ranked, skipped)
}
pub(crate) fn extract_pool_sticky_session_token(body_json: &serde_json::Value) -> Option<String> {
fn non_empty_str(value: Option<&serde_json::Value>) -> Option<&str> {
value
.and_then(serde_json::Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
}
let object = body_json.as_object()?;
non_empty_str(object.get("prompt_cache_key"))
.or_else(|| non_empty_str(object.get("conversation_id")))
.or_else(|| non_empty_str(object.get("conversationId")))
.or_else(|| non_empty_str(object.get("session_id")))
.or_else(|| non_empty_str(object.get("sessionId")))
.or_else(|| {
object
.get("metadata")
.and_then(serde_json::Value::as_object)
.and_then(|metadata| {
non_empty_str(metadata.get("session_id"))
.or_else(|| non_empty_str(metadata.get("conversation_id")))
})
})
.or_else(|| {
object
.get("conversationState")
.and_then(serde_json::Value::as_object)
.and_then(|state| {
non_empty_str(state.get("conversationId"))
.or_else(|| non_empty_str(state.get("sessionId")))
})
})
.map(ToOwned::to_owned)
}
fn current_local_execution_candidate_common_skip_reason_with_transport(
candidate: &SchedulerMinimalCandidateSelectionCandidate,
transport: &GatewayProviderTransportSnapshot,
requested_model: Option<&str>,
) -> Option<&'static str> {
let requested_model = requested_model.unwrap_or_default();
if !transport.provider.is_active {
return Some("provider_inactive");
}
if !transport.endpoint.is_active {
return Some("endpoint_inactive");
}
if !transport.key.is_active {
return Some("key_inactive");
}
let endpoint_api_format = transport.endpoint.api_format.trim();
if !candidate
.endpoint_api_format
.trim()
.eq_ignore_ascii_case(endpoint_api_format)
&& !api_format_matches(&candidate.endpoint_api_format, endpoint_api_format)
{
return Some("endpoint_api_format_changed");
}
if !transport_key_supports_api_format(transport, endpoint_api_format) {
return Some("key_api_format_disabled");
}
if !transport_key_allows_candidate_model(transport, requested_model, candidate) {
return Some("key_model_disabled");
}
None
}
fn candidate_is_ineligible_due_to_disabled_format_conversion(
transport: &GatewayProviderTransportSnapshot,
normalized_client_api_format: &str,
) -> bool {
let endpoint_api_format = transport.endpoint.api_format.trim();
if api_format_matches(endpoint_api_format, normalized_client_api_format) {
return false;
}
crate::ai_pipeline::conversion::request_conversion_kind(
normalized_client_api_format,
endpoint_api_format,
)
.is_some()
&& crate::ai_pipeline::conversion::request_conversion_requires_enable_flag(
normalized_client_api_format,
endpoint_api_format,
)
&& !crate::ai_pipeline::conversion::request_conversion_enabled_for_transport(
transport,
normalized_client_api_format,
endpoint_api_format,
)
}
fn current_local_execution_candidate_skip_reason_with_transport(
candidate: &SchedulerMinimalCandidateSelectionCandidate,
transport: &GatewayProviderTransportSnapshot,
normalized_client_api_format: &str,
requested_model: &str,
) -> Option<&'static str> {
if let Some(skip_reason) = current_local_execution_candidate_common_skip_reason_with_transport(
candidate,
transport,
Some(requested_model),
) {
return Some(skip_reason);
}
let endpoint_api_format = transport.endpoint.api_format.trim();
if api_format_matches(endpoint_api_format, normalized_client_api_format) {
return None;
}
if !crate::ai_pipeline::conversion::request_pair_allowed_for_transport(
transport,
normalized_client_api_format,
endpoint_api_format,
) {
return Some("transport_unsupported");
}
None
}
fn transport_key_supports_api_format(
transport: &GatewayProviderTransportSnapshot,
endpoint_api_format: &str,
) -> bool {
let provider_type = transport.provider.provider_type.trim();
let auth_type = transport.key.auth_type.trim();
let inherits_provider_api_formats = provider_type_is_fixed(provider_type)
&& (auth_type.eq_ignore_ascii_case("oauth")
|| (provider_type.eq_ignore_ascii_case("kiro")
&& auth_type.eq_ignore_ascii_case("bearer")
&& transport
.key
.decrypted_auth_config
.as_deref()
.map(str::trim)
.is_some_and(|value| !value.is_empty())));
if inherits_provider_api_formats {
return true;
}
match transport.key.api_formats.as_deref() {
None => true,
Some(formats) => formats
.iter()
.any(|value| api_format_matches(value, endpoint_api_format)),
}
}
fn normalize_api_format_alias(value: &str) -> String {
crate::ai_pipeline::normalize_legacy_openai_format_alias(value)
}
fn api_format_matches(left: &str, right: &str) -> bool {
normalize_api_format_alias(left) == normalize_api_format_alias(right)
}
fn transport_key_allows_candidate_model(
transport: &GatewayProviderTransportSnapshot,
requested_model: &str,
candidate: &SchedulerMinimalCandidateSelectionCandidate,
) -> bool {
let Some(allowed_models) = transport.key.allowed_models.as_deref() else {
return true;
};
let requested_model = requested_model.trim();
let global_model_name = candidate.global_model_name.trim();
let selected_provider_model_name = candidate.selected_provider_model_name.trim();
let mapping_matched_model = candidate
.mapping_matched_model
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty());
for allowed_model in allowed_models.iter().map(String::as_str).map(str::trim) {
if allowed_model.is_empty() {
continue;
}
if allowed_model == requested_model
|| allowed_model == global_model_name
|| allowed_model == selected_provider_model_name
|| mapping_matched_model.is_some_and(|value| value == allowed_model)
{
return true;
}
}
false
}
pub(crate) async fn read_candidate_transport_snapshot(
state: PlannerAppState<'_>,
candidate: &SchedulerMinimalCandidateSelectionCandidate,
) -> Option<GatewayProviderTransportSnapshot> {
match state
.read_provider_transport_snapshot(
&candidate.provider_id,
&candidate.endpoint_id,
&candidate.key_id,
)
.await
{
Ok(Some(transport)) => Some(transport),
Ok(None) => None,
Err(error) => {
warn!(
event_name = "candidate_eligibility_transport_load_failed",
log_type = "event",
provider_id = %candidate.provider_id,
endpoint_id = %candidate.endpoint_id,
key_id = %candidate.key_id,
error = ?error,
"failed to load provider transport while evaluating local candidate eligibility"
);
None
}
}
}
pub(crate) use super::candidate_resolution::*;
@@ -1,9 +1,10 @@
use aether_scheduler_core::SchedulerMinimalCandidateSelectionCandidate;
use aether_scheduler_core::SchedulerRankingOutcome;
use serde_json::Value;
use uuid::Uuid;
use crate::ai_pipeline::planner::candidate_affinity::remember_scheduler_affinity_for_candidate;
use crate::ai_pipeline::planner::candidate_eligibility::{
use crate::ai_pipeline::planner::candidate_resolution::{
EligibleLocalExecutionCandidate, SkippedLocalExecutionCandidate,
};
use crate::ai_pipeline::planner::failure_diagnostic::CandidateFailureDiagnostic;
@@ -70,7 +71,10 @@ pub(crate) fn remember_first_local_candidate_affinity(
}
fn should_persist_available_local_candidate(eligible: &EligibleLocalExecutionCandidate) -> bool {
eligible.orchestration.pool_key_index.is_none()
eligible
.orchestration
.pool_key_index
.is_none_or(|index| index == 0)
}
fn should_persist_skipped_local_candidate(candidate: &SkippedLocalExecutionCandidate) -> bool {
@@ -104,7 +108,10 @@ where
let candidate_index = candidate_index as u32;
let attempt_slots = local_attempt_slot_count(&eligible.transport);
let pool_key_index = eligible.orchestration.pool_key_index;
let extra_data = build_extra_data(&eligible);
let extra_data = local_candidate_extra_data_with_ranking(
build_extra_data(&eligible),
eligible.ranking.as_ref(),
);
let mut owned_eligible = Some(eligible);
for retry_index in 0..attempt_slots {
@@ -153,6 +160,54 @@ where
materialized
}
fn local_candidate_extra_data_with_ranking(
extra_data: Option<Value>,
ranking: Option<&SchedulerRankingOutcome>,
) -> Option<Value> {
let Some(ranking) = ranking else {
return extra_data;
};
let mut object = match extra_data {
Some(Value::Object(object)) => object,
Some(value) => {
let mut object = serde_json::Map::new();
object.insert("extra".to_string(), value);
object
}
None => serde_json::Map::new(),
};
object.insert(
"ranking_mode".to_string(),
Value::String(format!("{:?}", ranking.ranking_mode)),
);
object.insert(
"priority_mode".to_string(),
Value::String(format!("{:?}", ranking.priority_mode)),
);
object.insert(
"ranking_index".to_string(),
Value::Number(serde_json::Number::from(ranking.ranking_index as u64)),
);
object.insert(
"priority_slot".to_string(),
Value::Number(serde_json::Number::from(i64::from(ranking.priority_slot))),
);
if let Some(promoted_by) = ranking.promoted_by {
object.insert(
"promoted_by".to_string(),
Value::String(promoted_by.to_string()),
);
}
if let Some(demoted_by) = ranking.demoted_by {
object.insert(
"demoted_by".to_string(),
Value::String(demoted_by.to_string()),
);
}
Some(Value::Object(object))
}
pub(crate) async fn persist_available_local_execution_candidates_with_context<F>(
state: PlannerAppState<'_>,
trace_id: &str,
@@ -356,7 +411,10 @@ mod tests {
GatewayProviderTransportEndpoint, GatewayProviderTransportKey,
GatewayProviderTransportProvider,
};
use aether_scheduler_core::SchedulerMinimalCandidateSelectionCandidate;
use aether_scheduler_core::{
SchedulerMinimalCandidateSelectionCandidate, SchedulerPriorityMode, SchedulerRankingMode,
SchedulerRankingOutcome,
};
use serde_json::json;
use super::*;
@@ -456,11 +514,12 @@ mod tests {
candidate_group_id: pool_key_index.map(|_| "pool-group".to_string()),
pool_key_index,
},
ranking: None,
}
}
#[tokio::test]
async fn pool_candidates_are_not_persisted_as_available_before_attempt() {
async fn pool_group_representatives_are_persisted_as_available_before_attempt() {
let repository = Arc::new(InMemoryRequestCandidateRepository::default());
let app = AppState::new()
.expect("state should build")
@@ -478,6 +537,7 @@ mod tests {
None,
vec![
sample_eligible("pool-key", Some(0)),
sample_eligible("pool-key-internal", Some(1)),
sample_eligible("normal-key", None),
],
"persist should not fail",
@@ -485,13 +545,74 @@ mod tests {
)
.await;
assert_eq!(attempts.len(), 2);
assert_eq!(attempts.len(), 3);
let stored = app
.read_request_candidates_by_request_id("trace-pool-lazy")
.await
.expect("request candidates should read");
assert_eq!(stored.len(), 2);
assert_eq!(stored[0].key_id.as_deref(), Some("pool-key"));
assert_eq!(stored[0].candidate_index, 0);
assert_eq!(stored[1].key_id.as_deref(), Some("normal-key"));
assert_eq!(stored[1].candidate_index, 2);
}
#[tokio::test]
async fn available_candidates_persist_ranking_metadata_in_extra_data() {
let repository = Arc::new(InMemoryRequestCandidateRepository::default());
let app = AppState::new()
.expect("state should build")
.with_data_state_for_tests(
GatewayDataState::with_request_candidate_repository_for_tests(Arc::clone(
&repository,
)),
);
let mut eligible = sample_eligible("ranked-key", None);
eligible.ranking = Some(SchedulerRankingOutcome {
original_index: 1,
ranking_index: 0,
priority_mode: SchedulerPriorityMode::Provider,
ranking_mode: SchedulerRankingMode::CacheAffinity,
priority_slot: 7,
promoted_by: Some("cached_affinity"),
demoted_by: Some("cross_format"),
});
persist_available_local_execution_candidates(
PlannerAppState::new(&app),
"trace-ranking-extra-data",
"user-1",
"api-key-1",
None,
vec![eligible],
"persist should not fail",
|_| Some(json!({ "existing": "value" })),
)
.await;
let stored = app
.read_request_candidates_by_request_id("trace-ranking-extra-data")
.await
.expect("request candidates should read");
assert_eq!(stored.len(), 1);
assert_eq!(stored[0].key_id.as_deref(), Some("normal-key"));
let extra_data = stored[0]
.extra_data
.as_ref()
.and_then(serde_json::Value::as_object)
.expect("ranking metadata should persist as object extra data");
assert_eq!(extra_data.get("existing"), Some(&json!("value")));
assert_eq!(
extra_data.get("ranking_mode"),
Some(&json!("CacheAffinity"))
);
assert_eq!(extra_data.get("priority_mode"), Some(&json!("Provider")));
assert_eq!(extra_data.get("ranking_index"), Some(&json!(0)));
assert_eq!(extra_data.get("priority_slot"), Some(&json!(7)));
assert_eq!(
extra_data.get("promoted_by"),
Some(&json!("cached_affinity"))
);
assert_eq!(extra_data.get("demoted_by"), Some(&json!("cross_format")));
}
#[tokio::test]
@@ -2,7 +2,7 @@ use aether_contracts::ProxySnapshot;
use aether_scheduler_core::SchedulerMinimalCandidateSelectionCandidate;
use serde_json::{json, Map, Value};
use crate::ai_pipeline::planner::candidate_eligibility::EligibleLocalExecutionCandidate;
use crate::ai_pipeline::planner::candidate_resolution::EligibleLocalExecutionCandidate;
use crate::ai_pipeline::planner::passthrough::resolve_same_format_provider_transport_unsupported_reason_for_trace;
use crate::ai_pipeline::transport::{
body_rules_are_locally_supported, header_rules_are_locally_supported,
@@ -0,0 +1,414 @@
use std::sync::Arc;
use aether_provider_transport::provider_types::provider_type_is_fixed;
use tracing::warn;
use aether_scheduler_core::{SchedulerMinimalCandidateSelectionCandidate, SchedulerRankingOutcome};
use crate::ai_pipeline::{
GatewayAuthApiKeySnapshot, GatewayProviderTransportSnapshot, PlannerAppState,
};
use crate::orchestration::LocalExecutionCandidateMetadata;
use super::candidate_affinity::rank_eligible_local_execution_candidates;
use super::pool_scheduler::apply_local_execution_pool_scheduler;
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct EligibleLocalExecutionCandidate {
pub(crate) candidate: SchedulerMinimalCandidateSelectionCandidate,
pub(crate) transport: Arc<GatewayProviderTransportSnapshot>,
pub(crate) provider_api_format: String,
pub(crate) orchestration: LocalExecutionCandidateMetadata,
pub(crate) ranking: Option<SchedulerRankingOutcome>,
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct SkippedLocalExecutionCandidate {
pub(crate) candidate: SchedulerMinimalCandidateSelectionCandidate,
pub(crate) skip_reason: &'static str,
pub(crate) transport: Option<Arc<GatewayProviderTransportSnapshot>>,
pub(crate) extra_data: Option<serde_json::Value>,
}
impl SkippedLocalExecutionCandidate {
pub(crate) fn transport_ref(&self) -> Option<&GatewayProviderTransportSnapshot> {
self.transport.as_deref()
}
}
pub(crate) async fn filter_and_rank_local_execution_candidates(
state: PlannerAppState<'_>,
candidates: Vec<SchedulerMinimalCandidateSelectionCandidate>,
client_api_format: &str,
requested_model: &str,
auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
required_capabilities: Option<&serde_json::Value>,
sticky_session_token: Option<&str>,
) -> (
Vec<EligibleLocalExecutionCandidate>,
Vec<SkippedLocalExecutionCandidate>,
) {
let requested_model = requested_model.trim();
filter_and_rank_local_execution_candidates_with_gate(
state,
candidates,
client_api_format,
Some(requested_model),
auth_snapshot,
required_capabilities,
sticky_session_token,
|candidate, transport, normalized_client_api_format| {
current_local_execution_candidate_skip_reason_with_transport(
candidate,
transport,
normalized_client_api_format,
requested_model,
)
},
)
.await
}
pub(crate) async fn filter_and_rank_local_execution_candidates_without_transport_pair_gate(
state: PlannerAppState<'_>,
candidates: Vec<SchedulerMinimalCandidateSelectionCandidate>,
client_api_format: &str,
requested_model: Option<&str>,
auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
required_capabilities: Option<&serde_json::Value>,
sticky_session_token: Option<&str>,
) -> (
Vec<EligibleLocalExecutionCandidate>,
Vec<SkippedLocalExecutionCandidate>,
) {
let requested_model = requested_model.map(str::trim);
filter_and_rank_local_execution_candidates_with_gate(
state,
candidates,
client_api_format,
requested_model,
auth_snapshot,
required_capabilities,
sticky_session_token,
|candidate, transport, _normalized_client_api_format| {
current_local_execution_candidate_common_skip_reason_with_transport(
candidate,
transport,
requested_model,
)
},
)
.await
}
async fn filter_and_rank_local_execution_candidates_with_gate<F>(
state: PlannerAppState<'_>,
candidates: Vec<SchedulerMinimalCandidateSelectionCandidate>,
client_api_format: &str,
requested_model: Option<&str>,
auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
required_capabilities: Option<&serde_json::Value>,
sticky_session_token: Option<&str>,
runtime_skip_reason: F,
) -> (
Vec<EligibleLocalExecutionCandidate>,
Vec<SkippedLocalExecutionCandidate>,
)
where
F: Fn(
&SchedulerMinimalCandidateSelectionCandidate,
&GatewayProviderTransportSnapshot,
&str,
) -> Option<&'static str>,
{
let normalized_client_api_format = client_api_format.trim().to_ascii_lowercase();
let mut selectable = Vec::with_capacity(candidates.len());
let mut skipped = Vec::with_capacity(candidates.len());
for candidate in candidates {
let Some(transport) = read_candidate_transport_snapshot(state, &candidate).await else {
skipped.push(SkippedLocalExecutionCandidate {
candidate,
skip_reason: "transport_snapshot_missing",
transport: None,
extra_data: None,
});
continue;
};
let transport = Arc::new(transport);
match runtime_skip_reason(
&candidate,
transport.as_ref(),
normalized_client_api_format.as_str(),
) {
Some(skip_reason) => skipped.push(SkippedLocalExecutionCandidate {
candidate,
skip_reason,
transport: Some(transport),
extra_data: None,
}),
None => selectable.push(EligibleLocalExecutionCandidate {
provider_api_format: transport.endpoint.api_format.trim().to_ascii_lowercase(),
candidate,
transport,
orchestration: LocalExecutionCandidateMetadata::default(),
ranking: None,
}),
}
}
let ranked = rank_eligible_local_execution_candidates(
state,
selectable,
normalized_client_api_format.as_str(),
requested_model,
auth_snapshot,
required_capabilities,
)
.await;
let (ranked, pool_skipped) =
apply_local_execution_pool_scheduler(state, ranked, sticky_session_token).await;
skipped.extend(pool_skipped);
(ranked, skipped)
}
pub(crate) fn extract_pool_sticky_session_token(body_json: &serde_json::Value) -> Option<String> {
fn non_empty_str(value: Option<&serde_json::Value>) -> Option<&str> {
value
.and_then(serde_json::Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
}
let object = body_json.as_object()?;
non_empty_str(object.get("prompt_cache_key"))
.or_else(|| non_empty_str(object.get("conversation_id")))
.or_else(|| non_empty_str(object.get("conversationId")))
.or_else(|| non_empty_str(object.get("session_id")))
.or_else(|| non_empty_str(object.get("sessionId")))
.or_else(|| {
object
.get("metadata")
.and_then(serde_json::Value::as_object)
.and_then(|metadata| {
non_empty_str(metadata.get("session_id"))
.or_else(|| non_empty_str(metadata.get("conversation_id")))
})
})
.or_else(|| {
object
.get("conversationState")
.and_then(serde_json::Value::as_object)
.and_then(|state| {
non_empty_str(state.get("conversationId"))
.or_else(|| non_empty_str(state.get("sessionId")))
})
})
.map(ToOwned::to_owned)
}
fn current_local_execution_candidate_common_skip_reason_with_transport(
candidate: &SchedulerMinimalCandidateSelectionCandidate,
transport: &GatewayProviderTransportSnapshot,
requested_model: Option<&str>,
) -> Option<&'static str> {
let requested_model = requested_model.unwrap_or_default();
if !transport.provider.is_active {
return Some("provider_inactive");
}
if !transport.endpoint.is_active {
return Some("endpoint_inactive");
}
if !transport.key.is_active {
return Some("key_inactive");
}
let endpoint_api_format = transport.endpoint.api_format.trim();
if !candidate
.endpoint_api_format
.trim()
.eq_ignore_ascii_case(endpoint_api_format)
&& !api_format_matches(&candidate.endpoint_api_format, endpoint_api_format)
{
return Some("endpoint_api_format_changed");
}
if !transport_key_supports_api_format(transport, endpoint_api_format) {
return Some("key_api_format_disabled");
}
if !transport_key_allows_candidate_model(transport, requested_model, candidate) {
return Some("key_model_disabled");
}
None
}
fn disabled_format_conversion_skip_reason(
transport: &GatewayProviderTransportSnapshot,
normalized_client_api_format: &str,
) -> Option<&'static str> {
let endpoint_api_format = transport.endpoint.api_format.trim();
if api_format_matches(endpoint_api_format, normalized_client_api_format) {
return None;
}
if crate::ai_pipeline::conversion::request_conversion_kind(
normalized_client_api_format,
endpoint_api_format,
)
.is_none()
{
return None;
}
if crate::ai_pipeline::conversion::request_conversion_requires_enable_flag(
normalized_client_api_format,
endpoint_api_format,
) && !crate::ai_pipeline::conversion::request_conversion_enabled_for_transport(
transport,
normalized_client_api_format,
endpoint_api_format,
) {
return Some("format_conversion_disabled");
}
None
}
fn current_local_execution_candidate_skip_reason_with_transport(
candidate: &SchedulerMinimalCandidateSelectionCandidate,
transport: &GatewayProviderTransportSnapshot,
normalized_client_api_format: &str,
requested_model: &str,
) -> Option<&'static str> {
if let Some(skip_reason) = current_local_execution_candidate_common_skip_reason_with_transport(
candidate,
transport,
Some(requested_model),
) {
return Some(skip_reason);
}
let endpoint_api_format = transport.endpoint.api_format.trim();
if api_format_matches(endpoint_api_format, normalized_client_api_format) {
return None;
}
if let Some(skip_reason) =
disabled_format_conversion_skip_reason(transport, normalized_client_api_format)
{
return Some(skip_reason);
}
if !crate::ai_pipeline::conversion::request_pair_allowed_for_transport(
transport,
normalized_client_api_format,
endpoint_api_format,
) {
return Some("transport_unsupported");
}
None
}
fn transport_key_supports_api_format(
transport: &GatewayProviderTransportSnapshot,
endpoint_api_format: &str,
) -> bool {
let provider_type = transport.provider.provider_type.trim();
let auth_type = transport.key.auth_type.trim();
let inherits_provider_api_formats = provider_type_is_fixed(provider_type)
&& (auth_type.eq_ignore_ascii_case("oauth")
|| (provider_type.eq_ignore_ascii_case("kiro")
&& auth_type.eq_ignore_ascii_case("bearer")
&& transport
.key
.decrypted_auth_config
.as_deref()
.map(str::trim)
.is_some_and(|value| !value.is_empty())));
if inherits_provider_api_formats {
return true;
}
match transport.key.api_formats.as_deref() {
None => true,
Some(formats) => formats
.iter()
.any(|value| api_format_matches(value, endpoint_api_format)),
}
}
fn normalize_api_format_alias(value: &str) -> String {
crate::ai_pipeline::normalize_legacy_openai_format_alias(value)
}
fn api_format_matches(left: &str, right: &str) -> bool {
normalize_api_format_alias(left) == normalize_api_format_alias(right)
}
fn transport_key_allows_candidate_model(
transport: &GatewayProviderTransportSnapshot,
requested_model: &str,
candidate: &SchedulerMinimalCandidateSelectionCandidate,
) -> bool {
let Some(allowed_models) = transport.key.allowed_models.as_deref() else {
return true;
};
let requested_model = requested_model.trim();
let global_model_name = candidate.global_model_name.trim();
let selected_provider_model_name = candidate.selected_provider_model_name.trim();
let mapping_matched_model = candidate
.mapping_matched_model
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty());
for allowed_model in allowed_models.iter().map(String::as_str).map(str::trim) {
if allowed_model.is_empty() {
continue;
}
if allowed_model == requested_model
|| allowed_model == global_model_name
|| allowed_model == selected_provider_model_name
|| mapping_matched_model.is_some_and(|value| value == allowed_model)
{
return true;
}
}
false
}
pub(crate) async fn read_candidate_transport_snapshot(
state: PlannerAppState<'_>,
candidate: &SchedulerMinimalCandidateSelectionCandidate,
) -> Option<GatewayProviderTransportSnapshot> {
match state
.read_provider_transport_snapshot(
&candidate.provider_id,
&candidate.endpoint_id,
&candidate.key_id,
)
.await
{
Ok(Some(transport)) => Some(transport),
Ok(None) => None,
Err(error) => {
warn!(
event_name = "candidate_resolution_transport_load_failed",
log_type = "event",
provider_id = %candidate.provider_id,
endpoint_id = %candidate.endpoint_id,
key_id = %candidate.key_id,
error = ?error,
"failed to load provider transport while evaluating local candidate eligibility"
);
None
}
}
}
@@ -9,6 +9,7 @@ mod candidate_eligibility;
mod candidate_materialization;
mod candidate_metadata;
mod candidate_preparation;
mod candidate_resolution;
mod candidate_source;
mod common;
mod decision;
@@ -27,7 +28,7 @@ mod specialized;
mod standard;
mod state;
pub(crate) use self::candidate_eligibility::extract_pool_sticky_session_token;
pub(crate) use self::candidate_resolution::extract_pool_sticky_session_token;
pub(crate) use self::failure_diagnostic::{
CandidateFailureDiagnostic, CandidateFailureDiagnosticKind,
};
@@ -1,9 +1,5 @@
use tracing::warn;
use crate::ai_pipeline::planner::candidate_eligibility::{
extract_pool_sticky_session_token, filter_and_rank_local_execution_candidates,
SkippedLocalExecutionCandidate,
};
use crate::ai_pipeline::planner::candidate_materialization::{
persist_available_local_execution_candidates_with_context,
persist_skipped_local_execution_candidates_with_context,
@@ -14,6 +10,10 @@ use crate::ai_pipeline::planner::candidate_metadata::{
build_local_execution_candidate_contract_metadata_for_candidate,
LocalExecutionCandidateMetadataParts,
};
use crate::ai_pipeline::planner::candidate_resolution::{
extract_pool_sticky_session_token, filter_and_rank_local_execution_candidates,
SkippedLocalExecutionCandidate,
};
use crate::ai_pipeline::planner::common::extract_requested_model_from_request;
use crate::ai_pipeline::planner::decision_input::{
build_local_requested_model_decision_input, resolve_local_authenticated_decision_input,
@@ -112,6 +112,7 @@ pub(crate) async fn materialize_local_same_format_provider_candidate_attempts(
candidates,
spec_metadata.api_format,
&input.requested_model,
Some(&input.auth_snapshot),
input.required_capabilities.as_ref(),
sticky_session_token.as_deref(),
)
@@ -1,9 +1,9 @@
use std::sync::Arc;
use crate::ai_pipeline::planner::candidate_eligibility::EligibleLocalExecutionCandidate;
use crate::ai_pipeline::planner::candidate_preparation::{
resolve_candidate_mapped_model, resolve_candidate_oauth_auth, OauthPreparationContext,
};
use crate::ai_pipeline::planner::candidate_resolution::EligibleLocalExecutionCandidate;
use crate::ai_pipeline::planner::spec_metadata::local_same_format_provider_spec_metadata;
use crate::ai_pipeline::transport::kiro::KiroRequestAuth;
use crate::ai_pipeline::transport::vertex::resolve_local_vertex_api_key_query_auth;
@@ -7,7 +7,7 @@ use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKe
use serde_json::{Map, Value};
use tracing::warn;
use crate::ai_pipeline::planner::candidate_eligibility::{
use crate::ai_pipeline::planner::candidate_resolution::{
EligibleLocalExecutionCandidate, SkippedLocalExecutionCandidate,
};
use crate::ai_pipeline::PlannerAppState;
@@ -434,6 +434,7 @@ fn schedule_pool_group(
transport,
provider_api_format,
orchestration,
ranking,
} = eligible;
let key_id = candidate.key_id.clone();
let mut key_context = key_context_by_id.get(&key_id).cloned().unwrap_or_default();
@@ -495,6 +496,7 @@ fn schedule_pool_group(
transport,
provider_api_format,
orchestration,
ranking,
},
key_context,
original_index,
@@ -1040,7 +1042,7 @@ mod tests {
apply_local_execution_pool_scheduler_with_runtime_map, build_pool_catalog_key_context,
normalize_enabled_pool_presets, PoolCatalogKeyContext,
};
use crate::ai_pipeline::planner::candidate_eligibility::EligibleLocalExecutionCandidate;
use crate::ai_pipeline::planner::candidate_resolution::EligibleLocalExecutionCandidate;
use crate::ai_pipeline::PlannerAppState;
use crate::data::GatewayDataState;
use crate::handlers::shared::provider_pool::{
@@ -1836,6 +1838,7 @@ mod tests {
},
provider_api_format: "openai:chat".to_string(),
orchestration: LocalExecutionCandidateMetadata::default(),
ranking: None,
transport: Arc::new(crate::ai_pipeline::GatewayProviderTransportSnapshot {
provider: GatewayProviderTransportProvider {
id: provider_id.to_string(),
@@ -3,7 +3,6 @@ use serde_json::json;
use tracing::warn;
use crate::ai_pipeline::contracts::ExecutionRuntimeAuthContext;
use crate::ai_pipeline::planner::candidate_eligibility::filter_and_rank_local_execution_candidates_without_transport_pair_gate;
use crate::ai_pipeline::planner::candidate_materialization::{
mark_skipped_local_execution_candidate,
mark_skipped_local_execution_candidate_with_failure_diagnostic,
@@ -15,6 +14,7 @@ use crate::ai_pipeline::planner::candidate_metadata::{
build_local_execution_candidate_metadata,
build_local_execution_candidate_metadata_for_candidate, LocalExecutionCandidateMetadataParts,
};
use crate::ai_pipeline::planner::candidate_resolution::filter_and_rank_local_execution_candidates_without_transport_pair_gate;
use crate::ai_pipeline::planner::decision_input::{
build_local_authenticated_decision_input, resolve_local_authenticated_decision_input,
};
@@ -95,6 +95,7 @@ pub(super) async fn materialize_local_gemini_files_candidate_attempts(
candidates,
GEMINI_FILES_CLIENT_API_FORMAT,
None,
Some(&input.auth_snapshot),
input.required_capabilities.as_ref(),
None,
)
@@ -1,10 +1,6 @@
use tracing::warn;
use crate::ai_pipeline::contracts::ExecutionRuntimeAuthContext;
use crate::ai_pipeline::planner::candidate_eligibility::{
extract_pool_sticky_session_token, filter_and_rank_local_execution_candidates,
SkippedLocalExecutionCandidate,
};
use crate::ai_pipeline::planner::candidate_materialization::{
mark_skipped_local_execution_candidate,
mark_skipped_local_execution_candidate_with_failure_diagnostic,
@@ -16,6 +12,10 @@ use crate::ai_pipeline::planner::candidate_metadata::{
build_local_execution_candidate_metadata,
build_local_execution_candidate_metadata_for_candidate, LocalExecutionCandidateMetadataParts,
};
use crate::ai_pipeline::planner::candidate_resolution::{
extract_pool_sticky_session_token, filter_and_rank_local_execution_candidates,
SkippedLocalExecutionCandidate,
};
use crate::ai_pipeline::planner::decision_input::{
build_local_requested_model_decision_input, resolve_local_authenticated_decision_input,
};
@@ -157,6 +157,7 @@ async fn materialize_local_openai_image_candidate_attempts(
candidates,
api_format,
&input.requested_model,
Some(&input.auth_snapshot),
input.required_capabilities.as_ref(),
sticky_session_token.as_deref(),
)
@@ -3,10 +3,6 @@ use tracing::warn;
use super::{LocalVideoCreateFamily, LocalVideoCreateSpec};
use crate::ai_pipeline::contracts::ExecutionRuntimeAuthContext;
use crate::ai_pipeline::planner::candidate_eligibility::{
extract_pool_sticky_session_token, filter_and_rank_local_execution_candidates,
SkippedLocalExecutionCandidate,
};
use crate::ai_pipeline::planner::candidate_materialization::{
mark_skipped_local_execution_candidate,
mark_skipped_local_execution_candidate_with_failure_diagnostic,
@@ -18,6 +14,10 @@ use crate::ai_pipeline::planner::candidate_metadata::{
build_local_execution_candidate_metadata,
build_local_execution_candidate_metadata_for_candidate, LocalExecutionCandidateMetadataParts,
};
use crate::ai_pipeline::planner::candidate_resolution::{
extract_pool_sticky_session_token, filter_and_rank_local_execution_candidates,
SkippedLocalExecutionCandidate,
};
use crate::ai_pipeline::planner::common::extract_requested_model_from_request;
use crate::ai_pipeline::planner::decision_input::{
build_local_requested_model_decision_input, resolve_local_authenticated_decision_input,
@@ -169,6 +169,7 @@ async fn materialize_local_video_create_candidate_attempts(
candidates,
api_format,
&input.requested_model,
Some(&input.auth_snapshot),
input.required_capabilities.as_ref(),
sticky_session_token.as_deref(),
)
@@ -3,10 +3,6 @@ use std::collections::BTreeSet;
use tracing::warn;
use crate::ai_pipeline::conversion::{request_candidate_api_formats, request_conversion_kind};
use crate::ai_pipeline::planner::candidate_eligibility::{
extract_pool_sticky_session_token, filter_and_rank_local_execution_candidates,
SkippedLocalExecutionCandidate,
};
use crate::ai_pipeline::planner::candidate_materialization::{
persist_available_local_execution_candidates_with_context,
persist_skipped_local_execution_candidates_with_context,
@@ -17,6 +13,10 @@ use crate::ai_pipeline::planner::candidate_metadata::{
build_local_execution_candidate_contract_metadata_for_candidate,
LocalExecutionCandidateMetadataParts,
};
use crate::ai_pipeline::planner::candidate_resolution::{
extract_pool_sticky_session_token, filter_and_rank_local_execution_candidates,
SkippedLocalExecutionCandidate,
};
use crate::ai_pipeline::planner::candidate_source::auth_snapshot_allows_cross_format_candidate;
use crate::ai_pipeline::planner::common::extract_requested_model_from_request;
use crate::ai_pipeline::planner::decision_input::{
@@ -178,6 +178,7 @@ pub(super) async fn materialize_local_standard_candidate_attempts(
candidates,
spec_metadata.api_format,
&input.requested_model,
Some(&input.auth_snapshot),
input.required_capabilities.as_ref(),
sticky_session_token.as_deref(),
)
@@ -4,10 +4,10 @@ use std::sync::Arc;
use serde_json::Value;
use crate::ai_pipeline::conversion::{request_conversion_direct_auth, request_conversion_kind};
use crate::ai_pipeline::planner::candidate_eligibility::EligibleLocalExecutionCandidate;
use crate::ai_pipeline::planner::candidate_preparation::{
prepare_header_authenticated_candidate, OauthPreparationContext,
};
use crate::ai_pipeline::planner::candidate_resolution::EligibleLocalExecutionCandidate;
use crate::ai_pipeline::planner::common::OPENAI_CHAT_STREAM_PLAN_KIND;
use crate::ai_pipeline::planner::standard::{
apply_codex_openai_responses_special_headers, build_cross_format_openai_chat_request_body,
@@ -1,10 +1,6 @@
use aether_scheduler_core::SchedulerMinimalCandidateSelectionCandidate;
use crate::ai_pipeline::contracts::ExecutionRuntimeAuthContext;
use crate::ai_pipeline::planner::candidate_eligibility::{
extract_pool_sticky_session_token, filter_and_rank_local_execution_candidates,
SkippedLocalExecutionCandidate,
};
use crate::ai_pipeline::planner::candidate_materialization::{
mark_skipped_local_execution_candidate, mark_skipped_local_execution_candidate_with_extra_data,
mark_skipped_local_execution_candidate_with_failure_diagnostic,
@@ -17,6 +13,10 @@ use crate::ai_pipeline::planner::candidate_metadata::{
build_local_execution_candidate_contract_metadata_for_candidate,
LocalExecutionCandidateMetadataParts,
};
use crate::ai_pipeline::planner::candidate_resolution::{
extract_pool_sticky_session_token, filter_and_rank_local_execution_candidates,
SkippedLocalExecutionCandidate,
};
use crate::ai_pipeline::planner::materialization_policy::{
build_local_candidate_persistence_policy, LocalCandidatePersistencePolicyKind,
};
@@ -135,6 +135,7 @@ pub(crate) async fn materialize_local_openai_chat_candidate_attempts(
candidates,
"openai:chat",
&input.requested_model,
Some(&input.auth_snapshot),
input.required_capabilities.as_ref(),
sticky_session_token.as_deref(),
)
@@ -4,7 +4,7 @@ use aether_scheduler_core::SchedulerMinimalCandidateSelectionCandidate;
use super::super::{GatewayError, LocalOpenAiChatDecisionInput};
use crate::ai_pipeline::conversion::request_candidate_api_formats;
use crate::ai_pipeline::planner::candidate_eligibility::SkippedLocalExecutionCandidate;
use crate::ai_pipeline::planner::candidate_resolution::SkippedLocalExecutionCandidate;
use crate::ai_pipeline::planner::candidate_source::auth_snapshot_allows_cross_format_candidate;
use crate::ai_pipeline::PlannerAppState;
use crate::clock::current_unix_secs;
@@ -5,10 +5,10 @@ use serde_json::Value;
use tracing::debug;
use crate::ai_pipeline::conversion::{request_conversion_direct_auth, request_conversion_kind};
use crate::ai_pipeline::planner::candidate_eligibility::EligibleLocalExecutionCandidate;
use crate::ai_pipeline::planner::candidate_preparation::{
prepare_header_authenticated_candidate, OauthPreparationContext,
};
use crate::ai_pipeline::planner::candidate_resolution::EligibleLocalExecutionCandidate;
use crate::ai_pipeline::planner::common::force_upstream_streaming_for_provider;
use crate::ai_pipeline::planner::spec_metadata::local_openai_responses_spec_metadata;
use crate::ai_pipeline::planner::standard::{
@@ -5,10 +5,6 @@ use tracing::warn;
use crate::ai_pipeline::contracts::ExecutionRuntimeAuthContext;
use crate::ai_pipeline::conversion::{request_candidate_api_formats, request_conversion_kind};
use crate::ai_pipeline::planner::candidate_eligibility::{
extract_pool_sticky_session_token, filter_and_rank_local_execution_candidates,
SkippedLocalExecutionCandidate,
};
use crate::ai_pipeline::planner::candidate_materialization::{
mark_skipped_local_execution_candidate, mark_skipped_local_execution_candidate_with_extra_data,
mark_skipped_local_execution_candidate_with_failure_diagnostic,
@@ -21,6 +17,10 @@ use crate::ai_pipeline::planner::candidate_metadata::{
build_local_execution_candidate_contract_metadata_for_candidate,
LocalExecutionCandidateMetadataParts,
};
use crate::ai_pipeline::planner::candidate_resolution::{
extract_pool_sticky_session_token, filter_and_rank_local_execution_candidates,
SkippedLocalExecutionCandidate,
};
use crate::ai_pipeline::planner::candidate_source::auth_snapshot_allows_cross_format_candidate;
use crate::ai_pipeline::planner::common::extract_standard_requested_model;
use crate::ai_pipeline::planner::decision_input::{
@@ -234,6 +234,7 @@ pub(crate) async fn materialize_local_openai_responses_candidate_attempts(
candidates,
spec_metadata.api_format,
&input.requested_model,
Some(&input.auth_snapshot),
input.required_capabilities.as_ref(),
sticky_session_token.as_deref(),
)