mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-08 10:27:46 +08:00
feat(routing): make routing profiles the sole scheduler policy source
Bootstrap an enabled system-default routing group from the legacy scheduler config keys on startup, resolve the default ordering config from that group before falling back to the legacy keys, and stop merging keep_priority_on_conversion with the legacy flag when a policy is resolved. Thread the policy-derived ordering config into candidate preselection so it no longer reads system config independently. Add per-API-format key priority overrides so a key serving several formats keeps independent ordering, matching the legacy global_priority_by_format semantics. Expose keep_priority_on_conversion in the routing profile editor and read the effective policy in the model routing preview, monitoring metrics and provider page badge.
This commit is contained in:
@@ -1840,6 +1840,7 @@ fn routing_trace_for_candidate(
|
||||
CandidateKind::Provider => Some(candidate.key_id.clone()),
|
||||
CandidateKind::PoolGroup => None,
|
||||
},
|
||||
api_format: Some(candidate.endpoint_api_format.clone()),
|
||||
provider_priority: candidate.provider_priority,
|
||||
key_priority: candidate
|
||||
.key_global_priority_for_format
|
||||
|
||||
@@ -3,7 +3,7 @@ use aether_ai_serving::{
|
||||
AiCandidateRankingPort, AiRankableCandidateParts, AiRankingContextConfig,
|
||||
AiRankingSchedulingMode,
|
||||
};
|
||||
use aether_routing_core::{ResolvedRoutingPolicy, RoutingSchedulingMode, RoutingSetPriorityMode};
|
||||
use aether_routing_core::ResolvedRoutingPolicy;
|
||||
use async_trait::async_trait;
|
||||
use tokio::sync::Mutex;
|
||||
use tracing::warn;
|
||||
@@ -184,35 +184,16 @@ fn ai_ranking_scheduling_mode(mode: SchedulerSchedulingMode) -> AiRankingSchedul
|
||||
}
|
||||
}
|
||||
|
||||
/// Ordering config for a request. A resolved routing policy is authoritative
|
||||
/// and is never merged with legacy system-config values; without a policy the
|
||||
/// effective default (system-default routing group, then legacy keys) applies.
|
||||
pub(crate) async fn scheduler_ordering_config_for_routing_policy(
|
||||
state: PlannerAppState<'_>,
|
||||
routing_policy: Option<&ResolvedRoutingPolicy>,
|
||||
) -> SchedulerOrderingConfig {
|
||||
let system_config = read_scheduler_ordering_config_or_default(state).await;
|
||||
match routing_policy {
|
||||
Some(policy) => {
|
||||
let mut config = scheduler_ordering_config_from_routing_policy(policy);
|
||||
config.keep_priority_on_conversion |= system_config.keep_priority_on_conversion;
|
||||
config
|
||||
}
|
||||
None => system_config,
|
||||
}
|
||||
}
|
||||
|
||||
fn scheduler_ordering_config_from_routing_policy(
|
||||
policy: &ResolvedRoutingPolicy,
|
||||
) -> SchedulerOrderingConfig {
|
||||
SchedulerOrderingConfig {
|
||||
priority_mode: match policy.priority_mode {
|
||||
RoutingSetPriorityMode::Provider => SchedulerPriorityMode::Provider,
|
||||
RoutingSetPriorityMode::GlobalKey => SchedulerPriorityMode::GlobalKey,
|
||||
},
|
||||
scheduling_mode: match policy.scheduling_mode {
|
||||
RoutingSchedulingMode::FixedOrder => SchedulerSchedulingMode::FixedOrder,
|
||||
RoutingSchedulingMode::CacheAffinity => SchedulerSchedulingMode::CacheAffinity,
|
||||
RoutingSchedulingMode::LoadBalance => SchedulerSchedulingMode::LoadBalance,
|
||||
},
|
||||
keep_priority_on_conversion: policy.keep_priority_on_conversion,
|
||||
Some(policy) => SchedulerOrderingConfig::from_routing_policy(policy),
|
||||
None => read_scheduler_ordering_config_or_default(state).await,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -231,14 +212,26 @@ fn routing_overlaid_candidate(
|
||||
let overlaid_key_priority = match kind {
|
||||
LocalExecutionCandidateKind::SingleKey => policy
|
||||
.ranking_overlay
|
||||
.key_priority_overrides
|
||||
.get(candidate.key_id.as_str()),
|
||||
.key_priority_override_matching_format(candidate.key_id.as_str(), |format| {
|
||||
crate::ai_serving::api_format_alias_matches(
|
||||
format,
|
||||
candidate.endpoint_api_format.as_str(),
|
||||
)
|
||||
})
|
||||
.or_else(|| {
|
||||
policy
|
||||
.ranking_overlay
|
||||
.key_priority_overrides
|
||||
.get(candidate.key_id.as_str())
|
||||
.copied()
|
||||
}),
|
||||
LocalExecutionCandidateKind::PoolGroup => policy
|
||||
.ranking_overlay
|
||||
.pool_priority_overrides
|
||||
.get(candidate.provider_id.as_str()),
|
||||
.get(candidate.provider_id.as_str())
|
||||
.copied(),
|
||||
};
|
||||
if let Some(overlaid_key_priority) = overlaid_key_priority.copied() {
|
||||
if let Some(overlaid_key_priority) = overlaid_key_priority {
|
||||
overlaid.key_internal_priority = overlaid_key_priority;
|
||||
overlaid.key_global_priority_for_format = Some(overlaid_key_priority);
|
||||
}
|
||||
@@ -396,7 +389,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn routing_policy_inherits_global_conversion_priority_override() {
|
||||
async fn routing_policy_ignores_legacy_global_conversion_priority_override() {
|
||||
let data_state = GatewayDataState::default().with_system_config_values_for_tests([(
|
||||
"keep_priority_on_conversion".to_string(),
|
||||
json!(true),
|
||||
@@ -429,7 +422,10 @@ mod tests {
|
||||
ordering.scheduling_mode,
|
||||
crate::scheduler::config::SchedulerSchedulingMode::FixedOrder
|
||||
);
|
||||
assert!(ordering.keep_priority_on_conversion);
|
||||
assert!(
|
||||
!ordering.keep_priority_on_conversion,
|
||||
"a resolved routing policy must not inherit the legacy system-config flag"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -174,6 +174,8 @@ impl AiCandidatePreselectionPort for GatewayLocalCandidatePreselectionPort<'_> {
|
||||
self.ranking_seed,
|
||||
false,
|
||||
self.request_operation,
|
||||
self.routing_policy
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -1291,6 +1293,9 @@ impl<'a> LocalCandidatePreselectionPageCursor<'a> {
|
||||
.then_some(self.client_session_affinity.as_ref())
|
||||
.flatten(),
|
||||
self.ranking_seed,
|
||||
self.routing_policy
|
||||
.as_ref()
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
)
|
||||
.await?;
|
||||
let skipped_candidates = skipped_candidates
|
||||
@@ -2657,14 +2662,16 @@ mod tests {
|
||||
candidate_repository,
|
||||
)
|
||||
.with_encryption_key_for_tests("development-key")
|
||||
// Legacy keys deliberately disagree with the routing policy: the
|
||||
// resolved policy must be the only source of scheduler ordering.
|
||||
.with_system_config_values_for_tests([
|
||||
(
|
||||
"scheduling_mode".to_string(),
|
||||
serde_json::json!("fixed_order"),
|
||||
serde_json::json!("cache_affinity"),
|
||||
),
|
||||
(
|
||||
"keep_priority_on_conversion".to_string(),
|
||||
serde_json::json!(true),
|
||||
serde_json::json!(false),
|
||||
),
|
||||
]);
|
||||
let app = AppState::new()
|
||||
@@ -2681,7 +2688,7 @@ mod tests {
|
||||
resolved_model: "gpt-5.4-mini".to_string(),
|
||||
priority_mode: aether_routing_core::RoutingSetPriorityMode::Provider,
|
||||
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
|
||||
keep_priority_on_conversion: false,
|
||||
keep_priority_on_conversion: true,
|
||||
ranking_overlay: Default::default(),
|
||||
mutation_plan: Default::default(),
|
||||
pool_policy_overrides: Default::default(),
|
||||
|
||||
@@ -1073,6 +1073,7 @@ fn ensure_report_context_routing_trace(
|
||||
endpoint_id: decision.endpoint_id.clone().unwrap_or_default(),
|
||||
model_id,
|
||||
key_id,
|
||||
api_format: decision.provider_api_format.clone(),
|
||||
provider_priority,
|
||||
key_priority,
|
||||
},
|
||||
|
||||
@@ -26,6 +26,7 @@ use crate::ai_serving::{
|
||||
};
|
||||
use crate::client_session_affinity::client_session_affinity_from_api_request;
|
||||
use crate::clock::current_unix_secs;
|
||||
use crate::scheduler::config::SchedulerOrderingConfig;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
use super::{
|
||||
@@ -140,6 +141,10 @@ pub(crate) async fn materialize_local_same_format_provider_candidate_attempts(
|
||||
current_unix_secs(),
|
||||
false,
|
||||
spec.operation.map(|operation| operation.as_str()),
|
||||
input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
)
|
||||
.await?;
|
||||
let outcome = materialize_local_execution_candidates_with_serving(
|
||||
@@ -246,6 +251,10 @@ pub(crate) async fn build_local_same_format_provider_candidate_attempt_source<'a
|
||||
current_unix_secs(),
|
||||
false,
|
||||
spec.operation.map(|operation| operation.as_str()),
|
||||
input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ use crate::ai_serving::{
|
||||
};
|
||||
use crate::client_session_affinity::client_session_affinity_from_parts;
|
||||
use crate::clock::current_unix_secs;
|
||||
use crate::scheduler::config::SchedulerOrderingConfig;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
pub(super) use crate::ai_serving::planner::candidate_materialization::LocalExecutionCandidateAttempt as LocalGeminiFilesCandidateAttempt;
|
||||
@@ -108,6 +109,10 @@ pub(super) async fn materialize_local_gemini_files_candidate_attempts(
|
||||
Some(&input.auth_snapshot),
|
||||
input.client_session_affinity.as_ref(),
|
||||
current_unix_secs(),
|
||||
input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
)
|
||||
.await?;
|
||||
let outcome = materialize_local_execution_candidates_with_serving(
|
||||
@@ -181,6 +186,10 @@ pub(super) async fn build_local_gemini_files_candidate_attempt_source<'a>(
|
||||
Some(&input.auth_snapshot),
|
||||
input.client_session_affinity.as_ref(),
|
||||
current_unix_secs(),
|
||||
input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
)
|
||||
.await?;
|
||||
Ok(build_local_execution_candidate_attempt_source_with_serving(
|
||||
|
||||
@@ -27,6 +27,7 @@ use crate::ai_serving::{
|
||||
};
|
||||
use crate::client_session_affinity::client_session_affinity_from_parts;
|
||||
use crate::clock::current_unix_secs;
|
||||
use crate::scheduler::config::SchedulerOrderingConfig;
|
||||
use crate::{AppState, GatewayError};
|
||||
use aether_scheduler_core::SchedulerMinimalCandidateSelectionCandidate;
|
||||
|
||||
@@ -127,6 +128,10 @@ pub(super) async fn list_local_openai_image_candidate_attempts(
|
||||
input.client_session_affinity.as_ref(),
|
||||
current_unix_secs(),
|
||||
false,
|
||||
input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -201,6 +206,10 @@ pub(super) async fn build_local_openai_image_candidate_attempt_source<'a>(
|
||||
input.client_session_affinity.as_ref(),
|
||||
current_unix_secs(),
|
||||
false,
|
||||
input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -29,6 +29,7 @@ use crate::ai_serving::{
|
||||
};
|
||||
use crate::client_session_affinity::client_session_affinity_from_parts;
|
||||
use crate::clock::current_unix_secs;
|
||||
use crate::scheduler::config::SchedulerOrderingConfig;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
pub(super) use crate::ai_serving::planner::candidate_materialization::LocalExecutionCandidateAttempt as LocalVideoCreateCandidateAttempt;
|
||||
@@ -133,6 +134,10 @@ pub(super) async fn list_local_video_create_candidate_attempts(
|
||||
input.client_session_affinity.as_ref(),
|
||||
current_unix_secs(),
|
||||
false,
|
||||
input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -190,6 +195,10 @@ pub(super) async fn build_local_video_create_candidate_attempt_source<'a>(
|
||||
input.client_session_affinity.as_ref(),
|
||||
current_unix_secs(),
|
||||
false,
|
||||
input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -8,9 +8,14 @@ use crate::constants::{
|
||||
API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS, API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS,
|
||||
};
|
||||
use crate::scheduler::candidate::SchedulerSkippedCandidate;
|
||||
use crate::scheduler::config::SchedulerOrderingConfig;
|
||||
use crate::GatewayError;
|
||||
|
||||
impl<'a> PlannerAppState<'a> {
|
||||
/// `ordering_config` is the request's routing-policy derived scheduler
|
||||
/// config (see `SchedulerOrderingConfig::from_routing_policy`). `None`
|
||||
/// falls back to the runtime default.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn list_selectable_candidates(
|
||||
self,
|
||||
api_format: &str,
|
||||
@@ -21,6 +26,7 @@ impl<'a> PlannerAppState<'a> {
|
||||
client_session_affinity: Option<&ClientSessionAffinity>,
|
||||
now_unix_secs: u64,
|
||||
enable_model_directives: bool,
|
||||
ordering_config: Option<SchedulerOrderingConfig>,
|
||||
) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
|
||||
crate::scheduler::candidate::list_selectable_candidates(
|
||||
self.app().data.as_ref(),
|
||||
@@ -33,10 +39,12 @@ impl<'a> PlannerAppState<'a> {
|
||||
client_session_affinity,
|
||||
now_unix_secs,
|
||||
enable_model_directives,
|
||||
ordering_config,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn list_selectable_candidates_with_skip_reasons(
|
||||
self,
|
||||
api_format: &str,
|
||||
@@ -47,6 +55,7 @@ impl<'a> PlannerAppState<'a> {
|
||||
client_session_affinity: Option<&ClientSessionAffinity>,
|
||||
now_unix_secs: u64,
|
||||
enable_model_directives: bool,
|
||||
ordering_config: Option<SchedulerOrderingConfig>,
|
||||
) -> Result<
|
||||
(
|
||||
Vec<SchedulerMinimalCandidateSelectionCandidate>,
|
||||
@@ -64,10 +73,12 @@ impl<'a> PlannerAppState<'a> {
|
||||
now_unix_secs,
|
||||
enable_model_directives,
|
||||
None,
|
||||
ordering_config,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn list_selectable_candidates_with_skip_reasons_for_request_operation(
|
||||
self,
|
||||
api_format: &str,
|
||||
@@ -79,6 +90,7 @@ impl<'a> PlannerAppState<'a> {
|
||||
now_unix_secs: u64,
|
||||
enable_model_directives: bool,
|
||||
request_operation: Option<&str>,
|
||||
ordering_config: Option<SchedulerOrderingConfig>,
|
||||
) -> Result<
|
||||
(
|
||||
Vec<SchedulerMinimalCandidateSelectionCandidate>,
|
||||
@@ -103,6 +115,7 @@ impl<'a> PlannerAppState<'a> {
|
||||
attempt_now_unix_secs,
|
||||
enable_model_directives,
|
||||
request_operation,
|
||||
ordering_config,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -123,6 +136,7 @@ impl<'a> PlannerAppState<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn list_selectable_enumerated_candidates_with_skip_reasons(
|
||||
self,
|
||||
api_format: &str,
|
||||
@@ -132,6 +146,7 @@ impl<'a> PlannerAppState<'a> {
|
||||
auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
|
||||
client_session_affinity: Option<&ClientSessionAffinity>,
|
||||
now_unix_secs: u64,
|
||||
ordering_config: Option<SchedulerOrderingConfig>,
|
||||
) -> Result<
|
||||
(
|
||||
Vec<SchedulerMinimalCandidateSelectionCandidate>,
|
||||
@@ -148,10 +163,12 @@ impl<'a> PlannerAppState<'a> {
|
||||
auth_snapshot,
|
||||
client_session_affinity,
|
||||
now_unix_secs,
|
||||
ordering_config,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn list_selectable_candidates_for_required_capability_without_requested_model(
|
||||
self,
|
||||
candidate_api_format: &str,
|
||||
@@ -160,6 +177,7 @@ impl<'a> PlannerAppState<'a> {
|
||||
auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
|
||||
client_session_affinity: Option<&ClientSessionAffinity>,
|
||||
now_unix_secs: u64,
|
||||
ordering_config: Option<SchedulerOrderingConfig>,
|
||||
) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
|
||||
let wait_timeout = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS);
|
||||
let wait_interval = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS.max(1));
|
||||
@@ -176,6 +194,7 @@ impl<'a> PlannerAppState<'a> {
|
||||
auth_snapshot,
|
||||
client_session_affinity,
|
||||
attempt_now_unix_secs,
|
||||
ordering_config,
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user