i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-04 16:37:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(routing): make routing profiles the sole scheduler policy source

Browse Source 
Bootstrap an enabled system-default routing group from the legacy
scheduler config keys on startup, resolve the default ordering config
from that group before falling back to the legacy keys, and stop merging
keep_priority_on_conversion with the legacy flag when a policy is
resolved. Thread the policy-derived ordering config into candidate
preselection so it no longer reads system config independently.

Add per-API-format key priority overrides so a key serving several
formats keeps independent ordering, matching the legacy
global_priority_by_format semantics. Expose keep_priority_on_conversion
in the routing profile editor and read the effective policy in the
model routing preview, monitoring metrics and provider page badge.
This commit is contained in:
elky
2026-09-02 17:04:04 +08:00
parent 166236c2ee
commit 415b2da81b
32 changed files with 1001 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/aether-gateway/src/ai_serving/planner/candidate_materialization.rs
+1
View File
@@ -1840,6 +1840,7 @@ fn routing_trace_for_candidate(
CandidateKind::Provider => Some(candidate.key_id.clone()),
CandidateKind::PoolGroup => None,
},
api_format: Some(candidate.endpoint_api_format.clone()),
provider_priority: candidate.provider_priority,
key_priority: candidate
.key_global_priority_for_format
apps/aether-gateway/src/ai_serving/planner/candidate_ranking.rs
+27 -31
View File
@@ -3,7 +3,7 @@ use aether_ai_serving::{
AiCandidateRankingPort, AiRankableCandidateParts, AiRankingContextConfig,
AiRankingSchedulingMode,
};
use aether_routing_core::{ResolvedRoutingPolicy, RoutingSchedulingMode, RoutingSetPriorityMode};
use aether_routing_core::ResolvedRoutingPolicy;
use async_trait::async_trait;
use tokio::sync::Mutex;
use tracing::warn;
@@ -184,35 +184,16 @@ fn ai_ranking_scheduling_mode(mode: SchedulerSchedulingMode) -> AiRankingSchedul
}
}
/// Ordering config for a request. A resolved routing policy is authoritative
/// and is never merged with legacy system-config values; without a policy the
/// effective default (system-default routing group, then legacy keys) applies.
pub(crate) async fn scheduler_ordering_config_for_routing_policy(
state: PlannerAppState<'_>,
routing_policy: Option<&ResolvedRoutingPolicy>,
) -> SchedulerOrderingConfig {
let system_config = read_scheduler_ordering_config_or_default(state).await;
match routing_policy {
Some(policy) => {
let mut config = scheduler_ordering_config_from_routing_policy(policy);
config.keep_priority_on_conversion |= system_config.keep_priority_on_conversion;
config
}
None => system_config,
}
}
fn scheduler_ordering_config_from_routing_policy(
policy: &ResolvedRoutingPolicy,
) -> SchedulerOrderingConfig {
SchedulerOrderingConfig {
priority_mode: match policy.priority_mode {
RoutingSetPriorityMode::Provider => SchedulerPriorityMode::Provider,
RoutingSetPriorityMode::GlobalKey => SchedulerPriorityMode::GlobalKey,
},
scheduling_mode: match policy.scheduling_mode {
RoutingSchedulingMode::FixedOrder => SchedulerSchedulingMode::FixedOrder,
RoutingSchedulingMode::CacheAffinity => SchedulerSchedulingMode::CacheAffinity,
RoutingSchedulingMode::LoadBalance => SchedulerSchedulingMode::LoadBalance,
},
keep_priority_on_conversion: policy.keep_priority_on_conversion,
Some(policy) => SchedulerOrderingConfig::from_routing_policy(policy),
None => read_scheduler_ordering_config_or_default(state).await,
}
}
@@ -231,14 +212,26 @@ fn routing_overlaid_candidate(
let overlaid_key_priority = match kind {
LocalExecutionCandidateKind::SingleKey => policy
.ranking_overlay
.key_priority_overrides
.get(candidate.key_id.as_str()),
.key_priority_override_matching_format(candidate.key_id.as_str(), |format| {
crate::ai_serving::api_format_alias_matches(
format,
candidate.endpoint_api_format.as_str(),
)
})
.or_else(|| {
policy
.ranking_overlay
.key_priority_overrides
.get(candidate.key_id.as_str())
.copied()
}),
LocalExecutionCandidateKind::PoolGroup => policy
.ranking_overlay
.pool_priority_overrides
.get(candidate.provider_id.as_str()),
.get(candidate.provider_id.as_str())
.copied(),
};
if let Some(overlaid_key_priority) = overlaid_key_priority.copied() {
if let Some(overlaid_key_priority) = overlaid_key_priority {
overlaid.key_internal_priority = overlaid_key_priority;
overlaid.key_global_priority_for_format = Some(overlaid_key_priority);
}
@@ -396,7 +389,7 @@ mod tests {
}
#[tokio::test]
async fn routing_policy_inherits_global_conversion_priority_override() {
async fn routing_policy_ignores_legacy_global_conversion_priority_override() {
let data_state = GatewayDataState::default().with_system_config_values_for_tests([(
"keep_priority_on_conversion".to_string(),
json!(true),
@@ -429,7 +422,10 @@ mod tests {
ordering.scheduling_mode,
crate::scheduler::config::SchedulerSchedulingMode::FixedOrder
);
assert!(ordering.keep_priority_on_conversion);
assert!(
!ordering.keep_priority_on_conversion,
"a resolved routing policy must not inherit the legacy system-config flag"
);
}
#[test]
apps/aether-gateway/src/ai_serving/planner/candidate_source.rs
+10 -3
View File
@@ -174,6 +174,8 @@ impl AiCandidatePreselectionPort for GatewayLocalCandidatePreselectionPort<'_> {
self.ranking_seed,
false,
self.request_operation,
self.routing_policy
.map(SchedulerOrderingConfig::from_routing_policy),
)
.await?;
@@ -1291,6 +1293,9 @@ impl<'a> LocalCandidatePreselectionPageCursor<'a> {
.then_some(self.client_session_affinity.as_ref())
.flatten(),
self.ranking_seed,
self.routing_policy
.as_ref()
.map(SchedulerOrderingConfig::from_routing_policy),
)
.await?;
let skipped_candidates = skipped_candidates
@@ -2657,14 +2662,16 @@ mod tests {
candidate_repository,
)
.with_encryption_key_for_tests("development-key")
// Legacy keys deliberately disagree with the routing policy: the
// resolved policy must be the only source of scheduler ordering.
.with_system_config_values_for_tests([
(
"scheduling_mode".to_string(),
serde_json::json!("fixed_order"),
serde_json::json!("cache_affinity"),
),
(
"keep_priority_on_conversion".to_string(),
serde_json::json!(true),
serde_json::json!(false),
),
]);
let app = AppState::new()
@@ -2681,7 +2688,7 @@ mod tests {
resolved_model: "gpt-5.4-mini".to_string(),
priority_mode: aether_routing_core::RoutingSetPriorityMode::Provider,
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
keep_priority_on_conversion: false,
keep_priority_on_conversion: true,
ranking_overlay: Default::default(),
mutation_plan: Default::default(),
pool_policy_overrides: Default::default(),
apps/aether-gateway/src/ai_serving/planner/decision_input.rs
+1
View File
@@ -1073,6 +1073,7 @@ fn ensure_report_context_routing_trace(
endpoint_id: decision.endpoint_id.clone().unwrap_or_default(),
model_id,
key_id,
api_format: decision.provider_api_format.clone(),
provider_priority,
key_priority,
},
apps/aether-gateway/src/ai_serving/planner/passthrough/provider/family/candidates.rs
+9
View File
@@ -26,6 +26,7 @@ use crate::ai_serving::{
};
use crate::client_session_affinity::client_session_affinity_from_api_request;
use crate::clock::current_unix_secs;
use crate::scheduler::config::SchedulerOrderingConfig;
use crate::{AppState, GatewayError};
use super::{
@@ -140,6 +141,10 @@ pub(crate) async fn materialize_local_same_format_provider_candidate_attempts(
current_unix_secs(),
false,
spec.operation.map(|operation| operation.as_str()),
input
.routing_policy
.as_ref()
.map(SchedulerOrderingConfig::from_routing_policy),
)
.await?;
let outcome = materialize_local_execution_candidates_with_serving(
@@ -246,6 +251,10 @@ pub(crate) async fn build_local_same_format_provider_candidate_attempt_source<'a
current_unix_secs(),
false,
spec.operation.map(|operation| operation.as_str()),
input
.routing_policy
.as_ref()
.map(SchedulerOrderingConfig::from_routing_policy),
)
.await?;
apps/aether-gateway/src/ai_serving/planner/specialized/files/support.rs
+9
View File
@@ -26,6 +26,7 @@ use crate::ai_serving::{
};
use crate::client_session_affinity::client_session_affinity_from_parts;
use crate::clock::current_unix_secs;
use crate::scheduler::config::SchedulerOrderingConfig;
use crate::{AppState, GatewayError};
pub(super) use crate::ai_serving::planner::candidate_materialization::LocalExecutionCandidateAttempt as LocalGeminiFilesCandidateAttempt;
@@ -108,6 +109,10 @@ pub(super) async fn materialize_local_gemini_files_candidate_attempts(
Some(&input.auth_snapshot),
input.client_session_affinity.as_ref(),
current_unix_secs(),
input
.routing_policy
.as_ref()
.map(SchedulerOrderingConfig::from_routing_policy),
)
.await?;
let outcome = materialize_local_execution_candidates_with_serving(
@@ -181,6 +186,10 @@ pub(super) async fn build_local_gemini_files_candidate_attempt_source<'a>(
Some(&input.auth_snapshot),
input.client_session_affinity.as_ref(),
current_unix_secs(),
input
.routing_policy
.as_ref()
.map(SchedulerOrderingConfig::from_routing_policy),
)
.await?;
Ok(build_local_execution_candidate_attempt_source_with_serving(
apps/aether-gateway/src/ai_serving/planner/specialized/image/support.rs
+9
View File
@@ -27,6 +27,7 @@ use crate::ai_serving::{
};
use crate::client_session_affinity::client_session_affinity_from_parts;
use crate::clock::current_unix_secs;
use crate::scheduler::config::SchedulerOrderingConfig;
use crate::{AppState, GatewayError};
use aether_scheduler_core::SchedulerMinimalCandidateSelectionCandidate;
@@ -127,6 +128,10 @@ pub(super) async fn list_local_openai_image_candidate_attempts(
input.client_session_affinity.as_ref(),
current_unix_secs(),
false,
input
.routing_policy
.as_ref()
.map(SchedulerOrderingConfig::from_routing_policy),
)
.await
{
@@ -201,6 +206,10 @@ pub(super) async fn build_local_openai_image_candidate_attempt_source<'a>(
input.client_session_affinity.as_ref(),
current_unix_secs(),
false,
input
.routing_policy
.as_ref()
.map(SchedulerOrderingConfig::from_routing_policy),
)
.await
{
apps/aether-gateway/src/ai_serving/planner/specialized/video/support.rs
+9
View File
@@ -29,6 +29,7 @@ use crate::ai_serving::{
};
use crate::client_session_affinity::client_session_affinity_from_parts;
use crate::clock::current_unix_secs;
use crate::scheduler::config::SchedulerOrderingConfig;
use crate::{AppState, GatewayError};
pub(super) use crate::ai_serving::planner::candidate_materialization::LocalExecutionCandidateAttempt as LocalVideoCreateCandidateAttempt;
@@ -133,6 +134,10 @@ pub(super) async fn list_local_video_create_candidate_attempts(
input.client_session_affinity.as_ref(),
current_unix_secs(),
false,
input
.routing_policy
.as_ref()
.map(SchedulerOrderingConfig::from_routing_policy),
)
.await
{
@@ -190,6 +195,10 @@ pub(super) async fn build_local_video_create_candidate_attempt_source<'a>(
input.client_session_affinity.as_ref(),
current_unix_secs(),
false,
input
.routing_policy
.as_ref()
.map(SchedulerOrderingConfig::from_routing_policy),
)
.await
{
apps/aether-gateway/src/ai_serving/planner/state/scheduler.rs
+19
View File
@@ -8,9 +8,14 @@ use crate::constants::{
API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS, API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS,
};
use crate::scheduler::candidate::SchedulerSkippedCandidate;
use crate::scheduler::config::SchedulerOrderingConfig;
use crate::GatewayError;
impl<'a> PlannerAppState<'a> {
/// `ordering_config` is the request's routing-policy derived scheduler
/// config (see `SchedulerOrderingConfig::from_routing_policy`). `None`
/// falls back to the runtime default.
#[allow(clippy::too_many_arguments)]
pub(crate) async fn list_selectable_candidates(
self,
api_format: &str,
@@ -21,6 +26,7 @@ impl<'a> PlannerAppState<'a> {
client_session_affinity: Option<&ClientSessionAffinity>,
now_unix_secs: u64,
enable_model_directives: bool,
ordering_config: Option<SchedulerOrderingConfig>,
) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
crate::scheduler::candidate::list_selectable_candidates(
self.app().data.as_ref(),
@@ -33,10 +39,12 @@ impl<'a> PlannerAppState<'a> {
client_session_affinity,
now_unix_secs,
enable_model_directives,
ordering_config,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn list_selectable_candidates_with_skip_reasons(
self,
api_format: &str,
@@ -47,6 +55,7 @@ impl<'a> PlannerAppState<'a> {
client_session_affinity: Option<&ClientSessionAffinity>,
now_unix_secs: u64,
enable_model_directives: bool,
ordering_config: Option<SchedulerOrderingConfig>,
) -> Result<
(
Vec<SchedulerMinimalCandidateSelectionCandidate>,
@@ -64,10 +73,12 @@ impl<'a> PlannerAppState<'a> {
now_unix_secs,
enable_model_directives,
None,
ordering_config,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn list_selectable_candidates_with_skip_reasons_for_request_operation(
self,
api_format: &str,
@@ -79,6 +90,7 @@ impl<'a> PlannerAppState<'a> {
now_unix_secs: u64,
enable_model_directives: bool,
request_operation: Option<&str>,
ordering_config: Option<SchedulerOrderingConfig>,
) -> Result<
(
Vec<SchedulerMinimalCandidateSelectionCandidate>,
@@ -103,6 +115,7 @@ impl<'a> PlannerAppState<'a> {
attempt_now_unix_secs,
enable_model_directives,
request_operation,
ordering_config,
)
.await?;
@@ -123,6 +136,7 @@ impl<'a> PlannerAppState<'a> {
}
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn list_selectable_enumerated_candidates_with_skip_reasons(
self,
api_format: &str,
@@ -132,6 +146,7 @@ impl<'a> PlannerAppState<'a> {
auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
client_session_affinity: Option<&ClientSessionAffinity>,
now_unix_secs: u64,
ordering_config: Option<SchedulerOrderingConfig>,
) -> Result<
(
Vec<SchedulerMinimalCandidateSelectionCandidate>,
@@ -148,10 +163,12 @@ impl<'a> PlannerAppState<'a> {
auth_snapshot,
client_session_affinity,
now_unix_secs,
ordering_config,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn list_selectable_candidates_for_required_capability_without_requested_model(
self,
candidate_api_format: &str,
@@ -160,6 +177,7 @@ impl<'a> PlannerAppState<'a> {
auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
client_session_affinity: Option<&ClientSessionAffinity>,
now_unix_secs: u64,
ordering_config: Option<SchedulerOrderingConfig>,
) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
let wait_timeout = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS);
let wait_interval = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS.max(1));
@@ -176,6 +194,7 @@ impl<'a> PlannerAppState<'a> {
auth_snapshot,
client_session_affinity,
attempt_now_unix_secs,
ordering_config,
)
.await?;
apps/aether-gateway/src/dispatch/pool_scheduler.rs
+21 -12
View File
@@ -754,19 +754,28 @@ impl<'a> PoolKeyCursor<'a> {
return None;
}
};
let api_format = self.group.candidate.endpoint_api_format.as_str();
rows.sort_by(|left, right| {
let left_priority = self
.routing_overlay
.as_ref()
.map_or(left.key_internal_priority, |overlay| {
overlay.key_priority(&left.key_id, left.key_internal_priority)
});
let right_priority = self
.routing_overlay
.as_ref()
.map_or(right.key_internal_priority, |overlay| {
overlay.key_priority(&right.key_id, right.key_internal_priority)
});
let left_priority = self.routing_overlay.as_ref().map_or(
left.key_internal_priority,
|overlay| {
overlay.key_priority_for_format(
&left.key_id,
api_format,
left.key_internal_priority,
)
},
);
let right_priority = self.routing_overlay.as_ref().map_or(
right.key_internal_priority,
|overlay| {
overlay.key_priority_for_format(
&right.key_id,
api_format,
right.key_internal_priority,
)
},
);
left_priority
.cmp(&right_priority)
.then(left.key_id.cmp(&right.key_id))
apps/aether-gateway/src/handlers/admin/model/routing.rs
+7 -20
View File
@@ -67,27 +67,14 @@ pub(crate) async fn build_admin_global_model_routing_payload(
.push(key);
}
let scheduling_mode = state
.read_system_config_json_value("scheduling_mode")
// Effective default scheduling: system-default routing group first, then
// legacy system-config keys.
let ordering_config = crate::scheduler::config::read_scheduler_ordering_config(state.app())
.await
.ok()
.flatten()
.and_then(|value| value.as_str().map(ToOwned::to_owned))
.unwrap_or_else(|| "cache_affinity".to_string());
let priority_mode = state
.read_system_config_json_value("provider_priority_mode")
.await
.ok()
.flatten()
.and_then(|value| value.as_str().map(ToOwned::to_owned))
.unwrap_or_else(|| "provider".to_string());
let keep_priority_on_conversion = state
.read_system_config_json_value("keep_priority_on_conversion")
.await
.ok()
.flatten()
.and_then(|value| value.as_bool())
.unwrap_or(false);
.unwrap_or_default();
let scheduling_mode = ordering_config.scheduling_mode_str().to_string();
let priority_mode = ordering_config.priority_mode_str().to_string();
let keep_priority_on_conversion = ordering_config.keep_priority_on_conversion;
let now_unix_secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs())
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_store.rs
+4 -10
View File
@@ -264,16 +264,10 @@ async fn list_admin_monitoring_cache_affinity_records_matching(
pub(super) async fn build_admin_monitoring_cache_snapshot(
state: &AdminAppState<'_>,
) -> Result<AdminMonitoringCacheSnapshot, GatewayError> {
let scheduling_mode = state
.read_system_config_json_value("scheduling_mode")
.await?
.and_then(|value| value.as_str().map(ToOwned::to_owned))
.unwrap_or_else(|| "cache_affinity".to_string());
let provider_priority_mode = state
.read_system_config_json_value("provider_priority_mode")
.await?
.and_then(|value| value.as_str().map(ToOwned::to_owned))
.unwrap_or_else(|| "provider".to_string());
let ordering_config =
crate::scheduler::config::read_scheduler_ordering_config(state.app()).await?;
let scheduling_mode = ordering_config.scheduling_mode_str().to_string();
let provider_priority_mode = ordering_config.priority_mode_str().to_string();
let now = chrono::Utc::now();
let usage_summary = if state.has_usage_data_reader() {
apps/aether-gateway/src/main.rs
+16
View File
@@ -2096,6 +2096,22 @@ async fn run() -> Result<(), Box<dyn std::error::Error>> {
);
}
state.bootstrap_admin_from_env().await?;
match state.ensure_system_default_routing_group().await {
Ok(Some(group)) => {
info!(
group_id = %group.id,
group_name = %group.name,
"created system default routing group from legacy scheduler config"
);
}
Ok(None) => {}
Err(err) => {
warn!(
error = %err,
"failed to bootstrap system default routing group; scheduler falls back to legacy system config"
);
}
}
match state.prewarm_chat_pii_redaction_runtime_config().await {
Ok(enabled) => {
info!(
apps/aether-gateway/src/scheduler/candidate/mod.rs
+41 -18
View File
@@ -1,7 +1,9 @@
use self::selection::{
collect_selectable_candidates, collect_selectable_candidates_with_skip_reasons,
collect_selectable_candidates, collect_selectable_candidates_with_skip_reasons_and_ordering,
collect_selectable_enumerated_candidates_with_skip_reasons,
resolve_preselection_ordering_config,
};
use super::config::SchedulerOrderingConfig;
use super::state::SchedulerRuntimeState;
mod affinity;
@@ -53,6 +55,10 @@ enum RequiredCapabilityMatchMode {
Exclusive,
}
/// `ordering_config` carries the request's routing-policy derived scheduler
/// config. `None` falls back to the runtime default (system-default routing
/// group, then legacy system-config keys).
#[allow(clippy::too_many_arguments)]
pub(crate) async fn list_selectable_candidates(
selection_row_source: &(impl MinimalCandidateSelectionRowSource + Sync),
runtime_state: &impl SchedulerRuntimeState,
@@ -64,6 +70,7 @@ pub(crate) async fn list_selectable_candidates(
client_session_affinity: Option<&ClientSessionAffinity>,
now_unix_secs: u64,
enable_model_directives: bool,
ordering_config: Option<SchedulerOrderingConfig>,
) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
collect_selectable_candidates(
selection_row_source,
@@ -76,6 +83,7 @@ pub(crate) async fn list_selectable_candidates(
client_session_affinity,
now_unix_secs,
enable_model_directives,
ordering_config,
)
.await
}
@@ -87,6 +95,7 @@ pub(crate) fn is_exact_all_skipped_by_auth_limit(
selection::is_exact_all_skipped_by_auth_limit(selected, skipped)
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn list_selectable_candidates_with_skip_reasons(
selection_row_source: &(impl MinimalCandidateSelectionRowSource + Sync),
runtime_state: &impl SchedulerRuntimeState,
@@ -98,6 +107,7 @@ pub(crate) async fn list_selectable_candidates_with_skip_reasons(
client_session_affinity: Option<&ClientSessionAffinity>,
now_unix_secs: u64,
enable_model_directives: bool,
ordering_config: Option<SchedulerOrderingConfig>,
) -> Result<
(
Vec<SchedulerMinimalCandidateSelectionCandidate>,
@@ -105,7 +115,7 @@ pub(crate) async fn list_selectable_candidates_with_skip_reasons(
),
GatewayError,
> {
collect_selectable_candidates_with_skip_reasons(
collect_selectable_candidates_with_skip_reasons_and_ordering(
selection_row_source,
runtime_state,
api_format,
@@ -117,10 +127,12 @@ pub(crate) async fn list_selectable_candidates_with_skip_reasons(
now_unix_secs,
enable_model_directives,
None,
ordering_config,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn list_selectable_candidates_with_skip_reasons_for_request_operation(
selection_row_source: &(impl MinimalCandidateSelectionRowSource + Sync),
runtime_state: &impl SchedulerRuntimeState,
@@ -133,6 +145,7 @@ pub(crate) async fn list_selectable_candidates_with_skip_reasons_for_request_ope
now_unix_secs: u64,
enable_model_directives: bool,
request_operation: Option<&str>,
ordering_config: Option<SchedulerOrderingConfig>,
) -> Result<
(
Vec<SchedulerMinimalCandidateSelectionCandidate>,
@@ -140,7 +153,7 @@ pub(crate) async fn list_selectable_candidates_with_skip_reasons_for_request_ope
),
GatewayError,
> {
collect_selectable_candidates_with_skip_reasons(
collect_selectable_candidates_with_skip_reasons_and_ordering(
selection_row_source,
runtime_state,
api_format,
@@ -152,6 +165,7 @@ pub(crate) async fn list_selectable_candidates_with_skip_reasons_for_request_ope
now_unix_secs,
enable_model_directives,
request_operation,
ordering_config,
)
.await
}
@@ -166,6 +180,7 @@ pub(crate) async fn list_selectable_enumerated_candidates_with_skip_reasons(
auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
client_session_affinity: Option<&ClientSessionAffinity>,
now_unix_secs: u64,
ordering_config: Option<SchedulerOrderingConfig>,
) -> Result<
(
Vec<SchedulerMinimalCandidateSelectionCandidate>,
@@ -173,7 +188,8 @@ pub(crate) async fn list_selectable_enumerated_candidates_with_skip_reasons(
),
GatewayError,
> {
let ordering_config = runtime_state.read_scheduler_ordering_config().await?;
let ordering_config =
resolve_preselection_ordering_config(runtime_state, ordering_config).await?;
let priority_affinity_key = selection::scheduling_priority_affinity_key(
auth_snapshot,
client_session_affinity,
@@ -194,6 +210,7 @@ pub(crate) async fn list_selectable_enumerated_candidates_with_skip_reasons(
.await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn list_selectable_candidates_for_required_capability_without_requested_model(
selection_row_source: &(impl MinimalCandidateSelectionRowSource + Sync),
runtime_state: &impl SchedulerRuntimeState,
@@ -203,6 +220,7 @@ pub(crate) async fn list_selectable_candidates_for_required_capability_without_r
auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
client_session_affinity: Option<&ClientSessionAffinity>,
now_unix_secs: u64,
ordering_config: Option<SchedulerOrderingConfig>,
) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
Ok(
list_selectable_candidates_for_required_capability_without_requested_model_with_auth_limit_signal(
@@ -214,12 +232,14 @@ pub(crate) async fn list_selectable_candidates_for_required_capability_without_r
auth_snapshot,
client_session_affinity,
now_unix_secs,
ordering_config,
)
.await?
.0,
)
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn list_selectable_candidates_for_required_capability_without_requested_model_with_auth_limit_signal(
selection_row_source: &(impl MinimalCandidateSelectionRowSource + Sync),
runtime_state: &impl SchedulerRuntimeState,
@@ -229,6 +249,7 @@ pub(crate) async fn list_selectable_candidates_for_required_capability_without_r
auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
client_session_affinity: Option<&ClientSessionAffinity>,
now_unix_secs: u64,
ordering_config: Option<SchedulerOrderingConfig>,
) -> Result<(Vec<SchedulerMinimalCandidateSelectionCandidate>, bool), GatewayError> {
let normalized_api_format = normalize_api_format(candidate_api_format);
if normalized_api_format.is_empty() {
@@ -262,20 +283,22 @@ pub(crate) async fn list_selectable_candidates_for_required_capability_without_r
let mut all_attempts_blocked_by_auth_limit = !model_names.is_empty();
for global_model_name in model_names {
let (candidates, skipped_candidates) = collect_selectable_candidates_with_skip_reasons(
selection_row_source,
runtime_state,
&normalized_api_format,
&global_model_name,
require_streaming,
required_capabilities.as_ref(),
auth_snapshot,
client_session_affinity,
now_unix_secs,
false,
None,
)
.await?;
let (candidates, skipped_candidates) =
collect_selectable_candidates_with_skip_reasons_and_ordering(
selection_row_source,
runtime_state,
&normalized_api_format,
&global_model_name,
require_streaming,
required_capabilities.as_ref(),
auth_snapshot,
client_session_affinity,
now_unix_secs,
false,
None,
ordering_config,
)
.await?;
all_attempts_blocked_by_auth_limit &=
is_exact_all_skipped_by_auth_limit(&candidates, &skipped_candidates);
match capability_mode {
apps/aether-gateway/src/scheduler/candidate/selection.rs
+78 -17
View File
@@ -1,7 +1,7 @@
use crate::data::auth::GatewayAuthApiKeySnapshot;
use crate::data::candidate_selection::MinimalCandidateSelectionRowSource;
use crate::scheduler::affinity::SCHEDULER_AFFINITY_TTL;
use crate::scheduler::config::SchedulerSchedulingMode;
use crate::scheduler::config::{SchedulerOrderingConfig, SchedulerSchedulingMode};
use crate::GatewayError;
use aether_scheduler_core::ClientSessionAffinity;
@@ -115,6 +115,7 @@ pub(super) async fn select_minimal_candidate(
Ok(selected)
}
#[allow(clippy::too_many_arguments)]
pub(super) async fn collect_selectable_candidates(
selection_row_source: &(impl MinimalCandidateSelectionRowSource + Sync),
runtime_state: &impl SchedulerRuntimeState,
@@ -126,24 +127,32 @@ pub(super) async fn collect_selectable_candidates(
client_session_affinity: Option<&ClientSessionAffinity>,
now_unix_secs: u64,
enable_model_directives: bool,
ordering_config: Option<SchedulerOrderingConfig>,
) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
Ok(collect_selectable_candidates_with_skip_reasons(
selection_row_source,
runtime_state,
api_format,
global_model_name,
require_streaming,
required_capabilities,
auth_snapshot,
client_session_affinity,
now_unix_secs,
enable_model_directives,
None,
Ok(
collect_selectable_candidates_with_skip_reasons_and_ordering(
selection_row_source,
runtime_state,
api_format,
global_model_name,
require_streaming,
required_capabilities,
auth_snapshot,
client_session_affinity,
now_unix_secs,
enable_model_directives,
None,
ordering_config,
)
.await?
.0,
)
.await?
.0)
}
/// Legacy-shaped entrypoint that resolves the ordering config from the
/// runtime state. Prefer `collect_selectable_candidates_with_skip_reasons_and_ordering`
/// and pass the request's routing-policy config explicitly.
#[allow(clippy::too_many_arguments)]
pub(super) async fn collect_selectable_candidates_with_skip_reasons(
selection_row_source: &(impl MinimalCandidateSelectionRowSource + Sync),
runtime_state: &impl SchedulerRuntimeState,
@@ -163,7 +172,59 @@ pub(super) async fn collect_selectable_candidates_with_skip_reasons(
),
GatewayError,
> {
let ordering_config = runtime_state.read_scheduler_ordering_config().await?;
collect_selectable_candidates_with_skip_reasons_and_ordering(
selection_row_source,
runtime_state,
api_format,
global_model_name,
require_streaming,
required_capabilities,
auth_snapshot,
client_session_affinity,
now_unix_secs,
enable_model_directives,
request_operation,
None,
)
.await
}
/// Resolve the ordering config for a preselection pass: the routing-policy
/// derived config wins when the caller has one; otherwise fall back to the
/// runtime default (system-default routing group, then legacy keys).
pub(super) async fn resolve_preselection_ordering_config(
runtime_state: &impl SchedulerRuntimeState,
ordering_config: Option<SchedulerOrderingConfig>,
) -> Result<SchedulerOrderingConfig, GatewayError> {
match ordering_config {
Some(config) => Ok(config),
None => runtime_state.read_scheduler_ordering_config().await,
}
}
#[allow(clippy::too_many_arguments)]
pub(super) async fn collect_selectable_candidates_with_skip_reasons_and_ordering(
selection_row_source: &(impl MinimalCandidateSelectionRowSource + Sync),
runtime_state: &impl SchedulerRuntimeState,
api_format: &str,
global_model_name: &str,
require_streaming: bool,
required_capabilities: Option<&serde_json::Value>,
auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
client_session_affinity: Option<&ClientSessionAffinity>,
now_unix_secs: u64,
enable_model_directives: bool,
request_operation: Option<&str>,
ordering_config: Option<SchedulerOrderingConfig>,
) -> Result<
(
Vec<SchedulerMinimalCandidateSelectionCandidate>,
Vec<SchedulerSkippedCandidate>,
),
GatewayError,
> {
let ordering_config =
resolve_preselection_ordering_config(runtime_state, ordering_config).await?;
let priority_affinity_key = scheduling_priority_affinity_key(
auth_snapshot,
client_session_affinity,
@@ -205,7 +266,7 @@ pub(super) async fn collect_selectable_enumerated_candidates_with_skip_reasons(
auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
client_session_affinity: Option<&ClientSessionAffinity>,
now_unix_secs: u64,
ordering_config: crate::scheduler::config::SchedulerOrderingConfig,
ordering_config: SchedulerOrderingConfig,
priority_affinity_key: Option<&str>,
) -> Result<
(
apps/aether-gateway/src/scheduler/candidate/tests/required_capability.rs
+4
View File
@@ -65,6 +65,7 @@ async fn compatible_required_capability_prefers_matching_keys_without_hard_filte
None,
None,
100,
None,
)
.await
.expect("selection should succeed");
@@ -120,6 +121,7 @@ async fn exclusive_required_capability_keeps_hard_filtering_only_matching_keys()
None,
None,
100,
None,
)
.await
.expect("selection should succeed");
@@ -196,6 +198,7 @@ async fn required_capability_without_model_uses_session_scoped_affinity() {
Some(&auth_snapshot),
Some(&client_session_affinity),
100,
None,
)
.await
.expect("selection should succeed");
@@ -273,6 +276,7 @@ async fn required_capability_reports_auth_limit_signal_when_every_model_is_block
Some(&auth_snapshot),
None,
100,
None,
)
.await
.expect("selection should succeed");
apps/aether-gateway/src/scheduler/candidate/tests/selection.rs
+1
View File
@@ -75,6 +75,7 @@ async fn collect_selectable_candidates(
None,
now_unix_secs,
false,
None,
)
.await
}
apps/aether-gateway/src/scheduler/config.rs
+293
View File
@@ -1,4 +1,9 @@
use aether_data_contracts::repository::routing_profiles::RoutingGroupLookupKey;
use aether_routing_core::{
ResolvedRoutingPolicy, RoutingDefaultPolicy, RoutingSchedulingMode, RoutingSetPriorityMode,
};
use aether_scheduler_core::SchedulerPriorityMode;
use tracing::warn;
use crate::{AppState, GatewayError};
@@ -10,6 +15,23 @@ pub(crate) enum SchedulerSchedulingMode {
LoadBalance,
}
impl SchedulerSchedulingMode {
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::FixedOrder => "fixed_order",
Self::CacheAffinity => "cache_affinity",
Self::LoadBalance => "load_balance",
}
}
}
pub(crate) fn scheduler_priority_mode_as_str(mode: SchedulerPriorityMode) -> &'static str {
match mode {
SchedulerPriorityMode::Provider => "provider",
SchedulerPriorityMode::GlobalKey => "global_key",
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct SchedulerOrderingConfig {
pub(crate) priority_mode: SchedulerPriorityMode,
@@ -27,6 +49,64 @@ impl Default for SchedulerOrderingConfig {
}
}
impl SchedulerOrderingConfig {
/// Ordering config derived from a resolved routing policy. The policy is
/// the single source of truth: no legacy system-config value is merged in.
pub(crate) fn from_routing_policy(policy: &ResolvedRoutingPolicy) -> Self {
Self {
priority_mode: scheduler_priority_mode_from_routing(policy.priority_mode),
scheduling_mode: scheduler_scheduling_mode_from_routing(policy.scheduling_mode),
keep_priority_on_conversion: policy.keep_priority_on_conversion,
}
}
pub(crate) fn from_routing_default_policy(policy: &RoutingDefaultPolicy) -> Self {
Self {
priority_mode: scheduler_priority_mode_from_routing(policy.priority_mode),
scheduling_mode: scheduler_scheduling_mode_from_routing(policy.scheduling_mode),
keep_priority_on_conversion: policy.keep_priority_on_conversion,
}
}
pub(crate) fn to_routing_default_policy(self) -> RoutingDefaultPolicy {
RoutingDefaultPolicy {
priority_mode: match self.priority_mode {
SchedulerPriorityMode::Provider => RoutingSetPriorityMode::Provider,
SchedulerPriorityMode::GlobalKey => RoutingSetPriorityMode::GlobalKey,
},
scheduling_mode: match self.scheduling_mode {
SchedulerSchedulingMode::FixedOrder => RoutingSchedulingMode::FixedOrder,
SchedulerSchedulingMode::CacheAffinity => RoutingSchedulingMode::CacheAffinity,
SchedulerSchedulingMode::LoadBalance => RoutingSchedulingMode::LoadBalance,
},
keep_priority_on_conversion: self.keep_priority_on_conversion,
}
}
pub(crate) fn priority_mode_str(self) -> &'static str {
scheduler_priority_mode_as_str(self.priority_mode)
}
pub(crate) fn scheduling_mode_str(self) -> &'static str {
self.scheduling_mode.as_str()
}
}
fn scheduler_priority_mode_from_routing(mode: RoutingSetPriorityMode) -> SchedulerPriorityMode {
match mode {
RoutingSetPriorityMode::Provider => SchedulerPriorityMode::Provider,
RoutingSetPriorityMode::GlobalKey => SchedulerPriorityMode::GlobalKey,
}
}
fn scheduler_scheduling_mode_from_routing(mode: RoutingSchedulingMode) -> SchedulerSchedulingMode {
match mode {
RoutingSchedulingMode::FixedOrder => SchedulerSchedulingMode::FixedOrder,
RoutingSchedulingMode::CacheAffinity => SchedulerSchedulingMode::CacheAffinity,
RoutingSchedulingMode::LoadBalance => SchedulerSchedulingMode::LoadBalance,
}
}
pub(crate) fn parse_scheduler_priority_mode(
value: Option<&serde_json::Value>,
) -> SchedulerPriorityMode {
@@ -62,8 +142,62 @@ pub(crate) fn parse_scheduler_scheduling_mode(
}
}
/// Effective scheduler ordering config for requests that carry no resolved
/// routing policy.
///
/// Resolution order:
/// 1. the enabled system-default routing group's `default_policy`;
/// 2. the legacy system-config keys (`provider_priority_mode`,
/// `scheduling_mode`, `keep_priority_on_conversion`).
///
/// Step 2 only exists so deployments that never created a routing group keep
/// their behaviour; once the legacy keys are removed this function collapses
/// to step 1 plus `SchedulerOrderingConfig::default()`.
pub(crate) async fn read_scheduler_ordering_config(
state: &AppState,
) -> Result<SchedulerOrderingConfig, GatewayError> {
if let Some(config) = read_system_default_routing_ordering_config(state).await? {
return Ok(config);
}
read_legacy_scheduler_ordering_config(state).await
}
/// Ordering config from the enabled system-default routing group, if any.
pub(crate) async fn read_system_default_routing_ordering_config(
state: &AppState,
) -> Result<Option<SchedulerOrderingConfig>, GatewayError> {
let Some(group) = state
.find_routing_group(RoutingGroupLookupKey::SystemDefault)
.await?
.filter(|group| group.enabled)
else {
return Ok(None);
};
let default_policy = match group.config_json.get("default_policy") {
None | Some(serde_json::Value::Null) => RoutingDefaultPolicy::default(),
Some(value) => match serde_json::from_value::<RoutingDefaultPolicy>(value.clone()) {
Ok(policy) => policy,
Err(error) => {
warn!(
event_name = "scheduler_system_default_routing_policy_invalid",
log_type = "event",
group_id = %group.id,
error = %error,
"system default routing group has an invalid default_policy; ignoring it"
);
return Ok(None);
}
},
};
Ok(Some(SchedulerOrderingConfig::from_routing_default_policy(
&default_policy,
)))
}
/// Legacy system-config based ordering config. Kept only as a migration
/// fallback; see `read_scheduler_ordering_config`.
pub(crate) async fn read_legacy_scheduler_ordering_config(
state: &AppState,
) -> Result<SchedulerOrderingConfig, GatewayError> {
let priority_mode = parse_scheduler_priority_mode(
state
@@ -89,3 +223,162 @@ pub(crate) async fn read_scheduler_ordering_config(
keep_priority_on_conversion,
})
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use aether_data::repository::routing_profiles::InMemoryRoutingGroupRepository;
use aether_data_contracts::repository::routing_profiles::{
CreateRoutingGroupRecord, RoutingGroupLookupKey, RoutingGroupReadRepository,
RoutingGroupWriteRepository,
};
use serde_json::json;
use super::*;
use crate::data::GatewayDataState;
fn legacy_values() -> [(String, serde_json::Value); 3] {
[
("provider_priority_mode".to_string(), json!("global_key")),
("scheduling_mode".to_string(), json!("load_balance")),
("keep_priority_on_conversion".to_string(), json!(true)),
]
}
async fn create_system_default(
repository: &InMemoryRoutingGroupRepository,
enabled: bool,
config_json: serde_json::Value,
) {
repository
.create_routing_group(CreateRoutingGroupRecord {
id: "system-default".to_string(),
name: "system-default".to_string(),
description: None,
enabled,
is_system_default: true,
config_json,
version: 1,
created_at: 1,
updated_at: 1,
published_at: None,
})
.await
.unwrap();
}
#[tokio::test]
async fn system_default_routing_group_overrides_legacy_keys() {
let repository = Arc::new(InMemoryRoutingGroupRepository::default());
create_system_default(
&repository,
true,
json!({
"default_policy": {
"priority_mode": "provider",
"scheduling_mode": "fixed_order",
"keep_priority_on_conversion": false
}
}),
)
.await;
let state = AppState::new().unwrap().with_data_state_for_tests(
GatewayDataState::disabled()
.with_system_config_values_for_tests(legacy_values())
.with_routing_group_repository_for_tests(repository),
);
let config = read_scheduler_ordering_config(&state).await.unwrap();
assert_eq!(config.priority_mode, SchedulerPriorityMode::Provider);
assert_eq!(config.scheduling_mode, SchedulerSchedulingMode::FixedOrder);
assert!(!config.keep_priority_on_conversion);
}
#[tokio::test]
async fn missing_default_policy_in_system_default_group_uses_routing_defaults() {
let repository = Arc::new(InMemoryRoutingGroupRepository::default());
create_system_default(&repository, true, json!({})).await;
let state = AppState::new().unwrap().with_data_state_for_tests(
GatewayDataState::disabled()
.with_system_config_values_for_tests(legacy_values())
.with_routing_group_repository_for_tests(repository),
);
let config = read_scheduler_ordering_config(&state).await.unwrap();
assert_eq!(config, SchedulerOrderingConfig::default());
}
#[tokio::test]
async fn disabled_or_missing_system_default_group_falls_back_to_legacy_keys() {
let repository = Arc::new(InMemoryRoutingGroupRepository::default());
create_system_default(
&repository,
false,
json!({"default_policy": {"scheduling_mode": "fixed_order"}}),
)
.await;
let with_disabled_group = AppState::new().unwrap().with_data_state_for_tests(
GatewayDataState::disabled()
.with_system_config_values_for_tests(legacy_values())
.with_routing_group_repository_for_tests(repository),
);
let without_repository = AppState::new().unwrap().with_data_state_for_tests(
GatewayDataState::disabled().with_system_config_values_for_tests(legacy_values()),
);
for state in [with_disabled_group, without_repository] {
let config = read_scheduler_ordering_config(&state).await.unwrap();
assert_eq!(config.priority_mode, SchedulerPriorityMode::GlobalKey);
assert_eq!(config.scheduling_mode, SchedulerSchedulingMode::LoadBalance);
assert!(config.keep_priority_on_conversion);
}
}
#[tokio::test]
async fn bootstrap_creates_system_default_group_from_legacy_keys_once() {
let repository = Arc::new(InMemoryRoutingGroupRepository::default());
let state = AppState::new().unwrap().with_data_state_for_tests(
GatewayDataState::disabled()
.with_system_config_values_for_tests(legacy_values())
.with_routing_group_repository_for_tests(repository.clone()),
);
let created = state
.ensure_system_default_routing_group_inner()
.await
.unwrap()
.expect("first bootstrap should create the system default group");
assert!(created.enabled);
assert!(created.is_system_default);
assert_eq!(
created.config_json["default_policy"],
json!({
"priority_mode": "global_key",
"scheduling_mode": "load_balance",
"keep_priority_on_conversion": true
})
);
let second = state
.ensure_system_default_routing_group_inner()
.await
.unwrap();
assert!(second.is_none(), "bootstrap must be idempotent");
assert_eq!(
repository
.find_routing_group(RoutingGroupLookupKey::SystemDefault)
.await
.unwrap()
.map(|group| group.id),
Some(created.id)
);
let config = read_scheduler_ordering_config(&state).await.unwrap();
assert_eq!(config.priority_mode, SchedulerPriorityMode::GlobalKey);
assert_eq!(config.scheduling_mode, SchedulerSchedulingMode::LoadBalance);
assert!(config.keep_priority_on_conversion);
}
}
apps/aether-gateway/src/state/routing_profiles.rs
+78
View File
@@ -4,11 +4,89 @@ use aether_data_contracts::repository::routing_profiles::{
StoredRoutingGroup, StoredRoutingGroupBinding, StoredRoutingGroupVersion,
UpdateRoutingGroupBindingRecord, UpdateRoutingGroupRecord,
};
use aether_routing_core::RoutingGroupConfig;
use std::sync::Arc;
use tracing::warn;
use super::{AppState, GatewayError};
const BOOTSTRAP_SYSTEM_DEFAULT_ROUTING_GROUP_NAME: &str = "system-default";
impl AppState {
/// Make sure an enabled system-default routing group exists.
///
/// When none exists, one is created from the legacy scheduler system-config
/// keys so that removing those keys later does not change behaviour. Returns
/// the created group, or `None` when nothing had to be created (no routing
/// storage, no writer, or a system default already exists).
pub async fn ensure_system_default_routing_group(
&self,
) -> Result<Option<StoredRoutingGroup>, std::io::Error> {
self.ensure_system_default_routing_group_inner()
.await
.map_err(|err| std::io::Error::other(format!("{err:?}")))
}
pub(crate) async fn ensure_system_default_routing_group_inner(
&self,
) -> Result<Option<StoredRoutingGroup>, GatewayError> {
if !self.has_routing_group_data_reader() {
return Ok(None);
}
if self
.find_routing_group(RoutingGroupLookupKey::SystemDefault)
.await?
.is_some()
{
return Ok(None);
}
if !self.has_routing_group_data_writer() {
warn!(
event_name = "routing_system_default_bootstrap_skipped",
log_type = "event",
"no system default routing group exists and routing storage is read-only; scheduler falls back to legacy system config"
);
return Ok(None);
}
let legacy = crate::scheduler::config::read_legacy_scheduler_ordering_config(self).await?;
let config = RoutingGroupConfig {
default_policy: legacy.to_routing_default_policy(),
..RoutingGroupConfig::default()
};
let config_json = serde_json::to_value(config)
.map_err(|err| GatewayError::Internal(format!("serialize routing config: {err}")))?;
let name = if self
.find_routing_group(RoutingGroupLookupKey::Name(
BOOTSTRAP_SYSTEM_DEFAULT_ROUTING_GROUP_NAME,
))
.await?
.is_some()
{
format!(
"{BOOTSTRAP_SYSTEM_DEFAULT_ROUTING_GROUP_NAME}-{}",
&uuid::Uuid::new_v4().simple().to_string()[..8]
)
} else {
BOOTSTRAP_SYSTEM_DEFAULT_ROUTING_GROUP_NAME.to_string()
};
let now = crate::clock::current_unix_secs() as i64;
self.create_routing_group(CreateRoutingGroupRecord {
id: uuid::Uuid::new_v4().to_string(),
name,
description: Some("自动从旧版调度配置迁移生成的系统默认策略".to_string()),
enabled: true,
is_system_default: true,
config_json,
version: 1,
created_at: now,
updated_at: now,
published_at: Some(now),
})
.await
}
pub(crate) fn has_routing_group_data_reader(&self) -> bool {
self.data.has_routing_group_reader()
}