feat(routing): make routing profiles the sole scheduler policy source

Bootstrap an enabled system-default routing group from the legacy
scheduler config keys on startup, resolve the default ordering config
from that group before falling back to the legacy keys, and stop merging
keep_priority_on_conversion with the legacy flag when a policy is
resolved. Thread the policy-derived ordering config into candidate
preselection so it no longer reads system config independently.

Add per-API-format key priority overrides so a key serving several
formats keeps independent ordering, matching the legacy
global_priority_by_format semantics. Expose keep_priority_on_conversion
in the routing profile editor and read the effective policy in the
model routing preview, monitoring metrics and provider page badge.
This commit is contained in:
elky
2026-09-02 17:04:04 +08:00
parent 166236c2ee
commit 415b2da81b
32 changed files with 1001 additions and 128 deletions
@@ -81,6 +81,10 @@ pub enum RoutingAction {
SetKeyPriority {
key_id: String,
priority: i32,
/// When set, the override only applies to candidates served through
/// this API format; otherwise it applies to the key on every format.
#[serde(default, skip_serializing_if = "Option::is_none")]
api_format: Option<String>,
},
JsonPatchBody {
patch: Vec<RoutingJsonPatchOperation>,
+3 -3
View File
@@ -13,9 +13,9 @@ pub use actions::{
};
pub use conditions::{RoutingCondition, RoutingConditionContext, RoutingConditionOp};
pub use model::{
RoutingGroupBinding, RoutingGroupBindingSubject, RoutingGroupConfig, RoutingGroupRecord,
RoutingGroupVersionRecord, RoutingModelPolicy, RoutingPoolPolicyOverride, RoutingRule,
RoutingSchedulingPreset,
RoutingDefaultPolicy, RoutingGroupBinding, RoutingGroupBindingSubject, RoutingGroupConfig,
RoutingGroupRecord, RoutingGroupVersionRecord, RoutingModelPolicy, RoutingPoolPolicyOverride,
RoutingRule, RoutingSchedulingPreset,
};
pub use mutations::{
apply_json_patch_operations, validate_header_patch, validate_json_patch_operations,
+7
View File
@@ -44,6 +44,13 @@ pub struct RoutingModelPolicy {
pub provider_priority_overrides: BTreeMap<String, i32>,
#[serde(default)]
pub key_priority_overrides: BTreeMap<String, i32>,
/// Key priority overrides scoped to one API format: `api_format -> key_id -> priority`.
///
/// A key can serve several API formats and legacy `global_priority_by_format`
/// ranks it independently per format. Entries here take precedence over
/// `key_priority_overrides` when the candidate format matches.
#[serde(default)]
pub key_priority_overrides_by_format: BTreeMap<String, BTreeMap<String, i32>>,
#[serde(default)]
pub pool_priority_overrides: BTreeMap<String, i32>,
#[serde(default)]
+28 -6
View File
@@ -163,6 +163,13 @@ fn apply_model_policy(policy: &mut ResolvedRoutingPolicy, model_policy: &Routing
.iter()
.map(|(key, value)| (key.clone(), *value)),
);
for (api_format, overrides) in &model_policy.key_priority_overrides_by_format {
for (key_id, priority) in overrides {
policy
.ranking_overlay
.insert_key_priority_override_for_format(api_format, key_id.clone(), *priority);
}
}
policy.ranking_overlay.pool_priority_overrides.extend(
model_policy
.pool_priority_overrides
@@ -218,12 +225,27 @@ fn apply_action(
.provider_priority_overrides
.insert(provider_id.clone(), *priority);
}
RoutingAction::SetKeyPriority { key_id, priority } => {
policy
.ranking_overlay
.key_priority_overrides
.insert(key_id.clone(), *priority);
}
RoutingAction::SetKeyPriority {
key_id,
priority,
api_format,
} => match api_format
.as_deref()
.map(str::trim)
.filter(|f| !f.is_empty())
{
Some(api_format) => {
policy
.ranking_overlay
.insert_key_priority_override_for_format(api_format, key_id.clone(), *priority);
}
None => {
policy
.ranking_overlay
.key_priority_overrides
.insert(key_id.clone(), *priority);
}
},
RoutingAction::JsonPatchBody { patch } => {
validate_json_patch_operations(patch)
.map_err(|error| RoutingPolicyError::InvalidMutation(error.to_string()))?;
+96 -1
View File
@@ -21,6 +21,9 @@ pub struct RankingOverlay {
pub provider_priority_overrides: BTreeMap<String, i32>,
#[serde(default)]
pub key_priority_overrides: BTreeMap<String, i32>,
/// `api_format -> key_id -> priority`; see `RoutingModelPolicy`.
#[serde(default)]
pub key_priority_overrides_by_format: BTreeMap<String, BTreeMap<String, i32>>,
#[serde(default)]
pub pool_priority_overrides: BTreeMap<String, i32>,
}
@@ -40,6 +43,46 @@ impl RankingOverlay {
.unwrap_or(fallback)
}
/// Format-scoped key priority: a per-format override wins, then the
/// format-agnostic key override, then `fallback`.
pub fn key_priority_for_format(&self, key_id: &str, api_format: &str, fallback: i32) -> i32 {
self.key_priority_override_for_format(key_id, api_format)
.unwrap_or_else(|| self.key_priority(key_id, fallback))
}
/// Format-scoped key override using exact (case-insensitive) format match.
pub fn key_priority_override_for_format(&self, key_id: &str, api_format: &str) -> Option<i32> {
let api_format = api_format.trim();
self.key_priority_override_matching_format(key_id, |format| {
format.trim().eq_ignore_ascii_case(api_format)
})
}
/// Format-scoped key override where the caller decides how configured
/// format names match the candidate format (for alias-aware matching).
pub fn key_priority_override_matching_format(
&self,
key_id: &str,
mut format_matches: impl FnMut(&str) -> bool,
) -> Option<i32> {
self.key_priority_overrides_by_format
.iter()
.find(|(format, _)| format_matches(format))
.and_then(|(_, overrides)| overrides.get(key_id).copied())
}
pub fn insert_key_priority_override_for_format(
&mut self,
api_format: &str,
key_id: String,
priority: i32,
) {
self.key_priority_overrides_by_format
.entry(api_format.trim().to_ascii_lowercase())
.or_default()
.insert(key_id, priority);
}
pub fn pool_priority(&self, provider_id: &str, fallback: i32) -> i32 {
self.pool_priority_overrides
.get(provider_id)
@@ -89,6 +132,9 @@ pub struct RoutingCandidateFacts {
pub model_id: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub key_id: Option<String>,
/// Candidate API format used to resolve format-scoped key overrides.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub api_format: Option<String>,
pub provider_priority: i32,
pub key_priority: i32,
}
@@ -114,7 +160,12 @@ pub fn rank_vector_for_candidate(
CandidateKind::Provider => facts
.key_id
.as_deref()
.map(|key_id| overlay.key_priority(key_id, facts.key_priority))
.map(|key_id| match facts.api_format.as_deref() {
Some(api_format) => {
overlay.key_priority_for_format(key_id, api_format, facts.key_priority)
}
None => overlay.key_priority(key_id, facts.key_priority),
})
.unwrap_or(facts.key_priority),
CandidateKind::PoolGroup => {
overlay.pool_priority(&facts.provider_id, facts.key_priority)
@@ -142,6 +193,7 @@ mod tests {
endpoint_id: "endpoint-a".to_string(),
model_id: "model-a".to_string(),
key_id: Some("key-a".to_string()),
api_format: None,
provider_priority: 10,
key_priority: 20,
};
@@ -151,6 +203,47 @@ mod tests {
assert_eq!(vector.key_priority_after, 5);
}
#[test]
fn format_scoped_key_override_wins_over_key_override_for_matching_format() {
let mut overlay = RankingOverlay {
key_priority_overrides: BTreeMap::from([("key-a".to_string(), 5)]),
..RankingOverlay::default()
};
overlay.insert_key_priority_override_for_format("openai:chat", "key-a".to_string(), 1);
assert_eq!(
overlay.key_priority_for_format("key-a", "openai:chat", 20),
1
);
assert_eq!(
overlay.key_priority_for_format("key-a", "OpenAI:Chat", 20),
1
);
assert_eq!(
overlay.key_priority_for_format("key-a", "claude:messages", 20),
5
);
assert_eq!(
overlay.key_priority_for_format("key-b", "openai:chat", 20),
20
);
let facts = RoutingCandidateFacts {
candidate_kind: CandidateKind::Provider,
provider_id: "provider-a".to_string(),
endpoint_id: "endpoint-a".to_string(),
model_id: "model-a".to_string(),
key_id: Some("key-a".to_string()),
api_format: Some("openai:chat".to_string()),
provider_priority: 10,
key_priority: 20,
};
assert_eq!(
rank_vector_for_candidate(&overlay, &facts).key_priority_after,
1
);
}
#[test]
fn rank_vector_falls_back_to_existing_priorities() {
let facts = RoutingCandidateFacts {
@@ -159,6 +252,7 @@ mod tests {
endpoint_id: "endpoint-a".to_string(),
model_id: "model-a".to_string(),
key_id: Some("key-a".to_string()),
api_format: None,
provider_priority: 10,
key_priority: 20,
};
@@ -180,6 +274,7 @@ mod tests {
endpoint_id: "endpoint-a".to_string(),
model_id: "model-a".to_string(),
key_id: None,
api_format: None,
provider_priority: 10,
key_priority: 20,
};
@@ -285,6 +285,7 @@ mod tests {
RoutingAction::SetKeyPriority {
key_id: "key-1".to_string(),
priority: 1,
api_format: None,
},
"set_key_priority",
),