Restrict scheduler affinity to cache affinity mode

This commit is contained in:
fawney19
2026-05-11 14:06:49 +08:00
parent e91c874863
commit 247ea9d1bd
12 changed files with 515 additions and 65 deletions

View File

@@ -47,34 +47,50 @@ pub(super) async fn select_minimal_candidate(
enable_model_directives: bool,
) -> Result<Option<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
let affinity_epoch = runtime_state.scheduler_affinity_epoch();
let ordering_config = runtime_state.read_scheduler_ordering_config().await?;
let affinity_cache_key = build_scheduler_affinity_cache_key(
auth_snapshot,
api_format,
global_model_name,
client_session_affinity,
);
let selected = collect_selectable_candidates(
let priority_affinity_key =
scheduling_priority_affinity_key(auth_snapshot, ordering_config.scheduling_mode);
let candidates = enumerate_scheduler_candidates(
selection_row_source,
runtime_state,
api_format,
global_model_name,
require_streaming,
required_capabilities,
auth_snapshot,
client_session_affinity,
now_unix_secs,
enable_model_directives,
)
.await?;
let selected = collect_selectable_enumerated_candidates_with_skip_reasons(
runtime_state,
api_format,
global_model_name,
candidates,
required_capabilities,
auth_snapshot,
client_session_affinity,
now_unix_secs,
ordering_config,
priority_affinity_key,
)
.await?
.0
.into_iter()
.next();
if let Some(candidate) = selected.as_ref() {
remember_scheduler_affinity(
affinity_cache_key.as_deref(),
runtime_state,
candidate,
Some(affinity_epoch),
);
if ordering_config.scheduling_mode == SchedulerSchedulingMode::CacheAffinity {
if let Some(candidate) = selected.as_ref() {
remember_scheduler_affinity(
affinity_cache_key.as_deref(),
runtime_state,
candidate,
Some(affinity_epoch),
);
}
}
Ok(selected)
}

View File

@@ -22,6 +22,7 @@ use crate::data::candidate_selection::MinimalCandidateSelectionRowSource;
use crate::data::GatewayDataState;
use crate::{AppState, GatewayError};
use super::super::affinity::build_scheduler_affinity_cache_key;
use super::super::runtime::should_skip_provider_quota;
use super::super::selection::{
collect_selectable_candidates as collect_selectable_candidates_impl,
@@ -605,6 +606,46 @@ async fn cache_affinity_promotes_cached_scheduler_affinity_candidate_when_enable
assert_eq!(selected.key_id, "key-b");
}
#[tokio::test]
async fn load_balance_selection_does_not_remember_scheduler_affinity() {
let row = sample_row();
let candidates = Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
row,
]));
let quotas = Arc::new(InMemoryProviderQuotaRepository::seed(vec![]));
let state = AppState::new()
.expect("state should build")
.with_data_state_for_tests(
GatewayDataState::with_candidate_selection_and_quota_for_tests(candidates, quotas)
.with_system_config_values_for_tests(vec![(
"scheduling_mode".to_string(),
json!("load_balance"),
)]),
);
let auth_snapshot = sample_auth_snapshot("affinity-key-1");
let cache_key =
build_scheduler_affinity_cache_key(Some(&auth_snapshot), "openai:chat", "gpt-4.1", None)
.expect("scheduler affinity cache key should build");
let selected = select_candidate(
state.data.as_ref(),
&state,
"openai:chat",
"gpt-4.1",
false,
Some(&auth_snapshot),
100,
)
.await
.expect("selection should succeed")
.expect("candidate should exist");
assert_eq!(selected.key_id, "key-1");
assert!(state
.read_scheduler_affinity_target(cache_key.as_str(), Duration::from_secs(300))
.is_none());
}
#[tokio::test]
async fn load_balance_ignores_provider_priority_and_cached_affinity() {
let mut first = sample_row();