mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 17:30:23 +08:00
Restrict scheduler affinity to cache affinity mode
This commit is contained in:
@@ -47,34 +47,50 @@ pub(super) async fn select_minimal_candidate(
|
||||
enable_model_directives: bool,
|
||||
) -> Result<Option<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
|
||||
let affinity_epoch = runtime_state.scheduler_affinity_epoch();
|
||||
let ordering_config = runtime_state.read_scheduler_ordering_config().await?;
|
||||
let affinity_cache_key = build_scheduler_affinity_cache_key(
|
||||
auth_snapshot,
|
||||
api_format,
|
||||
global_model_name,
|
||||
client_session_affinity,
|
||||
);
|
||||
let selected = collect_selectable_candidates(
|
||||
let priority_affinity_key =
|
||||
scheduling_priority_affinity_key(auth_snapshot, ordering_config.scheduling_mode);
|
||||
let candidates = enumerate_scheduler_candidates(
|
||||
selection_row_source,
|
||||
runtime_state,
|
||||
api_format,
|
||||
global_model_name,
|
||||
require_streaming,
|
||||
required_capabilities,
|
||||
auth_snapshot,
|
||||
client_session_affinity,
|
||||
now_unix_secs,
|
||||
enable_model_directives,
|
||||
)
|
||||
.await?;
|
||||
let selected = collect_selectable_enumerated_candidates_with_skip_reasons(
|
||||
runtime_state,
|
||||
api_format,
|
||||
global_model_name,
|
||||
candidates,
|
||||
required_capabilities,
|
||||
auth_snapshot,
|
||||
client_session_affinity,
|
||||
now_unix_secs,
|
||||
ordering_config,
|
||||
priority_affinity_key,
|
||||
)
|
||||
.await?
|
||||
.0
|
||||
.into_iter()
|
||||
.next();
|
||||
if let Some(candidate) = selected.as_ref() {
|
||||
remember_scheduler_affinity(
|
||||
affinity_cache_key.as_deref(),
|
||||
runtime_state,
|
||||
candidate,
|
||||
Some(affinity_epoch),
|
||||
);
|
||||
if ordering_config.scheduling_mode == SchedulerSchedulingMode::CacheAffinity {
|
||||
if let Some(candidate) = selected.as_ref() {
|
||||
remember_scheduler_affinity(
|
||||
affinity_cache_key.as_deref(),
|
||||
runtime_state,
|
||||
candidate,
|
||||
Some(affinity_epoch),
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(selected)
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ use crate::data::candidate_selection::MinimalCandidateSelectionRowSource;
|
||||
use crate::data::GatewayDataState;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
use super::super::affinity::build_scheduler_affinity_cache_key;
|
||||
use super::super::runtime::should_skip_provider_quota;
|
||||
use super::super::selection::{
|
||||
collect_selectable_candidates as collect_selectable_candidates_impl,
|
||||
@@ -605,6 +606,46 @@ async fn cache_affinity_promotes_cached_scheduler_affinity_candidate_when_enable
|
||||
assert_eq!(selected.key_id, "key-b");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn load_balance_selection_does_not_remember_scheduler_affinity() {
|
||||
let row = sample_row();
|
||||
let candidates = Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
|
||||
row,
|
||||
]));
|
||||
let quotas = Arc::new(InMemoryProviderQuotaRepository::seed(vec![]));
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::with_candidate_selection_and_quota_for_tests(candidates, quotas)
|
||||
.with_system_config_values_for_tests(vec![(
|
||||
"scheduling_mode".to_string(),
|
||||
json!("load_balance"),
|
||||
)]),
|
||||
);
|
||||
let auth_snapshot = sample_auth_snapshot("affinity-key-1");
|
||||
let cache_key =
|
||||
build_scheduler_affinity_cache_key(Some(&auth_snapshot), "openai:chat", "gpt-4.1", None)
|
||||
.expect("scheduler affinity cache key should build");
|
||||
|
||||
let selected = select_candidate(
|
||||
state.data.as_ref(),
|
||||
&state,
|
||||
"openai:chat",
|
||||
"gpt-4.1",
|
||||
false,
|
||||
Some(&auth_snapshot),
|
||||
100,
|
||||
)
|
||||
.await
|
||||
.expect("selection should succeed")
|
||||
.expect("candidate should exist");
|
||||
|
||||
assert_eq!(selected.key_id, "key-1");
|
||||
assert!(state
|
||||
.read_scheduler_affinity_target(cache_key.as_str(), Duration::from_secs(300))
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn load_balance_ignores_provider_priority_and_cached_affinity() {
|
||||
let mut first = sample_row();
|
||||
|
||||
Reference in New Issue
Block a user