diff --git a/apps/aether-gateway/src/ai_serving/planner/standard/openai/chat/plans/stream.rs b/apps/aether-gateway/src/ai_serving/planner/standard/openai/chat/plans/stream.rs index 0f6ae98c6..c846b8df8 100644 --- a/apps/aether-gateway/src/ai_serving/planner/standard/openai/chat/plans/stream.rs +++ b/apps/aether-gateway/src/ai_serving/planner/standard/openai/chat/plans/stream.rs @@ -1,4 +1,4 @@ -use aether_routing_core::RoutingExecutionPolicy; +use aether_routing_core::{RoutingExecutionPolicy, RoutingSchedulingMode}; use async_trait::async_trait; use std::collections::VecDeque; use tracing::warn; @@ -207,7 +207,12 @@ impl LocalOpenAiChatStreamAttemptSource<'_> { async fn next_raw_attempt_with_target_select( &mut self, ) -> Result, GatewayError> { - let select_window = openai_chat_stream_target_select_window(); + let select_window = openai_chat_stream_target_select_window_for_mode( + self.input + .routing_policy + .as_ref() + .map(|policy| policy.scheduling_mode), + ); if select_window <= 1 { return self.next_raw_attempt_linear().await; } @@ -365,6 +370,15 @@ fn openai_chat_stream_target_select_window() -> usize { .clamp(1, MAX_OPENAI_CHAT_STREAM_TARGET_SELECT_WINDOW) } +fn openai_chat_stream_target_select_window_for_mode( + scheduling_mode: Option, +) -> usize { + if scheduling_mode == Some(RoutingSchedulingMode::FixedOrder) { + return 1; + } + openai_chat_stream_target_select_window() +} + #[derive(Clone, Copy)] struct TargetSelectCandidateIdentity<'a> { provider_id: &'a str, @@ -574,4 +588,14 @@ mod tests { assert_eq!(select_target_index(19, &choices), 1); } + + #[test] + fn fixed_order_disables_stream_target_selection() { + assert_eq!( + openai_chat_stream_target_select_window_for_mode(Some( + RoutingSchedulingMode::FixedOrder, + )), + 1 + ); + } }