mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-04 00:17:45 +08:00
fix(routing): preserve fixed order for streaming chat
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
use aether_routing_core::RoutingExecutionPolicy;
|
||||
use aether_routing_core::{RoutingExecutionPolicy, RoutingSchedulingMode};
|
||||
use async_trait::async_trait;
|
||||
use std::collections::VecDeque;
|
||||
use tracing::warn;
|
||||
@@ -207,7 +207,12 @@ impl LocalOpenAiChatStreamAttemptSource<'_> {
|
||||
async fn next_raw_attempt_with_target_select(
|
||||
&mut self,
|
||||
) -> Result<Option<LocalOpenAiChatCandidateAttempt>, GatewayError> {
|
||||
let select_window = openai_chat_stream_target_select_window();
|
||||
let select_window = openai_chat_stream_target_select_window_for_mode(
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.scheduling_mode),
|
||||
);
|
||||
if select_window <= 1 {
|
||||
return self.next_raw_attempt_linear().await;
|
||||
}
|
||||
@@ -365,6 +370,15 @@ fn openai_chat_stream_target_select_window() -> usize {
|
||||
.clamp(1, MAX_OPENAI_CHAT_STREAM_TARGET_SELECT_WINDOW)
|
||||
}
|
||||
|
||||
fn openai_chat_stream_target_select_window_for_mode(
|
||||
scheduling_mode: Option<RoutingSchedulingMode>,
|
||||
) -> usize {
|
||||
if scheduling_mode == Some(RoutingSchedulingMode::FixedOrder) {
|
||||
return 1;
|
||||
}
|
||||
openai_chat_stream_target_select_window()
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct TargetSelectCandidateIdentity<'a> {
|
||||
provider_id: &'a str,
|
||||
@@ -574,4 +588,14 @@ mod tests {
|
||||
|
||||
assert_eq!(select_target_index(19, &choices), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fixed_order_disables_stream_target_selection() {
|
||||
assert_eq!(
|
||||
openai_chat_stream_target_select_window_for_mode(Some(
|
||||
RoutingSchedulingMode::FixedOrder,
|
||||
)),
|
||||
1
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user