fix: preserve upstream stream accept negotiation

This commit is contained in:
stabey
2026-05-09 12:29:23 +08:00
parent 7e5a08e09d
commit 5f4fa4ce1f
4 changed files with 143 additions and 14 deletions

View File

@@ -1,8 +1,11 @@
use axum::body::Bytes; use axum::body::Bytes;
use crate::ai_serving::{ use crate::ai_serving::{
endpoint_config_forces_upstream_stream_policy as endpoint_config_forces_upstream_stream_policy_impl,
enforce_request_body_stream_field as enforce_request_body_stream_field_impl,
force_upstream_streaming_for_provider as force_upstream_streaming_for_provider_impl, force_upstream_streaming_for_provider as force_upstream_streaming_for_provider_impl,
is_json_request, parse_direct_request_body as parse_direct_request_body_impl, is_json_request, parse_direct_request_body as parse_direct_request_body_impl,
resolve_upstream_is_stream_from_endpoint_config as resolve_upstream_is_stream_from_endpoint_config_impl,
}; };
pub(crate) use crate::ai_serving::{ pub(crate) use crate::ai_serving::{
CLAUDE_CHAT_STREAM_PLAN_KIND, CLAUDE_CHAT_SYNC_PLAN_KIND, CLAUDE_CLI_STREAM_PLAN_KIND, CLAUDE_CHAT_STREAM_PLAN_KIND, CLAUDE_CHAT_SYNC_PLAN_KIND, CLAUDE_CLI_STREAM_PLAN_KIND,
@@ -46,7 +49,7 @@ pub(crate) fn resolve_upstream_is_stream_for_provider(
) -> bool { ) -> bool {
let hard_requires_streaming = hard_requires_streaming let hard_requires_streaming = hard_requires_streaming
|| force_upstream_streaming_for_provider(provider_type, provider_api_format); || force_upstream_streaming_for_provider(provider_type, provider_api_format);
aether_ai_formats::resolve_upstream_is_stream_from_endpoint_config( resolve_upstream_is_stream_from_endpoint_config_impl(
endpoint_config, endpoint_config,
client_is_stream, client_is_stream,
hard_requires_streaming, hard_requires_streaming,
@@ -56,7 +59,7 @@ pub(crate) fn resolve_upstream_is_stream_for_provider(
pub(crate) fn endpoint_config_forces_body_stream_field( pub(crate) fn endpoint_config_forces_body_stream_field(
endpoint_config: Option<&serde_json::Value>, endpoint_config: Option<&serde_json::Value>,
) -> bool { ) -> bool {
aether_ai_formats::endpoint_config_forces_upstream_stream_policy(endpoint_config) endpoint_config_forces_upstream_stream_policy_impl(endpoint_config)
} }
pub(crate) fn request_requires_body_stream_field( pub(crate) fn request_requires_body_stream_field(
@@ -75,7 +78,7 @@ pub(crate) fn enforce_provider_body_stream_policy(
upstream_is_stream: bool, upstream_is_stream: bool,
require_body_stream_field: bool, require_body_stream_field: bool,
) { ) {
aether_ai_formats::enforce_request_body_stream_field( enforce_request_body_stream_field_impl(
provider_request_body, provider_request_body,
provider_api_format, provider_api_format,
upstream_is_stream, upstream_is_stream,

View File

@@ -18,6 +18,18 @@ use crate::ai_serving::transport::{
}; };
use crate::{AiExecutionDecision, GatewayError}; use crate::{AiExecutionDecision, GatewayError};
fn effective_stream_accept_mode(
payload_upstream_is_stream: bool,
provider_request_body: &serde_json::Value,
) -> bool {
payload_upstream_is_stream
|| provider_request_body
.as_object()
.and_then(|body| body.get("stream"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false)
}
pub(crate) fn build_openai_chat_stream_plan_from_decision( pub(crate) fn build_openai_chat_stream_plan_from_decision(
parts: &http::request::Parts, parts: &http::request::Parts,
body_json: &serde_json::Value, body_json: &serde_json::Value,
@@ -80,6 +92,8 @@ pub(crate) fn build_openai_chat_stream_plan_from_decision(
} }
provider_request_body provider_request_body
}; };
let effective_upstream_is_stream =
effective_stream_accept_mode(payload.upstream_is_stream, &provider_request_body_value);
let extra_headers = std::mem::take(&mut payload.extra_headers); let extra_headers = std::mem::take(&mut payload.extra_headers);
let mut provider_request_headers = let mut provider_request_headers =
build_standard_plan_fallback_headers(StandardPlanFallbackHeadersInput { build_standard_plan_fallback_headers(StandardPlanFallbackHeadersInput {
@@ -93,9 +107,9 @@ pub(crate) fn build_openai_chat_stream_plan_from_decision(
content_type: payload.content_type.as_deref(), content_type: payload.content_type.as_deref(),
provider_api_format: core.provider_api_format.as_str(), provider_api_format: core.provider_api_format.as_str(),
client_api_format: core.client_api_format.as_str(), client_api_format: core.client_api_format.as_str(),
upstream_is_stream: payload.upstream_is_stream, upstream_is_stream: effective_upstream_is_stream,
build_from_request_when_empty: true, build_from_request_when_empty: true,
accept_policy: StandardPlanFallbackAcceptPolicy::TextEventStreamIfStreaming, accept_policy: StandardPlanFallbackAcceptPolicy::TextEventStreamIfStreamingOrWildcard,
}); });
let content_type = payload let content_type = payload
.content_type .content_type
@@ -168,14 +182,18 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
.as_ref() .as_ref()
.and_then(|context| context.get("envelope_name")) .and_then(|context| context.get("envelope_name"))
.and_then(serde_json::Value::as_str); .and_then(serde_json::Value::as_str);
let accept_policy = if payload.upstream_is_stream let effective_upstream_is_stream =
effective_stream_accept_mode(payload.upstream_is_stream, &provider_request_body_value);
let accept_policy = if effective_upstream_is_stream
&& provider_adaptation_requires_eventstream_accept( && provider_adaptation_requires_eventstream_accept(
envelope_name, envelope_name,
core.provider_api_format.as_str(), core.provider_api_format.as_str(),
) { ) {
StandardPlanFallbackAcceptPolicy::ProviderEventStreamIfMissing StandardPlanFallbackAcceptPolicy::ProviderEventStreamIfMissing
} else { } else if envelope_name.is_some() {
StandardPlanFallbackAcceptPolicy::TextEventStreamIfStreaming StandardPlanFallbackAcceptPolicy::TextEventStreamIfStreaming
} else {
StandardPlanFallbackAcceptPolicy::TextEventStreamIfStreamingOrWildcard
}; };
let mut provider_request_headers = let mut provider_request_headers =
build_standard_plan_fallback_headers(StandardPlanFallbackHeadersInput { build_standard_plan_fallback_headers(StandardPlanFallbackHeadersInput {
@@ -189,7 +207,7 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
content_type: payload.content_type.as_deref(), content_type: payload.content_type.as_deref(),
provider_api_format: core.provider_api_format.as_str(), provider_api_format: core.provider_api_format.as_str(),
client_api_format: core.client_api_format.as_str(), client_api_format: core.client_api_format.as_str(),
upstream_is_stream: payload.upstream_is_stream, upstream_is_stream: effective_upstream_is_stream,
build_from_request_when_empty: false, build_from_request_when_empty: false,
accept_policy, accept_policy,
}); });
@@ -231,7 +249,7 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
plan_url = %plan.url, plan_url = %plan.url,
client_api_format = %plan.client_api_format, client_api_format = %plan.client_api_format,
provider_api_format = %plan.provider_api_format, provider_api_format = %plan.provider_api_format,
upstream_is_stream = payload.upstream_is_stream, upstream_is_stream = effective_upstream_is_stream,
compact, compact,
"gateway built local openai responses stream execution plan" "gateway built local openai responses stream execution plan"
); );

View File

@@ -40,7 +40,8 @@ pub(crate) use aether_ai_formats::api::{
copy_request_number_field_as, core_error_background_report_kind, copy_request_number_field_as, core_error_background_report_kind,
core_error_default_client_api_format, core_success_background_report_kind, core_error_default_client_api_format, core_success_background_report_kind,
default_model_for_openai_image_operation, encode_done_sse, encode_json_sse, default_model_for_openai_image_operation, encode_done_sse, encode_json_sse,
encode_kiro_sse_events, estimate_kiro_tokens, extract_openai_text_content, encode_kiro_sse_events, endpoint_config_forces_upstream_stream_policy,
enforce_request_body_stream_field, estimate_kiro_tokens, extract_openai_text_content,
find_kiro_real_thinking_end_tag, find_kiro_real_thinking_end_tag_at_buffer_end, find_kiro_real_thinking_end_tag, find_kiro_real_thinking_end_tag_at_buffer_end,
find_kiro_real_thinking_start_tag, force_upstream_streaming_for_provider, find_kiro_real_thinking_start_tag, force_upstream_streaming_for_provider,
gemini_request_is_image_generation, implicit_sync_finalize_report_kind, gemini_request_is_image_generation, implicit_sync_finalize_report_kind,
@@ -80,7 +81,8 @@ pub(crate) use aether_ai_formats::api::{
resolve_local_video_sync_spec, resolve_openai_chat_max_tokens, resolve_local_video_sync_spec, resolve_openai_chat_max_tokens,
resolve_openai_responses_stream_spec, resolve_openai_responses_sync_spec, resolve_openai_responses_stream_spec, resolve_openai_responses_sync_spec,
resolve_requested_gemini_image_model_for_request, resolve_requested_gemini_image_model_for_request,
resolve_requested_openai_image_model_for_request, sanitize_request_path, resolve_requested_openai_image_model_for_request,
resolve_upstream_is_stream_from_endpoint_config, sanitize_request_path,
sanitize_request_path_and_query, sanitize_request_query_string, sanitize_request_path_and_query, sanitize_request_query_string,
stream_body_contains_error_event, supports_stream_execution_decision_kind, stream_body_contains_error_event, supports_stream_execution_decision_kind,
supports_sync_execution_decision_kind, sync_chat_response_conversion_kind, supports_sync_execution_decision_kind, sync_chat_response_conversion_kind,

View File

@@ -40,6 +40,7 @@ pub struct StandardProviderRequestHeaders {
pub enum StandardPlanFallbackAcceptPolicy { pub enum StandardPlanFallbackAcceptPolicy {
None, None,
TextEventStreamIfStreaming, TextEventStreamIfStreaming,
TextEventStreamIfStreamingOrWildcard,
TextEventStreamRequired, TextEventStreamRequired,
ProviderEventStreamIfMissing, ProviderEventStreamIfMissing,
} }
@@ -132,19 +133,55 @@ pub fn build_standard_plan_fallback_headers(
.or_insert_with(|| "text/event-stream".to_string()); .or_insert_with(|| "text/event-stream".to_string());
} }
} }
StandardPlanFallbackAcceptPolicy::TextEventStreamIfStreamingOrWildcard => {
if input.upstream_is_stream {
set_accept_if_missing_or_wildcard(&mut headers, "text/event-stream");
}
}
StandardPlanFallbackAcceptPolicy::TextEventStreamRequired => { StandardPlanFallbackAcceptPolicy::TextEventStreamRequired => {
headers.insert("accept".to_string(), "text/event-stream".to_string()); headers.insert("accept".to_string(), "text/event-stream".to_string());
} }
StandardPlanFallbackAcceptPolicy::ProviderEventStreamIfMissing => { StandardPlanFallbackAcceptPolicy::ProviderEventStreamIfMissing => {
headers set_accept_if_missing_or_wildcard(&mut headers, "application/vnd.amazon.eventstream");
.entry("accept".to_string())
.or_insert_with(|| "application/vnd.amazon.eventstream".to_string());
} }
} }
headers headers
} }
fn set_accept_if_missing_or_wildcard(headers: &mut BTreeMap<String, String>, value: &str) {
let Some(existing_key) = headers
.keys()
.find(|key| key.eq_ignore_ascii_case("accept"))
.cloned()
else {
headers.insert("accept".to_string(), value.to_string());
return;
};
if headers
.get(&existing_key)
.is_some_and(|existing_value| accept_is_wildcard_only(existing_value))
{
headers.insert(existing_key, value.to_string());
}
}
fn accept_is_wildcard_only(value: &str) -> bool {
let mut saw_value = false;
for raw_part in value.split(',') {
let media_type = raw_part.trim().split(';').next().unwrap_or_default().trim();
if media_type.is_empty() {
continue;
}
saw_value = true;
if media_type != "*/*" {
return false;
}
}
saw_value
}
pub fn apply_standard_provider_request_body_rules( pub fn apply_standard_provider_request_body_rules(
mut provider_request_body: Value, mut provider_request_body: Value,
body_rules: Option<&Value>, body_rules: Option<&Value>,
@@ -431,6 +468,75 @@ mod tests {
); );
} }
#[test]
fn stream_fallback_headers_treat_wildcard_accept_as_absent() {
let mut request_headers = HeaderMap::new();
request_headers.insert(http::header::ACCEPT, "*/*".parse().expect("header"));
let headers = build_standard_plan_fallback_headers(StandardPlanFallbackHeadersInput {
request_headers: &request_headers,
existing_provider_request_headers: BTreeMap::new(),
auth_header: Some("authorization"),
auth_value: Some("Bearer secret"),
extra_headers: &BTreeMap::new(),
content_type: Some("application/json"),
provider_api_format: "openai:chat",
client_api_format: "openai:chat",
upstream_is_stream: true,
build_from_request_when_empty: true,
accept_policy: StandardPlanFallbackAcceptPolicy::TextEventStreamIfStreamingOrWildcard,
});
assert_eq!(
headers.get("accept"),
Some(&"text/event-stream".to_string())
);
}
#[test]
fn stream_fallback_headers_preserve_wildcard_in_missing_only_mode() {
let mut request_headers = HeaderMap::new();
request_headers.insert(http::header::ACCEPT, "*/*".parse().expect("header"));
let headers = build_standard_plan_fallback_headers(StandardPlanFallbackHeadersInput {
request_headers: &request_headers,
existing_provider_request_headers: BTreeMap::new(),
auth_header: Some("authorization"),
auth_value: Some("Bearer secret"),
extra_headers: &BTreeMap::new(),
content_type: Some("application/json"),
provider_api_format: "gemini:generate_content",
client_api_format: "openai:responses",
upstream_is_stream: true,
build_from_request_when_empty: true,
accept_policy: StandardPlanFallbackAcceptPolicy::TextEventStreamIfStreaming,
});
assert_eq!(headers.get("accept"), Some(&"*/*".to_string()));
}
#[test]
fn stream_fallback_headers_preserve_explicit_accept() {
let mut existing_headers = BTreeMap::new();
existing_headers.insert("accept".to_string(), "application/json".to_string());
let headers = build_standard_plan_fallback_headers(StandardPlanFallbackHeadersInput {
request_headers: &HeaderMap::new(),
existing_provider_request_headers: existing_headers,
auth_header: Some("authorization"),
auth_value: Some("Bearer secret"),
extra_headers: &BTreeMap::new(),
content_type: Some("application/json"),
provider_api_format: "openai:chat",
client_api_format: "openai:chat",
upstream_is_stream: true,
build_from_request_when_empty: false,
accept_policy: StandardPlanFallbackAcceptPolicy::TextEventStreamIfStreamingOrWildcard,
});
assert_eq!(headers.get("accept"), Some(&"application/json".to_string()));
}
#[test] #[test]
fn plan_fallback_headers_preserve_empty_existing_mode() { fn plan_fallback_headers_preserve_empty_existing_mode() {
let headers = build_standard_plan_fallback_headers(StandardPlanFallbackHeadersInput { let headers = build_standard_plan_fallback_headers(StandardPlanFallbackHeadersInput {