fix(gateway): OpenAI 图像 finalize 统一返回 b64_json 并对齐 codex CLI 候选流式上游

- finalize 移除 response_format=url 的 data URL 分支,统一输出 b64_json
- image planner decision 依据请求体中的 stream 字段决定 upstream_is_stream
- openai chat sync plan 对 codex+openai:cli 候选强制上游流式
- 扩充 conversion registry 与 standard matrix 的全量 surface 对端测试
This commit is contained in:
fawney19
2026-04-24 19:01:11 +08:00
parent 4618184516
commit d0d71aa51a
6 changed files with 363 additions and 41 deletions

View File

@@ -106,13 +106,6 @@ fn maybe_build_local_openai_image_sync_finalize_response(
let Some(body_base64) = payload.body_base64.as_deref() else { let Some(body_base64) = payload.body_base64.as_deref() else {
return Ok(None); return Ok(None);
}; };
let response_format = report_context
.get("image_request")
.and_then(|value| value.get("response_format"))
.and_then(serde_json::Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or("url");
let default_output_format = report_context let default_output_format = report_context
.get("image_request") .get("image_request")
.and_then(|value| value.get("output_format")) .and_then(|value| value.get("output_format"))
@@ -227,21 +220,10 @@ fn maybe_build_local_openai_image_sync_finalize_response(
.get("output_format") .get("output_format")
.and_then(serde_json::Value::as_str) .and_then(serde_json::Value::as_str)
.unwrap_or(default_output_format); .unwrap_or(default_output_format);
if response_format.eq_ignore_ascii_case("b64_json") { serde_json::json!({
serde_json::json!({ "b64_json": b64_json,
"b64_json": b64_json, "revised_prompt": revised_prompt,
"revised_prompt": revised_prompt, })
})
} else {
serde_json::json!({
"url": format!(
"data:{};base64,{}",
image_output_mime_type(output_format),
b64_json
),
"revised_prompt": revised_prompt,
})
}
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let client_body_json = serde_json::json!({ let client_body_json = serde_json::json!({
@@ -259,14 +241,6 @@ fn maybe_build_local_openai_image_sync_finalize_response(
)?)) )?))
} }
fn image_output_mime_type(output_format: &str) -> &'static str {
match output_format.trim().to_ascii_lowercase().as_str() {
"jpeg" | "jpg" => "image/jpeg",
"webp" => "image/webp",
_ => "image/png",
}
}
#[cfg(test)] #[cfg(test)]
#[path = "../tests_sync.rs"] #[path = "../tests_sync.rs"]
mod tests; mod tests;

View File

@@ -1945,14 +1945,14 @@ async fn local_finalize_handles_openai_image_stream_response_from_output_item_do
} }
#[tokio::test] #[tokio::test]
async fn local_finalize_handles_openai_image_stream_response_with_url_response_format() { async fn local_finalize_returns_b64_json_even_when_url_response_format_requested() {
let payload = GatewaySyncReportRequest { let payload = GatewaySyncReportRequest {
trace_id: "trace-openai-image-finalize-url-123".to_string(), trace_id: "trace-openai-image-finalize-url-123".to_string(),
report_kind: "openai_image_sync_finalize".to_string(), report_kind: "openai_image_sync_finalize".to_string(),
report_context: Some(json!({ report_context: Some(json!({
"client_api_format": "openai:image", "client_api_format": "openai:image",
"provider_api_format": "openai:image", "provider_api_format": "openai:image",
"model": "gpt-image-1", "model": "dall-e-3",
"mapped_model": "gpt-5.4", "mapped_model": "gpt-5.4",
"image_request": { "image_request": {
"operation": "generate", "operation": "generate",
@@ -1994,12 +1994,113 @@ async fn local_finalize_handles_openai_image_stream_response_with_url_response_f
.expect("response body should read"); .expect("response body should read");
let response_json: serde_json::Value = let response_json: serde_json::Value =
serde_json::from_slice(&response_body).expect("response should be json"); serde_json::from_slice(&response_body).expect("response should be json");
assert_eq!( assert_eq!(response_json["data"][0]["b64_json"], "aGVsbG8=");
response_json["data"][0]["url"], assert!(response_json["data"][0].get("url").is_none());
"data:image/webp;base64,aGVsbG8="
);
assert_eq!( assert_eq!(
response_json["data"][0]["revised_prompt"], response_json["data"][0]["revised_prompt"],
"revised webp prompt" "revised webp prompt"
); );
} }
#[tokio::test]
async fn local_finalize_defaults_gpt_image_stream_response_to_b64_json() {
let payload = GatewaySyncReportRequest {
trace_id: "trace-openai-image-finalize-default-b64-123".to_string(),
report_kind: "openai_image_sync_finalize".to_string(),
report_context: Some(json!({
"client_api_format": "openai:image",
"provider_api_format": "openai:image",
"model": "gpt-image-2",
"mapped_model": "gpt-5.4",
"image_request": {
"operation": "generate",
"output_format": "png"
}
})),
status_code: 200,
headers: BTreeMap::from([(
"content-type".to_string(),
"text/event-stream".to_string(),
)]),
body_json: None,
client_body_json: None,
body_base64: Some(base64::engine::general_purpose::STANDARD.encode(
concat!(
"event: response.output_item.done\n",
"data: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"ig_default_123\",\"type\":\"image_generation_call\",\"output_format\":\"png\",\"result\":\"aGVsbG8=\"}}\n\n",
"event: response.completed\n",
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_img_default_123\",\"object\":\"response\",\"model\":\"gpt-5.4\",\"status\":\"completed\",\"output\":[]}}\n\n"
)
.as_bytes(),
)),
telemetry: None,
};
let outcome = maybe_build_local_core_sync_finalize_response(
"trace-openai-image-finalize-default-b64-123",
&test_decision(),
&payload,
)
.expect("image finalize should succeed")
.expect("image finalize should match");
let response_body = to_bytes(outcome.response.into_body(), usize::MAX)
.await
.expect("response body should read");
let response_json: serde_json::Value =
serde_json::from_slice(&response_body).expect("response should be json");
assert_eq!(response_json["data"][0]["b64_json"], "aGVsbG8=");
assert!(response_json["data"][0].get("url").is_none());
}
#[tokio::test]
async fn local_finalize_forces_gpt_image_stream_response_to_b64_json_even_when_url_requested() {
let payload = GatewaySyncReportRequest {
trace_id: "trace-openai-image-finalize-force-b64-123".to_string(),
report_kind: "openai_image_sync_finalize".to_string(),
report_context: Some(json!({
"client_api_format": "openai:image",
"provider_api_format": "openai:image",
"model": "gpt-image-2",
"mapped_model": "gpt-5.4",
"image_request": {
"operation": "generate",
"response_format": "url",
"output_format": "png"
}
})),
status_code: 200,
headers: BTreeMap::from([(
"content-type".to_string(),
"text/event-stream".to_string(),
)]),
body_json: None,
client_body_json: None,
body_base64: Some(base64::engine::general_purpose::STANDARD.encode(
concat!(
"event: response.output_item.done\n",
"data: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"ig_force_123\",\"type\":\"image_generation_call\",\"output_format\":\"png\",\"result\":\"aGVsbG8=\"}}\n\n",
"event: response.completed\n",
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_img_force_123\",\"object\":\"response\",\"model\":\"gpt-5.4\",\"status\":\"completed\",\"output\":[]}}\n\n"
)
.as_bytes(),
)),
telemetry: None,
};
let outcome = maybe_build_local_core_sync_finalize_response(
"trace-openai-image-finalize-force-b64-123",
&test_decision(),
&payload,
)
.expect("image finalize should succeed")
.expect("image finalize should match");
let response_body = to_bytes(outcome.response.into_body(), usize::MAX)
.await
.expect("response body should read");
let response_json: serde_json::Value =
serde_json::from_slice(&response_body).expect("response should be json");
assert_eq!(response_json["data"][0]["b64_json"], "aGVsbG8=");
assert!(response_json["data"][0].get("url").is_none());
}

View File

@@ -57,6 +57,11 @@ pub(super) async fn maybe_build_local_openai_image_decision_payload_for_candidat
extra_fields.insert("proxy".to_string(), proxy_value); extra_fields.insert("proxy".to_string(), proxy_value);
} }
extra_fields.insert("image_request".to_string(), resolved.input_summary.clone()); extra_fields.insert("image_request".to_string(), resolved.input_summary.clone());
let upstream_is_stream = resolved
.provider_request_body
.get("stream")
.and_then(serde_json::Value::as_bool)
.unwrap_or(spec_metadata.require_streaming);
let report_context = build_local_execution_report_context(LocalExecutionReportContextParts { let report_context = build_local_execution_report_context(LocalExecutionReportContextParts {
auth_context: &input.auth_context, auth_context: &input.auth_context,
request_id: trace_id, request_id: trace_id,
@@ -82,7 +87,7 @@ pub(super) async fn maybe_build_local_openai_image_decision_payload_for_candidat
original_request_body_json: Some(body_json), original_request_body_json: Some(body_json),
original_request_body_base64: body_base64, original_request_body_base64: body_base64,
client_requested_stream: spec_metadata.require_streaming, client_requested_stream: spec_metadata.require_streaming,
upstream_is_stream: spec_metadata.require_streaming, upstream_is_stream,
has_envelope: false, has_envelope: false,
needs_conversion: false, needs_conversion: false,
extra_fields, extra_fields,
@@ -117,7 +122,7 @@ pub(super) async fn maybe_build_local_openai_image_decision_payload_for_candidat
proxy, proxy,
tls_profile, tls_profile,
timeouts: resolve_transport_execution_timeouts(&transport), timeouts: resolve_transport_execution_timeouts(&transport),
upstream_is_stream: spec_metadata.require_streaming, upstream_is_stream,
report_kind: spec_metadata.report_kind.map(ToOwned::to_owned), report_kind: spec_metadata.report_kind.map(ToOwned::to_owned),
report_context: Some(report_context), report_context: Some(report_context),
auth_context: input.auth_context.clone(), auth_context: input.auth_context.clone(),

View File

@@ -10,12 +10,21 @@ use super::diagnostic::{
set_local_openai_chat_candidate_evaluation_diagnostic, set_local_openai_chat_miss_diagnostic, set_local_openai_chat_candidate_evaluation_diagnostic, set_local_openai_chat_miss_diagnostic,
}; };
use super::resolve::resolve_local_openai_chat_decision_input; use super::resolve::resolve_local_openai_chat_decision_input;
use crate::ai_pipeline::planner::common::OPENAI_CHAT_SYNC_PLAN_KIND; use crate::ai_pipeline::planner::common::{
force_upstream_streaming_for_provider, OPENAI_CHAT_SYNC_PLAN_KIND,
};
use crate::ai_pipeline::planner::plan_builders::{ use crate::ai_pipeline::planner::plan_builders::{
build_openai_chat_sync_plan_from_decision, LocalSyncPlanAndReport, build_openai_chat_sync_plan_from_decision, LocalSyncPlanAndReport,
}; };
use crate::ai_pipeline::planner::runtime_miss::apply_local_runtime_candidate_terminal_reason; use crate::ai_pipeline::planner::runtime_miss::apply_local_runtime_candidate_terminal_reason;
fn openai_chat_sync_upstream_is_stream_for_candidate(
provider_type: &str,
provider_api_format: &str,
) -> bool {
force_upstream_streaming_for_provider(provider_type, provider_api_format)
}
pub(crate) async fn build_local_openai_chat_sync_plan_and_reports( pub(crate) async fn build_local_openai_chat_sync_plan_and_reports(
state: &AppState, state: &AppState,
parts: &http::request::Parts, parts: &http::request::Parts,
@@ -88,6 +97,10 @@ pub(crate) async fn build_local_openai_chat_sync_plan_and_reports(
let mut plans = Vec::new(); let mut plans = Vec::new();
for attempt in attempts { for attempt in attempts {
let upstream_is_stream = openai_chat_sync_upstream_is_stream_for_candidate(
attempt.eligible.transport.provider.provider_type.as_str(),
attempt.eligible.provider_api_format.as_str(),
);
let Some(payload) = maybe_build_local_openai_chat_decision_payload_for_candidate( let Some(payload) = maybe_build_local_openai_chat_decision_payload_for_candidate(
state, state,
parts, parts,
@@ -97,7 +110,7 @@ pub(crate) async fn build_local_openai_chat_sync_plan_and_reports(
attempt, attempt,
OPENAI_CHAT_SYNC_PLAN_KIND, OPENAI_CHAT_SYNC_PLAN_KIND,
"openai_chat_sync_success", "openai_chat_sync_success",
false, upstream_is_stream,
) )
.await .await
else { else {
@@ -121,3 +134,24 @@ pub(crate) async fn build_local_openai_chat_sync_plan_and_reports(
Ok(plans) Ok(plans)
} }
#[cfg(test)]
mod tests {
use super::openai_chat_sync_upstream_is_stream_for_candidate;
#[test]
fn openai_chat_sync_forces_streaming_for_codex_openai_cli_candidates() {
assert!(openai_chat_sync_upstream_is_stream_for_candidate(
"codex",
"openai:cli"
));
assert!(!openai_chat_sync_upstream_is_stream_for_candidate(
"openai",
"openai:cli"
));
assert!(!openai_chat_sync_upstream_is_stream_for_candidate(
"codex",
"openai:chat"
));
}
}

View File

@@ -396,6 +396,25 @@ mod tests {
GatewayProviderTransportProvider, GatewayProviderTransportSnapshot, GatewayProviderTransportProvider, GatewayProviderTransportSnapshot,
}; };
const STANDARD_SURFACES: &[&str] = &[
"openai:chat",
"openai:cli",
"claude:chat",
"claude:cli",
"gemini:chat",
"gemini:cli",
];
fn expected_request_conversion_kind(provider_api_format: &str) -> RequestConversionKind {
match provider_api_format {
"openai:chat" => RequestConversionKind::ToOpenAIChat,
"openai:cli" => RequestConversionKind::ToOpenAIFamilyCli,
"claude:chat" | "claude:cli" => RequestConversionKind::ToClaudeStandard,
"gemini:chat" | "gemini:cli" => RequestConversionKind::ToGeminiStandard,
other => panic!("unexpected provider api format: {other}"),
}
}
#[test] #[test]
fn request_conversion_registry_supports_bidirectional_standard_matrix() { fn request_conversion_registry_supports_bidirectional_standard_matrix() {
assert_eq!( assert_eq!(
@@ -433,6 +452,27 @@ mod tests {
assert_eq!(request_conversion_kind("claude:chat", "claude:chat"), None); assert_eq!(request_conversion_kind("claude:chat", "claude:chat"), None);
} }
#[test]
fn request_conversion_registry_covers_all_standard_surface_pairs() {
for client_api_format in STANDARD_SURFACES {
for provider_api_format in STANDARD_SURFACES {
let actual = request_conversion_kind(client_api_format, provider_api_format);
if client_api_format == provider_api_format {
assert_eq!(
actual, None,
"{client_api_format} -> {provider_api_format} should be same-format"
);
} else {
assert_eq!(
actual,
Some(expected_request_conversion_kind(provider_api_format)),
"{client_api_format} -> {provider_api_format} should be routable"
);
}
}
}
}
#[test] #[test]
fn sync_response_conversion_registry_supports_bidirectional_standard_matrix() { fn sync_response_conversion_registry_supports_bidirectional_standard_matrix() {
assert_eq!( assert_eq!(
@@ -469,6 +509,57 @@ mod tests {
); );
} }
#[test]
fn sync_response_conversion_registry_covers_all_standard_surface_pairs() {
for provider_api_format in STANDARD_SURFACES {
for client_api_format in ["openai:chat", "claude:chat", "gemini:chat"] {
let actual =
sync_chat_response_conversion_kind(provider_api_format, client_api_format);
if *provider_api_format == client_api_format {
assert_eq!(
actual, None,
"{provider_api_format} -> {client_api_format} should be same-format"
);
} else {
let expected = match client_api_format {
"openai:chat" => SyncChatResponseConversionKind::ToOpenAIChat,
"claude:chat" => SyncChatResponseConversionKind::ToClaudeChat,
"gemini:chat" => SyncChatResponseConversionKind::ToGeminiChat,
other => panic!("unexpected chat client api format: {other}"),
};
assert_eq!(
actual,
Some(expected),
"{provider_api_format} -> {client_api_format} should finalize to chat"
);
}
}
for client_api_format in ["openai:cli", "claude:cli", "gemini:cli"] {
let actual =
sync_cli_response_conversion_kind(provider_api_format, client_api_format);
if *provider_api_format == client_api_format {
assert_eq!(
actual, None,
"{provider_api_format} -> {client_api_format} should be same-format"
);
} else {
let expected = match client_api_format {
"openai:cli" => SyncCliResponseConversionKind::ToOpenAIFamilyCli,
"claude:cli" => SyncCliResponseConversionKind::ToClaudeCli,
"gemini:cli" => SyncCliResponseConversionKind::ToGeminiCli,
other => panic!("unexpected cli client api format: {other}"),
};
assert_eq!(
actual,
Some(expected),
"{provider_api_format} -> {client_api_format} should finalize to cli"
);
}
}
}
}
#[test] #[test]
fn request_candidate_registry_excludes_compact_as_cross_format_target() { fn request_candidate_registry_excludes_compact_as_cross_format_target() {
assert_eq!( assert_eq!(

View File

@@ -150,7 +150,124 @@ pub fn build_standard_upstream_url(
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::build_standard_request_body; use super::build_standard_request_body;
use serde_json::json; use serde_json::{json, Value};
const STANDARD_SURFACES: &[&str] = &[
"openai:chat",
"openai:cli",
"claude:chat",
"claude:cli",
"gemini:chat",
"gemini:cli",
];
fn sample_request_for(api_format: &str) -> (Value, &'static str) {
match api_format {
"openai:chat" => (
json!({
"model": "source-model",
"messages": [
{"role": "system", "content": "Be concise."},
{"role": "user", "content": "Hello matrix"}
],
"max_tokens": 32
}),
"/v1/chat/completions",
),
"openai:cli" => (
json!({
"model": "source-model",
"instructions": "Be concise.",
"input": "Hello matrix",
"max_output_tokens": 32
}),
"/v1/responses",
),
"claude:chat" | "claude:cli" => (
json!({
"model": "source-model",
"system": "Be concise.",
"messages": [{
"role": "user",
"content": [{"type": "text", "text": "Hello matrix"}]
}],
"max_tokens": 32
}),
"/v1/messages",
),
"gemini:chat" | "gemini:cli" => (
json!({
"systemInstruction": {
"parts": [{"text": "Be concise."}]
},
"contents": [{
"role": "user",
"parts": [{"text": "Hello matrix"}]
}],
"generationConfig": {
"maxOutputTokens": 32
}
}),
"/v1beta/models/source-model:generateContent",
),
other => panic!("unexpected api format: {other}"),
}
}
fn assert_stream_flag(provider_api_format: &str, upstream_is_stream: bool, converted: &Value) {
match provider_api_format {
"openai:chat" | "openai:cli" | "claude:chat" | "claude:cli" => {
assert_eq!(
converted
.get("stream")
.and_then(Value::as_bool)
.unwrap_or(false),
upstream_is_stream,
"{provider_api_format} stream flag should follow upstream_is_stream"
);
}
"gemini:chat" | "gemini:cli" => {
assert!(
converted.get("stream").is_none(),
"gemini streaming is represented by endpoint URL, not request body"
);
}
other => panic!("unexpected provider api format: {other}"),
}
}
#[test]
fn builds_request_body_for_all_standard_surface_pairs_in_sync_and_stream_modes() {
for client_api_format in STANDARD_SURFACES {
let (request, request_path) = sample_request_for(client_api_format);
for provider_api_format in STANDARD_SURFACES {
for upstream_is_stream in [false, true] {
let converted = build_standard_request_body(
&request,
client_api_format,
"mapped-model",
"custom",
provider_api_format,
request_path,
upstream_is_stream,
None,
None,
)
.unwrap_or_else(|| {
panic!(
"{client_api_format} -> {provider_api_format} should build with upstream_is_stream={upstream_is_stream}"
)
});
assert_stream_flag(provider_api_format, upstream_is_stream, &converted);
assert!(
converted.to_string().contains("Hello matrix"),
"{client_api_format} -> {provider_api_format} should retain user content"
);
}
}
}
}
#[test] #[test]
fn builds_openai_chat_request_from_claude_chat_source() { fn builds_openai_chat_request_from_claude_chat_source() {