Merge remote-tracking branch 'origin/pr-483' into merge-pr-483

# Conflicts:
#	apps/aether-gateway/src/ai_serving/planner/standard/openai/chat/decision/payload.rs
#	apps/aether-gateway/src/ai_serving/planner/standard/openai/chat/decision/request.rs
#	apps/aether-gateway/src/ai_serving/planner/standard/openai/responses/decision/payload.rs
#	apps/aether-gateway/src/ai_serving/planner/standard/openai/responses/decision/request.rs
#	apps/aether-gateway/src/execution_runtime/chatgpt_web_image.rs
This commit is contained in:
fawney19
2026-05-19 02:23:14 +08:00
53 changed files with 8081 additions and 305 deletions
@@ -92,8 +92,7 @@ pub fn canonical_usage_from_openai_usage(value: Option<&Value>) -> Option<Canoni
let total_tokens = usage.get("total_tokens").and_then(Value::as_u64).unwrap_or(
input_tokens
.saturating_add(output_tokens)
.saturating_add(cache_creation_tokens)
.saturating_add(cache_read_tokens),
.saturating_add(reasoning_tokens),
);
if input_tokens == 0 && total_tokens > output_tokens {
input_tokens = total_tokens.saturating_sub(output_tokens);
@@ -150,8 +149,7 @@ pub fn canonical_usage_from_claude_usage(value: Option<&Value>) -> Option<Canoni
output_tokens,
total_tokens: input_tokens
.saturating_add(output_tokens)
.saturating_add(cache_creation_tokens)
.saturating_add(cache_read_tokens),
.saturating_add(reasoning_tokens),
cache_creation_tokens,
cache_creation_ephemeral_5m_tokens,
cache_creation_ephemeral_1h_tokens,
@@ -235,7 +233,7 @@ pub fn canonical_usage_from_gemini_usage(value: Option<&Value>) -> Option<Canoni
.unwrap_or(
input_tokens
.saturating_add(output_tokens)
.saturating_add(cache_read_tokens),
.saturating_add(reasoning_tokens),
);
Some(CanonicalUsage {
input_tokens,
@@ -8,6 +8,7 @@ use crate::formats::openai::chat::stream::{
OpenAIChatClientEmitter, OpenAIChatProviderState, OpenAIResponsesClientEmitter,
OpenAIResponsesProviderState,
};
use crate::formats::openai::image::stream::OpenAiImageStreamTerminalState;
use crate::formats::shared::error_body::{
build_core_error_body_for_client_format, LocalCoreSyncErrorKind,
};
@@ -97,7 +98,7 @@ impl StreamingStandardFormatMatrix {
#[derive(Default)]
pub struct StreamingStandardTerminalObserver {
provider: Option<ProviderStreamParser>,
provider: Option<TerminalStreamParser>,
latest_summary: Option<ExecutionStreamTerminalSummary>,
}
@@ -111,8 +112,17 @@ impl StreamingStandardTerminalObserver {
let Some(provider) = self.provider.as_mut() else {
return Ok(());
};
let frames = provider.push_line(report_context, line)?;
self.observe_frames(frames);
match provider {
TerminalStreamParser::Standard(provider) => {
let frames = provider.push_line(report_context, line)?;
self.observe_frames(frames);
}
TerminalStreamParser::OpenAIImage(provider) => {
if let Some(summary) = provider.push_line(report_context, line)? {
self.latest_summary = Some(summary);
}
}
}
Ok(())
}
@@ -124,8 +134,17 @@ impl StreamingStandardTerminalObserver {
let Some(provider) = self.provider.as_mut() else {
return Ok(self.latest_summary.clone());
};
let frames = provider.finish(report_context)?;
self.observe_frames(frames);
match provider {
TerminalStreamParser::Standard(provider) => {
let frames = provider.finish(report_context)?;
self.observe_frames(frames);
}
TerminalStreamParser::OpenAIImage(provider) => {
if let Some(summary) = provider.finish(report_context)? {
self.latest_summary = Some(summary);
}
}
}
Ok(self.latest_summary.clone())
}
@@ -153,7 +172,7 @@ impl StreamingStandardTerminalObserver {
return;
}
let provider_api_format = provider_api_format_for_context(report_context);
self.provider = ProviderStreamParser::for_api_format(provider_api_format.as_str());
self.provider = TerminalStreamParser::for_api_format(provider_api_format.as_str());
}
fn observe_frames(&mut self, frames: Vec<CanonicalStreamFrame>) {
@@ -194,6 +213,23 @@ impl StreamingStandardTerminalObserver {
}
}
enum TerminalStreamParser {
Standard(ProviderStreamParser),
OpenAIImage(OpenAiImageStreamTerminalState),
}
impl TerminalStreamParser {
fn for_api_format(provider_api_format: &str) -> Option<Self> {
if provider_api_format
.trim()
.eq_ignore_ascii_case("openai:image")
{
return Some(Self::OpenAIImage(OpenAiImageStreamTerminalState::default()));
}
ProviderStreamParser::for_api_format(provider_api_format).map(Self::Standard)
}
}
enum ProviderStreamParser {
OpenAIChat(OpenAIChatProviderState),
OpenAIResponses(OpenAIResponsesProviderState),
@@ -936,4 +972,86 @@ mod tests {
assert_eq!(summary.unknown_event_count, 1);
assert!(!summary.observed_finish);
}
#[test]
fn terminal_observer_tracks_openai_image_stream_usage() {
let mut report_context = report_context("openai:image", "openai:chat");
report_context["image_request"] = json!({
"size": "1024x1024",
"quality": "medium",
"output_format": "png",
});
let mut observer = StreamingStandardTerminalObserver::default();
observer
.push_line(
&report_context,
data_line(json!({
"type": "response.output_item.done",
"output_index": 0,
"item": {
"id": "ig_123",
"type": "image_generation_call",
"result": "aGVsbG8=",
},
})),
)
.expect("image output item should parse");
observer
.push_line(&report_context, b"\n".to_vec())
.expect("image output event should flush");
observer
.push_line(
&report_context,
data_line(json!({
"type": "response.completed",
"response": {
"id": "resp_image_123",
"model": "gpt-image-2",
"output": [],
"tool_usage": {
"image_gen": {
"input_tokens": 40,
"output_tokens": 60,
"total_tokens": 100,
},
},
},
})),
)
.expect("image completed should parse");
observer
.push_line(&report_context, b"\n".to_vec())
.expect("image completed event should flush");
let summary = observer
.finish(&report_context)
.expect("image summary should finish")
.expect("summary should exist");
let usage = summary
.standardized_usage
.expect("standardized usage should exist");
assert_eq!(summary.response_id.as_deref(), Some("resp_image_123"));
assert_eq!(summary.model.as_deref(), Some("gpt-image-2"));
assert_eq!(summary.finish_reason.as_deref(), Some("stop"));
assert!(summary.observed_finish);
assert_eq!(usage.input_tokens, 40);
assert_eq!(usage.output_tokens, 60);
assert_eq!(usage.request_count, 1);
assert_eq!(usage.dimensions.get("image_count"), Some(&json!(1)));
assert_eq!(usage.dimensions.get("total_tokens"), Some(&json!(100)));
assert_eq!(
usage.dimensions.get("image_size"),
Some(&json!("1024x1024"))
);
assert_eq!(
usage.dimensions.get("image_output_format"),
Some(&json!("png"))
);
assert_eq!(
usage.dimensions.get("image_quality"),
Some(&json!("medium"))
);
}
}
@@ -1,6 +1,6 @@
use serde_json::Value;
use crate::formats::openai::image::stream::OpenAiImageStreamState;
use crate::formats::openai::image::stream::{OpenAiImageChatStreamState, OpenAiImageStreamState};
use crate::formats::shared::model_directives::model_directive_display_model_from_report_context;
use crate::formats::shared::stream_core::StreamingStandardFormatMatrix;
use crate::formats::shared::AiSurfaceFinalizeError;
@@ -15,6 +15,7 @@ pub enum FinalizeStreamRewriteMode {
EnvelopeUnwrap,
ModelDirectiveDisplay,
OpenAiImage,
OpenAiImageToOpenAiChat,
Standard,
KiroToClaudeCli,
KiroToClaudeCliThenStandard,
@@ -57,6 +58,14 @@ pub fn resolve_finalize_stream_rewrite_mode(
.then_some(FinalizeStreamRewriteMode::KiroToClaudeCliThenStandard);
}
if provider_api_format == "openai:image" && client_api_format == "openai:chat" {
return Some(FinalizeStreamRewriteMode::OpenAiImageToOpenAiChat);
}
if provider_api_format == "openai:image" && client_api_format == "openai:image" {
return Some(FinalizeStreamRewriteMode::OpenAiImage);
}
if needs_conversion {
// CPA strategy: when provider and client share the same wire format
// (exact match or same family), pass through the stream verbatim.
@@ -73,10 +82,6 @@ pub fn resolve_finalize_stream_rewrite_mode(
.then_some(FinalizeStreamRewriteMode::Standard);
}
if provider_api_format == "openai:image" && client_api_format == "openai:image" {
return Some(FinalizeStreamRewriteMode::OpenAiImage);
}
if envelope_name.eq_ignore_ascii_case(KIRO_ENVELOPE_NAME) {
return (provider_api_format == "claude:messages"
&& client_api_format == "claude:messages")
@@ -106,6 +111,7 @@ enum AiSurfaceStreamRewriteState {
EnvelopeUnwrap,
ModelDirectiveDisplay,
OpenAiImage(Box<OpenAiImageStreamState>),
OpenAiImageToOpenAiChat(Box<OpenAiImageChatStreamState>),
Standard(Box<StreamingStandardFormatMatrix>),
KiroToClaudeCli(Box<KiroToClaudeCliStreamState>),
KiroToClaudeCliThenStandard {
@@ -132,6 +138,11 @@ pub fn maybe_build_ai_surface_stream_rewriter<'a>(
FinalizeStreamRewriteMode::OpenAiImage => {
AiSurfaceStreamRewriteState::OpenAiImage(Box::<OpenAiImageStreamState>::default())
}
FinalizeStreamRewriteMode::OpenAiImageToOpenAiChat => {
AiSurfaceStreamRewriteState::OpenAiImageToOpenAiChat(
Box::<OpenAiImageChatStreamState>::default(),
)
}
FinalizeStreamRewriteMode::Standard => {
AiSurfaceStreamRewriteState::Standard(Box::<StreamingStandardFormatMatrix>::default())
}
@@ -159,6 +170,9 @@ impl AiSurfaceStreamRewriter<'_> {
AiSurfaceStreamRewriteState::OpenAiImage(state) => {
state.push_chunk(self.report_context, chunk)
}
AiSurfaceStreamRewriteState::OpenAiImageToOpenAiChat(state) => {
state.push_chunk(self.report_context, chunk)
}
AiSurfaceStreamRewriteState::KiroToClaudeCli(state) => {
state.push_chunk(self.report_context, chunk)
}
@@ -183,6 +197,9 @@ impl AiSurfaceStreamRewriter<'_> {
pub fn finish(&mut self) -> Result<Vec<u8>, AiSurfaceFinalizeError> {
match &mut self.state {
AiSurfaceStreamRewriteState::OpenAiImage(state) => state.finish(self.report_context),
AiSurfaceStreamRewriteState::OpenAiImageToOpenAiChat(state) => {
state.finish(self.report_context)
}
AiSurfaceStreamRewriteState::KiroToClaudeCli(state) => {
state.finish(self.report_context)
}
@@ -228,6 +245,7 @@ impl AiSurfaceStreamRewriter<'_> {
transform_standard_line(state, self.report_context, line)
}
AiSurfaceStreamRewriteState::OpenAiImage(_)
| AiSurfaceStreamRewriteState::OpenAiImageToOpenAiChat(_)
| AiSurfaceStreamRewriteState::KiroToClaudeCli(_)
| AiSurfaceStreamRewriteState::KiroToClaudeCliThenStandard { .. } => Ok(Vec::new()),
}
@@ -686,4 +704,58 @@ data: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"thinki
Some(FinalizeStreamRewriteMode::OpenAiImage)
);
}
#[test]
fn rewrites_openai_image_stream_to_openai_chat_final_chunk() {
let report_context = json!({
"provider_api_format": "openai:image",
"client_api_format": "openai:chat",
"mapped_model": "gpt-image-2",
"request_id": "trace-image-chat-stream",
"needs_conversion": false,
});
assert_eq!(
resolve_finalize_stream_rewrite_mode(&report_context),
Some(FinalizeStreamRewriteMode::OpenAiImageToOpenAiChat)
);
let mut rewriter = maybe_build_ai_surface_stream_rewriter(Some(&report_context))
.expect("image to chat stream rewriter should exist");
let progress = rewriter
.push_chunk(
br#"event: response.image_generation_call.partial_image
data: {"type":"response.image_generation_call.partial_image","partial_image_b64":"cGFydGlhbA=="}
"#,
)
.expect("partial image should rewrite as progress");
let progress_text = String::from_utf8(progress).expect("progress output should be utf8");
assert!(progress_text.contains("\"object\":\"chat.completion.chunk\""));
assert!(!progress_text.contains("cGFydGlhbA=="));
let output_item = rewriter
.push_chunk(
br#"event: response.output_item.done
data: {"type":"response.output_item.done","item":{"type":"image_generation_call","id":"ig_1","result":"aGVsbG8=","output_format":"png"}}
"#,
)
.expect("output item should rewrite");
let output_item_text = String::from_utf8(output_item).expect("output item should be utf8");
assert!(output_item_text.is_empty());
let final_output = rewriter
.push_chunk(
br#"event: response.completed
data: {"type":"response.completed","response":{"id":"resp_123","model":"gpt-image-2","tool_usage":{"image_gen":{"total_tokens":0}},"output":[]}}
"#,
)
.expect("completed event should rewrite");
let final_text = String::from_utf8(final_output).expect("final output should be utf8");
assert!(final_text.contains("\"object\":\"chat.completion.chunk\""));
assert!(final_text.contains("![generated image](data:image/png;base64,aGVsbG8=)"));
assert!(final_text.contains("data: [DONE]"));
assert!(!final_text.contains("image_generation.completed"));
}
}
@@ -1,16 +1,21 @@
use std::borrow::Cow;
use aether_ai_formats::formats::conversion::response::{
convert_claude_response_to_openai_responses, convert_gemini_response_to_openai_responses,
convert_openai_chat_response_to_openai_responses,
};
use aether_contracts::{ExecutionStreamTerminalSummary, StandardizedUsage};
use serde_json::{json, Value};
use serde_json::{json, Map, Value};
use crate::formats::claude::messages::stream::ClaudeClientEmitter;
use crate::formats::gemini::generate_content::stream::GeminiClientEmitter;
use crate::formats::openai::chat::stream::{
OpenAIChatClientEmitter, OpenAIResponsesClientEmitter, OpenAIResponsesProviderState,
};
use crate::formats::shared::sse::encode_json_sse;
use crate::formats::shared::sse::{encode_done_sse, encode_json_sse};
use crate::formats::shared::stream_core::common::{
build_openai_chat_chunk, build_openai_chat_finish_chunk, build_openai_chat_usage_chunk,
};
use crate::formats::shared::stream_core::CanonicalStreamFrame;
use crate::formats::shared::AiSurfaceFinalizeError;
@@ -27,12 +32,25 @@ pub fn maybe_bridge_standard_sync_json_to_stream(
) -> Result<Option<SyncToStreamBridgeOutcome>, AiSurfaceFinalizeError> {
let provider_api_format = normalize_api_format(provider_api_format);
let client_api_format = normalize_api_format(client_api_format);
if client_api_format == "openai:image"
&& matches!(
provider_api_format.as_str(),
"openai:image" | "gemini:generate_content"
)
{
if provider_api_format == "openai:image" {
return match client_api_format.as_str() {
"openai:image" => {
maybe_bridge_openai_image_sync_json_to_stream(provider_body_json, report_context)
}
"openai:chat" => maybe_bridge_openai_image_sync_json_to_chat_stream(
provider_body_json,
report_context,
),
"openai:responses" | "openai:responses:compact" => {
maybe_bridge_openai_image_sync_json_to_responses_stream(
provider_body_json,
report_context,
)
}
_ => Ok(None),
};
}
if client_api_format == "openai:image" && provider_api_format == "gemini:generate_content" {
return maybe_bridge_openai_image_sync_json_to_stream(provider_body_json, report_context);
}
if !is_standard_api_format(provider_api_format.as_str())
@@ -72,49 +90,19 @@ fn maybe_bridge_openai_image_sync_json_to_stream(
provider_body_json: &Value,
report_context: Option<&Value>,
) -> Result<Option<SyncToStreamBridgeOutcome>, AiSurfaceFinalizeError> {
let provider_api_format = report_context
.and_then(|value| value.get("provider_api_format"))
.and_then(Value::as_str)
.map(str::trim)
.unwrap_or("openai:image");
let owned_response;
let provider_body_json = if provider_api_format == "gemini:generate_content" {
let Some(converted) =
crate::formats::shared::image_bridge::build_openai_image_response_from_gemini_response(
provider_body_json,
report_context,
)
else {
return Ok(None);
};
owned_response = converted;
&owned_response
} else if provider_body_json.get("output").is_some() && provider_body_json.get("data").is_none()
{
let Some(converted) = crate::formats::shared::image_bridge::build_openai_image_response_from_response_stream_sync_body(
provider_body_json,
report_context,
) else {
return Ok(None);
};
owned_response = converted;
&owned_response
} else {
provider_body_json
};
let Some(response) = provider_body_json.as_object() else {
return Ok(None);
};
let Some(image) = response
.get("data")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(Value::as_object)
.find_map(extract_openai_image_sync_b64_json)
let Some(provider_body_json) =
normalize_openai_image_sync_response(provider_body_json, report_context)?
else {
return Ok(None);
};
let Some(response) = provider_body_json.as_ref().as_object() else {
return Ok(None);
};
let outputs = collect_openai_image_outputs(response, report_context);
let Some(image) = outputs.iter().find_map(OpenAiImageOutput::b64_json) else {
return Ok(None);
};
let image_count = openai_image_response_image_count(response).max(outputs.len() as u64);
let usage = response.get("usage").cloned().unwrap_or(Value::Null);
let event_name = openai_image_completed_event_name(report_context);
let sse_body = encode_json_sse(
@@ -128,27 +116,461 @@ fn maybe_bridge_openai_image_sync_json_to_stream(
Ok(Some(SyncToStreamBridgeOutcome {
sse_body,
terminal_summary: Some(ExecutionStreamTerminalSummary {
standardized_usage: response
.get("usage")
.and_then(standardized_usage_from_openai_usage),
finish_reason: Some("stop".to_string()),
response_id: response
.get("id")
.and_then(Value::as_str)
.map(ToOwned::to_owned),
model: response
.get("model")
.and_then(Value::as_str)
.map(ToOwned::to_owned)
.or_else(|| image_bridge_model(report_context)),
observed_finish: true,
unknown_event_count: 0,
parser_error: None,
}),
terminal_summary: Some(openai_image_terminal_summary(
response,
report_context,
image_count,
)),
}))
}
fn maybe_bridge_openai_image_sync_json_to_chat_stream(
provider_body_json: &Value,
report_context: Option<&Value>,
) -> Result<Option<SyncToStreamBridgeOutcome>, AiSurfaceFinalizeError> {
let Some(provider_body_json) =
normalize_openai_image_sync_response(provider_body_json, report_context)?
else {
return Ok(None);
};
let Some(response) = provider_body_json.as_ref().as_object() else {
return Ok(None);
};
let outputs = collect_openai_image_outputs(response, report_context);
if outputs.is_empty() {
return Ok(None);
}
let image_count = openai_image_response_image_count(response).max(outputs.len() as u64);
let summary = openai_image_terminal_summary(response, report_context, image_count);
let response_id = openai_image_bridge_response_id(response, report_context, "chatcmpl-image");
let model = openai_image_bridge_response_model(response, report_context);
let content = outputs
.iter()
.enumerate()
.map(|(index, output)| output.markdown(index))
.collect::<Vec<_>>()
.join("\n\n");
let mut sse_body = Vec::new();
sse_body.extend(encode_json_sse(
None,
&build_openai_chat_chunk(&response_id, &model, content, None, None),
)?);
sse_body.extend(encode_json_sse(
None,
&build_openai_chat_finish_chunk(&response_id, &model, Some("stop")),
)?);
if let Some((input_tokens, output_tokens, total_tokens, reasoning_tokens)) = summary
.standardized_usage
.as_ref()
.and_then(openai_chat_usage_counts)
{
sse_body.extend(encode_json_sse(
None,
&build_openai_chat_usage_chunk(
&response_id,
&model,
input_tokens,
output_tokens,
total_tokens,
reasoning_tokens,
),
)?);
}
sse_body.extend(encode_done_sse());
Ok(Some(SyncToStreamBridgeOutcome {
sse_body,
terminal_summary: Some(summary),
}))
}
fn maybe_bridge_openai_image_sync_json_to_responses_stream(
provider_body_json: &Value,
report_context: Option<&Value>,
) -> Result<Option<SyncToStreamBridgeOutcome>, AiSurfaceFinalizeError> {
let Some(provider_body_json) =
normalize_openai_image_sync_response(provider_body_json, report_context)?
else {
return Ok(None);
};
let Some(response) = provider_body_json.as_ref().as_object() else {
return Ok(None);
};
let outputs = collect_openai_image_outputs(response, report_context);
if outputs.is_empty() {
return Ok(None);
}
let response_id = openai_image_bridge_response_id(response, report_context, "resp-image");
let model = openai_image_bridge_response_model(response, report_context);
let mut response_output = Vec::new();
for (index, output) in outputs.iter().enumerate() {
response_output.push(output.responses_image_generation_item(&response_id, index));
}
let mut response_object = Map::new();
response_object.insert("id".to_string(), Value::String(response_id.clone()));
response_object.insert("object".to_string(), Value::String("response".to_string()));
response_object.insert("model".to_string(), Value::String(model));
response_object.insert("status".to_string(), Value::String("completed".to_string()));
response_object.insert("output".to_string(), Value::Array(response_output.clone()));
if let Some(created) = response.get("created").and_then(Value::as_i64) {
response_object.insert("created_at".to_string(), json!(created));
}
if let Some(usage) = response.get("usage").filter(|value| value.is_object()) {
response_object.insert("usage".to_string(), usage.clone());
}
let mut sse_body = Vec::new();
for (index, item) in response_output.iter().enumerate() {
sse_body.extend(encode_json_sse(
Some("response.output_item.done"),
&json!({
"type": "response.output_item.done",
"output_index": index,
"item": item,
}),
)?);
}
sse_body.extend(encode_json_sse(
Some("response.completed"),
&json!({
"type": "response.completed",
"response": Value::Object(response_object),
}),
)?);
let image_count = openai_image_response_image_count(response).max(outputs.len() as u64);
Ok(Some(SyncToStreamBridgeOutcome {
sse_body,
terminal_summary: Some(openai_image_terminal_summary(
response,
report_context,
image_count,
)),
}))
}
#[derive(Clone, Debug)]
struct OpenAiImageOutput {
b64_json: Option<String>,
url: Option<String>,
mime_type: String,
output_format: Option<String>,
revised_prompt: Option<String>,
}
impl OpenAiImageOutput {
fn b64_json(&self) -> Option<String> {
self.b64_json
.clone()
.or_else(|| self.url.as_deref().and_then(extract_base64_from_data_url))
}
fn source_url(&self) -> Option<String> {
self.url.clone().or_else(|| {
self.b64_json
.as_ref()
.map(|value| format!("data:{};base64,{value}", self.mime_type))
})
}
fn markdown(&self, index: usize) -> String {
let alt = if index == 0 {
"generated image".to_string()
} else {
format!("generated image {}", index + 1)
};
match self.source_url() {
Some(url) => format!("![{alt}]({url})"),
None => String::new(),
}
}
fn responses_image_generation_item(&self, response_id: &str, index: usize) -> Value {
let mut item = Map::new();
item.insert(
"id".to_string(),
Value::String(format!("{response_id}_img_{index}")),
);
item.insert(
"type".to_string(),
Value::String("image_generation_call".to_string()),
);
item.insert("status".to_string(), Value::String("completed".to_string()));
if let Some(result) = self.b64_json().or_else(|| self.url.clone()) {
item.insert("result".to_string(), Value::String(result));
}
if let Some(output_format) = self.output_format.as_ref() {
item.insert(
"output_format".to_string(),
Value::String(output_format.clone()),
);
}
if let Some(revised_prompt) = self.revised_prompt.as_ref() {
item.insert(
"revised_prompt".to_string(),
Value::String(revised_prompt.clone()),
);
}
Value::Object(item)
}
}
fn normalize_openai_image_sync_response<'a>(
provider_body_json: &'a Value,
report_context: Option<&Value>,
) -> Result<Option<Cow<'a, Value>>, AiSurfaceFinalizeError> {
let provider_api_format = report_context
.and_then(|value| value.get("provider_api_format"))
.and_then(Value::as_str)
.map(str::trim)
.unwrap_or("openai:image");
if provider_api_format == "gemini:generate_content" {
let Some(converted) =
crate::formats::shared::image_bridge::build_openai_image_response_from_gemini_response(
provider_body_json,
report_context,
)
else {
return Ok(None);
};
return Ok(Some(Cow::Owned(converted)));
}
if provider_body_json.get("output").is_some() && provider_body_json.get("data").is_none() {
let Some(converted) = crate::formats::shared::image_bridge::build_openai_image_response_from_response_stream_sync_body(
provider_body_json,
report_context,
) else {
return Ok(None);
};
return Ok(Some(Cow::Owned(converted)));
}
Ok(Some(Cow::Borrowed(provider_body_json)))
}
fn collect_openai_image_outputs(
response: &Map<String, Value>,
report_context: Option<&Value>,
) -> Vec<OpenAiImageOutput> {
response
.get("data")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(Value::as_object)
.filter_map(|item| openai_image_output_from_item(item, report_context))
.collect()
}
fn openai_image_output_from_item(
item: &Map<String, Value>,
report_context: Option<&Value>,
) -> Option<OpenAiImageOutput> {
let b64_json = extract_openai_image_sync_b64_json(item);
let url = item
.get("url")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned);
if b64_json.is_none() && url.is_none() {
return None;
}
let output_format = item
.get("output_format")
.or_else(|| item.get("format"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.or_else(|| image_request_output_format(report_context));
let mime_type = url
.as_deref()
.and_then(extract_mime_type_from_data_url)
.or_else(|| {
output_format
.as_deref()
.map(mime_type_from_image_output_format)
})
.unwrap_or_else(|| "image/png".to_string());
let revised_prompt = item
.get("revised_prompt")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned);
Some(OpenAiImageOutput {
b64_json,
url,
mime_type,
output_format,
revised_prompt,
})
}
fn openai_image_response_image_count(response: &Map<String, Value>) -> u64 {
response
.get("data")
.and_then(Value::as_array)
.map(|items| items.len() as u64)
.unwrap_or(0)
}
fn openai_image_terminal_summary(
response: &Map<String, Value>,
report_context: Option<&Value>,
image_count: u64,
) -> ExecutionStreamTerminalSummary {
ExecutionStreamTerminalSummary {
standardized_usage: openai_image_standardized_usage(
response.get("usage"),
report_context,
image_count,
),
finish_reason: Some("stop".to_string()),
response_id: response
.get("id")
.and_then(Value::as_str)
.map(ToOwned::to_owned),
model: response
.get("model")
.and_then(Value::as_str)
.map(ToOwned::to_owned)
.or_else(|| image_bridge_model(report_context)),
observed_finish: true,
unknown_event_count: 0,
parser_error: None,
}
}
fn openai_image_standardized_usage(
usage: Option<&Value>,
report_context: Option<&Value>,
image_count: u64,
) -> Option<StandardizedUsage> {
let mut standardized_usage = usage
.and_then(standardized_usage_from_openai_usage)
.unwrap_or_else(StandardizedUsage::new);
if image_count > 0 {
standardized_usage.request_count = i64::try_from(image_count).unwrap_or(i64::MAX);
standardized_usage
.dimensions
.insert("image_count".to_string(), json!(image_count));
}
if let Some(output_format) = image_request_output_format(report_context) {
standardized_usage
.dimensions
.insert("image_output_format".to_string(), json!(output_format));
}
if let Some(size) = image_request_size(report_context) {
standardized_usage
.dimensions
.insert("image_size".to_string(), json!(size));
}
if let Some(quality) = image_request_quality(report_context) {
standardized_usage
.dimensions
.insert("image_quality".to_string(), json!(quality));
}
(standardized_usage.signal_score() > 0).then_some(standardized_usage)
}
fn openai_chat_usage_counts(usage: &StandardizedUsage) -> Option<(u64, u64, u64, u64)> {
let input_tokens = usage.input_tokens.max(0) as u64;
let output_tokens = usage.output_tokens.max(0) as u64;
let reasoning_tokens = usage.reasoning_tokens.max(0) as u64;
let total_tokens = usage
.dimensions
.get("total_tokens")
.and_then(Value::as_u64)
.unwrap_or_else(|| {
input_tokens
.saturating_add(output_tokens)
.saturating_add(reasoning_tokens)
});
(total_tokens > 0).then_some((input_tokens, output_tokens, total_tokens, reasoning_tokens))
}
fn openai_image_bridge_response_id(
response: &Map<String, Value>,
report_context: Option<&Value>,
fallback_prefix: &str,
) -> String {
response
.get("id")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.or_else(|| {
report_context
.and_then(|value| value.get("request_id"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(|value| format!("{fallback_prefix}-{value}"))
})
.unwrap_or_else(|| fallback_prefix.to_string())
}
fn openai_image_bridge_response_model(
response: &Map<String, Value>,
report_context: Option<&Value>,
) -> String {
response
.get("model")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.or_else(|| image_bridge_model(report_context))
.unwrap_or_else(|| "gpt-image".to_string())
}
fn image_request_output_format(report_context: Option<&Value>) -> Option<String> {
report_context
.and_then(|value| value.get("image_request"))
.and_then(|value| value.get("output_format"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
fn image_request_size(report_context: Option<&Value>) -> Option<String> {
report_context
.and_then(|value| value.get("image_request"))
.and_then(|value| value.get("size"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
fn image_request_quality(report_context: Option<&Value>) -> Option<String> {
report_context
.and_then(|value| value.get("image_request"))
.and_then(|value| value.get("quality"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
fn mime_type_from_image_output_format(output_format: &str) -> String {
match output_format.trim().to_ascii_lowercase().as_str() {
"jpg" | "jpeg" => "image/jpeg".to_string(),
"webp" => "image/webp".to_string(),
"png" => "image/png".to_string(),
value if !value.is_empty() => format!("image/{value}"),
_ => "image/png".to_string(),
}
}
fn normalize_api_format(value: &str) -> String {
aether_ai_formats::normalize_api_format_alias(value)
}
@@ -186,6 +608,14 @@ fn extract_base64_from_data_url(value: &str) -> Option<String> {
(!payload.trim().is_empty()).then(|| payload.trim().to_string())
}
fn extract_mime_type_from_data_url(value: &str) -> Option<String> {
let trimmed = value.trim();
let (metadata, _) = trimmed.split_once(',')?;
let mime_type = metadata.strip_prefix("data:")?.strip_suffix(";base64")?;
let mime_type = mime_type.trim();
(!mime_type.is_empty()).then(|| mime_type.to_string())
}
fn openai_image_completed_event_name(report_context: Option<&Value>) -> &'static str {
if openai_image_request_operation(report_context) == Some("edit") {
"image_edit.completed"
@@ -565,6 +995,76 @@ mod tests {
.cloned(),
Some(json!(100))
);
assert_eq!(
summary
.standardized_usage
.as_ref()
.and_then(|usage| usage.dimensions.get("image_count"))
.cloned(),
Some(json!(1))
);
}
#[test]
fn bridges_openai_image_sync_json_to_openai_chat_sse() {
let report_context = json!({
"provider_api_format": "openai:image",
"client_api_format": "openai:chat",
"mapped_model": "gpt-image-2",
"image_request": {
"operation": "generate",
"output_format": "png",
"size": "1024x1024",
"quality": "medium"
}
});
let outcome = maybe_bridge_standard_sync_json_to_stream(
&json!({
"id": "img_123",
"created": 1776971267,
"model": "gpt-image-2",
"data": [
{"b64_json": "aGVsbG8="},
{"b64_json": "d29ybGQ="}
],
"usage": {
"total_tokens": 100,
"input_tokens": 50,
"output_tokens": 50
}
}),
"openai:image",
"openai:chat",
Some(&report_context),
)
.expect("bridge should succeed")
.expect("bridge should produce sse");
let output = utf8(outcome.sse_body);
assert!(output.contains("\"object\":\"chat.completion.chunk\""));
assert!(output.contains("![generated image](data:image/png;base64,aGVsbG8=)"));
assert!(output.contains("![generated image 2](data:image/png;base64,d29ybGQ=)"));
assert!(output.contains("\"finish_reason\":\"stop\""));
assert!(output.contains("data: [DONE]"));
assert!(!output.contains("image_generation.completed"));
let summary = outcome
.terminal_summary
.expect("terminal summary should exist");
let usage = summary
.standardized_usage
.as_ref()
.expect("standard usage should exist");
assert_eq!(usage.request_count, 2);
assert_eq!(usage.dimensions.get("image_count"), Some(&json!(2)));
assert_eq!(
usage.dimensions.get("image_size"),
Some(&json!("1024x1024"))
);
assert_eq!(
usage.dimensions.get("image_quality"),
Some(&json!("medium"))
);
}
#[test]