mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-13 12:39:50 +08:00
Merge remote-tracking branch 'origin/pr-483' into merge-pr-483
# Conflicts: # apps/aether-gateway/src/ai_serving/planner/standard/openai/chat/decision/payload.rs # apps/aether-gateway/src/ai_serving/planner/standard/openai/chat/decision/request.rs # apps/aether-gateway/src/ai_serving/planner/standard/openai/responses/decision/payload.rs # apps/aether-gateway/src/ai_serving/planner/standard/openai/responses/decision/request.rs # apps/aether-gateway/src/execution_runtime/chatgpt_web_image.rs
This commit is contained in:
@@ -92,8 +92,7 @@ pub fn canonical_usage_from_openai_usage(value: Option<&Value>) -> Option<Canoni
|
||||
let total_tokens = usage.get("total_tokens").and_then(Value::as_u64).unwrap_or(
|
||||
input_tokens
|
||||
.saturating_add(output_tokens)
|
||||
.saturating_add(cache_creation_tokens)
|
||||
.saturating_add(cache_read_tokens),
|
||||
.saturating_add(reasoning_tokens),
|
||||
);
|
||||
if input_tokens == 0 && total_tokens > output_tokens {
|
||||
input_tokens = total_tokens.saturating_sub(output_tokens);
|
||||
@@ -150,8 +149,7 @@ pub fn canonical_usage_from_claude_usage(value: Option<&Value>) -> Option<Canoni
|
||||
output_tokens,
|
||||
total_tokens: input_tokens
|
||||
.saturating_add(output_tokens)
|
||||
.saturating_add(cache_creation_tokens)
|
||||
.saturating_add(cache_read_tokens),
|
||||
.saturating_add(reasoning_tokens),
|
||||
cache_creation_tokens,
|
||||
cache_creation_ephemeral_5m_tokens,
|
||||
cache_creation_ephemeral_1h_tokens,
|
||||
@@ -235,7 +233,7 @@ pub fn canonical_usage_from_gemini_usage(value: Option<&Value>) -> Option<Canoni
|
||||
.unwrap_or(
|
||||
input_tokens
|
||||
.saturating_add(output_tokens)
|
||||
.saturating_add(cache_read_tokens),
|
||||
.saturating_add(reasoning_tokens),
|
||||
);
|
||||
Some(CanonicalUsage {
|
||||
input_tokens,
|
||||
|
||||
@@ -8,6 +8,7 @@ use crate::formats::openai::chat::stream::{
|
||||
OpenAIChatClientEmitter, OpenAIChatProviderState, OpenAIResponsesClientEmitter,
|
||||
OpenAIResponsesProviderState,
|
||||
};
|
||||
use crate::formats::openai::image::stream::OpenAiImageStreamTerminalState;
|
||||
use crate::formats::shared::error_body::{
|
||||
build_core_error_body_for_client_format, LocalCoreSyncErrorKind,
|
||||
};
|
||||
@@ -97,7 +98,7 @@ impl StreamingStandardFormatMatrix {
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct StreamingStandardTerminalObserver {
|
||||
provider: Option<ProviderStreamParser>,
|
||||
provider: Option<TerminalStreamParser>,
|
||||
latest_summary: Option<ExecutionStreamTerminalSummary>,
|
||||
}
|
||||
|
||||
@@ -111,8 +112,17 @@ impl StreamingStandardTerminalObserver {
|
||||
let Some(provider) = self.provider.as_mut() else {
|
||||
return Ok(());
|
||||
};
|
||||
let frames = provider.push_line(report_context, line)?;
|
||||
self.observe_frames(frames);
|
||||
match provider {
|
||||
TerminalStreamParser::Standard(provider) => {
|
||||
let frames = provider.push_line(report_context, line)?;
|
||||
self.observe_frames(frames);
|
||||
}
|
||||
TerminalStreamParser::OpenAIImage(provider) => {
|
||||
if let Some(summary) = provider.push_line(report_context, line)? {
|
||||
self.latest_summary = Some(summary);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -124,8 +134,17 @@ impl StreamingStandardTerminalObserver {
|
||||
let Some(provider) = self.provider.as_mut() else {
|
||||
return Ok(self.latest_summary.clone());
|
||||
};
|
||||
let frames = provider.finish(report_context)?;
|
||||
self.observe_frames(frames);
|
||||
match provider {
|
||||
TerminalStreamParser::Standard(provider) => {
|
||||
let frames = provider.finish(report_context)?;
|
||||
self.observe_frames(frames);
|
||||
}
|
||||
TerminalStreamParser::OpenAIImage(provider) => {
|
||||
if let Some(summary) = provider.finish(report_context)? {
|
||||
self.latest_summary = Some(summary);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(self.latest_summary.clone())
|
||||
}
|
||||
|
||||
@@ -153,7 +172,7 @@ impl StreamingStandardTerminalObserver {
|
||||
return;
|
||||
}
|
||||
let provider_api_format = provider_api_format_for_context(report_context);
|
||||
self.provider = ProviderStreamParser::for_api_format(provider_api_format.as_str());
|
||||
self.provider = TerminalStreamParser::for_api_format(provider_api_format.as_str());
|
||||
}
|
||||
|
||||
fn observe_frames(&mut self, frames: Vec<CanonicalStreamFrame>) {
|
||||
@@ -194,6 +213,23 @@ impl StreamingStandardTerminalObserver {
|
||||
}
|
||||
}
|
||||
|
||||
enum TerminalStreamParser {
|
||||
Standard(ProviderStreamParser),
|
||||
OpenAIImage(OpenAiImageStreamTerminalState),
|
||||
}
|
||||
|
||||
impl TerminalStreamParser {
|
||||
fn for_api_format(provider_api_format: &str) -> Option<Self> {
|
||||
if provider_api_format
|
||||
.trim()
|
||||
.eq_ignore_ascii_case("openai:image")
|
||||
{
|
||||
return Some(Self::OpenAIImage(OpenAiImageStreamTerminalState::default()));
|
||||
}
|
||||
ProviderStreamParser::for_api_format(provider_api_format).map(Self::Standard)
|
||||
}
|
||||
}
|
||||
|
||||
enum ProviderStreamParser {
|
||||
OpenAIChat(OpenAIChatProviderState),
|
||||
OpenAIResponses(OpenAIResponsesProviderState),
|
||||
@@ -936,4 +972,86 @@ mod tests {
|
||||
assert_eq!(summary.unknown_event_count, 1);
|
||||
assert!(!summary.observed_finish);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn terminal_observer_tracks_openai_image_stream_usage() {
|
||||
let mut report_context = report_context("openai:image", "openai:chat");
|
||||
report_context["image_request"] = json!({
|
||||
"size": "1024x1024",
|
||||
"quality": "medium",
|
||||
"output_format": "png",
|
||||
});
|
||||
let mut observer = StreamingStandardTerminalObserver::default();
|
||||
|
||||
observer
|
||||
.push_line(
|
||||
&report_context,
|
||||
data_line(json!({
|
||||
"type": "response.output_item.done",
|
||||
"output_index": 0,
|
||||
"item": {
|
||||
"id": "ig_123",
|
||||
"type": "image_generation_call",
|
||||
"result": "aGVsbG8=",
|
||||
},
|
||||
})),
|
||||
)
|
||||
.expect("image output item should parse");
|
||||
observer
|
||||
.push_line(&report_context, b"\n".to_vec())
|
||||
.expect("image output event should flush");
|
||||
observer
|
||||
.push_line(
|
||||
&report_context,
|
||||
data_line(json!({
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "resp_image_123",
|
||||
"model": "gpt-image-2",
|
||||
"output": [],
|
||||
"tool_usage": {
|
||||
"image_gen": {
|
||||
"input_tokens": 40,
|
||||
"output_tokens": 60,
|
||||
"total_tokens": 100,
|
||||
},
|
||||
},
|
||||
},
|
||||
})),
|
||||
)
|
||||
.expect("image completed should parse");
|
||||
observer
|
||||
.push_line(&report_context, b"\n".to_vec())
|
||||
.expect("image completed event should flush");
|
||||
|
||||
let summary = observer
|
||||
.finish(&report_context)
|
||||
.expect("image summary should finish")
|
||||
.expect("summary should exist");
|
||||
let usage = summary
|
||||
.standardized_usage
|
||||
.expect("standardized usage should exist");
|
||||
|
||||
assert_eq!(summary.response_id.as_deref(), Some("resp_image_123"));
|
||||
assert_eq!(summary.model.as_deref(), Some("gpt-image-2"));
|
||||
assert_eq!(summary.finish_reason.as_deref(), Some("stop"));
|
||||
assert!(summary.observed_finish);
|
||||
assert_eq!(usage.input_tokens, 40);
|
||||
assert_eq!(usage.output_tokens, 60);
|
||||
assert_eq!(usage.request_count, 1);
|
||||
assert_eq!(usage.dimensions.get("image_count"), Some(&json!(1)));
|
||||
assert_eq!(usage.dimensions.get("total_tokens"), Some(&json!(100)));
|
||||
assert_eq!(
|
||||
usage.dimensions.get("image_size"),
|
||||
Some(&json!("1024x1024"))
|
||||
);
|
||||
assert_eq!(
|
||||
usage.dimensions.get("image_output_format"),
|
||||
Some(&json!("png"))
|
||||
);
|
||||
assert_eq!(
|
||||
usage.dimensions.get("image_quality"),
|
||||
Some(&json!("medium"))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::formats::openai::image::stream::OpenAiImageStreamState;
|
||||
use crate::formats::openai::image::stream::{OpenAiImageChatStreamState, OpenAiImageStreamState};
|
||||
use crate::formats::shared::model_directives::model_directive_display_model_from_report_context;
|
||||
use crate::formats::shared::stream_core::StreamingStandardFormatMatrix;
|
||||
use crate::formats::shared::AiSurfaceFinalizeError;
|
||||
@@ -15,6 +15,7 @@ pub enum FinalizeStreamRewriteMode {
|
||||
EnvelopeUnwrap,
|
||||
ModelDirectiveDisplay,
|
||||
OpenAiImage,
|
||||
OpenAiImageToOpenAiChat,
|
||||
Standard,
|
||||
KiroToClaudeCli,
|
||||
KiroToClaudeCliThenStandard,
|
||||
@@ -57,6 +58,14 @@ pub fn resolve_finalize_stream_rewrite_mode(
|
||||
.then_some(FinalizeStreamRewriteMode::KiroToClaudeCliThenStandard);
|
||||
}
|
||||
|
||||
if provider_api_format == "openai:image" && client_api_format == "openai:chat" {
|
||||
return Some(FinalizeStreamRewriteMode::OpenAiImageToOpenAiChat);
|
||||
}
|
||||
|
||||
if provider_api_format == "openai:image" && client_api_format == "openai:image" {
|
||||
return Some(FinalizeStreamRewriteMode::OpenAiImage);
|
||||
}
|
||||
|
||||
if needs_conversion {
|
||||
// CPA strategy: when provider and client share the same wire format
|
||||
// (exact match or same family), pass through the stream verbatim.
|
||||
@@ -73,10 +82,6 @@ pub fn resolve_finalize_stream_rewrite_mode(
|
||||
.then_some(FinalizeStreamRewriteMode::Standard);
|
||||
}
|
||||
|
||||
if provider_api_format == "openai:image" && client_api_format == "openai:image" {
|
||||
return Some(FinalizeStreamRewriteMode::OpenAiImage);
|
||||
}
|
||||
|
||||
if envelope_name.eq_ignore_ascii_case(KIRO_ENVELOPE_NAME) {
|
||||
return (provider_api_format == "claude:messages"
|
||||
&& client_api_format == "claude:messages")
|
||||
@@ -106,6 +111,7 @@ enum AiSurfaceStreamRewriteState {
|
||||
EnvelopeUnwrap,
|
||||
ModelDirectiveDisplay,
|
||||
OpenAiImage(Box<OpenAiImageStreamState>),
|
||||
OpenAiImageToOpenAiChat(Box<OpenAiImageChatStreamState>),
|
||||
Standard(Box<StreamingStandardFormatMatrix>),
|
||||
KiroToClaudeCli(Box<KiroToClaudeCliStreamState>),
|
||||
KiroToClaudeCliThenStandard {
|
||||
@@ -132,6 +138,11 @@ pub fn maybe_build_ai_surface_stream_rewriter<'a>(
|
||||
FinalizeStreamRewriteMode::OpenAiImage => {
|
||||
AiSurfaceStreamRewriteState::OpenAiImage(Box::<OpenAiImageStreamState>::default())
|
||||
}
|
||||
FinalizeStreamRewriteMode::OpenAiImageToOpenAiChat => {
|
||||
AiSurfaceStreamRewriteState::OpenAiImageToOpenAiChat(
|
||||
Box::<OpenAiImageChatStreamState>::default(),
|
||||
)
|
||||
}
|
||||
FinalizeStreamRewriteMode::Standard => {
|
||||
AiSurfaceStreamRewriteState::Standard(Box::<StreamingStandardFormatMatrix>::default())
|
||||
}
|
||||
@@ -159,6 +170,9 @@ impl AiSurfaceStreamRewriter<'_> {
|
||||
AiSurfaceStreamRewriteState::OpenAiImage(state) => {
|
||||
state.push_chunk(self.report_context, chunk)
|
||||
}
|
||||
AiSurfaceStreamRewriteState::OpenAiImageToOpenAiChat(state) => {
|
||||
state.push_chunk(self.report_context, chunk)
|
||||
}
|
||||
AiSurfaceStreamRewriteState::KiroToClaudeCli(state) => {
|
||||
state.push_chunk(self.report_context, chunk)
|
||||
}
|
||||
@@ -183,6 +197,9 @@ impl AiSurfaceStreamRewriter<'_> {
|
||||
pub fn finish(&mut self) -> Result<Vec<u8>, AiSurfaceFinalizeError> {
|
||||
match &mut self.state {
|
||||
AiSurfaceStreamRewriteState::OpenAiImage(state) => state.finish(self.report_context),
|
||||
AiSurfaceStreamRewriteState::OpenAiImageToOpenAiChat(state) => {
|
||||
state.finish(self.report_context)
|
||||
}
|
||||
AiSurfaceStreamRewriteState::KiroToClaudeCli(state) => {
|
||||
state.finish(self.report_context)
|
||||
}
|
||||
@@ -228,6 +245,7 @@ impl AiSurfaceStreamRewriter<'_> {
|
||||
transform_standard_line(state, self.report_context, line)
|
||||
}
|
||||
AiSurfaceStreamRewriteState::OpenAiImage(_)
|
||||
| AiSurfaceStreamRewriteState::OpenAiImageToOpenAiChat(_)
|
||||
| AiSurfaceStreamRewriteState::KiroToClaudeCli(_)
|
||||
| AiSurfaceStreamRewriteState::KiroToClaudeCliThenStandard { .. } => Ok(Vec::new()),
|
||||
}
|
||||
@@ -686,4 +704,58 @@ data: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"thinki
|
||||
Some(FinalizeStreamRewriteMode::OpenAiImage)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rewrites_openai_image_stream_to_openai_chat_final_chunk() {
|
||||
let report_context = json!({
|
||||
"provider_api_format": "openai:image",
|
||||
"client_api_format": "openai:chat",
|
||||
"mapped_model": "gpt-image-2",
|
||||
"request_id": "trace-image-chat-stream",
|
||||
"needs_conversion": false,
|
||||
});
|
||||
assert_eq!(
|
||||
resolve_finalize_stream_rewrite_mode(&report_context),
|
||||
Some(FinalizeStreamRewriteMode::OpenAiImageToOpenAiChat)
|
||||
);
|
||||
let mut rewriter = maybe_build_ai_surface_stream_rewriter(Some(&report_context))
|
||||
.expect("image to chat stream rewriter should exist");
|
||||
|
||||
let progress = rewriter
|
||||
.push_chunk(
|
||||
br#"event: response.image_generation_call.partial_image
|
||||
data: {"type":"response.image_generation_call.partial_image","partial_image_b64":"cGFydGlhbA=="}
|
||||
|
||||
"#,
|
||||
)
|
||||
.expect("partial image should rewrite as progress");
|
||||
let progress_text = String::from_utf8(progress).expect("progress output should be utf8");
|
||||
assert!(progress_text.contains("\"object\":\"chat.completion.chunk\""));
|
||||
assert!(!progress_text.contains("cGFydGlhbA=="));
|
||||
|
||||
let output_item = rewriter
|
||||
.push_chunk(
|
||||
br#"event: response.output_item.done
|
||||
data: {"type":"response.output_item.done","item":{"type":"image_generation_call","id":"ig_1","result":"aGVsbG8=","output_format":"png"}}
|
||||
|
||||
"#,
|
||||
)
|
||||
.expect("output item should rewrite");
|
||||
let output_item_text = String::from_utf8(output_item).expect("output item should be utf8");
|
||||
assert!(output_item_text.is_empty());
|
||||
|
||||
let final_output = rewriter
|
||||
.push_chunk(
|
||||
br#"event: response.completed
|
||||
data: {"type":"response.completed","response":{"id":"resp_123","model":"gpt-image-2","tool_usage":{"image_gen":{"total_tokens":0}},"output":[]}}
|
||||
|
||||
"#,
|
||||
)
|
||||
.expect("completed event should rewrite");
|
||||
let final_text = String::from_utf8(final_output).expect("final output should be utf8");
|
||||
assert!(final_text.contains("\"object\":\"chat.completion.chunk\""));
|
||||
assert!(final_text.contains(""));
|
||||
assert!(final_text.contains("data: [DONE]"));
|
||||
assert!(!final_text.contains("image_generation.completed"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,21 @@
|
||||
use std::borrow::Cow;
|
||||
|
||||
use aether_ai_formats::formats::conversion::response::{
|
||||
convert_claude_response_to_openai_responses, convert_gemini_response_to_openai_responses,
|
||||
convert_openai_chat_response_to_openai_responses,
|
||||
};
|
||||
use aether_contracts::{ExecutionStreamTerminalSummary, StandardizedUsage};
|
||||
use serde_json::{json, Value};
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
use crate::formats::claude::messages::stream::ClaudeClientEmitter;
|
||||
use crate::formats::gemini::generate_content::stream::GeminiClientEmitter;
|
||||
use crate::formats::openai::chat::stream::{
|
||||
OpenAIChatClientEmitter, OpenAIResponsesClientEmitter, OpenAIResponsesProviderState,
|
||||
};
|
||||
use crate::formats::shared::sse::encode_json_sse;
|
||||
use crate::formats::shared::sse::{encode_done_sse, encode_json_sse};
|
||||
use crate::formats::shared::stream_core::common::{
|
||||
build_openai_chat_chunk, build_openai_chat_finish_chunk, build_openai_chat_usage_chunk,
|
||||
};
|
||||
use crate::formats::shared::stream_core::CanonicalStreamFrame;
|
||||
use crate::formats::shared::AiSurfaceFinalizeError;
|
||||
|
||||
@@ -27,12 +32,25 @@ pub fn maybe_bridge_standard_sync_json_to_stream(
|
||||
) -> Result<Option<SyncToStreamBridgeOutcome>, AiSurfaceFinalizeError> {
|
||||
let provider_api_format = normalize_api_format(provider_api_format);
|
||||
let client_api_format = normalize_api_format(client_api_format);
|
||||
if client_api_format == "openai:image"
|
||||
&& matches!(
|
||||
provider_api_format.as_str(),
|
||||
"openai:image" | "gemini:generate_content"
|
||||
)
|
||||
{
|
||||
if provider_api_format == "openai:image" {
|
||||
return match client_api_format.as_str() {
|
||||
"openai:image" => {
|
||||
maybe_bridge_openai_image_sync_json_to_stream(provider_body_json, report_context)
|
||||
}
|
||||
"openai:chat" => maybe_bridge_openai_image_sync_json_to_chat_stream(
|
||||
provider_body_json,
|
||||
report_context,
|
||||
),
|
||||
"openai:responses" | "openai:responses:compact" => {
|
||||
maybe_bridge_openai_image_sync_json_to_responses_stream(
|
||||
provider_body_json,
|
||||
report_context,
|
||||
)
|
||||
}
|
||||
_ => Ok(None),
|
||||
};
|
||||
}
|
||||
if client_api_format == "openai:image" && provider_api_format == "gemini:generate_content" {
|
||||
return maybe_bridge_openai_image_sync_json_to_stream(provider_body_json, report_context);
|
||||
}
|
||||
if !is_standard_api_format(provider_api_format.as_str())
|
||||
@@ -72,49 +90,19 @@ fn maybe_bridge_openai_image_sync_json_to_stream(
|
||||
provider_body_json: &Value,
|
||||
report_context: Option<&Value>,
|
||||
) -> Result<Option<SyncToStreamBridgeOutcome>, AiSurfaceFinalizeError> {
|
||||
let provider_api_format = report_context
|
||||
.and_then(|value| value.get("provider_api_format"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.unwrap_or("openai:image");
|
||||
let owned_response;
|
||||
let provider_body_json = if provider_api_format == "gemini:generate_content" {
|
||||
let Some(converted) =
|
||||
crate::formats::shared::image_bridge::build_openai_image_response_from_gemini_response(
|
||||
provider_body_json,
|
||||
report_context,
|
||||
)
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
owned_response = converted;
|
||||
&owned_response
|
||||
} else if provider_body_json.get("output").is_some() && provider_body_json.get("data").is_none()
|
||||
{
|
||||
let Some(converted) = crate::formats::shared::image_bridge::build_openai_image_response_from_response_stream_sync_body(
|
||||
provider_body_json,
|
||||
report_context,
|
||||
) else {
|
||||
return Ok(None);
|
||||
};
|
||||
owned_response = converted;
|
||||
&owned_response
|
||||
} else {
|
||||
provider_body_json
|
||||
};
|
||||
let Some(response) = provider_body_json.as_object() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(image) = response
|
||||
.get("data")
|
||||
.and_then(Value::as_array)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter_map(Value::as_object)
|
||||
.find_map(extract_openai_image_sync_b64_json)
|
||||
let Some(provider_body_json) =
|
||||
normalize_openai_image_sync_response(provider_body_json, report_context)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(response) = provider_body_json.as_ref().as_object() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let outputs = collect_openai_image_outputs(response, report_context);
|
||||
let Some(image) = outputs.iter().find_map(OpenAiImageOutput::b64_json) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let image_count = openai_image_response_image_count(response).max(outputs.len() as u64);
|
||||
let usage = response.get("usage").cloned().unwrap_or(Value::Null);
|
||||
let event_name = openai_image_completed_event_name(report_context);
|
||||
let sse_body = encode_json_sse(
|
||||
@@ -128,27 +116,461 @@ fn maybe_bridge_openai_image_sync_json_to_stream(
|
||||
|
||||
Ok(Some(SyncToStreamBridgeOutcome {
|
||||
sse_body,
|
||||
terminal_summary: Some(ExecutionStreamTerminalSummary {
|
||||
standardized_usage: response
|
||||
.get("usage")
|
||||
.and_then(standardized_usage_from_openai_usage),
|
||||
finish_reason: Some("stop".to_string()),
|
||||
response_id: response
|
||||
.get("id")
|
||||
.and_then(Value::as_str)
|
||||
.map(ToOwned::to_owned),
|
||||
model: response
|
||||
.get("model")
|
||||
.and_then(Value::as_str)
|
||||
.map(ToOwned::to_owned)
|
||||
.or_else(|| image_bridge_model(report_context)),
|
||||
observed_finish: true,
|
||||
unknown_event_count: 0,
|
||||
parser_error: None,
|
||||
}),
|
||||
terminal_summary: Some(openai_image_terminal_summary(
|
||||
response,
|
||||
report_context,
|
||||
image_count,
|
||||
)),
|
||||
}))
|
||||
}
|
||||
|
||||
fn maybe_bridge_openai_image_sync_json_to_chat_stream(
|
||||
provider_body_json: &Value,
|
||||
report_context: Option<&Value>,
|
||||
) -> Result<Option<SyncToStreamBridgeOutcome>, AiSurfaceFinalizeError> {
|
||||
let Some(provider_body_json) =
|
||||
normalize_openai_image_sync_response(provider_body_json, report_context)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(response) = provider_body_json.as_ref().as_object() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let outputs = collect_openai_image_outputs(response, report_context);
|
||||
if outputs.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let image_count = openai_image_response_image_count(response).max(outputs.len() as u64);
|
||||
let summary = openai_image_terminal_summary(response, report_context, image_count);
|
||||
let response_id = openai_image_bridge_response_id(response, report_context, "chatcmpl-image");
|
||||
let model = openai_image_bridge_response_model(response, report_context);
|
||||
let content = outputs
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, output)| output.markdown(index))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n\n");
|
||||
|
||||
let mut sse_body = Vec::new();
|
||||
sse_body.extend(encode_json_sse(
|
||||
None,
|
||||
&build_openai_chat_chunk(&response_id, &model, content, None, None),
|
||||
)?);
|
||||
sse_body.extend(encode_json_sse(
|
||||
None,
|
||||
&build_openai_chat_finish_chunk(&response_id, &model, Some("stop")),
|
||||
)?);
|
||||
if let Some((input_tokens, output_tokens, total_tokens, reasoning_tokens)) = summary
|
||||
.standardized_usage
|
||||
.as_ref()
|
||||
.and_then(openai_chat_usage_counts)
|
||||
{
|
||||
sse_body.extend(encode_json_sse(
|
||||
None,
|
||||
&build_openai_chat_usage_chunk(
|
||||
&response_id,
|
||||
&model,
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
total_tokens,
|
||||
reasoning_tokens,
|
||||
),
|
||||
)?);
|
||||
}
|
||||
sse_body.extend(encode_done_sse());
|
||||
|
||||
Ok(Some(SyncToStreamBridgeOutcome {
|
||||
sse_body,
|
||||
terminal_summary: Some(summary),
|
||||
}))
|
||||
}
|
||||
|
||||
fn maybe_bridge_openai_image_sync_json_to_responses_stream(
|
||||
provider_body_json: &Value,
|
||||
report_context: Option<&Value>,
|
||||
) -> Result<Option<SyncToStreamBridgeOutcome>, AiSurfaceFinalizeError> {
|
||||
let Some(provider_body_json) =
|
||||
normalize_openai_image_sync_response(provider_body_json, report_context)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(response) = provider_body_json.as_ref().as_object() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let outputs = collect_openai_image_outputs(response, report_context);
|
||||
if outputs.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let response_id = openai_image_bridge_response_id(response, report_context, "resp-image");
|
||||
let model = openai_image_bridge_response_model(response, report_context);
|
||||
let mut response_output = Vec::new();
|
||||
for (index, output) in outputs.iter().enumerate() {
|
||||
response_output.push(output.responses_image_generation_item(&response_id, index));
|
||||
}
|
||||
|
||||
let mut response_object = Map::new();
|
||||
response_object.insert("id".to_string(), Value::String(response_id.clone()));
|
||||
response_object.insert("object".to_string(), Value::String("response".to_string()));
|
||||
response_object.insert("model".to_string(), Value::String(model));
|
||||
response_object.insert("status".to_string(), Value::String("completed".to_string()));
|
||||
response_object.insert("output".to_string(), Value::Array(response_output.clone()));
|
||||
if let Some(created) = response.get("created").and_then(Value::as_i64) {
|
||||
response_object.insert("created_at".to_string(), json!(created));
|
||||
}
|
||||
if let Some(usage) = response.get("usage").filter(|value| value.is_object()) {
|
||||
response_object.insert("usage".to_string(), usage.clone());
|
||||
}
|
||||
|
||||
let mut sse_body = Vec::new();
|
||||
for (index, item) in response_output.iter().enumerate() {
|
||||
sse_body.extend(encode_json_sse(
|
||||
Some("response.output_item.done"),
|
||||
&json!({
|
||||
"type": "response.output_item.done",
|
||||
"output_index": index,
|
||||
"item": item,
|
||||
}),
|
||||
)?);
|
||||
}
|
||||
sse_body.extend(encode_json_sse(
|
||||
Some("response.completed"),
|
||||
&json!({
|
||||
"type": "response.completed",
|
||||
"response": Value::Object(response_object),
|
||||
}),
|
||||
)?);
|
||||
|
||||
let image_count = openai_image_response_image_count(response).max(outputs.len() as u64);
|
||||
Ok(Some(SyncToStreamBridgeOutcome {
|
||||
sse_body,
|
||||
terminal_summary: Some(openai_image_terminal_summary(
|
||||
response,
|
||||
report_context,
|
||||
image_count,
|
||||
)),
|
||||
}))
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct OpenAiImageOutput {
|
||||
b64_json: Option<String>,
|
||||
url: Option<String>,
|
||||
mime_type: String,
|
||||
output_format: Option<String>,
|
||||
revised_prompt: Option<String>,
|
||||
}
|
||||
|
||||
impl OpenAiImageOutput {
|
||||
fn b64_json(&self) -> Option<String> {
|
||||
self.b64_json
|
||||
.clone()
|
||||
.or_else(|| self.url.as_deref().and_then(extract_base64_from_data_url))
|
||||
}
|
||||
|
||||
fn source_url(&self) -> Option<String> {
|
||||
self.url.clone().or_else(|| {
|
||||
self.b64_json
|
||||
.as_ref()
|
||||
.map(|value| format!("data:{};base64,{value}", self.mime_type))
|
||||
})
|
||||
}
|
||||
|
||||
fn markdown(&self, index: usize) -> String {
|
||||
let alt = if index == 0 {
|
||||
"generated image".to_string()
|
||||
} else {
|
||||
format!("generated image {}", index + 1)
|
||||
};
|
||||
match self.source_url() {
|
||||
Some(url) => format!(""),
|
||||
None => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn responses_image_generation_item(&self, response_id: &str, index: usize) -> Value {
|
||||
let mut item = Map::new();
|
||||
item.insert(
|
||||
"id".to_string(),
|
||||
Value::String(format!("{response_id}_img_{index}")),
|
||||
);
|
||||
item.insert(
|
||||
"type".to_string(),
|
||||
Value::String("image_generation_call".to_string()),
|
||||
);
|
||||
item.insert("status".to_string(), Value::String("completed".to_string()));
|
||||
if let Some(result) = self.b64_json().or_else(|| self.url.clone()) {
|
||||
item.insert("result".to_string(), Value::String(result));
|
||||
}
|
||||
if let Some(output_format) = self.output_format.as_ref() {
|
||||
item.insert(
|
||||
"output_format".to_string(),
|
||||
Value::String(output_format.clone()),
|
||||
);
|
||||
}
|
||||
if let Some(revised_prompt) = self.revised_prompt.as_ref() {
|
||||
item.insert(
|
||||
"revised_prompt".to_string(),
|
||||
Value::String(revised_prompt.clone()),
|
||||
);
|
||||
}
|
||||
Value::Object(item)
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_openai_image_sync_response<'a>(
|
||||
provider_body_json: &'a Value,
|
||||
report_context: Option<&Value>,
|
||||
) -> Result<Option<Cow<'a, Value>>, AiSurfaceFinalizeError> {
|
||||
let provider_api_format = report_context
|
||||
.and_then(|value| value.get("provider_api_format"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.unwrap_or("openai:image");
|
||||
if provider_api_format == "gemini:generate_content" {
|
||||
let Some(converted) =
|
||||
crate::formats::shared::image_bridge::build_openai_image_response_from_gemini_response(
|
||||
provider_body_json,
|
||||
report_context,
|
||||
)
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
return Ok(Some(Cow::Owned(converted)));
|
||||
}
|
||||
if provider_body_json.get("output").is_some() && provider_body_json.get("data").is_none() {
|
||||
let Some(converted) = crate::formats::shared::image_bridge::build_openai_image_response_from_response_stream_sync_body(
|
||||
provider_body_json,
|
||||
report_context,
|
||||
) else {
|
||||
return Ok(None);
|
||||
};
|
||||
return Ok(Some(Cow::Owned(converted)));
|
||||
}
|
||||
Ok(Some(Cow::Borrowed(provider_body_json)))
|
||||
}
|
||||
|
||||
fn collect_openai_image_outputs(
|
||||
response: &Map<String, Value>,
|
||||
report_context: Option<&Value>,
|
||||
) -> Vec<OpenAiImageOutput> {
|
||||
response
|
||||
.get("data")
|
||||
.and_then(Value::as_array)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter_map(Value::as_object)
|
||||
.filter_map(|item| openai_image_output_from_item(item, report_context))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn openai_image_output_from_item(
|
||||
item: &Map<String, Value>,
|
||||
report_context: Option<&Value>,
|
||||
) -> Option<OpenAiImageOutput> {
|
||||
let b64_json = extract_openai_image_sync_b64_json(item);
|
||||
let url = item
|
||||
.get("url")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned);
|
||||
if b64_json.is_none() && url.is_none() {
|
||||
return None;
|
||||
}
|
||||
let output_format = item
|
||||
.get("output_format")
|
||||
.or_else(|| item.get("format"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
.or_else(|| image_request_output_format(report_context));
|
||||
let mime_type = url
|
||||
.as_deref()
|
||||
.and_then(extract_mime_type_from_data_url)
|
||||
.or_else(|| {
|
||||
output_format
|
||||
.as_deref()
|
||||
.map(mime_type_from_image_output_format)
|
||||
})
|
||||
.unwrap_or_else(|| "image/png".to_string());
|
||||
let revised_prompt = item
|
||||
.get("revised_prompt")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned);
|
||||
|
||||
Some(OpenAiImageOutput {
|
||||
b64_json,
|
||||
url,
|
||||
mime_type,
|
||||
output_format,
|
||||
revised_prompt,
|
||||
})
|
||||
}
|
||||
|
||||
fn openai_image_response_image_count(response: &Map<String, Value>) -> u64 {
|
||||
response
|
||||
.get("data")
|
||||
.and_then(Value::as_array)
|
||||
.map(|items| items.len() as u64)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
fn openai_image_terminal_summary(
|
||||
response: &Map<String, Value>,
|
||||
report_context: Option<&Value>,
|
||||
image_count: u64,
|
||||
) -> ExecutionStreamTerminalSummary {
|
||||
ExecutionStreamTerminalSummary {
|
||||
standardized_usage: openai_image_standardized_usage(
|
||||
response.get("usage"),
|
||||
report_context,
|
||||
image_count,
|
||||
),
|
||||
finish_reason: Some("stop".to_string()),
|
||||
response_id: response
|
||||
.get("id")
|
||||
.and_then(Value::as_str)
|
||||
.map(ToOwned::to_owned),
|
||||
model: response
|
||||
.get("model")
|
||||
.and_then(Value::as_str)
|
||||
.map(ToOwned::to_owned)
|
||||
.or_else(|| image_bridge_model(report_context)),
|
||||
observed_finish: true,
|
||||
unknown_event_count: 0,
|
||||
parser_error: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn openai_image_standardized_usage(
|
||||
usage: Option<&Value>,
|
||||
report_context: Option<&Value>,
|
||||
image_count: u64,
|
||||
) -> Option<StandardizedUsage> {
|
||||
let mut standardized_usage = usage
|
||||
.and_then(standardized_usage_from_openai_usage)
|
||||
.unwrap_or_else(StandardizedUsage::new);
|
||||
if image_count > 0 {
|
||||
standardized_usage.request_count = i64::try_from(image_count).unwrap_or(i64::MAX);
|
||||
standardized_usage
|
||||
.dimensions
|
||||
.insert("image_count".to_string(), json!(image_count));
|
||||
}
|
||||
if let Some(output_format) = image_request_output_format(report_context) {
|
||||
standardized_usage
|
||||
.dimensions
|
||||
.insert("image_output_format".to_string(), json!(output_format));
|
||||
}
|
||||
if let Some(size) = image_request_size(report_context) {
|
||||
standardized_usage
|
||||
.dimensions
|
||||
.insert("image_size".to_string(), json!(size));
|
||||
}
|
||||
if let Some(quality) = image_request_quality(report_context) {
|
||||
standardized_usage
|
||||
.dimensions
|
||||
.insert("image_quality".to_string(), json!(quality));
|
||||
}
|
||||
(standardized_usage.signal_score() > 0).then_some(standardized_usage)
|
||||
}
|
||||
|
||||
fn openai_chat_usage_counts(usage: &StandardizedUsage) -> Option<(u64, u64, u64, u64)> {
|
||||
let input_tokens = usage.input_tokens.max(0) as u64;
|
||||
let output_tokens = usage.output_tokens.max(0) as u64;
|
||||
let reasoning_tokens = usage.reasoning_tokens.max(0) as u64;
|
||||
let total_tokens = usage
|
||||
.dimensions
|
||||
.get("total_tokens")
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or_else(|| {
|
||||
input_tokens
|
||||
.saturating_add(output_tokens)
|
||||
.saturating_add(reasoning_tokens)
|
||||
});
|
||||
(total_tokens > 0).then_some((input_tokens, output_tokens, total_tokens, reasoning_tokens))
|
||||
}
|
||||
|
||||
fn openai_image_bridge_response_id(
|
||||
response: &Map<String, Value>,
|
||||
report_context: Option<&Value>,
|
||||
fallback_prefix: &str,
|
||||
) -> String {
|
||||
response
|
||||
.get("id")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
.or_else(|| {
|
||||
report_context
|
||||
.and_then(|value| value.get("request_id"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(|value| format!("{fallback_prefix}-{value}"))
|
||||
})
|
||||
.unwrap_or_else(|| fallback_prefix.to_string())
|
||||
}
|
||||
|
||||
fn openai_image_bridge_response_model(
|
||||
response: &Map<String, Value>,
|
||||
report_context: Option<&Value>,
|
||||
) -> String {
|
||||
response
|
||||
.get("model")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
.or_else(|| image_bridge_model(report_context))
|
||||
.unwrap_or_else(|| "gpt-image".to_string())
|
||||
}
|
||||
|
||||
fn image_request_output_format(report_context: Option<&Value>) -> Option<String> {
|
||||
report_context
|
||||
.and_then(|value| value.get("image_request"))
|
||||
.and_then(|value| value.get("output_format"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
}
|
||||
|
||||
fn image_request_size(report_context: Option<&Value>) -> Option<String> {
|
||||
report_context
|
||||
.and_then(|value| value.get("image_request"))
|
||||
.and_then(|value| value.get("size"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
}
|
||||
|
||||
fn image_request_quality(report_context: Option<&Value>) -> Option<String> {
|
||||
report_context
|
||||
.and_then(|value| value.get("image_request"))
|
||||
.and_then(|value| value.get("quality"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
}
|
||||
|
||||
fn mime_type_from_image_output_format(output_format: &str) -> String {
|
||||
match output_format.trim().to_ascii_lowercase().as_str() {
|
||||
"jpg" | "jpeg" => "image/jpeg".to_string(),
|
||||
"webp" => "image/webp".to_string(),
|
||||
"png" => "image/png".to_string(),
|
||||
value if !value.is_empty() => format!("image/{value}"),
|
||||
_ => "image/png".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_api_format(value: &str) -> String {
|
||||
aether_ai_formats::normalize_api_format_alias(value)
|
||||
}
|
||||
@@ -186,6 +608,14 @@ fn extract_base64_from_data_url(value: &str) -> Option<String> {
|
||||
(!payload.trim().is_empty()).then(|| payload.trim().to_string())
|
||||
}
|
||||
|
||||
fn extract_mime_type_from_data_url(value: &str) -> Option<String> {
|
||||
let trimmed = value.trim();
|
||||
let (metadata, _) = trimmed.split_once(',')?;
|
||||
let mime_type = metadata.strip_prefix("data:")?.strip_suffix(";base64")?;
|
||||
let mime_type = mime_type.trim();
|
||||
(!mime_type.is_empty()).then(|| mime_type.to_string())
|
||||
}
|
||||
|
||||
fn openai_image_completed_event_name(report_context: Option<&Value>) -> &'static str {
|
||||
if openai_image_request_operation(report_context) == Some("edit") {
|
||||
"image_edit.completed"
|
||||
@@ -565,6 +995,76 @@ mod tests {
|
||||
.cloned(),
|
||||
Some(json!(100))
|
||||
);
|
||||
assert_eq!(
|
||||
summary
|
||||
.standardized_usage
|
||||
.as_ref()
|
||||
.and_then(|usage| usage.dimensions.get("image_count"))
|
||||
.cloned(),
|
||||
Some(json!(1))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bridges_openai_image_sync_json_to_openai_chat_sse() {
|
||||
let report_context = json!({
|
||||
"provider_api_format": "openai:image",
|
||||
"client_api_format": "openai:chat",
|
||||
"mapped_model": "gpt-image-2",
|
||||
"image_request": {
|
||||
"operation": "generate",
|
||||
"output_format": "png",
|
||||
"size": "1024x1024",
|
||||
"quality": "medium"
|
||||
}
|
||||
});
|
||||
let outcome = maybe_bridge_standard_sync_json_to_stream(
|
||||
&json!({
|
||||
"id": "img_123",
|
||||
"created": 1776971267,
|
||||
"model": "gpt-image-2",
|
||||
"data": [
|
||||
{"b64_json": "aGVsbG8="},
|
||||
{"b64_json": "d29ybGQ="}
|
||||
],
|
||||
"usage": {
|
||||
"total_tokens": 100,
|
||||
"input_tokens": 50,
|
||||
"output_tokens": 50
|
||||
}
|
||||
}),
|
||||
"openai:image",
|
||||
"openai:chat",
|
||||
Some(&report_context),
|
||||
)
|
||||
.expect("bridge should succeed")
|
||||
.expect("bridge should produce sse");
|
||||
|
||||
let output = utf8(outcome.sse_body);
|
||||
assert!(output.contains("\"object\":\"chat.completion.chunk\""));
|
||||
assert!(output.contains(""));
|
||||
assert!(output.contains(""));
|
||||
assert!(output.contains("\"finish_reason\":\"stop\""));
|
||||
assert!(output.contains("data: [DONE]"));
|
||||
assert!(!output.contains("image_generation.completed"));
|
||||
|
||||
let summary = outcome
|
||||
.terminal_summary
|
||||
.expect("terminal summary should exist");
|
||||
let usage = summary
|
||||
.standardized_usage
|
||||
.as_ref()
|
||||
.expect("standard usage should exist");
|
||||
assert_eq!(usage.request_count, 2);
|
||||
assert_eq!(usage.dimensions.get("image_count"), Some(&json!(2)));
|
||||
assert_eq!(
|
||||
usage.dimensions.get("image_size"),
|
||||
Some(&json!("1024x1024"))
|
||||
);
|
||||
assert_eq!(
|
||||
usage.dimensions.get("image_quality"),
|
||||
Some(&json!("medium"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user