i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 17:30:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(grok): add runtime image surfaces

Browse Source
This commit is contained in:
mayrain
2026-05-16 19:23:23 +08:00
parent 936e1ae37b
commit 5bf236957e
43 changed files with 7044 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
crates/aether-ai-formats/src/api.rs
View File

@@ -180,9 +180,10 @@ pub use crate::formats::{
request::{
build_chatgpt_web_image_request_body, build_openai_image_provider_request_body,
default_model_for_openai_image_operation, is_openai_image_stream_request,
normalize_openai_image_request, openai_image_operation_from_path,
resolve_requested_openai_image_model_for_request, ChatGptWebImageRequestError,
NormalizedOpenAiImageRequest, OpenAiImageOperation, OpenAiImageResponseFormat,
normalize_openai_image_request, normalize_openai_image_request_with_options,
openai_image_operation_from_path, resolve_requested_openai_image_model_for_request,
ChatGptWebImageRequestError, NormalizedOpenAiImageRequest, OpenAiImageNormalizeOptions,
OpenAiImageOperation, OpenAiImageResponseFormat,
},
spec::{
resolve_stream_spec as resolve_local_image_stream_spec,

6
crates/aether-ai-formats/src/formats/claude/messages/stream.rs
View File

@@ -669,6 +669,12 @@ impl ClaudeClientEmitter {
Ok(out)
}
CanonicalStreamEvent::ContentPart(part) => self.emit_content_part(part),
CanonicalStreamEvent::ImageGenerationCall { item, .. } => {
let Some(part) = content_part_from_openai_image_generation_item(&item) else {
return Ok(Vec::new());
};
self.emit_content_part(part)
}
CanonicalStreamEvent::ToolCallStart {
index,
call_id,

10
crates/aether-ai-formats/src/formats/gemini/generate_content/stream.rs
View File

@@ -485,6 +485,16 @@ impl GeminiClientEmitter {
None,
None,
),
CanonicalStreamEvent::ImageGenerationCall { item, .. } => {
let Some(part) = content_part_from_openai_image_generation_item(&item) else {
return Ok(Vec::new());
};
self.emit_candidate(
vec![gemini_part_from_canonical_content_part(part)],
None,
None,
)
}
CanonicalStreamEvent::ToolCallStart {
index,
call_id,

275
crates/aether-ai-formats/src/formats/openai/chat/stream.rs
View File

@@ -1,4 +1,4 @@
use std::collections::BTreeMap;
use std::collections::{BTreeMap, BTreeSet};
use serde_json::{json, Map, Value};
@@ -50,6 +50,7 @@ pub struct OpenAIResponsesProviderState {
tool_calls: BTreeMap<usize, OpenAIResponsesProviderToolState>,
tool_results: BTreeMap<usize, OpenAIResponsesProviderToolResultState>,
tool_index_by_key: BTreeMap<String, usize>,
image_item_keys: BTreeSet<String>,
last_tool_index: Option<usize>,
}
@@ -718,6 +719,55 @@ impl OpenAIResponsesProviderState {
}
}
fn emit_image_generation_item(
&mut self,
report_context: &Value,
out: &mut Vec<CanonicalStreamFrame>,
item: &Map<String, Value>,
output_index: Option<usize>,
final_item: bool,
) {
if item.get("type").and_then(Value::as_str) != Some("image_generation_call") {
return;
}
if !final_item
&& !item
.get("status")
.and_then(Value::as_str)
.is_some_and(|value| value.eq_ignore_ascii_case("completed"))
{
return;
}
let has_image_payload = item
.get("result")
.or_else(|| item.get("url"))
.and_then(Value::as_str)
.map(str::trim)
.is_some_and(|value| !value.is_empty());
if !has_image_payload {
return;
}
let index = output_index.unwrap_or(self.image_item_keys.len());
let key = item
.get("id")
.and_then(Value::as_str)
.map(ToOwned::to_owned)
.unwrap_or_else(|| format!("image_generation_call:{index}"));
if !self.image_item_keys.insert(key) {
return;
}
self.ensure_started(report_context, out);
let (id, model) = self.identity(report_context);
out.push(CanonicalStreamFrame {
id,
model,
event: CanonicalStreamEvent::ImageGenerationCall {
index,
item: Value::Object(item.clone()),
},
});
}
pub fn push_line(
&mut self,
report_context: &Value,
@@ -894,6 +944,15 @@ impl OpenAIResponsesProviderState {
"reasoning" => {
self.ensure_started(report_context, &mut out);
}
"image_generation_call" => {
self.emit_image_generation_item(
report_context,
&mut out,
item,
output_index,
false,
);
}
_ => {
out.push(self.unknown_frame(report_context, Value::Object(item.clone())));
}
@@ -1107,6 +1166,15 @@ impl OpenAIResponsesProviderState {
"reasoning" => {
self.emit_reasoning_item(report_context, &mut out, item);
}
"image_generation_call" => {
self.emit_image_generation_item(
report_context,
&mut out,
item,
output_index,
true,
);
}
_ => {
out.push(self.unknown_frame(report_context, Value::Object(item.clone())));
}
@@ -1152,6 +1220,15 @@ impl OpenAIResponsesProviderState {
"reasoning" => {
self.emit_reasoning_item(report_context, &mut out, item);
}
"image_generation_call" => {
self.emit_image_generation_item(
report_context,
&mut out,
item,
Some(output_index),
true,
);
}
_ => {
out.push(
self.unknown_frame(report_context, Value::Object(item.clone())),
@@ -1255,6 +1332,7 @@ pub struct OpenAIResponsesClientEmitter {
reasoning_summary_parts: Vec<String>,
tool_calls: BTreeMap<usize, OpenAIResponsesClientToolState>,
tool_results: BTreeMap<usize, OpenAIResponsesClientToolResultState>,
image_generation_items: BTreeMap<usize, Value>,
}
impl OpenAIChatClientEmitter {
@@ -1361,6 +1439,26 @@ impl OpenAIChatClientEmitter {
)?);
Ok(out)
}
CanonicalStreamEvent::ImageGenerationCall { item, .. } => {
let Some(part) = content_part_from_openai_image_generation_item(&item) else {
return Ok(Vec::new());
};
let placeholder = openai_stream_placeholder_for_content_part(&part);
let mut out = self.ensure_started()?;
out.extend(encode_json_sse(
None,
&build_openai_chat_chunk(
self.response_id
.as_deref()
.unwrap_or("chatcmpl-local-stream"),
self.model.as_deref().unwrap_or("unknown"),
placeholder,
None,
None,
),
)?);
Ok(out)
}
CanonicalStreamEvent::ToolCallStart {
index,
call_id,
@@ -1653,6 +1751,11 @@ impl OpenAIResponsesClientEmitter {
output_index
}
fn ensure_image_generation_output_index(&mut self, index: usize) -> usize {
self.next_output_index = self.next_output_index.max(index.saturating_add(1));
index
}
fn ensure_reasoning_item_started(&mut self) -> Result<Vec<u8>, AiSurfaceFinalizeError> {
let mut out = self.ensure_started()?;
let output_index = self.ensure_reasoning_output_index();
@@ -1956,6 +2059,42 @@ impl OpenAIResponsesClientEmitter {
Ok(out)
}
fn emit_image_generation_call_item(
&mut self,
index: usize,
item: Value,
) -> Result<Vec<u8>, AiSurfaceFinalizeError> {
let mut out = self.ensure_started()?;
let output_index = self.ensure_image_generation_output_index(index);
let mut item = item.as_object().cloned().unwrap_or_default();
item.insert(
"type".to_string(),
Value::String("image_generation_call".to_string()),
);
if !item.contains_key("id") {
item.insert(
"id".to_string(),
Value::String(format!("{}_ig_{}", self.response_id(), output_index)),
);
}
if !item.contains_key("status") {
item.insert("status".to_string(), Value::String("completed".to_string()));
}
let item = Value::Object(item);
self.image_generation_items
.insert(output_index, item.clone());
out.extend(self.encode_response_event(
"response.output_item.done",
json!({
"type": "response.output_item.done",
"response_id": self.response_id(),
"output_index": output_index,
"item": item,
}),
)?);
Ok(out)
}
fn completed_response(&self, usage: CanonicalUsage) -> Value {
let mut ordered_output = Vec::new();
let summary = if self.reasoning_summary_parts.is_empty() {
@@ -2058,6 +2197,9 @@ impl OpenAIResponsesClientEmitter {
ordered_output.push((output_index, Value::Object(item)));
}
}
for (output_index, item) in &self.image_generation_items {
ordered_output.push((*output_index, item.clone()));
}
ordered_output.sort_by_key(|(output_index, _)| *output_index);
let mut usage_payload = Map::new();
@@ -2183,6 +2325,9 @@ impl OpenAIResponsesClientEmitter {
)?);
Ok(out)
}
CanonicalStreamEvent::ImageGenerationCall { index, item } => {
self.emit_image_generation_call_item(index, item)
}
CanonicalStreamEvent::ToolCallStart {
index,
call_id,
@@ -2876,6 +3021,134 @@ mod tests {
)));
}
#[test]
fn openai_responses_provider_state_preserves_image_generation_calls() {
let mut state = OpenAIResponsesProviderState::default();
let report_context = json!({});
let frames = state
.push_line(
&report_context,
data_line(json!({
"type": "response.completed",
"response": {
"id": "resp_img_123",
"model": "gpt-image-2",
"output": [{
"id": "ig_123",
"type": "image_generation_call",
"status": "completed",
"output_format": "png",
"result": "aGVsbG8="
}],
"usage": {"input_tokens": 1, "output_tokens": 2, "total_tokens": 3}
}
})),
)
.expect("completed event should parse");
assert!(frames.iter().any(|frame| matches!(
frame.event,
CanonicalStreamEvent::ImageGenerationCall {
index: 0,
ref item,
} if item["type"] == json!("image_generation_call")
&& item["result"] == json!("aGVsbG8=")
)));
}
#[test]
fn openai_responses_provider_state_waits_for_final_image_generation_item() {
let mut state = OpenAIResponsesProviderState::default();
let report_context = json!({});
let added_frames = state
.push_line(
&report_context,
data_line(json!({
"type": "response.output_item.added",
"output_index": 0,
"item": {
"id": "ig_123",
"type": "image_generation_call",
"status": "generating",
"output_format": "png",
"result": "early"
}
})),
)
.expect("added event should parse");
assert!(!added_frames.iter().any(|frame| matches!(
frame.event,
CanonicalStreamEvent::ImageGenerationCall { .. }
)));
let done_frames = state
.push_line(
&report_context,
data_line(json!({
"type": "response.output_item.done",
"output_index": 0,
"item": {
"id": "ig_123",
"type": "image_generation_call",
"status": "completed",
"output_format": "png",
"result": "final"
}
})),
)
.expect("done event should parse");
assert!(done_frames.iter().any(|frame| matches!(
frame.event,
CanonicalStreamEvent::ImageGenerationCall {
index: 0,
ref item,
} if item["status"] == json!("completed") && item["result"] == json!("final")
)));
}
#[test]
fn openai_responses_client_emitter_emits_image_generation_call_events() {
let mut emitter = OpenAIResponsesClientEmitter::default();
let mut bytes = emitter
.emit(CanonicalStreamFrame {
id: "resp_img_123".to_string(),
model: "gpt-image-2".to_string(),
event: CanonicalStreamEvent::ImageGenerationCall {
index: 0,
item: json!({
"id": "ig_123",
"type": "image_generation_call",
"status": "completed",
"output_format": "png",
"result": "aGVsbG8="
}),
},
})
.expect("image event should encode");
bytes.extend(
emitter
.emit(CanonicalStreamFrame {
id: "resp_img_123".to_string(),
model: "gpt-image-2".to_string(),
event: CanonicalStreamEvent::Finish {
finish_reason: Some("stop".to_string()),
usage: None,
},
})
.expect("finish should encode"),
);
let sse = String::from_utf8(bytes).expect("sse should be utf8");
assert!(sse.contains("event: response.output_item.done\n"));
assert!(sse.contains("\"type\":\"image_generation_call\""));
assert!(sse.contains("\"result\":\"aGVsbG8=\""));
assert!(sse.contains("\"output\":["));
assert!(sse.contains("\"id\":\"ig_123\""));
}
#[test]
fn openai_responses_client_emitter_emits_function_call_output_events() {
let mut emitter = OpenAIResponsesClientEmitter::default();

199
crates/aether-ai-formats/src/formats/openai/image/request.rs
View File

@@ -3,15 +3,12 @@ use std::collections::BTreeMap;
use base64::Engine as _;
use serde_json::{json, Map, Number, Value};
use crate::formats::openai::responses::codex::{
CODEX_OPENAI_IMAGE_DEFAULT_MODEL, CODEX_OPENAI_IMAGE_DEFAULT_VARIATION_MODEL,
};
use crate::formats::openai::responses::codex::CODEX_OPENAI_IMAGE_DEFAULT_MODEL;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum OpenAiImageOperation {
Generate,
Edit,
Variation,
}
impl OpenAiImageOperation {
@@ -19,7 +16,6 @@ impl OpenAiImageOperation {
match self {
Self::Generate => "generate",
Self::Edit => "edit",
Self::Variation => "variation",
}
}
}
@@ -47,9 +43,31 @@ pub struct NormalizedOpenAiImageRequest {
prompt: Option<String>,
images: Vec<Value>,
tool: Map<String, Value>,
image_count: Option<u64>,
user: Option<String>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct OpenAiImageNormalizeOptions {
max_generation_count: u64,
}
impl Default for OpenAiImageNormalizeOptions {
fn default() -> Self {
Self {
max_generation_count: 1,
}
}
}
impl OpenAiImageNormalizeOptions {
pub fn with_max_generation_count(max_generation_count: u64) -> Self {
Self {
max_generation_count: max_generation_count.max(1),
}
}
}
pub const CHATGPT_WEB_IMAGE_MAX_AREA: u64 = 1_500_000;
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -123,7 +141,6 @@ pub fn build_chatgpt_web_image_request_body(
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or(match request.operation {
OpenAiImageOperation::Variation => "Create a faithful variation of the provided image.",
OpenAiImageOperation::Generate | OpenAiImageOperation::Edit => {
"Generate a high quality image."
}
@@ -244,7 +261,6 @@ pub fn openai_image_operation_from_path(path: &str) -> Option<OpenAiImageOperati
match path {
"/v1/images/generations" => Some(OpenAiImageOperation::Generate),
"/v1/images/edits" => Some(OpenAiImageOperation::Edit),
"/v1/images/variations" => Some(OpenAiImageOperation::Variation),
_ => None,
}
}
@@ -435,7 +451,6 @@ pub fn resolve_requested_openai_image_model_for_request(
pub fn default_model_for_openai_image_operation(operation: OpenAiImageOperation) -> &'static str {
match operation {
OpenAiImageOperation::Variation => CODEX_OPENAI_IMAGE_DEFAULT_VARIATION_MODEL,
OpenAiImageOperation::Generate | OpenAiImageOperation::Edit => {
CODEX_OPENAI_IMAGE_DEFAULT_MODEL
}
@@ -446,12 +461,26 @@ pub fn normalize_openai_image_request(
parts: &http::request::Parts,
body_json: &Value,
body_base64: Option<&str>,
) -> Option<NormalizedOpenAiImageRequest> {
normalize_openai_image_request_with_options(
parts,
body_json,
body_base64,
OpenAiImageNormalizeOptions::default(),
)
}
pub fn normalize_openai_image_request_with_options(
parts: &http::request::Parts,
body_json: &Value,
body_base64: Option<&str>,
options: OpenAiImageNormalizeOptions,
) -> Option<NormalizedOpenAiImageRequest> {
let operation = openai_image_operation_from_path(parts.uri.path())?;
if let Some(body_base64) = body_base64 {
normalize_openai_image_multipart_request(parts, body_base64, operation)
normalize_openai_image_multipart_request(parts, body_base64, operation, options)
} else {
normalize_openai_image_json_request(body_json, operation)
normalize_openai_image_json_request(body_json, operation, options)
}
}
@@ -486,12 +515,16 @@ pub fn build_openai_image_provider_request_body(request: &NormalizedOpenAiImageR
if let Some(user) = request.user.as_ref() {
body.insert("user".to_string(), Value::String(user.clone()));
}
if let Some(image_count) = request.image_count.filter(|value| *value > 1) {
body.insert("n".to_string(), Value::Number(Number::from(image_count)));
}
Value::Object(body)
}
fn normalize_openai_image_json_request(
body_json: &Value,
operation: OpenAiImageOperation,
options: OpenAiImageNormalizeOptions,
) -> Option<NormalizedOpenAiImageRequest> {
let object = body_json.as_object()?;
if object
@@ -502,10 +535,9 @@ fn normalize_openai_image_json_request(
{
return None;
}
if object
.get("n")
.and_then(image_request_count)
.is_some_and(|value| value != 1)
let image_count = object.get("n").and_then(image_request_count);
if image_count
.is_some_and(|value| value == 0 || value > max_count_for_operation(operation, options))
{
return None;
}
@@ -534,11 +566,7 @@ fn normalize_openai_image_json_request(
}
}
let mask = object.get("mask").and_then(normalize_mask_value);
if matches!(
operation,
OpenAiImageOperation::Edit | OpenAiImageOperation::Variation
) && images.is_empty()
{
if matches!(operation, OpenAiImageOperation::Edit) && images.is_empty() {
return None;
}
@@ -550,6 +578,7 @@ fn normalize_openai_image_json_request(
prompt,
images,
tool,
image_count,
user,
summary_json: build_image_request_summary_json(
operation,
@@ -564,6 +593,7 @@ fn normalize_openai_image_multipart_request(
parts: &http::request::Parts,
body_base64: &str,
operation: OpenAiImageOperation,
options: OpenAiImageNormalizeOptions,
) -> Option<NormalizedOpenAiImageRequest> {
let multipart_fields = parse_multipart_fields_from_base64(parts, body_base64)?;
let requested_model = normalize_requested_image_model(
@@ -572,9 +602,10 @@ fn normalize_openai_image_multipart_request(
if find_multipart_text_field(&multipart_fields, "style").is_some() {
return None;
}
if find_multipart_text_field(&multipart_fields, "n")
.and_then(|value| value.trim().parse::<u64>().ok())
.is_some_and(|value| value != 1)
let image_count = find_multipart_text_field(&multipart_fields, "n")
.and_then(|value| value.trim().parse::<u64>().ok());
if image_count
.is_some_and(|value| value == 0 || value > max_count_for_operation(operation, options))
{
return None;
}
@@ -635,11 +666,7 @@ fn normalize_openai_image_multipart_request(
}
}
if matches!(
operation,
OpenAiImageOperation::Edit | OpenAiImageOperation::Variation
) && images.is_empty()
{
if matches!(operation, OpenAiImageOperation::Edit) && images.is_empty() {
return None;
}
@@ -651,6 +678,7 @@ fn normalize_openai_image_multipart_request(
prompt,
images,
tool,
image_count,
user,
summary_json: build_image_request_summary_json(
operation,
@@ -677,10 +705,8 @@ fn normalize_prompt(
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned);
match operation {
OpenAiImageOperation::Generate | OpenAiImageOperation::Edit => prompt.map(Some),
OpenAiImageOperation::Variation => Some(prompt),
}
let _ = operation;
Some(prompt)
}
fn normalize_image_response_format(
@@ -804,7 +830,7 @@ fn build_tool_options(
Value::String(
match operation {
OpenAiImageOperation::Generate => "generate",
OpenAiImageOperation::Edit | OpenAiImageOperation::Variation => "edit",
OpenAiImageOperation::Edit => "edit",
}
.to_string(),
),
@@ -892,6 +918,16 @@ fn image_request_count(value: &Value) -> Option<u64> {
})
}
fn max_count_for_operation(
operation: OpenAiImageOperation,
options: OpenAiImageNormalizeOptions,
) -> u64 {
match operation {
OpenAiImageOperation::Generate => options.max_generation_count.max(1),
OpenAiImageOperation::Edit => 1,
}
}
fn normalize_image_value(value: &Value) -> Vec<Value> {
match value {
Value::Array(values) => values.iter().flat_map(normalize_image_value).collect(),
@@ -1101,7 +1137,9 @@ mod tests {
use super::{
build_chatgpt_web_image_request_body, build_openai_image_provider_request_body,
is_openai_image_stream_request, normalize_openai_image_request, OpenAiImageOperation,
is_openai_image_stream_request, normalize_openai_image_request,
normalize_openai_image_request_with_options, openai_image_operation_from_path,
OpenAiImageNormalizeOptions, OpenAiImageOperation,
};
use crate::formats::openai::image::spec::{resolve_stream_spec, resolve_sync_spec};
use crate::formats::openai::responses::codex::{
@@ -1168,7 +1206,7 @@ mod tests {
}
#[test]
fn normalize_variation_multipart_request_leaves_defaults_empty_until_codex_adapter() {
fn openai_image_variation_path_is_not_supported() {
let boundary = "boundary-variation-123";
let body = format!(
concat!(
@@ -1176,9 +1214,6 @@ mod tests {
"Content-Disposition: form-data; name=\"image\"; filename=\"image.png\"\r\n",
"Content-Type: image/png\r\n\r\n",
"hello\r\n",
"--{boundary}\r\n",
"Content-Disposition: form-data; name=\"response_format\"\r\n\r\n",
"url\r\n",
"--{boundary}--\r\n"
),
boundary = boundary,
@@ -1189,40 +1224,8 @@ mod tests {
Some(&format!("multipart/form-data; boundary={boundary}")),
);
let request = normalize_openai_image_request(&parts, &json!({}), Some(&body_base64))
.expect("variation request should normalize");
assert_eq!(request.operation, OpenAiImageOperation::Variation);
assert!(request.requested_model.is_none());
assert_eq!(request.summary_json["response_format"], json!("url"));
assert_eq!(
request.tool.get("action").and_then(|value| value.as_str()),
Some("edit")
);
assert!(request.tool.get("output_format").is_none());
assert_eq!(request.images.len(), 1);
let mut provider_request_body = build_openai_image_provider_request_body(&request);
apply_codex_openai_responses_special_body_edits(
&mut provider_request_body,
"codex",
"openai:image",
None,
None,
);
assert_eq!(
provider_request_body["input"][0]["content"][0]["text"],
json!("Create a faithful variation of the provided image.")
);
assert_eq!(
provider_request_body["model"],
json!(CODEX_OPENAI_IMAGE_INTERNAL_MODEL)
);
assert_eq!(
provider_request_body["tools"][0]["output_format"],
json!("png")
);
assert!(openai_image_operation_from_path("/v1/images/variations").is_none());
assert!(normalize_openai_image_request(&parts, &json!({}), Some(&body_base64)).is_none());
}
#[test]
@@ -1318,6 +1321,60 @@ mod tests {
);
}
#[test]
fn normalize_generate_json_request_keeps_allowed_multi_image_count() {
let parts = request_parts("/v1/images/generations", Some("application/json"));
let request = normalize_openai_image_request_with_options(
&parts,
&json!({
"model": "grok-imagine-image",
"prompt": "generate image",
"n": 4
}),
None,
OpenAiImageNormalizeOptions::with_max_generation_count(4),
)
.expect("grok generation request should allow n up to four");
let provider_request_body = build_openai_image_provider_request_body(&request);
assert_eq!(provider_request_body["n"], json!(4));
}
#[test]
fn normalize_generate_json_request_rejects_multi_image_count_by_default() {
let parts = request_parts("/v1/images/generations", Some("application/json"));
assert!(normalize_openai_image_request(
&parts,
&json!({
"model": "gpt-image-2",
"prompt": "generate image",
"n": 2
}),
None,
)
.is_none());
}
#[test]
fn normalize_edit_request_rejects_multi_image_count_even_with_generation_override() {
let parts = request_parts("/v1/images/edits", Some("application/json"));
assert!(normalize_openai_image_request_with_options(
&parts,
&json!({
"model": "grok-imagine-image-edit",
"prompt": "edit image",
"n": 2,
"image": {
"b64_json": "aGVsbG8=",
"mime_type": "image/png"
}
}),
None,
OpenAiImageNormalizeOptions::with_max_generation_count(4),
)
.is_none());
}
#[test]
fn build_generate_request_defaults_codex_image_tool_and_tool_choice() {
let parts = request_parts("/v1/images/generations", Some("application/json"));

49
crates/aether-ai-formats/src/formats/openai/responses/codex.rs
View File

@@ -44,6 +44,18 @@ fn is_openai_image_request(provider_api_format: &str) -> bool {
.eq_ignore_ascii_case("openai:image")
}
fn codex_openai_responses_body_uses_image_generation_tool(
body_object: &serde_json::Map<String, Value>,
) -> bool {
body_object
.get("tools")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(Value::as_object)
.any(|tool| tool.get("type").and_then(Value::as_str) == Some("image_generation"))
}
fn apply_codex_openai_image_tool_overrides(body_object: &mut serde_json::Map<String, Value>) {
let mut tool = body_object
.get("tools")
@@ -397,7 +409,9 @@ pub fn apply_codex_openai_responses_special_body_edits(
{
body_object.insert("instructions".to_string(), json!(""));
}
if is_openai_image_request(provider_api_format) {
if is_openai_image_request(provider_api_format)
|| codex_openai_responses_body_uses_image_generation_tool(body_object)
{
body_object.insert(
"model".to_string(),
json!(CODEX_OPENAI_IMAGE_INTERNAL_MODEL),
@@ -712,6 +726,39 @@ mod tests {
);
}
#[test]
fn codex_responses_image_tool_edits_force_internal_model_and_tool_defaults() {
let mut provider_request_body = json!({
"model": "gpt-image-2",
"input": "generate image",
"tools": [{
"type": "image_generation"
}]
});
apply_codex_openai_responses_special_body_edits(
&mut provider_request_body,
"codex",
"openai:responses",
None,
None,
);
assert_eq!(
provider_request_body["model"],
json!(CODEX_OPENAI_IMAGE_INTERNAL_MODEL)
);
assert_eq!(provider_request_body["stream"], json!(true));
assert_eq!(
provider_request_body["tools"][0]["type"],
json!("image_generation")
);
assert_eq!(
provider_request_body["tool_choice"]["type"],
json!("image_generation")
);
}
#[test]
fn codex_image_body_edits_preserve_edit_action_without_generate_defaults() {
let mut provider_request_body = json!({

79
crates/aether-ai-formats/src/formats/openai/responses/response.rs
View File

@@ -89,13 +89,37 @@ pub fn to_raw(canonical: &CanonicalResponse, report_context: &Value, _compact: b
for block in &canonical.content {
match block {
CanonicalContentBlock::Text { .. }
| CanonicalContentBlock::Image { .. }
| CanonicalContentBlock::File { .. }
| CanonicalContentBlock::Audio { .. } => {
if let Some(part) = canonical_content_block_to_openai_responses_part(block) {
message_content.push(part);
}
}
CanonicalContentBlock::Image {
data,
url,
media_type,
extensions,
..
} => {
if image_block_is_generation_call(extensions) {
flush_openai_responses_message_item(
&mut output,
&mut message_content,
&response_id,
&mut message_index,
);
output.push(openai_responses_image_generation_call_item(
&response_id,
output.len(),
data,
url,
media_type,
));
} else if let Some(part) = canonical_content_block_to_openai_responses_part(block) {
message_content.push(part);
}
}
CanonicalContentBlock::Thinking {
text,
encrypted_content,
@@ -248,3 +272,56 @@ pub fn to_raw(canonical: &CanonicalResponse, report_context: &Value, _compact: b
));
Value::Object(response)
}
fn image_block_is_generation_call(extensions: &BTreeMap<String, Value>) -> bool {
extensions
.get(OPENAI_RESPONSES_EXTENSION_NAMESPACE)
.or_else(|| extensions.get(OPENAI_RESPONSES_LEGACY_EXTENSION_NAMESPACE))
.and_then(|value| value.get("item_type"))
.and_then(Value::as_str)
.is_some_and(|value| value == "image_generation_call")
}
fn openai_responses_image_generation_call_item(
response_id: &str,
index: usize,
data: &Option<String>,
url: &Option<String>,
media_type: &Option<String>,
) -> Value {
let mut item = Map::new();
item.insert(
"id".to_string(),
Value::String(format!("{response_id}_ig_{index}")),
);
item.insert(
"type".to_string(),
Value::String("image_generation_call".to_string()),
);
item.insert("status".to_string(), Value::String("completed".to_string()));
item.insert("action".to_string(), Value::String("generate".to_string()));
item.insert(
"output_format".to_string(),
Value::String(openai_responses_output_format_from_mime_type(
media_type.as_deref().unwrap_or("image/png"),
)),
);
if let Some(data) = data.as_ref().filter(|value| !value.trim().is_empty()) {
item.insert("result".to_string(), Value::String(data.clone()));
} else if let Some(url) = url.as_ref().filter(|value| !value.trim().is_empty()) {
item.insert("url".to_string(), Value::String(url.clone()));
} else {
item.insert("result".to_string(), Value::String(String::new()));
}
Value::Object(item)
}
fn openai_responses_output_format_from_mime_type(mime_type: &str) -> String {
match mime_type.trim().to_ascii_lowercase().as_str() {
"image/jpeg" | "image/jpg" => "jpeg",
"image/webp" => "webp",
"image/gif" => "gif",
_ => "png",
}
.to_string()
}

90
crates/aether-ai-formats/src/formats/shared/image_bridge.rs
View File

@@ -1,6 +1,5 @@
use serde_json::{json, Map, Number, Value};
use crate::formats::openai::responses::codex::CODEX_OPENAI_IMAGE_DEFAULT_VARIATION_PROMPT;
use crate::formats::shared::model_directives::extract_gemini_model_from_path;
#[derive(Clone, Debug, PartialEq)]
@@ -32,11 +31,6 @@ pub fn build_gemini_image_request_body_from_openai_image_request(
}
let prompt = normalized_request_prompt(normalized_request)
.or_else(|| {
(normalized_request.operation
== crate::formats::openai::image::request::OpenAiImageOperation::Variation)
.then(|| CODEX_OPENAI_IMAGE_DEFAULT_VARIATION_PROMPT.to_string())
})
.unwrap_or_else(|| "Generate a high quality image.".to_string());
let mut parts = Vec::new();
if !prompt.trim().is_empty() {
@@ -393,20 +387,7 @@ pub fn build_openai_image_response_from_response_stream_sync_body(
let output = provider_body_json.get("output").and_then(Value::as_array)?;
let images = output
.iter()
.filter_map(|item| {
if item.get("type").and_then(Value::as_str) != Some("image_generation_call") {
return None;
}
let b64_json = item
.get("result")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
Some(json!({
"b64_json": b64_json,
"revised_prompt": item.get("revised_prompt").cloned().unwrap_or(Value::Null),
}))
})
.filter_map(openai_response_image_generation_item_to_image_data)
.collect::<Vec<_>>();
if images.is_empty() {
return None;
@@ -439,6 +420,48 @@ pub fn build_openai_image_response_from_response_stream_sync_body(
Some(Value::Object(response))
}
fn openai_response_image_generation_item_to_image_data(item: &Value) -> Option<Value> {
if item.get("type").and_then(Value::as_str) != Some("image_generation_call") {
return None;
}
let result = item
.get("result")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty());
let url = item
.get("url")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty());
let mut image = Map::new();
match result {
Some(value) if value.starts_with("data:") => {
let (_, b64_json) = parse_data_url(value)?;
image.insert("b64_json".to_string(), Value::String(b64_json));
}
Some(value) if value.starts_with("http://") || value.starts_with("https://") => {
image.insert("url".to_string(), Value::String(value.to_string()));
}
Some(value) => {
image.insert("b64_json".to_string(), Value::String(value.to_string()));
}
None => {
let url = url?;
if let Some((_, b64_json)) = parse_data_url(url) {
image.insert("b64_json".to_string(), Value::String(b64_json));
} else {
image.insert("url".to_string(), Value::String(url.to_string()));
}
}
}
image.insert(
"revised_prompt".to_string(),
item.get("revised_prompt").cloned().unwrap_or(Value::Null),
);
Some(Value::Object(image))
}
pub fn build_openai_image_provider_body_from_response_stream_sync_body(
provider_body_json: &Value,
report_context: Option<&Value>,
@@ -882,7 +905,9 @@ mod tests {
build_gemini_image_request_body_from_openai_image_request,
build_gemini_image_response_from_openai_image_response,
build_openai_image_request_body_from_gemini_image_request,
build_openai_image_response_from_gemini_response, gemini_request_is_image_generation,
build_openai_image_response_from_gemini_response,
build_openai_image_response_from_response_stream_sync_body,
gemini_request_is_image_generation,
};
use crate::formats::openai::image::request::normalize_openai_image_request;
@@ -1013,6 +1038,29 @@ mod tests {
assert_eq!(converted["usage"]["total_tokens"], 3);
}
#[test]
fn converts_responses_image_generation_url_to_openai_image_url() {
let converted = build_openai_image_response_from_response_stream_sync_body(
&json!({
"created_at": 1776839946,
"model": "gpt-image-2",
"output": [{
"type": "image_generation_call",
"status": "completed",
"url": "https://assets.example/generated.png"
}]
}),
None,
)
.expect("response image output should convert");
assert_eq!(
converted["data"][0]["url"],
"https://assets.example/generated.png"
);
assert!(converted["data"][0].get("b64_json").is_none());
}
#[test]
fn converts_openai_image_response_to_gemini_image_response() {
let converted = build_gemini_image_response_from_openai_image_response(

7
crates/aether-ai-formats/src/formats/shared/routing.rs
View File

@@ -193,10 +193,7 @@ pub fn resolve_execution_runtime_sync_plan_kind(
if route_family == Some("openai")
&& route_kind == Some("image")
&& *method == Method::POST
&& matches!(
path,
"/v1/images/generations" | "/v1/images/edits" | "/v1/images/variations"
)
&& matches!(path, "/v1/images/generations" | "/v1/images/edits")
{
return Some(OPENAI_IMAGE_SYNC_PLAN_KIND);
}
@@ -761,7 +758,7 @@ mod tests {
&Method::POST,
"/v1/images/variations",
),
Some(OPENAI_IMAGE_SYNC_PLAN_KIND)
None
);
assert!(supports_sync_execution_decision_kind(
OPENAI_IMAGE_SYNC_PLAN_KIND

51
crates/aether-ai-formats/src/formats/shared/stream_core/common.rs
View File

@@ -160,6 +160,57 @@ pub fn canonical_usage_from_claude_usage(value: Option<&Value>) -> Option<Canoni
})
}
pub fn content_part_from_openai_image_generation_item(
item: &Value,
) -> Option<CanonicalContentPart> {
let item = item.as_object()?;
let result = item
.get("result")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty());
let url = item
.get("url")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty());
let image = if let Some(result) = result {
if result.starts_with("data:image/")
|| result.starts_with("http://")
|| result.starts_with("https://")
{
result.to_string()
} else {
let mime_type = item
.get("mime_type")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.or_else(|| {
item.get("output_format")
.and_then(Value::as_str)
.map(openai_image_output_format_to_mime_type)
})
.unwrap_or_else(|| "image/png".to_string());
format!("data:{mime_type};base64,{result}")
}
} else {
url?.to_string()
};
Some(CanonicalContentPart::ImageUrl(image))
}
fn openai_image_output_format_to_mime_type(output_format: &str) -> String {
match output_format.trim().to_ascii_lowercase().as_str() {
"jpeg" | "jpg" => "image/jpeg",
"webp" => "image/webp",
"gif" => "image/gif",
_ => "image/png",
}
.to_string()
}
pub fn canonical_usage_from_gemini_usage(value: Option<&Value>) -> Option<CanonicalUsage> {
let usage = value?.as_object()?;
let input_tokens = usage

39
crates/aether-ai-formats/src/formats/shared/sync_products.rs
View File

@@ -22,8 +22,8 @@ use crate::formats::gemini::generate_content::stream::GeminiProviderState;
use crate::formats::shared::model_directives::model_directive_display_model_from_report_context;
use crate::formats::shared::response::remove_empty_pages_from_tool_arguments;
use crate::formats::shared::stream_core::common::{
map_openai_finish_reason_to_gemini, parse_json_arguments_value, CanonicalContentPart,
CanonicalStreamEvent, CanonicalUsage,
content_part_from_openai_image_generation_item, map_openai_finish_reason_to_gemini,
parse_json_arguments_value, CanonicalContentPart, CanonicalStreamEvent, CanonicalUsage,
};
#[derive(Clone, Debug, PartialEq)]
@@ -1577,6 +1577,7 @@ pub fn aggregate_openai_responses_stream_sync_response(body: &[u8]) -> Option<Va
let mut message_states: BTreeMap<usize, OpenAIResponsesSyncMessageState> = BTreeMap::new();
let mut reasoning_states: BTreeMap<usize, OpenAIResponsesSyncReasoningState> = BTreeMap::new();
let mut tool_states: BTreeMap<usize, OpenAIResponsesSyncToolState> = BTreeMap::new();
let mut image_items: BTreeMap<usize, Value> = BTreeMap::new();
let mut item_output_indexes = BTreeMap::<String, usize>::new();
for event in events {
@@ -1734,6 +1735,9 @@ pub fn aggregate_openai_responses_stream_sync_response(body: &[u8]) -> Option<Va
item,
);
}
"image_generation_call" => {
image_items.insert(output_index, Value::Object(item.clone()));
}
_ => {}
}
}
@@ -1833,6 +1837,9 @@ pub fn aggregate_openai_responses_stream_sync_response(body: &[u8]) -> Option<Va
item,
);
}
"image_generation_call" => {
image_items.insert(output_index, Value::Object(item.clone()));
}
_ => {}
}
}
@@ -1874,6 +1881,7 @@ pub fn aggregate_openai_responses_stream_sync_response(body: &[u8]) -> Option<Va
.keys()
.chain(reasoning_states.keys())
.chain(tool_states.keys())
.chain(image_items.keys())
.copied()
.collect::<Vec<_>>();
output_indexes.sort_unstable();
@@ -1897,6 +1905,9 @@ pub fn aggregate_openai_responses_stream_sync_response(body: &[u8]) -> Option<Va
if let Some(state) = tool_states.remove(&output_index) {
output.push(materialize_openai_responses_tool_item(output_index, state));
}
if let Some(item) = image_items.remove(&output_index) {
output.push(item);
}
}
response.insert("output".to_string(), Value::Array(output));
}
@@ -2541,6 +2552,11 @@ pub fn aggregate_gemini_stream_sync_response(body: &[u8]) -> Option<Value> {
CanonicalStreamEvent::ContentPart(part) => {
parts.push(gemini_sync_part_from_canonical_content_part(part));
}
CanonicalStreamEvent::ImageGenerationCall { item, .. } => {
if let Some(part) = content_part_from_openai_image_generation_item(&item) {
parts.push(gemini_sync_part_from_canonical_content_part(part));
}
}
CanonicalStreamEvent::ToolCallStart {
index,
call_id,
@@ -3464,6 +3480,25 @@ mod tests {
assert_eq!(result["output"][0]["content"][0]["text"], "Authoritative");
}
#[test]
fn reconstructs_openai_responses_image_generation_call_from_output_item_done() {
let body = concat!(
"event: response.created\n",
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_image_123\",\"object\":\"response\",\"model\":\"gpt-5.4-mini\",\"status\":\"in_progress\",\"output\":[]}}\n\n",
"event: response.output_item.done\n",
"data: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"ig_123\",\"type\":\"image_generation_call\",\"status\":\"completed\",\"output_format\":\"png\",\"result\":\"aGVsbG8=\"}}\n\n",
"event: response.completed\n",
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_image_123\",\"object\":\"response\",\"model\":\"gpt-5.4-mini\",\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":1,\"output_tokens\":2,\"total_tokens\":3}}}\n\n",
);
let result = aggregate_openai_responses_stream_sync_response(body.as_bytes())
.expect("openai-responses stream should aggregate into a sync body");
assert_eq!(result["output"][0]["type"], "image_generation_call");
assert_eq!(result["output"][0]["result"], "aGVsbG8=");
assert_eq!(result["output"][0]["output_format"], "png");
}
#[test]
fn reconstructs_openai_responses_multi_part_message_content_order() {
let body = concat!(

120
crates/aether-ai-formats/src/protocol/canonical.rs
View File

@@ -1762,6 +1762,11 @@ pub(crate) fn openai_responses_output_to_canonical_blocks(
),
});
}
"image_generation_call" => {
blocks.push(openai_responses_image_generation_call_to_block(
item_object,
)?);
}
"output_text" | "text" | "output_image" | "image_url" | "file" | "input_file"
| "input_audio" => blocks.push(openai_responses_part_to_canonical_block(item)?),
_ => blocks.push(CanonicalContentBlock::Unknown {
@@ -1774,6 +1779,79 @@ pub(crate) fn openai_responses_output_to_canonical_blocks(
Some(blocks)
}
fn openai_responses_image_generation_call_to_block(
item_object: &Map<String, Value>,
) -> Option<CanonicalContentBlock> {
let result = item_object
.get("result")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty());
let url = item_object
.get("url")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty());
let raw_image = result.or(url)?;
let fallback_media_type = item_object
.get("mime_type")
.and_then(Value::as_str)
.map(ToOwned::to_owned)
.or_else(|| {
item_object
.get("output_format")
.and_then(Value::as_str)
.map(openai_responses_output_format_to_mime_type)
});
let (media_type, data, url) = if raw_image.starts_with("data:image/") {
split_data_url(Some(raw_image.to_string()), fallback_media_type)
} else if raw_image.starts_with("http://") || raw_image.starts_with("https://") {
(fallback_media_type, None, Some(raw_image.to_string()))
} else if result.is_some() {
(
fallback_media_type.or_else(|| Some("image/png".to_string())),
Some(raw_image.to_string()),
None,
)
} else {
(fallback_media_type, None, Some(raw_image.to_string()))
};
let mut extensions = openai_responses_extensions(
item_object,
&[
"type",
"id",
"status",
"action",
"result",
"url",
"output_format",
"mime_type",
],
);
canonical_extension_object_mut(&mut extensions, OPENAI_RESPONSES_EXTENSION_NAMESPACE).insert(
"item_type".to_string(),
Value::String("image_generation_call".to_string()),
);
Some(CanonicalContentBlock::Image {
data,
url,
media_type,
detail: None,
extensions,
})
}
fn openai_responses_output_format_to_mime_type(output_format: &str) -> String {
match output_format.trim().to_ascii_lowercase().as_str() {
"jpeg" | "jpg" => "image/jpeg",
"webp" => "image/webp",
"gif" => "image/gif",
_ => "image/png",
}
.to_string()
}
pub(crate) fn openai_responses_part_to_canonical_block(
part: &Value,
) -> Option<CanonicalContentBlock> {
@@ -5327,6 +5405,48 @@ mod tests {
assert_eq!(rebuilt["service_tier"], "flex");
}
#[test]
fn openai_responses_image_generation_call_becomes_canonical_image_block() {
let response = json!({
"id": "resp_img",
"model": "gpt-image-2",
"status": "completed",
"output": [{
"id": "ig_1",
"type": "image_generation_call",
"status": "completed",
"output_format": "png",
"result": "aW1hZ2U="
}]
});
let canonical =
from_openai_responses_to_canonical_response(&response).expect("canonical response");
assert!(matches!(
canonical.content[0],
CanonicalContentBlock::Image { ref data, ref media_type, .. }
if data.as_deref() == Some("aW1hZ2U=")
&& media_type.as_deref() == Some("image/png")
));
let rebuilt_chat = canonical_to_openai_chat_response(&canonical);
assert_eq!(
rebuilt_chat["choices"][0]["message"]["content"][0]["type"],
json!("image_url")
);
assert_eq!(
rebuilt_chat["choices"][0]["message"]["content"][0]["image_url"]["url"],
json!("data:image/png;base64,aW1hZ2U=")
);
let rebuilt_responses = canonical_to_openai_responses_response(&canonical, &json!({}));
assert_eq!(
rebuilt_responses["output"][0]["type"],
json!("image_generation_call")
);
assert_eq!(rebuilt_responses["output"][0]["result"], json!("aW1hZ2U="));
}
#[test]
fn claude_request_adapter_preserves_cache_thinking_tools_and_builtin_extensions() {
let request = json!({

4
crates/aether-ai-formats/src/protocol/stream.rs
View File

@@ -38,6 +38,10 @@ pub enum CanonicalStreamEvent {
ReasoningSummaryDone,
ReasoningSignature(String),
ContentPart(CanonicalContentPart),
ImageGenerationCall {
index: usize,
item: Value,
},
ToolCallStart {
index: usize,
call_id: String,