i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 09:20:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ai-pipeline): 支持 reasoning signature 及媒体内容块的跨格式流式转换

Browse Source 
- 在 CanonicalStreamEvent 中新增 ReasoningSignature 和 ContentPart 变体，CanonicalContentPart 枚举覆盖图片/文件/音频
- Gemini 流解析器提取 thoughtSignature，并对 inlineData/fileData 等非文本 part 生成 ContentPart 事件
- Claude 流聚合支持 thinking_delta / signature_delta，输出 thinking block 携带 signature 字段
- Gemini 同步响应聚合重写，支持媒体 part 和 reasoning signature 的完整还原
- 跨格式矩阵（gemini↔claude↔openai）补全图片块双向转换及 reasoning signature 传递
- 请求转换层（to/from openai_chat）补全 Claude/Gemini 的 thinking、图片、工具调用字段映射
- 新增/扩展测试：inline image 双向重写、thinking signature 聚合、跨格式 sync product 媒体字段
This commit is contained in:
fawney19
2026-04-24 02:18:34 +08:00
parent 0f94f92c37
commit 581bc03d4e
28 changed files with 4155 additions and 499 deletions
Download Patch File Download Diff File Expand all files Collapse all files

302
crates/aether-ai-pipeline/src/conversion/request/from_openai_chat/claude.rs
View File

@@ -5,8 +5,8 @@ use super::super::to_openai_chat::{extract_openai_text_content, parse_openai_too
use super::shared::parse_openai_tool_arguments;
use crate::planner::openai::{
copy_request_number_field, extract_openai_reasoning_effort,
map_openai_reasoning_effort_to_thinking_budget, parse_openai_stop_sequences,
resolve_openai_chat_max_tokens,
map_openai_reasoning_effort_to_claude_output, map_openai_reasoning_effort_to_thinking_budget,
parse_openai_stop_sequences, resolve_openai_chat_max_tokens,
};
pub fn convert_openai_chat_request_to_claude_request(
@@ -37,17 +37,18 @@ pub fn convert_openai_chat_request_to_claude_request(
"user" => {
let blocks = convert_openai_content_to_claude_blocks(
message_object.get("content"),
true,
ClaudeMessageRole::User,
)?;
if !blocks.is_empty() {
messages.push(build_claude_message("user", blocks));
}
}
"assistant" => {
let mut blocks = convert_openai_content_to_claude_blocks(
let mut blocks = extract_openai_reasoning_to_claude_blocks(message_object);
blocks.extend(convert_openai_content_to_claude_blocks(
message_object.get("content"),
false,
)?;
ClaudeMessageRole::Assistant,
)?);
if let Some(tool_calls) =
message_object.get("tool_calls").and_then(Value::as_array)
{
@@ -159,14 +160,30 @@ pub fn convert_openai_chat_request_to_claude_request(
}),
);
}
if let Some(output_effort) =
map_openai_reasoning_effort_to_claude_output(reasoning_effort.as_str())
{
output.insert(
"output_config".to_string(),
json!({
"effort": output_effort,
}),
);
}
}
Some(Value::Object(output))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ClaudeMessageRole {
User,
Assistant,
}
fn convert_openai_content_to_claude_blocks(
content: Option<&Value>,
allow_images: bool,
role: ClaudeMessageRole,
) -> Option<Vec<Value>> {
match content {
None | Some(Value::Null) => Some(Vec::new()),
@@ -187,14 +204,14 @@ fn convert_openai_content_to_claude_blocks(
.and_then(Value::as_str)
.unwrap_or_default();
match part_type {
"text" | "input_text" => {
"text" | "input_text" | "output_text" => {
if let Some(text) = part_object.get("text").and_then(Value::as_str) {
if !text.trim().is_empty() {
blocks.push(json!({ "type": "text", "text": text }));
}
}
}
"image_url" | "input_image" if allow_images => {
"image_url" | "input_image" | "output_image" => {
let url = part_object
.get("image_url")
.and_then(|value| {
@@ -206,23 +223,36 @@ fn convert_openai_content_to_claude_blocks(
.map(ToOwned::to_owned)
})
})
.or_else(|| {
part_object
.get("url")
.and_then(Value::as_str)
.map(ToOwned::to_owned)
})
.filter(|value| !value.trim().is_empty())?;
if let Some((media_type, data)) = parse_data_url(url.as_str()) {
blocks.push(json!({
"type": "image",
"source": {
"type": "base64",
"media_type": media_type,
"data": data,
}
}));
if role == ClaudeMessageRole::User {
if let Some((media_type, data)) = parse_data_url(url.as_str()) {
blocks.push(json!({
"type": "image",
"source": {
"type": "base64",
"media_type": media_type,
"data": data,
}
}));
} else {
blocks.push(json!({
"type": "image",
"source": {
"type": "url",
"url": url,
}
}));
}
} else {
blocks.push(json!({
"type": "image",
"source": {
"type": "url",
"url": url,
}
"type": "text",
"text": assistant_image_placeholder(url.as_str()),
}));
}
}
@@ -244,6 +274,42 @@ fn convert_openai_content_to_claude_blocks(
}
}));
}
} else if let Some(file_id) = file_object
.get("file_id")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
{
blocks.push(json!({
"type": "text",
"text": format!("[File: {file_id}]"),
}));
}
}
"input_audio" => {
let audio_object = part_object
.get("input_audio")
.and_then(Value::as_object)
.unwrap_or(part_object);
let data = audio_object
.get("data")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty());
let format = audio_object
.get("format")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty());
if let (Some(data), Some(format)) = (data, format) {
blocks.push(json!({
"type": "document",
"source": {
"type": "base64",
"media_type": format!("audio/{format}"),
"data": data,
}
}));
}
}
_ => {}
@@ -255,6 +321,77 @@ fn convert_openai_content_to_claude_blocks(
}
}
fn extract_openai_reasoning_to_claude_blocks(message: &Map<String, Value>) -> Vec<Value> {
let mut blocks = Vec::new();
if let Some(reasoning_parts) = message.get("reasoning_parts").and_then(Value::as_array) {
for reasoning_part in reasoning_parts {
let Some(reasoning_object) = reasoning_part.as_object() else {
continue;
};
match reasoning_object
.get("type")
.and_then(Value::as_str)
.unwrap_or("thinking")
{
"thinking" => {
let thinking = reasoning_object
.get("thinking")
.or_else(|| reasoning_object.get("text"))
.and_then(Value::as_str)
.map(str::trim)
.unwrap_or_default();
if thinking.is_empty() {
continue;
}
let mut block = Map::new();
block.insert("type".to_string(), Value::String("thinking".to_string()));
block.insert("thinking".to_string(), Value::String(thinking.to_string()));
if let Some(signature) = reasoning_object
.get("signature")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
{
block.insert(
"signature".to_string(),
Value::String(signature.to_string()),
);
}
blocks.push(Value::Object(block));
}
"redacted_thinking" => {
let data = reasoning_object
.get("data")
.and_then(Value::as_str)
.map(str::trim)
.unwrap_or_default();
if !data.is_empty() {
blocks.push(json!({
"type": "redacted_thinking",
"data": data,
}));
}
}
_ => {}
}
}
}
if !blocks.is_empty() {
return blocks;
}
if let Some(reasoning_content) = message
.get("reasoning_content")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
{
blocks.push(json!({
"type": "thinking",
"thinking": reasoning_content,
}));
}
blocks
}
fn convert_openai_tools_to_claude(
tools: Option<&Value>,
web_search_options: Option<&Value>,
@@ -491,6 +628,14 @@ fn parse_data_url(value: &str) -> Option<(String, String)> {
Some((media_type.to_string(), data.to_string()))
}
fn assistant_image_placeholder(url: &str) -> String {
if url.starts_with("data:") {
"[Image]".to_string()
} else {
format!("[Image: {url}]")
}
}
#[cfg(test)]
mod tests {
use super::convert_openai_chat_request_to_claude_request;
@@ -555,4 +700,115 @@ mod tests {
})
);
}
#[test]
fn converts_openai_multipart_reasoning_and_file_id_to_claude_request() {
let request = json!({
"model": "gpt-5.4",
"messages": [
{
"role": "user",
"content": [
{ "type": "text", "text": "Read this" },
{
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,iVBORw0KGgo="
}
},
{
"type": "file",
"file": {
"file_data": "data:application/pdf;base64,JVBERi0x"
}
}
]
},
{
"role": "assistant",
"reasoning_content": "step by step",
"reasoning_parts": [
{
"type": "thinking",
"thinking": "step by step",
"signature": "sig_123"
},
{
"type": "redacted_thinking",
"data": "redacted_blob"
}
],
"content": [
{
"type": "image_url",
"image_url": {
"url": "https://example.com/diagram.png"
}
},
{ "type": "file", "file": { "file_id": "file_123" } },
{ "type": "text", "text": "done" }
],
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": {
"name": "lookup",
"arguments": "\"tokyo\""
}
}]
}
],
"reasoning_effort": "xhigh"
});
let converted =
convert_openai_chat_request_to_claude_request(&request, "claude-sonnet-4-5", false)
.expect("request should convert");
assert_eq!(
converted["messages"][0]["content"],
json!([
{ "type": "text", "text": "Read this" },
{
"type": "image",
"source": {
"type": "base64",
"media_type": "image/png",
"data": "iVBORw0KGgo="
}
},
{
"type": "document",
"source": {
"type": "base64",
"media_type": "application/pdf",
"data": "JVBERi0x"
}
}
])
);
assert_eq!(converted["messages"][1]["content"][0]["type"], "thinking");
assert_eq!(
converted["messages"][1]["content"][0]["signature"],
"sig_123"
);
assert_eq!(
converted["messages"][1]["content"][1]["type"],
"redacted_thinking"
);
assert_eq!(
converted["messages"][1]["content"][2]["text"],
"[Image: https://example.com/diagram.png]"
);
assert_eq!(
converted["messages"][1]["content"][3]["text"],
"[File: file_123]"
);
assert_eq!(
converted["messages"][1]["content"][5]["input"],
json!({"raw": "tokyo"})
);
assert_eq!(converted["thinking"]["budget_tokens"], 8192);
assert_eq!(converted["output_config"]["effort"], "max");
}
}

277
crates/aether-ai-pipeline/src/conversion/request/from_openai_chat/gemini.rs
View File

@@ -39,7 +39,7 @@ pub fn convert_openai_chat_request_to_gemini_request(
"user" => {
let parts = convert_openai_content_to_gemini_parts(
message_object.get("content"),
true,
OpenAiToGeminiRole::User,
)?;
if !parts.is_empty() {
contents.push(json!({
@@ -51,8 +51,12 @@ pub fn convert_openai_chat_request_to_gemini_request(
"assistant" => {
let mut parts = convert_openai_content_to_gemini_parts(
message_object.get("content"),
false,
OpenAiToGeminiRole::Assistant,
)?;
let reasoning_parts = extract_openai_reasoning_to_gemini_parts(message_object);
if !reasoning_parts.is_empty() {
parts.splice(0..0, reasoning_parts);
}
if let Some(tool_calls) =
message_object.get("tool_calls").and_then(Value::as_array)
{
@@ -249,14 +253,105 @@ pub fn convert_openai_chat_request_to_gemini_request(
.or_insert(thinking_config);
}
}
if let Some(gemini) = extra_body.get("gemini").and_then(Value::as_object) {
let existing = output
.entry("generationConfig".to_string())
.or_insert_with(|| Value::Object(Map::new()))
.as_object_mut()?;
if let Some(extra_config) = gemini
.get("generation_config_extra")
.or_else(|| gemini.get("generationConfigExtra"))
.and_then(Value::as_object)
{
for (key, value) in extra_config {
existing.entry(key.clone()).or_insert_with(|| value.clone());
}
}
if let Some(safety_settings) = gemini
.get("safety_settings")
.or_else(|| gemini.get("safetySettings"))
.cloned()
{
output.insert("safetySettings".to_string(), safety_settings);
}
if let Some(cached_content) = gemini
.get("cached_content")
.or_else(|| gemini.get("cachedContent"))
.cloned()
{
output.insert("cachedContent".to_string(), cached_content);
}
}
}
Some(Value::Object(output))
}
fn extract_openai_reasoning_to_gemini_parts(message: &Map<String, Value>) -> Vec<Value> {
let mut parts = Vec::new();
if let Some(reasoning_parts) = message.get("reasoning_parts").and_then(Value::as_array) {
for reasoning_part in reasoning_parts {
let Some(reasoning_object) = reasoning_part.as_object() else {
continue;
};
if reasoning_object
.get("type")
.and_then(Value::as_str)
.is_some_and(|value| value != "thinking")
{
continue;
}
let thinking = reasoning_object
.get("thinking")
.or_else(|| reasoning_object.get("text"))
.and_then(Value::as_str)
.map(str::trim)
.unwrap_or_default();
if thinking.is_empty() {
continue;
}
let mut part = Map::new();
part.insert("text".to_string(), Value::String(thinking.to_string()));
part.insert("thought".to_string(), Value::Bool(true));
if let Some(signature) = reasoning_object
.get("signature")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
{
part.insert(
"thoughtSignature".to_string(),
Value::String(signature.to_string()),
);
}
parts.push(Value::Object(part));
}
}
if !parts.is_empty() {
return parts;
}
if let Some(reasoning_content) = message
.get("reasoning_content")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
{
parts.push(json!({
"text": reasoning_content,
"thought": true,
}));
}
parts
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum OpenAiToGeminiRole {
User,
Assistant,
}
fn convert_openai_content_to_gemini_parts(
content: Option<&Value>,
allow_images: bool,
_role: OpenAiToGeminiRole,
) -> Option<Vec<Value>> {
match content {
None | Some(Value::Null) => Some(Vec::new()),
@@ -277,14 +372,14 @@ fn convert_openai_content_to_gemini_parts(
.and_then(Value::as_str)
.unwrap_or_default();
match part_type {
"text" | "input_text" => {
"text" | "input_text" | "output_text" => {
if let Some(text) = part_object.get("text").and_then(Value::as_str) {
if !text.trim().is_empty() {
converted.push(json!({ "text": text }));
}
}
}
"image_url" | "input_image" if allow_images => {
"image_url" | "input_image" | "output_image" => {
let image = part_object
.get("image_url")
.and_then(|value| {
@@ -334,6 +429,39 @@ fn convert_openai_content_to_gemini_parts(
}
}));
}
} else if let Some(file_id) = file_object
.get("file_id")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
{
converted.push(json!({
"text": format!("[File: {file_id}]"),
}));
}
}
"input_audio" => {
let audio_object = part_object
.get("input_audio")
.and_then(Value::as_object)
.unwrap_or(part_object);
let data = audio_object
.get("data")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty());
let format = audio_object
.get("format")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty());
if let (Some(data), Some(format)) = (data, format) {
converted.push(json!({
"inlineData": {
"mimeType": format!("audio/{format}"),
"data": data,
}
}));
}
}
_ => {}
@@ -1137,4 +1265,143 @@ mod tests {
assert_eq!(tools.len(), 1);
assert_eq!(tools[0], json!({ "googleSearch": {} }));
}
#[test]
fn converts_openai_reasoning_multimodal_and_passthrough_to_gemini_request() {
let request = json!({
"model": "gpt-5.4",
"messages": [
{
"role": "user",
"content": [
{ "type": "text", "text": "Read this" },
{
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,iVBORw0KGgo="
}
},
{
"type": "file",
"file": {
"file_data": "data:application/pdf;base64,JVBERi0x"
}
},
{
"type": "input_audio",
"input_audio": {
"data": "SUQz",
"format": "mp3"
}
}
]
},
{
"role": "assistant",
"reasoning_content": "step by step",
"reasoning_parts": [
{
"type": "thinking",
"thinking": "step by step",
"signature": "sig_123"
}
],
"content": [
{
"type": "image_url",
"image_url": {
"url": "https://example.com/cat.png"
}
},
{
"type": "file",
"file": { "file_id": "file_123" }
},
{ "type": "text", "text": "done" }
],
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": {
"name": "lookup",
"arguments": "\"tokyo\""
}
}]
}
],
"reasoning_effort": "high",
"extra_body": {
"google": {
"response_modalities": ["TEXT", "IMAGE"],
"thinking_config": {
"includeThoughts": true,
"thinkingBudget": 2048
}
},
"gemini": {
"generation_config_extra": {
"presencePenalty": 0.5
},
"safety_settings": [
{ "category": "HARM_CATEGORY_HATE_SPEECH" }
],
"cached_content": "cached/abc"
}
}
});
let converted =
convert_openai_chat_request_to_gemini_request(&request, "gemini-2.5-pro", false)
.expect("request should convert");
assert_eq!(
converted["contents"][0],
json!({
"role": "user",
"parts": [
{ "text": "Read this" },
{ "inlineData": { "mimeType": "image/png", "data": "iVBORw0KGgo=" } },
{ "inlineData": { "mimeType": "application/pdf", "data": "JVBERi0x" } },
{ "inlineData": { "mimeType": "audio/mp3", "data": "SUQz" } }
]
})
);
assert_eq!(converted["contents"][1]["role"], "model");
assert_eq!(converted["contents"][1]["parts"][0]["thought"], true);
assert_eq!(
converted["contents"][1]["parts"][0]["thoughtSignature"],
"sig_123"
);
assert_eq!(
converted["contents"][1]["parts"][1],
json!({
"fileData": {
"fileUri": "https://example.com/cat.png",
"mimeType": "image/png"
}
})
);
assert_eq!(
converted["contents"][1]["parts"][2]["text"],
"[File: file_123]"
);
assert_eq!(
converted["contents"][1]["parts"][4]["functionCall"]["args"],
json!({ "raw": "tokyo" })
);
assert_eq!(
converted["generationConfig"]["thinkingConfig"]["thinkingBudget"],
4096
);
assert_eq!(
converted["generationConfig"]["responseModalities"],
json!(["TEXT", "IMAGE"])
);
assert_eq!(converted["generationConfig"]["presencePenalty"], 0.5);
assert_eq!(
converted["safetySettings"],
json!([{ "category": "HARM_CATEGORY_HATE_SPEECH" }])
);
assert_eq!(converted["cachedContent"], "cached/abc");
}
}

6
crates/aether-ai-pipeline/src/conversion/request/from_openai_chat/shared.rs
View File

@@ -10,12 +10,12 @@ pub(super) fn parse_openai_tool_arguments(arguments: Option<&Value>) -> Option<V
} else {
match serde_json::from_str::<Value>(trimmed) {
Ok(Value::Object(object)) => Some(Value::Object(object)),
Ok(other) => Some(json!({ "input": other })),
Err(_) => Some(json!({ "input": trimmed })),
Ok(other) => Some(json!({ "raw": other })),
Err(_) => Some(json!({ "raw": trimmed })),
}
}
}
Some(other) => Some(json!({ "input": other })),
Some(other) => Some(json!({ "raw": other })),
None => Some(json!({})),
}
}

517
crates/aether-ai-pipeline/src/conversion/request/to_openai_chat/claude.rs
View File

@@ -32,82 +32,16 @@ pub fn normalize_claude_request_to_openai_chat_request(body_json: &Value) -> Opt
.to_ascii_lowercase();
match role.as_str() {
"user" => {
let mut text_segments = Vec::new();
if let Some(content) = message_object.get("content") {
for block in normalize_claude_content_blocks(content)? {
match block {
ClaudeNormalizedBlock::Text(text) => {
if !text.trim().is_empty() {
text_segments.push(text);
}
}
ClaudeNormalizedBlock::ToolResult {
tool_use_id,
content,
} => {
messages.push(json!({
"role": "tool",
"tool_call_id": tool_use_id,
"content": content,
}));
}
ClaudeNormalizedBlock::ToolUse { .. } => {}
}
}
}
let text = text_segments.join("\n\n");
if !text.trim().is_empty() {
messages.push(json!({
"role": "user",
"content": text,
}));
}
append_claude_user_message_to_openai_messages(
message_object.get("content"),
&mut messages,
)?;
}
"assistant" => {
let mut text_segments = Vec::new();
let mut tool_calls = Vec::new();
if let Some(content) = message_object.get("content") {
for block in normalize_claude_content_blocks(content)? {
match block {
ClaudeNormalizedBlock::Text(text) => {
if !text.trim().is_empty() {
text_segments.push(text);
}
}
ClaudeNormalizedBlock::ToolUse { id, name, input } => {
let tool_use_id = id.unwrap_or_else(|| {
let generated =
format!("toolu_auto_{next_generated_tool_use_index}");
next_generated_tool_use_index += 1;
generated
});
tool_calls.push(json!({
"id": tool_use_id,
"type": "function",
"function": {
"name": name,
"arguments": canonical_json_string(input.unwrap_or(Value::Object(Map::new()))),
}
}));
}
ClaudeNormalizedBlock::ToolResult { .. } => {}
}
}
}
let mut assistant = Map::new();
assistant.insert("role".to_string(), Value::String("assistant".to_string()));
assistant.insert(
"content".to_string(),
if text_segments.is_empty() && !tool_calls.is_empty() {
Value::Null
} else {
Value::String(text_segments.join("\n\n"))
},
);
if !tool_calls.is_empty() {
assistant.insert("tool_calls".to_string(), Value::Array(tool_calls));
}
messages.push(Value::Object(assistant));
messages.push(normalize_claude_assistant_message_to_openai_message(
message_object.get("content"),
&mut next_generated_tool_use_index,
)?);
}
_ => {}
}
@@ -123,8 +57,22 @@ pub fn normalize_claude_request_to_openai_chat_request(body_json: &Value) -> Opt
output.insert(passthrough_key.to_string(), value.clone());
}
}
if output.get("stop").is_none() {
if let Some(stop_sequences) = request
.get("stop_sequences")
.cloned()
.filter(|value| !value.is_null())
{
output.insert("stop".to_string(), stop_sequences);
}
}
if output.get("reasoning_effort").is_none() {
if let Some(thinking_budget) = request
if let Some(reasoning_effort) = extract_claude_output_reasoning_effort(request) {
output.insert(
"reasoning_effort".to_string(),
Value::String(reasoning_effort.to_string()),
);
} else if let Some(thinking_budget) = request
.get("thinking")
.and_then(Value::as_object)
.and_then(|thinking| thinking.get("budget_tokens"))
@@ -162,6 +110,16 @@ pub fn normalize_claude_request_to_openai_chat_request(body_json: &Value) -> Opt
#[derive(Debug)]
enum ClaudeNormalizedBlock {
Text(String),
Thinking {
text: String,
signature: Option<String>,
},
RedactedThinking {
data: String,
},
ImageUrl(String),
FileData(String),
FileUrl(String),
ToolUse {
id: Option<String>,
name: String,
@@ -181,13 +139,97 @@ fn normalize_claude_content_blocks(content: &Value) -> Option<Vec<ClaudeNormaliz
for block in blocks {
let block = block.as_object()?;
match block.get("type")?.as_str()? {
"text" | "thinking" => {
"text" => {
let text = block
.get("text")
.and_then(Value::as_str)
.unwrap_or_default();
normalized.push(ClaudeNormalizedBlock::Text(text.to_string()));
}
"thinking" => {
let thinking = block
.get("thinking")
.and_then(Value::as_str)
.or_else(|| block.get("text").and_then(Value::as_str))
.unwrap_or_default();
normalized.push(ClaudeNormalizedBlock::Thinking {
text: thinking.to_string(),
signature: block
.get("signature")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned),
});
}
"redacted_thinking" => {
let data = block
.get("data")
.and_then(Value::as_str)
.unwrap_or_default();
normalized.push(ClaudeNormalizedBlock::RedactedThinking {
data: data.to_string(),
});
}
"image" => {
let source = block.get("source")?.as_object()?;
match source.get("type")?.as_str()? {
"base64" => {
let media_type = source
.get("media_type")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
let data = source
.get("data")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
normalized.push(ClaudeNormalizedBlock::ImageUrl(build_data_url(
media_type, data,
)));
}
"url" => {
let url = source
.get("url")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?
.to_string();
normalized.push(ClaudeNormalizedBlock::ImageUrl(url));
}
_ => {}
}
}
"document" => {
let source = block.get("source")?.as_object()?;
match source.get("type")?.as_str()? {
"base64" => {
let media_type = source
.get("media_type")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
let data = source
.get("data")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
normalized.push(ClaudeNormalizedBlock::FileData(build_data_url(
media_type, data,
)));
}
"url" => {
let url = source
.get("url")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?
.to_string();
normalized.push(ClaudeNormalizedBlock::FileUrl(url));
}
_ => {}
}
}
"tool_use" => {
let name = block
.get("name")
@@ -227,6 +269,216 @@ fn normalize_claude_content_blocks(content: &Value) -> Option<Vec<ClaudeNormaliz
}
}
fn append_claude_user_message_to_openai_messages(
content: Option<&Value>,
messages: &mut Vec<Value>,
) -> Option<()> {
let Some(content) = content else {
return Some(());
};
let mut pending_parts = Vec::new();
for block in normalize_claude_content_blocks(content)? {
match block {
ClaudeNormalizedBlock::Text(text) | ClaudeNormalizedBlock::Thinking { text, .. } => {
push_openai_text_part(&mut pending_parts, text);
}
ClaudeNormalizedBlock::RedactedThinking { .. } => {}
ClaudeNormalizedBlock::ImageUrl(url) => {
pending_parts.push(build_openai_image_part(url));
}
ClaudeNormalizedBlock::FileData(file_data) => {
pending_parts.push(build_openai_file_part(file_data));
}
ClaudeNormalizedBlock::FileUrl(url) => {
push_openai_text_part(&mut pending_parts, format!("[File: {url}]"));
}
ClaudeNormalizedBlock::ToolResult {
tool_use_id,
content,
} => {
flush_openai_user_content_parts(&mut pending_parts, messages);
messages.push(json!({
"role": "tool",
"tool_call_id": tool_use_id,
"content": content,
}));
}
ClaudeNormalizedBlock::ToolUse { .. } => {}
}
}
flush_openai_user_content_parts(&mut pending_parts, messages);
Some(())
}
fn normalize_claude_assistant_message_to_openai_message(
content: Option<&Value>,
next_generated_tool_use_index: &mut usize,
) -> Option<Value> {
let mut reasoning_segments = Vec::new();
let mut reasoning_parts = Vec::new();
let mut content_parts = Vec::new();
let mut tool_calls = Vec::new();
if let Some(content) = content {
for block in normalize_claude_content_blocks(content)? {
match block {
ClaudeNormalizedBlock::Text(text) => {
push_openai_text_part(&mut content_parts, text);
}
ClaudeNormalizedBlock::Thinking { text, signature } => {
if !text.trim().is_empty() {
reasoning_segments.push(text.clone());
}
let mut reasoning_part = Map::new();
reasoning_part
.insert("type".to_string(), Value::String("thinking".to_string()));
reasoning_part.insert("thinking".to_string(), Value::String(text));
if let Some(signature) = signature {
reasoning_part.insert("signature".to_string(), Value::String(signature));
}
reasoning_parts.push(Value::Object(reasoning_part));
}
ClaudeNormalizedBlock::RedactedThinking { data } => {
if data.trim().is_empty() {
continue;
}
reasoning_parts.push(json!({
"type": "redacted_thinking",
"data": data,
}));
}
ClaudeNormalizedBlock::ImageUrl(url) => {
content_parts.push(build_openai_image_part(url));
}
ClaudeNormalizedBlock::FileData(file_data) => {
content_parts.push(build_openai_file_part(file_data));
}
ClaudeNormalizedBlock::FileUrl(url) => {
push_openai_text_part(&mut content_parts, format!("[File: {url}]"));
}
ClaudeNormalizedBlock::ToolUse { id, name, input } => {
let tool_use_id = id.unwrap_or_else(|| {
let generated = format!("toolu_auto_{next_generated_tool_use_index}");
*next_generated_tool_use_index += 1;
generated
});
tool_calls.push(json!({
"id": tool_use_id,
"type": "function",
"function": {
"name": name,
"arguments": canonical_json_string(input.unwrap_or(Value::Object(Map::new()))),
}
}));
}
ClaudeNormalizedBlock::ToolResult { .. } => {}
}
}
}
let mut assistant = Map::new();
assistant.insert("role".to_string(), Value::String("assistant".to_string()));
assistant.insert(
"content".to_string(),
match build_openai_content_value(content_parts) {
Some(content) => content,
None if !tool_calls.is_empty() => Value::Null,
None => Value::String(String::new()),
},
);
if !reasoning_segments.is_empty() {
assistant.insert(
"reasoning_content".to_string(),
Value::String(reasoning_segments.join("")),
);
}
if !reasoning_parts.is_empty() {
assistant.insert("reasoning_parts".to_string(), Value::Array(reasoning_parts));
}
if !tool_calls.is_empty() {
assistant.insert("tool_calls".to_string(), Value::Array(tool_calls));
}
Some(Value::Object(assistant))
}
fn flush_openai_user_content_parts(pending_parts: &mut Vec<Value>, messages: &mut Vec<Value>) {
let parts = std::mem::take(pending_parts);
let Some(content) = build_openai_content_value(parts) else {
return;
};
messages.push(json!({
"role": "user",
"content": content,
}));
}
fn build_openai_content_value(parts: Vec<Value>) -> Option<Value> {
if parts.is_empty() {
return None;
}
if parts
.iter()
.all(|part| part.get("type").and_then(Value::as_str) == Some("text"))
{
let text = parts
.iter()
.filter_map(|part| part.get("text").and_then(Value::as_str))
.collect::<Vec<_>>()
.join("\n\n");
return Some(Value::String(text));
}
Some(Value::Array(parts))
}
fn push_openai_text_part(parts: &mut Vec<Value>, text: String) {
if text.trim().is_empty() {
return;
}
parts.push(json!({
"type": "text",
"text": text,
}));
}
fn build_openai_image_part(url: String) -> Value {
json!({
"type": "image_url",
"image_url": {
"url": url,
}
})
}
fn build_openai_file_part(file_data: String) -> Value {
json!({
"type": "file",
"file": {
"file_data": file_data,
}
})
}
fn build_data_url(media_type: &str, data: &str) -> String {
format!("data:{media_type};base64,{data}")
}
fn extract_claude_output_reasoning_effort(request: &Map<String, Value>) -> Option<&'static str> {
match request
.get("output_config")
.and_then(Value::as_object)
.and_then(|config| config.get("effort"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?
.to_ascii_lowercase()
.as_str()
{
"low" => Some("low"),
"medium" => Some("medium"),
"high" => Some("high"),
"max" | "xhigh" => Some("xhigh"),
_ => None,
}
}
fn extract_claude_system_text(system: Option<&Value>) -> Option<String> {
let system = system?;
let text = match system {
@@ -326,6 +578,7 @@ fn extract_claude_web_search_options(tools: Option<&Value>) -> Option<Value> {
if !tool_type.starts_with("web_search") {
continue;
}
let found = true;
let mut options = Map::new();
if let Some(max_uses) = tool.get("max_uses").and_then(Value::as_u64) {
let search_context_size = if max_uses <= 1 {
@@ -357,7 +610,7 @@ fn extract_claude_web_search_options(tools: Option<&Value>) -> Option<Value> {
);
}
}
if !options.is_empty() {
if found {
return Some(Value::Object(options));
}
}
@@ -540,4 +793,114 @@ mod tests {
assert_eq!(normalized["parallel_tool_calls"], false);
assert!(normalized.get("tools").is_none());
}
#[test]
fn normalizes_claude_media_thinking_and_stop_sequences() {
let request = json!({
"model": "claude-sonnet-4-5",
"messages": [
{
"role": "user",
"content": [
{ "type": "text", "text": "See attachment" },
{
"type": "image",
"source": {
"type": "url",
"url": "https://example.com/cat.png"
}
},
{
"type": "document",
"source": {
"type": "base64",
"media_type": "application/pdf",
"data": "JVBERi0x"
}
}
]
},
{
"role": "assistant",
"content": [
{
"type": "thinking",
"thinking": "need context first",
"signature": "sig_123"
},
{ "type": "redacted_thinking", "data": "redacted_blob" },
{ "type": "text", "text": "Working on it" }
]
}
],
"stop_sequences": ["END"],
"output_config": { "effort": "max" }
});
let normalized = normalize_claude_request_to_openai_chat_request(&request)
.expect("request should convert");
assert_eq!(normalized["stop"], json!(["END"]));
assert_eq!(normalized["reasoning_effort"], "xhigh");
assert_eq!(
normalized["messages"][0]["content"],
json!([
{ "type": "text", "text": "See attachment" },
{
"type": "image_url",
"image_url": { "url": "https://example.com/cat.png" }
},
{
"type": "file",
"file": {
"file_data": "data:application/pdf;base64,JVBERi0x"
}
}
])
);
assert_eq!(
normalized["messages"][1]["reasoning_content"],
"need context first"
);
assert_eq!(
normalized["messages"][1]["reasoning_parts"],
json!([
{
"type": "thinking",
"thinking": "need context first",
"signature": "sig_123"
},
{
"type": "redacted_thinking",
"data": "redacted_blob"
}
])
);
assert_eq!(normalized["messages"][1]["content"], "Working on it");
}
#[test]
fn preserves_default_claude_web_search_tool_as_empty_openai_options() {
let request = json!({
"model": "claude-sonnet-4-5",
"tools": [
{
"type": "web_search_20250305",
"name": "web_search"
}
],
"messages": [
{
"role": "user",
"content": "find something"
}
]
});
let normalized = normalize_claude_request_to_openai_chat_request(&request)
.expect("request should convert");
assert_eq!(normalized["web_search_options"], json!({}));
assert!(normalized.get("tools").is_none());
}
}

705
crates/aether-ai-pipeline/src/conversion/request/to_openai_chat/gemini.rs
View File

@@ -3,6 +3,29 @@ use serde_json::{json, Map, Value};
use super::shared::canonical_json_string;
use crate::planner::openai::map_thinking_budget_to_openai_reasoning_effort;
const GEMINI_MAPPED_GENERATION_CONFIG_KEYS: &[&str] = &[
"maxOutputTokens",
"max_output_tokens",
"temperature",
"topP",
"top_p",
"topK",
"top_k",
"candidateCount",
"candidate_count",
"seed",
"stopSequences",
"stop_sequences",
"thinkingConfig",
"thinking_config",
"responseMimeType",
"response_mime_type",
"responseSchema",
"response_schema",
"responseModalities",
"response_modalities",
];
pub fn normalize_gemini_request_to_openai_chat_request(
body_json: &Value,
request_path: &str,
@@ -45,94 +68,8 @@ pub fn normalize_gemini_request_to_openai_chat_request(
.to_ascii_lowercase();
let parts = content_object.get("parts").and_then(Value::as_array)?;
match role.as_str() {
"model" => {
let mut text_segments = Vec::new();
let mut tool_calls = Vec::new();
for (index, part) in parts.iter().enumerate() {
let part = part.as_object()?;
if let Some(text) = part.get("text").and_then(Value::as_str) {
if !text.trim().is_empty() {
text_segments.push(text.to_string());
}
} else if let Some(function_call) =
part.get("functionCall").and_then(Value::as_object)
{
let name = function_call
.get("name")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
let id = function_call
.get("id")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.unwrap_or_else(|| format!("toolu_{}_{}", name, index));
tool_calls.push(json!({
"id": id,
"type": "function",
"function": {
"name": name,
"arguments": canonical_json_string(function_call.get("args").cloned().unwrap_or(Value::Object(Map::new()))),
}
}));
}
}
let mut assistant = Map::new();
assistant.insert("role".to_string(), Value::String("assistant".to_string()));
assistant.insert(
"content".to_string(),
if text_segments.is_empty() && !tool_calls.is_empty() {
Value::Null
} else {
Value::String(text_segments.join("\n\n"))
},
);
if !tool_calls.is_empty() {
assistant.insert("tool_calls".to_string(), Value::Array(tool_calls));
}
messages.push(Value::Object(assistant));
}
_ => {
let mut text_segments = Vec::new();
for part in parts {
let part = part.as_object()?;
if let Some(text) = part.get("text").and_then(Value::as_str) {
if !text.trim().is_empty() {
text_segments.push(text.to_string());
}
} else if let Some(function_response) =
part.get("functionResponse").and_then(Value::as_object)
{
let name = function_response
.get("name")
.and_then(Value::as_str)
.unwrap_or("tool");
let response_value = function_response
.get("response")
.cloned()
.unwrap_or(Value::Object(Map::new()));
let tool_call_id = function_response
.get("id")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.unwrap_or_else(|| format!("toolu_{}", name));
messages.push(json!({
"role": "tool",
"tool_call_id": tool_call_id,
"content": response_value,
}));
}
}
let text = text_segments.join("\n\n");
if !text.trim().is_empty() {
messages.push(json!({
"role": "user",
"content": text,
}));
}
}
"model" => messages.push(normalize_gemini_model_parts_to_openai_message(parts)?),
_ => append_gemini_user_parts_to_openai_messages(parts, &mut messages)?,
}
}
}
@@ -142,33 +79,64 @@ pub fn normalize_gemini_request_to_openai_chat_request(
.get("generationConfig")
.or_else(|| request.get("generation_config"))
.and_then(Value::as_object);
let mut google_extra = Map::new();
let mut gemini_extra = Map::new();
if let Some(generation_config) = generation_config {
if let Some(value) = generation_config.get("maxOutputTokens").cloned() {
if let Some(value) =
generation_config_value(generation_config, "maxOutputTokens", "max_output_tokens")
.cloned()
{
output.insert("max_completion_tokens".to_string(), value);
}
if let Some(value) = generation_config.get("temperature").cloned() {
output.insert("temperature".to_string(), value);
}
if let Some(value) = generation_config.get("topP").cloned() {
if let Some(value) = generation_config_value(generation_config, "topP", "top_p").cloned() {
output.insert("top_p".to_string(), value);
}
if let Some(value) = generation_config.get("topK").cloned() {
if let Some(value) = generation_config_value(generation_config, "topK", "top_k").cloned() {
output.insert("top_k".to_string(), value);
}
if let Some(value) = generation_config.get("candidateCount").cloned() {
if let Some(value) =
generation_config_value(generation_config, "candidateCount", "candidate_count").cloned()
{
output.insert("n".to_string(), value);
}
if let Some(value) = generation_config.get("seed").cloned() {
output.insert("seed".to_string(), value);
}
if let Some(value) = generation_config.get("stopSequences").cloned() {
if let Some(value) =
generation_config_value(generation_config, "stopSequences", "stop_sequences").cloned()
{
output.insert("stop".to_string(), value);
}
if let Some(thinking_budget) = generation_config
.get("thinkingConfig")
.and_then(Value::as_object)
.and_then(|thinking| thinking.get("thinkingBudget"))
.and_then(Value::as_u64)
if let Some(thinking_config) =
generation_config_value(generation_config, "thinkingConfig", "thinking_config")
.and_then(Value::as_object)
{
google_extra.insert(
"thinking_config".to_string(),
Value::Object(thinking_config.clone()),
);
}
if let Some(response_modalities) = generation_config_value(
generation_config,
"responseModalities",
"response_modalities",
)
.cloned()
{
google_extra.insert("response_modalities".to_string(), response_modalities);
}
if let Some(thinking_budget) =
generation_config_value(generation_config, "thinkingConfig", "thinking_config")
.and_then(Value::as_object)
.and_then(|thinking| {
thinking
.get("thinkingBudget")
.or_else(|| thinking.get("thinking_budget"))
})
.and_then(Value::as_u64)
{
output.insert(
"reasoning_effort".to_string(),
@@ -177,12 +145,13 @@ pub fn normalize_gemini_request_to_openai_chat_request(
),
);
}
if generation_config
.get("responseMimeType")
if generation_config_value(generation_config, "responseMimeType", "response_mime_type")
.and_then(Value::as_str)
.is_some_and(|value| value == "application/json")
{
let response_format = if let Some(schema) = generation_config.get("responseSchema") {
let response_format = if let Some(schema) =
generation_config_value(generation_config, "responseSchema", "response_schema")
{
json!({
"type": "json_schema",
"json_schema": {
@@ -195,23 +164,431 @@ pub fn normalize_gemini_request_to_openai_chat_request(
};
output.insert("response_format".to_string(), response_format);
}
let mut generation_config_extra = Map::new();
for (key, value) in generation_config {
if GEMINI_MAPPED_GENERATION_CONFIG_KEYS
.iter()
.any(|candidate| candidate == &key.as_str())
{
continue;
}
generation_config_extra.insert(key.clone(), value.clone());
}
if !generation_config_extra.is_empty() {
gemini_extra.insert(
"generation_config_extra".to_string(),
Value::Object(generation_config_extra),
);
}
}
if let Some(value) = request.get("stream").cloned() {
output.insert("stream".to_string(), value);
}
if let Some(value) = request
.get("safetySettings")
.or_else(|| request.get("safety_settings"))
.cloned()
{
gemini_extra.insert("safety_settings".to_string(), value);
}
if let Some(value) = request
.get("cachedContent")
.or_else(|| request.get("cached_content"))
.cloned()
{
gemini_extra.insert("cached_content".to_string(), value);
}
if let Some(tools) = normalize_gemini_tools_to_openai(request.get("tools"))? {
output.insert("tools".to_string(), Value::Array(tools));
}
if let Some(web_search_options) = extract_gemini_web_search_options(request.get("tools")) {
output.insert("web_search_options".to_string(), web_search_options);
}
if let Some(tool_choice) = normalize_gemini_tool_choice_to_openai(request.get("toolConfig"))? {
if let Some(tool_choice) = normalize_gemini_tool_choice_to_openai(
request
.get("toolConfig")
.or_else(|| request.get("tool_config")),
)? {
output.insert("tool_choice".to_string(), tool_choice);
}
if !google_extra.is_empty() || !gemini_extra.is_empty() {
let mut extra_body = Map::new();
if !google_extra.is_empty() {
extra_body.insert("google".to_string(), Value::Object(google_extra));
}
if !gemini_extra.is_empty() {
extra_body.insert("gemini".to_string(), Value::Object(gemini_extra));
}
output.insert("extra_body".to_string(), Value::Object(extra_body));
}
Some(Value::Object(output))
}
#[derive(Debug)]
enum GeminiNormalizedPart {
Text(String),
Thinking {
text: String,
signature: Option<String>,
},
ImageUrl(String),
FileData(String),
FileUrl(String),
AudioData {
data: String,
format: String,
},
ToolUse {
id: Option<String>,
name: String,
input: Value,
},
ToolResult {
tool_use_id: String,
content: Value,
},
}
fn append_gemini_user_parts_to_openai_messages(
parts: &[Value],
messages: &mut Vec<Value>,
) -> Option<()> {
let mut pending_parts = Vec::new();
for part in normalize_gemini_parts(parts)? {
match part {
GeminiNormalizedPart::Text(text) | GeminiNormalizedPart::Thinking { text, .. } => {
push_openai_text_part(&mut pending_parts, text);
}
GeminiNormalizedPart::ImageUrl(url) => {
pending_parts.push(build_openai_image_part(url));
}
GeminiNormalizedPart::FileData(file_data) => {
pending_parts.push(build_openai_file_part(file_data));
}
GeminiNormalizedPart::FileUrl(url) => {
push_openai_text_part(&mut pending_parts, format!("[File: {url}]"));
}
GeminiNormalizedPart::AudioData { data, format } => {
pending_parts.push(build_openai_audio_part(data, format));
}
GeminiNormalizedPart::ToolResult {
tool_use_id,
content,
} => {
flush_openai_user_content_parts(&mut pending_parts, messages);
messages.push(json!({
"role": "tool",
"tool_call_id": tool_use_id,
"content": content,
}));
}
GeminiNormalizedPart::ToolUse { .. } => {}
}
}
flush_openai_user_content_parts(&mut pending_parts, messages);
Some(())
}
fn normalize_gemini_model_parts_to_openai_message(parts: &[Value]) -> Option<Value> {
let mut reasoning_segments = Vec::new();
let mut reasoning_parts = Vec::new();
let mut content_parts = Vec::new();
let mut tool_calls = Vec::new();
for (index, part) in normalize_gemini_parts(parts)?.into_iter().enumerate() {
match part {
GeminiNormalizedPart::Text(text) => {
push_openai_text_part(&mut content_parts, text);
}
GeminiNormalizedPart::Thinking { text, signature } => {
if !text.trim().is_empty() {
reasoning_segments.push(text.clone());
}
let mut reasoning_part = Map::new();
reasoning_part.insert("type".to_string(), Value::String("thinking".to_string()));
reasoning_part.insert("thinking".to_string(), Value::String(text));
if let Some(signature) = signature {
reasoning_part.insert("signature".to_string(), Value::String(signature));
}
reasoning_parts.push(Value::Object(reasoning_part));
}
GeminiNormalizedPart::ImageUrl(url) => {
content_parts.push(build_openai_image_part(url));
}
GeminiNormalizedPart::FileData(file_data) => {
content_parts.push(build_openai_file_part(file_data));
}
GeminiNormalizedPart::FileUrl(url) => {
push_openai_text_part(&mut content_parts, format!("[File: {url}]"));
}
GeminiNormalizedPart::AudioData { data, format } => {
content_parts.push(build_openai_audio_part(data, format));
}
GeminiNormalizedPart::ToolUse { id, name, input } => {
let tool_id = id.unwrap_or_else(|| format!("toolu_{}_{}", name, index));
tool_calls.push(json!({
"id": tool_id,
"type": "function",
"function": {
"name": name,
"arguments": canonical_json_string(input),
}
}));
}
GeminiNormalizedPart::ToolResult { .. } => {}
}
}
let mut assistant = Map::new();
assistant.insert("role".to_string(), Value::String("assistant".to_string()));
assistant.insert(
"content".to_string(),
match build_openai_content_value(content_parts) {
Some(content) => content,
None if !tool_calls.is_empty() => Value::Null,
None => Value::String(String::new()),
},
);
if !reasoning_segments.is_empty() {
assistant.insert(
"reasoning_content".to_string(),
Value::String(reasoning_segments.join("")),
);
}
if !reasoning_parts.is_empty() {
assistant.insert("reasoning_parts".to_string(), Value::Array(reasoning_parts));
}
if !tool_calls.is_empty() {
assistant.insert("tool_calls".to_string(), Value::Array(tool_calls));
}
Some(Value::Object(assistant))
}
fn normalize_gemini_parts(parts: &[Value]) -> Option<Vec<GeminiNormalizedPart>> {
let mut normalized = Vec::new();
for part in parts {
let part = part.as_object()?;
if let Some(text) = part.get("text").and_then(Value::as_str) {
if part
.get("thought")
.and_then(Value::as_bool)
.unwrap_or(false)
{
normalized.push(GeminiNormalizedPart::Thinking {
text: text.to_string(),
signature: part
.get("thoughtSignature")
.or_else(|| part.get("thought_signature"))
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned),
});
} else {
normalized.push(GeminiNormalizedPart::Text(text.to_string()));
}
continue;
}
if let Some(inline_data) = part
.get("inlineData")
.or_else(|| part.get("inline_data"))
.and_then(Value::as_object)
{
let mime_type = inline_data
.get("mimeType")
.or_else(|| inline_data.get("mime_type"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
let data = inline_data
.get("data")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
if mime_type.starts_with("image/") {
normalized.push(GeminiNormalizedPart::ImageUrl(build_data_url(
mime_type, data,
)));
} else if let Some(format) = mime_type.strip_prefix("audio/") {
normalized.push(GeminiNormalizedPart::AudioData {
data: data.to_string(),
format: format.to_string(),
});
} else {
normalized.push(GeminiNormalizedPart::FileData(build_data_url(
mime_type, data,
)));
}
continue;
}
if let Some(file_data) = part
.get("fileData")
.or_else(|| part.get("file_data"))
.and_then(Value::as_object)
{
let file_uri = file_data
.get("fileUri")
.or_else(|| file_data.get("file_uri"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
let mime_type = file_data
.get("mimeType")
.or_else(|| file_data.get("mime_type"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty());
if mime_type.is_some_and(|value| value.starts_with("image/")) {
normalized.push(GeminiNormalizedPart::ImageUrl(file_uri.to_string()));
} else {
normalized.push(GeminiNormalizedPart::FileUrl(file_uri.to_string()));
}
continue;
}
if let Some(function_call) = part
.get("functionCall")
.or_else(|| part.get("function_call"))
.and_then(Value::as_object)
{
let name = function_call
.get("name")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?
.to_string();
normalized.push(GeminiNormalizedPart::ToolUse {
id: function_call
.get("id")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned),
name,
input: function_call
.get("args")
.cloned()
.unwrap_or_else(|| Value::Object(Map::new())),
});
continue;
}
if let Some(function_response) = part
.get("functionResponse")
.or_else(|| part.get("function_response"))
.and_then(Value::as_object)
{
let tool_use_id = function_response
.get("id")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.or_else(|| {
function_response
.get("name")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
})?;
let response_value = function_response
.get("response")
.cloned()
.unwrap_or_else(|| Value::Object(Map::new()));
let content = match response_value {
Value::Object(mut object) => object
.remove("result")
.unwrap_or_else(|| Value::Object(object)),
other => other,
};
normalized.push(GeminiNormalizedPart::ToolResult {
tool_use_id,
content,
});
}
}
Some(normalized)
}
fn flush_openai_user_content_parts(pending_parts: &mut Vec<Value>, messages: &mut Vec<Value>) {
let parts = std::mem::take(pending_parts);
let Some(content) = build_openai_content_value(parts) else {
return;
};
messages.push(json!({
"role": "user",
"content": content,
}));
}
fn build_openai_content_value(parts: Vec<Value>) -> Option<Value> {
if parts.is_empty() {
return None;
}
if parts
.iter()
.all(|part| part.get("type").and_then(Value::as_str) == Some("text"))
{
let text = parts
.iter()
.filter_map(|part| part.get("text").and_then(Value::as_str))
.collect::<Vec<_>>()
.join("\n\n");
return Some(Value::String(text));
}
Some(Value::Array(parts))
}
fn push_openai_text_part(parts: &mut Vec<Value>, text: String) {
if text.trim().is_empty() {
return;
}
parts.push(json!({
"type": "text",
"text": text,
}));
}
fn build_openai_image_part(url: String) -> Value {
json!({
"type": "image_url",
"image_url": {
"url": url,
}
})
}
fn build_openai_file_part(file_data: String) -> Value {
json!({
"type": "file",
"file": {
"file_data": file_data,
}
})
}
fn build_openai_audio_part(data: String, format: String) -> Value {
json!({
"type": "input_audio",
"input_audio": {
"data": data,
"format": format,
}
})
}
fn build_data_url(mime_type: &str, data: &str) -> String {
format!("data:{mime_type};base64,{data}")
}
fn generation_config_value<'a>(
generation_config: &'a Map<String, Value>,
camel: &str,
snake: &str,
) -> Option<&'a Value> {
generation_config
.get(camel)
.or_else(|| generation_config.get(snake))
}
fn extract_gemini_system_text(system_instruction: Option<&Value>) -> Option<String> {
let system_instruction = system_instruction?;
match system_instruction {
@@ -458,4 +835,128 @@ mod tests {
])
);
}
#[test]
fn normalizes_gemini_multimodal_thought_and_passthrough_config() {
let request = json!({
"contents": [
{
"role": "user",
"parts": [
{ "text": "Look at these" },
{ "inlineData": { "mimeType": "image/png", "data": "iVBORw0KGgo=" } },
{ "inline_data": { "mime_type": "application/pdf", "data": "JVBERi0x" } },
{ "inlineData": { "mimeType": "audio/mp3", "data": "SUQz" } },
{
"functionResponse": {
"name": "lookup",
"id": "call_1",
"response": { "result": { "city": "Shanghai" } }
}
}
]
},
{
"role": "model",
"parts": [
{ "text": "reasoning", "thought": true, "thoughtSignature": "sig_123" },
{ "text": "done" },
{ "fileData": { "fileUri": "https://example.com/cat.png", "mimeType": "image/png" } },
{ "fileData": { "fileUri": "https://example.com/report.pdf", "mimeType": "application/pdf" } },
{
"functionCall": {
"name": "lookup",
"id": "call_1",
"args": { "city": "Shanghai" }
}
}
]
}
],
"generation_config": {
"max_output_tokens": 128,
"stop_sequences": ["END"],
"thinking_config": {
"includeThoughts": true,
"thinkingBudget": 4096
},
"responseModalities": ["TEXT", "IMAGE"],
"candidate_count": 2,
"presencePenalty": 0.5
},
"safetySettings": [{ "category": "HARM_CATEGORY_HATE_SPEECH" }],
"cachedContent": "cached/123"
});
let normalized = normalize_gemini_request_to_openai_chat_request(
&request,
"/v1beta/models/gemini-2.5-pro:generateContent",
)
.expect("request should convert");
assert_eq!(normalized["model"], "gemini-2.5-pro");
assert_eq!(
normalized["messages"][2]["reasoning_parts"],
json!([
{
"type": "thinking",
"thinking": "reasoning",
"signature": "sig_123"
}
])
);
assert_eq!(
normalized["messages"][0]["content"],
json!([
{ "type": "text", "text": "Look at these" },
{
"type": "image_url",
"image_url": { "url": "data:image/png;base64,iVBORw0KGgo=" }
},
{
"type": "file",
"file": { "file_data": "data:application/pdf;base64,JVBERi0x" }
},
{
"type": "input_audio",
"input_audio": { "data": "SUQz", "format": "mp3" }
}
])
);
assert_eq!(normalized["messages"][1]["role"], "tool");
assert_eq!(normalized["messages"][1]["tool_call_id"], "call_1");
assert_eq!(
normalized["messages"][1]["content"],
json!({ "city": "Shanghai" })
);
assert_eq!(normalized["messages"][2]["reasoning_content"], "reasoning");
assert_eq!(
normalized["messages"][2]["content"],
json!([
{ "type": "text", "text": "done" },
{
"type": "image_url",
"image_url": { "url": "https://example.com/cat.png" }
},
{ "type": "text", "text": "[File: https://example.com/report.pdf]" }
])
);
assert_eq!(normalized["messages"][2]["tool_calls"][0]["id"], "call_1");
assert_eq!(normalized["max_completion_tokens"], 128);
assert_eq!(normalized["n"], 2);
assert_eq!(normalized["stop"], json!(["END"]));
assert_eq!(normalized["reasoning_effort"], "high");
assert_eq!(
normalized["extra_body"]["google"]["response_modalities"],
json!(["TEXT", "IMAGE"])
);
assert_eq!(
normalized["extra_body"]["gemini"]["generation_config_extra"]["presencePenalty"],
0.5
);
assert_eq!(
normalized["extra_body"]["gemini"]["cached_content"],
"cached/123"
);
}
}

302
crates/aether-ai-pipeline/src/conversion/response/from_openai_chat/claude_chat.rs
View File

@@ -1,8 +1,6 @@
use serde_json::{json, Value};
use super::shared::{
build_generated_tool_call_id, extract_openai_assistant_text, parse_openai_function_arguments,
};
use super::shared::{build_generated_tool_call_id, parse_openai_function_arguments};
pub fn convert_openai_chat_response_to_claude_chat(
body_json: &Value,
@@ -12,24 +10,10 @@ pub fn convert_openai_chat_response_to_claude_chat(
let choices = body.get("choices")?.as_array()?;
let first_choice = choices.first()?.as_object()?;
let message = first_choice.get("message")?.as_object()?;
let mut content = Vec::new();
if let Some(reasoning_content) = message.get("reasoning_content").and_then(Value::as_str) {
if !reasoning_content.trim().is_empty() {
content.push(json!({
"type": "thinking",
"thinking": reasoning_content,
}));
}
}
if let Some(text) = extract_openai_assistant_text(message.get("content")) {
if !text.trim().is_empty() {
content.push(json!({
"type": "text",
"text": text,
}));
}
}
let mut content = extract_openai_reasoning_to_claude_blocks(message);
content.extend(convert_openai_assistant_content_to_claude_blocks(
message.get("content"),
)?);
if let Some(tool_call_values) = message.get("tool_calls").and_then(Value::as_array) {
for (index, tool_call) in tool_call_values.iter().enumerate() {
let tool_call = tool_call.as_object()?;
@@ -121,6 +105,225 @@ pub fn convert_openai_chat_response_to_claude_chat(
})
}
fn extract_openai_reasoning_to_claude_blocks(
message: &serde_json::Map<String, Value>,
) -> Vec<Value> {
let mut blocks = Vec::new();
if let Some(reasoning_parts) = message.get("reasoning_parts").and_then(Value::as_array) {
for reasoning_part in reasoning_parts {
let Some(reasoning_object) = reasoning_part.as_object() else {
continue;
};
match reasoning_object
.get("type")
.and_then(Value::as_str)
.unwrap_or("thinking")
{
"thinking" => {
let thinking = reasoning_object
.get("thinking")
.or_else(|| reasoning_object.get("text"))
.and_then(Value::as_str)
.map(str::trim)
.unwrap_or_default();
if thinking.is_empty() {
continue;
}
let mut block = serde_json::Map::new();
block.insert("type".to_string(), Value::String("thinking".to_string()));
block.insert("thinking".to_string(), Value::String(thinking.to_string()));
if let Some(signature) = reasoning_object
.get("signature")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
{
block.insert(
"signature".to_string(),
Value::String(signature.to_string()),
);
}
blocks.push(Value::Object(block));
}
"redacted_thinking" => {
let data = reasoning_object
.get("data")
.and_then(Value::as_str)
.map(str::trim)
.unwrap_or_default();
if !data.is_empty() {
blocks.push(json!({
"type": "redacted_thinking",
"data": data,
}));
}
}
_ => {}
}
}
}
if !blocks.is_empty() {
return blocks;
}
if let Some(reasoning_content) = message
.get("reasoning_content")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
{
blocks.push(json!({
"type": "thinking",
"thinking": reasoning_content,
}));
}
blocks
}
fn convert_openai_assistant_content_to_claude_blocks(
content: Option<&Value>,
) -> Option<Vec<Value>> {
match content {
None | Some(Value::Null) => Some(Vec::new()),
Some(Value::String(text)) => {
if text.trim().is_empty() {
Some(Vec::new())
} else {
Some(vec![json!({
"type": "text",
"text": text,
})])
}
}
Some(Value::Array(parts)) => {
let mut blocks = Vec::new();
for part in parts {
let part = part.as_object()?;
let part_type = part
.get("type")
.and_then(Value::as_str)
.unwrap_or_default()
.trim()
.to_ascii_lowercase();
match part_type.as_str() {
"text" | "output_text" => {
if let Some(text) = part.get("text").and_then(Value::as_str) {
if !text.trim().is_empty() {
blocks.push(json!({
"type": "text",
"text": text,
}));
}
}
}
"image_url" | "output_image" => {
if let Some(url) = extract_openai_image_url(part) {
if let Some((media_type, data)) = parse_data_url(url.as_str()) {
blocks.push(json!({
"type": "image",
"source": {
"type": "base64",
"media_type": media_type,
"data": data,
}
}));
} else {
blocks.push(json!({
"type": "image",
"source": {
"type": "url",
"url": url,
}
}));
}
}
}
"file" => {
let file = part.get("file").and_then(Value::as_object).unwrap_or(part);
if let Some(file_data) = file.get("file_data").and_then(Value::as_str) {
if let Some((media_type, data)) = parse_data_url(file_data) {
blocks.push(json!({
"type": "document",
"source": {
"type": "base64",
"media_type": media_type,
"data": data,
}
}));
}
} else if let Some(file_id) = file
.get("file_id")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
{
blocks.push(json!({
"type": "text",
"text": format!("[File: {file_id}]"),
}));
}
}
"input_audio" => {
let audio = part
.get("input_audio")
.and_then(Value::as_object)
.unwrap_or(part);
let data = audio
.get("data")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty());
let format = audio
.get("format")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty());
if let (Some(data), Some(format)) = (data, format) {
blocks.push(json!({
"type": "document",
"source": {
"type": "base64",
"media_type": format!("audio/{format}"),
"data": data,
}
}));
}
}
_ => {}
}
}
Some(blocks)
}
_ => None,
}
}
fn extract_openai_image_url(part: &serde_json::Map<String, Value>) -> Option<String> {
part.get("image_url")
.and_then(|value| {
value.as_str().map(ToOwned::to_owned).or_else(|| {
value
.as_object()
.and_then(|object| object.get("url"))
.and_then(Value::as_str)
.map(ToOwned::to_owned)
})
})
.or_else(|| {
part.get("url")
.and_then(Value::as_str)
.map(ToOwned::to_owned)
})
}
fn parse_data_url(value: &str) -> Option<(String, String)> {
let rest = value.strip_prefix("data:")?;
let (meta, data) = rest.split_once(",")?;
let media_type = meta.strip_suffix(";base64")?;
if media_type.trim().is_empty() || data.trim().is_empty() {
return None;
}
Some((media_type.to_string(), data.to_string()))
}
#[cfg(test)]
mod tests {
use super::convert_openai_chat_response_to_claude_chat;
@@ -136,7 +339,18 @@ mod tests {
"message": {
"role": "assistant",
"content": "hello",
"reasoning_content": "step by step"
"reasoning_content": "step by step",
"reasoning_parts": [
{
"type": "thinking",
"thinking": "step by step",
"signature": "sig_123"
},
{
"type": "redacted_thinking",
"data": "redacted_blob"
}
]
},
"finish_reason": "stop"
}],
@@ -155,7 +369,51 @@ mod tests {
assert_eq!(converted["content"][0]["type"], "thinking");
assert_eq!(converted["content"][0]["thinking"], "step by step");
assert_eq!(converted["content"][0]["signature"], "sig_123");
assert_eq!(converted["content"][1]["type"], "redacted_thinking");
assert_eq!(converted["usage"]["cache_read_input_tokens"], 3);
assert_eq!(converted["usage"]["cache_creation_input_tokens"], 2);
}
#[test]
fn converts_openai_multipart_content_into_claude_blocks() {
let response = json!({
"id": "chatcmpl_img_123",
"model": "gpt-5.4",
"choices": [{
"index": 0,
"message": {
"role": "assistant",
"content": [
{ "type": "text", "text": "See attached." },
{
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,iVBORw0KGgo="
}
},
{
"type": "file",
"file": {
"file_data": "data:application/pdf;base64,JVBERi0x"
}
}
]
},
"finish_reason": "stop"
}]
});
let converted = convert_openai_chat_response_to_claude_chat(&response, &json!({}))
.expect("response should convert");
assert_eq!(converted["content"][0]["type"], "text");
assert_eq!(converted["content"][1]["type"], "image");
assert_eq!(converted["content"][1]["source"]["media_type"], "image/png");
assert_eq!(converted["content"][2]["type"], "document");
assert_eq!(
converted["content"][2]["source"]["media_type"],
"application/pdf"
);
}
}

337
crates/aether-ai-pipeline/src/conversion/response/from_openai_chat/gemini_chat.rs
View File

@@ -1,8 +1,6 @@
use serde_json::{json, Value};
use super::shared::{
build_generated_tool_call_id, extract_openai_assistant_text, parse_openai_function_arguments,
};
use super::shared::{build_generated_tool_call_id, parse_openai_function_arguments};
pub fn convert_openai_chat_response_to_gemini_chat(
body_json: &Value,
@@ -14,21 +12,10 @@ pub fn convert_openai_chat_response_to_gemini_chat(
for choice in choices {
let choice = choice.as_object()?;
let message = choice.get("message")?.as_object()?;
let mut parts = Vec::new();
if let Some(reasoning_content) = message.get("reasoning_content").and_then(Value::as_str) {
if !reasoning_content.trim().is_empty() {
parts.push(json!({
"text": reasoning_content,
"thought": true,
}));
}
}
if let Some(text) = extract_openai_assistant_text(message.get("content")) {
if !text.trim().is_empty() {
parts.push(json!({ "text": text }));
}
}
let mut parts = extract_openai_reasoning_to_gemini_parts(message);
parts.extend(convert_openai_assistant_content_to_gemini_parts(
message.get("content"),
)?);
if let Some(tool_call_values) = message.get("tool_calls").and_then(Value::as_array) {
for (index, tool_call) in tool_call_values.iter().enumerate() {
let tool_call = tool_call.as_object()?;
@@ -120,6 +107,214 @@ pub fn convert_openai_chat_response_to_gemini_chat(
}))
}
fn extract_openai_reasoning_to_gemini_parts(
message: &serde_json::Map<String, Value>,
) -> Vec<Value> {
let mut parts = Vec::new();
if let Some(reasoning_parts) = message.get("reasoning_parts").and_then(Value::as_array) {
for reasoning_part in reasoning_parts {
let Some(reasoning_object) = reasoning_part.as_object() else {
continue;
};
if reasoning_object
.get("type")
.and_then(Value::as_str)
.is_some_and(|value| value != "thinking")
{
continue;
}
let thinking = reasoning_object
.get("thinking")
.or_else(|| reasoning_object.get("text"))
.and_then(Value::as_str)
.map(str::trim)
.unwrap_or_default();
if thinking.is_empty() {
continue;
}
let mut part = serde_json::Map::new();
part.insert("text".to_string(), Value::String(thinking.to_string()));
part.insert("thought".to_string(), Value::Bool(true));
if let Some(signature) = reasoning_object
.get("signature")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
{
part.insert(
"thoughtSignature".to_string(),
Value::String(signature.to_string()),
);
}
parts.push(Value::Object(part));
}
}
if !parts.is_empty() {
return parts;
}
if let Some(reasoning_content) = message
.get("reasoning_content")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
{
parts.push(json!({
"text": reasoning_content,
"thought": true,
}));
}
parts
}
fn convert_openai_assistant_content_to_gemini_parts(content: Option<&Value>) -> Option<Vec<Value>> {
match content {
None | Some(Value::Null) => Some(Vec::new()),
Some(Value::String(text)) => {
if text.trim().is_empty() {
Some(Vec::new())
} else {
Some(vec![json!({ "text": text })])
}
}
Some(Value::Array(parts)) => {
let mut converted = Vec::new();
for part in parts {
let part = part.as_object()?;
let part_type = part
.get("type")
.and_then(Value::as_str)
.unwrap_or_default()
.trim()
.to_ascii_lowercase();
match part_type.as_str() {
"text" | "output_text" => {
if let Some(text) = part.get("text").and_then(Value::as_str) {
if !text.trim().is_empty() {
converted.push(json!({ "text": text }));
}
}
}
"image_url" | "output_image" => {
let image_url = part
.get("image_url")
.and_then(|value| {
value.as_str().map(ToOwned::to_owned).or_else(|| {
value
.as_object()
.and_then(|object| object.get("url"))
.and_then(Value::as_str)
.map(ToOwned::to_owned)
})
})
.or_else(|| {
part.get("url")
.and_then(Value::as_str)
.map(ToOwned::to_owned)
})?;
if let Some((mime_type, data)) = parse_data_url(image_url.as_str()) {
converted.push(json!({
"inlineData": {
"mimeType": mime_type,
"data": data,
}
}));
} else {
converted.push(json!({
"fileData": {
"fileUri": image_url,
"mimeType": guess_media_type_from_reference(image_url.as_str(), "image/jpeg"),
}
}));
}
}
"file" | "input_file" => {
let file_object =
part.get("file").and_then(Value::as_object).unwrap_or(part);
if let Some(file_data) =
file_object.get("file_data").and_then(Value::as_str)
{
if let Some((mime_type, data)) = parse_data_url(file_data) {
converted.push(json!({
"inlineData": {
"mimeType": mime_type,
"data": data,
}
}));
}
} else if let Some(file_id) = file_object
.get("file_id")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
{
converted.push(json!({
"text": format!("[File: {file_id}]"),
}));
}
}
"input_audio" => {
let audio_object = part
.get("input_audio")
.and_then(Value::as_object)
.unwrap_or(part);
let data = audio_object
.get("data")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty());
let format = audio_object
.get("format")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty());
if let (Some(data), Some(format)) = (data, format) {
converted.push(json!({
"inlineData": {
"mimeType": format!("audio/{format}"),
"data": data,
}
}));
}
}
_ => {}
}
}
Some(converted)
}
_ => None,
}
}
fn parse_data_url(value: &str) -> Option<(String, String)> {
let rest = value.strip_prefix("data:")?;
let (meta, data) = rest.split_once(",")?;
let mime_type = meta.strip_suffix(";base64")?;
if mime_type.trim().is_empty() || data.trim().is_empty() {
return None;
}
Some((mime_type.to_string(), data.to_string()))
}
fn guess_media_type_from_reference(reference: &str, default_mime: &str) -> String {
let normalized = reference
.split('?')
.next()
.unwrap_or(reference)
.to_ascii_lowercase();
if normalized.ends_with(".png") {
"image/png".to_string()
} else if normalized.ends_with(".gif") {
"image/gif".to_string()
} else if normalized.ends_with(".webp") {
"image/webp".to_string()
} else if normalized.ends_with(".jpg") || normalized.ends_with(".jpeg") {
"image/jpeg".to_string()
} else if normalized.ends_with(".pdf") {
"application/pdf".to_string()
} else {
default_mime.to_string()
}
}
#[cfg(test)]
mod tests {
use super::convert_openai_chat_response_to_gemini_chat;
@@ -136,7 +331,14 @@ mod tests {
"message": {
"role": "assistant",
"content": "hello",
"reasoning_content": "step by step"
"reasoning_content": "step by step",
"reasoning_parts": [
{
"type": "thinking",
"thinking": "step by step",
"signature": "sig_123"
}
]
},
"finish_reason": "stop"
},
@@ -181,6 +383,10 @@ mod tests {
converted["candidates"][0]["content"]["parts"][0]["thought"],
true
);
assert_eq!(
converted["candidates"][0]["content"]["parts"][0]["thoughtSignature"],
"sig_123"
);
assert_eq!(
converted["candidates"][1]["content"]["parts"][0]["functionCall"]["name"],
"lookup"
@@ -188,4 +394,97 @@ mod tests {
assert_eq!(converted["usageMetadata"]["candidatesTokenCount"], 5);
assert_eq!(converted["usageMetadata"]["thoughtsTokenCount"], 2);
}
#[test]
fn preserves_multimodal_openai_content_in_gemini_response() {
let response = json!({
"id": "chatcmpl_mm_123",
"model": "gpt-5.4",
"choices": [{
"index": 0,
"message": {
"role": "assistant",
"reasoning_content": "step by step",
"content": [
{ "type": "text", "text": "Attached." },
{
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,iVBORw0KGgo="
}
},
{
"type": "file",
"file": {
"file_data": "data:application/pdf;base64,JVBERi0x"
}
},
{
"type": "input_audio",
"input_audio": {
"data": "SUQz",
"format": "mp3"
}
},
{
"type": "file",
"file": { "file_id": "file_123" }
}
]
},
"finish_reason": "stop"
}],
"usage": {
"prompt_tokens": 4,
"completion_tokens": 3,
"completion_tokens_details": { "reasoning_tokens": 1 },
"total_tokens": 7
}
});
let converted = convert_openai_chat_response_to_gemini_chat(&response, &json!({}))
.expect("response should convert");
assert_eq!(
converted["candidates"][0]["content"]["parts"][0]["thought"],
true
);
assert_eq!(
converted["candidates"][0]["content"]["parts"][1],
json!({ "text": "Attached." })
);
assert_eq!(
converted["candidates"][0]["content"]["parts"][2],
json!({
"inlineData": {
"mimeType": "image/png",
"data": "iVBORw0KGgo="
}
})
);
assert_eq!(
converted["candidates"][0]["content"]["parts"][3],
json!({
"inlineData": {
"mimeType": "application/pdf",
"data": "JVBERi0x"
}
})
);
assert_eq!(
converted["candidates"][0]["content"]["parts"][4],
json!({
"inlineData": {
"mimeType": "audio/mp3",
"data": "SUQz"
}
})
);
assert_eq!(
converted["candidates"][0]["content"]["parts"][5],
json!({ "text": "[File: file_123]" })
);
assert_eq!(converted["usageMetadata"]["candidatesTokenCount"], 2);
assert_eq!(converted["usageMetadata"]["thoughtsTokenCount"], 1);
}
}

117
crates/aether-ai-pipeline/src/conversion/response/from_openai_chat/openai_cli.rs
View File

@@ -97,6 +97,14 @@ pub fn convert_openai_chat_response_to_openai_cli(
}
message_content.push(image_part);
}
} else if matches!(part_type.as_str(), "file" | "input_file") {
if let Some(file_part) = build_openai_cli_file_part(part) {
message_content.push(file_part);
}
} else if part_type == "input_audio" {
if let Some(audio_part) = build_openai_cli_input_audio_part(part) {
message_content.push(audio_part);
}
}
}
}
@@ -238,6 +246,49 @@ pub fn convert_openai_chat_response_to_openai_cli(
Some(response)
}
fn build_openai_cli_file_part(part: &serde_json::Map<String, Value>) -> Option<Value> {
let file_object = part.get("file").and_then(Value::as_object).unwrap_or(part);
let mut file = serde_json::Map::new();
for key in ["file_data", "file_id", "filename"] {
if let Some(value) = file_object
.get(key)
.cloned()
.filter(|value| !value.is_null())
{
file.insert(key.to_string(), value);
}
}
if file.is_empty() {
return None;
}
Some(json!({
"type": "file",
"file": Value::Object(file),
}))
}
fn build_openai_cli_input_audio_part(part: &serde_json::Map<String, Value>) -> Option<Value> {
let audio_object = part
.get("input_audio")
.and_then(Value::as_object)
.unwrap_or(part);
let data = audio_object
.get("data")
.cloned()
.filter(|value| value.as_str().is_some_and(|value| !value.trim().is_empty()))?;
let format = audio_object
.get("format")
.cloned()
.filter(|value| value.as_str().is_some_and(|value| !value.trim().is_empty()))?;
Some(json!({
"type": "input_audio",
"input_audio": {
"data": data,
"format": format,
}
}))
}
#[cfg(test)]
mod tests {
use super::convert_openai_chat_response_to_openai_cli;
@@ -321,4 +372,70 @@ mod tests {
json!({"reasoning_tokens": 0})
);
}
#[test]
fn preserves_file_and_audio_parts_when_converting_to_responses() {
let response = json!({
"id": "chatcmpl_mm_123",
"object": "chat.completion",
"model": "gpt-5",
"choices": [{
"index": 0,
"message": {
"role": "assistant",
"content": [
{ "type": "text", "text": "Attached." },
{
"type": "file",
"file": {
"file_data": "data:application/pdf;base64,JVBERi0x",
"filename": "report.pdf"
}
},
{
"type": "input_audio",
"input_audio": {
"data": "SUQz",
"format": "mp3"
}
}
]
},
"finish_reason": "stop"
}],
"usage": {
"prompt_tokens": 4,
"completion_tokens": 2,
"total_tokens": 6
}
});
let converted = convert_openai_chat_response_to_openai_cli(&response, &json!({}), false)
.expect("chat response should convert to responses");
assert_eq!(
converted["output"][0]["content"],
json!([
{
"type": "output_text",
"text": "Attached.",
"annotations": []
},
{
"type": "file",
"file": {
"file_data": "data:application/pdf;base64,JVBERi0x",
"filename": "report.pdf"
}
},
{
"type": "input_audio",
"input_audio": {
"data": "SUQz",
"format": "mp3"
}
}
])
);
}
}

44
crates/aether-ai-pipeline/src/conversion/response/from_openai_chat/shared.rs
View File

@@ -114,38 +114,22 @@ pub fn build_openai_cli_response_with_content(
})
}
pub(super) fn extract_openai_assistant_text(content: Option<&Value>) -> Option<String> {
match content? {
Value::Null => Some(String::new()),
Value::String(text) => Some(text.clone()),
Value::Array(parts) => {
let mut text = String::new();
for part in parts {
let part = part.as_object()?;
let part_type = part
.get("type")
.and_then(Value::as_str)
.unwrap_or_default()
.trim()
.to_ascii_lowercase();
if matches!(part_type.as_str(), "text" | "output_text") {
if let Some(piece) = part.get("text").and_then(Value::as_str) {
text.push_str(piece);
}
}
}
Some(text)
}
_ => None,
}
}
pub(super) fn parse_openai_function_arguments(arguments: Option<&Value>) -> Option<Value> {
match arguments.cloned().unwrap_or(Value::Object(Map::new())) {
Value::String(text) => serde_json::from_str(&text)
.ok()
.or(Some(Value::String(text))),
other => Some(other),
Value::Object(object) => Some(Value::Object(object)),
Value::String(text) => {
let trimmed = text.trim();
if trimmed.is_empty() {
Some(Value::Object(Map::new()))
} else {
match serde_json::from_str::<Value>(trimmed) {
Ok(Value::Object(object)) => Some(Value::Object(object)),
Ok(other) => Some(json!({ "raw": other })),
Err(_) => Some(json!({ "raw": text })),
}
}
}
other => Some(json!({ "raw": other })),
}
}

257
crates/aether-ai-pipeline/src/conversion/response/to_openai_chat/claude_chat.rs
View File

@@ -9,22 +9,53 @@ pub fn convert_claude_chat_response_to_openai_chat(
let body = body_json.as_object()?;
let content = body.get("content")?.as_array()?;
let mut text = String::new();
let mut content_parts = Vec::new();
let mut reasoning_content = String::new();
let mut reasoning_parts = Vec::new();
let mut tool_calls = Vec::new();
let mut has_non_text_content = false;
for (index, block) in content.iter().enumerate() {
let block = block.as_object()?;
match block.get("type")?.as_str()? {
"text" => {
text.push_str(block.get("text")?.as_str()?);
let piece = block.get("text")?.as_str()?;
push_openai_text_part(&mut text, &mut content_parts, piece);
}
"thinking" => {
if let Some(piece) = block
let piece = block
.get("thinking")
.and_then(Value::as_str)
.or_else(|| block.get("text").and_then(Value::as_str))
{
.unwrap_or_default();
if !piece.is_empty() {
reasoning_content.push_str(piece);
}
let mut reasoning_part = Map::new();
reasoning_part.insert("type".to_string(), Value::String("thinking".to_string()));
reasoning_part.insert("thinking".to_string(), Value::String(piece.to_string()));
if let Some(signature) = block
.get("signature")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
{
reasoning_part.insert(
"signature".to_string(),
Value::String(signature.to_string()),
);
}
reasoning_parts.push(Value::Object(reasoning_part));
}
"redacted_thinking" => {
let data = block
.get("data")
.and_then(Value::as_str)
.unwrap_or_default();
if !data.is_empty() {
reasoning_parts.push(json!({
"type": "redacted_thinking",
"data": data,
}));
}
}
"tool_use" => {
let tool_name = block.get("name")?.as_str()?;
@@ -44,6 +75,21 @@ pub fn convert_claude_chat_response_to_openai_chat(
}
}));
}
"image" => {
content_parts.push(convert_claude_image_block_to_openai_part(block)?);
has_non_text_content = true;
}
"document" => {
let part = convert_claude_document_block_to_openai_part(block)?;
if part.get("type").and_then(Value::as_str) == Some("text") {
if let Some(piece) = part.get("text").and_then(Value::as_str) {
push_openai_text_part(&mut text, &mut content_parts, piece);
}
} else {
content_parts.push(part);
has_non_text_content = true;
}
}
_ => continue,
}
}
@@ -77,8 +123,10 @@ pub fn convert_claude_chat_response_to_openai_chat(
.get("id")
.and_then(Value::as_str)
.unwrap_or("chatcmpl-local-finalize");
let message_content = if text.is_empty() && !tool_calls.is_empty() {
let message_content = if content_parts.is_empty() && !tool_calls.is_empty() {
Value::Null
} else if has_non_text_content {
Value::Array(content_parts)
} else {
Value::String(text)
};
@@ -91,6 +139,9 @@ pub fn convert_claude_chat_response_to_openai_chat(
Value::String(reasoning_content),
);
}
if !reasoning_parts.is_empty() {
message.insert("reasoning_parts".to_string(), Value::Array(reasoning_parts));
}
if !tool_calls.is_empty() {
message.insert("tool_calls".to_string(), Value::Array(tool_calls));
}
@@ -141,6 +192,108 @@ pub fn convert_claude_chat_response_to_openai_chat(
})
}
fn push_openai_text_part(text: &mut String, content_parts: &mut Vec<Value>, piece: &str) {
if piece.is_empty() {
return;
}
text.push_str(piece);
content_parts.push(json!({
"type": "text",
"text": piece,
}));
}
fn convert_claude_image_block_to_openai_part(
block: &serde_json::Map<String, Value>,
) -> Option<Value> {
let source = block.get("source")?.as_object()?;
match source.get("type")?.as_str()? {
"base64" => {
let media_type = source
.get("media_type")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
let data = source
.get("data")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
Some(json!({
"type": "image_url",
"image_url": {
"url": build_data_url(media_type, data),
}
}))
}
"url" => {
let url = source
.get("url")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
Some(json!({
"type": "image_url",
"image_url": {
"url": url,
}
}))
}
_ => None,
}
}
fn convert_claude_document_block_to_openai_part(
block: &serde_json::Map<String, Value>,
) -> Option<Value> {
let source = block.get("source")?.as_object()?;
match source.get("type")?.as_str()? {
"base64" => {
let media_type = source
.get("media_type")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
let data = source
.get("data")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
if let Some(format) = media_type.strip_prefix("audio/") {
return Some(json!({
"type": "input_audio",
"input_audio": {
"data": data,
"format": format,
}
}));
}
Some(json!({
"type": "file",
"file": {
"file_data": build_data_url(media_type, data),
}
}))
}
"url" => {
let url = source
.get("url")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
Some(json!({
"type": "text",
"text": format!("[File: {url}]"),
}))
}
_ => None,
}
}
fn build_data_url(media_type: &str, data: &str) -> String {
format!("data:{media_type};base64,{data}")
}
#[cfg(test)]
mod tests {
use super::convert_claude_chat_response_to_openai_chat;
@@ -152,7 +305,7 @@ mod tests {
"id": "msg_123",
"model": "claude-sonnet-4-5",
"content": [
{ "type": "thinking", "thinking": "step by step" },
{ "type": "thinking", "thinking": "step by step", "signature": "sig_123" },
{ "type": "text", "text": "hello" }
],
"stop_reason": "end_turn",
@@ -176,6 +329,16 @@ mod tests {
converted["choices"][0]["message"]["reasoning_content"],
"step by step"
);
assert_eq!(
converted["choices"][0]["message"]["reasoning_parts"],
json!([
{
"type": "thinking",
"thinking": "step by step",
"signature": "sig_123"
}
])
);
assert_eq!(
converted["usage"]["prompt_tokens_details"]["cached_tokens"],
3
@@ -186,4 +349,88 @@ mod tests {
);
assert_eq!(converted["service_tier"], "default");
}
#[test]
fn converts_claude_multimodal_content_into_openai_chat_parts() {
let response = json!({
"id": "msg_multimodal_123",
"model": "claude-sonnet-4-5",
"content": [
{ "type": "thinking", "thinking": "step by step" },
{ "type": "text", "text": "See attached." },
{
"type": "image",
"source": {
"type": "url",
"url": "https://example.com/cat.png"
}
},
{
"type": "document",
"source": {
"type": "base64",
"media_type": "application/pdf",
"data": "JVBERi0x"
}
},
{
"type": "document",
"source": {
"type": "base64",
"media_type": "audio/mp3",
"data": "SUQz"
}
},
{
"type": "tool_use",
"id": "call_1",
"name": "lookup",
"input": { "city": "Shanghai" }
}
],
"stop_reason": "tool_use",
"usage": {
"input_tokens": 9,
"output_tokens": 4
}
});
let converted = convert_claude_chat_response_to_openai_chat(&response, &json!({}))
.expect("response should convert");
assert_eq!(
converted["choices"][0]["message"]["reasoning_content"],
"step by step"
);
assert_eq!(
converted["choices"][0]["message"]["content"],
json!([
{ "type": "text", "text": "See attached." },
{
"type": "image_url",
"image_url": {
"url": "https://example.com/cat.png"
}
},
{
"type": "file",
"file": {
"file_data": "data:application/pdf;base64,JVBERi0x"
}
},
{
"type": "input_audio",
"input_audio": {
"data": "SUQz",
"format": "mp3"
}
}
])
);
assert_eq!(converted["choices"][0]["finish_reason"], "tool_calls");
assert_eq!(
converted["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"],
"{\"city\":\"Shanghai\"}"
);
}
}

85
crates/aether-ai-pipeline/src/conversion/response/to_openai_chat/claude_cli.rs
View File

@@ -1,87 +1,12 @@
use serde_json::{json, Value};
use serde_json::Value;
use super::super::from_openai_chat::{
build_openai_cli_response_with_reasoning, OpenAiCliResponseUsage,
};
use super::shared::{build_generated_tool_call_id, canonicalize_tool_arguments};
use super::super::from_openai_chat::convert_openai_chat_response_to_openai_cli;
use super::claude_chat::convert_claude_chat_response_to_openai_chat;
pub fn convert_claude_cli_response_to_openai_cli(
body_json: &Value,
report_context: &Value,
) -> Option<Value> {
let body = body_json.as_object()?;
let content = body.get("content")?.as_array()?;
let mut text = String::new();
let mut reasoning_summaries = Vec::new();
let mut function_calls = Vec::new();
for (index, block) in content.iter().enumerate() {
let block = block.as_object()?;
match block.get("type")?.as_str()? {
"text" => {
text.push_str(block.get("text")?.as_str()?);
}
"thinking" => {
if let Some(piece) = block
.get("thinking")
.and_then(Value::as_str)
.or_else(|| block.get("text").and_then(Value::as_str))
{
if !piece.trim().is_empty() {
reasoning_summaries.push(piece.to_string());
}
}
}
"tool_use" => {
let tool_name = block.get("name")?.as_str()?;
let call_id = block
.get("id")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.unwrap_or_else(|| build_generated_tool_call_id(index));
let arguments = canonicalize_tool_arguments(block.get("input").cloned());
function_calls.push(json!({
"type": "function_call",
"call_id": call_id,
"name": tool_name,
"arguments": arguments,
}));
}
_ => continue,
}
}
let usage = body.get("usage").and_then(Value::as_object);
let prompt_tokens = usage
.and_then(|value| value.get("input_tokens"))
.and_then(Value::as_u64)
.unwrap_or(0);
let output_tokens = usage
.and_then(|value| value.get("output_tokens"))
.and_then(Value::as_u64)
.unwrap_or(0);
let total_tokens = prompt_tokens + output_tokens;
let model = body
.get("model")
.and_then(Value::as_str)
.or_else(|| report_context.get("mapped_model").and_then(Value::as_str))
.or_else(|| report_context.get("model").and_then(Value::as_str))
.unwrap_or("unknown");
let response_id = body
.get("id")
.and_then(Value::as_str)
.unwrap_or("resp-local-finalize");
Some(build_openai_cli_response_with_reasoning(
response_id,
model,
&text,
reasoning_summaries,
function_calls,
OpenAiCliResponseUsage {
prompt_tokens,
output_tokens,
total_tokens,
},
))
let canonical = convert_claude_chat_response_to_openai_chat(body_json, report_context)?;
convert_openai_chat_response_to_openai_cli(&canonical, report_context, false)
}

159
crates/aether-ai-pipeline/src/conversion/response/to_openai_chat/gemini_chat.rs
View File

@@ -18,6 +18,7 @@ pub fn convert_gemini_chat_response_to_openai_chat(
let mut text = String::new();
let mut content_parts = Vec::new();
let mut reasoning_content = String::new();
let mut reasoning_parts = Vec::new();
let mut tool_calls = Vec::new();
let mut has_non_text_content = false;
for (index, part) in parts.iter().enumerate() {
@@ -29,6 +30,22 @@ pub fn convert_gemini_chat_response_to_openai_chat(
.unwrap_or(false)
{
reasoning_content.push_str(piece);
let mut reasoning_part = Map::new();
reasoning_part
.insert("type".to_string(), Value::String("thinking".to_string()));
reasoning_part.insert("thinking".to_string(), Value::String(piece.to_string()));
if let Some(signature) = part
.get("thoughtSignature")
.or_else(|| part.get("thought_signature"))
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
{
reasoning_part.insert(
"signature".to_string(),
Value::String(signature.to_string()),
);
}
reasoning_parts.push(Value::Object(reasoning_part));
} else {
text.push_str(piece);
content_parts.push(json!({
@@ -60,14 +77,15 @@ pub fn convert_gemini_chat_response_to_openai_chat(
"type": "text",
"text": rendered_text,
}));
} else if let Some(image_url) = extract_gemini_image_url(part) {
content_parts.push(json!({
"type": "image_url",
"image_url": {
"url": image_url,
} else if let Some(content_part) = convert_gemini_part_to_openai_content_part(part) {
if content_part.get("type").and_then(Value::as_str) == Some("text") {
if let Some(piece) = content_part.get("text").and_then(Value::as_str) {
text.push_str(piece);
}
}));
has_non_text_content = true;
} else {
has_non_text_content = true;
}
content_parts.push(content_part);
} else {
continue;
}
@@ -100,6 +118,9 @@ pub fn convert_gemini_chat_response_to_openai_chat(
Value::String(reasoning_content),
);
}
if !reasoning_parts.is_empty() {
message.insert("reasoning_parts".to_string(), Value::Array(reasoning_parts));
}
if !tool_calls.is_empty() {
message.insert("tool_calls".to_string(), Value::Array(tool_calls));
}
@@ -177,6 +198,69 @@ fn render_gemini_textual_part(part: &Map<String, Value>) -> Option<String> {
None
}
fn convert_gemini_part_to_openai_content_part(part: &Map<String, Value>) -> Option<Value> {
if let Some(image_url) = extract_gemini_image_url(part) {
return Some(json!({
"type": "image_url",
"image_url": {
"url": image_url,
}
}));
}
if let Some(inline_data) = part
.get("inlineData")
.or_else(|| part.get("inline_data"))
.and_then(Value::as_object)
{
let mime_type = inline_data
.get("mimeType")
.or_else(|| inline_data.get("mime_type"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
let data = inline_data
.get("data")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
if let Some(format) = mime_type.strip_prefix("audio/") {
return Some(json!({
"type": "input_audio",
"input_audio": {
"data": data,
"format": format,
}
}));
}
return Some(json!({
"type": "file",
"file": {
"file_data": format!("data:{mime_type};base64,{data}"),
}
}));
}
if let Some(file_data) = part
.get("fileData")
.or_else(|| part.get("file_data"))
.and_then(Value::as_object)
{
let file_uri = file_data
.get("fileUri")
.or_else(|| file_data.get("file_uri"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
return Some(json!({
"type": "text",
"text": format!("[File: {file_uri}]"),
}));
}
None
}
#[cfg(test)]
mod tests {
use super::convert_gemini_chat_response_to_openai_chat;
@@ -193,7 +277,7 @@ mod tests {
"finishReason": "RECITATION",
"content": {
"parts": [
{ "text": "thinking", "thought": true },
{ "text": "thinking", "thought": true, "thoughtSignature": "sig_123" },
{ "executableCode": { "language": "python", "code": "print(1)" } },
{ "codeExecutionResult": { "output": "1" } }
]
@@ -226,6 +310,16 @@ mod tests {
converted["choices"][0]["message"]["reasoning_content"],
"thinking"
);
assert_eq!(
converted["choices"][0]["message"]["reasoning_parts"],
json!([
{
"type": "thinking",
"thinking": "thinking",
"signature": "sig_123"
}
])
);
let content = converted["choices"][0]["message"]["content"]
.as_str()
.expect("content should be string");
@@ -238,4 +332,53 @@ mod tests {
);
assert_eq!(converted["usage"]["completion_tokens"], 7);
}
#[test]
fn preserves_gemini_multimodal_parts_in_openai_chat_response() {
let response = json!({
"responseId": "resp_mm_123",
"modelVersion": "gemini-2.5-pro",
"candidates": [{
"index": 0,
"finishReason": "STOP",
"content": {
"parts": [
{ "text": "Attached." },
{ "inlineData": { "mimeType": "image/png", "data": "iVBORw0KGgo=" } },
{ "inlineData": { "mimeType": "application/pdf", "data": "JVBERi0x" } },
{ "inline_data": { "mime_type": "audio/mp3", "data": "SUQz" } },
{ "fileData": { "fileUri": "https://example.com/report.pdf", "mimeType": "application/pdf" } }
]
}
}],
"usageMetadata": {
"promptTokenCount": 4,
"candidatesTokenCount": 2,
"totalTokenCount": 6
}
});
let converted = convert_gemini_chat_response_to_openai_chat(&response, &json!({}))
.expect("response should convert");
assert_eq!(
converted["choices"][0]["message"]["content"],
json!([
{ "type": "text", "text": "Attached." },
{
"type": "image_url",
"image_url": { "url": "data:image/png;base64,iVBORw0KGgo=" }
},
{
"type": "file",
"file": { "file_data": "data:application/pdf;base64,JVBERi0x" }
},
{
"type": "input_audio",
"input_audio": { "data": "SUQz", "format": "mp3" }
},
{ "type": "text", "text": "[File: https://example.com/report.pdf]" }
])
);
}
}

108
crates/aether-ai-pipeline/src/conversion/response/to_openai_chat/gemini_cli.rs
View File

@@ -1,110 +1,12 @@
use serde_json::{json, Value};
use serde_json::Value;
use super::super::from_openai_chat::{
build_openai_cli_response_with_content, OpenAiCliResponseUsage,
};
use super::shared::{
build_generated_tool_call_id, canonicalize_tool_arguments, extract_gemini_image_url,
};
use super::super::from_openai_chat::convert_openai_chat_response_to_openai_cli;
use super::gemini_chat::convert_gemini_chat_response_to_openai_chat;
pub fn convert_gemini_cli_response_to_openai_cli(
body_json: &Value,
report_context: &Value,
) -> Option<Value> {
let body = body_json.as_object()?;
let candidates = body.get("candidates")?.as_array()?;
let first_candidate = candidates.first()?.as_object()?;
let content = first_candidate.get("content")?.as_object()?;
let parts = content.get("parts")?.as_array()?;
let mut message_content = Vec::new();
let mut reasoning_summaries = Vec::new();
let mut function_calls = Vec::new();
for (index, part) in parts.iter().enumerate() {
let part = part.as_object()?;
if let Some(piece) = part.get("text").and_then(Value::as_str) {
if part
.get("thought")
.and_then(Value::as_bool)
.unwrap_or(false)
{
if !piece.trim().is_empty() {
reasoning_summaries.push(piece.to_string());
}
} else {
message_content.push(json!({
"type": "output_text",
"text": piece,
"annotations": []
}));
}
} else if let Some(function_call) = part.get("functionCall").and_then(Value::as_object) {
let tool_name = function_call.get("name")?.as_str()?;
let call_id = function_call
.get("id")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.unwrap_or_else(|| build_generated_tool_call_id(index));
let arguments = canonicalize_tool_arguments(function_call.get("args").cloned());
function_calls.push(json!({
"type": "function_call",
"call_id": call_id,
"name": tool_name,
"arguments": arguments,
}));
} else if let Some(image_url) = extract_gemini_image_url(part) {
message_content.push(json!({
"type": "output_image",
"image_url": image_url,
}));
} else {
continue;
}
}
let usage = body.get("usageMetadata").and_then(Value::as_object);
let prompt_tokens = usage
.and_then(|value| value.get("promptTokenCount"))
.and_then(Value::as_u64)
.unwrap_or(0);
let output_tokens = usage
.map(|value| {
value
.get("candidatesTokenCount")
.and_then(Value::as_u64)
.unwrap_or(0)
+ value
.get("thoughtsTokenCount")
.and_then(Value::as_u64)
.unwrap_or(0)
})
.unwrap_or(0);
let total_tokens = usage
.and_then(|value| value.get("totalTokenCount"))
.and_then(Value::as_u64)
.unwrap_or(prompt_tokens + output_tokens);
let model = body
.get("modelVersion")
.and_then(Value::as_str)
.or_else(|| report_context.get("mapped_model").and_then(Value::as_str))
.or_else(|| report_context.get("model").and_then(Value::as_str))
.unwrap_or("unknown");
let response_id = body
.get("responseId")
.or_else(|| body.get("_v1internal_response_id"))
.and_then(Value::as_str)
.unwrap_or("resp-local-finalize");
Some(build_openai_cli_response_with_content(
response_id,
model,
message_content,
reasoning_summaries,
function_calls,
OpenAiCliResponseUsage {
prompt_tokens,
output_tokens,
total_tokens,
},
))
let canonical = convert_gemini_chat_response_to_openai_chat(body_json, report_context)?;
convert_openai_chat_response_to_openai_cli(&canonical, report_context, false)
}

117
crates/aether-ai-pipeline/src/conversion/response/to_openai_chat/openai_cli.rs
View File

@@ -80,6 +80,18 @@ pub fn convert_openai_cli_response_to_openai_chat(
}));
has_non_text_content = true;
}
} else if part_type == "file" {
if let Some(file_part) = extract_openai_response_file(part_object) {
content_parts.push(file_part);
has_non_text_content = true;
}
} else if part_type == "input_audio" {
if let Some(audio_part) =
extract_openai_response_input_audio(part_object)
{
content_parts.push(audio_part);
has_non_text_content = true;
}
}
}
}
@@ -298,6 +310,52 @@ fn extract_openai_response_image(
Some((image_url, detail))
}
fn extract_openai_response_file(part_object: &Map<String, Value>) -> Option<Value> {
let file_object = part_object
.get("file")
.and_then(Value::as_object)
.unwrap_or(part_object);
let mut file = Map::new();
for key in ["file_data", "file_id", "filename"] {
if let Some(value) = file_object
.get(key)
.cloned()
.filter(|value| !value.is_null())
{
file.insert(key.to_string(), value);
}
}
if file.is_empty() {
return None;
}
Some(json!({
"type": "file",
"file": Value::Object(file),
}))
}
fn extract_openai_response_input_audio(part_object: &Map<String, Value>) -> Option<Value> {
let audio_object = part_object
.get("input_audio")
.and_then(Value::as_object)
.unwrap_or(part_object);
let data = audio_object
.get("data")
.cloned()
.filter(|value| value.as_str().is_some_and(|value| !value.trim().is_empty()))?;
let format = audio_object
.get("format")
.cloned()
.filter(|value| value.as_str().is_some_and(|value| !value.trim().is_empty()))?;
Some(json!({
"type": "input_audio",
"input_audio": {
"data": data,
"format": format,
}
}))
}
fn offset_annotation_indices(annotation: &Value, offset: i64) -> Value {
let Some(object) = annotation.as_object() else {
return annotation.clone();
@@ -374,4 +432,63 @@ mod tests {
json!({"reasoning_tokens": 1})
);
}
#[test]
fn preserves_file_and_audio_parts_when_converting_to_chat() {
let response = json!({
"id": "resp_mm_123",
"object": "response",
"model": "gpt-5",
"output": [{
"type": "message",
"role": "assistant",
"content": [
{ "type": "output_text", "text": "Attached." },
{
"type": "file",
"file": {
"file_data": "data:application/pdf;base64,JVBERi0x",
"filename": "report.pdf"
}
},
{
"type": "input_audio",
"input_audio": {
"data": "SUQz",
"format": "mp3"
}
}
]
}],
"usage": {
"input_tokens": 4,
"output_tokens": 2,
"total_tokens": 6
}
});
let converted = convert_openai_cli_response_to_openai_chat(&response, &json!({}))
.expect("responses response should convert to chat");
assert_eq!(
converted["choices"][0]["message"]["content"],
json!([
{ "type": "text", "text": "Attached." },
{
"type": "file",
"file": {
"file_data": "data:application/pdf;base64,JVBERi0x",
"filename": "report.pdf"
}
},
{
"type": "input_audio",
"input_audio": {
"data": "SUQz",
"format": "mp3"
}
}
])
);
}
}

18
crates/aether-ai-pipeline/src/conversion/response/to_openai_chat/shared.rs
View File

@@ -13,17 +13,28 @@ pub(super) fn canonicalize_tool_arguments(value: Option<Value>) -> String {
}
pub(super) fn extract_gemini_image_url(part: &Map<String, Value>) -> Option<String> {
if let Some(inline_data) = part.get("inlineData").and_then(Value::as_object) {
let mime_type = inline_data.get("mimeType").and_then(Value::as_str)?;
if let Some(inline_data) = part
.get("inlineData")
.or_else(|| part.get("inline_data"))
.and_then(Value::as_object)
{
let mime_type = inline_data
.get("mimeType")
.or_else(|| inline_data.get("mime_type"))
.and_then(Value::as_str)?;
if !mime_type.starts_with("image/") {
return None;
}
let data = inline_data.get("data").and_then(Value::as_str)?;
return Some(format!("data:{mime_type};base64,{data}"));
}
let file_data = part.get("fileData").and_then(Value::as_object)?;
let file_data = part
.get("fileData")
.or_else(|| part.get("file_data"))
.and_then(Value::as_object)?;
if file_data
.get("mimeType")
.or_else(|| file_data.get("mime_type"))
.and_then(Value::as_str)
.is_some_and(|mime_type| !mime_type.starts_with("image/"))
{
@@ -31,6 +42,7 @@ pub(super) fn extract_gemini_image_url(part: &Map<String, Value>) -> Option<Stri
}
file_data
.get("fileUri")
.or_else(|| file_data.get("file_uri"))
.and_then(Value::as_str)
.map(ToOwned::to_owned)
}