i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 01:10:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/pr/524'

Browse Source
This commit is contained in:
fawney19
2026-05-20 23:17:31 +08:00
parent d49b0a8a45 779877acd0
commit 4d0c654822
29 changed files with 1370 additions and 978 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
apps/aether-gateway/src/ai_serving/planner/specialized/image/request.rs
View File

@@ -19,9 +19,10 @@ use crate::ai_serving::{
apply_codex_openai_responses_special_body_edits, apply_codex_openai_responses_special_headers,
build_chatgpt_web_image_request_body,
build_gemini_image_request_body_from_openai_image_request,
build_openai_image_provider_request_body, default_model_for_openai_image_operation,
normalize_openai_image_request, request_conversion_direct_auth, CandidateFailureDiagnostic,
GatewayProviderTransportSnapshot, PlannerAppState, RequestConversionKind,
build_openai_image_api_provider_request_body, build_openai_image_provider_request_body,
default_model_for_openai_image_operation, normalize_openai_image_request,
request_conversion_direct_auth, CandidateFailureDiagnostic, GatewayProviderTransportSnapshot,
PlannerAppState, RequestConversionKind,
};
use crate::image_capabilities::openai_image_normalize_options_for_provider;
use crate::AppState;
@@ -160,21 +161,31 @@ pub(super) async fn resolve_local_openai_image_candidate_payload_parts(
.provider_type
.trim()
.eq_ignore_ascii_case("grok");
let is_codex = transport
.provider
.provider_type
.trim()
.eq_ignore_ascii_case("codex");
let transport_profile = crate::ai_serving::transport::resolve_transport_profile(transport);
let upstream_url = if is_chatgpt_web {
chatgpt_web_image_internal_url(&transport.endpoint.base_url)
} else if is_grok {
build_grok_upstream_url(transport, GROK_CHAT_PATH)
} else {
build_openai_image_upstream_url(transport, parts.uri.query())
build_openai_image_upstream_url(transport, Some(parts.uri.path()), parts.uri.query())
};
let mut provider_request_body = if is_chatgpt_web {
match build_chatgpt_web_image_request_body(parts, body_json, body_base64) {
Ok(body) => body,
Err(err) => err.to_error_json(),
}
} else {
} else if is_codex || is_grok {
build_openai_image_provider_request_body(&normalized_request)
} else {
build_openai_image_api_provider_request_body(
&normalized_request,
Some(prepared_candidate.mapped_model.as_str()),
)
};
if !is_chatgpt_web {
apply_codex_openai_responses_special_body_edits(

3
apps/aether-gateway/src/ai_serving/planner/standard/family/request.rs
View File

@@ -625,7 +625,8 @@ async fn resolve_local_gemini_image_to_openai_image_candidate_payload_parts(
};
let upstream_is_stream = true;
let upstream_url = build_openai_image_upstream_url(transport, None);
let upstream_url =
build_openai_image_upstream_url(transport, Some("/v1/images/generations"), None);
let effective_headers = input.effective_headers(&parts.headers);
let Some(mut provider_request_headers) =
build_openai_image_headers(ProviderOpenAiImageHeadersInput {

1
apps/aether-gateway/src/ai_serving/planner/standard/openai/chat/decision.rs
View File

@@ -9,7 +9,6 @@ pub(super) use self::payload::maybe_build_local_openai_chat_decision_payload_for
pub(super) use self::support::{
build_lazy_local_openai_chat_candidate_attempt_source,
build_local_openai_chat_candidate_attempt_source,
build_local_openai_chat_image_candidate_attempt_source,
materialize_local_openai_chat_candidate_attempts, LocalOpenAiChatCandidateAttempt,
LocalOpenAiChatCandidateAttemptSource, LocalOpenAiChatDecisionInput,
};

540
apps/aether-gateway/src/ai_serving/planner/standard/openai/chat/decision/request.rs
View File

@@ -4,7 +4,7 @@ use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use aether_contracts::ResolvedTransportProfile;
use serde_json::{json, Value};
use serde_json::Value;
use crate::ai_serving::planner::candidate_preparation::{
prepare_header_authenticated_candidate, prepare_header_authenticated_candidate_from_auth,
@@ -16,10 +16,9 @@ use crate::ai_serving::planner::common::{
request_requires_body_stream_field, OPENAI_CHAT_STREAM_PLAN_KIND,
};
use crate::ai_serving::planner::standard::{
apply_codex_openai_responses_special_body_edits, apply_codex_openai_responses_special_headers,
build_cross_format_openai_chat_request_body, build_cross_format_openai_chat_upstream_url,
build_local_openai_chat_request_body, build_local_openai_chat_upstream_url,
request_body_build_failure_extra_data,
apply_codex_openai_responses_special_headers, build_cross_format_openai_chat_request_body,
build_cross_format_openai_chat_upstream_url, build_local_openai_chat_request_body,
build_local_openai_chat_upstream_url, request_body_build_failure_extra_data,
};
use crate::ai_serving::transport::auth::resolve_local_openai_bearer_auth;
use crate::ai_serving::transport::kiro::{
@@ -30,10 +29,8 @@ use crate::ai_serving::transport::kiro::{
use crate::ai_serving::transport::local_openai_chat_transport_unsupported_reason;
use crate::ai_serving::transport::{
build_grok_browser_headers, build_grok_upstream_url, build_kiro_cross_format_upstream_url,
build_openai_image_headers, build_openai_image_upstream_url,
build_standard_provider_request_headers, openai_image_transport_unsupported_reason,
resolve_openai_image_auth, GrokHeaderInput, ProviderOpenAiImageHeadersInput,
StandardProviderRequestHeadersInput, GROK_CHAT_PATH,
build_standard_provider_request_headers, GrokHeaderInput, StandardProviderRequestHeadersInput,
GROK_CHAT_PATH,
};
use crate::ai_serving::{
ai_local_execution_contract_for_formats, request_conversion_direct_auth,
@@ -501,20 +498,6 @@ pub(crate) async fn resolve_local_openai_chat_candidate_payload_parts(
};
let provider_api_format = provider_api_format.trim().to_ascii_lowercase();
if provider_api_format == "openai:image" {
return resolve_openai_chat_to_openai_image_payload_parts(
state,
parts,
trace_id,
body_json,
input,
eligible,
candidate_index,
candidate_id,
upstream_is_stream,
)
.await;
}
let Some(conversion_kind) =
request_conversion_kind("openai:chat", provider_api_format.as_str())
@@ -806,449 +789,6 @@ pub(crate) async fn resolve_local_openai_chat_candidate_payload_parts(
}))
}
#[allow(clippy::too_many_arguments)]
async fn resolve_openai_chat_to_openai_image_payload_parts(
state: &AppState,
parts: &http::request::Parts,
trace_id: &str,
body_json: &serde_json::Value,
input: &LocalOpenAiChatDecisionInput,
eligible: &EligibleLocalExecutionCandidate,
candidate_index: u32,
candidate_id: &str,
upstream_is_stream: bool,
) -> Result<Option<LocalOpenAiChatCandidatePayloadParts>, GatewayError> {
let candidate = &eligible.candidate;
let transport = &eligible.transport;
let provider_api_format = "openai:image";
if let Some(skip_reason) =
openai_image_transport_unsupported_reason(transport, provider_api_format)
{
mark_skipped_local_openai_chat_candidate(
state,
input,
trace_id,
candidate,
candidate_index,
candidate_id,
skip_reason,
)
.await;
return Ok(None);
}
let prepared_candidate = match prepare_header_authenticated_candidate(
crate::ai_serving::PlannerAppState::new(state),
transport,
candidate,
resolve_openai_image_auth(transport),
OauthPreparationContext {
trace_id,
api_format: provider_api_format,
operation: "openai_chat_image_bridge",
},
)
.await
{
Ok(prepared) => prepared,
Err(skip_reason) => {
mark_skipped_local_openai_chat_candidate(
state,
input,
trace_id,
candidate,
candidate_index,
candidate_id,
skip_reason,
)
.await;
return Ok(None);
}
};
let is_chatgpt_web = transport
.provider
.provider_type
.trim()
.eq_ignore_ascii_case("chatgpt_web");
let Some((mut provider_request_body, image_request_summary)) = (if is_chatgpt_web {
build_chatgpt_web_image_provider_body_from_openai_chat_body(
body_json,
&input.requested_model,
)
} else {
build_openai_image_provider_body_from_openai_chat_body(
body_json,
&input.requested_model,
upstream_is_stream,
)
}) else {
mark_skipped_local_openai_chat_candidate_with_extra_data(
state,
input,
trace_id,
candidate,
candidate_index,
candidate_id,
"provider_request_body_build_failed",
request_body_build_failure_extra_data(body_json, "openai:chat", provider_api_format),
)
.await;
return Ok(None);
};
if !is_chatgpt_web {
apply_codex_openai_responses_special_body_edits(
&mut provider_request_body,
transport.provider.provider_type.as_str(),
provider_api_format,
transport.endpoint.body_rules.as_ref(),
Some(candidate.key_id.as_str()),
);
}
let upstream_url = if is_chatgpt_web {
chatgpt_web_image_internal_url(&transport.endpoint.base_url)
} else {
build_openai_image_upstream_url(transport, parts.uri.query())
};
let Some(mut provider_request_headers) =
build_openai_image_headers(ProviderOpenAiImageHeadersInput {
headers: &parts.headers,
auth_header: &prepared_candidate.auth_header,
auth_value: &prepared_candidate.auth_value,
header_rules: transport.endpoint.header_rules.as_ref(),
provider_request_body: &provider_request_body,
original_request_body: body_json,
})
else {
mark_skipped_local_openai_chat_candidate_with_failure_diagnostic(
state,
input,
trace_id,
candidate,
candidate_index,
candidate_id,
"transport_header_rules_apply_failed",
CandidateFailureDiagnostic::header_rules_apply_failed(
"openai:chat",
provider_api_format,
"openai_chat_image_bridge_headers",
),
)
.await;
return Ok(None);
};
if is_chatgpt_web {
provider_request_headers.insert("x-aether-chatgpt-web-image".to_string(), "1".to_string());
} else {
apply_codex_openai_responses_special_headers(
&mut provider_request_headers,
&provider_request_body,
&parts.headers,
transport.provider.provider_type.as_str(),
provider_api_format,
Some(trace_id),
transport.key.decrypted_auth_config.as_deref(),
);
}
let (execution_strategy, conversion_mode) =
ai_local_execution_contract_for_formats("openai:chat", provider_api_format);
Ok(Some(LocalOpenAiChatCandidatePayloadParts {
client_api_format: "openai:chat".to_string(),
auth_header: prepared_candidate.auth_header,
auth_value: prepared_candidate.auth_value,
mapped_model: prepared_candidate.mapped_model,
provider_api_format: provider_api_format.to_string(),
provider_request_body,
provider_request_headers,
upstream_url,
execution_strategy,
conversion_mode,
report_kind: "openai_chat_stream_success".to_string(),
envelope_name: None,
transport: Arc::clone(transport),
request_redacted: false,
transport_profile: None,
image_request_summary: Some(image_request_summary),
}))
}
fn build_openai_image_provider_body_from_openai_chat_body(
body_json: &Value,
requested_model: &str,
upstream_is_stream: bool,
) -> Option<(Value, Value)> {
let (prompt, images) = collect_openai_chat_image_prompt_and_images(body_json)?;
let operation = if images.is_empty() {
"generate"
} else {
"edit"
};
let mut image_options = serde_json::Map::new();
copy_openai_chat_image_option(body_json, &mut image_options, "size");
copy_openai_chat_image_option(body_json, &mut image_options, "quality");
copy_openai_chat_image_option(body_json, &mut image_options, "background");
copy_openai_chat_image_option(body_json, &mut image_options, "output_format");
copy_openai_chat_image_option(body_json, &mut image_options, "output_compression");
copy_openai_chat_image_option(body_json, &mut image_options, "moderation");
copy_openai_chat_image_option(body_json, &mut image_options, "input_fidelity");
copy_openai_chat_image_option(body_json, &mut image_options, "partial_images");
let input = if images.is_empty() {
serde_json::json!([{
"role": "user",
"content": prompt,
}])
} else {
let mut content = vec![serde_json::json!({
"type": "input_text",
"text": prompt,
})];
content.extend(images);
serde_json::json!([{
"role": "user",
"content": content,
}])
};
let mut body = serde_json::Map::new();
if let Some(model) = body_json
.get("model")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.or_else(|| {
let requested_model = requested_model.trim();
(!requested_model.is_empty()).then_some(requested_model)
})
{
body.insert("model".to_string(), Value::String(model.to_string()));
}
body.insert("input".to_string(), input);
let mut image_tool = image_options.clone();
image_tool.insert(
"type".to_string(),
Value::String("image_generation".to_string()),
);
body.insert(
"tools".to_string(),
Value::Array(vec![Value::Object(image_tool)]),
);
if upstream_is_stream {
body.insert("stream".to_string(), Value::Bool(true));
}
if let Some(user) = body_json
.get("user")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
{
body.insert("user".to_string(), Value::String(user.to_string()));
}
let mut summary = serde_json::Map::new();
summary.insert(
"operation".to_string(),
Value::String(operation.to_string()),
);
for key in ["output_format", "partial_images", "size", "quality"] {
if let Some(value) = image_options.get(key) {
summary.insert(key.to_string(), value.clone());
}
}
Some((Value::Object(body), Value::Object(summary)))
}
fn build_chatgpt_web_image_provider_body_from_openai_chat_body(
body_json: &Value,
requested_model: &str,
) -> Option<(Value, Value)> {
let (prompt, images) = collect_openai_chat_image_prompt_and_images(body_json)?;
let operation = if images.is_empty() {
"generate"
} else {
"edit"
};
let size = body_json
.get("size")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or("1024x1024");
let output_format = body_json
.get("output_format")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or("png");
let quality = body_json
.get("quality")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or("medium");
let model = body_json
.get("model")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or_else(|| requested_model.trim());
let web_model = body_json
.get("web_model")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or("gpt-5-5-thinking");
let image_urls = openai_image_inputs_as_urls(&images);
let body = json!({
"operation": operation,
"model": if model.is_empty() { "gpt-image-2" } else { model },
"web_model": web_model,
"prompt": prompt,
"size": size,
"ratio": chatgpt_web_ratio_for_size(size),
"output_format": output_format,
"images": image_urls,
});
let summary = json!({
"operation": operation,
"output_format": output_format,
"size": size,
"quality": quality,
});
Some((body, summary))
}
fn copy_openai_chat_image_option(
body_json: &Value,
image_options: &mut serde_json::Map<String, Value>,
key: &str,
) {
if let Some(value) = body_json.get(key) {
image_options.insert(key.to_string(), value.clone());
}
}
fn collect_openai_chat_image_prompt_and_images(body_json: &Value) -> Option<(String, Vec<Value>)> {
let messages = body_json.get("messages").and_then(Value::as_array)?;
let mut prompt_parts = Vec::new();
let mut images = Vec::new();
for message in messages.iter().filter_map(Value::as_object) {
let role = message
.get("role")
.and_then(Value::as_str)
.map(str::trim)
.unwrap_or_default();
let content = message.get("content");
if matches!(role, "system" | "developer" | "user") {
if let Some(text) = crate::ai_serving::extract_openai_text_content(content)
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
{
prompt_parts.push(text);
}
}
if role == "user" {
collect_openai_chat_image_inputs(content, &mut images);
}
}
let prompt = prompt_parts.join("\n").trim().to_string();
(!prompt.is_empty()).then_some((prompt, images))
}
fn collect_openai_chat_image_inputs(content: Option<&Value>, images: &mut Vec<Value>) {
let Some(parts) = content.and_then(Value::as_array) else {
return;
};
for part in parts.iter().filter_map(Value::as_object) {
let part_type = part
.get("type")
.and_then(Value::as_str)
.map(str::trim)
.unwrap_or_default();
if matches!(part_type, "image_url" | "input_image") {
if let Some(url) = part
.get("image_url")
.and_then(|value| {
value
.as_str()
.or_else(|| value.get("url").and_then(Value::as_str))
})
.map(str::trim)
.filter(|value| !value.is_empty())
{
images.push(serde_json::json!({
"type": "input_image",
"image_url": url,
}));
} else if let Some(file_id) = part
.get("file_id")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
{
images.push(serde_json::json!({
"type": "input_image",
"file_id": file_id,
}));
}
}
}
}
fn openai_image_inputs_as_urls(images: &[Value]) -> Vec<Value> {
images
.iter()
.filter_map(|image| {
image
.get("image_url")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(|value| Value::String(value.to_string()))
})
.collect()
}
fn chatgpt_web_ratio_for_size(size: &str) -> String {
let Some((width, height)) = size.split_once('x') else {
return "1:1".to_string();
};
let Ok(width) = width.trim().parse::<u64>() else {
return "1:1".to_string();
};
let Ok(height) = height.trim().parse::<u64>() else {
return "1:1".to_string();
};
if width == 0 || height == 0 {
return "1:1".to_string();
}
let divisor = gcd(width, height);
format!("{}:{}", width / divisor, height / divisor)
}
fn gcd(mut left: u64, mut right: u64) -> u64 {
while right != 0 {
let next = left % right;
left = right;
right = next;
}
left.max(1)
}
fn chatgpt_web_image_internal_url(base_url: &str) -> String {
let base_url = base_url.trim().trim_end_matches('/');
let base_url = if base_url.is_empty() {
"https://chatgpt.com"
} else {
base_url
};
format!("{base_url}/__aether/chatgpt-web-image")
}
#[allow(clippy::too_many_arguments)]
async fn build_kiro_openai_chat_cross_format_payload_parts(
state: &AppState,
@@ -1472,71 +1012,3 @@ fn redaction_mask_error_to_gateway_error(error: RedactionMaskError) -> GatewayEr
},
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn chatgpt_web_chat_image_bridge_body_uses_internal_web_shape() {
let body_json = json!({
"model": "gpt-image-2",
"messages": [
{"role": "system", "content": "Use crisp vector-like shapes."},
{
"role": "user",
"content": [
{"type": "text", "text": "Draw a glass city"},
{"type": "image_url", "image_url": {"url": "https://example.com/ref.png"}}
]
}
],
"size": "1536x1024",
"output_format": "webp",
"web_model": "gpt-5-image-test"
});
let (provider_body, summary) =
build_chatgpt_web_image_provider_body_from_openai_chat_body(&body_json, "gpt-image-2")
.expect("chat image body should convert");
assert_eq!(provider_body["operation"], "edit");
assert_eq!(provider_body["model"], "gpt-image-2");
assert_eq!(provider_body["web_model"], "gpt-5-image-test");
assert_eq!(
provider_body["prompt"],
"Use crisp vector-like shapes.\nDraw a glass city"
);
assert_eq!(provider_body["size"], "1536x1024");
assert_eq!(provider_body["ratio"], "3:2");
assert_eq!(provider_body["output_format"], "webp");
assert_eq!(provider_body["images"][0], "https://example.com/ref.png");
assert_eq!(summary["operation"], "edit");
assert_eq!(summary["output_format"], "webp");
}
#[test]
fn openai_chat_image_bridge_body_injects_image_generation_tool() {
let body_json = json!({
"model": "gpt-image-2",
"messages": [
{"role": "user", "content": "Draw a glass city"}
],
"size": "1024x1024",
"output_format": "png"
});
let (provider_body, summary) =
build_openai_image_provider_body_from_openai_chat_body(&body_json, "gpt-image-2", true)
.expect("chat image body should convert");
assert_eq!(provider_body["tools"][0]["type"], "image_generation");
assert_eq!(provider_body["tools"][0]["size"], "1024x1024");
assert_eq!(provider_body["tools"][0]["output_format"], "png");
assert_eq!(provider_body["model"], "gpt-image-2");
assert_eq!(provider_body["stream"], true);
assert_eq!(provider_body["input"][0]["content"], "Draw a glass city");
assert_eq!(summary["operation"], "generate");
assert_eq!(summary["output_format"], "png");
}
}

99
apps/aether-gateway/src/ai_serving/planner/standard/openai/chat/decision/support.rs
View File

@@ -14,10 +14,7 @@ use crate::ai_serving::planner::candidate_metadata::{
LocalExecutionCandidateMetadataParts,
};
use crate::ai_serving::planner::candidate_resolution::SkippedLocalExecutionCandidate;
use crate::ai_serving::planner::candidate_source::{
preselect_local_execution_candidates_for_api_formats_with_serving,
LocalCandidatePreselectionKeyMode,
};
use crate::ai_serving::planner::candidate_source::LocalCandidatePreselectionKeyMode;
use crate::ai_serving::planner::materialization_policy::{
build_local_candidate_persistence_policy, LocalCandidatePersistencePolicyKind,
};
@@ -26,7 +23,7 @@ use crate::ai_serving::{
ai_local_execution_contract_for_formats, extract_pool_sticky_session_token,
ExecutionRuntimeAuthContext, PlannerAppState,
};
use crate::{AppState, GatewayError};
use crate::AppState;
pub(crate) use crate::ai_serving::planner::candidate_materialization::LocalExecutionCandidateAttempt as LocalOpenAiChatCandidateAttempt;
pub(crate) use crate::ai_serving::planner::candidate_materialization::LocalExecutionCandidateAttemptSource as LocalOpenAiChatCandidateAttemptSource;
@@ -357,95 +354,3 @@ pub(crate) async fn build_lazy_local_openai_chat_candidate_attempt_source<'a>(
)
.await
}
pub(crate) async fn build_local_openai_chat_image_candidate_attempt_source<'a>(
state: &'a AppState,
trace_id: &str,
input: &LocalOpenAiChatDecisionInput,
body_json: &serde_json::Value,
) -> Result<(LocalOpenAiChatCandidateAttemptSource<'a>, usize), GatewayError> {
let planner_state = PlannerAppState::new(state);
let sticky_session_token = extract_pool_sticky_session_token(body_json);
let auth_context: &ExecutionRuntimeAuthContext = &input.auth_context;
let persistence_policy = build_local_candidate_persistence_policy(
auth_context,
input.required_capabilities.as_ref(),
LocalCandidatePersistencePolicyKind::OpenAiChatDecision,
);
let preselection = preselect_local_execution_candidates_for_api_formats_with_serving(
planner_state,
"openai:chat",
&input.requested_model,
false,
input.required_capabilities.as_ref(),
&input.auth_snapshot,
input.routing_policy.as_ref(),
input.client_session_affinity.as_ref(),
false,
LocalCandidatePreselectionKeyMode::ProviderEndpointKeyModelAndApiFormat,
vec!["openai:image".to_string()],
)
.await?;
Ok(build_local_execution_candidate_attempt_source_with_serving(
planner_state,
trace_id,
"openai:chat",
Some(&input.requested_model),
Some(&input.auth_snapshot),
input.client_session_affinity.as_ref(),
input.required_capabilities.as_ref(),
input.routing_policy.as_ref(),
sticky_session_token.as_deref(),
input.request_auth_channel.as_deref(),
persistence_policy,
preselection.candidates,
preselection.skipped_candidates,
LocalCandidateResolutionMode::WithoutTransportPairGate,
|eligible| {
let provider_api_format = eligible.provider_api_format.clone();
let (execution_strategy, conversion_mode) =
ai_local_execution_contract_for_formats("openai:chat", &provider_api_format);
Some(build_local_execution_candidate_contract_metadata(
LocalExecutionCandidateMetadataParts {
eligible,
provider_api_format: provider_api_format.as_str(),
client_api_format: "openai:chat",
extra_fields: serde_json::Map::new(),
},
execution_strategy,
conversion_mode,
eligible.candidate.endpoint_api_format.trim(),
))
},
|mut skipped_candidate| {
let provider_api_format = skipped_candidate
.transport
.as_ref()
.map(|transport| transport.endpoint.api_format.trim().to_ascii_lowercase())
.unwrap_or_else(|| {
skipped_candidate
.candidate
.endpoint_api_format
.trim()
.to_ascii_lowercase()
});
let (execution_strategy, conversion_mode) =
ai_local_execution_contract_for_formats("openai:chat", &provider_api_format);
skipped_candidate.extra_data = Some(
build_local_execution_candidate_contract_metadata_for_candidate(
&skipped_candidate.candidate,
skipped_candidate.transport_ref(),
provider_api_format.as_str(),
"openai:chat",
serde_json::Map::new(),
execution_strategy,
conversion_mode,
provider_api_format.as_str(),
),
);
skipped_candidate
},
)
.await)
}

1
apps/aether-gateway/src/ai_serving/planner/standard/openai/chat/mod.rs
View File

@@ -11,7 +11,6 @@ mod plans;
use self::decision::{
build_lazy_local_openai_chat_candidate_attempt_source,
build_local_openai_chat_image_candidate_attempt_source,
maybe_build_local_openai_chat_decision_payload_for_candidate, LocalOpenAiChatCandidateAttempt,
LocalOpenAiChatCandidateAttemptSource, LocalOpenAiChatDecisionInput,
};

57
apps/aether-gateway/src/ai_serving/planner/standard/openai/chat/plans/stream.rs
View File

@@ -1,10 +1,8 @@
use async_trait::async_trait;
use tracing::warn;
use super::super::super::openai_request_is_image_generation_intent;
use super::super::{
build_lazy_local_openai_chat_candidate_attempt_source,
build_local_openai_chat_image_candidate_attempt_source,
maybe_build_local_openai_chat_decision_payload_for_candidate, AppState, GatewayControlDecision,
GatewayError, LocalOpenAiChatCandidateAttempt, LocalOpenAiChatCandidateAttemptSource,
LocalOpenAiChatDecisionInput,
@@ -51,53 +49,14 @@ pub(crate) async fn build_local_openai_chat_stream_attempt_source<'a>(
};
let effective_body_json = input.effective_body_json(body_json).clone();
let image_generation_intent =
openai_request_is_image_generation_intent(&input.requested_model, body_json);
let (mut candidates, mut candidate_count) = if image_generation_intent {
let (image_candidates, image_candidate_count) =
build_local_openai_chat_image_candidate_attempt_source(
state,
trace_id,
&input,
&effective_body_json,
)
.await?;
if image_candidate_count > 0 {
(image_candidates, image_candidate_count)
} else {
build_lazy_local_openai_chat_candidate_attempt_source(
state,
trace_id,
&input,
&effective_body_json,
true,
)
.await
}
} else {
build_lazy_local_openai_chat_candidate_attempt_source(
state,
trace_id,
&input,
&effective_body_json,
true,
)
.await
};
if !image_generation_intent && candidate_count == 0 {
let (image_candidates, image_candidate_count) =
build_local_openai_chat_image_candidate_attempt_source(
state,
trace_id,
&input,
&effective_body_json,
)
.await?;
if image_candidate_count > 0 {
candidates = image_candidates;
candidate_count = image_candidate_count;
}
}
let (candidates, candidate_count) = build_lazy_local_openai_chat_candidate_attempt_source(
state,
trace_id,
&input,
&effective_body_json,
true,
)
.await;
if candidate_count == 0 {
set_local_openai_chat_candidate_evaluation_diagnostic(
state,

96
apps/aether-gateway/src/ai_serving/planner/standard/openai/image_intent.rs
View File

@@ -1,75 +1,55 @@
pub(crate) fn openai_request_is_image_generation_intent(
requested_model: &str,
body_json: &serde_json::Value,
use serde_json::Value;
pub(super) fn openai_request_is_image_generation_intent(
_requested_model: &str,
body_json: &Value,
) -> bool {
openai_model_is_image_generation(requested_model)
|| body_json
.get("model")
.and_then(serde_json::Value::as_str)
.is_some_and(openai_model_is_image_generation)
|| openai_tool_choice_selects_image_generation(body_json.get("tool_choice"))
request_forces_image_generation_tool(body_json)
}
fn openai_model_is_image_generation(model: &str) -> bool {
model.trim().to_ascii_lowercase().starts_with("gpt-image-")
fn request_forces_image_generation_tool(body_json: &Value) -> bool {
body_json
.get("tool_choice")
.is_some_and(value_is_image_generation_tool)
}
fn openai_tool_choice_selects_image_generation(choice: Option<&serde_json::Value>) -> bool {
let Some(choice) = choice else {
return false;
};
if let Some(value) = choice.as_str() {
return value.trim().eq_ignore_ascii_case("image_generation");
}
let Some(object) = choice.as_object() else {
return false;
};
object
fn value_is_image_generation_tool(value: &Value) -> bool {
value
.get("type")
.and_then(serde_json::Value::as_str)
.is_some_and(|value| value.trim().eq_ignore_ascii_case("image_generation"))
|| object
.get("tool")
.and_then(|value| value.get("type"))
.and_then(serde_json::Value::as_str)
.is_some_and(|value| value.trim().eq_ignore_ascii_case("image_generation"))
|| object
.get("function")
.and_then(|value| value.get("name"))
.and_then(serde_json::Value::as_str)
.is_some_and(|value| value.trim().eq_ignore_ascii_case("image_generation"))
.and_then(Value::as_str)
.is_some_and(|tool_type| tool_type.trim().eq_ignore_ascii_case("image_generation"))
}
#[cfg(test)]
mod tests {
use super::openai_request_is_image_generation_intent;
use serde_json::json;
use super::*;
#[test]
fn detects_openai_image_generation_intent_like_compat_proxies() {
assert!(openai_request_is_image_generation_intent(
"GPT-IMAGE-2",
&json!({})
));
assert!(openai_request_is_image_generation_intent(
"gpt-5",
&json!({"model":"gpt-image-2"})
));
assert!(openai_request_is_image_generation_intent(
"gpt-5",
&json!({"tool_choice":{"function":{"name":"image_generation"}}})
));
assert!(openai_request_is_image_generation_intent(
"gpt-5",
&json!({"tool_choice":{"type":"image_generation"}})
));
fn tools_declaration_without_tool_choice_does_not_trigger_image_generation() {
let body_json = serde_json::json!({
"model": "gpt-image-2",
"input": "Draw a mountain observatory",
"tools": [{"type": "image_generation"}]
});
assert!(!openai_request_is_image_generation_intent(
"gpt-5",
&json!({"tools":[{"type":"image_generation"}]})
"gpt-image-2",
&body_json
));
assert!(!openai_request_is_image_generation_intent(
"gpt-5",
&json!({"messages":[{"role":"user","content":"hello"}]})
}
#[test]
fn explicit_image_generation_tool_choice_triggers_image_generation() {
let body_json = serde_json::json!({
"model": "gpt-image-2",
"input": "Draw a mountain observatory",
"tools": [{"type": "image_generation"}],
"tool_choice": {"type": "image_generation"}
});
assert!(openai_request_is_image_generation_intent(
"gpt-image-2",
&body_json
));
}
}

2
apps/aether-gateway/src/ai_serving/planner/standard/openai/mod.rs
View File

@@ -17,7 +17,7 @@ pub(crate) use chat::{
set_local_openai_chat_execution_exhausted_diagnostic,
};
pub(crate) use embedding::maybe_build_sync_local_openai_embedding_decision_payload;
pub(super) use image_intent::openai_request_is_image_generation_intent;
use image_intent::openai_request_is_image_generation_intent;
pub(crate) use responses::{
build_local_openai_responses_stream_attempt_source_for_kind,
build_local_openai_responses_stream_plan_and_reports_for_kind,

8
apps/aether-gateway/src/ai_serving/planner/standard/openai/responses/decision/request.rs
View File

@@ -114,7 +114,7 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts(
.trim()
.eq_ignore_ascii_case("grok");
if provider_api_format.eq_ignore_ascii_case("openai:image") {
if !is_grok && provider_api_format.eq_ignore_ascii_case("openai:image") {
return resolve_openai_responses_to_openai_image_payload_parts(
state,
parts,
@@ -739,7 +739,11 @@ async fn resolve_openai_responses_to_openai_image_payload_parts(
let upstream_url = if is_chatgpt_web {
chatgpt_web_image_internal_url(&transport.endpoint.base_url)
} else {
build_openai_image_upstream_url(transport, parts.uri.query())
build_openai_image_upstream_url(
transport,
Some("/v1/images/generations"),
parts.uri.query(),
)
};
let Some(mut provider_request_headers) =
build_openai_image_headers(ProviderOpenAiImageHeadersInput {

1
apps/aether-gateway/src/ai_serving/pure/mod.rs
View File

@@ -20,6 +20,7 @@ pub(crate) use aether_ai_formats::api::{
build_local_openai_responses_request_body,
build_local_openai_responses_request_body_with_model_directives,
build_local_success_background_report, build_local_success_conversion_background_report,
build_openai_image_api_provider_request_body,
build_openai_image_provider_body_from_response_stream_sync_body,
build_openai_image_provider_request_body,
build_openai_image_request_body_from_gemini_image_request,

224
apps/aether-gateway/src/execution_runtime/stream/execution.rs
View File

@@ -113,6 +113,7 @@ use crate::{
const OPENAI_IMAGE_STREAM_PLAN_KIND: &str = "openai_image_stream";
const SSE_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(15);
const SSE_KEEPALIVE_BYTES: &[u8] = b": aether-keepalive\n\n";
const SSE_CONTROL_FILTER_MAX_BUFFER_BYTES: usize = 1024 * 1024;
const STREAM_IDLE_LOG_INTERVAL: Duration = Duration::from_secs(60);
const STREAM_IDLE_LOG_INTERVAL_MS: u64 = 60_000;
const REWRITTEN_STREAM_PREFETCH_TIMEOUT: Duration = Duration::from_millis(750);
@@ -1279,10 +1280,13 @@ fn build_sse_body_stream(
keepalive_interval: Duration,
) -> impl futures_util::Stream<Item = Result<Bytes, IoError>> + Send + 'static {
stream! {
let mut upstream_control_filter = emit_keepalive.then(SseControlBlockFilter::default);
let mut sent_prefetched_chunk = false;
for chunk in prefetched_chunks_for_body {
sent_prefetched_chunk = true;
yield Ok(chunk);
if let Some(chunk) = filter_upstream_sse_control_chunk(&mut upstream_control_filter, chunk) {
sent_prefetched_chunk = true;
yield Ok(chunk);
}
}
if emit_keepalive {
@@ -1299,13 +1303,23 @@ fn build_sse_body_stream(
let Some(item) = item else {
break;
};
yield item;
match item {
Ok(chunk) => {
if let Some(chunk) = filter_upstream_sse_control_chunk(&mut upstream_control_filter, chunk) {
yield Ok(chunk);
}
}
Err(err) => yield Err(err),
}
}
_ = keepalive.tick() => {
yield Ok(Bytes::from_static(SSE_KEEPALIVE_BYTES));
}
}
}
if let Some(chunk) = flush_upstream_sse_control_filter(&mut upstream_control_filter) {
yield Ok(chunk);
}
} else {
while let Some(item) = rx.recv().await {
yield item;
@@ -1314,10 +1328,118 @@ fn build_sse_body_stream(
}
}
#[derive(Default)]
struct SseControlBlockFilter {
buffered: Vec<u8>,
}
impl SseControlBlockFilter {
fn push_chunk(&mut self, chunk: &[u8]) -> Vec<u8> {
if chunk.is_empty() {
return Vec::new();
}
self.buffered.extend_from_slice(chunk);
let mut output = Vec::new();
while let Some((block_end, separator_len)) = find_sse_block_boundary(&self.buffered) {
let block = self
.buffered
.drain(..block_end + separator_len)
.collect::<Vec<_>>();
if sse_block_has_data_line(&block) {
output.extend(block);
}
}
if self.buffered.len() > SSE_CONTROL_FILTER_MAX_BUFFER_BYTES {
output.extend(std::mem::take(&mut self.buffered));
}
output
}
fn finish(&mut self) -> Vec<u8> {
if self.buffered.is_empty() {
return Vec::new();
}
let block = std::mem::take(&mut self.buffered);
if sse_block_has_data_line(&block) {
block
} else {
Vec::new()
}
}
}
fn filter_upstream_sse_control_chunk(
filter: &mut Option<SseControlBlockFilter>,
chunk: Bytes,
) -> Option<Bytes> {
let Some(filter) = filter.as_mut() else {
return Some(chunk);
};
let filtered = filter.push_chunk(chunk.as_ref());
(!filtered.is_empty()).then(|| Bytes::from(filtered))
}
fn flush_upstream_sse_control_filter(filter: &mut Option<SseControlBlockFilter>) -> Option<Bytes> {
let filtered = filter.as_mut()?.finish();
(!filtered.is_empty()).then(|| Bytes::from(filtered))
}
fn find_sse_block_boundary(buffer: &[u8]) -> Option<(usize, usize)> {
let lf = buffer
.windows(2)
.position(|window| window == b"\n\n")
.map(|index| (index, 2));
let crlf = buffer
.windows(4)
.position(|window| window == b"\r\n\r\n")
.map(|index| (index, 4));
match (lf, crlf) {
(Some(lf), Some(crlf)) => Some(if lf.0 <= crlf.0 { lf } else { crlf }),
(Some(lf), None) => Some(lf),
(None, Some(crlf)) => Some(crlf),
(None, None) => None,
}
}
fn sse_block_has_data_line(block: &[u8]) -> bool {
let Ok(text) = std::str::from_utf8(block) else {
return true;
};
text.lines()
.any(|line| line.trim_start().starts_with("data:"))
}
fn stream_chunk_contains_sse_done(chunk: &[u8]) -> bool {
std::str::from_utf8(chunk)
.ok()
.is_some_and(|text| text.lines().any(|line| line.trim() == "data: [DONE]"))
std::str::from_utf8(chunk).ok().is_some_and(|text| {
text.lines().any(|line| {
let line = line.trim();
if matches!(
line,
"data: [DONE]" | "event: message_stop" | "event: response.completed"
) {
return true;
}
let Some(data) = line.strip_prefix("data:").map(str::trim) else {
return false;
};
data == "[DONE]"
|| serde_json::from_str::<serde_json::Value>(data).is_ok_and(|value| {
value
.get("type")
.and_then(serde_json::Value::as_str)
.is_some_and(|event_type| {
matches!(event_type, "message_stop" | "response.completed")
})
})
})
})
}
async fn next_stream_frame<R>(
@@ -3364,7 +3486,7 @@ mod tests {
build_sse_body_stream, execute_execution_runtime_stream, execute_stream_from_frame_stream,
maybe_apply_kiro_prompt_cache_usage_to_stream_summary, merge_stream_terminal_summary,
should_limit_direct_finalize_prefetch, should_probe_success_failover_before_stream,
should_skip_direct_finalize_prefetch,
should_skip_direct_finalize_prefetch, stream_chunk_contains_sse_done,
};
use crate::control::GatewayControlDecision;
use crate::tunnel::{tunnel_protocol, TunnelProxyConn};
@@ -3381,6 +3503,20 @@ mod tests {
.with_execution_runtime_candidate(true)
}
#[test]
fn detects_client_visible_sse_terminal_events() {
assert!(stream_chunk_contains_sse_done(b"data: [DONE]\n\n"));
assert!(stream_chunk_contains_sse_done(
b"event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"
));
assert!(stream_chunk_contains_sse_done(
b"event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{}}\n\n"
));
assert!(!stream_chunk_contains_sse_done(
b"event: content_block_delta\ndata: {\"type\":\"content_block_delta\"}\n\n"
));
}
fn tunnel_proxy_snapshot(base_url: String) -> aether_contracts::ProxySnapshot {
aether_contracts::ProxySnapshot {
enabled: Some(true),
@@ -4116,6 +4252,80 @@ mod tests {
assert_eq!(second.as_ref(), b": aether-keepalive\n\n");
}
#[tokio::test]
async fn sse_body_stream_drops_upstream_control_only_blocks() {
let (_tx, rx) = mpsc::channel::<Result<Bytes, std::io::Error>>(1);
let mut body_stream = Box::pin(build_sse_body_stream(
vec![
Bytes::from_static(b": upstream-keepalive\n\n"),
Bytes::from_static(b"event: ping\nid: 1\nretry: 1000\n\n"),
Bytes::from_static(
b"event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"delta\":\"hi\"}\n\n",
),
],
rx,
true,
Duration::from_secs(60),
));
let chunk = tokio::time::timeout(Duration::from_millis(50), body_stream.next())
.await
.expect("business chunk should arrive")
.expect("stream should yield business chunk")
.expect("business chunk should be ok");
let text = std::str::from_utf8(chunk.as_ref()).expect("chunk should be utf8");
assert!(text.contains("event: response.output_text.delta"));
assert!(text.contains("data: {\"type\":\"response.output_text.delta\""));
assert!(!text.contains("upstream-keepalive"));
assert!(!text.contains("event: ping"));
assert!(!text.contains("retry: 1000"));
}
#[tokio::test]
async fn sse_body_stream_filters_control_blocks_across_chunk_boundaries() {
let (_tx, rx) = mpsc::channel::<Result<Bytes, std::io::Error>>(1);
let mut body_stream = Box::pin(build_sse_body_stream(
vec![
Bytes::from_static(b": upstream-keepalive\n"),
Bytes::from_static(b"\n"),
Bytes::from_static(b"event: response.created\n"),
Bytes::from_static(b"data: {\"type\":\"response.created\"}\n\n"),
],
rx,
true,
Duration::from_secs(60),
));
let chunk = tokio::time::timeout(Duration::from_millis(50), body_stream.next())
.await
.expect("business chunk should arrive")
.expect("stream should yield business chunk")
.expect("business chunk should be ok");
let text = std::str::from_utf8(chunk.as_ref()).expect("chunk should be utf8");
assert_eq!(
text,
"event: response.created\ndata: {\"type\":\"response.created\"}\n\n"
);
}
#[tokio::test]
async fn sse_body_stream_uses_local_keepalive_when_prefetched_blocks_are_control_only() {
let (_tx, rx) = mpsc::channel::<Result<Bytes, std::io::Error>>(1);
let mut body_stream = Box::pin(build_sse_body_stream(
vec![Bytes::from_static(b": upstream-keepalive\n\n")],
rx,
true,
Duration::from_secs(60),
));
let first = tokio::time::timeout(Duration::from_millis(50), body_stream.next())
.await
.expect("local keepalive should arrive")
.expect("stream should yield local keepalive")
.expect("local keepalive should be ok");
assert_eq!(first.as_ref(), b": aether-keepalive\n\n");
}
#[tokio::test]
async fn openai_image_stream_total_timeout_emits_image_failed_event() {
let state = AppState::new().expect("app state should build");

73
apps/aether-gateway/src/execution_runtime/submission.rs
View File

@@ -1,9 +1,7 @@
#[cfg(test)]
use crate::ai_serving::api::core_success_background_report_kind;
use crate::ai_serving::api::{
build_core_error_body_for_client_format, core_error_background_report_kind,
core_error_default_client_api_format, is_core_error_finalize_kind,
maybe_compile_sync_finalize_response,
core_error_default_client_api_format, core_success_background_report_kind,
is_core_error_finalize_kind, maybe_compile_sync_finalize_response,
normalize_provider_private_response_value as unwrap_local_finalize_response_value,
LocalCoreSyncErrorKind,
};
@@ -36,6 +34,12 @@ pub(super) fn maybe_build_local_core_error_response(
return Ok(None);
};
let status_source_json = resolve_local_sync_source_body_json(payload)?;
if payload.status_code < 400
&& !has_nested_error(&response_body_json)
&& !status_source_json.as_ref().is_some_and(has_nested_error)
{
return Ok(None);
}
let mut response_headers = payload.headers.clone();
response_headers.remove("content-encoding");
@@ -81,8 +85,9 @@ fn build_local_sync_response_from_json(
payload: &GatewaySyncReportRequest,
body_json: serde_json::Value,
) -> Result<Response<Body>, GatewayError> {
let status_code = if is_core_error_finalize_kind(payload.report_kind.as_str())
|| has_nested_error(&body_json)
let body_has_error = has_nested_error(&body_json);
let status_code = if body_has_error
|| (payload.status_code >= 400 && is_core_error_finalize_kind(payload.report_kind.as_str()))
{
resolve_local_sync_error_status_code(payload.status_code, &body_json)
} else {
@@ -621,11 +626,30 @@ pub(crate) async fn submit_local_core_error_or_sync_finalize(
build_local_core_sync_finalize_fallback_response(trace_id, decision, &payload)?
};
if let Some(error_report_kind) =
let response_status = response.status();
if response_status.is_success() {
if let Some(success_report_kind) =
core_success_background_report_kind(payload.report_kind.as_str())
{
let mut report_payload = payload.clone();
report_payload.report_kind = success_report_kind.to_string();
report_payload.status_code = response_status.as_u16();
spawn_sync_report(state.clone(), report_payload);
} else {
warn!(
event_name = "local_core_finalize_missing_success_report_mapping",
log_type = "event",
trace_id = %trace_id,
report_kind = %payload.report_kind,
"gateway built local core finalize success response without background success report mapping"
);
}
} else if let Some(error_report_kind) =
resolve_core_error_background_report_kind(payload.report_kind.as_str())
{
let mut report_payload = payload.clone();
report_payload.report_kind = error_report_kind;
report_payload.status_code = response_status.as_u16();
spawn_sync_report(state.clone(), report_payload);
} else {
warn!(
@@ -819,4 +843,39 @@ mod tests {
"unexpected message: {message}"
);
}
#[tokio::test]
async fn submit_local_core_finalize_keeps_http_200_for_success_image_body() {
let payload = core_finalize_payload(
"openai_image_sync_finalize",
"openai:image",
"openai:image",
200,
json!({
"created": 1779273523,
"data": [{
"b64_json": "aGVsbG8="
}]
}),
);
let state = AppState::new().expect("state should build");
let response = submit_local_core_error_or_sync_finalize(
&state,
"trace-image-success-200",
&test_decision(),
payload,
)
.await
.expect("response should build");
assert_eq!(response.status(), http::StatusCode::OK);
let body: serde_json::Value = serde_json::from_slice(
&to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read"),
)
.expect("body should decode");
assert_eq!(body["data"][0]["b64_json"], "aGVsbG8=");
}
}

53
apps/aether-gateway/src/handlers/admin/provider/query/models/model_test.rs
View File

@@ -1486,6 +1486,17 @@ fn provider_query_decode_execution_body(
.and_then(|value| base64::engine::general_purpose::STANDARD.decode(value).ok())
}
fn provider_query_execution_json_body(result: &aether_contracts::ExecutionResult) -> Option<Value> {
result
.body
.as_ref()
.and_then(|body| body.json_body.clone())
.or_else(|| {
provider_query_decode_execution_body(result)
.and_then(|body| serde_json::from_slice::<Value>(&body).ok())
})
}
fn provider_query_aggregate_standard_stream_sync_response(
provider_api_format: &str,
body: &[u8],
@@ -1505,10 +1516,7 @@ fn provider_query_standard_execution_response_body(
provider_api_format: &str,
result: &aether_contracts::ExecutionResult,
) -> Option<Value> {
let body = result
.body
.as_ref()
.and_then(|body| body.json_body.clone())
let body = provider_query_execution_json_body(result)
.or_else(|| {
provider_query_decode_execution_body(result).and_then(|body| {
provider_query_aggregate_standard_stream_sync_response(provider_api_format, &body)
@@ -1527,10 +1535,8 @@ fn provider_query_standard_execution_response_body(
fn provider_query_extract_error_message(
result: &aether_contracts::ExecutionResult,
) -> Option<String> {
result
.body
provider_query_execution_json_body(result)
.as_ref()
.and_then(|body| body.json_body.as_ref())
.and_then(Value::as_object)
.and_then(|value| {
value
@@ -1589,7 +1595,7 @@ async fn provider_query_finalize_kiro_result(
})),
status_code: result.status_code,
headers: result.headers.clone(),
body_json: result.body.as_ref().and_then(|body| body.json_body.clone()),
body_json: provider_query_execution_json_body(result),
client_body_json: None,
body_base64: result
.body
@@ -1821,6 +1827,7 @@ fn provider_query_chatgpt_web_image_internal_url(base_url: &str) -> String {
fn provider_query_openai_image_test_upstream_url(
transport: &AdminGatewayProviderTransportSnapshot,
request_path: Option<&str>,
request_query: Option<&str>,
) -> String {
if transport
@@ -1841,7 +1848,11 @@ fn provider_query_openai_image_test_upstream_url(
crate::provider_transport::GROK_CHAT_PATH,
)
} else {
crate::provider_transport::build_openai_image_upstream_url(transport, request_query)
crate::provider_transport::build_openai_image_upstream_url(
transport,
request_path,
request_query,
)
}
}
@@ -1874,7 +1885,7 @@ async fn provider_query_finalize_openai_image_result(
})),
status_code: result.status_code,
headers: result.headers.clone(),
body_json: result.body.as_ref().and_then(|body| body.json_body.clone()),
body_json: provider_query_execution_json_body(result),
client_body_json: None,
body_base64: result
.body
@@ -1967,13 +1978,23 @@ async fn provider_query_execute_openai_image_test_candidate(
.provider_type
.trim()
.eq_ignore_ascii_case("grok");
let is_codex = transport
.provider
.provider_type
.trim()
.eq_ignore_ascii_case("codex");
let mut provider_request_body = if is_chatgpt_web {
match crate::ai_serving::build_chatgpt_web_image_request_body(&parts, &request_body, None) {
Ok(body) => body,
Err(err) => err.to_error_json(),
}
} else {
} else if is_codex || is_grok {
crate::ai_serving::build_openai_image_provider_request_body(&normalized_request)
} else {
crate::ai_serving::build_openai_image_api_provider_request_body(
&normalized_request,
Some(candidate.effective_model.as_str()),
)
};
if !is_chatgpt_web {
crate::ai_serving::apply_codex_openai_responses_special_body_edits(
@@ -2075,7 +2096,11 @@ async fn provider_query_execute_openai_image_test_candidate(
} else {
normalized_request.summary_json.clone()
};
let request_url = provider_query_openai_image_test_upstream_url(&transport, parts.uri.query());
let request_url = provider_query_openai_image_test_upstream_url(
&transport,
Some(parts.uri.path()),
parts.uri.query(),
);
let upstream_is_stream = provider_request_body
.get("stream")
.and_then(Value::as_bool)
@@ -2158,9 +2183,9 @@ async fn provider_query_execute_openai_image_test_candidate(
&result,
)
.await?
.or_else(|| result.body.as_ref().and_then(|body| body.json_body.clone()))
.or_else(|| provider_query_execution_json_body(&result))
} else {
result.body.as_ref().and_then(|body| body.json_body.clone())
provider_query_execution_json_body(&result)
};
let did_fail = result.status_code >= 400;
let error_message = if did_fail {

62
apps/aether-gateway/src/handlers/admin/provider/query/models/model_test/tests.rs
View File

@@ -90,6 +90,42 @@ fn provider_query_test_request_body_defaults_missing_model() {
assert_eq!(body["model"], json!("fallback-model"));
}
#[test]
fn provider_query_execution_json_body_decodes_stream_encoded_json_response() {
use base64::Engine as _;
let body = json!({
"created": 1,
"data": [{
"url": "https://example.test/image.png"
}]
});
let encoded_body = base64::engine::general_purpose::STANDARD.encode(
serde_json::to_vec(&body).expect("test body should serialize"),
);
let result = aether_contracts::ExecutionResult {
request_id: "request-1".to_string(),
candidate_id: None,
status_code: 200,
headers: std::collections::BTreeMap::from([(
"content-type".to_string(),
"application/json".to_string(),
)]),
body: Some(aether_contracts::ResponseBody {
json_body: None,
body_bytes_b64: Some(encoded_body),
}),
telemetry: None,
error: None,
};
assert_eq!(provider_query_execution_json_body(&result), Some(body.clone()));
assert_eq!(
provider_query_standard_execution_response_body("openai:image", &result),
Some(body)
);
}
#[test]
fn provider_query_test_request_body_fills_empty_conversation() {
let payload = json!({
@@ -850,17 +886,39 @@ fn provider_query_grok_image_test_uses_grok_app_chat_upstream_url() {
let transport = sample_openai_image_transport("grok");
assert_eq!(
provider_query_openai_image_test_upstream_url(&transport, Some("trace=1")),
provider_query_openai_image_test_upstream_url(
&transport,
Some("/v1/images/generations"),
Some("trace=1"),
),
"https://grok.com/rest/app-chat/conversations/new"
);
}
#[test]
fn provider_query_custom_image_test_uses_images_upstream_url() {
let transport = sample_openai_image_transport("custom");
assert_eq!(
provider_query_openai_image_test_upstream_url(
&transport,
Some("/v1/images/generations"),
Some("trace=1"),
),
"https://grok.com/v1/images/generations?trace=1"
);
}
#[test]
fn provider_query_chatgpt_web_image_test_uses_internal_upstream_url() {
let transport = sample_openai_image_transport("chatgpt_web");
assert_eq!(
provider_query_openai_image_test_upstream_url(&transport, Some("trace=1")),
provider_query_openai_image_test_upstream_url(
&transport,
Some("/v1/images/generations"),
Some("trace=1"),
),
"https://grok.com/__aether/chatgpt-web-image"
);
}

79
apps/aether-gateway/src/tests/ai_execute/stream/image.rs
View File

@@ -13,7 +13,6 @@ use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadReposi
use aether_data_contracts::repository::candidate_selection::{
StoredMinimalCandidateSelectionRow, StoredProviderModelMapping,
};
use aether_data_contracts::repository::candidates::RequestCandidateReadRepository;
use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
};
@@ -751,7 +750,6 @@ struct SeenImageBridgeExecutionPlan {
url: String,
plan_stream: bool,
auth_header: String,
chatgpt_web_marker: String,
body_json: serde_json::Value,
}
@@ -1017,12 +1015,6 @@ fn capture_image_bridge_execution_plan(
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
chatgpt_web_marker: payload
.get("headers")
.and_then(|value| value.get("x-aether-chatgpt-web-image"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
body_json: payload
.get("body")
.and_then(|value| value.get("json_body"))
@@ -1065,75 +1057,6 @@ fn image_bridge_execution_runtime(
)
}
#[tokio::test]
async fn gateway_routes_openai_chat_stream_image_intent_to_openai_image_plan_without_streaming_support(
) {
let seen_execution_plan = Arc::new(Mutex::new(None::<SeenImageBridgeExecutionPlan>));
let execution_runtime = image_bridge_execution_runtime(Arc::clone(&seen_execution_plan));
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let (gateway_url, gateway_handle, client_api_key, request_candidate_repository) =
start_image_bridge_gateway(
"chat-stream-image-bridge",
"image-provider",
"custom",
"https://images.example.com",
execution_runtime_url,
)
.await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/chat/completions"))
.header(http::header::CONTENT_TYPE, "application/json")
.header(http::header::AUTHORIZATION, format!("Bearer {client_api_key}"))
.header(TRACE_ID_HEADER, "trace-chat-stream-image-bridge-123")
.body(
r#"{"model":"gpt-image-2","messages":[{"role":"user","content":"Draw a city made of glass"}],"stream":true,"size":"1024x1024"}"#,
)
.send()
.await
.expect("request should succeed");
let status = response.status();
let response_text = response.text().await.expect("body should read");
let stored_candidates = request_candidate_repository
.list_by_request_id("trace-chat-stream-image-bridge-123")
.await
.expect("request candidates should read");
assert_eq!(
status,
StatusCode::OK,
"{response_text}\n{stored_candidates:#?}"
);
assert!(response_text.contains("\"object\":\"chat.completion.chunk\""));
assert!(response_text.contains("![generated image](data:image/png;base64,aGVsbG8=)"));
assert!(response_text.contains("data: [DONE]"));
assert!(!response_text.contains("image_generation.completed"));
let seen_plan = seen_execution_plan
.lock()
.expect("mutex should lock")
.clone()
.expect("execution plan should be captured");
assert_eq!(seen_plan.trace_id, "trace-chat-stream-image-bridge-123");
assert_eq!(seen_plan.client_api_format, "openai:chat");
assert_eq!(seen_plan.provider_api_format, "openai:image");
assert_eq!(seen_plan.url, "https://images.example.com/v1/responses");
assert!(seen_plan.plan_stream);
assert_eq!(seen_plan.auth_header, "Bearer sk-upstream-image-bridge");
assert_eq!(seen_plan.chatgpt_web_marker, "");
assert_eq!(seen_plan.body_json["model"], "gpt-image-2");
assert_eq!(seen_plan.body_json["stream"], true);
assert_eq!(
seen_plan.body_json["input"][0]["content"],
"Draw a city made of glass"
);
assert_eq!(seen_plan.body_json["tools"][0]["type"], "image_generation");
assert_eq!(seen_plan.body_json["tools"][0]["size"], "1024x1024");
gateway_handle.abort();
execution_runtime_handle.abort();
}
#[tokio::test]
async fn gateway_routes_openai_responses_stream_image_intent_to_openai_image_plan_without_streaming_support(
) {
@@ -1156,7 +1079,7 @@ async fn gateway_routes_openai_responses_stream_image_intent_to_openai_image_pla
.header(http::header::AUTHORIZATION, format!("Bearer {client_api_key}"))
.header(TRACE_ID_HEADER, "trace-responses-stream-image-bridge-123")
.body(
r#"{"model":"gpt-image-2","input":"Draw a mountain observatory","tools":[{"type":"image_generation","size":"1024x1024"}],"stream":true}"#,
r#"{"model":"gpt-image-2","input":"Draw a mountain observatory","tools":[{"type":"image_generation","size":"1024x1024"}],"tool_choice":{"type":"image_generation"},"stream":true}"#,
)
.send()
.await

13
crates/aether-ai-formats/src/api.rs
View File

@@ -185,12 +185,13 @@ pub use crate::formats::{
},
openai::image::{
request::{
build_chatgpt_web_image_request_body, build_openai_image_provider_request_body,
default_model_for_openai_image_operation, is_openai_image_stream_request,
normalize_openai_image_request, normalize_openai_image_request_with_options,
openai_image_operation_from_path, resolve_requested_openai_image_model_for_request,
ChatGptWebImageRequestError, NormalizedOpenAiImageRequest, OpenAiImageNormalizeOptions,
OpenAiImageOperation, OpenAiImageResponseFormat,
build_chatgpt_web_image_request_body, build_openai_image_api_provider_request_body,
build_openai_image_provider_request_body, default_model_for_openai_image_operation,
is_openai_image_stream_request, normalize_openai_image_request,
normalize_openai_image_request_with_options, openai_image_operation_from_path,
resolve_requested_openai_image_model_for_request, ChatGptWebImageRequestError,
NormalizedOpenAiImageRequest, OpenAiImageNormalizeOptions, OpenAiImageOperation,
OpenAiImageResponseFormat,
},
spec::{
resolve_stream_spec as resolve_local_image_stream_spec,

34
crates/aether-ai-formats/src/formats/claude/messages/stream.rs
View File

@@ -753,13 +753,7 @@ impl ClaudeClientEmitter {
}),
);
let usage = usage.unwrap_or_default();
payload.insert(
"usage".to_string(),
json!({
"input_tokens": usage.input_tokens,
"output_tokens": usage.output_tokens,
}),
);
payload.insert("usage".to_string(), claude_usage_from_usage(&usage));
out.extend(encode_json_sse(
Some("message_delta"),
&Value::Object(payload),
@@ -1289,6 +1283,32 @@ mod tests {
assert!(sse.contains("\"usage\":{\"input_tokens\":0,\"output_tokens\":0}"));
}
#[test]
fn claude_client_emitter_includes_cache_usage_in_finish_events() {
let mut emitter = ClaudeClientEmitter::default();
let bytes = emitter
.emit(CanonicalStreamFrame {
id: "msg_cache".to_string(),
model: "claude-sonnet-4-5".to_string(),
event: CanonicalStreamEvent::Finish {
finish_reason: Some("stop".to_string()),
usage: Some(CanonicalUsage {
input_tokens: 10,
output_tokens: 2,
total_tokens: 12,
cache_creation_tokens: 5,
cache_read_tokens: 4,
..CanonicalUsage::default()
}),
},
})
.expect("finish should encode");
let sse = String::from_utf8(bytes).expect("sse should be utf8");
assert!(sse.contains("\"cache_creation_input_tokens\":5"));
assert!(sse.contains("\"cache_read_input_tokens\":4"));
}
#[test]
fn claude_client_emitter_emits_image_blocks_for_media_parts() {
let mut emitter = ClaudeClientEmitter::default();

25
crates/aether-ai-formats/src/formats/gemini/generate_content/stream.rs
View File

@@ -404,27 +404,10 @@ impl GeminiClientEmitter {
Value::Array(vec![Value::Object(candidate)]),
);
if let Some(usage) = usage {
let visible_output_tokens = usage.output_tokens.saturating_sub(usage.reasoning_tokens);
let mut usage_metadata = Map::new();
usage_metadata.insert(
"promptTokenCount".to_string(),
Value::from(usage.input_tokens),
response.insert(
"usageMetadata".to_string(),
gemini_usage_metadata_from_usage(&usage),
);
usage_metadata.insert(
"candidatesTokenCount".to_string(),
Value::from(visible_output_tokens),
);
usage_metadata.insert(
"totalTokenCount".to_string(),
Value::from(usage.total_tokens),
);
if usage.reasoning_tokens > 0 {
usage_metadata.insert(
"thoughtsTokenCount".to_string(),
Value::from(usage.reasoning_tokens),
);
}
response.insert("usageMetadata".to_string(), Value::Object(usage_metadata));
}
encode_json_sse(None, &Value::Object(response))
}
@@ -1000,6 +983,7 @@ mod tests {
output_tokens: 3,
reasoning_tokens: 1,
total_tokens: 4,
cache_read_tokens: 5,
..CanonicalUsage::default()
}),
},
@@ -1012,6 +996,7 @@ mod tests {
assert!(sse.contains("\"thoughtSignature\":\"sig_123\""));
assert!(sse.contains("\"thoughtsTokenCount\":1"));
assert!(sse.contains("\"candidatesTokenCount\":2"));
assert!(sse.contains("\"cachedContentTokenCount\":5"));
assert!(sse.contains("\"finishReason\":\"STOP\""));
}

32
crates/aether-ai-formats/src/formats/openai/chat/stream.rs
View File

@@ -1566,15 +1566,12 @@ impl OpenAIChatClientEmitter {
if let Some(usage) = usage {
out.extend(encode_json_sse(
None,
&build_openai_chat_usage_chunk(
&build_openai_chat_usage_chunk_from_usage(
self.response_id
.as_deref()
.unwrap_or("chatcmpl-local-stream"),
self.model.as_deref().unwrap_or("unknown"),
usage.input_tokens,
usage.output_tokens,
usage.total_tokens,
usage.reasoning_tokens,
&usage,
),
)?);
}
@@ -2202,20 +2199,6 @@ impl OpenAIResponsesClientEmitter {
}
ordered_output.sort_by_key(|(output_index, _)| *output_index);
let mut usage_payload = Map::new();
usage_payload.insert("input_tokens".to_string(), Value::from(usage.input_tokens));
usage_payload.insert(
"output_tokens".to_string(),
Value::from(usage.output_tokens),
);
usage_payload.insert("total_tokens".to_string(), Value::from(usage.total_tokens));
if usage.reasoning_tokens > 0 {
usage_payload.insert(
"output_tokens_details".to_string(),
json!({ "reasoning_tokens": usage.reasoning_tokens }),
);
}
json!({
"id": self.response_id(),
"object": "response",
@@ -2225,7 +2208,7 @@ impl OpenAIResponsesClientEmitter {
.into_iter()
.map(|(_, item)| item)
.collect::<Vec<_>>(),
"usage": usage_payload,
"usage": openai_responses_usage_from_usage(&usage),
})
}
@@ -3297,6 +3280,8 @@ mod tests {
input_tokens: 1,
output_tokens: 2,
total_tokens: 3,
cache_creation_tokens: 5,
cache_read_tokens: 4,
reasoning_tokens: 1,
..CanonicalUsage::default()
}),
@@ -3311,6 +3296,8 @@ mod tests {
assert!(sse.contains("\"prompt_tokens\":1"));
assert!(sse.contains("\"completion_tokens\":2"));
assert!(sse.contains("\"completion_tokens_details\":{\"reasoning_tokens\":1}"));
assert!(sse.contains("\"cached_creation_tokens\":5"));
assert!(sse.contains("\"cached_tokens\":4"));
assert!(sse.contains("\"total_tokens\":3"));
assert!(sse.contains("data: [DONE]\n\n"));
}
@@ -3418,6 +3405,8 @@ mod tests {
input_tokens: 1,
output_tokens: 2,
total_tokens: 3,
cache_creation_tokens: 5,
cache_read_tokens: 4,
reasoning_tokens: 1,
..CanonicalUsage::default()
}),
@@ -3430,6 +3419,9 @@ mod tests {
assert!(sse.contains("\"type\":\"reasoning\""));
assert!(sse.contains("\"text\":\"because\""));
assert!(sse.contains("\"output_tokens_details\":{\"reasoning_tokens\":1}"));
assert!(sse.contains("\"input_tokens_details\""));
assert!(sse.contains("\"cached_creation_tokens\":5"));
assert!(sse.contains("\"cached_tokens\":4"));
}
#[test]

84
crates/aether-ai-formats/src/formats/openai/image/request.rs
View File

@@ -44,6 +44,7 @@ pub struct NormalizedOpenAiImageRequest {
images: Vec<Value>,
tool: Map<String, Value>,
image_count: Option<u64>,
stream: Option<bool>,
user: Option<String>,
}
@@ -521,6 +522,54 @@ pub fn build_openai_image_provider_request_body(request: &NormalizedOpenAiImageR
Value::Object(body)
}
pub fn build_openai_image_api_provider_request_body(
request: &NormalizedOpenAiImageRequest,
mapped_model: Option<&str>,
) -> Value {
let model = mapped_model
.map(str::trim)
.filter(|value| !value.is_empty())
.or(request.requested_model.as_deref())
.unwrap_or_else(|| default_model_for_openai_image_operation(request.operation));
let mut body = Map::new();
body.insert("model".to_string(), Value::String(model.to_string()));
if let Some(prompt) = request.prompt.as_ref() {
body.insert("prompt".to_string(), Value::String(prompt.clone()));
}
if let Some(image_count) = request.image_count {
body.insert("n".to_string(), Value::Number(Number::from(image_count)));
}
if let Some(user) = request.user.as_ref() {
body.insert("user".to_string(), Value::String(user.clone()));
}
if let Some(stream) = request.stream {
body.insert("stream".to_string(), Value::Bool(stream));
}
for (key, value) in &request.tool {
match key.as_str() {
"type" | "action" => {}
"input_image_mask" => {
body.insert("mask".to_string(), value.clone());
}
_ => {
body.insert(key.clone(), value.clone());
}
}
}
if let Some(response_format) = request.summary_json.get("response_format") {
body.entry("response_format".to_string())
.or_insert_with(|| response_format.clone());
}
if !request.images.is_empty() {
if request.images.len() == 1 {
body.insert("image".to_string(), request.images[0].clone());
} else {
body.insert("images".to_string(), Value::Array(request.images.clone()));
}
}
Value::Object(body)
}
fn normalize_openai_image_json_request(
body_json: &Value,
operation: OpenAiImageOperation,
@@ -549,6 +598,7 @@ fn normalize_openai_image_json_request(
let output_format =
normalize_output_format(object.get("output_format").and_then(Value::as_str))?;
let partial_images = normalize_partial_images(object.get("partial_images"))?;
let stream = object.get("stream").and_then(value_as_bool);
let user = object
.get("user")
.and_then(Value::as_str)
@@ -579,6 +629,7 @@ fn normalize_openai_image_json_request(
images,
tool,
image_count,
stream,
user,
summary_json: build_image_request_summary_json(
operation,
@@ -627,6 +678,9 @@ fn normalize_openai_image_multipart_request(
.map(Value::String)
.as_ref(),
)?;
let stream = find_multipart_text_field(&multipart_fields, "stream")
.as_deref()
.and_then(parse_bool_string);
let user = find_multipart_text_field(&multipart_fields, "user")
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty());
@@ -679,6 +733,7 @@ fn normalize_openai_image_multipart_request(
images,
tool,
image_count,
stream,
user,
summary_json: build_image_request_summary_json(
operation,
@@ -1478,6 +1533,35 @@ mod tests {
);
}
#[test]
fn build_image_api_provider_request_body_keeps_images_api_shape() {
let parts = request_parts("/v1/images/generations", Some("application/json"));
let request = normalize_openai_image_request_with_options(
&parts,
&json!({
"model": "grok-imagine-image-lite",
"prompt": "draw a cat",
"n": 1,
"size": "1024x1024",
"stream": true
}),
None,
OpenAiImageNormalizeOptions::with_max_generation_count(4),
)
.expect("generation request should normalize");
let provider_request_body =
build_openai_image_api_provider_request_body(&request, Some("mapped-image-model"));
assert_eq!(provider_request_body["model"], "mapped-image-model");
assert_eq!(provider_request_body["prompt"], "draw a cat");
assert_eq!(provider_request_body["n"], 1);
assert_eq!(provider_request_body["size"], "1024x1024");
assert_eq!(provider_request_body["stream"], true);
assert!(provider_request_body.get("input").is_none());
assert!(provider_request_body.get("tools").is_none());
}
#[test]
fn chatgpt_web_accepts_1k_tier_and_1024_size() {
let parts = request_parts("/v1/images/generations", Some("application/json"));

108
crates/aether-ai-formats/src/formats/openai/image/stream.rs
View File

@@ -8,7 +8,8 @@ use crate::contracts::OPENAI_IMAGE_SYNC_FINALIZE_REPORT_KIND;
use crate::formats::openai::responses::codex::CODEX_OPENAI_IMAGE_DEFAULT_OUTPUT_FORMAT;
use crate::formats::shared::sse::{encode_done_sse, encode_json_sse};
use crate::formats::shared::stream_core::common::{
build_openai_chat_chunk, build_openai_chat_finish_chunk, build_openai_chat_usage_chunk,
build_openai_chat_chunk, build_openai_chat_finish_chunk,
build_openai_chat_usage_chunk_with_cache,
};
use crate::formats::shared::AiSurfaceFinalizeError;
@@ -501,18 +502,26 @@ impl OpenAiImageChatStreamState {
None,
&build_openai_chat_finish_chunk(&response_id, &model, Some("stop")),
)?);
if let Some((input_tokens, output_tokens, total_tokens, reasoning_tokens)) =
openai_image_chat_usage_counts(usage)
if let Some((
input_tokens,
output_tokens,
total_tokens,
reasoning_tokens,
cache_creation_tokens,
cache_read_tokens,
)) = openai_image_chat_usage_counts(usage)
{
output.extend(encode_json_sse(
None,
&build_openai_chat_usage_chunk(
&build_openai_chat_usage_chunk_with_cache(
&response_id,
&model,
input_tokens,
output_tokens,
total_tokens,
reasoning_tokens,
cache_creation_tokens,
cache_read_tokens,
),
)?);
}
@@ -900,7 +909,7 @@ fn image_chat_markdown(frame: &OpenAiImageChatFrame) -> String {
)
}
fn openai_image_chat_usage_counts(usage: Option<&Value>) -> Option<(u64, u64, u64, u64)> {
fn openai_image_chat_usage_counts(usage: Option<&Value>) -> Option<(u64, u64, u64, u64, u64, u64)> {
let usage = usage.and_then(Value::as_object)?;
let mut input_tokens = usage
.get("input_tokens")
@@ -912,6 +921,30 @@ fn openai_image_chat_usage_counts(usage: Option<&Value>) -> Option<(u64, u64, u6
.or_else(|| usage.get("completion_tokens"))
.and_then(Value::as_u64)
.unwrap_or(0);
let cache_creation_tokens = usage
.get("cache_creation_input_tokens")
.and_then(Value::as_u64)
.or_else(|| {
usage
.get("input_tokens_details")
.or_else(|| usage.get("prompt_tokens_details"))
.and_then(Value::as_object)
.and_then(|details| details.get("cached_creation_tokens"))
.and_then(Value::as_u64)
})
.unwrap_or(0);
let cache_read_tokens = usage
.get("cache_read_input_tokens")
.and_then(Value::as_u64)
.or_else(|| {
usage
.get("input_tokens_details")
.or_else(|| usage.get("prompt_tokens_details"))
.and_then(Value::as_object)
.and_then(|details| details.get("cached_tokens"))
.and_then(Value::as_u64)
})
.unwrap_or(0);
let total_tokens = usage
.get("total_tokens")
.and_then(Value::as_u64)
@@ -919,7 +952,14 @@ fn openai_image_chat_usage_counts(usage: Option<&Value>) -> Option<(u64, u64, u6
if input_tokens == 0 && total_tokens > output_tokens {
input_tokens = total_tokens.saturating_sub(output_tokens);
}
(total_tokens > 0).then_some((input_tokens, output_tokens, total_tokens, 0))
(total_tokens > 0).then_some((
input_tokens,
output_tokens,
total_tokens,
0,
cache_creation_tokens,
cache_read_tokens,
))
}
fn image_failure_error(event: &Value) -> Value {
@@ -1146,6 +1186,12 @@ pub fn maybe_build_openai_image_sync_finalize_product(
return Ok(None);
}
if let Some(provider_body_json) = body_json {
if openai_image_response_has_standard_data(provider_body_json) {
return Ok(Some(OpenAiImageSyncFinalizeProduct {
client_body_json: provider_body_json.clone(),
provider_body_json: provider_body_json.clone(),
}));
}
if provider_body_json.get("output").is_some() && provider_body_json.get("data").is_none() {
let Some(client_body_json) = crate::formats::shared::image_bridge::build_openai_image_response_from_response_stream_sync_body(
provider_body_json,
@@ -1279,6 +1325,25 @@ pub fn maybe_build_openai_image_sync_finalize_product(
}))
}
fn openai_image_response_has_standard_data(body_json: &Value) -> bool {
body_json
.get("data")
.and_then(Value::as_array)
.is_some_and(|items| {
items.iter().any(|item| {
item.as_object().is_some_and(|object| {
["b64_json", "url"].iter().any(|field| {
object
.get(*field)
.and_then(Value::as_str)
.map(str::trim)
.is_some_and(|value| !value.is_empty())
})
})
})
})
}
#[cfg(test)]
mod tests {
use base64::Engine as _;
@@ -1487,4 +1552,35 @@ mod tests {
"revised history prompt"
);
}
#[test]
fn sync_finalize_accepts_standard_openai_image_response() {
let provider_body = json!({
"created": 1779273523,
"data": [{
"b64_json": "aGVsbG8=",
"revised_prompt": "draw a small cat"
}]
});
let product = maybe_build_openai_image_sync_finalize_product(
"openai_image_sync_finalize",
200,
Some(&json!({
"client_api_format": "openai:image",
"provider_api_format": "openai:image",
"image_request": {
"operation": "generate",
"response_format": "b64_json"
}
})),
Some(&provider_body),
None,
)
.expect("standard image response should finalize")
.expect("standard image response should match");
assert_eq!(product.client_body_json["created"], 1779273523);
assert_eq!(product.client_body_json["data"][0]["b64_json"], "aGVsbG8=");
assert_eq!(product.provider_body_json, provider_body);
}
}

196
crates/aether-ai-formats/src/formats/shared/stream_core/common.rs
View File

@@ -344,6 +344,156 @@ pub fn build_openai_chat_usage_chunk(
total_tokens: u64,
reasoning_tokens: u64,
) -> Value {
build_openai_chat_usage_chunk_with_cache(
id,
model,
prompt_tokens,
completion_tokens,
total_tokens,
reasoning_tokens,
0,
0,
)
}
pub fn build_openai_chat_usage_chunk_with_cache(
id: &str,
model: &str,
prompt_tokens: u64,
completion_tokens: u64,
total_tokens: u64,
reasoning_tokens: u64,
cache_creation_tokens: u64,
cache_read_tokens: u64,
) -> Value {
let usage = openai_chat_usage_payload(
prompt_tokens,
completion_tokens,
total_tokens,
reasoning_tokens,
cache_creation_tokens,
cache_read_tokens,
);
json!({
"id": id,
"object": "chat.completion.chunk",
"model": model,
"choices": [],
"usage": usage,
})
}
pub fn build_openai_chat_usage_chunk_from_usage(
id: &str,
model: &str,
usage: &CanonicalUsage,
) -> Value {
build_openai_chat_usage_chunk_with_cache(
id,
model,
usage.input_tokens,
usage.output_tokens,
usage.total_tokens,
usage.reasoning_tokens,
cache_creation_tokens_for_usage(usage),
usage.cache_read_tokens,
)
}
pub fn openai_responses_usage_from_usage(usage: &CanonicalUsage) -> Value {
let mut output = Map::new();
output.insert("input_tokens".to_string(), Value::from(usage.input_tokens));
output.insert(
"output_tokens".to_string(),
Value::from(usage.output_tokens),
);
output.insert("total_tokens".to_string(), Value::from(usage.total_tokens));
if usage.reasoning_tokens > 0 {
output.insert(
"output_tokens_details".to_string(),
json!({ "reasoning_tokens": usage.reasoning_tokens }),
);
}
insert_openai_token_details(
&mut output,
"input_tokens_details",
cache_creation_tokens_for_usage(usage),
usage.cache_read_tokens,
);
Value::Object(output)
}
pub fn claude_usage_from_usage(usage: &CanonicalUsage) -> Value {
let mut output = Map::new();
output.insert("input_tokens".to_string(), Value::from(usage.input_tokens));
output.insert(
"output_tokens".to_string(),
Value::from(usage.output_tokens),
);
if usage.cache_read_tokens > 0 {
output.insert(
"cache_read_input_tokens".to_string(),
Value::from(usage.cache_read_tokens),
);
}
let cache_creation_tokens = cache_creation_tokens_for_usage(usage);
if cache_creation_tokens > 0 {
output.insert(
"cache_creation_input_tokens".to_string(),
Value::from(cache_creation_tokens),
);
}
if usage.cache_creation_ephemeral_5m_tokens > 0 || usage.cache_creation_ephemeral_1h_tokens > 0
{
output.insert(
"cache_creation".to_string(),
json!({
"ephemeral_5m_input_tokens": usage.cache_creation_ephemeral_5m_tokens,
"ephemeral_1h_input_tokens": usage.cache_creation_ephemeral_1h_tokens,
}),
);
}
Value::Object(output)
}
pub fn gemini_usage_metadata_from_usage(usage: &CanonicalUsage) -> Value {
let visible_output_tokens = usage.output_tokens.saturating_sub(usage.reasoning_tokens);
let mut output = Map::new();
output.insert(
"promptTokenCount".to_string(),
Value::from(usage.input_tokens),
);
output.insert(
"candidatesTokenCount".to_string(),
Value::from(visible_output_tokens),
);
output.insert(
"totalTokenCount".to_string(),
Value::from(usage.total_tokens),
);
if usage.reasoning_tokens > 0 {
output.insert(
"thoughtsTokenCount".to_string(),
Value::from(usage.reasoning_tokens),
);
}
if usage.cache_read_tokens > 0 {
output.insert(
"cachedContentTokenCount".to_string(),
Value::from(usage.cache_read_tokens),
);
}
Value::Object(output)
}
fn openai_chat_usage_payload(
prompt_tokens: u64,
completion_tokens: u64,
total_tokens: u64,
reasoning_tokens: u64,
cache_creation_tokens: u64,
cache_read_tokens: u64,
) -> Map<String, Value> {
let mut usage = Map::new();
usage.insert("prompt_tokens".to_string(), Value::from(prompt_tokens));
usage.insert(
@@ -357,11 +507,43 @@ pub fn build_openai_chat_usage_chunk(
json!({ "reasoning_tokens": reasoning_tokens }),
);
}
json!({
"id": id,
"object": "chat.completion.chunk",
"model": model,
"choices": [],
"usage": usage,
})
insert_openai_token_details(
&mut usage,
"prompt_tokens_details",
cache_creation_tokens,
cache_read_tokens,
);
usage
}
fn insert_openai_token_details(
output: &mut Map<String, Value>,
details_key: &str,
cache_creation_tokens: u64,
cache_read_tokens: u64,
) {
if cache_creation_tokens == 0 && cache_read_tokens == 0 {
return;
}
let mut details = Map::new();
if cache_read_tokens > 0 {
details.insert("cached_tokens".to_string(), Value::from(cache_read_tokens));
}
if cache_creation_tokens > 0 {
details.insert(
"cached_creation_tokens".to_string(),
Value::from(cache_creation_tokens),
);
}
output.insert(details_key.to_string(), Value::Object(details));
}
fn cache_creation_tokens_for_usage(usage: &CanonicalUsage) -> u64 {
if usage.cache_creation_tokens > 0 {
usage.cache_creation_tokens
} else {
usage
.cache_creation_ephemeral_5m_tokens
.saturating_add(usage.cache_creation_ephemeral_1h_tokens)
}
}

297
crates/aether-ai-formats/src/formats/shared/sync_to_stream.rs
View File

@@ -14,9 +14,12 @@ use crate::formats::openai::chat::stream::{
};
use crate::formats::shared::sse::{encode_done_sse, encode_json_sse};
use crate::formats::shared::stream_core::common::{
build_openai_chat_chunk, build_openai_chat_finish_chunk, build_openai_chat_usage_chunk,
build_openai_chat_chunk, build_openai_chat_finish_chunk,
build_openai_chat_usage_chunk_with_cache,
};
use crate::formats::shared::stream_core::{
CanonicalStreamFrame, StreamingStandardFormatMatrix, StreamingStandardTerminalObserver,
};
use crate::formats::shared::stream_core::CanonicalStreamFrame;
use crate::formats::shared::AiSurfaceFinalizeError;
pub struct SyncToStreamBridgeOutcome {
@@ -32,6 +35,14 @@ pub fn maybe_bridge_standard_sync_json_to_stream(
) -> Result<Option<SyncToStreamBridgeOutcome>, AiSurfaceFinalizeError> {
let provider_api_format = normalize_api_format(provider_api_format);
let client_api_format = normalize_api_format(client_api_format);
if let Some(outcome) = maybe_bridge_aether_sse_response_capture_to_stream(
provider_body_json,
provider_api_format.as_str(),
client_api_format.as_str(),
report_context,
)? {
return Ok(Some(outcome));
}
if provider_api_format == "openai:image" {
return match client_api_format.as_str() {
"openai:image" => {
@@ -161,20 +172,29 @@ fn maybe_bridge_openai_image_sync_json_to_chat_stream(
None,
&build_openai_chat_finish_chunk(&response_id, &model, Some("stop")),
)?);
if let Some((input_tokens, output_tokens, total_tokens, reasoning_tokens)) = summary
if let Some((
input_tokens,
output_tokens,
total_tokens,
reasoning_tokens,
cache_creation_tokens,
cache_read_tokens,
)) = summary
.standardized_usage
.as_ref()
.and_then(openai_chat_usage_counts)
{
sse_body.extend(encode_json_sse(
None,
&build_openai_chat_usage_chunk(
&build_openai_chat_usage_chunk_with_cache(
&response_id,
&model,
input_tokens,
output_tokens,
total_tokens,
reasoning_tokens,
cache_creation_tokens,
cache_read_tokens,
),
)?);
}
@@ -479,10 +499,12 @@ fn openai_image_standardized_usage(
(standardized_usage.signal_score() > 0).then_some(standardized_usage)
}
fn openai_chat_usage_counts(usage: &StandardizedUsage) -> Option<(u64, u64, u64, u64)> {
fn openai_chat_usage_counts(usage: &StandardizedUsage) -> Option<(u64, u64, u64, u64, u64, u64)> {
let input_tokens = usage.input_tokens.max(0) as u64;
let output_tokens = usage.output_tokens.max(0) as u64;
let reasoning_tokens = usage.reasoning_tokens.max(0) as u64;
let cache_creation_tokens = usage.cache_creation_tokens.max(0) as u64;
let cache_read_tokens = usage.cache_read_tokens.max(0) as u64;
let total_tokens = usage
.dimensions
.get("total_tokens")
@@ -492,7 +514,14 @@ fn openai_chat_usage_counts(usage: &StandardizedUsage) -> Option<(u64, u64, u64,
.saturating_add(output_tokens)
.saturating_add(reasoning_tokens)
});
(total_tokens > 0).then_some((input_tokens, output_tokens, total_tokens, reasoning_tokens))
(total_tokens > 0).then_some((
input_tokens,
output_tokens,
total_tokens,
reasoning_tokens,
cache_creation_tokens,
cache_read_tokens,
))
}
fn openai_image_bridge_response_id(
@@ -586,6 +615,173 @@ fn is_standard_api_format(value: &str) -> bool {
)
}
fn maybe_bridge_aether_sse_response_capture_to_stream(
provider_body_json: &Value,
provider_api_format: &str,
client_api_format: &str,
report_context: Option<&Value>,
) -> Result<Option<SyncToStreamBridgeOutcome>, AiSurfaceFinalizeError> {
let Some(object) = provider_body_json.as_object() else {
return Ok(None);
};
let status_code = object
.get("status_code")
.or_else(|| object.get("statusCode"))
.and_then(Value::as_u64)
.unwrap_or(0);
if !(200..300).contains(&status_code) {
return Ok(None);
}
let Some(headers) = object.get("headers").and_then(Value::as_object) else {
return Ok(None);
};
let content_type = response_capture_header(headers, "content-type").unwrap_or_default();
if !content_type
.to_ascii_lowercase()
.contains("text/event-stream")
{
return Ok(None);
}
let Some(body_text) = object.get("body").and_then(Value::as_str) else {
return Ok(None);
};
if body_text.trim().is_empty() || body_text.contains("...[truncated]") {
return Ok(None);
}
let captured_api_format =
response_capture_header(headers, "x-aether-control-endpoint-signature")
.map(normalize_api_format)
.or_else(|| infer_sse_body_api_format(body_text))
.unwrap_or_else(|| provider_api_format.to_string());
if !is_standard_api_format(captured_api_format.as_str())
|| !is_standard_api_format(client_api_format)
{
return Ok(None);
}
let bridge_context = build_bridge_report_context(
report_context,
captured_api_format.as_str(),
client_api_format,
);
let sse_body = if captured_api_format == client_api_format {
body_text.as_bytes().to_vec()
} else {
rewrite_sse_body_between_formats(
body_text.as_bytes(),
captured_api_format.as_str(),
client_api_format,
&bridge_context,
)?
};
let terminal_summary = observe_sse_terminal_summary(
body_text.as_bytes(),
captured_api_format.as_str(),
&bridge_context,
)?;
Ok(Some(SyncToStreamBridgeOutcome {
sse_body,
terminal_summary,
}))
}
fn response_capture_header<'a>(headers: &'a Map<String, Value>, name: &str) -> Option<&'a str> {
headers
.iter()
.find(|(key, _)| key.eq_ignore_ascii_case(name))
.and_then(|(_, value)| value.as_str())
.map(str::trim)
.filter(|value| !value.is_empty())
}
fn infer_sse_body_api_format(body_text: &str) -> Option<String> {
if body_text.contains("event: message_start")
|| body_text.contains("\"type\":\"message_start\"")
{
return Some("claude:messages".to_string());
}
if body_text.contains("event: response.") || body_text.contains("\"type\":\"response.") {
return Some("openai:responses".to_string());
}
if body_text.contains("data: [DONE]")
|| body_text.contains("\"object\":\"chat.completion.chunk\"")
{
return Some("openai:chat".to_string());
}
if body_text.contains("\"candidates\"") && body_text.contains("\"finishReason\"") {
return Some("gemini:generate_content".to_string());
}
None
}
fn rewrite_sse_body_between_formats(
body: &[u8],
provider_api_format: &str,
client_api_format: &str,
report_context: &Value,
) -> Result<Vec<u8>, AiSurfaceFinalizeError> {
let mut context =
build_bridge_report_context(Some(report_context), provider_api_format, client_api_format);
if let Some(object) = context.as_object_mut() {
object.insert(
"provider_stream_event_api_format".to_string(),
Value::String(provider_api_format.to_string()),
);
}
let mut matrix = StreamingStandardFormatMatrix::default();
let mut out = Vec::new();
for_each_sse_line(body, |line| {
out.extend(matrix.transform_line(&context, line)?);
Ok(())
})?;
out.extend(matrix.finish(&context)?);
Ok(out)
}
fn observe_sse_terminal_summary(
body: &[u8],
provider_api_format: &str,
report_context: &Value,
) -> Result<Option<ExecutionStreamTerminalSummary>, AiSurfaceFinalizeError> {
let mut context = build_bridge_report_context(
Some(report_context),
provider_api_format,
provider_api_format,
);
if let Some(object) = context.as_object_mut() {
object.insert(
"provider_stream_event_api_format".to_string(),
Value::String(provider_api_format.to_string()),
);
}
let mut observer = StreamingStandardTerminalObserver::default();
for_each_sse_line(body, |line| observer.push_line(&context, line))?;
observer.finish(&context)
}
fn for_each_sse_line<F>(body: &[u8], mut on_line: F) -> Result<(), AiSurfaceFinalizeError>
where
F: FnMut(Vec<u8>) -> Result<(), AiSurfaceFinalizeError>,
{
let mut start = 0usize;
for (index, byte) in body.iter().enumerate() {
if *byte == b'\n' {
on_line(body[start..=index].to_vec())?;
start = index + 1;
}
}
if start < body.len() {
on_line(body[start..].to_vec())?;
}
Ok(())
}
fn extract_openai_image_sync_b64_json(item: &serde_json::Map<String, Value>) -> Option<String> {
item.get("b64_json")
.and_then(Value::as_str)
@@ -1030,7 +1226,11 @@ mod tests {
"usage": {
"total_tokens": 100,
"input_tokens": 50,
"output_tokens": 50
"output_tokens": 50,
"input_tokens_details": {
"cached_tokens": 20,
"cached_creation_tokens": 10
}
}
}),
"openai:image",
@@ -1045,6 +1245,8 @@ mod tests {
assert!(output.contains("![generated image](data:image/png;base64,aGVsbG8=)"));
assert!(output.contains("![generated image 2](data:image/png;base64,d29ybGQ=)"));
assert!(output.contains("\"finish_reason\":\"stop\""));
assert!(output.contains("\"cached_tokens\":20"));
assert!(output.contains("\"cached_creation_tokens\":10"));
assert!(output.contains("data: [DONE]"));
assert!(!output.contains("image_generation.completed"));
@@ -1101,4 +1303,85 @@ mod tests {
assert!(output.contains("\"b64_json\":\"d29ybGQ=\""));
assert!(output.contains("\"total_tokens\":9"));
}
#[test]
fn bridges_aether_sse_response_capture_to_same_client_stream() {
let captured_body = concat!(
": aether-keepalive\n\n",
"event: message_start\n",
"data: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"gpt-5.5\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0}}}\n\n",
"event: content_block_start\n",
"data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n",
"event: content_block_delta\n",
"data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"hello\"}}\n\n",
"event: message_delta\n",
"data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"input_tokens\":1,\"output_tokens\":2}}\n\n",
"event: message_stop\n",
"data: {\"type\":\"message_stop\"}\n\n",
);
let outcome = maybe_bridge_standard_sync_json_to_stream(
&json!({
"status_code": 200,
"headers": {
"content-type": "text/event-stream",
"x-aether-control-endpoint-signature": "claude:messages"
},
"body": captured_body
}),
"openai:responses",
"claude:messages",
None,
)
.expect("bridge should succeed")
.expect("capture should bridge");
let output = utf8(outcome.sse_body);
assert!(output.contains("event: message_start"));
assert!(output.contains("event: message_stop"));
assert!(!output.contains("status_code"));
assert_eq!(
outcome
.terminal_summary
.as_ref()
.and_then(|summary| summary.finish_reason.as_deref()),
Some("end_turn")
);
}
#[test]
fn rewrites_aether_sse_response_capture_to_requested_client_stream() {
let captured_body = concat!(
"event: message_start\n",
"data: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"gpt-5.5\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0}}}\n\n",
"event: content_block_start\n",
"data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"call_1\",\"name\":\"Edit\",\"input\":{}}}\n\n",
"event: content_block_delta\n",
"data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"file_path\\\":\\\"/tmp/a.txt\\\"}\"}}\n\n",
"event: message_delta\n",
"data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\"},\"usage\":{\"input_tokens\":1,\"output_tokens\":2}}\n\n",
"event: message_stop\n",
"data: {\"type\":\"message_stop\"}\n\n",
);
let outcome = maybe_bridge_standard_sync_json_to_stream(
&json!({
"status_code": 200,
"headers": {
"content-type": "text/event-stream",
"x-aether-control-endpoint-signature": "claude:messages"
},
"body": captured_body
}),
"claude:messages",
"openai:responses",
None,
)
.expect("bridge should succeed")
.expect("capture should bridge");
let output = utf8(outcome.sse_body);
assert!(output.contains("event: response.output_item.added"));
assert!(output.contains("event: response.function_call_arguments.delta"));
assert!(output.contains("event: response.completed"));
assert!(!output.contains("status_code"));
}
}

35
crates/aether-provider-transport/src/openai_image/mod.rs
View File

@@ -7,7 +7,7 @@ use crate::grok::{is_grok_provider_transport, resolve_grok_session_auth};
use crate::policy::local_standard_transport_unsupported_reason_with_network;
use crate::rules::apply_local_header_rules_with_request_headers;
use crate::snapshot::GatewayProviderTransportSnapshot;
use crate::url::build_openai_responses_url;
use crate::url::{build_openai_image_url, build_openai_responses_url};
#[derive(Debug, Clone, Copy)]
pub struct ProviderOpenAiImageHeadersInput<'a> {
@@ -56,9 +56,18 @@ pub fn resolve_openai_image_auth(
pub fn build_openai_image_upstream_url(
transport: &GatewayProviderTransportSnapshot,
request_path: Option<&str>,
request_query: Option<&str>,
) -> String {
build_openai_responses_url(&transport.endpoint.base_url, request_query, false)
if transport
.provider
.provider_type
.trim()
.eq_ignore_ascii_case("codex")
{
return build_openai_responses_url(&transport.endpoint.base_url, request_query, false);
}
build_openai_image_url(&transport.endpoint.base_url, request_path, request_query)
}
pub fn build_openai_image_headers(
@@ -155,12 +164,30 @@ mod tests {
}
#[test]
fn builds_openai_image_url_on_responses_surface() {
let url = build_openai_image_upstream_url(&sample_transport(), Some("trace=1"));
fn codex_openai_image_url_stays_on_responses_surface() {
let url = build_openai_image_upstream_url(
&sample_transport(),
Some("/v1/images/generations"),
Some("trace=1"),
);
assert_eq!(url, "https://api.openai.com/v1/responses?trace=1");
}
#[test]
fn standard_openai_image_url_uses_images_surface() {
let mut transport = sample_transport();
transport.provider.provider_type = "openai".to_string();
let url = build_openai_image_upstream_url(
&transport,
Some("/v1/images/generations"),
Some("trace=1"),
);
assert_eq!(url, "https://api.openai.com/v1/images/generations?trace=1");
}
#[test]
fn chatgpt_web_is_supported_by_dedicated_openai_image_transport_policy() {
let mut transport = sample_transport();

55
crates/aether-provider-transport/src/url.rs
View File

@@ -41,6 +41,43 @@ pub fn build_openai_responses_url(
url
}
pub fn build_openai_image_url(
upstream_base_url: &str,
request_path: Option<&str>,
query: Option<&str>,
) -> String {
let (trimmed, base_query) = split_base_url_query(upstream_base_url);
let trimmed = trimmed.trim_end_matches('/');
let suffix = openai_image_path_suffix(request_path);
let mut url = if openai_image_base_includes_operation_path(trimmed) {
trimmed.to_string()
} else if trimmed.ends_with("/v1") || google_openai_compat_base_includes_api_root(trimmed) {
format!("{trimmed}{suffix}")
} else {
format!("{trimmed}/v1{suffix}")
};
append_merged_query(&mut url, base_query, None, query, &[]);
url
}
fn openai_image_path_suffix(request_path: Option<&str>) -> &'static str {
match request_path
.map(str::trim)
.map(|value| value.trim_end_matches('/'))
{
Some("/v1/images/edits") | Some("/images/edits") => "/images/edits",
_ => "/images/generations",
}
}
fn openai_image_base_includes_operation_path(base_url: &str) -> bool {
let path = Url::parse(base_url)
.ok()
.map(|url| url.path().trim_end_matches('/').to_string())
.unwrap_or_else(|| base_url.trim_end_matches('/').to_string());
path.ends_with("/images/generations") || path.ends_with("/images/edits")
}
pub fn build_claude_messages_url(upstream_base_url: &str, query: Option<&str>) -> String {
let (trimmed, base_query) = split_base_url_query(upstream_base_url);
let trimmed = trimmed.trim_end_matches('/');
@@ -307,7 +344,7 @@ fn merge_query_string(
mod tests {
use super::{
build_gemini_content_url, build_gemini_files_passthrough_url,
build_gemini_video_predict_long_running_url, build_openai_chat_url,
build_gemini_video_predict_long_running_url, build_openai_chat_url, build_openai_image_url,
build_openai_responses_url, build_passthrough_path_url,
normalize_gemini_content_action_path,
};
@@ -353,6 +390,22 @@ mod tests {
);
}
#[test]
fn openai_image_url_uses_images_surface() {
assert_eq!(
build_openai_image_url(
"https://api.openai.example/v1?tenant=demo",
Some("/v1/images/generations"),
Some("trace=1")
),
"https://api.openai.example/v1/images/generations?tenant=demo&trace=1"
);
assert_eq!(
build_openai_image_url("https://api.openai.example", Some("/v1/images/edits"), None),
"https://api.openai.example/v1/images/edits"
);
}
#[test]
fn merges_base_url_query_for_dynamic_gemini_content_urls() {
assert_eq!(

52
frontend/src/features/providers/components/provider-tabs/ModelTestDialog.vue
View File

@@ -688,36 +688,6 @@
</TabsContent>
<TabsContent value="response-body">
<div
v-if="selectedInspectionImagePreviews.length > 0"
class="mb-3 rounded-md border border-border/60 bg-muted/20 p-3"
>
<div class="mb-3 flex items-center justify-between gap-3 text-xs text-muted-foreground">
<span>图片预览</span>
<span>{{ selectedInspectionImagePreviews.length }} 张</span>
</div>
<div class="grid gap-3 sm:grid-cols-2 xl:grid-cols-3">
<button
v-for="(preview, index) in selectedInspectionImagePreviews"
:key="`${preview.src}-${index}`"
type="button"
class="group block overflow-hidden rounded-md border border-border/60 bg-background text-left transition-colors hover:border-primary/60 focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-primary/70"
@click="openImagePreview(preview)"
>
<div class="aspect-square w-full overflow-hidden bg-muted/30">
<img
:src="preview.src"
:alt="preview.label"
class="h-full w-full object-contain"
loading="lazy"
>
</div>
<div class="border-t border-border/60 px-2 py-1 text-[11px] text-muted-foreground">
{{ preview.label }}
</div>
</button>
</div>
</div>
<JsonContent
:data="selectedInspectionAttempt.response_body"
view-mode="formatted"
@@ -935,11 +905,17 @@ const activeImagePreview = ref<ModelTestImagePreview | null>(null)
watch(() => props.result, () => {
showAllAttempts.value = false
inspectionTab.value = 'request-body'
inspectionExpandDepth.value = 0
inspectionCopiedStates.value = {}
const defaultAttempt = inspectableAttempts.value[0] ?? resultAttempts.value[0] ?? null
const defaultAttempt = resultImageAttempt.value
?? resultAttempts.value.find(attempt => attempt.status === 'success')
?? inspectableAttempts.value[0]
?? resultAttempts.value[0]
?? null
selectedInspectionKey.value = defaultAttempt ? inspectionKey(defaultAttempt) : null
inspectionTab.value = defaultAttempt && attemptImagePreviews(defaultAttempt).length > 0
? 'response-body'
: 'request-body'
})
const shouldCollapseAttempts = computed(() => resultAttempts.value.length > 20)
@@ -1245,11 +1221,11 @@ const selectedInspectionAttempt = computed(() => {
return inspectableAttempts.value[0] ?? resultAttempts.value[0] ?? null
})
const selectedInspectionImagePreviews = computed(() => (
selectedInspectionAttempt.value
? extractModelTestImagePreviews(selectedInspectionAttempt.value.response_body)
: []
))
const resultImageAttempt = computed(() => {
return resultAttempts.value.find(attempt => attempt.status === 'success' && attemptImagePreviews(attempt).length > 0)
?? resultAttempts.value.find(attempt => attemptImagePreviews(attempt).length > 0)
?? null
})
const resultWinningTitle = computed(() => {
const summary = resultSummary.value
@@ -1432,7 +1408,7 @@ function inspectionKey(attempt: TestAttemptDetail): string {
function selectInspectionAttempt(attempt: TestAttemptDetail) {
selectedInspectionKey.value = inspectionKey(attempt)
inspectionTab.value = 'request-body'
inspectionTab.value = attemptImagePreviews(attempt).length > 0 ? 'response-body' : 'request-body'
}
function hasDebugData(attempt: TestAttemptDetail): boolean {

66
frontend/src/features/providers/components/provider-tabs/__tests__/model-test-request.spec.ts
View File

@@ -479,6 +479,23 @@ describe('extractModelTestResponsePreview', () => {
})).toBe('图片：https://example.com/generated.png')
})
it('extracts renderable URL image previews from OpenAI image responses', () => {
expect(extractModelTestImagePreviews({
data: [
{
url: 'https://example.com/generated.png',
revised_prompt: 'A generated image',
},
],
})).toEqual([
{
src: 'https://example.com/generated.png',
label: '图片 1',
source: 'url',
},
])
})
it('summarizes base64 image responses without dumping the image payload', () => {
expect(extractModelTestResponsePreview({
data: [
@@ -506,6 +523,55 @@ describe('extractModelTestResponsePreview', () => {
])
})
it('accepts root-relative URLs in OpenAI image responses', () => {
expect(extractModelTestImagePreviews({
data: [
{
url: '/v1/files/image?id=2af14311-a0cb-4bbf-ae20-d1fcf44e0479',
},
],
})).toEqual([
{
src: '/v1/files/image?id=2af14311-a0cb-4bbf-ae20-d1fcf44e0479',
label: '图片 1',
source: 'url',
},
])
})
it('parses stringified JSON image responses', () => {
expect(extractModelTestImagePreviews(JSON.stringify({
data: [
{
url: 'https://example.com/generated.png',
},
],
}))).toEqual([
{
src: 'https://example.com/generated.png',
label: '图片 1',
source: 'url',
},
])
})
it('parses stringified JSON base64 image responses', () => {
expect(extractModelTestImagePreviews(JSON.stringify({
data: [
{
b64_json: 'aGVsbG8=',
mime_type: 'image/jpeg',
},
],
}))).toEqual([
{
src: 'data:image/jpeg;base64,aGVsbG8=',
label: '图片 1',
source: 'base64',
},
])
})
it('extracts image previews from nested response image urls', () => {
expect(extractModelTestImagePreviews({
output: [

31
frontend/src/features/providers/components/provider-tabs/model-test-preview.ts
View File

@@ -10,16 +10,17 @@ export type ModelTestImagePreview = {
}
export function extractModelTestResponsePreview(responseBody: unknown): string | null {
const text = extractResponseText(responseBody)
const normalizedBody = normalizeModelTestResponseBody(responseBody)
const text = extractResponseText(normalizedBody)
if (text) return text
const reasoning = extractResponseReasoning(responseBody)
const reasoning = extractResponseReasoning(normalizedBody)
if (reasoning) return `推理：${reasoning}`
const image = extractImagePreview(responseBody)
const image = extractImagePreview(normalizedBody)
if (image) return image
const summary = extractResponseSummary(responseBody)
const summary = extractResponseSummary(normalizedBody)
if (summary) return summary
return null
@@ -27,10 +28,26 @@ export function extractModelTestResponsePreview(responseBody: unknown): string |
export function extractModelTestImagePreviews(responseBody: unknown): ModelTestImagePreview[] {
const previews: ModelTestImagePreview[] = []
collectImagePreviews(responseBody, previews, new Set(), 0)
collectImagePreviews(normalizeModelTestResponseBody(responseBody), previews, new Set(), 0)
return previews
}
function normalizeModelTestResponseBody(value: unknown): unknown {
if (typeof value !== 'string') return value
const trimmed = value.trim()
if (!trimmed) return value
if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
try {
return JSON.parse(trimmed)
} catch {
return value
}
}
return value
}
function isJsonRecord(value: unknown): value is JsonRecord {
return Boolean(value) && typeof value === 'object' && !Array.isArray(value)
}
@@ -146,6 +163,7 @@ function collectImagePreviews(
value.output,
value.images,
value.content,
value.item,
]
for (const nested of nestedValues) {
collectImagePreviews(nested, previews, seen, depth + 1)
@@ -203,6 +221,9 @@ function imageUrlToPreview(value: unknown, source: 'url'): ModelTestImagePreview
if (url.startsWith('data:image/')) {
return { src: url, label: 'base64', source: 'base64' }
}
if (url.startsWith('/')) {
return { src: url, label: 'URL', source }
}
try {
const parsed = new URL(url)