i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-11 19:59:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/pr-483' into merge-pr-483

Browse Source 
# Conflicts:
#	apps/aether-gateway/src/ai_serving/planner/standard/openai/chat/decision/payload.rs
#	apps/aether-gateway/src/ai_serving/planner/standard/openai/chat/decision/request.rs
#	apps/aether-gateway/src/ai_serving/planner/standard/openai/responses/decision/payload.rs
#	apps/aether-gateway/src/ai_serving/planner/standard/openai/responses/decision/request.rs
#	apps/aether-gateway/src/execution_runtime/chatgpt_web_image.rs
This commit is contained in:
fawney19
2026-05-19 02:23:14 +08:00
parent afedd90c80 c58ce63fc3
commit c563c192b7
53 changed files with 8081 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/aether-gateway/src/ai_serving/planner/standard/openai/responses/decision/payload.rs
+25
View File
@@ -81,6 +81,30 @@ pub(crate) async fn maybe_build_local_openai_responses_decision_payload_for_cand
if let Some(envelope_name) = resolved.envelope_name {
extra_fields.insert("envelope_name".to_string(), json!(envelope_name));
}
if let Some(image_request_summary) = resolved.image_request_summary.as_ref() {
extra_fields.insert("image_request".to_string(), image_request_summary.clone());
}
if resolved
.provider_api_format
.eq_ignore_ascii_case("openai:image")
&& resolved
.transport
.provider
.provider_type
.trim()
.eq_ignore_ascii_case("chatgpt_web")
{
extra_fields.insert("chatgpt_web_image".to_string(), json!(true));
extra_fields.insert(
"local_failover_policy".to_string(),
json!({
"stop_status_codes": [400, 401, 403, 429, 500, 502, 503, 504],
"error_stop_patterns": [
{ "pattern": ".*" }
]
}),
);
}
insert_provider_stream_event_api_format(
&mut extra_fields,
resolved.transport.provider.provider_type.as_str(),
@@ -179,6 +203,7 @@ pub(crate) async fn maybe_build_local_openai_responses_decision_payload_for_cand
upstream_is_stream,
transport,
transport_profile: _,
image_request_summary: _,
} = resolved;
let mut decision = build_ai_execution_decision_response(AiExecutionDecisionResponseParts {
apps/aether-gateway/src/ai_serving/planner/standard/openai/responses/decision/request.rs
+482 -4
View File
@@ -2,7 +2,7 @@ use std::collections::BTreeMap;
use std::sync::Arc;
use aether_contracts::ResolvedTransportProfile;
use serde_json::Value;
use serde_json::{json, Value};
use tracing::debug;
use crate::ai_serving::planner::candidate_preparation::{
@@ -16,7 +16,8 @@ use crate::ai_serving::planner::common::{
};
use crate::ai_serving::planner::spec_metadata::local_openai_responses_spec_metadata;
use crate::ai_serving::planner::standard::{
apply_codex_openai_responses_special_headers, build_cross_format_openai_responses_request_body,
apply_codex_openai_responses_special_body_edits, apply_codex_openai_responses_special_headers,
build_cross_format_openai_responses_request_body,
build_cross_format_openai_responses_upstream_url, build_local_openai_responses_request_body,
build_local_openai_responses_upstream_url, request_body_build_failure_extra_data,
};
@@ -37,9 +38,11 @@ use crate::ai_serving::transport::kiro::{
};
use crate::ai_serving::transport::{
build_grok_browser_headers, build_grok_upstream_url, build_kiro_cross_format_upstream_url,
build_openai_image_headers, build_openai_image_upstream_url,
build_standard_provider_request_headers,
local_standard_transport_unsupported_reason_with_network, GrokHeaderInput,
StandardProviderRequestHeadersInput, GROK_CHAT_PATH,
local_standard_transport_unsupported_reason_with_network,
openai_image_transport_unsupported_reason, resolve_openai_image_auth, GrokHeaderInput,
ProviderOpenAiImageHeadersInput, StandardProviderRequestHeadersInput, GROK_CHAT_PATH,
};
use crate::ai_serving::{
ai_local_execution_contract_for_formats, request_conversion_direct_auth,
@@ -81,6 +84,7 @@ pub(crate) struct LocalOpenAiResponsesCandidatePayloadParts {
pub(super) upstream_is_stream: bool,
pub(super) transport: Arc<GatewayProviderTransportSnapshot>,
pub(super) transport_profile: Option<ResolvedTransportProfile>,
pub(super) image_request_summary: Option<Value>,
}
#[allow(clippy::too_many_arguments)]
@@ -110,6 +114,21 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts(
.trim()
.eq_ignore_ascii_case("grok");
if provider_api_format.eq_ignore_ascii_case("openai:image") {
return resolve_openai_responses_to_openai_image_payload_parts(
state,
parts,
trace_id,
body_json,
input,
eligible,
candidate_index,
candidate_id,
spec,
)
.await;
}
let same_format = api_format_alias_matches(provider_api_format, &client_api_format);
let conversion_kind = request_conversion_kind(spec_metadata.api_format, provider_api_format);
let transport_unsupported_reason = if is_grok
@@ -596,6 +615,7 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts(
upstream_is_stream,
transport: Arc::clone(transport),
transport_profile,
image_request_summary: None,
})
}
@@ -603,6 +623,463 @@ fn api_format_alias_matches(left: &str, right: &str) -> bool {
crate::ai_serving::api_format_alias_matches(left, right)
}
#[allow(clippy::too_many_arguments)]
async fn resolve_openai_responses_to_openai_image_payload_parts(
state: &AppState,
parts: &http::request::Parts,
trace_id: &str,
body_json: &serde_json::Value,
input: &LocalOpenAiResponsesDecisionInput,
eligible: &EligibleLocalExecutionCandidate,
candidate_index: u32,
candidate_id: &str,
spec: LocalOpenAiResponsesSpec,
) -> Option<LocalOpenAiResponsesCandidatePayloadParts> {
let spec_metadata = local_openai_responses_spec_metadata(spec);
let candidate = &eligible.candidate;
let transport = &eligible.transport;
let provider_api_format = "openai:image";
if let Some(skip_reason) =
openai_image_transport_unsupported_reason(transport, provider_api_format)
{
mark_skipped_local_openai_responses_candidate(
state,
input,
trace_id,
candidate,
candidate_index,
candidate_id,
skip_reason,
)
.await;
return None;
}
let prepared_candidate = match prepare_header_authenticated_candidate(
PlannerAppState::new(state),
transport,
candidate,
resolve_openai_image_auth(transport),
OauthPreparationContext {
trace_id,
api_format: provider_api_format,
operation: "openai_responses_image_bridge",
},
)
.await
{
Ok(prepared) => prepared,
Err(skip_reason) => {
mark_skipped_local_openai_responses_candidate(
state,
input,
trace_id,
candidate,
candidate_index,
candidate_id,
skip_reason,
)
.await;
return None;
}
};
let is_chatgpt_web = transport
.provider
.provider_type
.trim()
.eq_ignore_ascii_case("chatgpt_web");
let upstream_is_stream = resolve_upstream_is_stream_for_provider(
transport.endpoint.config.as_ref(),
transport.provider.provider_type.as_str(),
provider_api_format,
spec_metadata.require_streaming,
false,
);
let Some((mut provider_request_body, image_request_summary)) = (if is_chatgpt_web {
build_chatgpt_web_image_provider_body_from_openai_responses_body(
body_json,
&input.requested_model,
)
} else {
build_openai_image_provider_body_from_openai_responses_body(
body_json,
&input.requested_model,
upstream_is_stream,
)
}) else {
mark_skipped_local_openai_responses_candidate_with_extra_data(
state,
input,
trace_id,
candidate,
candidate_index,
candidate_id,
"provider_request_body_build_failed",
request_body_build_failure_extra_data(
body_json,
spec_metadata.api_format,
provider_api_format,
),
)
.await;
return None;
};
if !is_chatgpt_web {
apply_codex_openai_responses_special_body_edits(
&mut provider_request_body,
transport.provider.provider_type.as_str(),
provider_api_format,
transport.endpoint.body_rules.as_ref(),
Some(candidate.key_id.as_str()),
);
}
let upstream_url = if is_chatgpt_web {
chatgpt_web_image_internal_url(&transport.endpoint.base_url)
} else {
build_openai_image_upstream_url(transport, parts.uri.query())
};
let Some(mut provider_request_headers) =
build_openai_image_headers(ProviderOpenAiImageHeadersInput {
headers: &parts.headers,
auth_header: &prepared_candidate.auth_header,
auth_value: &prepared_candidate.auth_value,
header_rules: transport.endpoint.header_rules.as_ref(),
provider_request_body: &provider_request_body,
original_request_body: body_json,
})
else {
mark_skipped_local_openai_responses_candidate_with_failure_diagnostic(
state,
input,
trace_id,
candidate,
candidate_index,
candidate_id,
"transport_header_rules_apply_failed",
CandidateFailureDiagnostic::header_rules_apply_failed(
spec_metadata.api_format,
provider_api_format,
"openai_responses_image_bridge_headers",
),
)
.await;
return None;
};
if is_chatgpt_web {
provider_request_headers.insert("x-aether-chatgpt-web-image".to_string(), "1".to_string());
} else {
apply_codex_openai_responses_special_headers(
&mut provider_request_headers,
&provider_request_body,
&parts.headers,
transport.provider.provider_type.as_str(),
provider_api_format,
Some(trace_id),
transport.key.decrypted_auth_config.as_deref(),
);
}
let (execution_strategy, conversion_mode) =
ai_local_execution_contract_for_formats(spec_metadata.api_format, provider_api_format);
Some(LocalOpenAiResponsesCandidatePayloadParts {
auth_header: prepared_candidate.auth_header,
auth_value: prepared_candidate.auth_value,
mapped_model: prepared_candidate.mapped_model,
provider_api_format: provider_api_format.to_string(),
provider_request_body,
provider_request_headers,
upstream_url,
execution_strategy,
conversion_mode,
is_antigravity: false,
envelope_name: None,
upstream_is_stream,
transport: Arc::clone(transport),
transport_profile: None,
image_request_summary: Some(image_request_summary),
})
}
fn build_openai_image_provider_body_from_openai_responses_body(
body_json: &Value,
requested_model: &str,
upstream_is_stream: bool,
) -> Option<(Value, Value)> {
let object = body_json.as_object()?;
let input = object.get("input")?.clone();
let mut tool = openai_responses_image_generation_tool(object).unwrap_or_else(|| {
serde_json::Map::from_iter([("type".to_string(), json!("image_generation"))])
});
tool.entry("type".to_string())
.or_insert_with(|| json!("image_generation"));
tool.entry("action".to_string())
.or_insert_with(|| json!("generate"));
let mut body = serde_json::Map::new();
body.insert("input".to_string(), input);
body.insert(
"tools".to_string(),
Value::Array(vec![Value::Object(tool.clone())]),
);
if let Some(model) = object
.get("model")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.or_else(|| {
let requested_model = requested_model.trim();
(!requested_model.is_empty()).then_some(requested_model)
})
{
body.insert("model".to_string(), Value::String(model.to_string()));
}
for key in [
"user",
"metadata",
"include",
"parallel_tool_calls",
"store",
] {
if let Some(value) = object.get(key) {
body.insert(key.to_string(), value.clone());
}
}
if upstream_is_stream {
body.insert("stream".to_string(), Value::Bool(true));
} else if let Some(value) = object.get("stream") {
body.insert("stream".to_string(), value.clone());
}
let mut summary = serde_json::Map::new();
summary.insert(
"operation".to_string(),
tool.get("action")
.cloned()
.unwrap_or_else(|| json!("generate")),
);
for key in ["output_format", "partial_images", "size", "quality"] {
if let Some(value) = tool.get(key).or_else(|| object.get(key)) {
summary.insert(key.to_string(), value.clone());
}
}
Some((Value::Object(body), Value::Object(summary)))
}
fn openai_responses_image_generation_tool(
object: &serde_json::Map<String, Value>,
) -> Option<serde_json::Map<String, Value>> {
object
.get("tools")
.and_then(Value::as_array)?
.iter()
.filter_map(Value::as_object)
.find(|tool| {
tool.get("type")
.and_then(Value::as_str)
.is_some_and(|value| value.trim().eq_ignore_ascii_case("image_generation"))
})
.cloned()
}
fn build_chatgpt_web_image_provider_body_from_openai_responses_body(
body_json: &Value,
requested_model: &str,
) -> Option<(Value, Value)> {
let object = body_json.as_object()?;
let (prompt, images) = collect_openai_responses_image_prompt_and_images(object.get("input"))?;
let operation = if images.is_empty() {
"generate"
} else {
"edit"
};
let tool = openai_responses_image_generation_tool(object);
let size = image_option_string(tool.as_ref(), object, "size").unwrap_or("1024x1024");
let output_format =
image_option_string(tool.as_ref(), object, "output_format").unwrap_or("png");
let quality = image_option_string(tool.as_ref(), object, "quality").unwrap_or("medium");
let model = object
.get("model")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or_else(|| requested_model.trim());
let web_model = image_option_string(tool.as_ref(), object, "web_model")
.or_else(|| image_option_string(tool.as_ref(), object, "model"))
.unwrap_or("gpt-5-5-thinking");
let image_urls = openai_image_inputs_as_urls(&images);
let body = json!({
"operation": operation,
"model": if model.is_empty() { "gpt-image-2" } else { model },
"web_model": web_model,
"prompt": prompt,
"size": size,
"ratio": chatgpt_web_ratio_for_size(size),
"output_format": output_format,
"images": image_urls,
});
let summary = json!({
"operation": operation,
"output_format": output_format,
"size": size,
"quality": quality,
});
Some((body, summary))
}
fn image_option_string<'a>(
tool: Option<&'a serde_json::Map<String, Value>>,
object: &'a serde_json::Map<String, Value>,
key: &str,
) -> Option<&'a str> {
tool.and_then(|tool| tool.get(key))
.or_else(|| object.get(key))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
}
fn collect_openai_responses_image_prompt_and_images(
input: Option<&Value>,
) -> Option<(String, Vec<Value>)> {
let input = input?;
let mut prompt_parts = Vec::new();
let mut images = Vec::new();
collect_openai_responses_image_input(input, &mut prompt_parts, &mut images);
let prompt = prompt_parts.join("\n").trim().to_string();
(!prompt.is_empty()).then_some((prompt, images))
}
fn collect_openai_responses_image_input(
value: &Value,
prompt_parts: &mut Vec<String>,
images: &mut Vec<Value>,
) {
match value {
Value::String(text) => {
let text = text.trim();
if !text.is_empty() {
prompt_parts.push(text.to_string());
}
}
Value::Array(items) => {
for item in items {
collect_openai_responses_image_input(item, prompt_parts, images);
}
}
Value::Object(object) => {
let item_type = object
.get("type")
.and_then(Value::as_str)
.map(str::trim)
.unwrap_or_default();
if matches!(item_type, "input_text" | "text") {
if let Some(text) = object
.get("text")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
{
prompt_parts.push(text.to_string());
}
} else if matches!(item_type, "input_image" | "image_url") {
collect_openai_image_input_object(object, images);
}
if let Some(content) = object.get("content") {
collect_openai_responses_image_input(content, prompt_parts, images);
}
}
_ => {}
}
}
fn collect_openai_image_input_object(
object: &serde_json::Map<String, Value>,
images: &mut Vec<Value>,
) {
if let Some(url) = object
.get("image_url")
.and_then(|value| {
value
.as_str()
.or_else(|| value.get("url").and_then(Value::as_str))
})
.or_else(|| object.get("url").and_then(Value::as_str))
.map(str::trim)
.filter(|value| !value.is_empty())
{
images.push(json!({
"type": "input_image",
"image_url": url,
}));
} else if let Some(file_id) = object
.get("file_id")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
{
images.push(json!({
"type": "input_image",
"file_id": file_id,
}));
}
}
fn openai_image_inputs_as_urls(images: &[Value]) -> Vec<Value> {
images
.iter()
.filter_map(|image| {
image
.get("image_url")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(|value| Value::String(value.to_string()))
})
.collect()
}
fn chatgpt_web_ratio_for_size(size: &str) -> String {
let Some((width, height)) = size.split_once('x') else {
return "1:1".to_string();
};
let Ok(width) = width.trim().parse::<u64>() else {
return "1:1".to_string();
};
let Ok(height) = height.trim().parse::<u64>() else {
return "1:1".to_string();
};
if width == 0 || height == 0 {
return "1:1".to_string();
}
let divisor = gcd(width, height);
format!("{}:{}", width / divisor, height / divisor)
}
fn gcd(mut left: u64, mut right: u64) -> u64 {
while right != 0 {
let next = left % right;
left = right;
right = next;
}
left.max(1)
}
fn chatgpt_web_image_internal_url(base_url: &str) -> String {
let base_url = base_url.trim().trim_end_matches('/');
let base_url = if base_url.is_empty() {
"https://chatgpt.com"
} else {
base_url
};
format!("{base_url}/__aether/chatgpt-web-image")
}
#[allow(clippy::too_many_arguments)]
async fn build_kiro_openai_responses_payload_parts(
state: &AppState,
@@ -748,5 +1225,6 @@ async fn build_kiro_openai_responses_payload_parts(
upstream_is_stream,
transport: Arc::clone(transport),
transport_profile: None,
image_request_summary: None,
})
}
apps/aether-gateway/src/ai_serving/planner/standard/openai/responses/decision/support.rs
+110
View File
@@ -15,6 +15,7 @@ use crate::ai_serving::planner::candidate_metadata::{
LocalExecutionCandidateMetadataParts,
};
use crate::ai_serving::planner::candidate_source::{
preselect_local_execution_candidates_for_api_formats_with_serving,
preselect_local_execution_candidates_with_serving, LocalCandidatePreselectionKeyMode,
};
use crate::ai_serving::planner::common::extract_standard_requested_model;
@@ -36,6 +37,7 @@ use crate::ai_serving::{
use crate::client_session_affinity::client_session_affinity_from_parts;
use crate::{AppState, GatewayError};
use super::super::super::openai_request_is_image_generation_intent;
use super::LocalOpenAiResponsesSpec;
pub(crate) use crate::ai_serving::planner::candidate_materialization::LocalExecutionCandidateAttempt as LocalOpenAiResponsesCandidateAttempt;
@@ -265,6 +267,16 @@ pub(crate) async fn build_local_openai_responses_candidate_attempt_source<'a>(
input.required_capabilities.as_ref(),
LocalCandidatePersistencePolicyKind::OpenAiResponsesDecision,
);
if openai_request_is_image_generation_intent(&input.requested_model, body_json) {
let (image_candidates, image_candidate_count) =
build_local_openai_responses_image_candidate_attempt_source(
state, trace_id, input, body_json, spec,
)
.await?;
if image_candidate_count > 0 {
return Ok((image_candidates, image_candidate_count));
}
}
Ok(
build_lazy_requested_model_execution_candidate_attempt_source_with_serving(
planner_state,
@@ -335,6 +347,104 @@ pub(crate) async fn build_local_openai_responses_candidate_attempt_source<'a>(
)
}
pub(crate) async fn build_local_openai_responses_image_candidate_attempt_source<'a>(
state: &'a AppState,
trace_id: &str,
input: &LocalOpenAiResponsesDecisionInput,
body_json: &serde_json::Value,
spec: LocalOpenAiResponsesSpec,
) -> Result<(LocalOpenAiResponsesCandidateAttemptSource<'a>, usize), GatewayError> {
let spec_metadata = local_openai_responses_spec_metadata(spec);
let planner_state = PlannerAppState::new(state);
let sticky_session_token = extract_pool_sticky_session_token(body_json);
let auth_context: &ExecutionRuntimeAuthContext = &input.auth_context;
let persistence_policy = build_local_candidate_persistence_policy(
auth_context,
input.required_capabilities.as_ref(),
LocalCandidatePersistencePolicyKind::OpenAiResponsesDecision,
);
let preselection = preselect_local_execution_candidates_for_api_formats_with_serving(
planner_state,
spec_metadata.api_format,
&input.requested_model,
false,
input.required_capabilities.as_ref(),
&input.auth_snapshot,
input.routing_policy.as_ref(),
input.client_session_affinity.as_ref(),
true,
LocalCandidatePreselectionKeyMode::ProviderEndpointKeyModelAndApiFormat,
vec!["openai:image".to_string()],
)
.await?;
Ok(build_local_execution_candidate_attempt_source_with_serving(
planner_state,
trace_id,
spec_metadata.api_format,
Some(&input.requested_model),
Some(&input.auth_snapshot),
input.client_session_affinity.as_ref(),
input.required_capabilities.as_ref(),
input.routing_policy.as_ref(),
sticky_session_token.as_deref(),
input.request_auth_channel.as_deref(),
persistence_policy,
preselection.candidates,
preselection.skipped_candidates,
LocalCandidateResolutionMode::WithoutTransportPairGate,
move |eligible| {
let provider_api_format = eligible.provider_api_format.clone();
let (execution_strategy, conversion_mode) = ai_local_execution_contract_for_formats(
spec_metadata.api_format,
&provider_api_format,
);
Some(build_local_execution_candidate_contract_metadata(
LocalExecutionCandidateMetadataParts {
eligible,
provider_api_format: provider_api_format.as_str(),
client_api_format: spec_metadata.api_format,
extra_fields: serde_json::Map::new(),
},
execution_strategy,
conversion_mode,
eligible.candidate.endpoint_api_format.as_str(),
))
},
move |mut skipped_candidate| {
let provider_api_format = skipped_candidate
.transport
.as_ref()
.map(|transport| transport.endpoint.api_format.trim().to_ascii_lowercase())
.unwrap_or_else(|| {
skipped_candidate
.candidate
.endpoint_api_format
.trim()
.to_ascii_lowercase()
});
let (execution_strategy, conversion_mode) = ai_local_execution_contract_for_formats(
spec_metadata.api_format,
&provider_api_format,
);
skipped_candidate.extra_data = Some(
build_local_execution_candidate_contract_metadata_for_candidate(
&skipped_candidate.candidate,
skipped_candidate.transport_ref(),
provider_api_format.as_str(),
spec_metadata.api_format,
serde_json::Map::new(),
execution_strategy,
conversion_mode,
provider_api_format.as_str(),
),
);
skipped_candidate
},
)
.await)
}
pub(crate) async fn mark_skipped_local_openai_responses_candidate(
state: &AppState,
input: &LocalOpenAiResponsesDecisionInput,