i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-12 14:10:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge PR #669: align GPT-5.6 and Codex request protocols

Browse Source
This commit is contained in:
elky
2026-07-12 21:50:20 +08:00
parent 7f61bb43c7 25c49dd804
commit e58621a735
313 changed files with 29823 additions and 5178 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/aether-gateway/src/execution_runtime/fallback.rs
+1
View File
@@ -941,6 +941,7 @@ mod tests {
success_failover_patterns: Vec::new(),
error_stop_patterns: Vec::new(),
stop_cyber_policy_errors: false,
retry_client_errors_by_default: true,
}
);
}
apps/aether-gateway/src/execution_runtime/grok.rs
+1
View File
@@ -1042,6 +1042,7 @@ fn grok_stream_terminal_summary(
finish_reason: Some("stop".to_string()),
response_id: None,
model: plan.model_name.clone(),
provider_actual_service_tier: None,
observed_finish: true,
unknown_event_count: 0,
parser_error: None,
apps/aether-gateway/src/execution_runtime/server.rs
+41 -3
View File
@@ -359,8 +359,7 @@ impl IntoResponse for ExecutionRuntimeAppError {
return build_overloaded_response(&self.0.to_string());
}
ExecutionRuntimeServerError::Transport(
ExecutionRuntimeTransportError::StreamUnsupported
| ExecutionRuntimeTransportError::RequestBodyRequired
ExecutionRuntimeTransportError::RequestBodyRequired
| ExecutionRuntimeTransportError::BodyDecode(_)
| ExecutionRuntimeTransportError::UnsupportedContentEncoding(_)
| ExecutionRuntimeTransportError::ProxyUnsupported
@@ -397,7 +396,9 @@ mod tests {
build_execution_runtime_router_with_request_concurrency_limit,
build_execution_runtime_router_with_request_gates, DISTRIBUTED_REQUEST_GATE_NAME,
};
use aether_contracts::{ExecutionPlan, ExecutionTimeouts, RequestBody};
use aether_contracts::{
ExecutionPlan, ExecutionTimeouts, RequestBody, StreamFrame, StreamFrameType,
};
use aether_runtime_state::{
MemoryRuntimeStateConfig, RuntimeSemaphore, RuntimeSemaphoreConfig, RuntimeState,
};
@@ -459,6 +460,43 @@ mod tests {
}
}
#[tokio::test]
async fn execution_runtime_stream_endpoint_carries_non_stream_upstream_plan() {
let upstream = Router::new().route(
"/sync-json",
any(|| async { axum::Json(serde_json::json!({"ok": true})) }),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let runtime = build_execution_runtime_router_with_request_concurrency_limit(None);
let (runtime_url, runtime_handle) = start_server(runtime).await;
let mut plan = stream_plan(format!("{upstream_url}/sync-json"));
plan.stream = false;
let response = reqwest::Client::new()
.post(format!("{runtime_url}/v1/execute/stream"))
.json(&plan)
.send()
.await
.expect("execution request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let body = response.text().await.expect("frame body should read");
let frame_types = body
.lines()
.map(|line| {
serde_json::from_str::<StreamFrame>(line)
.expect("execution runtime frame should decode")
.frame_type
})
.collect::<Vec<_>>();
assert!(frame_types.contains(&StreamFrameType::Headers));
assert!(frame_types.contains(&StreamFrameType::Data));
assert!(frame_types.contains(&StreamFrameType::Eof));
runtime_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn execution_runtime_rejects_second_in_flight_stream_request_with_overload() {
let upstream_hits = Arc::new(AtomicUsize::new(0));
apps/aether-gateway/src/execution_runtime/stream/execution.rs
+11 -3
View File
@@ -851,6 +851,9 @@ fn merge_stream_terminal_summary(
if current_summary.model.is_none() {
current_summary.model = observed.model;
}
if observed.provider_actual_service_tier.is_some() {
current_summary.provider_actual_service_tier = observed.provider_actual_service_tier;
}
current_summary.observed_finish |= observed.observed_finish;
current_summary.unknown_event_count = current_summary
.unknown_event_count
@@ -5484,9 +5487,8 @@ async fn execute_stream_from_frame_stream(
telemetry = Some(frame_telemetry);
}
StreamFramePayload::Eof { summary } => {
if summary.is_some() {
stream_terminal_summary = summary;
}
stream_terminal_summary =
merge_stream_terminal_summary(stream_terminal_summary.take(), summary);
break;
}
StreamFramePayload::Error { error } => {
@@ -6293,12 +6295,14 @@ mod tests {
Some(ExecutionStreamTerminalSummary {
standardized_usage: Some(runtime_usage),
model: Some("gpt-5.5".to_string()),
provider_actual_service_tier: Some("priority".to_string()),
unknown_event_count: 1,
..ExecutionStreamTerminalSummary::default()
}),
Some(ExecutionStreamTerminalSummary {
standardized_usage: Some(observed_usage),
response_id: Some("resp_123".to_string()),
provider_actual_service_tier: Some("default".to_string()),
observed_finish: true,
unknown_event_count: 2,
..ExecutionStreamTerminalSummary::default()
@@ -6313,6 +6317,10 @@ mod tests {
assert_eq!(usage.output_tokens, 137);
assert_eq!(merged.model.as_deref(), Some("gpt-5.5"));
assert_eq!(merged.response_id.as_deref(), Some("resp_123"));
assert_eq!(
merged.provider_actual_service_tier.as_deref(),
Some("default")
);
assert!(merged.observed_finish);
assert_eq!(merged.unknown_event_count, 3);
}
apps/aether-gateway/src/execution_runtime/stream_pump.rs
+70 -33
View File
@@ -676,6 +676,14 @@ fn should_buffer_non_stream_response(
return false;
}
if report_context
.get("upstream_is_stream")
.and_then(Value::as_bool)
== Some(false)
{
return true;
}
headers
.get("content-length")
.and_then(|value| value.trim().parse::<u64>().ok())
@@ -1134,22 +1142,36 @@ mod tests {
}
#[test]
fn buffers_non_sse_response_only_when_content_length_is_known() {
let report_context = serde_json::json!({
fn buffers_declared_non_stream_responses_without_relying_on_content_length() {
let streaming_context = serde_json::json!({
"provider_api_format": "openai:chat",
"client_api_format": "openai:chat",
"upstream_is_stream": true,
});
let non_stream_context = serde_json::json!({
"provider_api_format": "openai:image",
"client_api_format": "openai:responses",
"upstream_is_stream": false,
});
assert!(!should_buffer_non_stream_response(
&BTreeMap::from([("content-type".into(), "application/json".into())]),
&report_context
&streaming_context
));
assert!(should_buffer_non_stream_response(
&BTreeMap::from([("content-type".into(), "application/json".into())]),
&non_stream_context
));
assert!(should_buffer_non_stream_response(
&BTreeMap::from([
("content-type".into(), "application/json".into()),
("content-length".into(), "128".into()),
]),
&report_context
&streaming_context
));
assert!(!should_buffer_non_stream_response(
&BTreeMap::from([("content-type".into(), "text/event-stream".into())]),
&non_stream_context
));
}
@@ -1567,33 +1589,48 @@ mod tests {
.await
.expect("listener should bind");
let addr = listener.local_addr().expect("local addr should resolve");
let generated_image = "a".repeat(32 * 1024);
let expected_image = generated_image.clone();
let server = tokio::spawn(async move {
let app = Router::new().route(
"/responses",
post(|| async {
let body = serde_json::json!({
"created": 1776971267_u64,
"data": [{
"b64_json": "aGVsbG8="
}],
"usage": {
"total_tokens": 100,
"input_tokens": 50,
"output_tokens": 50,
"input_tokens_details": {
"text_tokens": 10,
"image_tokens": 40
"/images/generations",
post(move || {
let generated_image = generated_image.clone();
async move {
let body = serde_json::json!({
"created": 1776971267_u64,
"data": [{
"b64_json": generated_image
}],
"usage": {
"total_tokens": 100,
"input_tokens": 50,
"output_tokens": 50,
"input_tokens_details": {
"text_tokens": 10,
"image_tokens": 40
}
}
}
});
let mut response = axum::http::Response::new(Body::from(
serde_json::to_vec(&body).expect("json should encode"),
));
response.headers_mut().insert(
header::CONTENT_TYPE,
HeaderValue::from_static("application/json"),
);
response
});
let encoded = serde_json::to_vec(&body).expect("json should encode");
let chunks = encoded
.chunks(4096)
.map(Bytes::copy_from_slice)
.collect::<Vec<_>>();
let chunked_body = stream! {
for chunk in chunks {
yield Ok::<Bytes, Infallible>(chunk);
tokio::task::yield_now().await;
}
};
let mut response =
axum::http::Response::new(Body::from_stream(chunked_body));
response.headers_mut().insert(
header::CONTENT_TYPE,
HeaderValue::from_static("application/json"),
);
response
}
}),
);
axum::serve(listener, app)
@@ -1611,16 +1648,15 @@ mod tests {
endpoint_id: "endpoint-1".to_string(),
key_id: "key-1".to_string(),
method: "POST".to_string(),
url: format!("http://{addr}/responses"),
url: format!("http://{addr}/images/generations"),
headers: BTreeMap::new(),
content_type: None,
content_encoding: None,
body: RequestBody::from_json(serde_json::json!({
"model": "gpt-image-1",
"prompt": "poster",
"stream": true
"prompt": "poster"
})),
stream: true,
stream: false,
client_api_format: "openai:image".to_string(),
provider_api_format: "openai:image".to_string(),
model_name: Some("gpt-image-1".into()),
@@ -1673,7 +1709,8 @@ mod tests {
let bridged_text = String::from_utf8(bridged_body).expect("bridged body should be utf8");
assert!(bridged_text.contains("event: image_generation.completed"));
assert!(bridged_text.contains("\"type\":\"image_generation.completed\""));
assert!(bridged_text.contains("\"b64_json\":\"aGVsbG8=\""));
assert!(bridged_text.contains(&format!("\"b64_json\":\"{expected_image}\"")));
assert!(bridged_text.len() > 32 * 1024);
assert!(bridged_text.contains("\"total_tokens\":100"));
let eof_frame = frames
apps/aether-gateway/src/execution_runtime/tests.rs
+1
View File
@@ -43,6 +43,7 @@ fn missing_exact_provider_request_payload(decision_kind: &str) -> AiExecutionDec
request_id: Some("req_123".to_string()),
candidate_id: Some("cand_123".to_string()),
provider_name: Some("provider".to_string()),
provider_type: None,
provider_id: Some("provider_id".to_string()),
endpoint_id: Some("endpoint_id".to_string()),
key_id: Some("key_id".to_string()),
apps/aether-gateway/src/execution_runtime/transport.rs
+126 -76
View File
@@ -57,8 +57,8 @@ const TUNNEL_RELAY_PATH_PREFIX: &str = "/api/internal/tunnel/relay";
const DEFAULT_TUNNEL_TIMEOUT_MS: u64 = 60_000;
const DEFAULT_STREAM_FIRST_BYTE_TIMEOUT_MS: u64 = 30_000;
const DEFAULT_NON_STREAM_TOTAL_TIMEOUT_MS: u64 = 300_000;
const DEFAULT_CODEX_COMPACT_TOTAL_TIMEOUT_MS: u64 = 1_200_000;
const MIN_TUNNEL_TIMEOUT_SECS: u64 = 1;
const MAX_TUNNEL_TIMEOUT_SECS: u64 = 300;
const DIRECT_REQWEST_H2_CLIENT_SHARDS_ENV: &str = "AETHER_GATEWAY_DIRECT_REQWEST_H2_CLIENT_SHARDS";
const DIRECT_REQWEST_CLIENT_SHARDS_ENV: &str = "AETHER_GATEWAY_DIRECT_REQWEST_CLIENT_SHARDS";
const DIRECT_REQWEST_H2_TARGET_STREAMS_PER_CLIENT_ENV: &str =
@@ -512,8 +512,6 @@ pub(crate) fn format_hyper_error_chain(err: &dyn std::error::Error) -> String {
#[derive(Debug, Error)]
pub(crate) enum ExecutionRuntimeTransportError {
#[error("stream execution is not supported for this plan")]
StreamUnsupported,
#[error("request body must contain json_body or body_bytes_b64")]
RequestBodyRequired,
#[error("request body base64 is invalid: {0}")]
@@ -681,10 +679,6 @@ impl DirectSyncExecutionRuntime {
&self,
plan: &ExecutionPlan,
) -> Result<DirectUpstreamStreamExecution, ExecutionRuntimeTransportError> {
if !plan.stream {
return Err(ExecutionRuntimeTransportError::StreamUnsupported);
}
let build_body_started_at = Instant::now();
let body_bytes = build_request_body(plan)?;
observe_gateway_stage_ms(
@@ -835,6 +829,7 @@ fn build_stream_summary_report_context(plan: &ExecutionPlan) -> Value {
"provider_api_format": plan.provider_api_format,
"client_api_format": plan.client_api_format,
"model": plan.model_name,
"upstream_is_stream": plan.stream,
})
}
@@ -2125,20 +2120,17 @@ pub(crate) fn build_request_body(
Vec::new()
};
if should_gzip_request_body(plan) && plan.body.json_body.is_some() {
body_bytes = gzip_bytes(&body_bytes)?;
if plan.body.json_body.is_some() {
body_bytes = match normalize_content_encoding(plan.content_encoding.as_deref()).as_deref() {
Some("gzip") => gzip_bytes(&body_bytes)?,
Some("zstd") => zstd_bytes(&body_bytes)?,
_ => body_bytes,
};
}
Ok(body_bytes)
}
fn should_gzip_request_body(plan: &ExecutionPlan) -> bool {
matches!(
normalize_content_encoding(plan.content_encoding.as_deref()).as_deref(),
Some("gzip")
)
}
fn normalize_content_encoding(value: Option<&str>) -> Option<String> {
value
.map(str::trim)
@@ -2156,6 +2148,11 @@ fn gzip_bytes(body_bytes: &[u8]) -> Result<Vec<u8>, ExecutionRuntimeTransportErr
.map_err(|err| ExecutionRuntimeTransportError::RelayError(err.to_string()))
}
fn zstd_bytes(body_bytes: &[u8]) -> Result<Vec<u8>, ExecutionRuntimeTransportError> {
zstd::stream::encode_all(std::io::Cursor::new(body_bytes), 3)
.map_err(|err| ExecutionRuntimeTransportError::RelayError(err.to_string()))
}
fn build_relay_client(
timeouts: Option<&aether_contracts::ExecutionTimeouts>,
) -> Result<reqwest::Client, ExecutionRuntimeTransportError> {
@@ -2218,28 +2215,49 @@ fn resolve_tunnel_first_byte_timeout(plan: &ExecutionPlan) -> Option<Duration> {
})
}
fn resolve_non_stream_total_timeout(plan: &ExecutionPlan) -> Option<Duration> {
if plan.stream {
pub(crate) fn resolve_non_stream_total_timeout_for_request(
is_stream: bool,
provider_api_format: &str,
timeouts: Option<&aether_contracts::ExecutionTimeouts>,
) -> Option<Duration> {
if is_stream {
return None;
}
let timeout_ms = plan
.timeouts
.as_ref()
let default_timeout_ms =
if crate::ai_serving::is_openai_responses_compact_format(provider_api_format) {
DEFAULT_CODEX_COMPACT_TOTAL_TIMEOUT_MS
} else {
DEFAULT_NON_STREAM_TOTAL_TIMEOUT_MS
};
let timeout_ms = timeouts
.and_then(|timeouts| timeouts.total_ms)
.unwrap_or(DEFAULT_NON_STREAM_TOTAL_TIMEOUT_MS);
.unwrap_or(default_timeout_ms);
Some(Duration::from_millis(timeout_ms.max(1)))
}
fn resolve_non_stream_total_timeout(plan: &ExecutionPlan) -> Option<Duration> {
resolve_non_stream_total_timeout_for_request(
plan.stream,
&plan.provider_api_format,
plan.timeouts.as_ref(),
)
}
pub(crate) fn resolve_stream_first_byte_timeout_for_request(
is_stream: bool,
timeouts: Option<&aether_contracts::ExecutionTimeouts>,
) -> Option<Duration> {
if !is_stream {
return None;
}
let timeout_ms = timeouts
.and_then(|timeouts| timeouts.first_byte_ms)
.unwrap_or(DEFAULT_STREAM_FIRST_BYTE_TIMEOUT_MS);
Some(Duration::from_millis(timeout_ms.max(1)))
}
pub(crate) fn resolve_stream_first_byte_timeout(plan: &ExecutionPlan) -> Option<Duration> {
if !plan.stream {
return None;
}
let timeout_ms = plan
.timeouts
.as_ref()
.and_then(|timeouts| timeouts.first_byte_ms)
.unwrap_or(DEFAULT_STREAM_FIRST_BYTE_TIMEOUT_MS);
Some(Duration::from_millis(timeout_ms.max(1)))
resolve_stream_first_byte_timeout_for_request(plan.stream, plan.timeouts.as_ref())
}
pub(crate) async fn with_non_stream_total_timeout<T, F>(
@@ -2359,7 +2377,10 @@ fn resolve_tunnel_timeout_metadata(plan: &ExecutionPlan) -> TunnelTimeoutMetadat
fn timeout_ms_to_secs(ms: u64) -> u64 {
let secs = ms.div_ceil(1_000);
secs.clamp(MIN_TUNNEL_TIMEOUT_SECS, MAX_TUNNEL_TIMEOUT_SECS)
secs.clamp(
MIN_TUNNEL_TIMEOUT_SECS,
aether_contracts::MAX_EXECUTION_REQUEST_TIMEOUT_SECS,
)
}
fn resolve_tunnel_node_id(proxy: Option<&ProxySnapshot>) -> Option<String> {
@@ -3552,7 +3573,7 @@ pub(crate) fn build_request_headers(
let mut out = HeaderMap::new();
let normalized_content_encoding = normalize_content_encoding(content_encoding);
if let Some(encoding) = normalized_content_encoding.as_deref() {
if encoding != "gzip" && !allow_passthrough_content_encoding {
if !matches!(encoding, "gzip" | "zstd") && !allow_passthrough_content_encoding {
return Err(ExecutionRuntimeTransportError::UnsupportedContentEncoding(
encoding.to_string(),
));
@@ -4637,6 +4658,25 @@ mod tests {
assert_eq!(timeout, std::time::Duration::from_secs(300));
}
#[test]
fn codex_compact_uses_the_full_unary_timeout_by_default() {
let mut plan = tunnel_timeout_plan(false);
plan.provider_api_format = "openai:responses:compact".to_string();
plan.timeouts = None;
let timeout = resolve_non_stream_total_timeout(&plan)
.expect("Codex Compact should have a total timeout");
let meta = build_direct_tunnel_request_meta(
&plan,
&reqwest::header::HeaderMap::new(),
ExecutionTransportControls::default(),
);
assert_eq!(timeout, std::time::Duration::from_secs(1_200));
assert_eq!(meta.request_timeout_ms, Some(1_200_000));
assert_eq!(meta.timeout, 1_200);
}
#[test]
fn tunnel_request_meta_uses_non_stream_default_instead_of_first_byte_default() {
let mut plan = tunnel_timeout_plan(false);
@@ -6323,13 +6363,21 @@ mod tests {
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_string();
let mut decoder = flate2::read::GzDecoder::new(body.as_ref());
let mut decoded = String::new();
decoder
.read_to_string(&mut decoded)
.expect("gzip body should decode");
let decoded = match header_encoding.as_str() {
"gzip" => {
let mut decoder = flate2::read::GzDecoder::new(body.as_ref());
let mut decoded = Vec::new();
decoder
.read_to_end(&mut decoded)
.expect("gzip body should decode");
decoded
}
"zstd" => zstd::stream::decode_all(std::io::Cursor::new(body.as_ref()))
.expect("zstd body should decode"),
encoding => panic!("unexpected content encoding: {encoding}"),
};
let decoded_json: serde_json::Value =
serde_json::from_str(&decoded).expect("decoded json should parse");
serde_json::from_slice(&decoded).expect("decoded json should parse");
(
axum::http::StatusCode::OK,
Json(json!({
@@ -6346,45 +6394,47 @@ mod tests {
});
let execution_runtime = DirectSyncExecutionRuntime::new();
let result = execution_runtime
.execute_sync(&ExecutionPlan {
request_id: "req-gzip-1".into(),
candidate_id: Some("cand-1".into()),
provider_name: Some("openai".into()),
provider_id: "prov-1".into(),
endpoint_id: "ep-1".into(),
key_id: "key-1".into(),
method: "POST".into(),
url: format!("http://{addr}/chat"),
headers: BTreeMap::from([("content-type".into(), "application/json".into())]),
content_type: Some("application/json".into()),
content_encoding: Some("gzip".into()),
body: RequestBody::from_json(json!({"model": "gpt-4.1"})),
stream: false,
client_api_format: "openai:chat".into(),
provider_api_format: "openai:chat".into(),
model_name: Some("gpt-4.1".into()),
proxy: None,
transport_profile: None,
timeouts: Some(ExecutionTimeouts {
connect_ms: Some(5_000),
total_ms: Some(LOCAL_HTTP_SUCCESS_TIMEOUT_MS),
..ExecutionTimeouts::default()
}),
})
.await
.expect("gzip sync execution should succeed");
for encoding in ["gzip", "zstd"] {
let result = execution_runtime
.execute_sync(&ExecutionPlan {
request_id: format!("req-{encoding}-1"),
candidate_id: Some("cand-1".into()),
provider_name: Some("openai".into()),
provider_id: "prov-1".into(),
endpoint_id: "ep-1".into(),
key_id: "key-1".into(),
method: "POST".into(),
url: format!("http://{addr}/chat"),
headers: BTreeMap::from([("content-type".into(), "application/json".into())]),
content_type: Some("application/json".into()),
content_encoding: Some(encoding.into()),
body: RequestBody::from_json(json!({"model": "gpt-4.1"})),
stream: false,
client_api_format: "openai:chat".into(),
provider_api_format: "openai:chat".into(),
model_name: Some("gpt-4.1".into()),
proxy: None,
transport_profile: None,
timeouts: Some(ExecutionTimeouts {
connect_ms: Some(5_000),
total_ms: Some(LOCAL_HTTP_SUCCESS_TIMEOUT_MS),
..ExecutionTimeouts::default()
}),
})
.await
.expect("compressed sync execution should succeed");
assert_eq!(result.status_code, 200);
assert_eq!(
result.body.and_then(|body| body.json_body),
Some(json!({
"content_encoding": encoding,
"body": {"model": "gpt-4.1"},
}))
);
}
server.abort();
assert_eq!(result.status_code, 200);
assert_eq!(
result.body.and_then(|body| body.json_body),
Some(json!({
"content_encoding": "gzip",
"body": {"model": "gpt-4.1"},
}))
);
}
#[tokio::test]