i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 09:20:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 大规模模块拆分与代码精简，新增 ai-pipeline/data-contracts 独立 crate

Browse Source 
- 新增 aether-ai-pipeline 和 aether-data-contracts crate，将 pipeline 逻辑与数据契约从 gateway 中解耦
- 重构 admin handlers：拆分单体模块为 auth/billing/endpoint/features/model/observability/provider/system 等独立子模块
- 合并 chat/cli 重复代码路径：精简 conversion、finalize、planner 中的 sync/chat/cli 分支
- 重构 scheduler/executor/data 层，引入 facade 模式降低模块间耦合
- 移除冗余的 intent 模块，将 plan_fallback/policy/stream_path/sync_path 迁移至 executor
- 前端适配：调整 admin API 调用和 provider 模型测试对话框
This commit is contained in:
fawney19
2026-04-07 02:50:19 +08:00
parent 763ff03a7b
commit 5d96d6673b
732 changed files with 28593 additions and 20666 deletions
Download Patch File Download Diff File Expand all files Collapse all files

316
apps/aether-gateway/src/execution_runtime/fallback.rs Normal file
View File

@@ -0,0 +1,316 @@
use aether_contracts::ExecutionResult;
fn is_local_candidate_attempt(report_context: Option<&serde_json::Value>) -> bool {
report_context
.and_then(serde_json::Value::as_object)
.and_then(|context| context.get("candidate_index"))
.and_then(serde_json::Value::as_u64)
.is_some()
}
fn is_retryable_local_upstream_status(status_code: u16) -> bool {
status_code == 429 || status_code >= 500
}
pub(crate) fn should_retry_next_local_candidate_sync(
plan_kind: &str,
report_context: Option<&serde_json::Value>,
result: &ExecutionResult,
) -> bool {
is_local_candidate_attempt(report_context)
&& plan_kind == "openai_chat_sync"
&& is_retryable_local_upstream_status(result.status_code)
}
pub(crate) fn should_fallback_to_control_sync(
plan_kind: &str,
result: &ExecutionResult,
body_json: Option<&serde_json::Value>,
has_body_bytes: bool,
explicit_finalize: bool,
mapped_error_finalize: bool,
) -> bool {
if explicit_finalize
&& matches!(
plan_kind,
"openai_video_delete_sync" | "openai_video_cancel_sync" | "gemini_video_cancel_sync"
)
{
return false;
}
if !matches!(
plan_kind,
"openai_video_create_sync"
| "openai_video_remix_sync"
| "gemini_video_create_sync"
| "openai_chat_sync"
| "openai_cli_sync"
| "openai_compact_sync"
| "claude_chat_sync"
| "gemini_chat_sync"
| "claude_cli_sync"
| "gemini_cli_sync"
) {
return false;
}
if explicit_finalize {
return result.status_code < 400 && body_json.is_none() && !has_body_bytes;
}
if mapped_error_finalize {
return false;
}
if result.status_code >= 400 {
return true;
}
let Some(body_json) = body_json else {
return true;
};
body_json.get("error").is_some()
}
pub(crate) fn should_finalize_sync_response(report_kind: Option<&str>) -> bool {
report_kind.is_some_and(|kind| kind.ends_with("_finalize"))
}
pub(crate) fn resolve_core_sync_error_finalize_report_kind(
plan_kind: &str,
result: &ExecutionResult,
body_json: Option<&serde_json::Value>,
) -> Option<String> {
let has_embedded_error = body_json.is_some_and(|value| value.get("error").is_some());
if result.status_code < 400 && !has_embedded_error {
return None;
}
let report_kind = match plan_kind {
"openai_chat_sync" => "openai_chat_sync_finalize",
"openai_cli_sync" => "openai_cli_sync_finalize",
"openai_compact_sync" => "openai_compact_sync_finalize",
"claude_chat_sync" => "claude_chat_sync_finalize",
"gemini_chat_sync" => "gemini_chat_sync_finalize",
"claude_cli_sync" => "claude_cli_sync_finalize",
"gemini_cli_sync" => "gemini_cli_sync_finalize",
_ => return None,
};
Some(report_kind.to_string())
}
pub(crate) fn should_retry_next_local_candidate_stream(
plan_kind: &str,
report_context: Option<&serde_json::Value>,
status_code: u16,
) -> bool {
is_local_candidate_attempt(report_context)
&& plan_kind == "openai_chat_stream"
&& is_retryable_local_upstream_status(status_code)
}
pub(crate) fn should_fallback_to_control_stream(
plan_kind: &str,
status_code: u16,
mapped_error_finalize: bool,
) -> bool {
if mapped_error_finalize {
return false;
}
matches!(
plan_kind,
"openai_chat_stream"
| "claude_chat_stream"
| "gemini_chat_stream"
| "openai_cli_stream"
| "openai_compact_stream"
| "claude_cli_stream"
| "gemini_cli_stream"
) && status_code >= 400
}
pub(crate) fn resolve_core_stream_error_finalize_report_kind(
plan_kind: &str,
status_code: u16,
) -> Option<String> {
if status_code < 400 {
return None;
}
let report_kind = match plan_kind {
"openai_chat_stream" => "openai_chat_sync_finalize",
"claude_chat_stream" => "claude_chat_sync_finalize",
"gemini_chat_stream" => "gemini_chat_sync_finalize",
"openai_cli_stream" => "openai_cli_sync_finalize",
"openai_compact_stream" => "openai_compact_sync_finalize",
"claude_cli_stream" => "claude_cli_sync_finalize",
"gemini_cli_stream" => "gemini_cli_sync_finalize",
_ => return None,
};
Some(report_kind.to_string())
}
pub(crate) fn resolve_core_stream_direct_finalize_report_kind(plan_kind: &str) -> Option<String> {
let report_kind = match plan_kind {
"openai_chat_stream" => "openai_chat_sync_finalize",
"claude_chat_stream" => "claude_chat_sync_finalize",
"gemini_chat_stream" => "gemini_chat_sync_finalize",
"openai_cli_stream" => "openai_cli_sync_finalize",
"openai_compact_stream" => "openai_compact_sync_finalize",
"claude_cli_stream" => "claude_cli_sync_finalize",
"gemini_cli_stream" => "gemini_cli_sync_finalize",
_ => return None,
};
Some(report_kind.to_string())
}
#[cfg(test)]
mod tests {
use aether_contracts::ExecutionResult;
use super::{
resolve_core_stream_error_finalize_report_kind,
resolve_core_sync_error_finalize_report_kind, should_fallback_to_control_stream,
should_fallback_to_control_sync, should_retry_next_local_candidate_stream,
should_retry_next_local_candidate_sync,
};
#[test]
fn sync_failover_marks_chat_errors() {
let result = ExecutionResult {
request_id: "req-1".to_string(),
candidate_id: None,
status_code: 502,
headers: Default::default(),
body: None,
telemetry: None,
error: None,
};
assert!(should_fallback_to_control_sync(
"openai_chat_sync",
&result,
None,
false,
false,
false,
));
assert_eq!(
resolve_core_sync_error_finalize_report_kind("openai_chat_sync", &result, None),
Some("openai_chat_sync_finalize".to_string())
);
}
#[test]
fn stream_failover_marks_chat_errors() {
assert!(should_fallback_to_control_stream(
"openai_chat_stream",
502,
false,
));
assert_eq!(
resolve_core_stream_error_finalize_report_kind("openai_chat_stream", 502),
Some("openai_chat_sync_finalize".to_string())
);
}
#[test]
fn sync_retry_next_candidate_is_local_openai_chat_only() {
let result = ExecutionResult {
request_id: "req-1".to_string(),
candidate_id: None,
status_code: 502,
headers: Default::default(),
body: None,
telemetry: None,
error: None,
};
let local_report_context = serde_json::json!({
"candidate_index": 0,
"retry_index": 0,
});
assert!(should_retry_next_local_candidate_sync(
"openai_chat_sync",
Some(&local_report_context),
&result,
));
assert!(!should_retry_next_local_candidate_sync(
"openai_chat_sync",
None,
&result,
));
assert!(!should_retry_next_local_candidate_sync(
"claude_chat_sync",
None,
&result,
));
}
#[test]
fn sync_retry_next_candidate_treats_rate_limit_as_retryable() {
let result = ExecutionResult {
request_id: "req-1".to_string(),
candidate_id: None,
status_code: 429,
headers: Default::default(),
body: None,
telemetry: None,
error: None,
};
let local_report_context = serde_json::json!({
"candidate_index": 0,
"retry_index": 0,
});
assert!(should_retry_next_local_candidate_sync(
"openai_chat_sync",
Some(&local_report_context),
&result,
));
}
#[test]
fn stream_retry_next_candidate_is_local_openai_chat_only() {
let local_report_context = serde_json::json!({
"candidate_index": 0,
"retry_index": 0,
});
assert!(should_retry_next_local_candidate_stream(
"openai_chat_stream",
Some(&local_report_context),
502,
));
assert!(!should_retry_next_local_candidate_stream(
"openai_chat_stream",
None,
502,
));
assert!(!should_retry_next_local_candidate_stream(
"claude_chat_stream",
Some(&local_report_context),
502,
));
}
#[test]
fn stream_retry_next_candidate_treats_rate_limit_as_retryable() {
let local_report_context = serde_json::json!({
"candidate_index": 0,
"retry_index": 0,
});
assert!(should_retry_next_local_candidate_stream(
"openai_chat_stream",
Some(&local_report_context),
429,
));
}
}

14
apps/aether-gateway/src/execution_runtime/mod.rs
View File

@@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
mod constants;
mod fallback;
pub(crate) mod ndjson;
#[cfg(test)]
pub(crate) mod remote_compat;
@@ -17,19 +18,24 @@ pub(crate) mod transport;
pub(crate) use self::constants::{
MAX_ERROR_BODY_BYTES, MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES,
};
pub(crate) use self::fallback::{
resolve_core_stream_direct_finalize_report_kind,
resolve_core_stream_error_finalize_report_kind, resolve_core_sync_error_finalize_report_kind,
should_fallback_to_control_stream, should_fallback_to_control_sync,
should_finalize_sync_response, should_retry_next_local_candidate_stream,
should_retry_next_local_candidate_sync,
};
pub use server::{
build_execution_runtime_router, build_execution_runtime_router_with_request_concurrency_limit,
build_execution_runtime_router_with_request_gates, serve_execution_runtime_tcp,
serve_execution_runtime_unix,
};
pub(crate) use stream::{
execute_execution_runtime_stream, maybe_execute_via_execution_runtime_stream,
};
pub(crate) use stream::execute_execution_runtime_stream;
pub(crate) use stream_pump::build_direct_execution_frame_stream;
pub(crate) use sync::{
execute_execution_runtime_sync, maybe_build_local_sync_finalize_response,
maybe_build_local_video_error_response, maybe_build_local_video_success_outcome,
maybe_execute_via_execution_runtime_sync, resolve_local_sync_error_background_report_kind,
resolve_local_sync_error_background_report_kind,
resolve_local_sync_success_background_report_kind, LocalVideoSyncSuccessOutcome,
};
pub(crate) use transport::{

67
apps/aether-gateway/src/execution_runtime/stream/execution.rs
View File

@@ -1,6 +1,7 @@
use std::io::Error as IoError;
use aether_contracts::{ExecutionPlan, ExecutionTelemetry, StreamFramePayload};
use aether_data_contracts::repository::candidates::RequestCandidateStatus;
use async_stream::stream;
use axum::body::{Body, Bytes};
use axum::http::Response;
@@ -11,7 +12,7 @@ use serde_json::Value;
use tokio::sync::mpsc;
use tokio_util::codec::{FramedRead, LinesCodec};
use tokio_util::io::StreamReader;
use tracing::{debug, warn};
use tracing::{debug, info, warn};
use super::error::{
build_execution_runtime_error_response, collect_error_body, decode_stream_error_body,
@@ -30,6 +31,7 @@ use crate::ai_pipeline::finalize::maybe_build_stream_response_rewriter;
use crate::api::response::{
attach_control_metadata_headers, build_client_response, build_client_response_from_parts,
};
use crate::clock::current_unix_secs as current_request_candidate_unix_secs;
use crate::constants::{CONTROL_CANDIDATE_ID_HEADER, CONTROL_REQUEST_ID_HEADER};
use crate::control::GatewayControlDecision;
use crate::execution_runtime::build_direct_execution_frame_stream;
@@ -41,14 +43,16 @@ use crate::execution_runtime::submission::{
use crate::execution_runtime::transport::{
DirectSyncExecutionRuntime, DirectUpstreamStreamExecution,
};
use crate::execution_runtime::{MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES};
use crate::scheduler::{
current_unix_secs as current_request_candidate_unix_secs,
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
use crate::execution_runtime::{
resolve_core_stream_direct_finalize_report_kind,
resolve_core_stream_error_finalize_report_kind, should_fallback_to_control_stream,
should_retry_next_local_candidate_stream,
};
use crate::execution_runtime::{MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES};
use crate::log_ids::short_request_id;
use crate::request_candidate_runtime::{
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
};
use crate::usage::submit_stream_report;
use crate::usage::{GatewayStreamReportRequest, GatewaySyncReportRequest};
use crate::{AppState, GatewayError};
@@ -64,6 +68,7 @@ pub(crate) async fn execute_execution_runtime_stream(
mut report_context: Option<serde_json::Value>,
) -> Result<Option<Response<Body>>, GatewayError> {
ensure_execution_request_candidate_slot(state, &mut plan, &mut report_context).await;
let plan_request_id_for_log = short_request_id(plan.request_id.as_str());
#[cfg(not(test))]
{
let execution = match DirectSyncExecutionRuntime::new()
@@ -72,11 +77,11 @@ pub(crate) async fn execute_execution_runtime_stream(
{
Ok(execution) => execution,
Err(err) => {
warn!(
info!(
event_name = "stream_execution_runtime_unavailable",
log_type = "ops",
trace_id = %trace_id,
request_id = %plan.request_id,
request_id = %plan_request_id_for_log,
candidate_id = ?plan.candidate_id,
error = %err,
"gateway in-process stream execution unavailable"
@@ -109,11 +114,11 @@ pub(crate) async fn execute_execution_runtime_stream(
{
Ok(execution) => execution,
Err(err) => {
warn!(
info!(
event_name = "stream_execution_runtime_unavailable",
log_type = "ops",
trace_id = %trace_id,
request_id = %plan.request_id,
request_id = %plan_request_id_for_log,
candidate_id = ?plan.candidate_id,
error = %err,
"gateway in-process stream execution unavailable"
@@ -149,7 +154,7 @@ pub(crate) async fn execute_execution_runtime_stream(
event_name = "stream_execution_runtime_remote_unavailable",
log_type = "ops",
trace_id = %trace_id,
request_id = %plan.request_id,
request_id = %plan_request_id_for_log,
candidate_id = ?plan.candidate_id,
error = ?err,
"gateway remote execution runtime stream unavailable"
@@ -164,7 +169,7 @@ pub(crate) async fn execute_execution_runtime_stream(
state,
&plan,
report_context.as_ref(),
aether_data::repository::candidates::RequestCandidateStatus::Failed,
RequestCandidateStatus::Failed,
Some(response.status().as_u16()),
Some("execution_runtime_http_error".to_string()),
Some(format!(
@@ -212,6 +217,7 @@ async fn execute_stream_from_frame_stream(
frame_stream: BoxStream<'static, Result<Bytes, IoError>>,
) -> Result<Option<Response<Body>>, GatewayError> {
let request_id = plan.request_id.as_str();
let request_id_for_log = short_request_id(request_id);
let candidate_id = plan.candidate_id.as_deref();
let reader = StreamReader::new(frame_stream);
let mut lines = FramedRead::new(reader, LinesCodec::new());
@@ -235,7 +241,7 @@ async fn execute_stream_from_frame_stream(
state,
&plan,
report_context.as_ref(),
aether_data::repository::candidates::RequestCandidateStatus::Failed,
RequestCandidateStatus::Failed,
Some(status_code),
Some("retryable_upstream_status".to_string()),
Some(format!(
@@ -250,7 +256,7 @@ async fn execute_stream_from_frame_stream(
event_name = "local_stream_candidate_retry_scheduled",
log_type = "event",
trace_id = %trace_id,
request_id,
request_id = %request_id_for_log,
status_code,
"gateway local stream decision retrying next candidate after retryable execution runtime status"
);
@@ -270,7 +276,7 @@ async fn execute_stream_from_frame_stream(
state,
&plan,
report_context.as_ref(),
aether_data::repository::candidates::RequestCandidateStatus::Failed,
RequestCandidateStatus::Failed,
Some(status_code),
Some("control_fallback".to_string()),
Some(format!(
@@ -316,7 +322,7 @@ async fn execute_stream_from_frame_stream(
state,
&plan,
report_context.as_ref(),
aether_data::repository::candidates::RequestCandidateStatus::Failed,
RequestCandidateStatus::Failed,
Some(status_code),
Some("execution_runtime_stream_error".to_string()),
Some(format!(
@@ -626,7 +632,7 @@ async fn execute_stream_from_frame_stream(
state,
&plan,
report_context.as_ref(),
aether_data::repository::candidates::RequestCandidateStatus::Streaming,
RequestCandidateStatus::Streaming,
Some(status_code),
None,
None,
@@ -653,6 +659,7 @@ async fn execute_stream_from_frame_stream(
let direct_stream_finalize_kind_owned = direct_stream_finalize_kind.clone();
let candidate_started_unix_secs_for_report = candidate_started_unix_secs;
let request_id_for_report = request_id.to_string();
let request_id_for_report_log = short_request_id(request_id);
let candidate_id_for_report = candidate_id.map(ToOwned::to_owned);
tokio::spawn(async move {
let mut provider_buffered_body = provider_prefetched_body_for_report;
@@ -671,7 +678,7 @@ async fn execute_stream_from_frame_stream(
event_name = "stream_execution_frame_decode_failed",
log_type = "ops",
trace_id = %trace_id_owned,
request_id = %request_id_for_report,
request_id = %request_id_for_report_log,
candidate_id = ?candidate_id_for_report.as_deref(),
error = ?err,
"gateway failed to decode execution runtime stream frame"
@@ -697,7 +704,7 @@ async fn execute_stream_from_frame_stream(
event_name = "stream_execution_chunk_decode_failed",
log_type = "ops",
trace_id = %trace_id_owned,
request_id = %request_id_for_report,
request_id = %request_id_for_report_log,
candidate_id = ?candidate_id_for_report.as_deref(),
error = %err,
"gateway failed to decode execution runtime chunk"
@@ -731,7 +738,7 @@ async fn execute_stream_from_frame_stream(
event_name = "stream_execution_chunk_normalize_failed",
log_type = "ops",
trace_id = %trace_id_owned,
request_id = %request_id_for_report,
request_id = %request_id_for_report_log,
candidate_id = ?candidate_id_for_report.as_deref(),
error = ?err,
"gateway failed to normalize execution runtime stream chunk"
@@ -756,7 +763,7 @@ async fn execute_stream_from_frame_stream(
event_name = "stream_execution_chunk_rewrite_failed",
log_type = "ops",
trace_id = %trace_id_owned,
request_id = %request_id_for_report,
request_id = %request_id_for_report_log,
candidate_id = ?candidate_id_for_report.as_deref(),
error = ?err,
"gateway failed to rewrite execution runtime stream chunk"
@@ -783,7 +790,7 @@ async fn execute_stream_from_frame_stream(
event_name = "stream_execution_downstream_disconnected",
log_type = "ops",
trace_id = %trace_id_owned,
request_id = %request_id_for_report,
request_id = %request_id_for_report_log,
candidate_id = ?candidate_id_for_report.as_deref(),
"gateway stream downstream dropped; stopping execution runtime stream forwarding"
);
@@ -804,7 +811,7 @@ async fn execute_stream_from_frame_stream(
event_name = "stream_execution_error_frame",
log_type = "ops",
trace_id = %trace_id_owned,
request_id = %request_id_for_report,
request_id = %request_id_for_report_log,
candidate_id = ?candidate_id_for_report.as_deref(),
error = %error.message,
"execution runtime stream emitted error frame"
@@ -839,7 +846,7 @@ async fn execute_stream_from_frame_stream(
event_name = "stream_execution_normalized_flush_rewrite_failed",
log_type = "ops",
trace_id = %trace_id_owned,
request_id = %request_id_for_report,
request_id = %request_id_for_report_log,
candidate_id = ?candidate_id_for_report.as_deref(),
error = ?err,
"gateway failed to rewrite normalized private stream chunk during flush"
@@ -864,7 +871,7 @@ async fn execute_stream_from_frame_stream(
event_name = "stream_execution_downstream_flush_disconnected",
log_type = "ops",
trace_id = %trace_id_owned,
request_id = %request_id_for_report,
request_id = %request_id_for_report_log,
candidate_id = ?candidate_id_for_report.as_deref(),
"gateway stream downstream dropped while flushing private stream normalization"
);
@@ -878,7 +885,7 @@ async fn execute_stream_from_frame_stream(
event_name = "stream_execution_normalization_flush_failed",
log_type = "ops",
trace_id = %trace_id_owned,
request_id = %request_id_for_report,
request_id = %request_id_for_report_log,
candidate_id = ?candidate_id_for_report.as_deref(),
error = ?err,
"gateway failed to flush private stream normalization"
@@ -903,7 +910,7 @@ async fn execute_stream_from_frame_stream(
event_name = "stream_execution_downstream_rewrite_flush_disconnected",
log_type = "ops",
trace_id = %trace_id_owned,
request_id = %request_id_for_report,
request_id = %request_id_for_report_log,
candidate_id = ?candidate_id_for_report.as_deref(),
"gateway stream downstream dropped while flushing local stream rewrite"
);
@@ -916,7 +923,7 @@ async fn execute_stream_from_frame_stream(
event_name = "stream_execution_rewrite_flush_failed",
log_type = "ops",
trace_id = %trace_id_owned,
request_id = %request_id_for_report,
request_id = %request_id_for_report_log,
candidate_id = ?candidate_id_for_report.as_deref(),
error = ?err,
"gateway failed to flush local stream rewrite"
@@ -974,7 +981,7 @@ async fn execute_stream_from_frame_stream(
&state_for_report,
&plan_for_report,
report_context_owned.as_ref(),
aether_data::repository::candidates::RequestCandidateStatus::Cancelled,
RequestCandidateStatus::Cancelled,
Some(499),
Some("downstream_disconnect".to_string()),
Some("client disconnected before stream completion".to_string()),
@@ -1029,7 +1036,7 @@ async fn execute_stream_from_frame_stream(
&state_for_report,
&plan_for_report,
report_context_owned.as_ref(),
aether_data::repository::candidates::RequestCandidateStatus::Success,
RequestCandidateStatus::Success,
Some(status_code),
None,
None,
@@ -1048,7 +1055,7 @@ async fn execute_stream_from_frame_stream(
event_name = "execution_report_submit_failed",
log_type = "ops",
trace_id = %trace_id_owned,
request_id = %request_id_for_report,
request_id = %request_id_for_report_log,
candidate_id = ?candidate_id_for_report.as_deref(),
report_scope = "stream",
error = ?err,

13
apps/aether-gateway/src/execution_runtime/stream/execution_failures.rs
View File

@@ -1,4 +1,5 @@
use aether_contracts::{ExecutionError, ExecutionPlan, ExecutionTelemetry};
use aether_data_contracts::repository::candidates::RequestCandidateStatus;
use axum::body::Body;
use axum::http::Response;
use base64::Engine as _;
@@ -6,14 +7,13 @@ use serde_json::{Map, Value};
use tracing::warn;
use crate::api::response::attach_control_metadata_headers;
use crate::clock::current_unix_secs as current_request_candidate_unix_secs;
use crate::control::GatewayControlDecision;
use crate::execution_runtime::submission::{
resolve_core_error_background_report_kind, submit_local_core_error_or_sync_finalize,
};
use crate::scheduler::{
current_unix_secs as current_request_candidate_unix_secs,
record_report_request_candidate_status,
};
use crate::log_ids::short_request_id;
use crate::request_candidate_runtime::record_report_request_candidate_status;
use crate::usage::submit_sync_report;
use crate::{usage::GatewaySyncReportRequest, AppState, GatewayError};
@@ -126,7 +126,7 @@ async fn record_stream_sync_failure(
record_report_request_candidate_status(
state,
report_context,
aether_data::repository::candidates::RequestCandidateStatus::Failed,
RequestCandidateStatus::Failed,
Some(failure.status_code),
Some(failure.error_type.clone()),
Some(failure.error_message.clone()),
@@ -220,11 +220,12 @@ pub(super) async fn submit_midstream_stream_failure(
)
.await;
if let Err(err) = submit_sync_report(state, trace_id, payload).await {
let request_id = short_request_id(plan.request_id.as_str());
warn!(
event_name = "execution_report_submit_failed",
log_type = "ops",
trace_id = %trace_id,
request_id = %plan.request_id,
request_id = %request_id,
candidate_id = ?plan.candidate_id,
report_scope = "stream_failure",
error = ?err,

44
apps/aether-gateway/src/execution_runtime/stream/mod.rs
View File

@@ -1,48 +1,4 @@
use axum::body::{Body, Bytes};
use axum::http::Response;
use crate::control::GatewayControlDecision;
use crate::{AppState, GatewayError};
mod error;
mod execution;
pub(crate) use execution::execute_execution_runtime_stream;
pub(crate) async fn maybe_execute_via_execution_runtime_stream(
state: &AppState,
parts: &http::request::Parts,
body_bytes: &Bytes,
trace_id: &str,
decision: Option<&GatewayControlDecision>,
) -> Result<Option<Response<Body>>, GatewayError> {
let Some(decision) = decision else {
return Ok(None);
};
#[cfg(not(test))]
{
let _ = state;
if parts.method != http::Method::POST {
return Ok(None);
}
return crate::executor::maybe_execute_stream_local_path(
state, parts, body_bytes, trace_id, decision,
)
.await;
}
#[cfg(test)]
{
if state
.execution_runtime_override_base_url()
.unwrap_or_default()
.is_empty()
&& parts.method != http::Method::POST
{
return Ok(None);
}
crate::executor::maybe_execute_stream_local_path(
state, parts, body_bytes, trace_id, decision,
)
.await
}
}

146
apps/aether-gateway/src/execution_runtime/submission.rs
View File

@@ -29,34 +29,10 @@ pub(super) fn maybe_build_local_core_error_response(
return Ok(None);
}
let Some(body_json) = payload.body_json.as_ref() else {
return Ok(None);
};
let mut body_json = body_json.clone();
if let Some(report_context) = payload.report_context.as_ref() {
if let Some(unwrapped) =
unwrap_local_finalize_response_value(body_json.clone(), report_context)?
{
body_json = unwrapped;
}
}
let Some(body_object) = body_json.as_object() else {
return Ok(None);
};
if !body_object.contains_key("error")
&& !body_object
.get("type")
.and_then(|value| value.as_str())
.is_some_and(|value| value == "error")
{
return Ok(None);
}
let Some(response_body_json) = build_best_effort_local_core_error_body(payload, &body_json)?
else {
let Some(response_body_json) = resolve_local_core_error_response_body_json(payload)? else {
return Ok(None);
};
let status_source_json = resolve_local_sync_source_body_json(payload)?;
let mut response_headers = payload.headers.clone();
response_headers.remove("content-encoding");
@@ -68,7 +44,11 @@ pub(super) fn maybe_build_local_core_error_response(
response_headers.insert("content-length".to_string(), body_bytes.len().to_string());
Ok(Some(build_client_response_from_parts(
resolve_local_sync_error_status_code(payload.status_code, &body_json),
status_source_json
.as_ref()
.map_or(payload.status_code, |body_json| {
resolve_local_sync_error_status_code(payload.status_code, body_json)
}),
&response_headers,
Body::from(body_bytes),
trace_id,
@@ -83,25 +63,13 @@ fn maybe_resolve_local_sync_response_body_json(
return Ok(Some(client_body_json));
}
let Some(mut body_json) = payload.body_json.clone() else {
return Ok(None);
};
if let Some(report_context) = payload.report_context.as_ref() {
if let Some(unwrapped) =
unwrap_local_finalize_response_value(body_json.clone(), report_context)?
{
body_json = unwrapped;
}
}
if is_core_error_finalize_kind(payload.report_kind.as_str()) {
if let Some(converted) = build_best_effort_local_core_error_body(payload, &body_json)? {
if let Some(converted) = resolve_local_core_error_response_body_json(payload)? {
return Ok(Some(converted));
}
}
Ok(Some(body_json))
resolve_local_sync_source_body_json(payload)
}
fn build_local_sync_response_from_json(
@@ -216,6 +184,102 @@ pub(crate) fn build_best_effort_local_core_error_body(
))
}
pub(crate) fn resolve_local_core_error_response_body_json(
payload: &GatewaySyncReportRequest,
) -> Result<Option<serde_json::Value>, GatewayError> {
if !is_core_error_finalize_kind(payload.report_kind.as_str()) {
return Ok(None);
}
if let Some(client_body_json) = payload.client_body_json.clone() {
return Ok(Some(client_body_json));
}
if let Some(body_json) = resolve_local_sync_source_body_json(payload)? {
if let Some(converted) = build_best_effort_local_core_error_body(payload, &body_json)? {
return Ok(Some(converted));
}
return Ok(Some(body_json));
}
let Some(body_text) = decode_local_sync_body_text(payload)? else {
return Ok(None);
};
let client_api_format = resolve_local_sync_client_api_format(payload);
if client_api_format.is_empty() {
return Ok(None);
}
let kind =
classify_local_sync_error_kind(payload.status_code, None, None, None, body_text.as_str());
Ok(build_core_error_body_for_client_format(
&client_api_format,
body_text.as_str(),
None,
kind,
))
}
fn resolve_local_sync_source_body_json(
payload: &GatewaySyncReportRequest,
) -> Result<Option<serde_json::Value>, GatewayError> {
let body_json = if let Some(body_json) = payload.body_json.clone() {
body_json
} else if let Some(body_base64) = payload.body_base64.as_deref() {
let body_bytes = base64::engine::general_purpose::STANDARD
.decode(body_base64)
.map_err(|err| GatewayError::Internal(err.to_string()))?;
let stripped = strip_utf8_bom_and_ws(&body_bytes);
let Ok(body_json) = serde_json::from_slice::<serde_json::Value>(stripped) else {
return Ok(None);
};
body_json
} else {
return Ok(None);
};
if let Some(report_context) = payload.report_context.as_ref() {
if let Some(unwrapped) =
unwrap_local_finalize_response_value(body_json.clone(), report_context)?
{
return Ok(Some(unwrapped));
}
}
Ok(Some(body_json))
}
fn decode_local_sync_body_text(
payload: &GatewaySyncReportRequest,
) -> Result<Option<String>, GatewayError> {
let Some(body_base64) = payload.body_base64.as_deref() else {
return Ok(None);
};
let body_bytes = base64::engine::general_purpose::STANDARD
.decode(body_base64)
.map_err(|err| GatewayError::Internal(err.to_string()))?;
let stripped = strip_utf8_bom_and_ws(&body_bytes);
let body_text = String::from_utf8_lossy(stripped).trim().to_string();
if body_text.is_empty() {
return Ok(None);
}
Ok(Some(body_text))
}
fn resolve_local_sync_client_api_format(payload: &GatewaySyncReportRequest) -> String {
let default_api_format = core_error_default_client_api_format(payload.report_kind.as_str())
.unwrap_or_default()
.to_string();
payload
.report_context
.as_ref()
.and_then(|value| value.get("client_api_format"))
.and_then(|value| value.as_str())
.unwrap_or(default_api_format.as_str())
.trim()
.to_ascii_lowercase()
}
pub(crate) fn resolve_core_error_background_report_kind(report_kind: &str) -> Option<String> {
core_error_background_report_kind(report_kind).map(ToOwned::to_owned)
}

40
apps/aether-gateway/src/execution_runtime/sync/execution.rs
View File

@@ -1,6 +1,8 @@
use std::collections::BTreeMap;
use aether_contracts::{ExecutionPlan, ExecutionResult, ExecutionTelemetry};
use aether_data_contracts::repository::candidates::RequestCandidateStatus;
use aether_scheduler_core::execution_error_details;
use axum::body::Body;
use axum::http::Response;
use base64::Engine as _;
@@ -11,18 +13,20 @@ use crate::ai_pipeline::finalize::maybe_build_sync_finalize_outcome;
use crate::api::response::{
attach_control_metadata_headers, build_client_response, build_client_response_from_parts,
};
use crate::clock::current_unix_secs as current_request_candidate_unix_secs;
use crate::constants::{CONTROL_CANDIDATE_ID_HEADER, CONTROL_REQUEST_ID_HEADER};
use crate::control::GatewayControlDecision;
#[cfg(test)]
use crate::execution_runtime::remote_compat::post_sync_plan_to_remote_execution_runtime;
use crate::execution_runtime::submission::submit_local_core_error_or_sync_finalize;
use crate::execution_runtime::transport::DirectSyncExecutionRuntime;
use crate::scheduler::{
current_unix_secs as current_request_candidate_unix_secs,
ensure_execution_request_candidate_slot, execution_error_details,
record_local_request_candidate_status, resolve_core_sync_error_finalize_report_kind,
should_fallback_to_control_sync, should_finalize_sync_response,
should_retry_next_local_candidate_sync,
use crate::execution_runtime::{
resolve_core_sync_error_finalize_report_kind, should_fallback_to_control_sync,
should_finalize_sync_response, should_retry_next_local_candidate_sync,
};
use crate::log_ids::short_request_id;
use crate::request_candidate_runtime::{
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
};
use crate::usage::{spawn_sync_report, submit_sync_report};
use crate::video_tasks::VideoTaskSyncReportMode;
@@ -77,6 +81,7 @@ pub(crate) async fn execute_execution_runtime_sync(
) -> Result<Option<Response<Body>>, GatewayError> {
ensure_execution_request_candidate_slot(state, &mut plan, &mut report_context).await;
let plan_request_id = plan.request_id.as_str();
let plan_request_id_for_log = short_request_id(plan_request_id);
let plan_candidate_id = plan.candidate_id.as_deref();
#[cfg(not(test))]
let result = {
@@ -90,7 +95,7 @@ pub(crate) async fn execute_execution_runtime_sync(
event_name = "sync_execution_runtime_unavailable",
log_type = "ops",
trace_id = %trace_id,
request_id = %plan_request_id,
request_id = %plan_request_id_for_log,
candidate_id = ?plan_candidate_id,
error = %err,
"gateway in-process sync execution unavailable"
@@ -115,7 +120,7 @@ pub(crate) async fn execute_execution_runtime_sync(
event_name = "sync_execution_runtime_unavailable",
log_type = "ops",
trace_id = %trace_id,
request_id = %plan_request_id,
request_id = %plan_request_id_for_log,
candidate_id = ?plan_candidate_id,
error = %err,
"gateway in-process sync execution unavailable"
@@ -158,7 +163,7 @@ pub(crate) async fn execute_execution_runtime_sync(
state,
&plan,
report_context.as_ref(),
aether_data::repository::candidates::RequestCandidateStatus::Failed,
RequestCandidateStatus::Failed,
Some(result.status_code),
result_error_type.clone(),
result_error_message.clone(),
@@ -171,7 +176,7 @@ pub(crate) async fn execute_execution_runtime_sync(
event_name = "local_sync_candidate_retry_scheduled",
log_type = "event",
trace_id = %trace_id,
request_id = %plan_request_id,
request_id = %plan_request_id_for_log,
status_code = result.status_code,
"gateway local sync decision retrying next candidate after retryable execution runtime result"
);
@@ -180,6 +185,7 @@ pub(crate) async fn execute_execution_runtime_sync(
let request_id = (!result.request_id.trim().is_empty())
.then_some(result.request_id.as_str())
.or(Some(plan_request_id));
let request_id_for_log = short_request_id(request_id.unwrap_or("-"));
let candidate_id = result.candidate_id.as_deref().or(plan_candidate_id);
let mut headers = result.headers.clone();
let (body_bytes, body_json, body_base64) = decode_execution_result_body(&result, &mut headers)?;
@@ -223,7 +229,7 @@ pub(crate) async fn execute_execution_runtime_sync(
state,
&plan,
report_context.as_ref(),
aether_data::repository::candidates::RequestCandidateStatus::Failed,
RequestCandidateStatus::Failed,
Some(result.status_code),
result_error_type.clone(),
result_error_message.clone(),
@@ -245,9 +251,9 @@ pub(crate) async fn execute_execution_runtime_sync(
&plan,
report_context.as_ref(),
if result.status_code >= 400 {
aether_data::repository::candidates::RequestCandidateStatus::Failed
RequestCandidateStatus::Failed
} else {
aether_data::repository::candidates::RequestCandidateStatus::Success
RequestCandidateStatus::Success
},
Some(result.status_code),
result_error_type.clone(),
@@ -407,7 +413,7 @@ pub(crate) async fn execute_execution_runtime_sync(
event_name = "local_video_finalize_missing_success_report_mapping",
log_type = "ops",
trace_id = %trace_id,
request_id = request_id.unwrap_or("-"),
request_id = %request_id_for_log,
candidate_id = ?candidate_id,
report_kind = %payload.report_kind,
"gateway local video finalize produced response without background success report mapping"
@@ -449,7 +455,7 @@ pub(crate) async fn execute_execution_runtime_sync(
event_name = "local_video_finalize_missing_error_report_mapping",
log_type = "ops",
trace_id = %trace_id,
request_id = request_id.unwrap_or("-"),
request_id = %request_id_for_log,
candidate_id = ?candidate_id,
report_kind = %payload.report_kind,
"gateway local video finalize produced response without background error report mapping"
@@ -580,7 +586,7 @@ async fn execute_sync_via_remote_execution_runtime(
event_name = "sync_execution_runtime_remote_unavailable",
log_type = "ops",
trace_id = %trace_id,
request_id = %plan_request_id,
request_id = %short_request_id(plan_request_id),
candidate_id = ?plan_candidate_id,
error = ?err,
"gateway remote execution runtime sync unavailable"
@@ -595,7 +601,7 @@ async fn execute_sync_via_remote_execution_runtime(
state,
plan,
report_context,
aether_data::repository::candidates::RequestCandidateStatus::Failed,
RequestCandidateStatus::Failed,
Some(response.status().as_u16()),
Some("execution_runtime_http_error".to_string()),
Some(format!(

44
apps/aether-gateway/src/execution_runtime/sync/mod.rs
View File

@@ -1,9 +1,3 @@
use axum::body::{Body, Bytes};
use axum::http::Response;
use crate::control::GatewayControlDecision;
use crate::{AppState, GatewayError};
mod execution;
pub(crate) use execution::execute_execution_runtime_sync;
@@ -14,41 +8,3 @@ pub(crate) use execution::{
maybe_build_local_video_success_outcome, resolve_local_sync_error_background_report_kind,
resolve_local_sync_success_background_report_kind, LocalVideoSyncSuccessOutcome,
};
pub(crate) async fn maybe_execute_via_execution_runtime_sync(
state: &AppState,
parts: &http::request::Parts,
body_bytes: &Bytes,
trace_id: &str,
decision: Option<&GatewayControlDecision>,
) -> Result<Option<Response<Body>>, GatewayError> {
let Some(decision) = decision else {
return Ok(None);
};
#[cfg(not(test))]
{
let _ = state;
if parts.method != http::Method::POST {
return Ok(None);
}
return crate::executor::maybe_execute_sync_local_path(
state, parts, body_bytes, trace_id, decision,
)
.await;
}
#[cfg(test)]
{
if state
.execution_runtime_override_base_url()
.unwrap_or_default()
.is_empty()
&& parts.method != http::Method::POST
{
return Ok(None);
}
crate::executor::maybe_execute_sync_local_path(
state, parts, body_bytes, trace_id, decision,
)
.await
}
}

72
apps/aether-gateway/src/execution_runtime/tests.rs
View File

@@ -1,5 +1,6 @@
use aether_contracts::{ExecutionPlan, RequestBody};
use axum::http::Request;
use base64::Engine as _;
use serde_json::json;
use crate::ai_pipeline::contracts::GatewayControlSyncDecisionResponse;
@@ -11,13 +12,15 @@ use crate::ai_pipeline::planner::plan_builders::{
};
use crate::execution_runtime::submission::{
build_best_effort_local_core_error_body, resolve_core_error_background_report_kind,
resolve_core_success_background_report_kind,
resolve_core_success_background_report_kind, resolve_local_core_error_response_body_json,
};
use crate::execution_runtime::{
resolve_local_sync_error_background_report_kind,
resolve_local_sync_success_background_report_kind,
};
use crate::intent::{should_bypass_intent_decision, should_bypass_intent_plan};
use crate::executor::{
should_bypass_execution_runtime_decision, should_bypass_execution_runtime_plan,
};
use crate::usage::GatewaySyncReportRequest;
fn test_parts() -> http::request::Parts {
@@ -226,6 +229,67 @@ fn build_best_effort_local_core_error_body_converts_claude_cli_error_to_openai_c
);
}
#[test]
fn resolve_local_core_error_response_body_json_parses_body_base64_json_for_cross_format_cli_error()
{
let mut payload = core_finalize_payload(
"claude_cli_sync_finalize",
"claude:cli",
"openai:cli",
401,
json!({}),
);
payload.body_json = None;
payload.body_base64 = Some(
base64::engine::general_purpose::STANDARD
.encode(r#"{"error":{"message":"invalid auth token","type":"authentication_error"}}"#),
);
let resolved = resolve_local_core_error_response_body_json(&payload)
.expect("resolution should not error")
.expect("resolution should produce a client error body");
assert_eq!(
resolved,
json!({
"type": "error",
"error": {
"message": "invalid auth token",
"type": "authentication_error"
}
})
);
}
#[test]
fn resolve_local_core_error_response_body_json_builds_client_error_from_plain_text_body() {
let mut payload = core_finalize_payload(
"claude_cli_sync_finalize",
"claude:cli",
"openai:cli",
400,
json!({}),
);
payload.body_json = None;
payload.body_base64 =
Some(base64::engine::general_purpose::STANDARD.encode("invalid model for this endpoint"));
let resolved = resolve_local_core_error_response_body_json(&payload)
.expect("resolution should not error")
.expect("resolution should produce a client error body");
assert_eq!(
resolved,
json!({
"type": "error",
"error": {
"message": "invalid model for this endpoint",
"type": "invalid_request_error"
}
})
);
}
#[test]
fn resolve_local_sync_success_background_report_kind_maps_video_finalize_kinds() {
let cases = [
@@ -527,7 +591,7 @@ fn bypasses_execution_runtime_for_codex_backendapi_variant() {
payload.client_api_format = Some("openai:cli".to_string());
payload.upstream_url = Some("https://chatgpt.com/backendapi/codex/responses".to_string());
assert!(should_bypass_intent_decision(&payload));
assert!(should_bypass_execution_runtime_decision(&payload));
}
#[test]
@@ -554,5 +618,5 @@ fn bypasses_execution_runtime_for_codex_plan_variant() {
timeouts: None,
};
assert!(should_bypass_intent_plan(&plan));
assert!(should_bypass_execution_runtime_plan(&plan));
}