mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 01:10:23 +08:00
refactor: 移除独立 hub/proxy/executor/gateway crate，统一为 gateway tunnel 架构
- 删除 aether-hub、aether-proxy 独立项目及其 Dockerfile/配置 - 删除 crates/aether-executor 和 crates/aether-gateway 全部模块 - 新增 apps/ 目录作为应用入口 - 将 hub 概念重构为 gateway tunnel transport - 将 executor 重构为 execution runtime - 新增 tunnel.rs 合约定义和 testkit tunnel/execution_runtime 模块 - 更新 Python 服务层和测试适配新架构命名
This commit is contained in:
fawney19
617
apps/aether-gateway/src/execution_runtime/sync/execution.rs
Normal file
617
apps/aether-gateway/src/execution_runtime/sync/execution.rs
Normal file
@@ -0,0 +1,617 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aether_contracts::{ExecutionPlan, ExecutionResult, ExecutionTelemetry};
|
||||
use axum::body::Body;
|
||||
use axum::http::Response;
|
||||
use base64::Engine as _;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::gateway::constants::{CONTROL_CANDIDATE_ID_HEADER, CONTROL_REQUEST_ID_HEADER};
|
||||
#[cfg(test)]
|
||||
use crate::gateway::execution_runtime::remote_compat::post_sync_plan_to_remote_execution_runtime;
|
||||
use crate::gateway::execution_runtime::submission::submit_local_core_error_or_sync_finalize;
|
||||
use crate::gateway::execution_runtime::transport::DirectSyncExecutionRuntime;
|
||||
use crate::gateway::request_candidates::{
|
||||
current_unix_secs as current_request_candidate_unix_secs,
|
||||
ensure_execution_request_candidate_slot, execution_error_details,
|
||||
record_local_request_candidate_status,
|
||||
};
|
||||
use crate::gateway::scheduler::{
|
||||
resolve_core_sync_error_finalize_report_kind, should_fallback_to_control_sync,
|
||||
should_finalize_sync_response, should_retry_next_local_candidate_sync,
|
||||
};
|
||||
use crate::gateway::usage::{spawn_sync_report, submit_sync_report};
|
||||
use crate::gateway::video_tasks::VideoTaskSyncReportMode;
|
||||
use crate::gateway::{
|
||||
attach_control_metadata_headers, build_client_response, build_client_response_from_parts,
|
||||
maybe_build_sync_finalize_outcome, AppState, GatewayControlDecision, GatewayError,
|
||||
GatewaySyncReportRequest,
|
||||
};
|
||||
|
||||
#[path = "execution/policy.rs"]
|
||||
mod policy;
|
||||
#[path = "execution/response.rs"]
|
||||
mod response;
|
||||
|
||||
use policy::decode_execution_result_body;
|
||||
pub(crate) use response::{
|
||||
maybe_build_local_sync_finalize_response, maybe_build_local_video_error_response,
|
||||
maybe_build_local_video_success_outcome, resolve_local_sync_error_background_report_kind,
|
||||
resolve_local_sync_success_background_report_kind, LocalVideoSyncSuccessOutcome,
|
||||
};
|
||||
|
||||
struct ImplicitSyncFinalizeOutcome {
|
||||
payload: GatewaySyncReportRequest,
|
||||
outcome: crate::gateway::ai_pipeline::finalize::LocalCoreSyncFinalizeOutcome,
|
||||
}
|
||||
|
||||
async fn record_sync_terminal_usage(
|
||||
state: &AppState,
|
||||
plan: &ExecutionPlan,
|
||||
report_context: Option<&serde_json::Value>,
|
||||
payload: &GatewaySyncReportRequest,
|
||||
) {
|
||||
state
|
||||
.usage_runtime
|
||||
.record_sync_terminal(state.data.as_ref(), plan, report_context, payload)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
enum RemoteSyncFallbackOutcome {
|
||||
Executed(ExecutionResult),
|
||||
ClientResponse(Response<Body>),
|
||||
Unavailable,
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)] // internal function, grouping would add unnecessary indirection
|
||||
pub(crate) async fn execute_execution_runtime_sync(
|
||||
state: &AppState,
|
||||
request_path: &str,
|
||||
mut plan: ExecutionPlan,
|
||||
trace_id: &str,
|
||||
decision: &GatewayControlDecision,
|
||||
plan_kind: &str,
|
||||
report_kind: Option<String>,
|
||||
mut report_context: Option<serde_json::Value>,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
ensure_execution_request_candidate_slot(state, &mut plan, &mut report_context).await;
|
||||
let plan_request_id = plan.request_id.as_str();
|
||||
let plan_candidate_id = plan.candidate_id.as_deref();
|
||||
#[cfg(not(test))]
|
||||
let result = {
|
||||
match DirectSyncExecutionRuntime::new()
|
||||
.execute_sync(plan.clone())
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
trace_id = %trace_id,
|
||||
request_id = %plan_request_id,
|
||||
candidate_id = ?plan_candidate_id,
|
||||
error = %err,
|
||||
"gateway in-process sync execution unavailable"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
};
|
||||
#[cfg(test)]
|
||||
let result = {
|
||||
let remote_execution_runtime_base_url = state
|
||||
.test_remote_execution_runtime_base_url()
|
||||
.unwrap_or_default();
|
||||
if remote_execution_runtime_base_url.trim().is_empty() {
|
||||
match DirectSyncExecutionRuntime::new()
|
||||
.execute_sync(plan.clone())
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
trace_id = %trace_id,
|
||||
request_id = %plan_request_id,
|
||||
candidate_id = ?plan_candidate_id,
|
||||
error = %err,
|
||||
"gateway in-process sync execution unavailable"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let remote_outcome = execute_sync_via_remote_execution_runtime(
|
||||
state,
|
||||
remote_execution_runtime_base_url,
|
||||
trace_id,
|
||||
decision,
|
||||
&plan,
|
||||
plan_request_id,
|
||||
plan_candidate_id,
|
||||
report_context.as_ref(),
|
||||
)
|
||||
.await?;
|
||||
match remote_outcome {
|
||||
RemoteSyncFallbackOutcome::Executed(result) => result,
|
||||
RemoteSyncFallbackOutcome::ClientResponse(response) => return Ok(Some(response)),
|
||||
RemoteSyncFallbackOutcome::Unavailable => return Ok(None),
|
||||
}
|
||||
}
|
||||
};
|
||||
let result_body_json = result
|
||||
.body
|
||||
.as_ref()
|
||||
.and_then(|body| body.json_body.as_ref());
|
||||
let (result_error_type, result_error_message) =
|
||||
execution_error_details(result.error.as_ref(), result_body_json);
|
||||
let result_latency_ms = result
|
||||
.telemetry
|
||||
.as_ref()
|
||||
.and_then(|telemetry| telemetry.elapsed_ms);
|
||||
if should_retry_next_local_candidate_sync(plan_kind, report_context.as_ref(), &result) {
|
||||
let terminal_unix_secs = current_request_candidate_unix_secs();
|
||||
record_local_request_candidate_status(
|
||||
state,
|
||||
&plan,
|
||||
report_context.as_ref(),
|
||||
aether_data::repository::candidates::RequestCandidateStatus::Failed,
|
||||
Some(result.status_code),
|
||||
result_error_type.clone(),
|
||||
result_error_message.clone(),
|
||||
result_latency_ms,
|
||||
Some(terminal_unix_secs),
|
||||
Some(terminal_unix_secs),
|
||||
)
|
||||
.await;
|
||||
warn!(
|
||||
trace_id = %trace_id,
|
||||
request_id = %plan_request_id,
|
||||
status_code = result.status_code,
|
||||
"gateway local sync decision retrying next candidate after retryable execution runtime result"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
let request_id = (!result.request_id.trim().is_empty())
|
||||
.then_some(result.request_id.as_str())
|
||||
.or(Some(plan_request_id));
|
||||
let candidate_id = result.candidate_id.as_deref().or(plan_candidate_id);
|
||||
let mut headers = result.headers.clone();
|
||||
let (body_bytes, body_json, body_base64) = decode_execution_result_body(&result, &mut headers)?;
|
||||
let has_body_bytes = body_base64.is_some();
|
||||
let explicit_finalize = should_finalize_sync_response(report_kind.as_deref());
|
||||
let mapped_error_finalize_kind =
|
||||
resolve_core_sync_error_finalize_report_kind(plan_kind, &result, body_json.as_ref());
|
||||
let implicit_finalize = if explicit_finalize || mapped_error_finalize_kind.is_some() {
|
||||
None
|
||||
} else {
|
||||
maybe_build_implicit_sync_finalize_outcome(
|
||||
trace_id,
|
||||
decision,
|
||||
plan_kind,
|
||||
report_context.clone(),
|
||||
result.status_code,
|
||||
headers.clone(),
|
||||
body_json.clone(),
|
||||
body_base64.clone(),
|
||||
result.telemetry.clone(),
|
||||
)?
|
||||
};
|
||||
let finalize_report_kind = if explicit_finalize {
|
||||
report_kind.clone()
|
||||
} else if let Some(implicit_finalize) = implicit_finalize.as_ref() {
|
||||
Some(implicit_finalize.payload.report_kind.clone())
|
||||
} else {
|
||||
mapped_error_finalize_kind.clone()
|
||||
};
|
||||
|
||||
if should_fallback_to_control_sync(
|
||||
plan_kind,
|
||||
&result,
|
||||
body_json.as_ref(),
|
||||
has_body_bytes,
|
||||
explicit_finalize || implicit_finalize.is_some(),
|
||||
mapped_error_finalize_kind.is_some(),
|
||||
) {
|
||||
let terminal_unix_secs = current_request_candidate_unix_secs();
|
||||
record_local_request_candidate_status(
|
||||
state,
|
||||
&plan,
|
||||
report_context.as_ref(),
|
||||
aether_data::repository::candidates::RequestCandidateStatus::Failed,
|
||||
Some(result.status_code),
|
||||
result_error_type.clone(),
|
||||
result_error_message.clone(),
|
||||
result_latency_ms,
|
||||
Some(terminal_unix_secs),
|
||||
Some(terminal_unix_secs),
|
||||
)
|
||||
.await;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
state
|
||||
.usage_runtime
|
||||
.record_pending(state.data.as_ref(), &plan, report_context.as_ref())
|
||||
.await;
|
||||
let terminal_unix_secs = current_request_candidate_unix_secs();
|
||||
record_local_request_candidate_status(
|
||||
state,
|
||||
&plan,
|
||||
report_context.as_ref(),
|
||||
if result.status_code >= 400 {
|
||||
aether_data::repository::candidates::RequestCandidateStatus::Failed
|
||||
} else {
|
||||
aether_data::repository::candidates::RequestCandidateStatus::Success
|
||||
},
|
||||
Some(result.status_code),
|
||||
result_error_type.clone(),
|
||||
result_error_message.clone(),
|
||||
result_latency_ms,
|
||||
Some(terminal_unix_secs),
|
||||
Some(terminal_unix_secs),
|
||||
)
|
||||
.await;
|
||||
|
||||
let base_usage_payload = GatewaySyncReportRequest {
|
||||
trace_id: trace_id.to_string(),
|
||||
report_kind: finalize_report_kind
|
||||
.clone()
|
||||
.or_else(|| report_kind.clone())
|
||||
.unwrap_or_default(),
|
||||
report_context: report_context.clone(),
|
||||
status_code: result.status_code,
|
||||
headers: headers.clone(),
|
||||
body_json: body_json.clone(),
|
||||
client_body_json: None,
|
||||
body_base64: body_base64.clone(),
|
||||
telemetry: result.telemetry.clone(),
|
||||
};
|
||||
|
||||
if let Some(finalize_report_kind) = finalize_report_kind {
|
||||
if let Some(implicit_finalize) = implicit_finalize {
|
||||
let usage_payload = implicit_finalize
|
||||
.outcome
|
||||
.background_report
|
||||
.as_ref()
|
||||
.unwrap_or(&implicit_finalize.payload);
|
||||
record_sync_terminal_usage(state, &plan, report_context.as_ref(), usage_payload).await;
|
||||
if let Some(report_payload) = implicit_finalize.outcome.background_report {
|
||||
spawn_sync_report(state.clone(), trace_id.to_string(), report_payload);
|
||||
} else {
|
||||
warn!(
|
||||
trace_id = %trace_id,
|
||||
report_kind = %implicit_finalize.payload.report_kind,
|
||||
"gateway implicit local core finalize produced response without background success report mapping"
|
||||
);
|
||||
}
|
||||
return Ok(Some(attach_control_metadata_headers(
|
||||
implicit_finalize.outcome.response,
|
||||
request_id,
|
||||
candidate_id,
|
||||
)?));
|
||||
}
|
||||
|
||||
let payload = GatewaySyncReportRequest {
|
||||
trace_id: trace_id.to_string(),
|
||||
report_kind: finalize_report_kind,
|
||||
report_context,
|
||||
status_code: result.status_code,
|
||||
headers: headers.clone(),
|
||||
body_json: body_json.clone(),
|
||||
client_body_json: None,
|
||||
body_base64: body_base64.clone(),
|
||||
telemetry: result.telemetry.clone(),
|
||||
};
|
||||
if let Some(outcome) = maybe_build_sync_finalize_outcome(trace_id, decision, &payload)? {
|
||||
let usage_payload = outcome.background_report.as_ref().unwrap_or(&payload);
|
||||
record_sync_terminal_usage(
|
||||
state,
|
||||
&plan,
|
||||
payload.report_context.as_ref(),
|
||||
usage_payload,
|
||||
)
|
||||
.await;
|
||||
if let Some(report_payload) = outcome.background_report {
|
||||
spawn_sync_report(state.clone(), trace_id.to_string(), report_payload);
|
||||
} else {
|
||||
warn!(
|
||||
trace_id = %trace_id,
|
||||
report_kind = %payload.report_kind,
|
||||
"gateway local core finalize produced response without background success report mapping"
|
||||
);
|
||||
}
|
||||
return Ok(Some(attach_control_metadata_headers(
|
||||
outcome.response,
|
||||
request_id,
|
||||
candidate_id,
|
||||
)?));
|
||||
}
|
||||
if let Some(outcome) = maybe_build_local_video_success_outcome(
|
||||
trace_id,
|
||||
decision,
|
||||
&payload,
|
||||
&state.video_tasks,
|
||||
&plan,
|
||||
)? {
|
||||
record_sync_terminal_usage(
|
||||
state,
|
||||
&plan,
|
||||
payload.report_context.as_ref(),
|
||||
&outcome.report_payload,
|
||||
)
|
||||
.await;
|
||||
if let Some(snapshot) = outcome.local_task_snapshot.clone() {
|
||||
state.video_tasks.record_snapshot(snapshot.clone());
|
||||
let _ = state.upsert_video_task_snapshot(&snapshot).await?;
|
||||
}
|
||||
match outcome.report_mode {
|
||||
VideoTaskSyncReportMode::InlineSync => {
|
||||
submit_sync_report(state, trace_id, outcome.report_payload).await?;
|
||||
}
|
||||
VideoTaskSyncReportMode::Background => {
|
||||
spawn_sync_report(state.clone(), trace_id.to_string(), outcome.report_payload);
|
||||
}
|
||||
}
|
||||
return Ok(Some(attach_control_metadata_headers(
|
||||
outcome.response,
|
||||
request_id,
|
||||
candidate_id,
|
||||
)?));
|
||||
}
|
||||
if let Some(response) =
|
||||
maybe_build_local_sync_finalize_response(trace_id, decision, &payload)?
|
||||
{
|
||||
let usage_payload = if let Some(success_report_kind) =
|
||||
resolve_local_sync_success_background_report_kind(payload.report_kind.as_str())
|
||||
{
|
||||
let mut report_payload = payload.clone();
|
||||
report_payload.report_kind = success_report_kind;
|
||||
report_payload
|
||||
} else {
|
||||
payload.clone()
|
||||
};
|
||||
record_sync_terminal_usage(
|
||||
state,
|
||||
&plan,
|
||||
payload.report_context.as_ref(),
|
||||
&usage_payload,
|
||||
)
|
||||
.await;
|
||||
state
|
||||
.video_tasks
|
||||
.apply_finalize_mutation(request_path, payload.report_kind.as_str());
|
||||
if let Some(snapshot) = state
|
||||
.video_tasks
|
||||
.snapshot_for_route(decision.route_family.as_deref(), request_path)
|
||||
{
|
||||
let _ = state.upsert_video_task_snapshot(&snapshot).await?;
|
||||
}
|
||||
if let Some(success_report_kind) =
|
||||
resolve_local_sync_success_background_report_kind(payload.report_kind.as_str())
|
||||
{
|
||||
let mut report_payload = usage_payload;
|
||||
report_payload.report_kind = success_report_kind;
|
||||
spawn_sync_report(state.clone(), trace_id.to_string(), report_payload);
|
||||
} else {
|
||||
warn!(
|
||||
trace_id = %trace_id,
|
||||
report_kind = %payload.report_kind,
|
||||
"gateway local video finalize produced response without background success report mapping"
|
||||
);
|
||||
}
|
||||
return Ok(Some(attach_control_metadata_headers(
|
||||
response,
|
||||
request_id,
|
||||
candidate_id,
|
||||
)?));
|
||||
}
|
||||
if let Some(response) =
|
||||
maybe_build_local_video_error_response(trace_id, decision, &payload)?
|
||||
{
|
||||
let usage_payload = if let Some(error_report_kind) =
|
||||
resolve_local_sync_error_background_report_kind(payload.report_kind.as_str())
|
||||
{
|
||||
let mut report_payload = payload.clone();
|
||||
report_payload.report_kind = error_report_kind;
|
||||
report_payload
|
||||
} else {
|
||||
payload.clone()
|
||||
};
|
||||
record_sync_terminal_usage(
|
||||
state,
|
||||
&plan,
|
||||
payload.report_context.as_ref(),
|
||||
&usage_payload,
|
||||
)
|
||||
.await;
|
||||
if let Some(error_report_kind) =
|
||||
resolve_local_sync_error_background_report_kind(payload.report_kind.as_str())
|
||||
{
|
||||
let mut report_payload = usage_payload;
|
||||
report_payload.report_kind = error_report_kind;
|
||||
spawn_sync_report(state.clone(), trace_id.to_string(), report_payload);
|
||||
} else {
|
||||
warn!(
|
||||
trace_id = %trace_id,
|
||||
report_kind = %payload.report_kind,
|
||||
"gateway local video finalize produced response without background error report mapping"
|
||||
);
|
||||
}
|
||||
return Ok(Some(attach_control_metadata_headers(
|
||||
response,
|
||||
request_id,
|
||||
candidate_id,
|
||||
)?));
|
||||
}
|
||||
record_sync_terminal_usage(state, &plan, payload.report_context.as_ref(), &payload).await;
|
||||
let response =
|
||||
submit_local_core_error_or_sync_finalize(state, trace_id, decision, payload).await?;
|
||||
return Ok(Some(attach_control_metadata_headers(
|
||||
response,
|
||||
request_id,
|
||||
candidate_id,
|
||||
)?));
|
||||
}
|
||||
|
||||
record_sync_terminal_usage(state, &plan, report_context.as_ref(), &base_usage_payload).await;
|
||||
if let Some(report_kind) = report_kind {
|
||||
let report = GatewaySyncReportRequest {
|
||||
trace_id: trace_id.to_string(),
|
||||
report_kind,
|
||||
report_context,
|
||||
status_code: result.status_code,
|
||||
headers: headers.clone(),
|
||||
body_json: body_json.clone(),
|
||||
client_body_json: None,
|
||||
body_base64: body_base64.clone(),
|
||||
telemetry: result.telemetry.clone(),
|
||||
};
|
||||
spawn_sync_report(state.clone(), trace_id.to_string(), report);
|
||||
}
|
||||
|
||||
let request_id_header: Option<&str> = request_id
|
||||
.map(str::trim)
|
||||
.filter(|value: &&str| !value.is_empty());
|
||||
if let Some(request_id) = request_id_header {
|
||||
headers.insert(
|
||||
CONTROL_REQUEST_ID_HEADER.to_string(),
|
||||
request_id.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
let candidate_id_header: Option<&str> = candidate_id
|
||||
.map(str::trim)
|
||||
.filter(|value: &&str| !value.is_empty());
|
||||
if let Some(candidate_id) = candidate_id_header {
|
||||
headers.insert(
|
||||
CONTROL_CANDIDATE_ID_HEADER.to_string(),
|
||||
candidate_id.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
Ok(Some(build_client_response_from_parts(
|
||||
result.status_code,
|
||||
&headers,
|
||||
Body::from(body_bytes),
|
||||
trace_id,
|
||||
Some(decision),
|
||||
)?))
|
||||
}
|
||||
|
||||
fn resolve_implicit_sync_finalize_report_kind(plan_kind: &str) -> Option<&'static str> {
|
||||
match plan_kind {
|
||||
"openai_chat_sync" => Some("openai_chat_sync_finalize"),
|
||||
"claude_chat_sync" => Some("claude_chat_sync_finalize"),
|
||||
"gemini_chat_sync" => Some("gemini_chat_sync_finalize"),
|
||||
"openai_cli_sync" => Some("openai_cli_sync_finalize"),
|
||||
"openai_compact_sync" => Some("openai_compact_sync_finalize"),
|
||||
"claude_cli_sync" => Some("claude_cli_sync_finalize"),
|
||||
"gemini_cli_sync" => Some("gemini_cli_sync_finalize"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)] // mirrors sync execution context
|
||||
fn maybe_build_implicit_sync_finalize_outcome(
|
||||
trace_id: &str,
|
||||
decision: &GatewayControlDecision,
|
||||
plan_kind: &str,
|
||||
report_context: Option<serde_json::Value>,
|
||||
status_code: u16,
|
||||
headers: BTreeMap<String, String>,
|
||||
body_json: Option<serde_json::Value>,
|
||||
body_base64: Option<String>,
|
||||
telemetry: Option<ExecutionTelemetry>,
|
||||
) -> Result<Option<ImplicitSyncFinalizeOutcome>, GatewayError> {
|
||||
if status_code >= 400 || body_json.is_some() || body_base64.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let Some(report_kind) = resolve_implicit_sync_finalize_report_kind(plan_kind) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let payload = GatewaySyncReportRequest {
|
||||
trace_id: trace_id.to_string(),
|
||||
report_kind: report_kind.to_string(),
|
||||
report_context,
|
||||
status_code,
|
||||
headers,
|
||||
body_json,
|
||||
client_body_json: None,
|
||||
body_base64,
|
||||
telemetry,
|
||||
};
|
||||
let Some(outcome) = maybe_build_sync_finalize_outcome(trace_id, decision, &payload)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
Ok(Some(ImplicitSyncFinalizeOutcome { payload, outcome }))
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)] // internal helper mirroring execute path context
|
||||
#[cfg(test)]
|
||||
async fn execute_sync_via_remote_execution_runtime(
|
||||
state: &AppState,
|
||||
remote_execution_runtime_base_url: &str,
|
||||
trace_id: &str,
|
||||
decision: &GatewayControlDecision,
|
||||
plan: &ExecutionPlan,
|
||||
plan_request_id: &str,
|
||||
plan_candidate_id: Option<&str>,
|
||||
report_context: Option<&serde_json::Value>,
|
||||
) -> Result<RemoteSyncFallbackOutcome, GatewayError> {
|
||||
let response = match post_sync_plan_to_remote_execution_runtime(
|
||||
state,
|
||||
remote_execution_runtime_base_url,
|
||||
Some(trace_id),
|
||||
plan,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(response) => response,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
trace_id = %trace_id,
|
||||
request_id = %plan_request_id,
|
||||
candidate_id = ?plan_candidate_id,
|
||||
error = ?err,
|
||||
"gateway remote execution runtime sync unavailable"
|
||||
);
|
||||
return Ok(RemoteSyncFallbackOutcome::Unavailable);
|
||||
}
|
||||
};
|
||||
|
||||
if response.status() != http::StatusCode::OK {
|
||||
let terminal_unix_secs = current_request_candidate_unix_secs();
|
||||
record_local_request_candidate_status(
|
||||
state,
|
||||
plan,
|
||||
report_context,
|
||||
aether_data::repository::candidates::RequestCandidateStatus::Failed,
|
||||
Some(response.status().as_u16()),
|
||||
Some("execution_runtime_http_error".to_string()),
|
||||
Some(format!(
|
||||
"execution runtime returned HTTP {}",
|
||||
response.status()
|
||||
)),
|
||||
None,
|
||||
Some(terminal_unix_secs),
|
||||
Some(terminal_unix_secs),
|
||||
)
|
||||
.await;
|
||||
return Ok(RemoteSyncFallbackOutcome::ClientResponse(
|
||||
attach_control_metadata_headers(
|
||||
build_client_response(response, trace_id, Some(decision))?,
|
||||
Some(plan_request_id),
|
||||
plan_candidate_id,
|
||||
)?,
|
||||
));
|
||||
}
|
||||
|
||||
response
|
||||
.json()
|
||||
.await
|
||||
.map(RemoteSyncFallbackOutcome::Executed)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aether_contracts::ExecutionResult;
|
||||
use base64::Engine as _;
|
||||
|
||||
use crate::gateway::GatewayError;
|
||||
|
||||
type DecodedBody = (Vec<u8>, Option<serde_json::Value>, Option<String>);
|
||||
|
||||
pub(super) fn decode_execution_result_body(
|
||||
result: &ExecutionResult,
|
||||
headers: &mut BTreeMap<String, String>,
|
||||
) -> Result<DecodedBody, GatewayError> {
|
||||
let Some(body) = result.body.as_ref() else {
|
||||
return Ok((Vec::new(), None, None));
|
||||
};
|
||||
|
||||
if let Some(json_body) = body.json_body.clone() {
|
||||
headers
|
||||
.entry("content-type".to_string())
|
||||
.or_insert_with(|| "application/json".to_string());
|
||||
let bytes = serde_json::to_vec(&json_body)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
headers.insert("content-length".to_string(), bytes.len().to_string());
|
||||
return Ok((bytes, Some(json_body), None));
|
||||
}
|
||||
|
||||
if let Some(body_bytes_b64) = body.body_bytes_b64.clone() {
|
||||
let bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(&body_bytes_b64)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
return Ok((bytes, None, Some(body_bytes_b64)));
|
||||
}
|
||||
|
||||
Ok((Vec::new(), None, None))
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aether_contracts::ExecutionPlan;
|
||||
use axum::body::Body;
|
||||
use axum::http::Response;
|
||||
use serde_json::json;
|
||||
|
||||
use crate::gateway::video_tasks::{LocalVideoTaskSnapshot, VideoTaskSyncReportMode};
|
||||
use crate::gateway::VideoTaskService;
|
||||
use crate::gateway::{
|
||||
build_client_response_from_parts, GatewayControlDecision, GatewayError,
|
||||
GatewaySyncReportRequest,
|
||||
};
|
||||
|
||||
pub(crate) struct LocalVideoSyncSuccessOutcome {
|
||||
pub(crate) response: Response<Body>,
|
||||
pub(crate) report_payload: GatewaySyncReportRequest,
|
||||
pub(crate) report_mode: VideoTaskSyncReportMode,
|
||||
pub(crate) local_task_snapshot: Option<LocalVideoTaskSnapshot>,
|
||||
}
|
||||
|
||||
fn cloned_report_context_object(
|
||||
payload: &GatewaySyncReportRequest,
|
||||
) -> serde_json::Map<String, serde_json::Value> {
|
||||
payload
|
||||
.report_context
|
||||
.clone()
|
||||
.and_then(|value| value.as_object().cloned())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn build_local_video_success_response(
|
||||
trace_id: &str,
|
||||
decision: &GatewayControlDecision,
|
||||
body_json: &serde_json::Value,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let body_bytes =
|
||||
serde_json::to_vec(body_json).map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let mut headers = BTreeMap::new();
|
||||
headers.insert("content-type".to_string(), "application/json".to_string());
|
||||
headers.insert("content-length".to_string(), body_bytes.len().to_string());
|
||||
build_client_response_from_parts(
|
||||
http::StatusCode::OK.as_u16(),
|
||||
&headers,
|
||||
Body::from(body_bytes),
|
||||
trace_id,
|
||||
Some(decision),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn maybe_build_local_video_success_outcome(
|
||||
trace_id: &str,
|
||||
decision: &GatewayControlDecision,
|
||||
payload: &GatewaySyncReportRequest,
|
||||
video_tasks: &VideoTaskService,
|
||||
plan: &ExecutionPlan,
|
||||
) -> Result<Option<LocalVideoSyncSuccessOutcome>, GatewayError> {
|
||||
if payload.status_code >= 400 {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let provider_body = match payload
|
||||
.body_json
|
||||
.as_ref()
|
||||
.and_then(serde_json::Value::as_object)
|
||||
{
|
||||
Some(value) => value,
|
||||
None => return Ok(None),
|
||||
};
|
||||
let mut report_context = cloned_report_context_object(payload);
|
||||
let Some(plan) = video_tasks.prepare_sync_success(
|
||||
payload.report_kind.as_str(),
|
||||
provider_body,
|
||||
&report_context,
|
||||
plan,
|
||||
) else {
|
||||
return Ok(None);
|
||||
};
|
||||
plan.apply_to_report_context(&mut report_context);
|
||||
let client_body_json = plan.client_body_json();
|
||||
|
||||
let response = build_local_video_success_response(trace_id, decision, &client_body_json)?;
|
||||
let report_payload = GatewaySyncReportRequest {
|
||||
trace_id: payload.trace_id.clone(),
|
||||
report_kind: plan.success_report_kind().to_string(),
|
||||
report_context: Some(serde_json::Value::Object(report_context)),
|
||||
status_code: payload.status_code,
|
||||
headers: payload.headers.clone(),
|
||||
body_json: payload.body_json.clone(),
|
||||
client_body_json: Some(client_body_json),
|
||||
body_base64: None,
|
||||
telemetry: payload.telemetry.clone(),
|
||||
};
|
||||
|
||||
Ok(Some(LocalVideoSyncSuccessOutcome {
|
||||
response,
|
||||
report_payload,
|
||||
report_mode: plan.report_mode(),
|
||||
local_task_snapshot: matches!(plan.report_mode(), VideoTaskSyncReportMode::Background)
|
||||
.then(|| plan.to_snapshot()),
|
||||
}))
|
||||
}
|
||||
|
||||
pub(crate) fn maybe_build_local_sync_finalize_response(
|
||||
trace_id: &str,
|
||||
decision: &GatewayControlDecision,
|
||||
payload: &GatewaySyncReportRequest,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
let (status_code, body_json) = match payload.report_kind.as_str() {
|
||||
"openai_video_delete_sync_finalize" => {
|
||||
if payload.status_code >= 400 && payload.status_code != 404 {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(task_id) = payload
|
||||
.report_context
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("task_id"))
|
||||
.and_then(|value| value.as_str())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
(
|
||||
http::StatusCode::OK,
|
||||
json!({
|
||||
"id": task_id,
|
||||
"object": "video",
|
||||
"deleted": true,
|
||||
}),
|
||||
)
|
||||
}
|
||||
"openai_video_cancel_sync_finalize" | "gemini_video_cancel_sync_finalize" => {
|
||||
if payload.status_code >= 400 {
|
||||
return Ok(None);
|
||||
}
|
||||
(http::StatusCode::OK, json!({}))
|
||||
}
|
||||
_ => return Ok(None),
|
||||
};
|
||||
|
||||
let body_bytes =
|
||||
serde_json::to_vec(&body_json).map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let mut headers = BTreeMap::new();
|
||||
headers.insert("content-type".to_string(), "application/json".to_string());
|
||||
headers.insert("content-length".to_string(), body_bytes.len().to_string());
|
||||
|
||||
Ok(Some(build_client_response_from_parts(
|
||||
status_code.as_u16(),
|
||||
&headers,
|
||||
Body::from(body_bytes),
|
||||
trace_id,
|
||||
Some(decision),
|
||||
)?))
|
||||
}
|
||||
|
||||
pub(crate) fn maybe_build_local_video_error_response(
|
||||
trace_id: &str,
|
||||
decision: &GatewayControlDecision,
|
||||
payload: &GatewaySyncReportRequest,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
if !matches!(
|
||||
payload.report_kind.as_str(),
|
||||
"openai_video_create_sync_finalize"
|
||||
| "openai_video_remix_sync_finalize"
|
||||
| "gemini_video_create_sync_finalize"
|
||||
| "openai_video_delete_sync_finalize"
|
||||
| "openai_video_cancel_sync_finalize"
|
||||
| "gemini_video_cancel_sync_finalize"
|
||||
) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if payload.status_code < 400 {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let response_body = payload.body_json.clone().unwrap_or_else(|| json!({}));
|
||||
let body_bytes = serde_json::to_vec(&response_body)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
|
||||
let mut response_headers = payload.headers.clone();
|
||||
response_headers.remove("content-encoding");
|
||||
response_headers.remove("content-length");
|
||||
response_headers.insert("content-type".to_string(), "application/json".to_string());
|
||||
response_headers.insert("content-length".to_string(), body_bytes.len().to_string());
|
||||
|
||||
Ok(Some(build_client_response_from_parts(
|
||||
payload.status_code,
|
||||
&response_headers,
|
||||
Body::from(body_bytes),
|
||||
trace_id,
|
||||
Some(decision),
|
||||
)?))
|
||||
}
|
||||
|
||||
pub(crate) fn resolve_local_sync_success_background_report_kind(
|
||||
report_kind: &str,
|
||||
) -> Option<String> {
|
||||
let mapped = match report_kind {
|
||||
"openai_video_delete_sync_finalize" => "openai_video_delete_sync_success",
|
||||
"openai_video_cancel_sync_finalize" => "openai_video_cancel_sync_success",
|
||||
"gemini_video_cancel_sync_finalize" => "gemini_video_cancel_sync_success",
|
||||
_ => return None,
|
||||
};
|
||||
Some(mapped.to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn resolve_local_sync_error_background_report_kind(report_kind: &str) -> Option<String> {
|
||||
let mapped = match report_kind {
|
||||
"openai_video_create_sync_finalize" => "openai_video_create_sync_error",
|
||||
"openai_video_remix_sync_finalize" => "openai_video_remix_sync_error",
|
||||
"gemini_video_create_sync_finalize" => "gemini_video_create_sync_error",
|
||||
"openai_video_delete_sync_finalize" => "openai_video_delete_sync_error",
|
||||
"openai_video_cancel_sync_finalize" => "openai_video_cancel_sync_error",
|
||||
"gemini_video_cancel_sync_finalize" => "gemini_video_cancel_sync_error",
|
||||
_ => return None,
|
||||
};
|
||||
Some(mapped.to_string())
|
||||
}
|
||||
Reference in New Issue
Block a user