i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 01:10:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: 新增 frontdoor 执行回环守卫与多项可观测性增强

Browse Source 
- 新增 frontdoor_loop_guard 模块，检测并拒绝 execution runtime 回环到本地网关的请求（HTTP 508）
- candidate loop 引入 span tracking、执行尝试日志与流式看门狗超时
- 本地故障转移策略支持从 report_context 加载，新增 append_local_failover_policy_to_value
- runtime tracing 美化：移除 identity 前缀，按 span 深度树形缩进，target 固定宽度展示
- Codex OpenAI CLI 补齐 chatgpt-account-id/x-client-request-id/session_id/conversation_id 请求头
- OpenAI CLI same/cross-format 聚合规则放宽以支持 openai:compact 客户端格式，并过滤 error-like 响应体
- auth/proxy/finalize 日志补充 user_id/api_key_id/api_key_name/balance_remaining 等字段
- 启动日志拆分为 starting/ready/config 三段，新增 resolve_bind_http_base_url
- access_log middleware 将生成的 trace_id 回注到下游请求头
- Cargo.toml 启用 serde_json preserve_order 特性
This commit is contained in:
fawney19
2026-04-11 01:50:24 +08:00
parent 3f057628b7
commit 6144473ebe
38 changed files with 1957 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files

155
apps/aether-gateway/src/execution_runtime/fallback.rs
View File

@@ -2,7 +2,10 @@ use std::collections::BTreeSet;
use aether_contracts::{ExecutionPlan, ExecutionResult};
use regex::Regex;
use serde_json::{json, Value};
use tracing::debug;
use crate::provider_transport::GatewayProviderTransportSnapshot;
use crate::AppState;
fn local_candidate_index(report_context: Option<&serde_json::Value>) -> Option<u64> {
@@ -32,12 +35,22 @@ struct LocalFailoverRegexRule {
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LocalFailoverDecision {
pub(crate) enum LocalFailoverDecision {
UseDefault,
RetryNextCandidate,
StopLocalFailover,
}
impl LocalFailoverDecision {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::UseDefault => "use_default",
Self::RetryNextCandidate => "retry_next_candidate",
Self::StopLocalFailover => "stop_local_failover",
}
}
}
pub(crate) async fn should_retry_next_local_candidate_sync(
state: &AppState,
plan: &ExecutionPlan,
@@ -169,8 +182,14 @@ pub(crate) async fn should_retry_next_local_candidate_stream(
response_text: Option<&str>,
) -> bool {
matches!(
resolve_local_failover_decision(state, plan, report_context, status_code, response_text)
.await,
resolve_local_candidate_failover_decision_stream(
state,
plan,
report_context,
status_code,
response_text,
)
.await,
LocalFailoverDecision::RetryNextCandidate
)
}
@@ -184,12 +203,28 @@ pub(crate) async fn should_stop_local_candidate_failover_stream(
response_text: Option<&str>,
) -> bool {
matches!(
resolve_local_failover_decision(state, plan, report_context, status_code, response_text)
.await,
resolve_local_candidate_failover_decision_stream(
state,
plan,
report_context,
status_code,
response_text,
)
.await,
LocalFailoverDecision::StopLocalFailover
)
}
pub(crate) async fn resolve_local_candidate_failover_decision_stream(
state: &AppState,
plan: &ExecutionPlan,
report_context: Option<&serde_json::Value>,
status_code: u16,
response_text: Option<&str>,
) -> LocalFailoverDecision {
resolve_local_failover_decision(state, plan, report_context, status_code, response_text).await
}
pub(crate) fn local_failover_response_text(
body_json: Option<&serde_json::Value>,
body_bytes: &[u8],
@@ -217,7 +252,7 @@ async fn resolve_local_failover_decision(
let Some(candidate_index) = local_candidate_index(report_context) else {
return LocalFailoverDecision::UseDefault;
};
let policy = resolve_local_failover_policy(state, plan).await;
let policy = resolve_local_failover_policy(state, plan, report_context).await;
let response_text = response_text
.map(str::trim)
.filter(|value| !value.is_empty());
@@ -269,7 +304,27 @@ async fn resolve_local_failover_decision(
async fn resolve_local_failover_policy(
state: &AppState,
plan: &ExecutionPlan,
report_context: Option<&serde_json::Value>,
) -> LocalFailoverPolicy {
if let Some(policy) = local_failover_policy_from_report_context(report_context) {
debug!(
event_name = "local_failover_policy_loaded",
log_type = "debug",
request_id = %plan.request_id,
provider_id = %plan.provider_id,
endpoint_id = %plan.endpoint_id,
key_id = %plan.key_id,
source = "report_context",
max_retries = ?policy.max_retries,
stop_status_code_count = policy.stop_status_codes.len(),
continue_status_code_count = policy.continue_status_codes.len(),
success_failover_pattern_count = policy.success_failover_patterns.len(),
error_stop_pattern_count = policy.error_stop_patterns.len(),
"gateway loaded local failover policy from report context"
);
return policy;
}
let transport = match state
.read_provider_transport_snapshot(&plan.provider_id, &plan.endpoint_id, &plan.key_id)
.await
@@ -277,7 +332,28 @@ async fn resolve_local_failover_policy(
Ok(Some(transport)) => transport,
Ok(None) | Err(_) => return LocalFailoverPolicy::default(),
};
let policy = local_failover_policy_from_transport(&transport);
debug!(
event_name = "local_failover_policy_loaded",
log_type = "debug",
request_id = %plan.request_id,
provider_id = %plan.provider_id,
endpoint_id = %plan.endpoint_id,
key_id = %plan.key_id,
source = "transport_snapshot",
max_retries = ?policy.max_retries,
stop_status_code_count = policy.stop_status_codes.len(),
continue_status_code_count = policy.continue_status_codes.len(),
success_failover_pattern_count = policy.success_failover_patterns.len(),
error_stop_pattern_count = policy.error_stop_patterns.len(),
"gateway loaded local failover policy from transport snapshot"
);
policy
}
fn local_failover_policy_from_transport(
transport: &GatewayProviderTransportSnapshot,
) -> LocalFailoverPolicy {
let rules = transport
.provider
.config
@@ -337,6 +413,69 @@ async fn resolve_local_failover_policy(
}
}
fn local_failover_policy_from_report_context(
report_context: Option<&Value>,
) -> Option<LocalFailoverPolicy> {
let object = report_context
.and_then(Value::as_object)?
.get("local_failover_policy")?
.as_object()?;
Some(LocalFailoverPolicy {
max_retries: object.get("max_retries").and_then(parse_u64_value),
stop_status_codes: object
.get("stop_status_codes")
.map(parse_status_code_list)
.unwrap_or_default(),
continue_status_codes: object
.get("continue_status_codes")
.map(parse_status_code_list)
.unwrap_or_default(),
success_failover_patterns: parse_regex_rules(object, "success_failover_patterns"),
error_stop_patterns: parse_regex_rules(object, "error_stop_patterns"),
})
}
fn parse_status_code_list(value: &Value) -> BTreeSet<u16> {
value
.as_array()
.into_iter()
.flat_map(|values| values.iter())
.filter_map(|value| parse_u64_value(value).and_then(|value| u16::try_from(value).ok()))
.collect()
}
fn local_failover_policy_to_value(policy: &LocalFailoverPolicy) -> Value {
json!({
"max_retries": policy.max_retries,
"stop_status_codes": policy.stop_status_codes.iter().copied().collect::<Vec<_>>(),
"continue_status_codes": policy.continue_status_codes.iter().copied().collect::<Vec<_>>(),
"success_failover_patterns": policy.success_failover_patterns.iter().map(local_failover_regex_rule_to_value).collect::<Vec<_>>(),
"error_stop_patterns": policy.error_stop_patterns.iter().map(local_failover_regex_rule_to_value).collect::<Vec<_>>(),
})
}
fn local_failover_regex_rule_to_value(rule: &LocalFailoverRegexRule) -> Value {
json!({
"pattern": rule.pattern,
"status_codes": rule.status_codes.iter().copied().collect::<Vec<_>>(),
})
}
pub(crate) fn append_local_failover_policy_to_value(
value: Value,
transport: &GatewayProviderTransportSnapshot,
) -> Value {
let Value::Object(mut object) = value else {
return value;
};
object.insert(
"local_failover_policy".to_string(),
local_failover_policy_to_value(&local_failover_policy_from_transport(transport)),
);
Value::Object(object)
}
fn parse_regex_rules(
rules: &serde_json::Map<String, serde_json::Value>,
key: &str,
@@ -800,7 +939,7 @@ mod tests {
let plan = sample_plan();
let runtime = tokio::runtime::Runtime::new().expect("runtime should build");
let policy = runtime.block_on(resolve_local_failover_policy(&state, &plan));
let policy = runtime.block_on(resolve_local_failover_policy(&state, &plan, None));
assert_eq!(
policy,
LocalFailoverPolicy {
@@ -894,7 +1033,7 @@ mod tests {
let plan = sample_plan();
let runtime = tokio::runtime::Runtime::new().expect("runtime should build");
let policy = runtime.block_on(resolve_local_failover_policy(&state, &plan));
let policy = runtime.block_on(resolve_local_failover_policy(&state, &plan, None));
assert_eq!(
policy.success_failover_patterns,
vec![LocalFailoverRegexRule {

12
apps/aether-gateway/src/execution_runtime/mod.rs
View File

@@ -19,12 +19,14 @@ pub(crate) use self::constants::{
MAX_ERROR_BODY_BYTES, MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES,
};
pub(crate) use self::fallback::{
local_failover_response_text, resolve_core_stream_direct_finalize_report_kind,
append_local_failover_policy_to_value, local_failover_response_text,
resolve_core_stream_direct_finalize_report_kind,
resolve_core_stream_error_finalize_report_kind, resolve_core_sync_error_finalize_report_kind,
should_fallback_to_control_stream, should_fallback_to_control_sync,
should_finalize_sync_response, should_retry_next_local_candidate_stream,
should_retry_next_local_candidate_sync, should_stop_local_candidate_failover_stream,
should_stop_local_candidate_failover_sync,
resolve_local_candidate_failover_decision_stream, should_fallback_to_control_stream,
should_fallback_to_control_sync, should_finalize_sync_response,
should_retry_next_local_candidate_stream, should_retry_next_local_candidate_sync,
should_stop_local_candidate_failover_stream, should_stop_local_candidate_failover_sync,
LocalFailoverDecision,
};
pub use server::{
build_execution_runtime_router, build_execution_runtime_router_with_request_concurrency_limit,

210
apps/aether-gateway/src/execution_runtime/stream/execution.rs
View File

@@ -3,7 +3,9 @@ use std::io::Error as IoError;
use aether_contracts::{ExecutionPlan, ExecutionTelemetry, StreamFrame, StreamFramePayload};
use aether_data_contracts::repository::candidates::RequestCandidateStatus;
use aether_scheduler_core::SchedulerRequestCandidateStatusUpdate;
use aether_scheduler_core::{
parse_request_candidate_report_context, SchedulerRequestCandidateStatusUpdate,
};
use async_stream::stream;
use axum::body::{Body, Bytes};
use axum::http::Response;
@@ -47,8 +49,9 @@ use crate::execution_runtime::transport::{
};
use crate::execution_runtime::{
local_failover_response_text, resolve_core_stream_direct_finalize_report_kind,
resolve_core_stream_error_finalize_report_kind, should_fallback_to_control_stream,
should_retry_next_local_candidate_stream, should_stop_local_candidate_failover_stream,
resolve_core_stream_error_finalize_report_kind,
resolve_local_candidate_failover_decision_stream, should_fallback_to_control_stream,
should_retry_next_local_candidate_stream, LocalFailoverDecision,
};
use crate::execution_runtime::{MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES};
use crate::log_ids::short_request_id;
@@ -75,6 +78,14 @@ pub(crate) async fn execute_execution_runtime_stream(
.record_pending(state.data.as_ref(), &plan, report_context.as_ref())
.await;
let plan_request_id_for_log = short_request_id(plan.request_id.as_str());
let provider_name = plan.provider_name.as_deref().unwrap_or("-");
let endpoint_id = plan.endpoint_id.as_str();
let key_id = plan.key_id.as_str();
let model_name = plan.model_name.as_deref().unwrap_or("-");
let candidate_index = parse_request_candidate_report_context(report_context.as_ref())
.and_then(|context| context.candidate_index)
.map(|value| value.to_string())
.unwrap_or_else(|| "-".to_string());
#[cfg(not(test))]
{
let execution = match DirectSyncExecutionRuntime::new()
@@ -89,6 +100,11 @@ pub(crate) async fn execute_execution_runtime_stream(
trace_id = %trace_id,
request_id = %plan_request_id_for_log,
candidate_id = ?plan.candidate_id,
provider_name,
endpoint_id,
key_id,
model_name,
candidate_index = candidate_index.as_str(),
error = %err,
"gateway in-process stream execution unavailable"
);
@@ -126,6 +142,11 @@ pub(crate) async fn execute_execution_runtime_stream(
trace_id = %trace_id,
request_id = %plan_request_id_for_log,
candidate_id = ?plan.candidate_id,
provider_name,
endpoint_id,
key_id,
model_name,
candidate_index = candidate_index.as_str(),
error = %err,
"gateway in-process stream execution unavailable"
);
@@ -252,6 +273,37 @@ fn should_refresh_stream_usage_telemetry(
|| (next_elapsed.is_some() && next_elapsed != previous_elapsed)
}
fn should_skip_direct_finalize_prefetch(
direct_stream_finalize_kind: Option<&str>,
content_type: Option<&str>,
provider_api_format: &str,
client_api_format: &str,
has_private_stream_normalizer: bool,
has_local_stream_rewriter: bool,
) -> bool {
if direct_stream_finalize_kind.is_none()
|| has_private_stream_normalizer
|| has_local_stream_rewriter
{
return false;
}
if !provider_api_format.eq_ignore_ascii_case(client_api_format) {
return false;
}
let content_type = content_type
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or_default()
.to_ascii_lowercase();
if content_type.is_empty() {
return true;
}
!(content_type.contains("json") || content_type.ends_with("+json"))
}
async fn probe_local_stream_success_failover_text<R>(
buffered_frames: &mut VecDeque<StreamFrame>,
lines: &mut FramedRead<R, LinesCodec>,
@@ -294,6 +346,12 @@ async fn execute_stream_from_frame_stream(
let request_id = plan.request_id.as_str();
let request_id_for_log = short_request_id(request_id);
let candidate_id = plan.candidate_id.as_deref();
let provider_name = plan.provider_name.as_deref().unwrap_or("-");
let model_name = plan.model_name.as_deref().unwrap_or("-");
let candidate_index = parse_request_candidate_report_context(report_context.as_ref())
.and_then(|context| context.candidate_index)
.map(|value| value.to_string())
.unwrap_or_else(|| "-".to_string());
let reader = StreamReader::new(frame_stream);
let mut lines = FramedRead::new(reader, LinesCodec::new());
@@ -310,7 +368,6 @@ async fn execute_stream_from_frame_stream(
));
};
let mut buffered_frames = VecDeque::new();
if status_code == 200 {
let success_probe_text =
probe_local_stream_success_failover_text(&mut buffered_frames, &mut lines).await?;
@@ -349,6 +406,11 @@ async fn execute_stream_from_frame_stream(
trace_id = %trace_id,
request_id = %request_id_for_log,
status_code,
provider_name = provider_name,
endpoint_id = %plan.endpoint_id,
key_id = %plan.key_id,
model_name,
candidate_index = candidate_index.as_str(),
"gateway local stream decision retrying next candidate after success failover rule match"
);
return Ok(None);
@@ -363,26 +425,31 @@ async fn execute_stream_from_frame_stream(
let (body_json, body_base64) = decode_stream_error_body(&headers, &error_body);
let error_response_text =
local_failover_response_text(body_json.as_ref(), &error_body, None);
let stop_local_failover = should_stop_local_candidate_failover_stream(
let failover_decision = resolve_local_candidate_failover_decision_stream(
state,
&plan,
plan_kind,
report_context.as_ref(),
status_code,
error_response_text.as_deref(),
)
.await;
if !stop_local_failover
&& should_retry_next_local_candidate_stream(
state,
&plan,
plan_kind,
report_context.as_ref(),
status_code,
error_response_text.as_deref(),
)
.await
{
debug!(
event_name = "execution_runtime_stream_failover_decided",
log_type = "debug",
trace_id = %trace_id,
request_id = %request_id_for_log,
candidate_id = ?candidate_id,
plan_kind,
status_code,
provider_name,
endpoint_id = %plan.endpoint_id,
key_id = %plan.key_id,
model_name,
candidate_index = candidate_index.as_str(),
failover_decision = failover_decision.as_str(),
"gateway resolved execution runtime stream failover decision"
);
if matches!(failover_decision, LocalFailoverDecision::RetryNextCandidate) {
let terminal_unix_secs = current_request_candidate_unix_ms();
record_local_request_candidate_status(
state,
@@ -407,12 +474,17 @@ async fn execute_stream_from_frame_stream(
trace_id = %trace_id,
request_id = %request_id_for_log,
status_code,
provider_name = provider_name,
endpoint_id = %plan.endpoint_id,
key_id = %plan.key_id,
model_name,
candidate_index = candidate_index.as_str(),
"gateway local stream decision retrying next candidate after retryable execution runtime status"
);
return Ok(None);
}
if !stop_local_failover
if !matches!(failover_decision, LocalFailoverDecision::StopLocalFailover)
&& should_fallback_to_control_stream(
plan_kind,
status_code,
@@ -529,13 +601,44 @@ async fn execute_stream_from_frame_stream(
headers.remove("content-length");
headers.insert("content-type".to_string(), "text/event-stream".to_string());
}
let content_type = headers.get("content-type").map(String::as_str);
let skip_direct_finalize_prefetch = should_skip_direct_finalize_prefetch(
direct_stream_finalize_kind.as_deref(),
content_type,
plan.provider_api_format.as_str(),
plan.client_api_format.as_str(),
private_stream_normalizer.is_some(),
local_stream_rewriter.is_some(),
);
let mut prefetched_chunks: Vec<Bytes> = Vec::new();
let mut provider_prefetched_body = Vec::new();
let mut prefetched_body = Vec::new();
let mut prefetched_inspection_body = Vec::new();
let mut prefetched_telemetry: Option<ExecutionTelemetry> = None;
let mut reached_eof = false;
if let Some(ref report_kind) = direct_stream_finalize_kind {
if skip_direct_finalize_prefetch {
debug!(
event_name = "execution_runtime_stream_prefetch_skipped",
log_type = "debug",
trace_id = %trace_id,
request_id = %request_id_for_log,
candidate_id = ?candidate_id,
plan_kind,
provider_name,
endpoint_id = %plan.endpoint_id,
key_id = %plan.key_id,
model_name,
candidate_index = candidate_index.as_str(),
content_type = content_type.unwrap_or("-"),
provider_api_format = plan.provider_api_format.as_str(),
client_api_format = plan.client_api_format.as_str(),
"gateway skipped direct finalize prefetch for same-format passthrough stream"
);
}
if let Some(report_kind) = direct_stream_finalize_kind
.as_ref()
.filter(|_| !skip_direct_finalize_prefetch)
{
while prefetched_chunks.len() < MAX_STREAM_PREFETCH_FRAMES
&& prefetched_inspection_body.len() < MAX_STREAM_PREFETCH_BYTES
{
@@ -609,6 +712,22 @@ async fn execute_stream_from_frame_stream(
inspect_prefetched_stream_body(&headers, &prefetched_inspection_body);
match inspection {
StreamPrefetchInspection::EmbeddedError(body_json) => {
debug!(
event_name = "execution_runtime_stream_prefetch_embedded_error_detected",
log_type = "debug",
trace_id = %trace_id,
request_id = %request_id_for_log,
candidate_id = ?candidate_id,
plan_kind,
report_kind,
provider_name,
endpoint_id = %plan.endpoint_id,
key_id = %plan.key_id,
model_name,
candidate_index = candidate_index.as_str(),
provider_prefetched_body_bytes = provider_prefetched_body.len(),
"gateway detected embedded error while prefetching execution runtime stream"
);
let payload = GatewaySyncReportRequest {
trace_id: trace_id.to_string(),
report_kind: report_kind.clone(),
@@ -1260,3 +1379,56 @@ async fn execute_stream_from_frame_stream(
Some(decision),
)?))
}
#[cfg(test)]
mod tests {
use super::should_skip_direct_finalize_prefetch;
#[test]
fn skips_prefetch_for_same_format_passthrough_event_streams() {
assert!(should_skip_direct_finalize_prefetch(
Some("claude_cli_sync_finalize"),
Some("text/event-stream"),
"claude:cli",
"claude:cli",
false,
false,
));
}
#[test]
fn skips_prefetch_for_same_format_passthrough_streams_without_content_type() {
assert!(should_skip_direct_finalize_prefetch(
Some("claude_cli_sync_finalize"),
None,
"claude:cli",
"claude:cli",
false,
false,
));
}
#[test]
fn keeps_prefetch_for_same_format_json_streams() {
assert!(!should_skip_direct_finalize_prefetch(
Some("claude_cli_sync_finalize"),
Some("application/json"),
"claude:cli",
"claude:cli",
false,
false,
));
}
#[test]
fn keeps_prefetch_for_cross_format_or_rewritten_streams() {
assert!(!should_skip_direct_finalize_prefetch(
Some("claude_cli_sync_finalize"),
Some("text/event-stream"),
"openai:chat",
"claude:cli",
false,
true,
));
}
}

28
apps/aether-gateway/src/execution_runtime/sync/execution.rs
View File

@@ -2,7 +2,10 @@ use std::collections::BTreeMap;
use aether_contracts::{ExecutionPlan, ExecutionResult, ExecutionTelemetry};
use aether_data_contracts::repository::candidates::RequestCandidateStatus;
use aether_scheduler_core::{execution_error_details, SchedulerRequestCandidateStatusUpdate};
use aether_scheduler_core::{
execution_error_details, parse_request_candidate_report_context,
SchedulerRequestCandidateStatusUpdate,
};
use axum::body::Body;
use axum::http::Response;
use base64::Engine as _;
@@ -86,6 +89,14 @@ pub(crate) async fn execute_execution_runtime_sync(
let plan_request_id = plan.request_id.as_str();
let plan_request_id_for_log = short_request_id(plan_request_id);
let plan_candidate_id = plan.candidate_id.as_deref();
let provider_name = plan.provider_name.as_deref().unwrap_or("-");
let endpoint_id = plan.endpoint_id.as_str();
let key_id = plan.key_id.as_str();
let model_name = plan.model_name.as_deref().unwrap_or("-");
let candidate_index = parse_request_candidate_report_context(report_context.as_ref())
.and_then(|context| context.candidate_index)
.map(|value| value.to_string())
.unwrap_or_else(|| "-".to_string());
let candidate_started_unix_secs = current_request_candidate_unix_ms();
state
.usage_runtime
@@ -105,6 +116,11 @@ pub(crate) async fn execute_execution_runtime_sync(
trace_id = %trace_id,
request_id = %plan_request_id_for_log,
candidate_id = ?plan_candidate_id,
provider_name,
endpoint_id,
key_id,
model_name,
candidate_index = candidate_index.as_str(),
error = %err,
"gateway in-process sync execution unavailable"
);
@@ -130,6 +146,11 @@ pub(crate) async fn execute_execution_runtime_sync(
trace_id = %trace_id,
request_id = %plan_request_id_for_log,
candidate_id = ?plan_candidate_id,
provider_name,
endpoint_id,
key_id,
model_name,
candidate_index = candidate_index.as_str(),
error = %err,
"gateway in-process sync execution unavailable"
);
@@ -215,6 +236,11 @@ pub(crate) async fn execute_execution_runtime_sync(
trace_id = %trace_id,
request_id = %plan_request_id_for_log,
status_code = result.status_code,
provider_name,
endpoint_id,
key_id,
model_name,
candidate_index = candidate_index.as_str(),
"gateway local sync decision retrying next candidate after retryable execution runtime result"
);
return Ok(None);

85
apps/aether-gateway/src/execution_runtime/transport.rs
View File

@@ -16,8 +16,13 @@ use serde::Serialize;
use serde_json::Value;
use thiserror::Error;
use crate::constants::{
EXECUTION_RUNTIME_LOOP_GUARD_HEADER, EXECUTION_RUNTIME_LOOP_GUARD_VALUE,
EXECUTION_RUNTIME_LOOP_GUARD_VIA_TOKEN,
};
#[cfg(test)]
use crate::execution_runtime::remote_compat::execute_sync_plan_via_remote_execution_runtime;
use crate::frontdoor_loop_guard::gateway_frontdoor_self_loop_guard_error;
use crate::{AppState, GatewayError};
const HUB_RELAY_CONTENT_TYPE: &str = "application/vnd.aether.tunnel-envelope";
@@ -242,12 +247,17 @@ async fn send_request(
plan: &ExecutionPlan,
body_bytes: Vec<u8>,
) -> Result<reqwest::Response, ExecutionRuntimeTransportError> {
if let Some(detail) = gateway_frontdoor_self_loop_guard_error(plan.url.as_str()) {
return Err(ExecutionRuntimeTransportError::UpstreamRequest(detail));
}
let method = plan.method.parse::<reqwest::Method>()?;
let headers = build_request_headers(
&plan.headers,
plan.content_encoding.as_deref(),
plan.body.body_bytes_b64.is_some(),
)?;
let headers = append_execution_loop_guard_header(headers);
let total_timeout = plan
.timeouts
.as_ref()
@@ -274,6 +284,34 @@ async fn send_request(
})
}
fn append_execution_loop_guard_header(mut headers: HeaderMap) -> HeaderMap {
headers.insert(
HeaderName::from_static(EXECUTION_RUNTIME_LOOP_GUARD_HEADER),
HeaderValue::from_static(EXECUTION_RUNTIME_LOOP_GUARD_VALUE),
);
let via_name = HeaderName::from_static("via");
let via_value = headers
.get(&via_name)
.and_then(|value| value.to_str().ok())
.map(str::trim)
.filter(|value| !value.is_empty())
.map(|value| {
if value
.to_ascii_lowercase()
.contains(EXECUTION_RUNTIME_LOOP_GUARD_VIA_TOKEN)
{
value.to_string()
} else {
format!("{value}, 1.1 {EXECUTION_RUNTIME_LOOP_GUARD_VIA_TOKEN}")
}
})
.unwrap_or_else(|| format!("1.1 {EXECUTION_RUNTIME_LOOP_GUARD_VIA_TOKEN}"));
if let Ok(value) = HeaderValue::from_str(via_value.as_str()) {
headers.insert(via_name, value);
}
headers
}
async fn send_via_tunnel_relay(
plan: &ExecutionPlan,
method: reqwest::Method,
@@ -647,6 +685,53 @@ mod tests {
use serde_json::json;
use super::DirectSyncExecutionRuntime;
use crate::frontdoor_loop_guard::{
frontdoor_self_loop_public_ai_path, gateway_frontdoor_self_loop_guard_error_with_bind,
gateway_frontdoor_self_loop_guard_matches_with_bind,
};
#[test]
fn gateway_frontdoor_self_loop_guard_matches_loopback_public_ai_route() {
assert!(gateway_frontdoor_self_loop_guard_matches_with_bind(
"0.0.0.0:8084",
"http://127.0.0.1:8084/v1/messages"
));
assert!(gateway_frontdoor_self_loop_guard_matches_with_bind(
"0.0.0.0:8084",
"http://localhost:8084/v1/responses"
));
}
#[test]
fn gateway_frontdoor_self_loop_guard_ignores_non_ai_routes() {
assert!(!gateway_frontdoor_self_loop_guard_matches_with_bind(
"0.0.0.0:8084",
"http://127.0.0.1:8084/_gateway/health"
));
assert!(!frontdoor_self_loop_public_ai_path("/_gateway/health"));
}
#[test]
fn gateway_frontdoor_self_loop_guard_ignores_different_ports() {
assert!(!gateway_frontdoor_self_loop_guard_matches_with_bind(
"0.0.0.0:8084",
"http://127.0.0.1:9999/v1/messages"
));
}
#[test]
fn gateway_frontdoor_self_loop_guard_reports_clear_error() {
assert_eq!(
gateway_frontdoor_self_loop_guard_error_with_bind(
"0.0.0.0:8084",
"http://localhost:8084/v1/responses"
),
Some(
"upstream execution target resolves back to the local aether-gateway frontdoor: http://localhost:8084/v1/responses"
.to_string()
)
);
}
fn tunnel_proxy_snapshot(base_url: String) -> aether_contracts::ProxySnapshot {
aether_contracts::ProxySnapshot {