i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 17:30:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: 新增 frontdoor 执行回环守卫与多项可观测性增强

Browse Source 
- 新增 frontdoor_loop_guard 模块，检测并拒绝 execution runtime 回环到本地网关的请求（HTTP 508）
- candidate loop 引入 span tracking、执行尝试日志与流式看门狗超时
- 本地故障转移策略支持从 report_context 加载，新增 append_local_failover_policy_to_value
- runtime tracing 美化：移除 identity 前缀，按 span 深度树形缩进，target 固定宽度展示
- Codex OpenAI CLI 补齐 chatgpt-account-id/x-client-request-id/session_id/conversation_id 请求头
- OpenAI CLI same/cross-format 聚合规则放宽以支持 openai:compact 客户端格式，并过滤 error-like 响应体
- auth/proxy/finalize 日志补充 user_id/api_key_id/api_key_name/balance_remaining 等字段
- 启动日志拆分为 starting/ready/config 三段，新增 resolve_bind_http_base_url
- access_log middleware 将生成的 trace_id 回注到下游请求头
- Cargo.toml 启用 serde_json preserve_order 特性
This commit is contained in:
fawney19
2026-04-11 01:50:24 +08:00
parent 3f057628b7
commit 6144473ebe
38 changed files with 1957 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
apps/aether-gateway/src/handlers/admin/model/global/payloads.rs
View File

@@ -28,6 +28,7 @@ pub(crate) fn build_admin_global_model_response(
"config": global_model.config.clone(),
"provider_count": provider_count,
"active_provider_count": active_provider_count,
"usage_count": 0,
"created_at": timestamp_or_now(global_model.created_at_unix_ms, now_unix_secs),
"updated_at": timestamp_or_now(global_model.updated_at_unix_secs, now_unix_secs),
})

3
apps/aether-gateway/src/handlers/admin/observability/monitoring/usage_helpers.rs
View File

@@ -2,6 +2,7 @@ use aether_admin::observability::usage::admin_usage_is_failed;
use aether_data_contracts::repository::usage::StoredRequestUsageAudit;
pub(super) fn admin_monitoring_usage_is_error(item: &StoredRequestUsageAudit) -> bool {
item.status.trim().eq_ignore_ascii_case("error") || admin_usage_is_failed(item)
item.status.trim().eq_ignore_ascii_case("error")
|| admin_usage_is_failed(item)
|| item.error_category.is_some()
}

28
apps/aether-gateway/src/handlers/admin/provider/query/models.rs
View File

@@ -125,11 +125,11 @@ fn provider_query_selected_fetch_endpoints(
}
async fn provider_query_read_cached_models(
state: &AppState,
state: &AdminAppState<'_>,
provider_id: &str,
key_id: &str,
) -> Option<Vec<Value>> {
let runner = state.redis_kv_runner()?;
let runner = state.app().redis_kv_runner()?;
let cache_key = runner
.keyspace()
.key(&format!("upstream_models:{provider_id}:{key_id}"));
@@ -148,11 +148,11 @@ async fn provider_query_read_cached_models(
}
async fn provider_query_fetch_models_from_transport(
state: &AppState,
state: &AdminAppState<'_>,
transport: &crate::provider_transport::GatewayProviderTransportSnapshot,
) -> Result<Vec<Value>, String> {
let plan = build_models_fetch_execution_plan(state, transport).await?;
let result = execution_runtime::execute_execution_runtime_sync_plan(state, None, &plan)
let plan = build_models_fetch_execution_plan(state.app(), transport).await?;
let result = execution_runtime::execute_execution_runtime_sync_plan(state.app(), None, &plan)
.await
.map_err(|err| format!("{err:?}"))?;
@@ -182,7 +182,7 @@ async fn provider_query_fetch_models_from_transport(
}
async fn provider_query_fetch_models_for_key(
state: &AppState,
state: &AdminAppState<'_>,
provider: &StoredProviderCatalogProvider,
endpoints: &[StoredProviderCatalogEndpoint],
key: &StoredProviderCatalogKey,
@@ -213,6 +213,7 @@ async fn provider_query_fetch_models_for_key(
let mut all_errors = Vec::new();
for endpoint in selected_endpoints {
let Some(transport) = state
.app()
.read_provider_transport_snapshot(&provider.id, &endpoint.id, &key.id)
.await?
else {
@@ -231,7 +232,7 @@ async fn provider_query_fetch_models_for_key(
let unique_models = aggregate_models_for_cache(&all_models);
if !unique_models.is_empty() {
<AppState as ModelFetchRuntimeState>::write_upstream_models_cache(
state,
state.app(),
&provider.id,
&key.id,
&unique_models,
@@ -296,7 +297,7 @@ pub(crate) async fn build_admin_provider_query_models_response(
};
let result = provider_query_fetch_models_for_key(
state.app(),
state,
&provider,
&endpoints,
selected_key,
@@ -329,14 +330,9 @@ pub(crate) async fn build_admin_provider_query_models_response(
let mut cache_hit_count = 0usize;
let mut fetch_count = 0usize;
for key in active_keys {
let result = provider_query_fetch_models_for_key(
state.app(),
&provider,
&endpoints,
key,
force_refresh,
)
.await?;
let result =
provider_query_fetch_models_for_key(state, &provider, &endpoints, key, force_refresh)
.await?;
all_models.extend(result.models);
if let Some(error) = result.error {
all_errors.push(format!(

13
apps/aether-gateway/src/handlers/proxy/finalize.rs
View File

@@ -87,6 +87,13 @@ pub(super) fn finalize_gateway_response(
.filter(|value| !value.trim().is_empty())
.unwrap_or("-")
.to_string();
let auth_context = control_decision.and_then(|decision| decision.auth_context.as_ref());
let user_id = auth_context
.map(|auth_context| auth_context.user_id.as_str())
.unwrap_or("-");
let api_key_id = auth_context
.map(|auth_context| auth_context.api_key_id.as_str())
.unwrap_or("-");
let status_code = response.status().as_u16();
emit_admin_audit(
&mut response,
@@ -106,6 +113,8 @@ pub(super) fn finalize_gateway_response(
remote_addr = %remote_addr,
method = %method,
path = %path_and_query,
user_id,
api_key_id,
route_class,
execution_path,
dependency_reason = dependency_reason.as_str(),
@@ -124,6 +133,8 @@ pub(super) fn finalize_gateway_response(
remote_addr = %remote_addr,
method = %method,
path = %path_and_query,
user_id,
api_key_id,
route_class,
execution_path,
dependency_reason = dependency_reason.as_str(),
@@ -142,6 +153,8 @@ pub(super) fn finalize_gateway_response(
remote_addr = %remote_addr,
method = %method,
path = %path_and_query,
user_id,
api_key_id,
route_class,
execution_path,
dependency_reason = dependency_reason.as_str(),

105
apps/aether-gateway/src/handlers/proxy/mod.rs
View File

@@ -15,9 +15,10 @@ use crate::constants::{
EXECUTION_PATH_CONTROL_EXECUTE_SYNC, EXECUTION_PATH_DISTRIBUTED_OVERLOADED,
EXECUTION_PATH_EXECUTION_RUNTIME_STREAM, EXECUTION_PATH_EXECUTION_RUNTIME_SYNC,
EXECUTION_PATH_LOCAL_AI_PUBLIC, EXECUTION_PATH_LOCAL_AUTH_DENIED,
EXECUTION_PATH_LOCAL_EXECUTION_RUNTIME_MISS, EXECUTION_PATH_LOCAL_OVERLOADED,
EXECUTION_PATH_LOCAL_PROXY_PASSTHROUGH_REMOVED, EXECUTION_PATH_LOCAL_RATE_LIMITED,
EXECUTION_PATH_LOCAL_ROUTE_NOT_FOUND, EXECUTION_PATH_PUBLIC_PROXY_PASSTHROUGH,
EXECUTION_PATH_LOCAL_EXECUTION_LOOP_DETECTED, EXECUTION_PATH_LOCAL_EXECUTION_RUNTIME_MISS,
EXECUTION_PATH_LOCAL_OVERLOADED, EXECUTION_PATH_LOCAL_PROXY_PASSTHROUGH_REMOVED,
EXECUTION_PATH_LOCAL_RATE_LIMITED, EXECUTION_PATH_LOCAL_ROUTE_NOT_FOUND,
EXECUTION_PATH_PUBLIC_PROXY_PASSTHROUGH, EXECUTION_RUNTIME_LOOP_GUARD_HEADER,
FORWARDED_FOR_HEADER, FORWARDED_HOST_HEADER, FORWARDED_PROTO_HEADER, GATEWAY_HEADER,
LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER, TRACE_ID_HEADER,
TRUSTED_AUTH_ACCESS_ALLOWED_HEADER, TRUSTED_AUTH_API_KEY_ID_HEADER,
@@ -33,6 +34,9 @@ use crate::executor::{
maybe_execute_stream_request, maybe_execute_sync_request,
record_failed_usage_for_exhausted_request, LocalExecutionRequestOutcome,
};
use crate::frontdoor_loop_guard::{
frontdoor_self_loop_public_ai_path, request_has_execution_runtime_loop_guard,
};
use crate::handlers::shared::{
build_admin_proxy_auth_required_response, build_unhandled_admin_proxy_response,
local_proxy_route_requires_buffered_body, request_enables_control_execute,
@@ -48,7 +52,7 @@ use axum::body::{to_bytes, Body, Bytes};
use axum::extract::{ConnectInfo, Request, State};
use axum::http::{self, header::HeaderName, header::HeaderValue, Response};
use std::time::Instant;
use tracing::{info, warn};
use tracing::{debug, info, warn};
const OPENAI_CHAT_LOCAL_EXECUTION_RUNTIME_MISS_DETAIL: &str =
"OpenAI chat execution runtime miss did not match a Rust execution path";
@@ -67,8 +71,23 @@ const GEMINI_FILES_LOCAL_EXECUTION_RUNTIME_MISS_DETAIL: &str =
const LOCAL_ROUTE_NOT_FOUND_DETAIL: &str = "Route not found";
const LOCAL_PROXY_PASSTHROUGH_REMOVED_DETAIL: &str =
"Route matched a removed compatibility passthrough; implement it in Rust or retire the route";
const LOCAL_EXECUTION_LOOP_DETECTED_DETAIL: &str =
"Gateway detected an execution runtime request loop back into the local frontdoor";
const EXECUTION_PATH_TUNNEL_AFFINITY_FORWARD: &str = "tunnel_affinity_forward";
fn local_execution_outcome_label(outcome: &LocalExecutionRequestOutcome) -> &'static str {
match outcome {
LocalExecutionRequestOutcome::Responded(_) => "responded",
LocalExecutionRequestOutcome::Exhausted(_) => "exhausted",
LocalExecutionRequestOutcome::NoPath => "no_path",
}
}
fn request_hits_execution_loop_guard(parts: &http::request::Parts) -> bool {
request_has_execution_runtime_loop_guard(&parts.headers)
&& frontdoor_self_loop_public_ai_path(parts.uri.path())
}
fn execution_runtime_candidate_header_value(decision: &GatewayControlDecision) -> &'static str {
if decision.is_execution_runtime_candidate() {
"true"
@@ -322,6 +341,43 @@ pub(crate) async fn proxy_request(
let (parts, body) = request.into_parts();
let trace_id = extract_or_generate_trace_id(&parts.headers);
state.clear_local_execution_runtime_miss_diagnostic(&trace_id);
if request_hits_execution_loop_guard(&parts) {
warn!(
event_name = "frontdoor_execution_loop_detected",
log_type = "ops",
trace_id = %trace_id,
method = %parts.method,
path = %parts
.uri
.path_and_query()
.map(|value| value.as_str())
.unwrap_or("/"),
loop_guard_header = EXECUTION_RUNTIME_LOOP_GUARD_HEADER,
"gateway rejected execution runtime request loop into frontdoor"
);
let response = build_local_http_error_response(
&trace_id,
None,
http::StatusCode::LOOP_DETECTED,
LOCAL_EXECUTION_LOOP_DETECTED_DETAIL,
)?;
return Ok(finalize_gateway_response(
&state,
response,
&trace_id,
&remote_addr,
&parts.method,
parts
.uri
.path_and_query()
.map(|value| value.as_str())
.unwrap_or("/"),
None,
EXECUTION_PATH_LOCAL_EXECUTION_LOOP_DETECTED,
&started_at,
request_permit.take(),
));
}
let request_context_started_at = Instant::now();
let request_context = crate::control::resolve_public_request_context(
&state,
@@ -593,6 +649,27 @@ pub(crate) async fn proxy_request(
.check_and_consume(&state, control_decision)
.await?;
if let FrontdoorUserRpmOutcome::Rejected(rejection) = &rate_limit_outcome {
let auth_context = control_decision.and_then(|decision| decision.auth_context.as_ref());
let user_id = auth_context
.map(|auth_context| auth_context.user_id.as_str())
.unwrap_or("-");
let api_key_id = auth_context
.map(|auth_context| auth_context.api_key_id.as_str())
.unwrap_or("-");
let path_and_query = request_context.request_path_and_query();
info!(
event_name = "frontdoor_user_rpm_rejected",
log_type = "event",
trace_id = %trace_id,
method = %parts.method,
path = %path_and_query,
user_id,
api_key_id,
scope = rejection.scope,
limit = rejection.limit,
retry_after = rejection.retry_after,
"gateway rejected request at frontdoor user rpm limit"
);
let response =
build_local_user_rpm_limited_response(&trace_id, control_decision, rejection)?;
return Ok(finalize_gateway_response_with_context(
@@ -649,15 +726,29 @@ pub(crate) async fn proxy_request(
let stream_request = request_wants_stream(&request_context, buffered_body);
let mut local_execution_exhaustion = None;
if stream_request {
match maybe_execute_stream_request(
let stream_outcome = maybe_execute_stream_request(
&state,
&parts,
buffered_body,
&trace_id,
control_decision,
)
.await?
{
.await?;
debug!(
event_name = "proxy_stream_local_execute_outcome",
log_type = "debug",
trace_id = %trace_id,
outcome = local_execution_outcome_label(&stream_outcome),
route_family = control_decision
.and_then(|decision| decision.route_family.as_deref())
.unwrap_or("-"),
route_kind = control_decision
.and_then(|decision| decision.route_kind.as_deref())
.unwrap_or("-"),
request_path = %request_context.request_path_and_query(),
"gateway local stream execution returned to proxy"
);
match stream_outcome {
LocalExecutionRequestOutcome::Responded(execution_runtime_response) => {
state.clear_local_execution_runtime_miss_diagnostic(&trace_id);
return Ok(finalize_gateway_response_with_context(