feat: 新增 frontdoor 执行回环守卫与多项可观测性增强

- 新增 frontdoor_loop_guard 模块，检测并拒绝 execution runtime 回环到本地网关的请求（HTTP 508） - candidate loop 引入 span tracking、执行尝试日志与流式看门狗超时 - 本地故障转移策略支持从 report_context 加载，新增 append_local_failover_policy_to_value - runtime tracing 美化：移除 identity 前缀，按 span 深度树形缩进，target 固定宽度展示 - Codex OpenAI CLI 补齐 chatgpt-account-id/x-client-request-id/session_id/conversation_id 请求头 - OpenAI CLI same/cross-format 聚合规则放宽以支持 openai:compact 客户端格式，并过滤 error-like 响应体 - auth/proxy/finalize 日志补充 user_id/api_key_id/api_key_name/balance_remaining 等字段 - 启动日志拆分为 starting/ready/config 三段，新增 resolve_bind_http_base_url - access_log middleware 将生成的 trace_id 回注到下游请求头 - Cargo.toml 启用 serde_json preserve_order 特性
2026-09-02 17:30:23 +08:00 · 2026-04-11 01:50:24 +08:00
parent 3f057628b7
commit 6144473ebe
38 changed files with 1957 additions and 421 deletions
--- a/apps/aether-gateway/src/tests/ai_execute/finalize_local.rs
+++ b/apps/aether-gateway/src/tests/ai_execute/finalize_local.rs
@@ -124,7 +124,7 @@ async fn gateway_executes_openai_chat_sync_upstream_stream_via_local_finalize_re
        .with_transport_fields(
            true,
            false,
-            false,
+            true,
            None,
            Some(2),
            None,
@@ -614,7 +614,7 @@ async fn gateway_executes_openai_chat_cross_format_upstream_stream_via_local_fin
        .with_transport_fields(
            true,
            false,
-            false,
+            true,
            None,
            Some(2),
            None,
@@ -861,9 +861,12 @@ async fn gateway_executes_openai_chat_cross_format_upstream_stream_via_local_fin
        .await
        .expect("request should succeed");
    let elapsed = started_at.elapsed();
+    let response_status = response.status();
+    let response_body = response.text().await.expect("body should read");

-    assert_eq!(response.status(), StatusCode::OK);
-    let response_json: serde_json::Value = response.json().await.expect("body should parse");
+    assert_eq!(response_status, StatusCode::OK);
+    let response_json: serde_json::Value =
+        serde_json::from_str(&response_body).expect("body should parse");
    assert_eq!(
        response_json,
        json!({
@@ -1047,7 +1050,7 @@ async fn gateway_executes_openai_chat_cross_format_tool_use_upstream_stream_via_
        .with_transport_fields(
            true,
            false,
-            false,
+            true,
            None,
            Some(2),
            None,
@@ -1311,9 +1314,12 @@ async fn gateway_executes_openai_chat_cross_format_tool_use_upstream_stream_via_
        .await
        .expect("request should succeed");
    let elapsed = started_at.elapsed();
+    let response_status = response.status();
+    let response_body = response.text().await.expect("body should read");

-    assert_eq!(response.status(), StatusCode::OK);
-    let response_json: serde_json::Value = response.json().await.expect("body should parse");
+    assert_eq!(response_status, StatusCode::OK);
+    let response_json: serde_json::Value =
+        serde_json::from_str(&response_body).expect("body should parse");
    assert_eq!(
        response_json,
        json!({
@@ -1497,7 +1503,7 @@ async fn gateway_skips_openai_chat_antigravity_cross_format_sync_candidate_as_tr
        .with_transport_fields(
            true,
            false,
-            false,
+            true,
            None,
            Some(2),
            None,
@@ -1857,7 +1863,7 @@ async fn gateway_executes_openai_chat_cross_format_claude_upstream_sync_via_loca
        .with_transport_fields(
            true,
            false,
-            false,
+            true,
            None,
            Some(2),
            None,
@@ -2064,9 +2070,12 @@ async fn gateway_executes_openai_chat_cross_format_claude_upstream_sync_via_loca
        .await
        .expect("request should succeed");
    let elapsed = started_at.elapsed();
+    let response_status = response.status();
+    let response_body = response.text().await.expect("body should read");

-    assert_eq!(response.status(), StatusCode::OK);
-    let response_json: serde_json::Value = response.json().await.expect("body should parse");
+    assert_eq!(response_status, StatusCode::OK);
+    let response_json: serde_json::Value =
+        serde_json::from_str(&response_body).expect("body should parse");
    assert_eq!(
        response_json,
        json!({
@@ -2205,7 +2214,7 @@ async fn gateway_executes_openai_chat_cross_format_gemini_upstream_sync_via_loca
        .with_transport_fields(
            true,
            false,
-            false,
+            true,
            None,
            Some(2),
            None,
--- a/apps/aether-gateway/src/tests/ai_execute/finalize_local_cli/cross_format.rs
+++ b/apps/aether-gateway/src/tests/ai_execute/finalize_local_cli/cross_format.rs
@@ -122,7 +122,7 @@ async fn gateway_executes_openai_cli_cross_format_upstream_stream_via_local_fina
        .with_transport_fields(
            true,
            false,
-            false,
+            true,
            None,
            Some(2),
            None,
@@ -381,9 +381,12 @@ async fn gateway_executes_openai_cli_cross_format_upstream_stream_via_local_fina
        .await
        .expect("request should succeed");
    let elapsed = started_at.elapsed();
+    let response_status = response.status();
+    let response_body = response.text().await.expect("body should read");

-    assert_eq!(response.status(), StatusCode::OK);
-    let response_json: serde_json::Value = response.json().await.expect("body should parse");
+    assert_eq!(response_status, StatusCode::OK);
+    let response_json: serde_json::Value =
+        serde_json::from_str(&response_body).expect("body should parse");
    assert_eq!(
        response_json,
        json!({
@@ -577,7 +580,7 @@ async fn gateway_executes_openai_cli_cross_format_function_call_upstream_stream_
        .with_transport_fields(
            true,
            false,
-            false,
+            true,
            None,
            Some(2),
            None,
@@ -1046,7 +1049,7 @@ async fn gateway_executes_openai_cli_antigravity_cross_format_upstream_stream_vi
        .with_transport_fields(
            true,
            false,
-            false,
+            true,
            None,
            Some(2),
            None,
--- a/apps/aether-gateway/src/tests/ai_execute/stream/decision.rs
+++ b/apps/aether-gateway/src/tests/ai_execute/stream/decision.rs
@@ -116,7 +116,7 @@ async fn gateway_executes_openai_chat_stream_via_local_decision_gate_without_exe
        .with_transport_fields(
            true,
            false,
-            false,
+            true,
            None,
            Some(2),
            None,
@@ -540,7 +540,7 @@ async fn gateway_executes_openai_chat_stream_via_local_openai_cli_cross_format_c
        .with_transport_fields(
            true,
            false,
-            false,
+            true,
            None,
            Some(2),
            None,
@@ -816,7 +816,11 @@ async fn gateway_executes_openai_chat_stream_via_local_openai_cli_cross_format_c
            provider_catalog_repository,
            Arc::clone(&request_candidate_repository),
            DEVELOPMENT_ENCRYPTION_KEY,
-        ),
+        )
+        .with_system_config_values_for_tests(vec![(
+            "provider_priority_mode".to_string(),
+            json!("global_key"),
+        )]),
    );
    let gateway = build_router_with_state(gateway_state);
    let (gateway_url, gateway_handle) = start_server(gateway).await;
@@ -1310,7 +1314,11 @@ async fn gateway_executes_openai_chat_stream_with_custom_path_via_local_decision
            provider_catalog_repository,
            Arc::clone(&request_candidate_repository),
            DEVELOPMENT_ENCRYPTION_KEY,
-        ),
+        )
+        .with_system_config_values_for_tests(vec![(
+            "provider_priority_mode".to_string(),
+            json!("global_key"),
+        )]),
    );
    let gateway = build_router_with_state(gateway_state);
    let (gateway_url, gateway_handle) = start_server(gateway).await;
@@ -1409,7 +1417,8 @@ async fn gateway_executes_openai_chat_stream_with_custom_path_via_local_decision
 }

 #[tokio::test]
-async fn gateway_retries_next_local_openai_chat_stream_candidate_with_local_failover_only() {
+async fn gateway_retries_next_local_openai_chat_stream_candidate_after_retryable_429_execution_runtime_status(
+) {
    #[derive(Debug, Clone)]
    struct SeenExecutionRuntimeStreamRequest {
        trace_id: String,
@@ -1716,7 +1725,8 @@ async fn gateway_retries_next_local_openai_chat_stream_candidate_with_local_fail

                let frames = if attempt == 1 {
                    concat!(
-                        "{\"type\":\"headers\",\"payload\":{\"kind\":\"headers\",\"status_code\":502,\"headers\":{\"content-type\":\"application/json\"}}}\n",
+                        "{\"type\":\"headers\",\"payload\":{\"kind\":\"headers\",\"status_code\":429,\"headers\":{\"content-type\":\"application/json\"}}}\n",
+                        "{\"type\":\"data\",\"payload\":{\"kind\":\"data\",\"text\":\"{\\\"error\\\":{\\\"message\\\":\\\"rate limited\\\",\\\"type\\\":\\\"rate_limit_error\\\"}}\"}}\n",
                        "{\"type\":\"eof\",\"payload\":{\"kind\":\"eof\"}}\n"
                    )
                } else {
@@ -1812,7 +1822,11 @@ async fn gateway_retries_next_local_openai_chat_stream_candidate_with_local_fail
            provider_catalog_repository,
            Arc::clone(&request_candidate_repository),
            DEVELOPMENT_ENCRYPTION_KEY,
-        ),
+        )
+        .with_system_config_values_for_tests(vec![(
+            "provider_priority_mode".to_string(),
+            json!("global_key"),
+        )]),
    );
    let gateway = build_router_with_state(gateway_state);
    let (gateway_url, gateway_handle) = start_server(gateway).await;
@@ -1887,11 +1901,15 @@ async fn gateway_retries_next_local_openai_chat_stream_candidate_with_local_fail
    assert_eq!(stored_candidates.len(), 2);
    assert_eq!(stored_candidates[0].candidate_index, 0);
    assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Failed);
-    assert_eq!(stored_candidates[0].status_code, Some(502));
+    assert_eq!(stored_candidates[0].status_code, Some(429));
    assert_eq!(
        stored_candidates[0].error_type.as_deref(),
        Some("retryable_upstream_status")
    );
+    assert_eq!(
+        stored_candidates[0].error_message.as_deref(),
+        Some("execution runtime stream returned retryable status 429")
+    );
    assert_eq!(stored_candidates[1].candidate_index, 1);
    assert_eq!(stored_candidates[1].status, RequestCandidateStatus::Success);
    assert_eq!(stored_candidates[1].status_code, Some(200));
--- a/apps/aether-gateway/src/tests/ai_execute/stream_provider.rs
+++ b/apps/aether-gateway/src/tests/ai_execute/stream_provider.rs
@@ -572,7 +572,8 @@ async fn gateway_executes_kiro_claude_cli_stream_via_local_provider_catalog_cand
 }

 #[tokio::test]
-async fn gateway_executes_claude_cli_stream_via_local_decision_gate_with_local_stream_decision() {
+async fn gateway_executes_claude_cli_stream_via_local_decision_gate_without_waiting_for_same_format_prefetch(
+) {
    #[derive(Debug, Clone)]
    struct SeenExecutionRuntimeStreamRequest {
        trace_id: String,
@@ -869,15 +870,24 @@ async fn gateway_executes_claude_cli_stream_via_local_decision_gate_with_local_s
                            .unwrap_or_default()
                            .to_string(),
                    });
-                let frames = concat!(
-                    "{\"type\":\"headers\",\"payload\":{\"kind\":\"headers\",\"status_code\":200,\"headers\":{\"content-type\":\"text/event-stream\"}}}\n",
-                    "{\"type\":\"data\",\"payload\":{\"kind\":\"data\",\"text\":\"event: message_start\\ndata: {\\\"type\\\":\\\"message_start\\\"}\\n\\n\"}}\n",
-                    "{\"type\":\"telemetry\",\"payload\":{\"kind\":\"telemetry\",\"telemetry\":{\"elapsed_ms\":31,\"ttfb_ms\":11,\"upstream_bytes\":37}}}\n",
-                    "{\"type\":\"eof\",\"payload\":{\"kind\":\"eof\"}}\n"
-                );
+                let body_stream = async_stream::stream! {
+                    yield Ok::<Bytes, std::convert::Infallible>(Bytes::from_static(
+                        b"{\"type\":\"headers\",\"payload\":{\"kind\":\"headers\",\"status_code\":200,\"headers\":{\"content-type\":\"text/event-stream\"}}}\n"
+                    ));
+                    yield Ok::<Bytes, std::convert::Infallible>(Bytes::from_static(
+                        b"{\"type\":\"data\",\"payload\":{\"kind\":\"data\",\"text\":\"event: message_start\\ndata: {\\\"type\\\":\\\"message_start\\\"}\\n\\n\"}}\n"
+                    ));
+                    tokio::time::sleep(std::time::Duration::from_millis(250)).await;
+                    yield Ok::<Bytes, std::convert::Infallible>(Bytes::from_static(
+                        b"{\"type\":\"telemetry\",\"payload\":{\"kind\":\"telemetry\",\"telemetry\":{\"elapsed_ms\":31,\"ttfb_ms\":11,\"upstream_bytes\":37}}}\n"
+                    ));
+                    yield Ok::<Bytes, std::convert::Infallible>(Bytes::from_static(
+                        b"{\"type\":\"eof\",\"payload\":{\"kind\":\"eof\"}}\n"
+                    ));
+                };
                let mut response = Response::builder()
                    .status(StatusCode::OK)
-                    .body(Body::from(frames))
+                    .body(Body::from_stream(body_stream))
                    .expect("response should build");
                response.headers_mut().insert(
                    http::header::CONTENT_TYPE,
@@ -918,7 +928,7 @@ async fn gateway_executes_claude_cli_stream_via_local_decision_gate_with_local_s
    let gateway = build_router_with_state(gateway_state);
    let (gateway_url, gateway_handle) = start_server(gateway).await;

-    let response = reqwest::Client::new()
+    let mut response = reqwest::Client::new()
        .post(format!("{gateway_url}/v1/messages"))
        .header(http::header::CONTENT_TYPE, "application/json")
        .header(
@@ -935,8 +945,16 @@ async fn gateway_executes_claude_cli_stream_via_local_decision_gate_with_local_s

    assert_eq!(response.status(), StatusCode::OK);
    assert_eq!(
-        response.text().await.expect("body should read"),
-        "event: message_start\ndata: {\"type\":\"message_start\"}\n\n"
+        tokio::time::timeout(std::time::Duration::from_millis(100), response.chunk())
+            .await
+            .expect("same-format passthrough should yield first chunk before eof")
+            .expect("first chunk should read")
+            .expect("first chunk should exist"),
+        Bytes::from_static(b"event: message_start\ndata: {\"type\":\"message_start\"}\n\n")
+    );
+    assert_eq!(
+        response.text().await.expect("remaining body should read"),
+        ""
    );

    let seen_execution_runtime_request = seen_execution_runtime
--- a/apps/aether-gateway/src/tests/ai_execute/stream_provider_gemini/local_cli.rs
+++ b/apps/aether-gateway/src/tests/ai_execute/stream_provider_gemini/local_cli.rs
@@ -1836,9 +1836,32 @@ async fn gateway_executes_antigravity_gemini_cli_stream_via_local_decision_gate_
        .expect("request should succeed");

    assert_eq!(response.status(), StatusCode::OK);
+    let response_text = response.text().await.expect("body should read");
+    let payload = response_text
+        .trim()
+        .strip_prefix("data: ")
+        .expect("response should start with sse data prefix");
+    let response_json: serde_json::Value =
+        serde_json::from_str(payload).expect("stream payload should parse");
    assert_eq!(
-        response.text().await.expect("body should read"),
-        "data: {\"_v1internal_response_id\":\"resp_antigravity_cli_local_stream_123\",\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"Hello Antigravity Stream\"}],\"role\":\"model\"},\"finishReason\":\"STOP\",\"index\":0}],\"modelVersion\":\"claude-sonnet-4-5\",\"usageMetadata\":{\"candidatesTokenCount\":3,\"promptTokenCount\":2,\"totalTokenCount\":5}}\n\n"
+        response_json,
+        json!({
+            "_v1internal_response_id": "resp_antigravity_cli_local_stream_123",
+            "candidates": [{
+                "content": {
+                    "parts": [{"text": "Hello Antigravity Stream"}],
+                    "role": "model"
+                },
+                "finishReason": "STOP",
+                "index": 0
+            }],
+            "modelVersion": "claude-sonnet-4-5",
+            "usageMetadata": {
+                "promptTokenCount": 2,
+                "candidatesTokenCount": 3,
+                "totalTokenCount": 5
+            }
+        })
    );

    let seen_refresh_request = seen_refresh
--- a/apps/aether-gateway/src/tests/ai_execute/sync/chat/failover.rs
+++ b/apps/aether-gateway/src/tests/ai_execute/sync/chat/failover.rs
@@ -331,7 +331,11 @@ async fn gateway_skips_unsupported_local_openai_chat_sync_candidate_before_tryin
            provider_catalog_repository,
            Arc::clone(&request_candidate_repository),
            DEVELOPMENT_ENCRYPTION_KEY,
-        ),
+        )
+        .with_system_config_values_for_tests(vec![(
+            "provider_priority_mode".to_string(),
+            json!("global_key"),
+        )]),
    );
    let gateway = build_router_with_state(gateway_state);
    let (gateway_url, gateway_handle) = start_server(gateway).await;
@@ -1058,7 +1062,11 @@ async fn gateway_retries_next_local_openai_chat_sync_candidate_with_local_failov
            provider_catalog_repository,
            Arc::clone(&request_candidate_repository),
            DEVELOPMENT_ENCRYPTION_KEY,
-        ),
+        )
+        .with_system_config_values_for_tests(vec![(
+            "provider_priority_mode".to_string(),
+            json!("global_key"),
+        )]),
    );
    let gateway = build_router_with_state(gateway_state);
    let (gateway_url, gateway_handle) = start_server(gateway).await;
--- a/apps/aether-gateway/src/tests/proxy.rs
+++ b/apps/aether-gateway/src/tests/proxy.rs
@@ -3,10 +3,12 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
 use super::{
    any, build_router, build_router_with_state, json, start_server, to_bytes, AppState, Arc, Body,
    HeaderValue, Json, Mutex, Request, Response, Router, StatusCode, DEPENDENCY_REASON_HEADER,
-    EXECUTION_PATH_HEADER, EXECUTION_PATH_LOCAL_ROUTE_NOT_FOUND, FORWARDED_FOR_HEADER,
-    GATEWAY_HEADER, TRACE_ID_HEADER, TRUSTED_AUTH_ACCESS_ALLOWED_HEADER,
-    TRUSTED_AUTH_API_KEY_ID_HEADER, TRUSTED_AUTH_USER_ID_HEADER,
-    TUNNEL_AFFINITY_FORWARDED_BY_HEADER, TUNNEL_AFFINITY_OWNER_INSTANCE_HEADER,
+    EXECUTION_PATH_HEADER, EXECUTION_PATH_LOCAL_EXECUTION_LOOP_DETECTED,
+    EXECUTION_PATH_LOCAL_ROUTE_NOT_FOUND, EXECUTION_RUNTIME_LOOP_GUARD_HEADER,
+    EXECUTION_RUNTIME_LOOP_GUARD_VALUE, FORWARDED_FOR_HEADER, GATEWAY_HEADER, TRACE_ID_HEADER,
+    TRUSTED_AUTH_ACCESS_ALLOWED_HEADER, TRUSTED_AUTH_API_KEY_ID_HEADER,
+    TRUSTED_AUTH_USER_ID_HEADER, TUNNEL_AFFINITY_FORWARDED_BY_HEADER,
+    TUNNEL_AFFINITY_OWNER_INSTANCE_HEADER,
 };

 use aether_data::repository::auth::{
@@ -246,6 +248,69 @@ async fn gateway_preserves_existing_trace_id_on_unknown_local_not_found() {
    upstream_handle.abort();
 }

+#[tokio::test]
+async fn gateway_rejects_execution_runtime_loop_guarded_ai_request() {
+    let gateway = build_router().expect("gateway should build");
+    let (gateway_url, gateway_handle) = start_server(gateway).await;
+
+    let response = reqwest::Client::new()
+        .post(format!("{gateway_url}/v1/responses"))
+        .header(TRACE_ID_HEADER, "trace-loop-guard-123")
+        .header(
+            EXECUTION_RUNTIME_LOOP_GUARD_HEADER,
+            EXECUTION_RUNTIME_LOOP_GUARD_VALUE,
+        )
+        .header(http::header::CONTENT_TYPE, "application/json")
+        .body(r#"{"model":"gpt-5.4","input":"hello"}"#)
+        .send()
+        .await
+        .expect("request should succeed");
+
+    assert_eq!(response.status(), StatusCode::LOOP_DETECTED);
+    assert_eq!(
+        response
+            .headers()
+            .get(EXECUTION_PATH_HEADER)
+            .and_then(|value| value.to_str().ok()),
+        Some(EXECUTION_PATH_LOCAL_EXECUTION_LOOP_DETECTED)
+    );
+    let payload: serde_json::Value = response.json().await.expect("body should parse");
+    assert_eq!(payload["error"]["type"], "http_error");
+    assert_eq!(
+        payload["error"]["message"],
+        "Gateway detected an execution runtime request loop back into the local frontdoor"
+    );
+
+    gateway_handle.abort();
+}
+
+#[tokio::test]
+async fn gateway_rejects_execution_runtime_via_guarded_ai_request() {
+    let gateway = build_router().expect("gateway should build");
+    let (gateway_url, gateway_handle) = start_server(gateway).await;
+
+    let response = reqwest::Client::new()
+        .post(format!("{gateway_url}/v1/messages"))
+        .header(TRACE_ID_HEADER, "trace-loop-via-123")
+        .header("via", "1.1 aether-execution-runtime")
+        .header(http::header::CONTENT_TYPE, "application/json")
+        .body(r#"{"model":"claude-sonnet-4","messages":[{"role":"user","content":"hello"}]}"#)
+        .send()
+        .await
+        .expect("request should succeed");
+
+    assert_eq!(response.status(), StatusCode::LOOP_DETECTED);
+    assert_eq!(
+        response
+            .headers()
+            .get(EXECUTION_PATH_HEADER)
+            .and_then(|value| value.to_str().ok()),
+        Some(EXECUTION_PATH_LOCAL_EXECUTION_LOOP_DETECTED)
+    );
+
+    gateway_handle.abort();
+}
+
 #[tokio::test]
 async fn gateway_forwards_public_request_to_remote_tunnel_owner_before_fallback_probe() {
    #[derive(Debug, Clone)]