feat: 新增 frontdoor 执行回环守卫与多项可观测性增强

- 新增 frontdoor_loop_guard 模块,检测并拒绝 execution runtime 回环到本地网关的请求(HTTP 508)
- candidate loop 引入 span tracking、执行尝试日志与流式看门狗超时
- 本地故障转移策略支持从 report_context 加载,新增 append_local_failover_policy_to_value
- runtime tracing 美化:移除 identity 前缀,按 span 深度树形缩进,target 固定宽度展示
- Codex OpenAI CLI 补齐 chatgpt-account-id/x-client-request-id/session_id/conversation_id 请求头
- OpenAI CLI same/cross-format 聚合规则放宽以支持 openai:compact 客户端格式,并过滤 error-like 响应体
- auth/proxy/finalize 日志补充 user_id/api_key_id/api_key_name/balance_remaining 等字段
- 启动日志拆分为 starting/ready/config 三段,新增 resolve_bind_http_base_url
- access_log middleware 将生成的 trace_id 回注到下游请求头
- Cargo.toml 启用 serde_json preserve_order 特性
This commit is contained in:
fawney19
2026-04-11 01:50:24 +08:00
parent 3f057628b7
commit 6144473ebe
38 changed files with 1957 additions and 421 deletions

View File

@@ -288,7 +288,11 @@ mod tests {
#[test]
fn explicit_completed_status_wins_over_legacy_failure_fields() {
let item = sample_usage("completed", Some(429), Some("rate limited on first attempt"));
let item = sample_usage(
"completed",
Some(429),
Some("rate limited on first attempt"),
);
assert!(!admin_usage_is_failed(&item));
assert!(!admin_usage_matches_status(&item, Some("failed")));
assert!(admin_usage_matches_status(&item, Some("completed")));

View File

@@ -211,9 +211,14 @@ pub fn maybe_build_openai_cli_cross_format_sync_product_from_normalized_payload(
.trim()
.to_ascii_lowercase();
if client_api_format != "openai:cli"
|| sync_cli_response_conversion_kind(&provider_api_format, &client_api_format).is_none()
{
if !matches!(client_api_format.as_str(), "openai:cli" | "openai:compact") {
return Ok(None);
}
if !matches!(
provider_api_format.as_str(),
"openai:cli" | "claude:chat" | "claude:cli" | "gemini:chat" | "gemini:cli"
) {
return Ok(None);
}
@@ -434,15 +439,8 @@ fn maybe_build_openai_cli_same_family_sync_body(
.unwrap_or_default()
.trim()
.to_ascii_lowercase();
let needs_conversion = report_context
.get("needs_conversion")
.and_then(Value::as_bool)
.unwrap_or(false);
if !is_openai_cli_family_api_format(&provider_api_format)
|| !is_openai_cli_family_api_format(&client_api_format)
|| provider_api_format != client_api_format
|| needs_conversion
{
return None;
}
@@ -481,15 +479,8 @@ fn maybe_build_openai_cli_same_family_stream_sync_body(
.unwrap_or_default()
.trim()
.to_ascii_lowercase();
let needs_conversion = report_context
.get("needs_conversion")
.and_then(Value::as_bool)
.unwrap_or(false);
if !is_openai_cli_family_api_format(&provider_api_format)
|| !is_openai_cli_family_api_format(&client_api_format)
|| provider_api_format != client_api_format
|| needs_conversion
{
return Ok(None);
}
@@ -521,7 +512,8 @@ fn maybe_build_openai_cross_format_provider_body_from_normalized_payload(
None => None,
};
Ok(aggregated_stream_body.or_else(|| body_json.cloned()))
let provider_body_json = aggregated_stream_body.or_else(|| body_json.cloned());
Ok(provider_body_json.filter(|value| !is_error_like_sync_body(value)))
}
fn is_error_like_sync_body(value: &Value) -> bool {
@@ -1501,27 +1493,28 @@ mod tests {
}
#[test]
fn rejects_openai_cli_same_family_exact_same_stream_when_needs_conversion_is_true() {
fn accepts_openai_cli_same_family_stream_when_needs_conversion_is_true() {
let body = concat!(
"event: response.completed\n",
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_123\",\"object\":\"response\",\"model\":\"gpt-5\",\"status\":\"completed\",\"output\":[]}}\n\n",
);
let report_context = json!({
"provider_api_format": "openai:cli",
"client_api_format": "openai:cli",
"client_api_format": "openai:compact",
"needs_conversion": true,
});
let body_json = maybe_build_openai_cli_same_family_sync_body_from_normalized_payload(
"openai_cli_sync_finalize",
"openai_compact_sync_finalize",
200,
Some(&report_context),
None,
Some(&base64::engine::general_purpose::STANDARD.encode(body)),
)
.expect("openai-cli same-family guard should not error");
.expect("openai-cli same-family aggregation should not error")
.expect("aggregated body should exist");
assert!(body_json.is_none());
assert_eq!(body_json["id"], "resp_123");
}
#[test]
@@ -1736,6 +1729,60 @@ mod tests {
);
}
#[test]
fn rejects_openai_cli_cross_format_error_body_json() {
let report_context = json!({
"provider_api_format": "openai:cli",
"client_api_format": "openai:compact",
"model": "gpt-5",
"mapped_model": "gpt-5",
});
let provider_body_json = json!({
"error": {
"message": "quota reached",
"type": "rate_limit_error"
}
});
let product = maybe_build_openai_cli_cross_format_sync_product_from_normalized_payload(
"openai_compact_sync_finalize",
200,
Some(&report_context),
Some(&provider_body_json),
None,
)
.expect("openai-cli cross-format error guard should not error");
assert!(product.is_none());
}
#[test]
fn rejects_openai_cli_cross_format_for_openai_family_provider() {
let report_context = json!({
"provider_api_format": "openai:compact",
"client_api_format": "openai:cli",
"model": "gpt-5",
"mapped_model": "gpt-5",
});
let provider_body_json = json!({
"id": "resp_123",
"object": "response",
"status": "completed",
"output": []
});
let product = maybe_build_openai_cli_cross_format_sync_product_from_normalized_payload(
"openai_cli_sync_finalize",
200,
Some(&report_context),
Some(&provider_body_json),
None,
)
.expect("openai-cli cross-format openai-family guard should not error");
assert!(product.is_none());
}
#[test]
fn rejects_openai_chat_cross_format_for_unsupported_matrix() {
let report_context = json!({

View File

@@ -74,6 +74,17 @@ fn header_map_has_non_empty_value(headers: &http::HeaderMap, header_name: &str)
})
}
fn btree_map_has_non_empty_value(headers: &BTreeMap<String, String>, header_name: &str) -> bool {
let target = header_name.trim().to_ascii_lowercase();
if target.is_empty() {
return false;
}
headers
.iter()
.any(|(name, value)| name.trim().eq_ignore_ascii_case(&target) && !value.trim().is_empty())
}
fn extract_codex_account_id(decrypted_auth_config_raw: Option<&str>) -> Option<String> {
let raw = decrypted_auth_config_raw?.trim();
if raw.is_empty() {
@@ -187,7 +198,42 @@ pub fn apply_codex_openai_cli_special_headers(
.map(str::trim)
.filter(|value| !value.is_empty());
if !header_map_has_non_empty_value(original_headers, "session_id") {
provider_request_headers.insert("session_id".to_string(), prompt_cache_key.unwrap().to_string());
if !header_map_has_non_empty_value(original_headers, "chatgpt-account-id")
&& !btree_map_has_non_empty_value(provider_request_headers, "chatgpt-account-id")
{
if let Some(account_id) = extract_codex_account_id(decrypted_auth_config_raw) {
provider_request_headers.insert("chatgpt-account-id".to_string(), account_id);
}
}
if !header_map_has_non_empty_value(original_headers, "x-client-request-id")
&& !btree_map_has_non_empty_value(provider_request_headers, "x-client-request-id")
{
if let Some(request_id) = request_id.map(str::trim).filter(|value| !value.is_empty()) {
provider_request_headers
.insert("x-client-request-id".to_string(), request_id.to_string());
}
}
let short_session_id = prompt_cache_key.and_then(build_short_codex_header_id);
if !header_map_has_non_empty_value(original_headers, "session_id")
&& !btree_map_has_non_empty_value(provider_request_headers, "session_id")
{
if let Some(short_session_id) = short_session_id.as_deref() {
provider_request_headers.insert("session_id".to_string(), short_session_id.to_string());
}
}
if provider_api_format
.trim()
.eq_ignore_ascii_case("openai:cli")
&& !header_map_has_non_empty_value(original_headers, "conversation_id")
&& !btree_map_has_non_empty_value(provider_request_headers, "conversation_id")
{
if let Some(short_session_id) = short_session_id.as_deref() {
provider_request_headers
.insert("conversation_id".to_string(), short_session_id.to_string());
}
}
}

View File

@@ -13,6 +13,7 @@ pub fn should_skip_request_header(name: &str) -> bool {
| "upgrade"
| "x-aether-execution-path"
| "x-aether-dependency-reason"
| "x-aether-execution-loop-guard"
| "x-aether-control-execute-fallback"
| "x-aether-rate-limit-preflight"
)

View File

@@ -46,10 +46,6 @@ impl RuntimeLogIdentity {
instance_id: config.observability.instance_id.clone(),
}
}
fn node_role_display(&self) -> &str {
self.node_role.as_deref().unwrap_or("-")
}
}
#[derive(Debug, Clone)]
@@ -149,13 +145,16 @@ impl Visit for RuntimeFieldVisitor {
#[derive(Debug, Clone)]
struct PrettyRuntimeEventFormatter {
identity: RuntimeLogIdentity,
_identity: RuntimeLogIdentity,
ansi: bool,
}
impl PrettyRuntimeEventFormatter {
fn new(identity: RuntimeLogIdentity, ansi: bool) -> Self {
Self { identity, ansi }
Self {
_identity: identity,
ansi,
}
}
}
@@ -166,7 +165,7 @@ where
{
fn format_event(
&self,
_ctx: &FmtContext<'_, S, N>,
ctx: &FmtContext<'_, S, N>,
mut writer: Writer<'_>,
event: &Event<'_>,
) -> fmt::Result {
@@ -176,6 +175,7 @@ where
let level_color = self.ansi.then_some(level_ansi(meta.level()));
let message = fields.take_message();
let depth = ctx.event_scope().map(|scope| scope.count()).unwrap_or(0);
// timestamp (green)
write_colored(
@@ -189,21 +189,16 @@ where
write_colored(&mut writer, &format!("{:<8}", meta.level()), level_color)?;
// separator
write_separator(&mut writer, self.ansi)?;
// service:node_role (compact identity, dimmed)
let identity_label = format!(
"{}:{}",
self.identity.service,
self.identity.node_role_display(),
);
write_colored(&mut writer, &identity_label, self.ansi.then_some(ANSI_DIM))?;
// separator
write_separator(&mut writer, self.ansi)?;
// target (cyan, shortened to last 2 segments)
let short_target = shorten_target(meta.target(), 2);
write_colored(&mut writer, short_target, self.ansi.then_some(ANSI_CYAN))?;
// target (cyan, shortened/truncated to fixed width)
let target_cell = format_target_cell(meta.target(), TARGET_COLUMN_WIDTH);
write_colored(&mut writer, &target_cell, self.ansi.then_some(ANSI_CYAN))?;
// message (level color, after " - ")
if let Some(ref msg) = message {
write_colored(&mut writer, " - ", self.ansi.then_some(ANSI_DIM))?;
let prefix = span_tree_prefix(depth);
if !prefix.is_empty() {
write_colored(&mut writer, &prefix, self.ansi.then_some(ANSI_DIM))?;
}
write_colored(&mut writer, msg, level_color)?;
}
// remaining structured fields
@@ -233,13 +228,14 @@ where
{
fn format_event(
&self,
_ctx: &FmtContext<'_, S, N>,
ctx: &FmtContext<'_, S, N>,
mut writer: Writer<'_>,
event: &Event<'_>,
) -> fmt::Result {
let meta = event.metadata();
let mut fields = RuntimeFieldVisitor::default();
event.record(&mut fields);
let depth = ctx.event_scope().map(|scope| scope.count()).unwrap_or(0);
let mut payload = Map::new();
payload.insert(
@@ -271,6 +267,7 @@ where
"target".to_string(),
Value::String(meta.target().to_string()),
);
payload.insert("span_depth".to_string(), Value::from(depth as u64));
payload.insert(
"fields".to_string(),
Value::Object(fields.into_json_object()),
@@ -286,6 +283,8 @@ fn formatted_timestamp() -> String {
Local::now().format("%Y-%m-%d %H:%M:%S%.3f %:z").to_string()
}
const TARGET_COLUMN_WIDTH: usize = 24;
fn shorten_target(target: &str, max_segments: usize) -> &str {
let mut count = 0usize;
for (idx, _) in target.rmatch_indices("::") {
@@ -297,6 +296,38 @@ fn shorten_target(target: &str, max_segments: usize) -> &str {
target
}
fn span_tree_prefix(depth: usize) -> String {
if depth == 0 {
return String::new();
}
let mut prefix = String::new();
for _ in 0..depth.saturating_sub(1) {
prefix.push_str("");
}
prefix.push_str("├─ ");
prefix
}
fn format_target_cell(target: &str, width: usize) -> String {
let short_target = shorten_target(target, 2);
let len = short_target.chars().count();
if len <= width {
let padding = " ".repeat(width - len);
return format!("{short_target}{padding}");
}
let tail: String = short_target
.chars()
.rev()
.take(width.saturating_sub(1))
.collect::<Vec<_>>()
.into_iter()
.rev()
.collect();
format!("~{tail}")
}
const ANSI_RESET: &str = "\u{1b}[0m";
const ANSI_DIM: &str = "\u{1b}[2m";
const ANSI_BOLD: &str = "\u{1b}[1m";
@@ -806,9 +837,10 @@ fn select_log_files_for_cleanup(
#[cfg(test)]
mod tests {
use super::{
bucketed_log_path, cleanup_log_files, log_bucket_key, select_log_files_for_cleanup,
FileLoggingConfig, JsonRuntimeEventFormatter, LogFileCandidate, LogRotation,
PrettyRuntimeEventFormatter, RollingFileSink, RuntimeLogIdentity,
bucketed_log_path, cleanup_log_files, format_target_cell, log_bucket_key,
select_log_files_for_cleanup, FileLoggingConfig, JsonRuntimeEventFormatter,
LogFileCandidate, LogRotation, PrettyRuntimeEventFormatter, RollingFileSink,
RuntimeLogIdentity,
};
use chrono::{Local, TimeZone};
use std::fs;
@@ -955,7 +987,7 @@ mod tests {
}
#[test]
fn pretty_formatter_includes_service_identity_fields() {
fn pretty_formatter_omits_service_identity_fields() {
let writer = SharedBuffer::default();
let subscriber = tracing_subscriber::registry().with(
tracing_subscriber::fmt::layer()
@@ -972,17 +1004,26 @@ mod tests {
let dispatch = tracing::Dispatch::new(subscriber);
let _guard = tracing::dispatcher::set_default(&dispatch);
tracing::info!(event_name = "test_event", value = 7_u64, "hello");
tracing::info!(
target: "runtime::tracing",
event_name = "test_event",
value = 7_u64,
"hello"
);
let output = writer.contents();
assert!(
output.contains("test-service:frontdoor"),
"should contain compact identity"
!output.contains("test-service:frontdoor"),
"should not contain compact identity"
);
assert!(
output.contains(" | INFO"),
"should contain pipe-separated level"
);
assert!(
output.contains("runtime::tracing"),
"should contain shortened target"
);
assert!(
output.contains(" - hello"),
"should contain message after dash"
@@ -1017,13 +1058,95 @@ mod tests {
"should contain ANSI escape sequences"
);
assert!(
output.contains("test-service:frontdoor"),
"should contain compact identity"
!output.contains("test-service:frontdoor"),
"should not contain compact identity"
);
assert!(output.contains("colored"), "should contain message text");
assert!(output.contains("ansi_event"));
}
#[test]
fn pretty_formatter_adds_tree_prefix_inside_span() {
let writer = SharedBuffer::default();
let subscriber = tracing_subscriber::registry().with(
tracing_subscriber::fmt::layer()
.with_writer(writer.clone())
.event_format(PrettyRuntimeEventFormatter::new(
RuntimeLogIdentity {
service: "test-service",
node_role: Some("frontdoor".to_string()),
instance_id: Some("gateway-a".to_string()),
},
false,
)),
);
let dispatch = tracing::Dispatch::new(subscriber);
let _guard = tracing::dispatcher::set_default(&dispatch);
tracing::info_span!("candidates").in_scope(|| {
tracing::debug!(
target: "executor::candidate_loop",
event_name = "candidate_loop_started",
"inside span"
);
});
let output = writer.contents();
assert!(output.contains("executor::candidate_loop"));
assert!(output.contains(" - ├─ inside span"));
}
#[test]
fn pretty_formatter_keeps_target_column_aligned() {
let writer = SharedBuffer::default();
let subscriber = tracing_subscriber::registry().with(
tracing_subscriber::fmt::layer()
.with_writer(writer.clone())
.event_format(PrettyRuntimeEventFormatter::new(
RuntimeLogIdentity {
service: "test-service",
node_role: Some("frontdoor".to_string()),
instance_id: Some("gateway-a".to_string()),
},
false,
)),
);
let dispatch = tracing::Dispatch::new(subscriber);
let _guard = tracing::dispatcher::set_default(&dispatch);
tracing::info!(
target: "short::name",
event_name = "short_event",
"short message"
);
tracing::info!(
target: "root::supercalifragilistic::anotherverylongsegment",
event_name = "long_event",
"long message"
);
let output = writer.contents();
let lines = output.lines().collect::<Vec<_>>();
assert_eq!(lines.len(), 2, "expected exactly two log lines");
let first_dash = lines[0]
.find(" - ")
.expect("short line should contain message separator");
let second_dash = lines[1]
.find(" - ")
.expect("long line should contain message separator");
assert_eq!(
first_dash, second_dash,
"message separator should stay aligned"
);
assert!(
lines[1].contains(&format_target_cell(
"root::supercalifragilistic::anotherverylongsegment",
super::TARGET_COLUMN_WIDTH,
)),
"long target should be truncated into the fixed-width cell"
);
}
#[test]
fn json_formatter_includes_service_identity_fields() {
let writer = SharedBuffer::default();
@@ -1053,8 +1176,36 @@ mod tests {
assert_eq!(payload["service"], "test-service");
assert_eq!(payload["node_role"], "proxy");
assert_eq!(payload["instance_id"], "proxy-01");
assert_eq!(payload["span_depth"], 0);
assert_eq!(payload["fields"]["event_name"], "test_event");
assert_eq!(payload["fields"]["status"], "failed");
assert_eq!(payload["fields"]["status_code"], 502);
}
#[test]
fn json_formatter_includes_span_depth() {
let writer = SharedBuffer::default();
let subscriber = tracing_subscriber::registry().with(
tracing_subscriber::fmt::layer()
.json()
.with_writer(writer.clone())
.event_format(JsonRuntimeEventFormatter::new(RuntimeLogIdentity {
service: "test-service",
node_role: Some("proxy".to_string()),
instance_id: Some("proxy-01".to_string()),
})),
);
let dispatch = tracing::Dispatch::new(subscriber);
let _guard = tracing::dispatcher::set_default(&dispatch);
tracing::info_span!("request").in_scope(|| {
tracing::info!(event_name = "nested_event", "inside request span");
});
let output = writer.contents();
let payload: serde_json::Value =
serde_json::from_str(output.trim()).expect("json log line should parse");
assert_eq!(payload["span_depth"], 1);
assert_eq!(payload["fields"]["event_name"], "nested_event");
}
}