mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-12 06:00:20 +08:00
Merge pull request #809 from AAEE86/feat-detail-log
fix(gateway): redact sensitive error details in logs
This commit is contained in:
@@ -137,6 +137,10 @@ ADMIN_USERNAME=admin123456
|
||||
# 单请求解压后 Payload 上限(MiB),默认 256;显式 0 仍受 256 MiB 硬上限保护。
|
||||
# AETHER_MAX_REQUEST_BODY_MB=256
|
||||
# AETHER_GATEWAY_SECURITY_CACHE_TTL_MS=1000
|
||||
# 内部错误详情日志默认关闭;仅接受精确的小写 true(开启)和 false(关闭)。
|
||||
# 开启后 ERROR 日志输出不截断的错误详情,但仍移除 URL 凭据并遮盖常见密码、令牌等字段。
|
||||
# 脱敏无法覆盖任意敏感内容,排查完毕请关闭;未设置或无法识别的值均按关闭处理。
|
||||
# AETHER_GATEWAY_ERROR_DETAIL_LOGGING=false
|
||||
# AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB=64
|
||||
# AETHER_MAX_INTERNAL_BUFFERED_BODY_MB=64
|
||||
# AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY=1024
|
||||
|
||||
@@ -174,7 +174,7 @@ async fn authorize_operational_request(
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(error = ?err, "operational admin session authentication failed");
|
||||
warn!(error = %crate::error::redact_error_debug(&err), "operational admin session authentication failed");
|
||||
return operational_error_response(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"operational authentication unavailable",
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::http::{Response, StatusCode};
|
||||
use axum::response::IntoResponse;
|
||||
use axum::Json;
|
||||
use regex::Regex;
|
||||
use serde_json::json;
|
||||
use sha2::{Digest, Sha256};
|
||||
use tracing::warn;
|
||||
@@ -10,6 +13,134 @@ use crate::ai_serving::AiSurfaceFinalizeError;
|
||||
use crate::constants::*;
|
||||
use crate::insert_header_if_missing;
|
||||
|
||||
/// 开启后记录不截断但仍脱敏的内部错误详情,默认关闭。
|
||||
static GATEWAY_ERROR_DETAIL_LOGGING: LazyLock<bool> = LazyLock::new(|| {
|
||||
parse_gateway_error_detail_logging(
|
||||
std::env::var("AETHER_GATEWAY_ERROR_DETAIL_LOGGING")
|
||||
.ok()
|
||||
.as_deref(),
|
||||
)
|
||||
});
|
||||
|
||||
fn parse_gateway_error_detail_logging(value: Option<&str>) -> bool {
|
||||
// 仅接受精确的小写 true/false;未设置或无效值默认关闭详情日志。
|
||||
value
|
||||
.and_then(|value| value.parse::<bool>().ok())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
// 按 URL authority 的边界匹配 userinfo,避免跨过路径、查询串和片段中的 @。
|
||||
static ERROR_URL_USERINFO: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r#"(?i)([a-z][a-z0-9+.-]*://)[^\s/?\#"<>]*@"#)
|
||||
.expect("error URL userinfo regex should compile")
|
||||
});
|
||||
|
||||
static ERROR_CREDENTIAL_PREFIX: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(
|
||||
r#"(?ix)
|
||||
\b
|
||||
(?:password|passwd|pwd|(?:access|refresh|id|session|auth)[_-]?token|token|
|
||||
(?:client[_-]?)?secret|(?:api|access|secret|private)[_-]?key|
|
||||
(?:proxy[_-])?authorization)
|
||||
(?:\\*["'])?(?:\s|\\+[nrt])*[:=](?:\s|\\+[nrt])*
|
||||
(?:Some\((?:\s|\\+[nrt])*)?
|
||||
(?:(?:Bearer|Basic)(?:\s|\\+[nrt])+)?
|
||||
|(?:\b|\\+[nrt])Bearer(?:\s|\\+[nrt])+"#,
|
||||
)
|
||||
.expect("error credential prefix regex should compile")
|
||||
});
|
||||
|
||||
/// 检查是否启用了内部错误详情日志。
|
||||
pub(crate) fn gateway_error_detail_logging_enabled() -> bool {
|
||||
*GATEWAY_ERROR_DETAIL_LOGGING
|
||||
}
|
||||
|
||||
/// 日志摘要:移除 URL userinfo、常见凭据键值和 Bearer 内容,再限制为 256 字节。
|
||||
/// 这是自由文本的有限规则,不能保证识别任意敏感内容或编码后的字段名。
|
||||
pub(crate) fn redact_error_detail(error: &impl std::fmt::Display) -> String {
|
||||
redact_error_str(&error.to_string())
|
||||
}
|
||||
|
||||
/// 脱敏 Debug 格式的错误详情(用于未实现 Display 的错误类型)。
|
||||
pub(crate) fn redact_error_debug(error: &impl std::fmt::Debug) -> String {
|
||||
redact_error_str(&format!("{error:?}"))
|
||||
}
|
||||
|
||||
pub(crate) fn redact_error_str(message: &str) -> String {
|
||||
const MAX_LEN: usize = 256;
|
||||
// 必须先处理完整凭据,再截断;否则截断位置可能落在密码和 @host 之间。
|
||||
let mut redacted = redact_error_str_unbounded(message);
|
||||
if redacted.len() > MAX_LEN {
|
||||
let mut end = MAX_LEN;
|
||||
while !redacted.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
redacted.truncate(end);
|
||||
redacted.push_str("...");
|
||||
}
|
||||
redacted
|
||||
}
|
||||
|
||||
/// 详情日志仅取消长度限制,继续使用与摘要相同的凭据脱敏规则。
|
||||
fn redact_error_str_unbounded(message: &str) -> String {
|
||||
let urls_redacted = ERROR_URL_USERINFO.replace_all(message, "$1");
|
||||
let mut result = String::with_capacity(urls_redacted.len());
|
||||
let mut cursor = 0;
|
||||
let mut prefixes = ERROR_CREDENTIAL_PREFIX.find_iter(&urls_redacted).peekable();
|
||||
while let Some(prefix) = prefixes.next() {
|
||||
// 带引号的值可能包含 password= 等文本,已遮盖的内容不再重复处理。
|
||||
if prefix.start() < cursor {
|
||||
continue;
|
||||
}
|
||||
let value_start = prefix.end();
|
||||
// 未引用的值最多读到下一个凭据字段,避免吞掉它的开头却留下带空格的值。
|
||||
let unquoted_limit = prefixes
|
||||
.peek()
|
||||
.map_or(urls_redacted.len() - value_start, |next| {
|
||||
urls_redacted[value_start..next.start()]
|
||||
.trim_end_matches([',', ';', '&'])
|
||||
.len()
|
||||
});
|
||||
let value_end =
|
||||
value_start + credential_value_len(&urls_redacted[value_start..], unquoted_limit);
|
||||
result.push_str(&urls_redacted[cursor..value_start]);
|
||||
result.push_str("[REDACTED]");
|
||||
cursor = value_end;
|
||||
}
|
||||
result.push_str(&urls_redacted[cursor..]);
|
||||
result
|
||||
}
|
||||
|
||||
fn credential_value_len(value: &str, unquoted_limit: usize) -> usize {
|
||||
let bytes = value.as_bytes();
|
||||
let opening_slashes = bytes.iter().take_while(|byte| **byte == b'\\').count();
|
||||
if let Some(quote @ (b'"' | b'\'')) = bytes.get(opening_slashes) {
|
||||
// 仅同一转义层的引号可闭合;无法确认边界时多遮盖,避免泄露密码尾部。
|
||||
let mut slashes = 0;
|
||||
for (index, byte) in bytes.iter().enumerate().skip(opening_slashes + 1) {
|
||||
if byte == quote && slashes == opening_slashes {
|
||||
return index + 1;
|
||||
}
|
||||
slashes = if *byte == b'\\' { slashes + 1 } else { 0 };
|
||||
}
|
||||
// 不完整的引号内容整体遮盖,避免保留密码片段。
|
||||
return value.len();
|
||||
}
|
||||
|
||||
let mut escaped = false;
|
||||
// DSN 未引用密码中的标点也可能是凭据,只以未转义的空白为结束边界。
|
||||
for (index, ch) in value[..unquoted_limit].char_indices() {
|
||||
if ch == '\\' {
|
||||
escaped = true;
|
||||
} else if escaped {
|
||||
escaped = false;
|
||||
} else if ch.is_whitespace() {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
unquoted_limit
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) enum GatewayError {
|
||||
UpstreamUnavailable {
|
||||
@@ -211,13 +342,7 @@ impl IntoResponse for GatewayError {
|
||||
)
|
||||
.into_response(),
|
||||
Self::Internal(message) => {
|
||||
let error_fingerprint = gateway_error_fingerprint(&message);
|
||||
tracing::error!(
|
||||
event_name = "gateway_internal_error",
|
||||
error_fingerprint,
|
||||
error_length = message.len(),
|
||||
"internal gateway error hidden from client"
|
||||
);
|
||||
log_gateway_internal_error(&message, gateway_error_detail_logging_enabled());
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(json!({
|
||||
@@ -232,6 +357,26 @@ impl IntoResponse for GatewayError {
|
||||
}
|
||||
}
|
||||
|
||||
fn log_gateway_internal_error(message: &str, detail_logging: bool) {
|
||||
let error_fingerprint = gateway_error_fingerprint(message);
|
||||
if detail_logging {
|
||||
tracing::error!(
|
||||
event_name = "gateway_internal_error",
|
||||
error_fingerprint,
|
||||
error_length = message.len(),
|
||||
error_detail = %redact_error_str_unbounded(message),
|
||||
"internal gateway error hidden from client"
|
||||
);
|
||||
} else {
|
||||
tracing::error!(
|
||||
event_name = "gateway_internal_error",
|
||||
error_fingerprint,
|
||||
error_length = message.len(),
|
||||
"internal gateway error hidden from client"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn gateway_error_fingerprint(message: &str) -> String {
|
||||
let digest = Sha256::digest(message.as_bytes());
|
||||
format!("{:x}", digest)[..16].to_string()
|
||||
@@ -245,13 +390,68 @@ impl From<AiSurfaceFinalizeError> for GatewayError {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use axum::body::to_bytes;
|
||||
use axum::http::{header::RETRY_AFTER, StatusCode};
|
||||
use axum::response::IntoResponse;
|
||||
|
||||
use crate::constants::TRACE_ID_HEADER;
|
||||
|
||||
use super::{gateway_error_fingerprint, GatewayError};
|
||||
use super::{
|
||||
gateway_error_fingerprint, log_gateway_internal_error, parse_gateway_error_detail_logging,
|
||||
redact_error_debug, redact_error_detail, redact_error_str, GatewayError,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn detail_logging_only_accepts_lowercase_true_and_false() {
|
||||
assert!(parse_gateway_error_detail_logging(Some("true")));
|
||||
assert!(!parse_gateway_error_detail_logging(Some("false")));
|
||||
assert!(!parse_gateway_error_detail_logging(None));
|
||||
for value in [
|
||||
"1", "0", "yes", "no", "on", "off", "TRUE", "FALSE", "True", " true ", "true\n", "",
|
||||
"invalid",
|
||||
] {
|
||||
assert!(
|
||||
!parse_gateway_error_detail_logging(Some(value)),
|
||||
"value: {value:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct LogBuffer(Arc<Mutex<Vec<u8>>>);
|
||||
|
||||
impl std::io::Write for LogBuffer {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
self.0
|
||||
.lock()
|
||||
.expect("log buffer should lock")
|
||||
.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn internal_error_log(message: &str, detail_logging: bool) -> serde_json::Value {
|
||||
let buffer = LogBuffer::default();
|
||||
let writer = buffer.clone();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.json()
|
||||
.without_time()
|
||||
.with_max_level(tracing::Level::ERROR)
|
||||
.with_writer(move || writer.clone())
|
||||
.finish();
|
||||
// 使用线程局部日志捕获和显式开关,不修改进程环境以免干扰并发测试。
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
log_gateway_internal_error(message, detail_logging);
|
||||
});
|
||||
let bytes = buffer.0.lock().expect("log buffer should lock");
|
||||
serde_json::from_slice(&bytes).expect("internal error log should be JSON")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn internal_errors_do_not_expose_internal_details() {
|
||||
@@ -308,4 +508,292 @@ mod tests {
|
||||
Some(trace_id.as_str())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_strips_url_credentials() {
|
||||
let input = "connect failed: postgresql://admin:s3cret@db.internal:5432/aether";
|
||||
let redacted = redact_error_str(input);
|
||||
assert!(!redacted.contains("s3cret"));
|
||||
assert!(redacted.contains("postgresql://"));
|
||||
assert!(redacted.contains("db.internal"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_truncates_long_messages() {
|
||||
let long = "x".repeat(500);
|
||||
let redacted = redact_error_str(&long);
|
||||
assert_eq!(redacted, format!("{}...", "x".repeat(256)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_preserves_short_messages() {
|
||||
let input = "connection refused";
|
||||
assert_eq!(redact_error_str(input), input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_handles_multiple_urls() {
|
||||
for (input, expected) in [
|
||||
(
|
||||
"https://u:first@a/p,https://v:second@b/q",
|
||||
"https://a/p,https://b/q",
|
||||
),
|
||||
(
|
||||
"https://u:first@a,https://v:second@b",
|
||||
"https://a,https://b",
|
||||
),
|
||||
(
|
||||
"failed: https://user:token123@api.example.com/v1 and http://admin:pass@internal.io",
|
||||
"failed: https://api.example.com/v1 and http://internal.io",
|
||||
),
|
||||
] {
|
||||
assert_eq!(redact_error_str(input), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_handles_url_authority_boundaries_and_punctuation() {
|
||||
for (input, expected) in [
|
||||
(
|
||||
r#"url="postgres://u:p@h", retry=2"#,
|
||||
r#"url="postgres://h", retry=2"#,
|
||||
),
|
||||
(
|
||||
"(https://u:p@h?mode=test#detail)",
|
||||
"(https://h?mode=test#detail)",
|
||||
),
|
||||
(
|
||||
"postgres://u:p@ss@[::1]:5432/db",
|
||||
"postgres://[::1]:5432/db",
|
||||
),
|
||||
("https://u:p,a;s's@h/p", "https://h/p"),
|
||||
(
|
||||
"https://h/path@name?email=a@b#ref@c",
|
||||
"https://h/path@name?email=a@b#ref@c",
|
||||
),
|
||||
] {
|
||||
assert_eq!(redact_error_str(input), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_redacts_before_truncation() {
|
||||
let prefix = format!("{} ", "x".repeat(239));
|
||||
let input = format!("{prefix}postgres://u:supersecret@db/app");
|
||||
assert_eq!(
|
||||
redact_error_str(&input),
|
||||
format!("{prefix}postgres://db/ap...")
|
||||
);
|
||||
|
||||
let input = format!("password={} host=db", "secret".repeat(100));
|
||||
assert_eq!(redact_error_str(&input), "password=[REDACTED] host=db");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_preserves_utf8_and_whitespace() {
|
||||
let input = " first \n\tsecond \r\n";
|
||||
assert_eq!(redact_error_str(input), input);
|
||||
assert_eq!(
|
||||
redact_error_str(" failed:\n\tpostgres://u:p@h\r\n retry"),
|
||||
" failed:\n\tpostgres://h\r\n retry"
|
||||
);
|
||||
for length in [254, 255, 256] {
|
||||
let prefix = "x".repeat(length);
|
||||
assert_eq!(
|
||||
redact_error_str(&format!("{prefix}错误")),
|
||||
format!("{prefix}...")
|
||||
);
|
||||
}
|
||||
assert_eq!(redact_error_str(&"x".repeat(256)), "x".repeat(256));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_masks_common_credentials() {
|
||||
for key in [
|
||||
"password",
|
||||
"PASSWORD",
|
||||
"passwd",
|
||||
"pwd",
|
||||
"token",
|
||||
"access_token",
|
||||
"refresh-token",
|
||||
"idToken",
|
||||
"session_token",
|
||||
"auth-token",
|
||||
"secret",
|
||||
"client_secret",
|
||||
"clientSecret",
|
||||
"api_key",
|
||||
"api-key",
|
||||
"apiKey",
|
||||
"access_key",
|
||||
"secret_key",
|
||||
"private_key",
|
||||
"x-api-key",
|
||||
] {
|
||||
for separator in ["=", ":", " = ", "\t:\t"] {
|
||||
let input = format!("{key}{separator}test-credential retry=2");
|
||||
assert_eq!(
|
||||
redact_error_str(&input),
|
||||
format!("{key}{separator}[REDACTED] retry=2")
|
||||
);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
redact_error_str("password=one;token=two,secret=three&retry=2"),
|
||||
"password=[REDACTED];token=[REDACTED],secret=[REDACTED]"
|
||||
);
|
||||
assert_eq!(
|
||||
redact_error_str("https://u:p@host/path?token=abc&api_key=xyz#details"),
|
||||
"https://host/path?token=[REDACTED]&api_key=[REDACTED]"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_does_not_expose_punctuation_in_unquoted_passwords() {
|
||||
for secret in [
|
||||
"one#two", "one?two", "one&two", "one,two", "one;two", "one)two", "one\"two",
|
||||
] {
|
||||
assert_eq!(
|
||||
redact_error_str(&format!("password={secret} host=db")),
|
||||
"password=[REDACTED] host=db"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
redact_error_str("https://host/db?password=one?two&mode=test"),
|
||||
"https://host/db?password=[REDACTED]"
|
||||
);
|
||||
assert_eq!(
|
||||
redact_error_str(r#"password=one,token="two words" retry=2"#),
|
||||
"password=[REDACTED],token=[REDACTED] retry=2"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_masks_authorization_and_bearer_values() {
|
||||
for (input, expected) in [
|
||||
(
|
||||
"Authorization: Bearer short",
|
||||
"Authorization: Bearer [REDACTED]",
|
||||
),
|
||||
(
|
||||
"authorization=bEaReR\tabc.def",
|
||||
"authorization=bEaReR\t[REDACTED]",
|
||||
),
|
||||
(
|
||||
"Proxy-Authorization: Basic abc==",
|
||||
"Proxy-Authorization: Basic [REDACTED]",
|
||||
),
|
||||
(
|
||||
r#"{"Authorization": "Bearer secret value"}"#,
|
||||
r#"{"Authorization": [REDACTED]}"#,
|
||||
),
|
||||
(
|
||||
"error: BEARER a+/b==, retry=2",
|
||||
"error: BEARER [REDACTED] retry=2",
|
||||
),
|
||||
] {
|
||||
assert_eq!(redact_error_str(input), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_masks_quoted_and_escaped_values() {
|
||||
for (input, expected) in [
|
||||
(
|
||||
"password='space secret' host=db",
|
||||
"password=[REDACTED] host=db",
|
||||
),
|
||||
(
|
||||
r#"password="space \"secret" host=db"#,
|
||||
"password=[REDACTED] host=db",
|
||||
),
|
||||
(
|
||||
r"password=space\ secret host=db",
|
||||
"password=[REDACTED] host=db",
|
||||
),
|
||||
(
|
||||
r#"Error { password: "space secret", token: Some("option-secret") }"#,
|
||||
"Error { password: [REDACTED], token: Some([REDACTED]) }",
|
||||
),
|
||||
("password='unterminated secret", "password=[REDACTED]"),
|
||||
(
|
||||
r#"password="another unterminated secret"#,
|
||||
"password=[REDACTED]",
|
||||
),
|
||||
(
|
||||
r#"password="token=inner-secret" retry=2"#,
|
||||
"password=[REDACTED] retry=2",
|
||||
),
|
||||
] {
|
||||
assert_eq!(redact_error_str(input), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_and_debug_error_helpers_redact_real_formatted_values() {
|
||||
let input =
|
||||
r#"{"password": "space \"escaped-secret", "token": "token-secret", "retry": 2}"#;
|
||||
for redacted in [redact_error_detail(&input), redact_error_debug(&input)] {
|
||||
assert!(!redacted.contains("escaped-secret"));
|
||||
assert!(!redacted.contains("token-secret"));
|
||||
assert!(redacted.contains("retry"));
|
||||
assert_eq!(redacted.matches("[REDACTED]").count(), 2);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn debug_error_redaction_handles_escaped_whitespace_and_single_quotes() {
|
||||
for input in [
|
||||
"Authorization:\nBearer test-secret",
|
||||
"Authorization:\tBearer test-secret",
|
||||
"\nBearer test-secret",
|
||||
"Bearer\ntest-secret",
|
||||
r"password=space\ test-secret host=db",
|
||||
r"password='space \'test-secret' host=db",
|
||||
] {
|
||||
let redacted = redact_error_debug(&input);
|
||||
assert!(!redacted.contains("test-secret"), "redacted: {redacted}");
|
||||
assert!(redacted.contains("[REDACTED]"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internal_error_detail_logging_redacts_without_truncating() {
|
||||
let padding = "x".repeat(300);
|
||||
let message = format!("password=first-secret {padding}\nhttps://u:second-secret@db/path token=third-secret\nretry exhausted");
|
||||
let log = internal_error_log(&message, true);
|
||||
let fields = &log["fields"];
|
||||
assert_eq!(log["level"], "ERROR");
|
||||
assert_eq!(fields["event_name"], "gateway_internal_error");
|
||||
assert_eq!(fields["error_length"], message.len());
|
||||
assert_eq!(
|
||||
fields["error_fingerprint"],
|
||||
gateway_error_fingerprint(&message)
|
||||
);
|
||||
assert_eq!(
|
||||
fields["error_detail"],
|
||||
format!(
|
||||
"password=[REDACTED] {padding}\nhttps://db/path token=[REDACTED]\nretry exhausted"
|
||||
)
|
||||
);
|
||||
for secret in ["first-secret", "second-secret", "third-secret"] {
|
||||
assert!(!log.to_string().contains(secret));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internal_error_logging_omits_details_when_disabled() {
|
||||
let message = "password=internal-secret";
|
||||
let log = internal_error_log(message, false);
|
||||
let fields = &log["fields"];
|
||||
assert_eq!(fields["event_name"], "gateway_internal_error");
|
||||
assert_eq!(
|
||||
fields["error_fingerprint"],
|
||||
gateway_error_fingerprint(message)
|
||||
);
|
||||
assert_eq!(fields["error_length"], message.len());
|
||||
assert!(fields.get("error_detail").is_none());
|
||||
assert!(!log.to_string().contains("internal-secret"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -164,7 +164,7 @@ pub(crate) async fn refresh_oauth_plan_auth_for_retry(
|
||||
key_id = %plan.key_id,
|
||||
status_code,
|
||||
refresh_status_code,
|
||||
error = ?err,
|
||||
error = %crate::error::redact_error_debug(&err),
|
||||
"gateway failed to persist oauth retry refresh failure"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -253,7 +253,7 @@ pub(super) async fn build_admin_create_api_key_response(
|
||||
{
|
||||
tracing::error!(
|
||||
api_key_id = %created.api_key_id,
|
||||
error = ?error,
|
||||
error = %crate::error::redact_error_debug(&error),
|
||||
"standalone API key wallet provisioning cleanup failed"
|
||||
);
|
||||
return Err(error);
|
||||
@@ -266,7 +266,7 @@ pub(super) async fn build_admin_create_api_key_response(
|
||||
{
|
||||
tracing::error!(
|
||||
api_key_id = %created.api_key_id,
|
||||
error = ?cleanup_error,
|
||||
error = %crate::error::redact_error_debug(&cleanup_error),
|
||||
"standalone API key wallet provisioning cleanup failed"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@ impl AdminProviderOAuthDevicePollLease {
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
lock_key = %lock_key,
|
||||
error = ?error,
|
||||
error = %crate::error::redact_error_detail(&error),
|
||||
"gateway provider OAuth device poll lease acquisition failed"
|
||||
);
|
||||
AdminProviderOAuthDevicePollLeaseAcquire::Unavailable
|
||||
@@ -87,7 +87,7 @@ impl AdminProviderOAuthDevicePollLease {
|
||||
Err(error) => {
|
||||
tracing::error!(
|
||||
lock_key = %lease.key,
|
||||
error = ?error,
|
||||
error = %crate::error::redact_error_detail(&error),
|
||||
"gateway provider OAuth device poll final lease renewal failed"
|
||||
);
|
||||
Err(AdminProviderOAuthDevicePollLeaseFailure::Unavailable)
|
||||
@@ -106,7 +106,7 @@ impl AdminProviderOAuthDevicePollLease {
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
lock_key = %lease.key,
|
||||
error = ?error,
|
||||
error = %crate::error::redact_error_detail(&error),
|
||||
"gateway provider OAuth device poll lease release failed"
|
||||
);
|
||||
// Keep the lease in the guard so Drop can make one best-effort retry.
|
||||
@@ -128,7 +128,7 @@ impl Drop for AdminProviderOAuthDevicePollLease {
|
||||
if let Err(error) = runtime.lock_release(&lease).await {
|
||||
tracing::warn!(
|
||||
lock_key = %lease.key,
|
||||
error = ?error,
|
||||
error = %crate::error::redact_error_detail(&error),
|
||||
"gateway provider OAuth device poll lease Drop release failed"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -72,7 +72,7 @@ fn admin_management_token_internal_error_response(
|
||||
tracing::error!(
|
||||
event_name,
|
||||
trace_id,
|
||||
error = ?error,
|
||||
error = %crate::error::redact_error_debug(&error),
|
||||
"management token operation failed"
|
||||
);
|
||||
(
|
||||
|
||||
@@ -360,7 +360,7 @@ pub(super) async fn handle_auth_send_verification_code(
|
||||
if let Err(err) = send_auth_email(state, smtp_config, email_message).await {
|
||||
tracing::warn!(
|
||||
event_name = "auth_verification_email_send_failed",
|
||||
error = ?err,
|
||||
error = %crate::error::redact_error_debug(&err),
|
||||
"failed to send authentication verification email"
|
||||
);
|
||||
let _ = clear_auth_email_pending_code(state, &email).await;
|
||||
@@ -665,7 +665,7 @@ pub(super) async fn handle_auth_register(
|
||||
Err(err) => {
|
||||
tracing::warn!(
|
||||
event_name = "auth_email_registration_proof_consume_failed",
|
||||
error = ?err,
|
||||
error = %crate::error::redact_error_debug(&err),
|
||||
"failed to consume email registration proof"
|
||||
);
|
||||
return build_auth_error_response(
|
||||
@@ -931,7 +931,7 @@ pub(super) async fn handle_auth_verify_email(
|
||||
Err(err) => {
|
||||
tracing::warn!(
|
||||
event_name = "auth_email_verification_consume_failed",
|
||||
error = ?err,
|
||||
error = %crate::error::redact_error_debug(&err),
|
||||
"failed to consume email verification challenge"
|
||||
);
|
||||
false
|
||||
@@ -944,7 +944,7 @@ pub(super) async fn handle_auth_verify_email(
|
||||
Err(err) => {
|
||||
tracing::warn!(
|
||||
event_name = "auth_registration_proof_store_failed",
|
||||
error = ?err,
|
||||
error = %crate::error::redact_error_debug(&err),
|
||||
"failed to store email registration proof"
|
||||
);
|
||||
true
|
||||
|
||||
@@ -222,28 +222,28 @@ async fn read_auth_turnstile_config(
|
||||
.read_system_config_json_value("turnstile_enabled")
|
||||
.await
|
||||
.map_err(|err| {
|
||||
warn!(error = ?err, "turnstile enabled config lookup failed");
|
||||
warn!(error = %crate::error::redact_error_debug(&err), "turnstile enabled config lookup failed");
|
||||
AuthTurnstileFailure::ServiceUnavailable("人机验证服务暂不可用,请稍后重试")
|
||||
})?;
|
||||
let site_key = state
|
||||
.read_system_config_json_value("turnstile_site_key")
|
||||
.await
|
||||
.map_err(|err| {
|
||||
warn!(error = ?err, "turnstile site key config lookup failed");
|
||||
warn!(error = %crate::error::redact_error_debug(&err), "turnstile site key config lookup failed");
|
||||
AuthTurnstileFailure::ServiceUnavailable("人机验证服务暂不可用,请稍后重试")
|
||||
})?;
|
||||
let secret_key = state
|
||||
.read_system_config_json_value("turnstile_secret_key")
|
||||
.await
|
||||
.map_err(|err| {
|
||||
warn!(error = ?err, "turnstile secret key config lookup failed");
|
||||
warn!(error = %crate::error::redact_error_debug(&err), "turnstile secret key config lookup failed");
|
||||
AuthTurnstileFailure::ServiceUnavailable("人机验证服务暂不可用,请稍后重试")
|
||||
})?;
|
||||
let allowed_hostnames = state
|
||||
.read_system_config_json_value("turnstile_allowed_hostnames")
|
||||
.await
|
||||
.map_err(|err| {
|
||||
warn!(error = ?err, "turnstile hostname config lookup failed");
|
||||
warn!(error = %crate::error::redact_error_debug(&err), "turnstile hostname config lookup failed");
|
||||
AuthTurnstileFailure::ServiceUnavailable("人机验证服务暂不可用,请稍后重试")
|
||||
})?;
|
||||
|
||||
@@ -252,7 +252,7 @@ async fn read_auth_turnstile_config(
|
||||
decrypt_or_migrate_system_config_secret(state, "turnstile_secret_key", value)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
warn!(error = ?error, "turnstile secret key migration failed");
|
||||
warn!(error = %crate::error::redact_error_debug(&error), "turnstile secret key migration failed");
|
||||
AuthTurnstileFailure::ServiceUnavailable("人机验证服务暂不可用,请稍后重试")
|
||||
})?,
|
||||
),
|
||||
|
||||
@@ -207,7 +207,7 @@ pub(crate) async fn send_user_important_notification_email(
|
||||
match send_single_email_notification(smtp_config, user_email, ¬ification).await {
|
||||
Ok(()) => Ok(single_report("user_email", true, "用户邮件通知已发送")),
|
||||
Err(err) => {
|
||||
warn!(error = ?err, user_email = %user_email, "failed to send user notification email");
|
||||
warn!(error = %crate::error::redact_error_debug(&err), user_email = %user_email, "failed to send user notification email");
|
||||
Ok(single_report(
|
||||
"user_email",
|
||||
false,
|
||||
|
||||
@@ -431,7 +431,7 @@ async fn run_admin_system_purge_task(
|
||||
if let Err(record_err) = record_cleanup_run(&data, failed).await {
|
||||
warn!(error = %record_err, "failed to record admin system purge task failure");
|
||||
}
|
||||
warn!(error = ?err, kind = ?kind, "admin system purge task failed");
|
||||
warn!(error = %crate::error::redact_error_debug(&err), kind = ?kind, "admin system purge task failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -898,7 +898,7 @@ impl AppState {
|
||||
}
|
||||
Err(error) => {
|
||||
guard.fail(GatewayError::Internal(error.to_string()));
|
||||
warn!(error = %error, "background system config refresh failed");
|
||||
warn!(error = %crate::error::redact_error_detail(&error), "background system config refresh failed");
|
||||
}
|
||||
}
|
||||
drop(guard);
|
||||
|
||||
@@ -1932,7 +1932,7 @@ impl HubRouter {
|
||||
) {
|
||||
Ok(payload) => payload,
|
||||
Err(error) => {
|
||||
warn!(proxy_conn_id = proxy_conn_id, error = %error, "failed to decode heartbeat payload");
|
||||
warn!(proxy_conn_id = proxy_conn_id, error = %crate::error::redact_error_detail(&error), "failed to decode heartbeat payload");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -30,6 +30,7 @@ use dashmap::DashMap;
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::error::{redact_error_debug, redact_error_detail};
|
||||
use crate::{data::GatewayDataState, middleware};
|
||||
|
||||
pub use control_plane::ControlPlaneClient;
|
||||
@@ -2207,7 +2208,7 @@ pub async fn ws_proxy(
|
||||
{
|
||||
Ok(binding) => binding,
|
||||
Err(error) => {
|
||||
warn!(node_id = %node_id, error = %error, "proxy connection rejected: tunnel security key lookup unavailable");
|
||||
warn!(node_id = %node_id, error = %redact_error_detail(&error), "proxy connection rejected: tunnel security key lookup unavailable");
|
||||
return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response();
|
||||
}
|
||||
};
|
||||
@@ -2255,7 +2256,7 @@ pub async fn ws_proxy(
|
||||
requested_generation.clone(),
|
||||
),
|
||||
Err(error) => {
|
||||
warn!(node_id = %node_id, ?error, "proxy connection rejected: invalid secure tunnel handshake");
|
||||
warn!(node_id = %node_id, error = %redact_error_debug(&error), "proxy connection rejected: invalid secure tunnel handshake");
|
||||
return error.status_code().into_response();
|
||||
}
|
||||
}
|
||||
@@ -2282,7 +2283,7 @@ pub async fn ws_proxy(
|
||||
{
|
||||
Ok(credential) => credential,
|
||||
Err(error) => {
|
||||
warn!(node_id = %node_id, ?error, "proxy connection rejected: invalid management token");
|
||||
warn!(node_id = %node_id, error = %redact_error_debug(&error), "proxy connection rejected: invalid management token");
|
||||
return error.status_code().into_response();
|
||||
}
|
||||
};
|
||||
|
||||
@@ -14,6 +14,7 @@ use tracing::{debug, info, warn};
|
||||
|
||||
use super::hub::{ConnConfig, HubRouter, ProxyConn, ProxyManagementTokenCredential, SendStatus};
|
||||
use super::protocol;
|
||||
use crate::error::redact_error_detail;
|
||||
use aether_contracts::tunnel::{Frame, HelloPayload, MsgType};
|
||||
use aether_contracts::tunnel_security::{SecureFrameCodec, TunnelSecurityRole};
|
||||
|
||||
@@ -61,7 +62,7 @@ pub async fn handle_proxy_connection(
|
||||
) {
|
||||
Ok(codec) => Arc::new(codec),
|
||||
Err(error) => {
|
||||
warn!(conn_id, node_id = %node_id, error = %error, "secure tunnel codec initialization failed");
|
||||
warn!(conn_id, node_id = %node_id, error = %redact_error_detail(&error), "secure tunnel codec initialization failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
@@ -157,7 +158,7 @@ pub async fn handle_proxy_connection(
|
||||
let msg = match encrypt_message(msg, writer_security.as_deref()) {
|
||||
Ok(msg) => msg,
|
||||
Err(error) => {
|
||||
warn!(conn_id = writer_conn_id, error = %error, "failed to encrypt outbound proxy frame");
|
||||
warn!(conn_id = writer_conn_id, error = %redact_error_detail(&error), "failed to encrypt outbound proxy frame");
|
||||
break;
|
||||
}
|
||||
};
|
||||
@@ -366,7 +367,7 @@ async fn read_authenticated_proxy_hello(
|
||||
return None;
|
||||
}
|
||||
if let Err(error) = ws_tx.send(Message::Pong(payload)).await {
|
||||
warn!(conn_id, node_id = %node_id, error = %error, "failed to answer WebSocket ping before proxy authentication");
|
||||
warn!(conn_id, node_id = %node_id, error = %redact_error_detail(&error), "failed to answer WebSocket ping before proxy authentication");
|
||||
return None;
|
||||
}
|
||||
}
|
||||
@@ -380,7 +381,7 @@ async fn read_authenticated_proxy_hello(
|
||||
return None;
|
||||
}
|
||||
Some(Err(error)) => {
|
||||
warn!(conn_id, node_id = %node_id, error = %error, "proxy WebSocket failed before encrypted HELLO authentication");
|
||||
warn!(conn_id, node_id = %node_id, error = %redact_error_detail(&error), "proxy WebSocket failed before encrypted HELLO authentication");
|
||||
return None;
|
||||
}
|
||||
}
|
||||
@@ -471,7 +472,7 @@ async fn run_proxy_reader(
|
||||
let mut data = match decrypt_message(data, security.as_deref()) {
|
||||
Ok(data) => data,
|
||||
Err(error) => {
|
||||
warn!(conn_id = conn.id, error = %error, "failed to decrypt secure proxy frame");
|
||||
warn!(conn_id = conn.id, error = %redact_error_detail(&error), "failed to decrypt secure proxy frame");
|
||||
conn.request_close();
|
||||
break;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user