fix(gateway): redact sensitive error log details

This commit is contained in:
AAEE86
2026-09-09 13:27:20 +08:00
parent f2839ae6a7
commit 33ea4ebf15
15 changed files with 532 additions and 38 deletions
+4
View File
@@ -90,6 +90,10 @@ ADMIN_USERNAME=admin123456
# 单请求解压后 Payload 上限(MiB),默认 256;显式设为 0 才表示不限制。
# AETHER_MAX_REQUEST_BODY_MB=256
# AETHER_GATEWAY_SECURITY_CACHE_TTL_MS=1000
# 内部错误详情日志默认关闭;仅接受精确的小写 true(开启)和 false(关闭)。
# 开启后 ERROR 日志输出不截断的错误详情,但仍移除 URL 凭据并遮盖常见密码、令牌等字段。
# 脱敏无法覆盖任意敏感内容,排查完毕请关闭;未设置或无法识别的值均按关闭处理。
# AETHER_GATEWAY_ERROR_DETAIL_LOGGING=false
# AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB=64
# AETHER_MAX_INTERNAL_BUFFERED_BODY_MB=64
# AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY=1024
+1 -1
View File
@@ -174,7 +174,7 @@ async fn authorize_operational_request(
}
}
Err(err) => {
warn!(error = ?err, "operational admin session authentication failed");
warn!(error = %crate::error::redact_error_debug(&err), "operational admin session authentication failed");
return operational_error_response(
StatusCode::SERVICE_UNAVAILABLE,
"operational authentication unavailable",
+496 -8
View File
@@ -1,7 +1,10 @@
use std::sync::LazyLock;
use axum::body::Body;
use axum::http::{Response, StatusCode};
use axum::response::IntoResponse;
use axum::Json;
use regex::Regex;
use serde_json::json;
use sha2::{Digest, Sha256};
use tracing::warn;
@@ -10,6 +13,134 @@ use crate::ai_serving::AiSurfaceFinalizeError;
use crate::constants::*;
use crate::insert_header_if_missing;
/// 开启后记录不截断但仍脱敏的内部错误详情,默认关闭。
static GATEWAY_ERROR_DETAIL_LOGGING: LazyLock<bool> = LazyLock::new(|| {
parse_gateway_error_detail_logging(
std::env::var("AETHER_GATEWAY_ERROR_DETAIL_LOGGING")
.ok()
.as_deref(),
)
});
fn parse_gateway_error_detail_logging(value: Option<&str>) -> bool {
// 仅接受精确的小写 true/false;未设置或无效值默认关闭详情日志。
value
.and_then(|value| value.parse::<bool>().ok())
.unwrap_or(false)
}
// 按 URL authority 的边界匹配 userinfo,避免跨过路径、查询串和片段中的 @。
static ERROR_URL_USERINFO: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r#"(?i)([a-z][a-z0-9+.-]*://)[^\s/?\#"<>]*@"#)
.expect("error URL userinfo regex should compile")
});
static ERROR_CREDENTIAL_PREFIX: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(
r#"(?ix)
\b
(?:password|passwd|pwd|(?:access|refresh|id|session|auth)[_-]?token|token|
(?:client[_-]?)?secret|(?:api|access|secret|private)[_-]?key|
(?:proxy[_-])?authorization)
(?:\\*["'])?(?:\s|\\+[nrt])*[:=](?:\s|\\+[nrt])*
(?:Some\((?:\s|\\+[nrt])*)?
(?:(?:Bearer|Basic)(?:\s|\\+[nrt])+)?
|(?:\b|\\+[nrt])Bearer(?:\s|\\+[nrt])+"#,
)
.expect("error credential prefix regex should compile")
});
/// 检查是否启用了内部错误详情日志。
pub(crate) fn gateway_error_detail_logging_enabled() -> bool {
*GATEWAY_ERROR_DETAIL_LOGGING
}
/// 日志摘要:移除 URL userinfo、常见凭据键值和 Bearer 内容,再限制为 256 字节。
/// 这是自由文本的有限规则,不能保证识别任意敏感内容或编码后的字段名。
pub(crate) fn redact_error_detail(error: &impl std::fmt::Display) -> String {
redact_error_str(&error.to_string())
}
/// 脱敏 Debug 格式的错误详情(用于未实现 Display 的错误类型)。
pub(crate) fn redact_error_debug(error: &impl std::fmt::Debug) -> String {
redact_error_str(&format!("{error:?}"))
}
pub(crate) fn redact_error_str(message: &str) -> String {
const MAX_LEN: usize = 256;
// 必须先处理完整凭据,再截断;否则截断位置可能落在密码和 @host 之间。
let mut redacted = redact_error_str_unbounded(message);
if redacted.len() > MAX_LEN {
let mut end = MAX_LEN;
while !redacted.is_char_boundary(end) {
end -= 1;
}
redacted.truncate(end);
redacted.push_str("...");
}
redacted
}
/// 详情日志仅取消长度限制,继续使用与摘要相同的凭据脱敏规则。
fn redact_error_str_unbounded(message: &str) -> String {
let urls_redacted = ERROR_URL_USERINFO.replace_all(message, "$1");
let mut result = String::with_capacity(urls_redacted.len());
let mut cursor = 0;
let mut prefixes = ERROR_CREDENTIAL_PREFIX.find_iter(&urls_redacted).peekable();
while let Some(prefix) = prefixes.next() {
// 带引号的值可能包含 password= 等文本,已遮盖的内容不再重复处理。
if prefix.start() < cursor {
continue;
}
let value_start = prefix.end();
// 未引用的值最多读到下一个凭据字段,避免吞掉它的开头却留下带空格的值。
let unquoted_limit = prefixes
.peek()
.map_or(urls_redacted.len() - value_start, |next| {
urls_redacted[value_start..next.start()]
.trim_end_matches([',', ';', '&'])
.len()
});
let value_end =
value_start + credential_value_len(&urls_redacted[value_start..], unquoted_limit);
result.push_str(&urls_redacted[cursor..value_start]);
result.push_str("[REDACTED]");
cursor = value_end;
}
result.push_str(&urls_redacted[cursor..]);
result
}
fn credential_value_len(value: &str, unquoted_limit: usize) -> usize {
let bytes = value.as_bytes();
let opening_slashes = bytes.iter().take_while(|byte| **byte == b'\\').count();
if let Some(quote @ (b'"' | b'\'')) = bytes.get(opening_slashes) {
// 仅同一转义层的引号可闭合;无法确认边界时多遮盖,避免泄露密码尾部。
let mut slashes = 0;
for (index, byte) in bytes.iter().enumerate().skip(opening_slashes + 1) {
if byte == quote && slashes == opening_slashes {
return index + 1;
}
slashes = if *byte == b'\\' { slashes + 1 } else { 0 };
}
// 不完整的引号内容整体遮盖,避免保留密码片段。
return value.len();
}
let mut escaped = false;
// DSN 未引用密码中的标点也可能是凭据,只以未转义的空白为结束边界。
for (index, ch) in value[..unquoted_limit].char_indices() {
if ch == '\\' {
escaped = true;
} else if escaped {
escaped = false;
} else if ch.is_whitespace() {
return index;
}
}
unquoted_limit
}
#[derive(Debug, Clone)]
pub(crate) enum GatewayError {
UpstreamUnavailable {
@@ -211,13 +342,7 @@ impl IntoResponse for GatewayError {
)
.into_response(),
Self::Internal(message) => {
let error_fingerprint = gateway_error_fingerprint(&message);
tracing::error!(
event_name = "gateway_internal_error",
error_fingerprint,
error_length = message.len(),
"internal gateway error hidden from client"
);
log_gateway_internal_error(&message, gateway_error_detail_logging_enabled());
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({
@@ -232,6 +357,26 @@ impl IntoResponse for GatewayError {
}
}
fn log_gateway_internal_error(message: &str, detail_logging: bool) {
let error_fingerprint = gateway_error_fingerprint(message);
if detail_logging {
tracing::error!(
event_name = "gateway_internal_error",
error_fingerprint,
error_length = message.len(),
error_detail = %redact_error_str_unbounded(message),
"internal gateway error hidden from client"
);
} else {
tracing::error!(
event_name = "gateway_internal_error",
error_fingerprint,
error_length = message.len(),
"internal gateway error hidden from client"
);
}
}
fn gateway_error_fingerprint(message: &str) -> String {
let digest = Sha256::digest(message.as_bytes());
format!("{:x}", digest)[..16].to_string()
@@ -245,13 +390,68 @@ impl From<AiSurfaceFinalizeError> for GatewayError {
#[cfg(test)]
mod tests {
use std::sync::{Arc, Mutex};
use axum::body::to_bytes;
use axum::http::{header::RETRY_AFTER, StatusCode};
use axum::response::IntoResponse;
use crate::constants::TRACE_ID_HEADER;
use super::{gateway_error_fingerprint, GatewayError};
use super::{
gateway_error_fingerprint, log_gateway_internal_error, parse_gateway_error_detail_logging,
redact_error_debug, redact_error_detail, redact_error_str, GatewayError,
};
#[test]
fn detail_logging_only_accepts_lowercase_true_and_false() {
assert!(parse_gateway_error_detail_logging(Some("true")));
assert!(!parse_gateway_error_detail_logging(Some("false")));
assert!(!parse_gateway_error_detail_logging(None));
for value in [
"1", "0", "yes", "no", "on", "off", "TRUE", "FALSE", "True", " true ", "true\n", "",
"invalid",
] {
assert!(
!parse_gateway_error_detail_logging(Some(value)),
"value: {value:?}"
);
}
}
#[derive(Clone, Default)]
struct LogBuffer(Arc<Mutex<Vec<u8>>>);
impl std::io::Write for LogBuffer {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
self.0
.lock()
.expect("log buffer should lock")
.extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
fn internal_error_log(message: &str, detail_logging: bool) -> serde_json::Value {
let buffer = LogBuffer::default();
let writer = buffer.clone();
let subscriber = tracing_subscriber::fmt()
.json()
.without_time()
.with_max_level(tracing::Level::ERROR)
.with_writer(move || writer.clone())
.finish();
// 使用线程局部日志捕获和显式开关,不修改进程环境以免干扰并发测试。
tracing::subscriber::with_default(subscriber, || {
log_gateway_internal_error(message, detail_logging);
});
let bytes = buffer.0.lock().expect("log buffer should lock");
serde_json::from_slice(&bytes).expect("internal error log should be JSON")
}
#[tokio::test]
async fn internal_errors_do_not_expose_internal_details() {
@@ -308,4 +508,292 @@ mod tests {
Some(trace_id.as_str())
);
}
#[test]
fn redact_error_str_strips_url_credentials() {
let input = "connect failed: postgresql://admin:s3cret@db.internal:5432/aether";
let redacted = redact_error_str(input);
assert!(!redacted.contains("s3cret"));
assert!(redacted.contains("postgresql://"));
assert!(redacted.contains("db.internal"));
}
#[test]
fn redact_error_str_truncates_long_messages() {
let long = "x".repeat(500);
let redacted = redact_error_str(&long);
assert_eq!(redacted, format!("{}...", "x".repeat(256)));
}
#[test]
fn redact_error_str_preserves_short_messages() {
let input = "connection refused";
assert_eq!(redact_error_str(input), input);
}
#[test]
fn redact_error_str_handles_multiple_urls() {
for (input, expected) in [
(
"https://u:first@a/p,https://v:second@b/q",
"https://a/p,https://b/q",
),
(
"https://u:first@a,https://v:second@b",
"https://a,https://b",
),
(
"failed: https://user:token123@api.example.com/v1 and http://admin:pass@internal.io",
"failed: https://api.example.com/v1 and http://internal.io",
),
] {
assert_eq!(redact_error_str(input), expected);
}
}
#[test]
fn redact_error_str_handles_url_authority_boundaries_and_punctuation() {
for (input, expected) in [
(
r#"url="postgres://u:p@h", retry=2"#,
r#"url="postgres://h", retry=2"#,
),
(
"(https://u:p@h?mode=test#detail)",
"(https://h?mode=test#detail)",
),
(
"postgres://u:p@ss@[::1]:5432/db",
"postgres://[::1]:5432/db",
),
("https://u:p,a;s's@h/p", "https://h/p"),
(
"https://h/path@name?email=a@b#ref@c",
"https://h/path@name?email=a@b#ref@c",
),
] {
assert_eq!(redact_error_str(input), expected);
}
}
#[test]
fn redact_error_str_redacts_before_truncation() {
let prefix = format!("{} ", "x".repeat(239));
let input = format!("{prefix}postgres://u:supersecret@db/app");
assert_eq!(
redact_error_str(&input),
format!("{prefix}postgres://db/ap...")
);
let input = format!("password={} host=db", "secret".repeat(100));
assert_eq!(redact_error_str(&input), "password=[REDACTED] host=db");
}
#[test]
fn redact_error_str_preserves_utf8_and_whitespace() {
let input = " first \n\tsecond \r\n";
assert_eq!(redact_error_str(input), input);
assert_eq!(
redact_error_str(" failed:\n\tpostgres://u:p@h\r\n retry"),
" failed:\n\tpostgres://h\r\n retry"
);
for length in [254, 255, 256] {
let prefix = "x".repeat(length);
assert_eq!(
redact_error_str(&format!("{prefix}错误")),
format!("{prefix}...")
);
}
assert_eq!(redact_error_str(&"x".repeat(256)), "x".repeat(256));
}
#[test]
fn redact_error_str_masks_common_credentials() {
for key in [
"password",
"PASSWORD",
"passwd",
"pwd",
"token",
"access_token",
"refresh-token",
"idToken",
"session_token",
"auth-token",
"secret",
"client_secret",
"clientSecret",
"api_key",
"api-key",
"apiKey",
"access_key",
"secret_key",
"private_key",
"x-api-key",
] {
for separator in ["=", ":", " = ", "\t:\t"] {
let input = format!("{key}{separator}test-credential retry=2");
assert_eq!(
redact_error_str(&input),
format!("{key}{separator}[REDACTED] retry=2")
);
}
}
assert_eq!(
redact_error_str("password=one;token=two,secret=three&retry=2"),
"password=[REDACTED];token=[REDACTED],secret=[REDACTED]"
);
assert_eq!(
redact_error_str("https://u:p@host/path?token=abc&api_key=xyz#details"),
"https://host/path?token=[REDACTED]&api_key=[REDACTED]"
);
}
#[test]
fn redact_error_str_does_not_expose_punctuation_in_unquoted_passwords() {
for secret in [
"one#two", "one?two", "one&two", "one,two", "one;two", "one)two", "one\"two",
] {
assert_eq!(
redact_error_str(&format!("password={secret} host=db")),
"password=[REDACTED] host=db"
);
}
assert_eq!(
redact_error_str("https://host/db?password=one?two&mode=test"),
"https://host/db?password=[REDACTED]"
);
assert_eq!(
redact_error_str(r#"password=one,token="two words" retry=2"#),
"password=[REDACTED],token=[REDACTED] retry=2"
);
}
#[test]
fn redact_error_str_masks_authorization_and_bearer_values() {
for (input, expected) in [
(
"Authorization: Bearer short",
"Authorization: Bearer [REDACTED]",
),
(
"authorization=bEaReR\tabc.def",
"authorization=bEaReR\t[REDACTED]",
),
(
"Proxy-Authorization: Basic abc==",
"Proxy-Authorization: Basic [REDACTED]",
),
(
r#"{"Authorization": "Bearer secret value"}"#,
r#"{"Authorization": [REDACTED]}"#,
),
(
"error: BEARER a+/b==, retry=2",
"error: BEARER [REDACTED] retry=2",
),
] {
assert_eq!(redact_error_str(input), expected);
}
}
#[test]
fn redact_error_str_masks_quoted_and_escaped_values() {
for (input, expected) in [
(
"password='space secret' host=db",
"password=[REDACTED] host=db",
),
(
r#"password="space \"secret" host=db"#,
"password=[REDACTED] host=db",
),
(
r"password=space\ secret host=db",
"password=[REDACTED] host=db",
),
(
r#"Error { password: "space secret", token: Some("option-secret") }"#,
"Error { password: [REDACTED], token: Some([REDACTED]) }",
),
("password='unterminated secret", "password=[REDACTED]"),
(
r#"password="another unterminated secret"#,
"password=[REDACTED]",
),
(
r#"password="token=inner-secret" retry=2"#,
"password=[REDACTED] retry=2",
),
] {
assert_eq!(redact_error_str(input), expected);
}
}
#[test]
fn display_and_debug_error_helpers_redact_real_formatted_values() {
let input =
r#"{"password": "space \"escaped-secret", "token": "token-secret", "retry": 2}"#;
for redacted in [redact_error_detail(&input), redact_error_debug(&input)] {
assert!(!redacted.contains("escaped-secret"));
assert!(!redacted.contains("token-secret"));
assert!(redacted.contains("retry"));
assert_eq!(redacted.matches("[REDACTED]").count(), 2);
}
}
#[test]
fn debug_error_redaction_handles_escaped_whitespace_and_single_quotes() {
for input in [
"Authorization:\nBearer test-secret",
"Authorization:\tBearer test-secret",
"\nBearer test-secret",
"Bearer\ntest-secret",
r"password=space\ test-secret host=db",
r"password='space \'test-secret' host=db",
] {
let redacted = redact_error_debug(&input);
assert!(!redacted.contains("test-secret"), "redacted: {redacted}");
assert!(redacted.contains("[REDACTED]"));
}
}
#[test]
fn internal_error_detail_logging_redacts_without_truncating() {
let padding = "x".repeat(300);
let message = format!("password=first-secret {padding}\nhttps://u:second-secret@db/path token=third-secret\nretry exhausted");
let log = internal_error_log(&message, true);
let fields = &log["fields"];
assert_eq!(log["level"], "ERROR");
assert_eq!(fields["event_name"], "gateway_internal_error");
assert_eq!(fields["error_length"], message.len());
assert_eq!(
fields["error_fingerprint"],
gateway_error_fingerprint(&message)
);
assert_eq!(
fields["error_detail"],
format!(
"password=[REDACTED] {padding}\nhttps://db/path token=[REDACTED]\nretry exhausted"
)
);
for secret in ["first-secret", "second-secret", "third-secret"] {
assert!(!log.to_string().contains(secret));
}
}
#[test]
fn internal_error_logging_omits_details_when_disabled() {
let message = "password=internal-secret";
let log = internal_error_log(message, false);
let fields = &log["fields"];
assert_eq!(fields["event_name"], "gateway_internal_error");
assert_eq!(
fields["error_fingerprint"],
gateway_error_fingerprint(message)
);
assert_eq!(fields["error_length"], message.len());
assert!(fields.get("error_detail").is_none());
assert!(!log.to_string().contains("internal-secret"));
}
}
@@ -164,7 +164,7 @@ pub(crate) async fn refresh_oauth_plan_auth_for_retry(
key_id = %plan.key_id,
status_code,
refresh_status_code,
error = ?err,
error = %crate::error::redact_error_debug(&err),
"gateway failed to persist oauth retry refresh failure"
);
}
@@ -253,7 +253,7 @@ pub(super) async fn build_admin_create_api_key_response(
{
tracing::error!(
api_key_id = %created.api_key_id,
error = ?error,
error = %crate::error::redact_error_debug(&error),
"standalone API key wallet provisioning cleanup failed"
);
return Err(error);
@@ -266,7 +266,7 @@ pub(super) async fn build_admin_create_api_key_response(
{
tracing::error!(
api_key_id = %created.api_key_id,
error = ?cleanup_error,
error = %crate::error::redact_error_debug(&cleanup_error),
"standalone API key wallet provisioning cleanup failed"
);
}
@@ -47,7 +47,7 @@ impl AdminProviderOAuthDevicePollLease {
Err(error) => {
tracing::warn!(
lock_key = %lock_key,
error = ?error,
error = %crate::error::redact_error_detail(&error),
"gateway provider OAuth device poll lease acquisition failed"
);
AdminProviderOAuthDevicePollLeaseAcquire::Unavailable
@@ -87,7 +87,7 @@ impl AdminProviderOAuthDevicePollLease {
Err(error) => {
tracing::error!(
lock_key = %lease.key,
error = ?error,
error = %crate::error::redact_error_detail(&error),
"gateway provider OAuth device poll final lease renewal failed"
);
Err(AdminProviderOAuthDevicePollLeaseFailure::Unavailable)
@@ -106,7 +106,7 @@ impl AdminProviderOAuthDevicePollLease {
Err(error) => {
tracing::warn!(
lock_key = %lease.key,
error = ?error,
error = %crate::error::redact_error_detail(&error),
"gateway provider OAuth device poll lease release failed"
);
// Keep the lease in the guard so Drop can make one best-effort retry.
@@ -128,7 +128,7 @@ impl Drop for AdminProviderOAuthDevicePollLease {
if let Err(error) = runtime.lock_release(&lease).await {
tracing::warn!(
lock_key = %lease.key,
error = ?error,
error = %crate::error::redact_error_detail(&error),
"gateway provider OAuth device poll lease Drop release failed"
);
}
@@ -72,7 +72,7 @@ fn admin_management_token_internal_error_response(
tracing::error!(
event_name,
trace_id,
error = ?error,
error = %crate::error::redact_error_debug(&error),
"management token operation failed"
);
(
@@ -360,7 +360,7 @@ pub(super) async fn handle_auth_send_verification_code(
if let Err(err) = send_auth_email(state, smtp_config, email_message).await {
tracing::warn!(
event_name = "auth_verification_email_send_failed",
error = ?err,
error = %crate::error::redact_error_debug(&err),
"failed to send authentication verification email"
);
let _ = clear_auth_email_pending_code(state, &email).await;
@@ -665,7 +665,7 @@ pub(super) async fn handle_auth_register(
Err(err) => {
tracing::warn!(
event_name = "auth_email_registration_proof_consume_failed",
error = ?err,
error = %crate::error::redact_error_debug(&err),
"failed to consume email registration proof"
);
return build_auth_error_response(
@@ -931,7 +931,7 @@ pub(super) async fn handle_auth_verify_email(
Err(err) => {
tracing::warn!(
event_name = "auth_email_verification_consume_failed",
error = ?err,
error = %crate::error::redact_error_debug(&err),
"failed to consume email verification challenge"
);
false
@@ -944,7 +944,7 @@ pub(super) async fn handle_auth_verify_email(
Err(err) => {
tracing::warn!(
event_name = "auth_registration_proof_store_failed",
error = ?err,
error = %crate::error::redact_error_debug(&err),
"failed to store email registration proof"
);
true
@@ -222,28 +222,28 @@ async fn read_auth_turnstile_config(
.read_system_config_json_value("turnstile_enabled")
.await
.map_err(|err| {
warn!(error = ?err, "turnstile enabled config lookup failed");
warn!(error = %crate::error::redact_error_debug(&err), "turnstile enabled config lookup failed");
AuthTurnstileFailure::ServiceUnavailable("人机验证服务暂不可用,请稍后重试")
})?;
let site_key = state
.read_system_config_json_value("turnstile_site_key")
.await
.map_err(|err| {
warn!(error = ?err, "turnstile site key config lookup failed");
warn!(error = %crate::error::redact_error_debug(&err), "turnstile site key config lookup failed");
AuthTurnstileFailure::ServiceUnavailable("人机验证服务暂不可用,请稍后重试")
})?;
let secret_key = state
.read_system_config_json_value("turnstile_secret_key")
.await
.map_err(|err| {
warn!(error = ?err, "turnstile secret key config lookup failed");
warn!(error = %crate::error::redact_error_debug(&err), "turnstile secret key config lookup failed");
AuthTurnstileFailure::ServiceUnavailable("人机验证服务暂不可用,请稍后重试")
})?;
let allowed_hostnames = state
.read_system_config_json_value("turnstile_allowed_hostnames")
.await
.map_err(|err| {
warn!(error = ?err, "turnstile hostname config lookup failed");
warn!(error = %crate::error::redact_error_debug(&err), "turnstile hostname config lookup failed");
AuthTurnstileFailure::ServiceUnavailable("人机验证服务暂不可用,请稍后重试")
})?;
@@ -252,7 +252,7 @@ async fn read_auth_turnstile_config(
decrypt_or_migrate_system_config_secret(state, "turnstile_secret_key", value)
.await
.map_err(|error| {
warn!(error = ?error, "turnstile secret key migration failed");
warn!(error = %crate::error::redact_error_debug(&error), "turnstile secret key migration failed");
AuthTurnstileFailure::ServiceUnavailable("人机验证服务暂不可用,请稍后重试")
})?,
),
@@ -207,7 +207,7 @@ pub(crate) async fn send_user_important_notification_email(
match send_single_email_notification(smtp_config, user_email, &notification).await {
Ok(()) => Ok(single_report("user_email", true, "用户邮件通知已发送")),
Err(err) => {
warn!(error = ?err, user_email = %user_email, "failed to send user notification email");
warn!(error = %crate::error::redact_error_debug(&err), user_email = %user_email, "failed to send user notification email");
Ok(single_report(
"user_email",
false,
@@ -431,7 +431,7 @@ async fn run_admin_system_purge_task(
if let Err(record_err) = record_cleanup_run(&data, failed).await {
warn!(error = %record_err, "failed to record admin system purge task failure");
}
warn!(error = ?err, kind = ?kind, "admin system purge task failed");
warn!(error = %crate::error::redact_error_debug(&err), kind = ?kind, "admin system purge task failed");
}
}
}
+1 -1
View File
@@ -884,7 +884,7 @@ impl AppState {
}
Err(error) => {
guard.fail(GatewayError::Internal(error.to_string()));
warn!(error = %error, "background system config refresh failed");
warn!(error = %crate::error::redact_error_detail(&error), "background system config refresh failed");
}
}
drop(guard);
@@ -1932,7 +1932,7 @@ impl HubRouter {
) {
Ok(payload) => payload,
Err(error) => {
warn!(proxy_conn_id = proxy_conn_id, error = %error, "failed to decode heartbeat payload");
warn!(proxy_conn_id = proxy_conn_id, error = %crate::error::redact_error_detail(&error), "failed to decode heartbeat payload");
return;
}
};
@@ -30,6 +30,7 @@ use dashmap::DashMap;
use sha2::{Digest as _, Sha256};
use tracing::warn;
use crate::error::{redact_error_debug, redact_error_detail};
use crate::{data::GatewayDataState, middleware};
pub use control_plane::ControlPlaneClient;
@@ -2207,7 +2208,7 @@ pub async fn ws_proxy(
{
Ok(binding) => binding,
Err(error) => {
warn!(node_id = %node_id, error = %error, "proxy connection rejected: tunnel security key lookup unavailable");
warn!(node_id = %node_id, error = %redact_error_detail(&error), "proxy connection rejected: tunnel security key lookup unavailable");
return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response();
}
};
@@ -2255,7 +2256,7 @@ pub async fn ws_proxy(
requested_generation.clone(),
),
Err(error) => {
warn!(node_id = %node_id, ?error, "proxy connection rejected: invalid secure tunnel handshake");
warn!(node_id = %node_id, error = %redact_error_debug(&error), "proxy connection rejected: invalid secure tunnel handshake");
return error.status_code().into_response();
}
}
@@ -2282,7 +2283,7 @@ pub async fn ws_proxy(
{
Ok(credential) => credential,
Err(error) => {
warn!(node_id = %node_id, ?error, "proxy connection rejected: invalid management token");
warn!(node_id = %node_id, error = %redact_error_debug(&error), "proxy connection rejected: invalid management token");
return error.status_code().into_response();
}
};
@@ -14,6 +14,7 @@ use tracing::{debug, info, warn};
use super::hub::{ConnConfig, HubRouter, ProxyConn, ProxyManagementTokenCredential, SendStatus};
use super::protocol;
use crate::error::redact_error_detail;
use aether_contracts::tunnel::{Frame, HelloPayload, MsgType};
use aether_contracts::tunnel_security::{SecureFrameCodec, TunnelSecurityRole};
@@ -61,7 +62,7 @@ pub async fn handle_proxy_connection(
) {
Ok(codec) => Arc::new(codec),
Err(error) => {
warn!(conn_id, node_id = %node_id, error = %error, "secure tunnel codec initialization failed");
warn!(conn_id, node_id = %node_id, error = %redact_error_detail(&error), "secure tunnel codec initialization failed");
return;
}
};
@@ -157,7 +158,7 @@ pub async fn handle_proxy_connection(
let msg = match encrypt_message(msg, writer_security.as_deref()) {
Ok(msg) => msg,
Err(error) => {
warn!(conn_id = writer_conn_id, error = %error, "failed to encrypt outbound proxy frame");
warn!(conn_id = writer_conn_id, error = %redact_error_detail(&error), "failed to encrypt outbound proxy frame");
break;
}
};
@@ -366,7 +367,7 @@ async fn read_authenticated_proxy_hello(
return None;
}
if let Err(error) = ws_tx.send(Message::Pong(payload)).await {
warn!(conn_id, node_id = %node_id, error = %error, "failed to answer WebSocket ping before proxy authentication");
warn!(conn_id, node_id = %node_id, error = %redact_error_detail(&error), "failed to answer WebSocket ping before proxy authentication");
return None;
}
}
@@ -380,7 +381,7 @@ async fn read_authenticated_proxy_hello(
return None;
}
Some(Err(error)) => {
warn!(conn_id, node_id = %node_id, error = %error, "proxy WebSocket failed before encrypted HELLO authentication");
warn!(conn_id, node_id = %node_id, error = %redact_error_detail(&error), "proxy WebSocket failed before encrypted HELLO authentication");
return None;
}
}
@@ -471,7 +472,7 @@ async fn run_proxy_reader(
let mut data = match decrypt_message(data, security.as_deref()) {
Ok(data) => data,
Err(error) => {
warn!(conn_id = conn.id, error = %error, "failed to decrypt secure proxy frame");
warn!(conn_id = conn.id, error = %redact_error_detail(&error), "failed to decrypt secure proxy frame");
conn.request_close();
break;
}