Add proxy tunnel diagnostics and default logging

2026-09-02 01:10:23 +08:00 · 2026-05-15 19:43:59 +08:00
parent 1503986d40
commit 87e44479cc
17 changed files with 258 additions and 60 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -336,7 +336,7 @@ dependencies = [

 [[package]]
 name = "aether-proxy"
-version = "0.3.10"
+version = "0.3.11"
 dependencies = [
 "aether-contracts",
 "aether-gateway",
--- a/apps/aether-gateway/src/tunnel/embedded/proxy_conn.rs
+++ b/apps/aether-gateway/src/tunnel/embedded/proxy_conn.rs
@@ -62,18 +62,30 @@ pub async fn handle_proxy_connection(
                        match send_result {
                            Ok(Ok(())) => {}
                            Ok(Err(e)) => {
+                                let snapshot = writer_conn.outbound.snapshot();
                                warn!(
                                    conn_id = writer_conn_id,
                                    frames_sent = frames_sent,
+                                    queue_depth = snapshot.depth,
+                                    queue_capacity = snapshot.capacity,
+                                    stream_count = writer_conn.stream_count.load(std::sync::atomic::Ordering::Relaxed),
+                                    closing = writer_conn.outbound.is_closing(),
+                                    draining = writer_conn.is_draining(),
                                    error = %e,
                                    "writer ws_tx.send failed"
                                );
                                break;
                            }
                            Err(_) => {
+                                let snapshot = writer_conn.outbound.snapshot();
                                warn!(
                                    conn_id = writer_conn_id,
                                    frames_sent = frames_sent,
+                                    queue_depth = snapshot.depth,
+                                    queue_capacity = snapshot.capacity,
+                                    stream_count = writer_conn.stream_count.load(std::sync::atomic::Ordering::Relaxed),
+                                    closing = writer_conn.outbound.is_closing(),
+                                    draining = writer_conn.is_draining(),
                                    "writer ws_tx.send timed out"
                                );
                                break;
@@ -93,6 +105,17 @@ pub async fn handle_proxy_connection(
                },
                changed = close_rx.changed() => {
                    if changed.is_err() || *close_rx.borrow() {
+                        let snapshot = writer_conn.outbound.snapshot();
+                        info!(
+                            conn_id = writer_conn_id,
+                            frames_sent = frames_sent,
+                            queue_depth = snapshot.depth,
+                            queue_capacity = snapshot.capacity,
+                            stream_count = writer_conn.stream_count.load(std::sync::atomic::Ordering::Relaxed),
+                            closing = writer_conn.outbound.is_closing(),
+                            draining = writer_conn.is_draining(),
+                            "writer close signal received"
+                        );
                        break;
                    }
                }
@@ -104,7 +127,24 @@ pub async fn handle_proxy_connection(
            "writer task exiting"
        );
        writer_conn.request_close();
-        let _ = tokio::time::timeout(Duration::from_secs(5), ws_tx.close()).await;
+        match tokio::time::timeout(Duration::from_secs(5), ws_tx.close()).await {
+            Ok(Ok(())) => debug!(
+                conn_id = writer_conn_id,
+                frames_sent = frames_sent,
+                "writer WebSocket close completed"
+            ),
+            Ok(Err(error)) => warn!(
+                conn_id = writer_conn_id,
+                frames_sent = frames_sent,
+                error = %error,
+                "writer WebSocket close failed"
+            ),
+            Err(_) => warn!(
+                conn_id = writer_conn_id,
+                frames_sent = frames_sent,
+                "writer WebSocket close timed out"
+            ),
+        }
    });

    let ping_conn = conn.clone();
@@ -113,10 +153,34 @@ pub async fn handle_proxy_connection(
        loop {
            tokio::time::sleep(ping_interval).await;
            let ping = protocol::encode_ping();
-            if !matches!(
-                ping_conn.send(Message::Binary(ping.into())),
-                SendStatus::Queued
-            ) {
+            let status = ping_conn.send(Message::Binary(ping.into()));
+            if !matches!(status, SendStatus::Queued) {
+                let snapshot = ping_conn.outbound.snapshot();
+                match status {
+                    SendStatus::Closed => info!(
+                        conn_id = ping_conn.id,
+                        queue_depth = snapshot.depth,
+                        queue_capacity = snapshot.capacity,
+                        stream_count = ping_conn
+                            .stream_count
+                            .load(std::sync::atomic::Ordering::Relaxed),
+                        closing = ping_conn.outbound.is_closing(),
+                        draining = ping_conn.is_draining(),
+                        "ping task stopped because connection is closing"
+                    ),
+                    SendStatus::Congested => warn!(
+                        conn_id = ping_conn.id,
+                        queue_depth = snapshot.depth,
+                        queue_capacity = snapshot.capacity,
+                        stream_count = ping_conn
+                            .stream_count
+                            .load(std::sync::atomic::Ordering::Relaxed),
+                        closing = ping_conn.outbound.is_closing(),
+                        draining = ping_conn.is_draining(),
+                        "ping task stopped because outbound queue is congested"
+                    ),
+                    SendStatus::Queued => {}
+                }
                break;
            }
        }
@@ -130,6 +194,17 @@ pub async fn handle_proxy_connection(

    let _ = reader.await;
    ping_task.abort();
+    let snapshot = conn.outbound.snapshot();
+    info!(
+        conn_id = conn.id,
+        node_id = %conn.node_id,
+        queue_depth = snapshot.depth,
+        queue_capacity = snapshot.capacity,
+        stream_count = conn.stream_count.load(std::sync::atomic::Ordering::Relaxed),
+        closing = conn.outbound.is_closing(),
+        draining = conn.is_draining(),
+        "proxy connection cleanup starting"
+    );
    conn.request_close();
    hub.unregister_proxy(conn_id, &node_id);
    drop(conn);
--- a/apps/aether-proxy/Cargo.lock
+++ b/apps/aether-proxy/Cargo.lock
@@ -10,7 +10,7 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"

 [[package]]
 name = "aether-proxy"
-version = "0.2.4"
+version = "0.3.11"
 dependencies = [
 "anyhow",
 "arc-swap",
--- a/apps/aether-proxy/Cargo.toml
+++ b/apps/aether-proxy/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "aether-proxy"
-version = "0.3.10"
+version = "0.3.11"
 edition = "2021"
 description = "Tunnel proxy for Aether"

--- a/apps/aether-proxy/README.md
+++ b/apps/aether-proxy/README.md
@@ -15,13 +15,13 @@ Tunnel 模式下代理节点**无需对外监听端口**，仅需出站连接到
 <!-- DOWNLOAD_TABLE_START -->
 | Platform | Download |
 |----------|----------|
-| Linux x86_64 (GNU) | [aether-proxy-linux-amd64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.9/aether-proxy-linux-amd64.tar.gz) |
-| Linux ARM64 (GNU) | [aether-proxy-linux-arm64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.9/aether-proxy-linux-arm64.tar.gz) |
-| Linux x86_64 (musl) | [aether-proxy-linux-musl-amd64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.9/aether-proxy-linux-musl-amd64.tar.gz) |
-| Linux ARM64 (musl) | [aether-proxy-linux-musl-arm64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.9/aether-proxy-linux-musl-arm64.tar.gz) |
-| macOS x86_64 | [aether-proxy-macos-amd64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.9/aether-proxy-macos-amd64.tar.gz) |
-| macOS ARM64 | [aether-proxy-macos-arm64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.9/aether-proxy-macos-arm64.tar.gz) |
-| Windows x86_64 | [aether-proxy-windows-amd64.zip](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.9/aether-proxy-windows-amd64.zip) |
+| Linux x86_64 (GNU) | [aether-proxy-linux-amd64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.11/aether-proxy-linux-amd64.tar.gz) |
+| Linux ARM64 (GNU) | [aether-proxy-linux-arm64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.11/aether-proxy-linux-arm64.tar.gz) |
+| Linux x86_64 (musl) | [aether-proxy-linux-musl-amd64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.11/aether-proxy-linux-musl-amd64.tar.gz) |
+| Linux ARM64 (musl) | [aether-proxy-linux-musl-arm64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.11/aether-proxy-linux-musl-arm64.tar.gz) |
+| macOS x86_64 | [aether-proxy-macos-amd64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.11/aether-proxy-macos-amd64.tar.gz) |
+| macOS ARM64 | [aether-proxy-macos-arm64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.11/aether-proxy-macos-arm64.tar.gz) |
+| Windows x86_64 | [aether-proxy-windows-amd64.zip](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.11/aether-proxy-windows-amd64.zip) |
 <!-- DOWNLOAD_TABLE_END -->

 上表展示的是最新已发布版本的下载链接。从下一次 `proxy-v*` 发布开始，表格会自动补上 `Linux x86_64 (musl)` / `Linux ARM64 (musl)` 包，供 Alpine 等 musl 系统直接使用。
@@ -180,16 +180,16 @@ upstream_proxy_url = "socks5h://microwarp:1080"
 | 参数 | 环境变量 | 默认值 | 说明 |
 |------|----------|--------|------|
 | `--log-level` | `AETHER_PROXY_LOG_LEVEL` | `info` | 日志级别 |
-| `--log-destination` | `AETHER_PROXY_LOG_DESTINATION` | `stdout` | 输出到 `stdout`、文件或两者同时输出 |
-| `--log-dir` | `AETHER_PROXY_LOG_DIR` | 空 | 文件日志目录，`file/both` 时必填 |
+| `--log-destination` | `AETHER_PROXY_LOG_DESTINATION` | `both` | 输出到 `stdout`、文件或两者同时输出 |
+| `--log-dir` | `AETHER_PROXY_LOG_DIR` | `logs` | 文件日志目录，`file/both` 时必填 |
 | `--log-rotation` | `AETHER_PROXY_LOG_ROTATION` | `daily` | 文件日志按小时或按天轮转 |
 | `--log-retention-days` | `AETHER_PROXY_LOG_RETENTION_DAYS` | `7` | 文件日志保留天数 |
 | `--log-max-files` | `AETHER_PROXY_LOG_MAX_FILES` | `30` | 文件日志最多保留文件数 |

 ### 日志落点

- 默认 `AETHER_PROXY_LOG_DESTINATION=stdout`，日志交给容器日志驱动或宿主机服务管理器
- 需要落盘时改成 `file` 或 `both`，并设置 `AETHER_PROXY_LOG_DIR`；setup TUI 里用 `Save Logs to File` 开关即可
+- 默认 `AETHER_PROXY_LOG_DESTINATION=both`，同时输出到 stdout 和 `logs/` 文件目录
+- 需要只交给容器日志驱动或宿主机服务管理器时，可改成 `stdout`；setup TUI 里可用 `Save Logs to File` 开关关闭文件日志
 - 文件日志固定写普通文本，并支持 `hourly/daily` 轮转；默认按天轮换、保留 7 天，最多保留 30 个文件
 - 以 `systemd` 或 `OpenRC` 安装时默认会额外打开文件日志到 `/var/log/aether-proxy`
 - OpenRC 安装时，`aether-proxy logs` 实际读取 `/var/log/aether-proxy/current.log` 和 `/var/log/aether-proxy/error.log`；这些文件通常需要用 `sudo aether-proxy logs` 查看
--- a/apps/aether-proxy/src/config.rs
+++ b/apps/aether-proxy/src/config.rs
@@ -53,6 +53,7 @@ pub const DEFAULT_REDIRECT_REPLAY_BUDGET_BYTES: usize = 5_242_880;
 pub const DEFAULT_REDIRECT_REPLAY_BUDGET_HUMAN: &str = "5M";
 pub const DEFAULT_LOG_RETENTION_DAYS: u64 = 7;
 pub const DEFAULT_LOG_MAX_FILES: usize = 30;
+pub const DEFAULT_LOG_DIR: &str = "logs";
 pub const DEFAULT_TUNNEL_RECONNECT_BASE_MS: u64 = 50;
 pub const DEFAULT_TUNNEL_RECONNECT_MAX_MS: u64 = 250;
 pub const DEFAULT_TUNNEL_PING_INTERVAL_MS: u64 = 10_000;
@@ -487,12 +488,12 @@ pub struct Config {
        long,
        env = "AETHER_PROXY_LOG_DESTINATION",
        value_enum,
-        default_value = "stdout"
+        default_value = "both"
    )]
    pub log_destination: ProxyLogDestinationArg,

    /// Log directory when file logging is enabled
-    #[arg(long, env = "AETHER_PROXY_LOG_DIR")]
+    #[arg(long, env = "AETHER_PROXY_LOG_DIR", default_value = DEFAULT_LOG_DIR)]
    pub log_dir: Option<String>,

    /// Log rotation schedule for file logging
@@ -1359,6 +1360,36 @@ mod tests {
        );
    }

+    #[test]
+    fn proxy_logs_default_to_rotating_file_and_stdout() {
+        let config = Config::parse_from([
+            "aether-proxy",
+            "--aether-url",
+            "https://example.com",
+            "--management-token",
+            "ae_test",
+            "--node-name",
+            "proxy-test",
+        ]);
+
+        assert_eq!(config.log_destination, ProxyLogDestinationArg::Both);
+        assert_eq!(config.log_dir.as_deref(), Some(DEFAULT_LOG_DIR));
+        assert_eq!(config.log_rotation, ProxyLogRotationArg::Daily);
+        assert_eq!(config.log_retention_days, DEFAULT_LOG_RETENTION_DAYS);
+
+        let runtime = config
+            .service_runtime_config()
+            .expect("default file logging should be valid");
+        assert_eq!(runtime.observability.log_destination, LogDestination::Both);
+        let file_logging = runtime
+            .observability
+            .file_logging
+            .expect("file logging should be enabled by default");
+        assert_eq!(file_logging.dir, std::path::PathBuf::from(DEFAULT_LOG_DIR));
+        assert_eq!(file_logging.rotation, LogRotation::Daily);
+        assert_eq!(file_logging.retention_days, DEFAULT_LOG_RETENTION_DAYS);
+    }
+
    #[test]
    fn config_file_load_accepts_server_scoped_upstream_proxy_url() {
        let cfg = parse_config_file_content(
--- a/apps/aether-proxy/src/setup/tui.rs
+++ b/apps/aether-proxy/src/setup/tui.rs
@@ -171,7 +171,7 @@ impl App {
                Field {
                    label: "Save Logs to File",
                    key: "save_logs_to_file",
-                    value: "false".into(),
+                    value: "true".into(),
                    kind: FieldKind::Bool,
                    required: false,
                    help: "Write pretty .log files with daily rotation and 7-day retention",
--- a/apps/aether-proxy/src/state.rs
+++ b/apps/aether-proxy/src/state.rs
@@ -97,6 +97,7 @@ const TUNNEL_ERROR_MESSAGE_MAX_CHARS: usize = 320;
 #[derive(Debug, Clone, serde::Serialize)]
 pub struct TunnelErrorEvent {
    pub timestamp_unix_secs: u64,
+    pub timestamp_unix_ms: u64,
    pub category: String,
    pub message: String,
    pub severity: String,
@@ -240,8 +241,10 @@ impl TunnelMetrics {
        let message = normalize_error_field(message, TUNNEL_ERROR_MESSAGE_MAX_CHARS, "n/a");
        let diagnostic = classify_tunnel_error(category.as_str(), message.as_str());

+        let timestamp_unix_ms = now_unix_ms();
        let event = TunnelErrorEvent {
-            timestamp_unix_secs: now_unix_secs(),
+            timestamp_unix_secs: timestamp_unix_ms / 1_000,
+            timestamp_unix_ms,
            category,
            message,
            severity: diagnostic.severity.to_string(),
@@ -298,9 +301,13 @@ impl TunnelMetrics {
 }

 fn now_unix_secs() -> u64 {
+    now_unix_ms() / 1_000
+}
+
+fn now_unix_ms() -> u64 {
    SystemTime::now()
        .duration_since(UNIX_EPOCH)
-        .map(|d| d.as_secs())
+        .map(|d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX))
        .unwrap_or(0)
 }

--- a/apps/aether-proxy/src/tunnel/client.rs
+++ b/apps/aether-proxy/src/tunnel/client.rs
@@ -8,7 +8,7 @@ use tokio::sync::watch;
 use tokio_tungstenite::tungstenite::client::IntoClientRequest;
 use tokio_tungstenite::tungstenite::http;
 use tokio_tungstenite::tungstenite::protocol::WebSocketConfig;
-use tracing::{debug, warn};
+use tracing::{debug, info, warn};

 use crate::egress_proxy::{connect_target_via_proxy, ProxyConnectOptions, UpstreamProxyConfig};
 use crate::state::{AppState, ServerContext};
@@ -224,9 +224,30 @@ pub async fn connect_and_run(
        let _ = tokio::time::timeout(Duration::from_secs(35), writer_handle).await;
    }

-    server
-        .tunnel_metrics
-        .record_disconnect(connected_at.elapsed());
+    let connected_for = connected_at.elapsed();
+    match &outcome {
+        Ok(TunnelOutcome::Shutdown) => info!(
+            conn = conn_idx,
+            connected_duration_ms = connected_for.as_millis() as u64,
+            close_reason = "shutdown",
+            "tunnel session ending"
+        ),
+        Ok(TunnelOutcome::Disconnected) => info!(
+            conn = conn_idx,
+            connected_duration_ms = connected_for.as_millis() as u64,
+            close_reason = "disconnected",
+            "tunnel session ending"
+        ),
+        Err(error) => warn!(
+            conn = conn_idx,
+            connected_duration_ms = connected_for.as_millis() as u64,
+            close_reason = "error",
+            error = %error,
+            "tunnel session ending"
+        ),
+    }
+
+    server.tunnel_metrics.record_disconnect(connected_for);

    debug!("tunnel disconnected");
    outcome
@@ -250,14 +271,26 @@ fn spawn_drain_signal(
        }

        debug!(conn = conn_idx, "sending GOAWAY for tunnel drain");
-        let _ = tokio::time::timeout(
+        match tokio::time::timeout(
            Duration::from_millis(250),
            frame_tx.send(super::protocol::Frame::control(
                super::protocol::MsgType::GoAway,
                bytes::Bytes::new(),
            )),
        )
-        .await;
+        .await
+        {
+            Ok(Ok(())) => info!(conn = conn_idx, "sent GOAWAY for tunnel drain"),
+            Ok(Err(error)) => warn!(
+                conn = conn_idx,
+                error = ?error,
+                "failed to queue GOAWAY for tunnel drain"
+            ),
+            Err(_) => warn!(
+                conn = conn_idx,
+                "timed out queueing GOAWAY for tunnel drain"
+            ),
+        }
    })
 }

--- a/apps/aether-proxy/src/tunnel/heartbeat.rs
+++ b/apps/aether-proxy/src/tunnel/heartbeat.rs
@@ -536,6 +536,10 @@ mod tests {
            .pointer("/proxy_metadata/recent_tunnel_errors/0")
            .and_then(serde_json::Value::as_object)
            .expect("recent tunnel error should be reported");
+        assert!(recent_error
+            .get("timestamp_unix_ms")
+            .and_then(serde_json::Value::as_u64)
+            .is_some());
        assert_eq!(
            recent_error
                .get("component")
--- a/apps/aether-proxy/src/tunnel/writer.rs
+++ b/apps/aether-proxy/src/tunnel/writer.rs
@@ -179,10 +179,20 @@ async fn write_frame<S>(sink: &mut S, frame: Frame, tunnel_metrics: Option<&Tunn
 where
    S: SinkExt<Message, Error = tokio_tungstenite::tungstenite::Error> + Unpin + Send + 'static,
 {
+    let stream_id = frame.stream_id;
+    let msg_type = frame.msg_type;
+    let flags = frame.flags;
    let data = frame.encode();
    let wire_len = data.len().max(HEADER_SIZE);
    if let Err(e) = sink.send(Message::Binary(data.into())).await {
-        error!(error = %e, "failed to write frame to WebSocket");
+        error!(
+            stream_id = stream_id,
+            msg_type = ?msg_type,
+            flags = flags,
+            wire_len = wire_len,
+            error = %e,
+            "failed to write frame to WebSocket"
+        );
        if let Some(metrics) = tunnel_metrics {
            metrics.record_error("ws_write_error", &e.to_string());
        }
--- a/crates/aether-data/src/repository/proxy_nodes/memory.rs
+++ b/crates/aether-data/src/repository/proxy_nodes/memory.rs
@@ -8,13 +8,13 @@ use uuid::Uuid;

 use super::types::{
    bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
-    normalize_proxy_metadata, reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery,
-    ProxyNodeHeartbeatMutation, ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation,
-    ProxyNodeMetricsCleanupSummary, ProxyNodeMetricsStep, ProxyNodeReadRepository,
-    ProxyNodeRegistrationMutation, ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation,
-    ProxyNodeTunnelStatusMutation, ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket,
-    StoredProxyNode, StoredProxyNodeEvent, StoredProxyNodeMetricsBucket, TunnelMetricsSample,
-    PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
+    log_reported_tunnel_error_event, normalize_proxy_metadata,
+    reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery, ProxyNodeHeartbeatMutation,
+    ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation, ProxyNodeMetricsCleanupSummary,
+    ProxyNodeMetricsStep, ProxyNodeReadRepository, ProxyNodeRegistrationMutation,
+    ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation, ProxyNodeTunnelStatusMutation,
+    ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket, StoredProxyNode, StoredProxyNodeEvent,
+    StoredProxyNodeMetricsBucket, TunnelMetricsSample, PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
 };
 use crate::DataLayerError;

@@ -640,6 +640,7 @@ impl ProxyNodeWriteRepository for InMemoryProxyNodeRepository {

            let mut events = self.events.write().expect("proxy node repository lock");
            for error in &sample.recent_error_events {
+                log_reported_tunnel_error_event(&node.id, error, now_unix_secs);
                let event_id = Self::next_event_id(&events);
                events.push(StoredProxyNodeEvent {
                    id: event_id,
@@ -655,6 +656,7 @@ impl ProxyNodeWriteRepository for InMemoryProxyNodeRepository {
                        "summary": error.summary.as_deref(),
                        "operator_action": error.operator_action.as_deref(),
                        "timestamp_unix_secs": error.timestamp_unix_secs,
+                        "timestamp_unix_ms": error.timestamp_unix_ms,
                    })),
                    created_at_unix_ms: Some(if error.timestamp_unix_secs == 0 {
                        now_unix_secs
--- a/crates/aether-data/src/repository/proxy_nodes/mod.rs
+++ b/crates/aether-data/src/repository/proxy_nodes/mod.rs
@@ -10,7 +10,8 @@ pub use postgres::SqlxProxyNodeRepository;
 pub use sqlite::SqliteProxyNodeReadRepository;
 pub use types::{
    bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
-    normalize_proxy_node_scheduling_state, proxy_node_accepts_new_tunnels, proxy_reported_version,
+    log_reported_tunnel_error_event, normalize_proxy_node_scheduling_state,
+    proxy_node_accepts_new_tunnels, proxy_reported_version,
    reconcile_remote_config_after_heartbeat, remote_config_scheduling_state,
    remote_config_upgrade_target, ProxyNodeEventQuery, ProxyNodeHeartbeatMutation,
    ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation, ProxyNodeMetricsCleanupSummary,
--- a/crates/aether-data/src/repository/proxy_nodes/mysql.rs
+++ b/crates/aether-data/src/repository/proxy_nodes/mysql.rs
@@ -3,13 +3,13 @@ use sqlx::{mysql::MySqlRow, Row};

 use super::types::{
    bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
-    normalize_proxy_metadata, reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery,
-    ProxyNodeHeartbeatMutation, ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation,
-    ProxyNodeMetricsCleanupSummary, ProxyNodeMetricsStep, ProxyNodeReadRepository,
-    ProxyNodeRegistrationMutation, ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation,
-    ProxyNodeTunnelStatusMutation, ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket,
-    StoredProxyNode, StoredProxyNodeEvent, StoredProxyNodeMetricsBucket,
-    PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
+    log_reported_tunnel_error_event, normalize_proxy_metadata,
+    reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery, ProxyNodeHeartbeatMutation,
+    ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation, ProxyNodeMetricsCleanupSummary,
+    ProxyNodeMetricsStep, ProxyNodeReadRepository, ProxyNodeRegistrationMutation,
+    ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation, ProxyNodeTunnelStatusMutation,
+    ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket, StoredProxyNode, StoredProxyNodeEvent,
+    StoredProxyNodeMetricsBucket, PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
 };
 use crate::driver::mysql::MysqlPool;
 use crate::error::SqlResultExt;
@@ -831,6 +831,7 @@ WHERE is_manual = 0
            .await?;

            for error in &sample.recent_error_events {
+                log_reported_tunnel_error_event(&node.id, error, now_unix_secs);
                let detail = build_tunnel_error_event_detail(error);
                let event_metadata = serde_json::json!({
                    "source": "heartbeat",
@@ -841,6 +842,7 @@ WHERE is_manual = 0
                    "summary": error.summary.as_deref(),
                    "operator_action": error.operator_action.as_deref(),
                    "timestamp_unix_secs": error.timestamp_unix_secs,
+                    "timestamp_unix_ms": error.timestamp_unix_ms,
                });
                self.insert_event(
                    &node.id,
--- a/crates/aether-data/src/repository/proxy_nodes/postgres.rs
+++ b/crates/aether-data/src/repository/proxy_nodes/postgres.rs
@@ -5,13 +5,13 @@ use sqlx::{postgres::PgRow, PgPool, Row};

 use super::types::{
    bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
-    normalize_proxy_metadata, reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery,
-    ProxyNodeHeartbeatMutation, ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation,
-    ProxyNodeMetricsCleanupSummary, ProxyNodeMetricsStep, ProxyNodeReadRepository,
-    ProxyNodeRegistrationMutation, ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation,
-    ProxyNodeTunnelStatusMutation, ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket,
-    StoredProxyNode, StoredProxyNodeEvent, StoredProxyNodeMetricsBucket, TunnelMetricsSample,
-    PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
+    log_reported_tunnel_error_event, normalize_proxy_metadata,
+    reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery, ProxyNodeHeartbeatMutation,
+    ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation, ProxyNodeMetricsCleanupSummary,
+    ProxyNodeMetricsStep, ProxyNodeReadRepository, ProxyNodeRegistrationMutation,
+    ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation, ProxyNodeTunnelStatusMutation,
+    ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket, StoredProxyNode, StoredProxyNodeEvent,
+    StoredProxyNodeMetricsBucket, TunnelMetricsSample, PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
 };
 use crate::{
    error::{postgres_error, SqlxResultExt},
@@ -1219,6 +1219,7 @@ impl ProxyNodeWriteRepository for SqlxProxyNodeRepository {
            .await?;

            for error in &sample.recent_error_events {
+                log_reported_tunnel_error_event(&updated.id, error, now_unix_secs);
                let detail = build_tunnel_error_event_detail(error);
                let event_metadata = serde_json::json!({
                    "source": "heartbeat",
@@ -1229,6 +1230,7 @@ impl ProxyNodeWriteRepository for SqlxProxyNodeRepository {
                    "summary": error.summary.as_deref(),
                    "operator_action": error.operator_action.as_deref(),
                    "timestamp_unix_secs": error.timestamp_unix_secs,
+                    "timestamp_unix_ms": error.timestamp_unix_ms,
                });
                self.insert_event(
                    &updated.id,
--- a/crates/aether-data/src/repository/proxy_nodes/sqlite.rs
+++ b/crates/aether-data/src/repository/proxy_nodes/sqlite.rs
@@ -3,13 +3,13 @@ use sqlx::{sqlite::SqliteRow, Row};

 use super::types::{
    bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
-    normalize_proxy_metadata, reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery,
-    ProxyNodeHeartbeatMutation, ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation,
-    ProxyNodeMetricsCleanupSummary, ProxyNodeMetricsStep, ProxyNodeReadRepository,
-    ProxyNodeRegistrationMutation, ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation,
-    ProxyNodeTunnelStatusMutation, ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket,
-    StoredProxyNode, StoredProxyNodeEvent, StoredProxyNodeMetricsBucket,
-    PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
+    log_reported_tunnel_error_event, normalize_proxy_metadata,
+    reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery, ProxyNodeHeartbeatMutation,
+    ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation, ProxyNodeMetricsCleanupSummary,
+    ProxyNodeMetricsStep, ProxyNodeReadRepository, ProxyNodeRegistrationMutation,
+    ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation, ProxyNodeTunnelStatusMutation,
+    ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket, StoredProxyNode, StoredProxyNodeEvent,
+    StoredProxyNodeMetricsBucket, PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
 };
 use crate::driver::sqlite::SqlitePool;
 use crate::error::SqlResultExt;
@@ -833,6 +833,7 @@ WHERE is_manual = 0
            .await?;

            for error in &sample.recent_error_events {
+                log_reported_tunnel_error_event(&node.id, error, now_unix_secs);
                let detail = build_tunnel_error_event_detail(error);
                let event_metadata = serde_json::json!({
                    "source": "heartbeat",
@@ -843,6 +844,7 @@ WHERE is_manual = 0
                    "summary": error.summary.as_deref(),
                    "operator_action": error.operator_action.as_deref(),
                    "timestamp_unix_secs": error.timestamp_unix_secs,
+                    "timestamp_unix_ms": error.timestamp_unix_ms,
                });
                self.insert_event(
                    &node.id,
--- a/crates/aether-data/src/repository/proxy_nodes/types.rs
+++ b/crates/aether-data/src/repository/proxy_nodes/types.rs
@@ -322,6 +322,7 @@ pub const PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR: &str = "tunnel_err";
 #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 pub struct TunnelErrorEventRecord {
    pub timestamp_unix_secs: u64,
+    pub timestamp_unix_ms: Option<u64>,
    pub category: String,
    pub message: String,
    pub severity: Option<String>,
@@ -428,6 +429,28 @@ pub fn build_tunnel_error_event_detail(event: &TunnelErrorEventRecord) -> String
    )
 }

+pub fn log_reported_tunnel_error_event(
+    node_id: &str,
+    event: &TunnelErrorEventRecord,
+    received_at_unix_secs: u64,
+) {
+    tracing::warn!(
+        event_name = "proxy_tunnel_error_reported",
+        source = "heartbeat",
+        node_id = %node_id,
+        category = %event.category,
+        message = %event.message,
+        severity = ?event.severity,
+        component = ?event.component,
+        summary = ?event.summary,
+        operator_action = ?event.operator_action,
+        error_reported_at_unix_secs = event.timestamp_unix_secs,
+        error_reported_at_unix_ms = ?event.timestamp_unix_ms,
+        report_received_at_unix_secs = received_at_unix_secs,
+        "proxy reported tunnel error via heartbeat"
+    );
+}
+
 pub fn normalize_proxy_metadata(
    proxy_metadata: Option<&serde_json::Value>,
    proxy_version: Option<&str>,
@@ -495,6 +518,7 @@ fn extract_recent_tunnel_errors(proxy_metadata: Option<&Value>) -> Vec<TunnelErr
                    Some(TunnelErrorEventRecord {
                        timestamp_unix_secs: json_u64(item.get("timestamp_unix_secs"))
                            .unwrap_or_default(),
+                        timestamp_unix_ms: json_u64(item.get("timestamp_unix_ms")),
                        category: item
                            .get("category")
                            .and_then(Value::as_str)
@@ -865,6 +889,7 @@ mod tests {
                {"timestamp_unix_secs": 100, "category": "older", "message": "old"},
                {
                    "timestamp_unix_secs": 101,
+                    "timestamp_unix_ms": 101_999,
                    "category": "newer",
                    "message": "new",
                    "severity": "error",
@@ -898,6 +923,10 @@ mod tests {
            sample.recent_error_events[1].component.as_deref(),
            Some("tunnel_write")
        );
+        assert_eq!(
+            sample.recent_error_events[1].timestamp_unix_ms,
+            Some(101_999)
+        );
        assert_eq!(
            build_tunnel_error_event_detail(&sample.recent_error_events[1]),
            "[newer] WebSocket write failed because the peer closed or reset the connection"