Add proxy tunnel diagnostics and default logging

Cargo.lock

@@ -336,7 +336,7 @@ dependencies = [
 
 [[package]]
 name = "aether-proxy"
-version = "0.3.10"
+version = "0.3.11"
 dependencies = [
  "aether-contracts",
  "aether-gateway",

apps/aether-gateway/src/tunnel/embedded/proxy_conn.rs

@@ -62,18 +62,30 @@ pub async fn handle_proxy_connection(
                         match send_result {
                             Ok(Ok(())) => {}
                             Ok(Err(e)) => {
+                                let snapshot = writer_conn.outbound.snapshot();
                                 warn!(
                                     conn_id = writer_conn_id,
                                     frames_sent = frames_sent,
+                                    queue_depth = snapshot.depth,
+                                    queue_capacity = snapshot.capacity,
+                                    stream_count = writer_conn.stream_count.load(std::sync::atomic::Ordering::Relaxed),
+                                    closing = writer_conn.outbound.is_closing(),
+                                    draining = writer_conn.is_draining(),
                                     error = %e,
                                     "writer ws_tx.send failed"
                                 );
                                 break;
                             }
                             Err(_) => {
+                                let snapshot = writer_conn.outbound.snapshot();
                                 warn!(
                                     conn_id = writer_conn_id,
                                     frames_sent = frames_sent,
+                                    queue_depth = snapshot.depth,
+                                    queue_capacity = snapshot.capacity,
+                                    stream_count = writer_conn.stream_count.load(std::sync::atomic::Ordering::Relaxed),
+                                    closing = writer_conn.outbound.is_closing(),
+                                    draining = writer_conn.is_draining(),
                                     "writer ws_tx.send timed out"
                                 );
                                 break;
@@ -93,6 +105,17 @@ pub async fn handle_proxy_connection(
                 },
                 changed = close_rx.changed() => {
                     if changed.is_err() || *close_rx.borrow() {
+                        let snapshot = writer_conn.outbound.snapshot();
+                        info!(
+                            conn_id = writer_conn_id,
+                            frames_sent = frames_sent,
+                            queue_depth = snapshot.depth,
+                            queue_capacity = snapshot.capacity,
+                            stream_count = writer_conn.stream_count.load(std::sync::atomic::Ordering::Relaxed),
+                            closing = writer_conn.outbound.is_closing(),
+                            draining = writer_conn.is_draining(),
+                            "writer close signal received"
+                        );
                         break;
                     }
                 }
@@ -104,7 +127,24 @@ pub async fn handle_proxy_connection(
             "writer task exiting"
         );
         writer_conn.request_close();
-        let _ = tokio::time::timeout(Duration::from_secs(5), ws_tx.close()).await;
+        match tokio::time::timeout(Duration::from_secs(5), ws_tx.close()).await {
+            Ok(Ok(())) => debug!(
+                conn_id = writer_conn_id,
+                frames_sent = frames_sent,
+                "writer WebSocket close completed"
+            ),
+            Ok(Err(error)) => warn!(
+                conn_id = writer_conn_id,
+                frames_sent = frames_sent,
+                error = %error,
+                "writer WebSocket close failed"
+            ),
+            Err(_) => warn!(
+                conn_id = writer_conn_id,
+                frames_sent = frames_sent,
+                "writer WebSocket close timed out"
+            ),
+        }
     });
 
     let ping_conn = conn.clone();
@@ -113,10 +153,34 @@ pub async fn handle_proxy_connection(
         loop {
             tokio::time::sleep(ping_interval).await;
             let ping = protocol::encode_ping();
-            if !matches!(
+            let status = ping_conn.send(Message::Binary(ping.into()));
-                ping_conn.send(Message::Binary(ping.into())),
+            if !matches!(status, SendStatus::Queued) {
-                SendStatus::Queued
+                let snapshot = ping_conn.outbound.snapshot();
-            ) {
+                match status {
+                    SendStatus::Closed => info!(
+                        conn_id = ping_conn.id,
+                        queue_depth = snapshot.depth,
+                        queue_capacity = snapshot.capacity,
+                        stream_count = ping_conn
+                            .stream_count
+                            .load(std::sync::atomic::Ordering::Relaxed),
+                        closing = ping_conn.outbound.is_closing(),
+                        draining = ping_conn.is_draining(),
+                        "ping task stopped because connection is closing"
+                    ),
+                    SendStatus::Congested => warn!(
+                        conn_id = ping_conn.id,
+                        queue_depth = snapshot.depth,
+                        queue_capacity = snapshot.capacity,
+                        stream_count = ping_conn
+                            .stream_count
+                            .load(std::sync::atomic::Ordering::Relaxed),
+                        closing = ping_conn.outbound.is_closing(),
+                        draining = ping_conn.is_draining(),
+                        "ping task stopped because outbound queue is congested"
+                    ),
+                    SendStatus::Queued => {}
+                }
                 break;
             }
         }
@@ -130,6 +194,17 @@ pub async fn handle_proxy_connection(
 
     let _ = reader.await;
     ping_task.abort();
+    let snapshot = conn.outbound.snapshot();
+    info!(
+        conn_id = conn.id,
+        node_id = %conn.node_id,
+        queue_depth = snapshot.depth,
+        queue_capacity = snapshot.capacity,
+        stream_count = conn.stream_count.load(std::sync::atomic::Ordering::Relaxed),
+        closing = conn.outbound.is_closing(),
+        draining = conn.is_draining(),
+        "proxy connection cleanup starting"
+    );
     conn.request_close();
     hub.unregister_proxy(conn_id, &node_id);
     drop(conn);

apps/aether-proxy/Cargo.lock

@@ -10,7 +10,7 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
 
 [[package]]
 name = "aether-proxy"
-version = "0.2.4"
+version = "0.3.11"
 dependencies = [
  "anyhow",
  "arc-swap",

apps/aether-proxy/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "aether-proxy"
-version = "0.3.10"
+version = "0.3.11"
 edition = "2021"
 description = "Tunnel proxy for Aether"
 

apps/aether-proxy/README.md

@@ -15,13 +15,13 @@ Tunnel 模式下代理节点**无需对外监听端口**，仅需出站连接到
 <!-- DOWNLOAD_TABLE_START -->
 | Platform | Download |
 |----------|----------|
-| Linux x86_64 (GNU) | [aether-proxy-linux-amd64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.9/aether-proxy-linux-amd64.tar.gz) |
+| Linux x86_64 (GNU) | [aether-proxy-linux-amd64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.11/aether-proxy-linux-amd64.tar.gz) |
-| Linux ARM64 (GNU) | [aether-proxy-linux-arm64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.9/aether-proxy-linux-arm64.tar.gz) |
+| Linux ARM64 (GNU) | [aether-proxy-linux-arm64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.11/aether-proxy-linux-arm64.tar.gz) |
-| Linux x86_64 (musl) | [aether-proxy-linux-musl-amd64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.9/aether-proxy-linux-musl-amd64.tar.gz) |
+| Linux x86_64 (musl) | [aether-proxy-linux-musl-amd64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.11/aether-proxy-linux-musl-amd64.tar.gz) |
-| Linux ARM64 (musl) | [aether-proxy-linux-musl-arm64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.9/aether-proxy-linux-musl-arm64.tar.gz) |
+| Linux ARM64 (musl) | [aether-proxy-linux-musl-arm64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.11/aether-proxy-linux-musl-arm64.tar.gz) |
-| macOS x86_64 | [aether-proxy-macos-amd64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.9/aether-proxy-macos-amd64.tar.gz) |
+| macOS x86_64 | [aether-proxy-macos-amd64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.11/aether-proxy-macos-amd64.tar.gz) |
-| macOS ARM64 | [aether-proxy-macos-arm64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.9/aether-proxy-macos-arm64.tar.gz) |
+| macOS ARM64 | [aether-proxy-macos-arm64.tar.gz](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.11/aether-proxy-macos-arm64.tar.gz) |
-| Windows x86_64 | [aether-proxy-windows-amd64.zip](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.9/aether-proxy-windows-amd64.zip) |
+| Windows x86_64 | [aether-proxy-windows-amd64.zip](https://github.com/fawney19/Aether/releases/download/proxy-v0.3.11/aether-proxy-windows-amd64.zip) |
 <!-- DOWNLOAD_TABLE_END -->
 
 上表展示的是最新已发布版本的下载链接。从下一次 `proxy-v*` 发布开始，表格会自动补上 `Linux x86_64 (musl)` / `Linux ARM64 (musl)` 包，供 Alpine 等 musl 系统直接使用。
@@ -180,16 +180,16 @@ upstream_proxy_url = "socks5h://microwarp:1080"
 | 参数 | 环境变量 | 默认值 | 说明 |
 |------|----------|--------|------|
 | `--log-level` | `AETHER_PROXY_LOG_LEVEL` | `info` | 日志级别 |
-| `--log-destination` | `AETHER_PROXY_LOG_DESTINATION` | `stdout` | 输出到 `stdout`、文件或两者同时输出 |
+| `--log-destination` | `AETHER_PROXY_LOG_DESTINATION` | `both` | 输出到 `stdout`、文件或两者同时输出 |
-| `--log-dir` | `AETHER_PROXY_LOG_DIR` | 空 | 文件日志目录，`file/both` 时必填 |
+| `--log-dir` | `AETHER_PROXY_LOG_DIR` | `logs` | 文件日志目录，`file/both` 时必填 |
 | `--log-rotation` | `AETHER_PROXY_LOG_ROTATION` | `daily` | 文件日志按小时或按天轮转 |
 | `--log-retention-days` | `AETHER_PROXY_LOG_RETENTION_DAYS` | `7` | 文件日志保留天数 |
 | `--log-max-files` | `AETHER_PROXY_LOG_MAX_FILES` | `30` | 文件日志最多保留文件数 |
 
 ### 日志落点
 
-- 默认 `AETHER_PROXY_LOG_DESTINATION=stdout`，日志交给容器日志驱动或宿主机服务管理器
+- 默认 `AETHER_PROXY_LOG_DESTINATION=both`，同时输出到 stdout 和 `logs/` 文件目录
-- 需要落盘时改成 `file` 或 `both`，并设置 `AETHER_PROXY_LOG_DIR`；setup TUI 里用 `Save Logs to File` 开关即可
+- 需要只交给容器日志驱动或宿主机服务管理器时，可改成 `stdout`；setup TUI 里可用 `Save Logs to File` 开关关闭文件日志
 - 文件日志固定写普通文本，并支持 `hourly/daily` 轮转；默认按天轮换、保留 7 天，最多保留 30 个文件
 - 以 `systemd` 或 `OpenRC` 安装时默认会额外打开文件日志到 `/var/log/aether-proxy`
 - OpenRC 安装时，`aether-proxy logs` 实际读取 `/var/log/aether-proxy/current.log` 和 `/var/log/aether-proxy/error.log`；这些文件通常需要用 `sudo aether-proxy logs` 查看

apps/aether-proxy/src/config.rs

@@ -53,6 +53,7 @@ pub const DEFAULT_REDIRECT_REPLAY_BUDGET_BYTES: usize = 5_242_880;
 pub const DEFAULT_REDIRECT_REPLAY_BUDGET_HUMAN: &str = "5M";
 pub const DEFAULT_LOG_RETENTION_DAYS: u64 = 7;
 pub const DEFAULT_LOG_MAX_FILES: usize = 30;
+pub const DEFAULT_LOG_DIR: &str = "logs";
 pub const DEFAULT_TUNNEL_RECONNECT_BASE_MS: u64 = 50;
 pub const DEFAULT_TUNNEL_RECONNECT_MAX_MS: u64 = 250;
 pub const DEFAULT_TUNNEL_PING_INTERVAL_MS: u64 = 10_000;
@@ -487,12 +488,12 @@ pub struct Config {
         long,
         env = "AETHER_PROXY_LOG_DESTINATION",
         value_enum,
-        default_value = "stdout"
+        default_value = "both"
     )]
     pub log_destination: ProxyLogDestinationArg,
 
     /// Log directory when file logging is enabled
-    #[arg(long, env = "AETHER_PROXY_LOG_DIR")]
+    #[arg(long, env = "AETHER_PROXY_LOG_DIR", default_value = DEFAULT_LOG_DIR)]
     pub log_dir: Option<String>,
 
     /// Log rotation schedule for file logging
@@ -1359,6 +1360,36 @@ mod tests {
         );
     }
 
+    #[test]
+    fn proxy_logs_default_to_rotating_file_and_stdout() {
+        let config = Config::parse_from([
+            "aether-proxy",
+            "--aether-url",
+            "https://example.com",
+            "--management-token",
+            "ae_test",
+            "--node-name",
+            "proxy-test",
+        ]);
+
+        assert_eq!(config.log_destination, ProxyLogDestinationArg::Both);
+        assert_eq!(config.log_dir.as_deref(), Some(DEFAULT_LOG_DIR));
+        assert_eq!(config.log_rotation, ProxyLogRotationArg::Daily);
+        assert_eq!(config.log_retention_days, DEFAULT_LOG_RETENTION_DAYS);
+
+        let runtime = config
+            .service_runtime_config()
+            .expect("default file logging should be valid");
+        assert_eq!(runtime.observability.log_destination, LogDestination::Both);
+        let file_logging = runtime
+            .observability
+            .file_logging
+            .expect("file logging should be enabled by default");
+        assert_eq!(file_logging.dir, std::path::PathBuf::from(DEFAULT_LOG_DIR));
+        assert_eq!(file_logging.rotation, LogRotation::Daily);
+        assert_eq!(file_logging.retention_days, DEFAULT_LOG_RETENTION_DAYS);
+    }
+
     #[test]
     fn config_file_load_accepts_server_scoped_upstream_proxy_url() {
         let cfg = parse_config_file_content(

apps/aether-proxy/src/setup/tui.rs

@@ -171,7 +171,7 @@ impl App {
                 Field {
                     label: "Save Logs to File",
                     key: "save_logs_to_file",
-                    value: "false".into(),
+                    value: "true".into(),
                     kind: FieldKind::Bool,
                     required: false,
                     help: "Write pretty .log files with daily rotation and 7-day retention",

apps/aether-proxy/src/state.rs

@@ -97,6 +97,7 @@ const TUNNEL_ERROR_MESSAGE_MAX_CHARS: usize = 320;
 #[derive(Debug, Clone, serde::Serialize)]
 pub struct TunnelErrorEvent {
     pub timestamp_unix_secs: u64,
+    pub timestamp_unix_ms: u64,
     pub category: String,
     pub message: String,
     pub severity: String,
@@ -240,8 +241,10 @@ impl TunnelMetrics {
         let message = normalize_error_field(message, TUNNEL_ERROR_MESSAGE_MAX_CHARS, "n/a");
         let diagnostic = classify_tunnel_error(category.as_str(), message.as_str());
 
+        let timestamp_unix_ms = now_unix_ms();
         let event = TunnelErrorEvent {
-            timestamp_unix_secs: now_unix_secs(),
+            timestamp_unix_secs: timestamp_unix_ms / 1_000,
+            timestamp_unix_ms,
             category,
             message,
             severity: diagnostic.severity.to_string(),
@@ -298,9 +301,13 @@ impl TunnelMetrics {
 }
 
 fn now_unix_secs() -> u64 {
+    now_unix_ms() / 1_000
+}
+
+fn now_unix_ms() -> u64 {
     SystemTime::now()
         .duration_since(UNIX_EPOCH)
-        .map(|d| d.as_secs())
+        .map(|d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX))
         .unwrap_or(0)
 }
 

apps/aether-proxy/src/tunnel/client.rs

@@ -8,7 +8,7 @@ use tokio::sync::watch;
 use tokio_tungstenite::tungstenite::client::IntoClientRequest;
 use tokio_tungstenite::tungstenite::http;
 use tokio_tungstenite::tungstenite::protocol::WebSocketConfig;
-use tracing::{debug, warn};
+use tracing::{debug, info, warn};
 
 use crate::egress_proxy::{connect_target_via_proxy, ProxyConnectOptions, UpstreamProxyConfig};
 use crate::state::{AppState, ServerContext};
@@ -224,9 +224,30 @@ pub async fn connect_and_run(
         let _ = tokio::time::timeout(Duration::from_secs(35), writer_handle).await;
     }
 
-    server
+    let connected_for = connected_at.elapsed();
-        .tunnel_metrics
+    match &outcome {
-        .record_disconnect(connected_at.elapsed());
+        Ok(TunnelOutcome::Shutdown) => info!(
+            conn = conn_idx,
+            connected_duration_ms = connected_for.as_millis() as u64,
+            close_reason = "shutdown",
+            "tunnel session ending"
+        ),
+        Ok(TunnelOutcome::Disconnected) => info!(
+            conn = conn_idx,
+            connected_duration_ms = connected_for.as_millis() as u64,
+            close_reason = "disconnected",
+            "tunnel session ending"
+        ),
+        Err(error) => warn!(
+            conn = conn_idx,
+            connected_duration_ms = connected_for.as_millis() as u64,
+            close_reason = "error",
+            error = %error,
+            "tunnel session ending"
+        ),
+    }
+
+    server.tunnel_metrics.record_disconnect(connected_for);
 
     debug!("tunnel disconnected");
     outcome
@@ -250,14 +271,26 @@ fn spawn_drain_signal(
         }
 
         debug!(conn = conn_idx, "sending GOAWAY for tunnel drain");
-        let _ = tokio::time::timeout(
+        match tokio::time::timeout(
             Duration::from_millis(250),
             frame_tx.send(super::protocol::Frame::control(
                 super::protocol::MsgType::GoAway,
                 bytes::Bytes::new(),
             )),
         )
-        .await;
+        .await
+        {
+            Ok(Ok(())) => info!(conn = conn_idx, "sent GOAWAY for tunnel drain"),
+            Ok(Err(error)) => warn!(
+                conn = conn_idx,
+                error = ?error,
+                "failed to queue GOAWAY for tunnel drain"
+            ),
+            Err(_) => warn!(
+                conn = conn_idx,
+                "timed out queueing GOAWAY for tunnel drain"
+            ),
+        }
     })
 }
 

apps/aether-proxy/src/tunnel/heartbeat.rs

@@ -536,6 +536,10 @@ mod tests {
             .pointer("/proxy_metadata/recent_tunnel_errors/0")
             .and_then(serde_json::Value::as_object)
             .expect("recent tunnel error should be reported");
+        assert!(recent_error
+            .get("timestamp_unix_ms")
+            .and_then(serde_json::Value::as_u64)
+            .is_some());
         assert_eq!(
             recent_error
                 .get("component")

apps/aether-proxy/src/tunnel/writer.rs

@@ -179,10 +179,20 @@ async fn write_frame<S>(sink: &mut S, frame: Frame, tunnel_metrics: Option<&Tunn
 where
     S: SinkExt<Message, Error = tokio_tungstenite::tungstenite::Error> + Unpin + Send + 'static,
 {
+    let stream_id = frame.stream_id;
+    let msg_type = frame.msg_type;
+    let flags = frame.flags;
     let data = frame.encode();
     let wire_len = data.len().max(HEADER_SIZE);
     if let Err(e) = sink.send(Message::Binary(data.into())).await {
-        error!(error = %e, "failed to write frame to WebSocket");
+        error!(
+            stream_id = stream_id,
+            msg_type = ?msg_type,
+            flags = flags,
+            wire_len = wire_len,
+            error = %e,
+            "failed to write frame to WebSocket"
+        );
         if let Some(metrics) = tunnel_metrics {
             metrics.record_error("ws_write_error", &e.to_string());
         }

crates/aether-data/src/repository/proxy_nodes/memory.rs

@@ -8,13 +8,13 @@ use uuid::Uuid;
 
 use super::types::{
     bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
-    normalize_proxy_metadata, reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery,
+    log_reported_tunnel_error_event, normalize_proxy_metadata,
-    ProxyNodeHeartbeatMutation, ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation,
+    reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery, ProxyNodeHeartbeatMutation,
-    ProxyNodeMetricsCleanupSummary, ProxyNodeMetricsStep, ProxyNodeReadRepository,
+    ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation, ProxyNodeMetricsCleanupSummary,
-    ProxyNodeRegistrationMutation, ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation,
+    ProxyNodeMetricsStep, ProxyNodeReadRepository, ProxyNodeRegistrationMutation,
-    ProxyNodeTunnelStatusMutation, ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket,
+    ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation, ProxyNodeTunnelStatusMutation,
-    StoredProxyNode, StoredProxyNodeEvent, StoredProxyNodeMetricsBucket, TunnelMetricsSample,
+    ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket, StoredProxyNode, StoredProxyNodeEvent,
-    PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
+    StoredProxyNodeMetricsBucket, TunnelMetricsSample, PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
 };
 use crate::DataLayerError;
 
@@ -640,6 +640,7 @@ impl ProxyNodeWriteRepository for InMemoryProxyNodeRepository {
 
             let mut events = self.events.write().expect("proxy node repository lock");
             for error in &sample.recent_error_events {
+                log_reported_tunnel_error_event(&node.id, error, now_unix_secs);
                 let event_id = Self::next_event_id(&events);
                 events.push(StoredProxyNodeEvent {
                     id: event_id,
@@ -655,6 +656,7 @@ impl ProxyNodeWriteRepository for InMemoryProxyNodeRepository {
                         "summary": error.summary.as_deref(),
                         "operator_action": error.operator_action.as_deref(),
                         "timestamp_unix_secs": error.timestamp_unix_secs,
+                        "timestamp_unix_ms": error.timestamp_unix_ms,
                     })),
                     created_at_unix_ms: Some(if error.timestamp_unix_secs == 0 {
                         now_unix_secs

crates/aether-data/src/repository/proxy_nodes/mod.rs

@@ -10,7 +10,8 @@ pub use postgres::SqlxProxyNodeRepository;
 pub use sqlite::SqliteProxyNodeReadRepository;
 pub use types::{
     bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
-    normalize_proxy_node_scheduling_state, proxy_node_accepts_new_tunnels, proxy_reported_version,
+    log_reported_tunnel_error_event, normalize_proxy_node_scheduling_state,
+    proxy_node_accepts_new_tunnels, proxy_reported_version,
     reconcile_remote_config_after_heartbeat, remote_config_scheduling_state,
     remote_config_upgrade_target, ProxyNodeEventQuery, ProxyNodeHeartbeatMutation,
     ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation, ProxyNodeMetricsCleanupSummary,

crates/aether-data/src/repository/proxy_nodes/mysql.rs

@@ -3,13 +3,13 @@ use sqlx::{mysql::MySqlRow, Row};
 
 use super::types::{
     bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
-    normalize_proxy_metadata, reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery,
+    log_reported_tunnel_error_event, normalize_proxy_metadata,
-    ProxyNodeHeartbeatMutation, ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation,
+    reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery, ProxyNodeHeartbeatMutation,
-    ProxyNodeMetricsCleanupSummary, ProxyNodeMetricsStep, ProxyNodeReadRepository,
+    ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation, ProxyNodeMetricsCleanupSummary,
-    ProxyNodeRegistrationMutation, ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation,
+    ProxyNodeMetricsStep, ProxyNodeReadRepository, ProxyNodeRegistrationMutation,
-    ProxyNodeTunnelStatusMutation, ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket,
+    ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation, ProxyNodeTunnelStatusMutation,
-    StoredProxyNode, StoredProxyNodeEvent, StoredProxyNodeMetricsBucket,
+    ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket, StoredProxyNode, StoredProxyNodeEvent,
-    PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
+    StoredProxyNodeMetricsBucket, PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
 };
 use crate::driver::mysql::MysqlPool;
 use crate::error::SqlResultExt;
@@ -831,6 +831,7 @@ WHERE is_manual = 0
             .await?;
 
             for error in &sample.recent_error_events {
+                log_reported_tunnel_error_event(&node.id, error, now_unix_secs);
                 let detail = build_tunnel_error_event_detail(error);
                 let event_metadata = serde_json::json!({
                     "source": "heartbeat",
@@ -841,6 +842,7 @@ WHERE is_manual = 0
                     "summary": error.summary.as_deref(),
                     "operator_action": error.operator_action.as_deref(),
                     "timestamp_unix_secs": error.timestamp_unix_secs,
+                    "timestamp_unix_ms": error.timestamp_unix_ms,
                 });
                 self.insert_event(
                     &node.id,

crates/aether-data/src/repository/proxy_nodes/postgres.rs

@@ -5,13 +5,13 @@ use sqlx::{postgres::PgRow, PgPool, Row};
 
 use super::types::{
     bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
-    normalize_proxy_metadata, reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery,
+    log_reported_tunnel_error_event, normalize_proxy_metadata,
-    ProxyNodeHeartbeatMutation, ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation,
+    reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery, ProxyNodeHeartbeatMutation,
-    ProxyNodeMetricsCleanupSummary, ProxyNodeMetricsStep, ProxyNodeReadRepository,
+    ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation, ProxyNodeMetricsCleanupSummary,
-    ProxyNodeRegistrationMutation, ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation,
+    ProxyNodeMetricsStep, ProxyNodeReadRepository, ProxyNodeRegistrationMutation,
-    ProxyNodeTunnelStatusMutation, ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket,
+    ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation, ProxyNodeTunnelStatusMutation,
-    StoredProxyNode, StoredProxyNodeEvent, StoredProxyNodeMetricsBucket, TunnelMetricsSample,
+    ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket, StoredProxyNode, StoredProxyNodeEvent,
-    PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
+    StoredProxyNodeMetricsBucket, TunnelMetricsSample, PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
 };
 use crate::{
     error::{postgres_error, SqlxResultExt},
@@ -1219,6 +1219,7 @@ impl ProxyNodeWriteRepository for SqlxProxyNodeRepository {
             .await?;
 
             for error in &sample.recent_error_events {
+                log_reported_tunnel_error_event(&updated.id, error, now_unix_secs);
                 let detail = build_tunnel_error_event_detail(error);
                 let event_metadata = serde_json::json!({
                     "source": "heartbeat",
@@ -1229,6 +1230,7 @@ impl ProxyNodeWriteRepository for SqlxProxyNodeRepository {
                     "summary": error.summary.as_deref(),
                     "operator_action": error.operator_action.as_deref(),
                     "timestamp_unix_secs": error.timestamp_unix_secs,
+                    "timestamp_unix_ms": error.timestamp_unix_ms,
                 });
                 self.insert_event(
                     &updated.id,

crates/aether-data/src/repository/proxy_nodes/sqlite.rs

@@ -3,13 +3,13 @@ use sqlx::{sqlite::SqliteRow, Row};
 
 use super::types::{
     bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
-    normalize_proxy_metadata, reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery,
+    log_reported_tunnel_error_event, normalize_proxy_metadata,
-    ProxyNodeHeartbeatMutation, ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation,
+    reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery, ProxyNodeHeartbeatMutation,
-    ProxyNodeMetricsCleanupSummary, ProxyNodeMetricsStep, ProxyNodeReadRepository,
+    ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation, ProxyNodeMetricsCleanupSummary,
-    ProxyNodeRegistrationMutation, ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation,
+    ProxyNodeMetricsStep, ProxyNodeReadRepository, ProxyNodeRegistrationMutation,
-    ProxyNodeTunnelStatusMutation, ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket,
+    ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation, ProxyNodeTunnelStatusMutation,
-    StoredProxyNode, StoredProxyNodeEvent, StoredProxyNodeMetricsBucket,
+    ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket, StoredProxyNode, StoredProxyNodeEvent,
-    PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
+    StoredProxyNodeMetricsBucket, PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
 };
 use crate::driver::sqlite::SqlitePool;
 use crate::error::SqlResultExt;
@@ -833,6 +833,7 @@ WHERE is_manual = 0
             .await?;
 
             for error in &sample.recent_error_events {
+                log_reported_tunnel_error_event(&node.id, error, now_unix_secs);
                 let detail = build_tunnel_error_event_detail(error);
                 let event_metadata = serde_json::json!({
                     "source": "heartbeat",
@@ -843,6 +844,7 @@ WHERE is_manual = 0
                     "summary": error.summary.as_deref(),
                     "operator_action": error.operator_action.as_deref(),
                     "timestamp_unix_secs": error.timestamp_unix_secs,
+                    "timestamp_unix_ms": error.timestamp_unix_ms,
                 });
                 self.insert_event(
                     &node.id,

crates/aether-data/src/repository/proxy_nodes/types.rs

@@ -322,6 +322,7 @@ pub const PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR: &str = "tunnel_err";
 #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 pub struct TunnelErrorEventRecord {
     pub timestamp_unix_secs: u64,
+    pub timestamp_unix_ms: Option<u64>,
     pub category: String,
     pub message: String,
     pub severity: Option<String>,
@@ -428,6 +429,28 @@ pub fn build_tunnel_error_event_detail(event: &TunnelErrorEventRecord) -> String
     )
 }
 
+pub fn log_reported_tunnel_error_event(
+    node_id: &str,
+    event: &TunnelErrorEventRecord,
+    received_at_unix_secs: u64,
+) {
+    tracing::warn!(
+        event_name = "proxy_tunnel_error_reported",
+        source = "heartbeat",
+        node_id = %node_id,
+        category = %event.category,
+        message = %event.message,
+        severity = ?event.severity,
+        component = ?event.component,
+        summary = ?event.summary,
+        operator_action = ?event.operator_action,
+        error_reported_at_unix_secs = event.timestamp_unix_secs,
+        error_reported_at_unix_ms = ?event.timestamp_unix_ms,
+        report_received_at_unix_secs = received_at_unix_secs,
+        "proxy reported tunnel error via heartbeat"
+    );
+}
+
 pub fn normalize_proxy_metadata(
     proxy_metadata: Option<&serde_json::Value>,
     proxy_version: Option<&str>,
@@ -495,6 +518,7 @@ fn extract_recent_tunnel_errors(proxy_metadata: Option<&Value>) -> Vec<TunnelErr
                     Some(TunnelErrorEventRecord {
                         timestamp_unix_secs: json_u64(item.get("timestamp_unix_secs"))
                             .unwrap_or_default(),
+                        timestamp_unix_ms: json_u64(item.get("timestamp_unix_ms")),
                         category: item
                             .get("category")
                             .and_then(Value::as_str)
@@ -865,6 +889,7 @@ mod tests {
                 {"timestamp_unix_secs": 100, "category": "older", "message": "old"},
                 {
                     "timestamp_unix_secs": 101,
+                    "timestamp_unix_ms": 101_999,
                     "category": "newer",
                     "message": "new",
                     "severity": "error",
@@ -898,6 +923,10 @@ mod tests {
             sample.recent_error_events[1].component.as_deref(),
             Some("tunnel_write")
         );
+        assert_eq!(
+            sample.recent_error_events[1].timestamp_unix_ms,
+            Some(101_999)
+        );
         assert_eq!(
             build_tunnel_error_event_detail(&sample.recent_error_events[1]),
             "[newer] WebSocket write failed because the peer closed or reset the connection"