i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 17:30:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add proxy tunnel diagnostics and default logging

Browse Source
This commit is contained in:
fawney19
2026-05-15 19:43:59 +08:00
parent 1503986d40
commit 87e44479cc
17 changed files with 258 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
crates/aether-data/src/repository/proxy_nodes/memory.rs
View File

@@ -8,13 +8,13 @@ use uuid::Uuid;
use super::types::{
bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
normalize_proxy_metadata, reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery,
ProxyNodeHeartbeatMutation, ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation,
ProxyNodeMetricsCleanupSummary, ProxyNodeMetricsStep, ProxyNodeReadRepository,
ProxyNodeRegistrationMutation, ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation,
ProxyNodeTunnelStatusMutation, ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket,
StoredProxyNode, StoredProxyNodeEvent, StoredProxyNodeMetricsBucket, TunnelMetricsSample,
PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
log_reported_tunnel_error_event, normalize_proxy_metadata,
reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery, ProxyNodeHeartbeatMutation,
ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation, ProxyNodeMetricsCleanupSummary,
ProxyNodeMetricsStep, ProxyNodeReadRepository, ProxyNodeRegistrationMutation,
ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation, ProxyNodeTunnelStatusMutation,
ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket, StoredProxyNode, StoredProxyNodeEvent,
StoredProxyNodeMetricsBucket, TunnelMetricsSample, PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
};
use crate::DataLayerError;
@@ -640,6 +640,7 @@ impl ProxyNodeWriteRepository for InMemoryProxyNodeRepository {
let mut events = self.events.write().expect("proxy node repository lock");
for error in &sample.recent_error_events {
log_reported_tunnel_error_event(&node.id, error, now_unix_secs);
let event_id = Self::next_event_id(&events);
events.push(StoredProxyNodeEvent {
id: event_id,
@@ -655,6 +656,7 @@ impl ProxyNodeWriteRepository for InMemoryProxyNodeRepository {
"summary": error.summary.as_deref(),
"operator_action": error.operator_action.as_deref(),
"timestamp_unix_secs": error.timestamp_unix_secs,
"timestamp_unix_ms": error.timestamp_unix_ms,
})),
created_at_unix_ms: Some(if error.timestamp_unix_secs == 0 {
now_unix_secs

3
crates/aether-data/src/repository/proxy_nodes/mod.rs
View File

@@ -10,7 +10,8 @@ pub use postgres::SqlxProxyNodeRepository;
pub use sqlite::SqliteProxyNodeReadRepository;
pub use types::{
bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
normalize_proxy_node_scheduling_state, proxy_node_accepts_new_tunnels, proxy_reported_version,
log_reported_tunnel_error_event, normalize_proxy_node_scheduling_state,
proxy_node_accepts_new_tunnels, proxy_reported_version,
reconcile_remote_config_after_heartbeat, remote_config_scheduling_state,
remote_config_upgrade_target, ProxyNodeEventQuery, ProxyNodeHeartbeatMutation,
ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation, ProxyNodeMetricsCleanupSummary,

16
crates/aether-data/src/repository/proxy_nodes/mysql.rs
View File

@@ -3,13 +3,13 @@ use sqlx::{mysql::MySqlRow, Row};
use super::types::{
bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
normalize_proxy_metadata, reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery,
ProxyNodeHeartbeatMutation, ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation,
ProxyNodeMetricsCleanupSummary, ProxyNodeMetricsStep, ProxyNodeReadRepository,
ProxyNodeRegistrationMutation, ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation,
ProxyNodeTunnelStatusMutation, ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket,
StoredProxyNode, StoredProxyNodeEvent, StoredProxyNodeMetricsBucket,
PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
log_reported_tunnel_error_event, normalize_proxy_metadata,
reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery, ProxyNodeHeartbeatMutation,
ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation, ProxyNodeMetricsCleanupSummary,
ProxyNodeMetricsStep, ProxyNodeReadRepository, ProxyNodeRegistrationMutation,
ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation, ProxyNodeTunnelStatusMutation,
ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket, StoredProxyNode, StoredProxyNodeEvent,
StoredProxyNodeMetricsBucket, PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
};
use crate::driver::mysql::MysqlPool;
use crate::error::SqlResultExt;
@@ -831,6 +831,7 @@ WHERE is_manual = 0
.await?;
for error in &sample.recent_error_events {
log_reported_tunnel_error_event(&node.id, error, now_unix_secs);
let detail = build_tunnel_error_event_detail(error);
let event_metadata = serde_json::json!({
"source": "heartbeat",
@@ -841,6 +842,7 @@ WHERE is_manual = 0
"summary": error.summary.as_deref(),
"operator_action": error.operator_action.as_deref(),
"timestamp_unix_secs": error.timestamp_unix_secs,
"timestamp_unix_ms": error.timestamp_unix_ms,
});
self.insert_event(
&node.id,

16
crates/aether-data/src/repository/proxy_nodes/postgres.rs
View File

@@ -5,13 +5,13 @@ use sqlx::{postgres::PgRow, PgPool, Row};
use super::types::{
bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
normalize_proxy_metadata, reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery,
ProxyNodeHeartbeatMutation, ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation,
ProxyNodeMetricsCleanupSummary, ProxyNodeMetricsStep, ProxyNodeReadRepository,
ProxyNodeRegistrationMutation, ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation,
ProxyNodeTunnelStatusMutation, ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket,
StoredProxyNode, StoredProxyNodeEvent, StoredProxyNodeMetricsBucket, TunnelMetricsSample,
PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
log_reported_tunnel_error_event, normalize_proxy_metadata,
reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery, ProxyNodeHeartbeatMutation,
ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation, ProxyNodeMetricsCleanupSummary,
ProxyNodeMetricsStep, ProxyNodeReadRepository, ProxyNodeRegistrationMutation,
ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation, ProxyNodeTunnelStatusMutation,
ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket, StoredProxyNode, StoredProxyNodeEvent,
StoredProxyNodeMetricsBucket, TunnelMetricsSample, PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
};
use crate::{
error::{postgres_error, SqlxResultExt},
@@ -1219,6 +1219,7 @@ impl ProxyNodeWriteRepository for SqlxProxyNodeRepository {
.await?;
for error in &sample.recent_error_events {
log_reported_tunnel_error_event(&updated.id, error, now_unix_secs);
let detail = build_tunnel_error_event_detail(error);
let event_metadata = serde_json::json!({
"source": "heartbeat",
@@ -1229,6 +1230,7 @@ impl ProxyNodeWriteRepository for SqlxProxyNodeRepository {
"summary": error.summary.as_deref(),
"operator_action": error.operator_action.as_deref(),
"timestamp_unix_secs": error.timestamp_unix_secs,
"timestamp_unix_ms": error.timestamp_unix_ms,
});
self.insert_event(
&updated.id,

16
crates/aether-data/src/repository/proxy_nodes/sqlite.rs
View File

@@ -3,13 +3,13 @@ use sqlx::{sqlite::SqliteRow, Row};
use super::types::{
bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
normalize_proxy_metadata, reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery,
ProxyNodeHeartbeatMutation, ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation,
ProxyNodeMetricsCleanupSummary, ProxyNodeMetricsStep, ProxyNodeReadRepository,
ProxyNodeRegistrationMutation, ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation,
ProxyNodeTunnelStatusMutation, ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket,
StoredProxyNode, StoredProxyNodeEvent, StoredProxyNodeMetricsBucket,
PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
log_reported_tunnel_error_event, normalize_proxy_metadata,
reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery, ProxyNodeHeartbeatMutation,
ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation, ProxyNodeMetricsCleanupSummary,
ProxyNodeMetricsStep, ProxyNodeReadRepository, ProxyNodeRegistrationMutation,
ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation, ProxyNodeTunnelStatusMutation,
ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket, StoredProxyNode, StoredProxyNodeEvent,
StoredProxyNodeMetricsBucket, PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
};
use crate::driver::sqlite::SqlitePool;
use crate::error::SqlResultExt;
@@ -833,6 +833,7 @@ WHERE is_manual = 0
.await?;
for error in &sample.recent_error_events {
log_reported_tunnel_error_event(&node.id, error, now_unix_secs);
let detail = build_tunnel_error_event_detail(error);
let event_metadata = serde_json::json!({
"source": "heartbeat",
@@ -843,6 +844,7 @@ WHERE is_manual = 0
"summary": error.summary.as_deref(),
"operator_action": error.operator_action.as_deref(),
"timestamp_unix_secs": error.timestamp_unix_secs,
"timestamp_unix_ms": error.timestamp_unix_ms,
});
self.insert_event(
&node.id,

29
crates/aether-data/src/repository/proxy_nodes/types.rs
View File

@@ -322,6 +322,7 @@ pub const PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR: &str = "tunnel_err";
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct TunnelErrorEventRecord {
pub timestamp_unix_secs: u64,
pub timestamp_unix_ms: Option<u64>,
pub category: String,
pub message: String,
pub severity: Option<String>,
@@ -428,6 +429,28 @@ pub fn build_tunnel_error_event_detail(event: &TunnelErrorEventRecord) -> String
)
}
pub fn log_reported_tunnel_error_event(
node_id: &str,
event: &TunnelErrorEventRecord,
received_at_unix_secs: u64,
) {
tracing::warn!(
event_name = "proxy_tunnel_error_reported",
source = "heartbeat",
node_id = %node_id,
category = %event.category,
message = %event.message,
severity = ?event.severity,
component = ?event.component,
summary = ?event.summary,
operator_action = ?event.operator_action,
error_reported_at_unix_secs = event.timestamp_unix_secs,
error_reported_at_unix_ms = ?event.timestamp_unix_ms,
report_received_at_unix_secs = received_at_unix_secs,
"proxy reported tunnel error via heartbeat"
);
}
pub fn normalize_proxy_metadata(
proxy_metadata: Option<&serde_json::Value>,
proxy_version: Option<&str>,
@@ -495,6 +518,7 @@ fn extract_recent_tunnel_errors(proxy_metadata: Option<&Value>) -> Vec<TunnelErr
Some(TunnelErrorEventRecord {
timestamp_unix_secs: json_u64(item.get("timestamp_unix_secs"))
.unwrap_or_default(),
timestamp_unix_ms: json_u64(item.get("timestamp_unix_ms")),
category: item
.get("category")
.and_then(Value::as_str)
@@ -865,6 +889,7 @@ mod tests {
{"timestamp_unix_secs": 100, "category": "older", "message": "old"},
{
"timestamp_unix_secs": 101,
"timestamp_unix_ms": 101_999,
"category": "newer",
"message": "new",
"severity": "error",
@@ -898,6 +923,10 @@ mod tests {
sample.recent_error_events[1].component.as_deref(),
Some("tunnel_write")
);
assert_eq!(
sample.recent_error_events[1].timestamp_unix_ms,
Some(101_999)
);
assert_eq!(
build_tunnel_error_event_detail(&sample.recent_error_events[1]),
"[newer] WebSocket write failed because the peer closed or reset the connection"