i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-08 02:17:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(proxy): record tunnel stability metrics

Browse Source
This commit is contained in:
fawney19
2026-05-08 22:03:17 +08:00
parent 9a84a6ff6c
commit a703acd1fe
81 changed files with 7228 additions and 1109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/aether-data/src/repository/proxy_nodes/mysql.rs
+365 -6
View File
@@ -2,10 +2,14 @@ use async_trait::async_trait;
use sqlx::{mysql::MySqlRow, Row};
use super::types::{
normalize_proxy_metadata, reconcile_remote_config_after_heartbeat, ProxyNodeHeartbeatMutation,
ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation, ProxyNodeReadRepository,
bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
normalize_proxy_metadata, reconcile_remote_config_after_heartbeat, ProxyNodeEventQuery,
ProxyNodeHeartbeatMutation, ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation,
ProxyNodeMetricsCleanupSummary, ProxyNodeMetricsStep, ProxyNodeReadRepository,
ProxyNodeRegistrationMutation, ProxyNodeRemoteConfigMutation, ProxyNodeTrafficMutation,
ProxyNodeTunnelStatusMutation, ProxyNodeWriteRepository, StoredProxyNode, StoredProxyNodeEvent,
ProxyNodeTunnelStatusMutation, ProxyNodeWriteRepository, StoredProxyFleetMetricsBucket,
StoredProxyNode, StoredProxyNodeEvent, StoredProxyNodeMetricsBucket,
PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
};
use crate::driver::mysql::MysqlPool;
use crate::error::SqlResultExt;
@@ -148,17 +152,22 @@ ON DUPLICATE KEY UPDATE
node_id: &str,
event_type: &str,
detail: Option<&str>,
event_metadata: Option<&serde_json::Value>,
created_at_unix_secs: Option<u64>,
) -> Result<(), DataLayerError> {
sqlx::query(
r#"
INSERT INTO proxy_node_events (node_id, event_type, detail, created_at)
VALUES (?, ?, ?, ?)
INSERT INTO proxy_node_events (node_id, event_type, detail, event_metadata, created_at)
VALUES (?, ?, ?, ?, ?)
"#,
)
.bind(node_id)
.bind(event_type)
.bind(detail)
.bind(optional_json_to_string(
&event_metadata.cloned(),
"proxy_node_events.event_metadata",
)?)
.bind(created_at_unix_secs.unwrap_or_else(current_unix_secs) as i64)
.execute(&self.pool)
.await
@@ -166,6 +175,70 @@ VALUES (?, ?, ?, ?)
Ok(())
}
async fn upsert_metrics_bucket(
&self,
table: &str,
node_id: &str,
bucket_start: u64,
sample: &super::types::TunnelMetricsSample,
) -> Result<(), DataLayerError> {
sqlx::query(&format!(
r#"
INSERT INTO {table} (
node_id,
bucket_start_unix_secs,
samples,
uptime_samples,
active_connections_sum,
active_connections_max,
heartbeat_rtt_ms_sum,
heartbeat_rtt_ms_max,
connect_errors_delta,
disconnects_delta,
error_events_delta,
ws_in_bytes_delta,
ws_out_bytes_delta,
ws_in_frames_delta,
ws_out_frames_delta
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
samples = samples + VALUES(samples),
uptime_samples = uptime_samples + VALUES(uptime_samples),
active_connections_sum = active_connections_sum + VALUES(active_connections_sum),
active_connections_max = GREATEST(active_connections_max, VALUES(active_connections_max)),
heartbeat_rtt_ms_sum = heartbeat_rtt_ms_sum + VALUES(heartbeat_rtt_ms_sum),
heartbeat_rtt_ms_max = GREATEST(heartbeat_rtt_ms_max, VALUES(heartbeat_rtt_ms_max)),
connect_errors_delta = connect_errors_delta + VALUES(connect_errors_delta),
disconnects_delta = disconnects_delta + VALUES(disconnects_delta),
error_events_delta = error_events_delta + VALUES(error_events_delta),
ws_in_bytes_delta = ws_in_bytes_delta + VALUES(ws_in_bytes_delta),
ws_out_bytes_delta = ws_out_bytes_delta + VALUES(ws_out_bytes_delta),
ws_in_frames_delta = ws_in_frames_delta + VALUES(ws_in_frames_delta),
ws_out_frames_delta = ws_out_frames_delta + VALUES(ws_out_frames_delta)
"#
))
.bind(node_id)
.bind(i64::try_from(bucket_start).unwrap_or(i64::MAX))
.bind(sample.samples)
.bind(sample.uptime_samples)
.bind(sample.active_connections_sum)
.bind(sample.active_connections_max)
.bind(sample.heartbeat_rtt_ms_sum)
.bind(sample.heartbeat_rtt_ms_max)
.bind(sample.connect_errors_delta)
.bind(sample.disconnects_delta)
.bind(sample.error_events_delta)
.bind(sample.ws_in_bytes_delta)
.bind(sample.ws_out_bytes_delta)
.bind(sample.ws_in_frames_delta)
.bind(sample.ws_out_frames_delta)
.execute(&self.pool)
.await
.map_sql_err()?;
Ok(())
}
fn normalize_remote_config(
mutation: &ProxyNodeRemoteConfigMutation,
existing: Option<&serde_json::Value>,
@@ -298,6 +371,7 @@ SELECT
node_id,
event_type,
detail,
event_metadata,
created_at AS created_at_unix_ms
FROM proxy_node_events
WHERE node_id = ?
@@ -312,6 +386,152 @@ LIMIT ?
.map_sql_err()?;
rows.iter().map(map_proxy_node_event_row).collect()
}
async fn list_proxy_node_events_filtered(
&self,
node_id: &str,
query: &ProxyNodeEventQuery,
) -> Result<Vec<StoredProxyNodeEvent>, DataLayerError> {
let rows = sqlx::query(
r#"
SELECT
id,
node_id,
event_type,
detail,
event_metadata,
created_at AS created_at_unix_ms
FROM proxy_node_events
WHERE node_id = ?
AND (? IS NULL OR created_at >= ?)
AND (? IS NULL OR created_at <= ?)
AND (? IS NULL OR LOWER(event_type) = LOWER(?))
ORDER BY created_at DESC, id DESC
LIMIT ?
"#,
)
.bind(node_id)
.bind(
query
.from_unix_secs
.map(|v| i64::try_from(v).unwrap_or(i64::MAX)),
)
.bind(
query
.from_unix_secs
.map(|v| i64::try_from(v).unwrap_or(i64::MAX)),
)
.bind(
query
.to_unix_secs
.map(|v| i64::try_from(v).unwrap_or(i64::MAX)),
)
.bind(
query
.to_unix_secs
.map(|v| i64::try_from(v).unwrap_or(i64::MAX)),
)
.bind(query.event_type.as_deref())
.bind(query.event_type.as_deref())
.bind(i64::try_from(query.limit).unwrap_or(i64::MAX))
.fetch_all(&self.pool)
.await
.map_sql_err()?;
rows.iter().map(map_proxy_node_event_row).collect()
}
async fn list_proxy_node_metrics(
&self,
node_id: &str,
step: ProxyNodeMetricsStep,
from_unix_secs: u64,
to_unix_secs: u64,
limit: usize,
) -> Result<Vec<StoredProxyNodeMetricsBucket>, DataLayerError> {
let table = match step {
ProxyNodeMetricsStep::OneMinute => "proxy_node_metrics_1m",
ProxyNodeMetricsStep::OneHour => "proxy_node_metrics_1h",
};
let rows = sqlx::query(&format!(
r#"
SELECT
node_id,
bucket_start_unix_secs,
samples,
uptime_samples,
active_connections_sum,
active_connections_max,
heartbeat_rtt_ms_sum,
heartbeat_rtt_ms_max,
connect_errors_delta,
disconnects_delta,
error_events_delta,
ws_in_bytes_delta,
ws_out_bytes_delta,
ws_in_frames_delta,
ws_out_frames_delta
FROM {table}
WHERE node_id = ?
AND bucket_start_unix_secs >= ?
AND bucket_start_unix_secs <= ?
ORDER BY bucket_start_unix_secs ASC
LIMIT ?
"#
))
.bind(node_id)
.bind(i64::try_from(from_unix_secs).unwrap_or(i64::MAX))
.bind(i64::try_from(to_unix_secs).unwrap_or(i64::MAX))
.bind(i64::try_from(limit).unwrap_or(i64::MAX))
.fetch_all(&self.pool)
.await
.map_sql_err()?;
rows.iter().map(map_proxy_node_metric_row).collect()
}
async fn list_proxy_fleet_metrics(
&self,
step: ProxyNodeMetricsStep,
from_unix_secs: u64,
to_unix_secs: u64,
limit: usize,
) -> Result<Vec<StoredProxyFleetMetricsBucket>, DataLayerError> {
let table = match step {
ProxyNodeMetricsStep::OneMinute => "proxy_node_metrics_1m",
ProxyNodeMetricsStep::OneHour => "proxy_node_metrics_1h",
};
let rows = sqlx::query(&format!(
r#"
SELECT
bucket_start_unix_secs,
SUM(samples) AS samples,
SUM(uptime_samples) AS uptime_samples,
SUM(active_connections_sum) AS active_connections_sum,
MAX(active_connections_max) AS active_connections_max,
SUM(heartbeat_rtt_ms_sum) AS heartbeat_rtt_ms_sum,
MAX(heartbeat_rtt_ms_max) AS heartbeat_rtt_ms_max,
SUM(connect_errors_delta) AS connect_errors_delta,
SUM(disconnects_delta) AS disconnects_delta,
SUM(error_events_delta) AS error_events_delta,
SUM(ws_in_bytes_delta) AS ws_in_bytes_delta,
SUM(ws_out_bytes_delta) AS ws_out_bytes_delta,
SUM(ws_in_frames_delta) AS ws_in_frames_delta,
SUM(ws_out_frames_delta) AS ws_out_frames_delta
FROM {table}
WHERE bucket_start_unix_secs >= ?
AND bucket_start_unix_secs <= ?
GROUP BY bucket_start_unix_secs
ORDER BY bucket_start_unix_secs ASC
LIMIT ?
"#
))
.bind(i64::try_from(from_unix_secs).unwrap_or(i64::MAX))
.bind(i64::try_from(to_unix_secs).unwrap_or(i64::MAX))
.bind(i64::try_from(limit).unwrap_or(i64::MAX))
.fetch_all(&self.pool)
.await
.map_sql_err()?;
rows.iter().map(map_proxy_fleet_metric_row).collect()
}
}
#[async_trait]
@@ -540,7 +760,9 @@ WHERE is_manual = 0
));
}
let now = Some(current_unix_secs());
let previous_proxy_metadata = node.proxy_metadata.clone();
let now_unix_secs = current_unix_secs();
let now = Some(now_unix_secs);
node.last_heartbeat_at_unix_secs = now;
if node.status != "online" || !node.tunnel_connected {
node.status = "online".to_string();
@@ -584,7 +806,52 @@ WHERE is_manual = 0
node.config_version = node.config_version.saturating_add(1);
node.updated_at_unix_secs = now;
}
let tunnel_metrics_sample = build_tunnel_metrics_sample(
previous_proxy_metadata.as_ref(),
node.proxy_metadata.as_ref(),
node.active_connections,
node.tunnel_connected,
);
self.upsert_node(&node).await?;
if let Some(sample) = tunnel_metrics_sample.as_ref() {
self.upsert_metrics_bucket(
"proxy_node_metrics_1m",
&node.id,
bucket_start_unix_secs(now_unix_secs, ProxyNodeMetricsStep::OneMinute),
sample,
)
.await?;
self.upsert_metrics_bucket(
"proxy_node_metrics_1h",
&node.id,
bucket_start_unix_secs(now_unix_secs, ProxyNodeMetricsStep::OneHour),
sample,
)
.await?;
for error in &sample.recent_error_events {
let detail = build_tunnel_error_event_detail(error);
let event_metadata = serde_json::json!({
"source": "heartbeat",
"category": error.category,
"message": error.message,
"timestamp_unix_secs": error.timestamp_unix_secs,
});
self.insert_event(
&node.id,
PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR,
Some(detail.as_str()),
Some(&event_metadata),
Some(if error.timestamp_unix_secs == 0 {
now_unix_secs
} else {
error.timestamp_unix_secs
}),
)
.await?;
}
}
Ok(Some(node))
}
@@ -638,6 +905,7 @@ WHERE is_manual = 0
&mutation.node_id,
event_type,
Some(&format!("[stale_ignored] {event_detail}")),
None,
Some(current_unix_secs()),
)
.await?;
@@ -660,6 +928,7 @@ WHERE is_manual = 0
&mutation.node_id,
event_type,
Some(&event_detail),
None,
Some(event_time),
)
.await?;
@@ -691,6 +960,16 @@ WHERE is_manual = 0
.execute(&self.pool)
.await
.map_sql_err()?;
sqlx::query("DELETE FROM proxy_node_metrics_1m WHERE node_id = ?")
.bind(node_id)
.execute(&self.pool)
.await
.map_sql_err()?;
sqlx::query("DELETE FROM proxy_node_metrics_1h WHERE node_id = ?")
.bind(node_id)
.execute(&self.pool)
.await
.map_sql_err()?;
sqlx::query("DELETE FROM proxy_nodes WHERE id = ?")
.bind(node_id)
.execute(&self.pool)
@@ -747,6 +1026,33 @@ WHERE is_manual = 0
node.updated_at_unix_secs = Some(current_unix_secs());
self.upsert_node(&node).await
}
async fn cleanup_proxy_node_metrics(
&self,
retain_1m_from_unix_secs: u64,
retain_1h_from_unix_secs: u64,
) -> Result<ProxyNodeMetricsCleanupSummary, DataLayerError> {
let deleted_1m =
sqlx::query("DELETE FROM proxy_node_metrics_1m WHERE bucket_start_unix_secs < ?")
.bind(i64::try_from(retain_1m_from_unix_secs).unwrap_or(i64::MAX))
.execute(&self.pool)
.await
.map_sql_err()?
.rows_affected() as usize;
let deleted_1h =
sqlx::query("DELETE FROM proxy_node_metrics_1h WHERE bucket_start_unix_secs < ?")
.bind(i64::try_from(retain_1h_from_unix_secs).unwrap_or(i64::MAX))
.execute(&self.pool)
.await
.map_sql_err()?
.rows_affected() as usize;
Ok(ProxyNodeMetricsCleanupSummary {
deleted_1m_rows: deleted_1m,
deleted_1h_rows: deleted_1h,
})
}
}
fn optional_unix_secs(value: Option<i64>) -> Option<u64> {
@@ -861,10 +1167,63 @@ fn map_proxy_node_event_row(row: &MySqlRow) -> Result<StoredProxyNodeEvent, Data
node_id: row.try_get("node_id").map_sql_err()?,
event_type: row.try_get("event_type").map_sql_err()?,
detail: row.try_get("detail").map_sql_err()?,
event_metadata: optional_json_from_string(
row.try_get("event_metadata").map_sql_err()?,
"proxy_node_events.event_metadata",
)?,
created_at_unix_ms: optional_unix_secs(row.try_get("created_at_unix_ms").map_sql_err()?),
})
}
fn map_proxy_node_metric_row(
row: &MySqlRow,
) -> Result<StoredProxyNodeMetricsBucket, DataLayerError> {
Ok(StoredProxyNodeMetricsBucket {
node_id: row.try_get("node_id").map_sql_err()?,
bucket_start_unix_secs: optional_unix_secs(
row.try_get("bucket_start_unix_secs").map_sql_err()?,
)
.unwrap_or_default(),
samples: row.try_get("samples").map_sql_err()?,
uptime_samples: row.try_get("uptime_samples").map_sql_err()?,
active_connections_sum: row.try_get("active_connections_sum").map_sql_err()?,
active_connections_max: row.try_get("active_connections_max").map_sql_err()?,
heartbeat_rtt_ms_sum: row.try_get("heartbeat_rtt_ms_sum").map_sql_err()?,
heartbeat_rtt_ms_max: row.try_get("heartbeat_rtt_ms_max").map_sql_err()?,
connect_errors_delta: row.try_get("connect_errors_delta").map_sql_err()?,
disconnects_delta: row.try_get("disconnects_delta").map_sql_err()?,
error_events_delta: row.try_get("error_events_delta").map_sql_err()?,
ws_in_bytes_delta: row.try_get("ws_in_bytes_delta").map_sql_err()?,
ws_out_bytes_delta: row.try_get("ws_out_bytes_delta").map_sql_err()?,
ws_in_frames_delta: row.try_get("ws_in_frames_delta").map_sql_err()?,
ws_out_frames_delta: row.try_get("ws_out_frames_delta").map_sql_err()?,
})
}
fn map_proxy_fleet_metric_row(
row: &MySqlRow,
) -> Result<StoredProxyFleetMetricsBucket, DataLayerError> {
Ok(StoredProxyFleetMetricsBucket {
bucket_start_unix_secs: optional_unix_secs(
row.try_get("bucket_start_unix_secs").map_sql_err()?,
)
.unwrap_or_default(),
samples: row.try_get("samples").map_sql_err()?,
uptime_samples: row.try_get("uptime_samples").map_sql_err()?,
active_connections_sum: row.try_get("active_connections_sum").map_sql_err()?,
active_connections_max: row.try_get("active_connections_max").map_sql_err()?,
heartbeat_rtt_ms_sum: row.try_get("heartbeat_rtt_ms_sum").map_sql_err()?,
heartbeat_rtt_ms_max: row.try_get("heartbeat_rtt_ms_max").map_sql_err()?,
connect_errors_delta: row.try_get("connect_errors_delta").map_sql_err()?,
disconnects_delta: row.try_get("disconnects_delta").map_sql_err()?,
error_events_delta: row.try_get("error_events_delta").map_sql_err()?,
ws_in_bytes_delta: row.try_get("ws_in_bytes_delta").map_sql_err()?,
ws_out_bytes_delta: row.try_get("ws_out_bytes_delta").map_sql_err()?,
ws_in_frames_delta: row.try_get("ws_in_frames_delta").map_sql_err()?,
ws_out_frames_delta: row.try_get("ws_out_frames_delta").map_sql_err()?,
})
}
#[cfg(test)]
mod tests {
use super::MysqlProxyNodeReadRepository;