mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-05 08:57:46 +08:00
feat(proxy): surface node resource diagnostics
This commit is contained in:
@@ -158,6 +158,7 @@ pub async fn run(mut config: Config, servers: Vec<ServerEntry>) -> anyhow::Resul
|
||||
// Build a profile-keyed Hyper client pool for tunnel upstream requests.
|
||||
let upstream_client_pool =
|
||||
upstream_client::UpstreamClientPool::new(Arc::clone(&config), Arc::clone(&dns_cache));
|
||||
let resource_monitor = Arc::new(hardware::RuntimeResourceMonitor::new());
|
||||
|
||||
// Register with each Aether server and build per-server contexts.
|
||||
// Wrapped in Arc<Mutex> so retry_failed_registrations can append later.
|
||||
@@ -218,6 +219,7 @@ pub async fn run(mut config: Config, servers: Vec<ServerEntry>) -> anyhow::Resul
|
||||
dns_cache,
|
||||
upstream_client_pool,
|
||||
tunnel_tls_config,
|
||||
resource_monitor,
|
||||
stream_gate: None,
|
||||
distributed_stream_gate: None,
|
||||
};
|
||||
@@ -947,6 +949,7 @@ mod tests {
|
||||
dns_cache,
|
||||
upstream_client_pool,
|
||||
tunnel_tls_config: Arc::new(crate::tunnel::client::build_tls_config()),
|
||||
resource_monitor: Arc::new(crate::hardware::RuntimeResourceMonitor::new()),
|
||||
stream_gate: None,
|
||||
distributed_stream_gate: None,
|
||||
})
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
use std::sync::Mutex;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use serde::Serialize;
|
||||
use sysinfo::System;
|
||||
use sysinfo::{get_current_pid, Pid, ProcessesToUpdate, System};
|
||||
use tracing::info;
|
||||
|
||||
/// Hardware information collected at startup.
|
||||
@@ -17,6 +20,102 @@ pub struct HardwareInfo {
|
||||
pub estimated_max_concurrency: u64,
|
||||
}
|
||||
|
||||
/// Runtime resource usage sampled during heartbeat reporting.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct RuntimeResourceSnapshot {
|
||||
pub sampled_at_unix_secs: u64,
|
||||
pub system_cpu_usage_percent: f64,
|
||||
pub process_cpu_usage_percent: f64,
|
||||
pub memory_total_bytes: u64,
|
||||
pub memory_used_bytes: u64,
|
||||
pub memory_available_bytes: u64,
|
||||
pub memory_used_percent: f64,
|
||||
pub process_memory_bytes: u64,
|
||||
pub process_virtual_memory_bytes: u64,
|
||||
pub process_memory_percent: f64,
|
||||
pub load_average_1m: f64,
|
||||
pub load_average_5m: f64,
|
||||
pub load_average_15m: f64,
|
||||
pub system_uptime_secs: u64,
|
||||
pub process_uptime_secs: Option<u64>,
|
||||
}
|
||||
|
||||
/// Small, reusable sysinfo monitor. Keeping it alive between samples makes CPU
|
||||
/// usage deltas meaningful without re-enumerating the whole machine every time.
|
||||
pub struct RuntimeResourceMonitor {
|
||||
system: Mutex<System>,
|
||||
current_pid: Option<Pid>,
|
||||
}
|
||||
|
||||
impl RuntimeResourceMonitor {
|
||||
pub fn new() -> Self {
|
||||
let mut system = System::new_all();
|
||||
let current_pid = get_current_pid().ok();
|
||||
if let Some(pid) = current_pid {
|
||||
system.refresh_processes(ProcessesToUpdate::Some(&[pid]), true);
|
||||
}
|
||||
system.refresh_cpu_usage();
|
||||
system.refresh_memory();
|
||||
Self {
|
||||
system: Mutex::new(system),
|
||||
current_pid,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn snapshot(&self) -> RuntimeResourceSnapshot {
|
||||
let mut system = match self.system.lock() {
|
||||
Ok(guard) => guard,
|
||||
Err(poisoned) => poisoned.into_inner(),
|
||||
};
|
||||
|
||||
system.refresh_cpu_usage();
|
||||
system.refresh_memory();
|
||||
if let Some(pid) = self.current_pid {
|
||||
system.refresh_processes(ProcessesToUpdate::Some(&[pid]), true);
|
||||
}
|
||||
|
||||
let memory_total_bytes = system.total_memory();
|
||||
let memory_used_bytes = system.used_memory();
|
||||
let memory_available_bytes = system.available_memory();
|
||||
let (
|
||||
process_cpu_usage_percent,
|
||||
process_memory_bytes,
|
||||
process_virtual_memory_bytes,
|
||||
process_uptime_secs,
|
||||
) = self
|
||||
.current_pid
|
||||
.and_then(|pid| system.process(pid))
|
||||
.map(|process| {
|
||||
(
|
||||
process.cpu_usage() as f64,
|
||||
process.memory(),
|
||||
process.virtual_memory(),
|
||||
Some(process.run_time()),
|
||||
)
|
||||
})
|
||||
.unwrap_or((0.0, 0, 0, None));
|
||||
let load = System::load_average();
|
||||
|
||||
RuntimeResourceSnapshot {
|
||||
sampled_at_unix_secs: current_unix_secs(),
|
||||
system_cpu_usage_percent: system.global_cpu_usage() as f64,
|
||||
process_cpu_usage_percent,
|
||||
memory_total_bytes,
|
||||
memory_used_bytes,
|
||||
memory_available_bytes,
|
||||
memory_used_percent: ratio_percent(memory_used_bytes, memory_total_bytes),
|
||||
process_memory_bytes,
|
||||
process_virtual_memory_bytes,
|
||||
process_memory_percent: ratio_percent(process_memory_bytes, memory_total_bytes),
|
||||
load_average_1m: load.one,
|
||||
load_average_5m: load.five,
|
||||
load_average_15m: load.fifteen,
|
||||
system_uptime_secs: System::uptime(),
|
||||
process_uptime_secs,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Collect hardware information and estimate max concurrency.
|
||||
///
|
||||
/// Should be called once at startup -- hardware does not change at runtime.
|
||||
@@ -77,3 +176,18 @@ fn get_fd_limit() -> u64 {
|
||||
// Fallback for non-unix or error
|
||||
1024
|
||||
}
|
||||
|
||||
fn ratio_percent(value: u64, total: u64) -> f64 {
|
||||
if total == 0 {
|
||||
0.0
|
||||
} else {
|
||||
value as f64 * 100.0 / total as f64
|
||||
}
|
||||
}
|
||||
|
||||
fn current_unix_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|duration| duration.as_secs())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ use aether_runtime::{AdmissionPermit, ConcurrencyError, ConcurrencyGate, Concurr
|
||||
use aether_runtime_state::{RuntimeSemaphore, RuntimeSemaphoreError, RuntimeSemaphoreSnapshot};
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::hardware::RuntimeResourceMonitor;
|
||||
use crate::registration::client::AetherClient;
|
||||
use crate::runtime::SharedDynamicConfig;
|
||||
use crate::target_filter::DnsCache;
|
||||
@@ -24,6 +25,8 @@ pub struct AppState {
|
||||
pub upstream_client_pool: UpstreamClientPool,
|
||||
/// Shared TLS config for tunnel WebSocket connections (avoids re-parsing root CAs on each reconnect).
|
||||
pub tunnel_tls_config: Arc<rustls::ClientConfig>,
|
||||
/// Runtime CPU/memory monitor sampled by heartbeat payloads.
|
||||
pub resource_monitor: Arc<RuntimeResourceMonitor>,
|
||||
/// Optional per-process stream admission gate.
|
||||
pub stream_gate: Option<Arc<ConcurrencyGate>>,
|
||||
/// Optional cross-instance stream admission gate.
|
||||
@@ -96,6 +99,10 @@ pub struct TunnelErrorEvent {
|
||||
pub timestamp_unix_secs: u64,
|
||||
pub category: String,
|
||||
pub message: String,
|
||||
pub severity: String,
|
||||
pub component: String,
|
||||
pub summary: String,
|
||||
pub operator_action: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
@@ -229,11 +236,18 @@ impl TunnelMetrics {
|
||||
|
||||
pub fn record_error(&self, category: &str, message: &str) {
|
||||
self.error_events_total.fetch_add(1, Ordering::Release);
|
||||
let category = normalize_error_field(category, TUNNEL_ERROR_CATEGORY_MAX_CHARS, "unknown");
|
||||
let message = normalize_error_field(message, TUNNEL_ERROR_MESSAGE_MAX_CHARS, "n/a");
|
||||
let diagnostic = classify_tunnel_error(category.as_str(), message.as_str());
|
||||
|
||||
let event = TunnelErrorEvent {
|
||||
timestamp_unix_secs: now_unix_secs(),
|
||||
category: normalize_error_field(category, TUNNEL_ERROR_CATEGORY_MAX_CHARS, "unknown"),
|
||||
message: normalize_error_field(message, TUNNEL_ERROR_MESSAGE_MAX_CHARS, "n/a"),
|
||||
category,
|
||||
message,
|
||||
severity: diagnostic.severity.to_string(),
|
||||
component: diagnostic.component.to_string(),
|
||||
summary: diagnostic.summary.to_string(),
|
||||
operator_action: diagnostic.operator_action.to_string(),
|
||||
};
|
||||
|
||||
let mut recent_errors = match self.recent_errors.lock() {
|
||||
@@ -302,6 +316,95 @@ fn normalize_error_field(value: &str, max_chars: usize, fallback: &str) -> Strin
|
||||
normalized.chars().take(max_chars).collect()
|
||||
}
|
||||
|
||||
struct TunnelErrorDiagnostic {
|
||||
severity: &'static str,
|
||||
component: &'static str,
|
||||
summary: &'static str,
|
||||
operator_action: &'static str,
|
||||
}
|
||||
|
||||
fn classify_tunnel_error(category: &str, _message: &str) -> TunnelErrorDiagnostic {
|
||||
match category {
|
||||
"stale_timeout" => TunnelErrorDiagnostic {
|
||||
severity: "warning",
|
||||
component: "tunnel_read",
|
||||
summary: "No inbound tunnel frames before stale timeout",
|
||||
operator_action:
|
||||
"Check gateway or reverse-proxy idle timeouts, packet loss, and WebSocket ping/pong reachability. Increase AETHER_PROXY_TUNNEL_STALE_TIMEOUT_MS if the network is high-latency.",
|
||||
},
|
||||
"ws_write_error" => TunnelErrorDiagnostic {
|
||||
severity: "error",
|
||||
component: "tunnel_write",
|
||||
summary: "WebSocket write failed because the peer closed or reset the connection",
|
||||
operator_action:
|
||||
"Check gateway restarts, load balancer resets, NAT/firewall connection tracking, and whether the proxy is reconnecting successfully.",
|
||||
},
|
||||
"ws_ping_error" => TunnelErrorDiagnostic {
|
||||
severity: "error",
|
||||
component: "tunnel_write",
|
||||
summary: "WebSocket keepalive ping could not be sent",
|
||||
operator_action:
|
||||
"Check whether the peer closed the socket or an intermediary is dropping idle WebSocket connections.",
|
||||
},
|
||||
"ws_read_error" => TunnelErrorDiagnostic {
|
||||
severity: "error",
|
||||
component: "tunnel_read",
|
||||
summary: "WebSocket read failed",
|
||||
operator_action:
|
||||
"Check gateway logs and network stability around the same timestamp; compare with reconnect and heartbeat ACK counters.",
|
||||
},
|
||||
"tunnel_connect_error" => TunnelErrorDiagnostic {
|
||||
severity: "critical",
|
||||
component: "tunnel_connect",
|
||||
summary: "Tunnel connection attempt failed",
|
||||
operator_action:
|
||||
"Check Aether URL reachability, DNS, TLS, management token validity, and any configured AETHER_PROXY_AETHER_PROXY_URL.",
|
||||
},
|
||||
"frame_decode_error" => TunnelErrorDiagnostic {
|
||||
severity: "error",
|
||||
component: "tunnel_protocol",
|
||||
summary: "Received tunnel frame could not be decoded",
|
||||
operator_action:
|
||||
"Check proxy and gateway version compatibility and whether traffic is being modified by an intermediary.",
|
||||
},
|
||||
"stream_dispatch_timeout" => TunnelErrorDiagnostic {
|
||||
severity: "warning",
|
||||
component: "stream_dispatch",
|
||||
summary: "Request body frame could not be delivered to its stream handler in time",
|
||||
operator_action:
|
||||
"Check proxy CPU, memory, stream concurrency saturation, and slow upstream provider requests.",
|
||||
},
|
||||
"heartbeat_ack_empty" | "heartbeat_ack_parse" => TunnelErrorDiagnostic {
|
||||
severity: "warning",
|
||||
component: "heartbeat",
|
||||
summary: "Heartbeat ACK from gateway was missing or invalid",
|
||||
operator_action:
|
||||
"Check gateway heartbeat handler logs and proxy/gateway version compatibility.",
|
||||
},
|
||||
"writer_task_panic" | "writer_task_cancelled" => TunnelErrorDiagnostic {
|
||||
severity: "error",
|
||||
component: "tunnel_writer",
|
||||
summary: "Tunnel writer task exited unexpectedly",
|
||||
operator_action:
|
||||
"Check proxy logs for the preceding write or ping error and confirm the tunnel reconnect loop is active.",
|
||||
},
|
||||
"dispatcher_error" => TunnelErrorDiagnostic {
|
||||
severity: "error",
|
||||
component: "tunnel_dispatcher",
|
||||
summary: "Tunnel dispatcher exited with an error",
|
||||
operator_action:
|
||||
"Check the proxied request stream and gateway tunnel logs around the same timestamp.",
|
||||
},
|
||||
_ => TunnelErrorDiagnostic {
|
||||
severity: "info",
|
||||
component: "tunnel",
|
||||
summary: "Tunnel reported an unclassified error",
|
||||
operator_action:
|
||||
"Inspect the raw message and compare it with proxy, gateway, and network logs at the same time.",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
|
||||
pub enum ProxyAdmissionError {
|
||||
#[error("proxy stream admission saturated at {limit} for gate {gate}")]
|
||||
|
||||
@@ -240,6 +240,7 @@ async fn build_heartbeat_payload(
|
||||
let node_id = server.node_id.read().unwrap().clone();
|
||||
let tunnel_snapshot = server.tunnel_metrics.snapshot();
|
||||
let recent_errors = server.tunnel_metrics.recent_errors(8);
|
||||
let resource_usage = state.resource_monitor.snapshot();
|
||||
|
||||
let avg_latency_ms = if snapshot.requests > 0 {
|
||||
Some(snapshot.latency_ns as f64 / snapshot.requests as f64 / 1_000_000.0)
|
||||
@@ -291,6 +292,7 @@ async fn build_heartbeat_payload(
|
||||
"proxy_metadata": {
|
||||
"version": CURRENT_VERSION,
|
||||
"admission": admission,
|
||||
"resource_usage": resource_usage,
|
||||
"tunnel_metrics": {
|
||||
"connect_attempts": tunnel_snapshot.connect_attempts,
|
||||
"connect_successes": tunnel_snapshot.connect_successes,
|
||||
@@ -419,13 +421,13 @@ mod tests {
|
||||
use arc_swap::ArcSwap;
|
||||
use clap::Parser;
|
||||
|
||||
use super::{handle_ack, AckDecision};
|
||||
use super::{build_heartbeat_payload, handle_ack, AckDecision, HeartbeatSnapshot};
|
||||
use crate::registration::client::AetherClient;
|
||||
use crate::runtime::DynamicConfig;
|
||||
use crate::state::{ProxyMetrics, ServerContext, TunnelMetrics};
|
||||
use crate::state::{AppState, ProxyMetrics, ServerContext, TunnelMetrics};
|
||||
|
||||
fn sample_server() -> Arc<ServerContext> {
|
||||
let config = Arc::new(crate::config::Config::parse_from([
|
||||
fn sample_config() -> Arc<crate::config::Config> {
|
||||
Arc::new(crate::config::Config::parse_from([
|
||||
"aether-proxy",
|
||||
"--aether-url",
|
||||
"https://example.com",
|
||||
@@ -433,7 +435,11 @@ mod tests {
|
||||
"ae_test",
|
||||
"--node-name",
|
||||
"proxy-test",
|
||||
]));
|
||||
]))
|
||||
}
|
||||
|
||||
fn sample_server() -> Arc<ServerContext> {
|
||||
let config = sample_config();
|
||||
Arc::new(ServerContext {
|
||||
server_label: "heartbeat-test".to_string(),
|
||||
aether_url: config.aether_url.clone(),
|
||||
@@ -452,6 +458,24 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
fn sample_state(config: Arc<crate::config::Config>) -> AppState {
|
||||
let dns_cache = Arc::new(crate::target_filter::DnsCache::new(
|
||||
std::time::Duration::from_secs(config.dns_cache_ttl_secs),
|
||||
config.dns_cache_capacity,
|
||||
));
|
||||
AppState {
|
||||
config: Arc::clone(&config),
|
||||
dns_cache: Arc::clone(&dns_cache),
|
||||
upstream_client_pool: crate::upstream_client::UpstreamClientPool::new(
|
||||
config, dns_cache,
|
||||
),
|
||||
tunnel_tls_config: Arc::new(crate::tunnel::client::build_tls_config()),
|
||||
resource_monitor: Arc::new(crate::hardware::RuntimeResourceMonitor::new()),
|
||||
stream_gate: None,
|
||||
distributed_stream_gate: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heartbeat_ack_requires_heartbeat_id() {
|
||||
let server = sample_server();
|
||||
@@ -481,4 +505,48 @@ mod tests {
|
||||
));
|
||||
assert_eq!(server.dynamic.load().heartbeat_interval, 9);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heartbeat_payload_reports_resource_usage_and_tunnel_error_diagnostics() {
|
||||
let config = sample_config();
|
||||
let server = sample_server();
|
||||
server
|
||||
.tunnel_metrics
|
||||
.record_error("ws_write_error", "IO error: Connection reset by peer");
|
||||
let state = sample_state(config);
|
||||
|
||||
let payload = build_heartbeat_payload(
|
||||
&state,
|
||||
&server,
|
||||
"session-1",
|
||||
42,
|
||||
HeartbeatSnapshot::default(),
|
||||
)
|
||||
.await;
|
||||
let payload: serde_json::Value =
|
||||
serde_json::from_slice(&payload).expect("heartbeat payload should be JSON");
|
||||
let resource_usage = payload
|
||||
.pointer("/proxy_metadata/resource_usage")
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.expect("resource usage should be reported");
|
||||
assert!(resource_usage.contains_key("system_cpu_usage_percent"));
|
||||
assert!(resource_usage.contains_key("process_memory_bytes"));
|
||||
|
||||
let recent_error = payload
|
||||
.pointer("/proxy_metadata/recent_tunnel_errors/0")
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.expect("recent tunnel error should be reported");
|
||||
assert_eq!(
|
||||
recent_error
|
||||
.get("component")
|
||||
.and_then(serde_json::Value::as_str),
|
||||
Some("tunnel_write")
|
||||
);
|
||||
assert_eq!(
|
||||
recent_error
|
||||
.get("severity")
|
||||
.and_then(serde_json::Value::as_str),
|
||||
Some("error")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -462,6 +462,7 @@ mod tests {
|
||||
dns_cache,
|
||||
upstream_client_pool,
|
||||
tunnel_tls_config: Arc::new(crate::tunnel::client::build_tls_config()),
|
||||
resource_monitor: Arc::new(crate::hardware::RuntimeResourceMonitor::new()),
|
||||
stream_gate: None,
|
||||
distributed_stream_gate: None,
|
||||
})
|
||||
|
||||
@@ -2284,6 +2284,7 @@ mod tests {
|
||||
dns_cache,
|
||||
upstream_client_pool,
|
||||
tunnel_tls_config: Arc::new(build_tls_config()),
|
||||
resource_monitor: Arc::new(crate::hardware::RuntimeResourceMonitor::new()),
|
||||
stream_gate,
|
||||
distributed_stream_gate,
|
||||
})
|
||||
@@ -2314,6 +2315,7 @@ mod tests {
|
||||
dns_cache,
|
||||
upstream_client_pool,
|
||||
tunnel_tls_config: Arc::new(build_tls_config()),
|
||||
resource_monitor: Arc::new(crate::hardware::RuntimeResourceMonitor::new()),
|
||||
stream_gate: None,
|
||||
distributed_stream_gate: None,
|
||||
})
|
||||
|
||||
@@ -650,6 +650,10 @@ impl ProxyNodeWriteRepository for InMemoryProxyNodeRepository {
|
||||
"source": "heartbeat",
|
||||
"category": error.category,
|
||||
"message": error.message,
|
||||
"severity": error.severity.as_deref(),
|
||||
"component": error.component.as_deref(),
|
||||
"summary": error.summary.as_deref(),
|
||||
"operator_action": error.operator_action.as_deref(),
|
||||
"timestamp_unix_secs": error.timestamp_unix_secs,
|
||||
})),
|
||||
created_at_unix_ms: Some(if error.timestamp_unix_secs == 0 {
|
||||
|
||||
@@ -836,6 +836,10 @@ WHERE is_manual = 0
|
||||
"source": "heartbeat",
|
||||
"category": error.category,
|
||||
"message": error.message,
|
||||
"severity": error.severity.as_deref(),
|
||||
"component": error.component.as_deref(),
|
||||
"summary": error.summary.as_deref(),
|
||||
"operator_action": error.operator_action.as_deref(),
|
||||
"timestamp_unix_secs": error.timestamp_unix_secs,
|
||||
});
|
||||
self.insert_event(
|
||||
|
||||
@@ -1224,6 +1224,10 @@ impl ProxyNodeWriteRepository for SqlxProxyNodeRepository {
|
||||
"source": "heartbeat",
|
||||
"category": error.category,
|
||||
"message": error.message,
|
||||
"severity": error.severity.as_deref(),
|
||||
"component": error.component.as_deref(),
|
||||
"summary": error.summary.as_deref(),
|
||||
"operator_action": error.operator_action.as_deref(),
|
||||
"timestamp_unix_secs": error.timestamp_unix_secs,
|
||||
});
|
||||
self.insert_event(
|
||||
|
||||
@@ -838,6 +838,10 @@ WHERE is_manual = 0
|
||||
"source": "heartbeat",
|
||||
"category": error.category,
|
||||
"message": error.message,
|
||||
"severity": error.severity.as_deref(),
|
||||
"component": error.component.as_deref(),
|
||||
"summary": error.summary.as_deref(),
|
||||
"operator_action": error.operator_action.as_deref(),
|
||||
"timestamp_unix_secs": error.timestamp_unix_secs,
|
||||
});
|
||||
self.insert_event(
|
||||
|
||||
@@ -324,6 +324,10 @@ pub struct TunnelErrorEventRecord {
|
||||
pub timestamp_unix_secs: u64,
|
||||
pub category: String,
|
||||
pub message: String,
|
||||
pub severity: Option<String>,
|
||||
pub component: Option<String>,
|
||||
pub summary: Option<String>,
|
||||
pub operator_action: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
@@ -417,7 +421,11 @@ pub fn build_tunnel_metrics_sample(
|
||||
}
|
||||
|
||||
pub fn build_tunnel_error_event_detail(event: &TunnelErrorEventRecord) -> String {
|
||||
format!("[{}] {}", event.category, event.message)
|
||||
format!(
|
||||
"[{}] {}",
|
||||
event.category,
|
||||
event.summary.as_deref().unwrap_or(event.message.as_str())
|
||||
)
|
||||
}
|
||||
|
||||
pub fn normalize_proxy_metadata(
|
||||
@@ -497,6 +505,10 @@ fn extract_recent_tunnel_errors(proxy_metadata: Option<&Value>) -> Vec<TunnelErr
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("n/a")
|
||||
.to_string(),
|
||||
severity: json_string(item.get("severity")),
|
||||
component: json_string(item.get("component")),
|
||||
summary: json_string(item.get("summary")),
|
||||
operator_action: json_string(item.get("operator_action")),
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
@@ -512,6 +524,14 @@ fn json_u64(value: Option<&Value>) -> Option<u64> {
|
||||
})
|
||||
}
|
||||
|
||||
fn json_string(value: Option<&Value>) -> Option<String> {
|
||||
value
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToString::to_string)
|
||||
}
|
||||
|
||||
fn counter_delta_u64(previous: Option<u64>, current: u64) -> u64 {
|
||||
match previous {
|
||||
Some(previous) if current >= previous => current - previous,
|
||||
@@ -726,10 +746,11 @@ mod tests {
|
||||
use serde_json::json;
|
||||
|
||||
use super::{
|
||||
bucket_start_unix_secs, build_tunnel_metrics_sample, normalize_proxy_node_scheduling_state,
|
||||
proxy_node_accepts_new_tunnels, proxy_reported_version,
|
||||
reconcile_remote_config_after_heartbeat, remote_config_scheduling_state,
|
||||
remote_config_upgrade_target, ProxyNodeMetricsStep, StoredProxyNode,
|
||||
bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
|
||||
normalize_proxy_node_scheduling_state, proxy_node_accepts_new_tunnels,
|
||||
proxy_reported_version, reconcile_remote_config_after_heartbeat,
|
||||
remote_config_scheduling_state, remote_config_upgrade_target, ProxyNodeMetricsStep,
|
||||
StoredProxyNode,
|
||||
};
|
||||
|
||||
#[test]
|
||||
@@ -842,7 +863,15 @@ mod tests {
|
||||
},
|
||||
"recent_tunnel_errors": [
|
||||
{"timestamp_unix_secs": 100, "category": "older", "message": "old"},
|
||||
{"timestamp_unix_secs": 101, "category": "newer", "message": "new"}
|
||||
{
|
||||
"timestamp_unix_secs": 101,
|
||||
"category": "newer",
|
||||
"message": "new",
|
||||
"severity": "error",
|
||||
"component": "tunnel_write",
|
||||
"summary": "WebSocket write failed because the peer closed or reset the connection",
|
||||
"operator_action": "Check gateway restarts and network resets."
|
||||
}
|
||||
]
|
||||
});
|
||||
|
||||
@@ -860,6 +889,19 @@ mod tests {
|
||||
assert_eq!(sample.ws_out_frames_delta, 3);
|
||||
assert_eq!(sample.recent_error_events.len(), 2);
|
||||
assert_eq!(sample.recent_error_events[0].category, "older");
|
||||
assert_eq!(sample.recent_error_events[0].summary, None);
|
||||
assert_eq!(
|
||||
sample.recent_error_events[1].severity.as_deref(),
|
||||
Some("error")
|
||||
);
|
||||
assert_eq!(
|
||||
sample.recent_error_events[1].component.as_deref(),
|
||||
Some("tunnel_write")
|
||||
);
|
||||
assert_eq!(
|
||||
build_tunnel_error_event_detail(&sample.recent_error_events[1]),
|
||||
"[newer] WebSocket write failed because the peer closed or reset the connection"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -138,7 +138,10 @@
|
||||
>
|
||||
{{ eventTypeLabel(event.event_type) }}
|
||||
</Badge>
|
||||
<span class="text-muted-foreground truncate flex-1">{{ eventDetail(event) }}</span>
|
||||
<span
|
||||
class="text-muted-foreground truncate flex-1"
|
||||
:title="eventTooltip(event)"
|
||||
>{{ eventDetail(event) }}</span>
|
||||
<span class="text-[10px] text-muted-foreground/70 tabular-nums shrink-0">{{ formatTime(event.created_at || null) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
@@ -146,6 +149,35 @@
|
||||
</div>
|
||||
|
||||
<div class="grid gap-4 lg:grid-cols-2">
|
||||
<section class="rounded-lg border border-border/50 bg-background/70 p-3 min-w-0">
|
||||
<h5 class="text-xs font-semibold mb-3">
|
||||
硬件与资源
|
||||
</h5>
|
||||
<div class="grid grid-cols-2 sm:grid-cols-3 gap-x-4 gap-y-2">
|
||||
<div
|
||||
v-for="item in resourceItems"
|
||||
:key="item.label"
|
||||
class="min-w-0"
|
||||
>
|
||||
<div class="text-[11px] text-muted-foreground truncate">
|
||||
{{ item.label }}
|
||||
</div>
|
||||
<div
|
||||
class="text-xs font-medium tabular-nums truncate mt-0.5"
|
||||
:class="item.tone"
|
||||
>
|
||||
{{ item.value }}
|
||||
</div>
|
||||
<div
|
||||
v-if="item.hint"
|
||||
class="text-[10px] text-muted-foreground truncate mt-0.5"
|
||||
>
|
||||
{{ item.hint }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="rounded-lg border border-border/50 bg-background/70 p-3 min-w-0">
|
||||
<h5 class="text-xs font-semibold mb-3">
|
||||
实时快照
|
||||
@@ -234,6 +266,8 @@ const metricsSummary = computed(() => metrics.value?.summary ?? null)
|
||||
const bucketItems = computed(() => metrics.value?.items.slice(-24) ?? [])
|
||||
const recentEvents = computed(() => props.state?.events.slice(0, 8) ?? [])
|
||||
const metadata = computed(() => asRecord(detailNode.value.proxy_metadata))
|
||||
const hardwareInfo = computed(() => asRecord(detailNode.value.hardware_info))
|
||||
const resourceUsage = computed(() => asRecord(metadata.value?.resource_usage))
|
||||
const tunnelMetrics = computed(() => asRecord(metadata.value?.tunnel_metrics))
|
||||
|
||||
const loadedText = computed(() => {
|
||||
@@ -300,6 +334,82 @@ const snapshotItems = computed(() => {
|
||||
]
|
||||
})
|
||||
|
||||
const resourceItems = computed(() => {
|
||||
const hardware = hardwareInfo.value
|
||||
const resource = resourceUsage.value
|
||||
const memoryTotalBytes = numberField(resource, 'memory_total_bytes') ?? memoryMbToBytes(numberField(hardware, 'total_memory_mb'))
|
||||
const memoryUsedBytes = numberField(resource, 'memory_used_bytes')
|
||||
const memoryUsedPercent = numberField(resource, 'memory_used_percent') ?? percentFromBytes(memoryUsedBytes, memoryTotalBytes)
|
||||
const processMemoryBytes = numberField(resource, 'process_memory_bytes')
|
||||
const processMemoryPercent = numberField(resource, 'process_memory_percent')
|
||||
const systemCpu = numberField(resource, 'system_cpu_usage_percent')
|
||||
const processCpu = numberField(resource, 'process_cpu_usage_percent')
|
||||
const load1 = numberField(resource, 'load_average_1m')
|
||||
const load5 = numberField(resource, 'load_average_5m')
|
||||
const load15 = numberField(resource, 'load_average_15m')
|
||||
const cpuCores = numberField(hardware, 'cpu_cores')
|
||||
return [
|
||||
{
|
||||
label: '系统 CPU',
|
||||
value: formatUsagePercent(systemCpu),
|
||||
hint: '主机整体',
|
||||
tone: usageTone(systemCpu, 70, 90),
|
||||
},
|
||||
{
|
||||
label: '系统内存',
|
||||
value: formatUsagePercent(memoryUsedPercent),
|
||||
hint: memoryUsedBytes == null || memoryTotalBytes == null
|
||||
? ''
|
||||
: `${formatBytes(memoryUsedBytes)} / ${formatBytes(memoryTotalBytes)}`,
|
||||
tone: usageTone(memoryUsedPercent, 75, 90),
|
||||
},
|
||||
{
|
||||
label: 'Proxy CPU',
|
||||
value: formatUsagePercent(processCpu),
|
||||
hint: '当前进程',
|
||||
tone: usageTone(processCpu, 70, 90),
|
||||
},
|
||||
{
|
||||
label: 'Proxy 内存',
|
||||
value: processMemoryBytes == null ? '-' : formatBytes(processMemoryBytes),
|
||||
hint: processMemoryPercent == null ? '' : formatUsagePercent(processMemoryPercent),
|
||||
tone: usageTone(processMemoryPercent, 5, 15),
|
||||
},
|
||||
{
|
||||
label: '负载',
|
||||
value: formatLoadAverage(load1, load5, load15),
|
||||
hint: cpuCores == null ? '1/5/15m' : `${formatNumber(cpuCores)} 核`,
|
||||
tone: loadTone(load1, cpuCores),
|
||||
},
|
||||
{
|
||||
label: '运行时间',
|
||||
value: formatDurationSeconds(numberField(resource, 'process_uptime_secs')),
|
||||
hint: 'proxy 进程',
|
||||
tone: '',
|
||||
},
|
||||
{
|
||||
label: 'CPU 核心',
|
||||
value: cpuCores == null ? '-' : formatNumber(cpuCores),
|
||||
hint: stringField(hardware, 'os_info') || '',
|
||||
tone: '',
|
||||
},
|
||||
{
|
||||
label: '物理内存',
|
||||
value: memoryTotalBytes == null ? '-' : formatBytes(memoryTotalBytes),
|
||||
hint: '启动采集',
|
||||
tone: '',
|
||||
},
|
||||
{
|
||||
label: 'FD 限制',
|
||||
value: formatOptionalNumber(numberField(hardware, 'fd_limit')),
|
||||
hint: detailNode.value.estimated_max_concurrency == null
|
||||
? ''
|
||||
: `容量 ${formatNumber(detailNode.value.estimated_max_concurrency)}`,
|
||||
tone: '',
|
||||
},
|
||||
]
|
||||
})
|
||||
|
||||
const tunnelCounterItems = computed(() => [
|
||||
{ label: '建连尝试', value: formatTunnelNumber('connect_attempts') },
|
||||
{ label: '建连成功', value: formatTunnelNumber('connect_successes') },
|
||||
@@ -352,6 +462,10 @@ function formatNumber(value: number) {
|
||||
return String(Math.round(value))
|
||||
}
|
||||
|
||||
function formatOptionalNumber(value: number | null) {
|
||||
return value == null ? '-' : formatNumber(value)
|
||||
}
|
||||
|
||||
function formatDecimal(value: number | null) {
|
||||
if (value == null || !Number.isFinite(value)) return '-'
|
||||
if (value === 0) return '0'
|
||||
@@ -364,12 +478,24 @@ function formatPercent(value: number | null) {
|
||||
return `${(value * 100).toFixed(value >= 0.995 || value === 0 ? 0 : 1)}%`
|
||||
}
|
||||
|
||||
function formatUsagePercent(value: number | null) {
|
||||
if (value == null || !Number.isFinite(value)) return '-'
|
||||
if (value >= 100) return `${Math.round(value)}%`
|
||||
if (value >= 10) return `${value.toFixed(0)}%`
|
||||
return `${value.toFixed(1)}%`
|
||||
}
|
||||
|
||||
function formatMs(value: number | null) {
|
||||
if (value == null || !Number.isFinite(value) || value <= 0) return '-'
|
||||
if (value >= 1000) return `${(value / 1000).toFixed(1)}s`
|
||||
return `${Math.round(value)}ms`
|
||||
}
|
||||
|
||||
function formatDurationSeconds(value: number | null) {
|
||||
if (value == null || !Number.isFinite(value) || value <= 0) return '-'
|
||||
return formatDurationMs(value * 1000)
|
||||
}
|
||||
|
||||
function formatDurationMs(value: number | null) {
|
||||
if (value == null || !Number.isFinite(value) || value <= 0) return '-'
|
||||
const totalSeconds = Math.floor(value / 1000)
|
||||
@@ -399,6 +525,38 @@ function formatBytes(value: number) {
|
||||
return `${unit === 0 ? Math.round(size) : size.toFixed(1)} ${units[unit]}`
|
||||
}
|
||||
|
||||
function formatLoadAverage(one: number | null, five: number | null, fifteen: number | null) {
|
||||
if (one == null && five == null && fifteen == null) return '-'
|
||||
return [one, five, fifteen]
|
||||
.map(value => (value == null || !Number.isFinite(value)) ? '-' : value.toFixed(value >= 10 ? 0 : 2))
|
||||
.join(' / ')
|
||||
}
|
||||
|
||||
function memoryMbToBytes(value: number | null) {
|
||||
if (value == null || !Number.isFinite(value) || value <= 0) return null
|
||||
return value * 1024 * 1024
|
||||
}
|
||||
|
||||
function percentFromBytes(value: number | null, total: number | null) {
|
||||
if (value == null || total == null || !Number.isFinite(value) || !Number.isFinite(total) || total <= 0) return null
|
||||
return value * 100 / total
|
||||
}
|
||||
|
||||
function usageTone(value: number | null, warnAt: number, badAt: number) {
|
||||
if (value == null || !Number.isFinite(value)) return ''
|
||||
if (value >= badAt) return 'text-destructive'
|
||||
if (value >= warnAt) return 'text-yellow-600 dark:text-yellow-400'
|
||||
return ''
|
||||
}
|
||||
|
||||
function loadTone(load: number | null, cpuCores: number | null) {
|
||||
if (load == null || cpuCores == null || !Number.isFinite(load) || !Number.isFinite(cpuCores) || cpuCores <= 0) return ''
|
||||
const ratio = load / cpuCores
|
||||
if (ratio >= 1.5) return 'text-destructive'
|
||||
if (ratio >= 1) return 'text-yellow-600 dark:text-yellow-400'
|
||||
return ''
|
||||
}
|
||||
|
||||
function formatTime(iso: string | null) {
|
||||
if (!iso) return '-'
|
||||
const date = new Date(iso)
|
||||
@@ -473,6 +631,63 @@ function eventTypeVariant(type: string) {
|
||||
|
||||
function eventDetail(event: ProxyNodeEvent) {
|
||||
const metadata = asRecord(event.event_metadata)
|
||||
if (event.event_type === 'tunnel_err') {
|
||||
const category = stringField(metadata, 'category')
|
||||
const message = stringField(metadata, 'message')
|
||||
const summary = (category ? tunnelErrorSummary(category) : '') || stringField(metadata, 'summary') || category
|
||||
if (summary && message) return `${summary}:${message}`
|
||||
return summary || message || event.detail || '-'
|
||||
}
|
||||
return event.detail || stringField(metadata, 'message') || stringField(metadata, 'category') || '-'
|
||||
}
|
||||
|
||||
function eventTooltip(event: ProxyNodeEvent) {
|
||||
const metadata = asRecord(event.event_metadata)
|
||||
const parts = [eventDetail(event)]
|
||||
const category = stringField(metadata, 'category')
|
||||
const action = (category ? tunnelErrorAction(category) : '') || stringField(metadata, 'operator_action')
|
||||
if (action) parts.push(`建议:${action}`)
|
||||
const component = stringField(metadata, 'component')
|
||||
const severity = stringField(metadata, 'severity')
|
||||
if (component || severity) parts.push([component, severity].filter(Boolean).join(' / '))
|
||||
return parts.filter(Boolean).join('\n')
|
||||
}
|
||||
|
||||
function tunnelErrorSummary(category: string) {
|
||||
switch (category) {
|
||||
case 'stale_timeout': return '超时未收到隧道入站帧'
|
||||
case 'ws_write_error': return 'WebSocket 写入失败,对端重置或关闭'
|
||||
case 'ws_ping_error': return 'WebSocket 保活 Ping 发送失败'
|
||||
case 'ws_read_error': return 'WebSocket 读取失败'
|
||||
case 'tunnel_connect_error': return '隧道建连失败'
|
||||
case 'frame_decode_error': return '隧道帧解析失败'
|
||||
case 'stream_dispatch_timeout': return '请求流分发超时'
|
||||
case 'heartbeat_ack_empty': return '心跳确认为空'
|
||||
case 'heartbeat_ack_parse': return '心跳确认解析失败'
|
||||
case 'writer_task_panic': return '隧道写任务异常退出'
|
||||
case 'writer_task_cancelled': return '隧道写任务被取消'
|
||||
case 'dispatcher_error': return '隧道分发器错误'
|
||||
default: return ''
|
||||
}
|
||||
}
|
||||
|
||||
function tunnelErrorAction(category: string) {
|
||||
switch (category) {
|
||||
case 'stale_timeout': return '检查 gateway/反向代理 idle timeout、跨境链路抖动和 ping/pong 是否可达;网络抖动大时可调高 stale timeout。'
|
||||
case 'ws_write_error': return '检查 gateway 是否重启、负载均衡/NAT/防火墙是否重置长连接,并确认 proxy 是否已自动重连。'
|
||||
case 'ws_ping_error': return '检查中间代理是否清理空闲 WebSocket,或对端是否提前关闭连接。'
|
||||
case 'ws_read_error': return '对照同一时间的 gateway 日志和网络监控,确认是否存在链路中断。'
|
||||
case 'tunnel_connect_error': return '检查 Aether 地址、DNS、TLS、管理 token,以及 AETHER_PROXY_AETHER_PROXY_URL 配置。'
|
||||
case 'frame_decode_error': return '检查 proxy/gateway 版本兼容性,确认中间层没有改写 WebSocket 二进制帧。'
|
||||
case 'stream_dispatch_timeout': return '检查 proxy CPU/内存、并发上限和上游 provider 慢请求。'
|
||||
case 'heartbeat_ack_empty':
|
||||
case 'heartbeat_ack_parse':
|
||||
return '检查 gateway 心跳处理日志和 proxy/gateway 版本兼容性。'
|
||||
case 'writer_task_panic':
|
||||
case 'writer_task_cancelled':
|
||||
return '查看此前一条写入或 ping 错误,确认隧道重连循环仍在运行。'
|
||||
case 'dispatcher_error': return '检查同一时间的请求流和 gateway tunnel 日志。'
|
||||
default: return ''
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
Reference in New Issue
Block a user