mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-08 10:27:46 +08:00
feat(proxy): surface node resource diagnostics
This commit is contained in:
@@ -158,6 +158,7 @@ pub async fn run(mut config: Config, servers: Vec<ServerEntry>) -> anyhow::Resul
|
|||||||
// Build a profile-keyed Hyper client pool for tunnel upstream requests.
|
// Build a profile-keyed Hyper client pool for tunnel upstream requests.
|
||||||
let upstream_client_pool =
|
let upstream_client_pool =
|
||||||
upstream_client::UpstreamClientPool::new(Arc::clone(&config), Arc::clone(&dns_cache));
|
upstream_client::UpstreamClientPool::new(Arc::clone(&config), Arc::clone(&dns_cache));
|
||||||
|
let resource_monitor = Arc::new(hardware::RuntimeResourceMonitor::new());
|
||||||
|
|
||||||
// Register with each Aether server and build per-server contexts.
|
// Register with each Aether server and build per-server contexts.
|
||||||
// Wrapped in Arc<Mutex> so retry_failed_registrations can append later.
|
// Wrapped in Arc<Mutex> so retry_failed_registrations can append later.
|
||||||
@@ -218,6 +219,7 @@ pub async fn run(mut config: Config, servers: Vec<ServerEntry>) -> anyhow::Resul
|
|||||||
dns_cache,
|
dns_cache,
|
||||||
upstream_client_pool,
|
upstream_client_pool,
|
||||||
tunnel_tls_config,
|
tunnel_tls_config,
|
||||||
|
resource_monitor,
|
||||||
stream_gate: None,
|
stream_gate: None,
|
||||||
distributed_stream_gate: None,
|
distributed_stream_gate: None,
|
||||||
};
|
};
|
||||||
@@ -947,6 +949,7 @@ mod tests {
|
|||||||
dns_cache,
|
dns_cache,
|
||||||
upstream_client_pool,
|
upstream_client_pool,
|
||||||
tunnel_tls_config: Arc::new(crate::tunnel::client::build_tls_config()),
|
tunnel_tls_config: Arc::new(crate::tunnel::client::build_tls_config()),
|
||||||
|
resource_monitor: Arc::new(crate::hardware::RuntimeResourceMonitor::new()),
|
||||||
stream_gate: None,
|
stream_gate: None,
|
||||||
distributed_stream_gate: None,
|
distributed_stream_gate: None,
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
|
use std::sync::Mutex;
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use sysinfo::System;
|
use sysinfo::{get_current_pid, Pid, ProcessesToUpdate, System};
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
/// Hardware information collected at startup.
|
/// Hardware information collected at startup.
|
||||||
@@ -17,6 +20,102 @@ pub struct HardwareInfo {
|
|||||||
pub estimated_max_concurrency: u64,
|
pub estimated_max_concurrency: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Runtime resource usage sampled during heartbeat reporting.
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct RuntimeResourceSnapshot {
|
||||||
|
pub sampled_at_unix_secs: u64,
|
||||||
|
pub system_cpu_usage_percent: f64,
|
||||||
|
pub process_cpu_usage_percent: f64,
|
||||||
|
pub memory_total_bytes: u64,
|
||||||
|
pub memory_used_bytes: u64,
|
||||||
|
pub memory_available_bytes: u64,
|
||||||
|
pub memory_used_percent: f64,
|
||||||
|
pub process_memory_bytes: u64,
|
||||||
|
pub process_virtual_memory_bytes: u64,
|
||||||
|
pub process_memory_percent: f64,
|
||||||
|
pub load_average_1m: f64,
|
||||||
|
pub load_average_5m: f64,
|
||||||
|
pub load_average_15m: f64,
|
||||||
|
pub system_uptime_secs: u64,
|
||||||
|
pub process_uptime_secs: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Small, reusable sysinfo monitor. Keeping it alive between samples makes CPU
|
||||||
|
/// usage deltas meaningful without re-enumerating the whole machine every time.
|
||||||
|
pub struct RuntimeResourceMonitor {
|
||||||
|
system: Mutex<System>,
|
||||||
|
current_pid: Option<Pid>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RuntimeResourceMonitor {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
let mut system = System::new_all();
|
||||||
|
let current_pid = get_current_pid().ok();
|
||||||
|
if let Some(pid) = current_pid {
|
||||||
|
system.refresh_processes(ProcessesToUpdate::Some(&[pid]), true);
|
||||||
|
}
|
||||||
|
system.refresh_cpu_usage();
|
||||||
|
system.refresh_memory();
|
||||||
|
Self {
|
||||||
|
system: Mutex::new(system),
|
||||||
|
current_pid,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn snapshot(&self) -> RuntimeResourceSnapshot {
|
||||||
|
let mut system = match self.system.lock() {
|
||||||
|
Ok(guard) => guard,
|
||||||
|
Err(poisoned) => poisoned.into_inner(),
|
||||||
|
};
|
||||||
|
|
||||||
|
system.refresh_cpu_usage();
|
||||||
|
system.refresh_memory();
|
||||||
|
if let Some(pid) = self.current_pid {
|
||||||
|
system.refresh_processes(ProcessesToUpdate::Some(&[pid]), true);
|
||||||
|
}
|
||||||
|
|
||||||
|
let memory_total_bytes = system.total_memory();
|
||||||
|
let memory_used_bytes = system.used_memory();
|
||||||
|
let memory_available_bytes = system.available_memory();
|
||||||
|
let (
|
||||||
|
process_cpu_usage_percent,
|
||||||
|
process_memory_bytes,
|
||||||
|
process_virtual_memory_bytes,
|
||||||
|
process_uptime_secs,
|
||||||
|
) = self
|
||||||
|
.current_pid
|
||||||
|
.and_then(|pid| system.process(pid))
|
||||||
|
.map(|process| {
|
||||||
|
(
|
||||||
|
process.cpu_usage() as f64,
|
||||||
|
process.memory(),
|
||||||
|
process.virtual_memory(),
|
||||||
|
Some(process.run_time()),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.unwrap_or((0.0, 0, 0, None));
|
||||||
|
let load = System::load_average();
|
||||||
|
|
||||||
|
RuntimeResourceSnapshot {
|
||||||
|
sampled_at_unix_secs: current_unix_secs(),
|
||||||
|
system_cpu_usage_percent: system.global_cpu_usage() as f64,
|
||||||
|
process_cpu_usage_percent,
|
||||||
|
memory_total_bytes,
|
||||||
|
memory_used_bytes,
|
||||||
|
memory_available_bytes,
|
||||||
|
memory_used_percent: ratio_percent(memory_used_bytes, memory_total_bytes),
|
||||||
|
process_memory_bytes,
|
||||||
|
process_virtual_memory_bytes,
|
||||||
|
process_memory_percent: ratio_percent(process_memory_bytes, memory_total_bytes),
|
||||||
|
load_average_1m: load.one,
|
||||||
|
load_average_5m: load.five,
|
||||||
|
load_average_15m: load.fifteen,
|
||||||
|
system_uptime_secs: System::uptime(),
|
||||||
|
process_uptime_secs,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Collect hardware information and estimate max concurrency.
|
/// Collect hardware information and estimate max concurrency.
|
||||||
///
|
///
|
||||||
/// Should be called once at startup -- hardware does not change at runtime.
|
/// Should be called once at startup -- hardware does not change at runtime.
|
||||||
@@ -77,3 +176,18 @@ fn get_fd_limit() -> u64 {
|
|||||||
// Fallback for non-unix or error
|
// Fallback for non-unix or error
|
||||||
1024
|
1024
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn ratio_percent(value: u64, total: u64) -> f64 {
|
||||||
|
if total == 0 {
|
||||||
|
0.0
|
||||||
|
} else {
|
||||||
|
value as f64 * 100.0 / total as f64
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn current_unix_secs() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.map(|duration| duration.as_secs())
|
||||||
|
.unwrap_or(0)
|
||||||
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ use aether_runtime::{AdmissionPermit, ConcurrencyError, ConcurrencyGate, Concurr
|
|||||||
use aether_runtime_state::{RuntimeSemaphore, RuntimeSemaphoreError, RuntimeSemaphoreSnapshot};
|
use aether_runtime_state::{RuntimeSemaphore, RuntimeSemaphoreError, RuntimeSemaphoreSnapshot};
|
||||||
|
|
||||||
use crate::config::Config;
|
use crate::config::Config;
|
||||||
|
use crate::hardware::RuntimeResourceMonitor;
|
||||||
use crate::registration::client::AetherClient;
|
use crate::registration::client::AetherClient;
|
||||||
use crate::runtime::SharedDynamicConfig;
|
use crate::runtime::SharedDynamicConfig;
|
||||||
use crate::target_filter::DnsCache;
|
use crate::target_filter::DnsCache;
|
||||||
@@ -24,6 +25,8 @@ pub struct AppState {
|
|||||||
pub upstream_client_pool: UpstreamClientPool,
|
pub upstream_client_pool: UpstreamClientPool,
|
||||||
/// Shared TLS config for tunnel WebSocket connections (avoids re-parsing root CAs on each reconnect).
|
/// Shared TLS config for tunnel WebSocket connections (avoids re-parsing root CAs on each reconnect).
|
||||||
pub tunnel_tls_config: Arc<rustls::ClientConfig>,
|
pub tunnel_tls_config: Arc<rustls::ClientConfig>,
|
||||||
|
/// Runtime CPU/memory monitor sampled by heartbeat payloads.
|
||||||
|
pub resource_monitor: Arc<RuntimeResourceMonitor>,
|
||||||
/// Optional per-process stream admission gate.
|
/// Optional per-process stream admission gate.
|
||||||
pub stream_gate: Option<Arc<ConcurrencyGate>>,
|
pub stream_gate: Option<Arc<ConcurrencyGate>>,
|
||||||
/// Optional cross-instance stream admission gate.
|
/// Optional cross-instance stream admission gate.
|
||||||
@@ -96,6 +99,10 @@ pub struct TunnelErrorEvent {
|
|||||||
pub timestamp_unix_secs: u64,
|
pub timestamp_unix_secs: u64,
|
||||||
pub category: String,
|
pub category: String,
|
||||||
pub message: String,
|
pub message: String,
|
||||||
|
pub severity: String,
|
||||||
|
pub component: String,
|
||||||
|
pub summary: String,
|
||||||
|
pub operator_action: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, Default)]
|
#[derive(Debug, Clone, Copy, Default)]
|
||||||
@@ -229,11 +236,18 @@ impl TunnelMetrics {
|
|||||||
|
|
||||||
pub fn record_error(&self, category: &str, message: &str) {
|
pub fn record_error(&self, category: &str, message: &str) {
|
||||||
self.error_events_total.fetch_add(1, Ordering::Release);
|
self.error_events_total.fetch_add(1, Ordering::Release);
|
||||||
|
let category = normalize_error_field(category, TUNNEL_ERROR_CATEGORY_MAX_CHARS, "unknown");
|
||||||
|
let message = normalize_error_field(message, TUNNEL_ERROR_MESSAGE_MAX_CHARS, "n/a");
|
||||||
|
let diagnostic = classify_tunnel_error(category.as_str(), message.as_str());
|
||||||
|
|
||||||
let event = TunnelErrorEvent {
|
let event = TunnelErrorEvent {
|
||||||
timestamp_unix_secs: now_unix_secs(),
|
timestamp_unix_secs: now_unix_secs(),
|
||||||
category: normalize_error_field(category, TUNNEL_ERROR_CATEGORY_MAX_CHARS, "unknown"),
|
category,
|
||||||
message: normalize_error_field(message, TUNNEL_ERROR_MESSAGE_MAX_CHARS, "n/a"),
|
message,
|
||||||
|
severity: diagnostic.severity.to_string(),
|
||||||
|
component: diagnostic.component.to_string(),
|
||||||
|
summary: diagnostic.summary.to_string(),
|
||||||
|
operator_action: diagnostic.operator_action.to_string(),
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut recent_errors = match self.recent_errors.lock() {
|
let mut recent_errors = match self.recent_errors.lock() {
|
||||||
@@ -302,6 +316,95 @@ fn normalize_error_field(value: &str, max_chars: usize, fallback: &str) -> Strin
|
|||||||
normalized.chars().take(max_chars).collect()
|
normalized.chars().take(max_chars).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct TunnelErrorDiagnostic {
|
||||||
|
severity: &'static str,
|
||||||
|
component: &'static str,
|
||||||
|
summary: &'static str,
|
||||||
|
operator_action: &'static str,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn classify_tunnel_error(category: &str, _message: &str) -> TunnelErrorDiagnostic {
|
||||||
|
match category {
|
||||||
|
"stale_timeout" => TunnelErrorDiagnostic {
|
||||||
|
severity: "warning",
|
||||||
|
component: "tunnel_read",
|
||||||
|
summary: "No inbound tunnel frames before stale timeout",
|
||||||
|
operator_action:
|
||||||
|
"Check gateway or reverse-proxy idle timeouts, packet loss, and WebSocket ping/pong reachability. Increase AETHER_PROXY_TUNNEL_STALE_TIMEOUT_MS if the network is high-latency.",
|
||||||
|
},
|
||||||
|
"ws_write_error" => TunnelErrorDiagnostic {
|
||||||
|
severity: "error",
|
||||||
|
component: "tunnel_write",
|
||||||
|
summary: "WebSocket write failed because the peer closed or reset the connection",
|
||||||
|
operator_action:
|
||||||
|
"Check gateway restarts, load balancer resets, NAT/firewall connection tracking, and whether the proxy is reconnecting successfully.",
|
||||||
|
},
|
||||||
|
"ws_ping_error" => TunnelErrorDiagnostic {
|
||||||
|
severity: "error",
|
||||||
|
component: "tunnel_write",
|
||||||
|
summary: "WebSocket keepalive ping could not be sent",
|
||||||
|
operator_action:
|
||||||
|
"Check whether the peer closed the socket or an intermediary is dropping idle WebSocket connections.",
|
||||||
|
},
|
||||||
|
"ws_read_error" => TunnelErrorDiagnostic {
|
||||||
|
severity: "error",
|
||||||
|
component: "tunnel_read",
|
||||||
|
summary: "WebSocket read failed",
|
||||||
|
operator_action:
|
||||||
|
"Check gateway logs and network stability around the same timestamp; compare with reconnect and heartbeat ACK counters.",
|
||||||
|
},
|
||||||
|
"tunnel_connect_error" => TunnelErrorDiagnostic {
|
||||||
|
severity: "critical",
|
||||||
|
component: "tunnel_connect",
|
||||||
|
summary: "Tunnel connection attempt failed",
|
||||||
|
operator_action:
|
||||||
|
"Check Aether URL reachability, DNS, TLS, management token validity, and any configured AETHER_PROXY_AETHER_PROXY_URL.",
|
||||||
|
},
|
||||||
|
"frame_decode_error" => TunnelErrorDiagnostic {
|
||||||
|
severity: "error",
|
||||||
|
component: "tunnel_protocol",
|
||||||
|
summary: "Received tunnel frame could not be decoded",
|
||||||
|
operator_action:
|
||||||
|
"Check proxy and gateway version compatibility and whether traffic is being modified by an intermediary.",
|
||||||
|
},
|
||||||
|
"stream_dispatch_timeout" => TunnelErrorDiagnostic {
|
||||||
|
severity: "warning",
|
||||||
|
component: "stream_dispatch",
|
||||||
|
summary: "Request body frame could not be delivered to its stream handler in time",
|
||||||
|
operator_action:
|
||||||
|
"Check proxy CPU, memory, stream concurrency saturation, and slow upstream provider requests.",
|
||||||
|
},
|
||||||
|
"heartbeat_ack_empty" | "heartbeat_ack_parse" => TunnelErrorDiagnostic {
|
||||||
|
severity: "warning",
|
||||||
|
component: "heartbeat",
|
||||||
|
summary: "Heartbeat ACK from gateway was missing or invalid",
|
||||||
|
operator_action:
|
||||||
|
"Check gateway heartbeat handler logs and proxy/gateway version compatibility.",
|
||||||
|
},
|
||||||
|
"writer_task_panic" | "writer_task_cancelled" => TunnelErrorDiagnostic {
|
||||||
|
severity: "error",
|
||||||
|
component: "tunnel_writer",
|
||||||
|
summary: "Tunnel writer task exited unexpectedly",
|
||||||
|
operator_action:
|
||||||
|
"Check proxy logs for the preceding write or ping error and confirm the tunnel reconnect loop is active.",
|
||||||
|
},
|
||||||
|
"dispatcher_error" => TunnelErrorDiagnostic {
|
||||||
|
severity: "error",
|
||||||
|
component: "tunnel_dispatcher",
|
||||||
|
summary: "Tunnel dispatcher exited with an error",
|
||||||
|
operator_action:
|
||||||
|
"Check the proxied request stream and gateway tunnel logs around the same timestamp.",
|
||||||
|
},
|
||||||
|
_ => TunnelErrorDiagnostic {
|
||||||
|
severity: "info",
|
||||||
|
component: "tunnel",
|
||||||
|
summary: "Tunnel reported an unclassified error",
|
||||||
|
operator_action:
|
||||||
|
"Inspect the raw message and compare it with proxy, gateway, and network logs at the same time.",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
|
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
|
||||||
pub enum ProxyAdmissionError {
|
pub enum ProxyAdmissionError {
|
||||||
#[error("proxy stream admission saturated at {limit} for gate {gate}")]
|
#[error("proxy stream admission saturated at {limit} for gate {gate}")]
|
||||||
|
|||||||
@@ -240,6 +240,7 @@ async fn build_heartbeat_payload(
|
|||||||
let node_id = server.node_id.read().unwrap().clone();
|
let node_id = server.node_id.read().unwrap().clone();
|
||||||
let tunnel_snapshot = server.tunnel_metrics.snapshot();
|
let tunnel_snapshot = server.tunnel_metrics.snapshot();
|
||||||
let recent_errors = server.tunnel_metrics.recent_errors(8);
|
let recent_errors = server.tunnel_metrics.recent_errors(8);
|
||||||
|
let resource_usage = state.resource_monitor.snapshot();
|
||||||
|
|
||||||
let avg_latency_ms = if snapshot.requests > 0 {
|
let avg_latency_ms = if snapshot.requests > 0 {
|
||||||
Some(snapshot.latency_ns as f64 / snapshot.requests as f64 / 1_000_000.0)
|
Some(snapshot.latency_ns as f64 / snapshot.requests as f64 / 1_000_000.0)
|
||||||
@@ -291,6 +292,7 @@ async fn build_heartbeat_payload(
|
|||||||
"proxy_metadata": {
|
"proxy_metadata": {
|
||||||
"version": CURRENT_VERSION,
|
"version": CURRENT_VERSION,
|
||||||
"admission": admission,
|
"admission": admission,
|
||||||
|
"resource_usage": resource_usage,
|
||||||
"tunnel_metrics": {
|
"tunnel_metrics": {
|
||||||
"connect_attempts": tunnel_snapshot.connect_attempts,
|
"connect_attempts": tunnel_snapshot.connect_attempts,
|
||||||
"connect_successes": tunnel_snapshot.connect_successes,
|
"connect_successes": tunnel_snapshot.connect_successes,
|
||||||
@@ -419,13 +421,13 @@ mod tests {
|
|||||||
use arc_swap::ArcSwap;
|
use arc_swap::ArcSwap;
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
|
|
||||||
use super::{handle_ack, AckDecision};
|
use super::{build_heartbeat_payload, handle_ack, AckDecision, HeartbeatSnapshot};
|
||||||
use crate::registration::client::AetherClient;
|
use crate::registration::client::AetherClient;
|
||||||
use crate::runtime::DynamicConfig;
|
use crate::runtime::DynamicConfig;
|
||||||
use crate::state::{ProxyMetrics, ServerContext, TunnelMetrics};
|
use crate::state::{AppState, ProxyMetrics, ServerContext, TunnelMetrics};
|
||||||
|
|
||||||
fn sample_server() -> Arc<ServerContext> {
|
fn sample_config() -> Arc<crate::config::Config> {
|
||||||
let config = Arc::new(crate::config::Config::parse_from([
|
Arc::new(crate::config::Config::parse_from([
|
||||||
"aether-proxy",
|
"aether-proxy",
|
||||||
"--aether-url",
|
"--aether-url",
|
||||||
"https://example.com",
|
"https://example.com",
|
||||||
@@ -433,7 +435,11 @@ mod tests {
|
|||||||
"ae_test",
|
"ae_test",
|
||||||
"--node-name",
|
"--node-name",
|
||||||
"proxy-test",
|
"proxy-test",
|
||||||
]));
|
]))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sample_server() -> Arc<ServerContext> {
|
||||||
|
let config = sample_config();
|
||||||
Arc::new(ServerContext {
|
Arc::new(ServerContext {
|
||||||
server_label: "heartbeat-test".to_string(),
|
server_label: "heartbeat-test".to_string(),
|
||||||
aether_url: config.aether_url.clone(),
|
aether_url: config.aether_url.clone(),
|
||||||
@@ -452,6 +458,24 @@ mod tests {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn sample_state(config: Arc<crate::config::Config>) -> AppState {
|
||||||
|
let dns_cache = Arc::new(crate::target_filter::DnsCache::new(
|
||||||
|
std::time::Duration::from_secs(config.dns_cache_ttl_secs),
|
||||||
|
config.dns_cache_capacity,
|
||||||
|
));
|
||||||
|
AppState {
|
||||||
|
config: Arc::clone(&config),
|
||||||
|
dns_cache: Arc::clone(&dns_cache),
|
||||||
|
upstream_client_pool: crate::upstream_client::UpstreamClientPool::new(
|
||||||
|
config, dns_cache,
|
||||||
|
),
|
||||||
|
tunnel_tls_config: Arc::new(crate::tunnel::client::build_tls_config()),
|
||||||
|
resource_monitor: Arc::new(crate::hardware::RuntimeResourceMonitor::new()),
|
||||||
|
stream_gate: None,
|
||||||
|
distributed_stream_gate: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn heartbeat_ack_requires_heartbeat_id() {
|
fn heartbeat_ack_requires_heartbeat_id() {
|
||||||
let server = sample_server();
|
let server = sample_server();
|
||||||
@@ -481,4 +505,48 @@ mod tests {
|
|||||||
));
|
));
|
||||||
assert_eq!(server.dynamic.load().heartbeat_interval, 9);
|
assert_eq!(server.dynamic.load().heartbeat_interval, 9);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn heartbeat_payload_reports_resource_usage_and_tunnel_error_diagnostics() {
|
||||||
|
let config = sample_config();
|
||||||
|
let server = sample_server();
|
||||||
|
server
|
||||||
|
.tunnel_metrics
|
||||||
|
.record_error("ws_write_error", "IO error: Connection reset by peer");
|
||||||
|
let state = sample_state(config);
|
||||||
|
|
||||||
|
let payload = build_heartbeat_payload(
|
||||||
|
&state,
|
||||||
|
&server,
|
||||||
|
"session-1",
|
||||||
|
42,
|
||||||
|
HeartbeatSnapshot::default(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let payload: serde_json::Value =
|
||||||
|
serde_json::from_slice(&payload).expect("heartbeat payload should be JSON");
|
||||||
|
let resource_usage = payload
|
||||||
|
.pointer("/proxy_metadata/resource_usage")
|
||||||
|
.and_then(serde_json::Value::as_object)
|
||||||
|
.expect("resource usage should be reported");
|
||||||
|
assert!(resource_usage.contains_key("system_cpu_usage_percent"));
|
||||||
|
assert!(resource_usage.contains_key("process_memory_bytes"));
|
||||||
|
|
||||||
|
let recent_error = payload
|
||||||
|
.pointer("/proxy_metadata/recent_tunnel_errors/0")
|
||||||
|
.and_then(serde_json::Value::as_object)
|
||||||
|
.expect("recent tunnel error should be reported");
|
||||||
|
assert_eq!(
|
||||||
|
recent_error
|
||||||
|
.get("component")
|
||||||
|
.and_then(serde_json::Value::as_str),
|
||||||
|
Some("tunnel_write")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
recent_error
|
||||||
|
.get("severity")
|
||||||
|
.and_then(serde_json::Value::as_str),
|
||||||
|
Some("error")
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -462,6 +462,7 @@ mod tests {
|
|||||||
dns_cache,
|
dns_cache,
|
||||||
upstream_client_pool,
|
upstream_client_pool,
|
||||||
tunnel_tls_config: Arc::new(crate::tunnel::client::build_tls_config()),
|
tunnel_tls_config: Arc::new(crate::tunnel::client::build_tls_config()),
|
||||||
|
resource_monitor: Arc::new(crate::hardware::RuntimeResourceMonitor::new()),
|
||||||
stream_gate: None,
|
stream_gate: None,
|
||||||
distributed_stream_gate: None,
|
distributed_stream_gate: None,
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -2284,6 +2284,7 @@ mod tests {
|
|||||||
dns_cache,
|
dns_cache,
|
||||||
upstream_client_pool,
|
upstream_client_pool,
|
||||||
tunnel_tls_config: Arc::new(build_tls_config()),
|
tunnel_tls_config: Arc::new(build_tls_config()),
|
||||||
|
resource_monitor: Arc::new(crate::hardware::RuntimeResourceMonitor::new()),
|
||||||
stream_gate,
|
stream_gate,
|
||||||
distributed_stream_gate,
|
distributed_stream_gate,
|
||||||
})
|
})
|
||||||
@@ -2314,6 +2315,7 @@ mod tests {
|
|||||||
dns_cache,
|
dns_cache,
|
||||||
upstream_client_pool,
|
upstream_client_pool,
|
||||||
tunnel_tls_config: Arc::new(build_tls_config()),
|
tunnel_tls_config: Arc::new(build_tls_config()),
|
||||||
|
resource_monitor: Arc::new(crate::hardware::RuntimeResourceMonitor::new()),
|
||||||
stream_gate: None,
|
stream_gate: None,
|
||||||
distributed_stream_gate: None,
|
distributed_stream_gate: None,
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -650,6 +650,10 @@ impl ProxyNodeWriteRepository for InMemoryProxyNodeRepository {
|
|||||||
"source": "heartbeat",
|
"source": "heartbeat",
|
||||||
"category": error.category,
|
"category": error.category,
|
||||||
"message": error.message,
|
"message": error.message,
|
||||||
|
"severity": error.severity.as_deref(),
|
||||||
|
"component": error.component.as_deref(),
|
||||||
|
"summary": error.summary.as_deref(),
|
||||||
|
"operator_action": error.operator_action.as_deref(),
|
||||||
"timestamp_unix_secs": error.timestamp_unix_secs,
|
"timestamp_unix_secs": error.timestamp_unix_secs,
|
||||||
})),
|
})),
|
||||||
created_at_unix_ms: Some(if error.timestamp_unix_secs == 0 {
|
created_at_unix_ms: Some(if error.timestamp_unix_secs == 0 {
|
||||||
|
|||||||
@@ -836,6 +836,10 @@ WHERE is_manual = 0
|
|||||||
"source": "heartbeat",
|
"source": "heartbeat",
|
||||||
"category": error.category,
|
"category": error.category,
|
||||||
"message": error.message,
|
"message": error.message,
|
||||||
|
"severity": error.severity.as_deref(),
|
||||||
|
"component": error.component.as_deref(),
|
||||||
|
"summary": error.summary.as_deref(),
|
||||||
|
"operator_action": error.operator_action.as_deref(),
|
||||||
"timestamp_unix_secs": error.timestamp_unix_secs,
|
"timestamp_unix_secs": error.timestamp_unix_secs,
|
||||||
});
|
});
|
||||||
self.insert_event(
|
self.insert_event(
|
||||||
|
|||||||
@@ -1224,6 +1224,10 @@ impl ProxyNodeWriteRepository for SqlxProxyNodeRepository {
|
|||||||
"source": "heartbeat",
|
"source": "heartbeat",
|
||||||
"category": error.category,
|
"category": error.category,
|
||||||
"message": error.message,
|
"message": error.message,
|
||||||
|
"severity": error.severity.as_deref(),
|
||||||
|
"component": error.component.as_deref(),
|
||||||
|
"summary": error.summary.as_deref(),
|
||||||
|
"operator_action": error.operator_action.as_deref(),
|
||||||
"timestamp_unix_secs": error.timestamp_unix_secs,
|
"timestamp_unix_secs": error.timestamp_unix_secs,
|
||||||
});
|
});
|
||||||
self.insert_event(
|
self.insert_event(
|
||||||
|
|||||||
@@ -838,6 +838,10 @@ WHERE is_manual = 0
|
|||||||
"source": "heartbeat",
|
"source": "heartbeat",
|
||||||
"category": error.category,
|
"category": error.category,
|
||||||
"message": error.message,
|
"message": error.message,
|
||||||
|
"severity": error.severity.as_deref(),
|
||||||
|
"component": error.component.as_deref(),
|
||||||
|
"summary": error.summary.as_deref(),
|
||||||
|
"operator_action": error.operator_action.as_deref(),
|
||||||
"timestamp_unix_secs": error.timestamp_unix_secs,
|
"timestamp_unix_secs": error.timestamp_unix_secs,
|
||||||
});
|
});
|
||||||
self.insert_event(
|
self.insert_event(
|
||||||
|
|||||||
@@ -324,6 +324,10 @@ pub struct TunnelErrorEventRecord {
|
|||||||
pub timestamp_unix_secs: u64,
|
pub timestamp_unix_secs: u64,
|
||||||
pub category: String,
|
pub category: String,
|
||||||
pub message: String,
|
pub message: String,
|
||||||
|
pub severity: Option<String>,
|
||||||
|
pub component: Option<String>,
|
||||||
|
pub summary: Option<String>,
|
||||||
|
pub operator_action: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||||
@@ -417,7 +421,11 @@ pub fn build_tunnel_metrics_sample(
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn build_tunnel_error_event_detail(event: &TunnelErrorEventRecord) -> String {
|
pub fn build_tunnel_error_event_detail(event: &TunnelErrorEventRecord) -> String {
|
||||||
format!("[{}] {}", event.category, event.message)
|
format!(
|
||||||
|
"[{}] {}",
|
||||||
|
event.category,
|
||||||
|
event.summary.as_deref().unwrap_or(event.message.as_str())
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn normalize_proxy_metadata(
|
pub fn normalize_proxy_metadata(
|
||||||
@@ -497,6 +505,10 @@ fn extract_recent_tunnel_errors(proxy_metadata: Option<&Value>) -> Vec<TunnelErr
|
|||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
.unwrap_or("n/a")
|
.unwrap_or("n/a")
|
||||||
.to_string(),
|
.to_string(),
|
||||||
|
severity: json_string(item.get("severity")),
|
||||||
|
component: json_string(item.get("component")),
|
||||||
|
summary: json_string(item.get("summary")),
|
||||||
|
operator_action: json_string(item.get("operator_action")),
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
@@ -512,6 +524,14 @@ fn json_u64(value: Option<&Value>) -> Option<u64> {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn json_string(value: Option<&Value>) -> Option<String> {
|
||||||
|
value
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(ToString::to_string)
|
||||||
|
}
|
||||||
|
|
||||||
fn counter_delta_u64(previous: Option<u64>, current: u64) -> u64 {
|
fn counter_delta_u64(previous: Option<u64>, current: u64) -> u64 {
|
||||||
match previous {
|
match previous {
|
||||||
Some(previous) if current >= previous => current - previous,
|
Some(previous) if current >= previous => current - previous,
|
||||||
@@ -726,10 +746,11 @@ mod tests {
|
|||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
bucket_start_unix_secs, build_tunnel_metrics_sample, normalize_proxy_node_scheduling_state,
|
bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
|
||||||
proxy_node_accepts_new_tunnels, proxy_reported_version,
|
normalize_proxy_node_scheduling_state, proxy_node_accepts_new_tunnels,
|
||||||
reconcile_remote_config_after_heartbeat, remote_config_scheduling_state,
|
proxy_reported_version, reconcile_remote_config_after_heartbeat,
|
||||||
remote_config_upgrade_target, ProxyNodeMetricsStep, StoredProxyNode,
|
remote_config_scheduling_state, remote_config_upgrade_target, ProxyNodeMetricsStep,
|
||||||
|
StoredProxyNode,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -842,7 +863,15 @@ mod tests {
|
|||||||
},
|
},
|
||||||
"recent_tunnel_errors": [
|
"recent_tunnel_errors": [
|
||||||
{"timestamp_unix_secs": 100, "category": "older", "message": "old"},
|
{"timestamp_unix_secs": 100, "category": "older", "message": "old"},
|
||||||
{"timestamp_unix_secs": 101, "category": "newer", "message": "new"}
|
{
|
||||||
|
"timestamp_unix_secs": 101,
|
||||||
|
"category": "newer",
|
||||||
|
"message": "new",
|
||||||
|
"severity": "error",
|
||||||
|
"component": "tunnel_write",
|
||||||
|
"summary": "WebSocket write failed because the peer closed or reset the connection",
|
||||||
|
"operator_action": "Check gateway restarts and network resets."
|
||||||
|
}
|
||||||
]
|
]
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -860,6 +889,19 @@ mod tests {
|
|||||||
assert_eq!(sample.ws_out_frames_delta, 3);
|
assert_eq!(sample.ws_out_frames_delta, 3);
|
||||||
assert_eq!(sample.recent_error_events.len(), 2);
|
assert_eq!(sample.recent_error_events.len(), 2);
|
||||||
assert_eq!(sample.recent_error_events[0].category, "older");
|
assert_eq!(sample.recent_error_events[0].category, "older");
|
||||||
|
assert_eq!(sample.recent_error_events[0].summary, None);
|
||||||
|
assert_eq!(
|
||||||
|
sample.recent_error_events[1].severity.as_deref(),
|
||||||
|
Some("error")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
sample.recent_error_events[1].component.as_deref(),
|
||||||
|
Some("tunnel_write")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
build_tunnel_error_event_detail(&sample.recent_error_events[1]),
|
||||||
|
"[newer] WebSocket write failed because the peer closed or reset the connection"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -138,7 +138,10 @@
|
|||||||
>
|
>
|
||||||
{{ eventTypeLabel(event.event_type) }}
|
{{ eventTypeLabel(event.event_type) }}
|
||||||
</Badge>
|
</Badge>
|
||||||
<span class="text-muted-foreground truncate flex-1">{{ eventDetail(event) }}</span>
|
<span
|
||||||
|
class="text-muted-foreground truncate flex-1"
|
||||||
|
:title="eventTooltip(event)"
|
||||||
|
>{{ eventDetail(event) }}</span>
|
||||||
<span class="text-[10px] text-muted-foreground/70 tabular-nums shrink-0">{{ formatTime(event.created_at || null) }}</span>
|
<span class="text-[10px] text-muted-foreground/70 tabular-nums shrink-0">{{ formatTime(event.created_at || null) }}</span>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -146,6 +149,35 @@
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="grid gap-4 lg:grid-cols-2">
|
<div class="grid gap-4 lg:grid-cols-2">
|
||||||
|
<section class="rounded-lg border border-border/50 bg-background/70 p-3 min-w-0">
|
||||||
|
<h5 class="text-xs font-semibold mb-3">
|
||||||
|
硬件与资源
|
||||||
|
</h5>
|
||||||
|
<div class="grid grid-cols-2 sm:grid-cols-3 gap-x-4 gap-y-2">
|
||||||
|
<div
|
||||||
|
v-for="item in resourceItems"
|
||||||
|
:key="item.label"
|
||||||
|
class="min-w-0"
|
||||||
|
>
|
||||||
|
<div class="text-[11px] text-muted-foreground truncate">
|
||||||
|
{{ item.label }}
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
class="text-xs font-medium tabular-nums truncate mt-0.5"
|
||||||
|
:class="item.tone"
|
||||||
|
>
|
||||||
|
{{ item.value }}
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
v-if="item.hint"
|
||||||
|
class="text-[10px] text-muted-foreground truncate mt-0.5"
|
||||||
|
>
|
||||||
|
{{ item.hint }}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
<section class="rounded-lg border border-border/50 bg-background/70 p-3 min-w-0">
|
<section class="rounded-lg border border-border/50 bg-background/70 p-3 min-w-0">
|
||||||
<h5 class="text-xs font-semibold mb-3">
|
<h5 class="text-xs font-semibold mb-3">
|
||||||
实时快照
|
实时快照
|
||||||
@@ -234,6 +266,8 @@ const metricsSummary = computed(() => metrics.value?.summary ?? null)
|
|||||||
const bucketItems = computed(() => metrics.value?.items.slice(-24) ?? [])
|
const bucketItems = computed(() => metrics.value?.items.slice(-24) ?? [])
|
||||||
const recentEvents = computed(() => props.state?.events.slice(0, 8) ?? [])
|
const recentEvents = computed(() => props.state?.events.slice(0, 8) ?? [])
|
||||||
const metadata = computed(() => asRecord(detailNode.value.proxy_metadata))
|
const metadata = computed(() => asRecord(detailNode.value.proxy_metadata))
|
||||||
|
const hardwareInfo = computed(() => asRecord(detailNode.value.hardware_info))
|
||||||
|
const resourceUsage = computed(() => asRecord(metadata.value?.resource_usage))
|
||||||
const tunnelMetrics = computed(() => asRecord(metadata.value?.tunnel_metrics))
|
const tunnelMetrics = computed(() => asRecord(metadata.value?.tunnel_metrics))
|
||||||
|
|
||||||
const loadedText = computed(() => {
|
const loadedText = computed(() => {
|
||||||
@@ -300,6 +334,82 @@ const snapshotItems = computed(() => {
|
|||||||
]
|
]
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const resourceItems = computed(() => {
|
||||||
|
const hardware = hardwareInfo.value
|
||||||
|
const resource = resourceUsage.value
|
||||||
|
const memoryTotalBytes = numberField(resource, 'memory_total_bytes') ?? memoryMbToBytes(numberField(hardware, 'total_memory_mb'))
|
||||||
|
const memoryUsedBytes = numberField(resource, 'memory_used_bytes')
|
||||||
|
const memoryUsedPercent = numberField(resource, 'memory_used_percent') ?? percentFromBytes(memoryUsedBytes, memoryTotalBytes)
|
||||||
|
const processMemoryBytes = numberField(resource, 'process_memory_bytes')
|
||||||
|
const processMemoryPercent = numberField(resource, 'process_memory_percent')
|
||||||
|
const systemCpu = numberField(resource, 'system_cpu_usage_percent')
|
||||||
|
const processCpu = numberField(resource, 'process_cpu_usage_percent')
|
||||||
|
const load1 = numberField(resource, 'load_average_1m')
|
||||||
|
const load5 = numberField(resource, 'load_average_5m')
|
||||||
|
const load15 = numberField(resource, 'load_average_15m')
|
||||||
|
const cpuCores = numberField(hardware, 'cpu_cores')
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
label: '系统 CPU',
|
||||||
|
value: formatUsagePercent(systemCpu),
|
||||||
|
hint: '主机整体',
|
||||||
|
tone: usageTone(systemCpu, 70, 90),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: '系统内存',
|
||||||
|
value: formatUsagePercent(memoryUsedPercent),
|
||||||
|
hint: memoryUsedBytes == null || memoryTotalBytes == null
|
||||||
|
? ''
|
||||||
|
: `${formatBytes(memoryUsedBytes)} / ${formatBytes(memoryTotalBytes)}`,
|
||||||
|
tone: usageTone(memoryUsedPercent, 75, 90),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: 'Proxy CPU',
|
||||||
|
value: formatUsagePercent(processCpu),
|
||||||
|
hint: '当前进程',
|
||||||
|
tone: usageTone(processCpu, 70, 90),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: 'Proxy 内存',
|
||||||
|
value: processMemoryBytes == null ? '-' : formatBytes(processMemoryBytes),
|
||||||
|
hint: processMemoryPercent == null ? '' : formatUsagePercent(processMemoryPercent),
|
||||||
|
tone: usageTone(processMemoryPercent, 5, 15),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: '负载',
|
||||||
|
value: formatLoadAverage(load1, load5, load15),
|
||||||
|
hint: cpuCores == null ? '1/5/15m' : `${formatNumber(cpuCores)} 核`,
|
||||||
|
tone: loadTone(load1, cpuCores),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: '运行时间',
|
||||||
|
value: formatDurationSeconds(numberField(resource, 'process_uptime_secs')),
|
||||||
|
hint: 'proxy 进程',
|
||||||
|
tone: '',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: 'CPU 核心',
|
||||||
|
value: cpuCores == null ? '-' : formatNumber(cpuCores),
|
||||||
|
hint: stringField(hardware, 'os_info') || '',
|
||||||
|
tone: '',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: '物理内存',
|
||||||
|
value: memoryTotalBytes == null ? '-' : formatBytes(memoryTotalBytes),
|
||||||
|
hint: '启动采集',
|
||||||
|
tone: '',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: 'FD 限制',
|
||||||
|
value: formatOptionalNumber(numberField(hardware, 'fd_limit')),
|
||||||
|
hint: detailNode.value.estimated_max_concurrency == null
|
||||||
|
? ''
|
||||||
|
: `容量 ${formatNumber(detailNode.value.estimated_max_concurrency)}`,
|
||||||
|
tone: '',
|
||||||
|
},
|
||||||
|
]
|
||||||
|
})
|
||||||
|
|
||||||
const tunnelCounterItems = computed(() => [
|
const tunnelCounterItems = computed(() => [
|
||||||
{ label: '建连尝试', value: formatTunnelNumber('connect_attempts') },
|
{ label: '建连尝试', value: formatTunnelNumber('connect_attempts') },
|
||||||
{ label: '建连成功', value: formatTunnelNumber('connect_successes') },
|
{ label: '建连成功', value: formatTunnelNumber('connect_successes') },
|
||||||
@@ -352,6 +462,10 @@ function formatNumber(value: number) {
|
|||||||
return String(Math.round(value))
|
return String(Math.round(value))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatOptionalNumber(value: number | null) {
|
||||||
|
return value == null ? '-' : formatNumber(value)
|
||||||
|
}
|
||||||
|
|
||||||
function formatDecimal(value: number | null) {
|
function formatDecimal(value: number | null) {
|
||||||
if (value == null || !Number.isFinite(value)) return '-'
|
if (value == null || !Number.isFinite(value)) return '-'
|
||||||
if (value === 0) return '0'
|
if (value === 0) return '0'
|
||||||
@@ -364,12 +478,24 @@ function formatPercent(value: number | null) {
|
|||||||
return `${(value * 100).toFixed(value >= 0.995 || value === 0 ? 0 : 1)}%`
|
return `${(value * 100).toFixed(value >= 0.995 || value === 0 ? 0 : 1)}%`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatUsagePercent(value: number | null) {
|
||||||
|
if (value == null || !Number.isFinite(value)) return '-'
|
||||||
|
if (value >= 100) return `${Math.round(value)}%`
|
||||||
|
if (value >= 10) return `${value.toFixed(0)}%`
|
||||||
|
return `${value.toFixed(1)}%`
|
||||||
|
}
|
||||||
|
|
||||||
function formatMs(value: number | null) {
|
function formatMs(value: number | null) {
|
||||||
if (value == null || !Number.isFinite(value) || value <= 0) return '-'
|
if (value == null || !Number.isFinite(value) || value <= 0) return '-'
|
||||||
if (value >= 1000) return `${(value / 1000).toFixed(1)}s`
|
if (value >= 1000) return `${(value / 1000).toFixed(1)}s`
|
||||||
return `${Math.round(value)}ms`
|
return `${Math.round(value)}ms`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatDurationSeconds(value: number | null) {
|
||||||
|
if (value == null || !Number.isFinite(value) || value <= 0) return '-'
|
||||||
|
return formatDurationMs(value * 1000)
|
||||||
|
}
|
||||||
|
|
||||||
function formatDurationMs(value: number | null) {
|
function formatDurationMs(value: number | null) {
|
||||||
if (value == null || !Number.isFinite(value) || value <= 0) return '-'
|
if (value == null || !Number.isFinite(value) || value <= 0) return '-'
|
||||||
const totalSeconds = Math.floor(value / 1000)
|
const totalSeconds = Math.floor(value / 1000)
|
||||||
@@ -399,6 +525,38 @@ function formatBytes(value: number) {
|
|||||||
return `${unit === 0 ? Math.round(size) : size.toFixed(1)} ${units[unit]}`
|
return `${unit === 0 ? Math.round(size) : size.toFixed(1)} ${units[unit]}`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatLoadAverage(one: number | null, five: number | null, fifteen: number | null) {
|
||||||
|
if (one == null && five == null && fifteen == null) return '-'
|
||||||
|
return [one, five, fifteen]
|
||||||
|
.map(value => (value == null || !Number.isFinite(value)) ? '-' : value.toFixed(value >= 10 ? 0 : 2))
|
||||||
|
.join(' / ')
|
||||||
|
}
|
||||||
|
|
||||||
|
function memoryMbToBytes(value: number | null) {
|
||||||
|
if (value == null || !Number.isFinite(value) || value <= 0) return null
|
||||||
|
return value * 1024 * 1024
|
||||||
|
}
|
||||||
|
|
||||||
|
function percentFromBytes(value: number | null, total: number | null) {
|
||||||
|
if (value == null || total == null || !Number.isFinite(value) || !Number.isFinite(total) || total <= 0) return null
|
||||||
|
return value * 100 / total
|
||||||
|
}
|
||||||
|
|
||||||
|
function usageTone(value: number | null, warnAt: number, badAt: number) {
|
||||||
|
if (value == null || !Number.isFinite(value)) return ''
|
||||||
|
if (value >= badAt) return 'text-destructive'
|
||||||
|
if (value >= warnAt) return 'text-yellow-600 dark:text-yellow-400'
|
||||||
|
return ''
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadTone(load: number | null, cpuCores: number | null) {
|
||||||
|
if (load == null || cpuCores == null || !Number.isFinite(load) || !Number.isFinite(cpuCores) || cpuCores <= 0) return ''
|
||||||
|
const ratio = load / cpuCores
|
||||||
|
if (ratio >= 1.5) return 'text-destructive'
|
||||||
|
if (ratio >= 1) return 'text-yellow-600 dark:text-yellow-400'
|
||||||
|
return ''
|
||||||
|
}
|
||||||
|
|
||||||
function formatTime(iso: string | null) {
|
function formatTime(iso: string | null) {
|
||||||
if (!iso) return '-'
|
if (!iso) return '-'
|
||||||
const date = new Date(iso)
|
const date = new Date(iso)
|
||||||
@@ -473,6 +631,63 @@ function eventTypeVariant(type: string) {
|
|||||||
|
|
||||||
function eventDetail(event: ProxyNodeEvent) {
|
function eventDetail(event: ProxyNodeEvent) {
|
||||||
const metadata = asRecord(event.event_metadata)
|
const metadata = asRecord(event.event_metadata)
|
||||||
|
if (event.event_type === 'tunnel_err') {
|
||||||
|
const category = stringField(metadata, 'category')
|
||||||
|
const message = stringField(metadata, 'message')
|
||||||
|
const summary = (category ? tunnelErrorSummary(category) : '') || stringField(metadata, 'summary') || category
|
||||||
|
if (summary && message) return `${summary}:${message}`
|
||||||
|
return summary || message || event.detail || '-'
|
||||||
|
}
|
||||||
return event.detail || stringField(metadata, 'message') || stringField(metadata, 'category') || '-'
|
return event.detail || stringField(metadata, 'message') || stringField(metadata, 'category') || '-'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function eventTooltip(event: ProxyNodeEvent) {
|
||||||
|
const metadata = asRecord(event.event_metadata)
|
||||||
|
const parts = [eventDetail(event)]
|
||||||
|
const category = stringField(metadata, 'category')
|
||||||
|
const action = (category ? tunnelErrorAction(category) : '') || stringField(metadata, 'operator_action')
|
||||||
|
if (action) parts.push(`建议:${action}`)
|
||||||
|
const component = stringField(metadata, 'component')
|
||||||
|
const severity = stringField(metadata, 'severity')
|
||||||
|
if (component || severity) parts.push([component, severity].filter(Boolean).join(' / '))
|
||||||
|
return parts.filter(Boolean).join('\n')
|
||||||
|
}
|
||||||
|
|
||||||
|
function tunnelErrorSummary(category: string) {
|
||||||
|
switch (category) {
|
||||||
|
case 'stale_timeout': return '超时未收到隧道入站帧'
|
||||||
|
case 'ws_write_error': return 'WebSocket 写入失败,对端重置或关闭'
|
||||||
|
case 'ws_ping_error': return 'WebSocket 保活 Ping 发送失败'
|
||||||
|
case 'ws_read_error': return 'WebSocket 读取失败'
|
||||||
|
case 'tunnel_connect_error': return '隧道建连失败'
|
||||||
|
case 'frame_decode_error': return '隧道帧解析失败'
|
||||||
|
case 'stream_dispatch_timeout': return '请求流分发超时'
|
||||||
|
case 'heartbeat_ack_empty': return '心跳确认为空'
|
||||||
|
case 'heartbeat_ack_parse': return '心跳确认解析失败'
|
||||||
|
case 'writer_task_panic': return '隧道写任务异常退出'
|
||||||
|
case 'writer_task_cancelled': return '隧道写任务被取消'
|
||||||
|
case 'dispatcher_error': return '隧道分发器错误'
|
||||||
|
default: return ''
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function tunnelErrorAction(category: string) {
|
||||||
|
switch (category) {
|
||||||
|
case 'stale_timeout': return '检查 gateway/反向代理 idle timeout、跨境链路抖动和 ping/pong 是否可达;网络抖动大时可调高 stale timeout。'
|
||||||
|
case 'ws_write_error': return '检查 gateway 是否重启、负载均衡/NAT/防火墙是否重置长连接,并确认 proxy 是否已自动重连。'
|
||||||
|
case 'ws_ping_error': return '检查中间代理是否清理空闲 WebSocket,或对端是否提前关闭连接。'
|
||||||
|
case 'ws_read_error': return '对照同一时间的 gateway 日志和网络监控,确认是否存在链路中断。'
|
||||||
|
case 'tunnel_connect_error': return '检查 Aether 地址、DNS、TLS、管理 token,以及 AETHER_PROXY_AETHER_PROXY_URL 配置。'
|
||||||
|
case 'frame_decode_error': return '检查 proxy/gateway 版本兼容性,确认中间层没有改写 WebSocket 二进制帧。'
|
||||||
|
case 'stream_dispatch_timeout': return '检查 proxy CPU/内存、并发上限和上游 provider 慢请求。'
|
||||||
|
case 'heartbeat_ack_empty':
|
||||||
|
case 'heartbeat_ack_parse':
|
||||||
|
return '检查 gateway 心跳处理日志和 proxy/gateway 版本兼容性。'
|
||||||
|
case 'writer_task_panic':
|
||||||
|
case 'writer_task_cancelled':
|
||||||
|
return '查看此前一条写入或 ping 错误,确认隧道重连循环仍在运行。'
|
||||||
|
case 'dispatcher_error': return '检查同一时间的请求流和 gateway tunnel 日志。'
|
||||||
|
default: return ''
|
||||||
|
}
|
||||||
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
Reference in New Issue
Block a user