feat(proxy): surface node resource diagnostics

This commit is contained in:
fawney19
2026-05-11 02:59:18 +08:00
parent cc4512fbbb
commit b813498e40
12 changed files with 579 additions and 15 deletions
+3
View File
@@ -158,6 +158,7 @@ pub async fn run(mut config: Config, servers: Vec<ServerEntry>) -> anyhow::Resul
// Build a profile-keyed Hyper client pool for tunnel upstream requests.
let upstream_client_pool =
upstream_client::UpstreamClientPool::new(Arc::clone(&config), Arc::clone(&dns_cache));
let resource_monitor = Arc::new(hardware::RuntimeResourceMonitor::new());
// Register with each Aether server and build per-server contexts.
// Wrapped in Arc<Mutex> so retry_failed_registrations can append later.
@@ -218,6 +219,7 @@ pub async fn run(mut config: Config, servers: Vec<ServerEntry>) -> anyhow::Resul
dns_cache,
upstream_client_pool,
tunnel_tls_config,
resource_monitor,
stream_gate: None,
distributed_stream_gate: None,
};
@@ -947,6 +949,7 @@ mod tests {
dns_cache,
upstream_client_pool,
tunnel_tls_config: Arc::new(crate::tunnel::client::build_tls_config()),
resource_monitor: Arc::new(crate::hardware::RuntimeResourceMonitor::new()),
stream_gate: None,
distributed_stream_gate: None,
})
+115 -1
View File
@@ -1,5 +1,8 @@
use std::sync::Mutex;
use std::time::{SystemTime, UNIX_EPOCH};
use serde::Serialize;
use sysinfo::System;
use sysinfo::{get_current_pid, Pid, ProcessesToUpdate, System};
use tracing::info;
/// Hardware information collected at startup.
@@ -17,6 +20,102 @@ pub struct HardwareInfo {
pub estimated_max_concurrency: u64,
}
/// Runtime resource usage sampled during heartbeat reporting.
#[derive(Debug, Clone, Serialize)]
pub struct RuntimeResourceSnapshot {
pub sampled_at_unix_secs: u64,
pub system_cpu_usage_percent: f64,
pub process_cpu_usage_percent: f64,
pub memory_total_bytes: u64,
pub memory_used_bytes: u64,
pub memory_available_bytes: u64,
pub memory_used_percent: f64,
pub process_memory_bytes: u64,
pub process_virtual_memory_bytes: u64,
pub process_memory_percent: f64,
pub load_average_1m: f64,
pub load_average_5m: f64,
pub load_average_15m: f64,
pub system_uptime_secs: u64,
pub process_uptime_secs: Option<u64>,
}
/// Small, reusable sysinfo monitor. Keeping it alive between samples makes CPU
/// usage deltas meaningful without re-enumerating the whole machine every time.
pub struct RuntimeResourceMonitor {
system: Mutex<System>,
current_pid: Option<Pid>,
}
impl RuntimeResourceMonitor {
pub fn new() -> Self {
let mut system = System::new_all();
let current_pid = get_current_pid().ok();
if let Some(pid) = current_pid {
system.refresh_processes(ProcessesToUpdate::Some(&[pid]), true);
}
system.refresh_cpu_usage();
system.refresh_memory();
Self {
system: Mutex::new(system),
current_pid,
}
}
pub fn snapshot(&self) -> RuntimeResourceSnapshot {
let mut system = match self.system.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
system.refresh_cpu_usage();
system.refresh_memory();
if let Some(pid) = self.current_pid {
system.refresh_processes(ProcessesToUpdate::Some(&[pid]), true);
}
let memory_total_bytes = system.total_memory();
let memory_used_bytes = system.used_memory();
let memory_available_bytes = system.available_memory();
let (
process_cpu_usage_percent,
process_memory_bytes,
process_virtual_memory_bytes,
process_uptime_secs,
) = self
.current_pid
.and_then(|pid| system.process(pid))
.map(|process| {
(
process.cpu_usage() as f64,
process.memory(),
process.virtual_memory(),
Some(process.run_time()),
)
})
.unwrap_or((0.0, 0, 0, None));
let load = System::load_average();
RuntimeResourceSnapshot {
sampled_at_unix_secs: current_unix_secs(),
system_cpu_usage_percent: system.global_cpu_usage() as f64,
process_cpu_usage_percent,
memory_total_bytes,
memory_used_bytes,
memory_available_bytes,
memory_used_percent: ratio_percent(memory_used_bytes, memory_total_bytes),
process_memory_bytes,
process_virtual_memory_bytes,
process_memory_percent: ratio_percent(process_memory_bytes, memory_total_bytes),
load_average_1m: load.one,
load_average_5m: load.five,
load_average_15m: load.fifteen,
system_uptime_secs: System::uptime(),
process_uptime_secs,
}
}
}
/// Collect hardware information and estimate max concurrency.
///
/// Should be called once at startup -- hardware does not change at runtime.
@@ -77,3 +176,18 @@ fn get_fd_limit() -> u64 {
// Fallback for non-unix or error
1024
}
fn ratio_percent(value: u64, total: u64) -> f64 {
if total == 0 {
0.0
} else {
value as f64 * 100.0 / total as f64
}
}
fn current_unix_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs())
.unwrap_or(0)
}
+105 -2
View File
@@ -10,6 +10,7 @@ use aether_runtime::{AdmissionPermit, ConcurrencyError, ConcurrencyGate, Concurr
use aether_runtime_state::{RuntimeSemaphore, RuntimeSemaphoreError, RuntimeSemaphoreSnapshot};
use crate::config::Config;
use crate::hardware::RuntimeResourceMonitor;
use crate::registration::client::AetherClient;
use crate::runtime::SharedDynamicConfig;
use crate::target_filter::DnsCache;
@@ -24,6 +25,8 @@ pub struct AppState {
pub upstream_client_pool: UpstreamClientPool,
/// Shared TLS config for tunnel WebSocket connections (avoids re-parsing root CAs on each reconnect).
pub tunnel_tls_config: Arc<rustls::ClientConfig>,
/// Runtime CPU/memory monitor sampled by heartbeat payloads.
pub resource_monitor: Arc<RuntimeResourceMonitor>,
/// Optional per-process stream admission gate.
pub stream_gate: Option<Arc<ConcurrencyGate>>,
/// Optional cross-instance stream admission gate.
@@ -96,6 +99,10 @@ pub struct TunnelErrorEvent {
pub timestamp_unix_secs: u64,
pub category: String,
pub message: String,
pub severity: String,
pub component: String,
pub summary: String,
pub operator_action: String,
}
#[derive(Debug, Clone, Copy, Default)]
@@ -229,11 +236,18 @@ impl TunnelMetrics {
pub fn record_error(&self, category: &str, message: &str) {
self.error_events_total.fetch_add(1, Ordering::Release);
let category = normalize_error_field(category, TUNNEL_ERROR_CATEGORY_MAX_CHARS, "unknown");
let message = normalize_error_field(message, TUNNEL_ERROR_MESSAGE_MAX_CHARS, "n/a");
let diagnostic = classify_tunnel_error(category.as_str(), message.as_str());
let event = TunnelErrorEvent {
timestamp_unix_secs: now_unix_secs(),
category: normalize_error_field(category, TUNNEL_ERROR_CATEGORY_MAX_CHARS, "unknown"),
message: normalize_error_field(message, TUNNEL_ERROR_MESSAGE_MAX_CHARS, "n/a"),
category,
message,
severity: diagnostic.severity.to_string(),
component: diagnostic.component.to_string(),
summary: diagnostic.summary.to_string(),
operator_action: diagnostic.operator_action.to_string(),
};
let mut recent_errors = match self.recent_errors.lock() {
@@ -302,6 +316,95 @@ fn normalize_error_field(value: &str, max_chars: usize, fallback: &str) -> Strin
normalized.chars().take(max_chars).collect()
}
struct TunnelErrorDiagnostic {
severity: &'static str,
component: &'static str,
summary: &'static str,
operator_action: &'static str,
}
fn classify_tunnel_error(category: &str, _message: &str) -> TunnelErrorDiagnostic {
match category {
"stale_timeout" => TunnelErrorDiagnostic {
severity: "warning",
component: "tunnel_read",
summary: "No inbound tunnel frames before stale timeout",
operator_action:
"Check gateway or reverse-proxy idle timeouts, packet loss, and WebSocket ping/pong reachability. Increase AETHER_PROXY_TUNNEL_STALE_TIMEOUT_MS if the network is high-latency.",
},
"ws_write_error" => TunnelErrorDiagnostic {
severity: "error",
component: "tunnel_write",
summary: "WebSocket write failed because the peer closed or reset the connection",
operator_action:
"Check gateway restarts, load balancer resets, NAT/firewall connection tracking, and whether the proxy is reconnecting successfully.",
},
"ws_ping_error" => TunnelErrorDiagnostic {
severity: "error",
component: "tunnel_write",
summary: "WebSocket keepalive ping could not be sent",
operator_action:
"Check whether the peer closed the socket or an intermediary is dropping idle WebSocket connections.",
},
"ws_read_error" => TunnelErrorDiagnostic {
severity: "error",
component: "tunnel_read",
summary: "WebSocket read failed",
operator_action:
"Check gateway logs and network stability around the same timestamp; compare with reconnect and heartbeat ACK counters.",
},
"tunnel_connect_error" => TunnelErrorDiagnostic {
severity: "critical",
component: "tunnel_connect",
summary: "Tunnel connection attempt failed",
operator_action:
"Check Aether URL reachability, DNS, TLS, management token validity, and any configured AETHER_PROXY_AETHER_PROXY_URL.",
},
"frame_decode_error" => TunnelErrorDiagnostic {
severity: "error",
component: "tunnel_protocol",
summary: "Received tunnel frame could not be decoded",
operator_action:
"Check proxy and gateway version compatibility and whether traffic is being modified by an intermediary.",
},
"stream_dispatch_timeout" => TunnelErrorDiagnostic {
severity: "warning",
component: "stream_dispatch",
summary: "Request body frame could not be delivered to its stream handler in time",
operator_action:
"Check proxy CPU, memory, stream concurrency saturation, and slow upstream provider requests.",
},
"heartbeat_ack_empty" | "heartbeat_ack_parse" => TunnelErrorDiagnostic {
severity: "warning",
component: "heartbeat",
summary: "Heartbeat ACK from gateway was missing or invalid",
operator_action:
"Check gateway heartbeat handler logs and proxy/gateway version compatibility.",
},
"writer_task_panic" | "writer_task_cancelled" => TunnelErrorDiagnostic {
severity: "error",
component: "tunnel_writer",
summary: "Tunnel writer task exited unexpectedly",
operator_action:
"Check proxy logs for the preceding write or ping error and confirm the tunnel reconnect loop is active.",
},
"dispatcher_error" => TunnelErrorDiagnostic {
severity: "error",
component: "tunnel_dispatcher",
summary: "Tunnel dispatcher exited with an error",
operator_action:
"Check the proxied request stream and gateway tunnel logs around the same timestamp.",
},
_ => TunnelErrorDiagnostic {
severity: "info",
component: "tunnel",
summary: "Tunnel reported an unclassified error",
operator_action:
"Inspect the raw message and compare it with proxy, gateway, and network logs at the same time.",
},
}
}
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
pub enum ProxyAdmissionError {
#[error("proxy stream admission saturated at {limit} for gate {gate}")]
+73 -5
View File
@@ -240,6 +240,7 @@ async fn build_heartbeat_payload(
let node_id = server.node_id.read().unwrap().clone();
let tunnel_snapshot = server.tunnel_metrics.snapshot();
let recent_errors = server.tunnel_metrics.recent_errors(8);
let resource_usage = state.resource_monitor.snapshot();
let avg_latency_ms = if snapshot.requests > 0 {
Some(snapshot.latency_ns as f64 / snapshot.requests as f64 / 1_000_000.0)
@@ -291,6 +292,7 @@ async fn build_heartbeat_payload(
"proxy_metadata": {
"version": CURRENT_VERSION,
"admission": admission,
"resource_usage": resource_usage,
"tunnel_metrics": {
"connect_attempts": tunnel_snapshot.connect_attempts,
"connect_successes": tunnel_snapshot.connect_successes,
@@ -419,13 +421,13 @@ mod tests {
use arc_swap::ArcSwap;
use clap::Parser;
use super::{handle_ack, AckDecision};
use super::{build_heartbeat_payload, handle_ack, AckDecision, HeartbeatSnapshot};
use crate::registration::client::AetherClient;
use crate::runtime::DynamicConfig;
use crate::state::{ProxyMetrics, ServerContext, TunnelMetrics};
use crate::state::{AppState, ProxyMetrics, ServerContext, TunnelMetrics};
fn sample_server() -> Arc<ServerContext> {
let config = Arc::new(crate::config::Config::parse_from([
fn sample_config() -> Arc<crate::config::Config> {
Arc::new(crate::config::Config::parse_from([
"aether-proxy",
"--aether-url",
"https://example.com",
@@ -433,7 +435,11 @@ mod tests {
"ae_test",
"--node-name",
"proxy-test",
]));
]))
}
fn sample_server() -> Arc<ServerContext> {
let config = sample_config();
Arc::new(ServerContext {
server_label: "heartbeat-test".to_string(),
aether_url: config.aether_url.clone(),
@@ -452,6 +458,24 @@ mod tests {
})
}
fn sample_state(config: Arc<crate::config::Config>) -> AppState {
let dns_cache = Arc::new(crate::target_filter::DnsCache::new(
std::time::Duration::from_secs(config.dns_cache_ttl_secs),
config.dns_cache_capacity,
));
AppState {
config: Arc::clone(&config),
dns_cache: Arc::clone(&dns_cache),
upstream_client_pool: crate::upstream_client::UpstreamClientPool::new(
config, dns_cache,
),
tunnel_tls_config: Arc::new(crate::tunnel::client::build_tls_config()),
resource_monitor: Arc::new(crate::hardware::RuntimeResourceMonitor::new()),
stream_gate: None,
distributed_stream_gate: None,
}
}
#[test]
fn heartbeat_ack_requires_heartbeat_id() {
let server = sample_server();
@@ -481,4 +505,48 @@ mod tests {
));
assert_eq!(server.dynamic.load().heartbeat_interval, 9);
}
#[tokio::test]
async fn heartbeat_payload_reports_resource_usage_and_tunnel_error_diagnostics() {
let config = sample_config();
let server = sample_server();
server
.tunnel_metrics
.record_error("ws_write_error", "IO error: Connection reset by peer");
let state = sample_state(config);
let payload = build_heartbeat_payload(
&state,
&server,
"session-1",
42,
HeartbeatSnapshot::default(),
)
.await;
let payload: serde_json::Value =
serde_json::from_slice(&payload).expect("heartbeat payload should be JSON");
let resource_usage = payload
.pointer("/proxy_metadata/resource_usage")
.and_then(serde_json::Value::as_object)
.expect("resource usage should be reported");
assert!(resource_usage.contains_key("system_cpu_usage_percent"));
assert!(resource_usage.contains_key("process_memory_bytes"));
let recent_error = payload
.pointer("/proxy_metadata/recent_tunnel_errors/0")
.and_then(serde_json::Value::as_object)
.expect("recent tunnel error should be reported");
assert_eq!(
recent_error
.get("component")
.and_then(serde_json::Value::as_str),
Some("tunnel_write")
);
assert_eq!(
recent_error
.get("severity")
.and_then(serde_json::Value::as_str),
Some("error")
);
}
}
+1
View File
@@ -462,6 +462,7 @@ mod tests {
dns_cache,
upstream_client_pool,
tunnel_tls_config: Arc::new(crate::tunnel::client::build_tls_config()),
resource_monitor: Arc::new(crate::hardware::RuntimeResourceMonitor::new()),
stream_gate: None,
distributed_stream_gate: None,
})
@@ -2284,6 +2284,7 @@ mod tests {
dns_cache,
upstream_client_pool,
tunnel_tls_config: Arc::new(build_tls_config()),
resource_monitor: Arc::new(crate::hardware::RuntimeResourceMonitor::new()),
stream_gate,
distributed_stream_gate,
})
@@ -2314,6 +2315,7 @@ mod tests {
dns_cache,
upstream_client_pool,
tunnel_tls_config: Arc::new(build_tls_config()),
resource_monitor: Arc::new(crate::hardware::RuntimeResourceMonitor::new()),
stream_gate: None,
distributed_stream_gate: None,
})
@@ -650,6 +650,10 @@ impl ProxyNodeWriteRepository for InMemoryProxyNodeRepository {
"source": "heartbeat",
"category": error.category,
"message": error.message,
"severity": error.severity.as_deref(),
"component": error.component.as_deref(),
"summary": error.summary.as_deref(),
"operator_action": error.operator_action.as_deref(),
"timestamp_unix_secs": error.timestamp_unix_secs,
})),
created_at_unix_ms: Some(if error.timestamp_unix_secs == 0 {
@@ -836,6 +836,10 @@ WHERE is_manual = 0
"source": "heartbeat",
"category": error.category,
"message": error.message,
"severity": error.severity.as_deref(),
"component": error.component.as_deref(),
"summary": error.summary.as_deref(),
"operator_action": error.operator_action.as_deref(),
"timestamp_unix_secs": error.timestamp_unix_secs,
});
self.insert_event(
@@ -1224,6 +1224,10 @@ impl ProxyNodeWriteRepository for SqlxProxyNodeRepository {
"source": "heartbeat",
"category": error.category,
"message": error.message,
"severity": error.severity.as_deref(),
"component": error.component.as_deref(),
"summary": error.summary.as_deref(),
"operator_action": error.operator_action.as_deref(),
"timestamp_unix_secs": error.timestamp_unix_secs,
});
self.insert_event(
@@ -838,6 +838,10 @@ WHERE is_manual = 0
"source": "heartbeat",
"category": error.category,
"message": error.message,
"severity": error.severity.as_deref(),
"component": error.component.as_deref(),
"summary": error.summary.as_deref(),
"operator_action": error.operator_action.as_deref(),
"timestamp_unix_secs": error.timestamp_unix_secs,
});
self.insert_event(
@@ -324,6 +324,10 @@ pub struct TunnelErrorEventRecord {
pub timestamp_unix_secs: u64,
pub category: String,
pub message: String,
pub severity: Option<String>,
pub component: Option<String>,
pub summary: Option<String>,
pub operator_action: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
@@ -417,7 +421,11 @@ pub fn build_tunnel_metrics_sample(
}
pub fn build_tunnel_error_event_detail(event: &TunnelErrorEventRecord) -> String {
format!("[{}] {}", event.category, event.message)
format!(
"[{}] {}",
event.category,
event.summary.as_deref().unwrap_or(event.message.as_str())
)
}
pub fn normalize_proxy_metadata(
@@ -497,6 +505,10 @@ fn extract_recent_tunnel_errors(proxy_metadata: Option<&Value>) -> Vec<TunnelErr
.and_then(Value::as_str)
.unwrap_or("n/a")
.to_string(),
severity: json_string(item.get("severity")),
component: json_string(item.get("component")),
summary: json_string(item.get("summary")),
operator_action: json_string(item.get("operator_action")),
})
})
.collect::<Vec<_>>()
@@ -512,6 +524,14 @@ fn json_u64(value: Option<&Value>) -> Option<u64> {
})
}
fn json_string(value: Option<&Value>) -> Option<String> {
value
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToString::to_string)
}
fn counter_delta_u64(previous: Option<u64>, current: u64) -> u64 {
match previous {
Some(previous) if current >= previous => current - previous,
@@ -726,10 +746,11 @@ mod tests {
use serde_json::json;
use super::{
bucket_start_unix_secs, build_tunnel_metrics_sample, normalize_proxy_node_scheduling_state,
proxy_node_accepts_new_tunnels, proxy_reported_version,
reconcile_remote_config_after_heartbeat, remote_config_scheduling_state,
remote_config_upgrade_target, ProxyNodeMetricsStep, StoredProxyNode,
bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
normalize_proxy_node_scheduling_state, proxy_node_accepts_new_tunnels,
proxy_reported_version, reconcile_remote_config_after_heartbeat,
remote_config_scheduling_state, remote_config_upgrade_target, ProxyNodeMetricsStep,
StoredProxyNode,
};
#[test]
@@ -842,7 +863,15 @@ mod tests {
},
"recent_tunnel_errors": [
{"timestamp_unix_secs": 100, "category": "older", "message": "old"},
{"timestamp_unix_secs": 101, "category": "newer", "message": "new"}
{
"timestamp_unix_secs": 101,
"category": "newer",
"message": "new",
"severity": "error",
"component": "tunnel_write",
"summary": "WebSocket write failed because the peer closed or reset the connection",
"operator_action": "Check gateway restarts and network resets."
}
]
});
@@ -860,6 +889,19 @@ mod tests {
assert_eq!(sample.ws_out_frames_delta, 3);
assert_eq!(sample.recent_error_events.len(), 2);
assert_eq!(sample.recent_error_events[0].category, "older");
assert_eq!(sample.recent_error_events[0].summary, None);
assert_eq!(
sample.recent_error_events[1].severity.as_deref(),
Some("error")
);
assert_eq!(
sample.recent_error_events[1].component.as_deref(),
Some("tunnel_write")
);
assert_eq!(
build_tunnel_error_event_detail(&sample.recent_error_events[1]),
"[newer] WebSocket write failed because the peer closed or reset the connection"
);
}
#[test]
@@ -138,7 +138,10 @@
>
{{ eventTypeLabel(event.event_type) }}
</Badge>
<span class="text-muted-foreground truncate flex-1">{{ eventDetail(event) }}</span>
<span
class="text-muted-foreground truncate flex-1"
:title="eventTooltip(event)"
>{{ eventDetail(event) }}</span>
<span class="text-[10px] text-muted-foreground/70 tabular-nums shrink-0">{{ formatTime(event.created_at || null) }}</span>
</div>
</div>
@@ -146,6 +149,35 @@
</div>
<div class="grid gap-4 lg:grid-cols-2">
<section class="rounded-lg border border-border/50 bg-background/70 p-3 min-w-0">
<h5 class="text-xs font-semibold mb-3">
硬件与资源
</h5>
<div class="grid grid-cols-2 sm:grid-cols-3 gap-x-4 gap-y-2">
<div
v-for="item in resourceItems"
:key="item.label"
class="min-w-0"
>
<div class="text-[11px] text-muted-foreground truncate">
{{ item.label }}
</div>
<div
class="text-xs font-medium tabular-nums truncate mt-0.5"
:class="item.tone"
>
{{ item.value }}
</div>
<div
v-if="item.hint"
class="text-[10px] text-muted-foreground truncate mt-0.5"
>
{{ item.hint }}
</div>
</div>
</div>
</section>
<section class="rounded-lg border border-border/50 bg-background/70 p-3 min-w-0">
<h5 class="text-xs font-semibold mb-3">
实时快照
@@ -234,6 +266,8 @@ const metricsSummary = computed(() => metrics.value?.summary ?? null)
const bucketItems = computed(() => metrics.value?.items.slice(-24) ?? [])
const recentEvents = computed(() => props.state?.events.slice(0, 8) ?? [])
const metadata = computed(() => asRecord(detailNode.value.proxy_metadata))
const hardwareInfo = computed(() => asRecord(detailNode.value.hardware_info))
const resourceUsage = computed(() => asRecord(metadata.value?.resource_usage))
const tunnelMetrics = computed(() => asRecord(metadata.value?.tunnel_metrics))
const loadedText = computed(() => {
@@ -300,6 +334,82 @@ const snapshotItems = computed(() => {
]
})
const resourceItems = computed(() => {
const hardware = hardwareInfo.value
const resource = resourceUsage.value
const memoryTotalBytes = numberField(resource, 'memory_total_bytes') ?? memoryMbToBytes(numberField(hardware, 'total_memory_mb'))
const memoryUsedBytes = numberField(resource, 'memory_used_bytes')
const memoryUsedPercent = numberField(resource, 'memory_used_percent') ?? percentFromBytes(memoryUsedBytes, memoryTotalBytes)
const processMemoryBytes = numberField(resource, 'process_memory_bytes')
const processMemoryPercent = numberField(resource, 'process_memory_percent')
const systemCpu = numberField(resource, 'system_cpu_usage_percent')
const processCpu = numberField(resource, 'process_cpu_usage_percent')
const load1 = numberField(resource, 'load_average_1m')
const load5 = numberField(resource, 'load_average_5m')
const load15 = numberField(resource, 'load_average_15m')
const cpuCores = numberField(hardware, 'cpu_cores')
return [
{
label: '系统 CPU',
value: formatUsagePercent(systemCpu),
hint: '主机整体',
tone: usageTone(systemCpu, 70, 90),
},
{
label: '系统内存',
value: formatUsagePercent(memoryUsedPercent),
hint: memoryUsedBytes == null || memoryTotalBytes == null
? ''
: `${formatBytes(memoryUsedBytes)} / ${formatBytes(memoryTotalBytes)}`,
tone: usageTone(memoryUsedPercent, 75, 90),
},
{
label: 'Proxy CPU',
value: formatUsagePercent(processCpu),
hint: '当前进程',
tone: usageTone(processCpu, 70, 90),
},
{
label: 'Proxy 内存',
value: processMemoryBytes == null ? '-' : formatBytes(processMemoryBytes),
hint: processMemoryPercent == null ? '' : formatUsagePercent(processMemoryPercent),
tone: usageTone(processMemoryPercent, 5, 15),
},
{
label: '负载',
value: formatLoadAverage(load1, load5, load15),
hint: cpuCores == null ? '1/5/15m' : `${formatNumber(cpuCores)} 核`,
tone: loadTone(load1, cpuCores),
},
{
label: '运行时间',
value: formatDurationSeconds(numberField(resource, 'process_uptime_secs')),
hint: 'proxy 进程',
tone: '',
},
{
label: 'CPU 核心',
value: cpuCores == null ? '-' : formatNumber(cpuCores),
hint: stringField(hardware, 'os_info') || '',
tone: '',
},
{
label: '物理内存',
value: memoryTotalBytes == null ? '-' : formatBytes(memoryTotalBytes),
hint: '启动采集',
tone: '',
},
{
label: 'FD 限制',
value: formatOptionalNumber(numberField(hardware, 'fd_limit')),
hint: detailNode.value.estimated_max_concurrency == null
? ''
: `容量 ${formatNumber(detailNode.value.estimated_max_concurrency)}`,
tone: '',
},
]
})
const tunnelCounterItems = computed(() => [
{ label: '建连尝试', value: formatTunnelNumber('connect_attempts') },
{ label: '建连成功', value: formatTunnelNumber('connect_successes') },
@@ -352,6 +462,10 @@ function formatNumber(value: number) {
return String(Math.round(value))
}
function formatOptionalNumber(value: number | null) {
return value == null ? '-' : formatNumber(value)
}
function formatDecimal(value: number | null) {
if (value == null || !Number.isFinite(value)) return '-'
if (value === 0) return '0'
@@ -364,12 +478,24 @@ function formatPercent(value: number | null) {
return `${(value * 100).toFixed(value >= 0.995 || value === 0 ? 0 : 1)}%`
}
function formatUsagePercent(value: number | null) {
if (value == null || !Number.isFinite(value)) return '-'
if (value >= 100) return `${Math.round(value)}%`
if (value >= 10) return `${value.toFixed(0)}%`
return `${value.toFixed(1)}%`
}
function formatMs(value: number | null) {
if (value == null || !Number.isFinite(value) || value <= 0) return '-'
if (value >= 1000) return `${(value / 1000).toFixed(1)}s`
return `${Math.round(value)}ms`
}
function formatDurationSeconds(value: number | null) {
if (value == null || !Number.isFinite(value) || value <= 0) return '-'
return formatDurationMs(value * 1000)
}
function formatDurationMs(value: number | null) {
if (value == null || !Number.isFinite(value) || value <= 0) return '-'
const totalSeconds = Math.floor(value / 1000)
@@ -399,6 +525,38 @@ function formatBytes(value: number) {
return `${unit === 0 ? Math.round(size) : size.toFixed(1)} ${units[unit]}`
}
function formatLoadAverage(one: number | null, five: number | null, fifteen: number | null) {
if (one == null && five == null && fifteen == null) return '-'
return [one, five, fifteen]
.map(value => (value == null || !Number.isFinite(value)) ? '-' : value.toFixed(value >= 10 ? 0 : 2))
.join(' / ')
}
function memoryMbToBytes(value: number | null) {
if (value == null || !Number.isFinite(value) || value <= 0) return null
return value * 1024 * 1024
}
function percentFromBytes(value: number | null, total: number | null) {
if (value == null || total == null || !Number.isFinite(value) || !Number.isFinite(total) || total <= 0) return null
return value * 100 / total
}
function usageTone(value: number | null, warnAt: number, badAt: number) {
if (value == null || !Number.isFinite(value)) return ''
if (value >= badAt) return 'text-destructive'
if (value >= warnAt) return 'text-yellow-600 dark:text-yellow-400'
return ''
}
function loadTone(load: number | null, cpuCores: number | null) {
if (load == null || cpuCores == null || !Number.isFinite(load) || !Number.isFinite(cpuCores) || cpuCores <= 0) return ''
const ratio = load / cpuCores
if (ratio >= 1.5) return 'text-destructive'
if (ratio >= 1) return 'text-yellow-600 dark:text-yellow-400'
return ''
}
function formatTime(iso: string | null) {
if (!iso) return '-'
const date = new Date(iso)
@@ -473,6 +631,63 @@ function eventTypeVariant(type: string) {
function eventDetail(event: ProxyNodeEvent) {
const metadata = asRecord(event.event_metadata)
if (event.event_type === 'tunnel_err') {
const category = stringField(metadata, 'category')
const message = stringField(metadata, 'message')
const summary = (category ? tunnelErrorSummary(category) : '') || stringField(metadata, 'summary') || category
if (summary && message) return `${summary}:${message}`
return summary || message || event.detail || '-'
}
return event.detail || stringField(metadata, 'message') || stringField(metadata, 'category') || '-'
}
function eventTooltip(event: ProxyNodeEvent) {
const metadata = asRecord(event.event_metadata)
const parts = [eventDetail(event)]
const category = stringField(metadata, 'category')
const action = (category ? tunnelErrorAction(category) : '') || stringField(metadata, 'operator_action')
if (action) parts.push(`建议:${action}`)
const component = stringField(metadata, 'component')
const severity = stringField(metadata, 'severity')
if (component || severity) parts.push([component, severity].filter(Boolean).join(' / '))
return parts.filter(Boolean).join('\n')
}
function tunnelErrorSummary(category: string) {
switch (category) {
case 'stale_timeout': return '超时未收到隧道入站帧'
case 'ws_write_error': return 'WebSocket 写入失败,对端重置或关闭'
case 'ws_ping_error': return 'WebSocket 保活 Ping 发送失败'
case 'ws_read_error': return 'WebSocket 读取失败'
case 'tunnel_connect_error': return '隧道建连失败'
case 'frame_decode_error': return '隧道帧解析失败'
case 'stream_dispatch_timeout': return '请求流分发超时'
case 'heartbeat_ack_empty': return '心跳确认为空'
case 'heartbeat_ack_parse': return '心跳确认解析失败'
case 'writer_task_panic': return '隧道写任务异常退出'
case 'writer_task_cancelled': return '隧道写任务被取消'
case 'dispatcher_error': return '隧道分发器错误'
default: return ''
}
}
function tunnelErrorAction(category: string) {
switch (category) {
case 'stale_timeout': return '检查 gateway/反向代理 idle timeout、跨境链路抖动和 ping/pong 是否可达;网络抖动大时可调高 stale timeout。'
case 'ws_write_error': return '检查 gateway 是否重启、负载均衡/NAT/防火墙是否重置长连接,并确认 proxy 是否已自动重连。'
case 'ws_ping_error': return '检查中间代理是否清理空闲 WebSocket,或对端是否提前关闭连接。'
case 'ws_read_error': return '对照同一时间的 gateway 日志和网络监控,确认是否存在链路中断。'
case 'tunnel_connect_error': return '检查 Aether 地址、DNS、TLS、管理 token,以及 AETHER_PROXY_AETHER_PROXY_URL 配置。'
case 'frame_decode_error': return '检查 proxy/gateway 版本兼容性,确认中间层没有改写 WebSocket 二进制帧。'
case 'stream_dispatch_timeout': return '检查 proxy CPU/内存、并发上限和上游 provider 慢请求。'
case 'heartbeat_ack_empty':
case 'heartbeat_ack_parse':
return '检查 gateway 心跳处理日志和 proxy/gateway 版本兼容性。'
case 'writer_task_panic':
case 'writer_task_cancelled':
return '查看此前一条写入或 ping 错误,确认隧道重连循环仍在运行。'
case 'dispatcher_error': return '检查同一时间的请求流和 gateway tunnel 日志。'
default: return ''
}
}
</script>