feat(proxy): 节点状态简化、连接事件记录、可靠性指标与批量删除

- 移除 UNHEALTHY 中间状态,节点状态简化为 ONLINE/OFFLINE
- 新增 proxy_node_events 表记录 tunnel 连接/断开/错误事件
- 新增 failed_requests/dns_failures/stream_errors 可靠性指标(增量累加)
- tunnel 重连改为固定 1s 延迟,移除指数退避逻辑
- resolver/service 改为以 TunnelManager 内存状态判断节点可用性,避免 DB 竞态
- 修正 Claude cache_control 字段格式,使用 ttl 字段控制缓存时长
- 移除前端手动勾选 capability 的 UI,改为从价格配置自动推断
- 新增全局模型批量删除 API,替换前端并行单个删除
This commit is contained in:
fawney19
2026-02-28 13:52:32 +08:00
parent ecb16d345a
commit 54530faf03
25 changed files with 586 additions and 225 deletions
+6 -11
View File
@@ -202,19 +202,21 @@ pub struct Config {
#[arg(long, env = "AETHER_PROXY_LOG_JSON", default_value_t = false)]
pub log_json: bool,
/// WebSocket reconnect base delay in milliseconds
/// Deprecated: reconnect now uses a fixed 1s delay. Kept for config compatibility.
#[arg(
long,
env = "AETHER_PROXY_TUNNEL_RECONNECT_BASE_MS",
default_value_t = 500
default_value_t = 500,
hide = true
)]
pub tunnel_reconnect_base_ms: u64,
/// WebSocket reconnect max delay in milliseconds
/// Deprecated: reconnect now uses a fixed 1s delay. Kept for config compatibility.
#[arg(
long,
env = "AETHER_PROXY_TUNNEL_RECONNECT_MAX_MS",
default_value_t = 30000
default_value_t = 30000,
hide = true
)]
pub tunnel_reconnect_max_ms: u64,
@@ -288,13 +290,6 @@ impl Config {
if self.aether_retry_max_attempts == 0 {
anyhow::bail!("aether_retry_max_attempts must be >= 1");
}
if self.tunnel_reconnect_base_ms > self.tunnel_reconnect_max_ms {
anyhow::bail!(
"tunnel_reconnect_base_ms ({}) must be <= tunnel_reconnect_max_ms ({})",
self.tunnel_reconnect_base_ms,
self.tunnel_reconnect_max_ms
);
}
if self.upstream_connect_timeout_secs == 0 {
anyhow::bail!("upstream_connect_timeout_secs must be > 0");
}
-32
View File
@@ -1,6 +1,5 @@
//! WebSocket tunnel client: connect, authenticate, and run the tunnel.
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::Arc;
use std::time::Duration;
@@ -226,20 +225,6 @@ pub fn build_tls_config() -> rustls::ClientConfig {
.with_no_client_auth()
}
/// Calculate next reconnect delay with exponential backoff + jitter.
pub fn next_reconnect_delay(state: &Arc<AppState>, reconnect_attempts: &AtomicU32) -> Duration {
let attempt = reconnect_attempts.fetch_add(1, Ordering::Relaxed);
let base_ms = state.config.tunnel_reconnect_base_ms;
let max_ms = state.config.tunnel_reconnect_max_ms;
let delay_ms = base_ms.saturating_mul(1u64 << attempt.min(10)).min(max_ms);
let jitter = (delay_ms / 4).max(1);
let jitter_ms = rand_u64() % jitter;
Duration::from_millis(delay_ms + jitter_ms)
}
fn build_tunnel_url(server: &ServerContext) -> String {
let base = server.aether_url.trim_end_matches('/');
let ws_base = if base.starts_with("https://") {
@@ -251,20 +236,3 @@ fn build_tunnel_url(server: &ServerContext) -> String {
};
format!("{}/api/internal/proxy-tunnel", ws_base)
}
/// Simple pseudo-random u64 (no external crate needed).
fn rand_u64() -> u64 {
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{SystemTime, UNIX_EPOCH};
static COUNTER: AtomicU64 = AtomicU64::new(0);
let seed = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_nanos() as u64;
let cnt = COUNTER.fetch_add(1, Ordering::Relaxed);
let mut x = seed ^ cnt;
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
x
}
+6 -36
View File
@@ -5,7 +5,6 @@ pub mod protocol;
pub mod stream_handler;
pub mod writer;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::Arc;
use std::time::Duration;
@@ -14,10 +13,9 @@ use tracing::{error, info};
use crate::state::{AppState, ServerContext};
/// Minimum connection duration (seconds) to consider a session "stable".
/// If a connection lasts shorter than this, the backoff counter is NOT reset,
/// preventing rapid reconnect loops on persistently bad networks.
const MIN_STABLE_DURATION: Duration = Duration::from_secs(30);
/// Fixed reconnect delay -- short enough for fast recovery, long enough to
/// avoid CPU spin when the network is completely down.
const RECONNECT_DELAY: Duration = Duration::from_secs(1);
/// Run the tunnel mode main loop (connect, dispatch, reconnect).
///
@@ -31,42 +29,17 @@ pub async fn run(
) {
info!(server = %server.server_label, conn = conn_idx, "starting tunnel");
// Per-connection reconnect counter (avoids N connections interfering
// with each other's backoff via the shared ServerContext field).
let reconnect_attempts = AtomicU32::new(0);
loop {
let connect_start = tokio::time::Instant::now();
match client::connect_and_run(state, server, conn_idx, &mut shutdown).await {
Ok(client::TunnelOutcome::Shutdown) => {
info!(server = %server.server_label, conn = conn_idx, "tunnel shut down gracefully");
return;
}
Ok(client::TunnelOutcome::Disconnected) => {
let duration = connect_start.elapsed();
if duration >= MIN_STABLE_DURATION {
// Stable session -- reset backoff for quick reconnect
reconnect_attempts.store(0, Ordering::Release);
info!(
server = %server.server_label,
conn = conn_idx,
duration_secs = duration.as_secs(),
"tunnel disconnected after stable session"
);
} else {
// Short-lived session -- keep backoff increasing
info!(
server = %server.server_label,
conn = conn_idx,
duration_secs = duration.as_secs(),
"tunnel disconnected quickly, increasing backoff"
);
}
info!(server = %server.server_label, conn = conn_idx, "tunnel disconnected, reconnecting");
}
Err(e) => {
// Connection failed -- keep backoff increasing
error!(server = %server.server_label, conn = conn_idx, error = %e, "tunnel connection lost");
error!(server = %server.server_label, conn = conn_idx, error = %e, "tunnel connection error, reconnecting");
}
}
@@ -75,11 +48,8 @@ pub async fn run(
return;
}
let delay = client::next_reconnect_delay(state, &reconnect_attempts);
info!(server = %server.server_label, conn = conn_idx, delay_ms = delay.as_millis(), "reconnecting tunnel");
tokio::select! {
_ = tokio::time::sleep(delay) => {}
_ = tokio::time::sleep(RECONNECT_DELAY) => {}
_ = shutdown.changed() => {
info!(server = %server.server_label, conn = conn_idx, "shutdown requested during reconnect wait");
return;