mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-01 17:00:21 +08:00
feat(proxy-tunnel): 实现隧道连接池与 TCP 底层优化
Rust 端: - 支持每个 server 多条并行 WebSocket 连接 (tunnel_connections 配置) - 手动控制 TCP 连接: connect/handshake 超时、keepalive、NODELAY (socket2) - 预构建共享 TLS ClientConfig 避免每次重连重新解析根证书 - 增加 stale timeout 检测无数据连接,智能 backoff 按连接存活时长重置 - 每条连接独立 reconnect 计数器,仅主连接 (conn_idx=0) 发送心跳 - dispatcher 的错误帧和 PONG 改用 try_send 避免阻塞读循环 - stream_handler 增加 frame 发送超时保护防止写阻塞 Python 端: - TunnelManager 改为连接池,按 least-loaded 策略分配请求 - handle_incoming_frame 按连接实例路由,unregister 精确移除单条连接 - 心跳和 PONG 回复改为 fire-and-forget 避免阻塞主读循环 - send_frame 增加超时保护防止 TCP 写阻塞级联 - WebSocket 先 accept 再认证,auth 加超时 - 调整 idle timeout (90s) 和 ping 间隔 (15s)
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
//! Application lifecycle: initialization, task orchestration, and shutdown.
|
||||
|
||||
use std::sync::atomic::{AtomicU32, AtomicU64};
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -65,11 +65,19 @@ pub async fn run(mut config: Config, servers: Vec<ServerEntry>) -> anyhow::Resul
|
||||
));
|
||||
|
||||
// Build reqwest client for tunnel upstream requests (shared).
|
||||
let reqwest_client = reqwest::Client::builder()
|
||||
let mut reqwest_builder = reqwest::Client::builder()
|
||||
.pool_max_idle_per_host(config.upstream_pool_max_idle_per_host)
|
||||
.pool_idle_timeout(Duration::from_secs(config.upstream_pool_idle_timeout_secs))
|
||||
.connect_timeout(Duration::from_secs(config.upstream_connect_timeout_secs))
|
||||
.tcp_nodelay(config.upstream_tcp_nodelay)
|
||||
.tcp_nodelay(config.upstream_tcp_nodelay);
|
||||
|
||||
if config.upstream_tcp_keepalive_secs > 0 {
|
||||
reqwest_builder = reqwest_builder.tcp_keepalive(Some(Duration::from_secs(
|
||||
config.upstream_tcp_keepalive_secs,
|
||||
)));
|
||||
}
|
||||
|
||||
let reqwest_client = reqwest_builder
|
||||
.build()
|
||||
.expect("failed to build reqwest client");
|
||||
|
||||
@@ -106,7 +114,6 @@ pub async fn run(mut config: Config, servers: Vec<ServerEntry>) -> anyhow::Resul
|
||||
dynamic: Arc::new(RwLock::new(DynamicConfig::from_config(&config))),
|
||||
active_connections: Arc::new(AtomicU64::new(0)),
|
||||
metrics: Arc::new(ProxyMetrics::new()),
|
||||
reconnect_attempts: AtomicU32::new(0),
|
||||
}));
|
||||
}
|
||||
Err(e) => {
|
||||
@@ -125,10 +132,12 @@ pub async fn run(mut config: Config, servers: Vec<ServerEntry>) -> anyhow::Resul
|
||||
}
|
||||
|
||||
// Build shared application state
|
||||
let tunnel_tls_config = Arc::new(crate::tunnel::client::build_tls_config());
|
||||
let state = Arc::new(AppState {
|
||||
config: Arc::new(config),
|
||||
dns_cache,
|
||||
reqwest_client,
|
||||
tunnel_tls_config,
|
||||
});
|
||||
|
||||
// Shutdown signal channel
|
||||
@@ -139,15 +148,18 @@ pub async fn run(mut config: Config, servers: Vec<ServerEntry>) -> anyhow::Resul
|
||||
"running in tunnel mode"
|
||||
);
|
||||
|
||||
// Spawn one tunnel task per server
|
||||
// Spawn tunnel connections per server (pool_size connections each)
|
||||
let pool_size = state.config.tunnel_connections.max(1) as usize;
|
||||
let mut tunnel_handles = Vec::new();
|
||||
for server in &server_contexts {
|
||||
let s = Arc::clone(&state);
|
||||
let srv = Arc::clone(server);
|
||||
let rx = shutdown_rx.clone();
|
||||
tunnel_handles.push(tokio::spawn(async move {
|
||||
tunnel::run(&s, &srv, rx).await;
|
||||
}));
|
||||
for conn_idx in 0..pool_size {
|
||||
let s = Arc::clone(&state);
|
||||
let srv = Arc::clone(server);
|
||||
let rx = shutdown_rx.clone();
|
||||
tunnel_handles.push(tokio::spawn(async move {
|
||||
tunnel::run(&s, &srv, conn_idx, rx).await;
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for shutdown signal
|
||||
|
||||
@@ -225,6 +225,30 @@ pub struct Config {
|
||||
/// Maximum concurrent streams over tunnel (auto-detected from hardware if omitted)
|
||||
#[arg(long, env = "AETHER_PROXY_TUNNEL_MAX_STREAMS")]
|
||||
pub tunnel_max_streams: Option<u32>,
|
||||
|
||||
/// WebSocket tunnel TCP connect timeout in seconds
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_TUNNEL_CONNECT_TIMEOUT",
|
||||
default_value_t = 15
|
||||
)]
|
||||
pub tunnel_connect_timeout_secs: u64,
|
||||
|
||||
/// WebSocket tunnel TCP keepalive in seconds (0 disables)
|
||||
#[arg(long, env = "AETHER_PROXY_TUNNEL_TCP_KEEPALIVE", default_value_t = 30)]
|
||||
pub tunnel_tcp_keepalive_secs: u64,
|
||||
|
||||
/// WebSocket tunnel TCP_NODELAY
|
||||
#[arg(long, env = "AETHER_PROXY_TUNNEL_TCP_NODELAY", default_value_t = true)]
|
||||
pub tunnel_tcp_nodelay: bool,
|
||||
|
||||
/// Tunnel connection staleness timeout in seconds (triggers reconnect if no data received)
|
||||
#[arg(long, env = "AETHER_PROXY_TUNNEL_STALE_TIMEOUT", default_value_t = 45)]
|
||||
pub tunnel_stale_timeout_secs: u64,
|
||||
|
||||
/// Number of parallel WebSocket tunnel connections per server (connection pool)
|
||||
#[arg(long, env = "AETHER_PROXY_TUNNEL_CONNECTIONS", default_value_t = 3)]
|
||||
pub tunnel_connections: u32,
|
||||
}
|
||||
|
||||
/// Per-server connection config (used in multi-server TOML `[[servers]]`).
|
||||
@@ -306,6 +330,16 @@ pub struct ConfigFile {
|
||||
pub tunnel_ping_interval_secs: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tunnel_max_streams: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tunnel_connect_timeout_secs: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tunnel_tcp_keepalive_secs: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tunnel_tcp_nodelay: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tunnel_stale_timeout_secs: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tunnel_connections: Option<u32>,
|
||||
|
||||
/// Multi-server config: each entry connects to a separate Aether instance.
|
||||
/// When present, top-level aether_url/management_token are ignored for
|
||||
@@ -548,6 +582,20 @@ impl ConfigFile {
|
||||
self.tunnel_ping_interval_secs
|
||||
);
|
||||
set!("AETHER_PROXY_TUNNEL_MAX_STREAMS", self.tunnel_max_streams);
|
||||
set!(
|
||||
"AETHER_PROXY_TUNNEL_CONNECT_TIMEOUT",
|
||||
self.tunnel_connect_timeout_secs
|
||||
);
|
||||
set!(
|
||||
"AETHER_PROXY_TUNNEL_TCP_KEEPALIVE",
|
||||
self.tunnel_tcp_keepalive_secs
|
||||
);
|
||||
set!("AETHER_PROXY_TUNNEL_TCP_NODELAY", self.tunnel_tcp_nodelay);
|
||||
set!(
|
||||
"AETHER_PROXY_TUNNEL_STALE_TIMEOUT",
|
||||
self.tunnel_stale_timeout_secs
|
||||
);
|
||||
set!("AETHER_PROXY_TUNNEL_CONNECTIONS", self.tunnel_connections);
|
||||
|
||||
// allowed_ports needs special handling (comma-separated)
|
||||
if let Some(ref ports) = self.allowed_ports {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Shared application state passed to all subsystems.
|
||||
|
||||
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -16,6 +16,8 @@ pub struct AppState {
|
||||
pub dns_cache: Arc<DnsCache>,
|
||||
/// Reqwest client for tunnel upstream requests (shared).
|
||||
pub reqwest_client: reqwest::Client,
|
||||
/// Shared TLS config for tunnel WebSocket connections (avoids re-parsing root CAs on each reconnect).
|
||||
pub tunnel_tls_config: Arc<rustls::ClientConfig>,
|
||||
}
|
||||
|
||||
/// Per-server state: one instance per Aether server connection.
|
||||
@@ -38,8 +40,6 @@ pub struct ServerContext {
|
||||
pub active_connections: Arc<AtomicU64>,
|
||||
/// Per-server request/latency metrics.
|
||||
pub metrics: Arc<ProxyMetrics>,
|
||||
/// Reconnect attempt counter (reset on successful connection).
|
||||
pub reconnect_attempts: AtomicU32,
|
||||
}
|
||||
|
||||
/// Aggregate metrics for reporting to Aether.
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
//! WebSocket tunnel client: connect, authenticate, and run the tunnel.
|
||||
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::sync::watch;
|
||||
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
|
||||
use tokio_tungstenite::tungstenite::http;
|
||||
use tracing::{debug, info};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::state::{AppState, ServerContext};
|
||||
|
||||
@@ -22,16 +23,20 @@ pub enum TunnelOutcome {
|
||||
}
|
||||
|
||||
/// Connect to Aether's WebSocket tunnel endpoint and run until disconnected.
|
||||
///
|
||||
/// `conn_idx` identifies which connection in the pool this is (0-based).
|
||||
/// Only connection 0 sends heartbeats to avoid resetting shared metrics.
|
||||
pub async fn connect_and_run(
|
||||
state: &Arc<AppState>,
|
||||
server: &Arc<ServerContext>,
|
||||
conn_idx: usize,
|
||||
shutdown: &mut watch::Receiver<bool>,
|
||||
) -> Result<TunnelOutcome, anyhow::Error> {
|
||||
let ws_url = build_tunnel_url(server);
|
||||
info!(url = %ws_url, "connecting tunnel");
|
||||
info!(url = %ws_url, conn = conn_idx, "connecting tunnel");
|
||||
|
||||
// Build WebSocket request with auth headers
|
||||
let mut request = ws_url.into_client_request()?;
|
||||
let mut request = ws_url.clone().into_client_request()?;
|
||||
let headers = request.headers_mut();
|
||||
headers.insert(
|
||||
"Authorization",
|
||||
@@ -44,12 +49,59 @@ pub async fn connect_and_run(
|
||||
http::HeaderValue::from_str(&server.node_name)?,
|
||||
);
|
||||
|
||||
// Connect
|
||||
let (ws_stream, _response) = tokio_tungstenite::connect_async(request).await?;
|
||||
info!("tunnel connected");
|
||||
// Parse host:port from URL
|
||||
let uri: http::Uri = ws_url.parse()?;
|
||||
let host = uri
|
||||
.host()
|
||||
.ok_or_else(|| anyhow::anyhow!("missing host in tunnel URL"))?;
|
||||
let is_tls = uri.scheme_str() == Some("wss");
|
||||
let port = uri.port_u16().unwrap_or(if is_tls { 443 } else { 80 });
|
||||
|
||||
// Reset reconnect counter on success
|
||||
server.reconnect_attempts.store(0, Ordering::Relaxed);
|
||||
// TCP connect with timeout
|
||||
let connect_timeout = Duration::from_secs(state.config.tunnel_connect_timeout_secs);
|
||||
let tcp_stream = tokio::time::timeout(connect_timeout, TcpStream::connect((host, port)))
|
||||
.await
|
||||
.map_err(|_| {
|
||||
anyhow::anyhow!(
|
||||
"tunnel TCP connect timeout ({}s)",
|
||||
connect_timeout.as_secs()
|
||||
)
|
||||
})??;
|
||||
|
||||
// Configure TCP parameters via socket2
|
||||
configure_tcp_socket(&tcp_stream, state);
|
||||
|
||||
// WebSocket upgrade (with TLS if wss://)
|
||||
let connector = if is_tls {
|
||||
Some(tokio_tungstenite::Connector::Rustls(Arc::clone(
|
||||
&state.tunnel_tls_config,
|
||||
)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let handshake_timeout = Duration::from_secs(state.config.tunnel_connect_timeout_secs);
|
||||
let (ws_stream, _response) = tokio::time::timeout(
|
||||
handshake_timeout,
|
||||
tokio_tungstenite::client_async_tls_with_config(request, tcp_stream, None, connector),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
anyhow::anyhow!(
|
||||
"tunnel WebSocket handshake timeout ({}s)",
|
||||
handshake_timeout.as_secs()
|
||||
)
|
||||
})??;
|
||||
info!(
|
||||
conn = conn_idx,
|
||||
tcp_keepalive_secs = state.config.tunnel_tcp_keepalive_secs,
|
||||
tcp_nodelay = state.config.tunnel_tcp_nodelay,
|
||||
connect_timeout_secs = state.config.tunnel_connect_timeout_secs,
|
||||
stale_timeout_secs = state.config.tunnel_stale_timeout_secs,
|
||||
"tunnel connected"
|
||||
);
|
||||
|
||||
// NOTE: reconnect_attempts reset is handled by the caller (mod.rs)
|
||||
// based on how long the connection stayed alive.
|
||||
|
||||
// Split into read/write halves
|
||||
let (ws_sink, ws_read) = futures_util::StreamExt::split(ws_stream);
|
||||
@@ -58,13 +110,18 @@ pub async fn connect_and_run(
|
||||
let ping_interval = Duration::from_secs(state.config.tunnel_ping_interval_secs);
|
||||
let (frame_tx, mut writer_handle) = writer::spawn_writer(ws_sink, ping_interval);
|
||||
|
||||
// Spawn heartbeat task
|
||||
let hb_handle = heartbeat::spawn(
|
||||
Arc::clone(&state.config),
|
||||
Arc::clone(server),
|
||||
frame_tx.clone(),
|
||||
shutdown.clone(),
|
||||
);
|
||||
// Spawn heartbeat task (only for primary connection to avoid
|
||||
// resetting shared atomic metrics via swap(0))
|
||||
let hb_handle = if conn_idx == 0 {
|
||||
heartbeat::spawn(
|
||||
Arc::clone(&state.config),
|
||||
Arc::clone(server),
|
||||
frame_tx.clone(),
|
||||
shutdown.clone(),
|
||||
)
|
||||
} else {
|
||||
heartbeat::spawn_noop()
|
||||
};
|
||||
|
||||
// Run dispatcher (blocks until disconnect or shutdown).
|
||||
// Also watch for writer exit — if the write half dies (e.g. the peer
|
||||
@@ -81,7 +138,7 @@ pub async fn connect_and_run(
|
||||
}
|
||||
}
|
||||
_ = &mut writer_handle => {
|
||||
debug!("writer task exited, triggering reconnect");
|
||||
warn!("writer task exited, triggering reconnect");
|
||||
TunnelOutcome::Disconnected
|
||||
}
|
||||
_ = shutdown.changed() => {
|
||||
@@ -106,9 +163,39 @@ pub async fn connect_and_run(
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
/// Configure TCP keepalive and NODELAY on an established socket.
|
||||
fn configure_tcp_socket(stream: &TcpStream, state: &Arc<AppState>) {
|
||||
let sock_ref = socket2::SockRef::from(stream);
|
||||
|
||||
if state.config.tunnel_tcp_keepalive_secs > 0 {
|
||||
let keepalive = socket2::TcpKeepalive::new()
|
||||
.with_time(Duration::from_secs(state.config.tunnel_tcp_keepalive_secs))
|
||||
.with_interval(Duration::from_secs(5))
|
||||
.with_retries(3);
|
||||
if let Err(e) = sock_ref.set_tcp_keepalive(&keepalive) {
|
||||
warn!(error = %e, "failed to set TCP keepalive on tunnel socket");
|
||||
}
|
||||
}
|
||||
|
||||
if state.config.tunnel_tcp_nodelay {
|
||||
if let Err(e) = sock_ref.set_nodelay(true) {
|
||||
warn!(error = %e, "failed to set TCP_NODELAY on tunnel socket");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build rustls ClientConfig with system root certificates.
|
||||
pub fn build_tls_config() -> rustls::ClientConfig {
|
||||
let root_store =
|
||||
rustls::RootCertStore::from_iter(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
|
||||
rustls::ClientConfig::builder()
|
||||
.with_root_certificates(root_store)
|
||||
.with_no_client_auth()
|
||||
}
|
||||
|
||||
/// Calculate next reconnect delay with exponential backoff + jitter.
|
||||
pub fn next_reconnect_delay(state: &Arc<AppState>, server: &Arc<ServerContext>) -> Duration {
|
||||
let attempt = server.reconnect_attempts.fetch_add(1, Ordering::Relaxed);
|
||||
pub fn next_reconnect_delay(state: &Arc<AppState>, reconnect_attempts: &AtomicU32) -> Duration {
|
||||
let attempt = reconnect_attempts.fetch_add(1, Ordering::Relaxed);
|
||||
let base_ms = state.config.tunnel_reconnect_base_ms;
|
||||
let max_ms = state.config.tunnel_reconnect_max_ms;
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ use futures_util::StreamExt;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
use tracing::{debug, error, warn};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
use crate::state::{AppState, ServerContext};
|
||||
|
||||
@@ -37,11 +37,26 @@ where
|
||||
// Track spawned stream handlers so we can wait for them on shutdown
|
||||
let mut handler_handles: Vec<JoinHandle<()>> = Vec::new();
|
||||
let max_streams = state.config.tunnel_max_streams.unwrap_or(128) as usize;
|
||||
let stale_timeout = Duration::from_secs(state.config.tunnel_stale_timeout_secs);
|
||||
|
||||
// Track last time we received any data to detect stale connections
|
||||
let mut last_data_at = tokio::time::Instant::now();
|
||||
|
||||
let read_err = loop {
|
||||
let msg_result = match ws_stream.next().await {
|
||||
Some(r) => r,
|
||||
None => break None, // stream ended
|
||||
let msg_result = tokio::select! {
|
||||
msg = ws_stream.next() => {
|
||||
match msg {
|
||||
Some(r) => r,
|
||||
None => break None,
|
||||
}
|
||||
}
|
||||
_ = tokio::time::sleep_until(last_data_at + stale_timeout) => {
|
||||
warn!(
|
||||
stale_secs = stale_timeout.as_secs(),
|
||||
"tunnel connection stale, no data received"
|
||||
);
|
||||
break None;
|
||||
}
|
||||
};
|
||||
|
||||
let msg = match msg_result {
|
||||
@@ -52,12 +67,15 @@ where
|
||||
}
|
||||
};
|
||||
|
||||
// Any successfully received message proves the connection is alive
|
||||
last_data_at = tokio::time::Instant::now();
|
||||
|
||||
let data = match msg {
|
||||
Message::Binary(data) => Bytes::from(data),
|
||||
Message::Ping(_) => continue,
|
||||
Message::Pong(_) => continue,
|
||||
Message::Close(_) => {
|
||||
debug!("received WebSocket close");
|
||||
info!("received WebSocket close");
|
||||
break None;
|
||||
}
|
||||
_ => continue,
|
||||
@@ -78,14 +96,13 @@ where
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
warn!(stream_id = frame.stream_id, error = %e, "invalid request metadata");
|
||||
let _ = frame_tx
|
||||
.send(Frame::new(
|
||||
frame.stream_id,
|
||||
MsgType::StreamError,
|
||||
0,
|
||||
Bytes::from(format!("invalid request metadata: {e}")),
|
||||
))
|
||||
.await;
|
||||
// Use try_send to avoid blocking the read loop
|
||||
let _ = frame_tx.try_send(Frame::new(
|
||||
frame.stream_id,
|
||||
MsgType::StreamError,
|
||||
0,
|
||||
Bytes::from(format!("invalid request metadata: {e}")),
|
||||
));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
@@ -95,14 +112,12 @@ where
|
||||
stream_id = frame.stream_id,
|
||||
"max concurrent streams reached"
|
||||
);
|
||||
let _ = frame_tx
|
||||
.send(Frame::new(
|
||||
frame.stream_id,
|
||||
MsgType::StreamError,
|
||||
0,
|
||||
Bytes::from("max concurrent streams reached"),
|
||||
))
|
||||
.await;
|
||||
let _ = frame_tx.try_send(Frame::new(
|
||||
frame.stream_id,
|
||||
MsgType::StreamError,
|
||||
0,
|
||||
Bytes::from("max concurrent streams reached"),
|
||||
));
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -147,9 +162,8 @@ where
|
||||
}
|
||||
|
||||
MsgType::Ping => {
|
||||
let _ = frame_tx
|
||||
.send(Frame::control(MsgType::Pong, frame.payload))
|
||||
.await;
|
||||
// Use try_send to avoid blocking the read loop when writer is congested
|
||||
let _ = frame_tx.try_send(Frame::control(MsgType::Pong, frame.payload));
|
||||
}
|
||||
|
||||
MsgType::HeartbeatAck => {
|
||||
@@ -157,7 +171,7 @@ where
|
||||
}
|
||||
|
||||
MsgType::GoAway => {
|
||||
debug!("received GOAWAY");
|
||||
info!("received GOAWAY");
|
||||
break None;
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,15 @@ impl HeartbeatHandle {
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a no-op heartbeat handle that silently discards ACKs.
|
||||
/// Used for non-primary tunnel connections (conn_idx > 0) to avoid
|
||||
/// resetting shared atomic metrics via `swap(0)`.
|
||||
pub fn spawn_noop() -> HeartbeatHandle {
|
||||
let (ack_tx, _) = tokio::sync::mpsc::channel::<Bytes>(1);
|
||||
// receiver is immediately dropped; on_ack() calls will silently fail
|
||||
HeartbeatHandle { ack_tx }
|
||||
}
|
||||
|
||||
/// Spawn the heartbeat task. Returns a handle for forwarding ACKs.
|
||||
pub fn spawn(
|
||||
config: Arc<Config>,
|
||||
|
||||
@@ -5,47 +5,83 @@ pub mod protocol;
|
||||
pub mod stream_handler;
|
||||
pub mod writer;
|
||||
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::watch;
|
||||
use tracing::{error, info};
|
||||
|
||||
use crate::state::{AppState, ServerContext};
|
||||
|
||||
/// Minimum connection duration (seconds) to consider a session "stable".
|
||||
/// If a connection lasts shorter than this, the backoff counter is NOT reset,
|
||||
/// preventing rapid reconnect loops on persistently bad networks.
|
||||
const MIN_STABLE_DURATION: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Run the tunnel mode main loop (connect, dispatch, reconnect).
|
||||
///
|
||||
/// `conn_idx` identifies which connection in the pool this is (0-based).
|
||||
/// Only connection 0 sends heartbeats to avoid resetting shared metrics.
|
||||
pub async fn run(
|
||||
state: &Arc<AppState>,
|
||||
server: &Arc<ServerContext>,
|
||||
conn_idx: usize,
|
||||
mut shutdown: watch::Receiver<bool>,
|
||||
) {
|
||||
info!(server = %server.server_label, "starting tunnel");
|
||||
info!(server = %server.server_label, conn = conn_idx, "starting tunnel");
|
||||
|
||||
// Per-connection reconnect counter (avoids N connections interfering
|
||||
// with each other's backoff via the shared ServerContext field).
|
||||
let reconnect_attempts = AtomicU32::new(0);
|
||||
|
||||
loop {
|
||||
match client::connect_and_run(state, server, &mut shutdown).await {
|
||||
let connect_start = tokio::time::Instant::now();
|
||||
|
||||
match client::connect_and_run(state, server, conn_idx, &mut shutdown).await {
|
||||
Ok(client::TunnelOutcome::Shutdown) => {
|
||||
info!(server = %server.server_label, "tunnel shut down gracefully");
|
||||
info!(server = %server.server_label, conn = conn_idx, "tunnel shut down gracefully");
|
||||
return;
|
||||
}
|
||||
Ok(client::TunnelOutcome::Disconnected) => {
|
||||
info!(server = %server.server_label, "tunnel disconnected, will reconnect");
|
||||
let duration = connect_start.elapsed();
|
||||
if duration >= MIN_STABLE_DURATION {
|
||||
// Stable session -- reset backoff for quick reconnect
|
||||
reconnect_attempts.store(0, Ordering::Relaxed);
|
||||
info!(
|
||||
server = %server.server_label,
|
||||
conn = conn_idx,
|
||||
duration_secs = duration.as_secs(),
|
||||
"tunnel disconnected after stable session"
|
||||
);
|
||||
} else {
|
||||
// Short-lived session -- keep backoff increasing
|
||||
info!(
|
||||
server = %server.server_label,
|
||||
conn = conn_idx,
|
||||
duration_secs = duration.as_secs(),
|
||||
"tunnel disconnected quickly, increasing backoff"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!(server = %server.server_label, error = %e, "tunnel connection lost");
|
||||
// Connection failed -- keep backoff increasing
|
||||
error!(server = %server.server_label, conn = conn_idx, error = %e, "tunnel connection lost");
|
||||
}
|
||||
}
|
||||
|
||||
if *shutdown.borrow() {
|
||||
info!(server = %server.server_label, "shutdown requested, not reconnecting");
|
||||
info!(server = %server.server_label, conn = conn_idx, "shutdown requested, not reconnecting");
|
||||
return;
|
||||
}
|
||||
|
||||
let delay = client::next_reconnect_delay(state, server);
|
||||
info!(server = %server.server_label, delay_ms = delay.as_millis(), "reconnecting tunnel");
|
||||
let delay = client::next_reconnect_delay(state, &reconnect_attempts);
|
||||
info!(server = %server.server_label, conn = conn_idx, delay_ms = delay.as_millis(), "reconnecting tunnel");
|
||||
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(delay) => {}
|
||||
_ = shutdown.changed() => {
|
||||
info!(server = %server.server_label, "shutdown requested during reconnect wait");
|
||||
info!(server = %server.server_label, conn = conn_idx, "shutdown requested during reconnect wait");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,11 @@ use super::writer::FrameSender;
|
||||
/// Maximum response body chunk size per frame (32 KB).
|
||||
const MAX_CHUNK_SIZE: usize = 32 * 1024;
|
||||
|
||||
/// Timeout for sending a single frame to the writer channel.
|
||||
/// If the writer is congested (TCP backpressure), we abandon the stream
|
||||
/// rather than blocking indefinitely and exhausting the stream pool.
|
||||
const FRAME_SEND_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Handle a single stream: receive body, execute upstream, send response.
|
||||
pub async fn handle_stream(
|
||||
state: Arc<AppState>,
|
||||
@@ -39,6 +44,22 @@ pub async fn handle_stream(
|
||||
server.metrics.record_request(start.elapsed());
|
||||
}
|
||||
|
||||
/// Send a frame to the writer with a timeout. Returns false if send failed.
|
||||
async fn send_frame(tx: &FrameSender, frame: Frame) -> bool {
|
||||
match tokio::time::timeout(FRAME_SEND_TIMEOUT, tx.send(frame)).await {
|
||||
Ok(Ok(())) => true,
|
||||
Ok(Err(_)) => {
|
||||
// Channel closed (writer exited)
|
||||
false
|
||||
}
|
||||
Err(_) => {
|
||||
// Timeout — writer is congested
|
||||
warn!("frame send timeout (writer congested), abandoning stream");
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_stream_inner(
|
||||
state: &AppState,
|
||||
server: &ServerContext,
|
||||
@@ -190,14 +211,14 @@ async fn handle_stream_inner(
|
||||
headers: resp_headers,
|
||||
};
|
||||
let meta_json = serde_json::to_vec(&resp_meta).unwrap_or_default();
|
||||
let _ = frame_tx
|
||||
.send(Frame::new(
|
||||
stream_id,
|
||||
MsgType::ResponseHeaders,
|
||||
0,
|
||||
meta_json,
|
||||
))
|
||||
.await;
|
||||
if !send_frame(
|
||||
frame_tx,
|
||||
Frame::new(stream_id, MsgType::ResponseHeaders, 0, meta_json),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// Stream response body
|
||||
let mut stream = response.bytes_stream();
|
||||
@@ -205,19 +226,28 @@ async fn handle_stream_inner(
|
||||
match chunk_result {
|
||||
Ok(chunk) => {
|
||||
if chunk.len() <= MAX_CHUNK_SIZE {
|
||||
// 大多数 chunk 无需分割,直接零拷贝发送
|
||||
let _ = frame_tx
|
||||
.send(Frame::new(stream_id, MsgType::ResponseBody, 0, chunk))
|
||||
.await;
|
||||
if !send_frame(
|
||||
frame_tx,
|
||||
Frame::new(stream_id, MsgType::ResponseBody, 0, chunk),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
// 超大 chunk 按 MAX_CHUNK_SIZE 分割(使用 Bytes::slice 避免拷贝)
|
||||
// Split oversized chunks
|
||||
let mut offset = 0;
|
||||
while offset < chunk.len() {
|
||||
let end = (offset + MAX_CHUNK_SIZE).min(chunk.len());
|
||||
let slice = chunk.slice(offset..end);
|
||||
let _ = frame_tx
|
||||
.send(Frame::new(stream_id, MsgType::ResponseBody, 0, slice))
|
||||
.await;
|
||||
if !send_frame(
|
||||
frame_tx,
|
||||
Frame::new(stream_id, MsgType::ResponseBody, 0, slice),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return;
|
||||
}
|
||||
offset = end;
|
||||
}
|
||||
}
|
||||
@@ -231,27 +261,32 @@ async fn handle_stream_inner(
|
||||
}
|
||||
|
||||
// Send STREAM_END
|
||||
let _ = frame_tx
|
||||
.send(Frame::new(
|
||||
let _ = send_frame(
|
||||
frame_tx,
|
||||
Frame::new(
|
||||
stream_id,
|
||||
MsgType::StreamEnd,
|
||||
flags::END_STREAM,
|
||||
Bytes::new(),
|
||||
))
|
||||
.await;
|
||||
),
|
||||
)
|
||||
.await;
|
||||
|
||||
debug!(stream_id, status, "stream completed");
|
||||
}
|
||||
|
||||
async fn send_error(tx: &FrameSender, stream_id: u32, msg: &str) {
|
||||
let _ = tx
|
||||
.send(Frame::new(
|
||||
// Error frames use best-effort delivery — don't block if writer is congested
|
||||
let _ = send_frame(
|
||||
tx,
|
||||
Frame::new(
|
||||
stream_id,
|
||||
MsgType::StreamError,
|
||||
0,
|
||||
Bytes::from(msg.to_string()),
|
||||
))
|
||||
.await;
|
||||
),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
fn decompress_gzip(data: &[u8]) -> Result<Bytes, std::io::Error> {
|
||||
|
||||
Reference in New Issue
Block a user