mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 01:10:23 +08:00
refactor: 移除独立 hub/proxy/executor/gateway crate,统一为 gateway tunnel 架构
- 删除 aether-hub、aether-proxy 独立项目及其 Dockerfile/配置 - 删除 crates/aether-executor 和 crates/aether-gateway 全部模块 - 新增 apps/ 目录作为应用入口 - 将 hub 概念重构为 gateway tunnel transport - 将 executor 重构为 execution runtime - 新增 tunnel.rs 合约定义和 testkit tunnel/execution_runtime 模块 - 更新 Python 服务层和测试适配新架构命名
This commit is contained in:
356
apps/aether-proxy/src/app.rs
Normal file
356
apps/aether-proxy/src/app.rs
Normal file
@@ -0,0 +1,356 @@
|
||||
//! Application lifecycle: initialization, task orchestration, and shutdown.
|
||||
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::Duration;
|
||||
|
||||
use aether_runtime::{
|
||||
init_reloadable_tracing, wait_for_shutdown_signal, ConcurrencyGate, DistributedConcurrencyGate,
|
||||
LogFormat, RedisDistributedConcurrencyConfig,
|
||||
};
|
||||
use arc_swap::ArcSwap;
|
||||
use tokio::sync::{watch, Mutex};
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
use crate::config::{Config, ServerEntry};
|
||||
use crate::net;
|
||||
use crate::registration::client::AetherClient;
|
||||
use crate::runtime::{self, DynamicConfig};
|
||||
use crate::state::{AppState, ProxyMetrics, ServerContext};
|
||||
use crate::upstream_client;
|
||||
use crate::{hardware, target_filter, tunnel};
|
||||
|
||||
/// Run the full application lifecycle after config has been parsed.
|
||||
pub async fn run(mut config: Config, servers: Vec<ServerEntry>) -> anyhow::Result<()> {
|
||||
config.validate()?;
|
||||
init_tracing(&config);
|
||||
|
||||
info!(
|
||||
version = env!("CARGO_PKG_VERSION"),
|
||||
node_name = %config.node_name,
|
||||
server_count = servers.len(),
|
||||
"aether-proxy starting (tunnel mode)"
|
||||
);
|
||||
|
||||
// Resolve public IP (best-effort for region info)
|
||||
let public_ip = match &config.public_ip {
|
||||
Some(ip) => ip.clone(),
|
||||
None => net::detect_public_ip()
|
||||
.await
|
||||
.unwrap_or_else(|_| "0.0.0.0".to_string()),
|
||||
};
|
||||
|
||||
// Auto-detect region if not configured
|
||||
if config.node_region.is_none() {
|
||||
if let Some(region) = net::detect_region(&public_ip).await {
|
||||
config.node_region = Some(region);
|
||||
}
|
||||
}
|
||||
|
||||
// Collect hardware info (once at startup, sent during registration)
|
||||
let hw_info = hardware::collect();
|
||||
|
||||
// Auto-detect tunnel_max_streams from hardware if not explicitly set
|
||||
if config.tunnel_max_streams.is_none() {
|
||||
let auto = (hw_info.estimated_max_concurrency / 10).clamp(64, 1024) as u32;
|
||||
config.tunnel_max_streams = Some(auto);
|
||||
info!(
|
||||
tunnel_max_streams = auto,
|
||||
"auto-detected tunnel_max_streams from hardware"
|
||||
);
|
||||
}
|
||||
|
||||
info!(
|
||||
max_concurrency = hw_info.estimated_max_concurrency,
|
||||
"hardware info collected"
|
||||
);
|
||||
|
||||
let dns_cache = Arc::new(target_filter::DnsCache::new(
|
||||
Duration::from_secs(config.dns_cache_ttl_secs),
|
||||
config.dns_cache_capacity,
|
||||
));
|
||||
|
||||
// Build Hyper client for tunnel upstream requests (shared).
|
||||
// DNS still flows through validated addresses from DnsCache, while the
|
||||
// custom connector exposes per-request connect/TLS timing when available.
|
||||
let upstream_client = upstream_client::build_upstream_client(&config, Arc::clone(&dns_cache));
|
||||
|
||||
// Register with each Aether server and build per-server contexts.
|
||||
// Wrapped in Arc<Mutex> so retry_failed_registrations can append later.
|
||||
let server_contexts: Arc<Mutex<Vec<Arc<ServerContext>>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut failed_entries: Vec<(String, ServerEntry)> = Vec::new();
|
||||
for (i, entry) in servers.iter().enumerate() {
|
||||
let label = if servers.len() == 1 {
|
||||
"server".to_string()
|
||||
} else {
|
||||
format!("server-{}", i)
|
||||
};
|
||||
let node_name = entry
|
||||
.node_name
|
||||
.clone()
|
||||
.unwrap_or_else(|| config.node_name.clone());
|
||||
let client = Arc::new(AetherClient::new(
|
||||
&config,
|
||||
&entry.aether_url,
|
||||
&entry.management_token,
|
||||
));
|
||||
match client
|
||||
.register(&config, &node_name, &public_ip, Some(&hw_info))
|
||||
.await
|
||||
{
|
||||
Ok(node_id) => {
|
||||
info!(server = %label, node_id = %node_id, url = %entry.aether_url, node_name = %node_name, "registered");
|
||||
// Initialize dynamic config with per-server node_name (not global),
|
||||
// so that the heartbeat and reconnect use the correct name.
|
||||
let mut dynamic = DynamicConfig::from_config(&config);
|
||||
dynamic.node_name = node_name.clone();
|
||||
server_contexts.lock().await.push(Arc::new(ServerContext {
|
||||
server_label: label,
|
||||
aether_url: entry.aether_url.clone(),
|
||||
management_token: entry.management_token.clone(),
|
||||
node_name,
|
||||
node_id: Arc::new(RwLock::new(node_id)),
|
||||
aether_client: client,
|
||||
dynamic: Arc::new(ArcSwap::from_pointee(dynamic)),
|
||||
active_connections: Arc::new(AtomicU64::new(0)),
|
||||
metrics: Arc::new(ProxyMetrics::new()),
|
||||
}));
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
server = %label,
|
||||
url = %entry.aether_url,
|
||||
error = %e,
|
||||
"registration failed, will retry in background"
|
||||
);
|
||||
failed_entries.push((label, entry.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let ctx_count = server_contexts.lock().await.len();
|
||||
if ctx_count == 0 && failed_entries.is_empty() {
|
||||
anyhow::bail!("no servers configured");
|
||||
}
|
||||
if ctx_count == 0 {
|
||||
anyhow::bail!(
|
||||
"no servers registered successfully (all {} failed)",
|
||||
failed_entries.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Build shared application state
|
||||
let tunnel_tls_config = Arc::new(crate::tunnel::client::build_tls_config());
|
||||
let mut state = AppState {
|
||||
config: Arc::new(config),
|
||||
dns_cache,
|
||||
upstream_client,
|
||||
tunnel_tls_config,
|
||||
stream_gate: None,
|
||||
distributed_stream_gate: None,
|
||||
};
|
||||
if let Some(limit) = state.config.max_in_flight_streams {
|
||||
state = state
|
||||
.with_stream_concurrency_gate(Arc::new(ConcurrencyGate::new("proxy_streams", limit)));
|
||||
}
|
||||
if let Some(limit) = state.config.distributed_stream_limit {
|
||||
let redis_url = state
|
||||
.config
|
||||
.distributed_stream_redis_url
|
||||
.clone()
|
||||
.expect("distributed stream redis url should be validated");
|
||||
let distributed_gate = DistributedConcurrencyGate::new_redis(
|
||||
"proxy_streams_distributed",
|
||||
limit,
|
||||
RedisDistributedConcurrencyConfig {
|
||||
url: redis_url,
|
||||
key_prefix: state.config.distributed_stream_redis_key_prefix.clone(),
|
||||
lease_ttl_ms: state.config.distributed_stream_lease_ttl_ms,
|
||||
renew_interval_ms: state.config.distributed_stream_renew_interval_ms,
|
||||
command_timeout_ms: Some(state.config.distributed_stream_command_timeout_ms),
|
||||
},
|
||||
)?;
|
||||
state = state.with_distributed_stream_concurrency_gate(Arc::new(distributed_gate));
|
||||
}
|
||||
let state = Arc::new(state);
|
||||
|
||||
// Shutdown signal channel
|
||||
let (shutdown_tx, shutdown_rx) = watch::channel(false);
|
||||
|
||||
info!(
|
||||
active_servers = server_contexts.lock().await.len(),
|
||||
"running in tunnel mode"
|
||||
);
|
||||
|
||||
// Spawn tunnel connections per server (pool_size connections each)
|
||||
let pool_size = state.config.tunnel_connections.max(1) as usize;
|
||||
let mut tunnel_handles = Vec::new();
|
||||
for server in server_contexts.lock().await.iter() {
|
||||
for conn_idx in 0..pool_size {
|
||||
let s = Arc::clone(&state);
|
||||
let srv = Arc::clone(server);
|
||||
let rx = shutdown_rx.clone();
|
||||
tunnel_handles.push(tokio::spawn(async move {
|
||||
tunnel::run(&s, &srv, conn_idx, rx).await;
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
// Spawn background retry for failed server registrations
|
||||
if !failed_entries.is_empty() {
|
||||
let retry_state = Arc::clone(&state);
|
||||
let retry_contexts = Arc::clone(&server_contexts);
|
||||
let retry_public_ip = public_ip.clone();
|
||||
let retry_hw_info = hw_info.clone();
|
||||
let retry_shutdown = shutdown_rx.clone();
|
||||
let retry_pool_size = pool_size;
|
||||
tokio::spawn(async move {
|
||||
retry_failed_registrations(
|
||||
retry_state,
|
||||
retry_contexts,
|
||||
failed_entries,
|
||||
retry_public_ip,
|
||||
retry_hw_info,
|
||||
retry_pool_size,
|
||||
retry_shutdown,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
|
||||
// Wait for shutdown signal
|
||||
wait_for_shutdown().await;
|
||||
info!("shutdown signal received, cleaning up...");
|
||||
let _ = shutdown_tx.send(true);
|
||||
|
||||
// Graceful unregister from all servers (including retry-registered ones)
|
||||
for server in server_contexts.lock().await.iter() {
|
||||
let node_id = server.node_id.read().unwrap().clone();
|
||||
if let Err(e) = server.aether_client.unregister(&node_id).await {
|
||||
error!(
|
||||
server = %server.server_label,
|
||||
error = %e,
|
||||
"unregister failed during shutdown"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for all tunnel tasks
|
||||
for h in tunnel_handles {
|
||||
let _ = h.await;
|
||||
}
|
||||
|
||||
info!("aether-proxy stopped");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Retry interval for failed server registrations (5 minutes).
|
||||
const REGISTRATION_RETRY_INTERVAL: Duration = Duration::from_secs(300);
|
||||
/// Max registration retry attempts before giving up.
|
||||
const REGISTRATION_RETRY_MAX: u32 = 12;
|
||||
|
||||
/// Background task that retries registration for servers that failed at startup.
|
||||
async fn retry_failed_registrations(
|
||||
state: Arc<AppState>,
|
||||
server_contexts: Arc<Mutex<Vec<Arc<ServerContext>>>>,
|
||||
failed: Vec<(String, ServerEntry)>,
|
||||
public_ip: String,
|
||||
hw_info: crate::hardware::HardwareInfo,
|
||||
pool_size: usize,
|
||||
mut shutdown: watch::Receiver<bool>,
|
||||
) {
|
||||
for (label, entry) in &failed {
|
||||
let node_name = entry
|
||||
.node_name
|
||||
.clone()
|
||||
.unwrap_or_else(|| state.config.node_name.clone());
|
||||
let client = Arc::new(AetherClient::new(
|
||||
&state.config,
|
||||
&entry.aether_url,
|
||||
&entry.management_token,
|
||||
));
|
||||
|
||||
let mut attempt = 0u32;
|
||||
let node_id = loop {
|
||||
attempt += 1;
|
||||
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(REGISTRATION_RETRY_INTERVAL) => {}
|
||||
_ = shutdown.changed() => {
|
||||
info!(server = %label, "shutdown during registration retry");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
match client
|
||||
.register(&state.config, &node_name, &public_ip, Some(&hw_info))
|
||||
.await
|
||||
{
|
||||
Ok(id) => {
|
||||
info!(server = %label, node_id = %id, attempt, "registration retry succeeded");
|
||||
break id;
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
server = %label,
|
||||
attempt,
|
||||
max = REGISTRATION_RETRY_MAX,
|
||||
error = %e,
|
||||
"registration retry failed"
|
||||
);
|
||||
if attempt >= REGISTRATION_RETRY_MAX {
|
||||
error!(server = %label, "giving up registration after {} attempts", attempt);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Build server context and spawn tunnels
|
||||
let mut dynamic = DynamicConfig::from_config(&state.config);
|
||||
dynamic.node_name = node_name.clone();
|
||||
let server = Arc::new(ServerContext {
|
||||
server_label: label.clone(),
|
||||
aether_url: entry.aether_url.clone(),
|
||||
management_token: entry.management_token.clone(),
|
||||
node_name,
|
||||
node_id: Arc::new(RwLock::new(node_id)),
|
||||
aether_client: client,
|
||||
dynamic: Arc::new(ArcSwap::from_pointee(dynamic)),
|
||||
active_connections: Arc::new(AtomicU64::new(0)),
|
||||
metrics: Arc::new(ProxyMetrics::new()),
|
||||
});
|
||||
|
||||
// Add to shared list so shutdown can unregister this server
|
||||
server_contexts.lock().await.push(Arc::clone(&server));
|
||||
|
||||
for conn_idx in 0..pool_size {
|
||||
let s = Arc::clone(&state);
|
||||
let srv = Arc::clone(&server);
|
||||
let rx = shutdown.clone();
|
||||
tokio::spawn(async move {
|
||||
tunnel::run(&s, &srv, conn_idx, rx).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn init_tracing(config: &Config) {
|
||||
let format = if config.log_json {
|
||||
LogFormat::Json
|
||||
} else {
|
||||
LogFormat::Pretty
|
||||
};
|
||||
|
||||
let reloader = init_reloadable_tracing(&config.log_level, format)
|
||||
.expect("proxy tracing should initialize");
|
||||
runtime::set_log_reloader(reloader);
|
||||
}
|
||||
|
||||
async fn wait_for_shutdown() {
|
||||
wait_for_shutdown_signal()
|
||||
.await
|
||||
.expect("failed to install shutdown signal handler");
|
||||
}
|
||||
721
apps/aether-proxy/src/config.rs
Normal file
721
apps/aether-proxy/src/config.rs
Normal file
@@ -0,0 +1,721 @@
|
||||
use std::path::Path;
|
||||
|
||||
use clap::Parser;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Fields that existed in 0.1.x but were removed in 0.2.0.
|
||||
const LEGACY_ONLY_KEYS: &[&str] = &[
|
||||
"hmac_key",
|
||||
"listen_port",
|
||||
"timestamp_tolerance",
|
||||
"connect_timeout_secs",
|
||||
"tls_handshake_timeout_secs",
|
||||
"enable_tls",
|
||||
"tls_cert",
|
||||
"tls_key",
|
||||
];
|
||||
|
||||
/// Fields renamed from 0.1.x `delegate_*` to 0.2.0 `upstream_*`.
|
||||
const DELEGATE_TO_UPSTREAM: &[(&str, &str)] = &[
|
||||
(
|
||||
"delegate_connect_timeout_secs",
|
||||
"upstream_connect_timeout_secs",
|
||||
),
|
||||
(
|
||||
"delegate_pool_max_idle_per_host",
|
||||
"upstream_pool_max_idle_per_host",
|
||||
),
|
||||
(
|
||||
"delegate_pool_idle_timeout_secs",
|
||||
"upstream_pool_idle_timeout_secs",
|
||||
),
|
||||
("delegate_tcp_keepalive_secs", "upstream_tcp_keepalive_secs"),
|
||||
("delegate_tcp_nodelay", "upstream_tcp_nodelay"),
|
||||
];
|
||||
|
||||
/// Aether tunnel proxy.
|
||||
///
|
||||
/// Deployed on overseas VPS to relay API traffic for Aether instances
|
||||
/// behind the GFW. Connects to Aether via WebSocket tunnel, registers
|
||||
/// with Aether, and relays upstream requests.
|
||||
#[derive(Parser, Debug, Clone)]
|
||||
#[command(version, about)]
|
||||
pub struct Config {
|
||||
/// Aether server URL (e.g. https://aether.example.com)
|
||||
#[arg(long, env = "AETHER_PROXY_AETHER_URL")]
|
||||
pub aether_url: String,
|
||||
|
||||
/// Management Token for Aether admin API (ae_xxx)
|
||||
#[arg(long, env = "AETHER_PROXY_MANAGEMENT_TOKEN")]
|
||||
pub management_token: String,
|
||||
|
||||
/// Public IP address of this node (auto-detected if omitted)
|
||||
#[arg(long, env = "AETHER_PROXY_PUBLIC_IP")]
|
||||
pub public_ip: Option<String>,
|
||||
|
||||
/// Human-readable node name
|
||||
#[arg(long, env = "AETHER_PROXY_NODE_NAME", default_value = "proxy-01")]
|
||||
pub node_name: String,
|
||||
|
||||
/// Region label (e.g. ap-northeast-1)
|
||||
#[arg(long, env = "AETHER_PROXY_NODE_REGION")]
|
||||
pub node_region: Option<String>,
|
||||
|
||||
/// Heartbeat interval in seconds
|
||||
#[arg(long, env = "AETHER_PROXY_HEARTBEAT_INTERVAL", default_value_t = 30)]
|
||||
pub heartbeat_interval: u64,
|
||||
|
||||
/// Allowed destination ports (default: 80,443,8080,8443)
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_ALLOWED_PORTS",
|
||||
value_delimiter = ',',
|
||||
default_values_t = vec![80, 443, 8080, 8443]
|
||||
)]
|
||||
pub allowed_ports: Vec<u16>,
|
||||
|
||||
/// Aether API request timeout in seconds
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_AETHER_REQUEST_TIMEOUT",
|
||||
default_value_t = 10
|
||||
)]
|
||||
pub aether_request_timeout_secs: u64,
|
||||
|
||||
/// Aether API connect timeout in seconds
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_AETHER_CONNECT_TIMEOUT",
|
||||
default_value_t = 10
|
||||
)]
|
||||
pub aether_connect_timeout_secs: u64,
|
||||
|
||||
/// Aether API max idle connections per host
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_AETHER_POOL_MAX_IDLE_PER_HOST",
|
||||
default_value_t = 8
|
||||
)]
|
||||
pub aether_pool_max_idle_per_host: usize,
|
||||
|
||||
/// Aether API idle timeout in seconds
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_AETHER_POOL_IDLE_TIMEOUT",
|
||||
default_value_t = 90
|
||||
)]
|
||||
pub aether_pool_idle_timeout_secs: u64,
|
||||
|
||||
/// Aether API TCP keepalive in seconds (0 disables)
|
||||
#[arg(long, env = "AETHER_PROXY_AETHER_TCP_KEEPALIVE", default_value_t = 60)]
|
||||
pub aether_tcp_keepalive_secs: u64,
|
||||
|
||||
/// Aether API TCP_NODELAY
|
||||
#[arg(long, env = "AETHER_PROXY_AETHER_TCP_NODELAY", default_value_t = true)]
|
||||
pub aether_tcp_nodelay: bool,
|
||||
|
||||
/// Enable HTTP/2 when talking to Aether API
|
||||
#[arg(long, env = "AETHER_PROXY_AETHER_HTTP2", default_value_t = true)]
|
||||
pub aether_http2: bool,
|
||||
|
||||
/// Aether API retry attempts (including initial)
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_AETHER_RETRY_MAX_ATTEMPTS",
|
||||
default_value_t = 3
|
||||
)]
|
||||
pub aether_retry_max_attempts: u32,
|
||||
|
||||
/// Aether API retry base delay in milliseconds
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_AETHER_RETRY_BASE_DELAY_MS",
|
||||
default_value_t = 200
|
||||
)]
|
||||
pub aether_retry_base_delay_ms: u64,
|
||||
|
||||
/// Aether API retry max delay in milliseconds
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_AETHER_RETRY_MAX_DELAY_MS",
|
||||
default_value_t = 2000
|
||||
)]
|
||||
pub aether_retry_max_delay_ms: u64,
|
||||
|
||||
/// Maximum concurrent TCP connections (defaults to hardware estimate)
|
||||
#[arg(long, env = "AETHER_PROXY_MAX_CONCURRENT_CONNECTIONS")]
|
||||
pub max_concurrent_connections: Option<u64>,
|
||||
|
||||
/// Maximum in-flight tunneled streams accepted by this proxy instance.
|
||||
#[arg(long, env = "AETHER_PROXY_MAX_IN_FLIGHT_STREAMS")]
|
||||
pub max_in_flight_streams: Option<usize>,
|
||||
|
||||
/// Maximum in-flight tunneled streams admitted across all proxy instances.
|
||||
#[arg(long, env = "AETHER_PROXY_DISTRIBUTED_STREAM_LIMIT")]
|
||||
pub distributed_stream_limit: Option<usize>,
|
||||
|
||||
/// Redis URL used for cross-instance stream admission.
|
||||
#[arg(long, env = "AETHER_PROXY_DISTRIBUTED_STREAM_REDIS_URL")]
|
||||
pub distributed_stream_redis_url: Option<String>,
|
||||
|
||||
/// Optional key prefix for cross-instance stream admission state.
|
||||
#[arg(long, env = "AETHER_PROXY_DISTRIBUTED_STREAM_REDIS_KEY_PREFIX")]
|
||||
pub distributed_stream_redis_key_prefix: Option<String>,
|
||||
|
||||
/// Lease TTL in milliseconds for distributed stream admission permits.
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_DISTRIBUTED_STREAM_LEASE_TTL_MS",
|
||||
default_value_t = 30_000
|
||||
)]
|
||||
pub distributed_stream_lease_ttl_ms: u64,
|
||||
|
||||
/// Renew interval in milliseconds for distributed stream admission permits.
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_DISTRIBUTED_STREAM_RENEW_INTERVAL_MS",
|
||||
default_value_t = 10_000
|
||||
)]
|
||||
pub distributed_stream_renew_interval_ms: u64,
|
||||
|
||||
/// Command timeout in milliseconds for distributed stream admission Redis calls.
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_DISTRIBUTED_STREAM_COMMAND_TIMEOUT_MS",
|
||||
default_value_t = 1_000
|
||||
)]
|
||||
pub distributed_stream_command_timeout_ms: u64,
|
||||
|
||||
/// DNS cache TTL in seconds
|
||||
#[arg(long, env = "AETHER_PROXY_DNS_CACHE_TTL", default_value_t = 60)]
|
||||
pub dns_cache_ttl_secs: u64,
|
||||
|
||||
/// DNS cache capacity (entries)
|
||||
#[arg(long, env = "AETHER_PROXY_DNS_CACHE_CAPACITY", default_value_t = 1024)]
|
||||
pub dns_cache_capacity: usize,
|
||||
|
||||
/// Upstream HTTP client connect timeout in seconds
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_UPSTREAM_CONNECT_TIMEOUT",
|
||||
default_value_t = 30
|
||||
)]
|
||||
pub upstream_connect_timeout_secs: u64,
|
||||
|
||||
/// Upstream HTTP client max idle connections per host
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_UPSTREAM_POOL_MAX_IDLE_PER_HOST",
|
||||
default_value_t = 64
|
||||
)]
|
||||
pub upstream_pool_max_idle_per_host: usize,
|
||||
|
||||
/// Upstream HTTP client idle timeout in seconds
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_UPSTREAM_POOL_IDLE_TIMEOUT",
|
||||
default_value_t = 300
|
||||
)]
|
||||
pub upstream_pool_idle_timeout_secs: u64,
|
||||
|
||||
/// Upstream TCP keepalive in seconds (0 disables)
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_UPSTREAM_TCP_KEEPALIVE",
|
||||
default_value_t = 60
|
||||
)]
|
||||
pub upstream_tcp_keepalive_secs: u64,
|
||||
|
||||
/// Upstream TCP_NODELAY
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_UPSTREAM_TCP_NODELAY",
|
||||
default_value_t = true
|
||||
)]
|
||||
pub upstream_tcp_nodelay: bool,
|
||||
|
||||
/// Log level (trace, debug, info, warn, error)
|
||||
#[arg(long, env = "AETHER_PROXY_LOG_LEVEL", default_value = "info")]
|
||||
pub log_level: String,
|
||||
|
||||
/// Output logs as JSON
|
||||
#[arg(long, env = "AETHER_PROXY_LOG_JSON", default_value_t = false)]
|
||||
pub log_json: bool,
|
||||
|
||||
/// Tunnel reconnect base delay in milliseconds (used by exponential backoff)
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_TUNNEL_RECONNECT_BASE_MS",
|
||||
default_value_t = 500
|
||||
)]
|
||||
pub tunnel_reconnect_base_ms: u64,
|
||||
|
||||
/// Tunnel reconnect max delay in milliseconds (cap for exponential backoff)
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_TUNNEL_RECONNECT_MAX_MS",
|
||||
default_value_t = 30000
|
||||
)]
|
||||
pub tunnel_reconnect_max_ms: u64,
|
||||
|
||||
/// WebSocket tunnel ping interval in seconds
|
||||
#[arg(long, env = "AETHER_PROXY_TUNNEL_PING_INTERVAL", default_value_t = 15)]
|
||||
pub tunnel_ping_interval_secs: u64,
|
||||
|
||||
/// Maximum concurrent streams over tunnel (auto-detected from hardware if omitted)
|
||||
#[arg(long, env = "AETHER_PROXY_TUNNEL_MAX_STREAMS")]
|
||||
pub tunnel_max_streams: Option<u32>,
|
||||
|
||||
/// WebSocket tunnel TCP connect timeout in seconds
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_PROXY_TUNNEL_CONNECT_TIMEOUT",
|
||||
default_value_t = 15
|
||||
)]
|
||||
pub tunnel_connect_timeout_secs: u64,
|
||||
|
||||
/// WebSocket tunnel TCP keepalive in seconds (0 disables)
|
||||
#[arg(long, env = "AETHER_PROXY_TUNNEL_TCP_KEEPALIVE", default_value_t = 30)]
|
||||
pub tunnel_tcp_keepalive_secs: u64,
|
||||
|
||||
/// WebSocket tunnel TCP_NODELAY
|
||||
#[arg(long, env = "AETHER_PROXY_TUNNEL_TCP_NODELAY", default_value_t = true)]
|
||||
pub tunnel_tcp_nodelay: bool,
|
||||
|
||||
/// Tunnel connection staleness timeout in seconds (triggers reconnect if no data received)
|
||||
#[arg(long, env = "AETHER_PROXY_TUNNEL_STALE_TIMEOUT", default_value_t = 45)]
|
||||
pub tunnel_stale_timeout_secs: u64,
|
||||
|
||||
/// Number of parallel WebSocket tunnel connections per server (connection pool)
|
||||
#[arg(long, env = "AETHER_PROXY_TUNNEL_CONNECTIONS", default_value_t = 3)]
|
||||
pub tunnel_connections: u32,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
/// Validate configuration values are within sane ranges.
|
||||
/// Called after parsing to catch misconfigurations early.
|
||||
pub fn validate(&self) -> anyhow::Result<()> {
|
||||
if self.heartbeat_interval == 0 {
|
||||
anyhow::bail!("heartbeat_interval must be > 0");
|
||||
}
|
||||
if self.heartbeat_interval > 3600 {
|
||||
anyhow::bail!("heartbeat_interval must be <= 3600");
|
||||
}
|
||||
if self.allowed_ports.is_empty() {
|
||||
anyhow::bail!("allowed_ports must not be empty");
|
||||
}
|
||||
for &port in &self.allowed_ports {
|
||||
if port == 0 {
|
||||
anyhow::bail!("allowed_ports: port 0 is not valid");
|
||||
}
|
||||
}
|
||||
if self.tunnel_connect_timeout_secs == 0 {
|
||||
anyhow::bail!("tunnel_connect_timeout_secs must be > 0");
|
||||
}
|
||||
if self.tunnel_ping_interval_secs == 0 {
|
||||
anyhow::bail!("tunnel_ping_interval_secs must be > 0");
|
||||
}
|
||||
if self.tunnel_stale_timeout_secs <= self.tunnel_ping_interval_secs {
|
||||
anyhow::bail!(
|
||||
"tunnel_stale_timeout_secs ({}) must be > tunnel_ping_interval_secs ({})",
|
||||
self.tunnel_stale_timeout_secs,
|
||||
self.tunnel_ping_interval_secs
|
||||
);
|
||||
}
|
||||
if self.tunnel_connections == 0 {
|
||||
anyhow::bail!("tunnel_connections must be > 0");
|
||||
}
|
||||
if self.aether_retry_max_attempts == 0 {
|
||||
anyhow::bail!("aether_retry_max_attempts must be >= 1");
|
||||
}
|
||||
if self.upstream_connect_timeout_secs == 0 {
|
||||
anyhow::bail!("upstream_connect_timeout_secs must be > 0");
|
||||
}
|
||||
if matches!(self.max_in_flight_streams, Some(0)) {
|
||||
anyhow::bail!("max_in_flight_streams must be > 0");
|
||||
}
|
||||
if matches!(self.distributed_stream_limit, Some(0)) {
|
||||
anyhow::bail!("distributed_stream_limit must be > 0");
|
||||
}
|
||||
if self.distributed_stream_limit.is_some() && self.distributed_stream_redis_url.is_none() {
|
||||
anyhow::bail!(
|
||||
"distributed_stream_redis_url must be set when distributed_stream_limit is enabled"
|
||||
);
|
||||
}
|
||||
if self.distributed_stream_lease_ttl_ms == 0 {
|
||||
anyhow::bail!("distributed_stream_lease_ttl_ms must be > 0");
|
||||
}
|
||||
if self.distributed_stream_renew_interval_ms == 0 {
|
||||
anyhow::bail!("distributed_stream_renew_interval_ms must be > 0");
|
||||
}
|
||||
if self.distributed_stream_renew_interval_ms >= self.distributed_stream_lease_ttl_ms {
|
||||
anyhow::bail!(
|
||||
"distributed_stream_renew_interval_ms must be < distributed_stream_lease_ttl_ms"
|
||||
);
|
||||
}
|
||||
if self.distributed_stream_command_timeout_ms == 0 {
|
||||
anyhow::bail!("distributed_stream_command_timeout_ms must be > 0");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-server connection config (used in multi-server TOML `[[servers]]`).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ServerEntry {
|
||||
pub aether_url: String,
|
||||
pub management_token: String,
|
||||
/// Per-server node name override. Falls back to the global `node_name`.
|
||||
pub node_name: Option<String>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TOML config file support
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Serializable config for TOML file persistence.
|
||||
/// All fields are optional -- only populated values are written.
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
pub struct ConfigFile {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub aether_url: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub management_token: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub public_ip: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub node_name: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub node_region: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub heartbeat_interval: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub allowed_ports: Option<Vec<u16>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub aether_request_timeout_secs: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub aether_connect_timeout_secs: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub aether_pool_max_idle_per_host: Option<usize>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub aether_pool_idle_timeout_secs: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub aether_tcp_keepalive_secs: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub aether_tcp_nodelay: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub aether_http2: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub aether_retry_max_attempts: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub aether_retry_base_delay_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub aether_retry_max_delay_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub max_concurrent_connections: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub dns_cache_ttl_secs: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub dns_cache_capacity: Option<usize>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_connect_timeout_secs: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_pool_max_idle_per_host: Option<usize>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_pool_idle_timeout_secs: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_tcp_keepalive_secs: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_tcp_nodelay: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub log_level: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub log_json: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tunnel_reconnect_base_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tunnel_reconnect_max_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tunnel_ping_interval_secs: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tunnel_max_streams: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tunnel_connect_timeout_secs: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tunnel_tcp_keepalive_secs: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tunnel_tcp_nodelay: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tunnel_stale_timeout_secs: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tunnel_connections: Option<u32>,
|
||||
|
||||
/// Multi-server config: each entry connects to a separate Aether instance.
|
||||
/// When present, top-level aether_url/management_token are ignored for
|
||||
/// tunnel connections (but still injected as env for clap compatibility).
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub servers: Vec<ServerEntry>,
|
||||
}
|
||||
|
||||
impl ConfigFile {
|
||||
/// Load from a TOML file.
|
||||
pub fn load(path: &Path) -> anyhow::Result<Self> {
|
||||
let content = std::fs::read_to_string(path)?;
|
||||
Ok(toml::from_str(&content)?)
|
||||
}
|
||||
|
||||
/// Save to a TOML file.
|
||||
pub fn save(&self, path: &Path) -> anyhow::Result<()> {
|
||||
let content = toml::to_string_pretty(self)?;
|
||||
std::fs::write(path, content)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Detect and migrate a 0.1.x config file to 0.2.0 format in-place.
|
||||
///
|
||||
/// Returns `true` if migration was performed, `false` if already current.
|
||||
/// The original file is backed up as `<name>.v1.bak` before rewriting.
|
||||
pub fn migrate_legacy(path: &Path) -> anyhow::Result<bool> {
|
||||
let content = match std::fs::read_to_string(path) {
|
||||
Ok(c) => c,
|
||||
Err(_) => return Ok(false),
|
||||
};
|
||||
let mut table: toml::map::Map<String, toml::Value> = toml::from_str(&content)?;
|
||||
|
||||
// Detect legacy format: presence of any 0.1.x-only key.
|
||||
let is_legacy = LEGACY_ONLY_KEYS.iter().any(|k| table.contains_key(*k))
|
||||
|| DELEGATE_TO_UPSTREAM
|
||||
.iter()
|
||||
.any(|(old, _)| table.contains_key(*old));
|
||||
|
||||
if !is_legacy {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
// 1. Rename delegate_* -> upstream_* (carry over user-customized values)
|
||||
for &(old, new) in DELEGATE_TO_UPSTREAM {
|
||||
if let Some(val) = table.remove(old) {
|
||||
table.entry(new.to_string()).or_insert(val);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Build [[servers]] from top-level aether_url + management_token + node_name
|
||||
if !table.contains_key("servers") {
|
||||
let aether_url = table.get("aether_url").and_then(|v| v.as_str());
|
||||
let management_token = table.get("management_token").and_then(|v| v.as_str());
|
||||
if let (Some(url), Some(token)) = (aether_url, management_token) {
|
||||
let mut entry = toml::map::Map::new();
|
||||
entry.insert("aether_url".into(), toml::Value::String(url.to_string()));
|
||||
entry.insert(
|
||||
"management_token".into(),
|
||||
toml::Value::String(token.to_string()),
|
||||
);
|
||||
if let Some(name) = table.get("node_name").and_then(|v| v.as_str()) {
|
||||
entry.insert("node_name".into(), toml::Value::String(name.to_string()));
|
||||
}
|
||||
table.insert(
|
||||
"servers".into(),
|
||||
toml::Value::Array(vec![toml::Value::Table(entry)]),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Remove top-level fields that are now in [[servers]] or obsolete
|
||||
table.remove("aether_url");
|
||||
table.remove("management_token");
|
||||
table.remove("node_name");
|
||||
for &key in LEGACY_ONLY_KEYS {
|
||||
table.remove(key);
|
||||
}
|
||||
|
||||
// 4. Backup original file (abort migration if backup fails)
|
||||
let backup_path = path.with_extension("v1.bak");
|
||||
std::fs::copy(path, &backup_path).map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
"failed to backup config before migration: {} -> {}: {}",
|
||||
path.display(),
|
||||
backup_path.display(),
|
||||
e
|
||||
)
|
||||
})?;
|
||||
|
||||
// 5. Write migrated config
|
||||
let new_content = toml::to_string_pretty(&table)?;
|
||||
std::fs::write(path, &new_content)?;
|
||||
|
||||
eprintln!(" Config migrated from 0.1.x to 0.2.0 format.");
|
||||
eprintln!(" Backup saved: {}", backup_path.display());
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Resolve the effective server list.
|
||||
///
|
||||
/// If `[[servers]]` is present, use it. Otherwise fall back to the
|
||||
/// top-level `aether_url` + `management_token` as a single server.
|
||||
pub fn effective_servers(&self) -> Vec<ServerEntry> {
|
||||
if !self.servers.is_empty() {
|
||||
return self.servers.clone();
|
||||
}
|
||||
match (&self.aether_url, &self.management_token) {
|
||||
(Some(url), Some(token)) => vec![ServerEntry {
|
||||
aether_url: url.clone(),
|
||||
management_token: token.clone(),
|
||||
node_name: None,
|
||||
}],
|
||||
_ => vec![],
|
||||
}
|
||||
}
|
||||
|
||||
/// Inject values as environment variables so clap picks them up.
|
||||
///
|
||||
/// Only sets variables that are **not** already present in the
|
||||
/// environment, preserving the precedence: CLI > env > config file.
|
||||
pub fn inject_env(&self) {
|
||||
self.inject_env_inner(false);
|
||||
}
|
||||
|
||||
/// Inject values as environment variables, **overriding** any existing
|
||||
/// values. Used after setup to ensure the freshly-saved config takes
|
||||
/// effect before re-parsing.
|
||||
pub fn inject_env_override(&self) {
|
||||
self.inject_env_inner(true);
|
||||
}
|
||||
|
||||
fn inject_env_inner(&self, force: bool) {
|
||||
macro_rules! set {
|
||||
($env:expr, $val:expr) => {
|
||||
if let Some(ref v) = $val {
|
||||
if force || std::env::var($env).is_err() {
|
||||
std::env::set_var($env, v.to_string());
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// When top-level fields are absent, fall back to the first [[servers]]
|
||||
// entry so that clap's required `aether_url` / `management_token` are
|
||||
// satisfied even with the new config format.
|
||||
let first_server = self.servers.first();
|
||||
let aether_url = self
|
||||
.aether_url
|
||||
.as_deref()
|
||||
.or(first_server.map(|s| s.aether_url.as_str()));
|
||||
let management_token = self
|
||||
.management_token
|
||||
.as_deref()
|
||||
.or(first_server.map(|s| s.management_token.as_str()));
|
||||
let node_name = self
|
||||
.node_name
|
||||
.as_deref()
|
||||
.or(first_server.and_then(|s| s.node_name.as_deref()));
|
||||
|
||||
set!("AETHER_PROXY_AETHER_URL", aether_url);
|
||||
set!("AETHER_PROXY_MANAGEMENT_TOKEN", management_token);
|
||||
set!("AETHER_PROXY_PUBLIC_IP", self.public_ip);
|
||||
set!("AETHER_PROXY_NODE_NAME", node_name);
|
||||
set!("AETHER_PROXY_NODE_REGION", self.node_region);
|
||||
set!("AETHER_PROXY_HEARTBEAT_INTERVAL", self.heartbeat_interval);
|
||||
set!(
|
||||
"AETHER_PROXY_AETHER_REQUEST_TIMEOUT",
|
||||
self.aether_request_timeout_secs
|
||||
);
|
||||
set!(
|
||||
"AETHER_PROXY_AETHER_CONNECT_TIMEOUT",
|
||||
self.aether_connect_timeout_secs
|
||||
);
|
||||
set!(
|
||||
"AETHER_PROXY_AETHER_POOL_MAX_IDLE_PER_HOST",
|
||||
self.aether_pool_max_idle_per_host
|
||||
);
|
||||
set!(
|
||||
"AETHER_PROXY_AETHER_POOL_IDLE_TIMEOUT",
|
||||
self.aether_pool_idle_timeout_secs
|
||||
);
|
||||
set!(
|
||||
"AETHER_PROXY_AETHER_TCP_KEEPALIVE",
|
||||
self.aether_tcp_keepalive_secs
|
||||
);
|
||||
set!("AETHER_PROXY_AETHER_TCP_NODELAY", self.aether_tcp_nodelay);
|
||||
set!("AETHER_PROXY_AETHER_HTTP2", self.aether_http2);
|
||||
set!(
|
||||
"AETHER_PROXY_AETHER_RETRY_MAX_ATTEMPTS",
|
||||
self.aether_retry_max_attempts
|
||||
);
|
||||
set!(
|
||||
"AETHER_PROXY_AETHER_RETRY_BASE_DELAY_MS",
|
||||
self.aether_retry_base_delay_ms
|
||||
);
|
||||
set!(
|
||||
"AETHER_PROXY_AETHER_RETRY_MAX_DELAY_MS",
|
||||
self.aether_retry_max_delay_ms
|
||||
);
|
||||
set!(
|
||||
"AETHER_PROXY_MAX_CONCURRENT_CONNECTIONS",
|
||||
self.max_concurrent_connections
|
||||
);
|
||||
set!("AETHER_PROXY_DNS_CACHE_TTL", self.dns_cache_ttl_secs);
|
||||
set!("AETHER_PROXY_DNS_CACHE_CAPACITY", self.dns_cache_capacity);
|
||||
set!(
|
||||
"AETHER_PROXY_UPSTREAM_CONNECT_TIMEOUT",
|
||||
self.upstream_connect_timeout_secs
|
||||
);
|
||||
set!(
|
||||
"AETHER_PROXY_UPSTREAM_POOL_MAX_IDLE_PER_HOST",
|
||||
self.upstream_pool_max_idle_per_host
|
||||
);
|
||||
set!(
|
||||
"AETHER_PROXY_UPSTREAM_POOL_IDLE_TIMEOUT",
|
||||
self.upstream_pool_idle_timeout_secs
|
||||
);
|
||||
set!(
|
||||
"AETHER_PROXY_UPSTREAM_TCP_KEEPALIVE",
|
||||
self.upstream_tcp_keepalive_secs
|
||||
);
|
||||
set!(
|
||||
"AETHER_PROXY_UPSTREAM_TCP_NODELAY",
|
||||
self.upstream_tcp_nodelay
|
||||
);
|
||||
set!("AETHER_PROXY_LOG_LEVEL", self.log_level);
|
||||
set!("AETHER_PROXY_LOG_JSON", self.log_json);
|
||||
set!(
|
||||
"AETHER_PROXY_TUNNEL_RECONNECT_BASE_MS",
|
||||
self.tunnel_reconnect_base_ms
|
||||
);
|
||||
set!(
|
||||
"AETHER_PROXY_TUNNEL_RECONNECT_MAX_MS",
|
||||
self.tunnel_reconnect_max_ms
|
||||
);
|
||||
set!(
|
||||
"AETHER_PROXY_TUNNEL_PING_INTERVAL",
|
||||
self.tunnel_ping_interval_secs
|
||||
);
|
||||
set!("AETHER_PROXY_TUNNEL_MAX_STREAMS", self.tunnel_max_streams);
|
||||
set!(
|
||||
"AETHER_PROXY_TUNNEL_CONNECT_TIMEOUT",
|
||||
self.tunnel_connect_timeout_secs
|
||||
);
|
||||
set!(
|
||||
"AETHER_PROXY_TUNNEL_TCP_KEEPALIVE",
|
||||
self.tunnel_tcp_keepalive_secs
|
||||
);
|
||||
set!("AETHER_PROXY_TUNNEL_TCP_NODELAY", self.tunnel_tcp_nodelay);
|
||||
set!(
|
||||
"AETHER_PROXY_TUNNEL_STALE_TIMEOUT",
|
||||
self.tunnel_stale_timeout_secs
|
||||
);
|
||||
set!("AETHER_PROXY_TUNNEL_CONNECTIONS", self.tunnel_connections);
|
||||
|
||||
// allowed_ports needs special handling (comma-separated)
|
||||
if let Some(ref ports) = self.allowed_ports {
|
||||
if force || std::env::var("AETHER_PROXY_ALLOWED_PORTS").is_err() {
|
||||
let s: String = ports
|
||||
.iter()
|
||||
.map(|p| p.to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
std::env::set_var("AETHER_PROXY_ALLOWED_PORTS", s);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
79
apps/aether-proxy/src/hardware.rs
Normal file
79
apps/aether-proxy/src/hardware.rs
Normal file
@@ -0,0 +1,79 @@
|
||||
use serde::Serialize;
|
||||
use sysinfo::System;
|
||||
use tracing::info;
|
||||
|
||||
/// Hardware information collected at startup.
|
||||
///
|
||||
/// The struct is `Serialize`-able so it can be sent directly as the
|
||||
/// `hardware_info` JSON bag in the registration request. New fields
|
||||
/// can be added without database schema migrations.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct HardwareInfo {
|
||||
pub cpu_cores: u32,
|
||||
pub total_memory_mb: u64,
|
||||
pub os_info: String,
|
||||
pub fd_limit: u64,
|
||||
#[serde(skip)]
|
||||
pub estimated_max_concurrency: u64,
|
||||
}
|
||||
|
||||
/// Collect hardware information and estimate max concurrency.
|
||||
///
|
||||
/// Should be called once at startup -- hardware does not change at runtime.
|
||||
pub fn collect() -> HardwareInfo {
|
||||
let sys = System::new_all();
|
||||
|
||||
let cpu_cores = sys.cpus().len() as u32;
|
||||
let total_memory_mb = sys.total_memory() / (1024 * 1024);
|
||||
let os_info = format!(
|
||||
"{} {}",
|
||||
System::name().unwrap_or_else(|| "Unknown".into()),
|
||||
System::os_version().unwrap_or_default(),
|
||||
)
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
// Estimate max concurrent connections:
|
||||
// - Each tokio async task uses ~8-16 KB stack + heap buffers
|
||||
// - OS file descriptor limit is often the real bottleneck
|
||||
// - Conservative formula: min(fd_limit - 100, ram_mb * 40, cpu_cores * 2000)
|
||||
let fd_limit = get_fd_limit();
|
||||
let by_fd = fd_limit.saturating_sub(100);
|
||||
let by_ram = total_memory_mb.saturating_mul(40);
|
||||
let by_cpu = (cpu_cores as u64).saturating_mul(2000);
|
||||
let estimated_max_concurrency = by_fd.min(by_ram).min(by_cpu);
|
||||
|
||||
info!(
|
||||
cpu_cores,
|
||||
total_memory_mb,
|
||||
os_info = %os_info,
|
||||
fd_limit,
|
||||
estimated_max_concurrency,
|
||||
"hardware info collected"
|
||||
);
|
||||
|
||||
HardwareInfo {
|
||||
cpu_cores,
|
||||
total_memory_mb,
|
||||
os_info,
|
||||
fd_limit,
|
||||
estimated_max_concurrency,
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the soft file-descriptor limit (RLIMIT_NOFILE).
|
||||
fn get_fd_limit() -> u64 {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let mut rlim = libc::rlimit {
|
||||
rlim_cur: 0,
|
||||
rlim_max: 0,
|
||||
};
|
||||
let ret = unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &mut rlim) };
|
||||
if ret == 0 {
|
||||
return rlim.rlim_cur;
|
||||
}
|
||||
}
|
||||
// Fallback for non-unix or error
|
||||
1024
|
||||
}
|
||||
170
apps/aether-proxy/src/main.rs
Normal file
170
apps/aether-proxy/src/main.rs
Normal file
@@ -0,0 +1,170 @@
|
||||
#![allow(clippy::large_enum_variant)]
|
||||
|
||||
mod app;
|
||||
mod config;
|
||||
mod hardware;
|
||||
mod net;
|
||||
mod registration;
|
||||
mod runtime;
|
||||
mod setup;
|
||||
mod state;
|
||||
mod target_filter;
|
||||
mod tunnel;
|
||||
mod upstream_client;
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use clap::{CommandFactory, FromArgMatches, Parser};
|
||||
|
||||
use config::Config;
|
||||
|
||||
/// Default config file name.
|
||||
const DEFAULT_CONFIG: &str = "aether-proxy.toml";
|
||||
|
||||
/// Build the full clap command: Config args + discoverable subcommands.
|
||||
///
|
||||
/// `subcommand_negates_reqs` lets subcommands bypass the required Config
|
||||
/// flags so that e.g. `aether-proxy setup` doesn't demand `--aether-url`.
|
||||
fn build_command() -> clap::Command {
|
||||
Config::command()
|
||||
.subcommand(
|
||||
clap::Command::new("setup")
|
||||
.about("Interactive setup wizard (TUI)")
|
||||
.arg(
|
||||
clap::Arg::new("config_path")
|
||||
.help("Path to config file")
|
||||
.default_value(DEFAULT_CONFIG),
|
||||
),
|
||||
)
|
||||
.subcommand(clap::Command::new("start").about("Start the systemd service"))
|
||||
.subcommand(clap::Command::new("status").about("Show service status"))
|
||||
.subcommand(clap::Command::new("logs").about("Tail service logs"))
|
||||
.subcommand(clap::Command::new("restart").about("Restart the systemd service"))
|
||||
.subcommand(clap::Command::new("stop").about("Stop the systemd service"))
|
||||
.subcommand(clap::Command::new("uninstall").about("Uninstall the systemd service"))
|
||||
.subcommand(
|
||||
clap::Command::new("upgrade")
|
||||
.about("Self-upgrade from GitHub releases")
|
||||
.arg(clap::Arg::new("version").help("Target version (e.g. 0.2.0)")),
|
||||
)
|
||||
.subcommand_negates_reqs(true)
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
rustls::crypto::ring::default_provider()
|
||||
.install_default()
|
||||
.map_err(|_| anyhow::anyhow!("Failed to install rustls CryptoProvider"))?;
|
||||
|
||||
// Load config file as env-var defaults (before clap parsing)
|
||||
let config_file_path =
|
||||
std::env::var("AETHER_PROXY_CONFIG").unwrap_or_else(|_| DEFAULT_CONFIG.to_string());
|
||||
let config_path = std::path::Path::new(&config_file_path);
|
||||
if config_path.exists() {
|
||||
// Migrate legacy 0.1.x config to 0.2.0 format if needed
|
||||
if let Err(e) = config::ConfigFile::migrate_legacy(config_path) {
|
||||
eprintln!(" WARNING: config migration failed: {}", e);
|
||||
}
|
||||
if let Ok(file_cfg) = config::ConfigFile::load(config_path) {
|
||||
file_cfg.inject_env();
|
||||
}
|
||||
}
|
||||
|
||||
// Parse CLI (subcommands + config args in one pass)
|
||||
match build_command().try_get_matches() {
|
||||
Ok(matches) => match matches.subcommand() {
|
||||
Some(("setup", sub_m)) => {
|
||||
let path = sub_m
|
||||
.get_one::<String>("config_path")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from(DEFAULT_CONFIG));
|
||||
handle_setup_result(setup::run(path)?).await
|
||||
}
|
||||
Some(("start", _)) => setup::service::cmd_start(),
|
||||
Some(("status", _)) => setup::service::cmd_status(),
|
||||
Some(("logs", _)) => setup::service::cmd_logs(),
|
||||
Some(("restart", _)) => setup::service::cmd_restart(),
|
||||
Some(("stop", _)) => setup::service::cmd_stop(),
|
||||
Some(("uninstall", _)) => setup::service::cmd_uninstall(),
|
||||
Some(("upgrade", sub_m)) => {
|
||||
let version = sub_m.get_one::<String>("version").cloned();
|
||||
setup::upgrade::cmd_upgrade(version).await
|
||||
}
|
||||
Some(_) => unreachable!(),
|
||||
None => {
|
||||
// No subcommand — run the proxy with parsed config.
|
||||
let config = Config::from_arg_matches(&matches)?;
|
||||
run_proxy(config).await
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
if e.kind() == clap::error::ErrorKind::MissingRequiredArgument {
|
||||
eprintln!("Missing required config, launching setup wizard...\n");
|
||||
handle_setup_result(setup::run(PathBuf::from(&config_file_path))?).await
|
||||
} else {
|
||||
e.exit();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Decide what to do after the setup wizard completes.
|
||||
async fn handle_setup_result(outcome: setup::SetupOutcome) -> anyhow::Result<()> {
|
||||
match outcome {
|
||||
setup::SetupOutcome::ServiceInstalled => Ok(()),
|
||||
setup::SetupOutcome::ReadyToRun(config_path) => {
|
||||
// Reload config from the file that setup just wrote, overriding
|
||||
// any stale env vars from a previous config.
|
||||
match config::ConfigFile::load(&config_path) {
|
||||
Ok(file_cfg) => file_cfg.inject_env_override(),
|
||||
Err(e) => anyhow::bail!("failed to reload config after setup: {}", e),
|
||||
}
|
||||
// Parse from env-only (argv may still contain "setup" etc.)
|
||||
let config = Config::try_parse_from(["aether-proxy"])
|
||||
.map_err(|e| anyhow::anyhow!("config invalid after setup: {}", e))?;
|
||||
eprintln!(" Starting proxy...\n");
|
||||
run_proxy(config).await
|
||||
}
|
||||
setup::SetupOutcome::Cancelled => {
|
||||
eprintln!(" Setup cancelled.");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Start the proxy server, checking for systemd conflicts first.
|
||||
async fn run_proxy(config: Config) -> anyhow::Result<()> {
|
||||
// Warn if systemd service is already running (would cause port conflict).
|
||||
// Skip this check when we ARE the systemd service (INVOCATION_ID is set by systemd).
|
||||
if std::env::var_os("INVOCATION_ID").is_none() && setup::service::is_service_active() {
|
||||
eprintln!("Warning: systemd service is already running.");
|
||||
eprintln!("Use `./aether-proxy stop` to stop it first, or manage via subcommands:");
|
||||
eprintln!(" ./aether-proxy status / logs / restart / stop");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
// Resolve server list: prefer [[servers]] from TOML, fall back to CLI/env single server.
|
||||
let config_path =
|
||||
std::env::var("AETHER_PROXY_CONFIG").unwrap_or_else(|_| DEFAULT_CONFIG.to_string());
|
||||
let servers = if std::path::Path::new(&config_path).exists() {
|
||||
config::ConfigFile::load(std::path::Path::new(&config_path))
|
||||
.ok()
|
||||
.map(|f| f.effective_servers())
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or_else(|| {
|
||||
vec![config::ServerEntry {
|
||||
aether_url: config.aether_url.clone(),
|
||||
management_token: config.management_token.clone(),
|
||||
node_name: None,
|
||||
}]
|
||||
})
|
||||
} else {
|
||||
vec![config::ServerEntry {
|
||||
aether_url: config.aether_url.clone(),
|
||||
management_token: config.management_token.clone(),
|
||||
node_name: None,
|
||||
}]
|
||||
};
|
||||
|
||||
app::run(config, servers).await
|
||||
}
|
||||
90
apps/aether-proxy/src/net.rs
Normal file
90
apps/aether-proxy/src/net.rs
Normal file
@@ -0,0 +1,90 @@
|
||||
//! Network utility functions (public IP detection, region detection).
|
||||
//!
|
||||
//! These are standalone helpers not tied to any specific client or service.
|
||||
|
||||
use aether_http::{build_http_client, HttpClientConfig};
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// Auto-detect public IP by querying external services.
|
||||
pub async fn detect_public_ip() -> anyhow::Result<String> {
|
||||
let endpoints = [
|
||||
"https://api.ipify.org",
|
||||
"https://ifconfig.me/ip",
|
||||
"https://icanhazip.com",
|
||||
];
|
||||
|
||||
let client = build_http_client(&HttpClientConfig {
|
||||
request_timeout_ms: Some(5_000),
|
||||
user_agent: Some("aether-proxy/net".to_string()),
|
||||
..HttpClientConfig::default()
|
||||
})?;
|
||||
|
||||
for endpoint in &endpoints {
|
||||
match client.get(*endpoint).send().await {
|
||||
Ok(resp) if resp.status().is_success() => {
|
||||
let ip = resp.text().await?.trim().to_string();
|
||||
if !ip.is_empty() {
|
||||
info!(ip = %ip, source = %endpoint, "detected public IP");
|
||||
return Ok(ip);
|
||||
}
|
||||
}
|
||||
Ok(resp) => {
|
||||
debug!(endpoint = %endpoint, status = %resp.status(), "IP detection failed");
|
||||
}
|
||||
Err(e) => {
|
||||
debug!(endpoint = %endpoint, error = %e, "IP detection failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
anyhow::bail!("failed to detect public IP from any source; use --public-ip")
|
||||
}
|
||||
|
||||
/// Auto-detect geographic region from a public IP address.
|
||||
///
|
||||
/// Uses multiple providers with HTTPS preferred. Falls back to ip-api.com
|
||||
/// over plain HTTP (their free tier doesn't support HTTPS).
|
||||
/// This is best-effort and non-sensitive -- region detection should never
|
||||
/// block startup.
|
||||
pub async fn detect_region(ip: &str) -> Option<String> {
|
||||
// Try HTTPS provider first
|
||||
let https_url = format!("https://ipinfo.io/{}/country", ip);
|
||||
|
||||
let client = build_http_client(&HttpClientConfig {
|
||||
request_timeout_ms: Some(5_000),
|
||||
user_agent: Some("aether-proxy/net".to_string()),
|
||||
..HttpClientConfig::default()
|
||||
})
|
||||
.ok()?;
|
||||
|
||||
// Try ipinfo.io (HTTPS, returns plain text country code)
|
||||
if let Ok(resp) = client.get(&https_url).send().await {
|
||||
if resp.status().is_success() {
|
||||
if let Ok(text) = resp.text().await {
|
||||
let code = text.trim();
|
||||
if !code.is_empty() && code.len() <= 3 {
|
||||
info!(region = %code, ip = %ip, source = "ipinfo.io", "detected region");
|
||||
return Some(code.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: ip-api.com (HTTP only on free tier, non-sensitive data)
|
||||
let http_url = format!("http://ip-api.com/json/{}?fields=countryCode", ip);
|
||||
match client.get(&http_url).send().await {
|
||||
Ok(resp) if resp.status().is_success() => {
|
||||
let body: serde_json::Value = resp.json().await.ok()?;
|
||||
let code = body.get("countryCode")?.as_str()?;
|
||||
if code.is_empty() {
|
||||
return None;
|
||||
}
|
||||
info!(region = %code, ip = %ip, source = "ip-api.com", "detected region");
|
||||
Some(code.to_string())
|
||||
}
|
||||
_ => {
|
||||
debug!(ip = %ip, "region detection failed");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
235
apps/aether-proxy/src/registration/client.rs
Normal file
235
apps/aether-proxy/src/registration/client.rs
Normal file
@@ -0,0 +1,235 @@
|
||||
use aether_http::{build_http_client, jittered_delay_for_retry, HttpClientConfig, HttpRetryConfig};
|
||||
use reqwest::{Client, StatusCode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::time::sleep;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::hardware::HardwareInfo;
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct RegisterRequest {
|
||||
name: String,
|
||||
ip: String,
|
||||
port: u16,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
region: Option<String>,
|
||||
heartbeat_interval: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
hardware_info: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
estimated_max_concurrency: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
proxy_metadata: Option<serde_json::Value>,
|
||||
tunnel_mode: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct RegisterResponse {
|
||||
pub node_id: String,
|
||||
}
|
||||
|
||||
/// Remote configuration pushed by the Aether management backend.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct RemoteConfig {
|
||||
pub node_name: Option<String>,
|
||||
pub allowed_ports: Option<Vec<u16>>,
|
||||
pub log_level: Option<String>,
|
||||
pub heartbeat_interval: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct UnregisterRequest {
|
||||
node_id: String,
|
||||
}
|
||||
|
||||
/// Aether API client for proxy node lifecycle management.
|
||||
pub struct AetherClient {
|
||||
http: Client,
|
||||
base_url: String,
|
||||
token: String,
|
||||
retry: HttpRetryConfig,
|
||||
}
|
||||
|
||||
impl AetherClient {
|
||||
pub fn new(config: &Config, aether_url: &str, management_token: &str) -> Self {
|
||||
let http = build_http_client(&HttpClientConfig {
|
||||
connect_timeout_ms: Some(config.aether_connect_timeout_secs.saturating_mul(1_000)),
|
||||
request_timeout_ms: Some(config.aether_request_timeout_secs.saturating_mul(1_000)),
|
||||
pool_idle_timeout_ms: Some(config.aether_pool_idle_timeout_secs.saturating_mul(1_000)),
|
||||
pool_max_idle_per_host: Some(config.aether_pool_max_idle_per_host),
|
||||
tcp_keepalive_ms: if config.aether_tcp_keepalive_secs > 0 {
|
||||
Some(config.aether_tcp_keepalive_secs.saturating_mul(1_000))
|
||||
} else {
|
||||
None
|
||||
},
|
||||
tcp_nodelay: config.aether_tcp_nodelay,
|
||||
http2_adaptive_window: config.aether_http2,
|
||||
user_agent: Some(format!("aether-proxy/{}", env!("CARGO_PKG_VERSION"))),
|
||||
..HttpClientConfig::default()
|
||||
})
|
||||
.expect("failed to create HTTP client");
|
||||
|
||||
let retry = HttpRetryConfig {
|
||||
max_attempts: config.aether_retry_max_attempts,
|
||||
base_delay_ms: config.aether_retry_base_delay_ms,
|
||||
max_delay_ms: config.aether_retry_max_delay_ms,
|
||||
}
|
||||
.normalized();
|
||||
|
||||
Self {
|
||||
http,
|
||||
base_url: aether_url.trim_end_matches('/').to_string(),
|
||||
token: management_token.to_string(),
|
||||
retry,
|
||||
}
|
||||
}
|
||||
|
||||
/// Register this node with Aether (idempotent upsert by ip:port).
|
||||
///
|
||||
/// Returns the stable node_id assigned by Aether.
|
||||
pub async fn register(
|
||||
&self,
|
||||
config: &Config,
|
||||
node_name: &str,
|
||||
public_ip: &str,
|
||||
hw: Option<&HardwareInfo>,
|
||||
) -> anyhow::Result<String> {
|
||||
let url = format!("{}/api/admin/proxy-nodes/register", self.base_url);
|
||||
let body = RegisterRequest {
|
||||
name: node_name.to_string(),
|
||||
ip: public_ip.to_string(),
|
||||
port: 0,
|
||||
region: config.node_region.clone(),
|
||||
heartbeat_interval: config.heartbeat_interval,
|
||||
hardware_info: hw.and_then(|h| serde_json::to_value(h).ok()),
|
||||
estimated_max_concurrency: hw.map(|h| h.estimated_max_concurrency),
|
||||
proxy_metadata: Some(serde_json::json!({
|
||||
"version": env!("CARGO_PKG_VERSION"),
|
||||
})),
|
||||
tunnel_mode: true,
|
||||
};
|
||||
|
||||
info!(
|
||||
url = %url,
|
||||
name = %body.name,
|
||||
ip = %body.ip,
|
||||
"registering with Aether"
|
||||
);
|
||||
|
||||
let resp = self
|
||||
.send_with_retry(
|
||||
|| {
|
||||
self.http
|
||||
.post(&url)
|
||||
.header("Authorization", format!("Bearer {}", self.token))
|
||||
.json(&body)
|
||||
},
|
||||
"register",
|
||||
)
|
||||
.await?;
|
||||
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
let text = resp.text().await.unwrap_or_default();
|
||||
anyhow::bail!("register failed (HTTP {}): {}", status, text);
|
||||
}
|
||||
|
||||
let data: RegisterResponse = resp.json().await?;
|
||||
info!(node_id = %data.node_id, "registered successfully");
|
||||
Ok(data.node_id)
|
||||
}
|
||||
|
||||
/// Unregister this node from Aether (graceful shutdown).
|
||||
pub async fn unregister(&self, node_id: &str) -> anyhow::Result<()> {
|
||||
let url = format!("{}/api/admin/proxy-nodes/unregister", self.base_url);
|
||||
let body = UnregisterRequest {
|
||||
node_id: node_id.to_string(),
|
||||
};
|
||||
|
||||
info!(node_id = %node_id, "unregistering from Aether");
|
||||
|
||||
let resp = self
|
||||
.send_with_retry(
|
||||
|| {
|
||||
self.http
|
||||
.post(&url)
|
||||
.header("Authorization", format!("Bearer {}", self.token))
|
||||
.json(&body)
|
||||
},
|
||||
"unregister",
|
||||
)
|
||||
.await;
|
||||
|
||||
match resp {
|
||||
Ok(r) if r.status().is_success() => {
|
||||
info!(node_id = %node_id, "unregistered successfully");
|
||||
Ok(())
|
||||
}
|
||||
Ok(r) => {
|
||||
let text = r.text().await.unwrap_or_default();
|
||||
error!(body = %text, "unregister failed");
|
||||
anyhow::bail!("unregister failed: {}", text);
|
||||
}
|
||||
Err(e) => {
|
||||
// Best-effort during shutdown
|
||||
error!(error = %e, "unregister request failed");
|
||||
anyhow::bail!("unregister request failed: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_with_retry<F>(
|
||||
&self,
|
||||
mut make_req: F,
|
||||
label: &str,
|
||||
) -> Result<reqwest::Response, reqwest::Error>
|
||||
where
|
||||
F: FnMut() -> reqwest::RequestBuilder,
|
||||
{
|
||||
let mut attempt: u32 = 0;
|
||||
|
||||
loop {
|
||||
attempt = attempt.saturating_add(1);
|
||||
let resp = make_req().send().await;
|
||||
match resp {
|
||||
Ok(resp) => {
|
||||
if should_retry_status(resp.status()) && attempt < self.retry.max_attempts {
|
||||
let sleep_for = jittered_delay_for_retry(self.retry, attempt - 1);
|
||||
debug!(
|
||||
attempt,
|
||||
status = %resp.status(),
|
||||
sleep_ms = sleep_for.as_millis(),
|
||||
label,
|
||||
"Aether request retrying"
|
||||
);
|
||||
sleep(sleep_for).await;
|
||||
continue;
|
||||
}
|
||||
return Ok(resp);
|
||||
}
|
||||
Err(e) => {
|
||||
if attempt < self.retry.max_attempts {
|
||||
let sleep_for = jittered_delay_for_retry(self.retry, attempt - 1);
|
||||
debug!(
|
||||
attempt,
|
||||
error = %e,
|
||||
sleep_ms = sleep_for.as_millis(),
|
||||
label,
|
||||
"Aether request retrying"
|
||||
);
|
||||
sleep(sleep_for).await;
|
||||
continue;
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn should_retry_status(status: StatusCode) -> bool {
|
||||
status.is_server_error()
|
||||
|| status == StatusCode::TOO_MANY_REQUESTS
|
||||
|| status == StatusCode::REQUEST_TIMEOUT
|
||||
}
|
||||
1
apps/aether-proxy/src/registration/mod.rs
Normal file
1
apps/aether-proxy/src/registration/mod.rs
Normal file
@@ -0,0 +1 @@
|
||||
pub mod client;
|
||||
121
apps/aether-proxy/src/runtime.rs
Normal file
121
apps/aether-proxy/src/runtime.rs
Normal file
@@ -0,0 +1,121 @@
|
||||
//! Runtime-mutable configuration that can be updated remotely via heartbeat.
|
||||
//!
|
||||
//! Fields in [`DynamicConfig`] are initially populated from the static
|
||||
//! [`Config`](crate::config::Config) and may be overridden by the Aether
|
||||
//! management backend through the heartbeat response.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use tracing::info;
|
||||
|
||||
use crate::config::Config;
|
||||
|
||||
/// Configuration that can be changed at runtime without restart.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DynamicConfig {
|
||||
pub node_name: String,
|
||||
pub allowed_ports: Arc<HashSet<u16>>,
|
||||
pub log_level: String,
|
||||
pub heartbeat_interval: u64,
|
||||
/// Monotonically increasing version from the backend.
|
||||
/// `0` means no remote config has ever been applied.
|
||||
pub config_version: u64,
|
||||
}
|
||||
|
||||
impl DynamicConfig {
|
||||
/// Initialize from static config (startup defaults).
|
||||
pub fn from_config(config: &Config) -> Self {
|
||||
Self {
|
||||
node_name: config.node_name.clone(),
|
||||
allowed_ports: Arc::new(config.allowed_ports.iter().copied().collect()),
|
||||
log_level: config.log_level.clone(),
|
||||
heartbeat_interval: config.heartbeat_interval,
|
||||
config_version: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared dynamic config handle (lock-free reads via ArcSwap).
|
||||
pub type SharedDynamicConfig = Arc<ArcSwap<DynamicConfig>>;
|
||||
|
||||
// -- Log-level hot-reload -----
|
||||
|
||||
/// Global log-level reloader function, set during tracing init.
|
||||
type LogReloader = Box<dyn Fn(&str) + Send + Sync>;
|
||||
|
||||
static LOG_RELOADER: OnceLock<LogReloader> = OnceLock::new();
|
||||
|
||||
/// Register the log-level reload function (called once from `init_tracing`).
|
||||
pub fn set_log_reloader(f: LogReloader) {
|
||||
let _ = LOG_RELOADER.set(f);
|
||||
}
|
||||
|
||||
/// Apply a remote config update to the dynamic config.
|
||||
///
|
||||
/// Uses copy-on-write: loads the current snapshot, clones it, applies changes,
|
||||
/// and stores the new Arc. Reads are always lock-free.
|
||||
///
|
||||
/// Returns `true` if the config was actually changed.
|
||||
pub fn apply_remote_config(
|
||||
dynamic: &SharedDynamicConfig,
|
||||
remote: &crate::registration::client::RemoteConfig,
|
||||
version: u64,
|
||||
) -> bool {
|
||||
let current = dynamic.load();
|
||||
|
||||
if version <= current.config_version {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut new_cfg = (**current).clone();
|
||||
let mut changed = Vec::new();
|
||||
|
||||
if let Some(ref name) = remote.node_name {
|
||||
if *name != new_cfg.node_name {
|
||||
changed.push(format!("node_name -> {}", name));
|
||||
new_cfg.node_name = name.clone();
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ref ports) = remote.allowed_ports {
|
||||
let new_set: HashSet<u16> = ports.iter().copied().collect();
|
||||
if new_set != *new_cfg.allowed_ports {
|
||||
changed.push(format!("allowed_ports -> {:?}", ports));
|
||||
new_cfg.allowed_ports = Arc::new(new_set);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(interval) = remote.heartbeat_interval {
|
||||
if interval != new_cfg.heartbeat_interval {
|
||||
changed.push(format!("heartbeat_interval -> {}s", interval));
|
||||
new_cfg.heartbeat_interval = interval;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ref level) = remote.log_level {
|
||||
if *level != new_cfg.log_level {
|
||||
changed.push(format!("log_level -> {}", level));
|
||||
new_cfg.log_level = level.clone();
|
||||
// Hot-reload tracing filter
|
||||
if let Some(reloader) = LOG_RELOADER.get() {
|
||||
reloader(level);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let has_changes = !changed.is_empty();
|
||||
|
||||
if has_changes {
|
||||
new_cfg.config_version = version;
|
||||
info!(
|
||||
version,
|
||||
changes = %changed.join(", "),
|
||||
"remote config applied"
|
||||
);
|
||||
dynamic.store(Arc::new(new_cfg));
|
||||
}
|
||||
|
||||
has_changes
|
||||
}
|
||||
66
apps/aether-proxy/src/safe_dns.rs
Normal file
66
apps/aether-proxy/src/safe_dns.rs
Normal file
@@ -0,0 +1,66 @@
|
||||
//! Safe DNS resolver for reqwest that reuses validated addresses from DnsCache.
|
||||
//!
|
||||
//! This resolver ensures reqwest connects only to addresses that have been
|
||||
//! previously validated by `target_filter::validate_target()`, eliminating
|
||||
//! the TOCTTOU gap where DNS rebinding could redirect traffic to private IPs.
|
||||
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use reqwest::dns::{Addrs, Name, Resolve, Resolving};
|
||||
|
||||
use crate::target_filter::{self, DnsCache};
|
||||
|
||||
/// A DNS resolver that serves validated public addresses from the shared DnsCache.
|
||||
///
|
||||
/// When reqwest needs to resolve a hostname, this resolver returns addresses
|
||||
/// from the cache (populated by `validate_target()` during request validation).
|
||||
/// If the hostname is not in cache (shouldn't happen in normal flow), it
|
||||
/// performs a fresh resolution with private-IP filtering.
|
||||
pub struct SafeDnsResolver {
|
||||
dns_cache: Arc<DnsCache>,
|
||||
}
|
||||
|
||||
impl SafeDnsResolver {
|
||||
pub fn new(dns_cache: Arc<DnsCache>) -> Self {
|
||||
Self { dns_cache }
|
||||
}
|
||||
}
|
||||
|
||||
impl Resolve for SafeDnsResolver {
|
||||
fn resolve(&self, name: Name) -> Resolving {
|
||||
let dns_cache = Arc::clone(&self.dns_cache);
|
||||
Box::pin(async move {
|
||||
let host = name.as_str();
|
||||
|
||||
// Try cache first (should be populated by validate_target).
|
||||
// reqwest resolves by hostname only (no port), so use host-only lookup.
|
||||
if let Some(addrs) = dns_cache.get_by_host(host).await {
|
||||
let socket_addrs: Vec<SocketAddr> = (*addrs).clone();
|
||||
return Ok(Box::new(socket_addrs.into_iter()) as Addrs);
|
||||
}
|
||||
|
||||
// Fallback: resolve with private-IP filtering (defensive).
|
||||
// This path should rarely be hit since validate_target() runs first.
|
||||
// We don't know the real port here (reqwest Resolve only gives hostname),
|
||||
// so resolve directly without caching to avoid polluting the cache with
|
||||
// an incorrect port-based key.
|
||||
let addr_str = format!("{}:0", host);
|
||||
let resolved: Vec<SocketAddr> = tokio::net::lookup_host(&addr_str)
|
||||
.await
|
||||
.map_err(|e| -> Box<dyn std::error::Error + Send + Sync> { Box::new(e) })?
|
||||
.filter(|addr| !target_filter::is_private_ip(&addr.ip()))
|
||||
.collect();
|
||||
|
||||
if resolved.is_empty() {
|
||||
return Err(Box::new(std::io::Error::other(format!(
|
||||
"all resolved addresses for {} are private/reserved",
|
||||
host
|
||||
)))
|
||||
as Box<dyn std::error::Error + Send + Sync>);
|
||||
}
|
||||
|
||||
Ok(Box::new(resolved.into_iter()) as Addrs)
|
||||
})
|
||||
}
|
||||
}
|
||||
5
apps/aether-proxy/src/setup/mod.rs
Normal file
5
apps/aether-proxy/src/setup/mod.rs
Normal file
@@ -0,0 +1,5 @@
|
||||
pub(crate) mod service;
|
||||
mod tui;
|
||||
pub(crate) mod upgrade;
|
||||
|
||||
pub use self::tui::{run, SetupOutcome};
|
||||
256
apps/aether-proxy/src/setup/service.rs
Normal file
256
apps/aether-proxy/src/setup/service.rs
Normal file
@@ -0,0 +1,256 @@
|
||||
//! Systemd service installation for aether-proxy.
|
||||
//!
|
||||
//! Called from the setup TUI when the user enables "Install Service".
|
||||
//! The unit file points to the binary and config at their current
|
||||
//! absolute paths -- no files are copied.
|
||||
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
const UNIT_PATH: &str = "/etc/systemd/system/aether-proxy.service";
|
||||
const SERVICE_NAME: &str = "aether-proxy";
|
||||
|
||||
/// Whether systemd service installation is possible (systemd present + root).
|
||||
pub fn is_available() -> bool {
|
||||
is_systemd_available() && is_root()
|
||||
}
|
||||
|
||||
/// Install aether-proxy as a systemd service. Must be run as root.
|
||||
pub fn install_service(config_path: &Path) -> anyhow::Result<()> {
|
||||
if !is_systemd_available() {
|
||||
anyhow::bail!("systemd not available");
|
||||
}
|
||||
if !is_root() {
|
||||
anyhow::bail!("root required, use: sudo ./aether-proxy setup");
|
||||
}
|
||||
|
||||
let exe_path = std::env::current_exe()?.canonicalize()?;
|
||||
let exe_str = exe_path
|
||||
.to_str()
|
||||
.ok_or_else(|| anyhow::anyhow!("binary path contains invalid UTF-8"))?;
|
||||
|
||||
let config_abs = std::fs::canonicalize(config_path)?;
|
||||
let config_str = config_abs
|
||||
.to_str()
|
||||
.ok_or_else(|| anyhow::anyhow!("config path contains invalid UTF-8"))?;
|
||||
|
||||
let working_dir = config_abs
|
||||
.parent()
|
||||
.unwrap_or_else(|| Path::new("/"))
|
||||
.to_str()
|
||||
.unwrap_or("/");
|
||||
|
||||
// Stop existing service if running (ignore errors)
|
||||
if Path::new(UNIT_PATH).exists() {
|
||||
eprintln!(" Stopping existing service...");
|
||||
let _ = Command::new("systemctl")
|
||||
.args(["stop", SERVICE_NAME])
|
||||
.status();
|
||||
}
|
||||
|
||||
// Write unit file
|
||||
eprintln!(" Generating systemd unit file...");
|
||||
eprintln!(" Binary: {}", exe_str);
|
||||
eprintln!(" Config: {}", config_str);
|
||||
eprintln!(" WorkDir: {}", working_dir);
|
||||
|
||||
let unit_content = format!(
|
||||
"[Unit]\n\
|
||||
Description=Aether Proxy\n\
|
||||
After=network.target\n\
|
||||
\n\
|
||||
[Service]\n\
|
||||
Type=simple\n\
|
||||
WorkingDirectory={working_dir}\n\
|
||||
Environment=AETHER_PROXY_CONFIG={config_str}\n\
|
||||
ExecStart={exe_str}\n\
|
||||
Restart=on-failure\n\
|
||||
RestartSec=5\n\
|
||||
LimitNOFILE=65535\n\
|
||||
UMask=0077\n\
|
||||
\n\
|
||||
[Install]\n\
|
||||
WantedBy=multi-user.target\n",
|
||||
);
|
||||
std::fs::write(UNIT_PATH, &unit_content)?;
|
||||
|
||||
// Reload and enable
|
||||
eprintln!(" Enabling and starting service...");
|
||||
run_cmd("systemctl", &["daemon-reload"])?;
|
||||
run_cmd("systemctl", &["enable", "--now", SERVICE_NAME])?;
|
||||
|
||||
// Verify
|
||||
eprintln!();
|
||||
let output = Command::new("systemctl")
|
||||
.args(["is-active", SERVICE_NAME])
|
||||
.output()?;
|
||||
let state = String::from_utf8_lossy(&output.stdout).trim().to_string();
|
||||
|
||||
if state == "active" {
|
||||
eprintln!(" Service started successfully!");
|
||||
} else {
|
||||
eprintln!(" Service state: {} (check logs)", state);
|
||||
}
|
||||
|
||||
eprintln!();
|
||||
eprintln!(" Commands:");
|
||||
eprintln!(" ./aether-proxy status # service status");
|
||||
eprintln!(" ./aether-proxy logs # tail logs");
|
||||
eprintln!(" sudo ./aether-proxy restart # restart");
|
||||
eprintln!(" sudo ./aether-proxy stop # stop");
|
||||
eprintln!(" sudo ./aether-proxy uninstall # remove service");
|
||||
eprintln!();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn is_systemd_available() -> bool {
|
||||
Command::new("systemctl")
|
||||
.arg("--version")
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.map(|s| s.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
pub(crate) fn is_root() -> bool {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
unsafe { libc::geteuid() == 0 }
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a systemd unit file is currently installed.
|
||||
pub fn is_installed() -> bool {
|
||||
Path::new(UNIT_PATH).exists()
|
||||
}
|
||||
|
||||
/// Remove the systemd service (called from setup TUI when Install Service is toggled off).
|
||||
pub fn uninstall_service() -> anyhow::Result<()> {
|
||||
if !Path::new(UNIT_PATH).exists() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
eprintln!(" Stopping and removing existing service...");
|
||||
let _ = Command::new("systemctl")
|
||||
.args(["disable", "--now", SERVICE_NAME])
|
||||
.status();
|
||||
|
||||
std::fs::remove_file(UNIT_PATH)?;
|
||||
eprintln!(" Removed {}", UNIT_PATH);
|
||||
run_cmd("systemctl", &["daemon-reload"])?;
|
||||
eprintln!(" Service uninstalled.");
|
||||
eprintln!();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check if the systemd service is currently active.
|
||||
pub fn is_service_active() -> bool {
|
||||
std::path::Path::new(UNIT_PATH).exists()
|
||||
&& Command::new("systemctl")
|
||||
.args(["is-active", "--quiet", SERVICE_NAME])
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.map(|s| s.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
// ── CLI subcommands (systemd wrappers) ──────────────────────────────────────
|
||||
|
||||
fn ensure_service_installed() -> anyhow::Result<()> {
|
||||
if !std::path::Path::new(UNIT_PATH).exists() {
|
||||
anyhow::bail!("service not installed, run `sudo ./aether-proxy setup` first");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_root_and_service() -> anyhow::Result<()> {
|
||||
ensure_service_installed()?;
|
||||
if !is_root() {
|
||||
anyhow::bail!("root required, use: sudo ./aether-proxy <command>");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `aether-proxy status` -- show service status.
|
||||
pub fn cmd_status() -> anyhow::Result<()> {
|
||||
ensure_service_installed()?;
|
||||
let status = Command::new("systemctl")
|
||||
.args(["status", SERVICE_NAME])
|
||||
.status()?;
|
||||
// systemctl status returns non-zero when inactive; that's fine
|
||||
std::process::exit(status.code().unwrap_or(1));
|
||||
}
|
||||
|
||||
/// `aether-proxy logs` -- tail service logs.
|
||||
pub fn cmd_logs() -> anyhow::Result<()> {
|
||||
ensure_service_installed()?;
|
||||
let status = Command::new("journalctl")
|
||||
.args(["-u", SERVICE_NAME, "-f", "--no-pager", "-n", "100"])
|
||||
.status()?;
|
||||
std::process::exit(status.code().unwrap_or(1));
|
||||
}
|
||||
|
||||
/// `aether-proxy start` -- start the service.
|
||||
pub fn cmd_start() -> anyhow::Result<()> {
|
||||
ensure_root_and_service()?;
|
||||
run_cmd("systemctl", &["start", SERVICE_NAME])?;
|
||||
eprintln!(" Service started.");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `aether-proxy restart` -- restart the service.
|
||||
pub fn cmd_restart() -> anyhow::Result<()> {
|
||||
ensure_root_and_service()?;
|
||||
run_cmd("systemctl", &["restart", SERVICE_NAME])?;
|
||||
eprintln!(" Service restarted.");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `aether-proxy stop` -- stop the service.
|
||||
pub fn cmd_stop() -> anyhow::Result<()> {
|
||||
ensure_root_and_service()?;
|
||||
run_cmd("systemctl", &["stop", SERVICE_NAME])?;
|
||||
eprintln!(" Service stopped.");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `aether-proxy uninstall` -- disable and remove the systemd service.
|
||||
pub fn cmd_uninstall() -> anyhow::Result<()> {
|
||||
ensure_root_and_service()?;
|
||||
|
||||
eprintln!(" Stopping and disabling service...");
|
||||
let _ = Command::new("systemctl")
|
||||
.args(["disable", "--now", SERVICE_NAME])
|
||||
.status();
|
||||
|
||||
if std::path::Path::new(UNIT_PATH).exists() {
|
||||
std::fs::remove_file(UNIT_PATH)?;
|
||||
eprintln!(" Removed {}", UNIT_PATH);
|
||||
}
|
||||
|
||||
run_cmd("systemctl", &["daemon-reload"])?;
|
||||
eprintln!(" Service uninstalled.");
|
||||
eprintln!();
|
||||
eprintln!(" Config file and TLS certs are preserved. Remove manually if needed.");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn run_cmd(program: &str, args: &[&str]) -> anyhow::Result<()> {
|
||||
let display = format!("{} {}", program, args.join(" "));
|
||||
eprintln!(" > {}", display);
|
||||
|
||||
let status = Command::new(program).args(args).status()?;
|
||||
if !status.success() {
|
||||
anyhow::bail!("command failed: {}", display);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
868
apps/aether-proxy/src/setup/tui.rs
Normal file
868
apps/aether-proxy/src/setup/tui.rs
Normal file
@@ -0,0 +1,868 @@
|
||||
//! Interactive TUI for configuring aether-proxy.
|
||||
//!
|
||||
//! Launched via `aether-proxy setup [path]`. Presents a full-screen form
|
||||
//! backed by ratatui where the user can navigate fields, edit values, and
|
||||
//! save to a TOML config file. Supports multi-server configuration via
|
||||
//! a tabbed interface.
|
||||
|
||||
use std::io;
|
||||
use std::path::PathBuf;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crossterm::event::{self, Event, KeyCode, KeyEvent, KeyEventKind, KeyModifiers};
|
||||
use crossterm::execute;
|
||||
use crossterm::terminal::{self, EnterAlternateScreen, LeaveAlternateScreen};
|
||||
use ratatui::backend::CrosstermBackend;
|
||||
use ratatui::layout::{Constraint, Layout, Rect};
|
||||
use ratatui::style::{Color, Modifier, Style};
|
||||
use ratatui::text::{Line, Span};
|
||||
use ratatui::widgets::{Block, Borders, Paragraph};
|
||||
use ratatui::Frame;
|
||||
use ratatui::Terminal;
|
||||
|
||||
use crate::config::{ConfigFile, ServerEntry};
|
||||
|
||||
/// Outcome of the setup wizard, returned to the caller.
|
||||
pub enum SetupOutcome {
|
||||
/// Config saved; systemd service installed and started.
|
||||
ServiceInstalled,
|
||||
/// Config saved; no service -- caller should start the proxy directly.
|
||||
ReadyToRun(PathBuf),
|
||||
/// User quit without saving.
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
/// Column width reserved for the field label (chars).
|
||||
const LABEL_WIDTH: usize = 22;
|
||||
|
||||
// -- Field types --------------------------------------------------------------
|
||||
|
||||
#[derive(Clone, Copy, PartialEq)]
|
||||
enum FieldKind {
|
||||
Text,
|
||||
Secret,
|
||||
Bool,
|
||||
LogLevel,
|
||||
}
|
||||
|
||||
struct Field {
|
||||
label: &'static str,
|
||||
key: &'static str,
|
||||
value: String,
|
||||
kind: FieldKind,
|
||||
required: bool,
|
||||
help: &'static str,
|
||||
}
|
||||
// -- Server tab ---------------------------------------------------------------
|
||||
|
||||
/// A single server tab's editable fields.
|
||||
struct ServerTab {
|
||||
fields: Vec<Field>,
|
||||
}
|
||||
|
||||
impl ServerTab {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
fields: vec![
|
||||
Field {
|
||||
label: "Aether URL",
|
||||
key: "aether_url",
|
||||
value: String::new(),
|
||||
kind: FieldKind::Text,
|
||||
required: true,
|
||||
help: "Aether URL (e.g. https://aether.example.com)",
|
||||
},
|
||||
Field {
|
||||
label: "Management Token",
|
||||
key: "management_token",
|
||||
value: String::new(),
|
||||
kind: FieldKind::Secret,
|
||||
required: true,
|
||||
help: "Aether Management Token (ae_xxx)",
|
||||
},
|
||||
Field {
|
||||
label: "Node Name",
|
||||
key: "node_name",
|
||||
value: "proxy-01".into(),
|
||||
kind: FieldKind::Text,
|
||||
required: true,
|
||||
help: "Node name for identification in Aether dashboard",
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
fn from_entry(entry: &ServerEntry) -> Self {
|
||||
let mut tab = Self::new();
|
||||
tab.fields[0].value = entry.aether_url.clone();
|
||||
tab.fields[1].value = entry.management_token.clone();
|
||||
if let Some(ref name) = entry.node_name {
|
||||
tab.fields[2].value = name.clone();
|
||||
}
|
||||
tab
|
||||
}
|
||||
}
|
||||
|
||||
// -- App state ----------------------------------------------------------------
|
||||
|
||||
#[derive(PartialEq)]
|
||||
enum Mode {
|
||||
Normal,
|
||||
Editing,
|
||||
}
|
||||
|
||||
struct App {
|
||||
server_tabs: Vec<ServerTab>,
|
||||
active_tab: usize,
|
||||
global_fields: Vec<Field>,
|
||||
selected: usize,
|
||||
mode: Mode,
|
||||
edit_buffer: String,
|
||||
edit_cursor: usize,
|
||||
config_path: PathBuf,
|
||||
modified: bool,
|
||||
message: Option<(String, Instant, bool)>,
|
||||
scroll_offset: usize,
|
||||
saved_once: bool,
|
||||
pending_quit: bool,
|
||||
confirm_delete: bool,
|
||||
}
|
||||
impl App {
|
||||
fn new(config_path: PathBuf) -> Self {
|
||||
Self {
|
||||
server_tabs: vec![ServerTab::new()],
|
||||
active_tab: 0,
|
||||
global_fields: vec![
|
||||
Field {
|
||||
label: "Log Level",
|
||||
key: "log_level",
|
||||
value: "info".into(),
|
||||
kind: FieldKind::LogLevel,
|
||||
required: true,
|
||||
help: "Log level -- Enter to cycle: trace / debug / info / warn / error",
|
||||
},
|
||||
Field {
|
||||
label: "Log JSON",
|
||||
key: "log_json",
|
||||
value: "false".into(),
|
||||
kind: FieldKind::Bool,
|
||||
required: true,
|
||||
help: "Output logs as JSON -- Enter to toggle",
|
||||
},
|
||||
Field {
|
||||
label: "Install Service",
|
||||
key: "install_service",
|
||||
value: if super::service::is_available() {
|
||||
"true"
|
||||
} else {
|
||||
"false"
|
||||
}
|
||||
.into(),
|
||||
kind: FieldKind::Bool,
|
||||
required: true,
|
||||
help: "Install as systemd service (requires root) -- Enter to toggle",
|
||||
},
|
||||
],
|
||||
selected: 0,
|
||||
mode: Mode::Normal,
|
||||
edit_buffer: String::new(),
|
||||
edit_cursor: 0,
|
||||
config_path,
|
||||
modified: false,
|
||||
message: None,
|
||||
scroll_offset: 0,
|
||||
saved_once: false,
|
||||
pending_quit: false,
|
||||
confirm_delete: false,
|
||||
}
|
||||
}
|
||||
|
||||
// -- Field accessors (unified index across server + global) ---------------
|
||||
|
||||
fn server_field_count(&self) -> usize {
|
||||
self.server_tabs[self.active_tab].fields.len()
|
||||
}
|
||||
|
||||
fn total_field_count(&self) -> usize {
|
||||
self.server_field_count() + self.global_fields.len()
|
||||
}
|
||||
|
||||
fn selected_field(&self) -> &Field {
|
||||
let sc = self.server_field_count();
|
||||
if self.selected < sc {
|
||||
&self.server_tabs[self.active_tab].fields[self.selected]
|
||||
} else {
|
||||
&self.global_fields[self.selected - sc]
|
||||
}
|
||||
}
|
||||
|
||||
fn selected_field_mut(&mut self) -> &mut Field {
|
||||
let sc = self.server_field_count();
|
||||
if self.selected < sc {
|
||||
&mut self.server_tabs[self.active_tab].fields[self.selected]
|
||||
} else {
|
||||
&mut self.global_fields[self.selected - sc]
|
||||
}
|
||||
}
|
||||
|
||||
fn clamp_selection(&mut self) {
|
||||
let max = self.total_field_count();
|
||||
if self.selected >= max {
|
||||
self.selected = max.saturating_sub(1);
|
||||
}
|
||||
self.scroll_offset = 0;
|
||||
self.confirm_delete = false;
|
||||
}
|
||||
// -- Config <-> fields -----------------------------------------------------
|
||||
|
||||
fn load_from_file(&mut self) {
|
||||
if let Ok(cfg) = ConfigFile::load(&self.config_path) {
|
||||
self.apply_config(&cfg);
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_config(&mut self, cfg: &ConfigFile) {
|
||||
// Global fields
|
||||
for field in &mut self.global_fields {
|
||||
let val: Option<String> = match field.key {
|
||||
"log_level" => cfg.log_level.clone(),
|
||||
"log_json" => cfg.log_json.map(|v| v.to_string()),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(v) = val {
|
||||
field.value = v;
|
||||
}
|
||||
}
|
||||
|
||||
// Server tabs
|
||||
let servers = cfg.effective_servers();
|
||||
if servers.is_empty() {
|
||||
let mut tab = ServerTab::new();
|
||||
// Single-server fallback: use top-level node_name
|
||||
if let Some(ref name) = cfg.node_name {
|
||||
tab.fields[2].value = name.clone();
|
||||
}
|
||||
self.server_tabs = vec![tab];
|
||||
} else {
|
||||
self.server_tabs = servers.iter().map(ServerTab::from_entry).collect();
|
||||
// For single-server mode, node_name might be in top-level only
|
||||
if self.server_tabs.len() == 1 && self.server_tabs[0].fields[2].value.is_empty() {
|
||||
if let Some(ref name) = cfg.node_name {
|
||||
self.server_tabs[0].fields[2].value = name.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
self.active_tab = 0;
|
||||
self.selected = 0;
|
||||
self.scroll_offset = 0;
|
||||
}
|
||||
|
||||
fn to_config(&self) -> ConfigFile {
|
||||
let get_global = |key: &str| -> Option<String> {
|
||||
self.global_fields
|
||||
.iter()
|
||||
.find(|f| f.key == key)
|
||||
.map(|f| f.value.clone())
|
||||
.filter(|v| !v.is_empty())
|
||||
};
|
||||
|
||||
let get_tab = |tab: &ServerTab, key: &str| -> Option<String> {
|
||||
tab.fields
|
||||
.iter()
|
||||
.find(|f| f.key == key)
|
||||
.map(|f| f.value.clone())
|
||||
.filter(|v| !v.is_empty())
|
||||
};
|
||||
|
||||
let mut cfg = ConfigFile {
|
||||
log_level: get_global("log_level"),
|
||||
log_json: get_global("log_json").and_then(|v| v.parse().ok()),
|
||||
..ConfigFile::default()
|
||||
};
|
||||
|
||||
// Always write [[servers]] format; old top-level fields are read-only compat
|
||||
cfg.servers = self
|
||||
.server_tabs
|
||||
.iter()
|
||||
.map(|tab| ServerEntry {
|
||||
aether_url: get_tab(tab, "aether_url").unwrap_or_default(),
|
||||
management_token: get_tab(tab, "management_token").unwrap_or_default(),
|
||||
node_name: get_tab(tab, "node_name"),
|
||||
})
|
||||
.collect();
|
||||
cfg
|
||||
}
|
||||
|
||||
fn save(&mut self) -> anyhow::Result<()> {
|
||||
let cfg = self.to_config();
|
||||
cfg.save(&self.config_path)?;
|
||||
// Restrict config file permissions to owner-only (contains management token).
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let _ =
|
||||
std::fs::set_permissions(&self.config_path, std::fs::Permissions::from_mode(0o600));
|
||||
}
|
||||
self.modified = false;
|
||||
self.saved_once = true;
|
||||
self.message = Some((
|
||||
format!("saved to {}", self.config_path.display()),
|
||||
Instant::now(),
|
||||
false,
|
||||
));
|
||||
Ok(())
|
||||
}
|
||||
// -- Scrolling ---------------------------------------------------------------
|
||||
|
||||
fn ensure_visible(&mut self, visible_rows: usize) {
|
||||
if visible_rows == 0 {
|
||||
return;
|
||||
}
|
||||
// Account for separator line between server and global fields
|
||||
let display_row = if self.selected >= self.server_field_count() {
|
||||
self.selected + 1
|
||||
} else {
|
||||
self.selected
|
||||
};
|
||||
if display_row < self.scroll_offset {
|
||||
self.scroll_offset = display_row;
|
||||
} else if display_row >= self.scroll_offset + visible_rows {
|
||||
self.scroll_offset = display_row - visible_rows + 1;
|
||||
}
|
||||
}
|
||||
|
||||
// -- Key handling -------------------------------------------------------------
|
||||
|
||||
/// Returns `true` when the app should exit.
|
||||
fn handle_key(&mut self, key: KeyEvent) -> bool {
|
||||
// Expire old messages (but keep quit-confirmation messages alive)
|
||||
if let Some((_, when, _)) = &self.message {
|
||||
if !self.pending_quit && !self.confirm_delete && when.elapsed() > Duration::from_secs(4)
|
||||
{
|
||||
self.message = None;
|
||||
}
|
||||
}
|
||||
|
||||
match self.mode {
|
||||
Mode::Normal => self.handle_normal(key),
|
||||
Mode::Editing => {
|
||||
self.handle_edit(key);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_normal(&mut self, key: KeyEvent) -> bool {
|
||||
// -- Quit handling (with unsaved-changes confirmation) -----------------
|
||||
let is_quit_key = matches!(key.code, KeyCode::Char('q') | KeyCode::Esc);
|
||||
|
||||
if is_quit_key {
|
||||
if !self.modified || self.pending_quit {
|
||||
return true;
|
||||
}
|
||||
self.pending_quit = true;
|
||||
self.confirm_delete = false;
|
||||
self.message = Some((
|
||||
"unsaved changes! q again to discard, ^S to save".into(),
|
||||
Instant::now(),
|
||||
true,
|
||||
));
|
||||
return false;
|
||||
}
|
||||
|
||||
// Any other key cancels pending quit / pending delete
|
||||
if self.pending_quit {
|
||||
self.pending_quit = false;
|
||||
self.message = None;
|
||||
}
|
||||
if self.confirm_delete && !matches!(key.code, KeyCode::Delete | KeyCode::Char('x')) {
|
||||
self.confirm_delete = false;
|
||||
self.message = None;
|
||||
}
|
||||
|
||||
match key.code {
|
||||
KeyCode::Char('s')
|
||||
if key.modifiers.contains(KeyModifiers::CONTROL)
|
||||
|| key.modifiers.contains(KeyModifiers::SUPER) =>
|
||||
{
|
||||
if let Err(e) = self.save() {
|
||||
self.message = Some((format!("error: {}", e), Instant::now(), true));
|
||||
}
|
||||
}
|
||||
KeyCode::Up | KeyCode::Char('k') => {
|
||||
self.selected = self.selected.saturating_sub(1);
|
||||
}
|
||||
KeyCode::Down | KeyCode::Char('j') => {
|
||||
if self.selected + 1 < self.total_field_count() {
|
||||
self.selected += 1;
|
||||
}
|
||||
}
|
||||
KeyCode::Home => self.selected = 0,
|
||||
KeyCode::End => self.selected = self.total_field_count() - 1,
|
||||
KeyCode::Enter | KeyCode::Char(' ') => {
|
||||
let kind = self.selected_field().kind;
|
||||
let key_str = self.selected_field().key;
|
||||
let value = self.selected_field().value.clone();
|
||||
match kind {
|
||||
FieldKind::Bool => {
|
||||
let toggled = if value == "true" { "false" } else { "true" };
|
||||
if key_str == "install_service"
|
||||
&& toggled == "true"
|
||||
&& !super::service::is_available()
|
||||
{
|
||||
self.message = Some((
|
||||
"requires root with systemd, use: sudo aether-proxy setup".into(),
|
||||
Instant::now(),
|
||||
true,
|
||||
));
|
||||
} else {
|
||||
self.selected_field_mut().value = toggled.into();
|
||||
self.modified = true;
|
||||
}
|
||||
}
|
||||
FieldKind::LogLevel => {
|
||||
const LEVELS: &[&str] = &["trace", "debug", "info", "warn", "error"];
|
||||
let idx = LEVELS.iter().position(|l| *l == value).unwrap_or(2);
|
||||
self.selected_field_mut().value = LEVELS[(idx + 1) % LEVELS.len()].into();
|
||||
self.modified = true;
|
||||
}
|
||||
_ => {
|
||||
self.edit_buffer = value;
|
||||
self.edit_cursor = self.edit_buffer.chars().count();
|
||||
self.mode = Mode::Editing;
|
||||
}
|
||||
}
|
||||
}
|
||||
// -- Tab navigation --
|
||||
KeyCode::Tab => {
|
||||
if self.server_tabs.len() > 1 {
|
||||
self.active_tab = (self.active_tab + 1) % self.server_tabs.len();
|
||||
self.clamp_selection();
|
||||
}
|
||||
}
|
||||
KeyCode::BackTab => {
|
||||
if self.server_tabs.len() > 1 {
|
||||
self.active_tab = if self.active_tab == 0 {
|
||||
self.server_tabs.len() - 1
|
||||
} else {
|
||||
self.active_tab - 1
|
||||
};
|
||||
self.clamp_selection();
|
||||
}
|
||||
}
|
||||
KeyCode::Char(c @ '1'..='9') if !key.modifiers.contains(KeyModifiers::CONTROL) => {
|
||||
let idx = (c as usize) - ('1' as usize);
|
||||
if idx < self.server_tabs.len() && idx != self.active_tab {
|
||||
self.active_tab = idx;
|
||||
self.clamp_selection();
|
||||
}
|
||||
}
|
||||
// -- Add / remove server --
|
||||
KeyCode::Char('+') | KeyCode::Char('a') => {
|
||||
self.server_tabs.push(ServerTab::new());
|
||||
self.active_tab = self.server_tabs.len() - 1;
|
||||
self.selected = 0;
|
||||
self.scroll_offset = 0;
|
||||
self.modified = true;
|
||||
self.message = Some((
|
||||
format!("added server {}", self.server_tabs.len()),
|
||||
Instant::now(),
|
||||
false,
|
||||
));
|
||||
}
|
||||
KeyCode::Delete | KeyCode::Char('x') => {
|
||||
if self.server_tabs.len() <= 1 {
|
||||
self.message =
|
||||
Some(("cannot remove the last server".into(), Instant::now(), true));
|
||||
} else if self.confirm_delete {
|
||||
let removed = self.active_tab + 1;
|
||||
self.server_tabs.remove(self.active_tab);
|
||||
self.active_tab = self.active_tab.min(self.server_tabs.len() - 1);
|
||||
self.clamp_selection();
|
||||
self.modified = true;
|
||||
self.message =
|
||||
Some((format!("server {} removed", removed), Instant::now(), false));
|
||||
} else {
|
||||
self.confirm_delete = true;
|
||||
self.message = Some((
|
||||
"press Delete/x again to remove this server".into(),
|
||||
Instant::now(),
|
||||
true,
|
||||
));
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn handle_edit(&mut self, key: KeyEvent) {
|
||||
match key.code {
|
||||
KeyCode::Esc => {
|
||||
self.mode = Mode::Normal;
|
||||
}
|
||||
KeyCode::Enter => {
|
||||
if self.validate_edit() {
|
||||
self.selected_field_mut().value = self.edit_buffer.clone();
|
||||
self.modified = true;
|
||||
self.mode = Mode::Normal;
|
||||
} else {
|
||||
self.message = Some(("invalid format".into(), Instant::now(), true));
|
||||
}
|
||||
}
|
||||
KeyCode::Backspace => {
|
||||
if self.edit_cursor > 0 {
|
||||
self.edit_cursor -= 1;
|
||||
let byte = self.char_byte_pos(self.edit_cursor);
|
||||
self.edit_buffer.remove(byte);
|
||||
}
|
||||
}
|
||||
KeyCode::Delete => {
|
||||
if self.edit_cursor < self.edit_buffer.chars().count() {
|
||||
let byte = self.char_byte_pos(self.edit_cursor);
|
||||
self.edit_buffer.remove(byte);
|
||||
}
|
||||
}
|
||||
KeyCode::Left => {
|
||||
self.edit_cursor = self.edit_cursor.saturating_sub(1);
|
||||
}
|
||||
KeyCode::Right => {
|
||||
let len = self.edit_buffer.chars().count();
|
||||
if self.edit_cursor < len {
|
||||
self.edit_cursor += 1;
|
||||
}
|
||||
}
|
||||
KeyCode::Home => self.edit_cursor = 0,
|
||||
KeyCode::End => self.edit_cursor = self.edit_buffer.chars().count(),
|
||||
KeyCode::Char(c) => {
|
||||
let byte = self.char_byte_pos(self.edit_cursor);
|
||||
self.edit_buffer.insert(byte, c);
|
||||
self.edit_cursor += 1;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_edit(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// Byte offset of the char at `char_idx`.
|
||||
fn char_byte_pos(&self, char_idx: usize) -> usize {
|
||||
self.edit_buffer
|
||||
.char_indices()
|
||||
.nth(char_idx)
|
||||
.map(|(i, _)| i)
|
||||
.unwrap_or(self.edit_buffer.len())
|
||||
}
|
||||
}
|
||||
// -- Rendering ----------------------------------------------------------------
|
||||
|
||||
fn ui(f: &mut Frame, app: &mut App) {
|
||||
let area = f.area();
|
||||
|
||||
let title = if app.modified {
|
||||
" Aether Proxy Setup [*] "
|
||||
} else {
|
||||
" Aether Proxy Setup "
|
||||
};
|
||||
|
||||
let outer = Block::default()
|
||||
.borders(Borders::ALL)
|
||||
.title(title)
|
||||
.title_alignment(ratatui::layout::Alignment::Center)
|
||||
.border_style(Style::default().fg(Color::Cyan));
|
||||
|
||||
let inner = outer.inner(area);
|
||||
f.render_widget(outer, area);
|
||||
|
||||
// Split: fields | tab bar | footer
|
||||
let chunks = Layout::vertical([
|
||||
Constraint::Min(1),
|
||||
Constraint::Length(1),
|
||||
Constraint::Length(4),
|
||||
])
|
||||
.split(inner);
|
||||
|
||||
render_fields(f, app, chunks[0]);
|
||||
render_tab_bar(f, app, chunks[1]);
|
||||
render_footer(f, app, chunks[2]);
|
||||
}
|
||||
|
||||
fn render_fields(f: &mut Frame, app: &mut App, area: Rect) {
|
||||
let visible = area.height as usize;
|
||||
app.ensure_visible(visible);
|
||||
|
||||
let server_count = app.server_field_count();
|
||||
let mut lines: Vec<Line> = Vec::new();
|
||||
// display_row tracks the actual row index (including separator)
|
||||
let mut display_row: usize = 0;
|
||||
|
||||
// Server fields
|
||||
for i in 0..server_count {
|
||||
if display_row >= app.scroll_offset && display_row < app.scroll_offset + visible {
|
||||
lines.push(build_field_line(app, i, display_row));
|
||||
}
|
||||
display_row += 1;
|
||||
}
|
||||
|
||||
// Separator line
|
||||
if display_row >= app.scroll_offset && display_row < app.scroll_offset + visible {
|
||||
lines.push(Line::from(Span::styled(
|
||||
" ----------------------------------------",
|
||||
Style::default().fg(Color::DarkGray),
|
||||
)));
|
||||
}
|
||||
display_row += 1;
|
||||
|
||||
// Global fields
|
||||
for i in 0..app.global_fields.len() {
|
||||
let field_idx = server_count + i;
|
||||
if display_row >= app.scroll_offset && display_row < app.scroll_offset + visible {
|
||||
lines.push(build_field_line(app, field_idx, display_row));
|
||||
}
|
||||
display_row += 1;
|
||||
}
|
||||
|
||||
let paragraph = Paragraph::new(lines);
|
||||
f.render_widget(paragraph, area);
|
||||
|
||||
// Cursor position while editing
|
||||
if app.mode == Mode::Editing {
|
||||
let sel_display_row = if app.selected >= server_count {
|
||||
app.selected + 1
|
||||
} else {
|
||||
app.selected
|
||||
};
|
||||
let row_in_view = sel_display_row.saturating_sub(app.scroll_offset);
|
||||
let prefix: u16 = 3 + LABEL_WIDTH as u16 + 2;
|
||||
let cx = area.x + prefix + app.edit_cursor as u16;
|
||||
let cy = area.y + row_in_view as u16;
|
||||
if cx < area.x + area.width && cy < area.y + area.height {
|
||||
f.set_cursor_position((cx, cy));
|
||||
}
|
||||
}
|
||||
}
|
||||
fn build_field_line(app: &App, field_idx: usize, _display_row: usize) -> Line<'static> {
|
||||
let sc = app.server_field_count();
|
||||
let field = if field_idx < sc {
|
||||
&app.server_tabs[app.active_tab].fields[field_idx]
|
||||
} else {
|
||||
&app.global_fields[field_idx - sc]
|
||||
};
|
||||
|
||||
let selected = field_idx == app.selected;
|
||||
let indicator = if selected { " > " } else { " " };
|
||||
|
||||
let label_style = if selected {
|
||||
Style::default()
|
||||
.fg(Color::Cyan)
|
||||
.add_modifier(Modifier::BOLD)
|
||||
} else {
|
||||
Style::default().fg(Color::DarkGray)
|
||||
};
|
||||
|
||||
let padded_label = format!("{:<width$}", field.label, width = LABEL_WIDTH);
|
||||
|
||||
let (value_text, value_style) = if app.mode == Mode::Editing && selected {
|
||||
(app.edit_buffer.clone(), Style::default().fg(Color::Yellow))
|
||||
} else {
|
||||
field_display(field)
|
||||
};
|
||||
|
||||
Line::from(vec![
|
||||
Span::styled(indicator.to_string(), label_style),
|
||||
Span::styled(padded_label, label_style),
|
||||
Span::raw(" "),
|
||||
Span::styled(value_text, value_style),
|
||||
])
|
||||
}
|
||||
|
||||
/// Returns (display_text, style) for a field in normal mode.
|
||||
fn field_display(field: &Field) -> (String, Style) {
|
||||
if field.value.is_empty() {
|
||||
let text = if field.required {
|
||||
"(required)".into()
|
||||
} else {
|
||||
"-".into()
|
||||
};
|
||||
let color = if field.required {
|
||||
Color::Red
|
||||
} else {
|
||||
Color::DarkGray
|
||||
};
|
||||
return (text, Style::default().fg(color));
|
||||
}
|
||||
|
||||
match field.kind {
|
||||
FieldKind::Secret => (
|
||||
"*".repeat(field.value.len().min(20)),
|
||||
Style::default().fg(Color::White),
|
||||
),
|
||||
FieldKind::Bool => {
|
||||
if field.value == "true" {
|
||||
("[x] on".into(), Style::default().fg(Color::Green))
|
||||
} else {
|
||||
("[ ] off".into(), Style::default().fg(Color::DarkGray))
|
||||
}
|
||||
}
|
||||
FieldKind::LogLevel => {
|
||||
let color = match field.value.as_str() {
|
||||
"trace" => Color::Magenta,
|
||||
"debug" => Color::Blue,
|
||||
"info" => Color::Green,
|
||||
"warn" => Color::Yellow,
|
||||
"error" => Color::Red,
|
||||
_ => Color::White,
|
||||
};
|
||||
(field.value.clone(), Style::default().fg(color))
|
||||
}
|
||||
_ => (field.value.clone(), Style::default().fg(Color::White)),
|
||||
}
|
||||
}
|
||||
fn render_tab_bar(f: &mut Frame, app: &App, area: Rect) {
|
||||
let mut spans: Vec<Span> = Vec::new();
|
||||
spans.push(Span::raw(" "));
|
||||
|
||||
for (i, tab) in app.server_tabs.iter().enumerate() {
|
||||
let num = i + 1;
|
||||
let name = tab
|
||||
.fields
|
||||
.iter()
|
||||
.find(|f| f.key == "node_name")
|
||||
.filter(|f| !f.value.is_empty())
|
||||
.map(|f| f.value.clone())
|
||||
.unwrap_or_else(|| format!("Server {}", num));
|
||||
|
||||
let label = format!(" {} {} ", num, name);
|
||||
|
||||
if i == app.active_tab {
|
||||
spans.push(Span::styled(
|
||||
label,
|
||||
Style::default()
|
||||
.fg(Color::Black)
|
||||
.bg(Color::Cyan)
|
||||
.add_modifier(Modifier::BOLD),
|
||||
));
|
||||
} else {
|
||||
spans.push(Span::styled(label, Style::default().fg(Color::DarkGray)));
|
||||
}
|
||||
spans.push(Span::raw(" "));
|
||||
}
|
||||
|
||||
spans.push(Span::styled(" + Add ", Style::default().fg(Color::Green)));
|
||||
|
||||
f.render_widget(Paragraph::new(Line::from(spans)), area);
|
||||
}
|
||||
|
||||
fn render_footer(f: &mut Frame, app: &App, area: Rect) {
|
||||
let help = app.selected_field().help;
|
||||
|
||||
let keybindings = if app.mode == Mode::Editing {
|
||||
"Enter confirm Esc cancel"
|
||||
} else if app.server_tabs.len() > 1 {
|
||||
"j/k select Enter edit Tab switch + add x remove ^S save q quit"
|
||||
} else {
|
||||
"j/k select Enter edit + add server ^S save q quit"
|
||||
};
|
||||
|
||||
let mut status_spans: Vec<Span> = vec![Span::styled(
|
||||
format!(" {}", keybindings),
|
||||
Style::default().fg(Color::DarkGray),
|
||||
)];
|
||||
|
||||
if let Some((msg, _, is_err)) = &app.message {
|
||||
let color = if *is_err { Color::Red } else { Color::Green };
|
||||
status_spans.push(Span::raw(" "));
|
||||
status_spans.push(Span::styled(msg.clone(), Style::default().fg(color)));
|
||||
}
|
||||
|
||||
let footer_text = vec![
|
||||
Line::raw(""),
|
||||
Line::from(Span::styled(
|
||||
format!(" {}", help),
|
||||
Style::default().fg(Color::DarkGray),
|
||||
)),
|
||||
Line::from(status_spans),
|
||||
];
|
||||
|
||||
let footer = Paragraph::new(footer_text).block(
|
||||
Block::default()
|
||||
.borders(Borders::TOP)
|
||||
.border_style(Style::default().fg(Color::DarkGray)),
|
||||
);
|
||||
|
||||
f.render_widget(footer, area);
|
||||
}
|
||||
// -- Entry point --------------------------------------------------------------
|
||||
|
||||
pub fn run(config_path: PathBuf) -> anyhow::Result<SetupOutcome> {
|
||||
terminal::enable_raw_mode()?;
|
||||
let mut stdout = io::stdout();
|
||||
execute!(stdout, EnterAlternateScreen)?;
|
||||
let backend = CrosstermBackend::new(stdout);
|
||||
let mut terminal = Terminal::new(backend)?;
|
||||
|
||||
let mut app = App::new(config_path.clone());
|
||||
app.load_from_file();
|
||||
|
||||
let result = event_loop(&mut terminal, &mut app);
|
||||
|
||||
terminal::disable_raw_mode()?;
|
||||
execute!(terminal.backend_mut(), LeaveAlternateScreen)?;
|
||||
terminal.show_cursor()?;
|
||||
|
||||
result?;
|
||||
|
||||
// -- Post-TUI: decide outcome ---------------------------------------------
|
||||
|
||||
if !app.saved_once {
|
||||
return Ok(SetupOutcome::Cancelled);
|
||||
}
|
||||
|
||||
eprintln!();
|
||||
eprintln!(" Config saved to {}", config_path.display());
|
||||
eprintln!();
|
||||
|
||||
let wants_service = app
|
||||
.global_fields
|
||||
.iter()
|
||||
.find(|f| f.key == "install_service")
|
||||
.map(|f| f.value == "true")
|
||||
.unwrap_or(false);
|
||||
|
||||
if wants_service {
|
||||
match super::service::install_service(&config_path) {
|
||||
Ok(()) => return Ok(SetupOutcome::ServiceInstalled),
|
||||
Err(e) => {
|
||||
eprintln!(" Service install failed: {}", e);
|
||||
eprintln!(" Starting proxy directly instead.\n");
|
||||
}
|
||||
}
|
||||
} else if super::service::is_installed() {
|
||||
if let Err(e) = super::service::uninstall_service() {
|
||||
eprintln!(" Service uninstall failed: {}", e);
|
||||
eprintln!();
|
||||
}
|
||||
}
|
||||
|
||||
Ok(SetupOutcome::ReadyToRun(config_path))
|
||||
}
|
||||
|
||||
fn event_loop(
|
||||
terminal: &mut Terminal<CrosstermBackend<io::Stdout>>,
|
||||
app: &mut App,
|
||||
) -> anyhow::Result<()> {
|
||||
loop {
|
||||
terminal.draw(|f| ui(f, app))?;
|
||||
|
||||
if event::poll(Duration::from_millis(200))? {
|
||||
if let Event::Key(key) = event::read()? {
|
||||
if key.kind == KeyEventKind::Press && app.handle_key(key) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
416
apps/aether-proxy/src/setup/upgrade.rs
Normal file
416
apps/aether-proxy/src/setup/upgrade.rs
Normal file
@@ -0,0 +1,416 @@
|
||||
//! Self-upgrade for aether-proxy.
|
||||
//!
|
||||
//! Downloads a release from GitHub, verifies SHA256 checksum, and atomically
|
||||
//! replaces the running binary. Restarts the systemd service if active.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use aether_http::{apply_http_client_config, HttpClientConfig};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
const GITHUB_API_BASE: &str = "https://api.github.com";
|
||||
const GITHUB_REPO: &str = "fawney19/Aether";
|
||||
const CURRENT_VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
// ── GitHub API types ─────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct GithubRelease {
|
||||
tag_name: String,
|
||||
name: String,
|
||||
}
|
||||
|
||||
// ── Platform detection ───────────────────────────────────────────────────────
|
||||
|
||||
fn detect_platform() -> &'static str {
|
||||
if cfg!(target_os = "linux") && cfg!(target_arch = "x86_64") {
|
||||
"linux-amd64"
|
||||
} else if cfg!(target_os = "linux") && cfg!(target_arch = "aarch64") {
|
||||
"linux-arm64"
|
||||
} else if cfg!(target_os = "macos") && cfg!(target_arch = "x86_64") {
|
||||
"macos-amd64"
|
||||
} else if cfg!(target_os = "macos") && cfg!(target_arch = "aarch64") {
|
||||
"macos-arm64"
|
||||
} else if cfg!(target_os = "windows") && cfg!(target_arch = "x86_64") {
|
||||
"windows-amd64"
|
||||
} else {
|
||||
// All supported targets are covered above; this is unreachable for
|
||||
// any platform we actually build for.
|
||||
panic!("unsupported platform: compile-time target not in the supported matrix")
|
||||
}
|
||||
}
|
||||
|
||||
// ── GitHub HTTP client ───────────────────────────────────────────────────────
|
||||
|
||||
fn build_github_client() -> anyhow::Result<reqwest::Client> {
|
||||
let mut headers = reqwest::header::HeaderMap::new();
|
||||
|
||||
if let Ok(token) = std::env::var("GITHUB_TOKEN") {
|
||||
headers.insert(
|
||||
reqwest::header::AUTHORIZATION,
|
||||
reqwest::header::HeaderValue::from_str(&format!("Bearer {}", token))?,
|
||||
);
|
||||
}
|
||||
|
||||
headers.insert(
|
||||
reqwest::header::ACCEPT,
|
||||
reqwest::header::HeaderValue::from_static("application/vnd.github+json"),
|
||||
);
|
||||
|
||||
Ok(apply_http_client_config(
|
||||
reqwest::Client::builder().default_headers(headers),
|
||||
&HttpClientConfig {
|
||||
request_timeout_ms: Some(300_000),
|
||||
user_agent: Some(format!("aether-proxy/{}", CURRENT_VERSION)),
|
||||
..HttpClientConfig::default()
|
||||
},
|
||||
)
|
||||
.build()?)
|
||||
}
|
||||
|
||||
// ── Release fetching ─────────────────────────────────────────────────────────
|
||||
|
||||
async fn fetch_release(
|
||||
client: &reqwest::Client,
|
||||
version: Option<&str>,
|
||||
) -> anyhow::Result<GithubRelease> {
|
||||
match version {
|
||||
Some(ver) => {
|
||||
// Accept both "proxy-v0.2.0" and bare "0.2.0"
|
||||
let tag = if ver.starts_with("proxy-v") {
|
||||
ver.to_string()
|
||||
} else {
|
||||
format!("proxy-v{}", ver)
|
||||
};
|
||||
let url = format!(
|
||||
"{}/repos/{}/releases/tags/{}",
|
||||
GITHUB_API_BASE, GITHUB_REPO, tag
|
||||
);
|
||||
let resp = client.get(&url).send().await?;
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
anyhow::bail!("release '{}' not found (HTTP {}): {}", tag, status, body);
|
||||
}
|
||||
Ok(resp.json().await?)
|
||||
}
|
||||
None => {
|
||||
// List releases and find the latest proxy-v* tag
|
||||
let url = format!(
|
||||
"{}/repos/{}/releases?per_page=20",
|
||||
GITHUB_API_BASE, GITHUB_REPO
|
||||
);
|
||||
let resp = client.get(&url).send().await?;
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
anyhow::bail!("failed to list releases (HTTP {}): {}", status, body);
|
||||
}
|
||||
let releases: Vec<GithubRelease> = resp.json().await?;
|
||||
releases
|
||||
.into_iter()
|
||||
.find(|r| r.tag_name.starts_with("proxy-v"))
|
||||
.ok_or_else(|| anyhow::anyhow!("no proxy-v* release found"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Download via GitHub release direct links ─────────────────────────────────
|
||||
|
||||
/// Download a release asset via the public direct download URL:
|
||||
/// `https://github.com/{repo}/releases/download/{tag}/{filename}`
|
||||
async fn download_release_file(
|
||||
client: &reqwest::Client,
|
||||
tag: &str,
|
||||
filename: &str,
|
||||
) -> anyhow::Result<Vec<u8>> {
|
||||
let url = format!(
|
||||
"https://github.com/{}/releases/download/{}/{}",
|
||||
GITHUB_REPO, tag, filename
|
||||
);
|
||||
let resp = client
|
||||
.get(&url)
|
||||
.header(reqwest::header::ACCEPT, "application/octet-stream")
|
||||
.send()
|
||||
.await?;
|
||||
if !resp.status().is_success() {
|
||||
anyhow::bail!(
|
||||
"download failed for '{}' (HTTP {})",
|
||||
filename,
|
||||
resp.status(),
|
||||
);
|
||||
}
|
||||
Ok(resp.bytes().await?.to_vec())
|
||||
}
|
||||
|
||||
fn parse_checksum(sums_text: &str, filename: &str) -> anyhow::Result<String> {
|
||||
for line in sums_text.lines() {
|
||||
// Format: "<hash> <filename>" (GNU coreutils convention)
|
||||
let mut parts = line.split_ascii_whitespace();
|
||||
let (Some(hash), Some(name)) = (parts.next(), parts.next()) else {
|
||||
continue;
|
||||
};
|
||||
if name == filename || name.ends_with(filename) {
|
||||
return Ok(hash.to_lowercase());
|
||||
}
|
||||
}
|
||||
anyhow::bail!("checksum for '{}' not found in SHA256SUMS.txt", filename);
|
||||
}
|
||||
|
||||
async fn download_and_verify(
|
||||
client: &reqwest::Client,
|
||||
tag: &str,
|
||||
platform: &str,
|
||||
dest: &Path,
|
||||
) -> anyhow::Result<()> {
|
||||
let archive_name = format!("aether-proxy-{}.tar.gz", platform);
|
||||
|
||||
eprintln!(" Downloading {}...", archive_name);
|
||||
let (archive_bytes, checksum_bytes) = tokio::try_join!(
|
||||
download_release_file(client, tag, &archive_name),
|
||||
download_release_file(client, tag, "SHA256SUMS.txt"),
|
||||
)?;
|
||||
let checksum_text = String::from_utf8(checksum_bytes)?;
|
||||
|
||||
eprintln!(
|
||||
" Downloaded {} ({} bytes)",
|
||||
archive_name,
|
||||
archive_bytes.len()
|
||||
);
|
||||
|
||||
// Verify SHA256
|
||||
let expected_hash = parse_checksum(&checksum_text, &archive_name)?;
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(&archive_bytes);
|
||||
let actual_hash = hex::encode(hasher.finalize());
|
||||
|
||||
if actual_hash != expected_hash {
|
||||
anyhow::bail!(
|
||||
"SHA256 mismatch for {}:\n expected: {}\n actual: {}",
|
||||
archive_name,
|
||||
expected_hash,
|
||||
actual_hash
|
||||
);
|
||||
}
|
||||
eprintln!(" SHA256 verified: {}", &actual_hash[..16]);
|
||||
|
||||
extract_binary(&archive_bytes, dest)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── Archive extraction ───────────────────────────────────────────────────────
|
||||
|
||||
fn extract_binary(archive_bytes: &[u8], dest: &Path) -> anyhow::Result<()> {
|
||||
use flate2::read::GzDecoder;
|
||||
use tar::Archive;
|
||||
|
||||
// Guard against decompression bombs
|
||||
const MAX_BINARY_SIZE: u64 = 100 * 1024 * 1024; // 100 MB
|
||||
|
||||
let decoder = GzDecoder::new(archive_bytes);
|
||||
let mut archive = Archive::new(decoder);
|
||||
|
||||
let binary_name = if cfg!(target_os = "windows") {
|
||||
"aether-proxy.exe"
|
||||
} else {
|
||||
"aether-proxy"
|
||||
};
|
||||
|
||||
for entry in archive.entries()? {
|
||||
let mut entry = entry?;
|
||||
// Only accept regular files -- reject symlinks to prevent write-through attacks
|
||||
if entry.header().entry_type() != tar::EntryType::Regular {
|
||||
continue;
|
||||
}
|
||||
let path = entry.path()?;
|
||||
if path.file_name().and_then(|n| n.to_str()) == Some(binary_name) {
|
||||
let size = entry.header().size()?;
|
||||
if size > MAX_BINARY_SIZE {
|
||||
anyhow::bail!(
|
||||
"binary too large ({} bytes, max {} bytes)",
|
||||
size,
|
||||
MAX_BINARY_SIZE
|
||||
);
|
||||
}
|
||||
let mut file = std::fs::File::create(dest)?;
|
||||
std::io::copy(&mut entry, &mut file)?;
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
std::fs::set_permissions(dest, std::fs::Permissions::from_mode(0o755))?;
|
||||
}
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
anyhow::bail!("'{}' not found in archive", binary_name);
|
||||
}
|
||||
|
||||
// ── Atomic binary replacement ────────────────────────────────────────────────
|
||||
|
||||
fn atomic_replace(new_binary: &Path) -> anyhow::Result<PathBuf> {
|
||||
let current_exe = std::env::current_exe()?.canonicalize()?;
|
||||
let backup_path = current_exe.with_extension("bak");
|
||||
|
||||
// Remove stale backup
|
||||
let _ = std::fs::remove_file(&backup_path);
|
||||
|
||||
// current -> .bak
|
||||
std::fs::rename(¤t_exe, &backup_path).map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
"failed to backup current binary '{}' -> '{}': {}",
|
||||
current_exe.display(),
|
||||
backup_path.display(),
|
||||
e
|
||||
)
|
||||
})?;
|
||||
|
||||
// new -> current
|
||||
if let Err(e) = std::fs::rename(new_binary, ¤t_exe) {
|
||||
eprintln!(" ERROR: failed to place new binary, rolling back...");
|
||||
let _ = std::fs::rename(&backup_path, ¤t_exe);
|
||||
anyhow::bail!(
|
||||
"failed to install new binary '{}' -> '{}': {}",
|
||||
new_binary.display(),
|
||||
current_exe.display(),
|
||||
e
|
||||
);
|
||||
}
|
||||
|
||||
eprintln!(" Binary replaced: {}", current_exe.display());
|
||||
Ok(backup_path)
|
||||
}
|
||||
|
||||
// ── Public entry point ───────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum RestartMode {
|
||||
BestEffort,
|
||||
Required,
|
||||
}
|
||||
|
||||
async fn execute_upgrade(
|
||||
version: Option<&str>,
|
||||
require_root: bool,
|
||||
restart_mode: RestartMode,
|
||||
) -> anyhow::Result<()> {
|
||||
// Resolve exe path once; reuse throughout the function
|
||||
let current_exe = std::env::current_exe()?.canonicalize()?;
|
||||
let exe_dir = current_exe
|
||||
.parent()
|
||||
.ok_or_else(|| anyhow::anyhow!("cannot determine binary directory"))?;
|
||||
let temp_path = exe_dir.join(".aether-proxy.upgrade.tmp");
|
||||
|
||||
if require_root {
|
||||
if !super::service::is_root() {
|
||||
anyhow::bail!("automatic upgrade requires root privileges");
|
||||
}
|
||||
} else if !super::service::is_root() {
|
||||
// Check write permission to binary directory for manual upgrade mode.
|
||||
let test_path = exe_dir.join(".aether-proxy.write-test");
|
||||
match std::fs::File::create(&test_path) {
|
||||
Ok(_) => {
|
||||
let _ = std::fs::remove_file(&test_path);
|
||||
}
|
||||
Err(_) => {
|
||||
anyhow::bail!(
|
||||
"no write access to {}. Use: sudo aether-proxy upgrade",
|
||||
exe_dir.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let platform = detect_platform();
|
||||
eprintln!(" Platform: {}", platform);
|
||||
eprintln!(" Current version: {}", CURRENT_VERSION);
|
||||
|
||||
let client = build_github_client()?;
|
||||
let release = fetch_release(&client, version).await?;
|
||||
let target_tag = &release.tag_name;
|
||||
let target_semver = target_tag.strip_prefix("proxy-v").unwrap_or(target_tag);
|
||||
|
||||
eprintln!(" Target version: {} ({})", target_tag, release.name);
|
||||
|
||||
if target_semver == CURRENT_VERSION {
|
||||
eprintln!(
|
||||
" Already running version {}, nothing to do.",
|
||||
CURRENT_VERSION
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
eprintln!();
|
||||
eprintln!(" Upgrading: {} -> {}", CURRENT_VERSION, target_semver);
|
||||
eprintln!();
|
||||
|
||||
if let Err(e) = download_and_verify(&client, target_tag, platform, &temp_path).await {
|
||||
let _ = std::fs::remove_file(&temp_path);
|
||||
return Err(e);
|
||||
}
|
||||
let backup_path = match atomic_replace(&temp_path) {
|
||||
Ok(backup) => backup,
|
||||
Err(e) => {
|
||||
let _ = std::fs::remove_file(&temp_path);
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
|
||||
match restart_mode {
|
||||
RestartMode::BestEffort => {
|
||||
// Restart systemd service if running.
|
||||
// Use best-effort: binary is already replaced, so a restart failure should
|
||||
// not abort the whole upgrade -- the user can restart manually.
|
||||
if super::service::is_service_active() {
|
||||
if super::service::is_root() {
|
||||
eprintln!(" Restarting systemd service...");
|
||||
match super::service::run_cmd("systemctl", &["restart", "aether-proxy"]) {
|
||||
Ok(()) => eprintln!(" Service restarted."),
|
||||
Err(e) => {
|
||||
eprintln!(" WARNING: failed to restart service: {}", e);
|
||||
eprintln!(" Run manually: sudo systemctl restart aether-proxy");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
eprintln!(" Systemd service is active, but restart requires root.");
|
||||
eprintln!(" Run: sudo systemctl restart aether-proxy");
|
||||
eprintln!(" Skipping restart.");
|
||||
}
|
||||
} else {
|
||||
eprintln!(" No active systemd service detected, skipping restart.");
|
||||
}
|
||||
}
|
||||
RestartMode::Required => {
|
||||
if !super::service::is_root() {
|
||||
anyhow::bail!("automatic upgrade requires root privileges");
|
||||
}
|
||||
eprintln!(" Restarting systemd service...");
|
||||
super::service::run_cmd("systemctl", &["restart", "aether-proxy"])?;
|
||||
eprintln!(" Service restarted.");
|
||||
}
|
||||
}
|
||||
|
||||
eprintln!();
|
||||
eprintln!(" Upgrade complete!");
|
||||
eprintln!(
|
||||
" Backup kept at: {} (will be cleaned up on next upgrade)",
|
||||
backup_path.display()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `aether-proxy upgrade [version]` -- self-upgrade from GitHub releases.
|
||||
pub async fn cmd_upgrade(version: Option<String>) -> anyhow::Result<()> {
|
||||
execute_upgrade(version.as_deref(), false, RestartMode::BestEffort).await
|
||||
}
|
||||
|
||||
/// Perform automatic upgrade to a specific version.
|
||||
///
|
||||
/// This path is designed for server-pushed upgrades in systemd/root scenarios:
|
||||
/// it requires root and requires a successful `systemctl restart aether-proxy`.
|
||||
pub async fn perform_upgrade(version: &str) -> anyhow::Result<()> {
|
||||
execute_upgrade(Some(version), true, RestartMode::Required).await
|
||||
}
|
||||
183
apps/aether-proxy/src/state.rs
Normal file
183
apps/aether-proxy/src/state.rs
Normal file
@@ -0,0 +1,183 @@
|
||||
//! Shared application state passed to all subsystems.
|
||||
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::Duration;
|
||||
|
||||
use aether_runtime::{
|
||||
AdmissionPermit, ConcurrencyError, ConcurrencyGate, ConcurrencySnapshot,
|
||||
DistributedConcurrencyError, DistributedConcurrencyGate, DistributedConcurrencySnapshot,
|
||||
};
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::registration::client::AetherClient;
|
||||
use crate::runtime::SharedDynamicConfig;
|
||||
use crate::target_filter::DnsCache;
|
||||
use crate::upstream_client::UpstreamClient;
|
||||
|
||||
/// Central application state shared across all servers/tunnels.
|
||||
pub struct AppState {
|
||||
pub config: Arc<Config>,
|
||||
/// DNS cache for upstream target resolution (shared).
|
||||
pub dns_cache: Arc<DnsCache>,
|
||||
/// Hyper client for tunnel upstream requests with validated DNS and connection timing.
|
||||
pub upstream_client: UpstreamClient,
|
||||
/// Shared TLS config for tunnel WebSocket connections (avoids re-parsing root CAs on each reconnect).
|
||||
pub tunnel_tls_config: Arc<rustls::ClientConfig>,
|
||||
/// Optional per-process stream admission gate.
|
||||
pub stream_gate: Option<Arc<ConcurrencyGate>>,
|
||||
/// Optional cross-instance stream admission gate.
|
||||
pub distributed_stream_gate: Option<Arc<DistributedConcurrencyGate>>,
|
||||
}
|
||||
|
||||
/// Per-server state: one instance per Aether server connection.
|
||||
pub struct ServerContext {
|
||||
/// Human-readable label for logging (e.g. "server-0").
|
||||
pub server_label: String,
|
||||
/// Aether server URL for this connection.
|
||||
pub aether_url: String,
|
||||
/// Management token for this server.
|
||||
pub management_token: String,
|
||||
/// Resolved node name at registration time (per-server override or global fallback).
|
||||
/// After startup, the active node_name is read from `dynamic` (may be updated remotely).
|
||||
#[allow(dead_code)]
|
||||
pub node_name: String,
|
||||
/// Node ID assigned by this Aether server.
|
||||
pub node_id: Arc<RwLock<String>>,
|
||||
/// API client for this server.
|
||||
pub aether_client: Arc<AetherClient>,
|
||||
/// Dynamic config from this server's heartbeat ACKs.
|
||||
pub dynamic: SharedDynamicConfig,
|
||||
/// Per-server active connection count.
|
||||
pub active_connections: Arc<AtomicU64>,
|
||||
/// Per-server request/latency metrics.
|
||||
pub metrics: Arc<ProxyMetrics>,
|
||||
}
|
||||
|
||||
/// Aggregate metrics for reporting to Aether.
|
||||
pub struct ProxyMetrics {
|
||||
pub total_requests: AtomicU64,
|
||||
/// Cumulative connection-establishment latency in nanoseconds
|
||||
/// (DNS + TCP/TLS + TTFB, excludes response body streaming).
|
||||
pub total_latency_ns: AtomicU64,
|
||||
pub failed_requests: AtomicU64,
|
||||
pub dns_failures: AtomicU64,
|
||||
pub stream_errors: AtomicU64,
|
||||
}
|
||||
|
||||
impl ProxyMetrics {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
total_requests: AtomicU64::new(0),
|
||||
total_latency_ns: AtomicU64::new(0),
|
||||
failed_requests: AtomicU64::new(0),
|
||||
dns_failures: AtomicU64::new(0),
|
||||
stream_errors: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Record a completed request with its connection-establishment latency
|
||||
/// (DNS + TCP/TLS + TTFB, excludes response body streaming).
|
||||
pub fn record_request(&self, connect_elapsed: Duration) {
|
||||
let nanos = u64::try_from(connect_elapsed.as_nanos()).unwrap_or(u64::MAX);
|
||||
self.total_requests.fetch_add(1, Ordering::Release);
|
||||
self.total_latency_ns.fetch_add(nanos, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
|
||||
pub enum ProxyAdmissionError {
|
||||
#[error("proxy stream admission saturated at {limit} for gate {gate}")]
|
||||
Saturated { gate: &'static str, limit: usize },
|
||||
#[error("proxy stream admission unavailable for gate {gate}: {message}")]
|
||||
Unavailable {
|
||||
gate: &'static str,
|
||||
limit: usize,
|
||||
message: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
pub fn with_stream_concurrency_gate(mut self, gate: Arc<ConcurrencyGate>) -> Self {
|
||||
self.stream_gate = Some(gate);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_distributed_stream_concurrency_gate(
|
||||
mut self,
|
||||
gate: Arc<DistributedConcurrencyGate>,
|
||||
) -> Self {
|
||||
self.distributed_stream_gate = Some(gate);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn stream_concurrency_snapshot(&self) -> Option<ConcurrencySnapshot> {
|
||||
self.stream_gate.as_ref().map(|gate| gate.snapshot())
|
||||
}
|
||||
|
||||
pub async fn distributed_stream_concurrency_snapshot(
|
||||
&self,
|
||||
) -> Result<Option<DistributedConcurrencySnapshot>, DistributedConcurrencyError> {
|
||||
match &self.distributed_stream_gate {
|
||||
Some(gate) => gate.snapshot().await.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn try_acquire_stream_permit(
|
||||
&self,
|
||||
) -> Result<Option<AdmissionPermit>, ProxyAdmissionError> {
|
||||
let local = match &self.stream_gate {
|
||||
Some(gate) => Some(gate.try_acquire().map_err(|err| {
|
||||
match err {
|
||||
ConcurrencyError::Saturated { gate, limit } => {
|
||||
ProxyAdmissionError::Saturated { gate, limit }
|
||||
}
|
||||
ConcurrencyError::Closed { gate } => ProxyAdmissionError::Unavailable {
|
||||
gate,
|
||||
limit: self
|
||||
.stream_gate
|
||||
.as_ref()
|
||||
.map(|inner| inner.snapshot().limit)
|
||||
.unwrap_or(0),
|
||||
message: "local stream gate is closed".to_string(),
|
||||
},
|
||||
}
|
||||
})?),
|
||||
None => None,
|
||||
};
|
||||
|
||||
let distributed = match &self.distributed_stream_gate {
|
||||
Some(gate) => Some(gate.try_acquire().await.map_err(|err| {
|
||||
match err {
|
||||
DistributedConcurrencyError::Saturated { gate, limit } => {
|
||||
ProxyAdmissionError::Saturated { gate, limit }
|
||||
}
|
||||
DistributedConcurrencyError::Unavailable {
|
||||
gate,
|
||||
limit,
|
||||
message,
|
||||
} => ProxyAdmissionError::Unavailable {
|
||||
gate,
|
||||
limit,
|
||||
message,
|
||||
},
|
||||
DistributedConcurrencyError::InvalidConfiguration(message) => {
|
||||
ProxyAdmissionError::Unavailable {
|
||||
gate: "proxy_streams_distributed",
|
||||
limit: self
|
||||
.distributed_stream_gate
|
||||
.as_ref()
|
||||
.map(|inner| inner.limit())
|
||||
.unwrap_or(0),
|
||||
message,
|
||||
}
|
||||
}
|
||||
}
|
||||
})?),
|
||||
None => None,
|
||||
};
|
||||
|
||||
Ok(AdmissionPermit::from_parts(local, distributed))
|
||||
}
|
||||
}
|
||||
381
apps/aether-proxy/src/target_filter.rs
Normal file
381
apps/aether-proxy/src/target_filter.rs
Normal file
@@ -0,0 +1,381 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
/// Check if an IP address belongs to a private/reserved network.
|
||||
pub fn is_private_ip(ip: &IpAddr) -> bool {
|
||||
match ip {
|
||||
IpAddr::V4(v4) => is_private_ipv4(v4),
|
||||
IpAddr::V6(v6) => is_private_ipv6(v6),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_private_ipv4(ip: &Ipv4Addr) -> bool {
|
||||
let octets = ip.octets();
|
||||
// 10.0.0.0/8
|
||||
if octets[0] == 10 {
|
||||
return true;
|
||||
}
|
||||
// 172.16.0.0/12
|
||||
if octets[0] == 172 && (16..=31).contains(&octets[1]) {
|
||||
return true;
|
||||
}
|
||||
// 192.168.0.0/16
|
||||
if octets[0] == 192 && octets[1] == 168 {
|
||||
return true;
|
||||
}
|
||||
// 127.0.0.0/8
|
||||
if octets[0] == 127 {
|
||||
return true;
|
||||
}
|
||||
// 169.254.0.0/16 (link-local)
|
||||
if octets[0] == 169 && octets[1] == 254 {
|
||||
return true;
|
||||
}
|
||||
// 0.0.0.0/8
|
||||
if octets[0] == 0 {
|
||||
return true;
|
||||
}
|
||||
// 100.64.0.0/10 (CGNAT / shared address space)
|
||||
if octets[0] == 100 && (64..=127).contains(&octets[1]) {
|
||||
return true;
|
||||
}
|
||||
// 192.0.0.0/24 (IETF protocol assignments)
|
||||
if octets[0] == 192 && octets[1] == 0 && octets[2] == 0 {
|
||||
return true;
|
||||
}
|
||||
// 198.18.0.0/15 (benchmark testing)
|
||||
if octets[0] == 198 && (18..=19).contains(&octets[1]) {
|
||||
return true;
|
||||
}
|
||||
// 240.0.0.0/4 (reserved for future use)
|
||||
if octets[0] >= 240 {
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn is_private_ipv6(ip: &Ipv6Addr) -> bool {
|
||||
// ::1 loopback
|
||||
if ip.is_loopback() {
|
||||
return true;
|
||||
}
|
||||
// :: unspecified
|
||||
if ip.is_unspecified() {
|
||||
return true;
|
||||
}
|
||||
let segments = ip.segments();
|
||||
// fc00::/7 (ULA) - first byte is 0xfc or 0xfd
|
||||
if segments[0] & 0xfe00 == 0xfc00 {
|
||||
return true;
|
||||
}
|
||||
// fe80::/10 (link-local)
|
||||
if segments[0] & 0xffc0 == 0xfe80 {
|
||||
return true;
|
||||
}
|
||||
// IPv4-mapped IPv6 (::ffff:x.x.x.x) - check the embedded IPv4
|
||||
if let Some(v4) = ip.to_ipv4_mapped() {
|
||||
return is_private_ipv4(&v4);
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum FilterError {
|
||||
PrivateIp(IpAddr),
|
||||
PortNotAllowed(u16),
|
||||
DnsResolutionFailed(String),
|
||||
NoPublicAddrs(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for FilterError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::PrivateIp(ip) => write!(f, "target IP {} is in private/reserved range", ip),
|
||||
Self::PortNotAllowed(port) => write!(f, "port {} not in allowed list", port),
|
||||
Self::DnsResolutionFailed(host) => write!(f, "DNS resolution failed for {}", host),
|
||||
Self::NoPublicAddrs(host) => {
|
||||
write!(
|
||||
f,
|
||||
"all resolved addresses for {} are private/reserved",
|
||||
host
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct DnsCacheEntry {
|
||||
addrs: Arc<Vec<SocketAddr>>,
|
||||
expires_at: Instant,
|
||||
inserted_at: Instant,
|
||||
}
|
||||
|
||||
/// Lightweight DNS cache with TTL + capacity bounds.
|
||||
/// Stores all public resolved addresses per host (used by SafeDnsResolver
|
||||
/// to ensure reqwest connects to the same validated addresses).
|
||||
pub struct DnsCache {
|
||||
ttl: Duration,
|
||||
capacity: usize,
|
||||
entries: RwLock<HashMap<String, DnsCacheEntry>>,
|
||||
}
|
||||
|
||||
impl DnsCache {
|
||||
pub fn new(ttl: Duration, capacity: usize) -> Self {
|
||||
Self {
|
||||
ttl,
|
||||
capacity,
|
||||
entries: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Look up cached public addresses for a host (any port).
|
||||
///
|
||||
/// Used by `SafeDnsResolver` which only knows the hostname — returns the
|
||||
/// first unexpired entry whose key starts with `host:`.
|
||||
pub async fn get_by_host(&self, host: &str) -> Option<Arc<Vec<SocketAddr>>> {
|
||||
if self.capacity == 0 || self.ttl.is_zero() {
|
||||
return None;
|
||||
}
|
||||
let prefix = format!("{}:", host.to_ascii_lowercase());
|
||||
let now = Instant::now();
|
||||
let entries = self.entries.read().await;
|
||||
for (key, entry) in entries.iter() {
|
||||
if key.starts_with(&prefix) && entry.expires_at > now {
|
||||
return Some(Arc::clone(&entry.addrs));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Look up cached public addresses for a host + port.
|
||||
pub async fn get(&self, host: &str, port: u16) -> Option<Arc<Vec<SocketAddr>>> {
|
||||
if self.capacity == 0 || self.ttl.is_zero() {
|
||||
return None;
|
||||
}
|
||||
let key = Self::key(host, port);
|
||||
let now = Instant::now();
|
||||
|
||||
// Fast path: read lock for cache hit
|
||||
{
|
||||
let entries = self.entries.read().await;
|
||||
match entries.get(&key) {
|
||||
Some(entry) if entry.expires_at > now => return Some(Arc::clone(&entry.addrs)),
|
||||
None => return None,
|
||||
Some(_) => {} // expired, fall through to evict
|
||||
}
|
||||
}
|
||||
|
||||
// Slow path: write lock to remove expired entry
|
||||
let mut entries = self.entries.write().await;
|
||||
entries.remove(&key);
|
||||
None
|
||||
}
|
||||
|
||||
/// Insert resolved public addresses into cache.
|
||||
pub async fn insert(&self, host: &str, port: u16, addrs: Arc<Vec<SocketAddr>>) {
|
||||
if self.capacity == 0 || self.ttl.is_zero() || addrs.is_empty() {
|
||||
return;
|
||||
}
|
||||
let key = Self::key(host, port);
|
||||
let now = Instant::now();
|
||||
let mut entries = self.entries.write().await;
|
||||
entries.retain(|_, entry| entry.expires_at > now);
|
||||
while entries.len() >= self.capacity {
|
||||
let oldest_key = entries
|
||||
.iter()
|
||||
.min_by_key(|(_, entry)| entry.inserted_at)
|
||||
.map(|(key, _)| key.clone());
|
||||
if let Some(key) = oldest_key {
|
||||
entries.remove(&key);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
entries.insert(
|
||||
key,
|
||||
DnsCacheEntry {
|
||||
addrs,
|
||||
expires_at: now + self.ttl,
|
||||
inserted_at: now,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn key(host: &str, port: u16) -> String {
|
||||
format!("{}:{}", host.to_ascii_lowercase(), port)
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve a hostname to public (non-private) socket addresses.
|
||||
///
|
||||
/// Results are cached in `dns_cache`. Private/reserved IPs are filtered out.
|
||||
/// Returns an error if no public addresses remain after filtering.
|
||||
pub async fn resolve_public_addrs(
|
||||
host: &str,
|
||||
port: u16,
|
||||
dns_cache: &DnsCache,
|
||||
) -> Result<Vec<SocketAddr>, FilterError> {
|
||||
// Cache hit
|
||||
if let Some(addrs) = dns_cache.get(host, port).await {
|
||||
return Ok((*addrs).clone());
|
||||
}
|
||||
|
||||
// Async DNS resolution
|
||||
let addr_str = format!("{}:{}", host, port);
|
||||
let resolved: Vec<SocketAddr> = tokio::net::lookup_host(&addr_str)
|
||||
.await
|
||||
.map_err(|_| FilterError::DnsResolutionFailed(host.to_string()))?
|
||||
.collect();
|
||||
|
||||
if resolved.is_empty() {
|
||||
return Err(FilterError::DnsResolutionFailed(host.to_string()));
|
||||
}
|
||||
|
||||
// Filter out private/reserved addresses
|
||||
let public: Vec<SocketAddr> = resolved
|
||||
.into_iter()
|
||||
.filter(|addr| !is_private_ip(&addr.ip()))
|
||||
.collect();
|
||||
|
||||
if public.is_empty() {
|
||||
return Err(FilterError::NoPublicAddrs(host.to_string()));
|
||||
}
|
||||
|
||||
// Cache the validated public addresses
|
||||
let arc_addrs = Arc::new(public);
|
||||
dns_cache.insert(host, port, Arc::clone(&arc_addrs)).await;
|
||||
Ok((*arc_addrs).clone())
|
||||
}
|
||||
|
||||
/// Validate that the target host:port is allowed.
|
||||
///
|
||||
/// Performs port whitelist check, private IP filtering, and DNS resolution
|
||||
/// with caching. The resolved addresses are stored in the shared DnsCache
|
||||
/// so that the SafeDnsResolver can reuse them, eliminating the TOCTTOU gap.
|
||||
pub async fn validate_target(
|
||||
host: &str,
|
||||
port: u16,
|
||||
allowed_ports: &HashSet<u16>,
|
||||
dns_cache: &DnsCache,
|
||||
) -> Result<Vec<SocketAddr>, FilterError> {
|
||||
// Port whitelist check
|
||||
if !allowed_ports.contains(&port) {
|
||||
return Err(FilterError::PortNotAllowed(port));
|
||||
}
|
||||
|
||||
// Try parsing as IP directly (no DNS needed)
|
||||
if let Ok(ip) = host.parse::<IpAddr>() {
|
||||
if is_private_ip(&ip) {
|
||||
return Err(FilterError::PrivateIp(ip));
|
||||
}
|
||||
return Ok(vec![SocketAddr::new(ip, port)]);
|
||||
}
|
||||
|
||||
// Resolve and validate DNS (populates cache for SafeDnsResolver)
|
||||
resolve_public_addrs(host, port, dns_cache).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn ports() -> HashSet<u16> {
|
||||
[80, 443, 8080, 8443].into_iter().collect()
|
||||
}
|
||||
|
||||
fn cache() -> DnsCache {
|
||||
DnsCache::new(Duration::from_secs(60), 128)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_private_ipv4() {
|
||||
assert!(is_private_ip(&IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1))));
|
||||
assert!(is_private_ip(&IpAddr::V4(Ipv4Addr::new(172, 16, 0, 1))));
|
||||
assert!(is_private_ip(&IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1))));
|
||||
assert!(is_private_ip(&IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))));
|
||||
assert!(is_private_ip(&IpAddr::V4(Ipv4Addr::new(169, 254, 1, 1))));
|
||||
assert!(is_private_ip(&IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))));
|
||||
// CGNAT
|
||||
assert!(is_private_ip(&IpAddr::V4(Ipv4Addr::new(100, 64, 0, 1))));
|
||||
assert!(is_private_ip(&IpAddr::V4(Ipv4Addr::new(
|
||||
100, 127, 255, 254
|
||||
))));
|
||||
assert!(!is_private_ip(&IpAddr::V4(Ipv4Addr::new(
|
||||
100, 63, 255, 254
|
||||
))));
|
||||
// Benchmark testing
|
||||
assert!(is_private_ip(&IpAddr::V4(Ipv4Addr::new(198, 18, 0, 1))));
|
||||
// Reserved
|
||||
assert!(is_private_ip(&IpAddr::V4(Ipv4Addr::new(240, 0, 0, 1))));
|
||||
// Public
|
||||
assert!(!is_private_ip(&IpAddr::V4(Ipv4Addr::new(8, 8, 8, 8))));
|
||||
assert!(!is_private_ip(&IpAddr::V4(Ipv4Addr::new(203, 0, 113, 1))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_private_ipv6() {
|
||||
assert!(is_private_ip(&IpAddr::V6(Ipv6Addr::LOCALHOST)));
|
||||
assert!(is_private_ip(&IpAddr::V6(Ipv6Addr::UNSPECIFIED)));
|
||||
// fc00::1 (ULA)
|
||||
assert!(is_private_ip(&IpAddr::V6(Ipv6Addr::new(
|
||||
0xfc00, 0, 0, 0, 0, 0, 0, 1
|
||||
))));
|
||||
// fe80::1 (link-local)
|
||||
assert!(is_private_ip(&IpAddr::V6(Ipv6Addr::new(
|
||||
0xfe80, 0, 0, 0, 0, 0, 0, 1
|
||||
))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_port_not_allowed() {
|
||||
let cache = cache();
|
||||
let result = validate_target("8.8.8.8", 22, &ports(), &cache).await;
|
||||
assert!(matches!(result, Err(FilterError::PortNotAllowed(22))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_private_ip_blocked() {
|
||||
let cache = cache();
|
||||
let result = validate_target("127.0.0.1", 80, &ports(), &cache).await;
|
||||
assert!(matches!(result, Err(FilterError::PrivateIp(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_public_ip_allowed() {
|
||||
let cache = cache();
|
||||
let result = validate_target("8.8.8.8", 443, &ports(), &cache).await;
|
||||
assert!(result.is_ok());
|
||||
let addrs = result.unwrap();
|
||||
assert_eq!(addrs.len(), 1);
|
||||
assert_eq!(addrs[0].ip(), IpAddr::V4(Ipv4Addr::new(8, 8, 8, 8)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cache_stores_multiple_addrs() {
|
||||
let cache = cache();
|
||||
let addrs = vec![
|
||||
SocketAddr::new(IpAddr::V4(Ipv4Addr::new(1, 1, 1, 1)), 443),
|
||||
SocketAddr::new(IpAddr::V4(Ipv4Addr::new(1, 0, 0, 1)), 443),
|
||||
];
|
||||
cache
|
||||
.insert("example.com", 443, Arc::new(addrs.clone()))
|
||||
.await;
|
||||
let cached = cache.get("example.com", 443).await.unwrap();
|
||||
assert_eq!(*cached, addrs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cache_key_case_insensitive() {
|
||||
let cache = cache();
|
||||
let addrs = vec![SocketAddr::new(IpAddr::V4(Ipv4Addr::new(1, 1, 1, 1)), 443)];
|
||||
cache
|
||||
.insert("Example.COM", 443, Arc::new(addrs.clone()))
|
||||
.await;
|
||||
let cached = cache.get("example.com", 443).await.unwrap();
|
||||
assert_eq!(*cached, addrs);
|
||||
}
|
||||
}
|
||||
238
apps/aether-proxy/src/tunnel/client.rs
Normal file
238
apps/aether-proxy/src/tunnel/client.rs
Normal file
@@ -0,0 +1,238 @@
|
||||
//! WebSocket tunnel client: connect, authenticate, and run the tunnel.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::sync::watch;
|
||||
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
|
||||
use tokio_tungstenite::tungstenite::http;
|
||||
use tokio_tungstenite::tungstenite::protocol::WebSocketConfig;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::state::{AppState, ServerContext};
|
||||
|
||||
use super::{dispatcher, heartbeat, writer};
|
||||
|
||||
/// Outcome of a tunnel session.
|
||||
pub enum TunnelOutcome {
|
||||
/// Graceful shutdown requested by the local process.
|
||||
Shutdown,
|
||||
/// Remote side disconnected or connection lost — should reconnect.
|
||||
Disconnected,
|
||||
}
|
||||
|
||||
/// Connect to Aether's WebSocket tunnel endpoint and run until disconnected.
|
||||
///
|
||||
/// `conn_idx` identifies which connection in the pool this is (0-based).
|
||||
/// Only connection 0 sends heartbeats to avoid resetting shared metrics.
|
||||
pub async fn connect_and_run(
|
||||
state: &Arc<AppState>,
|
||||
server: &Arc<ServerContext>,
|
||||
conn_idx: usize,
|
||||
shutdown: &mut watch::Receiver<bool>,
|
||||
) -> Result<TunnelOutcome, anyhow::Error> {
|
||||
let ws_url = build_tunnel_url(server);
|
||||
info!(url = %ws_url, conn = conn_idx, "connecting tunnel");
|
||||
|
||||
// Build WebSocket request with auth headers
|
||||
let mut request = ws_url.clone().into_client_request()?;
|
||||
let headers = request.headers_mut();
|
||||
headers.insert(
|
||||
"Authorization",
|
||||
http::HeaderValue::from_str(&format!("Bearer {}", server.management_token))?,
|
||||
);
|
||||
let node_id = server.node_id.read().unwrap().clone();
|
||||
headers.insert("X-Node-Id", http::HeaderValue::from_str(&node_id)?);
|
||||
// Use dynamic node_name (may be updated by remote config) instead of
|
||||
// the static server.node_name, so that remote name changes take effect
|
||||
// on the next reconnect.
|
||||
let dynamic_node_name = server.dynamic.load().node_name.clone();
|
||||
headers.insert(
|
||||
"X-Node-Name",
|
||||
http::HeaderValue::from_str(&dynamic_node_name)?,
|
||||
);
|
||||
// Advertise per-connection max concurrent streams so the backend can
|
||||
// respect the proxy's capacity limit (backward-compatible: old backends
|
||||
// ignore this header).
|
||||
let max_streams = state.config.tunnel_max_streams.unwrap_or(128);
|
||||
headers.insert("X-Tunnel-Max-Streams", http::HeaderValue::from(max_streams));
|
||||
|
||||
// Parse host:port from URL
|
||||
let uri: http::Uri = ws_url.parse()?;
|
||||
let host = uri
|
||||
.host()
|
||||
.ok_or_else(|| anyhow::anyhow!("missing host in tunnel URL"))?;
|
||||
let is_tls = uri.scheme_str() == Some("wss");
|
||||
let port = uri.port_u16().unwrap_or(if is_tls { 443 } else { 80 });
|
||||
|
||||
// TCP connect with timeout
|
||||
let connect_timeout = Duration::from_secs(state.config.tunnel_connect_timeout_secs);
|
||||
let tcp_stream = tokio::time::timeout(connect_timeout, TcpStream::connect((host, port)))
|
||||
.await
|
||||
.map_err(|_| {
|
||||
anyhow::anyhow!(
|
||||
"tunnel TCP connect timeout ({}s)",
|
||||
connect_timeout.as_secs()
|
||||
)
|
||||
})??;
|
||||
|
||||
// Configure TCP parameters via socket2
|
||||
configure_tcp_socket(&tcp_stream, state);
|
||||
|
||||
// WebSocket upgrade (with TLS if wss://)
|
||||
let connector = if is_tls {
|
||||
Some(tokio_tungstenite::Connector::Rustls(Arc::clone(
|
||||
&state.tunnel_tls_config,
|
||||
)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// Match Python-side _MAX_FRAME_SIZE (64 MiB) to prevent tungstenite's
|
||||
// default 16 MiB limit from rejecting large AI API payloads (multi-image
|
||||
// base64 requests can exceed 16 MiB).
|
||||
let ws_config = WebSocketConfig {
|
||||
max_frame_size: Some(64 << 20),
|
||||
max_message_size: Some(64 << 20),
|
||||
..Default::default()
|
||||
};
|
||||
let handshake_timeout = Duration::from_secs(state.config.tunnel_connect_timeout_secs);
|
||||
let (ws_stream, _response) = tokio::time::timeout(
|
||||
handshake_timeout,
|
||||
tokio_tungstenite::client_async_tls_with_config(
|
||||
request,
|
||||
tcp_stream,
|
||||
Some(ws_config),
|
||||
connector,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
anyhow::anyhow!(
|
||||
"tunnel WebSocket handshake timeout ({}s)",
|
||||
handshake_timeout.as_secs()
|
||||
)
|
||||
})??;
|
||||
info!(
|
||||
conn = conn_idx,
|
||||
tcp_keepalive_secs = state.config.tunnel_tcp_keepalive_secs,
|
||||
tcp_nodelay = state.config.tunnel_tcp_nodelay,
|
||||
connect_timeout_secs = state.config.tunnel_connect_timeout_secs,
|
||||
stale_timeout_secs = state.config.tunnel_stale_timeout_secs,
|
||||
"tunnel connected"
|
||||
);
|
||||
|
||||
// NOTE: reconnect_attempts reset is handled by the caller (mod.rs)
|
||||
// based on how long the connection stayed alive.
|
||||
|
||||
// Split into read/write halves
|
||||
let (ws_sink, ws_read) = futures_util::StreamExt::split(ws_stream);
|
||||
|
||||
// Spawn writer task (with WebSocket ping keepalive)
|
||||
let ping_interval = Duration::from_secs(state.config.tunnel_ping_interval_secs);
|
||||
let (frame_tx, mut writer_handle) = writer::spawn_writer(ws_sink, ping_interval);
|
||||
|
||||
// Spawn heartbeat task (only for primary connection to avoid
|
||||
// resetting shared atomic metrics via swap(0))
|
||||
let hb_handle = if conn_idx == 0 {
|
||||
heartbeat::spawn(
|
||||
Arc::clone(state),
|
||||
Arc::clone(server),
|
||||
frame_tx.clone(),
|
||||
shutdown.clone(),
|
||||
)
|
||||
} else {
|
||||
heartbeat::spawn_noop()
|
||||
};
|
||||
|
||||
// Run dispatcher (blocks until disconnect or shutdown).
|
||||
// Also watch for writer exit — if the write half dies (e.g. the peer
|
||||
// closed the connection) but the read half stays open, dispatcher would
|
||||
// block forever on `ws_stream.next()`. Monitoring `writer_handle`
|
||||
// ensures we detect this and trigger a reconnect promptly.
|
||||
let state_clone = Arc::clone(state);
|
||||
let server_clone = Arc::clone(server);
|
||||
let outcome = tokio::select! {
|
||||
result = dispatcher::run(state_clone, server_clone, ws_read, frame_tx.clone(), hb_handle) => {
|
||||
match result {
|
||||
Ok(()) => TunnelOutcome::Disconnected,
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
writer_result = &mut writer_handle => {
|
||||
match writer_result {
|
||||
Ok(()) => warn!("writer task exited normally, triggering reconnect"),
|
||||
Err(e) => {
|
||||
if e.is_panic() {
|
||||
tracing::error!(error = %e, "writer task panicked, triggering reconnect");
|
||||
} else {
|
||||
warn!(error = %e, "writer task cancelled, triggering reconnect");
|
||||
}
|
||||
}
|
||||
}
|
||||
TunnelOutcome::Disconnected
|
||||
}
|
||||
_ = shutdown.changed() => {
|
||||
debug!("shutdown during tunnel dispatch");
|
||||
TunnelOutcome::Shutdown
|
||||
}
|
||||
};
|
||||
|
||||
// Drop our sender; the writer will exit once all stream handler clones
|
||||
// are also dropped (i.e. after they finish their in-flight work).
|
||||
drop(frame_tx);
|
||||
|
||||
// Wait for the writer task to finish with a generous timeout — the
|
||||
// dispatcher already waits up to 30s for stream handlers, so 35s here
|
||||
// covers that plus a small margin.
|
||||
// Skip if the writer already exited (the select branch that fired).
|
||||
if !writer_handle.is_finished() {
|
||||
let _ = tokio::time::timeout(Duration::from_secs(35), writer_handle).await;
|
||||
}
|
||||
|
||||
info!("tunnel disconnected");
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
/// Configure TCP keepalive and NODELAY on an established socket.
|
||||
fn configure_tcp_socket(stream: &TcpStream, state: &Arc<AppState>) {
|
||||
let sock_ref = socket2::SockRef::from(stream);
|
||||
|
||||
if state.config.tunnel_tcp_keepalive_secs > 0 {
|
||||
let keepalive = socket2::TcpKeepalive::new()
|
||||
.with_time(Duration::from_secs(state.config.tunnel_tcp_keepalive_secs))
|
||||
.with_interval(Duration::from_secs(5));
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
let keepalive = keepalive.with_retries(3);
|
||||
if let Err(e) = sock_ref.set_tcp_keepalive(&keepalive) {
|
||||
warn!(error = %e, "failed to set TCP keepalive on tunnel socket");
|
||||
}
|
||||
}
|
||||
|
||||
if state.config.tunnel_tcp_nodelay {
|
||||
if let Err(e) = sock_ref.set_nodelay(true) {
|
||||
warn!(error = %e, "failed to set TCP_NODELAY on tunnel socket");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build rustls ClientConfig with system root certificates.
|
||||
pub fn build_tls_config() -> rustls::ClientConfig {
|
||||
let root_store =
|
||||
rustls::RootCertStore::from_iter(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
|
||||
rustls::ClientConfig::builder()
|
||||
.with_root_certificates(root_store)
|
||||
.with_no_client_auth()
|
||||
}
|
||||
|
||||
fn build_tunnel_url(server: &ServerContext) -> String {
|
||||
let base = server.aether_url.trim_end_matches('/');
|
||||
let ws_base = if base.starts_with("https://") {
|
||||
base.replacen("https://", "wss://", 1)
|
||||
} else if base.starts_with("http://") {
|
||||
base.replacen("http://", "ws://", 1)
|
||||
} else {
|
||||
format!("wss://{}", base)
|
||||
};
|
||||
format!("{}/api/internal/proxy-tunnel", ws_base)
|
||||
}
|
||||
249
apps/aether-proxy/src/tunnel/dispatcher.rs
Normal file
249
apps/aether-proxy/src/tunnel/dispatcher.rs
Normal file
@@ -0,0 +1,249 @@
|
||||
//! Frame dispatcher: reads incoming WebSocket frames and routes them.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures_util::StreamExt;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
use crate::state::{AppState, ServerContext};
|
||||
|
||||
use super::heartbeat::HeartbeatHandle;
|
||||
use super::protocol::{decompress_if_gzip, Frame, MsgType, RequestMeta};
|
||||
use super::stream_handler;
|
||||
use super::writer::FrameSender;
|
||||
|
||||
/// Run the dispatcher loop, reading from the WebSocket stream.
|
||||
pub async fn run<S>(
|
||||
state: Arc<AppState>,
|
||||
server: Arc<ServerContext>,
|
||||
mut ws_stream: S,
|
||||
frame_tx: FrameSender,
|
||||
heartbeat: HeartbeatHandle,
|
||||
) -> Result<(), anyhow::Error>
|
||||
where
|
||||
S: StreamExt<Item = Result<Message, tokio_tungstenite::tungstenite::Error>>
|
||||
+ Unpin
|
||||
+ Send
|
||||
+ 'static,
|
||||
{
|
||||
// Active streams: stream_id -> body sender
|
||||
let mut streams: HashMap<u32, mpsc::Sender<Frame>> = HashMap::new();
|
||||
// Track spawned stream handlers so we can wait for them on shutdown
|
||||
let mut handler_handles: Vec<JoinHandle<()>> = Vec::new();
|
||||
let max_streams = state.config.tunnel_max_streams.unwrap_or(128) as usize;
|
||||
let mut frames_since_cleanup: u32 = 0;
|
||||
let stale_timeout = Duration::from_secs(state.config.tunnel_stale_timeout_secs);
|
||||
|
||||
// Track last time we received any data to detect stale connections
|
||||
let mut last_data_at = tokio::time::Instant::now();
|
||||
|
||||
let read_err = loop {
|
||||
let msg_result = tokio::select! {
|
||||
msg = ws_stream.next() => {
|
||||
match msg {
|
||||
Some(r) => r,
|
||||
None => break None,
|
||||
}
|
||||
}
|
||||
_ = tokio::time::sleep_until(last_data_at + stale_timeout) => {
|
||||
warn!(
|
||||
stale_secs = stale_timeout.as_secs(),
|
||||
"tunnel connection stale, no data received"
|
||||
);
|
||||
break None;
|
||||
}
|
||||
};
|
||||
|
||||
let msg = match msg_result {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
error!(error = %e, "WebSocket read error");
|
||||
break Some(e);
|
||||
}
|
||||
};
|
||||
|
||||
// Any successfully received message proves the connection is alive
|
||||
last_data_at = tokio::time::Instant::now();
|
||||
|
||||
let data = match msg {
|
||||
Message::Binary(data) => Bytes::from(data),
|
||||
Message::Ping(_) => continue,
|
||||
Message::Pong(_) => continue,
|
||||
Message::Close(_) => {
|
||||
info!("received WebSocket close");
|
||||
break None;
|
||||
}
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
let frame = match Frame::decode(data) {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
warn!(error = %e, "failed to decode frame");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
match frame.msg_type {
|
||||
MsgType::RequestHeaders => {
|
||||
// Decompress if the frame is gzip-compressed, then parse metadata
|
||||
let payload = match decompress_if_gzip(&frame) {
|
||||
Ok(p) => p,
|
||||
Err(e) => {
|
||||
warn!(stream_id = frame.stream_id, error = %e, "frame decompress failed");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let meta: RequestMeta = match serde_json::from_slice(&payload) {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
warn!(stream_id = frame.stream_id, error = %e, "invalid request metadata");
|
||||
// Use try_send to avoid blocking the read loop
|
||||
if frame_tx
|
||||
.try_send(Frame::new(
|
||||
frame.stream_id,
|
||||
MsgType::StreamError,
|
||||
0,
|
||||
Bytes::from(format!("invalid request metadata: {e}")),
|
||||
))
|
||||
.is_err()
|
||||
{
|
||||
warn!(
|
||||
stream_id = frame.stream_id,
|
||||
"writer channel full, StreamError dropped"
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if streams.len() >= max_streams {
|
||||
warn!(
|
||||
stream_id = frame.stream_id,
|
||||
"max concurrent streams reached"
|
||||
);
|
||||
if frame_tx
|
||||
.try_send(Frame::new(
|
||||
frame.stream_id,
|
||||
MsgType::StreamError,
|
||||
0,
|
||||
Bytes::from("max concurrent streams reached"),
|
||||
))
|
||||
.is_err()
|
||||
{
|
||||
warn!(
|
||||
stream_id = frame.stream_id,
|
||||
"writer channel full, StreamError dropped"
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Create body channel and spawn handler
|
||||
let (body_tx, body_rx) = mpsc::channel::<Frame>(64);
|
||||
streams.insert(frame.stream_id, body_tx);
|
||||
|
||||
let state_clone = Arc::clone(&state);
|
||||
let server_clone = Arc::clone(&server);
|
||||
let tx_clone = frame_tx.clone();
|
||||
let sid = frame.stream_id;
|
||||
let handle = tokio::spawn(async move {
|
||||
stream_handler::handle_stream(
|
||||
state_clone,
|
||||
server_clone,
|
||||
sid,
|
||||
meta,
|
||||
body_rx,
|
||||
tx_clone,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
handler_handles.push(handle);
|
||||
|
||||
debug!(stream_id = frame.stream_id, "new stream started");
|
||||
}
|
||||
|
||||
MsgType::RequestBody => {
|
||||
if let Some(tx) = streams.get(&frame.stream_id) {
|
||||
let is_end = frame.is_end_stream();
|
||||
let sid = frame.stream_id;
|
||||
let _ = tx.send(frame).await;
|
||||
if is_end {
|
||||
streams.remove(&sid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
MsgType::StreamEnd | MsgType::StreamError => {
|
||||
// Client-side cancellation or end
|
||||
if let Some(tx) = streams.remove(&frame.stream_id) {
|
||||
let _ = tx.send(frame).await;
|
||||
}
|
||||
}
|
||||
|
||||
MsgType::Ping => {
|
||||
// Use try_send to avoid blocking the read loop when writer is congested
|
||||
if frame_tx
|
||||
.try_send(Frame::control(MsgType::Pong, frame.payload))
|
||||
.is_err()
|
||||
{
|
||||
warn!("writer channel full, Pong dropped");
|
||||
}
|
||||
}
|
||||
|
||||
MsgType::HeartbeatAck => {
|
||||
heartbeat.on_ack(frame.payload).await;
|
||||
}
|
||||
|
||||
MsgType::GoAway => {
|
||||
info!("received GOAWAY");
|
||||
break None;
|
||||
}
|
||||
|
||||
_ => {
|
||||
debug!(msg_type = ?frame.msg_type, "ignoring unexpected frame type");
|
||||
}
|
||||
}
|
||||
|
||||
// Periodically clean up finished handles to avoid unbounded growth.
|
||||
// Trigger every 64 frames OR when the count exceeds max_streams.
|
||||
frames_since_cleanup += 1;
|
||||
if frames_since_cleanup >= 64 || handler_handles.len() > max_streams {
|
||||
handler_handles.retain(|h| !h.is_finished());
|
||||
frames_since_cleanup = 0;
|
||||
}
|
||||
};
|
||||
|
||||
// Drop body senders so stream handlers waiting on body_rx will unblock
|
||||
streams.clear();
|
||||
|
||||
// Wait for active stream handlers to finish so their frame_tx clones
|
||||
// are dropped before the writer closes the sink.
|
||||
drain_handlers(handler_handles).await;
|
||||
|
||||
match read_err {
|
||||
Some(e) => Err(e.into()),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait for all active stream handlers to finish (with a timeout).
|
||||
async fn drain_handlers(handles: Vec<JoinHandle<()>>) {
|
||||
if handles.is_empty() {
|
||||
return;
|
||||
}
|
||||
let count = handles.len();
|
||||
debug!(count, "waiting for active stream handlers to finish");
|
||||
let _ = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
for h in handles {
|
||||
let _ = h.await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
373
apps/aether-proxy/src/tunnel/heartbeat.rs
Normal file
373
apps/aether-proxy/src/tunnel/heartbeat.rs
Normal file
@@ -0,0 +1,373 @@
|
||||
//! Tunnel heartbeat: sends metrics over the tunnel, processes ACKs.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use std::time::SystemTime;
|
||||
use std::time::UNIX_EPOCH;
|
||||
|
||||
use bytes::Bytes;
|
||||
use tokio::sync::watch;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::registration::client::RemoteConfig;
|
||||
use crate::runtime;
|
||||
use crate::state::AppState;
|
||||
use crate::state::ServerContext;
|
||||
|
||||
use super::protocol::{Frame, MsgType};
|
||||
use super::writer::FrameSender;
|
||||
|
||||
const CURRENT_VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
static UPGRADE_IN_PROGRESS: AtomicBool = AtomicBool::new(false);
|
||||
static NON_ROOT_UPGRADE_WARNED: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
enum AckDecision {
|
||||
Accept {
|
||||
heartbeat_id: Option<u64>,
|
||||
upgrade_to: Option<String>,
|
||||
},
|
||||
Ignore,
|
||||
}
|
||||
|
||||
/// Handle for the dispatcher to forward HeartbeatAck frames.
|
||||
#[derive(Clone)]
|
||||
pub struct HeartbeatHandle {
|
||||
ack_tx: tokio::sync::mpsc::Sender<Bytes>,
|
||||
}
|
||||
|
||||
impl HeartbeatHandle {
|
||||
pub async fn on_ack(&self, payload: Bytes) {
|
||||
let _ = self.ack_tx.send(payload).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a no-op heartbeat handle that silently discards ACKs.
|
||||
/// Used for non-primary tunnel connections (conn_idx > 0) to avoid
|
||||
/// resetting shared atomic metrics via `swap(0)`.
|
||||
pub fn spawn_noop() -> HeartbeatHandle {
|
||||
let (ack_tx, _) = tokio::sync::mpsc::channel::<Bytes>(1);
|
||||
// receiver is immediately dropped; on_ack() calls will silently fail
|
||||
HeartbeatHandle { ack_tx }
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
struct HeartbeatSnapshot {
|
||||
requests: u64,
|
||||
latency_ns: u64,
|
||||
failed: u64,
|
||||
dns_failures: u64,
|
||||
stream_errors: u64,
|
||||
}
|
||||
|
||||
/// Spawn the heartbeat task. Returns a handle for forwarding ACKs.
|
||||
pub fn spawn(
|
||||
state: Arc<AppState>,
|
||||
server: Arc<ServerContext>,
|
||||
frame_tx: FrameSender,
|
||||
mut shutdown: watch::Receiver<bool>,
|
||||
) -> HeartbeatHandle {
|
||||
let (ack_tx, mut ack_rx) = tokio::sync::mpsc::channel::<Bytes>(4);
|
||||
|
||||
tokio::spawn(async move {
|
||||
// Read initial interval from dynamic config (may be updated by remote config).
|
||||
let initial_interval = Duration::from_secs(server.dynamic.load().heartbeat_interval);
|
||||
let mut current_interval = initial_interval;
|
||||
// At most one in-flight heartbeat snapshot is tracked at a time.
|
||||
// Snapshot is only cleared after receiving an ACK, which avoids losing
|
||||
// interval counters when ACK/frame delivery is temporarily unstable.
|
||||
let mut pending: Option<(u64, HeartbeatSnapshot)> = None;
|
||||
let mut next_heartbeat_id: u64 = 1;
|
||||
let heartbeat_session_id = format!(
|
||||
"{}-{}",
|
||||
std::process::id(),
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_nanos()
|
||||
);
|
||||
|
||||
// Skip first immediate tick by sleeping first.
|
||||
tokio::time::sleep(current_interval).await;
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(current_interval) => {
|
||||
let (heartbeat_id, snapshot) = if let Some((id, snap)) = pending {
|
||||
(id, snap)
|
||||
} else {
|
||||
let snap = collect_snapshot(&server);
|
||||
let id = next_heartbeat_id;
|
||||
next_heartbeat_id = next_heartbeat_id.wrapping_add(1);
|
||||
if next_heartbeat_id == 0 {
|
||||
next_heartbeat_id = 1;
|
||||
}
|
||||
pending = Some((id, snap));
|
||||
(id, snap)
|
||||
};
|
||||
|
||||
let payload = build_heartbeat_payload(
|
||||
&state,
|
||||
&server,
|
||||
&heartbeat_session_id,
|
||||
heartbeat_id,
|
||||
snapshot
|
||||
).await;
|
||||
let frame = Frame::control(MsgType::HeartbeatData, payload);
|
||||
if frame_tx.send(frame).await.is_err() {
|
||||
if let Some((_, snap)) = pending.take() {
|
||||
restore_snapshot(&server, snap);
|
||||
}
|
||||
break; // Writer closed
|
||||
}
|
||||
debug!("sent heartbeat data");
|
||||
|
||||
// Re-read interval from dynamic config (remote config may have
|
||||
// updated it since the last heartbeat).
|
||||
let new_interval = Duration::from_secs(
|
||||
server.dynamic.load().heartbeat_interval
|
||||
);
|
||||
if new_interval != current_interval {
|
||||
debug!(
|
||||
old_secs = current_interval.as_secs(),
|
||||
new_secs = new_interval.as_secs(),
|
||||
"heartbeat interval updated from dynamic config"
|
||||
);
|
||||
current_interval = new_interval;
|
||||
}
|
||||
}
|
||||
Some(ack_payload) = ack_rx.recv() => {
|
||||
match handle_ack(&server, &ack_payload) {
|
||||
AckDecision::Accept {
|
||||
heartbeat_id: ack_id,
|
||||
upgrade_to,
|
||||
} => {
|
||||
if let Some((pending_id, _)) = pending {
|
||||
match ack_id {
|
||||
Some(id) if id == pending_id => {
|
||||
pending = None;
|
||||
}
|
||||
None => {
|
||||
// Backward-compatible with servers that don't echo
|
||||
// heartbeat_id in ACK payload yet.
|
||||
pending = None;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
maybe_trigger_upgrade(upgrade_to);
|
||||
}
|
||||
AckDecision::Ignore => {}
|
||||
}
|
||||
}
|
||||
_ = shutdown.changed() => {
|
||||
debug!("heartbeat task shutting down");
|
||||
if let Some((_, snap)) = pending.take() {
|
||||
restore_snapshot(&server, snap);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
HeartbeatHandle { ack_tx }
|
||||
}
|
||||
|
||||
fn collect_snapshot(server: &ServerContext) -> HeartbeatSnapshot {
|
||||
HeartbeatSnapshot {
|
||||
requests: server.metrics.total_requests.swap(0, Ordering::AcqRel),
|
||||
latency_ns: server.metrics.total_latency_ns.swap(0, Ordering::AcqRel),
|
||||
failed: server.metrics.failed_requests.swap(0, Ordering::AcqRel),
|
||||
dns_failures: server.metrics.dns_failures.swap(0, Ordering::AcqRel),
|
||||
stream_errors: server.metrics.stream_errors.swap(0, Ordering::AcqRel),
|
||||
}
|
||||
}
|
||||
|
||||
fn restore_snapshot(server: &ServerContext, snap: HeartbeatSnapshot) {
|
||||
if snap.requests > 0 {
|
||||
server
|
||||
.metrics
|
||||
.total_requests
|
||||
.fetch_add(snap.requests, Ordering::Release);
|
||||
}
|
||||
if snap.latency_ns > 0 {
|
||||
server
|
||||
.metrics
|
||||
.total_latency_ns
|
||||
.fetch_add(snap.latency_ns, Ordering::Release);
|
||||
}
|
||||
if snap.failed > 0 {
|
||||
server
|
||||
.metrics
|
||||
.failed_requests
|
||||
.fetch_add(snap.failed, Ordering::Release);
|
||||
}
|
||||
if snap.dns_failures > 0 {
|
||||
server
|
||||
.metrics
|
||||
.dns_failures
|
||||
.fetch_add(snap.dns_failures, Ordering::Release);
|
||||
}
|
||||
if snap.stream_errors > 0 {
|
||||
server
|
||||
.metrics
|
||||
.stream_errors
|
||||
.fetch_add(snap.stream_errors, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
async fn build_heartbeat_payload(
|
||||
state: &AppState,
|
||||
server: &ServerContext,
|
||||
heartbeat_session_id: &str,
|
||||
heartbeat_id: u64,
|
||||
snapshot: HeartbeatSnapshot,
|
||||
) -> Bytes {
|
||||
let node_id = server.node_id.read().unwrap().clone();
|
||||
|
||||
let avg_latency_ms = if snapshot.requests > 0 {
|
||||
Some(snapshot.latency_ns as f64 / snapshot.requests as f64 / 1_000_000.0)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let local_admission = state.stream_concurrency_snapshot().map(|snapshot| {
|
||||
serde_json::json!({
|
||||
"limit": snapshot.limit,
|
||||
"in_flight": snapshot.in_flight,
|
||||
"available_permits": snapshot.available_permits,
|
||||
"high_watermark": snapshot.high_watermark,
|
||||
"rejected_total": snapshot.rejected,
|
||||
})
|
||||
});
|
||||
let distributed_admission = match state.distributed_stream_concurrency_snapshot().await {
|
||||
Ok(Some(snapshot)) => Some(serde_json::json!({
|
||||
"limit": snapshot.limit,
|
||||
"in_flight": snapshot.in_flight,
|
||||
"available_permits": snapshot.available_permits,
|
||||
"high_watermark": snapshot.high_watermark,
|
||||
"rejected_total": snapshot.rejected,
|
||||
})),
|
||||
Ok(None) => None,
|
||||
Err(err) => Some(serde_json::json!({
|
||||
"error": err.to_string(),
|
||||
})),
|
||||
};
|
||||
let admission = match (local_admission, distributed_admission) {
|
||||
(None, None) => None,
|
||||
(local, distributed) => Some(serde_json::json!({
|
||||
"local_streams": local,
|
||||
"distributed_streams": distributed,
|
||||
})),
|
||||
};
|
||||
|
||||
let payload = serde_json::json!({
|
||||
"node_id": node_id,
|
||||
"heartbeat_session_id": heartbeat_session_id,
|
||||
"heartbeat_id": heartbeat_id,
|
||||
"active_connections": server.active_connections.load(Ordering::Acquire),
|
||||
"total_requests": snapshot.requests,
|
||||
"avg_latency_ms": avg_latency_ms,
|
||||
"failed_requests": snapshot.failed,
|
||||
"dns_failures": snapshot.dns_failures,
|
||||
"stream_errors": snapshot.stream_errors,
|
||||
"proxy_metadata": {
|
||||
"version": CURRENT_VERSION,
|
||||
"admission": admission,
|
||||
},
|
||||
});
|
||||
|
||||
Bytes::from(serde_json::to_vec(&payload).unwrap_or_default())
|
||||
}
|
||||
|
||||
fn handle_ack(server: &ServerContext, payload: &[u8]) -> AckDecision {
|
||||
if payload.is_empty() {
|
||||
return AckDecision::Accept {
|
||||
heartbeat_id: None,
|
||||
upgrade_to: None,
|
||||
};
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct AckPayload {
|
||||
#[serde(default)]
|
||||
remote_config: Option<RemoteConfig>,
|
||||
#[serde(default)]
|
||||
config_version: u64,
|
||||
#[serde(default)]
|
||||
heartbeat_id: Option<u64>,
|
||||
#[serde(default)]
|
||||
upgrade_to: Option<String>,
|
||||
}
|
||||
|
||||
match serde_json::from_slice::<AckPayload>(payload) {
|
||||
Ok(ack) => {
|
||||
if let Some(ref rc) = ack.remote_config {
|
||||
runtime::apply_remote_config(&server.dynamic, rc, ack.config_version);
|
||||
}
|
||||
AckDecision::Accept {
|
||||
heartbeat_id: ack.heartbeat_id,
|
||||
upgrade_to: ack.upgrade_to.and_then(normalize_upgrade_target),
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(error = %e, "failed to parse heartbeat ACK");
|
||||
AckDecision::Ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_upgrade_target(raw: String) -> Option<String> {
|
||||
let trimmed = raw.trim();
|
||||
if trimmed.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let normalized = trimmed.strip_prefix("proxy-v").unwrap_or(trimmed);
|
||||
if normalized == CURRENT_VERSION {
|
||||
return None;
|
||||
}
|
||||
Some(normalized.to_string())
|
||||
}
|
||||
|
||||
fn maybe_trigger_upgrade(version: Option<String>) {
|
||||
let Some(target_version) = version else {
|
||||
return;
|
||||
};
|
||||
if !crate::setup::service::is_root() {
|
||||
if NON_ROOT_UPGRADE_WARNED
|
||||
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
|
||||
.is_ok()
|
||||
{
|
||||
warn!(
|
||||
target_version = %target_version,
|
||||
"remote upgrade skipped: root privileges are required"
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if UPGRADE_IN_PROGRESS
|
||||
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
|
||||
.is_err()
|
||||
{
|
||||
debug!(target_version = %target_version, "upgrade already in progress, ignoring");
|
||||
return;
|
||||
}
|
||||
|
||||
tokio::spawn(async move {
|
||||
info!(target_version = %target_version, "received remote upgrade instruction");
|
||||
match crate::setup::upgrade::perform_upgrade(&target_version).await {
|
||||
Ok(()) => {
|
||||
info!(target_version = %target_version, "remote upgrade finished");
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
target_version = %target_version,
|
||||
error = %e,
|
||||
"remote upgrade failed"
|
||||
);
|
||||
UPGRADE_IN_PROGRESS.store(false, Ordering::Release);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
496
apps/aether-proxy/src/tunnel/mod.rs
Normal file
496
apps/aether-proxy/src/tunnel/mod.rs
Normal file
@@ -0,0 +1,496 @@
|
||||
pub mod client;
|
||||
pub mod dispatcher;
|
||||
pub mod heartbeat;
|
||||
pub mod protocol;
|
||||
pub mod stream_handler;
|
||||
pub mod writer;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use tokio::sync::watch;
|
||||
use tracing::{error, info};
|
||||
|
||||
use crate::state::{AppState, ServerContext};
|
||||
|
||||
/// If a tunnel stays connected at least this long, treat the next disconnect
|
||||
/// as a non-failure and reset reconnect backoff.
|
||||
const STABLE_SESSION_RESET_AFTER: Duration = Duration::from_secs(30);
|
||||
/// Startup staggering step per secondary connection, used to avoid
|
||||
/// simultaneous bursts when a pool of tunnels starts together.
|
||||
const STARTUP_STAGGER_STEP_MS: u64 = 150;
|
||||
/// Upper bound for startup staggering.
|
||||
const MAX_STARTUP_STAGGER_MS: u64 = 1_500;
|
||||
/// Keep a tiny floor for repeated reconnects; first retry is still immediate.
|
||||
const MIN_RECONNECT_DELAY_MS: u64 = 50;
|
||||
/// Even under sustained failures, keep probing frequently so recovery is fast
|
||||
/// once cross-border network quality improves.
|
||||
const RECONNECT_PROBE_MAX_DELAY_MS: u64 = 3_000;
|
||||
|
||||
/// Run the tunnel mode main loop (connect, dispatch, reconnect).
|
||||
///
|
||||
/// `conn_idx` identifies which connection in the pool this is (0-based).
|
||||
/// Only connection 0 sends heartbeats to avoid resetting shared metrics.
|
||||
pub async fn run(
|
||||
state: &Arc<AppState>,
|
||||
server: &Arc<ServerContext>,
|
||||
conn_idx: usize,
|
||||
mut shutdown: watch::Receiver<bool>,
|
||||
) {
|
||||
info!(server = %server.server_label, conn = conn_idx, "starting tunnel");
|
||||
let reconnect_salt = compute_connection_salt(server, conn_idx);
|
||||
|
||||
let startup_delay = compute_startup_stagger(conn_idx, reconnect_salt);
|
||||
if !startup_delay.is_zero() {
|
||||
info!(
|
||||
server = %server.server_label,
|
||||
conn = conn_idx,
|
||||
delay_ms = startup_delay.as_millis(),
|
||||
"startup stagger before first connect"
|
||||
);
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(startup_delay) => {}
|
||||
_ = shutdown.changed() => {
|
||||
info!(server = %server.server_label, conn = conn_idx, "shutdown requested during startup stagger");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut consecutive_failures: u32 = 0;
|
||||
|
||||
loop {
|
||||
let started_at = Instant::now();
|
||||
match client::connect_and_run(state, server, conn_idx, &mut shutdown).await {
|
||||
Ok(client::TunnelOutcome::Shutdown) => {
|
||||
info!(server = %server.server_label, conn = conn_idx, "tunnel shut down gracefully");
|
||||
return;
|
||||
}
|
||||
Ok(client::TunnelOutcome::Disconnected) => {
|
||||
info!(server = %server.server_label, conn = conn_idx, "tunnel disconnected, reconnecting");
|
||||
}
|
||||
Err(e) => {
|
||||
error!(server = %server.server_label, conn = conn_idx, error = %e, "tunnel connection error, reconnecting");
|
||||
}
|
||||
}
|
||||
|
||||
if *shutdown.borrow() {
|
||||
info!(server = %server.server_label, conn = conn_idx, "shutdown requested, not reconnecting");
|
||||
return;
|
||||
}
|
||||
|
||||
// Reset backoff after a stable session to keep recovery snappy when
|
||||
// failures are only occasional.
|
||||
let connected_for = started_at.elapsed();
|
||||
if connected_for >= STABLE_SESSION_RESET_AFTER {
|
||||
consecutive_failures = 0;
|
||||
} else {
|
||||
consecutive_failures = consecutive_failures.saturating_add(1);
|
||||
}
|
||||
|
||||
let reconnect_delay = compute_reconnect_delay(
|
||||
state.config.tunnel_reconnect_base_ms,
|
||||
state.config.tunnel_reconnect_max_ms,
|
||||
consecutive_failures,
|
||||
reconnect_salt,
|
||||
);
|
||||
info!(
|
||||
server = %server.server_label,
|
||||
conn = conn_idx,
|
||||
failures = consecutive_failures,
|
||||
delay_ms = reconnect_delay.as_millis(),
|
||||
"waiting before reconnect"
|
||||
);
|
||||
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(reconnect_delay) => {}
|
||||
_ = shutdown.changed() => {
|
||||
info!(server = %server.server_label, conn = conn_idx, "shutdown requested during reconnect wait");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn compute_connection_salt(server: &ServerContext, conn_idx: usize) -> u64 {
|
||||
// FNV-1a style hash over server label + connection index.
|
||||
let mut h: u64 = 0xcbf29ce484222325;
|
||||
for &b in server.server_label.as_bytes() {
|
||||
h ^= b as u64;
|
||||
h = h.wrapping_mul(0x100000001b3);
|
||||
}
|
||||
h ^= conn_idx as u64;
|
||||
mix_u64(h)
|
||||
}
|
||||
|
||||
fn compute_startup_stagger(conn_idx: usize, salt: u64) -> Duration {
|
||||
if conn_idx == 0 {
|
||||
return Duration::ZERO;
|
||||
}
|
||||
let base = (conn_idx as u64).saturating_mul(STARTUP_STAGGER_STEP_MS);
|
||||
let jitter = mix_u64(salt) % 301; // 0..=300ms
|
||||
Duration::from_millis((base + jitter).min(MAX_STARTUP_STAGGER_MS))
|
||||
}
|
||||
|
||||
fn compute_reconnect_delay(
|
||||
base_ms: u64,
|
||||
max_ms: u64,
|
||||
consecutive_failures: u32,
|
||||
salt: u64,
|
||||
) -> Duration {
|
||||
// First retry should be immediate to maximize recovery speed on transient
|
||||
// blips (the user's primary expectation in poor networks).
|
||||
if consecutive_failures <= 1 {
|
||||
return Duration::ZERO;
|
||||
}
|
||||
|
||||
// Keep a sane minimum for repeated failures.
|
||||
let base_ms = base_ms.max(MIN_RECONNECT_DELAY_MS);
|
||||
let max_ms = max_ms.max(base_ms);
|
||||
let cap_ms = compute_reconnect_cap_ms(base_ms, max_ms, consecutive_failures)
|
||||
.min(RECONNECT_PROBE_MAX_DELAY_MS.max(base_ms));
|
||||
|
||||
// Equal-jitter: randomize in [cap/2, cap], preventing synchronized reconnect
|
||||
// storms while keeping reconnect latency bounded.
|
||||
if cap_ms <= 1 {
|
||||
return Duration::from_millis(cap_ms);
|
||||
}
|
||||
|
||||
let half = cap_ms / 2;
|
||||
let span = cap_ms - half;
|
||||
let now_nanos = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.subsec_nanos() as u64)
|
||||
.unwrap_or(0);
|
||||
let mixed = mix_u64(now_nanos ^ salt);
|
||||
let jitter = if span == 0 { 0 } else { mixed % (span + 1) };
|
||||
Duration::from_millis(half + jitter)
|
||||
}
|
||||
|
||||
fn compute_reconnect_cap_ms(base_ms: u64, max_ms: u64, consecutive_failures: u32) -> u64 {
|
||||
if consecutive_failures <= 1 {
|
||||
return base_ms.min(max_ms);
|
||||
}
|
||||
|
||||
let shift = (consecutive_failures - 1).min(31);
|
||||
let factor = 1u64 << shift;
|
||||
base_ms.saturating_mul(factor).min(max_ms)
|
||||
}
|
||||
|
||||
fn mix_u64(mut x: u64) -> u64 {
|
||||
// SplitMix64 finalizer - cheap bit mixing for pseudo-random jitter.
|
||||
x ^= x >> 30;
|
||||
x = x.wrapping_mul(0xbf58476d1ce4e5b9);
|
||||
x ^= x >> 27;
|
||||
x = x.wrapping_mul(0x94d049bb133111eb);
|
||||
x ^ (x >> 31)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use std::sync::{Arc, Once};
|
||||
use std::time::Duration;
|
||||
|
||||
use aether_gateway::{build_router_with_state, AppState as GatewayAppState};
|
||||
use arc_swap::ArcSwap;
|
||||
use axum::Router;
|
||||
use reqwest::StatusCode;
|
||||
use tokio::sync::watch;
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::registration::client::AetherClient;
|
||||
use crate::runtime::DynamicConfig;
|
||||
use crate::state::{AppState as ProxyAppState, ProxyMetrics, ServerContext};
|
||||
use crate::target_filter::DnsCache;
|
||||
use crate::tunnel::protocol;
|
||||
use crate::upstream_client;
|
||||
|
||||
use super::{
|
||||
compute_reconnect_cap_ms, compute_reconnect_delay, compute_startup_stagger, run,
|
||||
MAX_STARTUP_STAGGER_MS, RECONNECT_PROBE_MAX_DELAY_MS, STARTUP_STAGGER_STEP_MS,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn reconnect_cap_grows_exponentially_and_caps() {
|
||||
let base = 500;
|
||||
let max = 30_000;
|
||||
assert_eq!(compute_reconnect_cap_ms(base, max, 0), 500);
|
||||
assert_eq!(compute_reconnect_cap_ms(base, max, 1), 500);
|
||||
assert_eq!(compute_reconnect_cap_ms(base, max, 2), 1_000);
|
||||
assert_eq!(compute_reconnect_cap_ms(base, max, 3), 2_000);
|
||||
assert_eq!(compute_reconnect_cap_ms(base, max, 4), 4_000);
|
||||
assert_eq!(compute_reconnect_cap_ms(base, max, 5), 8_000);
|
||||
assert_eq!(compute_reconnect_cap_ms(base, max, 6), 16_000);
|
||||
assert_eq!(compute_reconnect_cap_ms(base, max, 7), 30_000);
|
||||
assert_eq!(compute_reconnect_cap_ms(base, max, 20), 30_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn startup_stagger_is_zero_for_primary_and_bounded_for_secondary() {
|
||||
assert_eq!(compute_startup_stagger(0, 42), Duration::ZERO);
|
||||
|
||||
let d1 = compute_startup_stagger(1, 42);
|
||||
let d2 = compute_startup_stagger(2, 42);
|
||||
|
||||
assert!(d1 >= Duration::from_millis(STARTUP_STAGGER_STEP_MS));
|
||||
assert!(d1 <= Duration::from_millis(MAX_STARTUP_STAGGER_MS));
|
||||
assert!(d2 >= Duration::from_millis(STARTUP_STAGGER_STEP_MS * 2));
|
||||
assert!(d2 <= Duration::from_millis(MAX_STARTUP_STAGGER_MS));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reconnect_delay_is_immediate_on_first_failure() {
|
||||
assert_eq!(compute_reconnect_delay(700, 45_000, 1, 123), Duration::ZERO);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reconnect_delay_stays_within_probe_ceiling_after_many_failures() {
|
||||
let d = compute_reconnect_delay(500, 45_000, 100, 12345);
|
||||
assert!(d <= Duration::from_millis(RECONNECT_PROBE_MAX_DELAY_MS));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn proxy_reconnects_after_gateway_restart() {
|
||||
ensure_rustls_provider();
|
||||
|
||||
let gateway_port = reserve_local_port().expect("gateway port should reserve");
|
||||
let gateway_base_url = format!("http://127.0.0.1:{gateway_port}");
|
||||
let (gateway_state, mut gateway_handle) = start_gateway_on_port(gateway_port)
|
||||
.await
|
||||
.expect("gateway should start");
|
||||
|
||||
let state = sample_state(sample_config(&gateway_base_url));
|
||||
let server = sample_server(&state, "node-recovery");
|
||||
let (shutdown_tx, shutdown_rx) = watch::channel(false);
|
||||
let proxy_task = tokio::spawn({
|
||||
let state = Arc::clone(&state);
|
||||
let server = Arc::clone(&server);
|
||||
async move {
|
||||
run(&state, &server, 0, shutdown_rx).await;
|
||||
}
|
||||
});
|
||||
|
||||
wait_until_relay_status(
|
||||
&gateway_base_url,
|
||||
"node-recovery",
|
||||
StatusCode::GATEWAY_TIMEOUT,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(gateway_state.force_close_all_tunnel_proxies(), 1);
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
gateway_handle.abort();
|
||||
|
||||
let (_restarted_gateway_state, restarted_gateway_handle) =
|
||||
start_gateway_on_port_retry(gateway_port)
|
||||
.await
|
||||
.expect("gateway should restart on fixed port");
|
||||
gateway_handle = restarted_gateway_handle;
|
||||
|
||||
wait_until_relay_status(
|
||||
&gateway_base_url,
|
||||
"node-recovery",
|
||||
StatusCode::GATEWAY_TIMEOUT,
|
||||
)
|
||||
.await;
|
||||
|
||||
let _ = shutdown_tx.send(true);
|
||||
tokio::time::timeout(Duration::from_secs(5), proxy_task)
|
||||
.await
|
||||
.expect("proxy task should stop")
|
||||
.expect("proxy task should join");
|
||||
gateway_handle.abort();
|
||||
}
|
||||
|
||||
async fn wait_until_relay_status(gateway_base_url: &str, node_id: &str, expected: StatusCode) {
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
||||
let mut last_observed = None::<String>;
|
||||
loop {
|
||||
if let Some((status, body)) = probe_relay_status(gateway_base_url, node_id).await {
|
||||
last_observed = Some(format!("{status} body={body}"));
|
||||
if status == expected {
|
||||
return;
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
tokio::time::Instant::now() < deadline,
|
||||
"relay status did not become {expected} within timeout; last={:?}",
|
||||
last_observed
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn probe_relay_status(
|
||||
gateway_base_url: &str,
|
||||
node_id: &str,
|
||||
) -> Option<(StatusCode, String)> {
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!(
|
||||
"{gateway_base_url}/api/internal/tunnel/relay/{node_id}"
|
||||
))
|
||||
.header("content-type", "application/octet-stream")
|
||||
.body(relay_probe_envelope())
|
||||
.send()
|
||||
.await
|
||||
.ok()?;
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
Some((status, body))
|
||||
}
|
||||
|
||||
fn relay_probe_envelope() -> Vec<u8> {
|
||||
let meta = protocol::RequestMeta {
|
||||
method: "GET".to_string(),
|
||||
url: "http://127.0.0.1:80/blocked".to_string(),
|
||||
headers: std::collections::HashMap::new(),
|
||||
timeout: 5,
|
||||
};
|
||||
let meta_json =
|
||||
serde_json::to_vec(&meta).expect("tunnel relay probe metadata should serialize");
|
||||
let mut envelope = Vec::with_capacity(4 + meta_json.len());
|
||||
envelope.extend_from_slice(&(meta_json.len() as u32).to_be_bytes());
|
||||
envelope.extend_from_slice(&meta_json);
|
||||
envelope
|
||||
}
|
||||
|
||||
async fn start_gateway_on_port(
|
||||
port: u16,
|
||||
) -> Result<(GatewayAppState, tokio::task::JoinHandle<()>), std::io::Error> {
|
||||
let state =
|
||||
GatewayAppState::new("http://127.0.0.1:9").expect("gateway test state should build");
|
||||
let router = build_router_with_state(state.clone());
|
||||
let handle = spawn_router_on_port(port, router).await?;
|
||||
Ok((state, handle))
|
||||
}
|
||||
|
||||
async fn start_gateway_on_port_retry(
|
||||
port: u16,
|
||||
) -> Result<(GatewayAppState, tokio::task::JoinHandle<()>), std::io::Error> {
|
||||
let mut attempts = 0usize;
|
||||
loop {
|
||||
match start_gateway_on_port(port).await {
|
||||
Ok(server) => return Ok(server),
|
||||
Err(err) => {
|
||||
attempts += 1;
|
||||
if attempts >= 20 {
|
||||
return Err(err);
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn spawn_router_on_port(
|
||||
port: u16,
|
||||
app: Router,
|
||||
) -> Result<tokio::task::JoinHandle<()>, std::io::Error> {
|
||||
let listener = tokio::net::TcpListener::bind(("127.0.0.1", port)).await?;
|
||||
Ok(tokio::spawn(async move {
|
||||
axum::serve(
|
||||
listener,
|
||||
app.into_make_service_with_connect_info::<std::net::SocketAddr>(),
|
||||
)
|
||||
.await
|
||||
.expect("gateway test server should run");
|
||||
}))
|
||||
}
|
||||
|
||||
fn reserve_local_port() -> Result<u16, std::io::Error> {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
|
||||
let port = listener.local_addr()?.port();
|
||||
drop(listener);
|
||||
Ok(port)
|
||||
}
|
||||
|
||||
fn sample_state(config: Config) -> Arc<ProxyAppState> {
|
||||
let config = Arc::new(config);
|
||||
let dns_cache = Arc::new(DnsCache::new(Duration::from_secs(60), 128));
|
||||
let upstream_client =
|
||||
upstream_client::build_upstream_client(&config, Arc::clone(&dns_cache));
|
||||
Arc::new(ProxyAppState {
|
||||
config,
|
||||
dns_cache,
|
||||
upstream_client,
|
||||
tunnel_tls_config: Arc::new(crate::tunnel::client::build_tls_config()),
|
||||
stream_gate: None,
|
||||
distributed_stream_gate: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn sample_server(state: &Arc<ProxyAppState>, node_id: &str) -> Arc<ServerContext> {
|
||||
let config = Arc::clone(&state.config);
|
||||
Arc::new(ServerContext {
|
||||
server_label: "gateway-owned-tunnel".to_string(),
|
||||
aether_url: config.aether_url.clone(),
|
||||
management_token: config.management_token.clone(),
|
||||
node_name: config.node_name.clone(),
|
||||
node_id: Arc::new(std::sync::RwLock::new(node_id.to_string())),
|
||||
aether_client: Arc::new(AetherClient::new(
|
||||
&config,
|
||||
&config.aether_url,
|
||||
&config.management_token,
|
||||
)),
|
||||
dynamic: Arc::new(ArcSwap::from_pointee(DynamicConfig::from_config(&config))),
|
||||
active_connections: Arc::new(AtomicU64::new(0)),
|
||||
metrics: Arc::new(ProxyMetrics::new()),
|
||||
})
|
||||
}
|
||||
|
||||
fn sample_config(aether_url: &str) -> Config {
|
||||
Config {
|
||||
aether_url: aether_url.to_string(),
|
||||
management_token: "token".to_string(),
|
||||
public_ip: None,
|
||||
node_name: "proxy-test".to_string(),
|
||||
node_region: None,
|
||||
heartbeat_interval: 1,
|
||||
allowed_ports: vec![80, 443],
|
||||
aether_request_timeout_secs: 10,
|
||||
aether_connect_timeout_secs: 2,
|
||||
aether_pool_max_idle_per_host: 8,
|
||||
aether_pool_idle_timeout_secs: 90,
|
||||
aether_tcp_keepalive_secs: 60,
|
||||
aether_tcp_nodelay: true,
|
||||
aether_http2: true,
|
||||
aether_retry_max_attempts: 1,
|
||||
aether_retry_base_delay_ms: 50,
|
||||
aether_retry_max_delay_ms: 100,
|
||||
max_concurrent_connections: None,
|
||||
max_in_flight_streams: None,
|
||||
distributed_stream_limit: None,
|
||||
distributed_stream_redis_url: None,
|
||||
distributed_stream_redis_key_prefix: None,
|
||||
distributed_stream_lease_ttl_ms: 30_000,
|
||||
distributed_stream_renew_interval_ms: 10_000,
|
||||
distributed_stream_command_timeout_ms: 1_000,
|
||||
dns_cache_ttl_secs: 60,
|
||||
dns_cache_capacity: 128,
|
||||
upstream_connect_timeout_secs: 30,
|
||||
upstream_pool_max_idle_per_host: 4,
|
||||
upstream_pool_idle_timeout_secs: 60,
|
||||
upstream_tcp_keepalive_secs: 60,
|
||||
upstream_tcp_nodelay: true,
|
||||
log_level: "info".to_string(),
|
||||
log_json: false,
|
||||
tunnel_reconnect_base_ms: 50,
|
||||
tunnel_reconnect_max_ms: 250,
|
||||
tunnel_ping_interval_secs: 1,
|
||||
tunnel_max_streams: Some(8),
|
||||
tunnel_connect_timeout_secs: 2,
|
||||
tunnel_tcp_keepalive_secs: 30,
|
||||
tunnel_tcp_nodelay: true,
|
||||
tunnel_stale_timeout_secs: 5,
|
||||
tunnel_connections: 1,
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_rustls_provider() {
|
||||
static INIT: Once = Once::new();
|
||||
INIT.call_once(|| {
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
});
|
||||
}
|
||||
}
|
||||
1
apps/aether-proxy/src/tunnel/protocol.rs
Normal file
1
apps/aether-proxy/src/tunnel/protocol.rs
Normal file
@@ -0,0 +1 @@
|
||||
pub use aether_contracts::tunnel::*;
|
||||
711
apps/aether-proxy/src/tunnel/stream_handler.rs
Normal file
711
apps/aether-proxy/src/tunnel/stream_handler.rs
Normal file
@@ -0,0 +1,711 @@
|
||||
//! Per-stream request handler.
|
||||
//!
|
||||
//! Receives request frames, executes the upstream HTTP request,
|
||||
//! and sends response frames back through the writer channel.
|
||||
|
||||
use std::io;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use aether_runtime::hold_admission_permit_until;
|
||||
use bytes::Bytes;
|
||||
use futures_util::stream;
|
||||
use futures_util::StreamExt;
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::body::Frame as BodyFrame;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::state::{AppState, ServerContext};
|
||||
use crate::target_filter;
|
||||
use crate::upstream_client;
|
||||
|
||||
use super::protocol::{
|
||||
compress_payload, decompress_if_gzip, flags, Frame as TunnelFrame, MsgType, RequestMeta,
|
||||
ResponseMeta,
|
||||
};
|
||||
use super::writer::FrameSender;
|
||||
|
||||
/// Maximum response body chunk size per frame (32 KB).
|
||||
const MAX_CHUNK_SIZE: usize = 32 * 1024;
|
||||
|
||||
/// Timeout for sending a single frame to the writer channel.
|
||||
/// If the writer is congested (TCP backpressure), we abandon the stream
|
||||
/// rather than blocking indefinitely and exhausting the stream pool.
|
||||
const FRAME_SEND_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Minimum allowed upstream request timeout (seconds).
|
||||
const MIN_TIMEOUT_SECS: u64 = 5;
|
||||
/// Maximum allowed upstream request timeout (seconds).
|
||||
const MAX_TIMEOUT_SECS: u64 = 300;
|
||||
|
||||
/// Headers that must not be forwarded to upstream (hop-by-hop or security-sensitive).
|
||||
///
|
||||
/// `host` and `content-length` are managed by the HTTP client (reqwest/hyper):
|
||||
/// - `host` → translated to `:authority` pseudo-header in HTTP/2; forwarding
|
||||
/// the original `host` alongside `:authority` triggers PROTOCOL_ERROR on
|
||||
/// strict H2 implementations (e.g. Google APIs).
|
||||
/// - `content-length` → recalculated by hyper from the actual body; a stale
|
||||
/// value from the tunnel (body may have been re-compressed) causes H2
|
||||
/// PROTOCOL_ERROR when it mismatches the real frame length.
|
||||
const BLOCKED_HEADERS: &[&str] = &[
|
||||
"connection",
|
||||
"content-length",
|
||||
"host",
|
||||
"keep-alive",
|
||||
"proxy-authenticate",
|
||||
"proxy-authorization",
|
||||
"proxy-connection",
|
||||
"te",
|
||||
"trailer",
|
||||
"transfer-encoding",
|
||||
"upgrade",
|
||||
];
|
||||
|
||||
/// Handle a single stream: receive body, execute upstream, send response.
|
||||
pub async fn handle_stream(
|
||||
state: Arc<AppState>,
|
||||
server: Arc<ServerContext>,
|
||||
stream_id: u32,
|
||||
meta: RequestMeta,
|
||||
body_rx: mpsc::Receiver<TunnelFrame>,
|
||||
frame_tx: FrameSender,
|
||||
) {
|
||||
let permit = match state.try_acquire_stream_permit().await {
|
||||
Ok(permit) => permit,
|
||||
Err(err) => {
|
||||
let message = match err {
|
||||
crate::state::ProxyAdmissionError::Saturated { .. } => "proxy overloaded",
|
||||
crate::state::ProxyAdmissionError::Unavailable { .. } => {
|
||||
"proxy admission unavailable"
|
||||
}
|
||||
};
|
||||
send_error(&frame_tx, stream_id, message).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
server.active_connections.fetch_add(1, Ordering::Release);
|
||||
|
||||
let connect_elapsed = hold_admission_permit_until(permit, async {
|
||||
handle_stream_inner(&state, &server, stream_id, meta, body_rx, &frame_tx).await
|
||||
})
|
||||
.await;
|
||||
|
||||
server.active_connections.fetch_sub(1, Ordering::Release);
|
||||
if let Some(d) = connect_elapsed {
|
||||
server.metrics.record_request(d);
|
||||
}
|
||||
}
|
||||
|
||||
/// Send a frame to the writer with a timeout. Returns false if send failed.
|
||||
async fn send_frame(tx: &FrameSender, frame: TunnelFrame) -> bool {
|
||||
match tokio::time::timeout(FRAME_SEND_TIMEOUT, tx.send(frame)).await {
|
||||
Ok(Ok(())) => true,
|
||||
Ok(Err(_)) => {
|
||||
// Channel closed (writer exited)
|
||||
false
|
||||
}
|
||||
Err(_) => {
|
||||
// Timeout — writer is congested
|
||||
warn!("frame send timeout (writer congested), abandoning stream");
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the connection-establishment duration (DNS + TCP/TLS + TTFB) if the
|
||||
/// upstream request succeeded, or `None` if the request never reached the
|
||||
/// response-headers stage.
|
||||
async fn handle_stream_inner(
|
||||
state: &AppState,
|
||||
server: &ServerContext,
|
||||
stream_id: u32,
|
||||
meta: RequestMeta,
|
||||
body_rx: mpsc::Receiver<TunnelFrame>,
|
||||
frame_tx: &FrameSender,
|
||||
) -> Option<Duration> {
|
||||
// Validate target
|
||||
let target_url = match url::Url::parse(&meta.url) {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
send_error(frame_tx, stream_id, &format!("invalid URL: {e}")).await;
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
// Only allow http/https schemes (block file://, data://, etc.)
|
||||
match target_url.scheme() {
|
||||
"http" | "https" => {}
|
||||
other => {
|
||||
send_error(
|
||||
frame_tx,
|
||||
stream_id,
|
||||
&format!("unsupported URL scheme: {other}"),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
||||
let host = match target_url.host_str() {
|
||||
Some(h) => h.to_string(),
|
||||
None => {
|
||||
send_error(frame_tx, stream_id, "missing host in URL").await;
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let port = target_url.port_or_known_default().unwrap_or(443);
|
||||
|
||||
// DNS + target validation (populates dns_cache for SafeDnsResolver)
|
||||
let connect_start = Instant::now();
|
||||
{
|
||||
let allowed_ports = Arc::clone(&server.dynamic.load().allowed_ports);
|
||||
if let Err(e) =
|
||||
target_filter::validate_target(&host, port, &allowed_ports, &state.dns_cache).await
|
||||
{
|
||||
server.metrics.dns_failures.fetch_add(1, Ordering::Release);
|
||||
send_error(frame_tx, stream_id, &format!("target blocked: {e}")).await;
|
||||
return None;
|
||||
}
|
||||
}
|
||||
let dns_ms = connect_start.elapsed().as_millis() as u64;
|
||||
|
||||
// Execute upstream request
|
||||
let client = &state.upstream_client;
|
||||
let timeout = Duration::from_secs(meta.timeout.clamp(MIN_TIMEOUT_SECS, MAX_TIMEOUT_SECS));
|
||||
let request_body_size = Arc::new(AtomicUsize::new(0));
|
||||
let request_body = build_streaming_request_body(body_rx, Arc::clone(&request_body_size));
|
||||
|
||||
let method: hyper::Method = meta.method.parse().unwrap_or(hyper::Method::GET);
|
||||
let mut request = match hyper::Request::builder()
|
||||
.method(method)
|
||||
.uri(meta.url.as_str())
|
||||
.body(request_body)
|
||||
{
|
||||
Ok(request) => request,
|
||||
Err(e) => {
|
||||
send_error(
|
||||
frame_tx,
|
||||
stream_id,
|
||||
&format!("invalid upstream request: {e}"),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
let headers = request.headers_mut();
|
||||
for (k, v) in &meta.headers {
|
||||
let k_lower = k.to_ascii_lowercase();
|
||||
if BLOCKED_HEADERS.contains(&k_lower.as_str()) {
|
||||
continue;
|
||||
}
|
||||
if let (Ok(name), Ok(value)) = (
|
||||
hyper::header::HeaderName::from_bytes(k.as_bytes()),
|
||||
hyper::header::HeaderValue::from_str(v),
|
||||
) {
|
||||
headers.insert(name, value);
|
||||
}
|
||||
}
|
||||
|
||||
let mut captured_connection = upstream_client::capture_connection(&mut request);
|
||||
let connection_start = Instant::now();
|
||||
let connection_capture = tokio::spawn(async move {
|
||||
let connected = captured_connection.wait_for_connection_metadata().await;
|
||||
connected
|
||||
.as_ref()
|
||||
.map(|_| connection_start.elapsed().as_millis() as u64)
|
||||
});
|
||||
|
||||
let upstream_start = Instant::now();
|
||||
let response = match tokio::time::timeout(timeout, client.request(request)).await {
|
||||
Ok(Ok(response)) => response,
|
||||
Ok(Err(e)) => {
|
||||
connection_capture.abort();
|
||||
server
|
||||
.metrics
|
||||
.failed_requests
|
||||
.fetch_add(1, Ordering::Release);
|
||||
let msg = if e.is_connect() {
|
||||
format!("upstream connect error: {e}")
|
||||
} else {
|
||||
format!("upstream error: {e}")
|
||||
};
|
||||
send_error(frame_tx, stream_id, &msg).await;
|
||||
return None;
|
||||
}
|
||||
Err(_) => {
|
||||
connection_capture.abort();
|
||||
server
|
||||
.metrics
|
||||
.failed_requests
|
||||
.fetch_add(1, Ordering::Release);
|
||||
send_error(frame_tx, stream_id, "upstream timeout").await;
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
// Capture connection-establishment duration (DNS + TCP/TLS + TTFB)
|
||||
// before proceeding to stream the response body.
|
||||
let connect_elapsed = connect_start.elapsed();
|
||||
|
||||
// Send RESPONSE_HEADERS
|
||||
let status = response.status().as_u16();
|
||||
let ttfb_ms = upstream_start.elapsed().as_millis() as u64;
|
||||
// Short timeout: on connection reuse hyper may never fire the connect
|
||||
// callback, so avoid blocking indefinitely.
|
||||
let connection_acquire_ms =
|
||||
match tokio::time::timeout(Duration::from_millis(100), connection_capture).await {
|
||||
Ok(Ok(ms)) => ms,
|
||||
Ok(Err(_)) => None, // JoinError (task panicked / cancelled)
|
||||
Err(_) => None, // timeout -- task is detached but lightweight
|
||||
};
|
||||
let request_timing =
|
||||
upstream_client::resolve_request_timing(&response, connection_acquire_ms, ttfb_ms);
|
||||
let mut resp_headers: Vec<(String, String)> = Vec::with_capacity(response.headers().len() + 1);
|
||||
for (k, v) in response.headers() {
|
||||
if let Ok(vs) = v.to_str() {
|
||||
resp_headers.push((k.as_str().to_string(), vs.to_string()));
|
||||
}
|
||||
}
|
||||
let timing = serde_json::json!({
|
||||
"dns_ms": dns_ms,
|
||||
"connection_acquire_ms": request_timing.connection_acquire_ms,
|
||||
"connection_reused": request_timing.connection_reused,
|
||||
"connect_ms": request_timing.connect_ms,
|
||||
"tls_ms": request_timing.tls_ms,
|
||||
"ttfb_ms": ttfb_ms,
|
||||
"upstream_ms": ttfb_ms,
|
||||
"response_wait_ms": request_timing.response_wait_ms,
|
||||
"upstream_processing_ms": request_timing.response_wait_ms,
|
||||
"timing_source": "instrumented_connector",
|
||||
"total_ms": connect_elapsed.as_millis() as u64,
|
||||
"body_size": request_body_size.load(Ordering::Relaxed),
|
||||
"mode": "tunnel",
|
||||
});
|
||||
resp_headers.push(("x-proxy-timing".to_string(), timing.to_string()));
|
||||
let resp_meta = ResponseMeta {
|
||||
status,
|
||||
headers: resp_headers,
|
||||
};
|
||||
let meta_json: Bytes = serde_json::to_vec(&resp_meta).unwrap_or_default().into();
|
||||
let (meta_payload, meta_flags) = compress_payload(meta_json);
|
||||
if !send_frame(
|
||||
frame_tx,
|
||||
TunnelFrame::new(
|
||||
stream_id,
|
||||
MsgType::ResponseHeaders,
|
||||
meta_flags,
|
||||
meta_payload,
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return Some(connect_elapsed);
|
||||
}
|
||||
|
||||
// Stream response body — relay upstream bytes through the tunnel.
|
||||
// Apply tunnel-level frame compression for chunks that benefit from it
|
||||
// (e.g. uncompressed SSE text). Already-compressed data (gzip/br from
|
||||
// upstream Content-Encoding) won't shrink further and will be sent as-is
|
||||
// thanks to the size check in compress_payload().
|
||||
let mut stream = response.into_body().into_data_stream();
|
||||
while let Some(chunk_result) = stream.next().await {
|
||||
match chunk_result {
|
||||
Ok(chunk) => {
|
||||
if chunk.len() <= MAX_CHUNK_SIZE {
|
||||
let (payload, extra_flags) = compress_payload(chunk);
|
||||
if !send_frame(
|
||||
frame_tx,
|
||||
TunnelFrame::new(stream_id, MsgType::ResponseBody, extra_flags, payload),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return Some(connect_elapsed);
|
||||
}
|
||||
} else {
|
||||
// Split oversized chunks, compress each slice
|
||||
let mut offset = 0;
|
||||
while offset < chunk.len() {
|
||||
let end = (offset + MAX_CHUNK_SIZE).min(chunk.len());
|
||||
let slice = chunk.slice(offset..end);
|
||||
let (payload, extra_flags) = compress_payload(slice);
|
||||
if !send_frame(
|
||||
frame_tx,
|
||||
TunnelFrame::new(
|
||||
stream_id,
|
||||
MsgType::ResponseBody,
|
||||
extra_flags,
|
||||
payload,
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return Some(connect_elapsed);
|
||||
}
|
||||
offset = end;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
server.metrics.stream_errors.fetch_add(1, Ordering::Release);
|
||||
warn!(stream_id, error = %e, "upstream body read error");
|
||||
send_error(frame_tx, stream_id, &format!("body read error: {e}")).await;
|
||||
return Some(connect_elapsed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Send STREAM_END
|
||||
let _ = send_frame(
|
||||
frame_tx,
|
||||
TunnelFrame::new(
|
||||
stream_id,
|
||||
MsgType::StreamEnd,
|
||||
flags::END_STREAM,
|
||||
Bytes::new(),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
|
||||
debug!(stream_id, status, "stream completed");
|
||||
Some(connect_elapsed)
|
||||
}
|
||||
|
||||
async fn send_error(tx: &FrameSender, stream_id: u32, msg: &str) {
|
||||
// Error frames use best-effort delivery — don't block if writer is congested
|
||||
let _ = send_frame(
|
||||
tx,
|
||||
TunnelFrame::new(
|
||||
stream_id,
|
||||
MsgType::StreamError,
|
||||
0,
|
||||
Bytes::from(msg.to_string()),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
fn build_streaming_request_body(
|
||||
body_rx: mpsc::Receiver<TunnelFrame>,
|
||||
body_size: Arc<AtomicUsize>,
|
||||
) -> upstream_client::UpstreamRequestBody {
|
||||
let body_stream = stream::unfold(
|
||||
(body_rx, body_size, false),
|
||||
|(mut body_rx, body_size, finished)| async move {
|
||||
if finished {
|
||||
return None;
|
||||
}
|
||||
|
||||
loop {
|
||||
let frame = match body_rx.recv().await {
|
||||
Some(frame) => frame,
|
||||
None => return None,
|
||||
};
|
||||
|
||||
match frame.msg_type {
|
||||
MsgType::RequestBody => {
|
||||
let end_stream = frame.is_end_stream();
|
||||
let payload = match decompress_if_gzip(&frame) {
|
||||
Ok(payload) => payload,
|
||||
Err(error) => {
|
||||
let err =
|
||||
io::Error::other(format!("gzip decompress failed: {error}"));
|
||||
return Some((Err(err), (body_rx, body_size, true)));
|
||||
}
|
||||
};
|
||||
|
||||
if payload.is_empty() {
|
||||
if end_stream {
|
||||
return None;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
body_size.fetch_add(payload.len(), Ordering::Relaxed);
|
||||
return Some((
|
||||
Ok(BodyFrame::data(payload)),
|
||||
(body_rx, body_size, end_stream),
|
||||
));
|
||||
}
|
||||
MsgType::StreamError => {
|
||||
let message = String::from_utf8(frame.payload.to_vec())
|
||||
.unwrap_or_else(|_| "client cancelled request body".to_string());
|
||||
return Some((Err(io::Error::other(message)), (body_rx, body_size, true)));
|
||||
}
|
||||
MsgType::StreamEnd => return None,
|
||||
_ => continue,
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
upstream_client::stream_request_body(body_stream)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use std::sync::Once;
|
||||
|
||||
use aether_runtime::{bounded_queue, ConcurrencyGate, DistributedConcurrencyGate};
|
||||
use arc_swap::ArcSwap;
|
||||
|
||||
use super::*;
|
||||
use crate::config::Config;
|
||||
use crate::registration::client::AetherClient;
|
||||
use crate::runtime::DynamicConfig;
|
||||
use crate::state::ProxyMetrics;
|
||||
use crate::target_filter::DnsCache;
|
||||
use crate::tunnel::client::build_tls_config;
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_request_body_yields_chunks_and_tracks_size() {
|
||||
let (tx, rx) = mpsc::channel(4);
|
||||
let body_size = Arc::new(AtomicUsize::new(0));
|
||||
let mut body = build_streaming_request_body(rx, Arc::clone(&body_size));
|
||||
|
||||
tx.send(TunnelFrame::new(
|
||||
1,
|
||||
MsgType::RequestBody,
|
||||
0,
|
||||
Bytes::from_static(b"abc"),
|
||||
))
|
||||
.await
|
||||
.expect("send first chunk");
|
||||
tx.send(TunnelFrame::new(
|
||||
1,
|
||||
MsgType::RequestBody,
|
||||
flags::END_STREAM,
|
||||
Bytes::from_static(b"def"),
|
||||
))
|
||||
.await
|
||||
.expect("send final chunk");
|
||||
drop(tx);
|
||||
|
||||
let first = body
|
||||
.frame()
|
||||
.await
|
||||
.expect("first frame")
|
||||
.expect("first frame ok")
|
||||
.into_data()
|
||||
.expect("first data frame");
|
||||
let second = body
|
||||
.frame()
|
||||
.await
|
||||
.expect("second frame")
|
||||
.expect("second frame ok")
|
||||
.into_data()
|
||||
.expect("second data frame");
|
||||
|
||||
assert_eq!(first, Bytes::from_static(b"abc"));
|
||||
assert_eq!(second, Bytes::from_static(b"def"));
|
||||
assert!(body.frame().await.is_none());
|
||||
assert_eq!(body_size.load(Ordering::Relaxed), 6);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_request_body_surfaces_client_cancel_as_error() {
|
||||
let (tx, rx) = mpsc::channel(4);
|
||||
let body_size = Arc::new(AtomicUsize::new(0));
|
||||
let mut body = build_streaming_request_body(rx, Arc::clone(&body_size));
|
||||
|
||||
tx.send(TunnelFrame::new(
|
||||
1,
|
||||
MsgType::StreamError,
|
||||
0,
|
||||
Bytes::from_static(b"client cancelled"),
|
||||
))
|
||||
.await
|
||||
.expect("send cancel frame");
|
||||
drop(tx);
|
||||
|
||||
let err = body
|
||||
.frame()
|
||||
.await
|
||||
.expect("error frame present")
|
||||
.expect_err("body should surface cancellation error");
|
||||
assert!(err.to_string().contains("client cancelled"));
|
||||
assert!(body.frame().await.is_none());
|
||||
assert_eq!(body_size.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_stream_when_local_admission_gate_is_saturated() {
|
||||
let gate = Arc::new(ConcurrencyGate::new("proxy_streams", 1));
|
||||
let _permit = gate.try_acquire().expect("first permit");
|
||||
let state = sample_state(Some(gate), None);
|
||||
let server = sample_server(&state);
|
||||
let (frame_tx, mut frame_rx) = bounded_queue::<TunnelFrame>(4);
|
||||
let (_body_tx, body_rx) = mpsc::channel(1);
|
||||
|
||||
handle_stream(
|
||||
Arc::clone(&state),
|
||||
server,
|
||||
7,
|
||||
sample_request_meta(),
|
||||
body_rx,
|
||||
frame_tx,
|
||||
)
|
||||
.await;
|
||||
|
||||
let frame = frame_rx.recv().await.expect("overload frame");
|
||||
assert_eq!(frame.stream_id, 7);
|
||||
assert_eq!(frame.msg_type, MsgType::StreamError);
|
||||
assert_eq!(frame.payload, Bytes::from_static(b"proxy overloaded"));
|
||||
assert_eq!(
|
||||
state
|
||||
.stream_gate
|
||||
.as_ref()
|
||||
.expect("stream gate")
|
||||
.snapshot()
|
||||
.rejected,
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_stream_when_distributed_admission_gate_is_saturated() {
|
||||
let gate = Arc::new(DistributedConcurrencyGate::new_in_memory(
|
||||
"proxy_streams_distributed",
|
||||
1,
|
||||
));
|
||||
let _permit = gate.try_acquire().await.expect("first permit");
|
||||
let state = sample_state(None, Some(gate));
|
||||
let server = sample_server(&state);
|
||||
let (frame_tx, mut frame_rx) = bounded_queue::<TunnelFrame>(4);
|
||||
let (_body_tx, body_rx) = mpsc::channel(1);
|
||||
|
||||
handle_stream(
|
||||
Arc::clone(&state),
|
||||
server,
|
||||
9,
|
||||
sample_request_meta(),
|
||||
body_rx,
|
||||
frame_tx,
|
||||
)
|
||||
.await;
|
||||
|
||||
let frame = frame_rx.recv().await.expect("overload frame");
|
||||
assert_eq!(frame.stream_id, 9);
|
||||
assert_eq!(frame.msg_type, MsgType::StreamError);
|
||||
assert_eq!(frame.payload, Bytes::from_static(b"proxy overloaded"));
|
||||
assert_eq!(
|
||||
state
|
||||
.distributed_stream_gate
|
||||
.as_ref()
|
||||
.expect("distributed gate")
|
||||
.snapshot()
|
||||
.await
|
||||
.expect("distributed snapshot")
|
||||
.rejected,
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
fn sample_request_meta() -> RequestMeta {
|
||||
RequestMeta {
|
||||
method: "GET".to_string(),
|
||||
url: "https://example.com/ok".to_string(),
|
||||
headers: HashMap::new(),
|
||||
timeout: 30,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_state(
|
||||
stream_gate: Option<Arc<ConcurrencyGate>>,
|
||||
distributed_stream_gate: Option<Arc<DistributedConcurrencyGate>>,
|
||||
) -> Arc<AppState> {
|
||||
ensure_rustls_provider();
|
||||
let config = Arc::new(sample_config());
|
||||
let dns_cache = Arc::new(DnsCache::new(Duration::from_secs(60), 128));
|
||||
let upstream_client =
|
||||
upstream_client::build_upstream_client(&config, Arc::clone(&dns_cache));
|
||||
Arc::new(AppState {
|
||||
config,
|
||||
dns_cache,
|
||||
upstream_client,
|
||||
tunnel_tls_config: Arc::new(build_tls_config()),
|
||||
stream_gate,
|
||||
distributed_stream_gate,
|
||||
})
|
||||
}
|
||||
|
||||
fn sample_server(state: &Arc<AppState>) -> Arc<ServerContext> {
|
||||
let config = Arc::clone(&state.config);
|
||||
Arc::new(ServerContext {
|
||||
server_label: "server".to_string(),
|
||||
aether_url: config.aether_url.clone(),
|
||||
management_token: config.management_token.clone(),
|
||||
node_name: config.node_name.clone(),
|
||||
node_id: Arc::new(std::sync::RwLock::new("node-1".to_string())),
|
||||
aether_client: Arc::new(AetherClient::new(
|
||||
&config,
|
||||
&config.aether_url,
|
||||
&config.management_token,
|
||||
)),
|
||||
dynamic: Arc::new(ArcSwap::from_pointee(DynamicConfig::from_config(&config))),
|
||||
active_connections: Arc::new(AtomicU64::new(0)),
|
||||
metrics: Arc::new(ProxyMetrics::new()),
|
||||
})
|
||||
}
|
||||
|
||||
fn sample_config() -> Config {
|
||||
Config {
|
||||
aether_url: "https://aether.example.com".to_string(),
|
||||
management_token: "token".to_string(),
|
||||
public_ip: None,
|
||||
node_name: "proxy-test".to_string(),
|
||||
node_region: None,
|
||||
heartbeat_interval: 30,
|
||||
allowed_ports: vec![80, 443],
|
||||
aether_request_timeout_secs: 10,
|
||||
aether_connect_timeout_secs: 10,
|
||||
aether_pool_max_idle_per_host: 8,
|
||||
aether_pool_idle_timeout_secs: 90,
|
||||
aether_tcp_keepalive_secs: 60,
|
||||
aether_tcp_nodelay: true,
|
||||
aether_http2: true,
|
||||
aether_retry_max_attempts: 3,
|
||||
aether_retry_base_delay_ms: 200,
|
||||
aether_retry_max_delay_ms: 2_000,
|
||||
max_concurrent_connections: None,
|
||||
max_in_flight_streams: None,
|
||||
distributed_stream_limit: None,
|
||||
distributed_stream_redis_url: None,
|
||||
distributed_stream_redis_key_prefix: None,
|
||||
distributed_stream_lease_ttl_ms: 30_000,
|
||||
distributed_stream_renew_interval_ms: 10_000,
|
||||
distributed_stream_command_timeout_ms: 1_000,
|
||||
dns_cache_ttl_secs: 60,
|
||||
dns_cache_capacity: 128,
|
||||
upstream_connect_timeout_secs: 30,
|
||||
upstream_pool_max_idle_per_host: 4,
|
||||
upstream_pool_idle_timeout_secs: 60,
|
||||
upstream_tcp_keepalive_secs: 60,
|
||||
upstream_tcp_nodelay: true,
|
||||
log_level: "info".to_string(),
|
||||
log_json: false,
|
||||
tunnel_reconnect_base_ms: 500,
|
||||
tunnel_reconnect_max_ms: 30_000,
|
||||
tunnel_ping_interval_secs: 15,
|
||||
tunnel_max_streams: Some(8),
|
||||
tunnel_connect_timeout_secs: 15,
|
||||
tunnel_tcp_keepalive_secs: 30,
|
||||
tunnel_tcp_nodelay: true,
|
||||
tunnel_stale_timeout_secs: 45,
|
||||
tunnel_connections: 1,
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_rustls_provider() {
|
||||
static INIT: Once = Once::new();
|
||||
INIT.call_once(|| {
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
});
|
||||
}
|
||||
}
|
||||
63
apps/aether-proxy/src/tunnel/writer.rs
Normal file
63
apps/aether-proxy/src/tunnel/writer.rs
Normal file
@@ -0,0 +1,63 @@
|
||||
//! Dedicated WebSocket writer task.
|
||||
//!
|
||||
//! All frame writes go through an mpsc channel to a single writer task,
|
||||
//! avoiding contention on the WebSocket sink. The writer also sends
|
||||
//! periodic WebSocket Ping frames to keep the connection alive through
|
||||
//! intermediary proxies (Nginx, Cloudflare, etc.).
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use aether_runtime::{bounded_queue, BoundedQueueSender};
|
||||
use futures_util::SinkExt;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
use tracing::{debug, error, trace};
|
||||
|
||||
use super::protocol::Frame;
|
||||
|
||||
/// Sender half — cloned by stream handlers and heartbeat.
|
||||
pub type FrameSender = BoundedQueueSender<Frame>;
|
||||
|
||||
/// Spawn the writer task. Returns the sender and a JoinHandle for cleanup.
|
||||
///
|
||||
/// `ping_interval` controls WebSocket-level Ping frequency (typically 15s).
|
||||
/// This keeps the connection alive through intermediary proxies/load-balancers.
|
||||
pub fn spawn_writer<S>(mut sink: S, ping_interval: Duration) -> (FrameSender, JoinHandle<()>)
|
||||
where
|
||||
S: SinkExt<Message, Error = tokio_tungstenite::tungstenite::Error> + Unpin + Send + 'static,
|
||||
{
|
||||
let (tx, mut rx) = bounded_queue::<Frame>(256);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let mut ping_ticker = tokio::time::interval(ping_interval);
|
||||
ping_ticker.tick().await; // skip first immediate tick
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
frame = rx.recv() => {
|
||||
match frame {
|
||||
Some(frame) => {
|
||||
let data = frame.encode();
|
||||
if let Err(e) = sink.send(Message::Binary(data.into())).await {
|
||||
error!(error = %e, "failed to write frame to WebSocket");
|
||||
break;
|
||||
}
|
||||
}
|
||||
None => break, // all senders dropped
|
||||
}
|
||||
}
|
||||
_ = ping_ticker.tick() => {
|
||||
if let Err(e) = sink.send(Message::Ping(vec![])).await {
|
||||
error!(error = %e, "failed to send WebSocket ping");
|
||||
break;
|
||||
}
|
||||
trace!("sent WebSocket ping");
|
||||
}
|
||||
}
|
||||
}
|
||||
debug!("writer task exiting");
|
||||
let _ = sink.close().await;
|
||||
});
|
||||
|
||||
(tx, handle)
|
||||
}
|
||||
444
apps/aether-proxy/src/upstream_client.rs
Normal file
444
apps/aether-proxy/src/upstream_client.rs
Normal file
@@ -0,0 +1,444 @@
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
use std::net::IpAddr;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures_util::Stream;
|
||||
use http_body_util::combinators::UnsyncBoxBody;
|
||||
use http_body_util::{BodyExt, StreamBody};
|
||||
use hyper::body::Frame;
|
||||
use hyper::rt;
|
||||
use hyper::Response;
|
||||
use hyper::Uri;
|
||||
pub use hyper_util::client::legacy::connect::capture_connection;
|
||||
use hyper_util::client::legacy::connect::dns::Name;
|
||||
use hyper_util::client::legacy::connect::{Connected, Connection, HttpConnector};
|
||||
use hyper_util::client::legacy::Client;
|
||||
use hyper_util::rt::{TokioExecutor, TokioIo, TokioTimer};
|
||||
use rustls::pki_types::ServerName;
|
||||
use rustls::ClientConfig;
|
||||
use tokio::net::TcpStream;
|
||||
use tokio_rustls::TlsConnector;
|
||||
use tower_service::Service;
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::target_filter::{self, DnsCache};
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
type PlainStream = TokioIo<TcpStream>;
|
||||
type TlsStream = TokioIo<tokio_rustls::client::TlsStream<TcpStream>>;
|
||||
|
||||
pub type UpstreamRequestBody = UnsyncBoxBody<Bytes, io::Error>;
|
||||
pub type UpstreamClient = Client<InstrumentedConnector, UpstreamRequestBody>;
|
||||
|
||||
pub fn stream_request_body<S>(stream: S) -> UpstreamRequestBody
|
||||
where
|
||||
S: Stream<Item = Result<Frame<Bytes>, io::Error>> + Send + 'static,
|
||||
{
|
||||
StreamBody::new(stream).boxed_unsync()
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct ConnectTiming {
|
||||
pub connect_ms: u64,
|
||||
pub tls_ms: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct RequestTiming {
|
||||
pub connection_acquire_ms: u64,
|
||||
pub connect_ms: u64,
|
||||
pub tls_ms: u64,
|
||||
pub response_wait_ms: u64,
|
||||
pub connection_reused: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ValidatedResolver {
|
||||
dns_cache: Arc<DnsCache>,
|
||||
}
|
||||
|
||||
impl ValidatedResolver {
|
||||
pub fn new(dns_cache: Arc<DnsCache>) -> Self {
|
||||
Self { dns_cache }
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ValidatedAddrs {
|
||||
inner: std::vec::IntoIter<std::net::SocketAddr>,
|
||||
}
|
||||
|
||||
impl Iterator for ValidatedAddrs {
|
||||
type Item = std::net::SocketAddr;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
self.inner.next()
|
||||
}
|
||||
}
|
||||
|
||||
impl Service<Name> for ValidatedResolver {
|
||||
type Response = ValidatedAddrs;
|
||||
type Error = io::Error;
|
||||
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn call(&mut self, name: Name) -> Self::Future {
|
||||
let dns_cache = Arc::clone(&self.dns_cache);
|
||||
let host = name.as_str().to_string();
|
||||
Box::pin(async move {
|
||||
if let Some(addrs) = dns_cache.get_by_host(&host).await {
|
||||
return Ok(ValidatedAddrs {
|
||||
inner: (*addrs).clone().into_iter(),
|
||||
});
|
||||
}
|
||||
|
||||
let resolved = target_filter::resolve_public_addrs(&host, 0, dns_cache.as_ref())
|
||||
.await
|
||||
.map_err(|err| io::Error::other(err.to_string()))?;
|
||||
Ok(ValidatedAddrs {
|
||||
inner: resolved.into_iter(),
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct InstrumentedConnector {
|
||||
http: HttpConnector<ValidatedResolver>,
|
||||
tls_config: Arc<ClientConfig>,
|
||||
}
|
||||
|
||||
impl Service<Uri> for InstrumentedConnector {
|
||||
type Response = TimedConn;
|
||||
type Error = BoxError;
|
||||
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
self.http.poll_ready(cx).map_err(Into::into)
|
||||
}
|
||||
|
||||
fn call(&mut self, dst: Uri) -> Self::Future {
|
||||
let scheme = dst.scheme_str().map(|value| value.to_ascii_lowercase());
|
||||
let tls_config = Arc::clone(&self.tls_config);
|
||||
let connecting = self.http.call(dst.clone());
|
||||
let connect_start = std::time::Instant::now();
|
||||
|
||||
Box::pin(async move {
|
||||
match scheme.as_deref() {
|
||||
Some("http") => {
|
||||
let tcp = connecting.await.map_err(|err| Box::new(err) as BoxError)?;
|
||||
let connect_ms = connect_start.elapsed().as_millis() as u64;
|
||||
Ok(TimedConn::new(
|
||||
MaybeHttpsStream::Http(tcp),
|
||||
ConnectTiming {
|
||||
connect_ms,
|
||||
tls_ms: 0,
|
||||
},
|
||||
))
|
||||
}
|
||||
Some("https") => {
|
||||
let server_name = resolve_server_name(&dst)?;
|
||||
let tcp = connecting.await.map_err(|err| Box::new(err) as BoxError)?;
|
||||
let connect_ms = connect_start.elapsed().as_millis() as u64;
|
||||
|
||||
let tls_start = std::time::Instant::now();
|
||||
let tls_stream = TlsConnector::from(tls_config)
|
||||
.connect(server_name, tcp.into_inner())
|
||||
.await
|
||||
.map_err(io::Error::other)?;
|
||||
let tls_ms = tls_start.elapsed().as_millis() as u64;
|
||||
|
||||
Ok(TimedConn::new(
|
||||
MaybeHttpsStream::Https(TokioIo::new(tls_stream)),
|
||||
ConnectTiming { connect_ms, tls_ms },
|
||||
))
|
||||
}
|
||||
Some(other) => Err(io::Error::other(format!("unsupported scheme {other}")).into()),
|
||||
None => Err(io::Error::other("missing scheme").into()),
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_upstream_client(config: &Config, dns_cache: Arc<DnsCache>) -> UpstreamClient {
|
||||
let mut http = HttpConnector::new_with_resolver(ValidatedResolver::new(dns_cache));
|
||||
http.enforce_http(false);
|
||||
http.set_connect_timeout(Some(Duration::from_secs(
|
||||
config.upstream_connect_timeout_secs,
|
||||
)));
|
||||
http.set_nodelay(config.upstream_tcp_nodelay);
|
||||
if config.upstream_tcp_keepalive_secs > 0 {
|
||||
http.set_keepalive(Some(Duration::from_secs(
|
||||
config.upstream_tcp_keepalive_secs,
|
||||
)));
|
||||
} else {
|
||||
http.set_keepalive(None);
|
||||
}
|
||||
|
||||
let connector = InstrumentedConnector {
|
||||
http,
|
||||
tls_config: build_tls_config(),
|
||||
};
|
||||
|
||||
let mut builder = Client::builder(TokioExecutor::new());
|
||||
builder.pool_max_idle_per_host(config.upstream_pool_max_idle_per_host);
|
||||
builder.pool_idle_timeout(Duration::from_secs(config.upstream_pool_idle_timeout_secs));
|
||||
builder.pool_timer(TokioTimer::new());
|
||||
builder.build(connector)
|
||||
}
|
||||
|
||||
pub fn resolve_request_timing<B>(
|
||||
response: &Response<B>,
|
||||
connection_acquire_ms: Option<u64>,
|
||||
ttfb_ms: u64,
|
||||
) -> RequestTiming {
|
||||
let raw = response
|
||||
.extensions()
|
||||
.get::<ConnectTiming>()
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
|
||||
let raw_connection_ms = raw.connect_ms.saturating_add(raw.tls_ms);
|
||||
let measured_acquire_ms = connection_acquire_ms.unwrap_or(raw_connection_ms.min(ttfb_ms));
|
||||
let likely_reused = measured_acquire_ms <= 5 && raw_connection_ms > 0;
|
||||
let connector_matches_request = raw_connection_ms <= measured_acquire_ms.saturating_add(25);
|
||||
|
||||
let (connect_ms, tls_ms) = if likely_reused || !connector_matches_request {
|
||||
(0, 0)
|
||||
} else {
|
||||
(raw.connect_ms, raw.tls_ms)
|
||||
};
|
||||
|
||||
RequestTiming {
|
||||
connection_acquire_ms: measured_acquire_ms,
|
||||
connect_ms,
|
||||
tls_ms,
|
||||
response_wait_ms: ttfb_ms.saturating_sub(measured_acquire_ms),
|
||||
connection_reused: likely_reused,
|
||||
}
|
||||
}
|
||||
|
||||
fn build_tls_config() -> Arc<ClientConfig> {
|
||||
let root_store =
|
||||
rustls::RootCertStore::from_iter(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
|
||||
let mut config = ClientConfig::builder()
|
||||
.with_root_certificates(root_store)
|
||||
.with_no_client_auth();
|
||||
config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()];
|
||||
Arc::new(config)
|
||||
}
|
||||
|
||||
fn resolve_server_name(uri: &Uri) -> Result<ServerName<'static>, BoxError> {
|
||||
let host = uri.host().ok_or_else(|| io::Error::other("missing host"))?;
|
||||
let host = host.trim_start_matches('[').trim_end_matches(']');
|
||||
|
||||
if let Ok(ip) = host.parse::<IpAddr>() {
|
||||
return Ok(ServerName::from(ip));
|
||||
}
|
||||
|
||||
Ok(ServerName::try_from(host.to_string())?)
|
||||
}
|
||||
|
||||
pub struct TimedConn {
|
||||
inner: MaybeHttpsStream,
|
||||
timing: ConnectTiming,
|
||||
}
|
||||
|
||||
impl TimedConn {
|
||||
fn new(inner: MaybeHttpsStream, timing: ConnectTiming) -> Self {
|
||||
Self { inner, timing }
|
||||
}
|
||||
}
|
||||
|
||||
impl Connection for TimedConn {
|
||||
fn connected(&self) -> Connected {
|
||||
self.inner.connected().extra(self.timing)
|
||||
}
|
||||
}
|
||||
|
||||
impl rt::Read for TimedConn {
|
||||
fn poll_read(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: rt::ReadBufCursor<'_>,
|
||||
) -> Poll<Result<(), io::Error>> {
|
||||
Pin::new(&mut self.inner).poll_read(cx, buf)
|
||||
}
|
||||
}
|
||||
|
||||
impl rt::Write for TimedConn {
|
||||
fn poll_write(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &[u8],
|
||||
) -> Poll<Result<usize, io::Error>> {
|
||||
Pin::new(&mut self.inner).poll_write(cx, buf)
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
|
||||
Pin::new(&mut self.inner).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
) -> Poll<Result<(), io::Error>> {
|
||||
Pin::new(&mut self.inner).poll_shutdown(cx)
|
||||
}
|
||||
|
||||
fn is_write_vectored(&self) -> bool {
|
||||
self.inner.is_write_vectored()
|
||||
}
|
||||
|
||||
fn poll_write_vectored(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
bufs: &[std::io::IoSlice<'_>],
|
||||
) -> Poll<Result<usize, io::Error>> {
|
||||
Pin::new(&mut self.inner).poll_write_vectored(cx, bufs)
|
||||
}
|
||||
}
|
||||
|
||||
pub enum MaybeHttpsStream {
|
||||
Http(PlainStream),
|
||||
Https(TlsStream),
|
||||
}
|
||||
|
||||
impl Connection for MaybeHttpsStream {
|
||||
fn connected(&self) -> Connected {
|
||||
match self {
|
||||
Self::Http(stream) => stream.connected(),
|
||||
Self::Https(stream) => {
|
||||
let (tcp, tls) = stream.inner().get_ref();
|
||||
if tls.alpn_protocol() == Some(b"h2") {
|
||||
tcp.connected().negotiated_h2()
|
||||
} else {
|
||||
tcp.connected()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl rt::Read for MaybeHttpsStream {
|
||||
fn poll_read(
|
||||
self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: rt::ReadBufCursor<'_>,
|
||||
) -> Poll<Result<(), io::Error>> {
|
||||
match Pin::get_mut(self) {
|
||||
Self::Http(stream) => Pin::new(stream).poll_read(cx, buf),
|
||||
Self::Https(stream) => Pin::new(stream).poll_read(cx, buf),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl rt::Write for MaybeHttpsStream {
|
||||
fn poll_write(
|
||||
self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buf: &[u8],
|
||||
) -> Poll<Result<usize, io::Error>> {
|
||||
match Pin::get_mut(self) {
|
||||
Self::Http(stream) => Pin::new(stream).poll_write(cx, buf),
|
||||
Self::Https(stream) => Pin::new(stream).poll_write(cx, buf),
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
|
||||
match Pin::get_mut(self) {
|
||||
Self::Http(stream) => Pin::new(stream).poll_flush(cx),
|
||||
Self::Https(stream) => Pin::new(stream).poll_flush(cx),
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
|
||||
match Pin::get_mut(self) {
|
||||
Self::Http(stream) => Pin::new(stream).poll_shutdown(cx),
|
||||
Self::Https(stream) => Pin::new(stream).poll_shutdown(cx),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_write_vectored(&self) -> bool {
|
||||
match self {
|
||||
Self::Http(stream) => stream.is_write_vectored(),
|
||||
Self::Https(stream) => stream.is_write_vectored(),
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_write_vectored(
|
||||
self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
bufs: &[std::io::IoSlice<'_>],
|
||||
) -> Poll<Result<usize, io::Error>> {
|
||||
match Pin::get_mut(self) {
|
||||
Self::Http(stream) => Pin::new(stream).poll_write_vectored(cx, bufs),
|
||||
Self::Https(stream) => Pin::new(stream).poll_write_vectored(cx, bufs),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use hyper::Response;
|
||||
|
||||
#[test]
|
||||
fn fresh_connection_uses_connector_breakdown() {
|
||||
let mut response = Response::new(());
|
||||
response.extensions_mut().insert(ConnectTiming {
|
||||
connect_ms: 80,
|
||||
tls_ms: 40,
|
||||
});
|
||||
|
||||
let timing = resolve_request_timing(&response, Some(125), 600);
|
||||
|
||||
assert_eq!(timing.connection_acquire_ms, 125);
|
||||
assert_eq!(timing.connect_ms, 80);
|
||||
assert_eq!(timing.tls_ms, 40);
|
||||
assert_eq!(timing.response_wait_ms, 475);
|
||||
assert!(!timing.connection_reused);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reused_connection_zeroes_stale_connect_timings() {
|
||||
let mut response = Response::new(());
|
||||
response.extensions_mut().insert(ConnectTiming {
|
||||
connect_ms: 70,
|
||||
tls_ms: 30,
|
||||
});
|
||||
|
||||
let timing = resolve_request_timing(&response, Some(0), 310);
|
||||
|
||||
assert_eq!(timing.connection_acquire_ms, 0);
|
||||
assert_eq!(timing.connect_ms, 0);
|
||||
assert_eq!(timing.tls_ms, 0);
|
||||
assert_eq!(timing.response_wait_ms, 310);
|
||||
assert!(timing.connection_reused);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn falls_back_to_connector_timings_when_capture_missing() {
|
||||
let mut response = Response::new(());
|
||||
response.extensions_mut().insert(ConnectTiming {
|
||||
connect_ms: 55,
|
||||
tls_ms: 25,
|
||||
});
|
||||
|
||||
let timing = resolve_request_timing(&response, None, 400);
|
||||
|
||||
assert_eq!(timing.connection_acquire_ms, 80);
|
||||
assert_eq!(timing.connect_ms, 55);
|
||||
assert_eq!(timing.tls_ms, 25);
|
||||
assert_eq!(timing.response_wait_ms, 320);
|
||||
assert!(!timing.connection_reused);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user