i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 01:10:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(proxy): record tunnel stability metrics

Browse Source
This commit is contained in:
fawney19
2026-05-08 22:03:17 +08:00
parent 9a84a6ff6c
commit a703acd1fe
81 changed files with 7229 additions and 1110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
apps/aether-proxy/README.md
View File

@@ -110,21 +110,24 @@ sudo aether-proxy uninstall
| `--upstream-pool-idle-timeout-secs` | `AETHER_PROXY_UPSTREAM_POOL_IDLE_TIMEOUT_SECS` | `300` | 连接池空闲超时（秒） |
| `--upstream-tcp-keepalive-secs` | `AETHER_PROXY_UPSTREAM_TCP_KEEPALIVE_SECS` | `60` | TCP keepalive（秒，0 关闭） |
| `--upstream-tcp-nodelay` | `AETHER_PROXY_UPSTREAM_TCP_NODELAY` | `true` | 启用 TCP_NODELAY |
| `--upstream-proxy-url` | `AETHER_PROXY_UPSTREAM_PROXY_URL` | 空 | 仅 provider 上游请求使用的出口代理，支持 `http://`、`socks5://`、`socks5h://` |
| `--upstream-proxy-url` | `AETHER_PROXY_UPSTREAM_PROXY_URL` | 空 | 仅 provider 上游请求使用的出口代理 |
| `--redirect-replay-budget-bytes` | `AETHER_PROXY_REDIRECT_REPLAY_BUDGET_BYTES` | `5M` | 307/308 请求体重放的预读预算，支持 `K/M/G`，`0` 表示禁用 body replay buffering |
`upstream_proxy_url` 只影响 `aether-proxy` 访问 OpenAI、Claude、Gemini 等 provider 的上游请求，不影响节点回连 Aether 服务器的 WebSocket tunnel。配合 WARP sidecar 时可填写：
出口代理支持 `http://`、`socks5://`、`socks5h://`。配合 WARP sidecar 时可填写：
```toml
upstream_proxy_url = "socks5h://microwarp:1080"
```
如果需要让 Aether 管理 API 和 WebSocket tunnel 也走代理，使用 `aether_proxy_url`。
#### Aether API 客户端
| 参数 | 环境变量 | 默认值 | 说明 |
|------|----------|--------|------|
| `--aether-request-timeout-secs` | `AETHER_PROXY_AETHER_REQUEST_TIMEOUT_SECS` | `10` | 请求总超时（秒） |
| `--aether-connect-timeout-secs` | `AETHER_PROXY_AETHER_CONNECT_TIMEOUT_SECS` | `10` | 建连超时（秒） |
| `--aether-proxy-url` | `AETHER_PROXY_AETHER_PROXY_URL` | 空 | Aether 注册、心跳和 WebSocket tunnel 回连使用的出口代理（默认不走代理） |
| `--aether-retry-max-attempts` | `AETHER_PROXY_AETHER_RETRY_MAX_ATTEMPTS` | `3` | 最大重试次数 |
#### DNS 与安全
@@ -154,6 +157,15 @@ upstream_proxy_url = "socks5h://microwarp:1080"
- 以 `systemd` 或 `OpenRC` 安装时默认会额外打开文件日志到 `/var/log/aether-proxy`
- OpenRC 安装时，`aether-proxy logs` 实际读取 `/var/log/aether-proxy/current.log` 和 `/var/log/aether-proxy/error.log`；这些文件通常需要用 `sudo aether-proxy logs` 查看
### 隧道健康上报（Heartbeat）
proxy 会在心跳 `proxy_metadata` 中主动上报隧道稳定性指标，便于后端直接入库/告警：
- `proxy_metadata.tunnel_metrics`：建连尝试/成功/失败、断开次数、累计在线时长、心跳 RTT、WebSocket 收发帧与字节等。
- `proxy_metadata.recent_tunnel_errors`：最近隧道异常事件（时间戳、类别、错误摘要，环形缓冲）。
说明：仅主连接（`conn=0`）发送 heartbeat，避免多条 tunnel 重复上报同一份全局指标。
### 多服务器配置
在 `aether-proxy.toml` 中使用 `[[servers]]` 配置 Aether 服务器。即使只有一个服务器，也必须写成一个 `[[servers]]` 条目；旧的顶层单服务器写法已不再支持。

12
apps/aether-proxy/src/app.rs
View File

@@ -17,7 +17,7 @@ use crate::config::{Config, ServerEntry, TunnelPoolSizing};
use crate::net;
use crate::registration::client::AetherClient;
use crate::runtime::{self, DynamicConfig};
use crate::state::{AppState, ProxyMetrics, ServerContext};
use crate::state::{AppState, ProxyMetrics, ServerContext, TunnelMetrics};
use crate::upstream_client;
use crate::{hardware, target_filter, tunnel};
@@ -80,6 +80,14 @@ pub async fn run(mut config: Config, servers: Vec<ServerEntry>) -> anyhow::Resul
server_count = servers.len(),
"aether-proxy starting (tunnel mode)"
);
if let Some(proxy_url) = config.effective_aether_proxy_url() {
if let Ok(proxy) = crate::egress_proxy::UpstreamProxyConfig::parse(proxy_url) {
info!(
aether_proxy_url = %proxy.redacted_url(),
"Aether control and tunnel egress proxy configured"
);
}
}
if let Some(proxy_url) = config
.upstream_proxy_url
.as_deref()
@@ -457,6 +465,7 @@ fn build_server_context(
dynamic: Arc::new(ArcSwap::from_pointee(dynamic)),
active_connections: Arc::new(AtomicU64::new(0)),
metrics: Arc::new(ProxyMetrics::new()),
tunnel_metrics: Arc::new(TunnelMetrics::new()),
})
}
@@ -960,6 +969,7 @@ mod tests {
aether_tcp_keepalive_secs: 60,
aether_tcp_nodelay: true,
aether_http2: true,
aether_proxy_url: None,
aether_retry_max_attempts: 1,
aether_retry_base_delay_ms: 50,
aether_retry_max_delay_ms: 100,

165
apps/aether-proxy/src/config.rs
View File

@@ -339,6 +339,11 @@ pub struct Config {
#[arg(long, env = "AETHER_PROXY_AETHER_HTTP2", default_value_t = true)]
pub aether_http2: bool,
/// Optional egress proxy used for Aether API registration and WebSocket tunnel reconnects.
/// Supported schemes: http, socks5, socks5h.
#[arg(long, env = "AETHER_PROXY_AETHER_PROXY_URL")]
pub aether_proxy_url: Option<String>,
/// Aether API retry attempts (including initial)
#[arg(
long,
@@ -680,12 +685,11 @@ impl Config {
if self.upstream_connect_timeout_secs == 0 {
anyhow::bail!("upstream_connect_timeout_secs must be > 0");
}
if let Some(proxy_url) = self
.upstream_proxy_url
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
{
if let Some(proxy_url) = normalized_proxy_url(&self.aether_proxy_url) {
crate::egress_proxy::UpstreamProxyConfig::parse(proxy_url)
.map_err(|err| anyhow::anyhow!("aether_proxy_url invalid: {err}"))?;
}
if let Some(proxy_url) = normalized_proxy_url(&self.upstream_proxy_url) {
crate::egress_proxy::UpstreamProxyConfig::parse(proxy_url)
.map_err(|err| anyhow::anyhow!("upstream_proxy_url invalid: {err}"))?;
}
@@ -740,6 +744,10 @@ impl Config {
Ok(Duration::from_millis(self.tunnel_stale_timeout_ms))
}
pub fn effective_aether_proxy_url(&self) -> Option<&str> {
normalized_proxy_url(&self.aether_proxy_url)
}
pub fn resolve_tunnel_pool_sizing(
&self,
hw_info: &HardwareInfo,
@@ -850,6 +858,8 @@ pub struct ConfigFile {
#[serde(skip_serializing_if = "Option::is_none")]
pub aether_http2: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub aether_proxy_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub aether_retry_max_attempts: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub aether_retry_base_delay_ms: Option<u64>,
@@ -929,8 +939,7 @@ impl ConfigFile {
/// Load from a TOML file.
pub fn load(path: &Path) -> anyhow::Result<Self> {
let content = std::fs::read_to_string(path)?;
reject_removed_config_keys(&content)?;
Ok(toml::from_str(&content)?)
parse_config_file_content(&content)
}
/// Save to a TOML file.
@@ -1006,6 +1015,7 @@ impl ConfigFile {
);
set!("AETHER_PROXY_AETHER_TCP_NODELAY", self.aether_tcp_nodelay);
set!("AETHER_PROXY_AETHER_HTTP2", self.aether_http2);
set!("AETHER_PROXY_AETHER_PROXY_URL", self.aether_proxy_url);
set!(
"AETHER_PROXY_AETHER_RETRY_MAX_ATTEMPTS",
self.aether_retry_max_attempts
@@ -1124,6 +1134,59 @@ impl ConfigFile {
}
}
fn parse_config_file_content(content: &str) -> anyhow::Result<ConfigFile> {
reject_removed_config_keys(content)?;
let mut value: toml::Value = toml::from_str(content)?;
promote_server_scoped_upstream_proxy_url(&mut value)?;
Ok(value.try_into()?)
}
fn normalized_proxy_url(value: &Option<String>) -> Option<&str> {
value
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
}
fn promote_server_scoped_upstream_proxy_url(value: &mut toml::Value) -> anyhow::Result<()> {
const KEY: &str = "upstream_proxy_url";
let Some(root) = value.as_table_mut() else {
return Ok(());
};
let mut promoted = root.get(KEY).cloned();
let Some(servers) = root.get_mut("servers").and_then(toml::Value::as_array_mut) else {
return Ok(());
};
for (index, server) in servers.iter_mut().enumerate() {
let Some(table) = server.as_table_mut() else {
continue;
};
let Some(server_value) = table.remove(KEY) else {
continue;
};
match promoted.as_ref() {
Some(existing) if existing != &server_value => {
anyhow::bail!(
"conflicting upstream_proxy_url values: top-level value and [[servers]] entry {} differ; configure it once at the top level",
index + 1
);
}
Some(_) => {}
None => promoted = Some(server_value),
}
}
if let Some(promoted) = promoted {
root.insert(KEY.to_string(), promoted);
}
Ok(())
}
fn reject_removed_config_keys(content: &str) -> anyhow::Result<()> {
let value: toml::Value = toml::from_str(content)?;
let Some(table) = value.as_table() else {
@@ -1233,6 +1296,92 @@ mod tests {
);
}
#[test]
fn config_file_deserializes_aether_proxy_url() {
let cfg: ConfigFile = toml::from_str("aether_proxy_url = \"socks5h://127.0.0.1:1080\"")
.expect("proxy URL toml");
assert_eq!(
cfg.aether_proxy_url.as_deref(),
Some("socks5h://127.0.0.1:1080")
);
}
#[test]
fn aether_proxy_url_requires_explicit_opt_in() {
let default_direct = Config::parse_from([
"aether-proxy",
"--aether-url",
"https://example.com",
"--management-token",
"ae_test",
"--node-name",
"proxy-test",
"--upstream-proxy-url",
"socks5h://127.0.0.1:1080",
]);
assert_eq!(default_direct.effective_aether_proxy_url(), None);
let explicit = Config::parse_from([
"aether-proxy",
"--aether-url",
"https://example.com",
"--management-token",
"ae_test",
"--node-name",
"proxy-test",
"--upstream-proxy-url",
"socks5h://127.0.0.1:1080",
"--aether-proxy-url",
"http://127.0.0.1:8080",
]);
assert_eq!(
explicit.effective_aether_proxy_url(),
Some("http://127.0.0.1:8080")
);
}
#[test]
fn config_file_load_accepts_server_scoped_upstream_proxy_url() {
let cfg = parse_config_file_content(
r#"
[[servers]]
aether_url = "https://aether.example.com"
upstream_proxy_url = "socks5://127.0.0.1:1080"
management_token = "ae_test"
node_name = "proxy-test"
"#,
)
.expect("server-scoped proxy URL should be promoted");
assert_eq!(
cfg.upstream_proxy_url.as_deref(),
Some("socks5://127.0.0.1:1080")
);
assert_eq!(cfg.servers.len(), 1);
assert_eq!(cfg.servers[0].aether_url, "https://aether.example.com");
}
#[test]
fn config_file_load_rejects_conflicting_server_scoped_upstream_proxy_url() {
let error = parse_config_file_content(
r#"
upstream_proxy_url = "socks5://127.0.0.1:1080"
[[servers]]
aether_url = "https://aether.example.com"
upstream_proxy_url = "socks5://127.0.0.1:1081"
management_token = "ae_test"
node_name = "proxy-test"
"#,
)
.expect_err("conflicting proxy URLs should be rejected");
assert!(
error.to_string().contains("conflicting upstream_proxy_url"),
"error should mention the conflicting key"
);
}
#[test]
fn config_file_rejects_removed_tunnel_seconds_keys() {
let error = reject_removed_config_keys("tunnel_ping_interval_secs = 5")

304
apps/aether-proxy/src/egress_proxy.rs
View File

@@ -1,4 +1,11 @@
use std::io;
use std::net::{IpAddr, SocketAddr};
use std::time::Duration;
use base64::Engine;
use socket2::{SockRef, TcpKeepalive};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpStream;
use url::Url;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -114,6 +121,303 @@ impl UpstreamProxyConfig {
}
}
#[derive(Debug, Clone, Copy)]
pub(crate) struct ProxyConnectOptions {
pub connect_timeout: Duration,
pub tcp_nodelay: bool,
pub tcp_keepalive: Option<Duration>,
}
pub(crate) async fn connect_target_via_proxy(
proxy: &UpstreamProxyConfig,
target_host: &str,
target_port: u16,
options: ProxyConnectOptions,
) -> io::Result<TcpStream> {
let mut tcp = connect_proxy_tcp(
proxy,
options.connect_timeout,
options.tcp_nodelay,
options.tcp_keepalive,
)
.await?;
match proxy.scheme() {
UpstreamProxyScheme::Http => {
http_connect(&mut tcp, &target_authority(target_host, target_port), proxy).await?;
}
UpstreamProxyScheme::Socks5 | UpstreamProxyScheme::Socks5h => {
socks5_connect(&mut tcp, proxy, target_host, target_port).await?;
}
}
Ok(tcp)
}
pub(crate) async fn connect_proxy_tcp(
proxy: &UpstreamProxyConfig,
connect_timeout: Duration,
tcp_nodelay: bool,
tcp_keepalive: Option<Duration>,
) -> io::Result<TcpStream> {
let resolved = tokio::time::timeout(
connect_timeout,
tokio::net::lookup_host((proxy.host(), proxy.port())),
)
.await
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "proxy DNS timeout"))?
.map_err(|err| io::Error::other(format!("proxy DNS failed: {err}")))?;
let mut last_error = None;
for addr in resolved {
match tokio::time::timeout(connect_timeout, TcpStream::connect(addr)).await {
Ok(Ok(stream)) => {
configure_tcp_stream(&stream, tcp_nodelay, tcp_keepalive)?;
return Ok(stream);
}
Ok(Err(error)) => last_error = Some(error),
Err(_) => {
last_error = Some(io::Error::new(
io::ErrorKind::TimedOut,
format!("proxy connect timeout: {addr}"),
));
}
}
}
Err(last_error.unwrap_or_else(|| io::Error::other("proxy DNS returned no addresses")))
}
fn configure_tcp_stream(
stream: &TcpStream,
tcp_nodelay: bool,
tcp_keepalive: Option<Duration>,
) -> io::Result<()> {
stream.set_nodelay(tcp_nodelay)?;
if let Some(keepalive) = tcp_keepalive {
let keepalive = TcpKeepalive::new().with_time(keepalive);
SockRef::from(stream).set_tcp_keepalive(&keepalive)?;
}
Ok(())
}
pub(crate) async fn http_connect(
stream: &mut TcpStream,
target_authority: &str,
proxy: &UpstreamProxyConfig,
) -> io::Result<()> {
let mut request = format!(
"CONNECT {target_authority} HTTP/1.1\r\nHost: {target_authority}\r\nProxy-Connection: Keep-Alive\r\n"
);
if let Some(auth) = proxy.basic_auth_header() {
request.push_str("Proxy-Authorization: ");
request.push_str(&auth);
request.push_str("\r\n");
}
request.push_str("\r\n");
stream.write_all(request.as_bytes()).await?;
stream.flush().await?;
let mut response = Vec::with_capacity(1024);
let mut chunk = [0u8; 1024];
loop {
if response.len() >= 16 * 1024 {
return Err(io::Error::other("proxy CONNECT response too large"));
}
let n = stream.read(&mut chunk).await?;
if n == 0 {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"proxy closed during CONNECT",
));
}
response.extend_from_slice(&chunk[..n]);
if response.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
let status_line_end = response
.windows(2)
.position(|window| window == b"\r\n")
.ok_or_else(|| io::Error::other("proxy CONNECT response missing status line"))?;
let status_line = std::str::from_utf8(&response[..status_line_end])
.map_err(|_| io::Error::other("proxy CONNECT status line is not UTF-8"))?;
let status = status_line.split_whitespace().nth(1).unwrap_or_default();
if status == "200" {
Ok(())
} else {
Err(io::Error::other(format!(
"proxy CONNECT failed: {status_line}"
)))
}
}
pub(crate) async fn socks5_connect(
stream: &mut TcpStream,
proxy: &UpstreamProxyConfig,
target_host: &str,
target_port: u16,
) -> io::Result<()> {
let requires_auth = proxy.username().is_some();
if requires_auth {
stream.write_all(&[0x05, 0x02, 0x00, 0x02]).await?;
} else {
stream.write_all(&[0x05, 0x01, 0x00]).await?;
}
let mut method_response = [0u8; 2];
stream.read_exact(&mut method_response).await?;
if method_response[0] != 0x05 {
return Err(io::Error::other("invalid SOCKS5 method response"));
}
match method_response[1] {
0x00 => {}
0x02 => socks5_authenticate(stream, proxy).await?,
0xff => return Err(io::Error::other("SOCKS5 proxy rejected all auth methods")),
method => {
return Err(io::Error::other(format!(
"SOCKS5 proxy selected unsupported auth method 0x{method:02x}"
)))
}
}
let address = socks5_target_address(target_host, target_port, proxy.uses_remote_dns()).await?;
stream.write_all(&address).await?;
let mut response = [0u8; 4];
stream.read_exact(&mut response).await?;
if response[0] != 0x05 {
return Err(io::Error::other("invalid SOCKS5 connect response"));
}
if response[1] != 0x00 {
return Err(io::Error::other(format!(
"SOCKS5 connect failed: {}",
socks5_reply_message(response[1])
)));
}
match response[3] {
0x01 => {
let mut ignored = [0u8; 4 + 2];
stream.read_exact(&mut ignored).await?;
}
0x03 => {
let mut len = [0u8; 1];
stream.read_exact(&mut len).await?;
let mut ignored = vec![0u8; len[0] as usize + 2];
stream.read_exact(&mut ignored).await?;
}
0x04 => {
let mut ignored = [0u8; 16 + 2];
stream.read_exact(&mut ignored).await?;
}
atyp => {
return Err(io::Error::other(format!(
"SOCKS5 proxy returned unsupported address type 0x{atyp:02x}"
)))
}
}
Ok(())
}
async fn socks5_authenticate(
stream: &mut TcpStream,
proxy: &UpstreamProxyConfig,
) -> io::Result<()> {
let username = proxy.username().unwrap_or_default().as_bytes();
let password = proxy.password().unwrap_or_default().as_bytes();
if username.len() > u8::MAX as usize || password.len() > u8::MAX as usize {
return Err(io::Error::other(
"SOCKS5 username/password must be at most 255 bytes",
));
}
let mut request = Vec::with_capacity(username.len() + password.len() + 3);
request.push(0x01);
request.push(username.len() as u8);
request.extend_from_slice(username);
request.push(password.len() as u8);
request.extend_from_slice(password);
stream.write_all(&request).await?;
let mut response = [0u8; 2];
stream.read_exact(&mut response).await?;
if response[0] != 0x01 || response[1] != 0x00 {
return Err(io::Error::other("SOCKS5 username/password auth failed"));
}
Ok(())
}
pub(crate) async fn socks5_target_address(
target_host: &str,
target_port: u16,
remote_dns: bool,
) -> io::Result<Vec<u8>> {
let mut request = vec![0x05, 0x01, 0x00];
if let Ok(ip) = target_host.parse::<IpAddr>() {
push_socks5_ip_address(&mut request, ip);
} else if remote_dns {
let host = target_host.as_bytes();
if host.len() > u8::MAX as usize {
return Err(io::Error::other("SOCKS5 target hostname is too long"));
}
request.push(0x03);
request.push(host.len() as u8);
request.extend_from_slice(host);
} else {
let mut resolved = tokio::net::lookup_host((target_host, target_port))
.await
.map_err(|err| io::Error::other(format!("SOCKS5 target DNS failed: {err}")))?;
let addr = resolved
.next()
.ok_or_else(|| io::Error::other("SOCKS5 target DNS returned no addresses"))?;
push_socks5_socket_address(&mut request, addr);
}
request.extend_from_slice(&target_port.to_be_bytes());
Ok(request)
}
fn push_socks5_socket_address(request: &mut Vec<u8>, addr: SocketAddr) {
push_socks5_ip_address(request, addr.ip());
}
fn push_socks5_ip_address(request: &mut Vec<u8>, ip: IpAddr) {
match ip {
IpAddr::V4(ip) => {
request.push(0x01);
request.extend_from_slice(&ip.octets());
}
IpAddr::V6(ip) => {
request.push(0x04);
request.extend_from_slice(&ip.octets());
}
}
}
fn socks5_reply_message(reply: u8) -> &'static str {
match reply {
0x01 => "general failure",
0x02 => "connection not allowed",
0x03 => "network unreachable",
0x04 => "host unreachable",
0x05 => "connection refused",
0x06 => "TTL expired",
0x07 => "command not supported",
0x08 => "address type not supported",
_ => "unknown error",
}
}
pub(crate) fn target_authority(host: &str, port: u16) -> String {
if host.contains(':') && !host.starts_with('[') {
format!("[{host}]:{port}")
} else {
format!("{host}:{port}")
}
}
fn non_empty_url_part(value: &str) -> Option<String> {
let value = value.trim();
if value.is_empty() {

1
apps/aether-proxy/src/registration/client.rs
View File

@@ -67,6 +67,7 @@ impl AetherClient {
tcp_nodelay: config.aether_tcp_nodelay,
http2_adaptive_window: config.aether_http2,
user_agent: Some(format!("aether-proxy/{}", env!("CARGO_PKG_VERSION"))),
proxy_url: config.effective_aether_proxy_url().map(str::to_string),
..HttpClientConfig::default()
})
.expect("failed to create HTTP client");

80
apps/aether-proxy/src/setup/tui.rs
View File

@@ -93,15 +93,6 @@ impl ServerTab {
required: true,
help: "Node name for identification in Aether dashboard",
},
Field {
label: "Upstream Proxy",
key: "upstream_proxy_url",
value: String::new(),
kind: FieldKind::Text,
required: false,
help:
"Optional provider egress proxy, e.g. http://127.0.0.1:8080 or socks5h://127.0.0.1:1080",
},
],
}
}
@@ -115,12 +106,6 @@ impl ServerTab {
}
tab
}
fn set_field_value(&mut self, key: &str, value: &str) {
if let Some(field) = self.fields.iter_mut().find(|field| field.key == key) {
field.value = value.to_string();
}
}
}
// -- App state ----------------------------------------------------------------
@@ -153,6 +138,15 @@ impl App {
server_tabs: vec![ServerTab::new()],
active_tab: 0,
global_fields: vec![
Field {
label: "Egress Proxy",
key: "upstream_proxy_url",
value: String::new(),
kind: FieldKind::Text,
required: false,
help:
"Optional egress proxy for Aether tunnel/API and provider requests, e.g. http://127.0.0.1:8080 or socks5h://127.0.0.1:1080",
},
Field {
label: "Install Service",
key: "install_service",
@@ -282,6 +276,7 @@ impl App {
"allow_private_targets" => cfg.allow_private_targets.map(|v| v.to_string()),
"heartbeat_interval" => cfg.heartbeat_interval.map(|v| v.to_string()),
"redirect_replay_budget_bytes" => cfg.redirect_replay_budget_bytes.clone(),
"upstream_proxy_url" => cfg.upstream_proxy_url.clone(),
_ => None,
};
if let Some(v) = val {
@@ -296,11 +291,6 @@ impl App {
} else {
self.server_tabs = servers.iter().map(ServerTab::from_entry).collect();
}
if let Some(proxy_url) = cfg.upstream_proxy_url.as_deref() {
for tab in &mut self.server_tabs {
tab.set_field_value("upstream_proxy_url", proxy_url);
}
}
self.active_tab = 0;
self.selected = 0;
self.scroll_offset = 0;
@@ -322,12 +312,6 @@ impl App {
.filter(|v| !v.is_empty())
}
fn get_shared_server_field(&self, key: &str) -> Option<String> {
self.server_tabs
.iter()
.find_map(|tab| Self::get_tab(tab, key))
}
fn toggle_enabled(&self, key: &str) -> bool {
self.get_global(key).as_deref() == Some("true")
}
@@ -373,7 +357,7 @@ impl App {
}
fn parse_optional_upstream_proxy_url(&self) -> anyhow::Result<Option<String>> {
let Some(raw) = self.get_shared_server_field("upstream_proxy_url") else {
let Some(raw) = self.get_global("upstream_proxy_url") else {
return Ok(None);
};
let trimmed = raw.trim();
@@ -381,7 +365,7 @@ impl App {
return Ok(None);
}
UpstreamProxyConfig::parse(trimmed)
.map_err(|err| anyhow::anyhow!("upstream proxy URL invalid: {err}"))?;
.map_err(|err| anyhow::anyhow!("egress proxy URL invalid: {err}"))?;
Ok(Some(trimmed.to_string()))
}
@@ -613,11 +597,7 @@ impl App {
}
// -- Add / remove server --
KeyCode::Char('+') | KeyCode::Char('a') => {
let upstream_proxy_url = self.get_shared_server_field("upstream_proxy_url");
let mut tab = ServerTab::new();
if let Some(proxy_url) = upstream_proxy_url.as_deref() {
tab.set_field_value("upstream_proxy_url", proxy_url);
}
let tab = ServerTab::new();
self.server_tabs.push(tab);
self.active_tab = self.server_tabs.len() - 1;
self.selected = 0;
@@ -694,14 +674,7 @@ impl App {
fn commit_edit_buffer(&mut self) -> bool {
if self.validate_edit() {
let key = self.selected_field().key;
if key == "upstream_proxy_url" {
for tab in &mut self.server_tabs {
tab.set_field_value(key, &self.edit_buffer);
}
} else {
self.selected_field_mut().value = self.edit_buffer.clone();
}
self.selected_field_mut().value = self.edit_buffer.clone();
self.modified = true;
self.mode = Mode::Normal;
true
@@ -1124,23 +1097,20 @@ mod tests {
}
#[test]
fn new_app_places_upstream_proxy_under_node_name() {
fn new_app_places_egress_proxy_in_global_fields() {
let app = sample_app();
let keys: Vec<&str> = app.server_tabs[0]
let server_keys: Vec<&str> = app.server_tabs[0]
.fields
.iter()
.map(|field| field.key)
.collect();
let global_keys: Vec<&str> = app.global_fields.iter().map(|field| field.key).collect();
assert_eq!(
keys,
vec![
"aether_url",
"management_token",
"node_name",
"upstream_proxy_url"
]
server_keys,
vec!["aether_url", "management_token", "node_name"]
);
assert_eq!(global_keys.first().copied(), Some("upstream_proxy_url"));
}
#[test]
@@ -1172,7 +1142,7 @@ mod tests {
set_global_field(&mut app, "allow_private_targets", "true");
set_global_field(&mut app, "heartbeat_interval", "45");
set_global_field(&mut app, "redirect_replay_budget_bytes", "6m");
set_server_field(&mut app, "upstream_proxy_url", "socks5h://127.0.0.1:1080");
set_global_field(&mut app, "upstream_proxy_url", "socks5h://127.0.0.1:1080");
let cfg = app.to_config().expect("config should serialize");
assert_eq!(cfg.allow_private_targets, Some(true));
@@ -1194,14 +1164,14 @@ mod tests {
}
#[test]
fn to_config_rejects_invalid_upstream_proxy_url() {
fn to_config_rejects_invalid_egress_proxy_url() {
let mut app = sample_app();
set_server_field(&mut app, "upstream_proxy_url", "ftp://proxy.example");
set_global_field(&mut app, "upstream_proxy_url", "ftp://proxy.example");
let error = app
.to_config()
.expect_err("invalid upstream proxy should be rejected");
assert!(error.to_string().contains("upstream proxy URL"));
.expect_err("invalid egress proxy should be rejected");
assert!(error.to_string().contains("egress proxy URL"));
}
#[test]

221
apps/aether-proxy/src/state.rs
View File

@@ -1,8 +1,10 @@
//! Shared application state passed to all subsystems.
use std::collections::VecDeque;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, RwLock};
use std::sync::{Arc, Mutex, RwLock};
use std::time::Duration;
use std::time::{SystemTime, UNIX_EPOCH};
use aether_runtime::{AdmissionPermit, ConcurrencyError, ConcurrencyGate, ConcurrencySnapshot};
use aether_runtime_state::{RuntimeSemaphore, RuntimeSemaphoreError, RuntimeSemaphoreSnapshot};
@@ -50,6 +52,8 @@ pub struct ServerContext {
pub active_connections: Arc<AtomicU64>,
/// Per-server request/latency metrics.
pub metrics: Arc<ProxyMetrics>,
/// Per-server tunnel stability/traffic metrics.
pub tunnel_metrics: Arc<TunnelMetrics>,
}
/// Aggregate metrics for reporting to Aether.
@@ -83,6 +87,221 @@ impl ProxyMetrics {
}
}
const RECENT_TUNNEL_ERROR_CAPACITY: usize = 64;
const TUNNEL_ERROR_CATEGORY_MAX_CHARS: usize = 48;
const TUNNEL_ERROR_MESSAGE_MAX_CHARS: usize = 320;
#[derive(Debug, Clone, serde::Serialize)]
pub struct TunnelErrorEvent {
pub timestamp_unix_secs: u64,
pub category: String,
pub message: String,
}
#[derive(Debug, Clone, Copy, Default)]
pub struct TunnelMetricsSnapshot {
pub connect_attempts: u64,
pub connect_successes: u64,
pub connect_errors: u64,
pub disconnects: u64,
pub last_connected_at_unix_secs: u64,
pub last_disconnected_at_unix_secs: u64,
pub last_connected_duration_ms: u64,
pub connected_duration_total_ms: u64,
pub heartbeat_sent: u64,
pub heartbeat_ack: u64,
pub heartbeat_rtt_last_ms: u64,
pub heartbeat_rtt_total_ms: u64,
pub ws_in_frames: u64,
pub ws_in_bytes: u64,
pub ws_out_frames: u64,
pub ws_out_bytes: u64,
pub error_events_total: u64,
}
impl TunnelMetricsSnapshot {
pub fn heartbeat_rtt_avg_ms(self) -> Option<f64> {
if self.heartbeat_ack == 0 {
None
} else {
Some(self.heartbeat_rtt_total_ms as f64 / self.heartbeat_ack as f64)
}
}
}
pub struct TunnelMetrics {
connect_attempts: AtomicU64,
connect_successes: AtomicU64,
connect_errors: AtomicU64,
disconnects: AtomicU64,
last_connected_at_unix_secs: AtomicU64,
last_disconnected_at_unix_secs: AtomicU64,
last_connected_duration_ms: AtomicU64,
connected_duration_total_ms: AtomicU64,
heartbeat_sent: AtomicU64,
heartbeat_ack: AtomicU64,
heartbeat_rtt_last_ms: AtomicU64,
heartbeat_rtt_total_ms: AtomicU64,
ws_in_frames: AtomicU64,
ws_in_bytes: AtomicU64,
ws_out_frames: AtomicU64,
ws_out_bytes: AtomicU64,
error_events_total: AtomicU64,
recent_errors: Mutex<VecDeque<TunnelErrorEvent>>,
}
impl TunnelMetrics {
pub fn new() -> Self {
Self {
connect_attempts: AtomicU64::new(0),
connect_successes: AtomicU64::new(0),
connect_errors: AtomicU64::new(0),
disconnects: AtomicU64::new(0),
last_connected_at_unix_secs: AtomicU64::new(0),
last_disconnected_at_unix_secs: AtomicU64::new(0),
last_connected_duration_ms: AtomicU64::new(0),
connected_duration_total_ms: AtomicU64::new(0),
heartbeat_sent: AtomicU64::new(0),
heartbeat_ack: AtomicU64::new(0),
heartbeat_rtt_last_ms: AtomicU64::new(0),
heartbeat_rtt_total_ms: AtomicU64::new(0),
ws_in_frames: AtomicU64::new(0),
ws_in_bytes: AtomicU64::new(0),
ws_out_frames: AtomicU64::new(0),
ws_out_bytes: AtomicU64::new(0),
error_events_total: AtomicU64::new(0),
recent_errors: Mutex::new(VecDeque::with_capacity(RECENT_TUNNEL_ERROR_CAPACITY)),
}
}
pub fn record_connect_attempt(&self) {
self.connect_attempts.fetch_add(1, Ordering::Release);
}
pub fn record_connect_success(&self) {
self.connect_successes.fetch_add(1, Ordering::Release);
self.last_connected_at_unix_secs
.store(now_unix_secs(), Ordering::Release);
}
pub fn record_connect_error(&self) {
self.connect_errors.fetch_add(1, Ordering::Release);
}
pub fn record_disconnect(&self, connected_for: Duration) {
let duration_ms = duration_to_millis_u64(connected_for);
self.disconnects.fetch_add(1, Ordering::Release);
self.last_disconnected_at_unix_secs
.store(now_unix_secs(), Ordering::Release);
self.last_connected_duration_ms
.store(duration_ms, Ordering::Release);
self.connected_duration_total_ms
.fetch_add(duration_ms, Ordering::Release);
}
pub fn record_heartbeat_sent(&self) {
self.heartbeat_sent.fetch_add(1, Ordering::Release);
}
pub fn record_heartbeat_ack(&self, rtt: Duration) {
let rtt_ms = duration_to_millis_u64(rtt);
self.heartbeat_ack.fetch_add(1, Ordering::Release);
self.heartbeat_rtt_last_ms.store(rtt_ms, Ordering::Release);
self.heartbeat_rtt_total_ms
.fetch_add(rtt_ms, Ordering::Release);
}
pub fn record_ws_incoming_frame(&self, payload_len: usize) {
self.ws_in_frames.fetch_add(1, Ordering::Release);
self.ws_in_bytes.fetch_add(
u64::try_from(payload_len).unwrap_or(u64::MAX),
Ordering::Release,
);
}
pub fn record_ws_outgoing_frame(&self, payload_len: usize) {
self.ws_out_frames.fetch_add(1, Ordering::Release);
self.ws_out_bytes.fetch_add(
u64::try_from(payload_len).unwrap_or(u64::MAX),
Ordering::Release,
);
}
pub fn record_error(&self, category: &str, message: &str) {
self.error_events_total.fetch_add(1, Ordering::Release);
let event = TunnelErrorEvent {
timestamp_unix_secs: now_unix_secs(),
category: normalize_error_field(category, TUNNEL_ERROR_CATEGORY_MAX_CHARS, "unknown"),
message: normalize_error_field(message, TUNNEL_ERROR_MESSAGE_MAX_CHARS, "n/a"),
};
let mut recent_errors = match self.recent_errors.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
if recent_errors.len() >= RECENT_TUNNEL_ERROR_CAPACITY {
recent_errors.pop_front();
}
recent_errors.push_back(event);
}
pub fn recent_errors(&self, limit: usize) -> Vec<TunnelErrorEvent> {
if limit == 0 {
return Vec::new();
}
let recent_errors = match self.recent_errors.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
let start = recent_errors.len().saturating_sub(limit);
recent_errors.iter().skip(start).cloned().collect()
}
pub fn snapshot(&self) -> TunnelMetricsSnapshot {
TunnelMetricsSnapshot {
connect_attempts: self.connect_attempts.load(Ordering::Acquire),
connect_successes: self.connect_successes.load(Ordering::Acquire),
connect_errors: self.connect_errors.load(Ordering::Acquire),
disconnects: self.disconnects.load(Ordering::Acquire),
last_connected_at_unix_secs: self.last_connected_at_unix_secs.load(Ordering::Acquire),
last_disconnected_at_unix_secs: self
.last_disconnected_at_unix_secs
.load(Ordering::Acquire),
last_connected_duration_ms: self.last_connected_duration_ms.load(Ordering::Acquire),
connected_duration_total_ms: self.connected_duration_total_ms.load(Ordering::Acquire),
heartbeat_sent: self.heartbeat_sent.load(Ordering::Acquire),
heartbeat_ack: self.heartbeat_ack.load(Ordering::Acquire),
heartbeat_rtt_last_ms: self.heartbeat_rtt_last_ms.load(Ordering::Acquire),
heartbeat_rtt_total_ms: self.heartbeat_rtt_total_ms.load(Ordering::Acquire),
ws_in_frames: self.ws_in_frames.load(Ordering::Acquire),
ws_in_bytes: self.ws_in_bytes.load(Ordering::Acquire),
ws_out_frames: self.ws_out_frames.load(Ordering::Acquire),
ws_out_bytes: self.ws_out_bytes.load(Ordering::Acquire),
error_events_total: self.error_events_total.load(Ordering::Acquire),
}
}
}
fn now_unix_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0)
}
fn duration_to_millis_u64(duration: Duration) -> u64 {
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
}
fn normalize_error_field(value: &str, max_chars: usize, fallback: &str) -> String {
let normalized = value.split_whitespace().collect::<Vec<_>>().join(" ");
if normalized.is_empty() {
return fallback.to_string();
}
normalized.chars().take(max_chars).collect()
}
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
pub enum ProxyAdmissionError {
#[error("proxy stream admission saturated at {limit} for gate {gate}")]

95
apps/aether-proxy/src/tunnel/client.rs
View File

@@ -1,7 +1,7 @@
//! WebSocket tunnel client: connect, authenticate, and run the tunnel.
use std::sync::Arc;
use std::time::Duration;
use std::time::{Duration, Instant};
use tokio::net::TcpStream;
use tokio::sync::watch;
@@ -10,6 +10,7 @@ use tokio_tungstenite::tungstenite::http;
use tokio_tungstenite::tungstenite::protocol::WebSocketConfig;
use tracing::{debug, warn};
use crate::egress_proxy::{connect_target_via_proxy, ProxyConnectOptions, UpstreamProxyConfig};
use crate::state::{AppState, ServerContext};
use super::{dispatcher, heartbeat, writer};
@@ -71,14 +72,7 @@ pub async fn connect_and_run(
.config
.tunnel_connect_timeout()
.expect("validated config should resolve tunnel connect timeout");
let tcp_stream = tokio::time::timeout(connect_timeout, TcpStream::connect((host, port)))
.await
.map_err(|_| {
anyhow::anyhow!(
"tunnel TCP connect timeout ({}ms)",
connect_timeout.as_millis()
)
})??;
let tcp_stream = connect_tunnel_tcp(state, host, port, connect_timeout).await?;
// Configure TCP parameters via socket2
configure_tcp_socket(&tcp_stream, state);
@@ -133,6 +127,8 @@ pub async fn connect_and_run(
ping_interval_ms = ping_interval.as_millis(),
"tunnel connected"
);
server.tunnel_metrics.record_connect_success();
let connected_at = Instant::now();
// NOTE: reconnect_attempts reset is handled by the caller (mod.rs)
// based on how long the connection stayed alive.
@@ -141,7 +137,11 @@ pub async fn connect_and_run(
let (ws_sink, ws_read) = futures_util::StreamExt::split(ws_stream);
// Spawn writer task (with WebSocket ping keepalive)
let (frame_tx, mut writer_handle) = writer::spawn_writer(ws_sink, ping_interval);
let (frame_tx, mut writer_handle) = writer::spawn_writer_with_metrics(
ws_sink,
ping_interval,
Some(Arc::clone(&server.tunnel_metrics)),
);
let drain_signal = spawn_drain_signal(conn_idx, frame_tx.clone(), drain.clone());
// Spawn heartbeat task (only for primary connection to avoid
@@ -174,8 +174,13 @@ pub async fn connect_and_run(
drain.clone(),
) => {
match result {
Ok(()) => TunnelOutcome::Disconnected,
Err(e) => return Err(e),
Ok(()) => Ok(TunnelOutcome::Disconnected),
Err(e) => {
server
.tunnel_metrics
.record_error("dispatcher_error", &e.to_string());
Err(e)
}
}
}
writer_result = &mut writer_handle => {
@@ -184,16 +189,22 @@ pub async fn connect_and_run(
Err(e) => {
if e.is_panic() {
tracing::error!(error = %e, "writer task panicked, triggering reconnect");
server
.tunnel_metrics
.record_error("writer_task_panic", &e.to_string());
} else {
warn!(error = %e, "writer task cancelled, triggering reconnect");
server
.tunnel_metrics
.record_error("writer_task_cancelled", &e.to_string());
}
}
}
TunnelOutcome::Disconnected
Ok(TunnelOutcome::Disconnected)
}
_ = shutdown.changed() => {
debug!("shutdown during tunnel dispatch");
TunnelOutcome::Shutdown
Ok(TunnelOutcome::Shutdown)
}
};
@@ -213,8 +224,12 @@ pub async fn connect_and_run(
let _ = tokio::time::timeout(Duration::from_secs(35), writer_handle).await;
}
server
.tunnel_metrics
.record_disconnect(connected_at.elapsed());
debug!("tunnel disconnected");
Ok(outcome)
outcome
}
fn spawn_drain_signal(
@@ -246,6 +261,56 @@ fn spawn_drain_signal(
})
}
async fn connect_tunnel_tcp(
state: &Arc<AppState>,
host: &str,
port: u16,
connect_timeout: Duration,
) -> Result<TcpStream, anyhow::Error> {
if let Some(proxy_url) = state.config.effective_aether_proxy_url() {
let proxy = UpstreamProxyConfig::parse(proxy_url)
.map_err(|err| anyhow::anyhow!("aether proxy URL invalid: {err}"))?;
debug!(
proxy_url = %proxy.redacted_url(),
host = %host,
port = port,
"connecting tunnel via Aether egress proxy"
);
return tokio::time::timeout(
connect_timeout,
connect_target_via_proxy(
&proxy,
host,
port,
ProxyConnectOptions {
connect_timeout,
tcp_nodelay: state.config.tunnel_tcp_nodelay,
tcp_keepalive: (state.config.tunnel_tcp_keepalive_secs > 0)
.then(|| Duration::from_secs(state.config.tunnel_tcp_keepalive_secs)),
},
),
)
.await
.map_err(|_| {
anyhow::anyhow!(
"tunnel proxy TCP connect timeout ({}ms)",
connect_timeout.as_millis()
)
})?
.map_err(anyhow::Error::from);
}
tokio::time::timeout(connect_timeout, TcpStream::connect((host, port)))
.await
.map_err(|_| {
anyhow::anyhow!(
"tunnel TCP connect timeout ({}ms)",
connect_timeout.as_millis()
)
})?
.map_err(anyhow::Error::from)
}
/// Configure TCP keepalive and NODELAY on an established socket.
fn configure_tcp_socket(stream: &TcpStream, state: &Arc<AppState>) {
let sock_ref = socket2::SockRef::from(stream);

19
apps/aether-proxy/src/tunnel/dispatcher.rs
View File

@@ -84,6 +84,10 @@ where
stale_ms = stale_timeout.as_millis(),
"tunnel connection stale, no data received"
);
server.tunnel_metrics.record_error(
"stale_timeout",
&format!("no tunnel frame received for {}ms", stale_timeout.as_millis()),
);
break None;
}
};
@@ -92,6 +96,9 @@ where
Ok(m) => m,
Err(e) => {
error!(error = %e, "WebSocket read error");
server
.tunnel_metrics
.record_error("ws_read_error", &e.to_string());
break Some(e);
}
};
@@ -100,7 +107,10 @@ where
last_data_at = tokio::time::Instant::now();
let data = match msg {
Message::Binary(data) => Bytes::from(data),
Message::Binary(data) => {
server.tunnel_metrics.record_ws_incoming_frame(data.len());
Bytes::from(data)
}
Message::Ping(_) => continue,
Message::Pong(_) => continue,
Message::Close(_) => {
@@ -114,6 +124,9 @@ where
Ok(f) => f,
Err(e) => {
warn!(error = %e, "failed to decode frame");
server
.tunnel_metrics
.record_error("frame_decode_error", &e.to_string());
continue;
}
};
@@ -223,6 +236,10 @@ where
if is_end || dispatch != StreamDispatchStatus::Delivered {
streams.remove(&sid);
if dispatch == StreamDispatchStatus::TimedOut {
server.tunnel_metrics.record_error(
"stream_dispatch_timeout",
&format!("request body dispatch timed out for stream {}", sid),
);
try_send_stream_error(
&frame_tx,
sid,

78
apps/aether-proxy/src/tunnel/heartbeat.rs
View File

@@ -8,6 +8,7 @@ use std::time::UNIX_EPOCH;
use bytes::Bytes;
use tokio::sync::watch;
use tokio::time::Instant;
use tracing::{debug, info, warn};
use crate::registration::client::RemoteConfig;
@@ -60,6 +61,13 @@ struct HeartbeatSnapshot {
stream_errors: u64,
}
#[derive(Debug, Clone, Copy)]
struct PendingHeartbeat {
heartbeat_id: u64,
snapshot: HeartbeatSnapshot,
sent_at: Option<Instant>,
}
/// Spawn the heartbeat task. Returns a handle for forwarding ACKs.
pub fn spawn(
state: Arc<AppState>,
@@ -76,7 +84,7 @@ pub fn spawn(
// At most one in-flight heartbeat snapshot is tracked at a time.
// Snapshot is only cleared after receiving an ACK, which avoids losing
// interval counters when ACK/frame delivery is temporarily unstable.
let mut pending: Option<(u64, HeartbeatSnapshot)> = None;
let mut pending: Option<PendingHeartbeat> = None;
let mut next_heartbeat_id: u64 = 1;
let heartbeat_session_id = format!(
"{}-{}",
@@ -93,8 +101,8 @@ pub fn spawn(
loop {
tokio::select! {
_ = tokio::time::sleep(current_interval) => {
let (heartbeat_id, snapshot) = if let Some((id, snap)) = pending {
(id, snap)
let pending_entry = if let Some(entry) = pending {
entry
} else {
let snap = collect_snapshot(&server);
let id = next_heartbeat_id;
@@ -102,24 +110,34 @@ pub fn spawn(
if next_heartbeat_id == 0 {
next_heartbeat_id = 1;
}
pending = Some((id, snap));
(id, snap)
let entry = PendingHeartbeat {
heartbeat_id: id,
snapshot: snap,
sent_at: None,
};
pending = Some(entry);
entry
};
let payload = build_heartbeat_payload(
&state,
&server,
&heartbeat_session_id,
heartbeat_id,
snapshot
pending_entry.heartbeat_id,
pending_entry.snapshot
).await;
let frame = Frame::control(MsgType::HeartbeatData, payload);
if frame_tx.send(frame).await.is_err() {
if let Some((_, snap)) = pending.take() {
restore_snapshot(&server, snap);
if let Some(entry) = pending.take() {
restore_snapshot(&server, entry.snapshot);
}
break; // Writer closed
}
server.tunnel_metrics.record_heartbeat_sent();
if let Some(mut entry) = pending {
entry.sent_at = Some(Instant::now());
pending = Some(entry);
}
debug!("sent heartbeat data");
// Re-read interval from dynamic config (remote config may have
@@ -142,8 +160,11 @@ pub fn spawn(
heartbeat_id: ack_id,
upgrade_to,
} => {
if let Some((pending_id, _)) = pending {
if ack_id == pending_id {
if let Some(entry) = pending {
if ack_id == entry.heartbeat_id {
if let Some(sent_at) = entry.sent_at {
server.tunnel_metrics.record_heartbeat_ack(sent_at.elapsed());
}
pending = None;
}
}
@@ -154,8 +175,8 @@ pub fn spawn(
}
_ = shutdown.changed() => {
debug!("heartbeat task shutting down");
if let Some((_, snap)) = pending.take() {
restore_snapshot(&server, snap);
if let Some(entry) = pending.take() {
restore_snapshot(&server, entry.snapshot);
}
break;
}
@@ -217,6 +238,8 @@ async fn build_heartbeat_payload(
snapshot: HeartbeatSnapshot,
) -> Bytes {
let node_id = server.node_id.read().unwrap().clone();
let tunnel_snapshot = server.tunnel_metrics.snapshot();
let recent_errors = server.tunnel_metrics.recent_errors(8);
let avg_latency_ms = if snapshot.requests > 0 {
Some(snapshot.latency_ns as f64 / snapshot.requests as f64 / 1_000_000.0)
@@ -268,6 +291,26 @@ async fn build_heartbeat_payload(
"proxy_metadata": {
"version": CURRENT_VERSION,
"admission": admission,
"tunnel_metrics": {
"connect_attempts": tunnel_snapshot.connect_attempts,
"connect_successes": tunnel_snapshot.connect_successes,
"connect_errors": tunnel_snapshot.connect_errors,
"disconnects": tunnel_snapshot.disconnects,
"last_connected_at_unix_secs": tunnel_snapshot.last_connected_at_unix_secs,
"last_disconnected_at_unix_secs": tunnel_snapshot.last_disconnected_at_unix_secs,
"last_connected_duration_ms": tunnel_snapshot.last_connected_duration_ms,
"connected_duration_total_ms": tunnel_snapshot.connected_duration_total_ms,
"heartbeat_sent": tunnel_snapshot.heartbeat_sent,
"heartbeat_ack": tunnel_snapshot.heartbeat_ack,
"heartbeat_rtt_last_ms": tunnel_snapshot.heartbeat_rtt_last_ms,
"heartbeat_rtt_avg_ms": tunnel_snapshot.heartbeat_rtt_avg_ms(),
"ws_in_frames": tunnel_snapshot.ws_in_frames,
"ws_in_bytes": tunnel_snapshot.ws_in_bytes,
"ws_out_frames": tunnel_snapshot.ws_out_frames,
"ws_out_bytes": tunnel_snapshot.ws_out_bytes,
"error_events_total": tunnel_snapshot.error_events_total,
},
"recent_tunnel_errors": recent_errors,
},
});
@@ -277,6 +320,9 @@ async fn build_heartbeat_payload(
fn handle_ack(server: &ServerContext, payload: &[u8]) -> AckDecision {
if payload.is_empty() {
warn!("received empty heartbeat ACK");
server
.tunnel_metrics
.record_error("heartbeat_ack_empty", "received empty heartbeat ACK");
return AckDecision::Ignore;
}
@@ -303,6 +349,9 @@ fn handle_ack(server: &ServerContext, payload: &[u8]) -> AckDecision {
}
Err(e) => {
warn!(error = %e, "failed to parse heartbeat ACK");
server
.tunnel_metrics
.record_error("heartbeat_ack_parse", &e.to_string());
AckDecision::Ignore
}
}
@@ -373,7 +422,7 @@ mod tests {
use super::{handle_ack, AckDecision};
use crate::registration::client::AetherClient;
use crate::runtime::DynamicConfig;
use crate::state::{ProxyMetrics, ServerContext};
use crate::state::{ProxyMetrics, ServerContext, TunnelMetrics};
fn sample_server() -> Arc<ServerContext> {
let config = Arc::new(crate::config::Config::parse_from([
@@ -399,6 +448,7 @@ mod tests {
dynamic: Arc::new(ArcSwap::from_pointee(DynamicConfig::from_config(&config))),
active_connections: Arc::new(AtomicU64::new(0)),
metrics: Arc::new(ProxyMetrics::new()),
tunnel_metrics: Arc::new(TunnelMetrics::new()),
})
}

9
apps/aether-proxy/src/tunnel/mod.rs
View File

@@ -76,6 +76,7 @@ pub async fn run(
info!(server = %server.server_label, conn = conn_idx, "tunnel drained, exiting slot");
return;
}
server.tunnel_metrics.record_connect_attempt();
let started_at = Instant::now();
match client::connect_and_run(state, server, conn_idx, &mut shutdown, drain.clone()).await {
Ok(client::TunnelOutcome::Shutdown) => {
@@ -86,6 +87,10 @@ pub async fn run(
debug!(server = %server.server_label, conn = conn_idx, "tunnel disconnected, reconnecting");
}
Err(e) => {
server.tunnel_metrics.record_connect_error();
server
.tunnel_metrics
.record_error("tunnel_connect_error", &e.to_string());
error!(server = %server.server_label, conn = conn_idx, error = %e, "tunnel connection error, reconnecting");
}
}
@@ -237,7 +242,7 @@ mod tests {
use crate::config::Config;
use crate::registration::client::AetherClient;
use crate::runtime::DynamicConfig;
use crate::state::{AppState as ProxyAppState, ProxyMetrics, ServerContext};
use crate::state::{AppState as ProxyAppState, ProxyMetrics, ServerContext, TunnelMetrics};
use crate::target_filter::DnsCache;
use crate::tunnel::protocol;
use crate::upstream_client;
@@ -478,6 +483,7 @@ mod tests {
dynamic: Arc::new(ArcSwap::from_pointee(DynamicConfig::from_config(&config))),
active_connections: Arc::new(AtomicU64::new(0)),
metrics: Arc::new(ProxyMetrics::new()),
tunnel_metrics: Arc::new(TunnelMetrics::new()),
})
}
@@ -498,6 +504,7 @@ mod tests {
aether_tcp_keepalive_secs: 60,
aether_tcp_nodelay: true,
aether_http2: true,
aether_proxy_url: None,
aether_retry_max_attempts: 1,
aether_retry_base_delay_ms: 50,
aether_retry_max_delay_ms: 100,

4
apps/aether-proxy/src/tunnel/stream_handler.rs
View File

@@ -1532,7 +1532,7 @@ mod tests {
use crate::config::Config;
use crate::registration::client::AetherClient;
use crate::runtime::DynamicConfig;
use crate::state::ProxyMetrics;
use crate::state::{ProxyMetrics, TunnelMetrics};
use crate::target_filter::DnsCache;
use crate::tunnel::client::build_tls_config;
@@ -2335,6 +2335,7 @@ mod tests {
dynamic: Arc::new(ArcSwap::from_pointee(DynamicConfig::from_config(&config))),
active_connections: Arc::new(AtomicU64::new(0)),
metrics: Arc::new(ProxyMetrics::new()),
tunnel_metrics: Arc::new(TunnelMetrics::new()),
})
}
@@ -2355,6 +2356,7 @@ mod tests {
aether_tcp_keepalive_secs: 60,
aether_tcp_nodelay: true,
aether_http2: true,
aether_proxy_url: None,
aether_retry_max_attempts: 3,
aether_retry_base_delay_ms: 200,
aether_retry_max_delay_ms: 2_000,

38
apps/aether-proxy/src/tunnel/writer.rs
View File

@@ -5,9 +5,10 @@
//! periodic WebSocket Ping frames to keep the connection alive through
//! intermediary proxies (Nginx, Cloudflare, etc.).
use std::sync::Arc;
use std::time::Duration;
use aether_contracts::tunnel::MsgType;
use aether_contracts::tunnel::{MsgType, HEADER_SIZE};
#[cfg(test)]
use aether_runtime::QueueSnapshot;
use aether_runtime::{bounded_queue, BoundedQueueSender, QueueSendError};
@@ -16,6 +17,8 @@ use tokio::task::JoinHandle;
use tokio_tungstenite::tungstenite::Message;
use tracing::{debug, error, trace};
use crate::state::TunnelMetrics;
use super::protocol::Frame;
const HIGH_PRIORITY_QUEUE_CAPACITY: usize = 64;
@@ -77,7 +80,20 @@ impl FrameSender {
///
/// `ping_interval` controls WebSocket-level Ping frequency (typically 15s).
/// This keeps the connection alive through intermediary proxies/load-balancers.
pub fn spawn_writer<S>(mut sink: S, ping_interval: Duration) -> (FrameSender, JoinHandle<()>)
#[cfg(test)]
pub fn spawn_writer<S>(sink: S, ping_interval: Duration) -> (FrameSender, JoinHandle<()>)
where
S: SinkExt<Message, Error = tokio_tungstenite::tungstenite::Error> + Unpin + Send + 'static,
{
spawn_writer_with_metrics(sink, ping_interval, None)
}
/// Spawn the writer task with optional tunnel metrics instrumentation.
pub fn spawn_writer_with_metrics<S>(
mut sink: S,
ping_interval: Duration,
tunnel_metrics: Option<Arc<TunnelMetrics>>,
) -> (FrameSender, JoinHandle<()>)
where
S: SinkExt<Message, Error = tokio_tungstenite::tungstenite::Error> + Unpin + Send + 'static,
{
@@ -93,7 +109,7 @@ where
loop {
if let Ok(frame) = high_rx.try_recv() {
if !write_frame(&mut sink, frame).await {
if !write_frame(&mut sink, frame, tunnel_metrics.as_deref()).await {
break;
}
continue;
@@ -107,7 +123,7 @@ where
frame = high_rx.recv(), if high_open => {
match frame {
Some(frame) => {
if !write_frame(&mut sink, frame).await {
if !write_frame(&mut sink, frame, tunnel_metrics.as_deref()).await {
break;
}
}
@@ -117,6 +133,9 @@ where
_ = ping_ticker.tick(), if high_open || normal_open => {
if let Err(e) = sink.send(Message::Ping(vec![])).await {
error!(error = %e, "failed to send WebSocket ping");
if let Some(metrics) = tunnel_metrics.as_deref() {
metrics.record_error("ws_ping_error", &e.to_string());
}
break;
}
trace!("sent WebSocket ping");
@@ -124,7 +143,7 @@ where
frame = normal_rx.recv(), if normal_open => {
match frame {
Some(frame) => {
if !write_frame(&mut sink, frame).await {
if !write_frame(&mut sink, frame, tunnel_metrics.as_deref()).await {
break;
}
}
@@ -156,15 +175,22 @@ fn classify_frame_priority(frame: &Frame) -> FramePriority {
}
}
async fn write_frame<S>(sink: &mut S, frame: Frame) -> bool
async fn write_frame<S>(sink: &mut S, frame: Frame, tunnel_metrics: Option<&TunnelMetrics>) -> bool
where
S: SinkExt<Message, Error = tokio_tungstenite::tungstenite::Error> + Unpin + Send + 'static,
{
let data = frame.encode();
let wire_len = data.len().max(HEADER_SIZE);
if let Err(e) = sink.send(Message::Binary(data.into())).await {
error!(error = %e, "failed to write frame to WebSocket");
if let Some(metrics) = tunnel_metrics {
metrics.record_error("ws_write_error", &e.to_string());
}
return false;
}
if let Some(metrics) = tunnel_metrics {
metrics.record_ws_outgoing_frame(wire_len);
}
true
}

274
apps/aether-proxy/src/upstream_client.rs
View File

@@ -2,7 +2,7 @@ use std::collections::HashMap;
use std::convert::Infallible;
use std::future::Future;
use std::io;
use std::net::{IpAddr, SocketAddr};
use std::net::IpAddr;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::Mutex;
@@ -28,14 +28,15 @@ use hyper_util::client::legacy::Client;
use hyper_util::rt::{TokioExecutor, TokioIo, TokioTimer};
use rustls::pki_types::ServerName;
use rustls::ClientConfig;
use socket2::{SockRef, TcpKeepalive};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpStream;
use tokio_rustls::TlsConnector;
use tower_service::Service;
use crate::config::Config;
use crate::egress_proxy::{UpstreamProxyConfig, UpstreamProxyScheme};
use crate::egress_proxy::{
connect_proxy_tcp, http_connect, socks5_connect, ProxyConnectOptions, UpstreamProxyConfig,
UpstreamProxyScheme,
};
use crate::target_filter::{self, DnsCache};
type BoxError = Box<dyn std::error::Error + Send + Sync>;
@@ -263,12 +264,6 @@ pub struct InstrumentedConnector {
tcp_keepalive: Option<Duration>,
}
struct ProxyConnectOptions {
connect_timeout: Duration,
tcp_nodelay: bool,
tcp_keepalive: Option<Duration>,
}
impl Service<Uri> for InstrumentedConnector {
type Response = TimedConn;
type Error = BoxError;
@@ -402,262 +397,6 @@ async fn connect_via_proxy(
}
}
async fn connect_proxy_tcp(
proxy: &UpstreamProxyConfig,
connect_timeout: Duration,
tcp_nodelay: bool,
tcp_keepalive: Option<Duration>,
) -> io::Result<TcpStream> {
let resolved = tokio::time::timeout(
connect_timeout,
tokio::net::lookup_host((proxy.host(), proxy.port())),
)
.await
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "proxy DNS timeout"))?
.map_err(|err| io::Error::other(format!("proxy DNS failed: {err}")))?;
let mut last_error = None;
for addr in resolved {
match tokio::time::timeout(connect_timeout, TcpStream::connect(addr)).await {
Ok(Ok(stream)) => {
configure_tcp_stream(&stream, tcp_nodelay, tcp_keepalive)?;
return Ok(stream);
}
Ok(Err(error)) => last_error = Some(error),
Err(_) => {
last_error = Some(io::Error::new(
io::ErrorKind::TimedOut,
format!("proxy connect timeout: {addr}"),
));
}
}
}
Err(last_error.unwrap_or_else(|| io::Error::other("proxy DNS returned no addresses")))
}
fn configure_tcp_stream(
stream: &TcpStream,
tcp_nodelay: bool,
tcp_keepalive: Option<Duration>,
) -> io::Result<()> {
stream.set_nodelay(tcp_nodelay)?;
if let Some(keepalive) = tcp_keepalive {
let keepalive = TcpKeepalive::new().with_time(keepalive);
SockRef::from(stream).set_tcp_keepalive(&keepalive)?;
}
Ok(())
}
async fn http_connect(
stream: &mut TcpStream,
target_authority: &str,
proxy: &UpstreamProxyConfig,
) -> io::Result<()> {
let mut request = format!(
"CONNECT {target_authority} HTTP/1.1\r\nHost: {target_authority}\r\nProxy-Connection: Keep-Alive\r\n"
);
if let Some(auth) = proxy.basic_auth_header() {
request.push_str("Proxy-Authorization: ");
request.push_str(&auth);
request.push_str("\r\n");
}
request.push_str("\r\n");
stream.write_all(request.as_bytes()).await?;
stream.flush().await?;
let mut response = Vec::with_capacity(1024);
let mut chunk = [0u8; 1024];
loop {
if response.len() >= 16 * 1024 {
return Err(io::Error::other("proxy CONNECT response too large"));
}
let n = stream.read(&mut chunk).await?;
if n == 0 {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"proxy closed during CONNECT",
));
}
response.extend_from_slice(&chunk[..n]);
if response.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
let status_line_end = response
.windows(2)
.position(|window| window == b"\r\n")
.ok_or_else(|| io::Error::other("proxy CONNECT response missing status line"))?;
let status_line = std::str::from_utf8(&response[..status_line_end])
.map_err(|_| io::Error::other("proxy CONNECT status line is not UTF-8"))?;
let status = status_line.split_whitespace().nth(1).unwrap_or_default();
if status == "200" {
Ok(())
} else {
Err(io::Error::other(format!(
"proxy CONNECT failed: {status_line}"
)))
}
}
async fn socks5_connect(
stream: &mut TcpStream,
proxy: &UpstreamProxyConfig,
target_host: &str,
target_port: u16,
) -> io::Result<()> {
let requires_auth = proxy.username().is_some();
if requires_auth {
stream.write_all(&[0x05, 0x02, 0x00, 0x02]).await?;
} else {
stream.write_all(&[0x05, 0x01, 0x00]).await?;
}
let mut method_response = [0u8; 2];
stream.read_exact(&mut method_response).await?;
if method_response[0] != 0x05 {
return Err(io::Error::other("invalid SOCKS5 method response"));
}
match method_response[1] {
0x00 => {}
0x02 => socks5_authenticate(stream, proxy).await?,
0xff => return Err(io::Error::other("SOCKS5 proxy rejected all auth methods")),
method => {
return Err(io::Error::other(format!(
"SOCKS5 proxy selected unsupported auth method 0x{method:02x}"
)))
}
}
let address = socks5_target_address(target_host, target_port, proxy.uses_remote_dns()).await?;
stream.write_all(&address).await?;
let mut response = [0u8; 4];
stream.read_exact(&mut response).await?;
if response[0] != 0x05 {
return Err(io::Error::other("invalid SOCKS5 connect response"));
}
if response[1] != 0x00 {
return Err(io::Error::other(format!(
"SOCKS5 connect failed: {}",
socks5_reply_message(response[1])
)));
}
match response[3] {
0x01 => {
let mut ignored = [0u8; 4 + 2];
stream.read_exact(&mut ignored).await?;
}
0x03 => {
let mut len = [0u8; 1];
stream.read_exact(&mut len).await?;
let mut ignored = vec![0u8; len[0] as usize + 2];
stream.read_exact(&mut ignored).await?;
}
0x04 => {
let mut ignored = [0u8; 16 + 2];
stream.read_exact(&mut ignored).await?;
}
atyp => {
return Err(io::Error::other(format!(
"SOCKS5 proxy returned unsupported address type 0x{atyp:02x}"
)))
}
}
Ok(())
}
async fn socks5_authenticate(
stream: &mut TcpStream,
proxy: &UpstreamProxyConfig,
) -> io::Result<()> {
let username = proxy.username().unwrap_or_default().as_bytes();
let password = proxy.password().unwrap_or_default().as_bytes();
if username.len() > u8::MAX as usize || password.len() > u8::MAX as usize {
return Err(io::Error::other(
"SOCKS5 username/password must be at most 255 bytes",
));
}
let mut request = Vec::with_capacity(username.len() + password.len() + 3);
request.push(0x01);
request.push(username.len() as u8);
request.extend_from_slice(username);
request.push(password.len() as u8);
request.extend_from_slice(password);
stream.write_all(&request).await?;
let mut response = [0u8; 2];
stream.read_exact(&mut response).await?;
if response[0] != 0x01 || response[1] != 0x00 {
return Err(io::Error::other("SOCKS5 username/password auth failed"));
}
Ok(())
}
async fn socks5_target_address(
target_host: &str,
target_port: u16,
remote_dns: bool,
) -> io::Result<Vec<u8>> {
let mut request = vec![0x05, 0x01, 0x00];
if let Ok(ip) = target_host.parse::<IpAddr>() {
push_socks5_ip_address(&mut request, ip);
} else if remote_dns {
let host = target_host.as_bytes();
if host.len() > u8::MAX as usize {
return Err(io::Error::other("SOCKS5 target hostname is too long"));
}
request.push(0x03);
request.push(host.len() as u8);
request.extend_from_slice(host);
} else {
let mut resolved = tokio::net::lookup_host((target_host, target_port))
.await
.map_err(|err| io::Error::other(format!("SOCKS5 target DNS failed: {err}")))?;
let addr = resolved
.next()
.ok_or_else(|| io::Error::other("SOCKS5 target DNS returned no addresses"))?;
push_socks5_socket_address(&mut request, addr);
}
request.extend_from_slice(&target_port.to_be_bytes());
Ok(request)
}
fn push_socks5_socket_address(request: &mut Vec<u8>, addr: SocketAddr) {
push_socks5_ip_address(request, addr.ip());
}
fn push_socks5_ip_address(request: &mut Vec<u8>, ip: IpAddr) {
match ip {
IpAddr::V4(ip) => {
request.push(0x01);
request.extend_from_slice(&ip.octets());
}
IpAddr::V6(ip) => {
request.push(0x04);
request.extend_from_slice(&ip.octets());
}
}
}
fn socks5_reply_message(reply: u8) -> &'static str {
match reply {
0x01 => "general failure",
0x02 => "connection not allowed",
0x03 => "network unreachable",
0x04 => "host unreachable",
0x05 => "connection refused",
0x06 => "TTL expired",
0x07 => "command not supported",
0x08 => "address type not supported",
_ => "unknown error",
}
}
fn uri_host(uri: &Uri) -> Result<String, io::Error> {
uri.host()
.map(|host| {
@@ -934,8 +673,11 @@ mod tests {
use clap::Parser;
use http_body_util::BodyExt;
use hyper::Response;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
use crate::egress_proxy::socks5_target_address;
#[test]
fn fresh_connection_uses_connector_breakdown() {
let mut response = Response::new(());