feat(aether-proxy): 网络层全面优化与代理耗时追踪

aether-proxy:
- 新增 20+ 可配置参数:Aether API/delegate/CONNECT 各阶段超时、连接池、TCP keepalive/nodelay
- AetherClient 支持指数退避重试(可配置次数/延迟)和 HTTP/2
- 新增 DNS 缓存(TTL + 容量限制)避免重复解析
- 新增并发连接数限制(Semaphore,默认基于硬件估算)
- CONNECT 隧道增加建连超时和升级超时
- TLS 增加握手超时、会话缓存(session ticket + memory cache)、ALPN 协商
- 新增 ProxyMetrics 收集请求计数和延迟,通过心跳上报
- delegate 响应注入 X-Proxy-Timing 头(dns_ms/upstream_ms/total_ms)

后端:
- StreamContext 和 handler 提取 X-Proxy-Timing 写入 proxy_info 追踪数据

前端:
- 请求时间线展示代理分阶段耗时(DNS/上游)
- Endpoint 添加按钮改为文字按钮样式
This commit is contained in:
fawney19
2026-02-11 11:30:29 +08:00
parent 68f5e7e502
commit ed3f208dda
20 changed files with 832 additions and 73 deletions

View File

@@ -767,6 +767,25 @@ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
[[package]]
name = "h2"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54"
dependencies = [
"atomic-waker",
"bytes",
"fnv",
"futures-core",
"futures-sink",
"http",
"indexmap",
"slab",
"tokio",
"tokio-util",
"tracing",
]
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.16.1" version = "0.16.1"
@@ -854,6 +873,7 @@ dependencies = [
"bytes", "bytes",
"futures-channel", "futures-channel",
"futures-core", "futures-core",
"h2",
"http", "http",
"http-body", "http-body",
"httparse", "httparse",
@@ -1845,6 +1865,7 @@ dependencies = [
"bytes", "bytes",
"futures-core", "futures-core",
"futures-util", "futures-util",
"h2",
"http", "http",
"http-body", "http-body",
"http-body-util", "http-body-util",

View File

@@ -9,7 +9,7 @@ tokio = { version = "1", features = ["full"] }
hyper = { version = "1", features = ["http1", "server"] } hyper = { version = "1", features = ["http1", "server"] }
hyper-util = { version = "0.1", features = ["tokio", "http1", "server"] } hyper-util = { version = "0.1", features = ["tokio", "http1", "server"] }
http-body-util = "0.1" http-body-util = "0.1"
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] } reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream", "http2"] }
futures-util = "0.3" futures-util = "0.3"
hmac = "0.12" hmac = "0.12"
sha2 = "0.10" sha2 = "0.10"

View File

@@ -61,6 +61,26 @@ sudo aether-proxy uninstall
| `--heartbeat-interval` | `AETHER_PROXY_HEARTBEAT_INTERVAL` | `30` | 心跳间隔(秒) | | `--heartbeat-interval` | `AETHER_PROXY_HEARTBEAT_INTERVAL` | `30` | 心跳间隔(秒) |
| `--allowed-ports` | `AETHER_PROXY_ALLOWED_PORTS` | `80,443,8080,8443` | 允许代理的目标端口 | | `--allowed-ports` | `AETHER_PROXY_ALLOWED_PORTS` | `80,443,8080,8443` | 允许代理的目标端口 |
| `--timestamp-tolerance` | `AETHER_PROXY_TIMESTAMP_TOLERANCE` | `300` | HMAC 时间戳容差(秒) | | `--timestamp-tolerance` | `AETHER_PROXY_TIMESTAMP_TOLERANCE` | `300` | HMAC 时间戳容差(秒) |
| `--aether-request-timeout` | `AETHER_PROXY_AETHER_REQUEST_TIMEOUT` | `10` | Aether API 请求总超时(秒) |
| `--aether-connect-timeout` | `AETHER_PROXY_AETHER_CONNECT_TIMEOUT` | `10` | Aether API 建连超时(秒) |
| `--aether-pool-max-idle-per-host` | `AETHER_PROXY_AETHER_POOL_MAX_IDLE_PER_HOST` | `8` | Aether API 每 Host 最大空闲连接数 |
| `--aether-pool-idle-timeout` | `AETHER_PROXY_AETHER_POOL_IDLE_TIMEOUT` | `90` | Aether API 连接池空闲超时(秒) |
| `--aether-tcp-keepalive` | `AETHER_PROXY_AETHER_TCP_KEEPALIVE` | `60` | Aether API TCP keepalive0 关闭) |
| `--aether-tcp-nodelay` | `AETHER_PROXY_AETHER_TCP_NODELAY` | `true` | Aether API 启用 TCP_NODELAY |
| `--aether-http2` | `AETHER_PROXY_AETHER_HTTP2` | `true` | Aether API 启用 HTTP/2 |
| `--aether-retry-max-attempts` | `AETHER_PROXY_AETHER_RETRY_MAX_ATTEMPTS` | `3` | Aether API 最大重试次数(含首次) |
| `--aether-retry-base-delay-ms` | `AETHER_PROXY_AETHER_RETRY_BASE_DELAY_MS` | `200` | Aether API 重试基础延迟(毫秒) |
| `--aether-retry-max-delay-ms` | `AETHER_PROXY_AETHER_RETRY_MAX_DELAY_MS` | `2000` | Aether API 重试最大延迟(毫秒) |
| `--max-concurrent-connections` | `AETHER_PROXY_MAX_CONCURRENT_CONNECTIONS` | 自动估算 | 最大并发连接数(默认硬件估算) |
| `--connect-timeout` | `AETHER_PROXY_CONNECT_TIMEOUT` | `30` | CONNECT 上游建连超时(秒) |
| `--tls-handshake-timeout` | `AETHER_PROXY_TLS_HANDSHAKE_TIMEOUT` | `10` | TLS 握手超时(秒) |
| `--dns-cache-ttl` | `AETHER_PROXY_DNS_CACHE_TTL` | `60` | DNS 缓存 TTL |
| `--dns-cache-capacity` | `AETHER_PROXY_DNS_CACHE_CAPACITY` | `1024` | DNS 缓存容量(条目数) |
| `--delegate-connect-timeout` | `AETHER_PROXY_DELEGATE_CONNECT_TIMEOUT` | `30` | delegate 上游建连超时(秒) |
| `--delegate-pool-max-idle-per-host` | `AETHER_PROXY_DELEGATE_POOL_MAX_IDLE_PER_HOST` | `64` | delegate 每 Host 最大空闲连接数 |
| `--delegate-pool-idle-timeout` | `AETHER_PROXY_DELEGATE_POOL_IDLE_TIMEOUT` | `300` | delegate 连接池空闲超时(秒) |
| `--delegate-tcp-keepalive` | `AETHER_PROXY_DELEGATE_TCP_KEEPALIVE` | `60` | delegate TCP keepalive0 关闭) |
| `--delegate-tcp-nodelay` | `AETHER_PROXY_DELEGATE_TCP_NODELAY` | `true` | delegate 启用 TCP_NODELAY |
| `--log-level` | `AETHER_PROXY_LOG_LEVEL` | `info` | 日志级别 | | `--log-level` | `AETHER_PROXY_LOG_LEVEL` | `info` | 日志级别 |
| `--log-json` | `AETHER_PROXY_LOG_JSON` | `false` | JSON 格式日志 | | `--log-json` | `AETHER_PROXY_LOG_JSON` | `false` | JSON 格式日志 |
| `--enable-tls` | `AETHER_PROXY_ENABLE_TLS` | `true` | 启用 TLS | | `--enable-tls` | `AETHER_PROXY_ENABLE_TLS` | `true` | 启用 TLS |

View File

@@ -5,16 +5,17 @@
use std::sync::atomic::AtomicU64; use std::sync::atomic::AtomicU64;
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
use std::time::Duration;
use tokio::signal; use tokio::signal;
use tokio::sync::watch; use tokio::sync::{watch, Semaphore};
use tracing::{error, info}; use tracing::{error, info};
use crate::config::Config; use crate::config::Config;
use crate::net; use crate::net;
use crate::registration::client::AetherClient; use crate::registration::client::AetherClient;
use crate::runtime::{self, DynamicConfig}; use crate::runtime::{self, DynamicConfig};
use crate::state::AppState; use crate::state::{AppState, ProxyMetrics};
use crate::{hardware, proxy}; use crate::{hardware, proxy};
/// Run the full application lifecycle after config has been parsed. /// Run the full application lifecycle after config has been parsed.
@@ -61,6 +62,23 @@ pub async fn run(mut config: Config) -> anyhow::Result<()> {
// Collect hardware info (once at startup) // Collect hardware info (once at startup)
let hw_info = hardware::collect(); let hw_info = hardware::collect();
let max_connections_raw = config
.max_concurrent_connections
.unwrap_or(hw_info.estimated_max_concurrency)
.max(1);
let max_connections = usize::try_from(max_connections_raw).unwrap_or(usize::MAX);
info!(
max_connections = max_connections_raw,
"connection limit configured"
);
let connection_semaphore = Arc::new(Semaphore::new(max_connections));
let metrics = Arc::new(ProxyMetrics::new());
let dns_cache = Arc::new(proxy::target_filter::DnsCache::new(
Duration::from_secs(config.dns_cache_ttl_secs),
config.dns_cache_capacity,
));
// Register with Aether // Register with Aether
let aether_client = Arc::new(AetherClient::new(&config)); let aether_client = Arc::new(AetherClient::new(&config));
let node_id = aether_client let node_id = aether_client
@@ -82,10 +100,21 @@ pub async fn run(mut config: Config) -> anyhow::Result<()> {
// No overall timeout — SSE streams can last indefinitely. // No overall timeout — SSE streams can last indefinitely.
// Connect timeout limits connection establishment; Aether controls // Connect timeout limits connection establishment; Aether controls
// first-byte / idle timeouts on its own side. // first-byte / idle timeouts on its own side.
let delegate_client = reqwest::Client::builder() let mut delegate_builder = reqwest::Client::builder()
.connect_timeout(std::time::Duration::from_secs(30)) .connect_timeout(Duration::from_secs(config.delegate_connect_timeout_secs))
.pool_max_idle_per_host(20) .pool_max_idle_per_host(config.delegate_pool_max_idle_per_host)
.pool_idle_timeout(std::time::Duration::from_secs(90)) .pool_idle_timeout(Duration::from_secs(config.delegate_pool_idle_timeout_secs))
.tcp_nodelay(config.delegate_tcp_nodelay);
if config.delegate_tcp_keepalive_secs > 0 {
delegate_builder = delegate_builder.tcp_keepalive(Some(Duration::from_secs(
config.delegate_tcp_keepalive_secs,
)));
} else {
delegate_builder = delegate_builder.tcp_keepalive(None);
}
let delegate_client = delegate_builder
.build() .build()
.expect("failed to create delegate HTTP client"); .expect("failed to create delegate HTTP client");
@@ -101,6 +130,9 @@ pub async fn run(mut config: Config) -> anyhow::Result<()> {
tls_acceptor, tls_acceptor,
delegate_client, delegate_client,
active_connections: Arc::new(AtomicU64::new(0)), active_connections: Arc::new(AtomicU64::new(0)),
connection_semaphore,
dns_cache,
metrics,
}); });
// Shutdown signal channel // Shutdown signal channel

View File

@@ -125,6 +125,26 @@ mod tests {
heartbeat_interval: 30, heartbeat_interval: 30,
allowed_ports: vec![80, 443], allowed_ports: vec![80, 443],
timestamp_tolerance: 300, timestamp_tolerance: 300,
aether_request_timeout_secs: 10,
aether_connect_timeout_secs: 10,
aether_pool_max_idle_per_host: 8,
aether_pool_idle_timeout_secs: 90,
aether_tcp_keepalive_secs: 60,
aether_tcp_nodelay: true,
aether_http2: true,
aether_retry_max_attempts: 3,
aether_retry_base_delay_ms: 200,
aether_retry_max_delay_ms: 2000,
max_concurrent_connections: None,
connect_timeout_secs: 30,
tls_handshake_timeout_secs: 10,
dns_cache_ttl_secs: 60,
dns_cache_capacity: 1024,
delegate_connect_timeout_secs: 30,
delegate_pool_max_idle_per_host: 64,
delegate_pool_idle_timeout_secs: 300,
delegate_tcp_keepalive_secs: 60,
delegate_tcp_nodelay: true,
log_level: "info".to_string(), log_level: "info".to_string(),
log_json: false, log_json: false,
enable_tls: false, enable_tls: false,

View File

@@ -44,13 +44,146 @@ pub struct Config {
pub heartbeat_interval: u64, pub heartbeat_interval: u64,
/// Allowed destination ports (default: 80,443,8080,8443) /// Allowed destination ports (default: 80,443,8080,8443)
#[arg(long, env = "AETHER_PROXY_ALLOWED_PORTS", value_delimiter = ',', default_values_t = vec![80, 443, 8080, 8443])] #[arg(
long,
env = "AETHER_PROXY_ALLOWED_PORTS",
value_delimiter = ',',
default_values_t = vec![80, 443, 8080, 8443]
)]
pub allowed_ports: Vec<u16>, pub allowed_ports: Vec<u16>,
/// Timestamp tolerance window in seconds for HMAC validation /// Timestamp tolerance window in seconds for HMAC validation
#[arg(long, env = "AETHER_PROXY_TIMESTAMP_TOLERANCE", default_value_t = 300)] #[arg(long, env = "AETHER_PROXY_TIMESTAMP_TOLERANCE", default_value_t = 300)]
pub timestamp_tolerance: u64, pub timestamp_tolerance: u64,
/// Aether API request timeout in seconds
#[arg(
long,
env = "AETHER_PROXY_AETHER_REQUEST_TIMEOUT",
default_value_t = 10
)]
pub aether_request_timeout_secs: u64,
/// Aether API connect timeout in seconds
#[arg(
long,
env = "AETHER_PROXY_AETHER_CONNECT_TIMEOUT",
default_value_t = 10
)]
pub aether_connect_timeout_secs: u64,
/// Aether API max idle connections per host
#[arg(
long,
env = "AETHER_PROXY_AETHER_POOL_MAX_IDLE_PER_HOST",
default_value_t = 8
)]
pub aether_pool_max_idle_per_host: usize,
/// Aether API idle timeout in seconds
#[arg(
long,
env = "AETHER_PROXY_AETHER_POOL_IDLE_TIMEOUT",
default_value_t = 90
)]
pub aether_pool_idle_timeout_secs: u64,
/// Aether API TCP keepalive in seconds (0 disables)
#[arg(long, env = "AETHER_PROXY_AETHER_TCP_KEEPALIVE", default_value_t = 60)]
pub aether_tcp_keepalive_secs: u64,
/// Aether API TCP_NODELAY
#[arg(long, env = "AETHER_PROXY_AETHER_TCP_NODELAY", default_value_t = true)]
pub aether_tcp_nodelay: bool,
/// Enable HTTP/2 when talking to Aether API
#[arg(long, env = "AETHER_PROXY_AETHER_HTTP2", default_value_t = true)]
pub aether_http2: bool,
/// Aether API retry attempts (including initial)
#[arg(
long,
env = "AETHER_PROXY_AETHER_RETRY_MAX_ATTEMPTS",
default_value_t = 3
)]
pub aether_retry_max_attempts: u32,
/// Aether API retry base delay in milliseconds
#[arg(
long,
env = "AETHER_PROXY_AETHER_RETRY_BASE_DELAY_MS",
default_value_t = 200
)]
pub aether_retry_base_delay_ms: u64,
/// Aether API retry max delay in milliseconds
#[arg(
long,
env = "AETHER_PROXY_AETHER_RETRY_MAX_DELAY_MS",
default_value_t = 2000
)]
pub aether_retry_max_delay_ms: u64,
/// Maximum concurrent TCP connections (defaults to hardware estimate)
#[arg(long, env = "AETHER_PROXY_MAX_CONCURRENT_CONNECTIONS")]
pub max_concurrent_connections: Option<u64>,
/// Upstream TCP connect timeout in seconds for CONNECT tunnels
#[arg(long, env = "AETHER_PROXY_CONNECT_TIMEOUT", default_value_t = 30)]
pub connect_timeout_secs: u64,
/// TLS handshake timeout in seconds for incoming TLS connections
#[arg(long, env = "AETHER_PROXY_TLS_HANDSHAKE_TIMEOUT", default_value_t = 10)]
pub tls_handshake_timeout_secs: u64,
/// DNS cache TTL in seconds
#[arg(long, env = "AETHER_PROXY_DNS_CACHE_TTL", default_value_t = 60)]
pub dns_cache_ttl_secs: u64,
/// DNS cache capacity (entries)
#[arg(long, env = "AETHER_PROXY_DNS_CACHE_CAPACITY", default_value_t = 1024)]
pub dns_cache_capacity: usize,
/// Delegate HTTP client connect timeout in seconds
#[arg(
long,
env = "AETHER_PROXY_DELEGATE_CONNECT_TIMEOUT",
default_value_t = 30
)]
pub delegate_connect_timeout_secs: u64,
/// Delegate HTTP client max idle connections per host
#[arg(
long,
env = "AETHER_PROXY_DELEGATE_POOL_MAX_IDLE_PER_HOST",
default_value_t = 64
)]
pub delegate_pool_max_idle_per_host: usize,
/// Delegate HTTP client idle timeout in seconds
#[arg(
long,
env = "AETHER_PROXY_DELEGATE_POOL_IDLE_TIMEOUT",
default_value_t = 300
)]
pub delegate_pool_idle_timeout_secs: u64,
/// Delegate TCP keepalive in seconds (0 disables)
#[arg(
long,
env = "AETHER_PROXY_DELEGATE_TCP_KEEPALIVE",
default_value_t = 60
)]
pub delegate_tcp_keepalive_secs: u64,
/// Delegate TCP_NODELAY
#[arg(
long,
env = "AETHER_PROXY_DELEGATE_TCP_NODELAY",
default_value_t = true
)]
pub delegate_tcp_nodelay: bool,
/// Log level (trace, debug, info, warn, error) /// Log level (trace, debug, info, warn, error)
#[arg(long, env = "AETHER_PROXY_LOG_LEVEL", default_value = "info")] #[arg(long, env = "AETHER_PROXY_LOG_LEVEL", default_value = "info")]
pub log_level: String, pub log_level: String,
@@ -109,6 +242,46 @@ pub struct ConfigFile {
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub timestamp_tolerance: Option<u64>, pub timestamp_tolerance: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub aether_request_timeout_secs: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub aether_connect_timeout_secs: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub aether_pool_max_idle_per_host: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub aether_pool_idle_timeout_secs: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub aether_tcp_keepalive_secs: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub aether_tcp_nodelay: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub aether_http2: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub aether_retry_max_attempts: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub aether_retry_base_delay_ms: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub aether_retry_max_delay_ms: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub max_concurrent_connections: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub connect_timeout_secs: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tls_handshake_timeout_secs: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub dns_cache_ttl_secs: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub dns_cache_capacity: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub delegate_connect_timeout_secs: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub delegate_pool_max_idle_per_host: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub delegate_pool_idle_timeout_secs: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub delegate_tcp_keepalive_secs: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub delegate_tcp_nodelay: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub log_level: Option<String>, pub log_level: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub log_json: Option<bool>, pub log_json: Option<bool>,
@@ -168,6 +341,71 @@ impl ConfigFile {
set!("AETHER_PROXY_NODE_REGION", self.node_region); set!("AETHER_PROXY_NODE_REGION", self.node_region);
set!("AETHER_PROXY_HEARTBEAT_INTERVAL", self.heartbeat_interval); set!("AETHER_PROXY_HEARTBEAT_INTERVAL", self.heartbeat_interval);
set!("AETHER_PROXY_TIMESTAMP_TOLERANCE", self.timestamp_tolerance); set!("AETHER_PROXY_TIMESTAMP_TOLERANCE", self.timestamp_tolerance);
set!(
"AETHER_PROXY_AETHER_REQUEST_TIMEOUT",
self.aether_request_timeout_secs
);
set!(
"AETHER_PROXY_AETHER_CONNECT_TIMEOUT",
self.aether_connect_timeout_secs
);
set!(
"AETHER_PROXY_AETHER_POOL_MAX_IDLE_PER_HOST",
self.aether_pool_max_idle_per_host
);
set!(
"AETHER_PROXY_AETHER_POOL_IDLE_TIMEOUT",
self.aether_pool_idle_timeout_secs
);
set!(
"AETHER_PROXY_AETHER_TCP_KEEPALIVE",
self.aether_tcp_keepalive_secs
);
set!("AETHER_PROXY_AETHER_TCP_NODELAY", self.aether_tcp_nodelay);
set!("AETHER_PROXY_AETHER_HTTP2", self.aether_http2);
set!(
"AETHER_PROXY_AETHER_RETRY_MAX_ATTEMPTS",
self.aether_retry_max_attempts
);
set!(
"AETHER_PROXY_AETHER_RETRY_BASE_DELAY_MS",
self.aether_retry_base_delay_ms
);
set!(
"AETHER_PROXY_AETHER_RETRY_MAX_DELAY_MS",
self.aether_retry_max_delay_ms
);
set!(
"AETHER_PROXY_MAX_CONCURRENT_CONNECTIONS",
self.max_concurrent_connections
);
set!("AETHER_PROXY_CONNECT_TIMEOUT", self.connect_timeout_secs);
set!(
"AETHER_PROXY_TLS_HANDSHAKE_TIMEOUT",
self.tls_handshake_timeout_secs
);
set!("AETHER_PROXY_DNS_CACHE_TTL", self.dns_cache_ttl_secs);
set!("AETHER_PROXY_DNS_CACHE_CAPACITY", self.dns_cache_capacity);
set!(
"AETHER_PROXY_DELEGATE_CONNECT_TIMEOUT",
self.delegate_connect_timeout_secs
);
set!(
"AETHER_PROXY_DELEGATE_POOL_MAX_IDLE_PER_HOST",
self.delegate_pool_max_idle_per_host
);
set!(
"AETHER_PROXY_DELEGATE_POOL_IDLE_TIMEOUT",
self.delegate_pool_idle_timeout_secs
);
set!(
"AETHER_PROXY_DELEGATE_TCP_KEEPALIVE",
self.delegate_tcp_keepalive_secs
);
set!(
"AETHER_PROXY_DELEGATE_TCP_NODELAY",
self.delegate_tcp_nodelay
);
set!("AETHER_PROXY_LOG_LEVEL", self.log_level); set!("AETHER_PROXY_LOG_LEVEL", self.log_level);
set!("AETHER_PROXY_LOG_JSON", self.log_json); set!("AETHER_PROXY_LOG_JSON", self.log_json);
set!("AETHER_PROXY_ENABLE_TLS", self.enable_tls); set!("AETHER_PROXY_ENABLE_TLS", self.enable_tls);

View File

@@ -1,14 +1,16 @@
use std::collections::HashSet; use std::collections::HashSet;
use std::sync::Arc; use std::sync::Arc;
use std::time::Duration;
use hyper::body::Incoming; use hyper::body::Incoming;
use hyper::{Request, Response}; use hyper::{Request, Response};
use tokio::net::TcpStream; use tokio::net::TcpStream;
use tokio::time::timeout;
use tracing::{debug, warn}; use tracing::{debug, warn};
use crate::auth; use crate::auth;
use crate::config::Config; use crate::config::Config;
use crate::proxy::target_filter; use crate::proxy::target_filter::{self, DnsCache};
/// Handle HTTP CONNECT tunnel requests. /// Handle HTTP CONNECT tunnel requests.
/// ///
@@ -18,6 +20,7 @@ pub async fn handle_connect(
config: Arc<Config>, config: Arc<Config>,
allowed_ports: &HashSet<u16>, allowed_ports: &HashSet<u16>,
timestamp_tolerance: u64, timestamp_tolerance: u64,
dns_cache: &DnsCache,
) -> Response<http_body_util::Empty<bytes::Bytes>> { ) -> Response<http_body_util::Empty<bytes::Bytes>> {
// Extract Proxy-Authorization header // Extract Proxy-Authorization header
let proxy_auth = req let proxy_auth = req
@@ -44,30 +47,43 @@ pub async fn handle_connect(
let port = authority.port_u16().unwrap_or(443); let port = authority.port_u16().unwrap_or(443);
// Target filter: private IP + port whitelist // Target filter: private IP + port whitelist
let target_addr = match target_filter::validate_target(&host, port, allowed_ports).await { let target_addr =
Ok(addr) => addr, match target_filter::validate_target(&host, port, allowed_ports, dns_cache).await {
Err(e) => { Ok(addr) => addr,
warn!(host = %host, port, error = %e, "CONNECT target rejected"); Err(e) => {
return forbidden(&e.to_string()); warn!(host = %host, port, error = %e, "CONNECT target rejected");
} return forbidden(&e.to_string());
}; }
};
debug!(target = %target_addr, "CONNECT tunnel establishing"); debug!(target = %target_addr, "CONNECT tunnel establishing");
// Connect to target // Connect to target
let target_stream = match TcpStream::connect(target_addr).await { let connect_timeout = Duration::from_secs(config.connect_timeout_secs);
Ok(s) => s, let target_stream = match timeout(connect_timeout, TcpStream::connect(target_addr)).await {
Err(e) => { Ok(Ok(s)) => s,
Ok(Err(e)) => {
warn!(target = %target_addr, error = %e, "CONNECT target connection failed"); warn!(target = %target_addr, error = %e, "CONNECT target connection failed");
return bad_gateway(&e.to_string()); return bad_gateway(&e.to_string());
} }
Err(_) => {
warn!(target = %target_addr, "CONNECT target connection timeout");
return gateway_timeout("connect timeout");
}
}; };
if let Err(e) = target_stream.set_nodelay(true) {
debug!(target = %target_addr, error = %e, "failed to set TCP_NODELAY");
}
// Respond 200 and upgrade connection to raw TCP tunnel // Respond 200 and upgrade connection to raw TCP tunnel
let target_display = target_addr.to_string(); let target_display = target_addr.to_string();
// Reuse connect_timeout for upgrade: both are connection-phase operations
// and should complete within the same order of magnitude.
let upgrade_timeout = Duration::from_secs(config.connect_timeout_secs);
tokio::task::spawn(async move { tokio::task::spawn(async move {
match hyper::upgrade::on(req).await { match timeout(upgrade_timeout, hyper::upgrade::on(req)).await {
Ok(upgraded) => { Ok(Ok(upgraded)) => {
let mut upgraded = hyper_util::rt::TokioIo::new(upgraded); let mut upgraded = hyper_util::rt::TokioIo::new(upgraded);
let mut target = target_stream; let mut target = target_stream;
@@ -85,9 +101,12 @@ pub async fn handle_connect(
} }
} }
} }
Err(e) => { Ok(Err(e)) => {
warn!(target = %target_display, error = %e, "CONNECT upgrade failed"); warn!(target = %target_display, error = %e, "CONNECT upgrade failed");
} }
Err(_) => {
warn!(target = %target_display, "CONNECT upgrade timeout");
}
} }
}); });
@@ -133,3 +152,12 @@ fn bad_gateway(msg: &str) -> Response<http_body_util::Empty<bytes::Bytes>> {
.body(http_body_util::Empty::new()) .body(http_body_util::Empty::new())
.unwrap() .unwrap()
} }
fn gateway_timeout(msg: &str) -> Response<http_body_util::Empty<bytes::Bytes>> {
Response::builder()
.status(504)
.header("Content-Length", "0")
.header("X-Error", msg)
.body(http_body_util::Empty::new())
.unwrap()
}

View File

@@ -1,6 +1,7 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::collections::HashSet; use std::collections::HashSet;
use std::sync::Arc; use std::sync::Arc;
use std::time::Instant;
use futures_util::TryStreamExt; use futures_util::TryStreamExt;
use http_body_util::{BodyExt, Full, Limited, StreamBody}; use http_body_util::{BodyExt, Full, Limited, StreamBody};
@@ -13,7 +14,7 @@ use url::Url;
use super::BoxBody; use super::BoxBody;
use crate::auth; use crate::auth;
use crate::config::Config; use crate::config::Config;
use crate::proxy::target_filter; use crate::proxy::target_filter::{self, DnsCache};
/// Delegation request payload sent by Aether. /// Delegation request payload sent by Aether.
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
@@ -36,8 +37,11 @@ pub async fn handle_delegate(
config: Arc<Config>, config: Arc<Config>,
allowed_ports: &HashSet<u16>, allowed_ports: &HashSet<u16>,
timestamp_tolerance: u64, timestamp_tolerance: u64,
dns_cache: &DnsCache,
http_client: &reqwest::Client, http_client: &reqwest::Client,
) -> Response<BoxBody> { ) -> Response<BoxBody> {
let total_start = Instant::now();
// Authenticate via Authorization header (same HMAC scheme as Proxy-Authorization) // Authenticate via Authorization header (same HMAC scheme as Proxy-Authorization)
let auth_header = req let auth_header = req
.headers() .headers()
@@ -86,10 +90,12 @@ pub async fn handle_delegate(
let port = parsed_url.port_or_known_default().unwrap_or(443); let port = parsed_url.port_or_known_default().unwrap_or(443);
if let Err(e) = target_filter::validate_target(&host, port, allowed_ports).await { let dns_start = Instant::now();
if let Err(e) = target_filter::validate_target(&host, port, allowed_ports, dns_cache).await {
warn!(host = %host, port, error = %e, "delegate target rejected"); warn!(host = %host, port, error = %e, "delegate target rejected");
return error_response(403, "target_not_allowed", &e.to_string()); return error_response(403, "target_not_allowed", &e.to_string());
} }
let dns_ms = dns_start.elapsed().as_millis() as u64;
debug!( debug!(
method = %delegate_req.method, method = %delegate_req.method,
@@ -110,8 +116,8 @@ pub async fn handle_delegate(
// NOTE: We intentionally do NOT set a per-request timeout here. // NOTE: We intentionally do NOT set a per-request timeout here.
// reqwest's `.timeout()` caps the *entire* request including body streaming, // reqwest's `.timeout()` caps the *entire* request including body streaming,
// which would truncate long-lived SSE streams. The delegate_client already // which would truncate long-lived SSE streams. The delegate_client already
// has a 30s connect_timeout for connection establishment, and Aether controls // has a configured connect_timeout for connection establishment, and Aether controls
// first-byte / idle timeouts on its own side via asyncio. // first-byte / idle timeouts on its own side via asyncio.
// Set headers (skip `host` — reqwest sets it from the URL automatically, // Set headers (skip `host` — reqwest sets it from the URL automatically,
@@ -129,6 +135,7 @@ pub async fn handle_delegate(
} }
// Send upstream request // Send upstream request
let upstream_start = Instant::now();
let upstream_resp = match upstream_req.send().await { let upstream_resp = match upstream_req.send().await {
Ok(resp) => resp, Ok(resp) => resp,
Err(e) => { Err(e) => {
@@ -142,12 +149,29 @@ pub async fn handle_delegate(
return error_response(502, "upstream_connection_failed", &safe_detail); return error_response(502, "upstream_connection_failed", &safe_detail);
} }
}; };
let upstream_ms = upstream_start.elapsed().as_millis() as u64;
// Build response: pass through upstream status + headers, stream body back // Build response: pass through upstream status + headers, stream body back
let status = upstream_resp.status().as_u16(); let status = upstream_resp.status().as_u16();
let upstream_headers = upstream_resp.headers().clone(); let upstream_headers = upstream_resp.headers().clone();
debug!(url = %delegate_req.url, status, "delegate upstream response"); let total_ms = total_start.elapsed().as_millis() as u64;
debug!(
url = %delegate_req.url,
status,
dns_ms,
upstream_ms,
total_ms,
"delegate upstream response"
);
// Inject proxy timing header for Aether to parse
let timing = serde_json::json!({
"dns_ms": dns_ms,
"upstream_ms": upstream_ms,
"total_ms": total_ms,
});
// Stream the response body // Stream the response body
let body_stream = upstream_resp let body_stream = upstream_resp
@@ -161,10 +185,14 @@ pub async fn handle_delegate(
for (name, value) in upstream_headers.iter() { for (name, value) in upstream_headers.iter() {
builder = builder.header(name, value); builder = builder.header(name, value);
} }
builder = builder.header("X-Proxy-Timing", timing.to_string());
builder builder.body(stream_body).unwrap_or_else(|_| {
.body(stream_body) Response::builder()
.unwrap_or_else(|_| Response::builder().status(500).body(super::empty_box_body()).unwrap()) .status(500)
.body(super::empty_box_body())
.unwrap()
})
} }
// ── Sanitisation ───────────────────────────────────────────────────────────── // ── Sanitisation ─────────────────────────────────────────────────────────────

View File

@@ -1,6 +1,7 @@
use std::net::SocketAddr; use std::net::SocketAddr;
use std::sync::atomic::Ordering; use std::sync::atomic::Ordering;
use std::sync::Arc; use std::sync::Arc;
use std::time::{Duration, Instant};
use http_body_util::BodyExt; use http_body_util::BodyExt;
use hyper::body::Incoming; use hyper::body::Incoming;
@@ -11,6 +12,7 @@ use hyper::{Method, Request, Response};
use hyper_util::rt::TokioIo; use hyper_util::rt::TokioIo;
use tokio::net::TcpListener; use tokio::net::TcpListener;
use tokio::sync::watch; use tokio::sync::watch;
use tokio::time::timeout;
use tracing::{debug, info, warn}; use tracing::{debug, info, warn};
use crate::proxy::{connect, delegate, tls, BoxBody}; use crate::proxy::{connect, delegate, tls, BoxBody};
@@ -39,6 +41,8 @@ pub async fn run(
info!(addr = %addr, "proxy server listening (HTTP only)"); info!(addr = %addr, "proxy server listening (HTTP only)");
} }
let handshake_timeout = Duration::from_secs(state.config.tls_handshake_timeout_secs);
loop { loop {
tokio::select! { tokio::select! {
result = listener.accept() => { result = listener.accept() => {
@@ -52,15 +56,38 @@ pub async fn run(
debug!(peer = %peer_addr, "new connection"); debug!(peer = %peer_addr, "new connection");
if let Err(e) = stream.set_nodelay(true) {
debug!(peer = %peer_addr, error = %e, "failed to set TCP_NODELAY");
}
let permit = match state.connection_semaphore.clone().try_acquire_owned() {
Ok(permit) => permit,
Err(_) => {
warn!(peer = %peer_addr, "connection rejected: limit reached");
continue;
}
};
let state = Arc::clone(state); let state = Arc::clone(state);
state.active_connections.fetch_add(1, Ordering::Relaxed); state.active_connections.fetch_add(1, Ordering::Relaxed);
tokio::task::spawn(async move { tokio::task::spawn(async move {
let _permit = permit;
// Dual-stack: peek first byte to decide TLS vs plain HTTP // Dual-stack: peek first byte to decide TLS vs plain HTTP
if let Some(ref acceptor) = state.tls_acceptor { if let Some(ref acceptor) = state.tls_acceptor {
if tls::is_tls_client_hello(&stream).await { let is_tls = match timeout(handshake_timeout, tls::is_tls_client_hello(&stream)).await {
match acceptor.clone().accept(stream).await { Ok(v) => v,
Ok(tls_stream) => { Err(_) => {
debug!(peer = %peer_addr, "TLS detection timeout");
state.active_connections.fetch_sub(1, Ordering::Relaxed);
return;
}
};
if is_tls {
match timeout(handshake_timeout, acceptor.clone().accept(stream)).await {
Ok(Ok(tls_stream)) => {
debug!(peer = %peer_addr, "TLS handshake ok"); debug!(peer = %peer_addr, "TLS handshake ok");
serve_connection( serve_connection(
TokioIo::new(tls_stream), TokioIo::new(tls_stream),
@@ -69,9 +96,12 @@ pub async fn run(
) )
.await; .await;
} }
Err(e) => { Ok(Err(e)) => {
debug!(peer = %peer_addr, error = %e, "TLS handshake failed"); debug!(peer = %peer_addr, error = %e, "TLS handshake failed");
} }
Err(_) => {
debug!(peer = %peer_addr, "TLS handshake timeout");
}
} }
state.active_connections.fetch_sub(1, Ordering::Relaxed); state.active_connections.fetch_sub(1, Ordering::Relaxed);
return; return;
@@ -107,13 +137,18 @@ where
let config = Arc::clone(&state.config); let config = Arc::clone(&state.config);
let dynamic = Arc::clone(&state.dynamic); let dynamic = Arc::clone(&state.dynamic);
let delegate_client = state.delegate_client.clone(); let delegate_client = state.delegate_client.clone();
let dns_cache = Arc::clone(&state.dns_cache);
let metrics = Arc::clone(&state.metrics);
let service = service_fn(move |req: Request<Incoming>| { let service = service_fn(move |req: Request<Incoming>| {
let config = Arc::clone(&config); let config = Arc::clone(&config);
let dynamic = Arc::clone(&dynamic); let dynamic = Arc::clone(&dynamic);
let delegate_client = delegate_client.clone(); let delegate_client = delegate_client.clone();
let dns_cache = Arc::clone(&dns_cache);
let metrics = Arc::clone(&metrics);
async move { async move {
let start = Instant::now();
// Snapshot current dynamic values (may be updated by remote config) // Snapshot current dynamic values (may be updated by remote config)
let (allowed_ports, timestamp_tolerance) = { let (allowed_ports, timestamp_tolerance) = {
let d = dynamic.read().unwrap(); let d = dynamic.read().unwrap();
@@ -121,13 +156,20 @@ where
}; };
if req.method() == Method::CONNECT { if req.method() == Method::CONNECT {
let resp = let resp = connect::handle_connect(
connect::handle_connect(req, config, &allowed_ports, timestamp_tolerance).await; req,
config,
&allowed_ports,
timestamp_tolerance,
dns_cache.as_ref(),
)
.await;
let resp = resp.map(|_| -> BoxBody { let resp = resp.map(|_| -> BoxBody {
http_body_util::Empty::new() http_body_util::Empty::new()
.map_err(|e| -> Box<dyn std::error::Error + Send + Sync> { match e {} }) .map_err(|e| -> Box<dyn std::error::Error + Send + Sync> { match e {} })
.boxed() .boxed()
}); });
metrics.record_request(start.elapsed());
Ok::<_, hyper::Error>(resp) Ok::<_, hyper::Error>(resp)
} else if req.uri().path() == "/_aether/delegate" && req.method() == hyper::Method::POST } else if req.uri().path() == "/_aether/delegate" && req.method() == hyper::Method::POST
{ {
@@ -136,19 +178,23 @@ where
config, config,
&allowed_ports, &allowed_ports,
timestamp_tolerance, timestamp_tolerance,
dns_cache.as_ref(),
&delegate_client, &delegate_client,
) )
.await; .await;
metrics.record_request(start.elapsed());
Ok(resp) Ok(resp)
} else { } else {
// Only CONNECT tunnels and /_aether/delegate are supported; // Only CONNECT tunnels and /_aether/delegate are supported;
// plain HTTP forward proxy was removed (all API traffic is HTTPS). // plain HTTP forward proxy was removed (all API traffic is HTTPS).
Ok(Response::builder() let resp = Response::builder()
.status(405) .status(405)
.header("Allow", "CONNECT") .header("Allow", "CONNECT")
.header("Content-Length", "0") .header("Content-Length", "0")
.body(crate::proxy::empty_box_body()) .body(crate::proxy::empty_box_body())
.unwrap()) .unwrap();
metrics.record_request(start.elapsed());
Ok(resp)
} }
} }
}); });

View File

@@ -1,5 +1,8 @@
use std::collections::HashSet; use std::collections::{HashMap, HashSet};
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use std::time::{Duration, Instant};
use tokio::sync::RwLock;
/// Check if an IP address belongs to a private/reserved network. /// Check if an IP address belongs to a private/reserved network.
fn is_private_ip(ip: &IpAddr) -> bool { fn is_private_ip(ip: &IpAddr) -> bool {
@@ -80,6 +83,85 @@ impl std::fmt::Display for FilterError {
} }
} }
struct DnsCacheEntry {
addr: SocketAddr,
expires_at: Instant,
inserted_at: Instant,
}
/// Lightweight DNS cache with TTL + capacity bounds.
pub struct DnsCache {
ttl: Duration,
capacity: usize,
entries: RwLock<HashMap<String, DnsCacheEntry>>,
}
impl DnsCache {
pub fn new(ttl: Duration, capacity: usize) -> Self {
Self {
ttl,
capacity,
entries: RwLock::new(HashMap::new()),
}
}
pub async fn get(&self, host: &str, port: u16) -> Option<SocketAddr> {
if self.capacity == 0 || self.ttl.is_zero() {
return None;
}
let key = Self::key(host, port);
let now = Instant::now();
// Fast path: read lock for cache hit
{
let entries = self.entries.read().await;
match entries.get(&key) {
Some(entry) if entry.expires_at > now => return Some(entry.addr),
None => return None,
Some(_) => {} // expired, fall through to evict
}
}
// Slow path: write lock to remove expired entry
let mut entries = self.entries.write().await;
entries.remove(&key);
None
}
pub async fn insert(&self, host: &str, port: u16, addr: SocketAddr) {
if self.capacity == 0 || self.ttl.is_zero() {
return;
}
let key = Self::key(host, port);
let now = Instant::now();
let mut entries = self.entries.write().await;
entries.retain(|_, entry| entry.expires_at > now);
while entries.len() >= self.capacity {
let oldest_key = entries
.iter()
.min_by_key(|(_, entry)| entry.inserted_at)
.map(|(key, _)| key.clone());
if let Some(key) = oldest_key {
entries.remove(&key);
} else {
break;
}
}
entries.insert(
key,
DnsCacheEntry {
addr,
expires_at: now + self.ttl,
inserted_at: now,
},
);
}
fn key(host: &str, port: u16) -> String {
format!("{}:{}", host, port)
}
}
/// Validate that the target host:port is allowed. /// Validate that the target host:port is allowed.
/// ///
/// Uses async DNS resolution (via `tokio::net::lookup_host`) to avoid /// Uses async DNS resolution (via `tokio::net::lookup_host`) to avoid
@@ -90,6 +172,7 @@ pub async fn validate_target(
host: &str, host: &str,
port: u16, port: u16,
allowed_ports: &HashSet<u16>, allowed_ports: &HashSet<u16>,
dns_cache: &DnsCache,
) -> Result<SocketAddr, FilterError> { ) -> Result<SocketAddr, FilterError> {
// Port whitelist check // Port whitelist check
if !allowed_ports.contains(&port) { if !allowed_ports.contains(&port) {
@@ -104,6 +187,10 @@ pub async fn validate_target(
return Ok(SocketAddr::new(ip, port)); return Ok(SocketAddr::new(ip, port));
} }
if let Some(addr) = dns_cache.get(host, port).await {
return Ok(addr);
}
// Async DNS resolution with private IP check (DNS rebinding protection) // Async DNS resolution with private IP check (DNS rebinding protection)
let addr_str = format!("{}:{}", host, port); let addr_str = format!("{}:{}", host, port);
let addrs: Vec<SocketAddr> = tokio::net::lookup_host(&addr_str) let addrs: Vec<SocketAddr> = tokio::net::lookup_host(&addr_str)
@@ -123,7 +210,9 @@ pub async fn validate_target(
} }
// Return the first valid address // Return the first valid address
Ok(addrs[0]) let selected = addrs[0];
dns_cache.insert(host, port, selected).await;
Ok(selected)
} }
#[cfg(test)] #[cfg(test)]
@@ -134,6 +223,10 @@ mod tests {
[80, 443, 8080, 8443].into_iter().collect() [80, 443, 8080, 8443].into_iter().collect()
} }
fn cache() -> DnsCache {
DnsCache::new(Duration::from_secs(60), 128)
}
#[test] #[test]
fn test_private_ipv4() { fn test_private_ipv4() {
assert!(is_private_ip(&IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)))); assert!(is_private_ip(&IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1))));
@@ -162,19 +255,22 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_port_not_allowed() { async fn test_port_not_allowed() {
let result = validate_target("8.8.8.8", 22, &ports()).await; let cache = cache();
let result = validate_target("8.8.8.8", 22, &ports(), &cache).await;
assert!(matches!(result, Err(FilterError::PortNotAllowed(22)))); assert!(matches!(result, Err(FilterError::PortNotAllowed(22))));
} }
#[tokio::test] #[tokio::test]
async fn test_private_ip_blocked() { async fn test_private_ip_blocked() {
let result = validate_target("127.0.0.1", 80, &ports()).await; let cache = cache();
let result = validate_target("127.0.0.1", 80, &ports(), &cache).await;
assert!(matches!(result, Err(FilterError::PrivateIp(_)))); assert!(matches!(result, Err(FilterError::PrivateIp(_))));
} }
#[tokio::test] #[tokio::test]
async fn test_public_ip_allowed() { async fn test_public_ip_allowed() {
let result = validate_target("8.8.8.8", 443, &ports()).await; let cache = cache();
let result = validate_target("8.8.8.8", 443, &ports(), &cache).await;
assert!(result.is_ok()); assert!(result.is_ok());
} }
} }

View File

@@ -9,6 +9,8 @@ use sha2::{Digest, Sha256};
use tokio_rustls::TlsAcceptor; use tokio_rustls::TlsAcceptor;
use tracing::{info, warn}; use tracing::{info, warn};
const SESSION_CACHE_SIZE: usize = 1024;
/// Generate a self-signed certificate if the files do not already exist. /// Generate a self-signed certificate if the files do not already exist.
/// ///
/// The certificate includes SANs: `localhost` and `aether-proxy`. /// The certificate includes SANs: `localhost` and `aether-proxy`.
@@ -73,10 +75,21 @@ pub fn build_tls_acceptor(cert_path: &Path, key_path: &Path) -> anyhow::Result<T
rustls_pemfile::private_key(&mut BufReader::new(key_file))? rustls_pemfile::private_key(&mut BufReader::new(key_file))?
.ok_or_else(|| anyhow::anyhow!("no private key found in {}", key_path.display()))?; .ok_or_else(|| anyhow::anyhow!("no private key found in {}", key_path.display()))?;
let config = rustls::ServerConfig::builder() let mut config = rustls::ServerConfig::builder()
.with_no_client_auth() .with_no_client_auth()
.with_single_cert(certs, key)?; .with_single_cert(certs, key)?;
config.alpn_protocols = vec![b"http/1.1".to_vec()];
config.session_storage = rustls::server::ServerSessionMemoryCache::new(SESSION_CACHE_SIZE);
match rustls::crypto::ring::Ticketer::new() {
Ok(ticketer) => {
config.ticketer = ticketer;
}
Err(e) => {
warn!(error = %e, "failed to init TLS ticketer; tickets disabled");
}
}
Ok(TlsAcceptor::from(Arc::new(config))) Ok(TlsAcceptor::from(Arc::new(config)))
} }

View File

@@ -1,5 +1,8 @@
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use reqwest::{Client, StatusCode}; use reqwest::{Client, StatusCode};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tokio::time::sleep;
use tracing::{debug, error, info, warn}; use tracing::{debug, error, info, warn};
use crate::config::Config; use crate::config::Config;
@@ -100,19 +103,44 @@ pub struct AetherClient {
http: Client, http: Client,
base_url: String, base_url: String,
token: String, token: String,
retry_max_attempts: u32,
retry_base_delay: Duration,
retry_max_delay: Duration,
} }
impl AetherClient { impl AetherClient {
pub fn new(config: &Config) -> Self { pub fn new(config: &Config) -> Self {
let http = Client::builder() let mut builder = Client::builder()
.timeout(std::time::Duration::from_secs(10)) .timeout(Duration::from_secs(config.aether_request_timeout_secs))
.build() .connect_timeout(Duration::from_secs(config.aether_connect_timeout_secs))
.expect("failed to create HTTP client"); .pool_max_idle_per_host(config.aether_pool_max_idle_per_host)
.pool_idle_timeout(Duration::from_secs(config.aether_pool_idle_timeout_secs))
.tcp_nodelay(config.aether_tcp_nodelay);
if config.aether_tcp_keepalive_secs > 0 {
builder =
builder.tcp_keepalive(Some(Duration::from_secs(config.aether_tcp_keepalive_secs)));
} else {
builder = builder.tcp_keepalive(None);
}
if config.aether_http2 {
builder = builder.http2_adaptive_window(true);
}
let http = builder.build().expect("failed to create HTTP client");
let retry_base_delay = Duration::from_millis(config.aether_retry_base_delay_ms);
let retry_max_delay =
Duration::from_millis(config.aether_retry_max_delay_ms).max(retry_base_delay);
Self { Self {
http, http,
base_url: config.aether_url.trim_end_matches('/').to_string(), base_url: config.aether_url.trim_end_matches('/').to_string(),
token: config.management_token.clone(), token: config.management_token.clone(),
retry_max_attempts: config.aether_retry_max_attempts.max(1),
retry_base_delay,
retry_max_delay,
} }
} }
@@ -149,11 +177,15 @@ impl AetherClient {
); );
let resp = self let resp = self
.http .send_with_retry(
.post(&url) || {
.header("Authorization", format!("Bearer {}", self.token)) self.http
.json(&body) .post(&url)
.send() .header("Authorization", format!("Bearer {}", self.token))
.json(&body)
},
"register",
)
.await?; .await?;
let status = resp.status(); let status = resp.status();
@@ -190,11 +222,15 @@ impl AetherClient {
debug!(node_id = %node_id, "sending heartbeat"); debug!(node_id = %node_id, "sending heartbeat");
let resp = self let resp = self
.http .send_with_retry(
.post(&url) || {
.header("Authorization", format!("Bearer {}", self.token)) self.http
.json(&body) .post(&url)
.send() .header("Authorization", format!("Bearer {}", self.token))
.json(&body)
},
"heartbeat",
)
.await .await
.map_err(|e| HeartbeatError::Other(e.into()))?; .map_err(|e| HeartbeatError::Other(e.into()))?;
@@ -247,11 +283,15 @@ impl AetherClient {
info!(node_id = %node_id, "unregistering from Aether"); info!(node_id = %node_id, "unregistering from Aether");
let resp = self let resp = self
.http .send_with_retry(
.post(&url) || {
.header("Authorization", format!("Bearer {}", self.token)) self.http
.json(&body) .post(&url)
.send() .header("Authorization", format!("Bearer {}", self.token))
.json(&body)
},
"unregister",
)
.await; .await;
match resp { match resp {
@@ -271,4 +311,75 @@ impl AetherClient {
} }
} }
} }
async fn send_with_retry<F>(
&self,
mut make_req: F,
label: &str,
) -> Result<reqwest::Response, reqwest::Error>
where
F: FnMut() -> reqwest::RequestBuilder,
{
let mut attempt: u32 = 0;
let mut delay = self.retry_base_delay;
loop {
attempt = attempt.saturating_add(1);
let resp = make_req().send().await;
match resp {
Ok(resp) => {
if should_retry_status(resp.status()) && attempt < self.retry_max_attempts {
let sleep_for = jitter_delay(delay);
debug!(
attempt,
status = %resp.status(),
sleep_ms = sleep_for.as_millis(),
label,
"Aether request retrying"
);
sleep(sleep_for).await;
let next_delay = delay.checked_mul(2).unwrap_or(self.retry_max_delay);
delay = std::cmp::min(next_delay, self.retry_max_delay);
continue;
}
return Ok(resp);
}
Err(e) => {
if attempt < self.retry_max_attempts {
let sleep_for = jitter_delay(delay);
debug!(
attempt,
error = %e,
sleep_ms = sleep_for.as_millis(),
label,
"Aether request retrying"
);
sleep(sleep_for).await;
let next_delay = delay.checked_mul(2).unwrap_or(self.retry_max_delay);
delay = std::cmp::min(next_delay, self.retry_max_delay);
continue;
}
return Err(e);
}
}
}
}
}
fn should_retry_status(status: StatusCode) -> bool {
status.is_server_error()
|| status == StatusCode::TOO_MANY_REQUESTS
|| status == StatusCode::REQUEST_TIMEOUT
}
fn jitter_delay(base: Duration) -> Duration {
if base.is_zero() {
return base;
}
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.subsec_nanos() as u64)
.unwrap_or(0);
let jitter_ms = nanos % 100;
base + Duration::from_millis(jitter_ms)
} }

View File

@@ -34,9 +34,24 @@ pub async fn run(state: &Arc<AppState>, mut shutdown_rx: watch::Receiver<bool>)
let current_node_id = state.node_id.read().unwrap().clone(); let current_node_id = state.node_id.read().unwrap().clone();
let active_conns = state.active_connections.load(Ordering::Relaxed) as i64; let active_conns = state.active_connections.load(Ordering::Relaxed) as i64;
// Swap-and-reset: report incremental metrics since last heartbeat
let interval_requests = state.metrics.total_requests.swap(0, Ordering::Relaxed);
let interval_latency_ns = state.metrics.total_latency_ns.swap(0, Ordering::Relaxed);
let interval_requests_i64 = i64::try_from(interval_requests).unwrap_or(i64::MAX);
let avg_latency_ms = if interval_requests > 0 {
Some(interval_latency_ns as f64 / interval_requests as f64 / 1_000_000.0)
} else {
None
};
match state match state
.aether_client .aether_client
.heartbeat(&current_node_id, Some(active_conns), None, None) .heartbeat(
&current_node_id,
Some(active_conns),
Some(interval_requests_i64),
avg_latency_ms,
)
.await .await
{ {
Ok(result) => { Ok(result) => {

View File

@@ -209,6 +209,26 @@ impl App {
heartbeat_interval: None, heartbeat_interval: None,
allowed_ports: None, allowed_ports: None,
timestamp_tolerance: None, timestamp_tolerance: None,
aether_request_timeout_secs: None,
aether_connect_timeout_secs: None,
aether_pool_max_idle_per_host: None,
aether_pool_idle_timeout_secs: None,
aether_tcp_keepalive_secs: None,
aether_tcp_nodelay: None,
aether_http2: None,
aether_retry_max_attempts: None,
aether_retry_base_delay_ms: None,
aether_retry_max_delay_ms: None,
max_concurrent_connections: None,
connect_timeout_secs: None,
tls_handshake_timeout_secs: None,
dns_cache_ttl_secs: None,
dns_cache_capacity: None,
delegate_connect_timeout_secs: None,
delegate_pool_max_idle_per_host: None,
delegate_pool_idle_timeout_secs: None,
delegate_tcp_keepalive_secs: None,
delegate_tcp_nodelay: None,
log_level: get("log_level"), log_level: get("log_level"),
log_json: get("log_json").and_then(|v| v.parse().ok()), log_json: get("log_json").and_then(|v| v.parse().ok()),
enable_tls: None, enable_tls: None,

View File

@@ -3,13 +3,16 @@
//! Consolidates the multiple `Arc<...>` parameters that were previously //! Consolidates the multiple `Arc<...>` parameters that were previously
//! threaded individually through proxy server, heartbeat, and handlers. //! threaded individually through proxy server, heartbeat, and handlers.
use std::sync::atomic::AtomicU64; use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
use std::time::Duration;
use tokio::sync::Semaphore;
use tokio_rustls::TlsAcceptor; use tokio_rustls::TlsAcceptor;
use crate::config::Config; use crate::config::Config;
use crate::hardware::HardwareInfo; use crate::hardware::HardwareInfo;
use crate::proxy::target_filter::DnsCache;
use crate::registration::client::AetherClient; use crate::registration::client::AetherClient;
use crate::runtime::SharedDynamicConfig; use crate::runtime::SharedDynamicConfig;
@@ -27,4 +30,31 @@ pub struct AppState {
pub delegate_client: reqwest::Client, pub delegate_client: reqwest::Client,
/// Active connection count for metrics reporting. /// Active connection count for metrics reporting.
pub active_connections: Arc<AtomicU64>, pub active_connections: Arc<AtomicU64>,
/// Connection concurrency limiter.
pub connection_semaphore: Arc<Semaphore>,
/// DNS cache for upstream target resolution.
pub dns_cache: Arc<DnsCache>,
/// Request/latency metrics for heartbeat.
pub metrics: Arc<ProxyMetrics>,
}
/// Aggregate metrics for reporting to Aether.
pub struct ProxyMetrics {
pub total_requests: AtomicU64,
pub total_latency_ns: AtomicU64,
}
impl ProxyMetrics {
pub fn new() -> Self {
Self {
total_requests: AtomicU64::new(0),
total_latency_ns: AtomicU64::new(0),
}
}
pub fn record_request(&self, elapsed: Duration) {
let nanos = u64::try_from(elapsed.as_nanos()).unwrap_or(u64::MAX);
self.total_requests.fetch_add(1, Ordering::Relaxed);
self.total_latency_ns.fetch_add(nanos, Ordering::Relaxed);
}
} }

View File

@@ -642,14 +642,13 @@
</SelectContent> </SelectContent>
</Select> </Select>
<Button <Button
variant="ghost" variant="outline"
size="icon" size="sm"
class="h-7 w-7 text-primary hover:text-primary" class="h-7 px-3"
title="添加"
:disabled="!newEndpoint.api_format || (!newEndpoint.base_url?.trim() && !provider?.website?.trim()) || addingEndpoint" :disabled="!newEndpoint.api_format || (!newEndpoint.base_url?.trim() && !provider?.website?.trim()) || addingEndpoint"
@click="handleAddEndpoint" @click="handleAddEndpoint"
> >
<Plus class="w-3.5 h-3.5" /> 添加
</Button> </Button>
</div> </div>
<!-- 卡片内容URL 配置 --> <!-- 卡片内容URL 配置 -->

View File

@@ -213,6 +213,10 @@
v-if="currentAttempt.extra_data.proxy.ttfb_ms != null" v-if="currentAttempt.extra_data.proxy.ttfb_ms != null"
class="text-xs text-muted-foreground ml-1" class="text-xs text-muted-foreground ml-1"
>{{ formatLatency(currentAttempt.extra_data.proxy.ttfb_ms) }}</span> >{{ formatLatency(currentAttempt.extra_data.proxy.ttfb_ms) }}</span>
<span
v-if="currentAttempt.extra_data.proxy.timing"
class="text-xs text-muted-foreground ml-1"
>(DNS {{ formatLatency(currentAttempt.extra_data.proxy.timing.dns_ms) }} / 上游 {{ formatLatency(currentAttempt.extra_data.proxy.timing.upstream_ms) }})</span>
</span> </span>
</div> </div>
<div <div

View File

@@ -40,7 +40,11 @@ from src.api.handlers.base.base_handler import (
from src.api.handlers.base.parsers import get_parser_for_format from src.api.handlers.base.parsers import get_parser_for_format
from src.api.handlers.base.request_builder import PassthroughRequestBuilder, get_provider_auth from src.api.handlers.base.request_builder import PassthroughRequestBuilder, get_provider_auth
from src.api.handlers.base.response_parser import ResponseParser from src.api.handlers.base.response_parser import ResponseParser
from src.api.handlers.base.stream_context import StreamContext, is_format_converted from src.api.handlers.base.stream_context import (
StreamContext,
extract_proxy_timing,
is_format_converted,
)
from src.api.handlers.base.stream_processor import StreamProcessor from src.api.handlers.base.stream_processor import StreamProcessor
from src.api.handlers.base.stream_telemetry import StreamTelemetryRecorder from src.api.handlers.base.stream_telemetry import StreamTelemetryRecorder
from src.api.handlers.base.upstream_stream_bridge import ( from src.api.handlers.base.upstream_stream_bridge import (
@@ -975,6 +979,7 @@ class ChatHandlerBase(BaseMessageHandler, ABC):
ctx.status_code = resp.status_code ctx.status_code = resp.status_code
ctx.response_headers = dict(resp.headers) ctx.response_headers = dict(resp.headers)
ctx.set_proxy_timing(ctx.response_headers)
if envelope: if envelope:
envelope.on_http_status(base_url=ctx.selected_base_url, status_code=ctx.status_code) envelope.on_http_status(base_url=ctx.selected_base_url, status_code=ctx.status_code)
@@ -1005,6 +1010,7 @@ class ChatHandlerBase(BaseMessageHandler, ABC):
resp = await http_client.post(**_pkw) resp = await http_client.post(**_pkw)
ctx.status_code = resp.status_code ctx.status_code = resp.status_code
ctx.response_headers = dict(resp.headers) ctx.response_headers = dict(resp.headers)
ctx.set_proxy_timing(ctx.response_headers)
if envelope: if envelope:
envelope.on_http_status( envelope.on_http_status(
base_url=ctx.selected_base_url, status_code=ctx.status_code base_url=ctx.selected_base_url, status_code=ctx.status_code
@@ -1168,6 +1174,7 @@ class ChatHandlerBase(BaseMessageHandler, ABC):
ctx.status_code = stream_response.status_code ctx.status_code = stream_response.status_code
ctx.response_headers = dict(stream_response.headers) ctx.response_headers = dict(stream_response.headers)
ctx.set_proxy_timing(ctx.response_headers)
if envelope: if envelope:
envelope.on_http_status( envelope.on_http_status(
base_url=ctx.selected_base_url, base_url=ctx.selected_base_url,
@@ -1643,6 +1650,7 @@ class ChatHandlerBase(BaseMessageHandler, ABC):
status_code = stream_resp.status_code status_code = stream_resp.status_code
response_headers = dict(stream_resp.headers) response_headers = dict(stream_resp.headers)
extract_proxy_timing(sync_proxy_info, response_headers)
if envelope: if envelope:
envelope.on_http_status( envelope.on_http_status(
@@ -1695,6 +1703,7 @@ class ChatHandlerBase(BaseMessageHandler, ABC):
status_code = resp.status_code status_code = resp.status_code
response_headers = dict(resp.headers) response_headers = dict(resp.headers)
extract_proxy_timing(sync_proxy_info, response_headers)
if envelope: if envelope:
envelope.on_http_status(base_url=selected_base_url_cached, status_code=status_code) envelope.on_http_status(base_url=selected_base_url_cached, status_code=status_code)

View File

@@ -43,7 +43,11 @@ from src.api.handlers.base.request_builder import PassthroughRequestBuilder, get
from src.api.handlers.base.response_parser import ( from src.api.handlers.base.response_parser import (
ResponseParser, ResponseParser,
) )
from src.api.handlers.base.stream_context import StreamContext, is_format_converted from src.api.handlers.base.stream_context import (
StreamContext,
extract_proxy_timing,
is_format_converted,
)
from src.api.handlers.base.upstream_stream_bridge import ( from src.api.handlers.base.upstream_stream_bridge import (
aggregate_upstream_stream_to_internal_response, aggregate_upstream_stream_to_internal_response,
) )
@@ -90,7 +94,7 @@ from src.utils.sse_parser import SSEEventParser
from src.utils.timeout import read_first_chunk_with_ttfb_timeout from src.utils.timeout import read_first_chunk_with_ttfb_timeout
# ============================================================================== # ==============================================================================
# SSE 行解析辅助函数 # 辅助函数
# ============================================================================== # ==============================================================================
@@ -946,6 +950,7 @@ class CliMessageHandlerBase(BaseMessageHandler):
ctx.status_code = resp.status_code ctx.status_code = resp.status_code
ctx.response_headers = dict(resp.headers) ctx.response_headers = dict(resp.headers)
ctx.set_proxy_timing(ctx.response_headers)
if envelope: if envelope:
envelope.on_http_status(base_url=ctx.selected_base_url, status_code=ctx.status_code) envelope.on_http_status(base_url=ctx.selected_base_url, status_code=ctx.status_code)
@@ -978,6 +983,7 @@ class CliMessageHandlerBase(BaseMessageHandler):
ctx.set_ttfb_ms(int((time.monotonic() - _connect_start) * 1000)) ctx.set_ttfb_ms(int((time.monotonic() - _connect_start) * 1000))
ctx.status_code = resp.status_code ctx.status_code = resp.status_code
ctx.response_headers = dict(resp.headers) ctx.response_headers = dict(resp.headers)
ctx.set_proxy_timing(ctx.response_headers)
if envelope: if envelope:
envelope.on_http_status( envelope.on_http_status(
base_url=ctx.selected_base_url, status_code=ctx.status_code base_url=ctx.selected_base_url, status_code=ctx.status_code
@@ -1142,6 +1148,7 @@ class CliMessageHandlerBase(BaseMessageHandler):
ctx.status_code = stream_response.status_code ctx.status_code = stream_response.status_code
ctx.response_headers = dict(stream_response.headers) ctx.response_headers = dict(stream_response.headers)
ctx.set_proxy_timing(ctx.response_headers)
logger.debug(f" └─ 收到响应: status={stream_response.status_code}") logger.debug(f" └─ 收到响应: status={stream_response.status_code}")
@@ -3079,6 +3086,7 @@ class CliMessageHandlerBase(BaseMessageHandler):
status_code = stream_resp.status_code status_code = stream_resp.status_code
response_headers = dict(stream_resp.headers) response_headers = dict(stream_resp.headers)
extract_proxy_timing(sync_proxy_info, response_headers)
if envelope: if envelope:
envelope.on_http_status( envelope.on_http_status(
@@ -3131,6 +3139,7 @@ class CliMessageHandlerBase(BaseMessageHandler):
status_code = resp.status_code status_code = resp.status_code
response_headers = dict(resp.headers) response_headers = dict(resp.headers)
extract_proxy_timing(sync_proxy_info, response_headers)
if envelope: if envelope:
envelope.on_http_status(base_url=selected_base_url_cached, status_code=status_code) envelope.on_http_status(base_url=selected_base_url_cached, status_code=status_code)

View File

@@ -10,6 +10,7 @@
from __future__ import annotations from __future__ import annotations
import json
import time import time
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any
@@ -18,6 +19,21 @@ if TYPE_CHECKING:
from src.core.api_format.conversion.stream_state import StreamState from src.core.api_format.conversion.stream_state import StreamState
def extract_proxy_timing(proxy_info: dict[str, Any] | None, headers: dict[str, str]) -> None:
"""从响应头中提取代理分阶段耗时X-Proxy-Timing写入 proxy_info"""
if proxy_info is None:
return
timing_raw = headers.get("x-proxy-timing")
if not timing_raw:
return
try:
timing = json.loads(timing_raw)
if isinstance(timing, dict):
proxy_info["timing"] = timing
except (json.JSONDecodeError, TypeError):
pass
def is_format_converted( def is_format_converted(
provider_api_format: str | None, provider_api_format: str | None,
client_api_format: str | None, client_api_format: str | None,
@@ -272,6 +288,10 @@ class StreamContext:
if self.proxy_info is not None: if self.proxy_info is not None:
self.proxy_info["ttfb_ms"] = ms self.proxy_info["ttfb_ms"] = ms
def set_proxy_timing(self, headers: dict[str, str]) -> None:
"""从代理响应头中提取分阶段耗时信息X-Proxy-Timing"""
extract_proxy_timing(self.proxy_info, headers)
def build_response_body(self, response_time_ms: int) -> dict[str, Any]: def build_response_body(self, response_time_ms: int) -> dict[str, Any]:
""" """
构建响应体元数据 构建响应体元数据