i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-03 01:40:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(proxy): improve tunnel throughput and observability

Browse Source
This commit is contained in:
fawney19
2026-05-19 23:49:36 +08:00
parent 57655bdb25
commit f5deed8709
30 changed files with 1306 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
crates/aether-testkit/src/bin/capacity_curve_baseline.rs
View File

@@ -7,9 +7,9 @@ use aether_contracts::{ExecutionPlan, ExecutionTimeouts, RequestBody};
use aether_gateway::tunnel_protocol as protocol;
use aether_testkit::{
fetch_prometheus_samples, find_metric_value_u64, init_test_runtime_for, run_http_load_probe,
ExecutionRuntimeHarness, ExecutionRuntimeHarnessConfig, GatewayHarness, GatewayHarnessConfig,
HttpLoadProbeConfig, HttpLoadProbeResponseMode, HttpLoadProbeResult, SpawnedServer,
TunnelHarness, TunnelHarnessConfig,
BenchmarkRuntimeSnapshot, ExecutionRuntimeHarness, ExecutionRuntimeHarnessConfig,
GatewayHarness, GatewayHarnessConfig, HttpLoadProbeConfig, HttpLoadProbeResponseMode,
HttpLoadProbeResult, SpawnedServer, TunnelHarness, TunnelHarnessConfig,
};
use axum::body::{to_bytes, Body, Bytes};
use axum::http::StatusCode;
@@ -84,9 +84,11 @@ struct CapacityCurvePointResult {
throughput_rps: u64,
p50_ms: u64,
p95_ms: u64,
p99_ms: u64,
max_ms: u64,
mean_ms: u64,
metrics: GateMetricSnapshot,
runtime: BenchmarkRuntimeSnapshot,
}
#[derive(Debug, Serialize)]
@@ -387,9 +389,11 @@ fn capacity_point(
throughput_rps,
p50_ms: result.p50_ms,
p95_ms: result.p95_ms,
p99_ms: result.p99_ms,
max_ms: result.max_ms,
mean_ms: result.mean_ms,
metrics,
runtime: result.runtime,
}
}
@@ -636,6 +640,12 @@ async fn connect_protocol_peer(
request
.headers_mut()
.insert("x-node-id", http::HeaderValue::from_static("node-baseline"));
request.headers_mut().insert(
aether_contracts::tunnel::TUNNEL_PROTOCOL_VERSION_HEADER,
http::HeaderValue::from_static(
aether_contracts::tunnel::CURRENT_TUNNEL_PROTOCOL_VERSION_STR,
),
);
request.headers_mut().insert(
"x-node-name",
http::HeaderValue::from_static("proxy-baseline"),

44
crates/aether-testkit/src/bin/failure_recovery_baseline.rs
View File

@@ -12,8 +12,8 @@ use aether_runtime_state::{
RedisClientConfig, RedisClientFactory, RedisLockRunner, RedisLockRunnerConfig,
};
use aether_testkit::{
init_test_runtime_for, reserve_local_port, ManagedPostgresServer, ManagedRedisServer,
TunnelHarness, TunnelHarnessConfig,
init_test_runtime_for, reserve_local_port, BenchmarkRuntimeSampler, BenchmarkRuntimeSnapshot,
ManagedPostgresServer, ManagedRedisServer, TunnelHarness, TunnelHarnessConfig,
};
use futures_util::{FutureExt, StreamExt};
use serde::Serialize;
@@ -89,9 +89,11 @@ struct RecoverySummary {
recovered_after_restart_ms: Option<u64>,
p50_ms: u64,
p95_ms: u64,
p99_ms: u64,
max_ms: u64,
mean_ms: u64,
phase_counts: PhaseCounts,
runtime: BenchmarkRuntimeSnapshot,
}
#[derive(Debug, Clone, Serialize)]
@@ -101,6 +103,7 @@ struct PostgresSlowQueryRecoveryReport {
recovery_claim_succeeded: bool,
recovery_claim_latency_ms: u64,
recovery_claimed_items: usize,
runtime: BenchmarkRuntimeSnapshot,
}
#[derive(Debug, Clone, Serialize)]
@@ -143,10 +146,14 @@ impl RecoveryCollector {
}
}
async fn summarize(&self, recovered_after_restart_ms: Option<u64>) -> RecoverySummary {
async fn summarize(
&self,
recovered_after_restart_ms: Option<u64>,
runtime: BenchmarkRuntimeSnapshot,
) -> RecoverySummary {
let mut latencies = self.latencies_ms.lock().await.clone();
latencies.sort_unstable();
let (p50_ms, p95_ms, max_ms, mean_ms) = summarize_latencies(&latencies);
let (p50_ms, p95_ms, p99_ms, max_ms, mean_ms) = summarize_latencies(&latencies);
let phase_counts = self.phase_counts.lock().await.clone();
RecoverySummary {
total_attempts: self.successful_attempts.load(Ordering::Acquire)
@@ -156,9 +163,11 @@ impl RecoveryCollector {
recovered_after_restart_ms,
p50_ms,
p95_ms,
p99_ms,
max_ms,
mean_ms,
phase_counts,
runtime,
}
}
}
@@ -236,6 +245,7 @@ async fn benchmark_redis_restart_recovery(
redis_server: Arc<Mutex<ManagedRedisServer>>,
config: &FailureRecoveryBaselineConfig,
) -> Result<RecoverySummary, Box<dyn std::error::Error>> {
let mut runtime_sampler = BenchmarkRuntimeSampler::new();
let redis_url = redis_server.lock().await.redis_url().to_string();
let factory = RedisClientFactory::new(RedisClientConfig {
url: redis_url,
@@ -334,7 +344,10 @@ async fn benchmark_redis_restart_recovery(
.map_err(std::io::Error::other)?;
Ok(collector
.summarize(load_optional_atomic_u64(&recovered_after_restart_ms))
.summarize(
load_optional_atomic_u64(&recovered_after_restart_ms),
runtime_sampler.snapshot(),
)
.await)
}
@@ -342,6 +355,7 @@ async fn benchmark_postgres_slow_query_recovery(
postgres_url: &str,
config: &FailureRecoveryBaselineConfig,
) -> Result<PostgresSlowQueryRecoveryReport, Box<dyn std::error::Error>> {
let mut runtime_sampler = BenchmarkRuntimeSampler::new();
let backend = PostgresBackend::from_config(PostgresPoolConfig {
database_url: postgres_url.to_string(),
min_connections: 1,
@@ -410,6 +424,7 @@ async fn benchmark_postgres_slow_query_recovery(
recovery_claim_succeeded: !claimed_ids.is_empty(),
recovery_claim_latency_ms,
recovery_claimed_items: claimed_ids.len(),
runtime: runtime_sampler.snapshot(),
})
}
@@ -444,6 +459,7 @@ async fn bootstrap_failure_recovery_lease_table(
async fn benchmark_tunnel_restart_recovery(
config: &FailureRecoveryBaselineConfig,
) -> Result<RecoverySummary, Box<dyn std::error::Error>> {
let mut runtime_sampler = BenchmarkRuntimeSampler::new();
let port = reserve_local_port()?;
let tunnel_config = TunnelHarnessConfig::default();
let initial_tunnel = TunnelHarness::start_on_port(tunnel_config.clone(), port).await?;
@@ -500,6 +516,12 @@ async fn benchmark_tunnel_restart_recovery(
.parse()
.map_err(|err| format!("failed to build x-node-id header: {err}"))?,
);
request.headers_mut().insert(
aether_contracts::tunnel::TUNNEL_PROTOCOL_VERSION_HEADER,
aether_contracts::tunnel::CURRENT_TUNNEL_PROTOCOL_VERSION_STR
.parse()
.expect("protocol version header value should be valid"),
);
request.headers_mut().insert(
"x-node-name",
format!("recovery-node-{worker_index}-{current}")
@@ -556,7 +578,10 @@ async fn benchmark_tunnel_restart_recovery(
.map_err(std::io::Error::other)?;
Ok(collector
.summarize(load_optional_atomic_u64(&recovered_after_restart_ms))
.summarize(
load_optional_atomic_u64(&recovered_after_restart_ms),
runtime_sampler.snapshot(),
)
.await)
}
@@ -596,15 +621,16 @@ fn load_optional_atomic_u64(value: &AtomicU64) -> Option<u64> {
}
}
fn summarize_latencies(latencies: &[u64]) -> (u64, u64, u64, u64) {
fn summarize_latencies(latencies: &[u64]) -> (u64, u64, u64, u64, u64) {
if latencies.is_empty() {
return (0, 0, 0, 0);
return (0, 0, 0, 0, 0);
}
let max_ms = *latencies.last().unwrap_or(&0);
let mean_ms = latencies.iter().sum::<u64>() / latencies.len() as u64;
let p50_ms = percentile(latencies, 50);
let p95_ms = percentile(latencies, 95);
(p50_ms, p95_ms, max_ms, mean_ms)
let p99_ms = percentile(latencies, 99);
(p50_ms, p95_ms, p99_ms, max_ms, mean_ms)
}
fn percentile(latencies: &[u64], percentile: u8) -> u64 {

104
crates/aether-testkit/src/bin/gateway_tunnel_stream_baseline.rs
View File

@@ -3,8 +3,9 @@ use std::time::Duration;
use aether_gateway::tunnel_protocol as protocol;
use aether_testkit::{
init_test_runtime_for, run_http_load_probe, HttpLoadProbeConfig, HttpLoadProbeResponseMode,
HttpLoadProbeResult, TunnelHarness, TunnelHarnessConfig,
fetch_prometheus_samples, find_metric_value_u64, init_test_runtime_for, run_http_load_probe,
HttpLoadProbeConfig, HttpLoadProbeResponseMode, HttpLoadProbeResult, TunnelHarness,
TunnelHarnessConfig,
};
use futures_util::{SinkExt, StreamExt};
use reqwest::Method;
@@ -38,6 +39,23 @@ impl Default for GatewayTunnelBaselineConfig {
struct GatewayTunnelBaselineReport {
suite: &'static str,
scenario: HttpLoadProbeResult,
tunnel_metrics: TunnelMetricsSnapshot,
}
#[derive(Debug, Serialize)]
struct TunnelMetricsSnapshot {
proxy_connections: u64,
active_streams: u64,
outbound_queue_depth_total: u64,
outbound_queue_depth_max: u64,
outbound_queue_capacity_total: u64,
outbound_queue_rejected_full_total: u64,
outbound_queue_rejected_closed_total: u64,
proxy_connection_congested_total: u64,
proxy_connection_write_latency_last_us_max: u64,
proxy_connection_write_latency_ewma_us_max: u64,
proxy_connections_protocol_v1: u64,
proxy_connections_protocol_v2: u64,
}
#[tokio::main]
@@ -81,11 +99,13 @@ async fn run_suite(
.await
.map_err(std::io::Error::other)?;
let tunnel_metrics = capture_tunnel_metrics(tunnel.base_url()).await?;
drop(peer);
Ok(GatewayTunnelBaselineReport {
suite: "gateway_tunnel_stream_baseline",
scenario: result,
tunnel_metrics,
})
}
@@ -127,6 +147,12 @@ async fn connect_protocol_peer(
request
.headers_mut()
.insert("x-node-id", http::HeaderValue::from_static("node-baseline"));
request.headers_mut().insert(
aether_contracts::tunnel::TUNNEL_PROTOCOL_VERSION_HEADER,
http::HeaderValue::from_static(
aether_contracts::tunnel::CURRENT_TUNNEL_PROTOCOL_VERSION_STR,
),
);
request.headers_mut().insert(
"x-node-name",
http::HeaderValue::from_static("proxy-baseline"),
@@ -162,6 +188,80 @@ async fn connect_protocol_peer(
}))
}
async fn capture_tunnel_metrics(
base_url: &str,
) -> Result<TunnelMetricsSnapshot, Box<dyn std::error::Error>> {
let samples = fetch_prometheus_samples(&format!("{base_url}/metrics"))
.await
.map_err(std::io::Error::other)?;
Ok(TunnelMetricsSnapshot {
proxy_connections: find_metric_value_u64(&samples, "tunnel_proxy_connections", &[])
.unwrap_or_default(),
active_streams: find_metric_value_u64(&samples, "tunnel_active_streams", &[])
.unwrap_or_default(),
outbound_queue_depth_total: find_metric_value_u64(
&samples,
"tunnel_proxy_outbound_queue_depth_total",
&[],
)
.unwrap_or_default(),
outbound_queue_depth_max: find_metric_value_u64(
&samples,
"tunnel_proxy_outbound_queue_depth_max",
&[],
)
.unwrap_or_default(),
outbound_queue_capacity_total: find_metric_value_u64(
&samples,
"tunnel_proxy_outbound_queue_capacity_total",
&[],
)
.unwrap_or_default(),
outbound_queue_rejected_full_total: find_metric_value_u64(
&samples,
"tunnel_proxy_outbound_queue_rejected_full_total",
&[],
)
.unwrap_or_default(),
outbound_queue_rejected_closed_total: find_metric_value_u64(
&samples,
"tunnel_proxy_outbound_queue_rejected_closed_total",
&[],
)
.unwrap_or_default(),
proxy_connection_congested_total: find_metric_value_u64(
&samples,
"tunnel_proxy_connection_congested_total",
&[],
)
.unwrap_or_default(),
proxy_connection_write_latency_last_us_max: find_metric_value_u64(
&samples,
"tunnel_proxy_connection_write_latency_last_us_max",
&[],
)
.unwrap_or_default(),
proxy_connection_write_latency_ewma_us_max: find_metric_value_u64(
&samples,
"tunnel_proxy_connection_write_latency_ewma_us_max",
&[],
)
.unwrap_or_default(),
proxy_connections_protocol_v1: find_metric_value_u64(
&samples,
"tunnel_proxy_connections_protocol_v1",
&[],
)
.unwrap_or_default(),
proxy_connections_protocol_v2: find_metric_value_u64(
&samples,
"tunnel_proxy_connections_protocol_v2",
&[],
)
.unwrap_or_default(),
})
}
async fn handle_binary_frame<S>(
sink: &mut S,
data: Vec<u8>,

29
crates/aether-testkit/src/bin/multi_instance_admission_baseline.rs
View File

@@ -8,10 +8,11 @@ use aether_runtime_state::{
RedisClientConfig, RuntimeSemaphore, RuntimeSemaphoreConfig, RuntimeState,
};
use aether_testkit::{
init_test_runtime_for, run_multi_url_http_load_probe, ExecutionRuntimeHarness,
ExecutionRuntimeHarnessConfig, GatewayHarness, GatewayHarnessConfig, HttpLoadProbeConfig,
HttpLoadProbeResponseMode, ManagedRedisServer, MultiUrlHttpLoadProbeResult, SpawnedServer,
TunnelHarness, TunnelHarnessConfig,
init_test_runtime_for, run_multi_url_http_load_probe, BenchmarkRuntimeSampler,
BenchmarkRuntimeSnapshot, ExecutionRuntimeHarness, ExecutionRuntimeHarnessConfig,
GatewayHarness, GatewayHarnessConfig, HttpLoadProbeConfig, HttpLoadProbeResponseMode,
ManagedRedisServer, MultiUrlHttpLoadProbeResult, SpawnedServer, TunnelHarness,
TunnelHarnessConfig,
};
use axum::body::to_bytes;
use axum::extract::Request;
@@ -85,9 +86,11 @@ struct WebSocketAdmissionProbeResult {
successful_attempts: usize,
p50_ms: u64,
p95_ms: u64,
p99_ms: u64,
max_ms: u64,
mean_ms: u64,
status_counts: BTreeMap<u16, usize>,
runtime: BenchmarkRuntimeSnapshot,
}
#[tokio::main]
@@ -443,6 +446,7 @@ async fn run_tunnel_proxy_connection_probe(
if urls.is_empty() {
return Err("tunnel proxy connection probe requires at least one target url".to_string());
}
let mut runtime_sampler = BenchmarkRuntimeSampler::new();
let next_attempt = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let latencies_ms = Arc::new(tokio::sync::Mutex::new(Vec::with_capacity(
config.tunnel_attempts,
@@ -489,6 +493,12 @@ async fn run_tunnel_proxy_connection_probe(
.parse()
.map_err(|err| format!("failed to build x-node-id header: {err}"))?,
);
request.headers_mut().insert(
aether_contracts::tunnel::TUNNEL_PROTOCOL_VERSION_HEADER,
aether_contracts::tunnel::CURRENT_TUNNEL_PROTOCOL_VERSION_STR
.parse()
.expect("protocol version header value should be valid"),
);
request.headers_mut().insert(
"x-node-name",
format!("baseline-node-{worker_index}-{current}")
@@ -557,7 +567,7 @@ async fn run_tunnel_proxy_connection_probe(
let mut latencies = latencies_ms.lock().await.clone();
latencies.sort_unstable();
let (p50_ms, p95_ms, max_ms, mean_ms) = summarize_latencies(&latencies);
let (p50_ms, p95_ms, p99_ms, max_ms, mean_ms) = summarize_latencies(&latencies);
let target_attempt_counts = target_attempt_counts.lock().await.clone();
let status_counts = status_counts.lock().await.clone();
@@ -572,21 +582,24 @@ async fn run_tunnel_proxy_connection_probe(
successful_attempts: successful_attempts.load(std::sync::atomic::Ordering::Acquire),
p50_ms,
p95_ms,
p99_ms,
max_ms,
mean_ms,
status_counts,
runtime: runtime_sampler.snapshot(),
})
}
fn summarize_latencies(latencies: &[u64]) -> (u64, u64, u64, u64) {
fn summarize_latencies(latencies: &[u64]) -> (u64, u64, u64, u64, u64) {
if latencies.is_empty() {
return (0, 0, 0, 0);
return (0, 0, 0, 0, 0);
}
let max_ms = *latencies.last().unwrap_or(&0);
let mean_ms = latencies.iter().sum::<u64>() / latencies.len() as u64;
let p50_ms = percentile(latencies, 50);
let p95_ms = percentile(latencies, 95);
(p50_ms, p95_ms, max_ms, mean_ms)
let p99_ms = percentile(latencies, 99);
(p50_ms, p95_ms, p99_ms, max_ms, mean_ms)
}
fn percentile(latencies: &[u64], percentile: u8) -> u64 {

6
crates/aether-testkit/src/bin/multi_instance_owner_relay_baseline.rs
View File

@@ -284,6 +284,12 @@ async fn connect_protocol_peer(
request
.headers_mut()
.insert("x-node-id", http::HeaderValue::from_static(NODE_ID));
request.headers_mut().insert(
aether_contracts::tunnel::TUNNEL_PROTOCOL_VERSION_HEADER,
http::HeaderValue::from_static(
aether_contracts::tunnel::CURRENT_TUNNEL_PROTOCOL_VERSION_STR,
),
);
request.headers_mut().insert(
"x-node-name",
http::HeaderValue::from_static("proxy-owner-relay-baseline"),