i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 01:10:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 移除独立 hub/proxy/executor/gateway crate，统一为 gateway tunnel 架构

Browse Source 
- 删除 aether-hub、aether-proxy 独立项目及其 Dockerfile/配置
- 删除 crates/aether-executor 和 crates/aether-gateway 全部模块
- 新增 apps/ 目录作为应用入口
- 将 hub 概念重构为 gateway tunnel transport
- 将 executor 重构为 execution runtime
- 新增 tunnel.rs 合约定义和 testkit tunnel/execution_runtime 模块
- 更新 Python 服务层和测试适配新架构命名
This commit is contained in:
fawney19
2026-04-03 14:59:58 +08:00
parent ddf18fed9a
commit 8f26e1a31f
983 changed files with 103098 additions and 105837 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/aether-testkit/Cargo.toml
View File

@@ -10,9 +10,7 @@ description = "Shared integration test helpers for Aether Rust services"
async-stream.workspace = true
aether-data.workspace = true
aether-contracts.workspace = true
aether-executor.workspace = true
aether-gateway.workspace = true
aether-hub.workspace = true
aether-http.workspace = true
aether-runtime.workspace = true
axum.workspace = true

80
crates/aether-testkit/src/bin/capacity_curve_baseline.rs
View File

@@ -4,12 +4,12 @@ use std::path::PathBuf;
use std::time::{Duration, Instant};
use aether_contracts::{ExecutionPlan, ExecutionTimeouts, RequestBody};
use aether_hub::protocol;
use aether_gateway::tunnel_protocol as protocol;
use aether_testkit::{
fetch_prometheus_samples, find_metric_value_u64, init_test_runtime_for, run_http_load_probe,
ExecutorHarness, ExecutorHarnessConfig, GatewayHarness, GatewayHarnessConfig,
HttpLoadProbeConfig, HttpLoadProbeResponseMode, HttpLoadProbeResult, HubHarness,
HubHarnessConfig, SpawnedServer,
ExecutionRuntimeHarness, ExecutionRuntimeHarnessConfig, GatewayHarness, GatewayHarnessConfig,
HttpLoadProbeConfig, HttpLoadProbeResponseMode, HttpLoadProbeResult, SpawnedServer,
TunnelHarness, TunnelHarnessConfig,
};
use axum::body::{to_bytes, Body, Bytes};
use axum::http::StatusCode;
@@ -23,13 +23,16 @@ use serde_json::json;
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tokio_tungstenite::tungstenite::Message;
const PROXY_TUNNEL_PATH: &str = "/api/internal/proxy-tunnel";
const TUNNEL_RELAY_PATH_PREFIX: &str = "/api/internal/tunnel/relay";
#[derive(Debug, Clone)]
struct CapacityCurveBaselineConfig {
points: Vec<usize>,
requests_per_point_multiplier: usize,
sync_delay: Duration,
stream_chunk_delay: Duration,
hub_hold: Duration,
tunnel_hold: Duration,
timeout: Duration,
saturation_latency_multiplier: u64,
output_path: Option<PathBuf>,
@@ -42,7 +45,7 @@ impl Default for CapacityCurveBaselineConfig {
requests_per_point_multiplier: 8,
sync_delay: Duration::from_millis(75),
stream_chunk_delay: Duration::from_millis(25),
hub_hold: Duration::from_millis(75),
tunnel_hold: Duration::from_millis(75),
timeout: Duration::from_secs(10),
saturation_latency_multiplier: 4,
output_path: None,
@@ -55,9 +58,9 @@ struct CapacityCurveBaselineReport {
suite: &'static str,
gateway_sync: CapacityCurveScenarioReport,
gateway_stream: CapacityCurveScenarioReport,
executor_sync: CapacityCurveScenarioReport,
executor_stream: CapacityCurveScenarioReport,
hub_tunnel_stream: CapacityCurveScenarioReport,
execution_runtime_sync: CapacityCurveScenarioReport,
execution_runtime_stream: CapacityCurveScenarioReport,
gateway_tunnel_stream: CapacityCurveScenarioReport,
}
#[derive(Debug, Serialize)]
@@ -148,23 +151,24 @@ async fn run_suite(
config,
)
.await?,
executor_sync: run_executor_curve(
"executor_sync",
"executor_requests",
execution_runtime_sync: run_execution_runtime_curve(
"execution_runtime_sync",
"execution_runtime_requests",
false,
upstream.base_url(),
config,
)
.await?,
executor_stream: run_executor_curve(
"executor_stream",
"executor_requests",
execution_runtime_stream: run_execution_runtime_curve(
"execution_runtime_stream",
"execution_runtime_requests",
true,
upstream.base_url(),
config,
)
.await?,
hub_tunnel_stream: run_hub_curve("hub_tunnel_stream", "hub_requests", config).await?,
gateway_tunnel_stream: run_tunnel_curve("gateway_tunnel_stream", "tunnel_requests", config)
.await?,
})
}
@@ -187,10 +191,11 @@ async fn run_gateway_curve(
for limit in &config.points {
let gateway = GatewayHarness::start(GatewayHarnessConfig {
upstream_base_url: upstream_base_url.to_string(),
control_base_url: None,
executor_base_url: None,
data_config: None,
max_in_flight_requests: Some(*limit),
distributed_request_gate: None,
tunnel_instance_id: None,
tunnel_relay_base_url: None,
})
.await?;
let total_requests = total_requests_for_limit(*limit, config.requests_per_point_multiplier);
@@ -229,7 +234,7 @@ async fn run_gateway_curve(
})
}
async fn run_executor_curve(
async fn run_execution_runtime_curve(
scenario_name: &str,
gate_name: &str,
stream: bool,
@@ -246,7 +251,7 @@ async fn run_executor_curve(
);
let mut points = Vec::new();
for limit in &config.points {
let executor = ExecutorHarness::start(ExecutorHarnessConfig {
let runtime = ExecutionRuntimeHarness::start(ExecutionRuntimeHarnessConfig {
max_in_flight_requests: Some(*limit),
distributed_request_gate: None,
})
@@ -255,7 +260,7 @@ async fn run_executor_curve(
let probe = execution_probe_config(
format!(
"{}/v1/execute/{}",
executor.base_url(),
runtime.base_url(),
if stream { "stream" } else { "sync" }
),
execution_plan(format!("{upstream_base_url}/v1/chat/completions"), stream),
@@ -269,7 +274,7 @@ async fn run_executor_curve(
.map_err(std::io::Error::other)?;
let duration_ms = started_at.elapsed().as_millis() as u64;
let metrics =
capture_gate_metrics(&format!("{}/metrics", executor.base_url()), gate_name).await?;
capture_gate_metrics(&format!("{}/metrics", runtime.base_url()), gate_name).await?;
points.push(capacity_point(
*limit,
total_requests,
@@ -288,17 +293,17 @@ async fn run_executor_curve(
})
}
async fn run_hub_curve(
async fn run_tunnel_curve(
scenario_name: &str,
gate_name: &str,
config: &CapacityCurveBaselineConfig,
) -> Result<CapacityCurveScenarioReport, Box<dyn std::error::Error>> {
let latency_budget_ms =
scenario_latency_budget_ms(config.hub_hold, config.saturation_latency_multiplier);
scenario_latency_budget_ms(config.tunnel_hold, config.saturation_latency_multiplier);
let mut points = Vec::new();
for limit in &config.points {
let relay_concurrency = (*limit).saturating_sub(1).max(1);
let hub = HubHarness::start(HubHarnessConfig {
let tunnel = TunnelHarness::start(TunnelHarnessConfig {
max_streams: (*limit).max(128),
ping_interval: Duration::from_secs(15),
idle_timeout: Duration::ZERO,
@@ -307,11 +312,14 @@ async fn run_hub_curve(
distributed_request_gate: None,
})
.await?;
let peer = connect_protocol_peer(hub.base_url(), config.hub_hold).await?;
let peer = connect_protocol_peer(tunnel.base_url(), config.tunnel_hold).await?;
let total_requests =
total_requests_for_limit(relay_concurrency, config.requests_per_point_multiplier);
let probe = HttpLoadProbeConfig {
url: format!("{}/local/relay/node-baseline", hub.base_url()),
url: format!(
"{tunnel_base}{TUNNEL_RELAY_PATH_PREFIX}/node-baseline",
tunnel_base = tunnel.base_url()
),
method: Method::POST,
headers: BTreeMap::from([(
"content-type".to_string(),
@@ -329,7 +337,7 @@ async fn run_hub_curve(
.map_err(std::io::Error::other)?;
let duration_ms = started_at.elapsed().as_millis() as u64;
let metrics =
capture_gate_metrics(&format!("{}/metrics", hub.base_url()), gate_name).await?;
capture_gate_metrics(&format!("{}/metrics", tunnel.base_url()), gate_name).await?;
points.push(capacity_point(
*limit,
total_requests,
@@ -610,10 +618,14 @@ fn relay_envelope() -> Vec<u8> {
}
async fn connect_protocol_peer(
hub_base_url: &str,
tunnel_base_url: &str,
hold: Duration,
) -> Result<tokio::task::JoinHandle<()>, Box<dyn std::error::Error>> {
let ws_url = format!("{}/proxy", hub_base_url.replace("http://", "ws://"));
let ws_url = format!(
"{}{}",
tunnel_base_url.replace("http://", "ws://"),
PROXY_TUNNEL_PATH
);
let request = ws_url.into_client_request()?;
let mut request = request;
request
@@ -751,9 +763,9 @@ fn parse_args(
next_value(&mut iter, "--stream-chunk-delay-ms")?.parse()?,
)
}
"--hub-hold-ms" => {
config.hub_hold =
Duration::from_millis(next_value(&mut iter, "--hub-hold-ms")?.parse()?)
"--tunnel-hold-ms" => {
config.tunnel_hold =
Duration::from_millis(next_value(&mut iter, "--tunnel-hold-ms")?.parse()?)
}
"--timeout-ms" => {
config.timeout =
@@ -804,6 +816,6 @@ fn next_value(
fn print_usage() {
eprintln!(
"usage: cargo run -p aether-testkit --bin capacity_curve_baseline -- [--points 8,16,32,64,128,256] [--requests-per-point-multiplier 8] [--sync-delay-ms 75] [--stream-chunk-delay-ms 25] [--hub-hold-ms 75] [--timeout-ms 10000] [--saturation-latency-multiplier 4] [--output /tmp/capacity_curve_baseline.json]"
"usage: cargo run -p aether-testkit --bin capacity_curve_baseline -- [--points 8,16,32,64,128,256] [--requests-per-point-multiplier 8] [--sync-delay-ms 75] [--stream-chunk-delay-ms 25] [--tunnel-hold-ms 75] [--timeout-ms 10000] [--saturation-latency-multiplier 4] [--output /tmp/capacity_curve_baseline.json]"
);
}

112
crates/aether-testkit/src/bin/failure_recovery_baseline.rs
View File

@@ -10,8 +10,8 @@ use aether_data::postgres::{
use aether_data::redis::{RedisClientConfig, RedisLockRunnerConfig};
use aether_data::{PostgresBackend, RedisBackend};
use aether_testkit::{
init_test_runtime_for, reserve_local_port, HubHarness, HubHarnessConfig, ManagedPostgresServer,
ManagedRedisServer,
init_test_runtime_for, reserve_local_port, ManagedPostgresServer, ManagedRedisServer,
TunnelHarness, TunnelHarnessConfig,
};
use futures_util::{FutureExt, StreamExt};
use serde::Serialize;
@@ -19,6 +19,8 @@ use tokio::sync::{oneshot, Mutex};
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tokio_tungstenite::tungstenite::Message;
const PROXY_TUNNEL_PATH: &str = "/api/internal/proxy-tunnel";
#[derive(Debug, Clone)]
struct FailureRecoveryBaselineConfig {
redis_attempts: usize,
@@ -27,11 +29,11 @@ struct FailureRecoveryBaselineConfig {
redis_downtime: Duration,
postgres_statement_timeout: Duration,
postgres_sleep: Duration,
hub_attempts: usize,
hub_concurrency: usize,
hub_hold: Duration,
hub_restart_delay: Duration,
hub_downtime: Duration,
tunnel_attempts: usize,
tunnel_concurrency: usize,
tunnel_hold: Duration,
tunnel_restart_delay: Duration,
tunnel_downtime: Duration,
timeout: Duration,
output_path: Option<PathBuf>,
redis_url: Option<String>,
@@ -47,11 +49,11 @@ impl Default for FailureRecoveryBaselineConfig {
redis_downtime: Duration::from_millis(150),
postgres_statement_timeout: Duration::from_millis(50),
postgres_sleep: Duration::from_millis(200),
hub_attempts: 60,
hub_concurrency: 4,
hub_hold: Duration::from_millis(50),
hub_restart_delay: Duration::from_millis(200),
hub_downtime: Duration::from_millis(150),
tunnel_attempts: 60,
tunnel_concurrency: 4,
tunnel_hold: Duration::from_millis(50),
tunnel_restart_delay: Duration::from_millis(200),
tunnel_downtime: Duration::from_millis(150),
timeout: Duration::from_secs(10),
output_path: None,
redis_url: None,
@@ -106,7 +108,7 @@ struct FailureRecoveryBaselineReport {
postgres_url: String,
redis_restart: RecoverySummary,
postgres_slow_query: PostgresSlowQueryRecoveryReport,
hub_restart: RecoverySummary,
tunnel_restart: RecoverySummary,
}
#[derive(Default)]
@@ -216,7 +218,7 @@ async fn run_suite(
let redis_restart = benchmark_redis_restart_recovery(redis_server.clone(), config).await?;
let postgres_slow_query =
benchmark_postgres_slow_query_recovery(postgres_server.database_url(), config).await?;
let hub_restart = benchmark_hub_restart_recovery(config).await?;
let tunnel_restart = benchmark_tunnel_restart_recovery(config).await?;
Ok(FailureRecoveryBaselineReport {
suite: "failure_recovery_baseline",
@@ -224,7 +226,7 @@ async fn run_suite(
postgres_url,
redis_restart,
postgres_slow_query,
hub_restart,
tunnel_restart,
})
}
@@ -431,13 +433,13 @@ async fn bootstrap_failure_recovery_lease_table(
Ok(())
}
async fn benchmark_hub_restart_recovery(
async fn benchmark_tunnel_restart_recovery(
config: &FailureRecoveryBaselineConfig,
) -> Result<RecoverySummary, Box<dyn std::error::Error>> {
let port = reserve_local_port()?;
let hub_config = HubHarnessConfig::default();
let initial_hub = HubHarness::start_on_port(hub_config.clone(), port).await?;
let ws_url = format!("ws://127.0.0.1:{port}/proxy");
let tunnel_config = TunnelHarnessConfig::default();
let initial_tunnel = TunnelHarness::start_on_port(tunnel_config.clone(), port).await?;
let ws_url = format!("ws://127.0.0.1:{port}{PROXY_TUNNEL_PATH}");
let collector = Arc::new(RecoveryCollector::default());
let next_attempt = Arc::new(AtomicUsize::new(0));
let phase = Arc::new(AtomicUsize::new(0));
@@ -445,25 +447,25 @@ async fn benchmark_hub_restart_recovery(
let restart_started = Arc::new(Mutex::new(None::<Instant>));
let (done_tx, done_rx) = oneshot::channel::<()>();
let hub_restart_delay = config.hub_restart_delay;
let hub_downtime = config.hub_downtime;
let tunnel_restart_delay = config.tunnel_restart_delay;
let tunnel_downtime = config.tunnel_downtime;
let phase_for_restart = phase.clone();
let restart_started_for_task = restart_started.clone();
let restart_task = tokio::spawn(async move {
tokio::time::sleep(hub_restart_delay).await;
tokio::time::sleep(tunnel_restart_delay).await;
phase_for_restart.store(1, Ordering::Release);
*restart_started_for_task.lock().await = Some(Instant::now());
drop(initial_hub);
tokio::time::sleep(hub_downtime).await;
let restarted_hub = start_hub_on_port_retry(hub_config, port).await?;
drop(initial_tunnel);
tokio::time::sleep(tunnel_downtime).await;
let restarted_tunnel = start_tunnel_on_port_retry(tunnel_config, port).await?;
phase_for_restart.store(2, Ordering::Release);
let _ = done_rx.await;
drop(restarted_hub);
drop(restarted_tunnel);
Ok::<(), String>(())
});
let mut workers = tokio::task::JoinSet::new();
for worker_index in 0..config.hub_concurrency {
for worker_index in 0..config.tunnel_concurrency {
let ws_url = ws_url.clone();
let next_attempt = next_attempt.clone();
let collector = collector.clone();
@@ -471,8 +473,8 @@ async fn benchmark_hub_restart_recovery(
let recovered_after_restart_ms = recovered_after_restart_ms.clone();
let restart_started = restart_started.clone();
let timeout = config.timeout;
let hold = config.hub_hold;
let total_attempts = config.hub_attempts;
let hold = config.tunnel_hold;
let total_attempts = config.tunnel_attempts;
workers.spawn(async move {
loop {
let current = next_attempt.fetch_add(1, Ordering::AcqRel);
@@ -536,13 +538,13 @@ async fn benchmark_hub_restart_recovery(
while let Some(result) = workers.join_next().await {
result
.map_err(|err| format!("hub recovery worker task failed: {err}"))?
.map_err(|err| format!("hub recovery worker failed: {err}"))?;
.map_err(|err| format!("tunnel recovery worker task failed: {err}"))?
.map_err(|err| format!("tunnel recovery worker failed: {err}"))?;
}
let _ = done_tx.send(());
restart_task
.await
.map_err(|err| format!("hub restart task failed: {err}"))?
.map_err(|err| format!("tunnel restart task failed: {err}"))?
.map_err(std::io::Error::other)?;
Ok(collector
@@ -550,18 +552,20 @@ async fn benchmark_hub_restart_recovery(
.await)
}
async fn start_hub_on_port_retry(
config: HubHarnessConfig,
async fn start_tunnel_on_port_retry(
config: TunnelHarnessConfig,
port: u16,
) -> Result<HubHarness, String> {
) -> Result<TunnelHarness, String> {
let mut attempts = 0usize;
loop {
match HubHarness::start_on_port(config.clone(), port).await {
Ok(hub) => return Ok(hub),
match TunnelHarness::start_on_port(config.clone(), port).await {
Ok(tunnel) => return Ok(tunnel),
Err(err) => {
attempts += 1;
if attempts >= 20 {
return Err(format!("failed to restart hub on fixed port {port}: {err}"));
return Err(format!(
"failed to restart tunnel on fixed port {port}: {err}"
));
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
@@ -635,23 +639,25 @@ fn parse_args(
config.postgres_sleep =
Duration::from_millis(next_value(&mut iter, "--postgres-sleep-ms")?.parse()?)
}
"--hub-attempts" => {
config.hub_attempts = next_value(&mut iter, "--hub-attempts")?.parse()?
"--tunnel-attempts" => {
config.tunnel_attempts = next_value(&mut iter, "--tunnel-attempts")?.parse()?
}
"--hub-concurrency" => {
config.hub_concurrency = next_value(&mut iter, "--hub-concurrency")?.parse()?
"--tunnel-concurrency" => {
config.tunnel_concurrency =
next_value(&mut iter, "--tunnel-concurrency")?.parse()?
}
"--hub-hold-ms" => {
config.hub_hold =
Duration::from_millis(next_value(&mut iter, "--hub-hold-ms")?.parse()?)
"--tunnel-hold-ms" => {
config.tunnel_hold =
Duration::from_millis(next_value(&mut iter, "--tunnel-hold-ms")?.parse()?)
}
"--hub-restart-delay-ms" => {
config.hub_restart_delay =
Duration::from_millis(next_value(&mut iter, "--hub-restart-delay-ms")?.parse()?)
"--tunnel-restart-delay-ms" => {
config.tunnel_restart_delay = Duration::from_millis(
next_value(&mut iter, "--tunnel-restart-delay-ms")?.parse()?,
)
}
"--hub-downtime-ms" => {
config.hub_downtime =
Duration::from_millis(next_value(&mut iter, "--hub-downtime-ms")?.parse()?)
"--tunnel-downtime-ms" => {
config.tunnel_downtime =
Duration::from_millis(next_value(&mut iter, "--tunnel-downtime-ms")?.parse()?)
}
"--timeout-ms" => {
config.timeout =
@@ -686,8 +692,8 @@ fn validate_config(
|| config.redis_concurrency == 0
|| config.postgres_statement_timeout.is_zero()
|| config.postgres_sleep.is_zero()
|| config.hub_attempts == 0
|| config.hub_concurrency == 0
|| config.tunnel_attempts == 0
|| config.tunnel_concurrency == 0
|| config.timeout.is_zero()
{
return Err("all failure recovery baseline numeric settings must be positive".into());

50
crates/aether-testkit/src/bin/hub_tunnel_stream_baseline.rs → crates/aether-testkit/src/bin/gateway_tunnel_stream_baseline.rs
View File

@@ -1,10 +1,10 @@
use std::path::PathBuf;
use std::time::Duration;
use aether_hub::protocol;
use aether_gateway::tunnel_protocol as protocol;
use aether_testkit::{
init_test_runtime_for, run_http_load_probe, HttpLoadProbeConfig, HttpLoadProbeResponseMode,
HttpLoadProbeResult, HubHarness, HubHarnessConfig,
HttpLoadProbeResult, TunnelHarness, TunnelHarnessConfig,
};
use futures_util::{SinkExt, StreamExt};
use reqwest::Method;
@@ -12,15 +12,18 @@ use serde::Serialize;
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tokio_tungstenite::tungstenite::Message;
const PROXY_TUNNEL_PATH: &str = "/api/internal/proxy-tunnel";
const TUNNEL_RELAY_PATH_PREFIX: &str = "/api/internal/tunnel/relay";
#[derive(Debug, Clone)]
struct HubTunnelBaselineConfig {
struct GatewayTunnelBaselineConfig {
total_requests: usize,
concurrency: usize,
timeout: Duration,
output_path: Option<PathBuf>,
}
impl Default for HubTunnelBaselineConfig {
impl Default for GatewayTunnelBaselineConfig {
fn default() -> Self {
Self {
total_requests: 200,
@@ -32,14 +35,14 @@ impl Default for HubTunnelBaselineConfig {
}
#[derive(Debug, Serialize)]
struct HubTunnelBaselineReport {
struct GatewayTunnelBaselineReport {
suite: &'static str,
scenario: HttpLoadProbeResult,
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
init_test_runtime_for("hub-tunnel-stream-baseline");
init_test_runtime_for("gateway-tunnel-stream-baseline");
let config = parse_args(std::env::args().skip(1).collect())?;
let report = run_suite(&config).await?;
let raw = serde_json::to_string_pretty(&report)?;
@@ -54,13 +57,16 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
}
async fn run_suite(
config: &HubTunnelBaselineConfig,
) -> Result<HubTunnelBaselineReport, Box<dyn std::error::Error>> {
let hub = HubHarness::start(HubHarnessConfig::default()).await?;
let peer = connect_protocol_peer(hub.base_url()).await?;
config: &GatewayTunnelBaselineConfig,
) -> Result<GatewayTunnelBaselineReport, Box<dyn std::error::Error>> {
let tunnel = TunnelHarness::start(TunnelHarnessConfig::default()).await?;
let peer = connect_protocol_peer(tunnel.base_url()).await?;
let result = run_http_load_probe(&HttpLoadProbeConfig {
url: format!("{}/local/relay/node-baseline", hub.base_url()),
url: format!(
"{tunnel_base}{TUNNEL_RELAY_PATH_PREFIX}/node-baseline",
tunnel_base = tunnel.base_url()
),
method: Method::POST,
headers: std::collections::BTreeMap::from([(
"content-type".to_string(),
@@ -77,8 +83,8 @@ async fn run_suite(
drop(peer);
Ok(HubTunnelBaselineReport {
suite: "hub_tunnel_stream_baseline",
Ok(GatewayTunnelBaselineReport {
suite: "gateway_tunnel_stream_baseline",
scenario: result,
})
}
@@ -93,7 +99,7 @@ fn relay_envelope() -> Vec<u8> {
)]),
timeout: 30,
};
let meta_json = serde_json::to_vec(&meta).expect("hub relay metadata should serialize");
let meta_json = serde_json::to_vec(&meta).expect("tunnel relay metadata should serialize");
let body = br#"{"model":"gpt-5","messages":[{"role":"user","content":"hello"}]}"#;
let mut envelope = Vec::with_capacity(4 + meta_json.len() + body.len());
envelope.extend_from_slice(&(meta_json.len() as u32).to_be_bytes());
@@ -103,9 +109,13 @@ fn relay_envelope() -> Vec<u8> {
}
async fn connect_protocol_peer(
hub_base_url: &str,
tunnel_base_url: &str,
) -> Result<tokio::task::JoinHandle<()>, Box<dyn std::error::Error>> {
let ws_url = format!("{}/proxy", hub_base_url.replace("http://", "ws://"));
let ws_url = format!(
"{}{}",
tunnel_base_url.replace("http://", "ws://"),
PROXY_TUNNEL_PATH
);
let request = ws_url.into_client_request()?;
let mut request = request;
request
@@ -211,8 +221,10 @@ where
Ok(())
}
fn parse_args(args: Vec<String>) -> Result<HubTunnelBaselineConfig, Box<dyn std::error::Error>> {
let mut config = HubTunnelBaselineConfig::default();
fn parse_args(
args: Vec<String>,
) -> Result<GatewayTunnelBaselineConfig, Box<dyn std::error::Error>> {
let mut config = GatewayTunnelBaselineConfig::default();
let mut iter = args.into_iter();
while let Some(arg) = iter.next() {
match arg.as_str() {
@@ -258,6 +270,6 @@ fn next_value(
fn print_usage() {
eprintln!(
"usage: cargo run -p aether-testkit --bin hub_tunnel_stream_baseline -- [--requests 200] [--concurrency 20] [--timeout-ms 10000] [--output /tmp/hub_tunnel_baseline.json]"
"usage: cargo run -p aether-testkit --bin gateway_tunnel_stream_baseline -- [--requests 200] [--concurrency 20] [--timeout-ms 10000] [--output /tmp/gateway_tunnel_baseline.json]"
);
}

183
crates/aether-testkit/src/bin/multi_instance_admission_baseline.rs
View File

@@ -6,9 +6,10 @@ use std::time::{Duration, Instant};
use aether_contracts::{ExecutionPlan, ExecutionTimeouts, RequestBody};
use aether_runtime::{DistributedConcurrencyGate, RedisDistributedConcurrencyConfig};
use aether_testkit::{
init_test_runtime_for, run_multi_url_http_load_probe, ExecutorHarness, ExecutorHarnessConfig,
GatewayHarness, GatewayHarnessConfig, HttpLoadProbeConfig, HttpLoadProbeResponseMode,
HubHarness, HubHarnessConfig, ManagedRedisServer, MultiUrlHttpLoadProbeResult, SpawnedServer,
init_test_runtime_for, run_multi_url_http_load_probe, ExecutionRuntimeHarness,
ExecutionRuntimeHarnessConfig, GatewayHarness, GatewayHarnessConfig, HttpLoadProbeConfig,
HttpLoadProbeResponseMode, ManagedRedisServer, MultiUrlHttpLoadProbeResult, SpawnedServer,
TunnelHarness, TunnelHarnessConfig,
};
use axum::body::to_bytes;
use axum::extract::Request;
@@ -22,18 +23,20 @@ use serde_json::json;
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tokio_tungstenite::tungstenite::Message;
const PROXY_TUNNEL_PATH: &str = "/api/internal/proxy-tunnel";
#[derive(Debug, Clone)]
struct MultiInstanceAdmissionBaselineConfig {
gateway_requests: usize,
gateway_concurrency: usize,
executor_requests: usize,
executor_concurrency: usize,
hub_attempts: usize,
hub_concurrency: usize,
hub_hold: Duration,
execution_runtime_requests: usize,
execution_runtime_concurrency: usize,
tunnel_attempts: usize,
tunnel_concurrency: usize,
tunnel_hold: Duration,
upstream_delay: Duration,
request_limit: usize,
hub_request_limit: usize,
tunnel_request_limit: usize,
timeout: Duration,
output_path: Option<PathBuf>,
redis_url: Option<String>,
@@ -44,14 +47,14 @@ impl Default for MultiInstanceAdmissionBaselineConfig {
Self {
gateway_requests: 200,
gateway_concurrency: 20,
executor_requests: 200,
executor_concurrency: 20,
hub_attempts: 40,
hub_concurrency: 10,
hub_hold: Duration::from_millis(100),
execution_runtime_requests: 200,
execution_runtime_concurrency: 20,
tunnel_attempts: 40,
tunnel_concurrency: 10,
tunnel_hold: Duration::from_millis(100),
upstream_delay: Duration::from_millis(100),
request_limit: 8,
hub_request_limit: 4,
tunnel_request_limit: 4,
timeout: Duration::from_secs(10),
output_path: None,
redis_url: None,
@@ -64,8 +67,8 @@ struct MultiInstanceAdmissionBaselineReport {
suite: &'static str,
redis_url: String,
gateway_sync: MultiUrlHttpLoadProbeResult,
executor_sync: MultiUrlHttpLoadProbeResult,
hub_proxy: WebSocketAdmissionProbeResult,
execution_runtime_sync: MultiUrlHttpLoadProbeResult,
tunnel_proxy: WebSocketAdmissionProbeResult,
}
#[derive(Debug, Clone, Serialize)]
@@ -123,9 +126,9 @@ async fn run_suite(
let (gateway_urls, _gateways) =
start_gateway_pair(&redis_url, upstream.base_url(), config).await?;
let (executor_urls, _executors) =
start_executor_pair(&redis_url, upstream.base_url(), config).await?;
let (hub_urls, _hubs) = start_hub_pair(&redis_url, config).await?;
let (execution_runtime_urls, _runtimes) =
start_execution_runtime_pair(&redis_url, upstream.base_url(), config).await?;
let (tunnel_urls, _tunnels) = start_tunnel_pair(&redis_url, config).await?;
let gateway_sync = run_multi_url_http_load_probe(
&gateway_sync_probe_config(&gateway_urls, config),
@@ -133,13 +136,13 @@ async fn run_suite(
)
.await
.map_err(std::io::Error::other)?;
let executor_sync = run_multi_url_http_load_probe(
&executor_sync_probe_config(&executor_urls, upstream.base_url(), config),
&executor_urls,
let execution_runtime_sync = run_multi_url_http_load_probe(
&execution_runtime_sync_probe_config(&execution_runtime_urls, upstream.base_url(), config),
&execution_runtime_urls,
)
.await
.map_err(std::io::Error::other)?;
let hub_proxy = run_hub_proxy_connection_probe(&hub_urls, config)
let tunnel_proxy = run_tunnel_proxy_connection_probe(&tunnel_urls, config)
.await
.map_err(std::io::Error::other)?;
@@ -147,8 +150,8 @@ async fn run_suite(
suite: "multi_instance_admission_baseline",
redis_url,
gateway_sync,
executor_sync,
hub_proxy,
execution_runtime_sync,
tunnel_proxy,
})
}
@@ -171,18 +174,20 @@ async fn start_gateway_pair(
)?;
let gateway_a = GatewayHarness::start(GatewayHarnessConfig {
upstream_base_url: upstream_base_url.to_string(),
control_base_url: None,
executor_base_url: None,
data_config: None,
max_in_flight_requests: None,
distributed_request_gate: Some(gate_a),
tunnel_instance_id: None,
tunnel_relay_base_url: None,
})
.await?;
let gateway_b = GatewayHarness::start(GatewayHarnessConfig {
upstream_base_url: upstream_base_url.to_string(),
control_base_url: None,
executor_base_url: None,
data_config: None,
max_in_flight_requests: None,
distributed_request_gate: Some(gate_b),
tunnel_instance_id: None,
tunnel_relay_base_url: None,
})
.await?;
Ok((
@@ -194,29 +199,29 @@ async fn start_gateway_pair(
))
}
async fn start_executor_pair(
async fn start_execution_runtime_pair(
redis_url: &str,
upstream_base_url: &str,
config: &MultiInstanceAdmissionBaselineConfig,
) -> Result<(Vec<String>, Vec<ExecutorHarness>), Box<dyn std::error::Error>> {
) -> Result<(Vec<String>, Vec<ExecutionRuntimeHarness>), Box<dyn std::error::Error>> {
let gate_a = distributed_request_gate(
"executor_requests_distributed",
"execution_runtime_requests_distributed",
config.request_limit,
redis_url,
"executor-a",
"execution-runtime-a",
)?;
let gate_b = distributed_request_gate(
"executor_requests_distributed",
"execution_runtime_requests_distributed",
config.request_limit,
redis_url,
"executor-a",
"execution-runtime-a",
)?;
let executor_a = ExecutorHarness::start(ExecutorHarnessConfig {
let runtime_a = ExecutionRuntimeHarness::start(ExecutionRuntimeHarnessConfig {
max_in_flight_requests: None,
distributed_request_gate: Some(gate_a),
})
.await?;
let executor_b = ExecutorHarness::start(ExecutorHarnessConfig {
let runtime_b = ExecutionRuntimeHarness::start(ExecutionRuntimeHarnessConfig {
max_in_flight_requests: None,
distributed_request_gate: Some(gate_b),
})
@@ -224,45 +229,53 @@ async fn start_executor_pair(
let _ = upstream_base_url;
Ok((
vec![
format!("{}/v1/execute/sync", executor_a.base_url()),
format!("{}/v1/execute/sync", executor_b.base_url()),
format!("{}/v1/execute/sync", runtime_a.base_url()),
format!("{}/v1/execute/sync", runtime_b.base_url()),
],
vec![executor_a, executor_b],
vec![runtime_a, runtime_b],
))
}
async fn start_hub_pair(
async fn start_tunnel_pair(
redis_url: &str,
config: &MultiInstanceAdmissionBaselineConfig,
) -> Result<(Vec<String>, Vec<HubHarness>), Box<dyn std::error::Error>> {
) -> Result<(Vec<String>, Vec<TunnelHarness>), Box<dyn std::error::Error>> {
let gate_a = distributed_request_gate(
"hub_requests_distributed",
config.hub_request_limit,
"tunnel_requests_distributed",
config.tunnel_request_limit,
redis_url,
"hub-a",
"tunnel-a",
)?;
let gate_b = distributed_request_gate(
"hub_requests_distributed",
config.hub_request_limit,
"tunnel_requests_distributed",
config.tunnel_request_limit,
redis_url,
"hub-a",
"tunnel-a",
)?;
let hub_a = HubHarness::start(HubHarnessConfig {
let tunnel_a = TunnelHarness::start(TunnelHarnessConfig {
distributed_request_gate: Some(gate_a),
..HubHarnessConfig::default()
..TunnelHarnessConfig::default()
})
.await?;
let hub_b = HubHarness::start(HubHarnessConfig {
let tunnel_b = TunnelHarness::start(TunnelHarnessConfig {
distributed_request_gate: Some(gate_b),
..HubHarnessConfig::default()
..TunnelHarnessConfig::default()
})
.await?;
Ok((
vec![
format!("{}/proxy", hub_a.base_url().replace("http://", "ws://")),
format!("{}/proxy", hub_b.base_url().replace("http://", "ws://")),
format!(
"{}{}",
tunnel_a.base_url().replace("http://", "ws://"),
PROXY_TUNNEL_PATH
),
format!(
"{}{}",
tunnel_b.base_url().replace("http://", "ws://"),
PROXY_TUNNEL_PATH
),
],
vec![hub_a, hub_b],
vec![tunnel_a, tunnel_b],
))
}
@@ -302,7 +315,7 @@ fn gateway_sync_probe_config(
probe
}
fn executor_sync_probe_config(
fn execution_runtime_sync_probe_config(
urls: &[String],
upstream_base_url: &str,
config: &MultiInstanceAdmissionBaselineConfig,
@@ -318,8 +331,8 @@ fn executor_sync_probe_config(
)))
.expect("execution plan should serialize"),
),
total_requests: config.executor_requests,
concurrency: config.executor_concurrency,
total_requests: config.execution_runtime_requests,
concurrency: config.execution_runtime_concurrency,
timeout: config.timeout,
response_mode: HttpLoadProbeResponseMode::FullBody,
}
@@ -409,16 +422,16 @@ fn build_delayed_upstream(delay: Duration) -> Router {
)
}
async fn run_hub_proxy_connection_probe(
async fn run_tunnel_proxy_connection_probe(
urls: &[String],
config: &MultiInstanceAdmissionBaselineConfig,
) -> Result<WebSocketAdmissionProbeResult, String> {
if urls.is_empty() {
return Err("hub proxy connection probe requires at least one target url".to_string());
return Err("tunnel proxy connection probe requires at least one target url".to_string());
}
let next_attempt = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let latencies_ms = Arc::new(tokio::sync::Mutex::new(Vec::with_capacity(
config.hub_attempts,
config.tunnel_attempts,
)));
let target_attempt_counts = Arc::new(tokio::sync::Mutex::new(BTreeMap::<String, usize>::new()));
let status_counts = Arc::new(tokio::sync::Mutex::new(BTreeMap::<u16, usize>::new()));
@@ -428,7 +441,7 @@ async fn run_hub_proxy_connection_probe(
let completed_attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let mut workers = tokio::task::JoinSet::new();
for worker_index in 0..config.hub_concurrency {
for worker_index in 0..config.tunnel_concurrency {
let urls = urls.to_vec();
let next_attempt = Arc::clone(&next_attempt);
let latencies_ms = Arc::clone(&latencies_ms);
@@ -439,8 +452,8 @@ async fn run_hub_proxy_connection_probe(
let successful_attempts = Arc::clone(&successful_attempts);
let completed_attempts = Arc::clone(&completed_attempts);
let timeout = config.timeout;
let hold = config.hub_hold;
let total_attempts = config.hub_attempts;
let hold = config.tunnel_hold;
let total_attempts = config.tunnel_attempts;
workers.spawn(async move {
loop {
@@ -524,8 +537,8 @@ async fn run_hub_proxy_connection_probe(
while let Some(result) = workers.join_next().await {
result
.map_err(|err| format!("hub admission worker task failed: {err}"))?
.map_err(|err| format!("hub admission worker failed: {err}"))?;
.map_err(|err| format!("tunnel admission worker task failed: {err}"))?
.map_err(|err| format!("tunnel admission worker failed: {err}"))?;
}
let mut latencies = latencies_ms.lock().await.clone();
@@ -537,8 +550,8 @@ async fn run_hub_proxy_connection_probe(
Ok(WebSocketAdmissionProbeResult {
target_urls: urls.to_vec(),
target_attempt_counts,
total_attempts: config.hub_attempts,
concurrency: config.hub_concurrency,
total_attempts: config.tunnel_attempts,
concurrency: config.tunnel_concurrency,
completed_attempts: completed_attempts.load(std::sync::atomic::Ordering::Acquire),
failed_attempts: failed_attempts.load(std::sync::atomic::Ordering::Acquire),
rejected_attempts: rejected_attempts.load(std::sync::atomic::Ordering::Acquire),
@@ -585,22 +598,23 @@ fn parse_args(
config.gateway_concurrency =
next_value(&mut iter, "--gateway-concurrency")?.parse()?
}
"--executor-requests" => {
config.executor_requests = next_value(&mut iter, "--executor-requests")?.parse()?
"--execution-runtime-requests" | "--executor-requests" => {
config.execution_runtime_requests = next_value(&mut iter, arg.as_str())?.parse()?
}
"--executor-concurrency" => {
config.executor_concurrency =
next_value(&mut iter, "--executor-concurrency")?.parse()?
"--execution-runtime-concurrency" | "--executor-concurrency" => {
config.execution_runtime_concurrency =
next_value(&mut iter, arg.as_str())?.parse()?
}
"--hub-attempts" => {
config.hub_attempts = next_value(&mut iter, "--hub-attempts")?.parse()?
"--tunnel-attempts" => {
config.tunnel_attempts = next_value(&mut iter, "--tunnel-attempts")?.parse()?
}
"--hub-concurrency" => {
config.hub_concurrency = next_value(&mut iter, "--hub-concurrency")?.parse()?
"--tunnel-concurrency" => {
config.tunnel_concurrency =
next_value(&mut iter, "--tunnel-concurrency")?.parse()?
}
"--hub-hold-ms" => {
config.hub_hold =
Duration::from_millis(next_value(&mut iter, "--hub-hold-ms")?.parse()?)
"--tunnel-hold-ms" => {
config.tunnel_hold =
Duration::from_millis(next_value(&mut iter, "--tunnel-hold-ms")?.parse()?)
}
"--upstream-delay-ms" => {
config.upstream_delay =
@@ -609,8 +623,9 @@ fn parse_args(
"--request-limit" => {
config.request_limit = next_value(&mut iter, "--request-limit")?.parse()?
}
"--hub-request-limit" => {
config.hub_request_limit = next_value(&mut iter, "--hub-request-limit")?.parse()?
"--tunnel-request-limit" => {
config.tunnel_request_limit =
next_value(&mut iter, "--tunnel-request-limit")?.parse()?
}
"--timeout-ms" => {
config.timeout =
@@ -651,6 +666,6 @@ fn next_value(
fn print_usage() {
eprintln!(
"usage: cargo run -p aether-testkit --bin multi_instance_admission_baseline -- [--gateway-requests 200] [--gateway-concurrency 20] [--executor-requests 200] [--executor-concurrency 20] [--hub-attempts 40] [--hub-concurrency 10] [--hub-hold-ms 100] [--upstream-delay-ms 100] [--request-limit 8] [--hub-request-limit 4] [--redis-url redis://127.0.0.1:6379/0] [--output /tmp/multi_instance_admission_baseline.json]"
"usage: cargo run -p aether-testkit --bin multi_instance_admission_baseline -- [--gateway-requests 200] [--gateway-concurrency 20] [--execution-runtime-requests 200] [--execution-runtime-concurrency 20] [--tunnel-attempts 40] [--tunnel-concurrency 10] [--tunnel-hold-ms 100] [--upstream-delay-ms 100] [--request-limit 8] [--tunnel-request-limit 4] [--redis-url redis://127.0.0.1:6379/0] [--output /tmp/multi_instance_admission_baseline.json]"
);
}

534
crates/aether-testkit/src/bin/multi_instance_owner_relay_baseline.rs Normal file
View File

@@ -0,0 +1,534 @@
use std::path::PathBuf;
use std::time::Duration;
use aether_gateway::tunnel_protocol as protocol;
use aether_gateway::GatewayDataConfig;
use aether_testkit::{
init_test_runtime_for, reserve_local_port, run_http_load_probe, wait_until, GatewayHarness,
GatewayHarnessConfig, HttpLoadProbeConfig, HttpLoadProbeResponseMode, HttpLoadProbeResult,
ManagedPostgresServer, ManagedRedisServer,
};
use futures_util::{SinkExt, StreamExt};
use reqwest::Method;
use serde::Serialize;
use sqlx::{Connection, Executor, PgConnection};
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tokio_tungstenite::tungstenite::Message;
const PROXY_TUNNEL_PATH: &str = "/api/internal/proxy-tunnel";
const TUNNEL_RELAY_PATH_PREFIX: &str = "/api/internal/tunnel/relay";
const NODE_ID: &str = "node-owner-relay-baseline";
#[derive(Debug, Clone)]
struct MultiInstanceOwnerRelayBaselineConfig {
total_requests: usize,
concurrency: usize,
timeout: Duration,
chunk_delay: Duration,
output_path: Option<PathBuf>,
redis_url: Option<String>,
postgres_url: Option<String>,
}
impl Default for MultiInstanceOwnerRelayBaselineConfig {
fn default() -> Self {
Self {
total_requests: 200,
concurrency: 20,
timeout: Duration::from_secs(10),
chunk_delay: Duration::ZERO,
output_path: None,
redis_url: None,
postgres_url: None,
}
}
}
#[derive(Debug, Serialize)]
struct MultiInstanceOwnerRelayBaselineReport {
suite: &'static str,
redis_url: String,
postgres_url: String,
owner_instance_id: &'static str,
forwarder_instance_id: &'static str,
direct_owner_relay: HttpLoadProbeResult,
remote_owner_relay: HttpLoadProbeResult,
relay_overhead_ms: RelayOverheadSnapshot,
}
#[derive(Debug, Serialize)]
struct RelayOverheadSnapshot {
p50_delta_ms: i64,
p95_delta_ms: i64,
max_delta_ms: i64,
mean_delta_ms: i64,
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
init_test_runtime_for("multi-instance-owner-relay-baseline");
let config = parse_args(std::env::args().skip(1).collect())?;
let report = run_suite(&config).await?;
let raw = serde_json::to_string_pretty(&report)?;
println!("{raw}");
if let Some(path) = config.output_path.as_ref() {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
std::fs::write(path, format!("{raw}\n"))?;
}
Ok(())
}
async fn run_suite(
config: &MultiInstanceOwnerRelayBaselineConfig,
) -> Result<MultiInstanceOwnerRelayBaselineReport, Box<dyn std::error::Error>> {
let managed_redis = if config.redis_url.is_none() {
Some(ManagedRedisServer::start().await?)
} else {
None
};
let redis_url = config
.redis_url
.clone()
.or_else(|| {
managed_redis
.as_ref()
.map(|server| server.redis_url().to_string())
})
.expect("redis url should be resolved");
let managed_postgres = if config.postgres_url.is_none() {
Some(ManagedPostgresServer::start().await?)
} else {
None
};
let postgres_url = config
.postgres_url
.clone()
.or_else(|| {
managed_postgres
.as_ref()
.map(|server| server.database_url().to_string())
})
.expect("postgres url should be resolved");
ensure_owner_relay_schema(&postgres_url).await?;
let key_prefix = format!("aether-owner-relay-baseline-{}", std::process::id());
let shared_data = GatewayDataConfig::from_postgres_url(postgres_url.clone(), false)
.with_redis_url(redis_url.clone(), Some(key_prefix));
let owner_port = reserve_local_port()?;
let forwarder_port = reserve_local_port()?;
let owner_base_url = format!("http://127.0.0.1:{owner_port}");
let forwarder_base_url = format!("http://127.0.0.1:{forwarder_port}");
let owner_gateway = GatewayHarness::start_on_port(
GatewayHarnessConfig {
upstream_base_url: "http://127.0.0.1:1".to_string(),
data_config: Some(shared_data.clone()),
max_in_flight_requests: None,
distributed_request_gate: None,
tunnel_instance_id: Some("gateway-owner".to_string()),
tunnel_relay_base_url: Some(owner_base_url.clone()),
},
owner_port,
)
.await?;
let forwarder_gateway = GatewayHarness::start_on_port(
GatewayHarnessConfig {
upstream_base_url: "http://127.0.0.1:1".to_string(),
data_config: Some(shared_data),
max_in_flight_requests: None,
distributed_request_gate: None,
tunnel_instance_id: Some("gateway-forwarder".to_string()),
tunnel_relay_base_url: Some(forwarder_base_url.clone()),
},
forwarder_port,
)
.await?;
let peer = connect_protocol_peer(owner_gateway.base_url(), config.chunk_delay).await?;
wait_for_owner_attachment(&forwarder_base_url).await?;
let direct_owner_relay = run_http_load_probe(&HttpLoadProbeConfig {
url: format!(
"{owner_base}{TUNNEL_RELAY_PATH_PREFIX}/{NODE_ID}",
owner_base = owner_gateway.base_url()
),
method: Method::POST,
headers: relay_headers(),
body: Some(relay_envelope()),
total_requests: config.total_requests,
concurrency: config.concurrency,
timeout: config.timeout,
response_mode: HttpLoadProbeResponseMode::FullBody,
})
.await
.map_err(std::io::Error::other)?;
let remote_owner_relay = run_http_load_probe(&HttpLoadProbeConfig {
url: format!(
"{forwarder_base}{TUNNEL_RELAY_PATH_PREFIX}/{NODE_ID}",
forwarder_base = forwarder_gateway.base_url()
),
method: Method::POST,
headers: relay_headers(),
body: Some(relay_envelope()),
total_requests: config.total_requests,
concurrency: config.concurrency,
timeout: config.timeout,
response_mode: HttpLoadProbeResponseMode::FullBody,
})
.await
.map_err(std::io::Error::other)?;
drop(peer);
drop(forwarder_gateway);
drop(owner_gateway);
Ok(MultiInstanceOwnerRelayBaselineReport {
suite: "multi_instance_owner_relay_baseline",
redis_url,
postgres_url,
owner_instance_id: "gateway-owner",
forwarder_instance_id: "gateway-forwarder",
relay_overhead_ms: RelayOverheadSnapshot {
p50_delta_ms: remote_owner_relay.p50_ms as i64 - direct_owner_relay.p50_ms as i64,
p95_delta_ms: remote_owner_relay.p95_ms as i64 - direct_owner_relay.p95_ms as i64,
max_delta_ms: remote_owner_relay.max_ms as i64 - direct_owner_relay.max_ms as i64,
mean_delta_ms: remote_owner_relay.mean_ms as i64 - direct_owner_relay.mean_ms as i64,
},
direct_owner_relay,
remote_owner_relay,
})
}
async fn ensure_owner_relay_schema(postgres_url: &str) -> Result<(), Box<dyn std::error::Error>> {
let mut connection = PgConnection::connect(postgres_url).await?;
connection
.execute(
r#"
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'proxynodestatus') THEN
CREATE TYPE proxynodestatus AS ENUM ('online', 'offline');
END IF;
END
$$;
"#,
)
.await?;
connection
.execute(
r#"
CREATE TABLE IF NOT EXISTS system_configs (
id VARCHAR(36) PRIMARY KEY,
key VARCHAR(100) UNIQUE NOT NULL,
value JSON NOT NULL,
description TEXT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
)
"#,
)
.await?;
connection
.execute(
r#"
CREATE TABLE IF NOT EXISTS proxy_nodes (
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(100) NOT NULL,
ip VARCHAR(512) NOT NULL,
port INTEGER NOT NULL,
region VARCHAR(100) NULL,
is_manual BOOLEAN NOT NULL DEFAULT FALSE,
proxy_url VARCHAR(500) NULL,
proxy_username VARCHAR(255) NULL,
proxy_password VARCHAR(500) NULL,
status proxynodestatus NOT NULL DEFAULT 'online',
registered_by VARCHAR(36) NULL,
last_heartbeat_at TIMESTAMPTZ NULL,
heartbeat_interval INTEGER NOT NULL DEFAULT 30,
active_connections INTEGER NOT NULL DEFAULT 0,
total_requests BIGINT NOT NULL DEFAULT 0,
avg_latency_ms DOUBLE PRECISION NULL,
failed_requests BIGINT NOT NULL DEFAULT 0,
dns_failures BIGINT NOT NULL DEFAULT 0,
stream_errors BIGINT NOT NULL DEFAULT 0,
proxy_metadata JSONB NULL,
hardware_info JSONB NULL,
estimated_max_concurrency INTEGER NULL,
tunnel_mode BOOLEAN NOT NULL DEFAULT FALSE,
tunnel_connected BOOLEAN NOT NULL DEFAULT FALSE,
tunnel_connected_at TIMESTAMPTZ NULL,
remote_config JSONB NULL,
config_version INTEGER NOT NULL DEFAULT 1,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
)
"#,
)
.await?;
connection
.execute(
r#"
CREATE TABLE IF NOT EXISTS proxy_node_events (
id BIGSERIAL PRIMARY KEY,
node_id VARCHAR(36) NOT NULL,
event_type TEXT NOT NULL,
detail TEXT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
)
"#,
)
.await?;
connection.close().await?;
Ok(())
}
async fn wait_for_owner_attachment(forwarder_base_url: &str) -> Result<(), String> {
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(2))
.build()
.map_err(|err| format!("failed to build readiness client: {err}"))?;
let target_url = format!("{forwarder_base_url}{TUNNEL_RELAY_PATH_PREFIX}/{NODE_ID}");
let ready = wait_until(Duration::from_secs(10), Duration::from_millis(100), || {
let client = client.clone();
let target_url = target_url.clone();
async move {
let response = client
.post(target_url)
.header("content-type", "application/octet-stream")
.body(relay_envelope())
.send()
.await;
match response {
Ok(response) if response.status().is_success() => match response.text().await {
Ok(body) => body == "owner-relay-ok",
Err(_) => false,
},
_ => false,
}
}
})
.await;
if ready {
Ok(())
} else {
Err("timed out waiting for owner attachment propagation".to_string())
}
}
fn relay_headers() -> std::collections::BTreeMap<String, String> {
std::collections::BTreeMap::from([(
"content-type".to_string(),
"application/octet-stream".to_string(),
)])
}
fn relay_envelope() -> Vec<u8> {
let meta = protocol::RequestMeta {
method: "POST".to_string(),
url: "https://owner-relay.example/v1/chat/completions".to_string(),
headers: std::collections::HashMap::from([(
"content-type".to_string(),
"application/json".to_string(),
)]),
timeout: 30,
};
let meta_json = serde_json::to_vec(&meta).expect("owner relay metadata should serialize");
let body = br#"{"model":"gpt-5","messages":[{"role":"user","content":"owner relay"}]}"#;
let mut envelope = Vec::with_capacity(4 + meta_json.len() + body.len());
envelope.extend_from_slice(&(meta_json.len() as u32).to_be_bytes());
envelope.extend_from_slice(&meta_json);
envelope.extend_from_slice(body);
envelope
}
async fn connect_protocol_peer(
gateway_base_url: &str,
chunk_delay: Duration,
) -> Result<tokio::task::JoinHandle<()>, Box<dyn std::error::Error>> {
let ws_url = format!(
"{}{}",
gateway_base_url.replace("http://", "ws://"),
PROXY_TUNNEL_PATH
);
let request = ws_url.into_client_request()?;
let mut request = request;
request
.headers_mut()
.insert("x-node-id", http::HeaderValue::from_static(NODE_ID));
request.headers_mut().insert(
"x-node-name",
http::HeaderValue::from_static("proxy-owner-relay-baseline"),
);
request.headers_mut().insert(
"x-tunnel-max-streams",
http::HeaderValue::from_static("256"),
);
let (socket, _response) = tokio_tungstenite::connect_async(request).await?;
let (mut sink, mut stream) = socket.split();
Ok(tokio::spawn(async move {
while let Some(message) = stream.next().await {
let Ok(message) = message else {
break;
};
match message {
Message::Binary(data) => {
if handle_binary_frame(&mut sink, data.to_vec(), chunk_delay)
.await
.is_err()
{
break;
}
}
Message::Ping(payload) => {
if sink.send(Message::Pong(payload)).await.is_err() {
break;
}
}
Message::Close(_) => break,
_ => {}
}
}
let _ = sink.close().await;
}))
}
async fn handle_binary_frame<S>(
sink: &mut S,
data: Vec<u8>,
chunk_delay: Duration,
) -> Result<(), tokio_tungstenite::tungstenite::Error>
where
S: SinkExt<Message, Error = tokio_tungstenite::tungstenite::Error> + Unpin,
{
let Some(header) = protocol::FrameHeader::parse(&data) else {
return Ok(());
};
match header.msg_type {
protocol::PING => {
let payload = protocol::frame_payload_by_header(&data, &header).unwrap_or(&[]);
sink.send(Message::Binary(protocol::encode_pong(payload).into()))
.await?;
}
protocol::REQUEST_HEADERS => {
let payload = protocol::decode_payload(&data, &header).unwrap_or_default();
let _ = serde_json::from_slice::<protocol::RequestMeta>(&payload);
}
protocol::REQUEST_BODY if header.flags & protocol::FLAG_END_STREAM != 0 => {
let response_meta = protocol::ResponseMeta {
status: 200,
headers: vec![(
"content-type".to_string(),
"text/plain; charset=utf-8".to_string(),
)],
};
let response_meta_json =
serde_json::to_vec(&response_meta).expect("response metadata should serialize");
sink.send(Message::Binary(
protocol::encode_frame(
header.stream_id,
protocol::RESPONSE_HEADERS,
0,
&response_meta_json,
)
.into(),
))
.await?;
for chunk in [b"owner-".as_slice(), b"relay-".as_slice(), b"ok".as_slice()] {
if !chunk_delay.is_zero() {
tokio::time::sleep(chunk_delay).await;
}
sink.send(Message::Binary(
protocol::encode_frame(header.stream_id, protocol::RESPONSE_BODY, 0, chunk)
.into(),
))
.await?;
}
sink.send(Message::Binary(
protocol::encode_frame(header.stream_id, protocol::STREAM_END, 0, &[]).into(),
))
.await?;
}
_ => {}
}
Ok(())
}
fn parse_args(
args: Vec<String>,
) -> Result<MultiInstanceOwnerRelayBaselineConfig, Box<dyn std::error::Error>> {
let mut config = MultiInstanceOwnerRelayBaselineConfig::default();
let mut iter = args.into_iter();
while let Some(arg) = iter.next() {
match arg.as_str() {
"--requests" => config.total_requests = next_value(&mut iter, "--requests")?.parse()?,
"--concurrency" => {
config.concurrency = next_value(&mut iter, "--concurrency")?.parse()?
}
"--timeout-ms" => {
config.timeout =
Duration::from_millis(next_value(&mut iter, "--timeout-ms")?.parse()?)
}
"--chunk-delay-ms" => {
config.chunk_delay =
Duration::from_millis(next_value(&mut iter, "--chunk-delay-ms")?.parse()?)
}
"--redis-url" => config.redis_url = Some(next_value(&mut iter, "--redis-url")?),
"--postgres-url" => {
config.postgres_url = Some(next_value(&mut iter, "--postgres-url")?)
}
"--output" => {
config.output_path = Some(PathBuf::from(next_value(&mut iter, "--output")?))
}
"--help" | "-h" => {
print_usage();
std::process::exit(0);
}
other => {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("unknown argument: {other}"),
)
.into());
}
}
}
if config.total_requests == 0 || config.concurrency == 0 || config.timeout.is_zero() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"owner relay baseline numeric settings must be positive",
)
.into());
}
Ok(config)
}
fn next_value(
iter: &mut impl Iterator<Item = String>,
flag: &str,
) -> Result<String, Box<dyn std::error::Error>> {
iter.next().ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("missing value for {flag}"),
)
.into()
})
}
fn print_usage() {
println!(
"usage: cargo run -p aether-testkit --bin multi_instance_owner_relay_baseline -- [--requests 200] [--concurrency 20] [--timeout-ms 10000] [--chunk-delay-ms 0] [--redis-url redis://127.0.0.1:6379/0] [--postgres-url postgres://127.0.0.1:5432/postgres] [--output /tmp/multi_instance_owner_relay_baseline.json]"
);
}

36
crates/aether-testkit/src/bin/single_instance_baseline.rs
View File

@@ -5,9 +5,9 @@ use std::time::Duration;
use aether_contracts::{ExecutionPlan, ExecutionTimeouts, RequestBody};
use aether_testkit::{
init_test_runtime_for, run_http_load_probe, ExecutorHarness, ExecutorHarnessConfig,
GatewayHarness, GatewayHarnessConfig, HttpLoadProbeConfig, HttpLoadProbeResponseMode,
HttpLoadProbeResult, SpawnedServer,
init_test_runtime_for, run_http_load_probe, ExecutionRuntimeHarness,
ExecutionRuntimeHarnessConfig, GatewayHarness, GatewayHarnessConfig, HttpLoadProbeConfig,
HttpLoadProbeResponseMode, HttpLoadProbeResult, SpawnedServer,
};
use axum::body::{to_bytes, Body, Bytes};
use axum::http::StatusCode;
@@ -74,7 +74,7 @@ async fn run_suite(
) -> Result<SingleInstanceBaselineReport, Box<dyn std::error::Error>> {
let upstream = SpawnedServer::start(build_fake_upstream()).await?;
let gateway = GatewayHarness::start(GatewayHarnessConfig::new(upstream.base_url())).await?;
let executor = ExecutorHarness::start(ExecutorHarnessConfig::default()).await?;
let runtime = ExecutionRuntimeHarness::start(ExecutionRuntimeHarnessConfig::default()).await?;
let gateway_sync = run_http_load_probe(&gateway_sync_probe_config(gateway.base_url(), config))
.await
@@ -83,15 +83,15 @@ async fn run_suite(
run_http_load_probe(&gateway_stream_probe_config(gateway.base_url(), config))
.await
.map_err(std::io::Error::other)?;
let executor_sync = run_http_load_probe(&executor_sync_probe_config(
executor.base_url(),
let execution_runtime_sync = run_http_load_probe(&execution_runtime_sync_probe_config(
runtime.base_url(),
upstream.base_url(),
config,
))
.await
.map_err(std::io::Error::other)?;
let executor_stream = run_http_load_probe(&executor_stream_probe_config(
executor.base_url(),
let execution_runtime_stream = run_http_load_probe(&execution_runtime_stream_probe_config(
runtime.base_url(),
upstream.base_url(),
config,
))
@@ -110,12 +110,12 @@ async fn run_suite(
result: gateway_stream,
},
NamedBaselineResult {
name: "executor_sync".to_string(),
result: executor_sync,
name: "execution_runtime_sync".to_string(),
result: execution_runtime_sync,
},
NamedBaselineResult {
name: "executor_stream".to_string(),
result: executor_stream,
name: "execution_runtime_stream".to_string(),
result: execution_runtime_stream,
},
],
})
@@ -151,13 +151,13 @@ fn gateway_stream_probe_config(
probe
}
fn executor_sync_probe_config(
executor_base_url: &str,
fn execution_runtime_sync_probe_config(
runtime_base_url: &str,
upstream_base_url: &str,
config: &SingleInstanceBaselineConfig,
) -> HttpLoadProbeConfig {
execution_probe_config(
format!("{executor_base_url}/v1/execute/sync"),
format!("{runtime_base_url}/v1/execute/sync"),
execution_plan(format!("{upstream_base_url}/v1/chat/completions"), false),
config.sync_requests,
config.sync_concurrency,
@@ -165,13 +165,13 @@ fn executor_sync_probe_config(
)
}
fn executor_stream_probe_config(
executor_base_url: &str,
fn execution_runtime_stream_probe_config(
runtime_base_url: &str,
upstream_base_url: &str,
config: &SingleInstanceBaselineConfig,
) -> HttpLoadProbeConfig {
execution_probe_config(
format!("{executor_base_url}/v1/execute/stream"),
format!("{runtime_base_url}/v1/execute/stream"),
execution_plan(format!("{upstream_base_url}/v1/chat/completions"), true),
config.stream_requests,
config.stream_concurrency,

23
crates/aether-testkit/src/executor.rs → crates/aether-testkit/src/execution_runtime.rs
View File

@@ -1,43 +1,46 @@
use aether_executor::server::build_router_with_request_gates;
use aether_gateway::build_execution_runtime_router_with_request_gates;
use aether_runtime::DistributedConcurrencyGate;
use crate::server::SpawnedServer;
#[derive(Debug, Clone, Default)]
pub struct ExecutorHarnessConfig {
pub struct ExecutionRuntimeHarnessConfig {
pub max_in_flight_requests: Option<usize>,
pub distributed_request_gate: Option<DistributedConcurrencyGate>,
}
#[derive(Debug)]
pub struct ExecutorHarness {
pub struct ExecutionRuntimeHarness {
server: SpawnedServer,
}
impl ExecutorHarness {
pub async fn start(config: ExecutorHarnessConfig) -> Result<Self, String> {
impl ExecutionRuntimeHarness {
pub async fn start(config: ExecutionRuntimeHarnessConfig) -> Result<Self, String> {
Self::start_with_server(config, None).await
}
pub async fn start_on_port(config: ExecutorHarnessConfig, port: u16) -> Result<Self, String> {
pub async fn start_on_port(
config: ExecutionRuntimeHarnessConfig,
port: u16,
) -> Result<Self, String> {
Self::start_with_server(config, Some(port)).await
}
async fn start_with_server(
config: ExecutorHarnessConfig,
config: ExecutionRuntimeHarnessConfig,
port: Option<u16>,
) -> Result<Self, String> {
let router = build_router_with_request_gates(
let router = build_execution_runtime_router_with_request_gates(
config.max_in_flight_requests,
config.distributed_request_gate,
);
let server = match port {
Some(port) => SpawnedServer::start_on_port(port, router)
.await
.map_err(|err| format!("failed to start executor harness: {err}"))?,
.map_err(|err| format!("failed to start execution runtime harness: {err}"))?,
None => SpawnedServer::start(router)
.await
.map_err(|err| format!("failed to start executor harness: {err}"))?,
.map_err(|err| format!("failed to start execution runtime harness: {err}"))?,
};
Ok(Self { server })
}

28
crates/aether-testkit/src/gateway.rs
View File

@@ -1,4 +1,4 @@
use aether_gateway::{build_router_with_state, AppState};
use aether_gateway::{build_router_with_state, AppState, GatewayDataConfig};
use aether_runtime::DistributedConcurrencyGate;
use crate::server::SpawnedServer;
@@ -6,20 +6,22 @@ use crate::server::SpawnedServer;
#[derive(Debug, Clone)]
pub struct GatewayHarnessConfig {
pub upstream_base_url: String,
pub control_base_url: Option<String>,
pub executor_base_url: Option<String>,
pub data_config: Option<GatewayDataConfig>,
pub max_in_flight_requests: Option<usize>,
pub distributed_request_gate: Option<DistributedConcurrencyGate>,
pub tunnel_instance_id: Option<String>,
pub tunnel_relay_base_url: Option<String>,
}
impl GatewayHarnessConfig {
pub fn new(upstream_base_url: impl Into<String>) -> Self {
Self {
upstream_base_url: upstream_base_url.into(),
control_base_url: None,
executor_base_url: None,
data_config: None,
max_in_flight_requests: None,
distributed_request_gate: None,
tunnel_instance_id: None,
tunnel_relay_base_url: None,
}
}
}
@@ -42,12 +44,16 @@ impl GatewayHarness {
config: GatewayHarnessConfig,
port: Option<u16>,
) -> Result<Self, String> {
let mut state = AppState::new_with_executor(
config.upstream_base_url,
config.control_base_url,
config.executor_base_url,
)
.map_err(|err| format!("failed to build gateway harness state: {err}"))?;
let mut state = AppState::new(config.upstream_base_url)
.map_err(|err| format!("failed to build gateway harness state: {err}"))?;
if let Some(data_config) = config.data_config {
state = state
.with_data_config(data_config)
.map_err(|err| format!("failed to configure gateway harness data state: {err}"))?;
}
if let Some(instance_id) = config.tunnel_instance_id {
state = state.with_tunnel_identity(instance_id, config.tunnel_relay_base_url);
}
if let Some(limit) = config.max_in_flight_requests {
state = state.with_request_concurrency_limit(limit);
}

8
crates/aether-testkit/src/lib.rs
View File

@@ -1,21 +1,20 @@
mod executor;
mod execution_runtime;
mod fixtures;
mod gateway;
mod http;
mod hub;
mod load;
mod metrics;
mod postgres;
mod redis;
mod server;
mod tracing;
mod tunnel;
mod wait;
pub use executor::{ExecutorHarness, ExecutorHarnessConfig};
pub use execution_runtime::{ExecutionRuntimeHarness, ExecutionRuntimeHarnessConfig};
pub use fixtures::test_trace_id;
pub use gateway::{GatewayHarness, GatewayHarnessConfig};
pub use http::{json_body, test_http_client, test_http_client_config};
pub use hub::{HubHarness, HubHarnessConfig};
pub use load::{
run_http_load_probe, run_multi_url_http_load_probe, HttpLoadProbeConfig,
HttpLoadProbeResponseMode, HttpLoadProbeResult, MultiUrlHttpLoadProbeResult,
@@ -27,4 +26,5 @@ pub use postgres::ManagedPostgresServer;
pub use redis::ManagedRedisServer;
pub use server::{reserve_local_port, SpawnedServer};
pub use tracing::{init_test_runtime, init_test_runtime_for, test_runtime_config};
pub use tunnel::{TunnelHarness, TunnelHarnessConfig};
pub use wait::wait_until;

31
crates/aether-testkit/src/hub.rs → crates/aether-testkit/src/tunnel.rs
View File

@@ -1,12 +1,15 @@
use std::time::Duration;
use aether_hub::{build_router_with_state, AppState, ConnConfig, ControlPlaneClient};
use aether_gateway::{
build_tunnel_runtime_router_with_state, TunnelConnConfig, TunnelControlPlaneClient,
TunnelRuntimeState,
};
use aether_runtime::DistributedConcurrencyGate;
use crate::server::SpawnedServer;
#[derive(Debug, Clone)]
pub struct HubHarnessConfig {
pub struct TunnelHarnessConfig {
pub max_streams: usize,
pub ping_interval: Duration,
pub idle_timeout: Duration,
@@ -15,7 +18,7 @@ pub struct HubHarnessConfig {
pub distributed_request_gate: Option<DistributedConcurrencyGate>,
}
impl Default for HubHarnessConfig {
impl Default for TunnelHarnessConfig {
fn default() -> Self {
Self {
max_streams: 128,
@@ -29,26 +32,26 @@ impl Default for HubHarnessConfig {
}
#[derive(Debug)]
pub struct HubHarness {
pub struct TunnelHarness {
server: SpawnedServer,
}
impl HubHarness {
pub async fn start(config: HubHarnessConfig) -> Result<Self, String> {
impl TunnelHarness {
pub async fn start(config: TunnelHarnessConfig) -> Result<Self, String> {
Self::start_with_server(config, None).await
}
pub async fn start_on_port(config: HubHarnessConfig, port: u16) -> Result<Self, String> {
pub async fn start_on_port(config: TunnelHarnessConfig, port: u16) -> Result<Self, String> {
Self::start_with_server(config, Some(port)).await
}
async fn start_with_server(
config: HubHarnessConfig,
config: TunnelHarnessConfig,
port: Option<u16>,
) -> Result<Self, String> {
let state = AppState::new(
ControlPlaneClient::disabled(),
ConnConfig {
let state = TunnelRuntimeState::new(
TunnelControlPlaneClient::disabled(),
TunnelConnConfig {
ping_interval: config.ping_interval,
idle_timeout: config.idle_timeout,
outbound_queue_capacity: config.outbound_queue_capacity,
@@ -61,14 +64,14 @@ impl HubHarness {
} else {
state
};
let router = build_router_with_state(state);
let router = build_tunnel_runtime_router_with_state(state);
let server = match port {
Some(port) => SpawnedServer::start_on_port(port, router)
.await
.map_err(|err| format!("failed to start hub harness: {err}"))?,
.map_err(|err| format!("failed to start tunnel harness: {err}"))?,
None => SpawnedServer::start(router)
.await
.map_err(|err| format!("failed to start hub harness: {err}"))?,
.map_err(|err| format!("failed to start tunnel harness: {err}"))?,
};
Ok(Self { server })
}