refactor(workspace): enforce layered crate boundaries

crates/aether-testing/integration/src/bin/capacity_curve_baseline.rs

@@ -0,0 +1,880 @@
+// Gateway-backed benchmark scenarios live outside the reusable testkit.
+use std::collections::BTreeMap;
+use std::convert::Infallible;
+use std::path::PathBuf;
+use std::time::{Duration, Instant};
+
+use aether_contracts::{ExecutionPlan, ExecutionTimeouts, RequestBody};
+use aether_gateway::tunnel_protocol as protocol;
+use aether_testkit::{
+    fetch_prometheus_samples, find_metric_value_u64, init_test_runtime_for, run_http_load_probe,
+    BenchmarkRuntimeSnapshot, ExecutionRuntimeHarness, ExecutionRuntimeHarnessConfig,
+    GatewayHarness, GatewayHarnessConfig, HttpLoadProbeConfig, HttpLoadProbeResponseMode,
+    HttpLoadProbeResult, SpawnedServer, TunnelHarness, TunnelHarnessConfig,
+    GATEWAY_HARNESS_API_KEY,
+};
+use axum::body::{to_bytes, Body, Bytes};
+use axum::http::StatusCode;
+use axum::response::{IntoResponse, Response};
+use axum::routing::any;
+use axum::{extract::Request, Json, Router};
+use futures_util::{SinkExt, StreamExt};
+use reqwest::Method;
+use serde::Serialize;
+use serde_json::json;
+use tokio_tungstenite::tungstenite::client::IntoClientRequest;
+use tokio_tungstenite::tungstenite::Message;
+
+const PROXY_TUNNEL_PATH: &str = "/api/internal/proxy-tunnel";
+const TUNNEL_RELAY_PATH_PREFIX: &str = "/api/internal/tunnel/relay";
+
+#[derive(Debug, Clone)]
+struct CapacityCurveBaselineConfig {
+    points: Vec<usize>,
+    requests_per_point_multiplier: usize,
+    sync_delay: Duration,
+    stream_chunk_delay: Duration,
+    tunnel_hold: Duration,
+    timeout: Duration,
+    saturation_latency_multiplier: u64,
+    output_path: Option<PathBuf>,
+}
+
+impl Default for CapacityCurveBaselineConfig {
+    fn default() -> Self {
+        Self {
+            points: vec![8, 16, 32, 64, 128, 256],
+            requests_per_point_multiplier: 8,
+            sync_delay: Duration::from_millis(75),
+            stream_chunk_delay: Duration::from_millis(25),
+            tunnel_hold: Duration::from_millis(75),
+            timeout: Duration::from_secs(10),
+            saturation_latency_multiplier: 4,
+            output_path: None,
+        }
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct CapacityCurveBaselineReport {
+    suite: &'static str,
+    gateway_sync: CapacityCurveScenarioReport,
+    gateway_stream: CapacityCurveScenarioReport,
+    execution_runtime_sync: CapacityCurveScenarioReport,
+    execution_runtime_stream: CapacityCurveScenarioReport,
+    gateway_tunnel_stream: CapacityCurveScenarioReport,
+}
+
+#[derive(Debug, Serialize)]
+struct CapacityCurveScenarioReport {
+    name: String,
+    gate: String,
+    latency_budget_ms: u64,
+    points: Vec<CapacityCurvePointResult>,
+    saturation_point: Option<CapacityCurveSaturationPoint>,
+}
+
+#[derive(Debug, Serialize)]
+struct CapacityCurvePointResult {
+    limit: usize,
+    concurrency: usize,
+    total_requests: usize,
+    duration_ms: u64,
+    successful_requests: usize,
+    rejected_requests: usize,
+    failed_requests: usize,
+    throughput_rps: u64,
+    p50_ms: u64,
+    p95_ms: u64,
+    p99_ms: u64,
+    max_ms: u64,
+    mean_ms: u64,
+    metrics: GateMetricSnapshot,
+    runtime: BenchmarkRuntimeSnapshot,
+}
+
+#[derive(Debug, Serialize)]
+struct CapacityCurveSaturationPoint {
+    limit: usize,
+    concurrency: usize,
+    reason: String,
+    p95_ms: u64,
+    rejected_requests: usize,
+    failed_requests: usize,
+    high_watermark: u64,
+}
+
+#[derive(Debug, Serialize)]
+struct GateMetricSnapshot {
+    in_flight: u64,
+    available_permits: u64,
+    high_watermark: u64,
+    rejected_total: u64,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    init_test_runtime_for("capacity-curve-baseline");
+    let config = parse_args(std::env::args().skip(1).collect())?;
+    let report = run_suite(&config).await?;
+    let raw = serde_json::to_string_pretty(&report)?;
+    println!("{raw}");
+    if let Some(path) = config.output_path.as_ref() {
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        std::fs::write(path, format!("{raw}\n"))?;
+    }
+    Ok(())
+}
+
+async fn run_suite(
+    config: &CapacityCurveBaselineConfig,
+) -> Result<CapacityCurveBaselineReport, Box<dyn std::error::Error>> {
+    let upstream = SpawnedServer::start(build_delayed_upstream(
+        config.sync_delay,
+        config.stream_chunk_delay,
+    ))
+    .await?;
+
+    Ok(CapacityCurveBaselineReport {
+        suite: "capacity_curve_baseline",
+        gateway_sync: run_gateway_curve(
+            "gateway_proxy_sync",
+            "gateway_requests",
+            false,
+            upstream.base_url(),
+            config,
+        )
+        .await?,
+        gateway_stream: run_gateway_curve(
+            "gateway_proxy_stream",
+            "gateway_requests",
+            true,
+            upstream.base_url(),
+            config,
+        )
+        .await?,
+        execution_runtime_sync: run_execution_runtime_curve(
+            "execution_runtime_sync",
+            "execution_runtime_requests",
+            false,
+            upstream.base_url(),
+            config,
+        )
+        .await?,
+        execution_runtime_stream: run_execution_runtime_curve(
+            "execution_runtime_stream",
+            "execution_runtime_requests",
+            true,
+            upstream.base_url(),
+            config,
+        )
+        .await?,
+        gateway_tunnel_stream: run_tunnel_curve("gateway_tunnel_stream", "tunnel_requests", config)
+            .await?,
+    })
+}
+
+async fn run_gateway_curve(
+    scenario_name: &str,
+    gate_name: &str,
+    stream: bool,
+    upstream_base_url: &str,
+    config: &CapacityCurveBaselineConfig,
+) -> Result<CapacityCurveScenarioReport, Box<dyn std::error::Error>> {
+    let latency_budget_ms = scenario_latency_budget_ms(
+        if stream {
+            config.stream_chunk_delay.saturating_mul(3u32)
+        } else {
+            config.sync_delay
+        },
+        config.saturation_latency_multiplier,
+    );
+    let mut points = Vec::new();
+    for limit in &config.points {
+        let gateway = GatewayHarness::start(GatewayHarnessConfig {
+            upstream_base_url: upstream_base_url.to_string(),
+            data_config: None,
+            max_in_flight_requests: Some(*limit),
+            distributed_request_gate: None,
+            tunnel_instance_id: None,
+            tunnel_relay_base_url: None,
+        })
+        .await?;
+        let total_requests = total_requests_for_limit(*limit, config.requests_per_point_multiplier);
+        let probe = chat_probe_config(
+            format!("{}/v1/chat/completions", gateway.base_url()),
+            stream,
+            total_requests,
+            *limit,
+            config.timeout,
+        );
+        let started_at = Instant::now();
+        let result = run_http_load_probe(&probe)
+            .await
+            .map_err(std::io::Error::other)?;
+        let duration_ms = started_at.elapsed().as_millis() as u64;
+        let metrics = capture_gate_metrics(
+            &format!("{}/_gateway/metrics", gateway.base_url()),
+            gate_name,
+        )
+        .await?;
+        points.push(capacity_point(
+            *limit,
+            total_requests,
+            duration_ms,
+            result,
+            metrics,
+        ));
+    }
+
+    Ok(CapacityCurveScenarioReport {
+        name: scenario_name.to_string(),
+        gate: gate_name.to_string(),
+        latency_budget_ms,
+        saturation_point: detect_saturation_point(&points, latency_budget_ms),
+        points,
+    })
+}
+
+async fn run_execution_runtime_curve(
+    scenario_name: &str,
+    gate_name: &str,
+    stream: bool,
+    upstream_base_url: &str,
+    config: &CapacityCurveBaselineConfig,
+) -> Result<CapacityCurveScenarioReport, Box<dyn std::error::Error>> {
+    let latency_budget_ms = scenario_latency_budget_ms(
+        if stream {
+            config.stream_chunk_delay.saturating_mul(3u32)
+        } else {
+            config.sync_delay
+        },
+        config.saturation_latency_multiplier,
+    );
+    let mut points = Vec::new();
+    for limit in &config.points {
+        let runtime = ExecutionRuntimeHarness::start(ExecutionRuntimeHarnessConfig {
+            max_in_flight_requests: Some(*limit),
+            distributed_request_gate: None,
+        })
+        .await?;
+        let total_requests = total_requests_for_limit(*limit, config.requests_per_point_multiplier);
+        let probe = execution_probe_config(
+            format!(
+                "{}/v1/execute/{}",
+                runtime.base_url(),
+                if stream { "stream" } else { "sync" }
+            ),
+            execution_plan(format!("{upstream_base_url}/v1/chat/completions"), stream),
+            total_requests,
+            *limit,
+            config.timeout,
+        );
+        let started_at = Instant::now();
+        let result = run_http_load_probe(&probe)
+            .await
+            .map_err(std::io::Error::other)?;
+        let duration_ms = started_at.elapsed().as_millis() as u64;
+        let metrics =
+            capture_gate_metrics(&format!("{}/metrics", runtime.base_url()), gate_name).await?;
+        points.push(capacity_point(
+            *limit,
+            total_requests,
+            duration_ms,
+            result,
+            metrics,
+        ));
+    }
+
+    Ok(CapacityCurveScenarioReport {
+        name: scenario_name.to_string(),
+        gate: gate_name.to_string(),
+        latency_budget_ms,
+        saturation_point: detect_saturation_point(&points, latency_budget_ms),
+        points,
+    })
+}
+
+async fn run_tunnel_curve(
+    scenario_name: &str,
+    gate_name: &str,
+    config: &CapacityCurveBaselineConfig,
+) -> Result<CapacityCurveScenarioReport, Box<dyn std::error::Error>> {
+    let latency_budget_ms =
+        scenario_latency_budget_ms(config.tunnel_hold, config.saturation_latency_multiplier);
+    let mut points = Vec::new();
+    for limit in &config.points {
+        let relay_concurrency = (*limit).saturating_sub(1).max(1);
+        let tunnel = TunnelHarness::start(TunnelHarnessConfig {
+            max_streams: (*limit).max(128),
+            ping_interval: Duration::from_secs(15),
+            outbound_queue_capacity: 128,
+            max_in_flight_requests: Some(*limit),
+            distributed_request_gate: None,
+        })
+        .await?;
+        let peer = connect_protocol_peer(tunnel.base_url(), config.tunnel_hold).await?;
+        let total_requests =
+            total_requests_for_limit(relay_concurrency, config.requests_per_point_multiplier);
+        let probe = HttpLoadProbeConfig {
+            url: format!(
+                "{tunnel_base}{TUNNEL_RELAY_PATH_PREFIX}/node-baseline",
+                tunnel_base = tunnel.base_url()
+            ),
+            method: Method::POST,
+            headers: BTreeMap::from([(
+                "content-type".to_string(),
+                "application/octet-stream".to_string(),
+            )]),
+            body: Some(relay_envelope()),
+            total_requests,
+            concurrency: relay_concurrency,
+            timeout: config.timeout,
+            response_mode: HttpLoadProbeResponseMode::FullBody,
+            ..HttpLoadProbeConfig::default()
+        };
+        let started_at = Instant::now();
+        let result = run_http_load_probe(&probe)
+            .await
+            .map_err(std::io::Error::other)?;
+        let duration_ms = started_at.elapsed().as_millis() as u64;
+        let metrics =
+            capture_gate_metrics(&format!("{}/metrics", tunnel.base_url()), gate_name).await?;
+        points.push(capacity_point(
+            *limit,
+            total_requests,
+            duration_ms,
+            result,
+            metrics,
+        ));
+        drop(peer);
+    }
+
+    Ok(CapacityCurveScenarioReport {
+        name: scenario_name.to_string(),
+        gate: gate_name.to_string(),
+        latency_budget_ms,
+        saturation_point: detect_saturation_point(&points, latency_budget_ms),
+        points,
+    })
+}
+
+fn capacity_point(
+    limit: usize,
+    total_requests: usize,
+    duration_ms: u64,
+    result: HttpLoadProbeResult,
+    metrics: GateMetricSnapshot,
+) -> CapacityCurvePointResult {
+    let rejected_requests = result.status_counts.get(&503).copied().unwrap_or_default();
+    let successful_requests = result
+        .status_counts
+        .iter()
+        .filter(|(status, _)| **status >= 200 && **status < 300)
+        .map(|(_, count)| *count)
+        .sum::<usize>();
+    let throughput_rps = if duration_ms == 0 {
+        successful_requests as u64
+    } else {
+        ((successful_requests as u64) * 1_000) / duration_ms.max(1)
+    };
+
+    CapacityCurvePointResult {
+        limit,
+        concurrency: result.concurrency,
+        total_requests,
+        duration_ms,
+        successful_requests,
+        rejected_requests,
+        failed_requests: result.failed_requests,
+        throughput_rps,
+        p50_ms: result.p50_ms,
+        p95_ms: result.p95_ms,
+        p99_ms: result.p99_ms,
+        max_ms: result.max_ms,
+        mean_ms: result.mean_ms,
+        metrics,
+        runtime: result.runtime,
+    }
+}
+
+fn detect_saturation_point(
+    points: &[CapacityCurvePointResult],
+    latency_budget_ms: u64,
+) -> Option<CapacityCurveSaturationPoint> {
+    points.iter().find_map(|point| {
+        let reason = if point.failed_requests > 0 {
+            Some("failures_observed")
+        } else if point.rejected_requests > 0 {
+            Some("admission_rejections_observed")
+        } else if point.p95_ms > latency_budget_ms {
+            Some("latency_budget_exceeded")
+        } else {
+            None
+        }?;
+        Some(CapacityCurveSaturationPoint {
+            limit: point.limit,
+            concurrency: point.concurrency,
+            reason: reason.to_string(),
+            p95_ms: point.p95_ms,
+            rejected_requests: point.rejected_requests,
+            failed_requests: point.failed_requests,
+            high_watermark: point.metrics.high_watermark,
+        })
+    })
+}
+
+async fn capture_gate_metrics(
+    metrics_url: &str,
+    gate_name: &str,
+) -> Result<GateMetricSnapshot, Box<dyn std::error::Error>> {
+    let samples = fetch_prometheus_samples(metrics_url)
+        .await
+        .map_err(std::io::Error::other)?;
+    Ok(GateMetricSnapshot {
+        in_flight: find_metric_value_u64(&samples, "concurrency_in_flight", &[("gate", gate_name)])
+            .unwrap_or_default(),
+        available_permits: find_metric_value_u64(
+            &samples,
+            "concurrency_available_permits",
+            &[("gate", gate_name)],
+        )
+        .unwrap_or_default(),
+        high_watermark: find_metric_value_u64(
+            &samples,
+            "concurrency_high_watermark",
+            &[("gate", gate_name)],
+        )
+        .unwrap_or_default(),
+        rejected_total: find_metric_value_u64(
+            &samples,
+            "concurrency_rejected_total",
+            &[("gate", gate_name)],
+        )
+        .unwrap_or_default(),
+    })
+}
+
+fn scenario_latency_budget_ms(base: Duration, multiplier: u64) -> u64 {
+    (base.as_millis() as u64).saturating_mul(multiplier.max(1))
+}
+
+fn total_requests_for_limit(limit: usize, multiplier: usize) -> usize {
+    limit.saturating_mul(multiplier.max(1))
+}
+
+fn execution_probe_config(
+    url: String,
+    plan: ExecutionPlan,
+    total_requests: usize,
+    concurrency: usize,
+    timeout: Duration,
+) -> HttpLoadProbeConfig {
+    HttpLoadProbeConfig {
+        url,
+        method: Method::POST,
+        headers: BTreeMap::from([("content-type".to_string(), "application/json".to_string())]),
+        body: Some(
+            serde_json::to_vec(&plan).expect("execution plan should serialize for capacity curve"),
+        ),
+        total_requests,
+        concurrency,
+        timeout,
+        response_mode: HttpLoadProbeResponseMode::FullBody,
+        ..HttpLoadProbeConfig::default()
+    }
+}
+
+fn chat_probe_config(
+    url: String,
+    stream: bool,
+    total_requests: usize,
+    concurrency: usize,
+    timeout: Duration,
+) -> HttpLoadProbeConfig {
+    HttpLoadProbeConfig {
+        url,
+        method: Method::POST,
+        headers: BTreeMap::from([
+            ("content-type".to_string(), "application/json".to_string()),
+            (
+                "authorization".to_string(),
+                format!("Bearer {GATEWAY_HARNESS_API_KEY}"),
+            ),
+        ]),
+        body: Some(
+            serde_json::to_vec(&json!({
+                "model": "gpt-5",
+                "messages": [{"role": "user", "content": "hello"}],
+                "stream": stream,
+            }))
+            .expect("chat body should serialize"),
+        ),
+        total_requests,
+        concurrency,
+        timeout,
+        response_mode: HttpLoadProbeResponseMode::FullBody,
+        ..HttpLoadProbeConfig::default()
+    }
+}
+
+fn execution_plan(url: String, stream: bool) -> ExecutionPlan {
+    ExecutionPlan {
+        request_id: if stream {
+            "capacity-curve-stream-request".to_string()
+        } else {
+            "capacity-curve-sync-request".to_string()
+        },
+        candidate_id: Some(if stream {
+            "capacity-curve-stream-candidate".to_string()
+        } else {
+            "capacity-curve-sync-candidate".to_string()
+        }),
+        provider_name: Some("openai".to_string()),
+        provider_id: "provider-capacity".to_string(),
+        endpoint_id: "endpoint-capacity".to_string(),
+        key_id: "key-capacity".to_string(),
+        method: "POST".to_string(),
+        url,
+        headers: BTreeMap::from([("content-type".to_string(), "application/json".to_string())]),
+        content_type: Some("application/json".to_string()),
+        content_encoding: None,
+        body: RequestBody::from_json(json!({
+            "model": "gpt-5",
+            "messages": [{"role": "user", "content": "hello"}],
+            "stream": stream,
+        })),
+        stream,
+        client_api_format: "openai:chat".to_string(),
+        provider_api_format: "openai:chat".to_string(),
+        model_name: Some("gpt-5".to_string()),
+        proxy: None,
+        transport_profile: None,
+        timeouts: Some(ExecutionTimeouts {
+            connect_ms: Some(2_000),
+            read_ms: Some(10_000),
+            first_byte_ms: Some(5_000),
+            total_ms: Some(10_000),
+            ..ExecutionTimeouts::default()
+        }),
+    }
+}
+
+fn build_delayed_upstream(sync_delay: Duration, stream_chunk_delay: Duration) -> Router {
+    Router::new().route(
+        "/v1/chat/completions",
+        any(move |request: Request| {
+            let sync_delay = sync_delay;
+            let stream_chunk_delay = stream_chunk_delay;
+            async move {
+                let (_parts, body) = request.into_parts();
+                let raw_body = to_bytes(body, usize::MAX)
+                    .await
+                    .expect("capacity upstream body should read");
+                let payload: serde_json::Value =
+                    serde_json::from_slice(&raw_body).unwrap_or_else(|_| json!({}));
+                let stream = payload
+                    .get("stream")
+                    .and_then(|value| value.as_bool())
+                    .unwrap_or(false);
+                if stream {
+                    let body = async_stream::stream! {
+                        tokio::time::sleep(stream_chunk_delay).await;
+                        yield Ok::<_, Infallible>(Bytes::from_static(
+                            b"data: {\"id\":\"chunk-1\",\"choices\":[{\"delta\":{\"content\":\"hel\"}}]}\n\n",
+                        ));
+                        tokio::time::sleep(stream_chunk_delay).await;
+                        yield Ok::<_, Infallible>(Bytes::from_static(
+                            b"data: {\"id\":\"chunk-2\",\"choices\":[{\"delta\":{\"content\":\"lo\"}}]}\n\n",
+                        ));
+                        tokio::time::sleep(stream_chunk_delay).await;
+                        yield Ok::<_, Infallible>(Bytes::from_static(b"data: [DONE]\n\n"));
+                    };
+                    Response::builder()
+                        .status(StatusCode::OK)
+                        .header(http::header::CONTENT_TYPE, "text/event-stream")
+                        .body(Body::from_stream(body))
+                        .expect("capacity upstream stream response should build")
+                } else {
+                    tokio::time::sleep(sync_delay).await;
+                    Json(json!({
+                        "id": "chatcmpl-capacity",
+                        "object": "chat.completion",
+                        "model": payload.get("model").and_then(|value| value.as_str()).unwrap_or("gpt-5"),
+                        "choices": [{"message": {"role": "assistant", "content": "hello"}}]
+                    }))
+                    .into_response()
+                }
+            }
+        }),
+    )
+}
+
+fn relay_envelope() -> Vec<u8> {
+    let meta = protocol::RequestMeta {
+        method: "POST".to_string(),
+        url: "https://capacity.example/v1/chat/completions".to_string(),
+        headers: std::collections::HashMap::from([(
+            "content-type".to_string(),
+            "application/json".to_string(),
+        )]),
+        stream: false,
+        request_timeout_ms: None,
+        stream_first_byte_timeout_ms: None,
+        timeout: 30,
+        follow_redirects: None,
+        http1_only: false,
+        provider_id: None,
+        endpoint_id: None,
+        key_id: None,
+        transport_profile: None,
+    };
+    let meta_json = serde_json::to_vec(&meta).expect("hub relay metadata should serialize");
+    let body = br#"{"model":"gpt-5","messages":[{"role":"user","content":"hello"}]}"#;
+    let mut envelope = Vec::with_capacity(4 + meta_json.len() + body.len());
+    envelope.extend_from_slice(&(meta_json.len() as u32).to_be_bytes());
+    envelope.extend_from_slice(&meta_json);
+    envelope.extend_from_slice(body);
+    envelope
+}
+
+async fn connect_protocol_peer(
+    tunnel_base_url: &str,
+    hold: Duration,
+) -> Result<tokio::task::JoinHandle<()>, Box<dyn std::error::Error>> {
+    let ws_url = format!(
+        "{}{}",
+        tunnel_base_url.replace("http://", "ws://"),
+        PROXY_TUNNEL_PATH
+    );
+    let request = ws_url.into_client_request()?;
+    let mut request = request;
+    request
+        .headers_mut()
+        .insert("x-node-id", http::HeaderValue::from_static("node-baseline"));
+    request.headers_mut().insert(
+        aether_contracts::tunnel::TUNNEL_PROTOCOL_VERSION_HEADER,
+        http::HeaderValue::from_static(
+            aether_contracts::tunnel::CURRENT_TUNNEL_PROTOCOL_VERSION_STR,
+        ),
+    );
+    request.headers_mut().insert(
+        "x-node-name",
+        http::HeaderValue::from_static("proxy-baseline"),
+    );
+    request.headers_mut().insert(
+        "x-tunnel-max-streams",
+        http::HeaderValue::from_static("512"),
+    );
+
+    let (socket, _response) = tokio_tungstenite::connect_async(request).await?;
+    let (mut sink, mut stream) = socket.split();
+    sink.send(Message::Binary(
+        protocol::encode_hello(&protocol::HelloPayload {
+            protocol_version: aether_contracts::tunnel::CURRENT_TUNNEL_PROTOCOL_VERSION,
+            capabilities: vec![
+                "flow-control".to_string(),
+                "reset-stream".to_string(),
+                "graceful-drain".to_string(),
+            ],
+            session_id: Some("capacity-curve-session".to_string()),
+            replica_id: Some("capacity-curve-replica".to_string()),
+        })
+        .into(),
+    ))
+    .await?;
+    sink.send(Message::Binary(
+        protocol::encode_settings(&protocol::SettingsPayload {
+            initial_stream_window_bytes: 4 * 1024 * 1024,
+            min_window_update_bytes: 1024 * 1024,
+            drain_deadline_ms: 30_000,
+        })
+        .into(),
+    ))
+    .await?;
+    Ok(tokio::spawn(async move {
+        while let Some(message) = stream.next().await {
+            let Ok(message) = message else {
+                break;
+            };
+            match message {
+                Message::Binary(data)
+                    if handle_binary_frame(&mut sink, data.to_vec(), hold)
+                        .await
+                        .is_err() =>
+                {
+                    break;
+                }
+                Message::Ping(payload)
+                    if sink.send(Message::Pong(payload.clone())).await.is_err() =>
+                {
+                    break;
+                }
+                Message::Close(_) => break,
+                _ => {}
+            }
+        }
+        let _ = sink.close().await;
+    }))
+}
+
+async fn handle_binary_frame<S>(
+    sink: &mut S,
+    data: Vec<u8>,
+    hold: Duration,
+) -> Result<(), tokio_tungstenite::tungstenite::Error>
+where
+    S: SinkExt<Message, Error = tokio_tungstenite::tungstenite::Error> + Unpin,
+{
+    let Some(header) = protocol::FrameHeader::parse(&data) else {
+        return Ok(());
+    };
+    match header.msg_type {
+        protocol::PING => {
+            let payload = protocol::frame_payload_by_header(&data, &header).unwrap_or(&[]);
+            sink.send(Message::Binary(protocol::encode_pong(payload).into()))
+                .await?;
+        }
+        protocol::REQUEST_HEADERS => {
+            let payload = protocol::decode_payload(&data, &header).unwrap_or_default();
+            let _ = serde_json::from_slice::<protocol::RequestMeta>(&payload);
+        }
+        protocol::REQUEST_BODY => {
+            let payload = protocol::decode_payload(&data, &header).unwrap_or_default();
+            if !payload.is_empty() {
+                sink.send(Message::Binary(
+                    protocol::encode_window_update(header.stream_id, payload.len() as u32).into(),
+                ))
+                .await?;
+            }
+            if header.flags & protocol::FLAG_END_STREAM == 0 {
+                return Ok(());
+            }
+            tokio::time::sleep(hold).await;
+            let response_meta = protocol::ResponseMeta {
+                status: 200,
+                headers: vec![(
+                    "content-type".to_string(),
+                    "text/plain; charset=utf-8".to_string(),
+                )],
+            };
+            let response_meta_json =
+                serde_json::to_vec(&response_meta).expect("response metadata should serialize");
+            sink.send(Message::Binary(
+                protocol::encode_frame(
+                    header.stream_id,
+                    protocol::RESPONSE_HEADERS,
+                    0,
+                    &response_meta_json,
+                )
+                .into(),
+            ))
+            .await?;
+
+            for chunk in [
+                b"capacity-".as_slice(),
+                b"tunnel-".as_slice(),
+                b"stream".as_slice(),
+            ] {
+                sink.send(Message::Binary(
+                    protocol::encode_frame(header.stream_id, protocol::RESPONSE_BODY, 0, chunk)
+                        .into(),
+                ))
+                .await?;
+            }
+
+            sink.send(Message::Binary(
+                protocol::encode_frame(header.stream_id, protocol::STREAM_END, 0, &[]).into(),
+            ))
+            .await?;
+        }
+        _ => {}
+    }
+    Ok(())
+}
+
+fn parse_args(
+    args: Vec<String>,
+) -> Result<CapacityCurveBaselineConfig, Box<dyn std::error::Error>> {
+    let mut config = CapacityCurveBaselineConfig::default();
+    let mut iter = args.into_iter();
+    while let Some(arg) = iter.next() {
+        match arg.as_str() {
+            "--points" => {
+                config.points = next_value(&mut iter, "--points")?
+                    .split(',')
+                    .filter(|value| !value.trim().is_empty())
+                    .map(|value| value.trim().parse::<usize>())
+                    .collect::<Result<Vec<_>, _>>()?;
+            }
+            "--requests-per-point-multiplier" => {
+                config.requests_per_point_multiplier =
+                    next_value(&mut iter, "--requests-per-point-multiplier")?.parse()?
+            }
+            "--sync-delay-ms" => {
+                config.sync_delay =
+                    Duration::from_millis(next_value(&mut iter, "--sync-delay-ms")?.parse()?)
+            }
+            "--stream-chunk-delay-ms" => {
+                config.stream_chunk_delay = Duration::from_millis(
+                    next_value(&mut iter, "--stream-chunk-delay-ms")?.parse()?,
+                )
+            }
+            "--tunnel-hold-ms" => {
+                config.tunnel_hold =
+                    Duration::from_millis(next_value(&mut iter, "--tunnel-hold-ms")?.parse()?)
+            }
+            "--timeout-ms" => {
+                config.timeout =
+                    Duration::from_millis(next_value(&mut iter, "--timeout-ms")?.parse()?)
+            }
+            "--saturation-latency-multiplier" => {
+                config.saturation_latency_multiplier =
+                    next_value(&mut iter, "--saturation-latency-multiplier")?.parse()?
+            }
+            "--output" => {
+                config.output_path = Some(PathBuf::from(next_value(&mut iter, "--output")?))
+            }
+            "--help" | "-h" => {
+                print_usage();
+                std::process::exit(0);
+            }
+            other => {
+                return Err(std::io::Error::new(
+                    std::io::ErrorKind::InvalidInput,
+                    format!("unknown argument: {other}"),
+                )
+                .into());
+            }
+        }
+    }
+    if config.points.is_empty() {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            "capacity curve requires at least one point",
+        )
+        .into());
+    }
+    Ok(config)
+}
+
+fn next_value(
+    iter: &mut impl Iterator<Item = String>,
+    flag: &str,
+) -> Result<String, Box<dyn std::error::Error>> {
+    iter.next().ok_or_else(|| {
+        std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            format!("missing value for {flag}"),
+        )
+        .into()
+    })
+}
+
+fn print_usage() {
+    eprintln!(
+        "usage: cargo run -p aether-integration-tests --bin capacity_curve_baseline -- [--points 8,16,32,64,128,256] [--requests-per-point-multiplier 8] [--sync-delay-ms 75] [--stream-chunk-delay-ms 25] [--tunnel-hold-ms 75] [--timeout-ms 10000] [--saturation-latency-multiplier 4] [--output /tmp/capacity_curve_baseline.json]"
+    );
+}

crates/aether-testing/integration/src/bin/dependency_pressure_baseline.rs

@@ -0,0 +1,794 @@
+// Gateway-backed benchmark scenarios live outside the reusable testkit.
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use aether_data::driver::postgres::{
+    DatabaseRecordId, PostgresLeaseClaimOptions, PostgresLeaseClaimSpec, PostgresLeaseRunnerConfig,
+    PostgresPoolConfig,
+};
+use aether_data::PostgresBackend;
+use aether_runtime_state::{
+    RedisClientConfig, RedisConsumerGroup, RedisConsumerName, RedisKeyspace, RedisLockLease,
+    RedisLockRunner, RedisLockRunnerConfig, RedisStreamName, RedisStreamReclaimConfig,
+    RedisStreamRunner, RedisStreamRunnerConfig,
+};
+use aether_testkit::{init_test_runtime_for, ManagedPostgresServer, ManagedRedisServer};
+use futures_util::stream::{self, StreamExt};
+use serde::Serialize;
+
+#[derive(Debug, Clone)]
+struct DependencyPressureBaselineConfig {
+    redis_lock_total: usize,
+    redis_lock_concurrency: usize,
+    redis_stream_total: usize,
+    redis_stream_concurrency: usize,
+    redis_reclaim_total: usize,
+    redis_reclaim_min_idle: Duration,
+    postgres_rows: usize,
+    postgres_lease_cycles: usize,
+    postgres_lease_concurrency: usize,
+    postgres_lease_batch_size: usize,
+    postgres_lease_ms: u64,
+    timeout: Duration,
+    output_path: Option<PathBuf>,
+    redis_url: Option<String>,
+    postgres_url: Option<String>,
+}
+
+impl Default for DependencyPressureBaselineConfig {
+    fn default() -> Self {
+        Self {
+            redis_lock_total: 1_000,
+            redis_lock_concurrency: 20,
+            redis_stream_total: 2_000,
+            redis_stream_concurrency: 20,
+            redis_reclaim_total: 256,
+            redis_reclaim_min_idle: Duration::from_millis(100),
+            postgres_rows: 512,
+            postgres_lease_cycles: 128,
+            postgres_lease_concurrency: 16,
+            postgres_lease_batch_size: 16,
+            postgres_lease_ms: 250,
+            timeout: Duration::from_secs(10),
+            output_path: None,
+            redis_url: None,
+            postgres_url: None,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize)]
+struct OperationSummary {
+    total_calls: usize,
+    total_items: usize,
+    failed_calls: usize,
+    p50_ms: u64,
+    p95_ms: u64,
+    max_ms: u64,
+    mean_ms: u64,
+}
+
+#[derive(Debug, Clone, Serialize)]
+struct RedisLockPressureReport {
+    acquire: OperationSummary,
+    renew: OperationSummary,
+    release: OperationSummary,
+}
+
+#[derive(Debug, Clone, Serialize)]
+struct RedisStreamPressureReport {
+    append: OperationSummary,
+    read_group: OperationSummary,
+    reclaim: OperationSummary,
+    ack: OperationSummary,
+}
+
+#[derive(Debug, Clone, Serialize)]
+struct PostgresLeasePressureReport {
+    claim: OperationSummary,
+    renew: OperationSummary,
+    release: OperationSummary,
+}
+
+#[derive(Debug, Clone, Serialize)]
+struct DependencyPressureBaselineReport {
+    suite: &'static str,
+    redis_url: String,
+    postgres_url: String,
+    redis_lock: RedisLockPressureReport,
+    redis_stream: RedisStreamPressureReport,
+    postgres_lease: PostgresLeasePressureReport,
+}
+
+#[derive(Default)]
+struct SummaryCollector {
+    latencies_ms: tokio::sync::Mutex<Vec<u64>>,
+    total_items: std::sync::atomic::AtomicUsize,
+    failed_calls: std::sync::atomic::AtomicUsize,
+    total_calls: std::sync::atomic::AtomicUsize,
+}
+
+impl SummaryCollector {
+    async fn record(&self, elapsed: Duration, items: usize, failed: bool) {
+        self.latencies_ms
+            .lock()
+            .await
+            .push(elapsed.as_millis() as u64);
+        self.total_calls
+            .fetch_add(1, std::sync::atomic::Ordering::AcqRel);
+        self.total_items
+            .fetch_add(items, std::sync::atomic::Ordering::AcqRel);
+        if failed {
+            self.failed_calls
+                .fetch_add(1, std::sync::atomic::Ordering::AcqRel);
+        }
+    }
+
+    async fn summarize(&self) -> OperationSummary {
+        let mut latencies = self.latencies_ms.lock().await.clone();
+        latencies.sort_unstable();
+        let (p50_ms, p95_ms, max_ms, mean_ms) = summarize_latencies(&latencies);
+        OperationSummary {
+            total_calls: self.total_calls.load(std::sync::atomic::Ordering::Acquire),
+            total_items: self.total_items.load(std::sync::atomic::Ordering::Acquire),
+            failed_calls: self.failed_calls.load(std::sync::atomic::Ordering::Acquire),
+            p50_ms,
+            p95_ms,
+            max_ms,
+            mean_ms,
+        }
+    }
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    init_test_runtime_for("dependency-pressure-baseline");
+    let config = parse_args(std::env::args().skip(1).collect())?;
+    let report = run_suite(&config).await?;
+    let raw = serde_json::to_string_pretty(&report)?;
+    println!("{raw}");
+    if let Some(path) = config.output_path.as_ref() {
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        std::fs::write(path, format!("{raw}\n"))?;
+    }
+    Ok(())
+}
+
+async fn run_suite(
+    config: &DependencyPressureBaselineConfig,
+) -> Result<DependencyPressureBaselineReport, Box<dyn std::error::Error>> {
+    let managed_redis = if config.redis_url.is_none() {
+        Some(ManagedRedisServer::start().await?)
+    } else {
+        None
+    };
+    let managed_postgres = if config.postgres_url.is_none() {
+        Some(ManagedPostgresServer::start().await?)
+    } else {
+        None
+    };
+
+    let redis_url = config
+        .redis_url
+        .clone()
+        .or_else(|| {
+            managed_redis
+                .as_ref()
+                .map(|server| server.redis_url().to_string())
+        })
+        .expect("redis url should resolve");
+    let postgres_url = config
+        .postgres_url
+        .clone()
+        .or_else(|| {
+            managed_postgres
+                .as_ref()
+                .map(|server| server.database_url().to_string())
+        })
+        .expect("postgres url should resolve");
+
+    let redis_config = RedisClientConfig {
+        url: redis_url.clone(),
+        key_prefix: Some(format!("aether-dependency-pressure-{}", std::process::id())),
+    };
+    let redis_keyspace = redis_config.keyspace();
+    let postgres_backend = PostgresBackend::from_config(PostgresPoolConfig {
+        database_url: postgres_url.clone(),
+        min_connections: 1,
+        max_connections: (config.postgres_lease_concurrency as u32).saturating_add(8),
+        acquire_timeout_ms: config.timeout.as_millis() as u64,
+        idle_timeout_ms: 60_000,
+        max_lifetime_ms: 10 * 60_000,
+        statement_cache_capacity: 128,
+        require_ssl: false,
+    })?;
+
+    bootstrap_postgres_lease_table(postgres_backend.pool_clone(), config).await?;
+
+    let lock_runner = RedisLockRunner::from_config(
+        redis_config.clone(),
+        RedisLockRunnerConfig {
+            command_timeout_ms: Some(config.timeout.as_millis() as u64),
+            default_ttl_ms: 5_000,
+        },
+    )
+    .await?;
+    let stream_runner = RedisStreamRunner::from_config(
+        redis_config,
+        RedisStreamRunnerConfig {
+            command_timeout_ms: Some(config.timeout.as_millis() as u64),
+            read_block_ms: Some(10),
+            read_count: 64,
+        },
+    )
+    .await?;
+    let lease_runner = postgres_backend.lease_runner(PostgresLeaseRunnerConfig {
+        statement_timeout_ms: Some(config.timeout.as_millis() as u64),
+        lock_timeout_ms: Some(1_000),
+    })?;
+
+    let redis_lock = benchmark_redis_lock(&redis_keyspace, &lock_runner, config).await?;
+    let redis_stream = benchmark_redis_stream(&redis_keyspace, &stream_runner, config).await?;
+    let postgres_lease = benchmark_postgres_lease(&lease_runner, config).await?;
+
+    Ok(DependencyPressureBaselineReport {
+        suite: "dependency_pressure_baseline",
+        redis_url,
+        postgres_url,
+        redis_lock,
+        redis_stream,
+        postgres_lease,
+    })
+}
+
+async fn bootstrap_postgres_lease_table(
+    pool: sqlx::PgPool,
+    config: &DependencyPressureBaselineConfig,
+) -> Result<(), Box<dyn std::error::Error>> {
+    sqlx::query("DROP TABLE IF EXISTS baseline_lease_jobs")
+        .execute(&pool)
+        .await?;
+    sqlx::query(
+        "CREATE TABLE baseline_lease_jobs (
+             id TEXT PRIMARY KEY,
+             status TEXT NOT NULL,
+             updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+             lease_owner TEXT,
+             lease_expires_at TIMESTAMPTZ
+         )",
+    )
+    .execute(&pool)
+    .await?;
+
+    let mut builder = sqlx::QueryBuilder::new("INSERT INTO baseline_lease_jobs (id, status) ");
+    builder.push_values(0..config.postgres_rows, |mut row, index| {
+        row.push_bind(format!("job-{index:05}")).push_bind("ready");
+    });
+    builder.build().execute(&pool).await?;
+
+    Ok(())
+}
+
+async fn benchmark_redis_lock(
+    keyspace: &RedisKeyspace,
+    runner: &RedisLockRunner,
+    config: &DependencyPressureBaselineConfig,
+) -> Result<RedisLockPressureReport, Box<dyn std::error::Error>> {
+    let acquire = Arc::new(SummaryCollector::default());
+    let renew = Arc::new(SummaryCollector::default());
+    let release = Arc::new(SummaryCollector::default());
+    let next = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+
+    stream::iter(0..config.redis_lock_concurrency)
+        .for_each_concurrent(config.redis_lock_concurrency, |_| {
+            let runner = runner.clone();
+            let acquire = acquire.clone();
+            let renew = renew.clone();
+            let release = release.clone();
+            let next = next.clone();
+            let keyspace = keyspace.clone();
+            async move {
+                loop {
+                    let index = next.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
+                    if index >= config.redis_lock_total {
+                        break;
+                    }
+                    let owner = format!("lock-owner-{index}");
+                    let key = keyspace.lock_key(&format!("dependency-pressure-{index}"));
+                    let acquire_started = Instant::now();
+                    match runner.try_acquire(&key, &owner, None).await {
+                        Ok(Some(lease)) => {
+                            acquire.record(acquire_started.elapsed(), 1, false).await;
+                            record_redis_lock_follow_up(&runner, &lease, &renew, &release).await;
+                        }
+                        Ok(None) => {
+                            acquire.record(acquire_started.elapsed(), 0, true).await;
+                        }
+                        Err(_) => {
+                            acquire.record(acquire_started.elapsed(), 0, true).await;
+                        }
+                    }
+                }
+            }
+        })
+        .await;
+
+    Ok(RedisLockPressureReport {
+        acquire: acquire.summarize().await,
+        renew: renew.summarize().await,
+        release: release.summarize().await,
+    })
+}
+
+async fn record_redis_lock_follow_up(
+    runner: &RedisLockRunner,
+    lease: &RedisLockLease,
+    renew: &SummaryCollector,
+    release: &SummaryCollector,
+) {
+    let renew_started = Instant::now();
+    let renew_ok = runner.renew(lease, None).await.unwrap_or(false);
+    renew
+        .record(renew_started.elapsed(), usize::from(renew_ok), !renew_ok)
+        .await;
+
+    let release_started = Instant::now();
+    let release_ok = runner.release(lease).await.unwrap_or(false);
+    release
+        .record(
+            release_started.elapsed(),
+            usize::from(release_ok),
+            !release_ok,
+        )
+        .await;
+}
+
+async fn benchmark_redis_stream(
+    keyspace: &RedisKeyspace,
+    runner: &RedisStreamRunner,
+    config: &DependencyPressureBaselineConfig,
+) -> Result<RedisStreamPressureReport, Box<dyn std::error::Error>> {
+    let stream = keyspace.stream_name("dependency-pressure");
+    let group = RedisConsumerGroup("dependency-group".to_string());
+    let consumer_a = RedisConsumerName("consumer-a".to_string());
+    let consumer_b = RedisConsumerName("consumer-b".to_string());
+    runner
+        .ensure_consumer_group(&stream, &group, "0-0")
+        .await
+        .map_err(std::io::Error::other)?;
+
+    let append = benchmark_redis_stream_append(runner, &stream, config).await?;
+    let (read_group, drained_ids) =
+        benchmark_redis_stream_read_group(runner, &stream, &group, &consumer_a, config).await?;
+    let ack = SummaryCollector::default();
+    benchmark_redis_stream_ack_into(&ack, runner, &stream, &group, &drained_ids).await?;
+    let (reclaim, reclaimed_ids) =
+        benchmark_redis_stream_reclaim(runner, &stream, &group, &consumer_a, &consumer_b, config)
+            .await?;
+    benchmark_redis_stream_ack_into(&ack, runner, &stream, &group, &reclaimed_ids).await?;
+
+    Ok(RedisStreamPressureReport {
+        append,
+        read_group,
+        reclaim,
+        ack: ack.summarize().await,
+    })
+}
+
+async fn benchmark_redis_stream_append(
+    runner: &RedisStreamRunner,
+    stream: &RedisStreamName,
+    config: &DependencyPressureBaselineConfig,
+) -> Result<OperationSummary, Box<dyn std::error::Error>> {
+    let collector = Arc::new(SummaryCollector::default());
+    let next = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+
+    stream::iter(0..config.redis_stream_concurrency)
+        .for_each_concurrent(config.redis_stream_concurrency, |_| {
+            let runner = runner.clone();
+            let stream = stream.clone();
+            let collector = collector.clone();
+            let next = next.clone();
+            async move {
+                loop {
+                    let index = next.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
+                    if index >= config.redis_stream_total {
+                        break;
+                    }
+                    let started = Instant::now();
+                    let result = runner
+                        .append_json(
+                            &stream,
+                            "payload",
+                            &serde_json::json!({
+                                "job_id": index,
+                                "kind": "dependency-pressure",
+                            }),
+                        )
+                        .await;
+                    collector
+                        .record(
+                            started.elapsed(),
+                            usize::from(result.is_ok()),
+                            result.is_err(),
+                        )
+                        .await;
+                }
+            }
+        })
+        .await;
+
+    Ok(collector.summarize().await)
+}
+
+async fn benchmark_redis_stream_read_group(
+    runner: &RedisStreamRunner,
+    stream: &RedisStreamName,
+    group: &RedisConsumerGroup,
+    consumer: &RedisConsumerName,
+    config: &DependencyPressureBaselineConfig,
+) -> Result<(OperationSummary, Vec<String>), Box<dyn std::error::Error>> {
+    let collector = SummaryCollector::default();
+    let mut ids = Vec::with_capacity(config.redis_stream_total);
+
+    while ids.len() < config.redis_stream_total {
+        let started = Instant::now();
+        match runner.read_group(stream, group, consumer).await {
+            Ok(entries) => {
+                let item_count = entries.len();
+                ids.extend(entries.into_iter().map(|entry| entry.id));
+                collector.record(started.elapsed(), item_count, false).await;
+            }
+            Err(_) => {
+                collector.record(started.elapsed(), 0, true).await;
+            }
+        }
+    }
+
+    Ok((collector.summarize().await, ids))
+}
+
+async fn benchmark_redis_stream_reclaim(
+    runner: &RedisStreamRunner,
+    stream: &RedisStreamName,
+    group: &RedisConsumerGroup,
+    consumer_a: &RedisConsumerName,
+    consumer_b: &RedisConsumerName,
+    config: &DependencyPressureBaselineConfig,
+) -> Result<(OperationSummary, Vec<String>), Box<dyn std::error::Error>> {
+    for index in 0..config.redis_reclaim_total {
+        runner
+            .append_json(
+                stream,
+                "payload",
+                &serde_json::json!({
+                    "job_id": format!("reclaim-{index}"),
+                    "kind": "dependency-pressure",
+                }),
+            )
+            .await
+            .map_err(std::io::Error::other)?;
+    }
+
+    let mut pending_ids = Vec::with_capacity(config.redis_reclaim_total);
+    while pending_ids.len() < config.redis_reclaim_total {
+        let entries = runner
+            .read_group(stream, group, consumer_a)
+            .await
+            .map_err(std::io::Error::other)?;
+        pending_ids.extend(entries.into_iter().map(|entry| entry.id));
+    }
+
+    tokio::time::sleep(config.redis_reclaim_min_idle + Duration::from_millis(20)).await;
+
+    let collector = SummaryCollector::default();
+    let mut reclaimed_ids = Vec::with_capacity(config.redis_reclaim_total);
+    let mut next_start_id = "0-0".to_string();
+
+    while reclaimed_ids.len() < config.redis_reclaim_total {
+        let started = Instant::now();
+        match runner
+            .claim_stale(
+                stream,
+                group,
+                consumer_b,
+                &next_start_id,
+                RedisStreamReclaimConfig {
+                    min_idle_ms: config.redis_reclaim_min_idle.as_millis() as u64,
+                    count: 64,
+                },
+            )
+            .await
+        {
+            Ok(result) => {
+                next_start_id = result.next_start_id.clone();
+                let item_count = result.entries.len();
+                reclaimed_ids.extend(result.entries.into_iter().map(|entry| entry.id));
+                collector.record(started.elapsed(), item_count, false).await;
+            }
+            Err(_) => {
+                collector.record(started.elapsed(), 0, true).await;
+            }
+        }
+    }
+
+    Ok((collector.summarize().await, reclaimed_ids))
+}
+
+async fn benchmark_redis_stream_ack_into(
+    collector: &SummaryCollector,
+    runner: &RedisStreamRunner,
+    stream: &RedisStreamName,
+    group: &RedisConsumerGroup,
+    ids: &[String],
+) -> Result<(), Box<dyn std::error::Error>> {
+    for chunk in ids.chunks(64) {
+        let started = Instant::now();
+        match runner.ack(stream, group, chunk).await {
+            Ok(acked) => {
+                collector.record(started.elapsed(), acked, false).await;
+            }
+            Err(_) => {
+                collector.record(started.elapsed(), 0, true).await;
+            }
+        }
+    }
+
+    Ok(())
+}
+
+async fn benchmark_postgres_lease(
+    runner: &aether_data::driver::postgres::PostgresLeaseRunner,
+    config: &DependencyPressureBaselineConfig,
+) -> Result<PostgresLeasePressureReport, Box<dyn std::error::Error>> {
+    let spec = PostgresLeaseClaimSpec {
+        table: "baseline_lease_jobs",
+        id_column: "id",
+        lease_owner_column: "lease_owner",
+        lease_expires_at_column: "lease_expires_at",
+        eligibility_predicate_sql: "status = 'ready'",
+        order_by_sql: "id ASC",
+    };
+    let claim_options = PostgresLeaseClaimOptions {
+        batch_size: config.postgres_lease_batch_size,
+        lease_ms: config.postgres_lease_ms,
+    };
+
+    let claim = Arc::new(SummaryCollector::default());
+    let renew = Arc::new(SummaryCollector::default());
+    let release = Arc::new(SummaryCollector::default());
+    let next_cycle = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+
+    stream::iter(0..config.postgres_lease_concurrency)
+        .for_each_concurrent(config.postgres_lease_concurrency, |worker_index| {
+            let runner = runner.clone();
+            let spec = spec.clone();
+            let claim = claim.clone();
+            let renew = renew.clone();
+            let release = release.clone();
+            let next_cycle = next_cycle.clone();
+            async move {
+                let owner = format!("lease-owner-{worker_index}");
+                loop {
+                    let cycle = next_cycle.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
+                    if cycle >= config.postgres_lease_cycles {
+                        break;
+                    }
+
+                    let claim_started = Instant::now();
+                    match runner.claim_ids(&spec, claim_options, &owner).await {
+                        Ok(ids) => {
+                            let item_count = ids.len();
+                            claim
+                                .record(claim_started.elapsed(), item_count, false)
+                                .await;
+                            if ids.is_empty() {
+                                tokio::time::sleep(Duration::from_millis(5)).await;
+                                continue;
+                            }
+                            record_postgres_lease_follow_up(
+                                &runner,
+                                &spec,
+                                &owner,
+                                &ids,
+                                config.postgres_lease_ms,
+                                &renew,
+                                &release,
+                            )
+                            .await;
+                        }
+                        Err(_) => {
+                            claim.record(claim_started.elapsed(), 0, true).await;
+                        }
+                    }
+                }
+            }
+        })
+        .await;
+
+    Ok(PostgresLeasePressureReport {
+        claim: claim.summarize().await,
+        renew: renew.summarize().await,
+        release: release.summarize().await,
+    })
+}
+
+async fn record_postgres_lease_follow_up(
+    runner: &aether_data::driver::postgres::PostgresLeaseRunner,
+    spec: &PostgresLeaseClaimSpec,
+    owner: &str,
+    ids: &[DatabaseRecordId],
+    lease_ms: u64,
+    renew: &SummaryCollector,
+    release: &SummaryCollector,
+) {
+    let renew_started = Instant::now();
+    match runner.renew_ids(spec, ids, owner, lease_ms).await {
+        Ok(renewed) => {
+            let item_count = renewed.len();
+            renew
+                .record(renew_started.elapsed(), item_count, false)
+                .await;
+        }
+        Err(_) => {
+            renew.record(renew_started.elapsed(), 0, true).await;
+        }
+    }
+
+    let release_started = Instant::now();
+    match runner.release_ids(spec, ids, owner).await {
+        Ok(released) => {
+            let item_count = released.len();
+            release
+                .record(release_started.elapsed(), item_count, false)
+                .await;
+        }
+        Err(_) => {
+            release.record(release_started.elapsed(), 0, true).await;
+        }
+    }
+}
+
+fn summarize_latencies(latencies: &[u64]) -> (u64, u64, u64, u64) {
+    if latencies.is_empty() {
+        return (0, 0, 0, 0);
+    }
+    let max_ms = *latencies.last().unwrap_or(&0);
+    let mean_ms = latencies.iter().sum::<u64>() / latencies.len() as u64;
+    let p50_ms = percentile(latencies, 50);
+    let p95_ms = percentile(latencies, 95);
+    (p50_ms, p95_ms, max_ms, mean_ms)
+}
+
+fn percentile(latencies: &[u64], percentile: u8) -> u64 {
+    if latencies.is_empty() {
+        return 0;
+    }
+    let last_index = latencies.len() - 1;
+    let rank = ((last_index as f64) * (percentile as f64 / 100.0)).round() as usize;
+    latencies[rank.min(last_index)]
+}
+
+fn parse_args(
+    args: Vec<String>,
+) -> Result<DependencyPressureBaselineConfig, Box<dyn std::error::Error>> {
+    let mut config = DependencyPressureBaselineConfig::default();
+    let mut iter = args.into_iter();
+    while let Some(arg) = iter.next() {
+        match arg.as_str() {
+            "--redis-lock-total" => {
+                config.redis_lock_total = iter
+                    .next()
+                    .ok_or("missing value for --redis-lock-total")?
+                    .parse()?;
+            }
+            "--redis-lock-concurrency" => {
+                config.redis_lock_concurrency = iter
+                    .next()
+                    .ok_or("missing value for --redis-lock-concurrency")?
+                    .parse()?;
+            }
+            "--redis-stream-total" => {
+                config.redis_stream_total = iter
+                    .next()
+                    .ok_or("missing value for --redis-stream-total")?
+                    .parse()?;
+            }
+            "--redis-stream-concurrency" => {
+                config.redis_stream_concurrency = iter
+                    .next()
+                    .ok_or("missing value for --redis-stream-concurrency")?
+                    .parse()?;
+            }
+            "--redis-reclaim-total" => {
+                config.redis_reclaim_total = iter
+                    .next()
+                    .ok_or("missing value for --redis-reclaim-total")?
+                    .parse()?;
+            }
+            "--redis-reclaim-min-idle-ms" => {
+                config.redis_reclaim_min_idle = Duration::from_millis(
+                    iter.next()
+                        .ok_or("missing value for --redis-reclaim-min-idle-ms")?
+                        .parse()?,
+                );
+            }
+            "--postgres-rows" => {
+                config.postgres_rows = iter
+                    .next()
+                    .ok_or("missing value for --postgres-rows")?
+                    .parse()?;
+            }
+            "--postgres-lease-cycles" => {
+                config.postgres_lease_cycles = iter
+                    .next()
+                    .ok_or("missing value for --postgres-lease-cycles")?
+                    .parse()?;
+            }
+            "--postgres-lease-concurrency" => {
+                config.postgres_lease_concurrency = iter
+                    .next()
+                    .ok_or("missing value for --postgres-lease-concurrency")?
+                    .parse()?;
+            }
+            "--postgres-lease-batch-size" => {
+                config.postgres_lease_batch_size = iter
+                    .next()
+                    .ok_or("missing value for --postgres-lease-batch-size")?
+                    .parse()?;
+            }
+            "--postgres-lease-ms" => {
+                config.postgres_lease_ms = iter
+                    .next()
+                    .ok_or("missing value for --postgres-lease-ms")?
+                    .parse()?;
+            }
+            "--timeout-ms" => {
+                config.timeout = Duration::from_millis(
+                    iter.next()
+                        .ok_or("missing value for --timeout-ms")?
+                        .parse()?,
+                );
+            }
+            "--output" => {
+                config.output_path = Some(PathBuf::from(
+                    iter.next().ok_or("missing value for --output")?,
+                ));
+            }
+            "--redis-url" => {
+                config.redis_url = Some(iter.next().ok_or("missing value for --redis-url")?);
+            }
+            "--postgres-url" => {
+                config.postgres_url = Some(iter.next().ok_or("missing value for --postgres-url")?);
+            }
+            other => {
+                return Err(format!("unknown argument: {other}").into());
+            }
+        }
+    }
+    validate_config(&config)?;
+    Ok(config)
+}
+
+fn validate_config(
+    config: &DependencyPressureBaselineConfig,
+) -> Result<(), Box<dyn std::error::Error>> {
+    if config.redis_lock_total == 0
+        || config.redis_lock_concurrency == 0
+        || config.redis_stream_total == 0
+        || config.redis_stream_concurrency == 0
+        || config.redis_reclaim_total == 0
+        || config.postgres_rows == 0
+        || config.postgres_lease_cycles == 0
+        || config.postgres_lease_concurrency == 0
+        || config.postgres_lease_batch_size == 0
+        || config.postgres_lease_ms == 0
+        || config.timeout.is_zero()
+    {
+        return Err("all dependency pressure baseline numeric settings must be positive".into());
+    }
+    Ok(())
+}

crates/aether-testing/integration/src/bin/failure_recovery_baseline.rs

@@ -0,0 +1,734 @@
+// Gateway-backed benchmark scenarios live outside the reusable testkit.
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use aether_data::driver::postgres::{
+    PostgresLeaseClaimOptions, PostgresLeaseClaimSpec, PostgresLeaseRunnerConfig,
+    PostgresPoolConfig, PostgresTransactionOptions,
+};
+use aether_data::{DataLayerError, PostgresBackend};
+use aether_runtime_state::{RedisClientConfig, RedisLockRunner, RedisLockRunnerConfig};
+use aether_testkit::{
+    init_test_runtime_for, reserve_local_port, BenchmarkRuntimeSampler, BenchmarkRuntimeSnapshot,
+    ManagedPostgresServer, ManagedRedisServer, TunnelHarness, TunnelHarnessConfig,
+};
+use futures_util::{FutureExt, StreamExt};
+use serde::Serialize;
+use tokio::sync::{oneshot, Mutex};
+use tokio_tungstenite::tungstenite::client::IntoClientRequest;
+use tokio_tungstenite::tungstenite::Message;
+
+const PROXY_TUNNEL_PATH: &str = "/api/internal/proxy-tunnel";
+
+#[derive(Debug, Clone)]
+struct FailureRecoveryBaselineConfig {
+    redis_attempts: usize,
+    redis_concurrency: usize,
+    redis_restart_delay: Duration,
+    redis_downtime: Duration,
+    postgres_statement_timeout: Duration,
+    postgres_sleep: Duration,
+    tunnel_attempts: usize,
+    tunnel_concurrency: usize,
+    tunnel_hold: Duration,
+    tunnel_restart_delay: Duration,
+    tunnel_downtime: Duration,
+    timeout: Duration,
+    output_path: Option<PathBuf>,
+    redis_url: Option<String>,
+    postgres_url: Option<String>,
+}
+
+impl Default for FailureRecoveryBaselineConfig {
+    fn default() -> Self {
+        Self {
+            redis_attempts: 400,
+            redis_concurrency: 10,
+            redis_restart_delay: Duration::from_millis(200),
+            redis_downtime: Duration::from_millis(150),
+            postgres_statement_timeout: Duration::from_millis(50),
+            postgres_sleep: Duration::from_millis(200),
+            tunnel_attempts: 60,
+            tunnel_concurrency: 4,
+            tunnel_hold: Duration::from_millis(50),
+            tunnel_restart_delay: Duration::from_millis(200),
+            tunnel_downtime: Duration::from_millis(150),
+            timeout: Duration::from_secs(10),
+            output_path: None,
+            redis_url: None,
+            postgres_url: None,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+enum FaultPhase {
+    Pre,
+    During,
+    Post,
+}
+
+#[derive(Debug, Default, Clone, Serialize)]
+struct PhaseCounts {
+    pre_successes: usize,
+    pre_failures: usize,
+    during_successes: usize,
+    during_failures: usize,
+    post_successes: usize,
+    post_failures: usize,
+}
+
+#[derive(Debug, Clone, Serialize)]
+struct RecoverySummary {
+    total_attempts: usize,
+    successful_attempts: usize,
+    failed_attempts: usize,
+    recovered_after_restart_ms: Option<u64>,
+    p50_ms: u64,
+    p95_ms: u64,
+    p99_ms: u64,
+    max_ms: u64,
+    mean_ms: u64,
+    phase_counts: PhaseCounts,
+    runtime: BenchmarkRuntimeSnapshot,
+}
+
+#[derive(Debug, Clone, Serialize)]
+struct PostgresSlowQueryRecoveryReport {
+    slow_query_timed_out: bool,
+    slow_query_latency_ms: u64,
+    recovery_claim_succeeded: bool,
+    recovery_claim_latency_ms: u64,
+    recovery_claimed_items: usize,
+    runtime: BenchmarkRuntimeSnapshot,
+}
+
+#[derive(Debug, Clone, Serialize)]
+struct FailureRecoveryBaselineReport {
+    suite: &'static str,
+    redis_url: String,
+    postgres_url: String,
+    redis_restart: RecoverySummary,
+    postgres_slow_query: PostgresSlowQueryRecoveryReport,
+    tunnel_restart: RecoverySummary,
+}
+
+#[derive(Default)]
+struct RecoveryCollector {
+    latencies_ms: Mutex<Vec<u64>>,
+    phase_counts: Mutex<PhaseCounts>,
+    successful_attempts: AtomicUsize,
+    failed_attempts: AtomicUsize,
+}
+
+impl RecoveryCollector {
+    async fn record(&self, phase: FaultPhase, success: bool, latency: Duration) {
+        self.latencies_ms
+            .lock()
+            .await
+            .push(latency.as_millis() as u64);
+        let mut counts = self.phase_counts.lock().await;
+        match (phase, success) {
+            (FaultPhase::Pre, true) => counts.pre_successes += 1,
+            (FaultPhase::Pre, false) => counts.pre_failures += 1,
+            (FaultPhase::During, true) => counts.during_successes += 1,
+            (FaultPhase::During, false) => counts.during_failures += 1,
+            (FaultPhase::Post, true) => counts.post_successes += 1,
+            (FaultPhase::Post, false) => counts.post_failures += 1,
+        }
+        if success {
+            self.successful_attempts.fetch_add(1, Ordering::AcqRel);
+        } else {
+            self.failed_attempts.fetch_add(1, Ordering::AcqRel);
+        }
+    }
+
+    async fn summarize(
+        &self,
+        recovered_after_restart_ms: Option<u64>,
+        runtime: BenchmarkRuntimeSnapshot,
+    ) -> RecoverySummary {
+        let mut latencies = self.latencies_ms.lock().await.clone();
+        latencies.sort_unstable();
+        let (p50_ms, p95_ms, p99_ms, max_ms, mean_ms) = summarize_latencies(&latencies);
+        let phase_counts = self.phase_counts.lock().await.clone();
+        RecoverySummary {
+            total_attempts: self.successful_attempts.load(Ordering::Acquire)
+                + self.failed_attempts.load(Ordering::Acquire),
+            successful_attempts: self.successful_attempts.load(Ordering::Acquire),
+            failed_attempts: self.failed_attempts.load(Ordering::Acquire),
+            recovered_after_restart_ms,
+            p50_ms,
+            p95_ms,
+            p99_ms,
+            max_ms,
+            mean_ms,
+            phase_counts,
+            runtime,
+        }
+    }
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    init_test_runtime_for("failure-recovery-baseline");
+    let config = parse_args(std::env::args().skip(1).collect())?;
+    let report = run_suite(&config).await?;
+    let raw = serde_json::to_string_pretty(&report)?;
+    println!("{raw}");
+    if let Some(path) = config.output_path.as_ref() {
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        std::fs::write(path, format!("{raw}\n"))?;
+    }
+    Ok(())
+}
+
+async fn run_suite(
+    config: &FailureRecoveryBaselineConfig,
+) -> Result<FailureRecoveryBaselineReport, Box<dyn std::error::Error>> {
+    let managed_redis = if config.redis_url.is_none() {
+        Some(ManagedRedisServer::start().await?)
+    } else {
+        None
+    };
+    let managed_postgres = if config.postgres_url.is_none() {
+        Some(ManagedPostgresServer::start().await?)
+    } else {
+        None
+    };
+    let redis_url = config
+        .redis_url
+        .clone()
+        .or_else(|| {
+            managed_redis
+                .as_ref()
+                .map(|server| server.redis_url().to_string())
+        })
+        .expect("redis url should resolve");
+    let postgres_url = config
+        .postgres_url
+        .clone()
+        .or_else(|| {
+            managed_postgres
+                .as_ref()
+                .map(|server| server.database_url().to_string())
+        })
+        .expect("postgres url should resolve");
+
+    let redis_server = Arc::new(Mutex::new(
+        managed_redis.ok_or("failure recovery baseline requires managed redis")?,
+    ));
+    let postgres_server =
+        managed_postgres.ok_or("failure recovery baseline requires managed postgres")?;
+
+    let redis_restart = benchmark_redis_restart_recovery(redis_server.clone(), config).await?;
+    let postgres_slow_query =
+        benchmark_postgres_slow_query_recovery(postgres_server.database_url(), config).await?;
+    let tunnel_restart = benchmark_tunnel_restart_recovery(config).await?;
+
+    Ok(FailureRecoveryBaselineReport {
+        suite: "failure_recovery_baseline",
+        redis_url,
+        postgres_url,
+        redis_restart,
+        postgres_slow_query,
+        tunnel_restart,
+    })
+}
+
+async fn benchmark_redis_restart_recovery(
+    redis_server: Arc<Mutex<ManagedRedisServer>>,
+    config: &FailureRecoveryBaselineConfig,
+) -> Result<RecoverySummary, Box<dyn std::error::Error>> {
+    let mut runtime_sampler = BenchmarkRuntimeSampler::new();
+    let redis_url = redis_server.lock().await.redis_url().to_string();
+    let redis_config = RedisClientConfig {
+        url: redis_url,
+        key_prefix: Some(format!("aether-failure-recovery-{}", std::process::id())),
+    };
+    let keyspace = redis_config.keyspace();
+    let runner = RedisLockRunner::from_config(
+        redis_config,
+        RedisLockRunnerConfig {
+            command_timeout_ms: Some(250),
+            default_ttl_ms: 1_000,
+        },
+    )
+    .await?;
+    let collector = Arc::new(RecoveryCollector::default());
+    let next_attempt = Arc::new(AtomicUsize::new(0));
+    let phase = Arc::new(AtomicUsize::new(0));
+    let recovered_after_restart_ms = Arc::new(AtomicU64::new(0));
+
+    let absolute_restart_started = Arc::new(Mutex::new(None::<Instant>));
+    let restart_phase = phase.clone();
+    let restart_started = absolute_restart_started.clone();
+    let server_for_restart = redis_server.clone();
+    let redis_restart_delay = config.redis_restart_delay;
+    let redis_downtime = config.redis_downtime;
+    let restart_task = tokio::spawn(async move {
+        tokio::time::sleep(redis_restart_delay).await;
+        restart_phase.store(1, Ordering::Release);
+        *restart_started.lock().await = Some(Instant::now());
+        {
+            let mut server = server_for_restart.lock().await;
+            server.stop().map_err(std::io::Error::other)?;
+        }
+        tokio::time::sleep(redis_downtime).await;
+        {
+            let mut server = server_for_restart.lock().await;
+            server
+                .restart()
+                .await
+                .map_err(|err| std::io::Error::other(err.to_string()))?;
+        }
+        restart_phase.store(2, Ordering::Release);
+        Ok::<(), std::io::Error>(())
+    });
+
+    let mut tasks = tokio::task::JoinSet::new();
+    for _ in 0..config.redis_concurrency {
+        let runner = runner.clone();
+        let keyspace = keyspace.clone();
+        let collector = collector.clone();
+        let next_attempt = next_attempt.clone();
+        let phase = phase.clone();
+        let restart_started = absolute_restart_started.clone();
+        let recovered_after_restart_ms = recovered_after_restart_ms.clone();
+        let total_attempts = config.redis_attempts;
+        tasks.spawn(async move {
+            loop {
+                let current = next_attempt.fetch_add(1, Ordering::AcqRel);
+                if current >= total_attempts {
+                    break;
+                }
+                let current_phase = classify_phase(phase.load(Ordering::Acquire));
+                let key = keyspace.lock_key(&format!("recovery-lock-{current}"));
+                let owner = format!("redis-owner-{current}");
+                let started = Instant::now();
+                let success = match runner.try_acquire(&key, &owner, Some(1_000)).await {
+                    Ok(Some(lease)) => runner.release(&lease).await.unwrap_or(false),
+                    Ok(None) => false,
+                    Err(_) => false,
+                };
+                if success && matches!(current_phase, FaultPhase::Post) {
+                    if let Some(restart_started_at) = *restart_started.lock().await {
+                        let elapsed = restart_started_at.elapsed().as_millis() as u64;
+                        let _ = recovered_after_restart_ms.compare_exchange(
+                            0,
+                            elapsed,
+                            Ordering::AcqRel,
+                            Ordering::Acquire,
+                        );
+                    }
+                }
+                collector
+                    .record(current_phase, success, started.elapsed())
+                    .await;
+                tokio::time::sleep(Duration::from_millis(10)).await;
+            }
+        });
+    }
+    while let Some(result) = tasks.join_next().await {
+        result.map_err(std::io::Error::other)?;
+    }
+    restart_task
+        .await
+        .map_err(|err| format!("redis restart task failed: {err}"))?
+        .map_err(std::io::Error::other)?;
+
+    Ok(collector
+        .summarize(
+            load_optional_atomic_u64(&recovered_after_restart_ms),
+            runtime_sampler.snapshot(),
+        )
+        .await)
+}
+
+async fn benchmark_postgres_slow_query_recovery(
+    postgres_url: &str,
+    config: &FailureRecoveryBaselineConfig,
+) -> Result<PostgresSlowQueryRecoveryReport, Box<dyn std::error::Error>> {
+    let mut runtime_sampler = BenchmarkRuntimeSampler::new();
+    let backend = PostgresBackend::from_config(PostgresPoolConfig {
+        database_url: postgres_url.to_string(),
+        min_connections: 1,
+        max_connections: 8,
+        acquire_timeout_ms: config.timeout.as_millis() as u64,
+        idle_timeout_ms: 60_000,
+        max_lifetime_ms: 10 * 60_000,
+        statement_cache_capacity: 64,
+        require_ssl: false,
+    })?;
+    bootstrap_failure_recovery_lease_table(backend.pool_clone()).await?;
+    let transaction_runner = backend.transaction_runner();
+    let lease_runner = backend.lease_runner(PostgresLeaseRunnerConfig {
+        statement_timeout_ms: Some(config.timeout.as_millis() as u64),
+        lock_timeout_ms: Some(1_000),
+    })?;
+    let postgres_sleep_secs = config.postgres_sleep.as_secs_f64();
+
+    let slow_query_started = Instant::now();
+    let slow_query_timed_out = transaction_runner
+        .run(
+            PostgresTransactionOptions {
+                statement_timeout_ms: Some(config.postgres_statement_timeout.as_millis() as u64),
+                ..PostgresTransactionOptions::read_write()
+            },
+            |tx| {
+                async move {
+                    sqlx::query("SELECT pg_sleep($1::double precision)")
+                        .bind(postgres_sleep_secs)
+                        .execute(&mut **tx)
+                        .await
+                        .map_err(DataLayerError::postgres)?;
+                    Ok(())
+                }
+                .boxed()
+            },
+        )
+        .await
+        .is_err();
+    let slow_query_latency_ms = slow_query_started.elapsed().as_millis() as u64;
+
+    let recovery_claim_started = Instant::now();
+    let claimed_ids = lease_runner
+        .claim_ids(
+            &PostgresLeaseClaimSpec {
+                table: "baseline_failure_lease_jobs",
+                id_column: "id",
+                lease_owner_column: "lease_owner",
+                lease_expires_at_column: "lease_expires_at",
+                eligibility_predicate_sql: "status = 'ready'",
+                order_by_sql: "id ASC",
+            },
+            PostgresLeaseClaimOptions {
+                batch_size: 8,
+                lease_ms: 250,
+            },
+            "recovery-owner",
+        )
+        .await
+        .unwrap_or_default();
+    let recovery_claim_latency_ms = recovery_claim_started.elapsed().as_millis() as u64;
+
+    Ok(PostgresSlowQueryRecoveryReport {
+        slow_query_timed_out,
+        slow_query_latency_ms,
+        recovery_claim_succeeded: !claimed_ids.is_empty(),
+        recovery_claim_latency_ms,
+        recovery_claimed_items: claimed_ids.len(),
+        runtime: runtime_sampler.snapshot(),
+    })
+}
+
+async fn bootstrap_failure_recovery_lease_table(
+    pool: sqlx::PgPool,
+) -> Result<(), Box<dyn std::error::Error>> {
+    sqlx::query("DROP TABLE IF EXISTS baseline_failure_lease_jobs")
+        .execute(&pool)
+        .await?;
+    sqlx::query(
+        "CREATE TABLE baseline_failure_lease_jobs (
+             id TEXT PRIMARY KEY,
+             status TEXT NOT NULL,
+             updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+             lease_owner TEXT,
+             lease_expires_at TIMESTAMPTZ
+         )",
+    )
+    .execute(&pool)
+    .await?;
+
+    let mut builder =
+        sqlx::QueryBuilder::new("INSERT INTO baseline_failure_lease_jobs (id, status) ");
+    builder.push_values(0..32, |mut row, index| {
+        row.push_bind(format!("recovery-job-{index:03}"))
+            .push_bind("ready");
+    });
+    builder.build().execute(&pool).await?;
+    Ok(())
+}
+
+async fn benchmark_tunnel_restart_recovery(
+    config: &FailureRecoveryBaselineConfig,
+) -> Result<RecoverySummary, Box<dyn std::error::Error>> {
+    let mut runtime_sampler = BenchmarkRuntimeSampler::new();
+    let port = reserve_local_port()?;
+    let tunnel_config = TunnelHarnessConfig::default();
+    let initial_tunnel = TunnelHarness::start_on_port(tunnel_config.clone(), port).await?;
+    let ws_url = format!("ws://127.0.0.1:{port}{PROXY_TUNNEL_PATH}");
+    let collector = Arc::new(RecoveryCollector::default());
+    let next_attempt = Arc::new(AtomicUsize::new(0));
+    let phase = Arc::new(AtomicUsize::new(0));
+    let recovered_after_restart_ms = Arc::new(AtomicU64::new(0));
+    let restart_started = Arc::new(Mutex::new(None::<Instant>));
+    let (done_tx, done_rx) = oneshot::channel::<()>();
+
+    let tunnel_restart_delay = config.tunnel_restart_delay;
+    let tunnel_downtime = config.tunnel_downtime;
+    let phase_for_restart = phase.clone();
+    let restart_started_for_task = restart_started.clone();
+    let restart_task = tokio::spawn(async move {
+        tokio::time::sleep(tunnel_restart_delay).await;
+        phase_for_restart.store(1, Ordering::Release);
+        *restart_started_for_task.lock().await = Some(Instant::now());
+        drop(initial_tunnel);
+        tokio::time::sleep(tunnel_downtime).await;
+        let restarted_tunnel = start_tunnel_on_port_retry(tunnel_config, port).await?;
+        phase_for_restart.store(2, Ordering::Release);
+        let _ = done_rx.await;
+        drop(restarted_tunnel);
+        Ok::<(), String>(())
+    });
+
+    let mut workers = tokio::task::JoinSet::new();
+    for worker_index in 0..config.tunnel_concurrency {
+        let ws_url = ws_url.clone();
+        let next_attempt = next_attempt.clone();
+        let collector = collector.clone();
+        let phase = phase.clone();
+        let recovered_after_restart_ms = recovered_after_restart_ms.clone();
+        let restart_started = restart_started.clone();
+        let timeout = config.timeout;
+        let hold = config.tunnel_hold;
+        let total_attempts = config.tunnel_attempts;
+        workers.spawn(async move {
+            loop {
+                let current = next_attempt.fetch_add(1, Ordering::AcqRel);
+                if current >= total_attempts {
+                    break;
+                }
+                let current_phase = classify_phase(phase.load(Ordering::Acquire));
+                let mut request = ws_url
+                    .clone()
+                    .into_client_request()
+                    .map_err(|err| format!("failed to build websocket request: {err}"))?;
+                request.headers_mut().insert(
+                    "x-node-id",
+                    format!("recovery-node-{worker_index}-{current}")
+                        .parse()
+                        .map_err(|err| format!("failed to build x-node-id header: {err}"))?,
+                );
+                request.headers_mut().insert(
+                    aether_contracts::tunnel::TUNNEL_PROTOCOL_VERSION_HEADER,
+                    aether_contracts::tunnel::CURRENT_TUNNEL_PROTOCOL_VERSION_STR
+                        .parse()
+                        .expect("protocol version header value should be valid"),
+                );
+                request.headers_mut().insert(
+                    "x-node-name",
+                    format!("recovery-node-{worker_index}-{current}")
+                        .parse()
+                        .map_err(|err| format!("failed to build x-node-name header: {err}"))?,
+                );
+
+                let started = Instant::now();
+                let success =
+                    match tokio::time::timeout(timeout, tokio_tungstenite::connect_async(request))
+                        .await
+                    {
+                        Ok(Ok((mut ws, _))) => {
+                            tokio::time::sleep(hold).await;
+                            let _ = ws.close(None).await;
+                            while let Some(message) = ws.next().await {
+                                if matches!(message, Ok(Message::Close(_))) || message.is_err() {
+                                    break;
+                                }
+                            }
+                            true
+                        }
+                        _ => false,
+                    };
+                if success && matches!(current_phase, FaultPhase::Post) {
+                    if let Some(restart_started_at) = *restart_started.lock().await {
+                        let elapsed = restart_started_at.elapsed().as_millis() as u64;
+                        let _ = recovered_after_restart_ms.compare_exchange(
+                            0,
+                            elapsed,
+                            Ordering::AcqRel,
+                            Ordering::Acquire,
+                        );
+                    }
+                }
+                collector
+                    .record(current_phase, success, started.elapsed())
+                    .await;
+                tokio::time::sleep(Duration::from_millis(10)).await;
+            }
+            Ok::<(), String>(())
+        });
+    }
+
+    while let Some(result) = workers.join_next().await {
+        result
+            .map_err(|err| format!("tunnel recovery worker task failed: {err}"))?
+            .map_err(|err| format!("tunnel recovery worker failed: {err}"))?;
+    }
+    let _ = done_tx.send(());
+    restart_task
+        .await
+        .map_err(|err| format!("tunnel restart task failed: {err}"))?
+        .map_err(std::io::Error::other)?;
+
+    Ok(collector
+        .summarize(
+            load_optional_atomic_u64(&recovered_after_restart_ms),
+            runtime_sampler.snapshot(),
+        )
+        .await)
+}
+
+async fn start_tunnel_on_port_retry(
+    config: TunnelHarnessConfig,
+    port: u16,
+) -> Result<TunnelHarness, String> {
+    let mut attempts = 0usize;
+    loop {
+        match TunnelHarness::start_on_port(config.clone(), port).await {
+            Ok(tunnel) => return Ok(tunnel),
+            Err(err) => {
+                attempts += 1;
+                if attempts >= 20 {
+                    return Err(format!(
+                        "failed to restart tunnel on fixed port {port}: {err}"
+                    ));
+                }
+                tokio::time::sleep(Duration::from_millis(50)).await;
+            }
+        }
+    }
+}
+
+fn classify_phase(value: usize) -> FaultPhase {
+    match value {
+        0 => FaultPhase::Pre,
+        1 => FaultPhase::During,
+        _ => FaultPhase::Post,
+    }
+}
+
+fn load_optional_atomic_u64(value: &AtomicU64) -> Option<u64> {
+    match value.load(Ordering::Acquire) {
+        0 => None,
+        millis => Some(millis),
+    }
+}
+
+fn summarize_latencies(latencies: &[u64]) -> (u64, u64, u64, u64, u64) {
+    if latencies.is_empty() {
+        return (0, 0, 0, 0, 0);
+    }
+    let max_ms = *latencies.last().unwrap_or(&0);
+    let mean_ms = latencies.iter().sum::<u64>() / latencies.len() as u64;
+    let p50_ms = percentile(latencies, 50);
+    let p95_ms = percentile(latencies, 95);
+    let p99_ms = percentile(latencies, 99);
+    (p50_ms, p95_ms, p99_ms, max_ms, mean_ms)
+}
+
+fn percentile(latencies: &[u64], percentile: u8) -> u64 {
+    if latencies.is_empty() {
+        return 0;
+    }
+    let last_index = latencies.len() - 1;
+    let rank = ((last_index as f64) * (percentile as f64 / 100.0)).round() as usize;
+    latencies[rank.min(last_index)]
+}
+
+fn parse_args(
+    args: Vec<String>,
+) -> Result<FailureRecoveryBaselineConfig, Box<dyn std::error::Error>> {
+    let mut config = FailureRecoveryBaselineConfig::default();
+    let mut iter = args.into_iter();
+    while let Some(arg) = iter.next() {
+        match arg.as_str() {
+            "--redis-attempts" => {
+                config.redis_attempts = next_value(&mut iter, "--redis-attempts")?.parse()?
+            }
+            "--redis-concurrency" => {
+                config.redis_concurrency = next_value(&mut iter, "--redis-concurrency")?.parse()?
+            }
+            "--redis-restart-delay-ms" => {
+                config.redis_restart_delay = Duration::from_millis(
+                    next_value(&mut iter, "--redis-restart-delay-ms")?.parse()?,
+                )
+            }
+            "--redis-downtime-ms" => {
+                config.redis_downtime =
+                    Duration::from_millis(next_value(&mut iter, "--redis-downtime-ms")?.parse()?)
+            }
+            "--postgres-statement-timeout-ms" => {
+                config.postgres_statement_timeout = Duration::from_millis(
+                    next_value(&mut iter, "--postgres-statement-timeout-ms")?.parse()?,
+                )
+            }
+            "--postgres-sleep-ms" => {
+                config.postgres_sleep =
+                    Duration::from_millis(next_value(&mut iter, "--postgres-sleep-ms")?.parse()?)
+            }
+            "--tunnel-attempts" => {
+                config.tunnel_attempts = next_value(&mut iter, "--tunnel-attempts")?.parse()?
+            }
+            "--tunnel-concurrency" => {
+                config.tunnel_concurrency =
+                    next_value(&mut iter, "--tunnel-concurrency")?.parse()?
+            }
+            "--tunnel-hold-ms" => {
+                config.tunnel_hold =
+                    Duration::from_millis(next_value(&mut iter, "--tunnel-hold-ms")?.parse()?)
+            }
+            "--tunnel-restart-delay-ms" => {
+                config.tunnel_restart_delay = Duration::from_millis(
+                    next_value(&mut iter, "--tunnel-restart-delay-ms")?.parse()?,
+                )
+            }
+            "--tunnel-downtime-ms" => {
+                config.tunnel_downtime =
+                    Duration::from_millis(next_value(&mut iter, "--tunnel-downtime-ms")?.parse()?)
+            }
+            "--timeout-ms" => {
+                config.timeout =
+                    Duration::from_millis(next_value(&mut iter, "--timeout-ms")?.parse()?)
+            }
+            "--redis-url" => config.redis_url = Some(next_value(&mut iter, "--redis-url")?),
+            "--postgres-url" => {
+                config.postgres_url = Some(next_value(&mut iter, "--postgres-url")?)
+            }
+            "--output" => {
+                config.output_path = Some(PathBuf::from(next_value(&mut iter, "--output")?))
+            }
+            other => return Err(format!("unknown argument: {other}").into()),
+        }
+    }
+    validate_config(&config)?;
+    Ok(config)
+}
+
+fn next_value(
+    iter: &mut impl Iterator<Item = String>,
+    flag: &str,
+) -> Result<String, Box<dyn std::error::Error>> {
+    iter.next()
+        .ok_or_else(|| format!("missing value for {flag}").into())
+}
+
+fn validate_config(
+    config: &FailureRecoveryBaselineConfig,
+) -> Result<(), Box<dyn std::error::Error>> {
+    if config.redis_attempts == 0
+        || config.redis_concurrency == 0
+        || config.postgres_statement_timeout.is_zero()
+        || config.postgres_sleep.is_zero()
+        || config.tunnel_attempts == 0
+        || config.tunnel_concurrency == 0
+        || config.timeout.is_zero()
+    {
+        return Err("all failure recovery baseline numeric settings must be positive".into());
+    }
+    Ok(())
+}

crates/aether-testing/integration/src/bin/gateway_openai_chat_c_compare.rs

@@ -0,0 +1,921 @@
+// Gateway-backed benchmark scenarios live outside the reusable testkit.
+use std::collections::BTreeMap;
+use std::convert::Infallible;
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use aether_gateway::build_router_with_state;
+use aether_gateway::testkit::{build_openai_chat_pressure_state, OpenAiChatPressureStateConfig};
+use aether_testkit::{
+    fetch_prometheus_samples, run_http_load_probe, run_multi_url_http_load_probe,
+    HttpLoadProbeConfig, HttpLoadProbeResponseMode, HttpLoadProbeResult,
+    MultiUrlHttpLoadProbeResult, PrometheusSample, SpawnedServer,
+};
+use axum::body::{to_bytes, Body, Bytes};
+use axum::extract::State;
+use axum::http::header::{AUTHORIZATION, CACHE_CONTROL, CONTENT_TYPE};
+use axum::http::{Request, StatusCode};
+use axum::response::{IntoResponse, Response};
+use axum::routing::{get, post};
+use axum::Router;
+use reqwest::Method;
+use serde::Serialize;
+
+const DEFAULT_REQUESTS: usize = 10_000;
+const DEFAULT_C1K: usize = 1_000;
+const DEFAULT_C1W: usize = 10_000;
+const DEFAULT_TARGETS: usize = 4;
+const DEFAULT_CLIENT_SHARDS: usize = 512;
+const DEFAULT_TIMEOUT_MS: u64 = 60_000;
+const DEFAULT_WARMUP_REQUESTS: usize = 1_000;
+const DEFAULT_WARMUP_CONCURRENCY: usize = 50;
+const BODY_LIMIT: usize = 1024 * 1024;
+
+#[derive(Debug, Clone)]
+struct Config {
+    requests: usize,
+    c1k: usize,
+    c1w: usize,
+    targets: usize,
+    client_shards: usize,
+    timeout_ms: u64,
+    warmup_requests: usize,
+    warmup_concurrency: usize,
+    first_body_hold_ms: u64,
+    output_path: Option<PathBuf>,
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Self {
+            requests: DEFAULT_REQUESTS,
+            c1k: DEFAULT_C1K,
+            c1w: DEFAULT_C1W,
+            targets: DEFAULT_TARGETS,
+            client_shards: DEFAULT_CLIENT_SHARDS,
+            timeout_ms: DEFAULT_TIMEOUT_MS,
+            warmup_requests: DEFAULT_WARMUP_REQUESTS,
+            warmup_concurrency: DEFAULT_WARMUP_CONCURRENCY,
+            first_body_hold_ms: 0,
+            output_path: None,
+        }
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct CompareReport {
+    suite: &'static str,
+    gateway_base_url: String,
+    gateway_metrics_url: String,
+    upstream_targets: Vec<String>,
+    config: ReportConfig,
+    warmup: Option<HttpLoadProbeResult>,
+    c1k: RunReport,
+    c1w: RunReport,
+    mock_only: MockOnlyCompareReport,
+}
+
+#[derive(Debug, Serialize)]
+struct ReportConfig {
+    requests: usize,
+    c1k: usize,
+    c1w: usize,
+    targets: usize,
+    client_shards: usize,
+    timeout_ms: u64,
+    warmup_requests: usize,
+    warmup_concurrency: usize,
+    first_body_hold_ms: u64,
+}
+
+#[derive(Debug, Serialize)]
+struct RunReport {
+    label: String,
+    load: HttpLoadProbeResult,
+    gateway_metrics: GatewayMetricDelta,
+    mock_metrics: Vec<MockMetricDelta>,
+}
+
+#[derive(Debug, Serialize)]
+struct MockOnlyCompareReport {
+    c1k: MultiUrlHttpLoadProbeResult,
+    c1w: MultiUrlHttpLoadProbeResult,
+}
+
+#[derive(Debug, Default, Serialize)]
+struct GatewayMetricDelta {
+    stream_pre_first_byte_spawn_total: u64,
+    request_candidate_queue_dropped_total: u64,
+    raw_candidates_scanned_total: u64,
+    payload_build_selected_total: u64,
+    payload_build_prefetch_avoided_total: u64,
+    selected_rank_sum: u64,
+    model_directive_cache_hit_total: u64,
+    model_directive_cache_miss_total: u64,
+    redaction_request_cache_hit_total: u64,
+    redaction_request_cache_miss_total: u64,
+    target_raw_seen_total: BTreeMap<String, u64>,
+    target_preselect_total: BTreeMap<String, u64>,
+    target_selected_total: BTreeMap<String, u64>,
+    target_max_in_flight: BTreeMap<String, u64>,
+    target_saturated_total: BTreeMap<String, u64>,
+    stage_deltas: BTreeMap<String, StageMetricDelta>,
+}
+
+#[derive(Debug, Default, Serialize)]
+struct StageMetricDelta {
+    count: u64,
+    sum_ms: u64,
+    max_ms_after: u64,
+    p95_bucket_ms: Option<u64>,
+    p99_bucket_ms: Option<u64>,
+}
+
+#[derive(Debug, Clone, Default, Serialize)]
+struct MockMetricSnapshot {
+    requests_total: u64,
+    completed_total: u64,
+    in_flight: u64,
+    max_in_flight: u64,
+    first_chunk_yield_total: u64,
+    request_body_read_sum_ms: u64,
+    request_body_read_max_ms: u64,
+    response_header_to_first_chunk_sum_ms: u64,
+    response_header_to_first_chunk_max_ms: u64,
+}
+
+#[derive(Debug, Serialize)]
+struct MockMetricDelta {
+    target: String,
+    requests_total: u64,
+    completed_total: u64,
+    in_flight_after: u64,
+    max_in_flight_after: u64,
+    first_chunk_yield_total: u64,
+    request_body_read_sum_ms: u64,
+    request_body_read_max_ms_after: u64,
+    response_header_to_first_chunk_sum_ms: u64,
+    response_header_to_first_chunk_max_ms_after: u64,
+}
+
+#[derive(Debug, Default)]
+struct MockTargetMetrics {
+    requests_total: AtomicU64,
+    completed_total: AtomicU64,
+    in_flight: AtomicUsize,
+    max_in_flight: AtomicUsize,
+    first_chunk_yield_total: AtomicU64,
+    request_body_read_sum_ms: AtomicU64,
+    request_body_read_max_ms: AtomicU64,
+    response_header_to_first_chunk_sum_ms: AtomicU64,
+    response_header_to_first_chunk_max_ms: AtomicU64,
+}
+
+#[derive(Clone)]
+struct MockTargetState {
+    label: String,
+    metrics: Arc<MockTargetMetrics>,
+    first_chunk_delay: Duration,
+}
+
+struct CompletionGuard {
+    metrics: Option<Arc<MockTargetMetrics>>,
+}
+
+impl CompletionGuard {
+    fn new(metrics: Arc<MockTargetMetrics>) -> Self {
+        Self {
+            metrics: Some(metrics),
+        }
+    }
+}
+
+impl Drop for CompletionGuard {
+    fn drop(&mut self) {
+        if let Some(metrics) = self.metrics.take() {
+            record_mock_completed(&metrics);
+        }
+    }
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let config = parse_args(std::env::args().skip(1).collect())?;
+    if config.targets == 0 {
+        return Err("targets must be positive".into());
+    }
+
+    let mut mock_servers = Vec::with_capacity(config.targets);
+    let mut mock_metrics = Vec::with_capacity(config.targets);
+    for index in 0..config.targets {
+        let metrics = Arc::new(MockTargetMetrics::default());
+        let state = MockTargetState {
+            label: format!("target-{index}"),
+            metrics: Arc::clone(&metrics),
+            first_chunk_delay: Duration::ZERO,
+        };
+        let server = SpawnedServer::start(mock_router(state)).await?;
+        mock_servers.push(server);
+        mock_metrics.push(metrics);
+    }
+
+    let upstream_targets = mock_servers
+        .iter()
+        .map(|server| format!("{}/v1", server.base_url()))
+        .collect::<Vec<_>>();
+    let mock_chat_urls = mock_servers
+        .iter()
+        .map(|server| format!("{}/v1/chat/completions", server.base_url()))
+        .collect::<Vec<_>>();
+    let mock_health_url = format!("{}/health", mock_servers[0].base_url());
+    let gateway_state = build_openai_chat_pressure_state(OpenAiChatPressureStateConfig::new(
+        upstream_targets.clone(),
+    ))
+    .map_err(std::io::Error::other)?;
+    let gateway = SpawnedServer::start(build_router_with_state(gateway_state)).await?;
+    let gateway_url = format!("{}/v1/chat/completions", gateway.base_url());
+    let gateway_health_url = format!("{}/_gateway/health", gateway.base_url());
+    let gateway_metrics_url = format!("{}/_gateway/metrics", gateway.base_url());
+
+    let warmup = if config.warmup_requests > 0 {
+        Some(
+            run_http_load_probe(&load_config(
+                &gateway_url,
+                &gateway_health_url,
+                &config,
+                config.warmup_requests,
+                config.warmup_concurrency,
+                0,
+            ))
+            .await
+            .map_err(std::io::Error::other)?,
+        )
+    } else {
+        None
+    };
+
+    let before_c1k = fetch_prometheus_samples(&gateway_metrics_url)
+        .await
+        .map_err(std::io::Error::other)?;
+    let mock_before_c1k = snapshot_mock_metrics(&mock_metrics);
+    let c1k_load = run_http_load_probe(&load_config(
+        &gateway_url,
+        &gateway_health_url,
+        &config,
+        config.requests,
+        config.c1k,
+        config.first_body_hold_ms,
+    ))
+    .await
+    .map_err(std::io::Error::other)?;
+    let after_c1k = fetch_prometheus_samples(&gateway_metrics_url)
+        .await
+        .map_err(std::io::Error::other)?;
+    let mock_after_c1k = snapshot_mock_metrics(&mock_metrics);
+
+    let before_c1w = after_c1k.clone();
+    let mock_before_c1w = mock_after_c1k.clone();
+    let c1w_load = run_http_load_probe(&load_config(
+        &gateway_url,
+        &gateway_health_url,
+        &config,
+        config.requests,
+        config.c1w,
+        config.first_body_hold_ms,
+    ))
+    .await
+    .map_err(std::io::Error::other)?;
+    let after_c1w = fetch_prometheus_samples(&gateway_metrics_url)
+        .await
+        .map_err(std::io::Error::other)?;
+    let mock_after_c1w = snapshot_mock_metrics(&mock_metrics);
+    let mock_only_c1k = run_multi_url_http_load_probe(
+        &load_config(
+            &mock_chat_urls[0],
+            &mock_health_url,
+            &config,
+            config.requests,
+            config.c1k,
+            config.first_body_hold_ms,
+        ),
+        &mock_chat_urls,
+    )
+    .await
+    .map_err(std::io::Error::other)?;
+    let mock_only_c1w = run_multi_url_http_load_probe(
+        &load_config(
+            &mock_chat_urls[0],
+            &mock_health_url,
+            &config,
+            config.requests,
+            config.c1w,
+            config.first_body_hold_ms,
+        ),
+        &mock_chat_urls,
+    )
+    .await
+    .map_err(std::io::Error::other)?;
+
+    let report = CompareReport {
+        suite: "gateway_openai_chat_c_compare",
+        gateway_base_url: gateway.base_url().to_string(),
+        gateway_metrics_url,
+        upstream_targets,
+        config: ReportConfig {
+            requests: config.requests,
+            c1k: config.c1k,
+            c1w: config.c1w,
+            targets: config.targets,
+            client_shards: config.client_shards,
+            timeout_ms: config.timeout_ms,
+            warmup_requests: config.warmup_requests,
+            warmup_concurrency: config.warmup_concurrency,
+            first_body_hold_ms: config.first_body_hold_ms,
+        },
+        warmup,
+        c1k: RunReport {
+            label: format!("c{}", config.c1k),
+            load: c1k_load,
+            gateway_metrics: gateway_metric_delta(&before_c1k, &after_c1k),
+            mock_metrics: mock_metric_delta(&mock_before_c1k, &mock_after_c1k),
+        },
+        c1w: RunReport {
+            label: format!("c{}", config.c1w),
+            load: c1w_load,
+            gateway_metrics: gateway_metric_delta(&before_c1w, &after_c1w),
+            mock_metrics: mock_metric_delta(&mock_before_c1w, &mock_after_c1w),
+        },
+        mock_only: MockOnlyCompareReport {
+            c1k: mock_only_c1k,
+            c1w: mock_only_c1w,
+        },
+    };
+
+    let raw = serde_json::to_string_pretty(&report)?;
+    println!("{raw}");
+    if let Some(path) = config.output_path.as_ref() {
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        std::fs::write(path, format!("{raw}\n"))?;
+    }
+
+    Ok(())
+}
+
+fn load_config(
+    gateway_url: &str,
+    gateway_health_url: &str,
+    config: &Config,
+    requests: usize,
+    concurrency: usize,
+    first_body_hold_ms: u64,
+) -> HttpLoadProbeConfig {
+    let mut headers = BTreeMap::new();
+    headers.insert(
+        CONTENT_TYPE.as_str().to_string(),
+        "application/json".to_string(),
+    );
+    headers.insert(
+        AUTHORIZATION.as_str().to_string(),
+        "Bearer sk-aether-openai-chat-pressure".to_string(),
+    );
+    HttpLoadProbeConfig {
+        url: gateway_url.to_string(),
+        warmup_url: Some(gateway_health_url.to_string()),
+        method: Method::POST,
+        headers,
+        header_sets: Vec::new(),
+        body: Some(
+            serde_json::json!({
+                "model": "gpt-5",
+                "messages": [{"role": "user", "content": "hello"}],
+                "stream": true
+            })
+            .to_string()
+            .into_bytes(),
+        ),
+        total_requests: requests,
+        concurrency,
+        warmup_connections: 0,
+        timeout: Duration::from_millis(config.timeout_ms),
+        connect_timeout: Some(Duration::from_millis(10_000)),
+        response_mode: HttpLoadProbeResponseMode::FirstBodyByte,
+        client_shards: config.client_shards,
+        pool_max_idle_per_host: Some(config.requests.max(concurrency).max(1024)),
+        start_ramp: Duration::ZERO,
+        http1_only: true,
+        http2_prior_knowledge: false,
+        first_body_hold: Duration::from_millis(first_body_hold_ms),
+    }
+}
+
+fn mock_router(state: MockTargetState) -> Router {
+    Router::new()
+        .route("/health", get(|| async { "ok" }))
+        .route("/v1/chat/completions", post(mock_chat_completions))
+        .with_state(state)
+}
+
+async fn mock_chat_completions(
+    State(state): State<MockTargetState>,
+    request: Request<Body>,
+) -> Response {
+    record_mock_started(&state.metrics);
+    let body_started_at = Instant::now();
+    let body = match to_bytes(request.into_body(), BODY_LIMIT).await {
+        Ok(body) => body,
+        Err(err) => {
+            record_mock_completed(&state.metrics);
+            return (
+                StatusCode::BAD_REQUEST,
+                format!("failed to read mock request body: {err}"),
+            )
+                .into_response();
+        }
+    };
+    record_mock_request_body_read(&state.metrics, body_started_at.elapsed());
+
+    let wants_stream = request_wants_stream(&body);
+    let model = request_model(&body).unwrap_or_else(|| "gpt-5-upstream".to_string());
+    if !wants_stream {
+        record_mock_completed(&state.metrics);
+        return axum::Json(serde_json::json!({
+            "id": "chatcmpl-pressure-mock",
+            "object": "chat.completion",
+            "model": model,
+            "choices": [{
+                "index": 0,
+                "message": {"role": "assistant", "content": "ok"},
+                "finish_reason": "stop"
+            }]
+        }))
+        .into_response();
+    }
+
+    let metrics = Arc::clone(&state.metrics);
+    let first_chunk_delay = state.first_chunk_delay;
+    let label = state.label.clone();
+    let stream = async_stream::stream! {
+        let _guard = CompletionGuard::new(Arc::clone(&metrics));
+        let header_created_at = Instant::now();
+        if !first_chunk_delay.is_zero() {
+            tokio::time::sleep(first_chunk_delay).await;
+        }
+        record_mock_first_chunk_yield(&metrics, header_created_at.elapsed());
+        let chunk = serde_json::json!({
+            "id": format!("chatcmpl-pressure-mock-{label}"),
+            "object": "chat.completion.chunk",
+            "model": model,
+            "choices": [{
+                "index": 0,
+                "delta": {"role": "assistant", "content": "x"},
+                "finish_reason": null
+            }]
+        });
+        yield Ok::<Bytes, Infallible>(Bytes::from(format!("data: {chunk}\n\n")));
+        yield Ok::<Bytes, Infallible>(Bytes::from_static(
+            b"data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n",
+        ));
+        yield Ok::<Bytes, Infallible>(Bytes::from_static(b"data: [DONE]\n\n"));
+    };
+
+    let mut response = Response::new(Body::from_stream(stream));
+    response.headers_mut().insert(
+        CONTENT_TYPE,
+        "text/event-stream; charset=utf-8"
+            .parse()
+            .expect("valid content-type"),
+    );
+    response.headers_mut().insert(
+        CACHE_CONTROL,
+        "no-cache".parse().expect("valid cache-control"),
+    );
+    response
+}
+
+fn request_wants_stream(body: &[u8]) -> bool {
+    serde_json::from_slice::<serde_json::Value>(body)
+        .ok()
+        .and_then(|value| value.get("stream").and_then(serde_json::Value::as_bool))
+        .unwrap_or(false)
+}
+
+fn request_model(body: &[u8]) -> Option<String> {
+    serde_json::from_slice::<serde_json::Value>(body)
+        .ok()
+        .and_then(|value| {
+            value
+                .get("model")
+                .and_then(serde_json::Value::as_str)
+                .map(str::to_string)
+        })
+}
+
+fn record_mock_started(metrics: &MockTargetMetrics) {
+    metrics.requests_total.fetch_add(1, Ordering::AcqRel);
+    let in_flight = metrics.in_flight.fetch_add(1, Ordering::AcqRel) + 1;
+    metrics.max_in_flight.fetch_max(in_flight, Ordering::AcqRel);
+}
+
+fn record_mock_completed(metrics: &MockTargetMetrics) {
+    metrics.completed_total.fetch_add(1, Ordering::AcqRel);
+    metrics.in_flight.fetch_sub(1, Ordering::AcqRel);
+}
+
+fn record_mock_request_body_read(metrics: &MockTargetMetrics, elapsed: Duration) {
+    let elapsed_ms = elapsed.as_millis() as u64;
+    metrics
+        .request_body_read_sum_ms
+        .fetch_add(elapsed_ms, Ordering::AcqRel);
+    metrics
+        .request_body_read_max_ms
+        .fetch_max(elapsed_ms, Ordering::AcqRel);
+}
+
+fn record_mock_first_chunk_yield(metrics: &MockTargetMetrics, elapsed: Duration) {
+    let elapsed_ms = elapsed.as_millis() as u64;
+    metrics
+        .first_chunk_yield_total
+        .fetch_add(1, Ordering::AcqRel);
+    metrics
+        .response_header_to_first_chunk_sum_ms
+        .fetch_add(elapsed_ms, Ordering::AcqRel);
+    metrics
+        .response_header_to_first_chunk_max_ms
+        .fetch_max(elapsed_ms, Ordering::AcqRel);
+}
+
+fn snapshot_mock_metrics(metrics: &[Arc<MockTargetMetrics>]) -> Vec<MockMetricSnapshot> {
+    metrics
+        .iter()
+        .map(|metrics| MockMetricSnapshot {
+            requests_total: metrics.requests_total.load(Ordering::Acquire),
+            completed_total: metrics.completed_total.load(Ordering::Acquire),
+            in_flight: metrics.in_flight.load(Ordering::Acquire) as u64,
+            max_in_flight: metrics.max_in_flight.load(Ordering::Acquire) as u64,
+            first_chunk_yield_total: metrics.first_chunk_yield_total.load(Ordering::Acquire),
+            request_body_read_sum_ms: metrics.request_body_read_sum_ms.load(Ordering::Acquire),
+            request_body_read_max_ms: metrics.request_body_read_max_ms.load(Ordering::Acquire),
+            response_header_to_first_chunk_sum_ms: metrics
+                .response_header_to_first_chunk_sum_ms
+                .load(Ordering::Acquire),
+            response_header_to_first_chunk_max_ms: metrics
+                .response_header_to_first_chunk_max_ms
+                .load(Ordering::Acquire),
+        })
+        .collect()
+}
+
+fn mock_metric_delta(
+    before: &[MockMetricSnapshot],
+    after: &[MockMetricSnapshot],
+) -> Vec<MockMetricDelta> {
+    after
+        .iter()
+        .enumerate()
+        .map(|(index, after)| {
+            let before = before.get(index).cloned().unwrap_or_default();
+            MockMetricDelta {
+                target: format!("target-{index}"),
+                requests_total: after.requests_total.saturating_sub(before.requests_total),
+                completed_total: after.completed_total.saturating_sub(before.completed_total),
+                in_flight_after: after.in_flight,
+                max_in_flight_after: after.max_in_flight,
+                first_chunk_yield_total: after
+                    .first_chunk_yield_total
+                    .saturating_sub(before.first_chunk_yield_total),
+                request_body_read_sum_ms: after
+                    .request_body_read_sum_ms
+                    .saturating_sub(before.request_body_read_sum_ms),
+                request_body_read_max_ms_after: after.request_body_read_max_ms,
+                response_header_to_first_chunk_sum_ms: after
+                    .response_header_to_first_chunk_sum_ms
+                    .saturating_sub(before.response_header_to_first_chunk_sum_ms),
+                response_header_to_first_chunk_max_ms_after: after
+                    .response_header_to_first_chunk_max_ms,
+            }
+        })
+        .collect()
+}
+
+fn gateway_metric_delta(
+    before: &[PrometheusSample],
+    after: &[PrometheusSample],
+) -> GatewayMetricDelta {
+    let stage_names = [
+        "frontdoor_handler_queue",
+        "frontdoor_admission",
+        "frontdoor_context",
+        "frontdoor_body_buffer",
+        "frontdoor_owner_forward",
+        "frontdoor_auth_model",
+        "frontdoor_rpm",
+        "frontdoor_rpm_system_default",
+        "frontdoor_rpm_runtime_check",
+        "frontdoor_rpm_memory_fallback",
+        "frontdoor_local_ai_public",
+        "frontdoor_execute_stream",
+        "frontdoor_to_stream_response_ready",
+        "frontdoor_to_stream_body_first_poll",
+        "frontdoor_to_stream_first_client_yield",
+        "openai_chat_stream_target_select",
+        "openai_chat_payload_parts_prepare",
+        "openai_chat_payload_model_directives",
+        "openai_chat_payload_redaction",
+        "openai_chat_payload_auth_prepare",
+        "openai_chat_payload_body_build",
+        "stream_candidate_execute",
+        "upstream_execution_gate_wait",
+        "stream_upstream_target_admission",
+        "stream_upstream_headers",
+        "direct_reqwest_request_send",
+        "direct_send_headers",
+        "direct_h2c_response_headers_wait",
+        "stream_first_data",
+        "stream_first_client_yield",
+        "stream_total",
+    ];
+
+    GatewayMetricDelta {
+        stream_pre_first_byte_spawn_total: counter_delta(
+            before,
+            after,
+            "stream_pre_first_byte_spawn_total",
+            &[],
+        ),
+        request_candidate_queue_dropped_total: counter_delta(
+            before,
+            after,
+            "request_candidate_queue_dropped_total",
+            &[],
+        ),
+        raw_candidates_scanned_total: counter_delta(
+            before,
+            after,
+            "openai_chat_stream_target_select_raw_candidates_scanned_total",
+            &[],
+        ),
+        payload_build_selected_total: counter_delta(
+            before,
+            after,
+            "openai_chat_stream_payload_build_selected_total",
+            &[],
+        ),
+        payload_build_prefetch_avoided_total: counter_delta(
+            before,
+            after,
+            "openai_chat_stream_payload_build_prefetch_avoided_total",
+            &[],
+        ),
+        selected_rank_sum: counter_delta(
+            before,
+            after,
+            "openai_chat_stream_target_select_selected_rank_sum",
+            &[],
+        ),
+        model_directive_cache_hit_total: counter_delta(
+            before,
+            after,
+            "openai_chat_model_directive_cache_hit_total",
+            &[],
+        ),
+        model_directive_cache_miss_total: counter_delta(
+            before,
+            after,
+            "openai_chat_model_directive_cache_miss_total",
+            &[],
+        ),
+        redaction_request_cache_hit_total: counter_delta(
+            before,
+            after,
+            "chat_pii_redaction_request_cache_hit_total",
+            &[],
+        ),
+        redaction_request_cache_miss_total: counter_delta(
+            before,
+            after,
+            "chat_pii_redaction_request_cache_miss_total",
+            &[],
+        ),
+        target_raw_seen_total: metric_delta_by_target(
+            before,
+            after,
+            "upstream_target_raw_seen_total",
+        ),
+        target_preselect_total: metric_delta_by_target(
+            before,
+            after,
+            "upstream_target_preselect_total",
+        ),
+        target_selected_total: metric_delta_by_target(
+            before,
+            after,
+            "upstream_target_selected_total",
+        ),
+        target_max_in_flight: metric_after_by_target(after, "upstream_target_max_in_flight"),
+        target_saturated_total: metric_delta_by_target(
+            before,
+            after,
+            "upstream_target_saturated_total",
+        ),
+        stage_deltas: stage_names
+            .iter()
+            .map(|stage| {
+                (
+                    (*stage).to_string(),
+                    StageMetricDelta {
+                        count: counter_delta(
+                            before,
+                            after,
+                            "gateway_stage_latency_count",
+                            &[("stage", stage)],
+                        ),
+                        sum_ms: counter_delta(
+                            before,
+                            after,
+                            "gateway_stage_latency_sum_ms",
+                            &[("stage", stage)],
+                        ),
+                        max_ms_after: metric_value(
+                            after,
+                            "gateway_stage_latency_max_ms",
+                            &[("stage", stage)],
+                        )
+                        .unwrap_or_default(),
+                        p95_bucket_ms: stage_latency_bucket_quantile(before, after, stage, 0.95),
+                        p99_bucket_ms: stage_latency_bucket_quantile(before, after, stage, 0.99),
+                    },
+                )
+            })
+            .collect(),
+    }
+}
+
+fn stage_latency_bucket_quantile(
+    before: &[PrometheusSample],
+    after: &[PrometheusSample],
+    stage: &str,
+    quantile: f64,
+) -> Option<u64> {
+    let count = counter_delta(
+        before,
+        after,
+        "gateway_stage_latency_count",
+        &[("stage", stage)],
+    );
+    if count == 0 {
+        return None;
+    }
+    let threshold = ((count as f64) * quantile).ceil().max(1.0) as u64;
+    let mut buckets = after
+        .iter()
+        .filter(|sample| {
+            metric_name_matches(&sample.name, "gateway_stage_latency_bucket")
+                && labels_match(sample, &[("stage", stage)])
+        })
+        .filter_map(|sample| {
+            let le_ms = sample.labels.get("le_ms")?.parse::<u64>().ok()?;
+            let after_value = sample.value.parse::<u64>().ok()?;
+            let before_value = metric_value(
+                before,
+                "gateway_stage_latency_bucket",
+                &[("stage", stage), ("le_ms", sample.labels.get("le_ms")?)],
+            )
+            .unwrap_or_default();
+            Some((le_ms, after_value.saturating_sub(before_value)))
+        })
+        .collect::<Vec<_>>();
+    buckets.sort_unstable_by_key(|(le_ms, _)| *le_ms);
+    buckets
+        .into_iter()
+        .find_map(|(le_ms, value)| (value >= threshold).then_some(le_ms))
+}
+
+fn counter_delta(
+    before: &[PrometheusSample],
+    after: &[PrometheusSample],
+    metric_name: &str,
+    labels: &[(&str, &str)],
+) -> u64 {
+    let before = metric_value(before, metric_name, labels).unwrap_or_default();
+    let after = metric_value(after, metric_name, labels).unwrap_or_default();
+    after.saturating_sub(before)
+}
+
+fn metric_value(
+    samples: &[PrometheusSample],
+    metric_name: &str,
+    labels: &[(&str, &str)],
+) -> Option<u64> {
+    samples
+        .iter()
+        .find(|sample| {
+            metric_name_matches(&sample.name, metric_name) && labels_match(sample, labels)
+        })
+        .and_then(|sample| sample.value.parse::<u64>().ok())
+}
+
+fn metric_delta_by_target(
+    before: &[PrometheusSample],
+    after: &[PrometheusSample],
+    metric_name: &str,
+) -> BTreeMap<String, u64> {
+    let mut values = BTreeMap::new();
+    for sample in after
+        .iter()
+        .filter(|sample| metric_name_matches(&sample.name, metric_name))
+    {
+        let Some(target) = sample.labels.get("target") else {
+            continue;
+        };
+        let before_value =
+            metric_value(before, metric_name, &[("target", target)]).unwrap_or_default();
+        let after_value = sample.value.parse::<u64>().unwrap_or_default();
+        values.insert(target.clone(), after_value.saturating_sub(before_value));
+    }
+    values
+}
+
+fn metric_after_by_target(
+    samples: &[PrometheusSample],
+    metric_name: &str,
+) -> BTreeMap<String, u64> {
+    samples
+        .iter()
+        .filter(|sample| metric_name_matches(&sample.name, metric_name))
+        .filter_map(|sample| {
+            Some((
+                sample.labels.get("target")?.clone(),
+                sample.value.parse::<u64>().ok()?,
+            ))
+        })
+        .collect()
+}
+
+fn metric_name_matches(actual: &str, expected: &str) -> bool {
+    actual == expected || actual.ends_with(&format!("_{expected}"))
+}
+
+fn labels_match(sample: &PrometheusSample, labels: &[(&str, &str)]) -> bool {
+    labels
+        .iter()
+        .all(|(key, value)| sample.labels.get(*key).map(String::as_str) == Some(*value))
+}
+
+fn parse_args(args: Vec<String>) -> Result<Config, Box<dyn std::error::Error>> {
+    let mut config = Config::default();
+    let mut iter = args.into_iter();
+    while let Some(arg) = iter.next() {
+        match arg.as_str() {
+            "--requests" => config.requests = next_value(&mut iter, "--requests")?.parse()?,
+            "--c1k" => config.c1k = next_value(&mut iter, "--c1k")?.parse()?,
+            "--c1w" => config.c1w = next_value(&mut iter, "--c1w")?.parse()?,
+            "--targets" => config.targets = next_value(&mut iter, "--targets")?.parse()?,
+            "--client-shards" => {
+                config.client_shards = next_value(&mut iter, "--client-shards")?.parse()?
+            }
+            "--timeout-ms" => config.timeout_ms = next_value(&mut iter, "--timeout-ms")?.parse()?,
+            "--warmup-requests" => {
+                config.warmup_requests = next_value(&mut iter, "--warmup-requests")?.parse()?
+            }
+            "--warmup-concurrency" => {
+                config.warmup_concurrency =
+                    next_value(&mut iter, "--warmup-concurrency")?.parse()?
+            }
+            "--first-body-hold-ms" => {
+                config.first_body_hold_ms =
+                    next_value(&mut iter, "--first-body-hold-ms")?.parse()?
+            }
+            "--output" => {
+                config.output_path = Some(PathBuf::from(next_value(&mut iter, "--output")?))
+            }
+            "--help" | "-h" => {
+                print_help();
+                std::process::exit(0);
+            }
+            other => return Err(format!("unknown argument: {other}").into()),
+        }
+    }
+    Ok(config)
+}
+
+fn next_value(
+    iter: &mut impl Iterator<Item = String>,
+    flag: &str,
+) -> Result<String, Box<dyn std::error::Error>> {
+    iter.next()
+        .ok_or_else(|| format!("missing value for {flag}").into())
+}
+
+fn print_help() {
+    println!(
+        "gateway_openai_chat_c_compare \
+         [--requests N] [--c1k N] [--c1w N] [--targets N] \
+         [--client-shards N] [--timeout-ms N] \
+         [--warmup-requests N] [--warmup-concurrency N] \
+         [--first-body-hold-ms N] [--output PATH]"
+    );
+}

crates/aether-testing/integration/src/bin/gateway_pressure_seed.rs

@@ -0,0 +1,757 @@
+// Gateway-backed benchmark scenarios live outside the reusable testkit.
+use std::env;
+use std::fs;
+use std::path::PathBuf;
+
+use aether_data::repository::auth::CreateStandaloneApiKeyRecord;
+use aether_data::repository::wallet::WalletLookupKey;
+use aether_data::{
+    DataBackends, DataLayerConfig, DatabaseDriver, SqlDatabaseConfig, SqlPoolConfig,
+};
+use aether_data_contracts::repository::global_models::{
+    CreateAdminGlobalModelRecord, UpdateAdminGlobalModelRecord, UpsertAdminProviderModelRecord,
+};
+use aether_data_contracts::repository::provider_catalog::{
+    StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
+};
+use serde_json::json;
+use sha2::Digest;
+
+const DEFAULT_POSTGRES_URL: &str = "postgresql://postgres:[email protected]:5432/aether";
+const DEFAULT_OUTPUT_ENV_PATH: &str = "/tmp/aether_local_env.sh";
+const DEFAULT_OUTPUT_KEY_PATH: &str = "/tmp/aether_fullchain_api_key";
+const DEFAULT_OUTPUT_KEY_LIST_PATH: &str = "/tmp/aether_fullchain_api_keys";
+const DEFAULT_PROVIDER_ID: &str = "provider-local-pressure-openai";
+const DEFAULT_ENDPOINT_ID: &str = "endpoint-local-pressure-openai-chat";
+const DEFAULT_PROVIDER_KEY_ID: &str = "provider-key-local-pressure-openai";
+const DEFAULT_GLOBAL_MODEL_ID: &str = "gm-local-pressure-gpt-5-mini";
+const DEFAULT_MODEL_ID: &str = "model-local-pressure-gpt-5-mini";
+const DEFAULT_API_KEY_ID: &str = "api-key-local-pressure";
+const DEFAULT_OPERATOR_ID: &str = "pressure-local";
+const DEFAULT_MODEL: &str = "gpt-5-mini";
+const DEFAULT_MOCK_UPSTREAM_BASE_URL: &str = "http://127.0.0.1:18181/v1";
+const DEFAULT_GATEWAY_BASE_URL: &str = "http://127.0.0.1:8084";
+const DEFAULT_API_KEY: &str = "sk-aether-local-pressure";
+
+#[derive(Debug, Clone)]
+struct Config {
+    database_url: String,
+    output_env_path: PathBuf,
+    output_key_path: PathBuf,
+    output_key_list_path: PathBuf,
+    provider_id: String,
+    endpoint_id: String,
+    provider_key_id: String,
+    global_model_id: String,
+    model_id: String,
+    api_key_id: String,
+    operator_id: String,
+    model: String,
+    mock_upstream_base_url: String,
+    gateway_base_url: String,
+    api_key: String,
+    api_key_count: usize,
+    provider_api_key: String,
+    postgres_min_connections: u32,
+    postgres_max_connections: u32,
+}
+
+impl Config {
+    fn from_env_and_args() -> Result<Self, String> {
+        let mut config = Self {
+            database_url: env_value("DATABASE_URL")
+                .or_else(|| env_value("AETHER_DATABASE_URL"))
+                .or_else(|| env_value("AETHER_GATEWAY_DATA_POSTGRES_URL"))
+                .unwrap_or_else(|| DEFAULT_POSTGRES_URL.to_string()),
+            output_env_path: PathBuf::from(
+                env_value("OUTPUT_ENV_PATH").unwrap_or_else(|| DEFAULT_OUTPUT_ENV_PATH.to_string()),
+            ),
+            output_key_path: PathBuf::from(
+                env_value("OUTPUT_KEY_PATH").unwrap_or_else(|| DEFAULT_OUTPUT_KEY_PATH.to_string()),
+            ),
+            output_key_list_path: PathBuf::from(
+                env_value("OUTPUT_KEY_LIST_PATH")
+                    .or_else(|| env_value("PRESSURE_API_KEY_LIST_FILE"))
+                    .unwrap_or_else(|| DEFAULT_OUTPUT_KEY_LIST_PATH.to_string()),
+            ),
+            provider_id: env_value("PRESSURE_PROVIDER_ID")
+                .unwrap_or_else(|| DEFAULT_PROVIDER_ID.to_string()),
+            endpoint_id: env_value("PRESSURE_ENDPOINT_ID")
+                .unwrap_or_else(|| DEFAULT_ENDPOINT_ID.to_string()),
+            provider_key_id: env_value("PRESSURE_PROVIDER_KEY_ID")
+                .unwrap_or_else(|| DEFAULT_PROVIDER_KEY_ID.to_string()),
+            global_model_id: env_value("PRESSURE_GLOBAL_MODEL_ID")
+                .unwrap_or_else(|| DEFAULT_GLOBAL_MODEL_ID.to_string()),
+            model_id: env_value("PRESSURE_MODEL_ID")
+                .unwrap_or_else(|| DEFAULT_MODEL_ID.to_string()),
+            api_key_id: env_value("PRESSURE_API_KEY_ID")
+                .unwrap_or_else(|| DEFAULT_API_KEY_ID.to_string()),
+            operator_id: env_value("PRESSURE_OPERATOR_ID")
+                .unwrap_or_else(|| DEFAULT_OPERATOR_ID.to_string()),
+            model: env_value("PRESSURE_MODEL").unwrap_or_else(|| DEFAULT_MODEL.to_string()),
+            mock_upstream_base_url: env_value("PRESSURE_MOCK_UPSTREAM_BASE_URL")
+                .unwrap_or_else(|| DEFAULT_MOCK_UPSTREAM_BASE_URL.to_string()),
+            gateway_base_url: env_value("GATEWAY_BASE_URL")
+                .unwrap_or_else(|| DEFAULT_GATEWAY_BASE_URL.to_string()),
+            api_key: env_value("AETHER_API_KEY").unwrap_or_else(|| DEFAULT_API_KEY.to_string()),
+            api_key_count: env_value("PRESSURE_API_KEY_COUNT")
+                .and_then(|value| value.parse::<usize>().ok())
+                .unwrap_or(1),
+            provider_api_key: env_value("PRESSURE_PROVIDER_API_KEY")
+                .unwrap_or_else(|| "dummy-local-pressure-provider-key".to_string()),
+            postgres_min_connections: 1,
+            postgres_max_connections: 8,
+        };
+
+        let args = env::args().skip(1).collect::<Vec<_>>();
+        let mut index = 0;
+        while index < args.len() {
+            let arg = &args[index];
+            match arg.as_str() {
+                "--database-url" => config.database_url = arg_value(&args, &mut index, arg)?,
+                "--output-env" => {
+                    config.output_env_path = PathBuf::from(arg_value(&args, &mut index, arg)?)
+                }
+                "--output-key" => {
+                    config.output_key_path = PathBuf::from(arg_value(&args, &mut index, arg)?)
+                }
+                "--output-key-list" => {
+                    config.output_key_list_path = PathBuf::from(arg_value(&args, &mut index, arg)?)
+                }
+                "--provider-id" => config.provider_id = arg_value(&args, &mut index, arg)?,
+                "--endpoint-id" => config.endpoint_id = arg_value(&args, &mut index, arg)?,
+                "--provider-key-id" => config.provider_key_id = arg_value(&args, &mut index, arg)?,
+                "--global-model-id" => config.global_model_id = arg_value(&args, &mut index, arg)?,
+                "--model-id" => config.model_id = arg_value(&args, &mut index, arg)?,
+                "--api-key-id" => config.api_key_id = arg_value(&args, &mut index, arg)?,
+                "--operator-id" => config.operator_id = arg_value(&args, &mut index, arg)?,
+                "--model" => config.model = arg_value(&args, &mut index, arg)?,
+                "--mock-upstream-base-url" => {
+                    config.mock_upstream_base_url = arg_value(&args, &mut index, arg)?
+                }
+                "--gateway-base-url" => {
+                    config.gateway_base_url = arg_value(&args, &mut index, arg)?
+                }
+                "--api-key" => config.api_key = arg_value(&args, &mut index, arg)?,
+                "--api-key-count" => {
+                    config.api_key_count = parse_usize(&arg_value(&args, &mut index, arg)?, arg)?
+                }
+                "--provider-api-key" => {
+                    config.provider_api_key = arg_value(&args, &mut index, arg)?
+                }
+                "--postgres-min-connections" => {
+                    config.postgres_min_connections =
+                        parse_u32(&arg_value(&args, &mut index, arg)?, arg)?
+                }
+                "--postgres-max-connections" => {
+                    config.postgres_max_connections =
+                        parse_u32(&arg_value(&args, &mut index, arg)?, arg)?
+                }
+                "--help" | "-h" => {
+                    print_help();
+                    std::process::exit(0);
+                }
+                _ => return Err(format!("unknown argument: {arg}")),
+            }
+            index += 1;
+        }
+
+        config.validate()?;
+        Ok(config)
+    }
+
+    fn validate(&self) -> Result<(), String> {
+        for (name, value) in [
+            ("database_url", &self.database_url),
+            ("provider_id", &self.provider_id),
+            ("endpoint_id", &self.endpoint_id),
+            ("provider_key_id", &self.provider_key_id),
+            ("global_model_id", &self.global_model_id),
+            ("model_id", &self.model_id),
+            ("api_key_id", &self.api_key_id),
+            ("operator_id", &self.operator_id),
+            ("model", &self.model),
+            ("mock_upstream_base_url", &self.mock_upstream_base_url),
+            ("gateway_base_url", &self.gateway_base_url),
+            ("api_key", &self.api_key),
+        ] {
+            if value.trim().is_empty() {
+                return Err(format!("{name} cannot be empty"));
+            }
+        }
+        if self.postgres_min_connections > self.postgres_max_connections {
+            return Err("postgres min connections cannot exceed max connections".to_string());
+        }
+        if self.api_key_count == 0 {
+            return Err("api_key_count must be positive".to_string());
+        }
+        Ok(())
+    }
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let config = Config::from_env_and_args().map_err(|err| format!("invalid config: {err}"))?;
+
+    let backends = DataBackends::from_config(DataLayerConfig::from_database(SqlDatabaseConfig {
+        driver: DatabaseDriver::Postgres,
+        url: config.database_url.clone(),
+        pool: SqlPoolConfig {
+            min_connections: config.postgres_min_connections,
+            max_connections: config.postgres_max_connections,
+            acquire_timeout_ms: 5_000,
+            idle_timeout_ms: 30_000,
+            max_lifetime_ms: 300_000,
+            statement_cache_capacity: 128,
+            require_ssl: false,
+        },
+    }))?;
+
+    seed_provider_catalog(&backends, &config).await?;
+    seed_models(&backends, &config).await?;
+    let operator_user_id = seed_operator_user(&backends, &config).await?;
+    seed_api_keys(&backends, &config, &operator_user_id).await?;
+    verify_candidate_selection(&backends, &config).await?;
+    write_outputs(&config)?;
+
+    println!("gateway pressure seed complete");
+    println!("provider_id={}", config.provider_id);
+    println!("endpoint_id={}", config.endpoint_id);
+    println!("provider_key_id={}", config.provider_key_id);
+    println!("model={}", config.model);
+    println!("api_key_id={}", config.api_key_id);
+    println!("api_key_count={}", config.api_key_count);
+    println!("env written to {}", config.output_env_path.display());
+    println!("api key written to {}", config.output_key_path.display());
+    println!(
+        "api key list written to {}",
+        config.output_key_list_path.display()
+    );
+
+    Ok(())
+}
+
+async fn seed_provider_catalog(
+    backends: &DataBackends,
+    config: &Config,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let reader = backends
+        .read()
+        .provider_catalog()
+        .ok_or("provider catalog reader unavailable")?;
+    let writer = backends
+        .write()
+        .provider_catalog()
+        .ok_or("provider catalog writer unavailable")?;
+
+    let provider = StoredProviderCatalogProvider::new(
+        config.provider_id.clone(),
+        "Local pressure OpenAI mock".to_string(),
+        Some("http://127.0.0.1:18181".to_string()),
+        "openai".to_string(),
+    )?
+    .with_transport_fields(
+        true,
+        false,
+        false,
+        None,
+        Some(0),
+        None,
+        Some(120.0),
+        Some(30.0),
+        None,
+    )
+    .with_routing_fields(0)
+    .with_description(Some(
+        "Local OpenAI-compatible mock provider for gateway pressure tests".to_string(),
+    ));
+
+    if reader
+        .list_providers_by_ids(std::slice::from_ref(&config.provider_id))
+        .await?
+        .is_empty()
+    {
+        writer.create_provider(&provider, None).await?;
+    } else {
+        writer.update_provider(&provider).await?;
+    }
+
+    let endpoint = StoredProviderCatalogEndpoint::new(
+        config.endpoint_id.clone(),
+        config.provider_id.clone(),
+        "openai:chat".to_string(),
+        Some("openai".to_string()),
+        Some("chat_completions".to_string()),
+        true,
+    )?
+    .with_transport_fields(
+        config.mock_upstream_base_url.clone(),
+        None,
+        None,
+        Some(0),
+        None,
+        None,
+        None,
+        None,
+    )?
+    .with_health_score(1.0);
+
+    if reader
+        .list_endpoints_by_ids(std::slice::from_ref(&config.endpoint_id))
+        .await?
+        .is_empty()
+    {
+        writer.create_endpoint(&endpoint).await?;
+    } else {
+        writer.update_endpoint(&endpoint).await?;
+    }
+
+    let provider_key = StoredProviderCatalogKey::new(
+        config.provider_key_id.clone(),
+        config.provider_id.clone(),
+        "Local pressure mock key".to_string(),
+        "api_key".to_string(),
+        Some(json!({"streaming": true})),
+        true,
+    )?
+    .with_transport_fields(
+        Some(json!(["openai:chat"])),
+        Some(config.provider_api_key.clone()),
+        None,
+        None,
+        None,
+        Some(json!([config.model.clone()])),
+        None,
+        None,
+        None,
+    )?
+    .with_rate_limit_fields(None, None, None, None, None, None, None, None, None)
+    .with_health_fields(
+        Some(json!({"openai:chat": {"status": "healthy"}})),
+        Some(json!({"openai:chat": {"state": "closed"}})),
+    );
+
+    if reader
+        .list_keys_by_ids(std::slice::from_ref(&config.provider_key_id))
+        .await?
+        .is_empty()
+    {
+        writer.create_key(&provider_key).await?;
+    } else {
+        writer.update_key(&provider_key).await?;
+    }
+
+    Ok(())
+}
+
+async fn seed_models(
+    backends: &DataBackends,
+    config: &Config,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let reader = backends
+        .read()
+        .global_models()
+        .ok_or("global model reader unavailable")?;
+    let writer = backends
+        .write()
+        .global_models()
+        .ok_or("global model writer unavailable")?;
+
+    let capabilities = Some(json!({
+        "streaming": true,
+        "chat": true
+    }));
+    let global_config = Some(json!({
+        "model_mappings": [config.model],
+        "pressure_seed": true
+    }));
+
+    if reader
+        .get_admin_global_model_by_id(&config.global_model_id)
+        .await?
+        .is_some()
+    {
+        writer
+            .update_admin_global_model(&UpdateAdminGlobalModelRecord::new(
+                config.global_model_id.clone(),
+                config.model.clone(),
+                true,
+                Some(0.0),
+                None,
+                capabilities.clone(),
+                global_config.clone(),
+            )?)
+            .await?;
+    } else {
+        writer
+            .create_admin_global_model(&CreateAdminGlobalModelRecord::new(
+                config.global_model_id.clone(),
+                config.model.clone(),
+                config.model.clone(),
+                true,
+                Some(0.0),
+                None,
+                capabilities.clone(),
+                global_config.clone(),
+            )?)
+            .await?;
+    }
+
+    let provider_model_mappings = Some(json!([
+        {
+            "name": config.model,
+            "priority": 0,
+            "api_formats": ["openai:chat"],
+            "endpoint_ids": [config.endpoint_id]
+        }
+    ]));
+    let provider_model = UpsertAdminProviderModelRecord::new(
+        config.model_id.clone(),
+        config.provider_id.clone(),
+        config.global_model_id.clone(),
+        config.model.clone(),
+        provider_model_mappings,
+        Some(0.0),
+        None,
+        Some(false),
+        Some(false),
+        Some(true),
+        Some(false),
+        Some(false),
+        true,
+        true,
+        Some(json!({"pressure_seed": true})),
+    )?;
+
+    if reader
+        .list_admin_provider_models_by_global_model_id(&config.global_model_id)
+        .await?
+        .iter()
+        .any(|model| model.id == config.model_id)
+    {
+        writer.update_admin_provider_model(&provider_model).await?;
+    } else {
+        writer.create_admin_provider_model(&provider_model).await?;
+    }
+
+    Ok(())
+}
+
+async fn seed_api_keys(
+    backends: &DataBackends,
+    config: &Config,
+    operator_user_id: &str,
+) -> Result<(), Box<dyn std::error::Error>> {
+    for index in 0..config.api_key_count {
+        seed_api_key(backends, config, operator_user_id, index).await?;
+    }
+    Ok(())
+}
+
+async fn seed_api_key(
+    backends: &DataBackends,
+    config: &Config,
+    operator_user_id: &str,
+    key_index: usize,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let auth_reader = backends
+        .read()
+        .auth_api_keys()
+        .ok_or("auth api key reader unavailable")?;
+    let auth_writer = backends
+        .write()
+        .auth_api_keys()
+        .ok_or("auth api key writer unavailable")?;
+    let wallet_reader = backends
+        .read()
+        .wallets()
+        .ok_or("wallet reader unavailable")?;
+
+    let api_key_id = pressure_api_key_id(config, key_index);
+    let api_key_value = pressure_api_key_value(config, key_index);
+
+    let existing = auth_reader
+        .find_export_standalone_api_key_by_id(&api_key_id)
+        .await?;
+    if existing.is_none() {
+        auth_writer
+            .create_standalone_api_key(CreateStandaloneApiKeyRecord {
+                user_id: operator_user_id.to_string(),
+                api_key_id: api_key_id.clone(),
+                key_hash: sha256_hex(&api_key_value),
+                key_encrypted: Some(api_key_value),
+                name: Some(format!("Local pressure API key {}", key_index + 1)),
+                allowed_providers: Some(vec![config.provider_id.clone()]),
+                allowed_api_formats: Some(vec!["openai:chat".to_string()]),
+                allowed_models: Some(vec![config.model.clone()]),
+                ip_rules: None,
+                rate_limit: Some(0),
+                concurrent_limit: None,
+                force_capabilities: None,
+                is_active: true,
+                expires_at_unix_secs: None,
+                auto_delete_on_expiry: false,
+                total_requests: 0,
+                total_tokens: 0,
+                total_cost_usd: 0.0,
+            })
+            .await?;
+    } else {
+        auth_writer
+            .update_standalone_api_key_basic(
+                aether_data::repository::auth::UpdateStandaloneApiKeyBasicRecord {
+                    api_key_id: api_key_id.clone(),
+                    name: Some(format!("Local pressure API key {}", key_index + 1)),
+                    rate_limit_present: true,
+                    rate_limit: Some(0),
+                    concurrent_limit_present: true,
+                    concurrent_limit: None,
+                    allowed_providers: Some(Some(vec![config.provider_id.clone()])),
+                    allowed_api_formats: Some(Some(vec!["openai:chat".to_string()])),
+                    allowed_models: Some(Some(vec![config.model.clone()])),
+                    ip_rules: Some(None),
+                    expires_at_present: true,
+                    expires_at_unix_secs: None,
+                    auto_delete_on_expiry_present: true,
+                    auto_delete_on_expiry: false,
+                },
+            )
+            .await?;
+        auth_writer
+            .set_standalone_api_key_active(&api_key_id, true)
+            .await?;
+    }
+
+    if wallet_reader
+        .find(WalletLookupKey::ApiKeyId(&api_key_id))
+        .await?
+        .is_none()
+    {
+        wallet_reader
+            .initialize_auth_api_key_wallet(&api_key_id, 0.0, true)
+            .await?;
+    } else {
+        wallet_reader
+            .update_auth_api_key_wallet_limit_mode(&api_key_id, "unlimited")
+            .await?;
+    }
+
+    Ok(())
+}
+
+async fn seed_operator_user(
+    backends: &DataBackends,
+    config: &Config,
+) -> Result<String, Box<dyn std::error::Error>> {
+    let user_reader = backends.read().users().ok_or("user reader unavailable")?;
+    let wallet_reader = backends
+        .read()
+        .wallets()
+        .ok_or("wallet reader unavailable")?;
+    let username = format!("{}-user", config.operator_id);
+
+    let user = match user_reader.find_user_auth_by_username(&username).await? {
+        Some(user) => user_reader
+            .update_local_auth_user_admin_fields(
+                &user.id,
+                Some("admin".to_string()),
+                true,
+                Some(vec![config.provider_id.clone()]),
+                true,
+                Some(vec!["openai:chat".to_string()]),
+                true,
+                Some(vec![config.model.clone()]),
+                true,
+                None,
+                Some(true),
+            )
+            .await?
+            .unwrap_or(user),
+        None => user_reader
+            .create_local_auth_user_with_settings(
+                Some(format!("{}@local.pressure", config.operator_id)),
+                true,
+                username,
+                "local-pressure-password-disabled".to_string(),
+                "admin".to_string(),
+                Some(vec![config.provider_id.clone()]),
+                Some(vec!["openai:chat".to_string()]),
+                Some(vec![config.model.clone()]),
+                None,
+            )
+            .await?
+            .ok_or("failed to create pressure operator user")?,
+    };
+
+    if wallet_reader
+        .find(WalletLookupKey::UserId(&user.id))
+        .await?
+        .is_none()
+    {
+        wallet_reader
+            .initialize_auth_user_wallet(&user.id, 0.0, true)
+            .await?;
+    } else {
+        wallet_reader
+            .update_auth_user_wallet_limit_mode(&user.id, "unlimited")
+            .await?;
+    }
+
+    Ok(user.id)
+}
+
+async fn verify_candidate_selection(
+    backends: &DataBackends,
+    config: &Config,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let reader = backends
+        .read()
+        .minimal_candidate_selection()
+        .ok_or("candidate selection reader unavailable")?;
+    let rows = reader
+        .list_for_exact_api_format_and_requested_model("openai:chat", &config.model)
+        .await?;
+    let has_pressure_row = rows.iter().any(|row| {
+        row.provider_id == config.provider_id
+            && row.endpoint_id == config.endpoint_id
+            && row.key_id == config.provider_key_id
+            && row.global_model_id == config.global_model_id
+            && row.model_id == config.model_id
+            && row.provider_is_active
+            && row.endpoint_is_active
+            && row.key_is_active
+            && row.model_is_active
+            && row.model_is_available
+    });
+    if !has_pressure_row {
+        return Err(format!(
+            "seeded candidate not visible for model {} and openai:chat",
+            config.model
+        )
+        .into());
+    }
+    Ok(())
+}
+
+fn write_outputs(config: &Config) -> Result<(), Box<dyn std::error::Error>> {
+    if let Some(parent) = config.output_env_path.parent() {
+        fs::create_dir_all(parent)?;
+    }
+    if let Some(parent) = config.output_key_path.parent() {
+        fs::create_dir_all(parent)?;
+    }
+    if let Some(parent) = config.output_key_list_path.parent() {
+        fs::create_dir_all(parent)?;
+    }
+
+    fs::write(&config.output_key_path, format!("{}\n", config.api_key))?;
+    let key_list = (0..config.api_key_count)
+        .map(|index| pressure_api_key_value(config, index))
+        .collect::<Vec<_>>()
+        .join("\n");
+    fs::write(&config.output_key_list_path, format!("{key_list}\n"))?;
+    let env_content = format!(
+        concat!(
+            "export AETHER_API_KEY_FILE={key_path}\n",
+            "export AETHER_API_KEY_LIST_FILE={key_list_path}\n",
+            "export AETHER_API_KEY=$(cat {key_path})\n",
+            "export GATEWAY_BASE_URL={gateway_base_url}\n",
+            "export TARGET_URL={gateway_base_url}/v1/chat/completions\n",
+            "export METRICS_URL={gateway_base_url}/_gateway/metrics\n",
+            "export PRESSURE_MODEL={model}\n",
+            "export PRESSURE_MOCK_UPSTREAM_BASE_URL={mock_upstream_base_url}\n",
+            "export PRESSURE_MOCK_UPSTREAM_METRICS_URL=http://127.0.0.1:18181/metrics\n"
+        ),
+        key_path = shell_escape(&config.output_key_path.display().to_string()),
+        key_list_path = shell_escape(&config.output_key_list_path.display().to_string()),
+        gateway_base_url = shell_escape(config.gateway_base_url.trim_end_matches('/')),
+        model = shell_escape(&config.model),
+        mock_upstream_base_url = shell_escape(&config.mock_upstream_base_url),
+    );
+    fs::write(&config.output_env_path, env_content)?;
+
+    Ok(())
+}
+
+fn sha256_hex(value: &str) -> String {
+    let mut hasher = sha2::Sha256::new();
+    hasher.update(value.as_bytes());
+    format!("{:x}", hasher.finalize())
+}
+
+fn pressure_api_key_id(config: &Config, index: usize) -> String {
+    if index == 0 {
+        config.api_key_id.clone()
+    } else {
+        format!("{}-{}", config.api_key_id, index + 1)
+    }
+}
+
+fn pressure_api_key_value(config: &Config, index: usize) -> String {
+    if index == 0 {
+        config.api_key.clone()
+    } else {
+        format!("{}-{}", config.api_key, index + 1)
+    }
+}
+
+fn shell_escape(value: &str) -> String {
+    if value
+        .chars()
+        .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '/' | '.' | ':' | '_' | '-'))
+    {
+        return value.to_string();
+    }
+    format!("'{}'", value.replace('\'', "'\\''"))
+}
+
+fn env_value(name: &str) -> Option<String> {
+    env::var(name)
+        .ok()
+        .map(|value| value.trim().to_string())
+        .filter(|value| !value.is_empty())
+}
+
+fn arg_value(args: &[String], index: &mut usize, name: &str) -> Result<String, String> {
+    *index += 1;
+    args.get(*index)
+        .filter(|value| !value.starts_with("--"))
+        .cloned()
+        .ok_or_else(|| format!("{name} requires a value"))
+}
+
+fn parse_u32(value: &str, name: &str) -> Result<u32, String> {
+    value
+        .parse::<u32>()
+        .map_err(|_| format!("{name} must be an unsigned integer"))
+}
+
+fn parse_usize(value: &str, name: &str) -> Result<usize, String> {
+    value
+        .parse::<usize>()
+        .map_err(|_| format!("{name} must be an unsigned integer"))
+}
+
+fn print_help() {
+    println!(
+        "Usage: cargo run -p aether-integration-tests --bin gateway_pressure_seed -- [options]\n\
+\n\
+Options:\n\
+  --database-url URL\n\
+  --output-env PATH\n\
+  --output-key PATH\n\
+  --output-key-list PATH\n\
+  --gateway-base-url URL\n\
+  --mock-upstream-base-url URL\n\
+  --model NAME\n\
+  --api-key VALUE\n\
+  --api-key-count N\n\
+  --provider-api-key VALUE\n\
+  --provider-id ID\n\
+  --endpoint-id ID\n\
+  --provider-key-id ID\n\
+  --global-model-id ID\n\
+  --model-id ID\n\
+  --api-key-id ID\n"
+    );
+}

crates/aether-testing/integration/src/bin/gateway_tunnel_stream_baseline.rs

@@ -0,0 +1,671 @@
+// Gateway-backed benchmark scenarios live outside the reusable testkit.
+use std::collections::BTreeMap;
+use std::path::PathBuf;
+use std::time::Duration;
+
+use aether_gateway::tunnel_protocol as protocol;
+use aether_testkit::{
+    fetch_prometheus_samples, find_metric_value_u64, init_test_runtime_for, run_http_load_probe,
+    HttpLoadProbeConfig, HttpLoadProbeResponseMode, HttpLoadProbeResult, PrometheusSample,
+    TunnelHarness, TunnelHarnessConfig,
+};
+use futures_util::{SinkExt, StreamExt};
+use reqwest::Method;
+use serde::Serialize;
+use tokio_tungstenite::tungstenite::client::IntoClientRequest;
+use tokio_tungstenite::tungstenite::Message;
+
+const PROXY_TUNNEL_PATH: &str = "/api/internal/proxy-tunnel";
+const TUNNEL_RELAY_PATH_PREFIX: &str = "/api/internal/tunnel/relay";
+
+#[derive(Debug, Clone)]
+struct GatewayTunnelBaselineConfig {
+    total_requests: usize,
+    concurrency: usize,
+    request_body_bytes: usize,
+    tunnel_connections: usize,
+    outbound_queue_capacity: usize,
+    close_one_connection_after: Option<Duration>,
+    require_acceptance: bool,
+    timeout: Duration,
+    output_path: Option<PathBuf>,
+}
+
+impl Default for GatewayTunnelBaselineConfig {
+    fn default() -> Self {
+        Self {
+            total_requests: 200,
+            concurrency: 20,
+            request_body_bytes: 6 * 1024 * 1024,
+            tunnel_connections: 4,
+            outbound_queue_capacity: 512,
+            close_one_connection_after: None,
+            require_acceptance: false,
+            timeout: Duration::from_secs(10),
+            output_path: None,
+        }
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct GatewayTunnelBaselineReport {
+    suite: &'static str,
+    config: GatewayTunnelEffectiveConfig,
+    scenario: HttpLoadProbeResult,
+    tunnel_metrics: TunnelMetricsSnapshot,
+    acceptance: AcceptanceReport,
+}
+
+#[derive(Debug, Serialize)]
+struct GatewayTunnelEffectiveConfig {
+    total_requests: usize,
+    concurrency: usize,
+    request_body_bytes: usize,
+    tunnel_connections: usize,
+    outbound_queue_capacity: usize,
+    close_one_connection_after_ms: Option<u64>,
+    timeout_ms: u64,
+}
+
+#[derive(Debug, Serialize)]
+struct TunnelMetricsSnapshot {
+    proxy_connections: u64,
+    active_streams: u64,
+    outbound_queue_depth_total: u64,
+    outbound_queue_depth_max: u64,
+    outbound_queue_capacity_total: u64,
+    outbound_queue_rejected_full_total: u64,
+    outbound_queue_rejected_closed_total: u64,
+    proxy_connection_congested_total: u64,
+    proxy_connection_write_latency_last_us_max: u64,
+    proxy_connection_write_latency_ewma_us_max: u64,
+    body_backpressure_total: u64,
+    flow_window_blocked_ms: u64,
+    connection_health_score: u64,
+    stream_reset_total: u64,
+    stream_reset_reasons: BTreeMap<String, u64>,
+    drain_total: u64,
+    drain_reasons: BTreeMap<String, u64>,
+    scheduler_selected_conn_total: u64,
+    proxy_connections_protocol_v1: u64,
+    proxy_connections_protocol_v2: u64,
+    proxy_connections_protocol_v3: u64,
+}
+
+#[derive(Debug, Serialize)]
+struct AcceptanceReport {
+    required: bool,
+    passed: bool,
+    success_rate_bps: u64,
+    min_success_rate_bps: u64,
+    congestion_free: bool,
+    no_queue_full_rejections: bool,
+    reasons: Vec<String>,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    init_test_runtime_for("gateway-tunnel-stream-baseline");
+    let config = parse_args(std::env::args().skip(1).collect())?;
+    let report = run_suite(&config).await?;
+    let raw = serde_json::to_string_pretty(&report)?;
+    println!("{raw}");
+    if let Some(path) = config.output_path.as_ref() {
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        std::fs::write(path, format!("{raw}\n"))?;
+    }
+    Ok(())
+}
+
+async fn run_suite(
+    config: &GatewayTunnelBaselineConfig,
+) -> Result<GatewayTunnelBaselineReport, Box<dyn std::error::Error>> {
+    let tunnel = TunnelHarness::start(TunnelHarnessConfig {
+        outbound_queue_capacity: config.outbound_queue_capacity,
+        ..TunnelHarnessConfig::default()
+    })
+    .await?;
+    let mut peers = connect_protocol_peers(tunnel.base_url(), config.tunnel_connections).await?;
+    let fault_injection = config.close_one_connection_after.map(|delay| {
+        let peer = peers.pop();
+        tokio::spawn(async move {
+            if let Some(peer) = peer {
+                tokio::time::sleep(delay).await;
+                peer.abort();
+            }
+        })
+    });
+
+    let result = run_http_load_probe(&HttpLoadProbeConfig {
+        url: format!(
+            "{tunnel_base}{TUNNEL_RELAY_PATH_PREFIX}/node-baseline",
+            tunnel_base = tunnel.base_url()
+        ),
+        method: Method::POST,
+        headers: std::collections::BTreeMap::from([(
+            "content-type".to_string(),
+            "application/octet-stream".to_string(),
+        )]),
+        body: Some(relay_envelope(config.request_body_bytes)),
+        total_requests: config.total_requests,
+        concurrency: config.concurrency,
+        timeout: config.timeout,
+        response_mode: HttpLoadProbeResponseMode::FullBody,
+        ..HttpLoadProbeConfig::default()
+    })
+    .await
+    .map_err(std::io::Error::other)?;
+
+    let tunnel_metrics = capture_tunnel_metrics(tunnel.base_url()).await?;
+    if let Some(task) = fault_injection {
+        let _ = task.await;
+    }
+    drop(peers);
+    let acceptance = evaluate_acceptance(config, &result, &tunnel_metrics);
+    if config.require_acceptance && !acceptance.passed {
+        return Err(std::io::Error::other(format!(
+            "gateway tunnel acceptance failed: {:?}",
+            acceptance.reasons
+        ))
+        .into());
+    }
+
+    Ok(GatewayTunnelBaselineReport {
+        suite: "gateway_tunnel_stream_baseline",
+        config: GatewayTunnelEffectiveConfig {
+            total_requests: config.total_requests,
+            concurrency: config.concurrency,
+            request_body_bytes: config.request_body_bytes,
+            tunnel_connections: config.tunnel_connections,
+            outbound_queue_capacity: config.outbound_queue_capacity,
+            close_one_connection_after_ms: config
+                .close_one_connection_after
+                .map(|duration| duration.as_millis() as u64),
+            timeout_ms: config.timeout.as_millis() as u64,
+        },
+        scenario: result,
+        tunnel_metrics,
+        acceptance,
+    })
+}
+
+fn relay_envelope(request_body_bytes: usize) -> Vec<u8> {
+    let meta = protocol::RequestMeta {
+        method: "POST".to_string(),
+        url: "https://baseline.example/v1/chat/completions".to_string(),
+        headers: std::collections::HashMap::from([(
+            "content-type".to_string(),
+            "application/json".to_string(),
+        )]),
+        stream: true,
+        request_timeout_ms: None,
+        stream_first_byte_timeout_ms: None,
+        timeout: 30,
+        follow_redirects: None,
+        http1_only: false,
+        provider_id: None,
+        endpoint_id: None,
+        key_id: None,
+        transport_profile: None,
+    };
+    let meta_json = serde_json::to_vec(&meta).expect("tunnel relay metadata should serialize");
+    let body = vec![b'x'; request_body_bytes];
+    let mut envelope = Vec::with_capacity(4 + meta_json.len() + body.len());
+    envelope.extend_from_slice(&(meta_json.len() as u32).to_be_bytes());
+    envelope.extend_from_slice(&meta_json);
+    envelope.extend_from_slice(&body);
+    envelope
+}
+
+async fn connect_protocol_peers(
+    tunnel_base_url: &str,
+    count: usize,
+) -> Result<Vec<tokio::task::JoinHandle<()>>, Box<dyn std::error::Error>> {
+    let count = count.max(1);
+    let mut peers = Vec::with_capacity(count);
+    for index in 0..count {
+        peers.push(connect_protocol_peer(tunnel_base_url, index).await?);
+    }
+    Ok(peers)
+}
+
+async fn connect_protocol_peer(
+    tunnel_base_url: &str,
+    index: usize,
+) -> Result<tokio::task::JoinHandle<()>, Box<dyn std::error::Error>> {
+    let ws_url = format!(
+        "{}{}",
+        tunnel_base_url.replace("http://", "ws://"),
+        PROXY_TUNNEL_PATH
+    );
+    let request = ws_url.into_client_request()?;
+    let mut request = request;
+    request
+        .headers_mut()
+        .insert("x-node-id", http::HeaderValue::from_static("node-baseline"));
+    request.headers_mut().insert(
+        aether_contracts::tunnel::TUNNEL_PROTOCOL_VERSION_HEADER,
+        http::HeaderValue::from_static(
+            aether_contracts::tunnel::CURRENT_TUNNEL_PROTOCOL_VERSION_STR,
+        ),
+    );
+    request.headers_mut().insert(
+        "x-node-name",
+        http::HeaderValue::from_str(&format!("proxy-baseline-{index}"))?,
+    );
+    request.headers_mut().insert(
+        "x-tunnel-max-streams",
+        http::HeaderValue::from_static("128"),
+    );
+
+    let (socket, _response) = tokio_tungstenite::connect_async(request).await?;
+    let (mut sink, mut stream) = socket.split();
+    sink.send(Message::Binary(
+        protocol::encode_hello(&protocol::HelloPayload {
+            protocol_version: aether_contracts::tunnel::CURRENT_TUNNEL_PROTOCOL_VERSION,
+            capabilities: vec![
+                "flow-control".to_string(),
+                "reset-stream".to_string(),
+                "graceful-drain".to_string(),
+            ],
+            session_id: Some(format!("baseline-session-{index}")),
+            replica_id: Some(format!("baseline-replica-{index}")),
+        })
+        .into(),
+    ))
+    .await?;
+    sink.send(Message::Binary(
+        protocol::encode_settings(&protocol::SettingsPayload {
+            initial_stream_window_bytes: 4 * 1024 * 1024,
+            min_window_update_bytes: 1024 * 1024,
+            drain_deadline_ms: 30_000,
+        })
+        .into(),
+    ))
+    .await?;
+    Ok(tokio::spawn(async move {
+        while let Some(message) = stream.next().await {
+            let Ok(message) = message else {
+                break;
+            };
+            match message {
+                Message::Binary(data)
+                    if handle_binary_frame(&mut sink, data.to_vec()).await.is_err() =>
+                {
+                    break;
+                }
+                Message::Ping(payload)
+                    if sink.send(Message::Pong(payload.clone())).await.is_err() =>
+                {
+                    break;
+                }
+                Message::Close(_) => break,
+                _ => {}
+            }
+        }
+        let _ = sink.close().await;
+    }))
+}
+
+async fn capture_tunnel_metrics(
+    base_url: &str,
+) -> Result<TunnelMetricsSnapshot, Box<dyn std::error::Error>> {
+    let samples = fetch_prometheus_samples(&format!("{base_url}/metrics"))
+        .await
+        .map_err(std::io::Error::other)?;
+    Ok(TunnelMetricsSnapshot {
+        proxy_connections: find_metric_value_u64(&samples, "tunnel_proxy_connections", &[])
+            .unwrap_or_default(),
+        active_streams: find_metric_value_u64(&samples, "tunnel_active_streams", &[])
+            .unwrap_or_default(),
+        outbound_queue_depth_total: find_metric_value_u64(
+            &samples,
+            "tunnel_proxy_outbound_queue_depth_total",
+            &[],
+        )
+        .unwrap_or_default(),
+        outbound_queue_depth_max: find_metric_value_u64(
+            &samples,
+            "tunnel_proxy_outbound_queue_depth_max",
+            &[],
+        )
+        .unwrap_or_default(),
+        outbound_queue_capacity_total: find_metric_value_u64(
+            &samples,
+            "tunnel_proxy_outbound_queue_capacity_total",
+            &[],
+        )
+        .unwrap_or_default(),
+        outbound_queue_rejected_full_total: find_metric_value_u64(
+            &samples,
+            "tunnel_proxy_outbound_queue_rejected_full_total",
+            &[],
+        )
+        .unwrap_or_default(),
+        outbound_queue_rejected_closed_total: find_metric_value_u64(
+            &samples,
+            "tunnel_proxy_outbound_queue_rejected_closed_total",
+            &[],
+        )
+        .unwrap_or_default(),
+        proxy_connection_congested_total: find_metric_value_u64(
+            &samples,
+            "tunnel_proxy_connection_congested_total",
+            &[],
+        )
+        .unwrap_or_default(),
+        proxy_connection_write_latency_last_us_max: find_metric_value_u64(
+            &samples,
+            "tunnel_proxy_connection_write_latency_last_us_max",
+            &[],
+        )
+        .unwrap_or_default(),
+        proxy_connection_write_latency_ewma_us_max: find_metric_value_u64(
+            &samples,
+            "tunnel_proxy_connection_write_latency_ewma_us_max",
+            &[],
+        )
+        .unwrap_or_default(),
+        body_backpressure_total: find_metric_value_u64(
+            &samples,
+            "tunnel_body_backpressure_total",
+            &[],
+        )
+        .unwrap_or_default(),
+        flow_window_blocked_ms: find_metric_value_u64(
+            &samples,
+            "tunnel_flow_window_blocked_ms",
+            &[],
+        )
+        .unwrap_or_default(),
+        connection_health_score: find_metric_value_u64(
+            &samples,
+            "tunnel_connection_health_score",
+            &[],
+        )
+        .unwrap_or_default(),
+        stream_reset_total: find_metric_value_u64(
+            &samples,
+            "tunnel_stream_reset_total",
+            &[("reason", "all")],
+        )
+        .unwrap_or_default(),
+        stream_reset_reasons: collect_reason_metrics(&samples, "tunnel_stream_reset_total"),
+        drain_total: find_metric_value_u64(&samples, "tunnel_drain_total", &[("reason", "all")])
+            .unwrap_or_default(),
+        drain_reasons: collect_reason_metrics(&samples, "tunnel_drain_total"),
+        scheduler_selected_conn_total: find_metric_value_u64(
+            &samples,
+            "tunnel_scheduler_selected_conn_total",
+            &[],
+        )
+        .unwrap_or_default(),
+        proxy_connections_protocol_v1: find_metric_value_u64(
+            &samples,
+            "tunnel_proxy_connections_protocol_v1",
+            &[],
+        )
+        .unwrap_or_default(),
+        proxy_connections_protocol_v2: find_metric_value_u64(
+            &samples,
+            "tunnel_proxy_connections_protocol_v2",
+            &[],
+        )
+        .unwrap_or_default(),
+        proxy_connections_protocol_v3: find_metric_value_u64(
+            &samples,
+            "tunnel_proxy_connections_protocol_v3",
+            &[],
+        )
+        .unwrap_or_default(),
+    })
+}
+
+fn collect_reason_metrics(
+    samples: &[PrometheusSample],
+    metric_name: &str,
+) -> BTreeMap<String, u64> {
+    samples
+        .iter()
+        .filter(|sample| {
+            (sample.name == metric_name || sample.name.ends_with(&format!("_{metric_name}")))
+                && sample
+                    .labels
+                    .get("reason")
+                    .is_some_and(|reason| reason != "all")
+        })
+        .filter_map(|sample| {
+            let reason = sample.labels.get("reason")?.clone();
+            let value = sample.value.parse::<u64>().ok()?;
+            Some((reason, value))
+        })
+        .collect()
+}
+
+fn evaluate_acceptance(
+    config: &GatewayTunnelBaselineConfig,
+    result: &HttpLoadProbeResult,
+    metrics: &TunnelMetricsSnapshot,
+) -> AcceptanceReport {
+    let min_success_rate_bps = if config.require_acceptance { 9_950 } else { 1 };
+    let successful_requests = result
+        .status_counts
+        .iter()
+        .filter(|(status, _)| (200u16..300u16).contains(status))
+        .map(|(_, count)| *count)
+        .sum::<usize>();
+    let success_rate_bps = if result.total_requests == 0 {
+        0
+    } else {
+        ((successful_requests as u128) * 10_000 / (result.total_requests as u128)) as u64
+    };
+    let congestion_free = metrics.proxy_connection_congested_total == 0;
+    let no_queue_full_rejections = metrics.outbound_queue_rejected_full_total == 0;
+    let mut reasons = Vec::new();
+    if success_rate_bps < min_success_rate_bps {
+        reasons.push(format!(
+            "success rate {} bps below required {} bps",
+            success_rate_bps, min_success_rate_bps
+        ));
+    }
+    if !congestion_free {
+        reasons.push(format!(
+            "connection congestion total is {}",
+            metrics.proxy_connection_congested_total
+        ));
+    }
+    if !no_queue_full_rejections {
+        reasons.push(format!(
+            "outbound queue full rejections total is {}",
+            metrics.outbound_queue_rejected_full_total
+        ));
+    }
+
+    AcceptanceReport {
+        required: config.require_acceptance,
+        passed: reasons.is_empty(),
+        success_rate_bps,
+        min_success_rate_bps,
+        congestion_free,
+        no_queue_full_rejections,
+        reasons,
+    }
+}
+
+async fn handle_binary_frame<S>(
+    sink: &mut S,
+    data: Vec<u8>,
+) -> Result<(), tokio_tungstenite::tungstenite::Error>
+where
+    S: SinkExt<Message, Error = tokio_tungstenite::tungstenite::Error> + Unpin,
+{
+    let Some(header) = protocol::FrameHeader::parse(&data) else {
+        return Ok(());
+    };
+    match header.msg_type {
+        protocol::PING => {
+            let payload = protocol::frame_payload_by_header(&data, &header).unwrap_or(&[]);
+            sink.send(Message::Binary(protocol::encode_pong(payload).into()))
+                .await?;
+        }
+        protocol::REQUEST_HEADERS => {
+            let payload = protocol::decode_payload(&data, &header).unwrap_or_default();
+            let _ = serde_json::from_slice::<protocol::RequestMeta>(&payload);
+        }
+        protocol::REQUEST_BODY => {
+            let payload = protocol::decode_payload(&data, &header).unwrap_or_default();
+            if !payload.is_empty() {
+                sink.send(Message::Binary(
+                    protocol::encode_window_update(header.stream_id, payload.len() as u32).into(),
+                ))
+                .await?;
+            }
+            if header.flags & protocol::FLAG_END_STREAM == 0 {
+                return Ok(());
+            }
+            let response_meta = protocol::ResponseMeta {
+                status: 200,
+                headers: vec![(
+                    "content-type".to_string(),
+                    "text/plain; charset=utf-8".to_string(),
+                )],
+            };
+            let response_meta_json =
+                serde_json::to_vec(&response_meta).expect("response metadata should serialize");
+            sink.send(Message::Binary(
+                protocol::encode_frame(
+                    header.stream_id,
+                    protocol::RESPONSE_HEADERS,
+                    0,
+                    &response_meta_json,
+                )
+                .into(),
+            ))
+            .await?;
+
+            for chunk in [
+                b"baseline-".as_slice(),
+                b"tunnel-".as_slice(),
+                b"stream".as_slice(),
+            ] {
+                sink.send(Message::Binary(
+                    protocol::encode_frame(header.stream_id, protocol::RESPONSE_BODY, 0, chunk)
+                        .into(),
+                ))
+                .await?;
+            }
+
+            sink.send(Message::Binary(
+                protocol::encode_frame(header.stream_id, protocol::STREAM_END, 0, &[]).into(),
+            ))
+            .await?;
+        }
+        _ => {}
+    }
+    Ok(())
+}
+
+fn parse_args(
+    args: Vec<String>,
+) -> Result<GatewayTunnelBaselineConfig, Box<dyn std::error::Error>> {
+    let mut config = GatewayTunnelBaselineConfig::default();
+    let mut iter = args.into_iter();
+    while let Some(arg) = iter.next() {
+        match arg.as_str() {
+            "--requests" => config.total_requests = next_value(&mut iter, "--requests")?.parse()?,
+            "--concurrency" => {
+                config.concurrency = next_value(&mut iter, "--concurrency")?.parse()?
+            }
+            "--body-bytes" => {
+                config.request_body_bytes = next_value(&mut iter, "--body-bytes")?.parse()?
+            }
+            "--tunnel-connections" => {
+                config.tunnel_connections =
+                    next_value(&mut iter, "--tunnel-connections")?.parse()?
+            }
+            "--outbound-queue-capacity" => {
+                config.outbound_queue_capacity =
+                    next_value(&mut iter, "--outbound-queue-capacity")?.parse()?
+            }
+            "--close-one-connection-after-ms" => {
+                config.close_one_connection_after = Some(Duration::from_millis(
+                    next_value(&mut iter, "--close-one-connection-after-ms")?.parse()?,
+                ))
+            }
+            "--require-acceptance" => {
+                config.require_acceptance = true;
+            }
+            "--timeout-ms" => {
+                config.timeout =
+                    Duration::from_millis(next_value(&mut iter, "--timeout-ms")?.parse()?)
+            }
+            "--output" => {
+                config.output_path = Some(PathBuf::from(next_value(&mut iter, "--output")?))
+            }
+            "--help" | "-h" => {
+                print_usage();
+                std::process::exit(0);
+            }
+            other => {
+                return Err(std::io::Error::new(
+                    std::io::ErrorKind::InvalidInput,
+                    format!("unknown argument: {other}"),
+                )
+                .into());
+            }
+        }
+    }
+    if config.request_body_bytes == 0 {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            "--body-bytes must be positive",
+        )
+        .into());
+    }
+    if config.tunnel_connections == 0 {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            "--tunnel-connections must be positive",
+        )
+        .into());
+    }
+    if config.outbound_queue_capacity == 0 {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            "--outbound-queue-capacity must be positive",
+        )
+        .into());
+    }
+    if config
+        .close_one_connection_after
+        .is_some_and(|duration| duration.is_zero())
+    {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            "--close-one-connection-after-ms must be positive",
+        )
+        .into());
+    }
+    Ok(config)
+}
+
+fn next_value(
+    iter: &mut impl Iterator<Item = String>,
+    flag: &str,
+) -> Result<String, Box<dyn std::error::Error>> {
+    iter.next().ok_or_else(|| {
+        std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            format!("missing value for {flag}"),
+        )
+        .into()
+    })
+}
+
+fn print_usage() {
+    eprintln!(
+        "usage: cargo run -p aether-integration-tests --bin gateway_tunnel_stream_baseline -- [--requests 200] [--concurrency 20] [--body-bytes 6291456] [--tunnel-connections 4] [--outbound-queue-capacity 512] [--close-one-connection-after-ms 1000] [--require-acceptance] [--timeout-ms 10000] [--output /tmp/gateway_tunnel_baseline.json]"
+    );
+}

crates/aether-testing/integration/src/bin/mock_openai_upstream.rs

@@ -0,0 +1,694 @@
+// Gateway-backed benchmark scenarios live outside the reusable testkit.
+use std::collections::BTreeMap;
+use std::convert::Infallible;
+use std::net::SocketAddr;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use axum::body::{to_bytes, Body, Bytes};
+use axum::extract::State;
+use axum::http::{header, HeaderValue, StatusCode};
+use axum::response::{IntoResponse, Response};
+use axum::routing::{get, post};
+use axum::Router;
+use serde_json::json;
+
+#[derive(Debug, Clone)]
+struct Config {
+    binds: Vec<SocketAddr>,
+    chunks: u64,
+    first_byte_delay: Duration,
+    chunk_delay: Duration,
+    payload_bytes: usize,
+    status: StatusCode,
+    assume_stream: bool,
+}
+
+const MAX_MOCK_REQUEST_BODY_BYTES: usize = 1024 * 1024;
+
+impl Default for Config {
+    fn default() -> Self {
+        Self {
+            binds: vec!["127.0.0.1:18181"
+                .parse()
+                .expect("default bind address should parse")],
+            chunks: 8,
+            first_byte_delay: Duration::from_millis(0),
+            chunk_delay: Duration::from_millis(20),
+            payload_bytes: 32,
+            status: StatusCode::OK,
+            assume_stream: false,
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+struct Metrics {
+    requests_total: AtomicU64,
+    completed_total: AtomicU64,
+    in_flight: AtomicU64,
+    max_in_flight: AtomicU64,
+    request_body_read_sum_ms: AtomicU64,
+    request_body_read_max_ms: AtomicU64,
+    accepted_to_response_header_sum_ms: AtomicU64,
+    accepted_to_response_header_max_ms: AtomicU64,
+    first_chunk_yield_total: AtomicU64,
+    response_header_to_first_chunk_sum_ms: AtomicU64,
+    response_header_to_first_chunk_max_ms: AtomicU64,
+    binds: BTreeMap<String, Arc<BindMetrics>>,
+}
+
+#[derive(Debug, Default)]
+struct BindMetrics {
+    requests_total: AtomicU64,
+    completed_total: AtomicU64,
+    in_flight: AtomicU64,
+    max_in_flight: AtomicU64,
+    request_body_read_sum_ms: AtomicU64,
+    request_body_read_max_ms: AtomicU64,
+    accepted_to_response_header_sum_ms: AtomicU64,
+    accepted_to_response_header_max_ms: AtomicU64,
+    first_chunk_yield_total: AtomicU64,
+    response_header_to_first_chunk_sum_ms: AtomicU64,
+    response_header_to_first_chunk_max_ms: AtomicU64,
+}
+
+#[derive(Debug, Clone)]
+struct App {
+    config: Config,
+    metrics: Arc<Metrics>,
+    bind_label: Arc<str>,
+}
+
+struct RequestCompletionGuard {
+    app: App,
+    completed: bool,
+}
+
+impl RequestCompletionGuard {
+    fn new(app: App) -> Self {
+        Self {
+            app,
+            completed: false,
+        }
+    }
+
+    fn complete(mut self) {
+        self.complete_once();
+    }
+
+    fn complete_once(&mut self) {
+        if self.completed {
+            return;
+        }
+        record_request_completed(&self.app);
+        self.completed = true;
+    }
+}
+
+impl Drop for RequestCompletionGuard {
+    fn drop(&mut self) {
+        self.complete_once();
+    }
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let config = parse_args(std::env::args().skip(1).collect())?;
+    let metrics = Arc::new(Metrics::for_binds(&config.binds));
+
+    serve_listeners(&config, metrics).await?;
+    Ok(())
+}
+
+async fn serve_listeners(
+    config: &Config,
+    metrics: Arc<Metrics>,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let mut listeners = Vec::with_capacity(config.binds.len());
+    for bind in &config.binds {
+        listeners.push((*bind, tokio::net::TcpListener::bind(bind).await?));
+    }
+    if listeners.len() == 1 {
+        let (bind, listener) = listeners
+            .into_iter()
+            .next()
+            .ok_or_else(|| std::io::Error::other("mock upstream listener set is empty"))?;
+        let app = build_router(config.clone(), Arc::clone(&metrics), bind);
+        eprintln!("mock OpenAI upstream listening on http://{bind}");
+        axum::serve(listener, app).await?;
+        return Ok(());
+    }
+
+    let mut servers = tokio::task::JoinSet::new();
+    for (bind, listener) in listeners {
+        let app = build_router(config.clone(), Arc::clone(&metrics), bind);
+        eprintln!("mock OpenAI upstream listening on http://{bind}");
+        servers.spawn(async move { axum::serve(listener, app).await });
+    }
+    if let Some(result) = servers.join_next().await {
+        servers.abort_all();
+        let serve_result = result
+            .map_err(|err| std::io::Error::other(format!("mock listener task failed: {err}")))?;
+        serve_result?;
+    }
+    Ok(())
+}
+
+fn build_router(config: Config, shared_metrics: Arc<Metrics>, bind: SocketAddr) -> Router {
+    Router::new()
+        .route("/health", get(health))
+        .route("/metrics", get(metrics))
+        .route("/v1/chat/completions", post(chat_completions))
+        .route("/chat/completions", post(chat_completions))
+        .route("/v1/responses", post(responses))
+        .route("/responses", post(responses))
+        .with_state(App {
+            config,
+            metrics: shared_metrics,
+            bind_label: Arc::from(bind.to_string()),
+        })
+}
+
+impl Metrics {
+    fn for_binds(binds: &[SocketAddr]) -> Self {
+        let mut metrics = Self::default();
+        for bind in binds {
+            metrics
+                .binds
+                .insert(bind.to_string(), Arc::new(BindMetrics::default()));
+        }
+        metrics
+    }
+
+    fn bind_metrics(&self, bind: &str) -> Option<Arc<BindMetrics>> {
+        self.binds.get(bind).cloned()
+    }
+}
+
+async fn health() -> impl IntoResponse {
+    (StatusCode::OK, "ok\n")
+}
+
+async fn metrics(State(app): State<App>) -> Response {
+    let mut body = format!(
+        concat!(
+            "# HELP mock_upstream_requests_total Total requests accepted by the mock upstream.\n",
+            "# TYPE mock_upstream_requests_total counter\n",
+            "mock_upstream_requests_total {}\n",
+            "# HELP mock_upstream_completed_total Total requests completed by the mock upstream.\n",
+            "# TYPE mock_upstream_completed_total counter\n",
+            "mock_upstream_completed_total {}\n",
+            "# HELP mock_upstream_in_flight Current in-flight requests/streams.\n",
+            "# TYPE mock_upstream_in_flight gauge\n",
+            "mock_upstream_in_flight {}\n",
+            "# HELP mock_upstream_max_in_flight Maximum in-flight requests/streams observed.\n",
+            "# TYPE mock_upstream_max_in_flight gauge\n",
+            "mock_upstream_max_in_flight {}\n",
+            "# HELP mock_upstream_first_chunk_yield_total Total first stream chunks yielded by the mock upstream.\n",
+            "# TYPE mock_upstream_first_chunk_yield_total counter\n",
+            "mock_upstream_first_chunk_yield_total {}\n",
+            "# HELP mock_upstream_request_body_read_sum_ms Total milliseconds spent reading request bodies after handler entry.\n",
+            "# TYPE mock_upstream_request_body_read_sum_ms counter\n",
+            "mock_upstream_request_body_read_sum_ms {}\n",
+            "# HELP mock_upstream_request_body_read_max_ms Maximum milliseconds spent reading a request body after handler entry.\n",
+            "# TYPE mock_upstream_request_body_read_max_ms gauge\n",
+            "mock_upstream_request_body_read_max_ms {}\n",
+            "# HELP mock_upstream_accepted_to_response_header_sum_ms Total milliseconds from request acceptance to response construction.\n",
+            "# TYPE mock_upstream_accepted_to_response_header_sum_ms counter\n",
+            "mock_upstream_accepted_to_response_header_sum_ms {}\n",
+            "# HELP mock_upstream_accepted_to_response_header_max_ms Maximum milliseconds from request acceptance to response construction.\n",
+            "# TYPE mock_upstream_accepted_to_response_header_max_ms gauge\n",
+            "mock_upstream_accepted_to_response_header_max_ms {}\n",
+            "# HELP mock_upstream_response_header_to_first_chunk_sum_ms Total milliseconds from response construction to first stream chunk yield.\n",
+            "# TYPE mock_upstream_response_header_to_first_chunk_sum_ms counter\n",
+            "mock_upstream_response_header_to_first_chunk_sum_ms {}\n",
+            "# HELP mock_upstream_response_header_to_first_chunk_max_ms Maximum milliseconds from response construction to first stream chunk yield.\n",
+            "# TYPE mock_upstream_response_header_to_first_chunk_max_ms gauge\n",
+            "mock_upstream_response_header_to_first_chunk_max_ms {}\n"
+        ),
+        app.metrics.requests_total.load(Ordering::Acquire),
+        app.metrics.completed_total.load(Ordering::Acquire),
+        app.metrics.in_flight.load(Ordering::Acquire),
+        app.metrics.max_in_flight.load(Ordering::Acquire),
+        app.metrics.first_chunk_yield_total.load(Ordering::Acquire),
+        app.metrics
+            .request_body_read_sum_ms
+            .load(Ordering::Acquire),
+        app.metrics
+            .request_body_read_max_ms
+            .load(Ordering::Acquire),
+        app.metrics
+            .accepted_to_response_header_sum_ms
+            .load(Ordering::Acquire),
+        app.metrics
+            .accepted_to_response_header_max_ms
+            .load(Ordering::Acquire),
+        app.metrics
+            .response_header_to_first_chunk_sum_ms
+            .load(Ordering::Acquire),
+        app.metrics
+            .response_header_to_first_chunk_max_ms
+            .load(Ordering::Acquire),
+    );
+    for (bind, metrics) in &app.metrics.binds {
+        let bind = prometheus_label_value(bind);
+        body.push_str(&format!(
+            concat!(
+                "mock_upstream_requests_total{{bind=\"{}\"}} {}\n",
+                "mock_upstream_completed_total{{bind=\"{}\"}} {}\n",
+                "mock_upstream_in_flight{{bind=\"{}\"}} {}\n",
+                "mock_upstream_max_in_flight{{bind=\"{}\"}} {}\n",
+                "mock_upstream_first_chunk_yield_total{{bind=\"{}\"}} {}\n",
+                "mock_upstream_request_body_read_sum_ms{{bind=\"{}\"}} {}\n",
+                "mock_upstream_request_body_read_max_ms{{bind=\"{}\"}} {}\n",
+                "mock_upstream_accepted_to_response_header_sum_ms{{bind=\"{}\"}} {}\n",
+                "mock_upstream_accepted_to_response_header_max_ms{{bind=\"{}\"}} {}\n",
+                "mock_upstream_response_header_to_first_chunk_sum_ms{{bind=\"{}\"}} {}\n",
+                "mock_upstream_response_header_to_first_chunk_max_ms{{bind=\"{}\"}} {}\n"
+            ),
+            bind,
+            metrics.requests_total.load(Ordering::Acquire),
+            bind,
+            metrics.completed_total.load(Ordering::Acquire),
+            bind,
+            metrics.in_flight.load(Ordering::Acquire),
+            bind,
+            metrics.max_in_flight.load(Ordering::Acquire),
+            bind,
+            metrics.first_chunk_yield_total.load(Ordering::Acquire),
+            bind,
+            metrics.request_body_read_sum_ms.load(Ordering::Acquire),
+            bind,
+            metrics.request_body_read_max_ms.load(Ordering::Acquire),
+            bind,
+            metrics
+                .accepted_to_response_header_sum_ms
+                .load(Ordering::Acquire),
+            bind,
+            metrics
+                .accepted_to_response_header_max_ms
+                .load(Ordering::Acquire),
+            bind,
+            metrics
+                .response_header_to_first_chunk_sum_ms
+                .load(Ordering::Acquire),
+            bind,
+            metrics
+                .response_header_to_first_chunk_max_ms
+                .load(Ordering::Acquire),
+        ));
+    }
+    (StatusCode::OK, body).into_response()
+}
+
+async fn chat_completions(State(app): State<App>, request: axum::extract::Request) -> Response {
+    let request_started_at = record_request_started(&app);
+    if app.config.status != StatusCode::OK {
+        record_response_header_created(&app, request_started_at.elapsed());
+        record_request_completed(&app);
+        return (app.config.status, "mock upstream error\n").into_response();
+    }
+    if app.config.assume_stream {
+        record_response_header_created(&app, request_started_at.elapsed());
+        return build_chat_sse_response(app);
+    }
+
+    let body = match read_request_body(&app, request).await {
+        Ok(body) => body,
+        Err(response) => {
+            record_response_header_created(&app, request_started_at.elapsed());
+            record_request_completed(&app);
+            return response;
+        }
+    };
+    let stream = request_wants_stream(&body);
+    if stream {
+        record_response_header_created(&app, request_started_at.elapsed());
+        return build_chat_sse_response(app);
+    }
+    let payload = json!({
+        "id": "chatcmpl-mock",
+        "object": "chat.completion",
+        "created": current_unix_secs(),
+        "model": request_model(&body).unwrap_or_else(|| "mock-model".to_string()),
+        "choices": [{
+            "index": 0,
+            "message": {
+                "role": "assistant",
+                "content": mock_payload(app.config.payload_bytes)
+            },
+            "finish_reason": "stop"
+        }],
+        "usage": {
+            "prompt_tokens": 1,
+            "completion_tokens": app.config.chunks.max(1),
+            "total_tokens": app.config.chunks.max(1) + 1
+        }
+    });
+    record_response_header_created(&app, request_started_at.elapsed());
+    record_request_completed(&app);
+    axum::Json(payload).into_response()
+}
+
+async fn responses(State(app): State<App>, request: axum::extract::Request) -> Response {
+    let request_started_at = record_request_started(&app);
+    if app.config.status != StatusCode::OK {
+        record_response_header_created(&app, request_started_at.elapsed());
+        record_request_completed(&app);
+        return (app.config.status, "mock upstream error\n").into_response();
+    }
+    if app.config.assume_stream {
+        record_response_header_created(&app, request_started_at.elapsed());
+        return build_responses_sse_response(app);
+    }
+
+    let body = match read_request_body(&app, request).await {
+        Ok(body) => body,
+        Err(response) => {
+            record_response_header_created(&app, request_started_at.elapsed());
+            record_request_completed(&app);
+            return response;
+        }
+    };
+    let stream = request_wants_stream(&body);
+    if stream {
+        record_response_header_created(&app, request_started_at.elapsed());
+        return build_responses_sse_response(app);
+    }
+    let payload = json!({
+        "id": "resp_mock",
+        "object": "response",
+        "created_at": current_unix_secs(),
+        "model": request_model(&body).unwrap_or_else(|| "mock-model".to_string()),
+        "output": [{
+            "type": "message",
+            "id": "msg_mock",
+            "role": "assistant",
+            "content": [{
+                "type": "output_text",
+                "text": mock_payload(app.config.payload_bytes)
+            }]
+        }],
+        "usage": {
+            "input_tokens": 1,
+            "output_tokens": app.config.chunks.max(1),
+            "total_tokens": app.config.chunks.max(1) + 1
+        }
+    });
+    record_response_header_created(&app, request_started_at.elapsed());
+    record_request_completed(&app);
+    axum::Json(payload).into_response()
+}
+
+fn build_chat_sse_response(app: App) -> Response {
+    let response_created_at = Instant::now();
+    let config = app.config.clone();
+    let mut completion = Some(RequestCompletionGuard::new(app.clone()));
+    let stream = async_stream::stream! {
+        if !config.first_byte_delay.is_zero() {
+            tokio::time::sleep(config.first_byte_delay).await;
+        }
+        for index in 0..config.chunks {
+            if index > 0 && !config.chunk_delay.is_zero() {
+                tokio::time::sleep(config.chunk_delay).await;
+            }
+            let payload = json!({
+                "id": "chatcmpl-mock",
+                "object": "chat.completion.chunk",
+                "created": current_unix_secs(),
+                "model": "mock-model",
+                "choices": [{
+                    "index": 0,
+                    "delta": {
+                        "content": mock_payload(config.payload_bytes)
+                    },
+                    "finish_reason": serde_json::Value::Null
+                }]
+            });
+            if index == 0 {
+                record_first_chunk_yield(&app, response_created_at.elapsed());
+            }
+            yield Ok::<Bytes, Infallible>(Bytes::from(format!("data: {payload}\n\n")));
+        }
+        yield Ok::<Bytes, Infallible>(Bytes::from(
+            "data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n",
+        ));
+        yield Ok::<Bytes, Infallible>(Bytes::from("data: [DONE]\n\n"));
+        if let Some(completion) = completion.take() {
+            completion.complete();
+        }
+    };
+    sse_response(Body::from_stream(stream))
+}
+
+fn build_responses_sse_response(app: App) -> Response {
+    let response_created_at = Instant::now();
+    let config = app.config.clone();
+    let mut completion = Some(RequestCompletionGuard::new(app.clone()));
+    let stream = async_stream::stream! {
+        if !config.first_byte_delay.is_zero() {
+            tokio::time::sleep(config.first_byte_delay).await;
+        }
+        record_first_chunk_yield(&app, response_created_at.elapsed());
+        yield Ok::<Bytes, Infallible>(Bytes::from(
+            "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_mock\",\"status\":\"in_progress\"}}\n\n",
+        ));
+        for index in 0..config.chunks {
+            if index > 0 && !config.chunk_delay.is_zero() {
+                tokio::time::sleep(config.chunk_delay).await;
+            }
+            let payload = json!({
+                "type": "response.output_text.delta",
+                "item_id": "msg_mock",
+                "output_index": 0,
+                "content_index": 0,
+                "delta": mock_payload(config.payload_bytes)
+            });
+            yield Ok::<Bytes, Infallible>(Bytes::from(format!("event: response.output_text.delta\ndata: {payload}\n\n")));
+        }
+        yield Ok::<Bytes, Infallible>(Bytes::from(
+            "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_mock\",\"status\":\"completed\"}}\n\n",
+        ));
+        yield Ok::<Bytes, Infallible>(Bytes::from("data: [DONE]\n\n"));
+        if let Some(completion) = completion.take() {
+            completion.complete();
+        }
+    };
+    sse_response(Body::from_stream(stream))
+}
+
+fn sse_response(body: Body) -> Response {
+    let mut response = Response::new(body);
+    let headers = response.headers_mut();
+    headers.insert(
+        header::CONTENT_TYPE,
+        HeaderValue::from_static("text/event-stream; charset=utf-8"),
+    );
+    headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-cache"));
+    response
+}
+
+async fn read_request_body(app: &App, request: axum::extract::Request) -> Result<Bytes, Response> {
+    let body_started_at = Instant::now();
+    let body = to_bytes(request.into_body(), MAX_MOCK_REQUEST_BODY_BYTES)
+        .await
+        .map_err(|err| {
+            (
+                StatusCode::BAD_REQUEST,
+                format!("mock upstream failed to read request body: {err}\n"),
+            )
+                .into_response()
+        });
+    record_request_body_read(app, body_started_at.elapsed());
+    body
+}
+
+fn request_wants_stream(body: &[u8]) -> bool {
+    serde_json::from_slice::<serde_json::Value>(body)
+        .ok()
+        .and_then(|value| value.get("stream").and_then(serde_json::Value::as_bool))
+        .unwrap_or(false)
+}
+
+fn request_model(body: &[u8]) -> Option<String> {
+    serde_json::from_slice::<serde_json::Value>(body)
+        .ok()
+        .and_then(|value| {
+            value
+                .get("model")
+                .and_then(serde_json::Value::as_str)
+                .map(str::to_string)
+        })
+}
+
+fn mock_payload(bytes: usize) -> String {
+    if bytes == 0 {
+        return String::new();
+    }
+    "x".repeat(bytes)
+}
+
+fn record_request_started(app: &App) -> Instant {
+    app.metrics.requests_total.fetch_add(1, Ordering::AcqRel);
+    let in_flight = app.metrics.in_flight.fetch_add(1, Ordering::AcqRel) + 1;
+    app.metrics
+        .max_in_flight
+        .fetch_max(in_flight, Ordering::AcqRel);
+    if let Some(bind) = app.metrics.bind_metrics(app.bind_label.as_ref()) {
+        bind.requests_total.fetch_add(1, Ordering::AcqRel);
+        let in_flight = bind.in_flight.fetch_add(1, Ordering::AcqRel) + 1;
+        bind.max_in_flight.fetch_max(in_flight, Ordering::AcqRel);
+    }
+    Instant::now()
+}
+
+fn record_request_completed(app: &App) {
+    app.metrics.completed_total.fetch_add(1, Ordering::AcqRel);
+    app.metrics.in_flight.fetch_sub(1, Ordering::AcqRel);
+    if let Some(bind) = app.metrics.bind_metrics(app.bind_label.as_ref()) {
+        bind.completed_total.fetch_add(1, Ordering::AcqRel);
+        bind.in_flight.fetch_sub(1, Ordering::AcqRel);
+    }
+}
+
+fn record_first_chunk_yield(app: &App, elapsed: Duration) {
+    let elapsed_ms = elapsed.as_millis() as u64;
+    app.metrics
+        .first_chunk_yield_total
+        .fetch_add(1, Ordering::AcqRel);
+    app.metrics
+        .response_header_to_first_chunk_sum_ms
+        .fetch_add(elapsed_ms, Ordering::AcqRel);
+    app.metrics
+        .response_header_to_first_chunk_max_ms
+        .fetch_max(elapsed_ms, Ordering::AcqRel);
+    if let Some(bind) = app.metrics.bind_metrics(app.bind_label.as_ref()) {
+        bind.first_chunk_yield_total.fetch_add(1, Ordering::AcqRel);
+        bind.response_header_to_first_chunk_sum_ms
+            .fetch_add(elapsed_ms, Ordering::AcqRel);
+        bind.response_header_to_first_chunk_max_ms
+            .fetch_max(elapsed_ms, Ordering::AcqRel);
+    }
+}
+
+fn record_request_body_read(app: &App, elapsed: Duration) {
+    let elapsed_ms = elapsed.as_millis() as u64;
+    app.metrics
+        .request_body_read_sum_ms
+        .fetch_add(elapsed_ms, Ordering::AcqRel);
+    app.metrics
+        .request_body_read_max_ms
+        .fetch_max(elapsed_ms, Ordering::AcqRel);
+    if let Some(bind) = app.metrics.bind_metrics(app.bind_label.as_ref()) {
+        bind.request_body_read_sum_ms
+            .fetch_add(elapsed_ms, Ordering::AcqRel);
+        bind.request_body_read_max_ms
+            .fetch_max(elapsed_ms, Ordering::AcqRel);
+    }
+}
+
+fn record_response_header_created(app: &App, elapsed: Duration) {
+    let elapsed_ms = elapsed.as_millis() as u64;
+    app.metrics
+        .accepted_to_response_header_sum_ms
+        .fetch_add(elapsed_ms, Ordering::AcqRel);
+    app.metrics
+        .accepted_to_response_header_max_ms
+        .fetch_max(elapsed_ms, Ordering::AcqRel);
+    if let Some(bind) = app.metrics.bind_metrics(app.bind_label.as_ref()) {
+        bind.accepted_to_response_header_sum_ms
+            .fetch_add(elapsed_ms, Ordering::AcqRel);
+        bind.accepted_to_response_header_max_ms
+            .fetch_max(elapsed_ms, Ordering::AcqRel);
+    }
+}
+
+fn prometheus_label_value(value: &str) -> String {
+    value
+        .replace('\\', r"\\")
+        .replace('\n', r"\n")
+        .replace('"', r#"\""#)
+}
+
+fn current_unix_secs() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|duration| duration.as_secs())
+        .unwrap_or(0)
+}
+
+fn parse_args(args: Vec<String>) -> Result<Config, Box<dyn std::error::Error>> {
+    let mut config = Config::default();
+    let mut binds_overridden = false;
+    let mut iter = args.into_iter();
+    while let Some(arg) = iter.next() {
+        match arg.as_str() {
+            "--bind" => {
+                if !binds_overridden {
+                    config.binds.clear();
+                    binds_overridden = true;
+                }
+                config.binds.push(next_value(&mut iter, "--bind")?.parse()?);
+            }
+            "--chunks" => config.chunks = next_value(&mut iter, "--chunks")?.parse()?,
+            "--first-byte-delay-ms" => {
+                config.first_byte_delay =
+                    Duration::from_millis(next_value(&mut iter, "--first-byte-delay-ms")?.parse()?)
+            }
+            "--chunk-delay-ms" => {
+                config.chunk_delay =
+                    Duration::from_millis(next_value(&mut iter, "--chunk-delay-ms")?.parse()?)
+            }
+            "--payload-bytes" => {
+                config.payload_bytes = next_value(&mut iter, "--payload-bytes")?.parse()?
+            }
+            "--status" => {
+                let status = next_value(&mut iter, "--status")?.parse::<u16>()?;
+                config.status = StatusCode::from_u16(status)?;
+            }
+            "--assume-stream" => {
+                config.assume_stream = true;
+            }
+            "--help" | "-h" => {
+                print_usage();
+                std::process::exit(0);
+            }
+            other => {
+                return Err(std::io::Error::new(
+                    std::io::ErrorKind::InvalidInput,
+                    format!("unknown argument: {other}"),
+                )
+                .into());
+            }
+        }
+    }
+    if config.binds.is_empty() {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            "at least one --bind is required",
+        )
+        .into());
+    }
+    Ok(config)
+}
+
+fn next_value(
+    iter: &mut impl Iterator<Item = String>,
+    flag: &str,
+) -> Result<String, Box<dyn std::error::Error>> {
+    iter.next().ok_or_else(|| {
+        std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            format!("missing value for {flag}"),
+        )
+        .into()
+    })
+}
+
+fn print_usage() {
+    eprintln!(
+        "usage: cargo run -p aether-integration-tests --bin mock_openai_upstream -- [--bind 127.0.0.1:18181]... [--chunks 8] [--first-byte-delay-ms 0] [--chunk-delay-ms 20] [--payload-bytes 32] [--status 200] [--assume-stream]"
+    );
+}

crates/aether-testing/integration/src/bin/multi_instance_admission_baseline.rs

@@ -0,0 +1,704 @@
+// Gateway-backed benchmark scenarios live outside the reusable testkit.
+use std::collections::BTreeMap;
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use aether_contracts::{ExecutionPlan, ExecutionTimeouts, RequestBody};
+use aether_runtime_state::{
+    RedisClientConfig, RuntimeSemaphore, RuntimeSemaphoreConfig, RuntimeState,
+};
+use aether_testkit::{
+    init_test_runtime_for, run_multi_url_http_load_probe, BenchmarkRuntimeSampler,
+    BenchmarkRuntimeSnapshot, ExecutionRuntimeHarness, ExecutionRuntimeHarnessConfig,
+    GatewayHarness, GatewayHarnessConfig, HttpLoadProbeConfig, HttpLoadProbeResponseMode,
+    ManagedRedisServer, MultiUrlHttpLoadProbeResult, SpawnedServer, TunnelHarness,
+    TunnelHarnessConfig, GATEWAY_HARNESS_API_KEY,
+};
+use axum::body::to_bytes;
+use axum::extract::Request;
+use axum::response::IntoResponse;
+use axum::routing::any;
+use axum::{Json, Router};
+use futures_util::StreamExt;
+use reqwest::Method;
+use serde::Serialize;
+use serde_json::json;
+use tokio_tungstenite::tungstenite::client::IntoClientRequest;
+use tokio_tungstenite::tungstenite::Message;
+
+const PROXY_TUNNEL_PATH: &str = "/api/internal/proxy-tunnel";
+
+#[derive(Debug, Clone)]
+struct MultiInstanceAdmissionBaselineConfig {
+    gateway_requests: usize,
+    gateway_concurrency: usize,
+    execution_runtime_requests: usize,
+    execution_runtime_concurrency: usize,
+    tunnel_attempts: usize,
+    tunnel_concurrency: usize,
+    tunnel_hold: Duration,
+    upstream_delay: Duration,
+    request_limit: usize,
+    tunnel_request_limit: usize,
+    timeout: Duration,
+    output_path: Option<PathBuf>,
+    redis_url: Option<String>,
+}
+
+impl Default for MultiInstanceAdmissionBaselineConfig {
+    fn default() -> Self {
+        Self {
+            gateway_requests: 200,
+            gateway_concurrency: 20,
+            execution_runtime_requests: 200,
+            execution_runtime_concurrency: 20,
+            tunnel_attempts: 40,
+            tunnel_concurrency: 10,
+            tunnel_hold: Duration::from_millis(100),
+            upstream_delay: Duration::from_millis(100),
+            request_limit: 8,
+            tunnel_request_limit: 4,
+            timeout: Duration::from_secs(10),
+            output_path: None,
+            redis_url: None,
+        }
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct MultiInstanceAdmissionBaselineReport {
+    suite: &'static str,
+    redis_url: String,
+    gateway_sync: MultiUrlHttpLoadProbeResult,
+    execution_runtime_sync: MultiUrlHttpLoadProbeResult,
+    tunnel_proxy: WebSocketAdmissionProbeResult,
+}
+
+#[derive(Debug, Clone, Serialize)]
+struct WebSocketAdmissionProbeResult {
+    target_urls: Vec<String>,
+    target_attempt_counts: BTreeMap<String, usize>,
+    total_attempts: usize,
+    concurrency: usize,
+    completed_attempts: usize,
+    failed_attempts: usize,
+    rejected_attempts: usize,
+    successful_attempts: usize,
+    p50_ms: u64,
+    p95_ms: u64,
+    p99_ms: u64,
+    max_ms: u64,
+    mean_ms: u64,
+    status_counts: BTreeMap<u16, usize>,
+    runtime: BenchmarkRuntimeSnapshot,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    init_test_runtime_for("multi-instance-admission-baseline");
+    let config = parse_args(std::env::args().skip(1).collect())?;
+    let report = run_suite(&config).await?;
+    let raw = serde_json::to_string_pretty(&report)?;
+    println!("{raw}");
+    if let Some(path) = config.output_path.as_ref() {
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        std::fs::write(path, format!("{raw}\n"))?;
+    }
+    Ok(())
+}
+
+async fn run_suite(
+    config: &MultiInstanceAdmissionBaselineConfig,
+) -> Result<MultiInstanceAdmissionBaselineReport, Box<dyn std::error::Error>> {
+    let managed_redis = if config.redis_url.is_none() {
+        Some(ManagedRedisServer::start().await?)
+    } else {
+        None
+    };
+    let redis_url = config
+        .redis_url
+        .clone()
+        .or_else(|| {
+            managed_redis
+                .as_ref()
+                .map(|server| server.redis_url().to_string())
+        })
+        .expect("redis url should be resolved");
+
+    let upstream = SpawnedServer::start(build_delayed_upstream(config.upstream_delay)).await?;
+
+    let (gateway_urls, _gateways) =
+        start_gateway_pair(&redis_url, upstream.base_url(), config).await?;
+    let (execution_runtime_urls, _runtimes) =
+        start_execution_runtime_pair(&redis_url, upstream.base_url(), config).await?;
+    let (tunnel_urls, _tunnels) = start_tunnel_pair(&redis_url, config).await?;
+
+    let gateway_sync = run_multi_url_http_load_probe(
+        &gateway_sync_probe_config(&gateway_urls, config),
+        &gateway_urls,
+    )
+    .await
+    .map_err(std::io::Error::other)?;
+    let execution_runtime_sync = run_multi_url_http_load_probe(
+        &execution_runtime_sync_probe_config(&execution_runtime_urls, upstream.base_url(), config),
+        &execution_runtime_urls,
+    )
+    .await
+    .map_err(std::io::Error::other)?;
+    let tunnel_proxy = run_tunnel_proxy_connection_probe(&tunnel_urls, config)
+        .await
+        .map_err(std::io::Error::other)?;
+
+    Ok(MultiInstanceAdmissionBaselineReport {
+        suite: "multi_instance_admission_baseline",
+        redis_url,
+        gateway_sync,
+        execution_runtime_sync,
+        tunnel_proxy,
+    })
+}
+
+async fn start_gateway_pair(
+    redis_url: &str,
+    upstream_base_url: &str,
+    config: &MultiInstanceAdmissionBaselineConfig,
+) -> Result<(Vec<String>, Vec<GatewayHarness>), Box<dyn std::error::Error>> {
+    let gate_a = distributed_request_gate(
+        "gateway_requests_distributed",
+        config.request_limit,
+        redis_url,
+        "gateway-a",
+    )
+    .await?;
+    let gate_b = distributed_request_gate(
+        "gateway_requests_distributed",
+        config.request_limit,
+        redis_url,
+        "gateway-b",
+    )
+    .await?;
+    let gateway_a = GatewayHarness::start(GatewayHarnessConfig {
+        upstream_base_url: upstream_base_url.to_string(),
+        data_config: None,
+        max_in_flight_requests: None,
+        distributed_request_gate: Some(gate_a),
+        tunnel_instance_id: None,
+        tunnel_relay_base_url: None,
+    })
+    .await?;
+    let gateway_b = GatewayHarness::start(GatewayHarnessConfig {
+        upstream_base_url: upstream_base_url.to_string(),
+        data_config: None,
+        max_in_flight_requests: None,
+        distributed_request_gate: Some(gate_b),
+        tunnel_instance_id: None,
+        tunnel_relay_base_url: None,
+    })
+    .await?;
+    Ok((
+        vec![
+            format!("{}/v1/chat/completions", gateway_a.base_url()),
+            format!("{}/v1/chat/completions", gateway_b.base_url()),
+        ],
+        vec![gateway_a, gateway_b],
+    ))
+}
+
+async fn start_execution_runtime_pair(
+    redis_url: &str,
+    upstream_base_url: &str,
+    config: &MultiInstanceAdmissionBaselineConfig,
+) -> Result<(Vec<String>, Vec<ExecutionRuntimeHarness>), Box<dyn std::error::Error>> {
+    let gate_a = distributed_request_gate(
+        "execution_runtime_requests_distributed",
+        config.request_limit,
+        redis_url,
+        "execution-runtime-a",
+    )
+    .await?;
+    let gate_b = distributed_request_gate(
+        "execution_runtime_requests_distributed",
+        config.request_limit,
+        redis_url,
+        "execution-runtime-b",
+    )
+    .await?;
+    let runtime_a = ExecutionRuntimeHarness::start(ExecutionRuntimeHarnessConfig {
+        max_in_flight_requests: None,
+        distributed_request_gate: Some(gate_a),
+    })
+    .await?;
+    let runtime_b = ExecutionRuntimeHarness::start(ExecutionRuntimeHarnessConfig {
+        max_in_flight_requests: None,
+        distributed_request_gate: Some(gate_b),
+    })
+    .await?;
+    let _ = upstream_base_url;
+    Ok((
+        vec![
+            format!("{}/v1/execute/sync", runtime_a.base_url()),
+            format!("{}/v1/execute/sync", runtime_b.base_url()),
+        ],
+        vec![runtime_a, runtime_b],
+    ))
+}
+
+async fn start_tunnel_pair(
+    redis_url: &str,
+    config: &MultiInstanceAdmissionBaselineConfig,
+) -> Result<(Vec<String>, Vec<TunnelHarness>), Box<dyn std::error::Error>> {
+    let gate_a = distributed_request_gate(
+        "tunnel_requests_distributed",
+        config.tunnel_request_limit,
+        redis_url,
+        "tunnel-a",
+    )
+    .await?;
+    let gate_b = distributed_request_gate(
+        "tunnel_requests_distributed",
+        config.tunnel_request_limit,
+        redis_url,
+        "tunnel-b",
+    )
+    .await?;
+    let tunnel_a = TunnelHarness::start(TunnelHarnessConfig {
+        distributed_request_gate: Some(gate_a),
+        ..TunnelHarnessConfig::default()
+    })
+    .await?;
+    let tunnel_b = TunnelHarness::start(TunnelHarnessConfig {
+        distributed_request_gate: Some(gate_b),
+        ..TunnelHarnessConfig::default()
+    })
+    .await?;
+    Ok((
+        vec![
+            format!(
+                "{}{}",
+                tunnel_a.base_url().replace("http://", "ws://"),
+                PROXY_TUNNEL_PATH
+            ),
+            format!(
+                "{}{}",
+                tunnel_b.base_url().replace("http://", "ws://"),
+                PROXY_TUNNEL_PATH
+            ),
+        ],
+        vec![tunnel_a, tunnel_b],
+    ))
+}
+
+async fn distributed_request_gate(
+    name: &'static str,
+    limit: usize,
+    redis_url: &str,
+    _instance_id: &str,
+) -> Result<RuntimeSemaphore, Box<dyn std::error::Error>> {
+    let runtime = RuntimeState::redis(
+        RedisClientConfig {
+            url: redis_url.to_string(),
+            key_prefix: Some(format!("aether-baseline-{}-{name}", std::process::id())),
+        },
+        Some(1_000),
+    )
+    .await?;
+    Ok(runtime.semaphore(
+        name,
+        limit,
+        RuntimeSemaphoreConfig {
+            lease_ttl_ms: 30_000,
+            renew_interval_ms: 10_000,
+            command_timeout_ms: Some(1_000),
+        },
+    )?)
+}
+
+fn gateway_sync_probe_config(
+    urls: &[String],
+    config: &MultiInstanceAdmissionBaselineConfig,
+) -> HttpLoadProbeConfig {
+    let mut probe = chat_probe_config(
+        urls[0].clone(),
+        config.gateway_requests,
+        config.gateway_concurrency,
+        config.timeout,
+    );
+    probe.response_mode = HttpLoadProbeResponseMode::FullBody;
+    probe
+}
+
+fn execution_runtime_sync_probe_config(
+    urls: &[String],
+    upstream_base_url: &str,
+    config: &MultiInstanceAdmissionBaselineConfig,
+) -> HttpLoadProbeConfig {
+    let _ = urls;
+    HttpLoadProbeConfig {
+        url: urls[0].clone(),
+        method: Method::POST,
+        headers: BTreeMap::from([("content-type".to_string(), "application/json".to_string())]),
+        body: Some(
+            serde_json::to_vec(&execution_plan(format!(
+                "{upstream_base_url}/v1/chat/completions"
+            )))
+            .expect("execution plan should serialize"),
+        ),
+        total_requests: config.execution_runtime_requests,
+        concurrency: config.execution_runtime_concurrency,
+        timeout: config.timeout,
+        response_mode: HttpLoadProbeResponseMode::FullBody,
+        ..HttpLoadProbeConfig::default()
+    }
+}
+
+fn chat_probe_config(
+    url: String,
+    total_requests: usize,
+    concurrency: usize,
+    timeout: Duration,
+) -> HttpLoadProbeConfig {
+    HttpLoadProbeConfig {
+        url,
+        method: Method::POST,
+        headers: BTreeMap::from([
+            ("content-type".to_string(), "application/json".to_string()),
+            (
+                "authorization".to_string(),
+                format!("Bearer {GATEWAY_HARNESS_API_KEY}"),
+            ),
+        ]),
+        body: Some(
+            serde_json::to_vec(&json!({
+                "model": "gpt-5",
+                "messages": [{"role": "user", "content": "hello"}],
+                "stream": false,
+            }))
+            .expect("chat body should serialize"),
+        ),
+        total_requests,
+        concurrency,
+        timeout,
+        response_mode: HttpLoadProbeResponseMode::FullBody,
+        ..HttpLoadProbeConfig::default()
+    }
+}
+
+fn execution_plan(url: String) -> ExecutionPlan {
+    ExecutionPlan {
+        request_id: "multi-instance-sync-request".to_string(),
+        candidate_id: Some("multi-instance-sync-candidate".to_string()),
+        provider_name: Some("openai".to_string()),
+        provider_id: "provider-baseline".to_string(),
+        endpoint_id: "endpoint-baseline".to_string(),
+        key_id: "key-baseline".to_string(),
+        method: "POST".to_string(),
+        url,
+        headers: BTreeMap::from([("content-type".to_string(), "application/json".to_string())]),
+        content_type: Some("application/json".to_string()),
+        content_encoding: None,
+        body: RequestBody::from_json(json!({
+            "model": "gpt-5",
+            "messages": [{"role": "user", "content": "hello"}],
+            "stream": false,
+        })),
+        stream: false,
+        client_api_format: "openai:chat".to_string(),
+        provider_api_format: "openai:chat".to_string(),
+        model_name: Some("gpt-5".to_string()),
+        proxy: None,
+        transport_profile: None,
+        timeouts: Some(ExecutionTimeouts {
+            connect_ms: Some(2_000),
+            read_ms: Some(10_000),
+            first_byte_ms: Some(5_000),
+            total_ms: Some(10_000),
+            ..ExecutionTimeouts::default()
+        }),
+    }
+}
+
+fn build_delayed_upstream(delay: Duration) -> Router {
+    Router::new().route(
+        "/v1/chat/completions",
+        any(move |request: Request| {
+            let delay = delay;
+            async move {
+                let (_parts, body) = request.into_parts();
+                let raw_body = to_bytes(body, usize::MAX)
+                    .await
+                    .expect("fake upstream body should read");
+                let payload: serde_json::Value =
+                    serde_json::from_slice(&raw_body).unwrap_or_else(|_| json!({}));
+                tokio::time::sleep(delay).await;
+                Json(json!({
+                    "id": "chatcmpl-distributed",
+                    "object": "chat.completion",
+                    "model": payload.get("model").and_then(|value| value.as_str()).unwrap_or("gpt-5"),
+                    "choices": [{"message": {"role": "assistant", "content": "hello"}}]
+                }))
+                .into_response()
+            }
+        }),
+    )
+}
+
+async fn run_tunnel_proxy_connection_probe(
+    urls: &[String],
+    config: &MultiInstanceAdmissionBaselineConfig,
+) -> Result<WebSocketAdmissionProbeResult, String> {
+    if urls.is_empty() {
+        return Err("tunnel proxy connection probe requires at least one target url".to_string());
+    }
+    let mut runtime_sampler = BenchmarkRuntimeSampler::new();
+    let next_attempt = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+    let latencies_ms = Arc::new(tokio::sync::Mutex::new(Vec::with_capacity(
+        config.tunnel_attempts,
+    )));
+    let target_attempt_counts = Arc::new(tokio::sync::Mutex::new(BTreeMap::<String, usize>::new()));
+    let status_counts = Arc::new(tokio::sync::Mutex::new(BTreeMap::<u16, usize>::new()));
+    let failed_attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+    let rejected_attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+    let successful_attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+    let completed_attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+
+    let mut workers = tokio::task::JoinSet::new();
+    for worker_index in 0..config.tunnel_concurrency {
+        let urls = urls.to_vec();
+        let next_attempt = Arc::clone(&next_attempt);
+        let latencies_ms = Arc::clone(&latencies_ms);
+        let target_attempt_counts = Arc::clone(&target_attempt_counts);
+        let status_counts = Arc::clone(&status_counts);
+        let failed_attempts = Arc::clone(&failed_attempts);
+        let rejected_attempts = Arc::clone(&rejected_attempts);
+        let successful_attempts = Arc::clone(&successful_attempts);
+        let completed_attempts = Arc::clone(&completed_attempts);
+        let timeout = config.timeout;
+        let hold = config.tunnel_hold;
+        let total_attempts = config.tunnel_attempts;
+
+        workers.spawn(async move {
+            loop {
+                let current = next_attempt.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
+                if current >= total_attempts {
+                    break;
+                }
+                let url = urls[current % urls.len()].clone();
+                {
+                    let mut counts = target_attempt_counts.lock().await;
+                    *counts.entry(url.clone()).or_insert(0) += 1;
+                }
+                let mut request = url
+                    .into_client_request()
+                    .map_err(|err| format!("failed to build websocket request: {err}"))?;
+                request.headers_mut().insert(
+                    "x-node-id",
+                    format!("baseline-node-{worker_index}-{current}")
+                        .parse()
+                        .map_err(|err| format!("failed to build x-node-id header: {err}"))?,
+                );
+                request.headers_mut().insert(
+                    aether_contracts::tunnel::TUNNEL_PROTOCOL_VERSION_HEADER,
+                    aether_contracts::tunnel::CURRENT_TUNNEL_PROTOCOL_VERSION_STR
+                        .parse()
+                        .expect("protocol version header value should be valid"),
+                );
+                request.headers_mut().insert(
+                    "x-node-name",
+                    format!("baseline-node-{worker_index}-{current}")
+                        .parse()
+                        .map_err(|err| format!("failed to build x-node-name header: {err}"))?,
+                );
+
+                let started_at = Instant::now();
+                match tokio::time::timeout(timeout, tokio_tungstenite::connect_async(request)).await
+                {
+                    Ok(Ok((mut ws, _response))) => {
+                        {
+                            let mut counts = status_counts.lock().await;
+                            *counts.entry(101).or_insert(0) += 1;
+                        }
+                        latencies_ms
+                            .lock()
+                            .await
+                            .push(started_at.elapsed().as_millis() as u64);
+                        successful_attempts.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
+                        completed_attempts.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
+                        tokio::time::sleep(hold).await;
+                        let _ = ws.close(None).await;
+                        while let Some(message) = ws.next().await {
+                            if matches!(message, Ok(Message::Close(_))) || message.is_err() {
+                                break;
+                            }
+                        }
+                    }
+                    Ok(Err(tokio_tungstenite::tungstenite::Error::Http(response))) => {
+                        let status = response.status().as_u16();
+                        {
+                            let mut counts = status_counts.lock().await;
+                            *counts.entry(status).or_insert(0) += 1;
+                        }
+                        latencies_ms
+                            .lock()
+                            .await
+                            .push(started_at.elapsed().as_millis() as u64);
+                        if status == 503 {
+                            rejected_attempts.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
+                        } else {
+                            failed_attempts.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
+                        }
+                        completed_attempts.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
+                    }
+                    Ok(Err(_)) | Err(_) => {
+                        latencies_ms
+                            .lock()
+                            .await
+                            .push(started_at.elapsed().as_millis() as u64);
+                        failed_attempts.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
+                        completed_attempts.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
+                    }
+                }
+            }
+            Ok::<(), String>(())
+        });
+    }
+
+    while let Some(result) = workers.join_next().await {
+        result
+            .map_err(|err| format!("tunnel admission worker task failed: {err}"))?
+            .map_err(|err| format!("tunnel admission worker failed: {err}"))?;
+    }
+
+    let mut latencies = latencies_ms.lock().await.clone();
+    latencies.sort_unstable();
+    let (p50_ms, p95_ms, p99_ms, max_ms, mean_ms) = summarize_latencies(&latencies);
+
+    let target_attempt_counts = target_attempt_counts.lock().await.clone();
+    let status_counts = status_counts.lock().await.clone();
+    Ok(WebSocketAdmissionProbeResult {
+        target_urls: urls.to_vec(),
+        target_attempt_counts,
+        total_attempts: config.tunnel_attempts,
+        concurrency: config.tunnel_concurrency,
+        completed_attempts: completed_attempts.load(std::sync::atomic::Ordering::Acquire),
+        failed_attempts: failed_attempts.load(std::sync::atomic::Ordering::Acquire),
+        rejected_attempts: rejected_attempts.load(std::sync::atomic::Ordering::Acquire),
+        successful_attempts: successful_attempts.load(std::sync::atomic::Ordering::Acquire),
+        p50_ms,
+        p95_ms,
+        p99_ms,
+        max_ms,
+        mean_ms,
+        status_counts,
+        runtime: runtime_sampler.snapshot(),
+    })
+}
+
+fn summarize_latencies(latencies: &[u64]) -> (u64, u64, u64, u64, u64) {
+    if latencies.is_empty() {
+        return (0, 0, 0, 0, 0);
+    }
+    let max_ms = *latencies.last().unwrap_or(&0);
+    let mean_ms = latencies.iter().sum::<u64>() / latencies.len() as u64;
+    let p50_ms = percentile(latencies, 50);
+    let p95_ms = percentile(latencies, 95);
+    let p99_ms = percentile(latencies, 99);
+    (p50_ms, p95_ms, p99_ms, max_ms, mean_ms)
+}
+
+fn percentile(latencies: &[u64], percentile: u8) -> u64 {
+    if latencies.is_empty() {
+        return 0;
+    }
+    let last_index = latencies.len() - 1;
+    let rank = ((last_index as f64) * (percentile as f64 / 100.0)).round() as usize;
+    latencies[rank.min(last_index)]
+}
+
+fn parse_args(
+    args: Vec<String>,
+) -> Result<MultiInstanceAdmissionBaselineConfig, Box<dyn std::error::Error>> {
+    let mut config = MultiInstanceAdmissionBaselineConfig::default();
+    let mut iter = args.into_iter();
+    while let Some(arg) = iter.next() {
+        match arg.as_str() {
+            "--gateway-requests" => {
+                config.gateway_requests = next_value(&mut iter, "--gateway-requests")?.parse()?
+            }
+            "--gateway-concurrency" => {
+                config.gateway_concurrency =
+                    next_value(&mut iter, "--gateway-concurrency")?.parse()?
+            }
+            "--execution-runtime-requests" | "--executor-requests" => {
+                config.execution_runtime_requests = next_value(&mut iter, arg.as_str())?.parse()?
+            }
+            "--execution-runtime-concurrency" | "--executor-concurrency" => {
+                config.execution_runtime_concurrency =
+                    next_value(&mut iter, arg.as_str())?.parse()?
+            }
+            "--tunnel-attempts" => {
+                config.tunnel_attempts = next_value(&mut iter, "--tunnel-attempts")?.parse()?
+            }
+            "--tunnel-concurrency" => {
+                config.tunnel_concurrency =
+                    next_value(&mut iter, "--tunnel-concurrency")?.parse()?
+            }
+            "--tunnel-hold-ms" => {
+                config.tunnel_hold =
+                    Duration::from_millis(next_value(&mut iter, "--tunnel-hold-ms")?.parse()?)
+            }
+            "--upstream-delay-ms" => {
+                config.upstream_delay =
+                    Duration::from_millis(next_value(&mut iter, "--upstream-delay-ms")?.parse()?)
+            }
+            "--request-limit" => {
+                config.request_limit = next_value(&mut iter, "--request-limit")?.parse()?
+            }
+            "--tunnel-request-limit" => {
+                config.tunnel_request_limit =
+                    next_value(&mut iter, "--tunnel-request-limit")?.parse()?
+            }
+            "--timeout-ms" => {
+                config.timeout =
+                    Duration::from_millis(next_value(&mut iter, "--timeout-ms")?.parse()?)
+            }
+            "--redis-url" => config.redis_url = Some(next_value(&mut iter, "--redis-url")?),
+            "--output" => {
+                config.output_path = Some(PathBuf::from(next_value(&mut iter, "--output")?))
+            }
+            "--help" | "-h" => {
+                print_usage();
+                std::process::exit(0);
+            }
+            other => {
+                return Err(std::io::Error::new(
+                    std::io::ErrorKind::InvalidInput,
+                    format!("unknown argument: {other}"),
+                )
+                .into());
+            }
+        }
+    }
+    Ok(config)
+}
+
+fn next_value(
+    iter: &mut impl Iterator<Item = String>,
+    flag: &str,
+) -> Result<String, Box<dyn std::error::Error>> {
+    iter.next().ok_or_else(|| {
+        std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            format!("missing value for {flag}"),
+        )
+        .into()
+    })
+}
+
+fn print_usage() {
+    eprintln!(
+        "usage: cargo run -p aether-integration-tests --bin multi_instance_admission_baseline -- [--gateway-requests 200] [--gateway-concurrency 20] [--execution-runtime-requests 200] [--execution-runtime-concurrency 20] [--tunnel-attempts 40] [--tunnel-concurrency 10] [--tunnel-hold-ms 100] [--upstream-delay-ms 100] [--request-limit 8] [--tunnel-request-limit 4] [--redis-url redis://127.0.0.1:6379/0] [--output /tmp/multi_instance_admission_baseline.json]"
+    );
+}

crates/aether-testing/integration/src/bin/multi_instance_owner_relay_baseline.rs

@@ -0,0 +1,467 @@
+// Gateway-backed benchmark scenarios live outside the reusable testkit.
+use std::path::PathBuf;
+use std::time::Duration;
+
+use aether_gateway::tunnel_protocol as protocol;
+use aether_gateway::GatewayDataConfig;
+use aether_testkit::{
+    init_test_runtime_for, prepare_aether_postgres_schema, reserve_local_port, run_http_load_probe,
+    wait_until, GatewayHarness, GatewayHarnessConfig, HttpLoadProbeConfig,
+    HttpLoadProbeResponseMode, HttpLoadProbeResult, ManagedPostgresServer, ManagedRedisServer,
+};
+use futures_util::{SinkExt, StreamExt};
+use reqwest::Method;
+use serde::Serialize;
+use tokio_tungstenite::tungstenite::client::IntoClientRequest;
+use tokio_tungstenite::tungstenite::Message;
+
+const PROXY_TUNNEL_PATH: &str = "/api/internal/proxy-tunnel";
+const TUNNEL_RELAY_PATH_PREFIX: &str = "/api/internal/tunnel/relay";
+const NODE_ID: &str = "node-owner-relay-baseline";
+
+#[derive(Debug, Clone)]
+struct MultiInstanceOwnerRelayBaselineConfig {
+    total_requests: usize,
+    concurrency: usize,
+    timeout: Duration,
+    chunk_delay: Duration,
+    output_path: Option<PathBuf>,
+    redis_url: Option<String>,
+    postgres_url: Option<String>,
+}
+
+impl Default for MultiInstanceOwnerRelayBaselineConfig {
+    fn default() -> Self {
+        Self {
+            total_requests: 200,
+            concurrency: 20,
+            timeout: Duration::from_secs(10),
+            chunk_delay: Duration::ZERO,
+            output_path: None,
+            redis_url: None,
+            postgres_url: None,
+        }
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct MultiInstanceOwnerRelayBaselineReport {
+    suite: &'static str,
+    redis_url: String,
+    postgres_url: String,
+    owner_instance_id: &'static str,
+    forwarder_instance_id: &'static str,
+    direct_owner_relay: HttpLoadProbeResult,
+    remote_owner_relay: HttpLoadProbeResult,
+    relay_overhead_ms: RelayOverheadSnapshot,
+}
+
+#[derive(Debug, Serialize)]
+struct RelayOverheadSnapshot {
+    p50_delta_ms: i64,
+    p95_delta_ms: i64,
+    max_delta_ms: i64,
+    mean_delta_ms: i64,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    init_test_runtime_for("multi-instance-owner-relay-baseline");
+    let config = parse_args(std::env::args().skip(1).collect())?;
+    let report = run_suite(&config).await?;
+    let raw = serde_json::to_string_pretty(&report)?;
+    println!("{raw}");
+    if let Some(path) = config.output_path.as_ref() {
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        std::fs::write(path, format!("{raw}\n"))?;
+    }
+    Ok(())
+}
+
+async fn run_suite(
+    config: &MultiInstanceOwnerRelayBaselineConfig,
+) -> Result<MultiInstanceOwnerRelayBaselineReport, Box<dyn std::error::Error>> {
+    let managed_redis = if config.redis_url.is_none() {
+        Some(ManagedRedisServer::start().await?)
+    } else {
+        None
+    };
+    let redis_url = config
+        .redis_url
+        .clone()
+        .or_else(|| {
+            managed_redis
+                .as_ref()
+                .map(|server| server.redis_url().to_string())
+        })
+        .expect("redis url should be resolved");
+
+    let managed_postgres = if config.postgres_url.is_none() {
+        Some(ManagedPostgresServer::start().await?)
+    } else {
+        None
+    };
+    let postgres_url = config
+        .postgres_url
+        .clone()
+        .or_else(|| {
+            managed_postgres
+                .as_ref()
+                .map(|server| server.database_url().to_string())
+        })
+        .expect("postgres url should be resolved");
+
+    prepare_aether_postgres_schema(&postgres_url).await?;
+
+    let key_prefix = format!("aether-owner-relay-baseline-{}", std::process::id());
+    let shared_data = GatewayDataConfig::from_postgres_url(postgres_url.clone(), false)
+        .with_redis_url(redis_url.clone(), Some(key_prefix));
+
+    let owner_port = reserve_local_port()?;
+    let forwarder_port = reserve_local_port()?;
+    let owner_base_url = format!("http://127.0.0.1:{owner_port}");
+    let forwarder_base_url = format!("http://127.0.0.1:{forwarder_port}");
+
+    let owner_gateway = GatewayHarness::start_on_port(
+        GatewayHarnessConfig {
+            upstream_base_url: "http://127.0.0.1:1".to_string(),
+            data_config: Some(shared_data.clone()),
+            max_in_flight_requests: None,
+            distributed_request_gate: None,
+            tunnel_instance_id: Some("gateway-owner".to_string()),
+            tunnel_relay_base_url: Some(owner_base_url.clone()),
+        },
+        owner_port,
+    )
+    .await?;
+    let forwarder_gateway = GatewayHarness::start_on_port(
+        GatewayHarnessConfig {
+            upstream_base_url: "http://127.0.0.1:1".to_string(),
+            data_config: Some(shared_data),
+            max_in_flight_requests: None,
+            distributed_request_gate: None,
+            tunnel_instance_id: Some("gateway-forwarder".to_string()),
+            tunnel_relay_base_url: Some(forwarder_base_url.clone()),
+        },
+        forwarder_port,
+    )
+    .await?;
+
+    let peer = connect_protocol_peer(owner_gateway.base_url(), config.chunk_delay).await?;
+
+    wait_for_owner_attachment(&forwarder_base_url).await?;
+
+    let direct_owner_relay = run_http_load_probe(&HttpLoadProbeConfig {
+        url: format!(
+            "{owner_base}{TUNNEL_RELAY_PATH_PREFIX}/{NODE_ID}",
+            owner_base = owner_gateway.base_url()
+        ),
+        method: Method::POST,
+        headers: relay_headers(),
+        body: Some(relay_envelope()),
+        total_requests: config.total_requests,
+        concurrency: config.concurrency,
+        timeout: config.timeout,
+        response_mode: HttpLoadProbeResponseMode::FullBody,
+        ..HttpLoadProbeConfig::default()
+    })
+    .await
+    .map_err(std::io::Error::other)?;
+
+    let remote_owner_relay = run_http_load_probe(&HttpLoadProbeConfig {
+        url: format!(
+            "{forwarder_base}{TUNNEL_RELAY_PATH_PREFIX}/{NODE_ID}",
+            forwarder_base = forwarder_gateway.base_url()
+        ),
+        method: Method::POST,
+        headers: relay_headers(),
+        body: Some(relay_envelope()),
+        total_requests: config.total_requests,
+        concurrency: config.concurrency,
+        timeout: config.timeout,
+        response_mode: HttpLoadProbeResponseMode::FullBody,
+        ..HttpLoadProbeConfig::default()
+    })
+    .await
+    .map_err(std::io::Error::other)?;
+
+    drop(peer);
+    drop(forwarder_gateway);
+    drop(owner_gateway);
+
+    Ok(MultiInstanceOwnerRelayBaselineReport {
+        suite: "multi_instance_owner_relay_baseline",
+        redis_url,
+        postgres_url,
+        owner_instance_id: "gateway-owner",
+        forwarder_instance_id: "gateway-forwarder",
+        relay_overhead_ms: RelayOverheadSnapshot {
+            p50_delta_ms: remote_owner_relay.p50_ms as i64 - direct_owner_relay.p50_ms as i64,
+            p95_delta_ms: remote_owner_relay.p95_ms as i64 - direct_owner_relay.p95_ms as i64,
+            max_delta_ms: remote_owner_relay.max_ms as i64 - direct_owner_relay.max_ms as i64,
+            mean_delta_ms: remote_owner_relay.mean_ms as i64 - direct_owner_relay.mean_ms as i64,
+        },
+        direct_owner_relay,
+        remote_owner_relay,
+    })
+}
+
+async fn wait_for_owner_attachment(forwarder_base_url: &str) -> Result<(), String> {
+    let client = reqwest::Client::builder()
+        .timeout(Duration::from_secs(2))
+        .build()
+        .map_err(|err| format!("failed to build readiness client: {err}"))?;
+    let target_url = format!("{forwarder_base_url}{TUNNEL_RELAY_PATH_PREFIX}/{NODE_ID}");
+    let ready = wait_until(Duration::from_secs(10), Duration::from_millis(100), || {
+        let client = client.clone();
+        let target_url = target_url.clone();
+        async move {
+            let response = client
+                .post(target_url)
+                .header("content-type", "application/octet-stream")
+                .body(relay_envelope())
+                .send()
+                .await;
+            match response {
+                Ok(response) if response.status().is_success() => match response.text().await {
+                    Ok(body) => body == "owner-relay-ok",
+                    Err(_) => false,
+                },
+                _ => false,
+            }
+        }
+    })
+    .await;
+    if ready {
+        Ok(())
+    } else {
+        Err("timed out waiting for owner attachment propagation".to_string())
+    }
+}
+
+fn relay_headers() -> std::collections::BTreeMap<String, String> {
+    std::collections::BTreeMap::from([(
+        "content-type".to_string(),
+        "application/octet-stream".to_string(),
+    )])
+}
+
+fn relay_envelope() -> Vec<u8> {
+    let meta = protocol::RequestMeta {
+        method: "POST".to_string(),
+        url: "https://owner-relay.example/v1/chat/completions".to_string(),
+        headers: std::collections::HashMap::from([(
+            "content-type".to_string(),
+            "application/json".to_string(),
+        )]),
+        stream: false,
+        request_timeout_ms: None,
+        stream_first_byte_timeout_ms: None,
+        timeout: 30,
+        follow_redirects: None,
+        http1_only: false,
+        provider_id: None,
+        endpoint_id: None,
+        key_id: None,
+        transport_profile: None,
+    };
+    let meta_json = serde_json::to_vec(&meta).expect("owner relay metadata should serialize");
+    let body = br#"{"model":"gpt-5","messages":[{"role":"user","content":"owner relay"}]}"#;
+    let mut envelope = Vec::with_capacity(4 + meta_json.len() + body.len());
+    envelope.extend_from_slice(&(meta_json.len() as u32).to_be_bytes());
+    envelope.extend_from_slice(&meta_json);
+    envelope.extend_from_slice(body);
+    envelope
+}
+
+async fn connect_protocol_peer(
+    gateway_base_url: &str,
+    chunk_delay: Duration,
+) -> Result<tokio::task::JoinHandle<()>, Box<dyn std::error::Error>> {
+    let ws_url = format!(
+        "{}{}",
+        gateway_base_url.replace("http://", "ws://"),
+        PROXY_TUNNEL_PATH
+    );
+    let request = ws_url.into_client_request()?;
+    let mut request = request;
+    request
+        .headers_mut()
+        .insert("x-node-id", http::HeaderValue::from_static(NODE_ID));
+    request.headers_mut().insert(
+        aether_contracts::tunnel::TUNNEL_PROTOCOL_VERSION_HEADER,
+        http::HeaderValue::from_static(
+            aether_contracts::tunnel::CURRENT_TUNNEL_PROTOCOL_VERSION_STR,
+        ),
+    );
+    request.headers_mut().insert(
+        "x-node-name",
+        http::HeaderValue::from_static("proxy-owner-relay-baseline"),
+    );
+    request.headers_mut().insert(
+        "x-tunnel-max-streams",
+        http::HeaderValue::from_static("256"),
+    );
+
+    let (socket, _response) = tokio_tungstenite::connect_async(request).await?;
+    let (mut sink, mut stream) = socket.split();
+    Ok(tokio::spawn(async move {
+        while let Some(message) = stream.next().await {
+            let Ok(message) = message else {
+                break;
+            };
+            match message {
+                Message::Binary(data)
+                    if handle_binary_frame(&mut sink, data.to_vec(), chunk_delay)
+                        .await
+                        .is_err() =>
+                {
+                    break;
+                }
+                Message::Ping(payload)
+                    if sink.send(Message::Pong(payload.clone())).await.is_err() =>
+                {
+                    break;
+                }
+                Message::Close(_) => break,
+                _ => {}
+            }
+        }
+        let _ = sink.close().await;
+    }))
+}
+
+async fn handle_binary_frame<S>(
+    sink: &mut S,
+    data: Vec<u8>,
+    chunk_delay: Duration,
+) -> Result<(), tokio_tungstenite::tungstenite::Error>
+where
+    S: SinkExt<Message, Error = tokio_tungstenite::tungstenite::Error> + Unpin,
+{
+    let Some(header) = protocol::FrameHeader::parse(&data) else {
+        return Ok(());
+    };
+    match header.msg_type {
+        protocol::PING => {
+            let payload = protocol::frame_payload_by_header(&data, &header).unwrap_or(&[]);
+            sink.send(Message::Binary(protocol::encode_pong(payload).into()))
+                .await?;
+        }
+        protocol::REQUEST_HEADERS => {
+            let payload = protocol::decode_payload(&data, &header).unwrap_or_default();
+            let _ = serde_json::from_slice::<protocol::RequestMeta>(&payload);
+        }
+        protocol::REQUEST_BODY if header.flags & protocol::FLAG_END_STREAM != 0 => {
+            let response_meta = protocol::ResponseMeta {
+                status: 200,
+                headers: vec![(
+                    "content-type".to_string(),
+                    "text/plain; charset=utf-8".to_string(),
+                )],
+            };
+            let response_meta_json =
+                serde_json::to_vec(&response_meta).expect("response metadata should serialize");
+            sink.send(Message::Binary(
+                protocol::encode_frame(
+                    header.stream_id,
+                    protocol::RESPONSE_HEADERS,
+                    0,
+                    &response_meta_json,
+                )
+                .into(),
+            ))
+            .await?;
+
+            for chunk in [b"owner-".as_slice(), b"relay-".as_slice(), b"ok".as_slice()] {
+                if !chunk_delay.is_zero() {
+                    tokio::time::sleep(chunk_delay).await;
+                }
+                sink.send(Message::Binary(
+                    protocol::encode_frame(header.stream_id, protocol::RESPONSE_BODY, 0, chunk)
+                        .into(),
+                ))
+                .await?;
+            }
+
+            sink.send(Message::Binary(
+                protocol::encode_frame(header.stream_id, protocol::STREAM_END, 0, &[]).into(),
+            ))
+            .await?;
+        }
+        _ => {}
+    }
+    Ok(())
+}
+
+fn parse_args(
+    args: Vec<String>,
+) -> Result<MultiInstanceOwnerRelayBaselineConfig, Box<dyn std::error::Error>> {
+    let mut config = MultiInstanceOwnerRelayBaselineConfig::default();
+    let mut iter = args.into_iter();
+    while let Some(arg) = iter.next() {
+        match arg.as_str() {
+            "--requests" => config.total_requests = next_value(&mut iter, "--requests")?.parse()?,
+            "--concurrency" => {
+                config.concurrency = next_value(&mut iter, "--concurrency")?.parse()?
+            }
+            "--timeout-ms" => {
+                config.timeout =
+                    Duration::from_millis(next_value(&mut iter, "--timeout-ms")?.parse()?)
+            }
+            "--chunk-delay-ms" => {
+                config.chunk_delay =
+                    Duration::from_millis(next_value(&mut iter, "--chunk-delay-ms")?.parse()?)
+            }
+            "--redis-url" => config.redis_url = Some(next_value(&mut iter, "--redis-url")?),
+            "--postgres-url" => {
+                config.postgres_url = Some(next_value(&mut iter, "--postgres-url")?)
+            }
+            "--output" => {
+                config.output_path = Some(PathBuf::from(next_value(&mut iter, "--output")?))
+            }
+            "--help" | "-h" => {
+                print_usage();
+                std::process::exit(0);
+            }
+            other => {
+                return Err(std::io::Error::new(
+                    std::io::ErrorKind::InvalidInput,
+                    format!("unknown argument: {other}"),
+                )
+                .into());
+            }
+        }
+    }
+
+    if config.total_requests == 0 || config.concurrency == 0 || config.timeout.is_zero() {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            "owner relay baseline numeric settings must be positive",
+        )
+        .into());
+    }
+
+    Ok(config)
+}
+
+fn next_value(
+    iter: &mut impl Iterator<Item = String>,
+    flag: &str,
+) -> Result<String, Box<dyn std::error::Error>> {
+    iter.next().ok_or_else(|| {
+        std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            format!("missing value for {flag}"),
+        )
+        .into()
+    })
+}
+
+fn print_usage() {
+    println!(
+        "usage: cargo run -p aether-integration-tests --bin multi_instance_owner_relay_baseline -- [--requests 200] [--concurrency 20] [--timeout-ms 10000] [--chunk-delay-ms 0] [--redis-url redis://127.0.0.1:6379/0] [--postgres-url postgres://127.0.0.1:5432/postgres] [--output /tmp/multi_instance_owner_relay_baseline.json]"
+    );
+}

crates/aether-testing/integration/src/bin/single_instance_baseline.rs

@@ -0,0 +1,382 @@
+// Gateway-backed benchmark scenarios live outside the reusable testkit.
+use std::collections::BTreeMap;
+use std::convert::Infallible;
+use std::path::PathBuf;
+use std::time::Duration;
+
+use aether_contracts::{ExecutionPlan, ExecutionTimeouts, RequestBody};
+use aether_testkit::{
+    init_test_runtime_for, run_http_load_probe, ExecutionRuntimeHarness,
+    ExecutionRuntimeHarnessConfig, GatewayHarness, GatewayHarnessConfig, HttpLoadProbeConfig,
+    HttpLoadProbeResponseMode, HttpLoadProbeResult, SpawnedServer, GATEWAY_HARNESS_API_KEY,
+};
+use axum::body::{to_bytes, Body, Bytes};
+use axum::http::StatusCode;
+use axum::response::{IntoResponse, Response};
+use axum::routing::any;
+use axum::{extract::Request, Json, Router};
+use reqwest::Method;
+use serde::Serialize;
+use serde_json::json;
+
+#[derive(Debug, Clone)]
+struct SingleInstanceBaselineConfig {
+    sync_requests: usize,
+    sync_concurrency: usize,
+    stream_requests: usize,
+    stream_concurrency: usize,
+    timeout: Duration,
+    output_path: Option<PathBuf>,
+}
+
+impl Default for SingleInstanceBaselineConfig {
+    fn default() -> Self {
+        Self {
+            sync_requests: 200,
+            sync_concurrency: 20,
+            stream_requests: 100,
+            stream_concurrency: 10,
+            timeout: Duration::from_secs(10),
+            output_path: None,
+        }
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct NamedBaselineResult {
+    name: String,
+    result: HttpLoadProbeResult,
+}
+
+#[derive(Debug, Serialize)]
+struct SingleInstanceBaselineReport {
+    suite: &'static str,
+    scenarios: Vec<NamedBaselineResult>,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    init_test_runtime_for("single-instance-baseline");
+    let config = parse_args(std::env::args().skip(1).collect())?;
+    let report = run_suite(&config).await?;
+    let raw = serde_json::to_string_pretty(&report)?;
+    println!("{raw}");
+    if let Some(path) = config.output_path.as_ref() {
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        std::fs::write(path, format!("{raw}\n"))?;
+    }
+    Ok(())
+}
+
+async fn run_suite(
+    config: &SingleInstanceBaselineConfig,
+) -> Result<SingleInstanceBaselineReport, Box<dyn std::error::Error>> {
+    let upstream = SpawnedServer::start(build_fake_upstream()).await?;
+    let gateway = GatewayHarness::start(GatewayHarnessConfig::new(upstream.base_url())).await?;
+    let runtime = ExecutionRuntimeHarness::start(ExecutionRuntimeHarnessConfig::default()).await?;
+
+    let gateway_sync = run_http_load_probe(&gateway_sync_probe_config(gateway.base_url(), config))
+        .await
+        .map_err(std::io::Error::other)?;
+    let gateway_stream =
+        run_http_load_probe(&gateway_stream_probe_config(gateway.base_url(), config))
+            .await
+            .map_err(std::io::Error::other)?;
+    let execution_runtime_sync = run_http_load_probe(&execution_runtime_sync_probe_config(
+        runtime.base_url(),
+        upstream.base_url(),
+        config,
+    ))
+    .await
+    .map_err(std::io::Error::other)?;
+    let execution_runtime_stream = run_http_load_probe(&execution_runtime_stream_probe_config(
+        runtime.base_url(),
+        upstream.base_url(),
+        config,
+    ))
+    .await
+    .map_err(std::io::Error::other)?;
+
+    Ok(SingleInstanceBaselineReport {
+        suite: "single_instance_baseline",
+        scenarios: vec![
+            NamedBaselineResult {
+                name: "gateway_proxy_sync".to_string(),
+                result: gateway_sync,
+            },
+            NamedBaselineResult {
+                name: "gateway_proxy_stream".to_string(),
+                result: gateway_stream,
+            },
+            NamedBaselineResult {
+                name: "execution_runtime_sync".to_string(),
+                result: execution_runtime_sync,
+            },
+            NamedBaselineResult {
+                name: "execution_runtime_stream".to_string(),
+                result: execution_runtime_stream,
+            },
+        ],
+    })
+}
+
+fn gateway_sync_probe_config(
+    gateway_base_url: &str,
+    config: &SingleInstanceBaselineConfig,
+) -> HttpLoadProbeConfig {
+    let mut probe = chat_probe_config(
+        format!("{gateway_base_url}/v1/chat/completions"),
+        false,
+        config.sync_requests,
+        config.sync_concurrency,
+        config.timeout,
+    );
+    probe.response_mode = HttpLoadProbeResponseMode::FullBody;
+    probe
+}
+
+fn gateway_stream_probe_config(
+    gateway_base_url: &str,
+    config: &SingleInstanceBaselineConfig,
+) -> HttpLoadProbeConfig {
+    let mut probe = chat_probe_config(
+        format!("{gateway_base_url}/v1/chat/completions"),
+        true,
+        config.stream_requests,
+        config.stream_concurrency,
+        config.timeout,
+    );
+    probe.response_mode = HttpLoadProbeResponseMode::FullBody;
+    probe
+}
+
+fn execution_runtime_sync_probe_config(
+    runtime_base_url: &str,
+    upstream_base_url: &str,
+    config: &SingleInstanceBaselineConfig,
+) -> HttpLoadProbeConfig {
+    execution_probe_config(
+        format!("{runtime_base_url}/v1/execute/sync"),
+        execution_plan(format!("{upstream_base_url}/v1/chat/completions"), false),
+        config.sync_requests,
+        config.sync_concurrency,
+        config.timeout,
+    )
+}
+
+fn execution_runtime_stream_probe_config(
+    runtime_base_url: &str,
+    upstream_base_url: &str,
+    config: &SingleInstanceBaselineConfig,
+) -> HttpLoadProbeConfig {
+    execution_probe_config(
+        format!("{runtime_base_url}/v1/execute/stream"),
+        execution_plan(format!("{upstream_base_url}/v1/chat/completions"), true),
+        config.stream_requests,
+        config.stream_concurrency,
+        config.timeout,
+    )
+}
+
+fn execution_probe_config(
+    url: String,
+    plan: ExecutionPlan,
+    total_requests: usize,
+    concurrency: usize,
+    timeout: Duration,
+) -> HttpLoadProbeConfig {
+    HttpLoadProbeConfig {
+        url,
+        method: Method::POST,
+        headers: BTreeMap::from([("content-type".to_string(), "application/json".to_string())]),
+        body: Some(
+            serde_json::to_vec(&plan).expect("execution plan should serialize for load probe"),
+        ),
+        total_requests,
+        concurrency,
+        timeout,
+        response_mode: HttpLoadProbeResponseMode::FullBody,
+        ..HttpLoadProbeConfig::default()
+    }
+}
+
+fn chat_probe_config(
+    url: String,
+    stream: bool,
+    total_requests: usize,
+    concurrency: usize,
+    timeout: Duration,
+) -> HttpLoadProbeConfig {
+    HttpLoadProbeConfig {
+        url,
+        method: Method::POST,
+        headers: BTreeMap::from([
+            ("content-type".to_string(), "application/json".to_string()),
+            (
+                "authorization".to_string(),
+                format!("Bearer {GATEWAY_HARNESS_API_KEY}"),
+            ),
+        ]),
+        body: Some(
+            serde_json::to_vec(&json!({
+                "model": "gpt-5",
+                "messages": [{"role": "user", "content": "hello"}],
+                "stream": stream,
+            }))
+            .expect("chat body should serialize"),
+        ),
+        total_requests,
+        concurrency,
+        timeout,
+        response_mode: HttpLoadProbeResponseMode::FullBody,
+        ..HttpLoadProbeConfig::default()
+    }
+}
+
+fn execution_plan(url: String, stream: bool) -> ExecutionPlan {
+    ExecutionPlan {
+        request_id: if stream {
+            "baseline-stream-request".to_string()
+        } else {
+            "baseline-sync-request".to_string()
+        },
+        candidate_id: Some(if stream {
+            "baseline-stream-candidate".to_string()
+        } else {
+            "baseline-sync-candidate".to_string()
+        }),
+        provider_name: Some("openai".to_string()),
+        provider_id: "provider-baseline".to_string(),
+        endpoint_id: "endpoint-baseline".to_string(),
+        key_id: "key-baseline".to_string(),
+        method: "POST".to_string(),
+        url,
+        headers: BTreeMap::from([("content-type".to_string(), "application/json".to_string())]),
+        content_type: Some("application/json".to_string()),
+        content_encoding: None,
+        body: RequestBody::from_json(json!({
+            "model": "gpt-5",
+            "messages": [{"role": "user", "content": "hello"}],
+            "stream": stream,
+        })),
+        stream,
+        client_api_format: "openai:chat".to_string(),
+        provider_api_format: "openai:chat".to_string(),
+        model_name: Some("gpt-5".to_string()),
+        proxy: None,
+        transport_profile: None,
+        timeouts: Some(ExecutionTimeouts {
+            connect_ms: Some(2_000),
+            read_ms: Some(10_000),
+            first_byte_ms: Some(5_000),
+            total_ms: Some(10_000),
+            ..ExecutionTimeouts::default()
+        }),
+    }
+}
+
+fn build_fake_upstream() -> Router {
+    Router::new().route(
+        "/v1/chat/completions",
+        any(|request: Request| async move {
+            let (_parts, body) = request.into_parts();
+            let raw_body = to_bytes(body, usize::MAX)
+                .await
+                .expect("fake upstream body should read");
+            let payload: serde_json::Value =
+                serde_json::from_slice(&raw_body).unwrap_or_else(|_| json!({}));
+            let stream = payload
+                .get("stream")
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false);
+            if stream {
+                let body = futures_util::stream::iter([
+                    Ok::<_, Infallible>(Bytes::from_static(
+                        b"data: {\"id\":\"chunk-1\",\"choices\":[{\"delta\":{\"content\":\"hel\"}}]}\n\n",
+                    )),
+                    Ok::<_, Infallible>(Bytes::from_static(
+                        b"data: {\"id\":\"chunk-2\",\"choices\":[{\"delta\":{\"content\":\"lo\"}}]}\n\n",
+                    )),
+                    Ok::<_, Infallible>(Bytes::from_static(b"data: [DONE]\n\n")),
+                ]);
+                Response::builder()
+                    .status(StatusCode::OK)
+                    .header(http::header::CONTENT_TYPE, "text/event-stream")
+                    .body(Body::from_stream(body))
+                    .expect("fake upstream stream response should build")
+            } else {
+                Json(json!({
+                    "id": "chatcmpl-baseline",
+                    "object": "chat.completion",
+                    "model": payload.get("model").and_then(|value| value.as_str()).unwrap_or("gpt-5"),
+                    "choices": [{"message": {"role": "assistant", "content": "hello"}}]
+                }))
+                .into_response()
+            }
+        }),
+    )
+}
+
+fn parse_args(
+    args: Vec<String>,
+) -> Result<SingleInstanceBaselineConfig, Box<dyn std::error::Error>> {
+    let mut config = SingleInstanceBaselineConfig::default();
+    let mut iter = args.into_iter();
+    while let Some(arg) = iter.next() {
+        match arg.as_str() {
+            "--sync-requests" => {
+                config.sync_requests = next_value(&mut iter, "--sync-requests")?.parse()?
+            }
+            "--sync-concurrency" => {
+                config.sync_concurrency = next_value(&mut iter, "--sync-concurrency")?.parse()?
+            }
+            "--stream-requests" => {
+                config.stream_requests = next_value(&mut iter, "--stream-requests")?.parse()?
+            }
+            "--stream-concurrency" => {
+                config.stream_concurrency =
+                    next_value(&mut iter, "--stream-concurrency")?.parse()?
+            }
+            "--timeout-ms" => {
+                config.timeout =
+                    Duration::from_millis(next_value(&mut iter, "--timeout-ms")?.parse()?)
+            }
+            "--output" => {
+                config.output_path = Some(PathBuf::from(next_value(&mut iter, "--output")?))
+            }
+            "--help" | "-h" => {
+                print_usage();
+                std::process::exit(0);
+            }
+            other => {
+                return Err(std::io::Error::new(
+                    std::io::ErrorKind::InvalidInput,
+                    format!("unknown argument: {other}"),
+                )
+                .into());
+            }
+        }
+    }
+    Ok(config)
+}
+
+fn next_value(
+    iter: &mut impl Iterator<Item = String>,
+    flag: &str,
+) -> Result<String, Box<dyn std::error::Error>> {
+    iter.next().ok_or_else(|| {
+        std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            format!("missing value for {flag}"),
+        )
+        .into()
+    })
+}
+
+fn print_usage() {
+    eprintln!(
+        "usage: cargo run -p aether-integration-tests --bin single_instance_baseline -- [--sync-requests 200] [--sync-concurrency 20] [--stream-requests 100] [--stream-concurrency 10] [--timeout-ms 10000] [--output /tmp/baseline.json]"
+    );
+}

crates/aether-testing/integration/src/bin/usage_aux_counter_hotspot_baseline.rs

@@ -0,0 +1,665 @@
+// Gateway-backed benchmark scenarios live outside the reusable testkit.
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::Arc;
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
+
+use aether_data::repository::usage::SqlxUsageReadRepository;
+use aether_data_contracts::repository::usage::{
+    ApiKeyLastUsedDelta, ManagementTokenCounterDelta, ProxyNodeCounterDelta,
+};
+use aether_testkit::{
+    init_test_runtime_for, prepare_aether_postgres_schema, ManagedPostgresServer,
+};
+use serde::Serialize;
+use sqlx::postgres::PgPoolOptions;
+use sqlx::{PgPool, Row};
+use tokio::sync::Mutex;
+
+const PROXY_NODE_ID: &str = "proxy-node-hotspot";
+const MANAGEMENT_TOKEN_ID: &str = "management-token-hotspot";
+const API_KEY_ID: &str = "api-key-last-used-hotspot";
+const USER_ID: &str = "usage-aux-hotspot-user";
+
+#[derive(Debug, Clone)]
+struct Config {
+    requests: usize,
+    concurrency: usize,
+    max_connections: u32,
+    flush_batch_size: usize,
+    flush_interval: Duration,
+    monitor_interval: Duration,
+    output_path: Option<PathBuf>,
+    postgres_url: Option<String>,
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Self {
+            requests: 2_000,
+            concurrency: 100,
+            max_connections: 64,
+            flush_batch_size: 1_000,
+            flush_interval: Duration::from_millis(100),
+            monitor_interval: Duration::from_millis(100),
+            output_path: None,
+            postgres_url: None,
+        }
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct Report {
+    suite: &'static str,
+    config: ReportConfig,
+    duration_ms: u64,
+    throughput_rps: u64,
+    completed_requests: usize,
+    failed_requests: usize,
+    p50_ms: u64,
+    p95_ms: u64,
+    max_ms: u64,
+    mean_ms: u64,
+    flush: FlushReport,
+    counters: CounterReport,
+    lock_monitor: LockMonitorReport,
+}
+
+#[derive(Debug, Serialize)]
+struct ReportConfig {
+    requests: usize,
+    concurrency: usize,
+    max_connections: u32,
+    flush_batch_size: usize,
+    flush_interval_ms: u64,
+    monitor_interval_ms: u64,
+    managed_postgres: bool,
+}
+
+#[derive(Debug, Serialize, Default)]
+struct FlushReport {
+    calls: usize,
+    rows_claimed: usize,
+    proxy_node_targets: usize,
+    management_token_targets: usize,
+    api_key_last_used_targets: usize,
+}
+
+#[derive(Debug, Serialize)]
+struct CounterReport {
+    outbox_pending_rows: i64,
+    outbox_processed_rows: i64,
+    proxy_total_requests: i64,
+    proxy_failed_requests: i64,
+    proxy_dns_failures: i64,
+    proxy_stream_errors: i64,
+    expected_proxy_total_requests: i64,
+    expected_proxy_failed_requests: i64,
+    expected_proxy_dns_failures: i64,
+    expected_proxy_stream_errors: i64,
+    management_token_usage_count: i64,
+    expected_management_token_usage_count: i64,
+    api_key_last_used_at: Option<i64>,
+    expected_min_api_key_last_used_at: i64,
+}
+
+#[derive(Debug, Serialize, Clone, Copy, Default)]
+struct LockMonitorReport {
+    samples: usize,
+    max_lock_waiters: i64,
+    max_proxy_node_update_waiters: i64,
+    max_management_token_update_waiters: i64,
+    max_api_key_update_waiters: i64,
+    max_oldest_lock_wait_ms: i64,
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+struct LockSample {
+    lock_waiters: i64,
+    proxy_node_update_waiters: i64,
+    management_token_update_waiters: i64,
+    api_key_update_waiters: i64,
+    oldest_lock_wait_ms: i64,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    init_test_runtime_for("usage-aux-counter-hotspot-baseline");
+    let config = parse_args(std::env::args().skip(1).collect())?;
+
+    let managed_postgres;
+    let database_url;
+    let _server;
+    if let Some(url) = config.postgres_url.as_ref() {
+        managed_postgres = false;
+        database_url = url.clone();
+        _server = None;
+    } else {
+        managed_postgres = true;
+        let server = ManagedPostgresServer::start().await?;
+        database_url = server.database_url().to_string();
+        _server = Some(server);
+    }
+
+    prepare_aether_postgres_schema(&database_url).await?;
+    let pool = PgPoolOptions::new()
+        .min_connections(1)
+        .max_connections(config.max_connections)
+        .acquire_timeout(Duration::from_secs(10))
+        .connect(&database_url)
+        .await?;
+    seed_hotspot_rows(&pool).await?;
+
+    let repository = SqlxUsageReadRepository::new(pool.clone());
+    let stop = Arc::new(AtomicBool::new(false));
+    let flush_report = Arc::new(Mutex::new(FlushReport::default()));
+    let lock_report = Arc::new(Mutex::new(LockMonitorReport::default()));
+
+    let flush_handle = spawn_flush_loop(
+        repository.clone(),
+        Arc::clone(&stop),
+        Arc::clone(&flush_report),
+        config.flush_batch_size,
+        config.flush_interval,
+    );
+    let monitor_handle = spawn_lock_monitor(
+        pool.clone(),
+        Arc::clone(&stop),
+        Arc::clone(&lock_report),
+        config.monitor_interval,
+    );
+
+    let started_at = Instant::now();
+    let load_result =
+        run_aux_counter_load(repository.clone(), config.requests, config.concurrency).await;
+    wait_for_outbox_drain(&pool, &repository, config.flush_batch_size).await?;
+    let duration_ms = started_at.elapsed().as_millis() as u64;
+
+    stop.store(true, Ordering::Release);
+    flush_handle.await??;
+    monitor_handle.await??;
+
+    let mut flush = flush_report.lock().await;
+    let final_flush = repository
+        .flush_usage_counter_deltas(config.flush_batch_size)
+        .await?;
+    flush.calls += 1;
+    flush.rows_claimed += final_flush.rows_claimed;
+    flush.proxy_node_targets += final_flush.proxy_node_targets;
+    flush.management_token_targets += final_flush.management_token_targets;
+    flush.api_key_last_used_targets += final_flush.api_key_last_used_targets;
+    drop(flush);
+
+    let counters = read_counters(&pool, config.requests).await?;
+    let latencies = load_result.latencies.lock().await.clone();
+    let (p50_ms, p95_ms, max_ms, mean_ms) = summarize_latencies(latencies);
+    let completed_requests = load_result.completed.load(Ordering::Acquire);
+    let throughput_rps = if duration_ms == 0 {
+        completed_requests as u64
+    } else {
+        ((completed_requests as u64) * 1_000) / duration_ms.max(1)
+    };
+
+    let report = Report {
+        suite: "usage_aux_counter_hotspot_baseline",
+        config: ReportConfig {
+            requests: config.requests,
+            concurrency: config.concurrency,
+            max_connections: config.max_connections,
+            flush_batch_size: config.flush_batch_size,
+            flush_interval_ms: config.flush_interval.as_millis() as u64,
+            monitor_interval_ms: config.monitor_interval.as_millis() as u64,
+            managed_postgres,
+        },
+        duration_ms,
+        throughput_rps,
+        completed_requests,
+        failed_requests: load_result.failed.load(Ordering::Acquire),
+        p50_ms,
+        p95_ms,
+        max_ms,
+        mean_ms,
+        flush: Arc::try_unwrap(flush_report)
+            .unwrap_or_else(|_| panic!("flush report still referenced"))
+            .into_inner(),
+        counters,
+        lock_monitor: Arc::try_unwrap(lock_report)
+            .unwrap_or_else(|_| panic!("lock report still referenced"))
+            .into_inner(),
+    };
+
+    let raw = serde_json::to_string_pretty(&report)?;
+    println!("{raw}");
+    if let Some(path) = config.output_path.as_ref() {
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        std::fs::write(path, format!("{raw}\n"))?;
+    }
+    Ok(())
+}
+
+#[derive(Clone)]
+struct LoadResult {
+    completed: Arc<AtomicUsize>,
+    failed: Arc<AtomicUsize>,
+    latencies: Arc<Mutex<Vec<u64>>>,
+}
+
+async fn run_aux_counter_load(
+    repository: SqlxUsageReadRepository,
+    requests: usize,
+    concurrency: usize,
+) -> LoadResult {
+    let next = Arc::new(AtomicUsize::new(0));
+    let completed = Arc::new(AtomicUsize::new(0));
+    let failed = Arc::new(AtomicUsize::new(0));
+    let latencies = Arc::new(Mutex::new(Vec::with_capacity(requests)));
+    let mut tasks = tokio::task::JoinSet::new();
+
+    for _ in 0..concurrency {
+        let repository = repository.clone();
+        let next = Arc::clone(&next);
+        let completed = Arc::clone(&completed);
+        let failed = Arc::clone(&failed);
+        let latencies = Arc::clone(&latencies);
+        tasks.spawn(async move {
+            loop {
+                let index = next.fetch_add(1, Ordering::AcqRel);
+                if index >= requests {
+                    break;
+                }
+
+                let started_at = Instant::now();
+                let result = enqueue_aux_counter_deltas(&repository, index).await;
+                latencies
+                    .lock()
+                    .await
+                    .push(started_at.elapsed().as_millis() as u64);
+                completed.fetch_add(1, Ordering::AcqRel);
+                if result.is_err() {
+                    failed.fetch_add(1, Ordering::AcqRel);
+                }
+            }
+        });
+    }
+
+    while let Some(result) = tasks.join_next().await {
+        if result.is_err() {
+            failed.fetch_add(1, Ordering::AcqRel);
+        }
+    }
+
+    LoadResult {
+        completed,
+        failed,
+        latencies,
+    }
+}
+
+async fn enqueue_aux_counter_deltas(
+    repository: &SqlxUsageReadRepository,
+    index: usize,
+) -> Result<(), aether_data::DataLayerError> {
+    repository
+        .enqueue_proxy_node_counter_delta(proxy_delta_for_index(index))
+        .await?;
+    repository
+        .enqueue_management_token_counter_delta(ManagementTokenCounterDelta {
+            token_id: MANAGEMENT_TOKEN_ID.to_string(),
+            usage_count_delta: 1,
+            last_used_at_unix_secs: Some(now_unix_secs().saturating_add(index as u64)),
+            last_used_ip: Some(format!("10.0.0.{}", (index % 250) + 1)),
+        })
+        .await?;
+    repository
+        .enqueue_api_key_last_used_delta(ApiKeyLastUsedDelta {
+            api_key_id: API_KEY_ID.to_string(),
+            last_used_at_unix_secs: now_unix_secs().saturating_add(index as u64),
+        })
+        .await?;
+    Ok(())
+}
+
+fn proxy_delta_for_index(index: usize) -> ProxyNodeCounterDelta {
+    ProxyNodeCounterDelta {
+        node_id: PROXY_NODE_ID.to_string(),
+        total_requests_delta: 1,
+        failed_requests_delta: if index.is_multiple_of(10) { 1 } else { 0 },
+        dns_failures_delta: if index.is_multiple_of(25) { 1 } else { 0 },
+        stream_errors_delta: if index.is_multiple_of(40) { 1 } else { 0 },
+    }
+}
+
+fn spawn_flush_loop(
+    repository: SqlxUsageReadRepository,
+    stop: Arc<AtomicBool>,
+    report: Arc<Mutex<FlushReport>>,
+    batch_size: usize,
+    interval: Duration,
+) -> tokio::task::JoinHandle<Result<(), aether_data::DataLayerError>> {
+    tokio::spawn(async move {
+        while !stop.load(Ordering::Acquire) {
+            let summary = repository.flush_usage_counter_deltas(batch_size).await?;
+            let mut report = report.lock().await;
+            report.calls += 1;
+            report.rows_claimed += summary.rows_claimed;
+            report.proxy_node_targets += summary.proxy_node_targets;
+            report.management_token_targets += summary.management_token_targets;
+            report.api_key_last_used_targets += summary.api_key_last_used_targets;
+            drop(report);
+            tokio::time::sleep(interval).await;
+        }
+        Ok(())
+    })
+}
+
+fn spawn_lock_monitor(
+    pool: PgPool,
+    stop: Arc<AtomicBool>,
+    report: Arc<Mutex<LockMonitorReport>>,
+    interval: Duration,
+) -> tokio::task::JoinHandle<Result<(), sqlx::Error>> {
+    tokio::spawn(async move {
+        while !stop.load(Ordering::Acquire) {
+            let sample = read_lock_sample(&pool).await?;
+            let mut report = report.lock().await;
+            report.samples += 1;
+            report.max_lock_waiters = report.max_lock_waiters.max(sample.lock_waiters);
+            report.max_proxy_node_update_waiters = report
+                .max_proxy_node_update_waiters
+                .max(sample.proxy_node_update_waiters);
+            report.max_management_token_update_waiters = report
+                .max_management_token_update_waiters
+                .max(sample.management_token_update_waiters);
+            report.max_api_key_update_waiters = report
+                .max_api_key_update_waiters
+                .max(sample.api_key_update_waiters);
+            report.max_oldest_lock_wait_ms = report
+                .max_oldest_lock_wait_ms
+                .max(sample.oldest_lock_wait_ms);
+            drop(report);
+            tokio::time::sleep(interval).await;
+        }
+        Ok(())
+    })
+}
+
+async fn wait_for_outbox_drain(
+    pool: &PgPool,
+    repository: &SqlxUsageReadRepository,
+    batch_size: usize,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let deadline = Instant::now() + Duration::from_secs(30);
+    loop {
+        let summary = repository.flush_usage_counter_deltas(batch_size).await?;
+        let pending: i64 = sqlx::query_scalar(
+            "SELECT COUNT(*)::BIGINT FROM usage_counter_deltas WHERE processed_at IS NULL",
+        )
+        .fetch_one(pool)
+        .await?;
+        if pending == 0 && summary.rows_claimed == 0 {
+            return Ok(());
+        }
+        if Instant::now() >= deadline {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::TimedOut,
+                format!("usage counter outbox did not drain; pending={pending}"),
+            )
+            .into());
+        }
+        tokio::time::sleep(Duration::from_millis(100)).await;
+    }
+}
+
+async fn seed_hotspot_rows(pool: &PgPool) -> Result<(), sqlx::Error> {
+    sqlx::query(
+        r#"
+INSERT INTO users (id, username, email_verified)
+VALUES ($1, 'usage-aux-hotspot', TRUE)
+ON CONFLICT (id) DO NOTHING
+"#,
+    )
+    .bind(USER_ID)
+    .execute(pool)
+    .await?;
+
+    sqlx::query(
+        r#"
+INSERT INTO api_keys (id, user_id, key_hash, name, is_active)
+VALUES ($1, $2, 'usage-aux-hotspot-hash', 'usage aux hotspot', TRUE)
+ON CONFLICT (id) DO UPDATE SET
+  last_used_at = NULL
+"#,
+    )
+    .bind(API_KEY_ID)
+    .bind(USER_ID)
+    .execute(pool)
+    .await?;
+
+    sqlx::query(
+        r#"
+INSERT INTO management_tokens (
+  id, user_id, name, token_hash, token_prefix, usage_count
+)
+VALUES ($1, $2, 'usage aux hotspot token', 'usage-aux-hotspot-token-hash', 'usageaux', 0)
+ON CONFLICT (id) DO UPDATE SET
+  usage_count = 0,
+  last_used_at = NULL,
+  last_used_ip = NULL
+"#,
+    )
+    .bind(MANAGEMENT_TOKEN_ID)
+    .bind(USER_ID)
+    .execute(pool)
+    .await?;
+
+    sqlx::query(
+        r#"
+INSERT INTO proxy_nodes (
+  id, name, ip, port, status, total_requests, failed_requests,
+  dns_failures, stream_errors
+)
+VALUES ($1, 'usage aux hotspot proxy', '127.0.0.1', 8080, 'online', 0, 0, 0, 0)
+ON CONFLICT (id) DO UPDATE SET
+  total_requests = 0,
+  failed_requests = 0,
+  dns_failures = 0,
+  stream_errors = 0
+"#,
+    )
+    .bind(PROXY_NODE_ID)
+    .execute(pool)
+    .await?;
+
+    sqlx::query(
+        r#"
+DELETE FROM usage_counter_deltas
+WHERE target_id IN ($1, $2, $3)
+"#,
+    )
+    .bind(PROXY_NODE_ID)
+    .bind(MANAGEMENT_TOKEN_ID)
+    .bind(API_KEY_ID)
+    .execute(pool)
+    .await?;
+    Ok(())
+}
+
+async fn read_lock_sample(pool: &PgPool) -> Result<LockSample, sqlx::Error> {
+    let row = sqlx::query(
+        r#"
+SELECT
+  COUNT(*) FILTER (WHERE wait_event_type = 'Lock')::BIGINT AS lock_waiters,
+  COUNT(*) FILTER (
+    WHERE wait_event_type = 'Lock' AND query LIKE 'UPDATE proxy_nodes%'
+  )::BIGINT AS proxy_node_update_waiters,
+  COUNT(*) FILTER (
+    WHERE wait_event_type = 'Lock' AND query LIKE 'UPDATE management_tokens%'
+  )::BIGINT AS management_token_update_waiters,
+  COUNT(*) FILTER (
+    WHERE wait_event_type = 'Lock' AND query LIKE 'UPDATE api_keys%'
+  )::BIGINT AS api_key_update_waiters,
+  COALESCE(
+    MAX(EXTRACT(EPOCH FROM (NOW() - query_start)) * 1000)
+      FILTER (WHERE wait_event_type = 'Lock'),
+    0
+  )::BIGINT AS oldest_lock_wait_ms
+FROM pg_stat_activity
+WHERE datname = current_database()
+"#,
+    )
+    .fetch_one(pool)
+    .await?;
+    Ok(LockSample {
+        lock_waiters: row.try_get("lock_waiters")?,
+        proxy_node_update_waiters: row.try_get("proxy_node_update_waiters")?,
+        management_token_update_waiters: row.try_get("management_token_update_waiters")?,
+        api_key_update_waiters: row.try_get("api_key_update_waiters")?,
+        oldest_lock_wait_ms: row.try_get("oldest_lock_wait_ms")?,
+    })
+}
+
+async fn read_counters(pool: &PgPool, requests: usize) -> Result<CounterReport, sqlx::Error> {
+    let row = sqlx::query(
+        r#"
+SELECT
+  (SELECT COUNT(*)::BIGINT FROM usage_counter_deltas WHERE processed_at IS NULL) AS outbox_pending_rows,
+  (SELECT COUNT(*)::BIGINT FROM usage_counter_deltas WHERE processed_at IS NOT NULL) AS outbox_processed_rows,
+  (SELECT total_requests::BIGINT FROM proxy_nodes WHERE id = $1) AS proxy_total_requests,
+  (SELECT failed_requests::BIGINT FROM proxy_nodes WHERE id = $1) AS proxy_failed_requests,
+  (SELECT dns_failures::BIGINT FROM proxy_nodes WHERE id = $1) AS proxy_dns_failures,
+  (SELECT stream_errors::BIGINT FROM proxy_nodes WHERE id = $1) AS proxy_stream_errors,
+  (SELECT usage_count::BIGINT FROM management_tokens WHERE id = $2) AS management_token_usage_count,
+  (
+    SELECT EXTRACT(EPOCH FROM last_used_at)::BIGINT
+    FROM api_keys
+    WHERE id = $3
+  ) AS api_key_last_used_at
+"#,
+    )
+    .bind(PROXY_NODE_ID)
+    .bind(MANAGEMENT_TOKEN_ID)
+    .bind(API_KEY_ID)
+    .fetch_one(pool)
+    .await?;
+
+    Ok(CounterReport {
+        outbox_pending_rows: row.try_get("outbox_pending_rows")?,
+        outbox_processed_rows: row.try_get("outbox_processed_rows")?,
+        proxy_total_requests: row.try_get("proxy_total_requests")?,
+        proxy_failed_requests: row.try_get("proxy_failed_requests")?,
+        proxy_dns_failures: row.try_get("proxy_dns_failures")?,
+        proxy_stream_errors: row.try_get("proxy_stream_errors")?,
+        expected_proxy_total_requests: requests as i64,
+        expected_proxy_failed_requests: count_every(requests, 10),
+        expected_proxy_dns_failures: count_every(requests, 25),
+        expected_proxy_stream_errors: count_every(requests, 40),
+        management_token_usage_count: row.try_get("management_token_usage_count")?,
+        expected_management_token_usage_count: requests as i64,
+        api_key_last_used_at: row.try_get("api_key_last_used_at")?,
+        expected_min_api_key_last_used_at: now_unix_secs().saturating_sub(5) as i64,
+    })
+}
+
+fn count_every(requests: usize, interval: usize) -> i64 {
+    if requests == 0 {
+        return 0;
+    }
+    ((requests - 1) / interval + 1) as i64
+}
+
+fn summarize_latencies(mut latencies: Vec<u64>) -> (u64, u64, u64, u64) {
+    if latencies.is_empty() {
+        return (0, 0, 0, 0);
+    }
+    latencies.sort_unstable();
+    let p50 = percentile(&latencies, 50);
+    let p95 = percentile(&latencies, 95);
+    let max = *latencies.last().unwrap_or(&0);
+    let mean = latencies.iter().sum::<u64>() / latencies.len() as u64;
+    (p50, p95, max, mean)
+}
+
+fn percentile(latencies: &[u64], percentile: usize) -> u64 {
+    let index = ((latencies.len() - 1) * percentile) / 100;
+    latencies[index]
+}
+
+fn now_unix_secs() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs()
+}
+
+fn parse_args(args: Vec<String>) -> Result<Config, Box<dyn std::error::Error>> {
+    let mut config = Config::default();
+    let mut iter = args.into_iter();
+    while let Some(arg) = iter.next() {
+        match arg.as_str() {
+            "--requests" => config.requests = next_value(&mut iter, "--requests")?.parse()?,
+            "--concurrency" => {
+                config.concurrency = next_value(&mut iter, "--concurrency")?.parse()?
+            }
+            "--max-connections" => {
+                config.max_connections = next_value(&mut iter, "--max-connections")?.parse()?
+            }
+            "--flush-batch-size" => {
+                config.flush_batch_size = next_value(&mut iter, "--flush-batch-size")?.parse()?
+            }
+            "--flush-interval-ms" => {
+                config.flush_interval =
+                    Duration::from_millis(next_value(&mut iter, "--flush-interval-ms")?.parse()?)
+            }
+            "--monitor-interval-ms" => {
+                config.monitor_interval =
+                    Duration::from_millis(next_value(&mut iter, "--monitor-interval-ms")?.parse()?)
+            }
+            "--postgres-url" => {
+                config.postgres_url = Some(next_value(&mut iter, "--postgres-url")?)
+            }
+            "--output" => {
+                config.output_path = Some(PathBuf::from(next_value(&mut iter, "--output")?))
+            }
+            "--help" | "-h" => {
+                print_usage();
+                std::process::exit(0);
+            }
+            other => {
+                return Err(std::io::Error::new(
+                    std::io::ErrorKind::InvalidInput,
+                    format!("unknown argument: {other}"),
+                )
+                .into());
+            }
+        }
+    }
+    if config.requests == 0 || config.concurrency == 0 || config.max_connections == 0 {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            "requests, concurrency, and max-connections must be positive",
+        )
+        .into());
+    }
+    Ok(config)
+}
+
+fn next_value(
+    iter: &mut impl Iterator<Item = String>,
+    flag: &str,
+) -> Result<String, Box<dyn std::error::Error>> {
+    iter.next().ok_or_else(|| {
+        std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            format!("missing value for {flag}"),
+        )
+        .into()
+    })
+}
+
+fn print_usage() {
+    eprintln!(
+        "usage: cargo run -p aether-integration-tests --bin usage_aux_counter_hotspot_baseline -- [--requests 2000] [--concurrency 100] [--max-connections 64] [--flush-batch-size 1000] [--flush-interval-ms 100] [--monitor-interval-ms 100] [--postgres-url postgres://...] [--output /tmp/usage_aux_counter_hotspot.json]"
+    );
+}

crates/aether-testing/integration/src/bin/usage_counter_hotspot_baseline.rs

@@ -0,0 +1,685 @@
+// Gateway-backed benchmark scenarios live outside the reusable testkit.
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::Arc;
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
+
+use aether_data::repository::usage::SqlxUsageReadRepository;
+use aether_data_contracts::repository::usage::UpsertUsageRecord;
+use aether_testkit::{
+    init_test_runtime_for, prepare_aether_postgres_schema, ManagedPostgresServer,
+};
+use serde::Serialize;
+use serde_json::json;
+use sqlx::postgres::PgPoolOptions;
+use sqlx::{PgPool, Row};
+use tokio::sync::Mutex;
+
+#[derive(Debug, Clone)]
+struct Config {
+    requests: usize,
+    concurrency: usize,
+    max_connections: u32,
+    flush_batch_size: usize,
+    flush_interval: Duration,
+    monitor_interval: Duration,
+    output_path: Option<PathBuf>,
+    postgres_url: Option<String>,
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Self {
+            requests: 2_000,
+            concurrency: 100,
+            max_connections: 64,
+            flush_batch_size: 1_000,
+            flush_interval: Duration::from_millis(100),
+            monitor_interval: Duration::from_millis(100),
+            output_path: None,
+            postgres_url: None,
+        }
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct Report {
+    suite: &'static str,
+    config: ReportConfig,
+    duration_ms: u64,
+    throughput_rps: u64,
+    completed_requests: usize,
+    failed_requests: usize,
+    p50_ms: u64,
+    p95_ms: u64,
+    max_ms: u64,
+    mean_ms: u64,
+    flush: FlushReport,
+    counters: CounterReport,
+    lock_monitor: LockMonitorReport,
+}
+
+#[derive(Debug, Serialize)]
+struct ReportConfig {
+    requests: usize,
+    concurrency: usize,
+    max_connections: u32,
+    flush_batch_size: usize,
+    flush_interval_ms: u64,
+    monitor_interval_ms: u64,
+    managed_postgres: bool,
+}
+
+#[derive(Debug, Serialize, Default)]
+struct FlushReport {
+    calls: usize,
+    rows_claimed: usize,
+    api_key_targets: usize,
+    provider_api_key_targets: usize,
+    model_targets: usize,
+    provider_monthly_targets: usize,
+    proxy_node_targets: usize,
+    management_token_targets: usize,
+    api_key_last_used_targets: usize,
+}
+
+#[derive(Debug, Serialize)]
+struct CounterReport {
+    usage_rows: i64,
+    outbox_pending_rows: i64,
+    outbox_processed_rows: i64,
+    api_key_total_requests: i64,
+    api_key_total_tokens: i64,
+    provider_key_request_count: i64,
+    provider_key_success_count: i64,
+    global_model_usage_count: i64,
+}
+
+#[derive(Debug, Serialize, Clone, Copy, Default)]
+struct LockMonitorReport {
+    samples: usize,
+    max_lock_waiters: i64,
+    max_api_key_update_waiters: i64,
+    max_provider_key_update_waiters: i64,
+    max_global_model_update_waiters: i64,
+    max_provider_update_waiters: i64,
+    max_oldest_lock_wait_ms: i64,
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+struct LockSample {
+    lock_waiters: i64,
+    api_key_update_waiters: i64,
+    provider_key_update_waiters: i64,
+    global_model_update_waiters: i64,
+    provider_update_waiters: i64,
+    oldest_lock_wait_ms: i64,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    init_test_runtime_for("usage-counter-hotspot-baseline");
+    let config = parse_args(std::env::args().skip(1).collect())?;
+
+    let managed_postgres;
+    let database_url;
+    let _server;
+    if let Some(url) = config.postgres_url.as_ref() {
+        managed_postgres = false;
+        database_url = url.clone();
+        _server = None;
+    } else {
+        managed_postgres = true;
+        let server = ManagedPostgresServer::start().await?;
+        database_url = server.database_url().to_string();
+        _server = Some(server);
+    }
+
+    prepare_aether_postgres_schema(&database_url).await?;
+    let pool = PgPoolOptions::new()
+        .min_connections(1)
+        .max_connections(config.max_connections)
+        .acquire_timeout(Duration::from_secs(10))
+        .connect(&database_url)
+        .await?;
+    seed_hotspot_rows(&pool).await?;
+
+    let repository = SqlxUsageReadRepository::new(pool.clone());
+    let stop = Arc::new(AtomicBool::new(false));
+    let flush_report = Arc::new(Mutex::new(FlushReport::default()));
+    let lock_report = Arc::new(Mutex::new(LockMonitorReport::default()));
+
+    let flush_handle = spawn_flush_loop(
+        repository.clone(),
+        Arc::clone(&stop),
+        Arc::clone(&flush_report),
+        config.flush_batch_size,
+        config.flush_interval,
+    );
+    let monitor_handle = spawn_lock_monitor(
+        pool.clone(),
+        Arc::clone(&stop),
+        Arc::clone(&lock_report),
+        config.monitor_interval,
+    );
+
+    let started_at = Instant::now();
+    let load_result = run_usage_load(repository, config.requests, config.concurrency).await;
+    wait_for_outbox_drain(
+        &pool,
+        &SqlxUsageReadRepository::new(pool.clone()),
+        config.flush_batch_size,
+    )
+    .await?;
+    let duration_ms = started_at.elapsed().as_millis() as u64;
+
+    stop.store(true, Ordering::Release);
+    flush_handle.await??;
+    monitor_handle.await??;
+
+    let mut flush = flush_report.lock().await;
+    let final_flush = SqlxUsageReadRepository::new(pool.clone())
+        .flush_usage_counter_deltas(config.flush_batch_size)
+        .await?;
+    flush.calls += 1;
+    flush.rows_claimed += final_flush.rows_claimed;
+    flush.api_key_targets += final_flush.api_key_targets;
+    flush.provider_api_key_targets += final_flush.provider_api_key_targets;
+    flush.model_targets += final_flush.model_targets;
+    flush.provider_monthly_targets += final_flush.provider_monthly_targets;
+    flush.proxy_node_targets += final_flush.proxy_node_targets;
+    flush.management_token_targets += final_flush.management_token_targets;
+    flush.api_key_last_used_targets += final_flush.api_key_last_used_targets;
+    drop(flush);
+
+    let counters = read_counters(&pool).await?;
+    let latencies = load_result.latencies.lock().await.clone();
+    let (p50_ms, p95_ms, max_ms, mean_ms) = summarize_latencies(latencies);
+    let completed_requests = load_result.completed.load(Ordering::Acquire);
+    let throughput_rps = if duration_ms == 0 {
+        completed_requests as u64
+    } else {
+        ((completed_requests as u64) * 1_000) / duration_ms.max(1)
+    };
+
+    let report = Report {
+        suite: "usage_counter_hotspot_baseline",
+        config: ReportConfig {
+            requests: config.requests,
+            concurrency: config.concurrency,
+            max_connections: config.max_connections,
+            flush_batch_size: config.flush_batch_size,
+            flush_interval_ms: config.flush_interval.as_millis() as u64,
+            monitor_interval_ms: config.monitor_interval.as_millis() as u64,
+            managed_postgres,
+        },
+        duration_ms,
+        throughput_rps,
+        completed_requests,
+        failed_requests: load_result.failed.load(Ordering::Acquire),
+        p50_ms,
+        p95_ms,
+        max_ms,
+        mean_ms,
+        flush: Arc::try_unwrap(flush_report)
+            .unwrap_or_else(|_| panic!("flush report still referenced"))
+            .into_inner(),
+        counters,
+        lock_monitor: Arc::try_unwrap(lock_report)
+            .unwrap_or_else(|_| panic!("lock report still referenced"))
+            .into_inner(),
+    };
+
+    let raw = serde_json::to_string_pretty(&report)?;
+    println!("{raw}");
+    if let Some(path) = config.output_path.as_ref() {
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        std::fs::write(path, format!("{raw}\n"))?;
+    }
+    Ok(())
+}
+
+#[derive(Clone)]
+struct LoadResult {
+    completed: Arc<AtomicUsize>,
+    failed: Arc<AtomicUsize>,
+    latencies: Arc<Mutex<Vec<u64>>>,
+}
+
+async fn run_usage_load(
+    repository: SqlxUsageReadRepository,
+    requests: usize,
+    concurrency: usize,
+) -> LoadResult {
+    let next = Arc::new(AtomicUsize::new(0));
+    let completed = Arc::new(AtomicUsize::new(0));
+    let failed = Arc::new(AtomicUsize::new(0));
+    let latencies = Arc::new(Mutex::new(Vec::with_capacity(requests)));
+    let mut tasks = tokio::task::JoinSet::new();
+
+    for _ in 0..concurrency {
+        let repository = repository.clone();
+        let next = Arc::clone(&next);
+        let completed = Arc::clone(&completed);
+        let failed = Arc::clone(&failed);
+        let latencies = Arc::clone(&latencies);
+        tasks.spawn(async move {
+            loop {
+                let index = next.fetch_add(1, Ordering::AcqRel);
+                if index >= requests {
+                    break;
+                }
+                let started_at = Instant::now();
+                let result = repository.upsert(usage_record(index)).await;
+                latencies
+                    .lock()
+                    .await
+                    .push(started_at.elapsed().as_millis() as u64);
+                completed.fetch_add(1, Ordering::AcqRel);
+                if result.is_err() {
+                    failed.fetch_add(1, Ordering::AcqRel);
+                }
+            }
+        });
+    }
+
+    while let Some(result) = tasks.join_next().await {
+        if result.is_err() {
+            failed.fetch_add(1, Ordering::AcqRel);
+        }
+    }
+
+    LoadResult {
+        completed,
+        failed,
+        latencies,
+    }
+}
+
+fn spawn_flush_loop(
+    repository: SqlxUsageReadRepository,
+    stop: Arc<AtomicBool>,
+    report: Arc<Mutex<FlushReport>>,
+    batch_size: usize,
+    interval: Duration,
+) -> tokio::task::JoinHandle<Result<(), aether_data::DataLayerError>> {
+    tokio::spawn(async move {
+        while !stop.load(Ordering::Acquire) {
+            let summary = repository.flush_usage_counter_deltas(batch_size).await?;
+            let mut report = report.lock().await;
+            report.calls += 1;
+            report.rows_claimed += summary.rows_claimed;
+            report.api_key_targets += summary.api_key_targets;
+            report.provider_api_key_targets += summary.provider_api_key_targets;
+            report.model_targets += summary.model_targets;
+            report.provider_monthly_targets += summary.provider_monthly_targets;
+            report.proxy_node_targets += summary.proxy_node_targets;
+            report.management_token_targets += summary.management_token_targets;
+            report.api_key_last_used_targets += summary.api_key_last_used_targets;
+            drop(report);
+            tokio::time::sleep(interval).await;
+        }
+        Ok(())
+    })
+}
+
+fn spawn_lock_monitor(
+    pool: PgPool,
+    stop: Arc<AtomicBool>,
+    report: Arc<Mutex<LockMonitorReport>>,
+    interval: Duration,
+) -> tokio::task::JoinHandle<Result<(), sqlx::Error>> {
+    tokio::spawn(async move {
+        while !stop.load(Ordering::Acquire) {
+            let sample = read_lock_sample(&pool).await?;
+            let mut report = report.lock().await;
+            report.samples += 1;
+            report.max_lock_waiters = report.max_lock_waiters.max(sample.lock_waiters);
+            report.max_api_key_update_waiters = report
+                .max_api_key_update_waiters
+                .max(sample.api_key_update_waiters);
+            report.max_provider_key_update_waiters = report
+                .max_provider_key_update_waiters
+                .max(sample.provider_key_update_waiters);
+            report.max_global_model_update_waiters = report
+                .max_global_model_update_waiters
+                .max(sample.global_model_update_waiters);
+            report.max_provider_update_waiters = report
+                .max_provider_update_waiters
+                .max(sample.provider_update_waiters);
+            report.max_oldest_lock_wait_ms = report
+                .max_oldest_lock_wait_ms
+                .max(sample.oldest_lock_wait_ms);
+            drop(report);
+            tokio::time::sleep(interval).await;
+        }
+        Ok(())
+    })
+}
+
+async fn wait_for_outbox_drain(
+    pool: &PgPool,
+    repository: &SqlxUsageReadRepository,
+    batch_size: usize,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let deadline = Instant::now() + Duration::from_secs(30);
+    loop {
+        let summary = repository.flush_usage_counter_deltas(batch_size).await?;
+        let pending: i64 = sqlx::query_scalar(
+            "SELECT COUNT(*)::BIGINT FROM usage_counter_deltas WHERE processed_at IS NULL",
+        )
+        .fetch_one(pool)
+        .await?;
+        if pending == 0 && summary.rows_claimed == 0 {
+            return Ok(());
+        }
+        if Instant::now() >= deadline {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::TimedOut,
+                format!("usage counter outbox did not drain; pending={pending}"),
+            )
+            .into());
+        }
+        tokio::time::sleep(Duration::from_millis(100)).await;
+    }
+}
+
+async fn seed_hotspot_rows(pool: &PgPool) -> Result<(), sqlx::Error> {
+    sqlx::query(
+        r#"
+INSERT INTO users (id, username, email_verified)
+VALUES ('user-hotspot', 'usage-hotspot', true)
+ON CONFLICT (id) DO NOTHING
+"#,
+    )
+    .execute(pool)
+    .await?;
+    sqlx::query(
+        r#"
+INSERT INTO api_keys (id, user_id, key_hash, name, is_active, total_requests, total_tokens, total_cost_usd)
+VALUES ('api-key-hotspot', 'user-hotspot', 'hash-hotspot', 'hotspot key', true, 0, 0, 0)
+ON CONFLICT (id) DO UPDATE SET
+  total_requests = 0,
+  total_tokens = 0,
+  total_cost_usd = 0,
+  last_used_at = NULL
+"#,
+    )
+    .execute(pool)
+    .await?;
+    sqlx::query(
+        r#"
+INSERT INTO providers (id, name, provider_type, monthly_used_usd)
+VALUES ('provider-hotspot', 'Hotspot Provider', 'openai', 0)
+ON CONFLICT (id) DO UPDATE SET monthly_used_usd = 0
+"#,
+    )
+    .execute(pool)
+    .await?;
+    sqlx::query(
+        r#"
+INSERT INTO provider_api_keys (
+  id, provider_id, name, total_tokens, total_cost_usd, request_count,
+  success_count, error_count, total_response_time_ms
+)
+VALUES ('provider-key-hotspot', 'provider-hotspot', 'Hotspot Provider Key', 0, 0, 0, 0, 0, 0)
+ON CONFLICT (id) DO UPDATE SET
+  total_tokens = 0,
+  total_cost_usd = 0,
+  request_count = 0,
+  success_count = 0,
+  error_count = 0,
+  total_response_time_ms = 0,
+  last_used_at = NULL
+"#,
+    )
+    .execute(pool)
+    .await?;
+    sqlx::query(
+        r#"
+INSERT INTO global_models (id, name, display_name, enabled, is_active, usage_count)
+VALUES ('model-hotspot', 'gpt-5', 'gpt-5', true, true, 0)
+ON CONFLICT (id) DO UPDATE SET usage_count = 0
+"#,
+    )
+    .execute(pool)
+    .await?;
+    Ok(())
+}
+
+fn usage_record(index: usize) -> UpsertUsageRecord {
+    let now_ms = now_unix_ms().saturating_add(index as u64);
+    let now_secs = now_ms / 1_000;
+    UpsertUsageRecord {
+        request_id: format!("usage-hotspot-{index:08}"),
+        user_id: Some("user-hotspot".to_string()),
+        api_key_id: Some("api-key-hotspot".to_string()),
+        username: None,
+        api_key_name: None,
+        provider_name: "openai".to_string(),
+        model: "gpt-5".to_string(),
+        target_model: None,
+        provider_id: Some("provider-hotspot".to_string()),
+        provider_endpoint_id: None,
+        provider_api_key_id: Some("provider-key-hotspot".to_string()),
+        request_type: Some("chat".to_string()),
+        api_format: Some("openai:chat".to_string()),
+        api_family: Some("openai".to_string()),
+        endpoint_kind: Some("chat".to_string()),
+        endpoint_api_format: Some("openai:chat".to_string()),
+        provider_api_family: Some("openai".to_string()),
+        provider_endpoint_kind: Some("chat".to_string()),
+        has_format_conversion: Some(false),
+        is_stream: Some(false),
+        input_tokens: Some(10),
+        output_tokens: Some(20),
+        total_tokens: Some(30),
+        cache_creation_input_tokens: None,
+        cache_creation_ephemeral_5m_input_tokens: None,
+        cache_creation_ephemeral_1h_input_tokens: None,
+        cache_read_input_tokens: None,
+        cache_creation_cost_usd: None,
+        cache_read_cost_usd: None,
+        output_price_per_1m: None,
+        total_cost_usd: Some(0.001),
+        actual_total_cost_usd: Some(0.001),
+        status_code: Some(200),
+        error_message: None,
+        error_category: None,
+        response_time_ms: Some(100),
+        first_byte_time_ms: Some(20),
+        status: "completed".to_string(),
+        billing_status: "pending".to_string(),
+        request_headers: None,
+        request_body: Some(json!({"model": "gpt-5"})),
+        request_body_ref: None,
+        request_body_state: None,
+        provider_request_headers: None,
+        provider_request_body: None,
+        provider_request_body_ref: None,
+        provider_request_body_state: None,
+        response_headers: None,
+        response_body: Some(json!({"id": format!("chatcmpl-{index}")})),
+        response_body_ref: None,
+        response_body_state: None,
+        client_response_headers: None,
+        client_response_body: None,
+        client_response_body_ref: None,
+        client_response_body_state: None,
+        candidate_id: Some(format!("candidate-{index:08}")),
+        candidate_index: Some(0),
+        key_name: None,
+        planner_kind: Some("hotspot_baseline".to_string()),
+        route_family: Some("openai".to_string()),
+        route_kind: Some("chat".to_string()),
+        execution_path: Some("testkit".to_string()),
+        local_execution_runtime_miss_reason: None,
+        request_metadata: None,
+        finalized_at_unix_secs: None,
+        created_at_unix_ms: Some(now_ms),
+        updated_at_unix_secs: now_secs,
+    }
+}
+
+async fn read_lock_sample(pool: &PgPool) -> Result<LockSample, sqlx::Error> {
+    let row = sqlx::query(
+        r#"
+SELECT
+  COUNT(*) FILTER (WHERE wait_event_type = 'Lock')::BIGINT AS lock_waiters,
+  COUNT(*) FILTER (
+    WHERE wait_event_type = 'Lock' AND query LIKE 'UPDATE api_keys%'
+  )::BIGINT AS api_key_update_waiters,
+  COUNT(*) FILTER (
+    WHERE wait_event_type = 'Lock' AND query LIKE 'UPDATE provider_api_keys%'
+  )::BIGINT AS provider_key_update_waiters,
+  COUNT(*) FILTER (
+    WHERE wait_event_type = 'Lock' AND query LIKE 'UPDATE global_models%'
+  )::BIGINT AS global_model_update_waiters,
+  COUNT(*) FILTER (
+    WHERE wait_event_type = 'Lock' AND query LIKE 'UPDATE providers%'
+  )::BIGINT AS provider_update_waiters,
+  COALESCE(
+    MAX(EXTRACT(EPOCH FROM (NOW() - query_start)) * 1000)
+      FILTER (WHERE wait_event_type = 'Lock'),
+    0
+  )::BIGINT AS oldest_lock_wait_ms
+FROM pg_stat_activity
+WHERE datname = current_database()
+"#,
+    )
+    .fetch_one(pool)
+    .await?;
+    Ok(LockSample {
+        lock_waiters: row.try_get("lock_waiters")?,
+        api_key_update_waiters: row.try_get("api_key_update_waiters")?,
+        provider_key_update_waiters: row.try_get("provider_key_update_waiters")?,
+        global_model_update_waiters: row.try_get("global_model_update_waiters")?,
+        provider_update_waiters: row.try_get("provider_update_waiters")?,
+        oldest_lock_wait_ms: row.try_get("oldest_lock_wait_ms")?,
+    })
+}
+
+async fn read_counters(pool: &PgPool) -> Result<CounterReport, sqlx::Error> {
+    let row = sqlx::query(
+        r#"
+SELECT
+  (SELECT COUNT(*)::BIGINT FROM usage) AS usage_rows,
+  (SELECT COUNT(*)::BIGINT FROM usage_counter_deltas WHERE processed_at IS NULL) AS outbox_pending_rows,
+  (SELECT COUNT(*)::BIGINT FROM usage_counter_deltas WHERE processed_at IS NOT NULL) AS outbox_processed_rows,
+  (SELECT COALESCE(total_requests, 0)::BIGINT FROM api_keys WHERE id = 'api-key-hotspot') AS api_key_total_requests,
+  (SELECT COALESCE(total_tokens, 0)::BIGINT FROM api_keys WHERE id = 'api-key-hotspot') AS api_key_total_tokens,
+  (SELECT COALESCE(request_count, 0)::BIGINT FROM provider_api_keys WHERE id = 'provider-key-hotspot') AS provider_key_request_count,
+  (SELECT COALESCE(success_count, 0)::BIGINT FROM provider_api_keys WHERE id = 'provider-key-hotspot') AS provider_key_success_count,
+  (SELECT COALESCE(usage_count, 0)::BIGINT FROM global_models WHERE name = 'gpt-5' LIMIT 1) AS global_model_usage_count
+"#,
+    )
+    .fetch_one(pool)
+    .await?;
+    Ok(CounterReport {
+        usage_rows: row.try_get("usage_rows")?,
+        outbox_pending_rows: row.try_get("outbox_pending_rows")?,
+        outbox_processed_rows: row.try_get("outbox_processed_rows")?,
+        api_key_total_requests: row.try_get("api_key_total_requests")?,
+        api_key_total_tokens: row.try_get("api_key_total_tokens")?,
+        provider_key_request_count: row.try_get("provider_key_request_count")?,
+        provider_key_success_count: row.try_get("provider_key_success_count")?,
+        global_model_usage_count: row.try_get("global_model_usage_count")?,
+    })
+}
+
+fn summarize_latencies(mut latencies: Vec<u64>) -> (u64, u64, u64, u64) {
+    if latencies.is_empty() {
+        return (0, 0, 0, 0);
+    }
+    latencies.sort_unstable();
+    let p50 = percentile(&latencies, 50);
+    let p95 = percentile(&latencies, 95);
+    let max = *latencies.last().unwrap_or(&0);
+    let mean = latencies.iter().sum::<u64>() / latencies.len() as u64;
+    (p50, p95, max, mean)
+}
+
+fn percentile(latencies: &[u64], percentile: usize) -> u64 {
+    let index = ((latencies.len() - 1) * percentile) / 100;
+    latencies[index]
+}
+
+fn now_unix_ms() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_millis() as u64
+}
+
+fn parse_args(args: Vec<String>) -> Result<Config, Box<dyn std::error::Error>> {
+    let mut config = Config::default();
+    let mut iter = args.into_iter();
+    while let Some(arg) = iter.next() {
+        match arg.as_str() {
+            "--requests" => config.requests = next_value(&mut iter, "--requests")?.parse()?,
+            "--concurrency" => {
+                config.concurrency = next_value(&mut iter, "--concurrency")?.parse()?
+            }
+            "--max-connections" => {
+                config.max_connections = next_value(&mut iter, "--max-connections")?.parse()?
+            }
+            "--flush-batch-size" => {
+                config.flush_batch_size = next_value(&mut iter, "--flush-batch-size")?.parse()?
+            }
+            "--flush-interval-ms" => {
+                config.flush_interval =
+                    Duration::from_millis(next_value(&mut iter, "--flush-interval-ms")?.parse()?)
+            }
+            "--monitor-interval-ms" => {
+                config.monitor_interval =
+                    Duration::from_millis(next_value(&mut iter, "--monitor-interval-ms")?.parse()?)
+            }
+            "--postgres-url" => {
+                config.postgres_url = Some(next_value(&mut iter, "--postgres-url")?)
+            }
+            "--output" => {
+                config.output_path = Some(PathBuf::from(next_value(&mut iter, "--output")?))
+            }
+            "--help" | "-h" => {
+                print_usage();
+                std::process::exit(0);
+            }
+            other => {
+                return Err(std::io::Error::new(
+                    std::io::ErrorKind::InvalidInput,
+                    format!("unknown argument: {other}"),
+                )
+                .into());
+            }
+        }
+    }
+    if config.requests == 0 || config.concurrency == 0 || config.max_connections == 0 {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            "requests, concurrency, and max-connections must be positive",
+        )
+        .into());
+    }
+    Ok(config)
+}
+
+fn next_value(
+    iter: &mut impl Iterator<Item = String>,
+    flag: &str,
+) -> Result<String, Box<dyn std::error::Error>> {
+    iter.next().ok_or_else(|| {
+        std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            format!("missing value for {flag}"),
+        )
+        .into()
+    })
+}
+
+fn print_usage() {
+    eprintln!(
+        "usage: cargo run -p aether-integration-tests --bin usage_counter_hotspot_baseline -- [--requests 2000] [--concurrency 100] [--max-connections 64] [--flush-batch-size 1000] [--flush-interval-ms 100] [--monitor-interval-ms 100] [--postgres-url postgres://...] [--output /tmp/usage_counter_hotspot.json]"
+    );
+}

crates/aether-testing/integration/src/bin/usage_settlement_hotspot_baseline.rs

@@ -0,0 +1,674 @@
+// Gateway-backed benchmark scenarios live outside the reusable testkit.
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::Arc;
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
+
+use aether_data::repository::settlement::SqlxSettlementRepository;
+use aether_data::repository::usage::SqlxUsageReadRepository;
+use aether_data_contracts::repository::settlement::{
+    SettlementWriteRepository, UsageSettlementInput,
+};
+use aether_testkit::{
+    init_test_runtime_for, prepare_aether_postgres_schema, ManagedPostgresServer,
+};
+use serde::Serialize;
+use sqlx::postgres::PgPoolOptions;
+use sqlx::{PgPool, Row};
+use tokio::sync::Mutex;
+
+const PROVIDER_ID: &str = "provider-hotspot";
+const REQUEST_PREFIX: &str = "settlement-hotspot";
+const COST_PER_REQUEST_USD: f64 = 0.001;
+
+#[derive(Debug, Clone)]
+struct Config {
+    requests: usize,
+    concurrency: usize,
+    max_connections: u32,
+    flush_batch_size: usize,
+    flush_interval: Duration,
+    monitor_interval: Duration,
+    output_path: Option<PathBuf>,
+    postgres_url: Option<String>,
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Self {
+            requests: 2_000,
+            concurrency: 100,
+            max_connections: 64,
+            flush_batch_size: 1_000,
+            flush_interval: Duration::from_millis(100),
+            monitor_interval: Duration::from_millis(100),
+            output_path: None,
+            postgres_url: None,
+        }
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct Report {
+    suite: &'static str,
+    config: ReportConfig,
+    duration_ms: u64,
+    throughput_rps: u64,
+    completed_requests: usize,
+    failed_requests: usize,
+    p50_ms: u64,
+    p95_ms: u64,
+    max_ms: u64,
+    mean_ms: u64,
+    flush: FlushReport,
+    counters: CounterReport,
+    lock_monitor: LockMonitorReport,
+}
+
+#[derive(Debug, Serialize)]
+struct ReportConfig {
+    requests: usize,
+    concurrency: usize,
+    max_connections: u32,
+    flush_batch_size: usize,
+    flush_interval_ms: u64,
+    monitor_interval_ms: u64,
+    managed_postgres: bool,
+}
+
+#[derive(Debug, Serialize, Default)]
+struct FlushReport {
+    calls: usize,
+    rows_claimed: usize,
+    api_key_targets: usize,
+    provider_api_key_targets: usize,
+    model_targets: usize,
+    provider_monthly_targets: usize,
+    proxy_node_targets: usize,
+    management_token_targets: usize,
+    api_key_last_used_targets: usize,
+}
+
+#[derive(Debug, Serialize)]
+struct CounterReport {
+    usage_rows: i64,
+    settled_usage_rows: i64,
+    settlement_snapshot_rows: i64,
+    outbox_pending_rows: i64,
+    outbox_processed_rows: i64,
+    provider_monthly_outbox_rows: i64,
+    provider_monthly_used_usd: f64,
+    expected_provider_monthly_used_usd: f64,
+}
+
+#[derive(Debug, Serialize, Clone, Copy, Default)]
+struct LockMonitorReport {
+    samples: usize,
+    max_lock_waiters: i64,
+    max_usage_update_waiters: i64,
+    max_wallet_update_waiters: i64,
+    max_provider_update_waiters: i64,
+    max_oldest_lock_wait_ms: i64,
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+struct LockSample {
+    lock_waiters: i64,
+    usage_update_waiters: i64,
+    wallet_update_waiters: i64,
+    provider_update_waiters: i64,
+    oldest_lock_wait_ms: i64,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    init_test_runtime_for("usage-settlement-hotspot-baseline");
+    let config = parse_args(std::env::args().skip(1).collect())?;
+
+    let managed_postgres;
+    let database_url;
+    let _server;
+    if let Some(url) = config.postgres_url.as_ref() {
+        managed_postgres = false;
+        database_url = url.clone();
+        _server = None;
+    } else {
+        managed_postgres = true;
+        let server = ManagedPostgresServer::start().await?;
+        database_url = server.database_url().to_string();
+        _server = Some(server);
+    }
+
+    prepare_aether_postgres_schema(&database_url).await?;
+    let pool = PgPoolOptions::new()
+        .min_connections(1)
+        .max_connections(config.max_connections)
+        .acquire_timeout(Duration::from_secs(10))
+        .connect(&database_url)
+        .await?;
+    seed_settlement_rows(&pool, config.requests).await?;
+
+    let usage_repository = SqlxUsageReadRepository::new(pool.clone());
+    let settlement_repository = SqlxSettlementRepository::new(pool.clone());
+    let stop = Arc::new(AtomicBool::new(false));
+    let flush_report = Arc::new(Mutex::new(FlushReport::default()));
+    let lock_report = Arc::new(Mutex::new(LockMonitorReport::default()));
+
+    let flush_handle = spawn_flush_loop(
+        usage_repository.clone(),
+        Arc::clone(&stop),
+        Arc::clone(&flush_report),
+        config.flush_batch_size,
+        config.flush_interval,
+    );
+    let monitor_handle = spawn_lock_monitor(
+        pool.clone(),
+        Arc::clone(&stop),
+        Arc::clone(&lock_report),
+        config.monitor_interval,
+    );
+
+    let started_at = Instant::now();
+    let load_result =
+        run_settlement_load(settlement_repository, config.requests, config.concurrency).await;
+    wait_for_outbox_drain(&pool, &usage_repository, config.flush_batch_size).await?;
+    let duration_ms = started_at.elapsed().as_millis() as u64;
+
+    stop.store(true, Ordering::Release);
+    flush_handle.await??;
+    monitor_handle.await??;
+
+    let mut flush = flush_report.lock().await;
+    let final_flush = usage_repository
+        .flush_usage_counter_deltas(config.flush_batch_size)
+        .await?;
+    flush.calls += 1;
+    flush.rows_claimed += final_flush.rows_claimed;
+    flush.api_key_targets += final_flush.api_key_targets;
+    flush.provider_api_key_targets += final_flush.provider_api_key_targets;
+    flush.model_targets += final_flush.model_targets;
+    flush.provider_monthly_targets += final_flush.provider_monthly_targets;
+    flush.proxy_node_targets += final_flush.proxy_node_targets;
+    flush.management_token_targets += final_flush.management_token_targets;
+    flush.api_key_last_used_targets += final_flush.api_key_last_used_targets;
+    drop(flush);
+
+    let counters = read_counters(&pool, config.requests).await?;
+    let latencies = load_result.latencies.lock().await.clone();
+    let (p50_ms, p95_ms, max_ms, mean_ms) = summarize_latencies(latencies);
+    let completed_requests = load_result.completed.load(Ordering::Acquire);
+    let throughput_rps = if duration_ms == 0 {
+        completed_requests as u64
+    } else {
+        ((completed_requests as u64) * 1_000) / duration_ms.max(1)
+    };
+
+    let report = Report {
+        suite: "usage_settlement_hotspot_baseline",
+        config: ReportConfig {
+            requests: config.requests,
+            concurrency: config.concurrency,
+            max_connections: config.max_connections,
+            flush_batch_size: config.flush_batch_size,
+            flush_interval_ms: config.flush_interval.as_millis() as u64,
+            monitor_interval_ms: config.monitor_interval.as_millis() as u64,
+            managed_postgres,
+        },
+        duration_ms,
+        throughput_rps,
+        completed_requests,
+        failed_requests: load_result.failed.load(Ordering::Acquire),
+        p50_ms,
+        p95_ms,
+        max_ms,
+        mean_ms,
+        flush: Arc::try_unwrap(flush_report)
+            .unwrap_or_else(|_| panic!("flush report still referenced"))
+            .into_inner(),
+        counters,
+        lock_monitor: Arc::try_unwrap(lock_report)
+            .unwrap_or_else(|_| panic!("lock report still referenced"))
+            .into_inner(),
+    };
+
+    let raw = serde_json::to_string_pretty(&report)?;
+    println!("{raw}");
+    if let Some(path) = config.output_path.as_ref() {
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        std::fs::write(path, format!("{raw}\n"))?;
+    }
+    Ok(())
+}
+
+#[derive(Clone)]
+struct LoadResult {
+    completed: Arc<AtomicUsize>,
+    failed: Arc<AtomicUsize>,
+    latencies: Arc<Mutex<Vec<u64>>>,
+}
+
+async fn run_settlement_load(
+    repository: SqlxSettlementRepository,
+    requests: usize,
+    concurrency: usize,
+) -> LoadResult {
+    let next = Arc::new(AtomicUsize::new(0));
+    let completed = Arc::new(AtomicUsize::new(0));
+    let failed = Arc::new(AtomicUsize::new(0));
+    let latencies = Arc::new(Mutex::new(Vec::with_capacity(requests)));
+    let mut tasks = tokio::task::JoinSet::new();
+
+    for _ in 0..concurrency {
+        let repository = repository.clone();
+        let next = Arc::clone(&next);
+        let completed = Arc::clone(&completed);
+        let failed = Arc::clone(&failed);
+        let latencies = Arc::clone(&latencies);
+        tasks.spawn(async move {
+            loop {
+                let index = next.fetch_add(1, Ordering::AcqRel);
+                if index >= requests {
+                    break;
+                }
+                let started_at = Instant::now();
+                let result = repository.settle_usage(settlement_input(index)).await;
+                latencies
+                    .lock()
+                    .await
+                    .push(started_at.elapsed().as_millis() as u64);
+                completed.fetch_add(1, Ordering::AcqRel);
+                match result {
+                    Ok(Some(settlement)) if settlement.billing_status == "settled" => {}
+                    _ => {
+                        failed.fetch_add(1, Ordering::AcqRel);
+                    }
+                }
+            }
+        });
+    }
+
+    while let Some(result) = tasks.join_next().await {
+        if result.is_err() {
+            failed.fetch_add(1, Ordering::AcqRel);
+        }
+    }
+
+    LoadResult {
+        completed,
+        failed,
+        latencies,
+    }
+}
+
+fn spawn_flush_loop(
+    repository: SqlxUsageReadRepository,
+    stop: Arc<AtomicBool>,
+    report: Arc<Mutex<FlushReport>>,
+    batch_size: usize,
+    interval: Duration,
+) -> tokio::task::JoinHandle<Result<(), aether_data::DataLayerError>> {
+    tokio::spawn(async move {
+        while !stop.load(Ordering::Acquire) {
+            let summary = repository.flush_usage_counter_deltas(batch_size).await?;
+            let mut report = report.lock().await;
+            report.calls += 1;
+            report.rows_claimed += summary.rows_claimed;
+            report.api_key_targets += summary.api_key_targets;
+            report.provider_api_key_targets += summary.provider_api_key_targets;
+            report.model_targets += summary.model_targets;
+            report.provider_monthly_targets += summary.provider_monthly_targets;
+            report.proxy_node_targets += summary.proxy_node_targets;
+            report.management_token_targets += summary.management_token_targets;
+            report.api_key_last_used_targets += summary.api_key_last_used_targets;
+            drop(report);
+            tokio::time::sleep(interval).await;
+        }
+        Ok(())
+    })
+}
+
+fn spawn_lock_monitor(
+    pool: PgPool,
+    stop: Arc<AtomicBool>,
+    report: Arc<Mutex<LockMonitorReport>>,
+    interval: Duration,
+) -> tokio::task::JoinHandle<Result<(), sqlx::Error>> {
+    tokio::spawn(async move {
+        while !stop.load(Ordering::Acquire) {
+            let sample = read_lock_sample(&pool).await?;
+            let mut report = report.lock().await;
+            report.samples += 1;
+            report.max_lock_waiters = report.max_lock_waiters.max(sample.lock_waiters);
+            report.max_usage_update_waiters = report
+                .max_usage_update_waiters
+                .max(sample.usage_update_waiters);
+            report.max_wallet_update_waiters = report
+                .max_wallet_update_waiters
+                .max(sample.wallet_update_waiters);
+            report.max_provider_update_waiters = report
+                .max_provider_update_waiters
+                .max(sample.provider_update_waiters);
+            report.max_oldest_lock_wait_ms = report
+                .max_oldest_lock_wait_ms
+                .max(sample.oldest_lock_wait_ms);
+            drop(report);
+            tokio::time::sleep(interval).await;
+        }
+        Ok(())
+    })
+}
+
+async fn wait_for_outbox_drain(
+    pool: &PgPool,
+    repository: &SqlxUsageReadRepository,
+    batch_size: usize,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let deadline = Instant::now() + Duration::from_secs(30);
+    loop {
+        let summary = repository.flush_usage_counter_deltas(batch_size).await?;
+        let pending: i64 = sqlx::query_scalar(
+            "SELECT COUNT(*)::BIGINT FROM usage_counter_deltas WHERE processed_at IS NULL",
+        )
+        .fetch_one(pool)
+        .await?;
+        if pending == 0 && summary.rows_claimed == 0 {
+            return Ok(());
+        }
+        if Instant::now() >= deadline {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::TimedOut,
+                format!("usage counter outbox did not drain; pending={pending}"),
+            )
+            .into());
+        }
+        tokio::time::sleep(Duration::from_millis(100)).await;
+    }
+}
+
+async fn seed_settlement_rows(pool: &PgPool, requests: usize) -> Result<(), sqlx::Error> {
+    sqlx::query(
+        r#"
+INSERT INTO providers (id, name, provider_type, monthly_used_usd)
+VALUES ($1, 'Hotspot Provider', 'openai', 0)
+ON CONFLICT (id) DO UPDATE SET monthly_used_usd = 0
+"#,
+    )
+    .bind(PROVIDER_ID)
+    .execute(pool)
+    .await?;
+
+    sqlx::query(
+        r#"
+DELETE FROM usage_counter_deltas
+WHERE request_id LIKE $1
+"#,
+    )
+    .bind(format!("{REQUEST_PREFIX}-%"))
+    .execute(pool)
+    .await?;
+    sqlx::query(
+        r#"
+DELETE FROM usage_settlement_snapshots
+WHERE request_id LIKE $1
+"#,
+    )
+    .bind(format!("{REQUEST_PREFIX}-%"))
+    .execute(pool)
+    .await?;
+    sqlx::query(
+        r#"
+DELETE FROM "usage"
+WHERE request_id LIKE $1
+"#,
+    )
+    .bind(format!("{REQUEST_PREFIX}-%"))
+    .execute(pool)
+    .await?;
+
+    sqlx::query(
+        r#"
+INSERT INTO "usage" (
+  id,
+  request_id,
+  provider_name,
+  model,
+  provider_id,
+  status,
+  billing_status,
+  total_cost_usd,
+  actual_total_cost_usd,
+  created_at_unix_ms,
+  updated_at_unix_secs
+)
+SELECT
+  'settlement-usage-' || LPAD(gs::TEXT, 8, '0'),
+  $2 || '-' || LPAD(gs::TEXT, 8, '0'),
+  'Hotspot Provider',
+  'gpt-5',
+  $3,
+  'completed',
+  'pending',
+  0,
+  $4,
+  $5 + gs,
+  $6
+FROM generate_series(0, $1::INTEGER - 1) AS gs
+"#,
+    )
+    .bind(i32::try_from(requests).unwrap_or(i32::MAX))
+    .bind(REQUEST_PREFIX)
+    .bind(PROVIDER_ID)
+    .bind(COST_PER_REQUEST_USD)
+    .bind(now_unix_ms() as i64)
+    .bind(now_unix_secs() as i64)
+    .execute(pool)
+    .await?;
+
+    Ok(())
+}
+
+fn settlement_input(index: usize) -> UsageSettlementInput {
+    UsageSettlementInput {
+        request_id: format!("{REQUEST_PREFIX}-{index:08}"),
+        user_id: None,
+        api_key_id: None,
+        api_key_is_standalone: false,
+        provider_id: Some(PROVIDER_ID.to_string()),
+        status: "completed".to_string(),
+        billing_status: "pending".to_string(),
+        total_cost_usd: 0.0,
+        actual_total_cost_usd: COST_PER_REQUEST_USD,
+        finalized_at_unix_secs: Some(now_unix_secs().saturating_add(index as u64)),
+    }
+}
+
+async fn read_lock_sample(pool: &PgPool) -> Result<LockSample, sqlx::Error> {
+    let row = sqlx::query(
+        r#"
+SELECT
+  COUNT(*) FILTER (WHERE wait_event_type = 'Lock')::BIGINT AS lock_waiters,
+  COUNT(*) FILTER (
+    WHERE wait_event_type = 'Lock' AND query LIKE '%UPDATE "usage"%'
+  )::BIGINT AS usage_update_waiters,
+  COUNT(*) FILTER (
+    WHERE wait_event_type = 'Lock' AND query LIKE '%UPDATE wallets%'
+  )::BIGINT AS wallet_update_waiters,
+  COUNT(*) FILTER (
+    WHERE wait_event_type = 'Lock' AND query LIKE '%UPDATE providers%'
+  )::BIGINT AS provider_update_waiters,
+  COALESCE(
+    MAX(EXTRACT(EPOCH FROM (NOW() - query_start)) * 1000)
+      FILTER (WHERE wait_event_type = 'Lock'),
+    0
+  )::BIGINT AS oldest_lock_wait_ms
+FROM pg_stat_activity
+WHERE datname = current_database()
+"#,
+    )
+    .fetch_one(pool)
+    .await?;
+    Ok(LockSample {
+        lock_waiters: row.try_get("lock_waiters")?,
+        usage_update_waiters: row.try_get("usage_update_waiters")?,
+        wallet_update_waiters: row.try_get("wallet_update_waiters")?,
+        provider_update_waiters: row.try_get("provider_update_waiters")?,
+        oldest_lock_wait_ms: row.try_get("oldest_lock_wait_ms")?,
+    })
+}
+
+async fn read_counters(pool: &PgPool, requests: usize) -> Result<CounterReport, sqlx::Error> {
+    let row = sqlx::query(
+        r#"
+SELECT
+  (SELECT COUNT(*)::BIGINT FROM "usage" WHERE request_id LIKE $1) AS usage_rows,
+  (
+    SELECT COUNT(*)::BIGINT
+    FROM "usage"
+    WHERE request_id LIKE $1 AND billing_status = 'settled'
+  ) AS settled_usage_rows,
+  (
+    SELECT COUNT(*)::BIGINT
+    FROM usage_settlement_snapshots
+    WHERE request_id LIKE $1
+  ) AS settlement_snapshot_rows,
+  (
+    SELECT COUNT(*)::BIGINT
+    FROM usage_counter_deltas
+    WHERE request_id LIKE $1 AND processed_at IS NULL
+  ) AS outbox_pending_rows,
+  (
+    SELECT COUNT(*)::BIGINT
+    FROM usage_counter_deltas
+    WHERE request_id LIKE $1 AND processed_at IS NOT NULL
+  ) AS outbox_processed_rows,
+  (
+    SELECT COUNT(*)::BIGINT
+    FROM usage_counter_deltas
+    WHERE request_id LIKE $1 AND kind = 'provider_monthly'
+  ) AS provider_monthly_outbox_rows,
+  (
+    SELECT CAST(monthly_used_usd AS DOUBLE PRECISION)
+    FROM providers
+    WHERE id = $2
+  ) AS provider_monthly_used_usd
+"#,
+    )
+    .bind(format!("{REQUEST_PREFIX}-%"))
+    .bind(PROVIDER_ID)
+    .fetch_one(pool)
+    .await?;
+    Ok(CounterReport {
+        usage_rows: row.try_get("usage_rows")?,
+        settled_usage_rows: row.try_get("settled_usage_rows")?,
+        settlement_snapshot_rows: row.try_get("settlement_snapshot_rows")?,
+        outbox_pending_rows: row.try_get("outbox_pending_rows")?,
+        outbox_processed_rows: row.try_get("outbox_processed_rows")?,
+        provider_monthly_outbox_rows: row.try_get("provider_monthly_outbox_rows")?,
+        provider_monthly_used_usd: row.try_get("provider_monthly_used_usd")?,
+        expected_provider_monthly_used_usd: (requests as f64) * COST_PER_REQUEST_USD,
+    })
+}
+
+fn summarize_latencies(mut latencies: Vec<u64>) -> (u64, u64, u64, u64) {
+    if latencies.is_empty() {
+        return (0, 0, 0, 0);
+    }
+    latencies.sort_unstable();
+    let p50 = percentile(&latencies, 50);
+    let p95 = percentile(&latencies, 95);
+    let max = *latencies.last().unwrap_or(&0);
+    let mean = latencies.iter().sum::<u64>() / latencies.len() as u64;
+    (p50, p95, max, mean)
+}
+
+fn percentile(latencies: &[u64], percentile: usize) -> u64 {
+    let index = ((latencies.len() - 1) * percentile) / 100;
+    latencies[index]
+}
+
+fn now_unix_ms() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_millis() as u64
+}
+
+fn now_unix_secs() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs()
+}
+
+fn parse_args(args: Vec<String>) -> Result<Config, Box<dyn std::error::Error>> {
+    let mut config = Config::default();
+    let mut iter = args.into_iter();
+    while let Some(arg) = iter.next() {
+        match arg.as_str() {
+            "--requests" => config.requests = next_value(&mut iter, "--requests")?.parse()?,
+            "--concurrency" => {
+                config.concurrency = next_value(&mut iter, "--concurrency")?.parse()?
+            }
+            "--max-connections" => {
+                config.max_connections = next_value(&mut iter, "--max-connections")?.parse()?
+            }
+            "--flush-batch-size" => {
+                config.flush_batch_size = next_value(&mut iter, "--flush-batch-size")?.parse()?
+            }
+            "--flush-interval-ms" => {
+                config.flush_interval =
+                    Duration::from_millis(next_value(&mut iter, "--flush-interval-ms")?.parse()?)
+            }
+            "--monitor-interval-ms" => {
+                config.monitor_interval =
+                    Duration::from_millis(next_value(&mut iter, "--monitor-interval-ms")?.parse()?)
+            }
+            "--postgres-url" => {
+                config.postgres_url = Some(next_value(&mut iter, "--postgres-url")?)
+            }
+            "--output" => {
+                config.output_path = Some(PathBuf::from(next_value(&mut iter, "--output")?))
+            }
+            "--help" | "-h" => {
+                print_usage();
+                std::process::exit(0);
+            }
+            other => {
+                return Err(std::io::Error::new(
+                    std::io::ErrorKind::InvalidInput,
+                    format!("unknown argument: {other}"),
+                )
+                .into());
+            }
+        }
+    }
+    if config.requests == 0 || config.concurrency == 0 || config.max_connections == 0 {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            "requests, concurrency, and max-connections must be positive",
+        )
+        .into());
+    }
+    Ok(config)
+}
+
+fn next_value(
+    iter: &mut impl Iterator<Item = String>,
+    flag: &str,
+) -> Result<String, Box<dyn std::error::Error>> {
+    iter.next().ok_or_else(|| {
+        std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            format!("missing value for {flag}"),
+        )
+        .into()
+    })
+}
+
+fn print_usage() {
+    eprintln!(
+        "usage: cargo run -p aether-integration-tests --bin usage_settlement_hotspot_baseline -- [--requests 2000] [--concurrency 100] [--max-connections 64] [--flush-batch-size 1000] [--flush-interval-ms 100] [--monitor-interval-ms 100] [--postgres-url postgres://...] [--output /tmp/usage_settlement_hotspot.json]"
+    );
+}