Improve gateway scheduling and runtime admission

crates/aether-testkit/src/bin/gateway_pressure_probe.rs

@@ -176,25 +176,56 @@ fn spawn_metrics_sampler(
 
 fn parse_args(args: Vec<String>) -> Result<Config, Box<dyn std::error::Error>> {
     let mut target_url: Option<String> = None;
+    let mut warmup_url: Option<String> = None;
     let mut metrics_url: Option<String> = None;
     let mut total_requests: Option<usize> = None;
     let mut concurrency: Option<usize> = None;
+    let mut warmup_connections: usize = 0;
     let mut timeout_ms: Option<u64> = None;
+    let mut connect_timeout_ms: Option<u64> = None;
+    let mut client_shards: Option<usize> = None;
+    let mut pool_max_idle_per_host: Option<usize> = None;
+    let mut start_ramp_ms: u64 = 0;
+    let mut first_body_hold_ms: u64 = 0;
     let mut sample_interval_ms: u64 = 500;
     let mut method = Method::GET;
     let mut headers = BTreeMap::new();
     let mut body: Option<Vec<u8>> = None;
     let mut response_mode = HttpLoadProbeResponseMode::HeadersOnly;
+    let mut http1_only = false;
+    let mut http2_prior_knowledge = false;
     let mut output_path = None;
 
     let mut iter = args.into_iter();
     while let Some(arg) = iter.next() {
         match arg.as_str() {
             "--url" => target_url = Some(next_value(&mut iter, "--url")?),
+            "--warmup-url" => warmup_url = Some(next_value(&mut iter, "--warmup-url")?),
             "--metrics-url" => metrics_url = Some(next_value(&mut iter, "--metrics-url")?),
             "--requests" => total_requests = Some(next_value(&mut iter, "--requests")?.parse()?),
             "--concurrency" => concurrency = Some(next_value(&mut iter, "--concurrency")?.parse()?),
+            "--warmup-connections" => {
+                warmup_connections = next_value(&mut iter, "--warmup-connections")?.parse()?
+            }
             "--timeout-ms" => timeout_ms = Some(next_value(&mut iter, "--timeout-ms")?.parse()?),
+            "--connect-timeout-ms" => {
+                connect_timeout_ms = Some(next_value(&mut iter, "--connect-timeout-ms")?.parse()?)
+            }
+            "--client-shards" => {
+                client_shards = Some(next_value(&mut iter, "--client-shards")?.parse()?)
+            }
+            "--pool-max-idle-per-host" => {
+                pool_max_idle_per_host =
+                    Some(next_value(&mut iter, "--pool-max-idle-per-host")?.parse()?)
+            }
+            "--start-ramp-ms" => {
+                start_ramp_ms = next_value(&mut iter, "--start-ramp-ms")?.parse()?
+            }
+            "--first-body-hold-ms" => {
+                first_body_hold_ms = next_value(&mut iter, "--first-body-hold-ms")?.parse()?
+            }
+            "--http1-only" => http1_only = true,
+            "--http2-prior-knowledge" => http2_prior_knowledge = true,
             "--sample-interval-ms" => {
                 sample_interval_ms = next_value(&mut iter, "--sample-interval-ms")?.parse()?
             }
@@ -247,9 +278,20 @@ fn parse_args(args: Vec<String>) -> Result<Config, Box<dyn std::error::Error>> {
         response_mode,
         ..HttpLoadProbeConfig::default()
     };
+    load.warmup_url = warmup_url;
+    load.warmup_connections = warmup_connections;
     if let Some(timeout_ms) = timeout_ms {
         load.timeout = Duration::from_millis(timeout_ms);
     }
+    load.connect_timeout = connect_timeout_ms.map(Duration::from_millis);
+    if let Some(client_shards) = client_shards {
+        load.client_shards = client_shards;
+    }
+    load.pool_max_idle_per_host = pool_max_idle_per_host;
+    load.start_ramp = Duration::from_millis(start_ramp_ms);
+    load.first_body_hold = Duration::from_millis(first_body_hold_ms);
+    load.http1_only = http1_only;
+    load.http2_prior_knowledge = http2_prior_knowledge;
     load.validate()
         .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err))?;
     if sample_interval_ms == 0 {
@@ -298,10 +340,15 @@ fn parse_response_mode(
 ) -> Result<HttpLoadProbeResponseMode, Box<dyn std::error::Error>> {
     match value.trim().to_ascii_lowercase().as_str() {
         "headers" | "headers-only" | "header" => Ok(HttpLoadProbeResponseMode::HeadersOnly),
+        "first-body-byte" | "first-body" | "first-byte" | "first-chunk" => {
+            Ok(HttpLoadProbeResponseMode::FirstBodyByte)
+        }
         "full" | "full-body" | "body" => Ok(HttpLoadProbeResponseMode::FullBody),
         other => Err(std::io::Error::new(
             std::io::ErrorKind::InvalidInput,
-            format!("unsupported --response-mode {other}; expected headers or full"),
+            format!(
+                "unsupported --response-mode {other}; expected headers, first-body-byte, or full"
+            ),
         )
         .into()),
     }
@@ -348,6 +395,6 @@ fn metric_name_matches(actual: &str, expected: &str) -> bool {
 
 fn print_usage() {
     eprintln!(
-        "usage: cargo run -p aether-testkit --bin gateway_pressure_probe -- --url <URL> --metrics-url <URL> --requests <N> --concurrency <N> [--method GET] [--timeout-ms 30000] [--sample-interval-ms 500] [-H 'Name: value'] [--body JSON | --body-file path] [--response-mode headers|full] [--output /tmp/gateway_pressure.json]"
+        "usage: cargo run -p aether-testkit --bin gateway_pressure_probe -- --url <URL> --metrics-url <URL> --requests <N> --concurrency <N> [--warmup-url <URL>] [--warmup-connections N] [--method GET] [--timeout-ms 30000] [--connect-timeout-ms 10000] [--client-shards 1] [--pool-max-idle-per-host N] [--start-ramp-ms 0] [--first-body-hold-ms 0] [--http1-only | --http2-prior-knowledge] [--sample-interval-ms 500] [-H 'Name: value'] [--body JSON | --body-file path] [--response-mode headers|first-body-byte|full] [--output /tmp/gateway_pressure.json]"
     );
 }

crates/aether-testkit/src/bin/http_load_probe.rs

@@ -15,21 +15,52 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
 fn parse_args(args: Vec<String>) -> Result<HttpLoadProbeConfig, Box<dyn std::error::Error>> {
     let mut url: Option<String> = None;
+    let mut warmup_url: Option<String> = None;
     let mut total_requests: Option<usize> = None;
     let mut concurrency: Option<usize> = None;
+    let mut warmup_connections: usize = 0;
     let mut timeout_ms: Option<u64> = None;
+    let mut connect_timeout_ms: Option<u64> = None;
+    let mut client_shards: Option<usize> = None;
+    let mut pool_max_idle_per_host: Option<usize> = None;
+    let mut start_ramp_ms: u64 = 0;
+    let mut first_body_hold_ms: u64 = 0;
     let mut method = Method::GET;
     let mut headers = std::collections::BTreeMap::new();
     let mut body: Option<Vec<u8>> = None;
     let mut response_mode = aether_testkit::HttpLoadProbeResponseMode::HeadersOnly;
+    let mut http1_only = false;
+    let mut http2_prior_knowledge = false;
 
     let mut iter = args.into_iter();
     while let Some(arg) = iter.next() {
         match arg.as_str() {
             "--url" => url = Some(next_value(&mut iter, "--url")?),
+            "--warmup-url" => warmup_url = Some(next_value(&mut iter, "--warmup-url")?),
             "--requests" => total_requests = Some(next_value(&mut iter, "--requests")?.parse()?),
             "--concurrency" => concurrency = Some(next_value(&mut iter, "--concurrency")?.parse()?),
+            "--warmup-connections" => {
+                warmup_connections = next_value(&mut iter, "--warmup-connections")?.parse()?
+            }
             "--timeout-ms" => timeout_ms = Some(next_value(&mut iter, "--timeout-ms")?.parse()?),
+            "--connect-timeout-ms" => {
+                connect_timeout_ms = Some(next_value(&mut iter, "--connect-timeout-ms")?.parse()?)
+            }
+            "--client-shards" => {
+                client_shards = Some(next_value(&mut iter, "--client-shards")?.parse()?)
+            }
+            "--pool-max-idle-per-host" => {
+                pool_max_idle_per_host =
+                    Some(next_value(&mut iter, "--pool-max-idle-per-host")?.parse()?)
+            }
+            "--start-ramp-ms" => {
+                start_ramp_ms = next_value(&mut iter, "--start-ramp-ms")?.parse()?
+            }
+            "--first-body-hold-ms" => {
+                first_body_hold_ms = next_value(&mut iter, "--first-body-hold-ms")?.parse()?
+            }
+            "--http1-only" => http1_only = true,
+            "--http2-prior-knowledge" => http2_prior_knowledge = true,
             "--method" => {
                 method = Method::from_bytes(next_value(&mut iter, "--method")?.as_bytes())?
             }
@@ -78,9 +109,20 @@ fn parse_args(args: Vec<String>) -> Result<HttpLoadProbeConfig, Box<dyn std::err
         response_mode,
         ..HttpLoadProbeConfig::default()
     };
+    config.warmup_url = warmup_url;
+    config.warmup_connections = warmup_connections;
     if let Some(timeout_ms) = timeout_ms {
         config.timeout = Duration::from_millis(timeout_ms);
     }
+    config.connect_timeout = connect_timeout_ms.map(Duration::from_millis);
+    if let Some(client_shards) = client_shards {
+        config.client_shards = client_shards;
+    }
+    config.pool_max_idle_per_host = pool_max_idle_per_host;
+    config.start_ramp = Duration::from_millis(start_ramp_ms);
+    config.first_body_hold = Duration::from_millis(first_body_hold_ms);
+    config.http1_only = http1_only;
+    config.http2_prior_knowledge = http2_prior_knowledge;
     config
         .validate()
         .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err))?;
@@ -115,10 +157,15 @@ fn parse_response_mode(
         "headers" | "headers-only" | "header" => {
             Ok(aether_testkit::HttpLoadProbeResponseMode::HeadersOnly)
         }
+        "first-body-byte" | "first-body" | "first-byte" | "first-chunk" => {
+            Ok(aether_testkit::HttpLoadProbeResponseMode::FirstBodyByte)
+        }
         "full" | "full-body" | "body" => Ok(aether_testkit::HttpLoadProbeResponseMode::FullBody),
         other => Err(std::io::Error::new(
             std::io::ErrorKind::InvalidInput,
-            format!("unsupported --response-mode {other}; expected headers or full"),
+            format!(
+                "unsupported --response-mode {other}; expected headers, first-body-byte, or full"
+            ),
         )
         .into()),
     }
@@ -139,6 +186,6 @@ fn next_value(
 
 fn print_usage() {
     eprintln!(
-        "usage: cargo run -p aether-testkit --bin http_load_probe -- --url <URL> --requests <N> --concurrency <N> [--method GET] [--timeout-ms 30000] [-H 'Name: value'] [--body JSON | --body-file path] [--response-mode headers|full]"
+        "usage: cargo run -p aether-testkit --bin http_load_probe -- --url <URL> --requests <N> --concurrency <N> [--warmup-url <URL>] [--warmup-connections N] [--method GET] [--timeout-ms 30000] [--connect-timeout-ms 10000] [--client-shards 1] [--pool-max-idle-per-host N] [--start-ramp-ms 0] [--first-body-hold-ms 0] [--http1-only | --http2-prior-knowledge] [-H 'Name: value'] [--body JSON | --body-file path] [--response-mode headers|first-body-byte|full]"
     );
 }

crates/aether-testkit/src/bin/mock_openai_upstream.rs

@@ -14,7 +14,7 @@ use serde_json::json;
 
 #[derive(Debug, Clone)]
 struct Config {
-    bind: SocketAddr,
+    binds: Vec<SocketAddr>,
     chunks: u64,
     first_byte_delay: Duration,
     chunk_delay: Duration,
@@ -25,9 +25,9 @@ struct Config {
 impl Default for Config {
     fn default() -> Self {
         Self {
-            bind: "127.0.0.1:18181"
+            binds: vec!["127.0.0.1:18181"
                 .parse()
-                .expect("default bind address should parse"),
+                .expect("default bind address should parse")],
             chunks: 8,
             first_byte_delay: Duration::from_millis(0),
             chunk_delay: Duration::from_millis(20),
@@ -68,9 +68,40 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .route("/responses", post(responses))
         .with_state(app_state);
 
-    let listener = tokio::net::TcpListener::bind(config.bind).await?;
-    eprintln!("mock OpenAI upstream listening on http://{}", config.bind);
-    axum::serve(listener, app).await?;
+    serve_listeners(&config.binds, app).await?;
+    Ok(())
+}
+
+async fn serve_listeners(
+    binds: &[SocketAddr],
+    app: Router,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let mut listeners = Vec::with_capacity(binds.len());
+    for bind in binds {
+        listeners.push((*bind, tokio::net::TcpListener::bind(bind).await?));
+    }
+    if listeners.len() == 1 {
+        let (bind, listener) = listeners
+            .into_iter()
+            .next()
+            .ok_or_else(|| std::io::Error::other("mock upstream listener set is empty"))?;
+        eprintln!("mock OpenAI upstream listening on http://{bind}");
+        axum::serve(listener, app).await?;
+        return Ok(());
+    }
+
+    let mut servers = tokio::task::JoinSet::new();
+    for (bind, listener) in listeners {
+        let app = app.clone();
+        eprintln!("mock OpenAI upstream listening on http://{bind}");
+        servers.spawn(async move { axum::serve(listener, app).await });
+    }
+    if let Some(result) = servers.join_next().await {
+        servers.abort_all();
+        let serve_result = result
+            .map_err(|err| std::io::Error::other(format!("mock listener task failed: {err}")))?;
+        serve_result?;
+    }
     Ok(())
 }
 
@@ -292,10 +323,17 @@ fn current_unix_secs() -> u64 {
 
 fn parse_args(args: Vec<String>) -> Result<Config, Box<dyn std::error::Error>> {
     let mut config = Config::default();
+    let mut binds_overridden = false;
     let mut iter = args.into_iter();
     while let Some(arg) = iter.next() {
         match arg.as_str() {
-            "--bind" => config.bind = next_value(&mut iter, "--bind")?.parse()?,
+            "--bind" => {
+                if !binds_overridden {
+                    config.binds.clear();
+                    binds_overridden = true;
+                }
+                config.binds.push(next_value(&mut iter, "--bind")?.parse()?);
+            }
             "--chunks" => config.chunks = next_value(&mut iter, "--chunks")?.parse()?,
             "--first-byte-delay-ms" => {
                 config.first_byte_delay =
@@ -325,6 +363,13 @@ fn parse_args(args: Vec<String>) -> Result<Config, Box<dyn std::error::Error>> {
             }
         }
     }
+    if config.binds.is_empty() {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidInput,
+            "at least one --bind is required",
+        )
+        .into());
+    }
     Ok(config)
 }
 
@@ -343,6 +388,6 @@ fn next_value(
 
 fn print_usage() {
     eprintln!(
-        "usage: cargo run -p aether-testkit --bin mock_openai_upstream -- [--bind 127.0.0.1:18181] [--chunks 8] [--first-byte-delay-ms 0] [--chunk-delay-ms 20] [--payload-bytes 32] [--status 200]"
+        "usage: cargo run -p aether-testkit --bin mock_openai_upstream -- [--bind 127.0.0.1:18181]... [--chunks 8] [--first-byte-delay-ms 0] [--chunk-delay-ms 20] [--payload-bytes 32] [--status 200]"
     );
 }

crates/aether-testkit/src/load.rs

@@ -1,4 +1,5 @@
 use std::collections::BTreeMap;
+use std::error::Error as StdError;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::time::{Duration, Instant};
@@ -9,36 +10,57 @@ use tokio::sync::Mutex;
 
 use crate::runtime::{BenchmarkRuntimeSampler, BenchmarkRuntimeSnapshot};
 
+const MAX_ERROR_SAMPLES: usize = 32;
+
 #[derive(Debug, Clone, Copy, Default, serde::Serialize, PartialEq, Eq)]
 pub enum HttpLoadProbeResponseMode {
     #[default]
     HeadersOnly,
+    FirstBodyByte,
     FullBody,
 }
 
 #[derive(Debug, Clone)]
 pub struct HttpLoadProbeConfig {
     pub url: String,
+    pub warmup_url: Option<String>,
     pub method: Method,
     pub headers: BTreeMap<String, String>,
     pub body: Option<Vec<u8>>,
     pub total_requests: usize,
     pub concurrency: usize,
+    pub warmup_connections: usize,
     pub timeout: Duration,
+    pub connect_timeout: Option<Duration>,
     pub response_mode: HttpLoadProbeResponseMode,
+    pub client_shards: usize,
+    pub pool_max_idle_per_host: Option<usize>,
+    pub start_ramp: Duration,
+    pub http1_only: bool,
+    pub http2_prior_knowledge: bool,
+    pub first_body_hold: Duration,
 }
 
 impl Default for HttpLoadProbeConfig {
     fn default() -> Self {
         Self {
             url: String::new(),
+            warmup_url: None,
             method: Method::GET,
             headers: BTreeMap::new(),
             body: None,
             total_requests: 100,
             concurrency: 10,
+            warmup_connections: 0,
             timeout: Duration::from_secs(30),
+            connect_timeout: None,
             response_mode: HttpLoadProbeResponseMode::HeadersOnly,
+            client_shards: 1,
+            pool_max_idle_per_host: None,
+            start_ramp: Duration::ZERO,
+            http1_only: false,
+            http2_prior_knowledge: false,
+            first_body_hold: Duration::ZERO,
         }
     }
 }
@@ -57,10 +79,33 @@ impl HttpLoadProbeConfig {
         if self.timeout.is_zero() {
             return Err("load probe timeout must be positive".to_string());
         }
+        if matches!(self.connect_timeout, Some(timeout) if timeout.is_zero()) {
+            return Err("load probe connect_timeout must be positive when set".to_string());
+        }
+        if self.client_shards == 0 {
+            return Err("load probe client_shards must be positive".to_string());
+        }
+        if self.http1_only && self.http2_prior_knowledge {
+            return Err(
+                "load probe cannot enable both http1_only and http2_prior_knowledge".to_string(),
+            );
+        }
         Ok(())
     }
 }
 
+#[derive(Debug, Clone, serde::Serialize, PartialEq, Eq)]
+pub struct HttpLoadProbeErrorSample {
+    pub request_index: usize,
+    pub url: String,
+    pub phase: String,
+    pub kind: String,
+    pub elapsed_ms: u64,
+    pub message: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub source: Option<String>,
+}
+
 #[derive(Debug, Clone, serde::Serialize, PartialEq, Eq)]
 pub struct HttpLoadProbeResult {
     pub url: String,
@@ -68,6 +113,18 @@ pub struct HttpLoadProbeResult {
     pub response_mode: HttpLoadProbeResponseMode,
     pub total_requests: usize,
     pub concurrency: usize,
+    pub warmup_connections: usize,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub warmup_url: Option<String>,
+    pub client_shards: usize,
+    pub start_ramp_ms: u64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub pool_max_idle_per_host: Option<usize>,
+    pub http1_only: bool,
+    pub http2_prior_knowledge: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub connect_timeout_ms: Option<u64>,
+    pub first_body_hold_ms: u64,
     pub duration_ms: u64,
     pub throughput_rps: u64,
     pub p99_ms: u64,
@@ -77,9 +134,23 @@ pub struct HttpLoadProbeResult {
     pub p95_ms: u64,
     pub max_ms: u64,
     pub mean_ms: u64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub headers_p50_ms: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub headers_p95_ms: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub headers_p99_ms: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub first_body_p50_ms: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub first_body_p95_ms: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub first_body_p99_ms: Option<u64>,
     pub runtime: BenchmarkRuntimeSnapshot,
     pub status_counts: BTreeMap<u16, usize>,
     pub error_counts: BTreeMap<String, usize>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub error_samples: Vec<HttpLoadProbeErrorSample>,
 }
 
 #[derive(Debug, Clone, serde::Serialize, PartialEq, Eq)]
@@ -90,6 +161,18 @@ pub struct MultiUrlHttpLoadProbeResult {
     pub response_mode: HttpLoadProbeResponseMode,
     pub total_requests: usize,
     pub concurrency: usize,
+    pub warmup_connections: usize,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub warmup_url: Option<String>,
+    pub client_shards: usize,
+    pub start_ramp_ms: u64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub pool_max_idle_per_host: Option<usize>,
+    pub http1_only: bool,
+    pub http2_prior_knowledge: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub connect_timeout_ms: Option<u64>,
+    pub first_body_hold_ms: u64,
     pub duration_ms: u64,
     pub throughput_rps: u64,
     pub p99_ms: u64,
@@ -99,9 +182,23 @@ pub struct MultiUrlHttpLoadProbeResult {
     pub p95_ms: u64,
     pub max_ms: u64,
     pub mean_ms: u64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub headers_p50_ms: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub headers_p95_ms: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub headers_p99_ms: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub first_body_p50_ms: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub first_body_p95_ms: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub first_body_p99_ms: Option<u64>,
     pub runtime: BenchmarkRuntimeSnapshot,
     pub status_counts: BTreeMap<u16, usize>,
     pub error_counts: BTreeMap<String, usize>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub error_samples: Vec<HttpLoadProbeErrorSample>,
 }
 
 pub async fn run_http_load_probe(
@@ -120,6 +217,15 @@ pub async fn run_http_load_probe(
             response_mode: result.response_mode,
             total_requests: result.total_requests,
             concurrency: result.concurrency,
+            warmup_connections: result.warmup_connections,
+            warmup_url: result.warmup_url,
+            client_shards: result.client_shards,
+            start_ramp_ms: result.start_ramp_ms,
+            pool_max_idle_per_host: result.pool_max_idle_per_host,
+            http1_only: result.http1_only,
+            http2_prior_knowledge: result.http2_prior_knowledge,
+            connect_timeout_ms: result.connect_timeout_ms,
+            first_body_hold_ms: result.first_body_hold_ms,
             duration_ms: result.duration_ms,
             throughput_rps: result.throughput_rps,
             p99_ms: result.p99_ms,
@@ -129,9 +235,16 @@ pub async fn run_http_load_probe(
             p95_ms: result.p95_ms,
             max_ms: result.max_ms,
             mean_ms: result.mean_ms,
+            headers_p50_ms: result.headers_p50_ms,
+            headers_p95_ms: result.headers_p95_ms,
+            headers_p99_ms: result.headers_p99_ms,
+            first_body_p50_ms: result.first_body_p50_ms,
+            first_body_p95_ms: result.first_body_p95_ms,
+            first_body_p99_ms: result.first_body_p99_ms,
             runtime: result.runtime,
             status_counts: result.status_counts,
             error_counts: result.error_counts,
+            error_samples: result.error_samples,
         })
 }
 
@@ -150,32 +263,38 @@ async fn run_http_load_probe_against_urls(
     config: &HttpLoadProbeConfig,
     urls: &[String],
 ) -> Result<MultiUrlHttpLoadProbeResult, String> {
-    let client = Client::builder()
-        .timeout(config.timeout)
-        .build()
-        .map_err(|err| format!("failed to build load probe http client: {err}"))?;
+    let clients = Arc::new(build_probe_clients(config)?);
     let total_requests = config.total_requests;
     let request_headers = build_headers(&config.headers)?;
     let request_body = config.body.clone().map(Arc::new);
     let response_mode = config.response_mode;
+    let first_body_hold = config.first_body_hold;
+    let start_ramp = config.start_ramp;
+    warmup_probe_connections(config, Arc::clone(&clients)).await?;
     let mut runtime_sampler = BenchmarkRuntimeSampler::new();
     let started_at = Instant::now();
 
     let next_request = Arc::new(AtomicUsize::new(0));
     let latencies_ms = Arc::new(Mutex::new(Vec::with_capacity(config.total_requests)));
+    let header_latencies_ms = Arc::new(Mutex::new(Vec::with_capacity(config.total_requests)));
+    let first_body_latencies_ms = Arc::new(Mutex::new(Vec::with_capacity(config.total_requests)));
     let status_counts = Arc::new(Mutex::new(BTreeMap::<u16, usize>::new()));
     let error_counts = Arc::new(Mutex::new(BTreeMap::<String, usize>::new()));
+    let error_samples = Arc::new(Mutex::new(Vec::<HttpLoadProbeErrorSample>::new()));
     let target_request_counts = Arc::new(Mutex::new(BTreeMap::<String, usize>::new()));
     let failed_requests = Arc::new(AtomicUsize::new(0));
     let completed_requests = Arc::new(AtomicUsize::new(0));
 
     let mut workers = tokio::task::JoinSet::new();
-    for _ in 0..config.concurrency {
-        let client = client.clone();
+    for worker_index in 0..config.concurrency {
+        let client = clients[worker_index % clients.len()].clone();
         let next_request = Arc::clone(&next_request);
         let latencies_ms = Arc::clone(&latencies_ms);
+        let header_latencies_ms = Arc::clone(&header_latencies_ms);
+        let first_body_latencies_ms = Arc::clone(&first_body_latencies_ms);
         let status_counts = Arc::clone(&status_counts);
         let error_counts = Arc::clone(&error_counts);
+        let error_samples = Arc::clone(&error_samples);
         let target_request_counts = Arc::clone(&target_request_counts);
         let failed_requests = Arc::clone(&failed_requests);
         let completed_requests = Arc::clone(&completed_requests);
@@ -183,8 +302,12 @@ async fn run_http_load_probe_against_urls(
         let urls = urls.to_vec();
         let request_headers = request_headers.clone();
         let request_body = request_body.clone();
+        let start_delay = worker_start_delay(start_ramp, worker_index, config.concurrency);
 
         workers.spawn(async move {
+            if !start_delay.is_zero() {
+                tokio::time::sleep(start_delay).await;
+            }
             loop {
                 let current = next_request.fetch_add(1, Ordering::AcqRel);
                 if current >= total_requests {
@@ -202,36 +325,62 @@ async fn run_http_load_probe_against_urls(
                 }
                 match request.send().await {
                     Ok(response) => {
+                        let headers_latency_ms = started_at.elapsed().as_millis() as u64;
                         let status = response.status().as_u16();
-                        let body_result = match response_mode {
-                            HttpLoadProbeResponseMode::HeadersOnly => Ok(()),
-                            HttpLoadProbeResponseMode::FullBody => response
-                                .bytes()
-                                .await
-                                .map(|_| ())
-                                .map_err(|err| classify_reqwest_error(&err)),
-                        };
-                        if let Err(error_kind) = body_result {
-                            failed_requests.fetch_add(1, Ordering::AcqRel);
-                            let mut counts = error_counts.lock().await;
-                            *counts.entry(error_kind).or_insert(0) += 1;
-                        } else {
-                            let mut counts = status_counts.lock().await;
-                            *counts.entry(status).or_insert(0) += 1;
-                            drop(counts);
-                            let mut target_counts = target_request_counts.lock().await;
-                            *target_counts.entry(url).or_insert(0) += 1;
+                        let body_result = observe_response_body(
+                            response,
+                            response_mode,
+                            started_at,
+                            first_body_hold,
+                        )
+                        .await;
+                        match body_result {
+                            Err(error) => {
+                                failed_requests.fetch_add(1, Ordering::AcqRel);
+                                record_load_error(
+                                    &error_counts,
+                                    &error_samples,
+                                    current,
+                                    &url,
+                                    started_at.elapsed().as_millis() as u64,
+                                    error,
+                                )
+                                .await;
+                            }
+                            Ok(observation) => {
+                                let mut counts = status_counts.lock().await;
+                                *counts.entry(status).or_insert(0) += 1;
+                                drop(counts);
+                                let mut target_counts = target_request_counts.lock().await;
+                                *target_counts.entry(url).or_insert(0) += 1;
+                                if let Some(first_body_latency_ms) =
+                                    observation.first_body_latency_ms
+                                {
+                                    first_body_latencies_ms
+                                        .lock()
+                                        .await
+                                        .push(first_body_latency_ms);
+                                }
+                            }
                         }
                         let latency_ms = started_at.elapsed().as_millis() as u64;
                         latencies_ms.lock().await.push(latency_ms);
+                        header_latencies_ms.lock().await.push(headers_latency_ms);
                         completed_requests.fetch_add(1, Ordering::AcqRel);
                     }
                     Err(err) => {
                         let latency_ms = started_at.elapsed().as_millis() as u64;
                         latencies_ms.lock().await.push(latency_ms);
                         failed_requests.fetch_add(1, Ordering::AcqRel);
-                        let mut counts = error_counts.lock().await;
-                        *counts.entry(classify_reqwest_error(&err)).or_insert(0) += 1;
+                        record_load_error(
+                            &error_counts,
+                            &error_samples,
+                            current,
+                            &url,
+                            latency_ms,
+                            classify_reqwest_error("send", &err),
+                        )
+                        .await;
                         completed_requests.fetch_add(1, Ordering::AcqRel);
                     }
                 }
@@ -245,10 +394,19 @@ async fn run_http_load_probe_against_urls(
 
     let status_counts = status_counts.lock().await.clone();
     let error_counts = error_counts.lock().await.clone();
+    let error_samples = error_samples.lock().await.clone();
     let target_request_counts = target_request_counts.lock().await.clone();
     let mut latencies = latencies_ms.lock().await.clone();
+    let mut header_latencies = header_latencies_ms.lock().await.clone();
+    let mut first_body_latencies = first_body_latencies_ms.lock().await.clone();
     latencies.sort_unstable();
+    header_latencies.sort_unstable();
+    first_body_latencies.sort_unstable();
     let (p50_ms, p95_ms, p99_ms, max_ms, mean_ms) = summarize_latencies(&latencies);
+    let (headers_p50_ms, headers_p95_ms, headers_p99_ms, _, _) =
+        summarize_latencies(&header_latencies);
+    let (first_body_p50_ms, first_body_p95_ms, first_body_p99_ms, _, _) =
+        summarize_latencies(&first_body_latencies);
     let duration_ms = started_at.elapsed().as_millis() as u64;
     let throughput_rps = if duration_ms == 0 {
         completed_requests.load(Ordering::Acquire) as u64
@@ -263,6 +421,17 @@ async fn run_http_load_probe_against_urls(
         response_mode: config.response_mode,
         total_requests: config.total_requests,
         concurrency: config.concurrency,
+        warmup_connections: config.warmup_connections,
+        warmup_url: config.warmup_url.clone(),
+        client_shards: config.client_shards,
+        start_ramp_ms: config.start_ramp.as_millis() as u64,
+        pool_max_idle_per_host: config.pool_max_idle_per_host,
+        http1_only: config.http1_only,
+        http2_prior_knowledge: config.http2_prior_knowledge,
+        connect_timeout_ms: config
+            .connect_timeout
+            .map(|timeout| timeout.as_millis() as u64),
+        first_body_hold_ms: config.first_body_hold.as_millis() as u64,
         duration_ms,
         throughput_rps,
         p99_ms,
@@ -272,32 +441,261 @@ async fn run_http_load_probe_against_urls(
         p95_ms,
         max_ms,
         mean_ms,
+        headers_p50_ms: (!header_latencies.is_empty()).then_some(headers_p50_ms),
+        headers_p95_ms: (!header_latencies.is_empty()).then_some(headers_p95_ms),
+        headers_p99_ms: (!header_latencies.is_empty()).then_some(headers_p99_ms),
+        first_body_p50_ms: (!first_body_latencies.is_empty()).then_some(first_body_p50_ms),
+        first_body_p95_ms: (!first_body_latencies.is_empty()).then_some(first_body_p95_ms),
+        first_body_p99_ms: (!first_body_latencies.is_empty()).then_some(first_body_p99_ms),
         runtime: runtime_sampler.snapshot(),
         status_counts,
         error_counts,
+        error_samples,
     })
 }
 
-fn classify_reqwest_error(err: &reqwest::Error) -> String {
-    if err.is_timeout() {
-        return "timeout".to_string();
+fn build_probe_clients(config: &HttpLoadProbeConfig) -> Result<Vec<Client>, String> {
+    let mut clients = Vec::with_capacity(config.client_shards);
+    for _ in 0..config.client_shards {
+        let mut builder = Client::builder().timeout(config.timeout);
+        if let Some(connect_timeout) = config.connect_timeout {
+            builder = builder.connect_timeout(connect_timeout);
+        }
+        if let Some(pool_max_idle_per_host) = config.pool_max_idle_per_host {
+            builder = builder.pool_max_idle_per_host(pool_max_idle_per_host);
+        }
+        if config.http1_only {
+            builder = builder.http1_only();
+        }
+        if config.http2_prior_knowledge {
+            builder = builder.http2_prior_knowledge();
+        }
+        clients.push(
+            builder
+                .build()
+                .map_err(|err| format!("failed to build load probe http client: {err}"))?,
+        );
     }
-    if err.is_connect() {
-        return "connect".to_string();
+    Ok(clients)
+}
+
+fn worker_start_delay(start_ramp: Duration, worker_index: usize, concurrency: usize) -> Duration {
+    if start_ramp.is_zero() || concurrency <= 1 || worker_index == 0 {
+        return Duration::ZERO;
     }
-    if err.is_body() {
-        return "body".to_string();
+    let ramp_nanos = start_ramp.as_nanos();
+    let offset_nanos = ramp_nanos
+        .saturating_mul(worker_index as u128)
+        .checked_div((concurrency - 1) as u128)
+        .unwrap_or_default();
+    Duration::from_nanos(offset_nanos.min(u64::MAX as u128) as u64)
+}
+
+async fn warmup_probe_connections(
+    config: &HttpLoadProbeConfig,
+    clients: Arc<Vec<Client>>,
+) -> Result<(), String> {
+    if config.warmup_connections == 0 {
+        return Ok(());
     }
-    if err.is_request() {
-        return "request".to_string();
+    let warmup_url = config.warmup_url.as_deref().unwrap_or(config.url.as_str());
+    let next_request = Arc::new(AtomicUsize::new(0));
+    let mut workers = tokio::task::JoinSet::new();
+    let concurrency = config.concurrency.min(config.warmup_connections).max(1);
+    for worker_index in 0..concurrency {
+        let client = clients[worker_index % clients.len()].clone();
+        let next_request = Arc::clone(&next_request);
+        let warmup_url = warmup_url.to_string();
+        let total = config.warmup_connections;
+        workers.spawn(async move {
+            loop {
+                let current = next_request.fetch_add(1, Ordering::AcqRel);
+                if current >= total {
+                    break;
+                }
+                let response = client
+                    .get(&warmup_url)
+                    .send()
+                    .await
+                    .map_err(|err| format!("warmup request failed: {err}"))?;
+                let response = response
+                    .error_for_status()
+                    .map_err(|err| format!("warmup request returned error status: {err}"))?;
+                response
+                    .bytes()
+                    .await
+                    .map_err(|err| format!("warmup response body failed: {err}"))?;
+            }
+            Ok::<(), String>(())
+        });
     }
-    if err.is_decode() {
-        return "decode".to_string();
+    while let Some(result) = workers.join_next().await {
+        result
+            .map_err(|err| format!("warmup worker task failed: {err}"))?
+            .map_err(|err| format!("failed to warm load probe connections: {err}"))?;
     }
-    if err.is_redirect() {
-        return "redirect".to_string();
+    Ok(())
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct ClassifiedLoadError {
+    key: String,
+    phase: String,
+    kind: String,
+    message: String,
+    source: Option<String>,
+}
+
+impl ClassifiedLoadError {
+    fn static_body(kind: &str, message: &str) -> Self {
+        Self {
+            key: format!("body:{kind}"),
+            phase: "body".to_string(),
+            kind: kind.to_string(),
+            message: message.to_string(),
+            source: None,
+        }
     }
-    "other".to_string()
+}
+
+async fn record_load_error(
+    error_counts: &Arc<Mutex<BTreeMap<String, usize>>>,
+    error_samples: &Arc<Mutex<Vec<HttpLoadProbeErrorSample>>>,
+    request_index: usize,
+    url: &str,
+    elapsed_ms: u64,
+    error: ClassifiedLoadError,
+) {
+    let mut counts = error_counts.lock().await;
+    *counts.entry(error.key).or_insert(0) += 1;
+    drop(counts);
+
+    let mut samples = error_samples.lock().await;
+    if samples.len() < MAX_ERROR_SAMPLES {
+        samples.push(HttpLoadProbeErrorSample {
+            request_index,
+            url: url.to_string(),
+            phase: error.phase,
+            kind: error.kind,
+            elapsed_ms,
+            message: error.message,
+            source: error.source,
+        });
+    }
+}
+
+fn classify_reqwest_error(phase: &str, err: &reqwest::Error) -> ClassifiedLoadError {
+    let kind = if err.is_timeout() && err.is_connect() {
+        "connect_timeout"
+    } else if err.is_timeout() && err.is_body() {
+        "body_timeout"
+    } else if err.is_timeout() {
+        "timeout"
+    } else if err.is_connect() {
+        "connect"
+    } else if err.is_body() {
+        "body"
+    } else if err.is_request() {
+        "request"
+    } else if err.is_decode() {
+        "decode"
+    } else if err.is_redirect() {
+        "redirect"
+    } else {
+        "other"
+    };
+
+    ClassifiedLoadError {
+        key: format!("{phase}:{kind}"),
+        phase: phase.to_string(),
+        kind: kind.to_string(),
+        message: compact_error_text(err.to_string(), 240),
+        source: error_source_chain(err, 240),
+    }
+}
+
+fn error_source_chain(err: &(dyn StdError + 'static), max_chars: usize) -> Option<String> {
+    let mut sources = Vec::new();
+    let mut next = err.source();
+    while let Some(source) = next {
+        sources.push(compact_error_text(source.to_string(), max_chars));
+        if sources.len() >= 4 {
+            break;
+        }
+        next = source.source();
+    }
+    (!sources.is_empty()).then(|| compact_error_text(sources.join(" | "), max_chars))
+}
+
+fn compact_error_text(value: impl AsRef<str>, max_chars: usize) -> String {
+    let mut compact = value
+        .as_ref()
+        .split_whitespace()
+        .collect::<Vec<_>>()
+        .join(" ");
+    if compact.chars().count() > max_chars {
+        compact = compact.chars().take(max_chars.saturating_sub(1)).collect();
+        compact.push_str("...");
+    }
+    compact
+}
+
+async fn observe_response_body(
+    mut response: reqwest::Response,
+    response_mode: HttpLoadProbeResponseMode,
+    started_at: Instant,
+    first_body_hold: Duration,
+) -> Result<BodyObservation, ClassifiedLoadError> {
+    match response_mode {
+        HttpLoadProbeResponseMode::HeadersOnly => Ok(BodyObservation::default()),
+        HttpLoadProbeResponseMode::FirstBodyByte => {
+            let first = response
+                .chunk()
+                .await
+                .map_err(|err| classify_reqwest_error("body", &err))?
+                .ok_or_else(|| {
+                    ClassifiedLoadError::static_body(
+                        "empty_body",
+                        "response body ended before the first chunk",
+                    )
+                })?;
+            let first_body_latency_ms = started_at.elapsed().as_millis() as u64;
+            drop(first);
+            if !first_body_hold.is_zero() {
+                tokio::time::sleep(first_body_hold).await;
+            }
+            Ok(BodyObservation {
+                first_body_latency_ms: Some(first_body_latency_ms),
+            })
+        }
+        HttpLoadProbeResponseMode::FullBody => {
+            let mut first_body_latency_ms = None;
+            while let Some(chunk) = response
+                .chunk()
+                .await
+                .map_err(|err| classify_reqwest_error("body", &err))?
+            {
+                if first_body_latency_ms.is_none() {
+                    first_body_latency_ms = Some(started_at.elapsed().as_millis() as u64);
+                }
+                drop(chunk);
+            }
+            if first_body_latency_ms.is_none() {
+                return Err(ClassifiedLoadError::static_body(
+                    "empty_body",
+                    "response body ended before the first chunk",
+                ));
+            }
+            Ok(BodyObservation {
+                first_body_latency_ms,
+            })
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+struct BodyObservation {
+    first_body_latency_ms: Option<u64>,
 }
 
 fn build_headers(headers: &BTreeMap<String, String>) -> Result<HeaderMap, String> {
@@ -337,7 +735,8 @@ fn percentile(latencies: &[u64], percentile: u8) -> u64 {
 #[cfg(test)]
 mod tests {
     use super::{
-        build_headers, summarize_latencies, HttpLoadProbeConfig, HttpLoadProbeResponseMode,
+        build_headers, summarize_latencies, worker_start_delay, HttpLoadProbeConfig,
+        HttpLoadProbeResponseMode,
     };
     use reqwest::Method;
     use std::collections::BTreeMap;
@@ -369,6 +768,25 @@ mod tests {
         }
         .validate()
         .is_err());
+        assert!(HttpLoadProbeConfig {
+            connect_timeout: Some(Duration::ZERO),
+            ..HttpLoadProbeConfig::default()
+        }
+        .validate()
+        .is_err());
+        assert!(HttpLoadProbeConfig {
+            client_shards: 0,
+            ..HttpLoadProbeConfig::default()
+        }
+        .validate()
+        .is_err());
+        assert!(HttpLoadProbeConfig {
+            http1_only: true,
+            http2_prior_knowledge: true,
+            ..HttpLoadProbeConfig::default()
+        }
+        .validate()
+        .is_err());
     }
 
     #[test]
@@ -386,12 +804,21 @@ mod tests {
     fn default_probe_config_is_reasonable() {
         let config = HttpLoadProbeConfig::default();
         assert_eq!(config.method, Method::GET);
+        assert!(config.warmup_url.is_none());
         assert!(config.headers.is_empty());
         assert!(config.body.is_none());
         assert_eq!(config.total_requests, 100);
         assert_eq!(config.concurrency, 10);
+        assert_eq!(config.warmup_connections, 0);
         assert_eq!(config.timeout, Duration::from_secs(30));
+        assert_eq!(config.connect_timeout, None);
         assert_eq!(config.response_mode, HttpLoadProbeResponseMode::HeadersOnly);
+        assert_eq!(config.client_shards, 1);
+        assert_eq!(config.pool_max_idle_per_host, None);
+        assert_eq!(config.start_ramp, Duration::ZERO);
+        assert!(!config.http1_only);
+        assert!(!config.http2_prior_knowledge);
+        assert_eq!(config.first_body_hold, Duration::ZERO);
     }
 
     #[test]
@@ -408,4 +835,24 @@ mod tests {
         let invalid = BTreeMap::from([("bad header".to_string(), "ok".to_string())]);
         assert!(build_headers(&invalid).is_err());
     }
+
+    #[test]
+    fn spreads_worker_start_delay_across_ramp() {
+        assert_eq!(
+            worker_start_delay(Duration::from_millis(900), 0, 10),
+            Duration::ZERO
+        );
+        assert_eq!(
+            worker_start_delay(Duration::from_millis(900), 5, 10),
+            Duration::from_millis(500)
+        );
+        assert_eq!(
+            worker_start_delay(Duration::from_millis(900), 9, 10),
+            Duration::from_millis(900)
+        );
+        assert_eq!(
+            worker_start_delay(Duration::from_millis(900), 3, 1),
+            Duration::ZERO
+        );
+    }
 }