fix: harden concurrency limits and high-RPM runtime paths

Bound request, stream, queue, and shutdown resource lifetimes. Reduce scheduler and Redis hot-path work and isolate database maintenance. Include regression coverage, load probes, and concurrency audit results.

crates/aether-testing/integration/Cargo.toml

@@ -13,6 +13,7 @@ aether-crypto.workspace = true
 aether-data.workspace = true
 aether-data-contracts.workspace = true
 aether-gateway = { workspace = true, features = ["testkit"] }
+aether-runtime.workspace = true
 aether-runtime-state.workspace = true
 aether-testkit = { workspace = true, features = ["gateway", "postgres"] }
 axum.workspace = true

crates/aether-testing/integration/src/bin/capacity_curve_baseline.rs

@@ -18,7 +18,7 @@ use axum::http::StatusCode;
 use axum::response::{IntoResponse, Response};
 use axum::routing::any;
 use axum::{extract::Request, Json, Router};
-use futures_util::{SinkExt, StreamExt};
+use futures_util::{stream::FuturesUnordered, SinkExt, StreamExt};
 use reqwest::Method;
 use serde::Serialize;
 use serde_json::json;
@@ -83,6 +83,8 @@ struct CapacityCurvePointResult {
     successful_requests: usize,
     rejected_requests: usize,
     failed_requests: usize,
+    status_counts: BTreeMap<u16, usize>,
+    non_success_status_samples: serde_json::Value,
     throughput_rps: u64,
     p50_ms: u64,
     p95_ms: u64,
@@ -112,8 +114,16 @@ struct GateMetricSnapshot {
     rejected_total: u64,
 }
 
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let _log_shutdown = aether_runtime::LogShutdownGuard::new();
+    let runtime = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .thread_stack_size(8 * 1024 * 1024)
+        .build()?;
+    runtime.block_on(run())
+}
+
+async fn run() -> Result<(), Box<dyn std::error::Error>> {
     init_test_runtime_for("capacity-curve-baseline");
     let config = parse_args(std::env::args().skip(1).collect())?;
     let report = run_suite(&config).await?;
@@ -215,11 +225,11 @@ async fn run_gateway_curve(
             .await
             .map_err(std::io::Error::other)?;
         let duration_ms = started_at.elapsed().as_millis() as u64;
-        let metrics = capture_gate_metrics(
-            &format!("{}/_gateway/metrics", gateway.base_url()),
-            gate_name,
-        )
-        .await?;
+        let samples = gateway
+            .metric_samples()
+            .await
+            .map_err(std::io::Error::other)?;
+        let metrics = gate_metrics(&samples, gate_name)?;
         points.push(capacity_point(
             *limit,
             total_requests,
@@ -319,6 +329,21 @@ async fn run_tunnel_curve(
         let peer = connect_protocol_peer(tunnel.base_url(), config.tunnel_hold).await?;
         let total_requests =
             total_requests_for_limit(relay_concurrency, config.requests_per_point_multiplier);
+        let envelope = relay_envelope();
+        let body_offset =
+            4 + u32::from_be_bytes(envelope[..4].try_into().expect("metadata length")) as usize;
+        verify_tunnel_fixture(&tunnel, &envelope, body_offset, config.timeout).await?;
+        let header_sets = (0..total_requests)
+            .map(|_| {
+                let mut headers =
+                    tunnel.relay_headers(&envelope[..body_offset], &envelope[body_offset..]);
+                headers.insert(
+                    "content-type".to_string(),
+                    "application/octet-stream".to_string(),
+                );
+                headers
+            })
+            .collect();
         let probe = HttpLoadProbeConfig {
             url: format!(
                 "{tunnel_base}{TUNNEL_RELAY_PATH_PREFIX}/node-baseline",
@@ -329,7 +354,8 @@ async fn run_tunnel_curve(
                 "content-type".to_string(),
                 "application/octet-stream".to_string(),
             )]),
-            body: Some(relay_envelope()),
+            header_sets,
+            body: Some(envelope),
             total_requests,
             concurrency: relay_concurrency,
             timeout: config.timeout,
@@ -350,7 +376,8 @@ async fn run_tunnel_curve(
             result,
             metrics,
         ));
-        drop(peer);
+        peer.abort();
+        let _ = peer.await;
     }
 
     Ok(CapacityCurveScenarioReport {
@@ -362,6 +389,58 @@ async fn run_tunnel_curve(
     })
 }
 
+async fn verify_tunnel_fixture(
+    tunnel: &TunnelHarness,
+    envelope: &[u8],
+    body_offset: usize,
+    timeout: Duration,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let client = reqwest::Client::builder().timeout(timeout).build()?;
+    let url = format!(
+        "{}{TUNNEL_RELAY_PATH_PREFIX}/{TUNNEL_HARNESS_NODE_ID}",
+        tunnel.base_url()
+    );
+    let unsigned = client.post(&url).body(envelope.to_vec()).send().await?;
+    if unsigned.status() != StatusCode::FORBIDDEN {
+        return Err(std::io::Error::other("unsigned tunnel preflight was not rejected").into());
+    }
+    let mut signed = client.post(&url).body(envelope.to_vec());
+    for (name, value) in tunnel.relay_headers(&envelope[..body_offset], &envelope[body_offset..]) {
+        signed = signed.header(name, value);
+    }
+    let signed = signed.build()?;
+    let mut tampered = signed
+        .try_clone()
+        .expect("buffered relay request should clone");
+    let mut tampered_body = envelope.to_vec();
+    *tampered_body
+        .last_mut()
+        .expect("relay body should be nonempty") ^= 1;
+    *tampered.body_mut() = Some(tampered_body.into());
+    if client.execute(tampered).await?.status() != StatusCode::FORBIDDEN {
+        return Err(std::io::Error::other("tampered tunnel preflight was not rejected").into());
+    }
+    let response = client
+        .execute(
+            signed
+                .try_clone()
+                .expect("buffered relay request should clone"),
+        )
+        .await?;
+    let status = response.status();
+    let body = response.text().await?;
+    if status != StatusCode::OK || body != "capacity-tunnel-stream" {
+        return Err(std::io::Error::other(format!(
+            "signed tunnel preflight failed: {status}: {body}"
+        ))
+        .into());
+    }
+    if client.execute(signed).await?.status() != StatusCode::FORBIDDEN {
+        return Err(std::io::Error::other("replayed tunnel preflight was not rejected").into());
+    }
+    Ok(())
+}
+
 fn capacity_point(
     limit: usize,
     total_requests: usize,
@@ -395,7 +474,10 @@ fn capacity_point(
         duration_ms,
         successful_requests,
         rejected_requests,
-        failed_requests: result.failed_requests,
+        failed_requests: total_requests.saturating_sub(successful_requests + rejected_requests),
+        status_counts: result.status_counts,
+        non_success_status_samples: serde_json::to_value(result.non_success_status_samples)
+            .expect("HTTP status samples should serialize"),
         throughput_rps,
         p50_ms: result.p50_ms,
         p95_ms: result.p95_ms,
@@ -440,27 +522,29 @@ async fn capture_gate_metrics(
     let samples = fetch_prometheus_samples(metrics_url)
         .await
         .map_err(std::io::Error::other)?;
+    gate_metrics(&samples, gate_name)
+}
+
+fn gate_metrics(
+    samples: &[aether_testkit::PrometheusSample],
+    gate_name: &str,
+) -> Result<GateMetricSnapshot, Box<dyn std::error::Error>> {
+    let required = |name| {
+        find_metric_value_u64(samples, name, &[("gate", gate_name)])
+            .or_else(|| {
+                find_metric_value_u64(
+                    samples,
+                    &format!("aether_testkit_{name}"),
+                    &[("gate", gate_name)],
+                )
+            })
+            .ok_or_else(|| std::io::Error::other(format!("missing {name} for gate {gate_name}")))
+    };
     Ok(GateMetricSnapshot {
-        in_flight: find_metric_value_u64(&samples, "concurrency_in_flight", &[("gate", gate_name)])
-            .unwrap_or_default(),
-        available_permits: find_metric_value_u64(
-            &samples,
-            "concurrency_available_permits",
-            &[("gate", gate_name)],
-        )
-        .unwrap_or_default(),
-        high_watermark: find_metric_value_u64(
-            &samples,
-            "concurrency_high_watermark",
-            &[("gate", gate_name)],
-        )
-        .unwrap_or_default(),
-        rejected_total: find_metric_value_u64(
-            &samples,
-            "concurrency_rejected_total",
-            &[("gate", gate_name)],
-        )
-        .unwrap_or_default(),
+        in_flight: required("concurrency_in_flight")?,
+        available_permits: required("concurrency_available_permits")?,
+        high_watermark: required("concurrency_high_watermark")?,
+        rejected_total: required("concurrency_rejected_total")?,
     })
 }
 
@@ -700,25 +784,35 @@ async fn connect_protocol_peer(
     ))
     .await?;
     Ok(tokio::spawn(async move {
-        while let Some(message) = stream.next().await {
-            let Ok(message) = message else {
-                break;
-            };
-            match message {
-                Message::Binary(data)
-                    if handle_binary_frame(&mut sink, data.to_vec(), hold)
-                        .await
-                        .is_err() =>
-                {
-                    break;
+        let mut responses = FuturesUnordered::new();
+        loop {
+            tokio::select! {
+                message = stream.next() => {
+                    match message {
+                        Some(Ok(Message::Binary(data))) => {
+                            match handle_binary_frame(&mut sink, data.to_vec()).await {
+                                Ok(Some(stream_id)) => responses.push(async move {
+                                    tokio::time::sleep(hold).await;
+                                    stream_id
+                                }),
+                                Ok(None) => {},
+                                Err(_) => break,
+                            }
+                        }
+                        Some(Ok(Message::Ping(payload))) => {
+                            if sink.send(Message::Pong(payload)).await.is_err() {
+                                break;
+                            }
+                        }
+                        None | Some(Err(_)) | Some(Ok(Message::Close(_))) => break,
+                        _ => {},
+                    }
                 }
-                Message::Ping(payload)
-                    if sink.send(Message::Pong(payload.clone())).await.is_err() =>
-                {
-                    break;
+                Some(stream_id) = responses.next(), if !responses.is_empty() => {
+                    if send_protocol_response(&mut sink, stream_id).await.is_err() {
+                        break;
+                    }
                 }
-                Message::Close(_) => break,
-                _ => {}
             }
         }
         let _ = sink.close().await;
@@ -728,13 +822,12 @@ async fn connect_protocol_peer(
 async fn handle_binary_frame<S>(
     sink: &mut S,
     data: Vec<u8>,
-    hold: Duration,
-) -> Result<(), tokio_tungstenite::tungstenite::Error>
+) -> Result<Option<u32>, tokio_tungstenite::tungstenite::Error>
 where
     S: SinkExt<Message, Error = tokio_tungstenite::tungstenite::Error> + Unpin,
 {
     let Some(header) = protocol::FrameHeader::parse(&data) else {
-        return Ok(());
+        return Ok(None);
     };
     match header.msg_type {
         protocol::PING => {
@@ -755,48 +848,57 @@ where
                 .await?;
             }
             if header.flags & protocol::FLAG_END_STREAM == 0 {
-                return Ok(());
+                return Ok(None);
             }
-            tokio::time::sleep(hold).await;
-            let response_meta = protocol::ResponseMeta {
-                status: 200,
-                headers: vec![(
-                    "content-type".to_string(),
-                    "text/plain; charset=utf-8".to_string(),
-                )],
-            };
-            let response_meta_json =
-                serde_json::to_vec(&response_meta).expect("response metadata should serialize");
-            sink.send(Message::Binary(
-                protocol::encode_frame(
-                    header.stream_id,
-                    protocol::RESPONSE_HEADERS,
-                    0,
-                    &response_meta_json,
-                )
-                .into(),
-            ))
-            .await?;
-
-            for chunk in [
-                b"capacity-".as_slice(),
-                b"tunnel-".as_slice(),
-                b"stream".as_slice(),
-            ] {
-                sink.send(Message::Binary(
-                    protocol::encode_frame(header.stream_id, protocol::RESPONSE_BODY, 0, chunk)
-                        .into(),
-                ))
-                .await?;
-            }
-
-            sink.send(Message::Binary(
-                protocol::encode_frame(header.stream_id, protocol::STREAM_END, 0, &[]).into(),
-            ))
-            .await?;
+            return Ok(Some(header.stream_id));
         }
         _ => {}
     }
+    Ok(None)
+}
+
+async fn send_protocol_response<S>(
+    sink: &mut S,
+    stream_id: u32,
+) -> Result<(), tokio_tungstenite::tungstenite::Error>
+where
+    S: SinkExt<Message, Error = tokio_tungstenite::tungstenite::Error> + Unpin,
+{
+    let response_meta = protocol::ResponseMeta {
+        status: 200,
+        headers: vec![(
+            "content-type".to_string(),
+            "text/plain; charset=utf-8".to_string(),
+        )],
+    };
+    let response_meta_json =
+        serde_json::to_vec(&response_meta).expect("response metadata should serialize");
+    sink.send(Message::Binary(
+        protocol::encode_frame(
+            stream_id,
+            protocol::RESPONSE_HEADERS,
+            0,
+            &response_meta_json,
+        )
+        .into(),
+    ))
+    .await?;
+
+    for chunk in [
+        b"capacity-".as_slice(),
+        b"tunnel-".as_slice(),
+        b"stream".as_slice(),
+    ] {
+        sink.send(Message::Binary(
+            protocol::encode_frame(stream_id, protocol::RESPONSE_BODY, 0, chunk).into(),
+        ))
+        .await?;
+    }
+
+    sink.send(Message::Binary(
+        protocol::encode_frame(stream_id, protocol::STREAM_END, 0, &[]).into(),
+    ))
+    .await?;
     Ok(())
 }
 

crates/aether-testing/integration/src/bin/dependency_pressure_baseline.rs

@@ -141,8 +141,13 @@ impl SummaryCollector {
     }
 }
 
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let _log_shutdown = aether_runtime::LogShutdownGuard::new();
+    run()
+}
+
 #[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
+async fn run() -> Result<(), Box<dyn std::error::Error>> {
     init_test_runtime_for("dependency-pressure-baseline");
     let config = parse_args(std::env::args().skip(1).collect())?;
     let report = run_suite(&config).await?;

crates/aether-testing/integration/src/bin/failure_recovery_baseline.rs

@@ -172,8 +172,13 @@ impl RecoveryCollector {
     }
 }
 
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let _log_shutdown = aether_runtime::LogShutdownGuard::new();
+    run()
+}
+
 #[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
+async fn run() -> Result<(), Box<dyn std::error::Error>> {
     init_test_runtime_for("failure-recovery-baseline");
     let config = parse_args(std::env::args().skip(1).collect())?;
     let report = run_suite(&config).await?;

crates/aether-testing/integration/src/bin/gateway_pressure_seed.rs

@@ -7,6 +7,7 @@ use std::io;
 use std::io::Write;
 use std::path::{Path, PathBuf};
 
+use aether_crypto::PythonFernetCompat;
 use aether_data::repository::auth::CreateStandaloneApiKeyRecord;
 use aether_data::repository::wallet::WalletLookupKey;
 use aether_data::{
@@ -16,6 +17,7 @@ use aether_data_contracts::repository::global_models::{
     CreateAdminGlobalModelRecord, UpdateAdminGlobalModelRecord, UpsertAdminProviderModelRecord,
 };
 use aether_data_contracts::repository::provider_catalog::{
+    ProviderCatalogKeyAdminCasUpdate, ProviderCatalogKeyOAuthCredentialFence,
     StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
 };
 use serde_json::json;
@@ -200,6 +202,12 @@ impl Config {
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let config = Config::from_env_and_args().map_err(|err| format!("invalid config: {err}"))?;
+    let encryption_key = env_value("AETHER_GATEWAY_DATA_ENCRYPTION_KEY")
+        .or_else(|| env_value("ENCRYPTION_KEY"))
+        .ok_or(
+            "set AETHER_GATEWAY_DATA_ENCRYPTION_KEY or ENCRYPTION_KEY to the gateway's encryption key before seeding",
+        )?;
+    let secret_cipher = PythonFernetCompat::from_secret(&encryption_key);
 
     let backends = DataBackends::from_config(DataLayerConfig::from_database(SqlDatabaseConfig {
         driver: DatabaseDriver::Postgres,
@@ -215,10 +223,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         },
     }))?;
 
-    seed_provider_catalog(&backends, &config).await?;
+    seed_provider_catalog(&backends, &config, &secret_cipher).await?;
     seed_models(&backends, &config).await?;
     let operator_user_id = seed_operator_user(&backends, &config).await?;
-    seed_api_keys(&backends, &config, &operator_user_id).await?;
+    seed_api_keys(&backends, &config, &operator_user_id, &secret_cipher).await?;
     verify_candidate_selection(&backends, &config).await?;
     write_outputs(&config)?;
 
@@ -243,6 +251,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 async fn seed_provider_catalog(
     backends: &DataBackends,
     config: &Config,
+    secret_cipher: &PythonFernetCompat,
 ) -> Result<(), Box<dyn std::error::Error>> {
     let reader = backends
         .read()
@@ -326,7 +335,7 @@ async fn seed_provider_catalog(
     )?
     .with_transport_fields(
         Some(json!(["openai:chat"])),
-        Some(config.provider_api_key.clone()),
+        Some(secret_cipher.encrypt_plaintext(&config.provider_api_key)?),
         None,
         None,
         None,
@@ -341,17 +350,40 @@ async fn seed_provider_catalog(
         Some(json!({"openai:chat": {"state": "closed"}})),
     );
 
-    if reader
-        .list_keys_by_ids(std::slice::from_ref(&config.provider_key_id))
-        .await?
-        .is_empty()
-    {
-        writer.create_key(&provider_key).await?;
-    } else {
-        writer.update_key(&provider_key).await?;
+    for _ in 0..8 {
+        let existing = reader
+            .list_keys_by_ids(std::slice::from_ref(&config.provider_key_id))
+            .await?
+            .into_iter()
+            .next();
+        let Some(existing) = existing else {
+            writer.create_key(&provider_key).await?;
+            return Ok(());
+        };
+        if existing.provider_id != config.provider_id {
+            return Err("existing pressure provider key belongs to a different provider".into());
+        }
+
+        // Randomized ciphertext changes on every seed. Fence against the observed
+        // credential and preserve runtime fields when rotating the configured key.
+        let update = ProviderCatalogKeyAdminCasUpdate {
+            expected_encrypted_auth_config: existing.encrypted_auth_config.clone(),
+            expected_credential: ProviderCatalogKeyOAuthCredentialFence {
+                encrypted_api_key: existing.encrypted_api_key,
+                auth_type: existing.auth_type,
+                provider_id: existing.provider_id,
+                provider_type: provider.provider_type.clone(),
+            },
+            key: provider_key.clone(),
+            codex_rotation: None,
+            reset_oauth_runtime: true,
+        };
+        if writer.compare_and_update_key_admin_state(&update).await? {
+            return Ok(());
+        }
     }
 
-    Ok(())
+    Err("pressure provider key changed repeatedly during seed; retry initialization".into())
 }
 
 fn pressure_provider_transport_config(mock_upstream_h2c: bool) -> Option<serde_json::Value> {
@@ -466,9 +498,10 @@ async fn seed_api_keys(
     backends: &DataBackends,
     config: &Config,
     operator_user_id: &str,
+    secret_cipher: &PythonFernetCompat,
 ) -> Result<(), Box<dyn std::error::Error>> {
     for index in 0..config.api_key_count {
-        seed_api_key(backends, config, operator_user_id, index).await?;
+        seed_api_key(backends, config, operator_user_id, index, secret_cipher).await?;
     }
     Ok(())
 }
@@ -478,6 +511,7 @@ async fn seed_api_key(
     config: &Config,
     operator_user_id: &str,
     key_index: usize,
+    secret_cipher: &PythonFernetCompat,
 ) -> Result<(), Box<dyn std::error::Error>> {
     let auth_reader = backends
         .read()
@@ -494,17 +528,28 @@ async fn seed_api_key(
 
     let api_key_id = pressure_api_key_id(config, key_index);
     let api_key_value = pressure_api_key_value(config, key_index);
+    let key_hash = sha256_hex(&api_key_value);
+    let key_encrypted = secret_cipher.encrypt_plaintext(&api_key_value)?;
 
     let existing = auth_reader
         .find_export_standalone_api_key_by_id(&api_key_id)
         .await?;
+    if existing
+        .as_ref()
+        .is_some_and(|record| record.key_hash != key_hash)
+    {
+        return Err(format!(
+            "existing pressure API key {api_key_id} has a different hash; use its original value or a new --api-key-id"
+        )
+        .into());
+    }
     if existing.is_none() {
         auth_writer
             .create_standalone_api_key(CreateStandaloneApiKeyRecord {
                 user_id: operator_user_id.to_string(),
                 api_key_id: api_key_id.clone(),
-                key_hash: sha256_hex(&api_key_value),
-                key_encrypted: Some(api_key_value),
+                key_hash,
+                key_encrypted: Some(key_encrypted),
                 name: Some(format!("Local pressure API key {}", key_index + 1)),
                 allowed_providers: Some(vec![config.provider_id.clone()]),
                 allowed_api_formats: Some(vec!["openai:chat".to_string()]),
@@ -526,8 +571,8 @@ async fn seed_api_key(
             .update_standalone_api_key_basic(
                 aether_data::repository::auth::UpdateStandaloneApiKeyBasicRecord {
                     api_key_id: api_key_id.clone(),
-                    key_encrypted: None,
-                    key_encrypted_present: false,
+                    key_encrypted: Some(key_encrypted),
+                    key_encrypted_present: true,
                     name: Some(format!("Local pressure API key {}", key_index + 1)),
                     name_present: true,
                     force_capabilities: None,
@@ -867,6 +912,8 @@ fn print_help() {
     println!(
         "Usage: cargo run -p aether-integration-tests --bin gateway_pressure_seed -- [options]\n\
 \n\
+The seed and gateway must share AETHER_GATEWAY_DATA_ENCRYPTION_KEY (or ENCRYPTION_KEY).\n\
+\n\
 Options:\n\
   --database-url URL\n\
   --output-env PATH\n\

crates/aether-testing/integration/src/bin/gateway_tunnel_stream_baseline.rs

@@ -104,8 +104,13 @@ struct AcceptanceReport {
     reasons: Vec<String>,
 }
 
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let _log_shutdown = aether_runtime::LogShutdownGuard::new();
+    run()
+}
+
 #[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
+async fn run() -> Result<(), Box<dyn std::error::Error>> {
     init_test_runtime_for("gateway-tunnel-stream-baseline");
     let config = parse_args(std::env::args().skip(1).collect())?;
     let report = run_suite(&config).await?;

crates/aether-testing/integration/src/bin/llm_stream_stability_baseline.rs

@@ -318,8 +318,13 @@ struct ProtocolPeer {
     stats: Arc<PeerStats>,
 }
 
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let _log_shutdown = aether_runtime::LogShutdownGuard::new();
+    run()
+}
+
 #[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
+async fn run() -> Result<(), Box<dyn std::error::Error>> {
     init_test_runtime_for("llm-stream-stability-baseline");
     let config = parse_args(std::env::args().skip(1).collect())?;
     config.validate().map_err(std::io::Error::other)?;

crates/aether-testing/integration/src/bin/mock_openai_upstream.rs

@@ -441,6 +441,7 @@ async fn chat_completions(State(app): State<App>, request: axum::extract::Reques
             completion
                 .take()
                 .expect("request completion guard should be present"),
+            false,
         );
     }
 
@@ -457,6 +458,14 @@ async fn chat_completions(State(app): State<App>, request: axum::extract::Reques
     };
     let stream = request_wants_stream(&body);
     if stream {
+        let include_usage = serde_json::from_slice::<serde_json::Value>(&body)
+            .ok()
+            .and_then(|value| {
+                value
+                    .pointer("/stream_options/include_usage")
+                    .and_then(serde_json::Value::as_bool)
+            })
+            .unwrap_or(false);
         record_response_header_created(&app, request_started.started_at.elapsed());
         return build_chat_sse_response(
             app,
@@ -464,6 +473,7 @@ async fn chat_completions(State(app): State<App>, request: axum::extract::Reques
             completion
                 .take()
                 .expect("request completion guard should be present"),
+            include_usage,
         );
     }
     // A stream truncation profile only applies after the request is known to be streaming.
@@ -609,6 +619,7 @@ fn build_chat_sse_response(
     app: App,
     profile: RequestProfile,
     completion: RequestCompletionGuard,
+    include_usage: bool,
 ) -> Response {
     let response_created_at = Instant::now();
     let config = app.config.clone();
@@ -659,6 +670,21 @@ fn build_chat_sse_response(
         yield Ok::<Bytes, std::io::Error>(Bytes::from(
             "data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n",
         ));
+        if include_usage {
+            let payload = json!({
+                "id": "chatcmpl-mock",
+                "object": "chat.completion.chunk",
+                "created": current_unix_secs(),
+                "model": "mock-model",
+                "choices": [],
+                "usage": {
+                    "prompt_tokens": 1,
+                    "completion_tokens": config.chunks.max(1),
+                    "total_tokens": config.chunks.max(1) + 1
+                }
+            });
+            yield Ok::<Bytes, std::io::Error>(Bytes::from(format!("data: {payload}\n\n")));
+        }
         yield Ok::<Bytes, std::io::Error>(Bytes::from("data: [DONE]\n\n"));
         if let Some(completion) = completion.take() {
             completion.complete();
@@ -1465,7 +1491,11 @@ mod tests {
     ) {
         let response = client
             .post(url)
-            .json(&json!({"stream": true, "model": "mock-test"}))
+            .json(&json!({
+                "stream": true,
+                "model": "mock-test",
+                "stream_options": {"include_usage": true}
+            }))
             .send()
             .await
             .expect("headers should arrive before the body error");
@@ -1505,6 +1535,78 @@ mod tests {
             !body.contains("[DONE]"),
             "truncated stream must not emit [DONE]"
         );
+        assert!(
+            !body.contains("\"usage\""),
+            "truncated stream must not emit terminal usage"
+        );
+    }
+
+    #[tokio::test]
+    async fn chat_stream_usage_is_opt_in_and_precedes_done() {
+        for chunks in [0, 3] {
+            for (include_usage, assume_stream) in [
+                (None, false),
+                (Some(false), false),
+                (Some(true), false),
+                (Some(true), true),
+            ] {
+                let config = Config {
+                    chunks,
+                    chunk_delay: Duration::ZERO,
+                    assume_stream,
+                    ..Default::default()
+                };
+                let app = App {
+                    metrics: Arc::new(Metrics::for_binds(&config.binds)),
+                    bind_label: Arc::from(config.binds[0].to_string()),
+                    config,
+                };
+                let mut payload = json!({"stream": true, "model": "mock-test"});
+                if let Some(include_usage) = include_usage {
+                    payload["stream_options"] = json!({"include_usage": include_usage});
+                }
+                let request = axum::http::Request::builder()
+                    .body(Body::from(payload.to_string()))
+                    .unwrap();
+                let response = chat_completions(State(app), request).await;
+                assert_eq!(response.status(), StatusCode::OK);
+                let body = to_bytes(response.into_body(), 16 * 1024).await.unwrap();
+                let body = std::str::from_utf8(&body).unwrap();
+                let frames = body
+                    .split("\n\n")
+                    .filter_map(|frame| frame.strip_prefix("data: "))
+                    .collect::<Vec<_>>();
+                assert_eq!(frames.last(), Some(&"[DONE]"));
+                let payloads = frames[..frames.len() - 1]
+                    .iter()
+                    .map(|frame| serde_json::from_str::<serde_json::Value>(frame).unwrap())
+                    .collect::<Vec<_>>();
+                let usage_chunks = payloads
+                    .iter()
+                    .filter(|payload| payload.get("usage").is_some())
+                    .collect::<Vec<_>>();
+                if include_usage == Some(true) && !assume_stream {
+                    assert_eq!(usage_chunks.len(), 1);
+                    assert_eq!(payloads.last(), Some(usage_chunks[0]));
+                    assert_eq!(usage_chunks[0]["object"], "chat.completion.chunk");
+                    assert_eq!(usage_chunks[0]["choices"], json!([]));
+                    assert_eq!(
+                        usage_chunks[0]["usage"],
+                        json!({
+                            "prompt_tokens": 1,
+                            "completion_tokens": chunks.max(1),
+                            "total_tokens": chunks.max(1) + 1
+                        })
+                    );
+                    assert_eq!(
+                        payloads[payloads.len() - 2]["choices"][0]["finish_reason"],
+                        "stop"
+                    );
+                } else {
+                    assert!(usage_chunks.is_empty());
+                }
+            }
+        }
     }
 
     async fn wait_for_completed(metrics: &Metrics, expected: u64) {

crates/aether-testing/integration/src/bin/multi_instance_admission_baseline.rs

@@ -94,8 +94,13 @@ struct WebSocketAdmissionProbeResult {
     runtime: BenchmarkRuntimeSnapshot,
 }
 
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let _log_shutdown = aether_runtime::LogShutdownGuard::new();
+    run()
+}
+
 #[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
+async fn run() -> Result<(), Box<dyn std::error::Error>> {
     init_test_runtime_for("multi-instance-admission-baseline");
     let config = parse_args(std::env::args().skip(1).collect())?;
     let report = run_suite(&config).await?;

crates/aether-testing/integration/src/bin/multi_instance_owner_relay_baseline.rs

@@ -66,8 +66,13 @@ struct RelayOverheadSnapshot {
     mean_delta_ms: i64,
 }
 
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let _log_shutdown = aether_runtime::LogShutdownGuard::new();
+    run()
+}
+
 #[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
+async fn run() -> Result<(), Box<dyn std::error::Error>> {
     init_test_runtime_for("multi-instance-owner-relay-baseline");
     let config = parse_args(std::env::args().skip(1).collect())?;
     let report = run_suite(&config).await?;

crates/aether-testing/integration/src/bin/single_instance_baseline.rs

@@ -54,8 +54,13 @@ struct SingleInstanceBaselineReport {
     scenarios: Vec<NamedBaselineResult>,
 }
 
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let _log_shutdown = aether_runtime::LogShutdownGuard::new();
+    run()
+}
+
 #[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
+async fn run() -> Result<(), Box<dyn std::error::Error>> {
     init_test_runtime_for("single-instance-baseline");
     let config = parse_args(std::env::args().skip(1).collect())?;
     let report = run_suite(&config).await?;

crates/aether-testing/integration/src/bin/usage_aux_counter_hotspot_baseline.rs

@@ -123,8 +123,13 @@ struct LockSample {
     oldest_lock_wait_ms: i64,
 }
 
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let _log_shutdown = aether_runtime::LogShutdownGuard::new();
+    run()
+}
+
 #[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
+async fn run() -> Result<(), Box<dyn std::error::Error>> {
     init_test_runtime_for("usage-aux-counter-hotspot-baseline");
     let config = parse_args(std::env::args().skip(1).collect())?;
 

crates/aether-testing/integration/src/bin/usage_counter_hotspot_baseline.rs

@@ -116,8 +116,13 @@ struct LockSample {
     oldest_lock_wait_ms: i64,
 }
 
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let _log_shutdown = aether_runtime::LogShutdownGuard::new();
+    run()
+}
+
 #[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
+async fn run() -> Result<(), Box<dyn std::error::Error>> {
     init_test_runtime_for("usage-counter-hotspot-baseline");
     let config = parse_args(std::env::args().skip(1).collect())?;
 
@@ -453,6 +458,7 @@ fn usage_record(index: usize) -> UpsertUsageRecord {
     let now_ms = now_unix_ms().saturating_add(index as u64);
     let now_secs = now_ms / 1_000;
     UpsertUsageRecord {
+        capture_retention: Default::default(),
         request_id: format!("usage-hotspot-{index:08}"),
         user_id: Some("user-hotspot".to_string()),
         api_key_id: Some("api-key-hotspot".to_string()),

crates/aether-testing/integration/src/bin/usage_settlement_hotspot_baseline.rs

@@ -18,6 +18,8 @@ use sqlx::{PgPool, Row};
 use tokio::sync::Mutex;
 
 const PROVIDER_ID: &str = "provider-hotspot";
+const USER_ID: &str = "settlement-hotspot-user";
+const WALLET_ID: &str = "settlement-hotspot-wallet";
 const REQUEST_PREFIX: &str = "settlement-hotspot";
 const COST_PER_REQUEST_USD: f64 = 0.001;
 
@@ -99,6 +101,7 @@ struct CounterReport {
     provider_monthly_outbox_rows: i64,
     provider_monthly_used_usd: f64,
     expected_provider_monthly_used_usd: f64,
+    wallet_consumed_usd: f64,
 }
 
 #[derive(Debug, Serialize, Clone, Copy, Default)]
@@ -120,8 +123,13 @@ struct LockSample {
     oldest_lock_wait_ms: i64,
 }
 
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let _log_shutdown = aether_runtime::LogShutdownGuard::new();
+    run()
+}
+
 #[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
+async fn run() -> Result<(), Box<dyn std::error::Error>> {
     init_test_runtime_for("usage-settlement-hotspot-baseline");
     let config = parse_args(std::env::args().skip(1).collect())?;
 
@@ -239,6 +247,24 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         }
         std::fs::write(path, format!("{raw}\n"))?;
     }
+    if report.failed_requests != 0
+        || report.counters.settled_usage_rows != config.requests as i64
+        || report.counters.settlement_snapshot_rows != config.requests as i64
+        || report.counters.outbox_pending_rows != 0
+        || (report.counters.provider_monthly_used_usd
+            - report.counters.expected_provider_monthly_used_usd)
+            .abs()
+            > 1e-8
+        || (report.counters.wallet_consumed_usd
+            - report.counters.expected_provider_monthly_used_usd)
+            .abs()
+            > 1e-8
+    {
+        return Err(std::io::Error::other(
+            "settlement baseline failed correctness checks; see report",
+        )
+        .into());
+    }
     Ok(())
 }
 
@@ -388,6 +414,24 @@ async fn wait_for_outbox_drain(
 }
 
 async fn seed_settlement_rows(pool: &PgPool, requests: usize) -> Result<(), sqlx::Error> {
+    sqlx::query(
+        "INSERT INTO users (id, username, email_verified) VALUES ($1, $1, true) ON CONFLICT (id) DO NOTHING",
+    )
+    .bind(USER_ID)
+    .execute(pool)
+    .await?;
+    sqlx::query(
+        r#"
+INSERT INTO wallets (id, user_id, balance, gift_balance, total_consumed, created_at, updated_at)
+VALUES ($1, $2, $3, 0, 0, NOW(), NOW())
+ON CONFLICT (id) DO UPDATE SET balance = EXCLUDED.balance, gift_balance = 0, total_consumed = 0
+"#,
+    )
+    .bind(WALLET_ID)
+    .bind(USER_ID)
+    .bind(requests as f64 * COST_PER_REQUEST_USD + 1.0)
+    .execute(pool)
+    .await?;
     sqlx::query(
         r#"
 INSERT INTO providers (id, name, provider_type, monthly_used_usd)
@@ -432,6 +476,7 @@ WHERE request_id LIKE $1
 INSERT INTO "usage" (
   id,
   request_id,
+  user_id,
   provider_name,
   model,
   provider_id,
@@ -445,6 +490,7 @@ INSERT INTO "usage" (
 SELECT
   'settlement-usage-' || LPAD(gs::TEXT, 8, '0'),
   $2 || '-' || LPAD(gs::TEXT, 8, '0'),
+  $7,
   'Hotspot Provider',
   'gpt-5',
   $3,
@@ -463,6 +509,7 @@ FROM generate_series(0, $1::INTEGER - 1) AS gs
     .bind(COST_PER_REQUEST_USD)
     .bind(now_unix_ms() as i64)
     .bind(now_unix_secs() as i64)
+    .bind(USER_ID)
     .execute(pool)
     .await?;
 
@@ -472,7 +519,7 @@ FROM generate_series(0, $1::INTEGER - 1) AS gs
 fn settlement_input(index: usize) -> UsageSettlementInput {
     UsageSettlementInput {
         request_id: format!("{REQUEST_PREFIX}-{index:08}"),
-        user_id: None,
+        user_id: Some(USER_ID.to_string()),
         api_key_id: None,
         api_key_is_standalone: false,
         provider_id: Some(PROVIDER_ID.to_string()),
@@ -552,11 +599,13 @@ SELECT
     SELECT CAST(monthly_used_usd AS DOUBLE PRECISION)
     FROM providers
     WHERE id = $2
-  ) AS provider_monthly_used_usd
+  ) AS provider_monthly_used_usd,
+  (SELECT CAST(total_consumed AS DOUBLE PRECISION) FROM wallets WHERE id = $3) AS wallet_consumed_usd
 "#,
     )
     .bind(format!("{REQUEST_PREFIX}-%"))
     .bind(PROVIDER_ID)
+    .bind(WALLET_ID)
     .fetch_one(pool)
     .await?;
     Ok(CounterReport {
@@ -568,6 +617,7 @@ SELECT
         provider_monthly_outbox_rows: row.try_get("provider_monthly_outbox_rows")?,
         provider_monthly_used_usd: row.try_get("provider_monthly_used_usd")?,
         expected_provider_monthly_used_usd: (requests as f64) * COST_PER_REQUEST_USD,
+        wallet_consumed_usd: row.try_get("wallet_consumed_usd")?,
     })
 }