mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-12 14:10:19 +08:00
refactor(workspace): enforce layered crate boundaries
This commit is contained in:
elky
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,191 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use aether_loadtools::{run_http_load_probe, HttpLoadProbeConfig};
|
||||
use reqwest::Method;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let config = parse_args(std::env::args().skip(1).collect())?;
|
||||
let result = run_http_load_probe(&config)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
println!("{}", serde_json::to_string_pretty(&result)?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_args(args: Vec<String>) -> Result<HttpLoadProbeConfig, Box<dyn std::error::Error>> {
|
||||
let mut url: Option<String> = None;
|
||||
let mut warmup_url: Option<String> = None;
|
||||
let mut total_requests: Option<usize> = None;
|
||||
let mut concurrency: Option<usize> = None;
|
||||
let mut warmup_connections: usize = 0;
|
||||
let mut timeout_ms: Option<u64> = None;
|
||||
let mut connect_timeout_ms: Option<u64> = None;
|
||||
let mut client_shards: Option<usize> = None;
|
||||
let mut pool_max_idle_per_host: Option<usize> = None;
|
||||
let mut start_ramp_ms: u64 = 0;
|
||||
let mut first_body_hold_ms: u64 = 0;
|
||||
let mut method = Method::GET;
|
||||
let mut headers = std::collections::BTreeMap::new();
|
||||
let mut body: Option<Vec<u8>> = None;
|
||||
let mut response_mode = aether_loadtools::HttpLoadProbeResponseMode::HeadersOnly;
|
||||
let mut http1_only = false;
|
||||
let mut http2_prior_knowledge = false;
|
||||
|
||||
let mut iter = args.into_iter();
|
||||
while let Some(arg) = iter.next() {
|
||||
match arg.as_str() {
|
||||
"--url" => url = Some(next_value(&mut iter, "--url")?),
|
||||
"--warmup-url" => warmup_url = Some(next_value(&mut iter, "--warmup-url")?),
|
||||
"--requests" => total_requests = Some(next_value(&mut iter, "--requests")?.parse()?),
|
||||
"--concurrency" => concurrency = Some(next_value(&mut iter, "--concurrency")?.parse()?),
|
||||
"--warmup-connections" => {
|
||||
warmup_connections = next_value(&mut iter, "--warmup-connections")?.parse()?
|
||||
}
|
||||
"--timeout-ms" => timeout_ms = Some(next_value(&mut iter, "--timeout-ms")?.parse()?),
|
||||
"--connect-timeout-ms" => {
|
||||
connect_timeout_ms = Some(next_value(&mut iter, "--connect-timeout-ms")?.parse()?)
|
||||
}
|
||||
"--client-shards" => {
|
||||
client_shards = Some(next_value(&mut iter, "--client-shards")?.parse()?)
|
||||
}
|
||||
"--pool-max-idle-per-host" => {
|
||||
pool_max_idle_per_host =
|
||||
Some(next_value(&mut iter, "--pool-max-idle-per-host")?.parse()?)
|
||||
}
|
||||
"--start-ramp-ms" => {
|
||||
start_ramp_ms = next_value(&mut iter, "--start-ramp-ms")?.parse()?
|
||||
}
|
||||
"--first-body-hold-ms" => {
|
||||
first_body_hold_ms = next_value(&mut iter, "--first-body-hold-ms")?.parse()?
|
||||
}
|
||||
"--http1-only" => http1_only = true,
|
||||
"--http2-prior-knowledge" => http2_prior_knowledge = true,
|
||||
"--method" => {
|
||||
method = Method::from_bytes(next_value(&mut iter, "--method")?.as_bytes())?
|
||||
}
|
||||
"--header" | "-H" => {
|
||||
let (name, value) = parse_header_arg(&next_value(&mut iter, "--header")?)?;
|
||||
headers.insert(name, value);
|
||||
}
|
||||
"--body" => body = Some(next_value(&mut iter, "--body")?.into_bytes()),
|
||||
"--body-file" => body = Some(std::fs::read(next_value(&mut iter, "--body-file")?)?),
|
||||
"--response-mode" => {
|
||||
response_mode = parse_response_mode(&next_value(&mut iter, "--response-mode")?)?
|
||||
}
|
||||
"--help" | "-h" => {
|
||||
print_usage();
|
||||
std::process::exit(0);
|
||||
}
|
||||
other => {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
format!("unknown argument: {other}"),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut config = HttpLoadProbeConfig {
|
||||
url: url.ok_or_else(|| {
|
||||
std::io::Error::new(std::io::ErrorKind::InvalidInput, "missing required --url")
|
||||
})?,
|
||||
total_requests: total_requests.ok_or_else(|| {
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"missing required --requests",
|
||||
)
|
||||
})?,
|
||||
concurrency: concurrency.ok_or_else(|| {
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"missing required --concurrency",
|
||||
)
|
||||
})?,
|
||||
method,
|
||||
headers,
|
||||
body,
|
||||
response_mode,
|
||||
..HttpLoadProbeConfig::default()
|
||||
};
|
||||
config.warmup_url = warmup_url;
|
||||
config.warmup_connections = warmup_connections;
|
||||
if let Some(timeout_ms) = timeout_ms {
|
||||
config.timeout = Duration::from_millis(timeout_ms);
|
||||
}
|
||||
config.connect_timeout = connect_timeout_ms.map(Duration::from_millis);
|
||||
if let Some(client_shards) = client_shards {
|
||||
config.client_shards = client_shards;
|
||||
}
|
||||
config.pool_max_idle_per_host = pool_max_idle_per_host;
|
||||
config.start_ramp = Duration::from_millis(start_ramp_ms);
|
||||
config.first_body_hold = Duration::from_millis(first_body_hold_ms);
|
||||
config.http1_only = http1_only;
|
||||
config.http2_prior_knowledge = http2_prior_knowledge;
|
||||
config
|
||||
.validate()
|
||||
.map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err))?;
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
fn parse_header_arg(value: &str) -> Result<(String, String), Box<dyn std::error::Error>> {
|
||||
let (name, value) = value
|
||||
.split_once(':')
|
||||
.or_else(|| value.split_once('='))
|
||||
.ok_or_else(|| {
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"--header expects `Name: value` or `Name=value`",
|
||||
)
|
||||
})?;
|
||||
let name = name.trim();
|
||||
if name.is_empty() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"--header name cannot be empty",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok((name.to_string(), value.trim().to_string()))
|
||||
}
|
||||
|
||||
fn parse_response_mode(
|
||||
value: &str,
|
||||
) -> Result<aether_loadtools::HttpLoadProbeResponseMode, Box<dyn std::error::Error>> {
|
||||
match value.trim().to_ascii_lowercase().as_str() {
|
||||
"headers" | "headers-only" | "header" => {
|
||||
Ok(aether_loadtools::HttpLoadProbeResponseMode::HeadersOnly)
|
||||
}
|
||||
"first-body-byte" | "first-body" | "first-byte" | "first-chunk" => {
|
||||
Ok(aether_loadtools::HttpLoadProbeResponseMode::FirstBodyByte)
|
||||
}
|
||||
"full" | "full-body" | "body" => Ok(aether_loadtools::HttpLoadProbeResponseMode::FullBody),
|
||||
other => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
format!(
|
||||
"unsupported --response-mode {other}; expected headers, first-body-byte, or full"
|
||||
),
|
||||
)
|
||||
.into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn next_value(
|
||||
iter: &mut impl Iterator<Item = String>,
|
||||
flag: &str,
|
||||
) -> Result<String, Box<dyn std::error::Error>> {
|
||||
iter.next().ok_or_else(|| {
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
format!("missing value for {flag}"),
|
||||
)
|
||||
.into()
|
||||
})
|
||||
}
|
||||
|
||||
fn print_usage() {
|
||||
eprintln!(
|
||||
"usage: cargo run -p aether-loadtools --bin http_load_probe -- --url <URL> --requests <N> --concurrency <N> [--warmup-url <URL>] [--warmup-connections N] [--method GET] [--timeout-ms 30000] [--connect-timeout-ms 10000] [--client-shards 1] [--pool-max-idle-per-host N] [--start-ramp-ms 0] [--first-body-hold-ms 0] [--http1-only | --http2-prior-knowledge] [-H 'Name: value'] [--body JSON | --body-file path] [--response-mode headers|first-body-byte|full]"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,435 @@
|
||||
use std::path::PathBuf;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use aether_loadtools::{init_load_runtime_for, ManagedRedisServer};
|
||||
use aether_runtime_state::{
|
||||
RedisClientConfig, RedisConsumerGroup, RedisConsumerName, RedisStreamName,
|
||||
RedisStreamReclaimConfig, RedisStreamRunner, RedisStreamRunnerConfig,
|
||||
};
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct RedisWorkerBaselineConfig {
|
||||
append_total: usize,
|
||||
append_concurrency: usize,
|
||||
reclaim_total: usize,
|
||||
reclaim_min_idle: Duration,
|
||||
output_path: Option<PathBuf>,
|
||||
redis_url: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for RedisWorkerBaselineConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
append_total: 1_000,
|
||||
append_concurrency: 20,
|
||||
reclaim_total: 128,
|
||||
reclaim_min_idle: Duration::from_millis(100),
|
||||
output_path: None,
|
||||
redis_url: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct OperationSummary {
|
||||
total_calls: usize,
|
||||
total_items: usize,
|
||||
failed_calls: usize,
|
||||
p50_ms: u64,
|
||||
p95_ms: u64,
|
||||
max_ms: u64,
|
||||
mean_ms: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct RedisWorkerBaselineReport {
|
||||
suite: &'static str,
|
||||
redis_url: String,
|
||||
append: OperationSummary,
|
||||
read_group: OperationSummary,
|
||||
reclaim: OperationSummary,
|
||||
ack: OperationSummary,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
init_load_runtime_for("redis-worker-baseline");
|
||||
let config = parse_args(std::env::args().skip(1).collect())?;
|
||||
let report = run_suite(&config).await?;
|
||||
let raw = serde_json::to_string_pretty(&report)?;
|
||||
println!("{raw}");
|
||||
if let Some(path) = config.output_path.as_ref() {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
std::fs::write(path, format!("{raw}\n"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_suite(
|
||||
config: &RedisWorkerBaselineConfig,
|
||||
) -> Result<RedisWorkerBaselineReport, Box<dyn std::error::Error>> {
|
||||
let managed_redis = if config.redis_url.is_none() {
|
||||
Some(ManagedRedisServer::start().await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let redis_url = config
|
||||
.redis_url
|
||||
.clone()
|
||||
.or_else(|| {
|
||||
managed_redis
|
||||
.as_ref()
|
||||
.map(|server| server.redis_url().to_string())
|
||||
})
|
||||
.expect("redis url should be resolved");
|
||||
|
||||
let redis_config = RedisClientConfig {
|
||||
url: redis_url.clone(),
|
||||
key_prefix: Some(format!("aether-baseline-{}", std::process::id())),
|
||||
};
|
||||
let keyspace = redis_config.keyspace();
|
||||
let stream = keyspace.stream_name("worker-baseline");
|
||||
let group = RedisConsumerGroup("worker-group".to_string());
|
||||
let consumer_a = RedisConsumerName("consumer-a".to_string());
|
||||
let consumer_b = RedisConsumerName("consumer-b".to_string());
|
||||
let runner = RedisStreamRunner::from_config(
|
||||
redis_config,
|
||||
RedisStreamRunnerConfig {
|
||||
command_timeout_ms: Some(2_000),
|
||||
read_block_ms: Some(10),
|
||||
read_count: 64,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
runner
|
||||
.ensure_consumer_group(&stream, &group, "0-0")
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
|
||||
let append = benchmark_append(&runner, &stream, config).await?;
|
||||
let (read_group, drained_ids) =
|
||||
benchmark_read_group(&runner, &stream, &group, &consumer_a, config).await?;
|
||||
let ack_read = benchmark_ack(&runner, &stream, &group, &drained_ids).await?;
|
||||
let (reclaim, reclaimed_ids) =
|
||||
benchmark_reclaim(&runner, &stream, &group, &consumer_a, &consumer_b, config).await?;
|
||||
let ack_reclaim = benchmark_ack(&runner, &stream, &group, &reclaimed_ids).await?;
|
||||
|
||||
Ok(RedisWorkerBaselineReport {
|
||||
suite: "redis_worker_baseline",
|
||||
redis_url,
|
||||
append,
|
||||
read_group,
|
||||
reclaim,
|
||||
ack: combine_summaries(&ack_read, &ack_reclaim),
|
||||
})
|
||||
}
|
||||
|
||||
async fn benchmark_append(
|
||||
runner: &RedisStreamRunner,
|
||||
stream: &RedisStreamName,
|
||||
config: &RedisWorkerBaselineConfig,
|
||||
) -> Result<OperationSummary, Box<dyn std::error::Error>> {
|
||||
let next = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let latencies = std::sync::Arc::new(tokio::sync::Mutex::new(Vec::with_capacity(
|
||||
config.append_total,
|
||||
)));
|
||||
let failed = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let mut tasks = tokio::task::JoinSet::new();
|
||||
|
||||
for _ in 0..config.append_concurrency {
|
||||
let runner = runner.clone();
|
||||
let stream = stream.clone();
|
||||
let next = next.clone();
|
||||
let latencies = latencies.clone();
|
||||
let failed = failed.clone();
|
||||
let total = config.append_total;
|
||||
tasks.spawn(async move {
|
||||
loop {
|
||||
let current = next.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
|
||||
if current >= total {
|
||||
break;
|
||||
}
|
||||
let started = Instant::now();
|
||||
let result = runner
|
||||
.append_json(
|
||||
&stream,
|
||||
"payload",
|
||||
&serde_json::json!({
|
||||
"job_id": current,
|
||||
"kind": "baseline",
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
latencies
|
||||
.lock()
|
||||
.await
|
||||
.push(started.elapsed().as_millis() as u64);
|
||||
if result.is_err() {
|
||||
failed.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
while let Some(result) = tasks.join_next().await {
|
||||
result.map_err(std::io::Error::other)?;
|
||||
}
|
||||
|
||||
let samples = latencies.lock().await.clone();
|
||||
Ok(summarize_operation(
|
||||
samples,
|
||||
config.append_total,
|
||||
failed.load(std::sync::atomic::Ordering::Acquire),
|
||||
))
|
||||
}
|
||||
|
||||
async fn benchmark_read_group(
|
||||
runner: &RedisStreamRunner,
|
||||
stream: &RedisStreamName,
|
||||
group: &RedisConsumerGroup,
|
||||
consumer: &RedisConsumerName,
|
||||
config: &RedisWorkerBaselineConfig,
|
||||
) -> Result<(OperationSummary, Vec<String>), Box<dyn std::error::Error>> {
|
||||
let mut latencies = Vec::new();
|
||||
let mut failed = 0usize;
|
||||
let mut ids = Vec::with_capacity(config.append_total);
|
||||
|
||||
while ids.len() < config.append_total {
|
||||
let started = Instant::now();
|
||||
match runner.read_group(stream, group, consumer).await {
|
||||
Ok(entries) => {
|
||||
latencies.push(started.elapsed().as_millis() as u64);
|
||||
ids.extend(entries.into_iter().map(|entry| entry.id));
|
||||
}
|
||||
Err(_) => {
|
||||
latencies.push(started.elapsed().as_millis() as u64);
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let summary = summarize_operation(latencies, ids.len(), failed);
|
||||
Ok((summary, ids))
|
||||
}
|
||||
|
||||
async fn benchmark_reclaim(
|
||||
runner: &RedisStreamRunner,
|
||||
stream: &RedisStreamName,
|
||||
group: &RedisConsumerGroup,
|
||||
consumer_a: &RedisConsumerName,
|
||||
consumer_b: &RedisConsumerName,
|
||||
config: &RedisWorkerBaselineConfig,
|
||||
) -> Result<(OperationSummary, Vec<String>), Box<dyn std::error::Error>> {
|
||||
for index in 0..config.reclaim_total {
|
||||
runner
|
||||
.append_json(
|
||||
stream,
|
||||
"payload",
|
||||
&serde_json::json!({
|
||||
"job_id": format!("reclaim-{index}"),
|
||||
"kind": "baseline",
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
}
|
||||
|
||||
let mut pending_ids = Vec::with_capacity(config.reclaim_total);
|
||||
while pending_ids.len() < config.reclaim_total {
|
||||
let entries = runner
|
||||
.read_group(stream, group, consumer_a)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
pending_ids.extend(entries.into_iter().map(|entry| entry.id));
|
||||
}
|
||||
|
||||
tokio::time::sleep(config.reclaim_min_idle + Duration::from_millis(20)).await;
|
||||
|
||||
let mut latencies = Vec::new();
|
||||
let mut failed = 0usize;
|
||||
let mut reclaimed_ids = Vec::with_capacity(config.reclaim_total);
|
||||
let mut next_start_id = "0-0".to_string();
|
||||
while reclaimed_ids.len() < config.reclaim_total {
|
||||
let started = Instant::now();
|
||||
match runner
|
||||
.claim_stale(
|
||||
stream,
|
||||
group,
|
||||
consumer_b,
|
||||
&next_start_id,
|
||||
RedisStreamReclaimConfig {
|
||||
min_idle_ms: config.reclaim_min_idle.as_millis() as u64,
|
||||
count: 64,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
latencies.push(started.elapsed().as_millis() as u64);
|
||||
next_start_id = result.next_start_id;
|
||||
reclaimed_ids.extend(result.entries.into_iter().map(|entry| entry.id));
|
||||
if next_start_id == "0-0" && reclaimed_ids.len() < config.reclaim_total {
|
||||
failed += 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
latencies.push(started.elapsed().as_millis() as u64);
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let summary = summarize_operation(latencies, reclaimed_ids.len(), failed);
|
||||
Ok((summary, reclaimed_ids))
|
||||
}
|
||||
|
||||
async fn benchmark_ack(
|
||||
runner: &RedisStreamRunner,
|
||||
stream: &RedisStreamName,
|
||||
group: &RedisConsumerGroup,
|
||||
ids: &[String],
|
||||
) -> Result<OperationSummary, Box<dyn std::error::Error>> {
|
||||
let mut latencies = Vec::new();
|
||||
let mut failed = 0usize;
|
||||
let mut acked = 0usize;
|
||||
for chunk in ids.chunks(64) {
|
||||
let started = Instant::now();
|
||||
match runner.ack(stream, group, chunk).await {
|
||||
Ok(count) => {
|
||||
latencies.push(started.elapsed().as_millis() as u64);
|
||||
acked += count;
|
||||
}
|
||||
Err(_) => {
|
||||
latencies.push(started.elapsed().as_millis() as u64);
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(summarize_operation(latencies, acked, failed))
|
||||
}
|
||||
|
||||
fn summarize_operation(
|
||||
latencies: Vec<u64>,
|
||||
total_items: usize,
|
||||
failed_calls: usize,
|
||||
) -> OperationSummary {
|
||||
if latencies.is_empty() {
|
||||
return OperationSummary {
|
||||
total_calls: 0,
|
||||
total_items,
|
||||
failed_calls,
|
||||
p50_ms: 0,
|
||||
p95_ms: 0,
|
||||
max_ms: 0,
|
||||
mean_ms: 0,
|
||||
};
|
||||
}
|
||||
let mut sorted = latencies;
|
||||
sorted.sort_unstable();
|
||||
let total_calls = sorted.len();
|
||||
let max_ms = *sorted.last().unwrap_or(&0);
|
||||
let mean_ms = sorted.iter().sum::<u64>() / total_calls as u64;
|
||||
let p50_ms = percentile(&sorted, 50);
|
||||
let p95_ms = percentile(&sorted, 95);
|
||||
OperationSummary {
|
||||
total_calls,
|
||||
total_items,
|
||||
failed_calls,
|
||||
p50_ms,
|
||||
p95_ms,
|
||||
max_ms,
|
||||
mean_ms,
|
||||
}
|
||||
}
|
||||
|
||||
fn combine_summaries(first: &OperationSummary, second: &OperationSummary) -> OperationSummary {
|
||||
let total_calls = first.total_calls + second.total_calls;
|
||||
let total_items = first.total_items + second.total_items;
|
||||
let failed_calls = first.failed_calls + second.failed_calls;
|
||||
let max_ms = first.max_ms.max(second.max_ms);
|
||||
let mean_ms = if total_calls == 0 {
|
||||
0
|
||||
} else {
|
||||
((first.mean_ms * first.total_calls as u64) + (second.mean_ms * second.total_calls as u64))
|
||||
/ total_calls as u64
|
||||
};
|
||||
OperationSummary {
|
||||
total_calls,
|
||||
total_items,
|
||||
failed_calls,
|
||||
p50_ms: first.p50_ms.min(second.p50_ms),
|
||||
p95_ms: first.p95_ms.max(second.p95_ms),
|
||||
max_ms,
|
||||
mean_ms,
|
||||
}
|
||||
}
|
||||
|
||||
fn percentile(latencies: &[u64], percentile: u8) -> u64 {
|
||||
if latencies.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
let last_index = latencies.len() - 1;
|
||||
let rank = ((last_index as f64) * (percentile as f64 / 100.0)).round() as usize;
|
||||
latencies[rank.min(last_index)]
|
||||
}
|
||||
|
||||
fn parse_args(args: Vec<String>) -> Result<RedisWorkerBaselineConfig, Box<dyn std::error::Error>> {
|
||||
let mut config = RedisWorkerBaselineConfig::default();
|
||||
let mut iter = args.into_iter();
|
||||
while let Some(arg) = iter.next() {
|
||||
match arg.as_str() {
|
||||
"--append-total" => {
|
||||
config.append_total = next_value(&mut iter, "--append-total")?.parse()?
|
||||
}
|
||||
"--append-concurrency" => {
|
||||
config.append_concurrency =
|
||||
next_value(&mut iter, "--append-concurrency")?.parse()?
|
||||
}
|
||||
"--reclaim-total" => {
|
||||
config.reclaim_total = next_value(&mut iter, "--reclaim-total")?.parse()?
|
||||
}
|
||||
"--reclaim-min-idle-ms" => {
|
||||
config.reclaim_min_idle =
|
||||
Duration::from_millis(next_value(&mut iter, "--reclaim-min-idle-ms")?.parse()?)
|
||||
}
|
||||
"--redis-url" => config.redis_url = Some(next_value(&mut iter, "--redis-url")?),
|
||||
"--output" => {
|
||||
config.output_path = Some(PathBuf::from(next_value(&mut iter, "--output")?))
|
||||
}
|
||||
"--help" | "-h" => {
|
||||
print_usage();
|
||||
std::process::exit(0);
|
||||
}
|
||||
other => {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
format!("unknown argument: {other}"),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
fn next_value(
|
||||
iter: &mut impl Iterator<Item = String>,
|
||||
flag: &str,
|
||||
) -> Result<String, Box<dyn std::error::Error>> {
|
||||
iter.next().ok_or_else(|| {
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
format!("missing value for {flag}"),
|
||||
)
|
||||
.into()
|
||||
})
|
||||
}
|
||||
|
||||
fn print_usage() {
|
||||
eprintln!(
|
||||
"usage: cargo run -p aether-loadtools --bin redis_worker_baseline -- [--append-total 1000] [--append-concurrency 20] [--reclaim-total 128] [--reclaim-min-idle-ms 100] [--redis-url redis://127.0.0.1:6379/0] [--output /tmp/redis_worker_baseline.json]"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,549 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use aether_loadtools::{init_load_runtime_for, ManagedRedisServer};
|
||||
use aether_runtime_state::{
|
||||
DataLayerError, RedisClientConfig, RedisRuntimeDiagnostics, RuntimeQueueStore,
|
||||
RuntimeSemaphoreConfig, RuntimeState,
|
||||
};
|
||||
use futures_util::stream::{self, StreamExt};
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct RuntimeRedisPressureConfig {
|
||||
kv_total: usize,
|
||||
kv_concurrency: usize,
|
||||
lock_total: usize,
|
||||
lock_concurrency: usize,
|
||||
semaphore_total: usize,
|
||||
semaphore_concurrency: usize,
|
||||
stream_total: usize,
|
||||
stream_concurrency: usize,
|
||||
blocking_probe_total: usize,
|
||||
blocking_probe_concurrency: usize,
|
||||
command_timeout_ms: u64,
|
||||
output_path: Option<PathBuf>,
|
||||
redis_url: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for RuntimeRedisPressureConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
kv_total: 20_000,
|
||||
kv_concurrency: 200,
|
||||
lock_total: 10_000,
|
||||
lock_concurrency: 100,
|
||||
semaphore_total: 5_000,
|
||||
semaphore_concurrency: 100,
|
||||
stream_total: 10_000,
|
||||
stream_concurrency: 100,
|
||||
blocking_probe_total: 1_000,
|
||||
blocking_probe_concurrency: 100,
|
||||
command_timeout_ms: 2_000,
|
||||
output_path: None,
|
||||
redis_url: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
struct OperationSummary {
|
||||
total_calls: usize,
|
||||
total_items: usize,
|
||||
failed_calls: usize,
|
||||
p50_ms: u64,
|
||||
p95_ms: u64,
|
||||
p99_ms: u64,
|
||||
max_ms: u64,
|
||||
mean_ms: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
struct RuntimeRedisPressureReport {
|
||||
suite: &'static str,
|
||||
redis_url: String,
|
||||
total_connections_before: Option<u64>,
|
||||
total_connections_after: Option<u64>,
|
||||
total_connections_delta: Option<i64>,
|
||||
connected_clients_after: Option<u64>,
|
||||
diagnostics_after: RedisRuntimeDiagnostics,
|
||||
kv: OperationSummary,
|
||||
lock: OperationSummary,
|
||||
semaphore: OperationSummary,
|
||||
stream_append: OperationSummary,
|
||||
stream_read_group: OperationSummary,
|
||||
stream_ack: OperationSummary,
|
||||
blocking_fast_lane_probe: OperationSummary,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct SummaryCollector {
|
||||
latencies_ms: tokio::sync::Mutex<Vec<u64>>,
|
||||
total_calls: AtomicUsize,
|
||||
total_items: AtomicUsize,
|
||||
failed_calls: AtomicUsize,
|
||||
}
|
||||
|
||||
impl SummaryCollector {
|
||||
async fn record(&self, elapsed: Duration, items: usize, failed: bool) {
|
||||
self.latencies_ms
|
||||
.lock()
|
||||
.await
|
||||
.push(elapsed.as_millis() as u64);
|
||||
self.total_calls.fetch_add(1, Ordering::AcqRel);
|
||||
self.total_items.fetch_add(items, Ordering::AcqRel);
|
||||
if failed {
|
||||
self.failed_calls.fetch_add(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
async fn summarize(&self) -> OperationSummary {
|
||||
let mut latencies = self.latencies_ms.lock().await.clone();
|
||||
latencies.sort_unstable();
|
||||
let total_calls = self.total_calls.load(Ordering::Acquire);
|
||||
OperationSummary {
|
||||
total_calls,
|
||||
total_items: self.total_items.load(Ordering::Acquire),
|
||||
failed_calls: self.failed_calls.load(Ordering::Acquire),
|
||||
p50_ms: percentile(&latencies, 50),
|
||||
p95_ms: percentile(&latencies, 95),
|
||||
p99_ms: percentile(&latencies, 99),
|
||||
max_ms: latencies.last().copied().unwrap_or_default(),
|
||||
mean_ms: if total_calls == 0 {
|
||||
0
|
||||
} else {
|
||||
latencies.iter().sum::<u64>() / total_calls as u64
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
init_load_runtime_for("runtime-redis-pressure");
|
||||
let config = parse_args(std::env::args().skip(1).collect())?;
|
||||
let report = run_suite(&config).await?;
|
||||
let raw = serde_json::to_string_pretty(&report)?;
|
||||
println!("{raw}");
|
||||
if let Some(path) = config.output_path.as_ref() {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
std::fs::write(path, format!("{raw}\n"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_suite(
|
||||
config: &RuntimeRedisPressureConfig,
|
||||
) -> Result<RuntimeRedisPressureReport, Box<dyn std::error::Error>> {
|
||||
let managed_redis = if config.redis_url.is_none() {
|
||||
Some(ManagedRedisServer::start().await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let redis_url = config
|
||||
.redis_url
|
||||
.clone()
|
||||
.or_else(|| {
|
||||
managed_redis
|
||||
.as_ref()
|
||||
.map(|server| server.redis_url().to_string())
|
||||
})
|
||||
.expect("redis url should resolve");
|
||||
let runtime = Arc::new(
|
||||
RuntimeState::redis(
|
||||
RedisClientConfig {
|
||||
url: redis_url.clone(),
|
||||
key_prefix: Some(format!("aether-runtime-pressure-{}", std::process::id())),
|
||||
},
|
||||
Some(config.command_timeout_ms),
|
||||
)
|
||||
.await?,
|
||||
);
|
||||
let before = runtime
|
||||
.redis_diagnostics()
|
||||
.await?
|
||||
.expect("redis diagnostics should be available");
|
||||
|
||||
let kv = benchmark_kv(runtime.clone(), config).await;
|
||||
let lock = benchmark_lock(runtime.clone(), config).await;
|
||||
let semaphore = benchmark_semaphore(runtime.clone(), config).await?;
|
||||
let (stream_append, stream_read_group, stream_ack) =
|
||||
benchmark_stream(runtime.clone(), config).await;
|
||||
let blocking_fast_lane_probe = benchmark_blocking_fast_lane(runtime.clone(), config).await?;
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
let after = runtime
|
||||
.redis_diagnostics()
|
||||
.await?
|
||||
.expect("redis diagnostics should be available");
|
||||
let total_connections_delta = match (
|
||||
before.total_connections_received,
|
||||
after.total_connections_received,
|
||||
) {
|
||||
(Some(before), Some(after)) => Some(after as i64 - before as i64),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
Ok(RuntimeRedisPressureReport {
|
||||
suite: "runtime_redis_pressure",
|
||||
redis_url,
|
||||
total_connections_before: before.total_connections_received,
|
||||
total_connections_after: after.total_connections_received,
|
||||
total_connections_delta,
|
||||
connected_clients_after: after.connected_clients,
|
||||
diagnostics_after: after,
|
||||
kv,
|
||||
lock,
|
||||
semaphore,
|
||||
stream_append,
|
||||
stream_read_group,
|
||||
stream_ack,
|
||||
blocking_fast_lane_probe,
|
||||
})
|
||||
}
|
||||
|
||||
async fn benchmark_kv(
|
||||
runtime: Arc<RuntimeState>,
|
||||
config: &RuntimeRedisPressureConfig,
|
||||
) -> OperationSummary {
|
||||
let collector = Arc::new(SummaryCollector::default());
|
||||
stream::iter(0..config.kv_total)
|
||||
.for_each_concurrent(config.kv_concurrency, |index| {
|
||||
let runtime = runtime.clone();
|
||||
let collector = collector.clone();
|
||||
async move {
|
||||
let started = Instant::now();
|
||||
let value = format!("value-{index}");
|
||||
let key = format!("pressure:kv:{index}");
|
||||
let result = async {
|
||||
runtime
|
||||
.kv_set(&key, value.clone(), Some(Duration::from_secs(60)))
|
||||
.await?;
|
||||
let actual = runtime.kv_get(&key).await?;
|
||||
if actual.as_deref() != Some(value.as_str()) {
|
||||
return Err(DataLayerError::UnexpectedValue(format!(
|
||||
"kv pressure mismatch for {key}"
|
||||
)));
|
||||
}
|
||||
runtime.kv_delete(&key).await?;
|
||||
Ok::<usize, DataLayerError>(3)
|
||||
}
|
||||
.await;
|
||||
let failed = result.is_err();
|
||||
collector
|
||||
.record(started.elapsed(), result.unwrap_or(0), failed)
|
||||
.await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
collector.summarize().await
|
||||
}
|
||||
|
||||
async fn benchmark_lock(
|
||||
runtime: Arc<RuntimeState>,
|
||||
config: &RuntimeRedisPressureConfig,
|
||||
) -> OperationSummary {
|
||||
let collector = Arc::new(SummaryCollector::default());
|
||||
stream::iter(0..config.lock_total)
|
||||
.for_each_concurrent(config.lock_concurrency, |index| {
|
||||
let runtime = runtime.clone();
|
||||
let collector = collector.clone();
|
||||
async move {
|
||||
let started = Instant::now();
|
||||
let key = format!("pressure:lock:{index}");
|
||||
let owner = format!("owner-{index}");
|
||||
let result = async {
|
||||
let Some(lease) = runtime
|
||||
.lock_try_acquire(&key, &owner, Duration::from_secs(30))
|
||||
.await?
|
||||
else {
|
||||
return Err(DataLayerError::UnexpectedValue(format!(
|
||||
"lock pressure acquire returned none for {key}"
|
||||
)));
|
||||
};
|
||||
if !runtime.lock_renew(&lease, Duration::from_secs(30)).await? {
|
||||
return Err(DataLayerError::UnexpectedValue(format!(
|
||||
"lock pressure renew returned false for {key}"
|
||||
)));
|
||||
}
|
||||
if !runtime.lock_release(&lease).await? {
|
||||
return Err(DataLayerError::UnexpectedValue(format!(
|
||||
"lock pressure release returned false for {key}"
|
||||
)));
|
||||
}
|
||||
Ok::<usize, DataLayerError>(3)
|
||||
}
|
||||
.await;
|
||||
let failed = result.is_err();
|
||||
collector
|
||||
.record(started.elapsed(), result.unwrap_or(0), failed)
|
||||
.await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
collector.summarize().await
|
||||
}
|
||||
|
||||
async fn benchmark_semaphore(
|
||||
runtime: Arc<RuntimeState>,
|
||||
config: &RuntimeRedisPressureConfig,
|
||||
) -> Result<OperationSummary, Box<dyn std::error::Error>> {
|
||||
let collector = Arc::new(SummaryCollector::default());
|
||||
let semaphore = Arc::new(runtime.semaphore(
|
||||
"redis_pressure",
|
||||
config.semaphore_concurrency.saturating_mul(4).max(1),
|
||||
RuntimeSemaphoreConfig {
|
||||
lease_ttl_ms: 10_000,
|
||||
renew_interval_ms: 5_000,
|
||||
command_timeout_ms: Some(config.command_timeout_ms),
|
||||
},
|
||||
)?);
|
||||
stream::iter(0..config.semaphore_total)
|
||||
.for_each_concurrent(config.semaphore_concurrency, |_| {
|
||||
let semaphore = semaphore.clone();
|
||||
let collector = collector.clone();
|
||||
async move {
|
||||
let started = Instant::now();
|
||||
let result = semaphore.try_acquire().await;
|
||||
let failed = result.is_err();
|
||||
drop(result);
|
||||
collector
|
||||
.record(started.elapsed(), usize::from(!failed), failed)
|
||||
.await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
Ok(collector.summarize().await)
|
||||
}
|
||||
|
||||
async fn benchmark_stream(
|
||||
runtime: Arc<RuntimeState>,
|
||||
config: &RuntimeRedisPressureConfig,
|
||||
) -> (OperationSummary, OperationSummary, OperationSummary) {
|
||||
let stream_name = "pressure-stream";
|
||||
let group = "pressure-workers";
|
||||
let consumer = "consumer-a";
|
||||
RuntimeQueueStore::ensure_consumer_group(runtime.as_ref(), stream_name, group, "0-0")
|
||||
.await
|
||||
.expect("stream consumer group should initialize");
|
||||
|
||||
let append = Arc::new(SummaryCollector::default());
|
||||
stream::iter(0..config.stream_total)
|
||||
.for_each_concurrent(config.stream_concurrency, |index| {
|
||||
let runtime = runtime.clone();
|
||||
let append = append.clone();
|
||||
async move {
|
||||
let started = Instant::now();
|
||||
let mut fields = BTreeMap::new();
|
||||
fields.insert("payload".to_string(), format!("stream-value-{index}"));
|
||||
let result = RuntimeQueueStore::append_fields_with_maxlen(
|
||||
runtime.as_ref(),
|
||||
stream_name,
|
||||
&fields,
|
||||
Some(config.stream_total.saturating_mul(2)),
|
||||
)
|
||||
.await;
|
||||
append
|
||||
.record(
|
||||
started.elapsed(),
|
||||
usize::from(result.is_ok()),
|
||||
result.is_err(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
let read = SummaryCollector::default();
|
||||
let mut ids = Vec::with_capacity(config.stream_total);
|
||||
while ids.len() < config.stream_total {
|
||||
let started = Instant::now();
|
||||
match RuntimeQueueStore::read_group(
|
||||
runtime.as_ref(),
|
||||
stream_name,
|
||||
group,
|
||||
consumer,
|
||||
128,
|
||||
Some(10),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(entries) => {
|
||||
let item_count = entries.len();
|
||||
ids.extend(entries.into_iter().map(|entry| entry.id));
|
||||
read.record(started.elapsed(), item_count, false).await;
|
||||
}
|
||||
Err(_) => {
|
||||
read.record(started.elapsed(), 0, true).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let ack = SummaryCollector::default();
|
||||
for chunk in ids.chunks(128) {
|
||||
let started = Instant::now();
|
||||
match RuntimeQueueStore::ack(runtime.as_ref(), stream_name, group, chunk).await {
|
||||
Ok(count) => ack.record(started.elapsed(), count, false).await,
|
||||
Err(_) => ack.record(started.elapsed(), 0, true).await,
|
||||
}
|
||||
}
|
||||
|
||||
(
|
||||
append.summarize().await,
|
||||
read.summarize().await,
|
||||
ack.summarize().await,
|
||||
)
|
||||
}
|
||||
|
||||
async fn benchmark_blocking_fast_lane(
|
||||
runtime: Arc<RuntimeState>,
|
||||
config: &RuntimeRedisPressureConfig,
|
||||
) -> Result<OperationSummary, Box<dyn std::error::Error>> {
|
||||
let stream_name = "pressure-blocking-empty";
|
||||
let group = "pressure-blocking-workers";
|
||||
RuntimeQueueStore::ensure_consumer_group(runtime.as_ref(), stream_name, group, "0-0").await?;
|
||||
let blocking_runtime = runtime.clone();
|
||||
let blocking = tokio::spawn(async move {
|
||||
RuntimeQueueStore::read_group(
|
||||
blocking_runtime.as_ref(),
|
||||
stream_name,
|
||||
group,
|
||||
"blocked-consumer",
|
||||
1,
|
||||
Some(1_000),
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
let summary = benchmark_fast_lane_probe(runtime, config).await;
|
||||
let _ = blocking.await?;
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
async fn benchmark_fast_lane_probe(
|
||||
runtime: Arc<RuntimeState>,
|
||||
config: &RuntimeRedisPressureConfig,
|
||||
) -> OperationSummary {
|
||||
let collector = Arc::new(SummaryCollector::default());
|
||||
stream::iter(0..config.blocking_probe_total)
|
||||
.for_each_concurrent(config.blocking_probe_concurrency, |index| {
|
||||
let runtime = runtime.clone();
|
||||
let collector = collector.clone();
|
||||
async move {
|
||||
let started = Instant::now();
|
||||
let key = format!("pressure:blocking-probe:{index}");
|
||||
let result = async {
|
||||
runtime
|
||||
.kv_set(&key, "ok", Some(Duration::from_secs(30)))
|
||||
.await?;
|
||||
let ok = runtime.kv_get(&key).await?.as_deref() == Some("ok");
|
||||
if !ok {
|
||||
return Err(DataLayerError::UnexpectedValue(format!(
|
||||
"blocking fast lane probe mismatch for {key}"
|
||||
)));
|
||||
}
|
||||
Ok::<usize, DataLayerError>(2)
|
||||
}
|
||||
.await;
|
||||
let failed = result.is_err();
|
||||
collector
|
||||
.record(started.elapsed(), result.unwrap_or(0), failed)
|
||||
.await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
collector.summarize().await
|
||||
}
|
||||
|
||||
fn percentile(latencies: &[u64], percentile: u8) -> u64 {
|
||||
if latencies.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
let last_index = latencies.len() - 1;
|
||||
let rank = ((last_index as f64) * (percentile as f64 / 100.0)).round() as usize;
|
||||
latencies[rank.min(last_index)]
|
||||
}
|
||||
|
||||
fn parse_args(args: Vec<String>) -> Result<RuntimeRedisPressureConfig, Box<dyn std::error::Error>> {
|
||||
let mut config = RuntimeRedisPressureConfig::default();
|
||||
let mut iter = args.into_iter();
|
||||
while let Some(arg) = iter.next() {
|
||||
match arg.as_str() {
|
||||
"--kv-total" => config.kv_total = next_value(&mut iter, "--kv-total")?.parse()?,
|
||||
"--kv-concurrency" => {
|
||||
config.kv_concurrency = next_value(&mut iter, "--kv-concurrency")?.parse()?
|
||||
}
|
||||
"--lock-total" => config.lock_total = next_value(&mut iter, "--lock-total")?.parse()?,
|
||||
"--lock-concurrency" => {
|
||||
config.lock_concurrency = next_value(&mut iter, "--lock-concurrency")?.parse()?
|
||||
}
|
||||
"--semaphore-total" => {
|
||||
config.semaphore_total = next_value(&mut iter, "--semaphore-total")?.parse()?
|
||||
}
|
||||
"--semaphore-concurrency" => {
|
||||
config.semaphore_concurrency =
|
||||
next_value(&mut iter, "--semaphore-concurrency")?.parse()?
|
||||
}
|
||||
"--stream-total" => {
|
||||
config.stream_total = next_value(&mut iter, "--stream-total")?.parse()?
|
||||
}
|
||||
"--stream-concurrency" => {
|
||||
config.stream_concurrency =
|
||||
next_value(&mut iter, "--stream-concurrency")?.parse()?
|
||||
}
|
||||
"--blocking-probe-total" => {
|
||||
config.blocking_probe_total =
|
||||
next_value(&mut iter, "--blocking-probe-total")?.parse()?
|
||||
}
|
||||
"--blocking-probe-concurrency" => {
|
||||
config.blocking_probe_concurrency =
|
||||
next_value(&mut iter, "--blocking-probe-concurrency")?.parse()?
|
||||
}
|
||||
"--command-timeout-ms" => {
|
||||
config.command_timeout_ms =
|
||||
next_value(&mut iter, "--command-timeout-ms")?.parse()?
|
||||
}
|
||||
"--redis-url" => config.redis_url = Some(next_value(&mut iter, "--redis-url")?),
|
||||
"--output" => {
|
||||
config.output_path = Some(PathBuf::from(next_value(&mut iter, "--output")?))
|
||||
}
|
||||
"--help" | "-h" => {
|
||||
print_usage();
|
||||
std::process::exit(0);
|
||||
}
|
||||
other => {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
format!("unknown argument: {other}"),
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
fn next_value(
|
||||
iter: &mut impl Iterator<Item = String>,
|
||||
flag: &str,
|
||||
) -> Result<String, Box<dyn std::error::Error>> {
|
||||
iter.next().ok_or_else(|| {
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
format!("missing value for {flag}"),
|
||||
)
|
||||
.into()
|
||||
})
|
||||
}
|
||||
|
||||
fn print_usage() {
|
||||
eprintln!(
|
||||
"usage: cargo run -p aether-loadtools --bin runtime_redis_pressure -- [--kv-total 20000] [--kv-concurrency 200] [--lock-total 10000] [--lock-concurrency 100] [--semaphore-total 5000] [--semaphore-concurrency 100] [--stream-total 10000] [--stream-concurrency 100] [--blocking-probe-total 1000] [--blocking-probe-concurrency 100] [--command-timeout-ms 2000] [--redis-url redis://127.0.0.1:6379/0] [--output /tmp/runtime_redis_pressure.json]"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
use aether_http::{build_http_client, HttpClientConfig};
|
||||
|
||||
pub fn json_body(value: serde_json::Value) -> serde_json::Value {
|
||||
value
|
||||
}
|
||||
|
||||
pub fn test_http_client_config() -> HttpClientConfig {
|
||||
HttpClientConfig {
|
||||
connect_timeout_ms: Some(1_000),
|
||||
request_timeout_ms: Some(5_000),
|
||||
user_agent: Some("aether-testkit".to_string()),
|
||||
..HttpClientConfig::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn test_http_client() -> reqwest::Client {
|
||||
build_http_client(&test_http_client_config()).expect("failed to build test HTTP client")
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
//! Standalone load and benchmark tooling.
|
||||
//!
|
||||
//! This crate intentionally has no dependency on gateway internals or the
|
||||
//! integration test harness, so ordinary load-tool builds remain lightweight.
|
||||
|
||||
mod http;
|
||||
mod load;
|
||||
mod metrics;
|
||||
mod runtime;
|
||||
|
||||
pub use aether_test_support::ManagedRedisServer;
|
||||
pub use http::{json_body, test_http_client, test_http_client_config};
|
||||
pub use load::{
|
||||
run_http_load_probe, run_multi_url_http_load_probe, HttpLoadProbeConfig,
|
||||
HttpLoadProbeErrorSample, HttpLoadProbeResponseMode, HttpLoadProbeResult,
|
||||
HttpLoadProbeStatusSample, MultiUrlHttpLoadProbeResult,
|
||||
};
|
||||
pub use metrics::{
|
||||
fetch_prometheus_samples, find_metric_value_u64, parse_prometheus_samples, PrometheusSample,
|
||||
};
|
||||
pub use runtime::{BenchmarkRuntimeSampler, BenchmarkRuntimeSnapshot};
|
||||
|
||||
pub fn init_load_runtime_for(service_name: &'static str) {
|
||||
let _ = aether_runtime::init_service_runtime(
|
||||
aether_runtime::ServiceRuntimeConfig::new(service_name, "aether_loadtools=debug")
|
||||
.with_metrics_namespace("aether_loadtools"),
|
||||
);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,234 @@
|
||||
// Prometheus scraping belongs to standalone load tooling, not gateway tests.
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PrometheusSample {
|
||||
pub name: String,
|
||||
pub labels: BTreeMap<String, String>,
|
||||
pub value: String,
|
||||
}
|
||||
|
||||
pub async fn fetch_prometheus_samples(url: &str) -> Result<Vec<PrometheusSample>, String> {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(5))
|
||||
.build()
|
||||
.map_err(|err| format!("failed to build metrics http client: {err}"))?;
|
||||
let response = client
|
||||
.get(url)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| format!("failed to fetch metrics from {url}: {err}"))?;
|
||||
let status = response.status();
|
||||
let body = response
|
||||
.text()
|
||||
.await
|
||||
.map_err(|err| format!("failed to read metrics body from {url}: {err}"))?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("metrics endpoint {url} returned {status}: {body}"));
|
||||
}
|
||||
Ok(parse_prometheus_samples(&body))
|
||||
}
|
||||
|
||||
pub fn parse_prometheus_samples(text: &str) -> Vec<PrometheusSample> {
|
||||
text.lines()
|
||||
.filter_map(parse_prometheus_line)
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
|
||||
pub fn find_metric_value_u64(
|
||||
samples: &[PrometheusSample],
|
||||
metric_name: &str,
|
||||
labels: &[(&str, &str)],
|
||||
) -> Option<u64> {
|
||||
samples
|
||||
.iter()
|
||||
.find(|sample| {
|
||||
metric_name_matches(&sample.name, metric_name) && labels_match(sample, labels)
|
||||
})
|
||||
.and_then(|sample| sample.value.parse::<u64>().ok())
|
||||
}
|
||||
|
||||
fn metric_name_matches(actual: &str, expected: &str) -> bool {
|
||||
actual == expected
|
||||
|| actual.strip_prefix("aether-gateway_") == Some(expected)
|
||||
|| actual.strip_prefix("aether_gateway_") == Some(expected)
|
||||
}
|
||||
|
||||
fn labels_match(sample: &PrometheusSample, labels: &[(&str, &str)]) -> bool {
|
||||
labels
|
||||
.iter()
|
||||
.all(|(key, value)| sample.labels.get(*key).map(|current| current.as_str()) == Some(*value))
|
||||
}
|
||||
|
||||
fn parse_prometheus_line(line: &str) -> Option<PrometheusSample> {
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() || trimmed.starts_with('#') {
|
||||
return None;
|
||||
}
|
||||
let (metric, value) = trimmed.rsplit_once(' ')?;
|
||||
let (name, labels) = if let Some((name, raw_labels)) = metric
|
||||
.split_once('{')
|
||||
.and_then(|(name, rest)| rest.strip_suffix('}').map(|labels| (name, labels)))
|
||||
{
|
||||
(name.to_string(), parse_labels(raw_labels))
|
||||
} else {
|
||||
(metric.to_string(), BTreeMap::new())
|
||||
};
|
||||
Some(PrometheusSample {
|
||||
name,
|
||||
labels,
|
||||
value: value.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_labels(raw: &str) -> BTreeMap<String, String> {
|
||||
let mut labels = BTreeMap::new();
|
||||
for pair in split_label_pairs(raw) {
|
||||
if let Some((key, value)) = pair.split_once('=') {
|
||||
labels.insert(
|
||||
key.trim().to_string(),
|
||||
unescape_label_value(value.trim().trim_matches('"')),
|
||||
);
|
||||
}
|
||||
}
|
||||
labels
|
||||
}
|
||||
|
||||
fn split_label_pairs(raw: &str) -> Vec<&str> {
|
||||
let mut pairs = Vec::new();
|
||||
let mut start = 0;
|
||||
let mut in_string = false;
|
||||
let mut escaped = false;
|
||||
for (index, ch) in raw.char_indices() {
|
||||
if escaped {
|
||||
escaped = false;
|
||||
continue;
|
||||
}
|
||||
match ch {
|
||||
'\\' if in_string => escaped = true,
|
||||
'"' => in_string = !in_string,
|
||||
',' if !in_string => {
|
||||
let pair = raw[start..index].trim();
|
||||
if !pair.is_empty() {
|
||||
pairs.push(pair);
|
||||
}
|
||||
start = index + ch.len_utf8();
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let pair = raw[start..].trim();
|
||||
if !pair.is_empty() {
|
||||
pairs.push(pair);
|
||||
}
|
||||
pairs
|
||||
}
|
||||
|
||||
fn unescape_label_value(value: &str) -> String {
|
||||
let mut output = String::with_capacity(value.len());
|
||||
let mut chars = value.chars();
|
||||
while let Some(ch) = chars.next() {
|
||||
if ch != '\\' {
|
||||
output.push(ch);
|
||||
continue;
|
||||
}
|
||||
match chars.next() {
|
||||
Some('n') => output.push('\n'),
|
||||
Some('\\') => output.push('\\'),
|
||||
Some('"') => output.push('"'),
|
||||
Some(next) => {
|
||||
output.push('\\');
|
||||
output.push(next);
|
||||
}
|
||||
None => output.push('\\'),
|
||||
}
|
||||
}
|
||||
output
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{find_metric_value_u64, parse_prometheus_samples};
|
||||
|
||||
#[test]
|
||||
fn parses_prometheus_samples_with_labels() {
|
||||
let samples = parse_prometheus_samples(
|
||||
r#"
|
||||
# HELP aether_gateway_concurrency_in_flight Current number of in-flight operations.
|
||||
# TYPE aether_gateway_concurrency_in_flight gauge
|
||||
aether_gateway_concurrency_in_flight{gate="gateway_requests"} 7
|
||||
aether_gateway_concurrency_rejected_total{gate="gateway_requests"} 12
|
||||
"#,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
find_metric_value_u64(
|
||||
&samples,
|
||||
"concurrency_in_flight",
|
||||
&[("gate", "gateway_requests")]
|
||||
),
|
||||
Some(7)
|
||||
);
|
||||
assert_eq!(
|
||||
find_metric_value_u64(
|
||||
&samples,
|
||||
"concurrency_rejected_total",
|
||||
&[("gate", "gateway_requests")]
|
||||
),
|
||||
Some(12)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_metric_lookup_does_not_match_background_pool_suffix() {
|
||||
let samples = parse_prometheus_samples(
|
||||
r#"
|
||||
aether-gateway_database_pool_usage_basis_points{driver="postgres"} 1429
|
||||
aether-gateway_background_database_pool_usage_basis_points{driver="postgres"} 10000
|
||||
"#,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
find_metric_value_u64(
|
||||
&samples,
|
||||
"database_pool_usage_basis_points",
|
||||
&[("driver", "postgres")]
|
||||
),
|
||||
Some(1429)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_quoted_label_values_containing_commas() {
|
||||
let samples = parse_prometheus_samples(
|
||||
r#"
|
||||
metric_with_sql{rank="1",query_prefix="SELECT id, name, created_at FROM request_candidates",state="active"} 2
|
||||
"#,
|
||||
);
|
||||
|
||||
assert_eq!(samples.len(), 1);
|
||||
assert_eq!(samples[0].labels.get("rank").map(String::as_str), Some("1"));
|
||||
assert_eq!(
|
||||
samples[0].labels.get("query_prefix").map(String::as_str),
|
||||
Some("SELECT id, name, created_at FROM request_candidates")
|
||||
);
|
||||
assert_eq!(
|
||||
samples[0].labels.get("state").map(String::as_str),
|
||||
Some("active")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unescapes_quoted_label_values() {
|
||||
let samples = parse_prometheus_samples(
|
||||
r#"
|
||||
metric_with_escape{message="bad\"line\nx\\y"} 1
|
||||
"#,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
samples[0].labels.get("message").map(String::as_str),
|
||||
Some("bad\"line\nx\\y")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
// Runtime sampling is shared by standalone load tools and integration tests.
|
||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use serde::Serialize;
|
||||
use sysinfo::{get_current_pid, Pid, ProcessesToUpdate, System};
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
|
||||
pub struct BenchmarkRuntimeSnapshot {
|
||||
pub sampled_at_unix_secs: u64,
|
||||
pub elapsed_ms: u64,
|
||||
pub system_cpu_usage_basis_points: u64,
|
||||
pub process_cpu_usage_basis_points: u64,
|
||||
pub memory_total_bytes: u64,
|
||||
pub memory_used_bytes: u64,
|
||||
pub memory_available_bytes: u64,
|
||||
pub memory_used_basis_points: u64,
|
||||
pub process_memory_bytes: u64,
|
||||
pub process_virtual_memory_bytes: u64,
|
||||
pub process_memory_basis_points: u64,
|
||||
pub fd_open_count: u64,
|
||||
pub fd_limit: u64,
|
||||
}
|
||||
|
||||
pub struct BenchmarkRuntimeSampler {
|
||||
started_at: Instant,
|
||||
system: System,
|
||||
current_pid: Option<Pid>,
|
||||
}
|
||||
|
||||
impl BenchmarkRuntimeSampler {
|
||||
pub fn new() -> Self {
|
||||
let mut system = System::new_all();
|
||||
let current_pid = get_current_pid().ok();
|
||||
if let Some(pid) = current_pid {
|
||||
system.refresh_processes(ProcessesToUpdate::Some(&[pid]), true);
|
||||
}
|
||||
system.refresh_cpu_usage();
|
||||
system.refresh_memory();
|
||||
Self {
|
||||
started_at: Instant::now(),
|
||||
system,
|
||||
current_pid,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn snapshot(&mut self) -> BenchmarkRuntimeSnapshot {
|
||||
self.system.refresh_cpu_usage();
|
||||
self.system.refresh_memory();
|
||||
if let Some(pid) = self.current_pid {
|
||||
self.system
|
||||
.refresh_processes(ProcessesToUpdate::Some(&[pid]), true);
|
||||
}
|
||||
|
||||
let memory_total_bytes = self.system.total_memory();
|
||||
let memory_used_bytes = self.system.used_memory();
|
||||
let memory_available_bytes = self.system.available_memory();
|
||||
let (process_cpu_usage_basis_points, process_memory_bytes, process_virtual_memory_bytes) =
|
||||
self.current_pid
|
||||
.and_then(|pid| self.system.process(pid))
|
||||
.map(|process| {
|
||||
(
|
||||
percent_to_basis_points(process.cpu_usage() as f64),
|
||||
process.memory(),
|
||||
process.virtual_memory(),
|
||||
)
|
||||
})
|
||||
.unwrap_or((0, 0, 0));
|
||||
|
||||
BenchmarkRuntimeSnapshot {
|
||||
sampled_at_unix_secs: current_unix_secs(),
|
||||
elapsed_ms: self.started_at.elapsed().as_millis() as u64,
|
||||
system_cpu_usage_basis_points: percent_to_basis_points(
|
||||
self.system.global_cpu_usage() as f64
|
||||
),
|
||||
process_cpu_usage_basis_points,
|
||||
memory_total_bytes,
|
||||
memory_used_bytes,
|
||||
memory_available_bytes,
|
||||
memory_used_basis_points: ratio_to_basis_points(memory_used_bytes, memory_total_bytes),
|
||||
process_memory_bytes,
|
||||
process_virtual_memory_bytes,
|
||||
process_memory_basis_points: ratio_to_basis_points(
|
||||
process_memory_bytes,
|
||||
memory_total_bytes,
|
||||
),
|
||||
fd_open_count: open_file_descriptors().unwrap_or(0),
|
||||
fd_limit: file_descriptor_limit(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for BenchmarkRuntimeSampler {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
fn current_unix_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|duration| duration.as_secs())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
fn percent_to_basis_points(value: f64) -> u64 {
|
||||
if !value.is_finite() || value.is_sign_negative() {
|
||||
0
|
||||
} else {
|
||||
(value * 100.0).round().clamp(0.0, u64::MAX as f64) as u64
|
||||
}
|
||||
}
|
||||
|
||||
fn ratio_to_basis_points(value: u64, total: u64) -> u64 {
|
||||
value.saturating_mul(10_000).checked_div(total).unwrap_or(0)
|
||||
}
|
||||
|
||||
fn open_file_descriptors() -> Option<u64> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
for dir in ["/proc/self/fd", "/dev/fd"] {
|
||||
if let Ok(entries) = std::fs::read_dir(dir) {
|
||||
return Some(entries.count() as u64);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn file_descriptor_limit() -> u64 {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let mut limit = libc::rlimit {
|
||||
rlim_cur: 0,
|
||||
rlim_max: 0,
|
||||
};
|
||||
let result = unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) };
|
||||
if result == 0 {
|
||||
return limit.rlim_cur;
|
||||
}
|
||||
}
|
||||
0
|
||||
}
|
||||
Reference in New Issue
Block a user