fix: harden concurrency limits and high-RPM runtime paths

Bound request, stream, queue, and shutdown resource lifetimes. Reduce scheduler and Redis hot-path work and isolate database maintenance. Include regression coverage, load probes, and concurrency audit results.
This commit is contained in:
elky
2026-09-10 08:14:58 +08:00
parent 361952ada9
commit ecc16673eb
149 changed files with 27963 additions and 1926 deletions
@@ -16,17 +16,36 @@ impl PostgresBackend {
table_names: &[&str],
) -> Result<DatabaseMaintenanceSummary, DataLayerError> {
let mut summary = DatabaseMaintenanceSummary::default();
if table_names.is_empty() {
return Ok(summary);
}
// VACUUM cannot run inside a transaction. Discard this connection on every
// exit path so its longer session deadlines never leak into request queries.
let mut conn = self.pool().acquire().await.map_postgres_err()?;
conn.close_on_drop();
sqlx::query("SET statement_timeout = '5min'")
.execute(&mut *conn)
.await
.map_postgres_err()?;
sqlx::query("SET lock_timeout = '30s'")
.execute(&mut *conn)
.await
.map_postgres_err()?;
for table_name in table_names {
let table_name = maintenance_identifier(table_name)?;
summary.attempted += 1;
let statement = format!("VACUUM ANALYZE \"{table_name}\"");
if sqlx::raw_sql(&statement)
.execute(self.pool())
match sqlx::query(&statement)
.execute(&mut *conn)
.await
.map_postgres_err()
.is_ok()
{
summary.succeeded += 1;
Ok(_) => summary.succeeded += 1,
Err(error) => tracing::warn!(
table_name,
error = %error,
"PostgreSQL table maintenance failed"
),
}
}
Ok(summary)
@@ -274,6 +274,40 @@ mod tests {
use super::PostgresBackend;
use crate::driver::postgres::{PostgresLeaseRunnerConfig, PostgresPoolConfig};
#[tokio::test]
async fn maintenance_and_aggregation_futures_are_send() {
fn assert_send(_: impl Send) {}
let backend = PostgresBackend::from_config(PostgresPoolConfig {
database_url: "postgres://localhost/aether".to_string(),
min_connections: 0,
..PostgresPoolConfig::default()
})
.unwrap();
let now = chrono::Utc::now();
let daily = crate::StatsDailyAggregationInput {
target_day_utc: now,
aggregated_at: now,
};
let hourly = crate::StatsHourlyAggregationInput {
target_hour_utc: now,
aggregated_at: now,
};
let wallet = crate::WalletDailyUsageAggregationInput {
billing_date: "2026-09-09".to_string(),
billing_timezone: "UTC".to_string(),
window_start_unix_secs: 0,
window_end_unix_secs: 86_400,
aggregated_at_unix_secs: 86_400,
};
// Drop without polling: these are compile-time checks for spawned workers.
assert_send(backend.run_table_maintenance(&["usage"]));
assert_send(backend.aggregate_stats_daily(&daily));
assert_send(backend.aggregate_stats_hourly(&hourly));
assert_send(backend.aggregate_wallet_daily_usage(&wallet));
}
#[tokio::test]
async fn backend_retains_config_and_pool() {
let config = PostgresPoolConfig {
@@ -66,6 +66,12 @@ async fn perform_stats_aggregation_for_day(
) -> Result<StatsDailyAggregationSummary, sqlx::Error> {
let day_end_utc = day_start_utc + chrono::Duration::days(1);
let mut tx = pool.begin().await?;
sqlx::query("SET LOCAL statement_timeout = '5min'")
.execute(&mut *tx)
.await?;
sqlx::query("SET LOCAL lock_timeout = '30s'")
.execute(&mut *tx)
.await?;
let aggregate_row = sqlx::query(SELECT_STATS_DAILY_AGGREGATE_SQL)
.bind(day_start_utc)
.bind(day_end_utc)
@@ -65,6 +65,12 @@ async fn perform_stats_hourly_aggregation_for_hour(
) -> Result<StatsHourlyAggregationSummary, sqlx::Error> {
let hour_end = hour_utc + chrono::Duration::hours(1);
let mut tx = pool.begin().await?;
sqlx::query("SET LOCAL statement_timeout = '5min'")
.execute(&mut *tx)
.await?;
sqlx::query("SET LOCAL lock_timeout = '30s'")
.execute(&mut *tx)
.await?;
let row = sqlx::query(SELECT_STATS_HOURLY_AGGREGATE_SQL)
.bind(hour_utc)
@@ -104,6 +104,14 @@ impl PostgresBackend {
let window_end = unix_secs_to_utc(input.window_end_unix_secs, "window_end")?;
let aggregated_at = unix_secs_to_utc(input.aggregated_at_unix_secs, "aggregated_at")?;
let mut tx = self.pool().begin().await.map_postgres_err()?;
sqlx::query("SET LOCAL statement_timeout = '5min'")
.execute(&mut *tx)
.await
.map_postgres_err()?;
sqlx::query("SET LOCAL lock_timeout = '30s'")
.execute(&mut *tx)
.await
.map_postgres_err()?;
let aggregated_wallets = sqlx::query(UPSERT_WALLET_DAILY_USAGE_LEDGER_SQL)
.bind(window_start)