mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-16 16:07:45 +08:00
fix: harden concurrency limits and high-RPM runtime paths
Bound request, stream, queue, and shutdown resource lifetimes. Reduce scheduler and Redis hot-path work and isolate database maintenance. Include regression coverage, load probes, and concurrency audit results.
This commit is contained in:
@@ -67,6 +67,21 @@ GROUP BY
|
||||
FLOOR(EXTRACT(EPOCH FROM (created_at - TO_TIMESTAMP($2))) / $4)::BIGINT
|
||||
"#;
|
||||
|
||||
const RUNTIME_CANDIDATE_COLUMNS: &str = r#"
|
||||
SELECT
|
||||
id, request_id, user_id, api_key_id,
|
||||
NULL::text AS username, NULL::text AS api_key_name,
|
||||
candidate_index, retry_index, provider_id, endpoint_id, key_id, status,
|
||||
NULL::text AS skip_reason, is_cached, status_code,
|
||||
NULL::text AS error_type, NULL::text AS error_message,
|
||||
latency_ms, concurrent_requests,
|
||||
NULL::jsonb AS extra_data, NULL::jsonb AS required_capabilities,
|
||||
CAST(EXTRACT(EPOCH FROM created_at) * 1000 AS BIGINT) AS created_at_unix_ms,
|
||||
CAST(EXTRACT(EPOCH FROM started_at) * 1000 AS BIGINT) AS started_at_unix_ms,
|
||||
CAST(EXTRACT(EPOCH FROM finished_at) * 1000 AS BIGINT) AS finished_at_unix_ms
|
||||
FROM request_candidates
|
||||
"#;
|
||||
|
||||
const UPSERT_SQL_TEMPLATE: &str = r#"
|
||||
INSERT INTO request_candidates (
|
||||
id,
|
||||
@@ -561,12 +576,29 @@ impl SqlxRequestCandidateReadRepository {
|
||||
pub async fn list_recent(
|
||||
&self,
|
||||
limit: usize,
|
||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||
self.list_recent_with_columns(limit, candidate_columns())
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn list_recent_runtime(
|
||||
&self,
|
||||
limit: usize,
|
||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||
self.list_recent_with_columns(limit, RUNTIME_CANDIDATE_COLUMNS)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_recent_with_columns(
|
||||
&self,
|
||||
limit: usize,
|
||||
columns: &'static str,
|
||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||
if limit == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let mut builder = QueryBuilder::<Postgres>::new(candidate_columns());
|
||||
let mut builder = QueryBuilder::<Postgres>::new(columns);
|
||||
builder.push(" ORDER BY created_at DESC");
|
||||
push_limit(
|
||||
&mut builder,
|
||||
@@ -1068,6 +1100,13 @@ impl RequestCandidateReadRepository for SqlxRequestCandidateReadRepository {
|
||||
Self::list_recent(self, limit).await
|
||||
}
|
||||
|
||||
async fn list_recent_runtime(
|
||||
&self,
|
||||
limit: usize,
|
||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||
Self::list_recent_runtime(self, limit).await
|
||||
}
|
||||
|
||||
async fn list_finalized_by_endpoint_ids_since(
|
||||
&self,
|
||||
endpoint_ids: &[String],
|
||||
@@ -1699,4 +1738,67 @@ VALUES ($1, $2, 0, 0, 'pending', $3, $4, $5::json, $6::json, $7, NOW())
|
||||
.await
|
||||
.expect("candidate NUL test rows should clean up");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires isolated AETHER_TEST_DATABASE_URL; uses a connection-local table"]
|
||||
async fn live_postgres_candidate_runtime_projection_preserves_metadata_and_admin_rows() {
|
||||
let database_url =
|
||||
std::env::var("AETHER_TEST_DATABASE_URL").expect("isolated test database");
|
||||
let pool = sqlx::postgres::PgPoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect(&database_url)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
r#"
|
||||
CREATE TEMP TABLE request_candidates (
|
||||
id text, request_id text, user_id text, api_key_id text,
|
||||
username text, api_key_name text, candidate_index integer, retry_index integer,
|
||||
provider_id text, endpoint_id text, key_id text, status text, skip_reason text,
|
||||
is_cached boolean, status_code integer, error_type text, error_message text,
|
||||
latency_ms integer, concurrent_requests integer, extra_data jsonb, required_capabilities jsonb,
|
||||
created_at timestamptz, started_at timestamptz, finished_at timestamptz
|
||||
)
|
||||
"#,
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
for index in 0..3 {
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO request_candidates VALUES (
|
||||
$1, $2, 'user', 'api-key', NULL, NULL, $3, 0,
|
||||
'provider', 'endpoint', 'key', 'failed', NULL, false, 500,
|
||||
'upstream_error', 'admin diagnostic', 20, 17, $4, '{"vision": true}'::jsonb,
|
||||
TO_TIMESTAMP(100 + $3), TO_TIMESTAMP(101 + $3), TO_TIMESTAMP(102 + $3)
|
||||
)
|
||||
"#,
|
||||
)
|
||||
.bind(format!("candidate-{index}"))
|
||||
.bind(format!("request-{index}"))
|
||||
.bind(index)
|
||||
.bind(json!({"upstream_response": {"body": "x".repeat(32_768)}}))
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let repository = SqlxRequestCandidateReadRepository::new(pool.clone());
|
||||
let full = repository.list_recent(2).await.unwrap();
|
||||
let runtime = repository.list_recent_runtime(2).await.unwrap();
|
||||
assert_eq!(
|
||||
runtime,
|
||||
full.iter()
|
||||
.map(|row| row.runtime_snapshot())
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
assert_eq!(runtime[0].id, "candidate-2");
|
||||
assert_eq!(runtime[0].concurrent_requests, Some(17));
|
||||
assert!(runtime[0].extra_data.is_none());
|
||||
assert!(runtime[0].error_message.is_none());
|
||||
assert!(full[0].extra_data.is_some());
|
||||
assert_eq!(repository.list_recent(2).await.unwrap(), full);
|
||||
assert!(repository.list_recent_runtime(0).await.unwrap().is_empty());
|
||||
pool.close().await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,7 +52,7 @@ pub use migrations::{
|
||||
run_migrations_with_bootstrap, BootstrapFuture, PostgresMigrationBootstrap, POSTGRES_MIGRATOR,
|
||||
};
|
||||
pub use oauth_providers::SqlxOAuthProviderRepository;
|
||||
pub use pool::{PostgresPool, PostgresPoolFactory};
|
||||
pub use pool::{acquire_postgres_migration_connection, PostgresPool, PostgresPoolFactory};
|
||||
pub use pool_scores::PostgresPoolMemberScoreRepository;
|
||||
pub use provider_catalog::SqlxProviderCatalogReadRepository;
|
||||
pub use proxy_nodes::SqlxProxyNodeRepository;
|
||||
|
||||
@@ -93,7 +93,7 @@ pub async fn run_migrations_with_bootstrap(
|
||||
pool: &PgPool,
|
||||
bootstrap: &dyn PostgresMigrationBootstrap,
|
||||
) -> Result<(), MigrateError> {
|
||||
let mut conn = pool.acquire().await?;
|
||||
let mut conn = crate::pool::acquire_postgres_migration_connection(pool).await?;
|
||||
|
||||
if POSTGRES_MIGRATOR.locking {
|
||||
conn.lock().await?;
|
||||
@@ -132,7 +132,7 @@ pub async fn prepare_database_for_startup_with_bootstrap(
|
||||
pool: &PgPool,
|
||||
bootstrap: &dyn PostgresMigrationBootstrap,
|
||||
) -> Result<Vec<PendingMigrationInfo>, MigrateError> {
|
||||
let mut conn = pool.acquire().await?;
|
||||
let mut conn = crate::pool::acquire_postgres_migration_connection(pool).await?;
|
||||
|
||||
if POSTGRES_MIGRATOR.locking {
|
||||
conn.lock().await?;
|
||||
|
||||
@@ -4,6 +4,70 @@ use sqlx::PgPool;
|
||||
use std::str::FromStr;
|
||||
use std::time::Duration;
|
||||
|
||||
const STATEMENT_TIMEOUT_ENV: &str = "AETHER_GATEWAY_DATA_POSTGRES_STATEMENT_TIMEOUT_MS";
|
||||
const LOCK_TIMEOUT_ENV: &str = "AETHER_GATEWAY_DATA_POSTGRES_LOCK_TIMEOUT_MS";
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct PostgresSessionTimeouts {
|
||||
statement_ms: u32,
|
||||
lock_ms: u32,
|
||||
}
|
||||
|
||||
impl PostgresSessionTimeouts {
|
||||
fn from_env() -> Result<Self, DataLayerError> {
|
||||
Ok(Self {
|
||||
statement_ms: read_timeout_env(STATEMENT_TIMEOUT_ENV, 30_000)?,
|
||||
lock_ms: read_timeout_env(LOCK_TIMEOUT_ENV, 3_000)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn apply(self, options: PgConnectOptions) -> PgConnectOptions {
|
||||
options.options([
|
||||
("statement_timeout", self.statement_ms),
|
||||
("lock_timeout", self.lock_ms),
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_timeout_ms(name: &str, value: &str) -> Result<u32, DataLayerError> {
|
||||
value
|
||||
.trim()
|
||||
.parse::<u32>()
|
||||
.ok()
|
||||
.filter(|value| *value <= i32::MAX as u32)
|
||||
.ok_or_else(|| {
|
||||
DataLayerError::InvalidConfiguration(format!(
|
||||
"{name} must be milliseconds in 0..=2147483647 (0 disables the timeout)"
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn read_timeout_env(name: &str, default: u32) -> Result<u32, DataLayerError> {
|
||||
match std::env::var(name) {
|
||||
Ok(value) => parse_timeout_ms(name, &value),
|
||||
Err(std::env::VarError::NotPresent) => Ok(default),
|
||||
Err(std::env::VarError::NotUnicode(_)) => Err(DataLayerError::InvalidConfiguration(
|
||||
format!("{name} must contain a valid integer"),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Migration and historical backfill connections are discarded on every exit path,
|
||||
/// including cancellation, so their relaxed deadlines cannot escape into request work.
|
||||
pub async fn acquire_postgres_migration_connection(
|
||||
pool: &PgPool,
|
||||
) -> Result<sqlx::pool::PoolConnection<sqlx::Postgres>, sqlx::Error> {
|
||||
let mut conn = pool.acquire().await?;
|
||||
conn.close_on_drop();
|
||||
sqlx::query("SET statement_timeout = 0")
|
||||
.execute(&mut *conn)
|
||||
.await?;
|
||||
sqlx::query("SET lock_timeout = 0")
|
||||
.execute(&mut *conn)
|
||||
.await?;
|
||||
Ok(conn)
|
||||
}
|
||||
|
||||
fn connect_options(config: &PostgresPoolConfig) -> Result<PgConnectOptions, DataLayerError> {
|
||||
config.validate()?;
|
||||
let options = PgConnectOptions::from_str(config.database_url.trim()).map_err(|err| {
|
||||
@@ -33,12 +97,16 @@ pub type PostgresPool = PgPool;
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PostgresPoolFactory {
|
||||
config: PostgresPoolConfig,
|
||||
timeouts: PostgresSessionTimeouts,
|
||||
}
|
||||
|
||||
impl PostgresPoolFactory {
|
||||
pub fn new(config: PostgresPoolConfig) -> Result<Self, DataLayerError> {
|
||||
config.validate()?;
|
||||
Ok(Self { config })
|
||||
Ok(Self {
|
||||
config,
|
||||
timeouts: PostgresSessionTimeouts::from_env()?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn config(&self) -> &PostgresPoolConfig {
|
||||
@@ -46,7 +114,7 @@ impl PostgresPoolFactory {
|
||||
}
|
||||
|
||||
pub fn connect_lazy(&self) -> Result<PostgresPool, DataLayerError> {
|
||||
let options = connect_options(&self.config)?;
|
||||
let options = self.timeouts.apply(connect_options(&self.config)?);
|
||||
Ok(PgPoolOptions::new()
|
||||
.min_connections(self.config.min_connections)
|
||||
.max_connections(self.config.max_connections)
|
||||
@@ -59,10 +127,171 @@ impl PostgresPoolFactory {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{connect_options, PostgresPoolFactory};
|
||||
use super::{connect_options, parse_timeout_ms, PostgresPoolFactory, PostgresSessionTimeouts};
|
||||
use crate::PostgresPoolConfig;
|
||||
use sqlx::postgres::PgSslMode;
|
||||
|
||||
#[tokio::test]
|
||||
async fn migration_connection_future_is_send() {
|
||||
fn assert_send(_: impl Send) {}
|
||||
|
||||
let pool = sqlx::postgres::PgPoolOptions::new()
|
||||
.connect_lazy("postgres://localhost/aether")
|
||||
.unwrap();
|
||||
assert_send(super::acquire_postgres_migration_connection(&pool));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validates_session_timeout_milliseconds() {
|
||||
assert_eq!(parse_timeout_ms("timeout", "0").unwrap(), 0);
|
||||
assert_eq!(parse_timeout_ms("timeout", " 3000 ").unwrap(), 3_000);
|
||||
assert_eq!(
|
||||
parse_timeout_ms("timeout", "2147483647").unwrap(),
|
||||
i32::MAX as u32
|
||||
);
|
||||
for invalid in ["", "-1", "3s", "2147483648", "4294967296"] {
|
||||
assert!(parse_timeout_ms("timeout", invalid).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_deadlines_preserve_unrelated_connection_options() {
|
||||
let options = PostgresSessionTimeouts {
|
||||
statement_ms: 30_000,
|
||||
lock_ms: 3_000,
|
||||
}
|
||||
.apply(sqlx::postgres::PgConnectOptions::new().options([("search_path", "audit")]));
|
||||
assert_eq!(
|
||||
options.get_options(),
|
||||
Some("-c search_path=audit -c statement_timeout=30000 -c lock_timeout=3000")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires an isolated AETHER_TEST_DATABASE_URL"]
|
||||
async fn live_session_deadlines_rollback_transactions_and_isolate_migration_overrides() {
|
||||
use crate::error::SqlxResultExt;
|
||||
use crate::{PostgresTransactionOptions, PostgresTransactionRunner};
|
||||
|
||||
let factory = PostgresPoolFactory {
|
||||
config: PostgresPoolConfig {
|
||||
database_url: std::env::var("AETHER_TEST_DATABASE_URL").expect("test database URL"),
|
||||
min_connections: 0,
|
||||
max_connections: 2,
|
||||
..PostgresPoolConfig::default()
|
||||
},
|
||||
timeouts: PostgresSessionTimeouts {
|
||||
statement_ms: 100,
|
||||
lock_ms: 40,
|
||||
},
|
||||
};
|
||||
let pool = factory.connect_lazy().unwrap();
|
||||
let table = format!("deadline_test_{}", uuid::Uuid::new_v4().simple());
|
||||
sqlx::query(&format!(
|
||||
"CREATE TABLE {table} (id INTEGER PRIMARY KEY, value INTEGER NOT NULL)"
|
||||
))
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(&format!("INSERT INTO {table} VALUES (1, 0)"))
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut blocker = pool.begin().await.unwrap();
|
||||
sqlx::query(&format!("UPDATE {table} SET value = 7 WHERE id = 1"))
|
||||
.execute(&mut *blocker)
|
||||
.await
|
||||
.unwrap();
|
||||
let runner = PostgresTransactionRunner::new(pool.clone());
|
||||
let insert = format!("INSERT INTO {table} VALUES (2, 2)");
|
||||
let update = format!("UPDATE {table} SET value = 9 WHERE id = 1");
|
||||
let started = std::time::Instant::now();
|
||||
let error = runner
|
||||
.run_read_write(|tx| {
|
||||
Box::pin(async move {
|
||||
sqlx::query(&insert)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_postgres_err()?;
|
||||
sqlx::query(&update)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_postgres_err()?;
|
||||
Ok(())
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("SQLSTATE 55P03"), "{error}");
|
||||
assert!(started.elapsed() < std::time::Duration::from_secs(2));
|
||||
blocker.rollback().await.unwrap();
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>(&format!("SELECT COUNT(*) FROM {table}"))
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i32>(&format!("SELECT value FROM {table} WHERE id = 1"))
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
|
||||
let error = sqlx::query("SELECT pg_sleep(0.3)")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.map_postgres_err()
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("SQLSTATE 57014"), "{error}");
|
||||
runner
|
||||
.run(
|
||||
PostgresTransactionOptions {
|
||||
statement_timeout_ms: Some(1_000),
|
||||
..PostgresTransactionOptions::read_write()
|
||||
},
|
||||
|tx| {
|
||||
Box::pin(async move {
|
||||
sqlx::query("SELECT pg_sleep(0.15)")
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_postgres_err()?;
|
||||
Ok(())
|
||||
})
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut migration = super::acquire_postgres_migration_connection(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("SELECT pg_sleep(0.15)")
|
||||
.execute(&mut *migration)
|
||||
.await
|
||||
.unwrap();
|
||||
drop(migration);
|
||||
for _ in 0..2 {
|
||||
let configured: i64 = sqlx::query_scalar(
|
||||
"SELECT setting::BIGINT FROM pg_settings WHERE name = 'statement_timeout'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
configured, 100,
|
||||
"relaxed/local overrides must not leak into pooled requests"
|
||||
);
|
||||
}
|
||||
sqlx::query(&format!("DROP TABLE {table}"))
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
pool.close().await;
|
||||
}
|
||||
|
||||
fn ssl_mode(url: &str, require_ssl: bool) -> PgSslMode {
|
||||
connect_options(&PostgresPoolConfig {
|
||||
database_url: url.to_string(),
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use async_trait::async_trait;
|
||||
use sqlx::{PgPool, Row};
|
||||
use sqlx::{PgPool, Postgres, QueryBuilder, Row};
|
||||
|
||||
use aether_data_contracts::repository::settlement::{
|
||||
finite_wallet_available_usd, plan_finite_wallet_debit, settlement_billable_cost_usd,
|
||||
@@ -297,6 +297,76 @@ fn usage_policy_subject_missing() -> DataLayerError {
|
||||
DataLayerError::InvalidInput("usage policy subject does not exist".to_string())
|
||||
}
|
||||
|
||||
fn usage_policy_window_aggregate_query(
|
||||
windows: impl Iterator<Item = (u64, u64)>,
|
||||
aggregate: &str,
|
||||
) -> Result<(QueryBuilder<'static, Postgres>, i64, i64), DataLayerError> {
|
||||
let mut builder = QueryBuilder::new("SELECT ");
|
||||
let mut earliest = i64::MAX;
|
||||
let mut latest = i64::MIN;
|
||||
for (index, (start, end)) in windows.enumerate() {
|
||||
let start = usage_policy_cost_i64(start, "usage policy window start")?;
|
||||
let end = usage_policy_cost_i64(end, "usage policy window end")?;
|
||||
earliest = earliest.min(start);
|
||||
latest = latest.max(end);
|
||||
if index > 0 {
|
||||
builder.push(", ");
|
||||
}
|
||||
builder
|
||||
.push("COALESCE(")
|
||||
.push(aggregate)
|
||||
.push(" FILTER (WHERE admitted_at >= TO_TIMESTAMP(")
|
||||
.push_bind(start)
|
||||
.push("::double precision) AND admitted_at < TO_TIMESTAMP(")
|
||||
.push_bind(end)
|
||||
.push("::double precision)), 0)::BIGINT");
|
||||
}
|
||||
Ok((builder, earliest, latest))
|
||||
}
|
||||
|
||||
async fn usage_policy_request_window_counts(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
input: &ReserveUsagePolicyRequestInput,
|
||||
) -> Result<sqlx::postgres::PgRow, DataLayerError> {
|
||||
let (mut query, earliest, latest) = usage_policy_window_aggregate_query(
|
||||
input
|
||||
.windows
|
||||
.iter()
|
||||
.map(|window| (window.starts_at_unix_secs, window.ends_at_unix_secs)),
|
||||
"COUNT(*)",
|
||||
)?;
|
||||
// The subject lock protects all windows. One bounded history scan replaces
|
||||
// repeated scans of overlapping windows without approximating their counts.
|
||||
query
|
||||
.push(" FROM usage_request_admissions WHERE subject_id = ")
|
||||
.push_bind(input.subject_id.clone())
|
||||
.push(" AND state = 'active' AND admitted_at >= TO_TIMESTAMP(")
|
||||
.push_bind(earliest)
|
||||
.push("::double precision) AND admitted_at < TO_TIMESTAMP(")
|
||||
.push_bind(latest)
|
||||
.push("::double precision)");
|
||||
query.build().fetch_one(&mut **tx).await.map_postgres_err()
|
||||
}
|
||||
|
||||
async fn usage_policy_cost_window_totals(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
input: &ReserveUsagePolicyCostInput,
|
||||
) -> Result<sqlx::postgres::PgRow, DataLayerError> {
|
||||
let (mut query, earliest, latest) = usage_policy_window_aggregate_query(
|
||||
input.windows.iter().map(|window| (window.starts_at_unix_secs, window.ends_at_unix_secs)),
|
||||
"SUM(CASE WHEN state = 'finalized' THEN COALESCE(actual_cost_units, 0) ELSE reserved_cost_units END)",
|
||||
)?;
|
||||
query.push(" FROM usage_cost_reservations WHERE subject_id = ")
|
||||
.push_bind(input.subject_id.clone())
|
||||
.push(" AND admitted_at >= TO_TIMESTAMP(").push_bind(earliest)
|
||||
.push("::double precision) AND admitted_at < TO_TIMESTAMP(").push_bind(latest)
|
||||
.push("::double precision) AND reservation_token <> ").push_bind(input.reservation_token.clone())
|
||||
.push(" AND (state = 'finalized' OR (state = 'reserved' AND reservation_expires_at > TO_TIMESTAMP(")
|
||||
.push_bind(usage_policy_cost_i64(input.admitted_at_unix_secs, "usage policy admitted_at")?)
|
||||
.push("::double precision)))");
|
||||
query.build().fetch_one(&mut **tx).await.map_postgres_err()
|
||||
}
|
||||
|
||||
fn settlement_from_row(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
) -> Result<StoredUsageSettlement, DataLayerError> {
|
||||
@@ -479,6 +549,8 @@ async fn consume_daily_quota_postgres(
|
||||
return Ok(DailyQuotaDebitResult::default());
|
||||
}
|
||||
let now = chrono::Utc::now();
|
||||
// Serialize each entitlement's debits. Read the shared plan's current overage policy
|
||||
// from this statement's snapshot without locking every subscriber's plan row.
|
||||
let entitlement_rows = sqlx::query(
|
||||
r#"
|
||||
SELECT
|
||||
@@ -494,7 +566,7 @@ WHERE user_plan_entitlements.user_id = $1
|
||||
ORDER BY user_plan_entitlements.expires_at ASC,
|
||||
user_plan_entitlements.created_at ASC,
|
||||
user_plan_entitlements.id ASC
|
||||
FOR UPDATE
|
||||
FOR UPDATE OF user_plan_entitlements
|
||||
"#,
|
||||
)
|
||||
.bind(user_id)
|
||||
@@ -655,31 +727,10 @@ WHERE event_token = $1
|
||||
});
|
||||
}
|
||||
|
||||
let window_counts = usage_policy_request_window_counts(tx, &input).await?;
|
||||
for (window_index, window) in input.windows.iter().enumerate() {
|
||||
let used_requests = sqlx::query_scalar::<_, i64>(
|
||||
r#"
|
||||
SELECT COUNT(*)::BIGINT
|
||||
FROM usage_request_admissions
|
||||
WHERE subject_id = $1
|
||||
AND state = 'active'
|
||||
AND admitted_at >= TO_TIMESTAMP($2::double precision)
|
||||
AND admitted_at < TO_TIMESTAMP($3::double precision)
|
||||
"#,
|
||||
)
|
||||
.bind(&input.subject_id)
|
||||
.bind(usage_policy_cost_i64(
|
||||
window.starts_at_unix_secs,
|
||||
"usage policy request window start",
|
||||
)?)
|
||||
.bind(usage_policy_cost_i64(
|
||||
window.ends_at_unix_secs,
|
||||
"usage policy request window end",
|
||||
)?)
|
||||
.fetch_one(&mut **tx)
|
||||
.await
|
||||
.map_postgres_err()?;
|
||||
let used_requests = usage_policy_cost_u64(
|
||||
used_requests,
|
||||
window_counts.try_get(window_index).map_postgres_err()?,
|
||||
"usage policy request used_requests",
|
||||
)?;
|
||||
if used_requests >= window.limit_requests {
|
||||
@@ -884,43 +935,12 @@ WHERE retain_until <= TO_TIMESTAMP($1::double precision)
|
||||
.unwrap_or(0);
|
||||
let target_reserved_cost_units =
|
||||
previous_reserved_cost_units.max(input.reserved_cost_units);
|
||||
let window_totals = usage_policy_cost_window_totals(tx, &input).await?;
|
||||
for (window_index, window) in input.windows.iter().enumerate() {
|
||||
let used_cost_units = sqlx::query_scalar::<_, i64>(
|
||||
r#"
|
||||
SELECT COALESCE(SUM(
|
||||
CASE
|
||||
WHEN state = 'finalized' THEN COALESCE(actual_cost_units, 0)
|
||||
WHEN state = 'reserved' AND reservation_expires_at > TO_TIMESTAMP($4::double precision)
|
||||
THEN reserved_cost_units
|
||||
ELSE 0
|
||||
END
|
||||
), 0)::BIGINT
|
||||
FROM usage_cost_reservations
|
||||
WHERE subject_id = $1
|
||||
AND admitted_at >= TO_TIMESTAMP($2::double precision)
|
||||
AND admitted_at < TO_TIMESTAMP($3::double precision)
|
||||
AND reservation_token <> $5
|
||||
"#,
|
||||
)
|
||||
.bind(&input.subject_id)
|
||||
.bind(usage_policy_cost_i64(
|
||||
window.starts_at_unix_secs,
|
||||
"usage policy window start",
|
||||
)?)
|
||||
.bind(usage_policy_cost_i64(
|
||||
window.ends_at_unix_secs,
|
||||
"usage policy window end",
|
||||
)?)
|
||||
.bind(usage_policy_cost_i64(
|
||||
input.admitted_at_unix_secs,
|
||||
"usage policy admitted_at",
|
||||
)?)
|
||||
.bind(&input.reservation_token)
|
||||
.fetch_one(&mut **tx)
|
||||
.await
|
||||
.map_postgres_err()?;
|
||||
let used_cost_units =
|
||||
usage_policy_cost_u64(used_cost_units, "usage policy used_cost_units")?;
|
||||
let used_cost_units = usage_policy_cost_u64(
|
||||
window_totals.try_get(window_index).map_postgres_err()?,
|
||||
"usage policy used_cost_units",
|
||||
)?;
|
||||
if used_cost_units
|
||||
.checked_add(target_reserved_cost_units)
|
||||
.is_none_or(|total| total > window.limit_cost_units)
|
||||
@@ -1415,6 +1435,264 @@ WHERE id = $1
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use futures_util::FutureExt;
|
||||
use std::panic::AssertUnwindSafe;
|
||||
|
||||
async fn isolated_settlement_test_pool() -> (sqlx::PgPool, String) {
|
||||
let database_url = std::env::var("AETHER_TEST_DATABASE_URL")
|
||||
.expect("AETHER_TEST_DATABASE_URL must point at the test database");
|
||||
let schema = format!("settlement_test_{}", uuid::Uuid::new_v4().simple());
|
||||
let options = database_url
|
||||
.parse::<sqlx::postgres::PgConnectOptions>()
|
||||
.expect("test database URL should parse")
|
||||
.options([("search_path", schema.as_str())]);
|
||||
let pool = sqlx::postgres::PgPoolOptions::new()
|
||||
.max_connections(2)
|
||||
.connect_with(options)
|
||||
.await
|
||||
.expect("test database should connect");
|
||||
sqlx::query(&format!("CREATE SCHEMA {schema}"))
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("isolated settlement schema should be created");
|
||||
// Separate connections must see the same fixture, so pg_temp cannot be used here.
|
||||
for table in [
|
||||
"billing_plans",
|
||||
"user_plan_entitlements",
|
||||
"entitlement_usage_ledgers",
|
||||
"users",
|
||||
"usage_request_admissions",
|
||||
"usage_cost_reservations",
|
||||
] {
|
||||
sqlx::query(&format!(
|
||||
"CREATE TABLE {table} (LIKE public.{table} INCLUDING ALL)"
|
||||
))
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("isolated settlement table should be created");
|
||||
}
|
||||
(pool, schema)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires AETHER_TEST_DATABASE_URL and PostgreSQL migrations"]
|
||||
async fn live_usage_policy_window_aggregates_preserve_exact_admission_and_idempotency() {
|
||||
use super::*;
|
||||
use aether_data_contracts::repository::settlement::{
|
||||
UsagePolicyCostWindow, UsagePolicyRequestWindow,
|
||||
};
|
||||
|
||||
let (pool, schema) = isolated_settlement_test_pool().await;
|
||||
let result = AssertUnwindSafe(async {
|
||||
sqlx::query("INSERT INTO users (id, username, email_verified) VALUES ('subject', 'subject', false)")
|
||||
.execute(&pool).await.unwrap();
|
||||
sqlx::raw_sql("INSERT INTO usage_request_admissions (request_id, subject_id, event_token, admitted_at, retain_until, state, released_at) VALUES
|
||||
('older', 'subject', 'older', TO_TIMESTAMP(50), TO_TIMESTAMP(500), 'active', NULL),
|
||||
('start', 'subject', 'start', TO_TIMESTAMP(100), TO_TIMESTAMP(500), 'active', NULL),
|
||||
('inside', 'subject', 'inside', TO_TIMESTAMP(150), TO_TIMESTAMP(500), 'active', NULL),
|
||||
('end', 'subject', 'end', TO_TIMESTAMP(200), TO_TIMESTAMP(500), 'active', NULL),
|
||||
('released', 'subject', 'released', TO_TIMESTAMP(150), TO_TIMESTAMP(500), 'released', TO_TIMESTAMP(170))")
|
||||
.execute(&pool).await.unwrap();
|
||||
let repo = SqlxSettlementRepository::new(pool.clone());
|
||||
let mut request = ReserveUsagePolicyRequestInput {
|
||||
request_id: "new".to_string(), subject_id: "subject".to_string(), event_token: "new".to_string(),
|
||||
admitted_at_unix_secs: 175, retain_until_unix_secs: 500,
|
||||
windows: vec![
|
||||
UsagePolicyRequestWindow { starts_at_unix_secs: 100, ends_at_unix_secs: 200, limit_requests: 2 },
|
||||
UsagePolicyRequestWindow { starts_at_unix_secs: 0, ends_at_unix_secs: 300, limit_requests: 4 },
|
||||
],
|
||||
};
|
||||
assert_eq!(repo.reserve_usage_policy_request(request.clone()).await.unwrap(),
|
||||
ReserveUsagePolicyRequestOutcome::Rejected { window_index: 0, limit_requests: 2, used_requests: 2 });
|
||||
request.windows[0].limit_requests = 3;
|
||||
assert_eq!(repo.reserve_usage_policy_request(request.clone()).await.unwrap(),
|
||||
ReserveUsagePolicyRequestOutcome::Rejected { window_index: 1, limit_requests: 4, used_requests: 4 });
|
||||
request.windows[1].limit_requests = 5;
|
||||
assert_eq!(repo.reserve_usage_policy_request(request.clone()).await.unwrap(), ReserveUsagePolicyRequestOutcome::Allowed);
|
||||
assert_eq!(repo.reserve_usage_policy_request(request.clone()).await.unwrap(), ReserveUsagePolicyRequestOutcome::Allowed);
|
||||
assert_eq!(sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM usage_request_admissions WHERE event_token = 'new'")
|
||||
.fetch_one(&pool).await.unwrap(), 1);
|
||||
repo.release_usage_policy_request_admission(ReleaseUsagePolicyRequestAdmissionInput {
|
||||
request_id: request.request_id.clone(), subject_id: request.subject_id.clone(), event_token: request.event_token.clone(), released_at_unix_secs: 180,
|
||||
}).await.unwrap();
|
||||
assert_eq!(repo.reserve_usage_policy_request(request).await.unwrap(), ReserveUsagePolicyRequestOutcome::AlreadyReleased);
|
||||
|
||||
sqlx::raw_sql("INSERT INTO usage_cost_reservations (request_id, subject_id, reservation_token, admitted_at, reserved_cost_units, actual_cost_units, state, reservation_expires_at, retain_until, finalized_at) VALUES
|
||||
('older', 'subject', 'older', TO_TIMESTAMP(50), 99, 11, 'finalized', TO_TIMESTAMP(160), TO_TIMESTAMP(500), TO_TIMESTAMP(160)),
|
||||
('start', 'subject', 'start', TO_TIMESTAMP(100), 99, 7, 'finalized', TO_TIMESTAMP(160), TO_TIMESTAMP(500), TO_TIMESTAMP(160)),
|
||||
('inside', 'subject', 'inside', TO_TIMESTAMP(150), 5, NULL, 'reserved', TO_TIMESTAMP(300), TO_TIMESTAMP(500), NULL),
|
||||
('expired', 'subject', 'expired', TO_TIMESTAMP(150), 99, NULL, 'reserved', TO_TIMESTAMP(175), TO_TIMESTAMP(500), NULL),
|
||||
('end', 'subject', 'end', TO_TIMESTAMP(200), 99, 13, 'finalized', TO_TIMESTAMP(300), TO_TIMESTAMP(500), TO_TIMESTAMP(250)),
|
||||
('released', 'subject', 'released', TO_TIMESTAMP(150), 99, 0, 'released', TO_TIMESTAMP(300), TO_TIMESTAMP(500), TO_TIMESTAMP(170))")
|
||||
.execute(&pool).await.unwrap();
|
||||
let mut cost = ReserveUsagePolicyCostInput {
|
||||
request_id: "cost".to_string(), subject_id: "subject".to_string(), reservation_token: "cost".to_string(),
|
||||
admitted_at_unix_secs: 175, reserved_cost_units: 3, reservation_expires_at_unix_secs: 400, retain_until_unix_secs: 500,
|
||||
windows: vec![
|
||||
UsagePolicyCostWindow { window_id: "short".to_string(), starts_at_unix_secs: 100, ends_at_unix_secs: 200, limit_cost_units: 14 },
|
||||
UsagePolicyCostWindow { window_id: "long".to_string(), starts_at_unix_secs: 0, ends_at_unix_secs: 300, limit_cost_units: 38 },
|
||||
],
|
||||
};
|
||||
assert_eq!(repo.reserve_usage_policy_cost(cost.clone()).await.unwrap(),
|
||||
ReserveUsagePolicyCostOutcome::Rejected { window_index: 0, limit_cost_units: 14, used_cost_units: 12 });
|
||||
cost.windows[0].limit_cost_units = 15;
|
||||
assert_eq!(repo.reserve_usage_policy_cost(cost.clone()).await.unwrap(),
|
||||
ReserveUsagePolicyCostOutcome::Rejected { window_index: 1, limit_cost_units: 38, used_cost_units: 36 });
|
||||
cost.windows[1].limit_cost_units = 39;
|
||||
let allowed = repo.reserve_usage_policy_cost(cost.clone()).await.unwrap();
|
||||
assert!(matches!(allowed, ReserveUsagePolicyCostOutcome::Allowed { .. }), "{allowed:?}");
|
||||
let repeated = repo.reserve_usage_policy_cost(cost.clone()).await.unwrap();
|
||||
assert!(matches!(repeated, ReserveUsagePolicyCostOutcome::Allowed { .. }), "{repeated:?}");
|
||||
cost.reserved_cost_units = 4;
|
||||
assert_eq!(repo.reserve_usage_policy_cost(cost).await.unwrap(),
|
||||
ReserveUsagePolicyCostOutcome::Rejected { window_index: 0, limit_cost_units: 15, used_cost_units: 12 });
|
||||
|
||||
sqlx::query("DELETE FROM usage_request_admissions").execute(&pool).await.unwrap();
|
||||
let make_request = |id: &str| ReserveUsagePolicyRequestInput {
|
||||
request_id: id.to_string(), subject_id: "subject".to_string(), event_token: id.to_string(),
|
||||
admitted_at_unix_secs: 175, retain_until_unix_secs: 500,
|
||||
windows: vec![UsagePolicyRequestWindow { starts_at_unix_secs: 0, ends_at_unix_secs: 300, limit_requests: 1 }],
|
||||
};
|
||||
let (first, second) = tokio::join!(repo.reserve_usage_policy_request(make_request("race-a")), repo.reserve_usage_policy_request(make_request("race-b")));
|
||||
let outcomes = [first.unwrap(), second.unwrap()];
|
||||
assert_eq!(outcomes.iter().filter(|outcome| matches!(outcome, ReserveUsagePolicyRequestOutcome::Allowed)).count(), 1);
|
||||
assert_eq!(outcomes.iter().filter(|outcome| matches!(outcome, ReserveUsagePolicyRequestOutcome::Rejected { used_requests: 1, .. })).count(), 1);
|
||||
}).catch_unwind().await;
|
||||
sqlx::query(&format!("DROP SCHEMA {schema} CASCADE"))
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
pool.close().await;
|
||||
if let Err(panic) = result {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires AETHER_TEST_DATABASE_URL and PostgreSQL migrations"]
|
||||
async fn live_daily_quota_serializes_each_entitlement_without_locking_shared_plan() {
|
||||
let (pool, schema) = isolated_settlement_test_pool().await;
|
||||
let result = AssertUnwindSafe(async {
|
||||
let grant = serde_json::json!([{
|
||||
"type": "daily_quota",
|
||||
"daily_quota_usd": 10.0,
|
||||
"reset_timezone": "UTC",
|
||||
"allow_wallet_overage": false,
|
||||
}]);
|
||||
sqlx::query(
|
||||
"INSERT INTO billing_plans (id, title, price_amount, duration_unit, duration_value, entitlements_json, created_at, updated_at) VALUES ('shared-plan', 'Shared plan', 10, 'month', 1, $1, NOW(), NOW())",
|
||||
)
|
||||
.bind(&grant)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("shared plan should insert");
|
||||
for user_id in ["user-a", "user-b"] {
|
||||
sqlx::query(
|
||||
"INSERT INTO user_plan_entitlements (id, user_id, plan_id, payment_order_id, starts_at, expires_at, entitlements_snapshot, created_at, updated_at) VALUES ($1, $1, 'shared-plan', $1, NOW() - INTERVAL '1 hour', NOW() + INTERVAL '1 day', $2, NOW(), NOW())",
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(&grant)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("user entitlement should insert");
|
||||
}
|
||||
|
||||
let mut first = pool.begin().await.expect("first transaction should start");
|
||||
let first_debit = super::consume_daily_quota_postgres(
|
||||
&mut first, "user-a", "request-a", 7.0, Some(0.0), false,
|
||||
)
|
||||
.await
|
||||
.expect("first user should consume quota");
|
||||
assert_eq!(first_debit.debited_usd, 7.0);
|
||||
assert!(!first_debit.insufficient);
|
||||
|
||||
let mut second = pool.begin().await.expect("second transaction should start");
|
||||
sqlx::query("SET LOCAL lock_timeout = '500ms'")
|
||||
.execute(&mut *second)
|
||||
.await
|
||||
.expect("lock timeout should be configured");
|
||||
let second_debit = super::consume_daily_quota_postgres(
|
||||
&mut second, "user-b", "request-b", 2.0, Some(0.0), false,
|
||||
)
|
||||
.await
|
||||
.expect("another user's quota must not wait for the shared plan");
|
||||
assert_eq!(second_debit.debited_usd, 2.0);
|
||||
assert!(!second_debit.insufficient);
|
||||
second.commit().await.expect("second debit should commit");
|
||||
|
||||
let mut same_user = pool.begin().await.expect("contending transaction should start");
|
||||
sqlx::query("SET LOCAL lock_timeout = '500ms'")
|
||||
.execute(&mut *same_user)
|
||||
.await
|
||||
.expect("lock timeout should be configured");
|
||||
let blocked = super::consume_daily_quota_postgres(
|
||||
&mut same_user, "user-a", "request-a-next", 2.0, Some(0.0), false,
|
||||
)
|
||||
.await
|
||||
.expect_err("the same entitlement must remain locked until commit");
|
||||
assert!(blocked.to_string().contains("SQLSTATE 55P03"), "{blocked}");
|
||||
same_user.rollback().await.expect("blocked transaction should roll back");
|
||||
first.commit().await.expect("first debit should commit");
|
||||
|
||||
let mut next = pool.begin().await.expect("next transaction should start");
|
||||
let next_debit = super::consume_daily_quota_postgres(
|
||||
&mut next, "user-a", "request-a-next", 2.0, Some(0.0), false,
|
||||
)
|
||||
.await
|
||||
.expect("same user should consume the remaining quota after commit");
|
||||
assert_eq!(next_debit.debited_usd, 2.0);
|
||||
assert!(!next_debit.insufficient);
|
||||
next.commit().await.expect("next debit should commit");
|
||||
let balance: (f64, f64) = sqlx::query_as(
|
||||
"SELECT balance_before::double precision, balance_after::double precision FROM entitlement_usage_ledgers WHERE request_id = 'request-a-next'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("next debit ledger should exist");
|
||||
assert_eq!(balance, (3.0, 1.0));
|
||||
|
||||
let mut held = pool.begin().await.expect("quota transaction should start");
|
||||
super::consume_daily_quota_postgres(
|
||||
&mut held, "user-a", "request-policy-before", 0.5, Some(0.0), false,
|
||||
)
|
||||
.await
|
||||
.expect("quota transaction should retain its entitlement lock");
|
||||
let mut edit = pool.begin().await.expect("plan edit transaction should start");
|
||||
sqlx::query("SET LOCAL lock_timeout = '500ms'")
|
||||
.execute(&mut *edit)
|
||||
.await
|
||||
.expect("plan edit timeout should be configured");
|
||||
sqlx::query(
|
||||
"UPDATE billing_plans SET entitlements_json = jsonb_set(entitlements_json, '{0,allow_wallet_overage}', 'true'::jsonb) WHERE id = 'shared-plan'",
|
||||
)
|
||||
.execute(&mut *edit)
|
||||
.await
|
||||
.expect("plan configuration edits must not wait for usage settlement");
|
||||
edit.commit().await.expect("plan edit should commit");
|
||||
held.rollback().await.expect("held quota debit should roll back");
|
||||
|
||||
let mut after_edit = pool.begin().await.expect("fresh transaction should start");
|
||||
let updated_policy = super::consume_daily_quota_postgres(
|
||||
&mut after_edit, "user-a", "request-policy-after", 2.0, Some(5.0), true,
|
||||
)
|
||||
.await
|
||||
.expect("fresh quota read should use current plan configuration");
|
||||
assert!(!updated_policy.insufficient);
|
||||
assert_eq!(updated_policy.debited_usd, 1.0);
|
||||
after_edit.rollback().await.expect("policy verification should roll back");
|
||||
})
|
||||
.catch_unwind()
|
||||
.await;
|
||||
sqlx::query(&format!("DROP SCHEMA {schema} CASCADE"))
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("isolated settlement schema should be removed");
|
||||
pool.close().await;
|
||||
if let Err(panic) = result {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finalize_usage_billing_sql_does_not_require_usage_updated_at_column() {
|
||||
assert!(!super::FINALIZE_USAGE_BILLING_SQL.contains("updated_at"));
|
||||
|
||||
@@ -34,6 +34,14 @@ impl PostgresTransactionOptions {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn maintenance() -> Self {
|
||||
Self {
|
||||
mode: TransactionMode::ReadWrite,
|
||||
statement_timeout_ms: Some(300_000),
|
||||
lock_timeout_ms: Some(30_000),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<(), DataLayerError> {
|
||||
if matches!(self.statement_timeout_ms, Some(0)) {
|
||||
return Err(DataLayerError::InvalidConfiguration(
|
||||
|
||||
@@ -34,7 +34,7 @@ use sqlx::{
|
||||
PgPool, Postgres, QueryBuilder, Row,
|
||||
};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::io::Write;
|
||||
use std::io::{BufWriter, Write};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{
|
||||
@@ -58,6 +58,9 @@ use aether_data_contracts::repository::usage::{
|
||||
use aether_data_contracts::DataLayerError;
|
||||
|
||||
pub mod cleanup;
|
||||
mod preparation;
|
||||
|
||||
use preparation::prepare_usage_in_background;
|
||||
|
||||
// Legacy inline body columns on public.usage are deprecated. Keep the threshold at zero so
|
||||
// newly captured bodies always spill to usage_body_blobs and resolve through usage_http_audits.
|
||||
@@ -2121,12 +2124,9 @@ impl PreparedPendingUsage {
|
||||
));
|
||||
}
|
||||
|
||||
// Keep the capture input separate from the accounting row. The persistence sanitizer
|
||||
// intentionally removes HTTP bodies/headers/states, but the pending batch still needs
|
||||
// those values to populate the canonical audit/blob tables.
|
||||
let capture_usage = usage.clone();
|
||||
let usage = sanitize_usage_for_persistence(usage);
|
||||
let prepared = prepare_usage_upsert_context(&capture_usage)?;
|
||||
// Prepare captures before the accounting sanitizer removes HTTP bodies/headers/states.
|
||||
let (usage, prepared) = prepare_usage_for_persistence(usage);
|
||||
let prepared = prepared?;
|
||||
let input_tokens = usage
|
||||
.input_tokens
|
||||
.map(to_i32)
|
||||
@@ -8450,10 +8450,10 @@ ORDER BY "usage".user_id ASC
|
||||
usage: UpsertUsageRecord,
|
||||
) -> Result<StoredRequestUsageAudit, DataLayerError> {
|
||||
usage.validate()?;
|
||||
// `usage` is the sanitized accounting projection; prepare the auxiliary capture and
|
||||
// snapshots from the original event so typed `none` markers can clear prior facts.
|
||||
let capture_usage = usage.clone();
|
||||
let usage = sanitize_usage_for_persistence(usage);
|
||||
// Move the event before cloning or compressing captures, and do not hold a connection
|
||||
// while preparing them. Stale lifecycle updates still ignore preparation errors below.
|
||||
let (usage, prepared) =
|
||||
prepare_usage_in_background(move || Ok(prepare_usage_for_persistence(usage))).await?;
|
||||
self.tx_runner
|
||||
.run_read_write(|tx| {
|
||||
Box::pin(async move {
|
||||
@@ -8519,7 +8519,7 @@ ORDER BY "usage".user_id ASC
|
||||
clear_provider_request_body,
|
||||
clear_response_body,
|
||||
clear_client_response_body,
|
||||
} = prepare_usage_upsert_context(&capture_usage)?;
|
||||
} = prepared?;
|
||||
let capture_update_allowed = recovers_terminal_failure
|
||||
|| usage_capture_update_allowed(
|
||||
previous_usage.as_ref().map(|stored| {
|
||||
@@ -8938,33 +8938,36 @@ ORDER BY "usage".user_id ASC
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut request_id_counts = BTreeMap::<String, usize>::new();
|
||||
for usage in &usages {
|
||||
*request_id_counts
|
||||
.entry(usage.request_id.clone())
|
||||
.or_default() += 1;
|
||||
}
|
||||
|
||||
// Duplicate request IDs must retain the caller's exact sequential merge order. They are
|
||||
// uncommon in lifecycle batches, so keep them on the canonical single-row path.
|
||||
let mut batch_rows = Vec::<(usize, PreparedPendingUsage)>::new();
|
||||
let mut fallback_rows = Vec::<(usize, UpsertUsageRecord)>::new();
|
||||
for (sequence, usage) in usages.into_iter().enumerate() {
|
||||
let original_usage = usage.clone();
|
||||
let prepared = PreparedPendingUsage::try_from_usage(usage)?;
|
||||
if request_id_counts
|
||||
.get(&prepared.usage.request_id)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
== 1
|
||||
{
|
||||
batch_rows.push((sequence, prepared));
|
||||
} else {
|
||||
// Preserve capture markers for the canonical fallback; that path performs the
|
||||
// sanitized bind only after preparing the auxiliary audit/blob state.
|
||||
fallback_rows.push((sequence, original_usage));
|
||||
let (batch_rows, mut fallback_rows) = prepare_usage_in_background(move || {
|
||||
let mut request_id_counts = BTreeMap::<String, usize>::new();
|
||||
for usage in &usages {
|
||||
*request_id_counts
|
||||
.entry(usage.request_id.clone())
|
||||
.or_default() += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Duplicate request IDs must retain the caller's exact sequential merge order.
|
||||
let mut batch_rows = Vec::<(usize, PreparedPendingUsage)>::new();
|
||||
let mut fallback_rows = Vec::<(usize, UpsertUsageRecord)>::new();
|
||||
for (sequence, usage) in usages.into_iter().enumerate() {
|
||||
let duplicate = request_id_counts
|
||||
.get(&usage.request_id)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
> 1;
|
||||
let original_usage = duplicate.then(|| usage.clone());
|
||||
let prepared = PreparedPendingUsage::try_from_usage(usage)?;
|
||||
if let Some(original_usage) = original_usage {
|
||||
// Preserve capture markers for the canonical fallback, including validation
|
||||
// of every row before starting the batch transaction.
|
||||
fallback_rows.push((sequence, original_usage));
|
||||
} else {
|
||||
batch_rows.push((sequence, prepared));
|
||||
}
|
||||
}
|
||||
Ok((batch_rows, fallback_rows))
|
||||
})
|
||||
.await?;
|
||||
|
||||
let mut inserted_request_ids = BTreeSet::<String>::new();
|
||||
if !batch_rows.is_empty() {
|
||||
@@ -10294,7 +10297,7 @@ RETURNING
|
||||
|
||||
pub async fn rebuild_api_key_usage_stats(&self) -> Result<u64, DataLayerError> {
|
||||
self.tx_runner
|
||||
.run_read_write(|tx| {
|
||||
.run(crate::PostgresTransactionOptions::maintenance(), |tx| {
|
||||
Box::pin(async move {
|
||||
sqlx::query(RESET_API_KEY_USAGE_STATS_SQL)
|
||||
.execute(&mut **tx)
|
||||
@@ -10313,7 +10316,7 @@ RETURNING
|
||||
|
||||
pub async fn rebuild_provider_api_key_usage_stats(&self) -> Result<u64, DataLayerError> {
|
||||
self.tx_runner
|
||||
.run_read_write(|tx| {
|
||||
.run(crate::PostgresTransactionOptions::maintenance(), |tx| {
|
||||
Box::pin(async move {
|
||||
sqlx::query(RESET_PROVIDER_API_KEY_USAGE_STATS_SQL)
|
||||
.execute(&mut **tx)
|
||||
@@ -12359,24 +12362,35 @@ fn prepare_usage_body_storage(value: Option<&Value>) -> Result<UsageBodyStorage,
|
||||
detached_blob_bytes: None,
|
||||
});
|
||||
};
|
||||
let bytes = serde_json::to_vec(value).map_err(|err| {
|
||||
DataLayerError::UnexpectedValue(format!("failed to serialize usage json: {err}"))
|
||||
})?;
|
||||
if bytes.len() == MAX_INLINE_USAGE_BODY_BYTES {
|
||||
return Ok(UsageBodyStorage {
|
||||
inline_json: Some(String::from_utf8(bytes).map_err(|err| {
|
||||
DataLayerError::UnexpectedValue(format!(
|
||||
"failed to encode inline usage body as utf-8: {err}"
|
||||
))
|
||||
})?),
|
||||
detached_blob_bytes: None,
|
||||
});
|
||||
}
|
||||
|
||||
let mut encoder = GzEncoder::new(Vec::new(), Compression::new(6));
|
||||
encoder.write_all(&bytes).map_err(|err| {
|
||||
DataLayerError::UnexpectedValue(format!("failed to compress usage json: {err}"))
|
||||
})?;
|
||||
if MAX_INLINE_USAGE_BODY_BYTES == 0 {
|
||||
// Coalesce serde's punctuation/escape writes without allocating a full JSON buffer.
|
||||
let mut writer = BufWriter::with_capacity(8 * 1024, &mut encoder);
|
||||
serde_json::to_writer(&mut writer, value).map_err(|err| {
|
||||
let operation = if err.is_io() { "compress" } else { "serialize" };
|
||||
DataLayerError::UnexpectedValue(format!("failed to {operation} usage json: {err}"))
|
||||
})?;
|
||||
writer.into_inner().map_err(|err| {
|
||||
DataLayerError::UnexpectedValue(format!("failed to compress usage json: {err}"))
|
||||
})?;
|
||||
} else {
|
||||
let bytes = serde_json::to_vec(value).map_err(|err| {
|
||||
DataLayerError::UnexpectedValue(format!("failed to serialize usage json: {err}"))
|
||||
})?;
|
||||
if bytes.len() == MAX_INLINE_USAGE_BODY_BYTES {
|
||||
return Ok(UsageBodyStorage {
|
||||
inline_json: Some(String::from_utf8(bytes).map_err(|err| {
|
||||
DataLayerError::UnexpectedValue(format!(
|
||||
"failed to encode inline usage body as utf-8: {err}"
|
||||
))
|
||||
})?),
|
||||
detached_blob_bytes: None,
|
||||
});
|
||||
}
|
||||
encoder.write_all(&bytes).map_err(|err| {
|
||||
DataLayerError::UnexpectedValue(format!("failed to compress usage json: {err}"))
|
||||
})?;
|
||||
}
|
||||
let detached_blob_bytes = encoder.finish().map_err(|err| {
|
||||
DataLayerError::UnexpectedValue(format!("failed to finish usage json compression: {err}"))
|
||||
})?;
|
||||
@@ -12429,11 +12443,40 @@ fn project_usage_request_metadata(
|
||||
}
|
||||
}
|
||||
|
||||
fn prepare_usage_for_persistence(
|
||||
mut usage: UpsertUsageRecord,
|
||||
) -> (
|
||||
UpsertUsageRecord,
|
||||
Result<PreparedUsageUpsert, DataLayerError>,
|
||||
) {
|
||||
// Capture controls and accounting metadata have different sanitizers. Move the
|
||||
// large payloads out before copying the metadata needed by both projections.
|
||||
let request_body = usage.request_body.take();
|
||||
let provider_request_body = usage.provider_request_body.take();
|
||||
let response_body = usage.response_body.take();
|
||||
let client_response_body = usage.client_response_body.take();
|
||||
let request_headers = usage.request_headers.take();
|
||||
let provider_request_headers = usage.provider_request_headers.take();
|
||||
let response_headers = usage.response_headers.take();
|
||||
let client_response_headers = usage.client_response_headers.take();
|
||||
let mut capture = usage.clone();
|
||||
capture.request_body = request_body;
|
||||
capture.provider_request_body = provider_request_body;
|
||||
capture.response_body = response_body;
|
||||
capture.client_response_body = client_response_body;
|
||||
capture.request_headers = request_headers;
|
||||
capture.provider_request_headers = provider_request_headers;
|
||||
capture.response_headers = response_headers;
|
||||
capture.client_response_headers = client_response_headers;
|
||||
capture.capture_retention = std::mem::take(&mut usage.capture_retention);
|
||||
let capture = sanitize_usage_capture_controls_for_persistence(capture);
|
||||
let prepared = prepare_usage_upsert_context(&capture);
|
||||
(sanitize_usage_for_persistence(usage), prepared)
|
||||
}
|
||||
|
||||
fn prepare_usage_upsert_context(
|
||||
usage: &UpsertUsageRecord,
|
||||
) -> Result<PreparedUsageUpsert, DataLayerError> {
|
||||
let usage = sanitize_usage_capture_controls_for_persistence(usage.clone());
|
||||
let usage = &usage;
|
||||
let replace_client_request_body_facts = request_body_capture_replaces_derived_facts(
|
||||
usage.request_body.as_ref(),
|
||||
usage.request_body_state,
|
||||
|
||||
@@ -0,0 +1,327 @@
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::time::Duration;
|
||||
|
||||
use aether_data_contracts::DataLayerError;
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
static USAGE_PREPARATION_EXECUTOR: OnceLock<UsagePreparationExecutor> = OnceLock::new();
|
||||
|
||||
pub(super) async fn prepare_usage_in_background<T: Send + 'static>(
|
||||
prepare: impl FnOnce() -> Result<T, DataLayerError> + Send + 'static,
|
||||
) -> Result<T, DataLayerError> {
|
||||
USAGE_PREPARATION_EXECUTOR
|
||||
.get_or_init(|| {
|
||||
UsagePreparationExecutor::new(4, 32, Duration::from_secs(1), Duration::from_secs(30))
|
||||
})
|
||||
.run(prepare)
|
||||
.await
|
||||
}
|
||||
|
||||
struct UsagePreparationExecutor {
|
||||
workers: Arc<Semaphore>,
|
||||
admitted: Arc<Semaphore>,
|
||||
queue_timeout: Duration,
|
||||
execution_timeout: Duration,
|
||||
}
|
||||
|
||||
impl UsagePreparationExecutor {
|
||||
fn new(
|
||||
workers: usize,
|
||||
admitted: usize,
|
||||
queue_timeout: Duration,
|
||||
execution_timeout: Duration,
|
||||
) -> Self {
|
||||
Self {
|
||||
workers: Arc::new(Semaphore::new(workers)),
|
||||
admitted: Arc::new(Semaphore::new(admitted)),
|
||||
queue_timeout,
|
||||
execution_timeout,
|
||||
}
|
||||
}
|
||||
|
||||
async fn run<T: Send + 'static>(
|
||||
&self,
|
||||
prepare: impl FnOnce() -> Result<T, DataLayerError> + Send + 'static,
|
||||
) -> Result<T, DataLayerError> {
|
||||
// Bound both running work and callers retaining input while waiting for a worker.
|
||||
// These limits count tasks, not bytes in the caller's original usage records.
|
||||
let admitted = self.admitted.clone().try_acquire_owned().map_err(|_| {
|
||||
DataLayerError::TimedOut("usage preparation capacity exhausted".to_string())
|
||||
})?;
|
||||
let worker = tokio::time::timeout(self.queue_timeout, self.workers.clone().acquire_owned())
|
||||
.await
|
||||
.map_err(|_| {
|
||||
DataLayerError::TimedOut(
|
||||
"timed out waiting for usage preparation worker".to_string(),
|
||||
)
|
||||
})?
|
||||
.map_err(|_| {
|
||||
DataLayerError::TimedOut("usage preparation workers unavailable".to_string())
|
||||
})?;
|
||||
|
||||
// The closure owns both permits even if its caller times out or is cancelled. It
|
||||
// prepares input only; detached completion must never begin a database transaction.
|
||||
let mut task = tokio::task::spawn_blocking(move || {
|
||||
let _admitted = admitted;
|
||||
let _worker = worker;
|
||||
prepare()
|
||||
});
|
||||
match tokio::time::timeout(self.execution_timeout, &mut task).await {
|
||||
Ok(result) => result.map_err(|error| {
|
||||
DataLayerError::TimedOut(format!("usage preparation worker failed: {error}"))
|
||||
})?,
|
||||
Err(_) => {
|
||||
// This cancels work still queued in Tokio; running blocking work keeps its
|
||||
// permits until it actually exits, since abort cannot stop a blocking thread.
|
||||
task.abort();
|
||||
Err(DataLayerError::TimedOut(
|
||||
"timed out preparing usage storage".to_string(),
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::mpsc;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn executor(
|
||||
admitted: usize,
|
||||
queue_timeout: Duration,
|
||||
execution_timeout: Duration,
|
||||
) -> Arc<UsagePreparationExecutor> {
|
||||
Arc::new(UsagePreparationExecutor::new(
|
||||
1,
|
||||
admitted,
|
||||
queue_timeout,
|
||||
execution_timeout,
|
||||
))
|
||||
}
|
||||
|
||||
async fn wait_for_worker_release(executor: &UsagePreparationExecutor) {
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while executor.workers.available_permits() != 1
|
||||
|| executor.admitted.available_permits() == 0
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("finished blocking work should release its permits");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn preparation_runs_off_the_runtime_thread_and_preserves_errors() {
|
||||
let executor = executor(1, Duration::from_secs(1), Duration::from_secs(2));
|
||||
let runtime_thread = std::thread::current().id();
|
||||
executor
|
||||
.run(move || {
|
||||
assert_ne!(std::thread::current().id(), runtime_thread);
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.expect("preparation should succeed");
|
||||
|
||||
let error = executor
|
||||
.run(|| Err::<(), _>(DataLayerError::InvalidInput("bad usage".to_string())))
|
||||
.await
|
||||
.expect_err("input errors must reach the caller");
|
||||
assert!(matches!(error, DataLayerError::InvalidInput(message) if message == "bad usage"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn saturated_admission_rejects_work_without_running_it() {
|
||||
let executor = executor(1, Duration::from_secs(1), Duration::from_secs(2));
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let first_executor = executor.clone();
|
||||
let first = tokio::spawn(async move {
|
||||
first_executor
|
||||
.run(move || {
|
||||
let _ = started_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
started_rx.await.expect("first job should start");
|
||||
|
||||
let error = executor
|
||||
.run(|| -> Result<(), DataLayerError> { panic!("rejected work must not execute") })
|
||||
.await
|
||||
.expect_err("admission should fail immediately");
|
||||
assert!(matches!(error, DataLayerError::TimedOut(message) if message.contains("capacity")));
|
||||
release_tx
|
||||
.send(())
|
||||
.expect("first job should still be alive");
|
||||
first.await.unwrap().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn blocking_worker_failure_is_retryable_and_releases_capacity() {
|
||||
let executor = executor(1, Duration::from_secs(1), Duration::from_secs(2));
|
||||
let error = executor
|
||||
.run(|| -> Result<(), DataLayerError> { panic!("simulated worker failure") })
|
||||
.await
|
||||
.expect_err("worker failure must reach the caller");
|
||||
assert!(
|
||||
matches!(error, DataLayerError::TimedOut(message) if message.contains("worker failed"))
|
||||
);
|
||||
executor
|
||||
.run(|| Ok(()))
|
||||
.await
|
||||
.expect("failed workers should release capacity");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn waiting_for_a_worker_has_a_deadline_and_never_starts_expired_work() {
|
||||
let executor = executor(2, Duration::from_millis(20), Duration::from_secs(2));
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let first_executor = executor.clone();
|
||||
let first = tokio::spawn(async move {
|
||||
first_executor
|
||||
.run(move || {
|
||||
let _ = started_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
started_rx.await.expect("first job should start");
|
||||
let ran = Arc::new(AtomicBool::new(false));
|
||||
let work_ran = ran.clone();
|
||||
let error = executor
|
||||
.run(move || {
|
||||
work_ran.store(true, Ordering::SeqCst);
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.expect_err("the queued job should time out");
|
||||
assert!(matches!(error, DataLayerError::TimedOut(message) if message.contains("waiting")));
|
||||
assert_eq!(executor.admitted.available_permits(), 1);
|
||||
release_tx
|
||||
.send(())
|
||||
.expect("first job should still be alive");
|
||||
first.await.unwrap().unwrap();
|
||||
assert!(!ran.load(Ordering::SeqCst));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellation_keeps_permits_until_running_blocking_work_exits() {
|
||||
let executor = executor(1, Duration::from_secs(1), Duration::from_secs(2));
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let first_executor = executor.clone();
|
||||
let first = tokio::spawn(async move {
|
||||
first_executor
|
||||
.run(move || {
|
||||
let _ = started_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
started_rx.await.expect("first job should start");
|
||||
first.abort();
|
||||
assert!(first.await.unwrap_err().is_cancelled());
|
||||
assert_eq!(executor.workers.available_permits(), 0);
|
||||
assert!(matches!(
|
||||
executor.run(|| Ok(())).await,
|
||||
Err(DataLayerError::TimedOut(_))
|
||||
));
|
||||
release_tx
|
||||
.send(())
|
||||
.expect("blocking work should outlive cancellation");
|
||||
wait_for_worker_release(&executor).await;
|
||||
executor
|
||||
.run(|| Ok(()))
|
||||
.await
|
||||
.expect("the executor should recover");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellation_keeps_capture_budget_until_blocking_input_is_dropped() {
|
||||
use aether_data_contracts::repository::usage::{
|
||||
usage_json_heap_estimate, UpsertUsageRecord, UsageCaptureMemoryBudget,
|
||||
};
|
||||
|
||||
let mut usage: UpsertUsageRecord = serde_json::from_value(serde_json::json!({
|
||||
"request_id": "req-cancelled-preparation",
|
||||
"provider_name": "test",
|
||||
"model": "test",
|
||||
"status": "completed",
|
||||
"billing_status": "pending",
|
||||
"updated_at_unix_secs": 100,
|
||||
"request_body": {"content": "retained".repeat(1024)}
|
||||
}))
|
||||
.unwrap();
|
||||
let bytes = std::mem::size_of::<serde_json::Value>()
|
||||
+ usage_json_heap_estimate(usage.request_body.as_ref().unwrap());
|
||||
let budget = Arc::new(UsageCaptureMemoryBudget::new(bytes));
|
||||
assert!(usage.capture_retention.reserve(Arc::clone(&budget), bytes));
|
||||
let executor = executor(1, Duration::from_secs(1), Duration::from_secs(2));
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let first_executor = Arc::clone(&executor);
|
||||
let first = tokio::spawn(async move {
|
||||
first_executor
|
||||
.run(move || {
|
||||
let _ = started_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
drop(usage);
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
started_rx.await.unwrap();
|
||||
first.abort();
|
||||
assert!(first.await.unwrap_err().is_cancelled());
|
||||
assert_eq!(budget.retained_bytes(), bytes);
|
||||
release_tx.send(()).unwrap();
|
||||
wait_for_worker_release(&executor).await;
|
||||
assert_eq!(budget.retained_bytes(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execution_timeout_keeps_permits_until_running_blocking_work_exits() {
|
||||
let executor = executor(1, Duration::from_secs(1), Duration::from_millis(20));
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let first_executor = executor.clone();
|
||||
let first = tokio::spawn(async move {
|
||||
first_executor
|
||||
.run(move || {
|
||||
let _ = started_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
started_rx.await.expect("first job should start");
|
||||
let error = first
|
||||
.await
|
||||
.unwrap()
|
||||
.expect_err("running work should time out");
|
||||
assert!(
|
||||
matches!(error, DataLayerError::TimedOut(message) if message.contains("preparing"))
|
||||
);
|
||||
assert_eq!(executor.workers.available_permits(), 0);
|
||||
assert!(matches!(
|
||||
executor.run(|| Ok(())).await,
|
||||
Err(DataLayerError::TimedOut(_))
|
||||
));
|
||||
release_tx
|
||||
.send(())
|
||||
.expect("blocking work should outlive timeout");
|
||||
wait_for_worker_release(&executor).await;
|
||||
executor
|
||||
.run(|| Ok(()))
|
||||
.await
|
||||
.expect("the executor should recover");
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@ use super::{
|
||||
attach_usage_routing_snapshot_metadata, attach_usage_settlement_pricing_snapshot_metadata,
|
||||
clear_previous_request_body_facts, inflate_usage_json_value,
|
||||
prepare_request_metadata_for_body_storage, prepare_usage_body_storage,
|
||||
prepare_usage_upsert_context, push_postgres_usage_websocket_filter,
|
||||
prepare_usage_for_persistence, push_postgres_usage_websocket_filter,
|
||||
request_body_capture_replaces_derived_facts, resolved_read_usage_body_ref,
|
||||
resolved_write_usage_body_ref, split_dashboard_daily_aggregate_range,
|
||||
split_dashboard_hourly_aggregate_range, usage_body_capture_state_for_storage, usage_body_ref,
|
||||
@@ -39,6 +39,7 @@ fn fast_clear_usage_record(
|
||||
terminal_service_tier: Option<&str>,
|
||||
) -> UpsertUsageRecord {
|
||||
UpsertUsageRecord {
|
||||
capture_retention: Default::default(),
|
||||
request_id: request_id.to_string(),
|
||||
user_id: None,
|
||||
api_key_id: None,
|
||||
@@ -204,7 +205,26 @@ async fn live_full_http_capture_round_trips_for_direct_and_batch_writes() {
|
||||
let repository = SqlxUsageReadRepository::new(factory.connect_lazy().unwrap());
|
||||
crate::run_migrations(repository.pool()).await.unwrap();
|
||||
|
||||
for batch in [false, true] {
|
||||
for write_mode in 0..3 {
|
||||
use aether_data_contracts::repository::usage::{
|
||||
usage_json_heap_estimate, UsageCaptureMemoryBudget,
|
||||
};
|
||||
let batch = write_mode != 0;
|
||||
let budget = Arc::new(UsageCaptureMemoryBudget::new(4 * 1024 * 1024));
|
||||
let retain_capture = |usage: &mut UpsertUsageRecord| {
|
||||
let bytes = [
|
||||
usage.request_body.as_ref(),
|
||||
usage.provider_request_body.as_ref(),
|
||||
usage.response_body.as_ref(),
|
||||
usage.client_response_body.as_ref(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|body| std::mem::size_of::<serde_json::Value>() + usage_json_heap_estimate(body))
|
||||
.sum();
|
||||
assert!(usage.capture_retention.reserve(Arc::clone(&budget), bytes));
|
||||
bytes
|
||||
};
|
||||
let request_id = format!("req-full-capture-{}", uuid::Uuid::new_v4().simple());
|
||||
let now_unix_secs = Utc::now().timestamp() as u64;
|
||||
let mut pending = fast_clear_usage_record(
|
||||
@@ -225,14 +245,18 @@ async fn live_full_http_capture_round_trips_for_direct_and_batch_writes() {
|
||||
pending.response_body_state = Some(UsageBodyCaptureState::Inline);
|
||||
pending.client_response_body = Some(json!("pending client response"));
|
||||
pending.client_response_body_state = Some(UsageBodyCaptureState::Inline);
|
||||
let pending_bytes = retain_capture(&mut pending);
|
||||
if batch {
|
||||
repository
|
||||
.upsert_pending_many(vec![pending.clone()])
|
||||
.await
|
||||
.unwrap();
|
||||
let records = if write_mode == 2 {
|
||||
vec![pending.clone(), pending.clone()]
|
||||
} else {
|
||||
vec![pending.clone()]
|
||||
};
|
||||
repository.upsert_pending_many(records).await.unwrap();
|
||||
} else {
|
||||
repository.upsert(pending.clone()).await.unwrap();
|
||||
}
|
||||
assert_eq!(budget.retained_bytes(), pending_bytes);
|
||||
for (field, expected) in [
|
||||
(UsageBodyField::RequestBody, pending.request_body.as_ref()),
|
||||
(
|
||||
@@ -274,7 +298,10 @@ async fn live_full_http_capture_round_trips_for_direct_and_batch_writes() {
|
||||
terminal.response_body_state = Some(UsageBodyCaptureState::Inline);
|
||||
terminal.client_response_body = Some(json!({"output": "final response"}));
|
||||
terminal.client_response_body_state = Some(UsageBodyCaptureState::Inline);
|
||||
let terminal_bytes = retain_capture(&mut terminal);
|
||||
repository.upsert(terminal.clone()).await.unwrap();
|
||||
assert_eq!(budget.retained_bytes(), pending_bytes + terminal_bytes);
|
||||
assert_eq!(budget.downgraded_total(), 0);
|
||||
|
||||
let stored = repository
|
||||
.find_by_request_id_shallow(&request_id)
|
||||
@@ -330,6 +357,9 @@ async fn live_full_http_capture_round_trips_for_direct_and_batch_writes() {
|
||||
.execute(repository.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
drop(pending);
|
||||
drop(terminal);
|
||||
assert_eq!(budget.retained_bytes(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2411,6 +2441,7 @@ async fn validates_upsert_before_hitting_database() {
|
||||
let repository = SqlxUsageReadRepository::new(pool);
|
||||
let result = repository
|
||||
.upsert(UpsertUsageRecord {
|
||||
capture_retention: Default::default(),
|
||||
request_id: "".to_string(),
|
||||
user_id: None,
|
||||
api_key_id: None,
|
||||
@@ -4324,6 +4355,96 @@ fn prepare_usage_body_storage_compresses_large_payloads() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_usage_body_storage_streams_json_shapes_into_compatible_gzip() {
|
||||
for payload in [
|
||||
serde_json::Value::Null,
|
||||
json!(false),
|
||||
json!(42),
|
||||
json!(["quoted\"text", "line\nbreak", "\u{4e2d}\u{6587}", null]),
|
||||
json!({
|
||||
"content": "escaped\n\"\\value".repeat(32 * 1024),
|
||||
"nested": {"values": [true, null, 1.25, -7]}
|
||||
}),
|
||||
] {
|
||||
let storage = prepare_usage_body_storage(Some(&payload)).expect("body should compress");
|
||||
assert!(storage.inline_json.is_none());
|
||||
let compressed = storage
|
||||
.detached_blob_bytes
|
||||
.expect("body should be detached");
|
||||
assert_eq!(
|
||||
inflate_usage_json_value(&compressed).expect("body should remain readable"),
|
||||
payload
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn managed_capture_preparation_moves_bodies_without_a_second_reservation() {
|
||||
use aether_data_contracts::repository::usage::{
|
||||
sanitize_usage_for_persistence, usage_json_heap_estimate, UsageCaptureMemoryBudget,
|
||||
};
|
||||
|
||||
let mut usage = fast_clear_usage_record(
|
||||
"req-managed-capture",
|
||||
"managed-capture",
|
||||
100,
|
||||
true,
|
||||
UsageBodyCaptureState::Inline,
|
||||
Some("priority"),
|
||||
);
|
||||
let bodies = [
|
||||
json!({"messages": [{"role": "user", "content": "request".repeat(4096)}]}),
|
||||
json!({"input": "provider request".repeat(4096), "service_tier": "priority"}),
|
||||
json!({"output": "provider response".repeat(4096)}),
|
||||
json!({"output": "client response".repeat(4096)}),
|
||||
];
|
||||
usage.request_body = Some(bodies[0].clone());
|
||||
usage.provider_request_body = Some(bodies[1].clone());
|
||||
usage.response_body = Some(bodies[2].clone());
|
||||
usage.client_response_body = Some(bodies[3].clone());
|
||||
usage.request_body_state = Some(UsageBodyCaptureState::Inline);
|
||||
usage.response_body_state = Some(UsageBodyCaptureState::Inline);
|
||||
usage.client_response_body_state = Some(UsageBodyCaptureState::Inline);
|
||||
usage.request_headers = Some(json!({"content-type": "application/json"}));
|
||||
usage.cache_read_input_tokens = Some(0);
|
||||
usage.total_cost_usd = Some(0.25);
|
||||
usage.actual_total_cost_usd = Some(0.125);
|
||||
let expected_accounting = sanitize_usage_for_persistence(usage.clone());
|
||||
let bytes = [
|
||||
usage.request_body.as_ref(),
|
||||
usage.provider_request_body.as_ref(),
|
||||
usage.response_body.as_ref(),
|
||||
usage.client_response_body.as_ref(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|body| std::mem::size_of::<serde_json::Value>() + usage_json_heap_estimate(body))
|
||||
.sum();
|
||||
let budget = Arc::new(UsageCaptureMemoryBudget::new(bytes));
|
||||
assert!(usage.capture_retention.reserve(Arc::clone(&budget), bytes));
|
||||
|
||||
let (accounting, prepared) = prepare_usage_for_persistence(usage);
|
||||
let prepared = prepared.expect("managed capture should prepare without cloning bodies");
|
||||
assert_eq!(budget.retained_bytes(), 0);
|
||||
assert_eq!(budget.downgraded_total(), 0);
|
||||
assert_eq!(accounting, expected_accounting);
|
||||
for (storage, expected) in [
|
||||
prepared.request_body_storage,
|
||||
prepared.provider_request_body_storage,
|
||||
prepared.response_body_storage,
|
||||
prepared.client_response_body_storage,
|
||||
]
|
||||
.into_iter()
|
||||
.zip(bodies)
|
||||
{
|
||||
assert_eq!(
|
||||
inflate_usage_json_value(storage.detached_blob_bytes.as_deref().unwrap()).unwrap(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_body_capture_state_for_storage_marks_detached_bodies_as_reference() {
|
||||
let payload = json!({"message": "hello"});
|
||||
@@ -4413,7 +4534,8 @@ fn explicit_none_capture_drops_residual_body_ref_and_incoming_fast_metadata_befo
|
||||
"provider_request_body_ref": "usage://request/req-none-residual/provider_request_body"
|
||||
}));
|
||||
|
||||
let prepared = prepare_usage_upsert_context(&usage).expect("usage should prepare");
|
||||
let (_, prepared) = prepare_usage_for_persistence(usage);
|
||||
let prepared = prepared.expect("usage should prepare");
|
||||
assert!(prepared.clear_provider_request_body);
|
||||
assert!(!prepared.provider_request_body_storage.has_detached_blob());
|
||||
assert_eq!(prepared.http_audit_refs.provider_request_body_ref, None);
|
||||
@@ -4805,6 +4927,7 @@ fn attach_usage_http_audit_body_refs_adds_missing_metadata_without_overwriting_e
|
||||
fn usage_routing_snapshot_from_usage_only_activates_for_routing_metadata() {
|
||||
let snapshot = usage_routing_snapshot_from_usage(
|
||||
&UpsertUsageRecord {
|
||||
capture_retention: Default::default(),
|
||||
request_id: "req-123".to_string(),
|
||||
user_id: None,
|
||||
api_key_id: None,
|
||||
@@ -4904,6 +5027,7 @@ fn usage_routing_snapshot_from_usage_only_activates_for_routing_metadata() {
|
||||
|
||||
let empty_snapshot = usage_routing_snapshot_from_usage(
|
||||
&UpsertUsageRecord {
|
||||
capture_retention: Default::default(),
|
||||
request_id: "req-124".to_string(),
|
||||
user_id: None,
|
||||
api_key_id: None,
|
||||
@@ -4982,6 +5106,7 @@ fn usage_routing_snapshot_from_usage_only_activates_for_routing_metadata() {
|
||||
fn usage_routing_snapshot_from_usage_prefers_typed_routing_fields_without_metadata() {
|
||||
let snapshot = usage_routing_snapshot_from_usage(
|
||||
&UpsertUsageRecord {
|
||||
capture_retention: Default::default(),
|
||||
request_id: "req-typed-routing-1".to_string(),
|
||||
user_id: None,
|
||||
api_key_id: None,
|
||||
@@ -5116,6 +5241,7 @@ fn attach_usage_routing_snapshot_metadata_adds_missing_keys_without_overwriting_
|
||||
fn usage_settlement_pricing_snapshot_from_usage_extracts_typed_billing_fields() {
|
||||
let snapshot = usage_settlement_pricing_snapshot_from_usage(
|
||||
&UpsertUsageRecord {
|
||||
capture_retention: Default::default(),
|
||||
request_id: "req-125".to_string(),
|
||||
user_id: None,
|
||||
api_key_id: None,
|
||||
|
||||
Reference in New Issue
Block a user