Add multi-database data layer

Introduce aether-data-schema and driver-specific schema generation for Postgres, MySQL, and SQLite.

Split data backends, lifecycle, repositories, and gateway runtime integration across database drivers.

Verified with cargo fmt --all --check, cargo clippy --workspace --all-targets -- -D warnings, and cargo test --workspace.

crates/aether-data/src/repository/usage/mod.rs

@@ -1,11 +1,349 @@
+macro_rules! impl_materialized_usage_read_repository {
+    ($repository:ty) => {
+        #[async_trait::async_trait]
+        impl $crate::repository::usage::UsageReadRepository for $repository {
+            async fn find_by_id(
+                &self,
+                id: &str,
+            ) -> Result<
+                Option<$crate::repository::usage::StoredRequestUsageAudit>,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::find_by_id(&repository, id).await
+            }
+
+            async fn list_by_ids(
+                &self,
+                ids: &[String],
+            ) -> Result<
+                Vec<$crate::repository::usage::StoredRequestUsageAudit>,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::list_by_ids(&repository, ids).await
+            }
+
+            async fn find_by_request_id(
+                &self,
+                request_id: &str,
+            ) -> Result<
+                Option<$crate::repository::usage::StoredRequestUsageAudit>,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::find_by_request_id(&repository, request_id).await
+            }
+
+            async fn resolve_body_ref(
+                &self,
+                body_ref: &str,
+            ) -> Result<Option<serde_json::Value>, $crate::DataLayerError> {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::resolve_body_ref(&repository, body_ref).await
+            }
+
+            async fn list_usage_audits(
+                &self,
+                query: &$crate::repository::usage::UsageAuditListQuery,
+            ) -> Result<
+                Vec<$crate::repository::usage::StoredRequestUsageAudit>,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::list_usage_audits(&repository, query).await
+            }
+
+            async fn count_usage_audits(
+                &self,
+                query: &$crate::repository::usage::UsageAuditListQuery,
+            ) -> Result<u64, $crate::DataLayerError> {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::count_usage_audits(&repository, query).await
+            }
+
+            async fn list_usage_audits_by_keyword_search(
+                &self,
+                query: &$crate::repository::usage::UsageAuditKeywordSearchQuery,
+            ) -> Result<
+                Vec<$crate::repository::usage::StoredRequestUsageAudit>,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::list_usage_audits_by_keyword_search(&repository, query).await
+            }
+
+            async fn count_usage_audits_by_keyword_search(
+                &self,
+                query: &$crate::repository::usage::UsageAuditKeywordSearchQuery,
+            ) -> Result<u64, $crate::DataLayerError> {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::count_usage_audits_by_keyword_search(&repository, query).await
+            }
+
+            async fn aggregate_usage_audits(
+                &self,
+                query: &$crate::repository::usage::UsageAuditAggregationQuery,
+            ) -> Result<
+                Vec<$crate::repository::usage::StoredUsageAuditAggregation>,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::aggregate_usage_audits(&repository, query).await
+            }
+
+            async fn summarize_usage_audits(
+                &self,
+                query: &$crate::repository::usage::UsageAuditSummaryQuery,
+            ) -> Result<
+                $crate::repository::usage::StoredUsageAuditSummary,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_audits(&repository, query).await
+            }
+
+            async fn summarize_usage_totals_by_user_ids(
+                &self,
+                user_ids: &[String],
+            ) -> Result<Vec<$crate::repository::usage::StoredUsageUserTotals>, $crate::DataLayerError>
+            {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_totals_by_user_ids(&repository, user_ids).await
+            }
+
+            async fn summarize_usage_cache_hit_summary(
+                &self,
+                query: &$crate::repository::usage::UsageCacheHitSummaryQuery,
+            ) -> Result<
+                $crate::repository::usage::StoredUsageCacheHitSummary,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_cache_hit_summary(&repository, query).await
+            }
+
+            async fn summarize_usage_settled_cost(
+                &self,
+                query: &$crate::repository::usage::UsageSettledCostSummaryQuery,
+            ) -> Result<
+                $crate::repository::usage::StoredUsageSettledCostSummary,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_settled_cost(&repository, query).await
+            }
+
+            async fn summarize_usage_cache_affinity_hit_summary(
+                &self,
+                query: &$crate::repository::usage::UsageCacheAffinityHitSummaryQuery,
+            ) -> Result<
+                $crate::repository::usage::StoredUsageCacheAffinityHitSummary,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_cache_affinity_hit_summary(&repository, query).await
+            }
+
+            async fn list_usage_cache_affinity_intervals(
+                &self,
+                query: &$crate::repository::usage::UsageCacheAffinityIntervalQuery,
+            ) -> Result<
+                Vec<$crate::repository::usage::StoredUsageCacheAffinityIntervalRow>,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::list_usage_cache_affinity_intervals(&repository, query).await
+            }
+
+            async fn summarize_dashboard_usage(
+                &self,
+                query: &$crate::repository::usage::UsageDashboardSummaryQuery,
+            ) -> Result<
+                $crate::repository::usage::StoredUsageDashboardSummary,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_dashboard_usage(&repository, query).await
+            }
+
+            async fn list_dashboard_daily_breakdown(
+                &self,
+                query: &$crate::repository::usage::UsageDashboardDailyBreakdownQuery,
+            ) -> Result<
+                Vec<$crate::repository::usage::StoredUsageDashboardDailyBreakdownRow>,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::list_dashboard_daily_breakdown(&repository, query).await
+            }
+
+            async fn summarize_dashboard_provider_counts(
+                &self,
+                query: &$crate::repository::usage::UsageDashboardProviderCountsQuery,
+            ) -> Result<
+                Vec<$crate::repository::usage::StoredUsageDashboardProviderCount>,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_dashboard_provider_counts(&repository, query).await
+            }
+
+            async fn summarize_usage_breakdown(
+                &self,
+                query: &$crate::repository::usage::UsageBreakdownSummaryQuery,
+            ) -> Result<
+                Vec<$crate::repository::usage::StoredUsageBreakdownSummaryRow>,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_breakdown(&repository, query).await
+            }
+
+            async fn count_monitoring_usage_errors(
+                &self,
+                query: &$crate::repository::usage::UsageMonitoringErrorCountQuery,
+            ) -> Result<u64, $crate::DataLayerError> {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::count_monitoring_usage_errors(&repository, query).await
+            }
+
+            async fn list_monitoring_usage_errors(
+                &self,
+                query: &$crate::repository::usage::UsageMonitoringErrorListQuery,
+            ) -> Result<
+                Vec<$crate::repository::usage::StoredRequestUsageAudit>,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::list_monitoring_usage_errors(&repository, query).await
+            }
+
+            async fn summarize_usage_error_distribution(
+                &self,
+                query: &$crate::repository::usage::UsageErrorDistributionQuery,
+            ) -> Result<
+                Vec<$crate::repository::usage::StoredUsageErrorDistributionRow>,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_error_distribution(&repository, query).await
+            }
+
+            async fn summarize_usage_performance_percentiles(
+                &self,
+                query: &$crate::repository::usage::UsagePerformancePercentilesQuery,
+            ) -> Result<
+                Vec<$crate::repository::usage::StoredUsagePerformancePercentilesRow>,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_performance_percentiles(&repository, query).await
+            }
+
+            async fn summarize_usage_cost_savings(
+                &self,
+                query: &$crate::repository::usage::UsageCostSavingsSummaryQuery,
+            ) -> Result<
+                $crate::repository::usage::StoredUsageCostSavingsSummary,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_cost_savings(&repository, query).await
+            }
+
+            async fn summarize_usage_time_series(
+                &self,
+                query: &$crate::repository::usage::UsageTimeSeriesQuery,
+            ) -> Result<
+                Vec<$crate::repository::usage::StoredUsageTimeSeriesBucket>,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_time_series(&repository, query).await
+            }
+
+            async fn summarize_usage_leaderboard(
+                &self,
+                query: &$crate::repository::usage::UsageLeaderboardQuery,
+            ) -> Result<
+                Vec<$crate::repository::usage::StoredUsageLeaderboardSummary>,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_leaderboard(&repository, query).await
+            }
+
+            async fn list_recent_usage_audits(
+                &self,
+                user_id: Option<&str>,
+                limit: usize,
+            ) -> Result<
+                Vec<$crate::repository::usage::StoredRequestUsageAudit>,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::list_recent_usage_audits(&repository, user_id, limit).await
+            }
+
+            async fn summarize_total_tokens_by_api_key_ids(
+                &self,
+                api_key_ids: &[String],
+            ) -> Result<std::collections::BTreeMap<String, u64>, $crate::DataLayerError> {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_total_tokens_by_api_key_ids(&repository, api_key_ids).await
+            }
+
+            async fn summarize_usage_by_provider_api_key_ids(
+                &self,
+                provider_api_key_ids: &[String],
+            ) -> Result<
+                std::collections::BTreeMap<
+                    String,
+                    $crate::repository::usage::StoredProviderApiKeyUsageSummary,
+                >,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_by_provider_api_key_ids(&repository, provider_api_key_ids).await
+            }
+
+            async fn summarize_provider_usage_since(
+                &self,
+                provider_id: &str,
+                since_unix_secs: u64,
+            ) -> Result<
+                $crate::repository::usage::StoredProviderUsageSummary,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_provider_usage_since(&repository, provider_id, since_unix_secs).await
+            }
+
+            async fn summarize_usage_daily_heatmap(
+                &self,
+                query: &$crate::repository::usage::UsageDailyHeatmapQuery,
+            ) -> Result<
+                Vec<$crate::repository::usage::StoredUsageDailySummary>,
+                $crate::DataLayerError,
+            > {
+                let repository = self.materialize_read_model().await?;
+                <$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_daily_heatmap(&repository, query).await
+            }
+        }
+    };
+}
+
 mod memory;
-mod sql;
+mod mysql;
+mod postgres;
+mod sqlite;
 
 #[allow(unused_imports)]
 pub(crate) use aether_data_contracts::repository::usage::{
-    StoredProviderApiKeyUsageSummary, StoredProviderUsageSummary, StoredProviderUsageWindow,
-    StoredRequestUsageAudit, StoredUsageAuditAggregation, StoredUsageAuditSummary,
-    StoredUsageBreakdownSummaryRow, StoredUsageCacheAffinityHitSummary,
+    PendingUsageCleanupSummary, StoredProviderApiKeyUsageSummary, StoredProviderUsageSummary,
+    StoredProviderUsageWindow, StoredRequestUsageAudit, StoredUsageAuditAggregation,
+    StoredUsageAuditSummary, StoredUsageBreakdownSummaryRow, StoredUsageCacheAffinityHitSummary,
     StoredUsageCacheAffinityIntervalRow, StoredUsageCacheHitSummary, StoredUsageCostSavingsSummary,
     StoredUsageDailySummary, StoredUsageDashboardDailyBreakdownRow,
     StoredUsageDashboardProviderCount, StoredUsageDashboardSummary,
@@ -15,16 +353,22 @@ pub(crate) use aether_data_contracts::repository::usage::{
     UsageAuditAggregationGroupBy, UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery,
     UsageAuditListQuery, UsageAuditSummaryQuery, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery,
     UsageCacheAffinityHitSummaryQuery, UsageCacheAffinityIntervalGroupBy,
-    UsageCacheAffinityIntervalQuery, UsageCacheHitSummaryQuery, UsageCostSavingsSummaryQuery,
-    UsageDailyHeatmapQuery, UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
+    UsageCacheAffinityIntervalQuery, UsageCacheHitSummaryQuery, UsageCleanupSummary,
+    UsageCleanupWindow, UsageCostSavingsSummaryQuery, UsageDailyHeatmapQuery,
+    UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
     UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy,
     UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery,
     UsagePerformancePercentilesQuery, UsageReadRepository, UsageRepository,
     UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity, UsageTimeSeriesQuery,
     UsageWriteRepository,
 };
+pub mod cleanup {
+    pub use super::postgres::cleanup::*;
+}
 pub use memory::InMemoryUsageReadRepository;
-pub use sql::SqlxUsageReadRepository;
+pub use mysql::{MysqlUsageReadRepository, MysqlUsageWriteRepository};
+pub use postgres::SqlxUsageReadRepository;
+pub use sqlite::{SqliteUsageReadRepository, SqliteUsageWriteRepository};
 
 #[derive(Debug, Clone, PartialEq, Default)]
 pub(crate) struct ApiKeyUsageContribution {

crates/aether-data/src/repository/usage/postgres/mod.rs

@@ -9,12 +9,12 @@ use aether_data_contracts::repository::usage::{
     UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery, UsageAuditSummaryQuery,
     UsageBodyCaptureState, UsageBodyField, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery,
     UsageCacheAffinityHitSummaryQuery, UsageCacheAffinityIntervalGroupBy,
-    UsageCacheAffinityIntervalQuery, UsageCacheHitSummaryQuery, UsageCostSavingsSummaryQuery,
-    UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
-    UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy,
-    UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery,
-    UsagePerformancePercentilesQuery, UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity,
-    UsageTimeSeriesQuery,
+    UsageCacheAffinityIntervalQuery, UsageCacheHitSummaryQuery, UsageCleanupSummary,
+    UsageCleanupWindow, UsageCostSavingsSummaryQuery, UsageDashboardDailyBreakdownQuery,
+    UsageDashboardProviderCountsQuery, UsageDashboardSummaryQuery, UsageErrorDistributionQuery,
+    UsageLeaderboardGroupBy, UsageLeaderboardQuery, UsageMonitoringErrorCountQuery,
+    UsageMonitoringErrorListQuery, UsagePerformancePercentilesQuery, UsageSettledCostSummaryQuery,
+    UsageTimeSeriesGranularity, UsageTimeSeriesQuery,
 };
 use async_trait::async_trait;
 use chrono::{DateTime, Utc};
@@ -35,16 +35,19 @@ use uuid::Uuid;
 use super::{
     api_key_usage_contribution, incoming_usage_can_recover_terminal_failure,
     provider_api_key_usage_contribution, strip_deprecated_usage_display_fields, ApiKeyUsageDelta,
-    ProviderApiKeyUsageDelta, StoredProviderApiKeyUsageSummary, StoredProviderUsageSummary,
-    StoredRequestUsageAudit, StoredUsageDailySummary, UpsertUsageRecord, UsageAuditListQuery,
-    UsageDailyHeatmapQuery, UsageReadRepository, UsageWriteRepository,
+    PendingUsageCleanupSummary, ProviderApiKeyUsageDelta, StoredProviderApiKeyUsageSummary,
+    StoredProviderUsageSummary, StoredRequestUsageAudit, StoredUsageDailySummary,
+    UpsertUsageRecord, UsageAuditListQuery, UsageDailyHeatmapQuery, UsageReadRepository,
+    UsageWriteRepository,
 };
-use crate::postgres::PostgresTransactionRunner;
+use crate::driver::postgres::PostgresTransactionRunner;
 use crate::{
     error::{postgres_error, SqlxResultExt},
     DataLayerError,
 };
 
+pub mod cleanup;
+
 // Legacy inline body columns on public.usage are deprecated. Keep the threshold at zero so
 // newly captured bodies always spill to usage_body_blobs and resolve through usage_http_audits.
 const MAX_INLINE_USAGE_BODY_BYTES: usize = 0;
@@ -1062,6 +1065,99 @@ const LIST_RECENT_USAGE_AUDITS_PREFIX: &str =
 
 const UPSERT_SQL: &str = include_str!("queries/upsert_sql.sql");
 
+const SELECT_STALE_PENDING_USAGE_BATCH_SQL: &str = r#"
+SELECT
+  usage.request_id,
+  usage.status,
+  COALESCE(usage_settlement_snapshots.billing_status, usage.billing_status) AS billing_status
+FROM usage
+LEFT JOIN usage_settlement_snapshots
+  ON usage_settlement_snapshots.request_id = usage.request_id
+WHERE usage.status IN ('pending', 'streaming')
+  AND usage.created_at < $1
+ORDER BY usage.created_at ASC, usage.request_id ASC
+LIMIT $2
+FOR UPDATE OF usage SKIP LOCKED
+"#;
+
+const SELECT_COMPLETED_PENDING_REQUEST_IDS_SQL: &str = r#"
+SELECT DISTINCT request_id
+FROM request_candidates
+WHERE request_id = ANY($1)
+  AND (
+    status = 'streaming'
+    OR (
+      status = 'success'
+      AND COALESCE(extra_data->>'stream_completed', 'false') = 'true'
+    )
+  )
+"#;
+
+const UPDATE_RECOVERED_STALE_USAGE_SQL: &str = r#"
+UPDATE usage
+SET status = 'completed',
+    status_code = 200,
+    error_message = NULL
+WHERE request_id = $1
+"#;
+
+const UPDATE_FAILED_STALE_USAGE_SQL: &str = r#"
+UPDATE usage
+SET status = 'failed',
+    status_code = 504,
+    error_message = $2
+WHERE request_id = $1
+"#;
+
+const UPDATE_FAILED_VOID_STALE_USAGE_SQL: &str = r#"
+WITH updated_usage AS (
+    UPDATE usage
+    SET status = 'failed',
+        status_code = 504,
+        error_message = $2,
+        billing_status = 'void',
+        finalized_at = $3,
+        total_cost_usd = 0,
+        request_cost_usd = 0,
+        actual_total_cost_usd = 0,
+        actual_request_cost_usd = 0
+    WHERE request_id = $1
+    RETURNING request_id
+)
+INSERT INTO usage_settlement_snapshots (
+    request_id,
+    billing_status,
+    finalized_at
+)
+SELECT request_id, 'void', $3
+FROM updated_usage
+ON CONFLICT (request_id)
+DO UPDATE SET
+    billing_status = EXCLUDED.billing_status,
+    finalized_at = COALESCE(
+        usage_settlement_snapshots.finalized_at,
+        EXCLUDED.finalized_at
+    ),
+    updated_at = NOW()
+"#;
+
+const UPDATE_RECOVERED_STREAMING_CANDIDATES_SQL: &str = r#"
+UPDATE request_candidates
+SET status = 'success',
+    finished_at = $2
+WHERE request_id = $1
+  AND status = 'streaming'
+"#;
+
+const UPDATE_FAILED_PENDING_CANDIDATES_SQL: &str = r#"
+UPDATE request_candidates
+SET status = 'failed',
+    finished_at = $2,
+    error_message = '请求超时（服务器可能已重启）'
+WHERE request_id = $1
+  AND status IN ('pending', 'streaming')
+"#;
+
 #[derive(Debug, Clone)]
 pub struct SqlxUsageReadRepository {
     pool: PgPool,
@@ -1141,12 +1237,12 @@ SELECT
   COALESCE(SUM(cache_creation_tokens), 0)::BIGINT AS cache_creation_tokens,
   COALESCE(SUM(cache_read_tokens), 0)::BIGINT AS cache_read_tokens,
   COALESCE(SUM(total_input_context), 0)::BIGINT AS total_input_context,
-  COALESCE(SUM(cache_creation_cost), 0) AS cache_creation_cost_usd,
-  COALESCE(SUM(cache_read_cost), 0) AS cache_read_cost_usd,
-  COALESCE(SUM(total_cost), 0) AS total_cost_usd,
-  COALESCE(SUM(actual_total_cost), 0) AS actual_total_cost_usd,
+  COALESCE(SUM(cache_creation_cost), 0)::DOUBLE PRECISION AS cache_creation_cost_usd,
+  COALESCE(SUM(cache_read_cost), 0)::DOUBLE PRECISION AS cache_read_cost_usd,
+  COALESCE(SUM(total_cost), 0)::DOUBLE PRECISION AS total_cost_usd,
+  COALESCE(SUM(actual_total_cost), 0)::DOUBLE PRECISION AS actual_total_cost_usd,
   COALESCE(SUM(error_requests), 0)::BIGINT AS error_requests,
-  COALESCE(SUM(response_time_sum_ms), 0) AS response_time_sum_ms,
+  COALESCE(SUM(response_time_sum_ms), 0)::DOUBLE PRECISION AS response_time_sum_ms,
   COALESCE(SUM(response_time_samples), 0)::BIGINT AS response_time_samples
 FROM stats_user_daily
 WHERE user_id = $1
@@ -1172,12 +1268,12 @@ SELECT
   COALESCE(SUM(cache_creation_tokens), 0)::BIGINT AS cache_creation_tokens,
   COALESCE(SUM(cache_read_tokens), 0)::BIGINT AS cache_read_tokens,
   COALESCE(SUM(total_input_context), 0)::BIGINT AS total_input_context,
-  COALESCE(SUM(cache_creation_cost), 0) AS cache_creation_cost_usd,
-  COALESCE(SUM(cache_read_cost), 0) AS cache_read_cost_usd,
-  COALESCE(SUM(total_cost), 0) AS total_cost_usd,
-  COALESCE(SUM(actual_total_cost), 0) AS actual_total_cost_usd,
+  COALESCE(SUM(cache_creation_cost), 0)::DOUBLE PRECISION AS cache_creation_cost_usd,
+  COALESCE(SUM(cache_read_cost), 0)::DOUBLE PRECISION AS cache_read_cost_usd,
+  COALESCE(SUM(total_cost), 0)::DOUBLE PRECISION AS total_cost_usd,
+  COALESCE(SUM(actual_total_cost), 0)::DOUBLE PRECISION AS actual_total_cost_usd,
   COALESCE(SUM(error_requests), 0)::BIGINT AS error_requests,
-  COALESCE(SUM(response_time_sum_ms), 0) AS response_time_sum_ms,
+  COALESCE(SUM(response_time_sum_ms), 0)::DOUBLE PRECISION AS response_time_sum_ms,
   COALESCE(SUM(response_time_samples), 0)::BIGINT AS response_time_samples
 FROM stats_daily
 WHERE date >= $1
@@ -2171,11 +2267,11 @@ SELECT
   COALESCE(SUM(cache_creation_ephemeral_1h_tokens), 0)::BIGINT
     AS cache_creation_ephemeral_1h_tokens,
   COALESCE(SUM(cache_read_tokens), 0)::BIGINT AS cache_read_tokens,
-  COALESCE(SUM(total_cost), 0) AS total_cost_usd,
-  COALESCE(SUM(actual_total_cost), 0) AS actual_total_cost_usd,
-  COALESCE(SUM(cache_creation_cost), 0) AS cache_creation_cost_usd,
-  COALESCE(SUM(cache_read_cost), 0) AS cache_read_cost_usd,
-  COALESCE(SUM(response_time_sum_ms), 0) AS total_response_time_ms,
+  COALESCE(SUM(total_cost), 0)::DOUBLE PRECISION AS total_cost_usd,
+  COALESCE(SUM(actual_total_cost), 0)::DOUBLE PRECISION AS actual_total_cost_usd,
+  COALESCE(SUM(cache_creation_cost), 0)::DOUBLE PRECISION AS cache_creation_cost_usd,
+  COALESCE(SUM(cache_read_cost), 0)::DOUBLE PRECISION AS cache_read_cost_usd,
+  COALESCE(SUM(response_time_sum_ms), 0)::DOUBLE PRECISION AS total_response_time_ms,
   COALESCE(SUM(error_requests), 0)::BIGINT AS error_requests
 FROM stats_user_daily
 WHERE user_id = $1
@@ -2203,11 +2299,11 @@ SELECT
   COALESCE(SUM(cache_creation_ephemeral_1h_tokens), 0)::BIGINT
     AS cache_creation_ephemeral_1h_tokens,
   COALESCE(SUM(cache_read_tokens), 0)::BIGINT AS cache_read_tokens,
-  COALESCE(SUM(total_cost), 0) AS total_cost_usd,
-  COALESCE(SUM(actual_total_cost), 0) AS actual_total_cost_usd,
-  COALESCE(SUM(cache_creation_cost), 0) AS cache_creation_cost_usd,
-  COALESCE(SUM(cache_read_cost), 0) AS cache_read_cost_usd,
-  COALESCE(SUM(response_time_sum_ms), 0) AS total_response_time_ms,
+  COALESCE(SUM(total_cost), 0)::DOUBLE PRECISION AS total_cost_usd,
+  COALESCE(SUM(actual_total_cost), 0)::DOUBLE PRECISION AS actual_total_cost_usd,
+  COALESCE(SUM(cache_creation_cost), 0)::DOUBLE PRECISION AS cache_creation_cost_usd,
+  COALESCE(SUM(cache_read_cost), 0)::DOUBLE PRECISION AS cache_read_cost_usd,
+  COALESCE(SUM(response_time_sum_ms), 0)::DOUBLE PRECISION AS total_response_time_ms,
   COALESCE(SUM(error_requests), 0)::BIGINT AS error_requests
 FROM stats_daily
 WHERE date >= $1
@@ -5749,13 +5845,13 @@ WHERE "usage".created_at >= $1
                 r#"
 SELECT
   date,
-  total_requests,
-  input_tokens,
-  output_tokens,
-  cache_creation_tokens,
-  cache_read_tokens,
-  total_cost,
-  COALESCE(actual_total_cost, 0) AS actual_total_cost
+  total_requests::BIGINT AS total_requests,
+  input_tokens::BIGINT AS input_tokens,
+  output_tokens::BIGINT AS output_tokens,
+  cache_creation_tokens::BIGINT AS cache_creation_tokens,
+  cache_read_tokens::BIGINT AS cache_read_tokens,
+  total_cost::DOUBLE PRECISION AS total_cost,
+  COALESCE(actual_total_cost, 0)::DOUBLE PRECISION AS actual_total_cost
 FROM stats_user_daily
 WHERE user_id = $1
   AND date >= $2
@@ -5772,13 +5868,13 @@ ORDER BY date ASC
                 r#"
 SELECT
   date,
-  total_requests,
-  input_tokens,
-  output_tokens,
-  cache_creation_tokens,
-  cache_read_tokens,
-  total_cost,
-  actual_total_cost
+  total_requests::BIGINT AS total_requests,
+  input_tokens::BIGINT AS input_tokens,
+  output_tokens::BIGINT AS output_tokens,
+  cache_creation_tokens::BIGINT AS cache_creation_tokens,
+  cache_read_tokens::BIGINT AS cache_read_tokens,
+  total_cost::DOUBLE PRECISION AS total_cost,
+  actual_total_cost::DOUBLE PRECISION AS actual_total_cost
 FROM stats_daily
 WHERE date >= $1
   AND date < $2
@@ -6586,6 +6682,134 @@ ORDER BY "usage".user_id ASC
             .await
     }
 
+    pub async fn cleanup_stale_pending_requests(
+        &self,
+        cutoff_unix_secs: u64,
+        now_unix_secs: u64,
+        timeout_minutes: u64,
+        batch_size: usize,
+    ) -> Result<PendingUsageCleanupSummary, DataLayerError> {
+        if batch_size == 0 {
+            return Ok(PendingUsageCleanupSummary::default());
+        }
+
+        let cutoff_timestamp = i64::try_from(cutoff_unix_secs).map_err(|_| {
+            DataLayerError::InvalidInput(format!(
+                "invalid stale pending usage cutoff: {cutoff_unix_secs}"
+            ))
+        })?;
+        let now_timestamp = i64::try_from(now_unix_secs).map_err(|_| {
+            DataLayerError::InvalidInput(format!(
+                "invalid stale pending usage timestamp: {now_unix_secs}"
+            ))
+        })?;
+        let cutoff = DateTime::<Utc>::from_timestamp(cutoff_timestamp, 0).ok_or_else(|| {
+            DataLayerError::InvalidInput(format!(
+                "invalid stale pending usage cutoff: {cutoff_unix_secs}"
+            ))
+        })?;
+        let now = DateTime::<Utc>::from_timestamp(now_timestamp, 0).ok_or_else(|| {
+            DataLayerError::InvalidInput(format!(
+                "invalid stale pending usage timestamp: {now_unix_secs}"
+            ))
+        })?;
+        let mut summary = PendingUsageCleanupSummary::default();
+        let batch_size_i64 = i64::try_from(batch_size).map_err(|_| {
+            DataLayerError::InvalidInput(format!(
+                "invalid stale pending usage batch size: {batch_size}"
+            ))
+        })?;
+
+        loop {
+            let mut tx = self.pool.begin().await.map_postgres_err()?;
+            let stale_rows = sqlx::query(SELECT_STALE_PENDING_USAGE_BATCH_SQL)
+                .bind(cutoff)
+                .bind(batch_size_i64)
+                .fetch_all(&mut *tx)
+                .await
+                .map_postgres_err()?;
+
+            if stale_rows.is_empty() {
+                tx.rollback().await.map_postgres_err()?;
+                break;
+            }
+
+            let stale_rows = stale_rows
+                .iter()
+                .map(|row| {
+                    Ok(StalePendingUsageRow {
+                        request_id: row.try_get("request_id").map_postgres_err()?,
+                        status: row.try_get("status").map_postgres_err()?,
+                        billing_status: row.try_get("billing_status").map_postgres_err()?,
+                    })
+                })
+                .collect::<Result<Vec<_>, DataLayerError>>()?;
+            let request_ids = stale_rows
+                .iter()
+                .map(|row| row.request_id.clone())
+                .collect::<Vec<_>>();
+            let completed_request_ids = if request_ids.is_empty() {
+                Vec::new()
+            } else {
+                sqlx::query(SELECT_COMPLETED_PENDING_REQUEST_IDS_SQL)
+                    .bind(request_ids)
+                    .fetch_all(&mut *tx)
+                    .await
+                    .map_postgres_err()?
+                    .iter()
+                    .map(|row| row.try_get("request_id").map_postgres_err())
+                    .collect::<Result<Vec<String>, DataLayerError>>()?
+            };
+
+            for row in stale_rows {
+                if completed_request_ids.contains(&row.request_id) {
+                    sqlx::query(UPDATE_RECOVERED_STALE_USAGE_SQL)
+                        .bind(&row.request_id)
+                        .execute(&mut *tx)
+                        .await
+                        .map_postgres_err()?;
+                    sqlx::query(UPDATE_RECOVERED_STREAMING_CANDIDATES_SQL)
+                        .bind(&row.request_id)
+                        .bind(now)
+                        .execute(&mut *tx)
+                        .await
+                        .map_postgres_err()?;
+                    summary.recovered += 1;
+                    continue;
+                }
+
+                let error_message = stale_pending_error_message(&row.status, timeout_minutes);
+                if row.billing_status == "pending" {
+                    sqlx::query(UPDATE_FAILED_VOID_STALE_USAGE_SQL)
+                        .bind(&row.request_id)
+                        .bind(&error_message)
+                        .bind(now)
+                        .execute(&mut *tx)
+                        .await
+                        .map_postgres_err()?;
+                } else {
+                    sqlx::query(UPDATE_FAILED_STALE_USAGE_SQL)
+                        .bind(&row.request_id)
+                        .bind(&error_message)
+                        .execute(&mut *tx)
+                        .await
+                        .map_postgres_err()?;
+                }
+                sqlx::query(UPDATE_FAILED_PENDING_CANDIDATES_SQL)
+                    .bind(&row.request_id)
+                    .bind(now)
+                    .execute(&mut *tx)
+                    .await
+                    .map_postgres_err()?;
+                summary.failed += 1;
+            }
+
+            tx.commit().await.map_postgres_err()?;
+        }
+
+        Ok(summary)
+    }
+
     pub async fn rebuild_api_key_usage_stats(&self) -> Result<u64, DataLayerError> {
         self.tx_runner
             .run_read_write(|tx| {
@@ -6858,6 +7082,42 @@ impl UsageWriteRepository for SqlxUsageReadRepository {
     async fn rebuild_provider_api_key_usage_stats(&self) -> Result<u64, DataLayerError> {
         Self::rebuild_provider_api_key_usage_stats(self).await
     }
+
+    async fn cleanup_stale_pending_requests(
+        &self,
+        cutoff_unix_secs: u64,
+        now_unix_secs: u64,
+        timeout_minutes: u64,
+        batch_size: usize,
+    ) -> Result<PendingUsageCleanupSummary, DataLayerError> {
+        Self::cleanup_stale_pending_requests(
+            self,
+            cutoff_unix_secs,
+            now_unix_secs,
+            timeout_minutes,
+            batch_size,
+        )
+        .await
+    }
+
+    async fn cleanup_usage(
+        &self,
+        window: &UsageCleanupWindow,
+        batch_size: usize,
+        auto_delete_expired_keys: bool,
+    ) -> Result<UsageCleanupSummary, DataLayerError> {
+        Self::cleanup_usage(self, window, batch_size, auto_delete_expired_keys).await
+    }
+}
+
+struct StalePendingUsageRow {
+    request_id: String,
+    status: String,
+    billing_status: String,
+}
+
+fn stale_pending_error_message(status: &str, timeout_minutes: u64) -> String {
+    format!("请求超时: 状态 '{status}' 超过 {timeout_minutes} 分钟未完成")
 }
 
 async fn find_usage_by_request_id_in_tx(

crates/aether-data/src/repository/usage/postgres/tests.rs

@@ -12,7 +12,7 @@ use super::{
     UsageHttpAuditRefs, UsageRoutingSnapshot, UsageSettlementPricingSnapshot,
     MAX_INLINE_USAGE_BODY_BYTES,
 };
-use crate::postgres::{PostgresPoolConfig, PostgresPoolFactory};
+use crate::driver::postgres::{PostgresPoolConfig, PostgresPoolFactory};
 use crate::repository::usage::UpsertUsageRecord;
 use aether_data_contracts::repository::usage::UsageBodyField;
 
@@ -382,6 +382,8 @@ fn usage_sql_summarize_usage_daily_heatmap_supports_daily_aggregates() {
     assert!(source.contains("summarize_usage_daily_heatmap_from_daily_aggregates"));
     assert!(source.contains("FROM stats_daily"));
     assert!(source.contains("FROM stats_user_daily"));
+    assert!(source.contains("total_requests::BIGINT AS total_requests"));
+    assert!(source.contains("total_cost::DOUBLE PRECISION AS total_cost"));
     assert!(
         source.contains("split_dashboard_daily_aggregate_range(start_utc, end_utc, cutoff_utc)")
     );