perf: queue request candidate persistence

apps/aether-gateway/src/state/app.rs

@@ -18,6 +18,7 @@ use super::super::cache::{
 use super::super::data::GatewayDataState;
 use super::super::fallback_metrics;
 use super::super::rate_limit::FrontdoorUserRpmLimiter;
+use super::super::request_candidate_queue::RequestCandidateQueueRuntime;
 use super::super::{provider_transport, usage};
 use super::{
     AdminBillingCollectorRecord, AdminBillingRuleRecord, AdminPaymentCallbackRecord,
@@ -135,6 +136,7 @@ pub struct AppState {
     pub(crate) dashboard_response_cache: Arc<DashboardResponseCache>,
     pub(crate) system_config_cache: Arc<SystemConfigCache>,
     pub(crate) fallback_metrics: Arc<fallback_metrics::GatewayFallbackMetrics>,
+    pub(crate) request_candidate_queue: Option<Arc<RequestCandidateQueueRuntime>>,
     pub(crate) frontdoor_cors: Option<Arc<FrontdoorCorsConfig>>,
     pub(crate) frontdoor_user_rpm: Arc<FrontdoorUserRpmLimiter>,
     pub(crate) tunnel: crate::tunnel::EmbeddedTunnelState,

apps/aether-gateway/src/state/core.rs

@@ -20,6 +20,7 @@ use aether_runtime_state::{
     RuntimeSemaphoreSnapshot, RuntimeState,
 };
 use aether_scheduler_core::PROVIDER_KEY_RPM_WINDOW_SECS;
+use tracing::warn;
 
 use super::{
     AppState, FrontdoorCorsConfig, FrontdoorRuntimeGuardConfig, LocalExecutionRuntimeMissDiagnostic,
@@ -38,6 +39,9 @@ use super::super::fallback_metrics;
 use super::super::fallback_metrics::{GatewayFallbackMetricKind, GatewayFallbackReason};
 use super::super::model_fetch::spawn_model_fetch_worker;
 use super::super::rate_limit::{FrontdoorUserRpmConfig, FrontdoorUserRpmLimiter};
+use super::super::request_candidate_queue::{
+    RequestCandidateQueueConfig, RequestCandidateQueueRuntime,
+};
 use super::super::router::RequestAdmissionError;
 use super::super::{control::GatewayControlDecision, error::GatewayError};
 use super::super::{provider_transport, usage};
@@ -180,6 +184,7 @@ impl AppState {
             self.runtime_state.clone(),
         );
         self.data = data;
+        self.configure_request_candidate_queue_from_env();
     }
 
     pub fn force_close_all_tunnel_proxies(&self) -> usize {
@@ -251,6 +256,7 @@ impl AppState {
             dashboard_response_cache: Arc::new(DashboardResponseCache::default()),
             system_config_cache: Arc::new(SystemConfigCache::default()),
             fallback_metrics: Arc::new(fallback_metrics::GatewayFallbackMetrics::default()),
+            request_candidate_queue: None,
             frontdoor_cors: None,
             frontdoor_user_rpm: Arc::new(FrontdoorUserRpmLimiter::new(
                 FrontdoorUserRpmConfig::default(),
@@ -423,6 +429,26 @@ impl AppState {
         self
     }
 
+    fn configure_request_candidate_queue_from_env(&mut self) {
+        let config = RequestCandidateQueueConfig::from_env();
+        self.request_candidate_queue = if config.async_enabled() {
+            if tokio::runtime::Handle::try_current().is_err() {
+                warn!(
+                    event_name = "request_candidate_async_queue_unavailable",
+                    log_type = "ops",
+                    "request candidate async queue requested outside a Tokio runtime; falling back to sync persistence"
+                );
+                None
+            } else {
+                self.data
+                    .request_candidate_writer()
+                    .map(|writer| RequestCandidateQueueRuntime::spawn(writer, config))
+            }
+        } else {
+            None
+        };
+    }
+
     pub fn with_distributed_request_concurrency_gate(mut self, gate: RuntimeSemaphore) -> Self {
         self.distributed_request_gate = Some(Arc::new(gate));
         self
@@ -918,6 +944,9 @@ impl AppState {
         if let Some(summary) = self.data.database_pool_summary() {
             samples.extend(database_pool_metric_samples(&summary));
         }
+        if let Some(queue) = self.request_candidate_queue.as_ref() {
+            samples.extend(queue.metric_samples());
+        }
         samples.extend(self.tunnel.metric_samples());
         samples.extend(self.fallback_metrics.metric_samples());
         samples

apps/aether-gateway/src/state/runtime/candidate_queries.rs

@@ -113,9 +113,67 @@ impl AppState {
         &self,
         candidate: candidates::UpsertRequestCandidateRecord,
     ) -> Result<Option<candidates::StoredRequestCandidate>, GatewayError> {
+        if let Some(queue) = self.request_candidate_queue.as_ref() {
+            let stored = stored_request_candidate_from_upsert(&candidate)?;
+            queue
+                .enqueue_or_fallback(candidate)
+                .await
+                .map_err(|err| GatewayError::Internal(err.to_string()))?;
+            return Ok(Some(stored));
+        }
+
         self.data
             .upsert_request_candidate(candidate)
             .await
             .map_err(|err| GatewayError::Internal(err.to_string()))
     }
 }
+
+fn stored_request_candidate_from_upsert(
+    candidate: &candidates::UpsertRequestCandidateRecord,
+) -> Result<candidates::StoredRequestCandidate, GatewayError> {
+    candidate
+        .validate()
+        .map_err(|err| GatewayError::Internal(err.to_string()))?;
+    candidates::StoredRequestCandidate::new(
+        candidate.id.clone(),
+        candidate.request_id.clone(),
+        candidate.user_id.clone(),
+        candidate.api_key_id.clone(),
+        candidate.username.clone(),
+        candidate.api_key_name.clone(),
+        candidate.candidate_index.try_into().unwrap_or(i32::MAX),
+        candidate.retry_index.try_into().unwrap_or(i32::MAX),
+        candidate.provider_id.clone(),
+        candidate.endpoint_id.clone(),
+        candidate.key_id.clone(),
+        candidate.status,
+        candidate.skip_reason.clone(),
+        candidate.is_cached.unwrap_or(false),
+        candidate.status_code.map(i32::from),
+        candidate.error_type.clone(),
+        candidate.error_message.clone(),
+        candidate
+            .latency_ms
+            .map(|value| i32::try_from(value).unwrap_or(i32::MAX)),
+        candidate
+            .concurrent_requests
+            .map(|value| i32::try_from(value).unwrap_or(i32::MAX)),
+        candidate.extra_data.clone(),
+        candidate.required_capabilities.clone(),
+        candidate
+            .created_at_unix_ms
+            .or(candidate.started_at_unix_ms)
+            .or(candidate.finished_at_unix_ms)
+            .unwrap_or_else(crate::clock::current_unix_ms)
+            .try_into()
+            .unwrap_or(i64::MAX),
+        candidate
+            .started_at_unix_ms
+            .map(|value| value.try_into().unwrap_or(i64::MAX)),
+        candidate
+            .finished_at_unix_ms
+            .map(|value| value.try_into().unwrap_or(i64::MAX)),
+    )
+    .map_err(|err| GatewayError::Internal(err.to_string()))
+}