Improve gateway scheduling and runtime admission

apps/aether-gateway/src/state/app.rs

@@ -2,6 +2,7 @@ use std::collections::HashMap;
 use std::sync::atomic::AtomicU64;
 use std::sync::Arc;
 use std::sync::Mutex as StdMutex;
+use std::sync::RwLock as StdRwLock;
 use std::time::Duration;
 
 use aether_data::repository::users::StoredUserGroup;
@@ -37,6 +38,15 @@ const MIN_LOCAL_EXECUTION_PLANNING_TIMEOUT_MS: u64 = 500;
 const MAX_LOCAL_EXECUTION_PLANNING_TIMEOUT_MS: u64 = 120_000;
 const LOCAL_EXECUTION_PLANNING_TIMEOUT_MS_ENV: &str =
     "AETHER_GATEWAY_LOCAL_EXECUTION_PLANNING_TIMEOUT_MS";
+const DEFAULT_CANDIDATE_PLANNING_GATE_LIMIT: usize = 1024;
+const DEFAULT_UPSTREAM_EXECUTION_GATE_LIMIT: usize = 2000;
+const DEFAULT_UPSTREAM_TARGET_GATE_LIMIT: usize = 2000;
+const DEFAULT_INTERNAL_GATE_QUEUE_BUDGET_MS: u64 = 250;
+const MAX_INTERNAL_GATE_QUEUE_BUDGET_MS: u64 = 5_000;
+const CANDIDATE_PLANNING_GATE_LIMIT_ENV: &str = "AETHER_GATEWAY_CANDIDATE_PLANNING_GATE_LIMIT";
+const UPSTREAM_EXECUTION_GATE_LIMIT_ENV: &str = "AETHER_GATEWAY_UPSTREAM_EXECUTION_GATE_LIMIT";
+const UPSTREAM_TARGET_GATE_LIMIT_ENV: &str = "AETHER_GATEWAY_UPSTREAM_TARGET_GATE_LIMIT";
+const INTERNAL_GATE_QUEUE_BUDGET_MS_ENV: &str = "AETHER_GATEWAY_INTERNAL_GATE_QUEUE_BUDGET_MS";
 
 #[cfg(test)]
 type TestExecutionRuntimeSyncOverrideFn = dyn Fn(
@@ -62,6 +72,10 @@ impl std::fmt::Debug for TestExecutionRuntimeSyncOverride {
 pub(crate) struct FrontdoorRuntimeGuardConfig {
     pub(crate) request_body_read_timeout: Duration,
     pub(crate) local_execution_planning_timeout: Duration,
+    pub(crate) internal_gate_queue_budget: Duration,
+    pub(crate) candidate_planning_gate_limit: Option<usize>,
+    pub(crate) upstream_execution_gate_limit: Option<usize>,
+    pub(crate) upstream_target_gate_limit: Option<usize>,
 }
 
 impl FrontdoorRuntimeGuardConfig {
@@ -79,6 +93,24 @@ impl FrontdoorRuntimeGuardConfig {
                 MIN_LOCAL_EXECUTION_PLANNING_TIMEOUT_MS,
                 MAX_LOCAL_EXECUTION_PLANNING_TIMEOUT_MS,
             ),
+            internal_gate_queue_budget: env_duration_ms(
+                INTERNAL_GATE_QUEUE_BUDGET_MS_ENV,
+                DEFAULT_INTERNAL_GATE_QUEUE_BUDGET_MS,
+                1,
+                MAX_INTERNAL_GATE_QUEUE_BUDGET_MS,
+            ),
+            candidate_planning_gate_limit: env_optional_usize(
+                CANDIDATE_PLANNING_GATE_LIMIT_ENV,
+                DEFAULT_CANDIDATE_PLANNING_GATE_LIMIT,
+            ),
+            upstream_execution_gate_limit: env_optional_usize(
+                UPSTREAM_EXECUTION_GATE_LIMIT_ENV,
+                DEFAULT_UPSTREAM_EXECUTION_GATE_LIMIT,
+            ),
+            upstream_target_gate_limit: env_optional_usize(
+                UPSTREAM_TARGET_GATE_LIMIT_ENV,
+                DEFAULT_UPSTREAM_TARGET_GATE_LIMIT,
+            ),
         }
     }
 
@@ -90,6 +122,12 @@ impl FrontdoorRuntimeGuardConfig {
         Self {
             request_body_read_timeout,
             local_execution_planning_timeout,
+            internal_gate_queue_budget: Duration::from_millis(
+                DEFAULT_INTERNAL_GATE_QUEUE_BUDGET_MS,
+            ),
+            candidate_planning_gate_limit: Some(DEFAULT_CANDIDATE_PLANNING_GATE_LIMIT),
+            upstream_execution_gate_limit: Some(DEFAULT_UPSTREAM_EXECUTION_GATE_LIMIT),
+            upstream_target_gate_limit: Some(DEFAULT_UPSTREAM_TARGET_GATE_LIMIT),
         }
     }
 }
@@ -104,6 +142,17 @@ fn env_duration_ms(key: &str, default_ms: u64, min_ms: u64, max_ms: u64) -> Dura
     Duration::from_millis(ms)
 }
 
+fn env_optional_usize(key: &str, default_value: usize) -> Option<usize> {
+    match std::env::var(key)
+        .ok()
+        .and_then(|value| value.trim().parse::<usize>().ok())
+    {
+        Some(0) => None,
+        Some(value) => Some(value.max(1)),
+        None => Some(default_value.max(1)),
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct AppState {
     #[cfg(test)]
@@ -117,6 +166,9 @@ pub struct AppState {
     pub(crate) video_task_poller: Option<VideoTaskPollerConfig>,
     pub(crate) frontdoor_runtime_guards: Arc<FrontdoorRuntimeGuardConfig>,
     pub(crate) request_gate: Option<Arc<ConcurrencyGate>>,
+    pub(crate) candidate_planning_gate: Option<Arc<ConcurrencyGate>>,
+    pub(crate) upstream_execution_gate: Option<Arc<ConcurrencyGate>>,
+    pub(crate) upstream_target_admission: Arc<crate::upstream_admission::UpstreamTargetAdmission>,
     pub(crate) distributed_request_gate: Option<Arc<RuntimeSemaphore>>,
     pub(crate) client: reqwest::Client,
     pub(crate) auth_context_cache: Arc<AuthContextCache>,
@@ -135,13 +187,17 @@ pub struct AppState {
     pub(crate) scheduler_affinity_epoch: Arc<AtomicU64>,
     pub(crate) dashboard_response_cache: Arc<DashboardResponseCache>,
     pub(crate) system_config_cache: Arc<SystemConfigCache>,
+    pub(crate) candidate_page_cache: Arc<super::super::cache::CandidatePageCache>,
+    pub(crate) candidate_resolved_page_cache: Arc<super::super::cache::CandidateResolvedPageCache>,
+    pub(crate) chat_pii_redaction_runtime_config_cache:
+        crate::privacy::ChatPiiRedactionRuntimeConfigCacheHandle,
     pub(crate) fallback_metrics: Arc<fallback_metrics::GatewayFallbackMetrics>,
     pub(crate) request_candidate_queue: Option<Arc<RequestCandidateQueueRuntime>>,
     pub(crate) frontdoor_cors: Option<Arc<FrontdoorCorsConfig>>,
     pub(crate) frontdoor_user_rpm: Arc<FrontdoorUserRpmLimiter>,
     pub(crate) tunnel: crate::tunnel::EmbeddedTunnelState,
     pub(crate) provider_transport_snapshot_cache:
-        Arc<StdMutex<HashMap<ProviderTransportSnapshotCacheKey, CachedProviderTransportSnapshot>>>,
+        Arc<StdRwLock<HashMap<ProviderTransportSnapshotCacheKey, CachedProviderTransportSnapshot>>>,
     pub(crate) provider_key_rpm_resets: Arc<StdMutex<HashMap<String, u64>>>,
     pub(crate) local_execution_runtime_miss_diagnostics:
         Arc<StdMutex<HashMap<String, LocalExecutionRuntimeMissDiagnostic>>>,

apps/aether-gateway/src/state/cache.rs

@@ -1,3 +1,4 @@
+use std::sync::Arc;
 use std::time::Duration;
 
 use super::super::provider_transport;
@@ -5,10 +6,11 @@ use super::super::provider_transport;
 pub(crate) const AUTH_API_KEY_LAST_USED_TTL: Duration = Duration::from_secs(60);
 pub(crate) const AUTH_API_KEY_LAST_USED_MAX_ENTRIES: usize = 10_000;
 pub(crate) const PROVIDER_TRANSPORT_SNAPSHOT_CACHE_TTL: Duration = Duration::from_secs(1);
+pub(crate) const PROVIDER_TRANSPORT_SNAPSHOT_CACHE_STALE_TTL: Duration = Duration::from_secs(30);
 pub(crate) const PROVIDER_TRANSPORT_SNAPSHOT_CACHE_MAX_ENTRIES: usize = 1_024;
 
 #[derive(Debug, Clone)]
 pub(crate) struct CachedProviderTransportSnapshot {
     pub(crate) loaded_at: std::time::Instant,
-    pub(crate) snapshot: provider_transport::GatewayProviderTransportSnapshot,
+    pub(crate) snapshot: Arc<provider_transport::GatewayProviderTransportSnapshot>,
 }

apps/aether-gateway/src/state/core.rs

@@ -32,7 +32,7 @@ use super::super::async_task::{
 use super::super::cache::{
     AuthApiKeyLastUsedCache, AuthContextCache, AuthSnapshotCache, DashboardResponseCache,
     DirectPlanBypassCache, JsonValueCache, SchedulerAffinityCache, SchedulerAffinitySnapshotEntry,
-    SchedulerAffinityTarget, SystemConfigCache, ValueCache,
+    SchedulerAffinityTarget, SystemConfigCache, SystemConfigInflightRegistration, ValueCache,
 };
 use super::super::data::{GatewayDataConfig, GatewayDataState};
 use super::super::fallback_metrics;
@@ -78,6 +78,7 @@ const AUTH_AFFECTING_SYSTEM_CONFIG_KEYS: &[&str] = &[
     crate::constants::ANTIGRAVITY_BEARER_BRIDGE_CONFIG_KEY,
 ];
 const FRONTDOOR_RPM_AFFECTING_SYSTEM_CONFIG_KEYS: &[&str] = &["rate_limit_per_minute"];
+const CHAT_PII_REDACTION_SYSTEM_CONFIG_PREFIX: &str = "module.chat_pii_redaction.";
 
 fn system_config_key_affects_scheduler(key: &str) -> bool {
     let key = key.trim();
@@ -94,7 +95,19 @@ fn system_config_key_affects_frontdoor_rpm(key: &str) -> bool {
     FRONTDOOR_RPM_AFFECTING_SYSTEM_CONFIG_KEYS.contains(&key)
 }
 
+fn system_config_key_affects_chat_pii_redaction(key: &str) -> bool {
+    key.trim()
+        .starts_with(CHAT_PII_REDACTION_SYSTEM_CONFIG_PREFIX)
+}
+
 impl AppState {
+    pub async fn prewarm_chat_pii_redaction_runtime_config(&self) -> Result<bool, String> {
+        crate::privacy::read_chat_pii_redaction_runtime_config(self)
+            .await
+            .map(|config| config.enabled)
+            .map_err(|err| format!("{err:?}"))
+    }
+
     fn usage_worker_queue_for(
         runtime_state: &Arc<RuntimeState>,
     ) -> Option<Arc<dyn RuntimeQueueStore>> {
@@ -172,6 +185,7 @@ impl AppState {
         self.clear_provider_transport_snapshot_cache();
         self.invalidate_scheduler_affinity_cache();
         self.invalidate_auth_context_cache();
+        self.candidate_resolved_page_cache.clear();
         self.system_config_cache.clear();
         self.frontdoor_user_rpm.clear_system_default_cache();
         let data = Arc::new(
@@ -179,6 +193,8 @@ impl AppState {
                 .clone()
                 .with_usage_worker_queue(Self::usage_worker_queue_for(&self.runtime_state)),
         );
+        self.candidate_page_cache.clear();
+        self.candidate_resolved_page_cache.clear();
         self.tunnel = crate::tunnel::EmbeddedTunnelState::with_data_and_runtime_state(
             Arc::clone(&data),
             self.runtime_state.clone(),
@@ -222,6 +238,7 @@ impl AppState {
             http2_adaptive_window: true,
             ..HttpClientConfig::default()
         })?;
+        let frontdoor_runtime_guards = Arc::new(FrontdoorRuntimeGuardConfig::from_env());
         Ok(Self {
             #[cfg(test)]
             execution_runtime_override_base_url: execution_runtime_override_base_url
@@ -236,8 +253,20 @@ impl AppState {
                 VideoTaskTruthSourceMode::PythonSyncReport,
             )),
             video_task_poller: None,
-            frontdoor_runtime_guards: Arc::new(FrontdoorRuntimeGuardConfig::from_env()),
+            frontdoor_runtime_guards: Arc::clone(&frontdoor_runtime_guards),
             request_gate: None,
+            candidate_planning_gate: frontdoor_runtime_guards
+                .candidate_planning_gate_limit
+                .map(|limit| Arc::new(ConcurrencyGate::new("gateway_candidate_planning", limit))),
+            upstream_execution_gate: frontdoor_runtime_guards
+                .upstream_execution_gate_limit
+                .map(|limit| Arc::new(ConcurrencyGate::new("gateway_upstream_execution", limit))),
+            upstream_target_admission: Arc::new(
+                crate::upstream_admission::UpstreamTargetAdmission::new(
+                    frontdoor_runtime_guards.upstream_target_gate_limit,
+                    frontdoor_runtime_guards.internal_gate_queue_budget,
+                ),
+            ),
             distributed_request_gate: None,
             client,
             auth_context_cache: Arc::new(AuthContextCache::default()),
@@ -255,6 +284,12 @@ impl AppState {
             scheduler_affinity_epoch: Arc::new(AtomicU64::new(0)),
             dashboard_response_cache: Arc::new(DashboardResponseCache::default()),
             system_config_cache: Arc::new(SystemConfigCache::default()),
+            candidate_page_cache: Arc::new(crate::cache::CandidatePageCache::default()),
+            candidate_resolved_page_cache: Arc::new(
+                crate::cache::CandidateResolvedPageCache::default(),
+            ),
+            chat_pii_redaction_runtime_config_cache:
+                crate::privacy::new_chat_pii_redaction_runtime_config_cache(),
             fallback_metrics: Arc::new(fallback_metrics::GatewayFallbackMetrics::default()),
             request_candidate_queue: None,
             frontdoor_cors: None,
@@ -265,7 +300,7 @@ impl AppState {
                 data,
                 runtime_state.clone(),
             ),
-            provider_transport_snapshot_cache: Arc::new(StdMutex::new(HashMap::new())),
+            provider_transport_snapshot_cache: Arc::new(std::sync::RwLock::new(HashMap::new())),
             provider_key_rpm_resets: Arc::new(StdMutex::new(HashMap::new())),
             local_execution_runtime_miss_diagnostics: Arc::new(StdMutex::new(HashMap::new())),
             admin_monitoring_error_stats_reset_at: Arc::new(StdMutex::new(None)),
@@ -535,19 +570,44 @@ impl AppState {
             return Ok(value);
         }
 
-        let _guard = self.system_config_cache.load_guard().await;
-        if let Some(value) = self.system_config_cache.get(key, SYSTEM_CONFIG_CACHE_TTL) {
-            return Ok(value);
+        loop {
+            let notified = self.system_config_cache.notified();
+            match self.system_config_cache.register_load(key) {
+                SystemConfigInflightRegistration::Bypass => {
+                    let value = self
+                        .data
+                        .find_system_config_value(key)
+                        .await
+                        .map_err(|err| GatewayError::Internal(err.to_string()))?;
+                    self.system_config_cache.insert(
+                        key.to_string(),
+                        value.clone(),
+                        SYSTEM_CONFIG_CACHE_TTL,
+                    );
+                    return Ok(value);
+                }
+                SystemConfigInflightRegistration::Follower => {
+                    notified.await;
+                    if let Some(value) = self.system_config_cache.get(key, SYSTEM_CONFIG_CACHE_TTL)
+                    {
+                        return Ok(value);
+                    }
+                }
+                SystemConfigInflightRegistration::Leader(_guard) => {
+                    let value = self
+                        .data
+                        .find_system_config_value(key)
+                        .await
+                        .map_err(|err| GatewayError::Internal(err.to_string()))?;
+                    self.system_config_cache.insert(
+                        key.to_string(),
+                        value.clone(),
+                        SYSTEM_CONFIG_CACHE_TTL,
+                    );
+                    return Ok(value);
+                }
+            }
         }
-
-        let value = self
-            .data
-            .find_system_config_value(key)
-            .await
-            .map_err(|err| GatewayError::Internal(err.to_string()))?;
-        self.system_config_cache
-            .insert(key.to_string(), value.clone(), SYSTEM_CONFIG_CACHE_TTL);
-        Ok(value)
     }
 
     pub(crate) async fn upsert_system_config_json_value(
@@ -606,12 +666,19 @@ impl AppState {
         if deleted && system_config_key_affects_frontdoor_rpm(key) {
             self.frontdoor_user_rpm.clear_system_default_cache();
         }
+        if deleted && system_config_key_affects_chat_pii_redaction(key) {
+            crate::privacy::clear_chat_pii_redaction_runtime_config_cache(
+                &self.chat_pii_redaction_runtime_config_cache,
+            );
+        }
         Ok(deleted)
     }
 
     pub(crate) fn invalidate_provider_routing_caches(&self) {
         self.data.clear_minimal_candidate_selection_cache();
         self.data.clear_provider_catalog_cache();
+        self.candidate_page_cache.clear();
+        self.candidate_resolved_page_cache.clear();
         self.clear_provider_transport_snapshot_cache();
         self.invalidate_scheduler_affinity_cache();
     }
@@ -619,6 +686,8 @@ impl AppState {
     pub(crate) fn invalidate_provider_health_routing_caches(&self) {
         self.data.clear_minimal_candidate_selection_cache();
         self.data.clear_provider_catalog_cache();
+        self.candidate_page_cache.clear();
+        self.candidate_resolved_page_cache.clear();
         self.clear_provider_transport_snapshot_cache();
     }
 
@@ -631,6 +700,8 @@ impl AppState {
         self.auth_api_key_feature_settings_cache.clear();
         self.provider_quota_snapshot_cache.clear();
         self.user_groups_for_user_cache.clear();
+        self.candidate_page_cache.clear();
+        self.candidate_resolved_page_cache.clear();
     }
 
     fn remember_system_config_write(&self, key: &str, value: Option<serde_json::Value>) {
@@ -645,6 +716,11 @@ impl AppState {
         if system_config_key_affects_frontdoor_rpm(key) {
             self.frontdoor_user_rpm.clear_system_default_cache();
         }
+        if system_config_key_affects_chat_pii_redaction(key) {
+            crate::privacy::clear_chat_pii_redaction_runtime_config_cache(
+                &self.chat_pii_redaction_runtime_config_cache,
+            );
+        }
     }
 
     pub(crate) async fn read_admin_system_stats(
@@ -908,6 +984,18 @@ impl AppState {
         self.request_gate.as_ref().map(|gate| gate.snapshot())
     }
 
+    pub(crate) fn candidate_planning_concurrency_snapshot(&self) -> Option<ConcurrencySnapshot> {
+        self.candidate_planning_gate
+            .as_ref()
+            .map(|gate| gate.snapshot())
+    }
+
+    pub(crate) fn upstream_execution_concurrency_snapshot(&self) -> Option<ConcurrencySnapshot> {
+        self.upstream_execution_gate
+            .as_ref()
+            .map(|gate| gate.snapshot())
+    }
+
     pub(crate) async fn distributed_request_concurrency_snapshot(
         &self,
     ) -> Result<Option<RuntimeSemaphoreSnapshot>, RuntimeSemaphoreError> {
@@ -922,6 +1010,12 @@ impl AppState {
         if let Some(snapshot) = self.request_concurrency_snapshot() {
             samples.extend(snapshot.to_metric_samples("gateway_requests"));
         }
+        if let Some(snapshot) = self.candidate_planning_concurrency_snapshot() {
+            samples.extend(snapshot.to_metric_samples("gateway_candidate_planning"));
+        }
+        if let Some(snapshot) = self.upstream_execution_concurrency_snapshot() {
+            samples.extend(snapshot.to_metric_samples("gateway_upstream_execution"));
+        }
         if let Some(gate) = self.distributed_request_gate.as_ref() {
             match gate.snapshot().await {
                 Ok(snapshot) => {
@@ -947,6 +1041,12 @@ impl AppState {
         if let Some(queue) = self.request_candidate_queue.as_ref() {
             samples.extend(queue.metric_samples());
         }
+        samples.extend(
+            crate::execution_runtime::transport::direct_reqwest_client_cache_metric_samples(),
+        );
+        samples.extend(self.upstream_target_admission.metric_samples());
+        samples.extend(crate::cache::candidate_page_cache_metric_samples());
+        samples.extend(crate::stage_metrics::gateway_stage_metric_samples());
         samples.extend(self.tunnel.metric_samples());
         samples.extend(self.fallback_metrics.metric_samples());
         samples
@@ -1131,6 +1231,8 @@ impl AppState {
             .fetch_add(1, Ordering::AcqRel)
             .saturating_add(1);
         self.scheduler_affinity_cache.clear();
+        self.candidate_page_cache.clear();
+        self.candidate_resolved_page_cache.clear();
         next_epoch
     }
 

apps/aether-gateway/src/state/mod.rs

@@ -31,7 +31,7 @@ pub(crate) use self::app::FrontdoorRuntimeGuardConfig;
 pub(crate) use self::cache::{
     CachedProviderTransportSnapshot, AUTH_API_KEY_LAST_USED_MAX_ENTRIES,
     AUTH_API_KEY_LAST_USED_TTL, PROVIDER_TRANSPORT_SNAPSHOT_CACHE_MAX_ENTRIES,
-    PROVIDER_TRANSPORT_SNAPSHOT_CACHE_TTL,
+    PROVIDER_TRANSPORT_SNAPSHOT_CACHE_STALE_TTL, PROVIDER_TRANSPORT_SNAPSHOT_CACHE_TTL,
 };
 pub use self::cors::FrontdoorCorsConfig;
 pub(crate) use self::types::{

apps/aether-gateway/src/state/oauth.rs

@@ -1,7 +1,7 @@
 use super::{
     provider_transport_snapshot_looks_refreshed, AppState, CachedProviderTransportSnapshot,
     GatewayError, ProviderTransportSnapshotCacheKey, PROVIDER_TRANSPORT_SNAPSHOT_CACHE_MAX_ENTRIES,
-    PROVIDER_TRANSPORT_SNAPSHOT_CACHE_TTL,
+    PROVIDER_TRANSPORT_SNAPSHOT_CACHE_STALE_TTL,
 };
 use crate::handlers::shared::default_provider_key_status_snapshot;
 use crate::provider_transport::LocalOAuthHttpExecutor;
@@ -20,6 +20,7 @@ use serde_json::{json, Map, Value};
 use sha2::{Digest, Sha256};
 use std::collections::BTreeMap;
 use std::io::Read;
+use std::sync::Arc;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 use aether_crypto::encrypt_python_fernet_plaintext;
@@ -479,39 +480,49 @@ impl<'a> provider_transport::LocalOAuthHttpExecutor for GatewayLocalOAuthHttpExe
 impl AppState {
     pub(crate) fn clear_provider_transport_snapshot_cache(&self) {
         self.provider_transport_snapshot_cache
-            .lock()
+            .write()
             .expect("provider transport snapshot cache should lock")
             .clear();
     }
 
-    fn get_cached_provider_transport_snapshot(
+    fn get_cached_provider_transport_snapshot_arc(
         &self,
         cache_key: &ProviderTransportSnapshotCacheKey,
-    ) -> Option<provider_transport::GatewayProviderTransportSnapshot> {
-        let mut cache = self
-            .provider_transport_snapshot_cache
-            .lock()
-            .expect("provider transport snapshot cache should lock");
-        let cached = cache.get(cache_key).cloned()?;
-        if cached.loaded_at.elapsed() <= PROVIDER_TRANSPORT_SNAPSHOT_CACHE_TTL {
+    ) -> Option<Arc<provider_transport::GatewayProviderTransportSnapshot>> {
+        let cached = {
+            let cache = self
+                .provider_transport_snapshot_cache
+                .read()
+                .expect("provider transport snapshot cache should lock");
+            cache.get(cache_key).cloned()
+        }?;
+        if cached.loaded_at.elapsed() <= PROVIDER_TRANSPORT_SNAPSHOT_CACHE_STALE_TTL {
             return Some(cached.snapshot);
         }
-        cache.remove(cache_key);
+        let mut cache = self
+            .provider_transport_snapshot_cache
+            .write()
+            .expect("provider transport snapshot cache should lock");
+        if cache.get(cache_key).is_some_and(|entry| {
+            entry.loaded_at.elapsed() > PROVIDER_TRANSPORT_SNAPSHOT_CACHE_STALE_TTL
+        }) {
+            cache.remove(cache_key);
+        }
         None
     }
 
     fn put_cached_provider_transport_snapshot(
         &self,
         cache_key: ProviderTransportSnapshotCacheKey,
-        snapshot: provider_transport::GatewayProviderTransportSnapshot,
+        snapshot: Arc<provider_transport::GatewayProviderTransportSnapshot>,
     ) {
         let mut cache = self
             .provider_transport_snapshot_cache
-            .lock()
+            .write()
             .expect("provider transport snapshot cache should lock");
         if cache.len() >= PROVIDER_TRANSPORT_SNAPSHOT_CACHE_MAX_ENTRIES {
             cache.retain(|_, entry| {
-                entry.loaded_at.elapsed() <= PROVIDER_TRANSPORT_SNAPSHOT_CACHE_TTL
+                entry.loaded_at.elapsed() <= PROVIDER_TRANSPORT_SNAPSHOT_CACHE_STALE_TTL
             });
             if cache.len() >= PROVIDER_TRANSPORT_SNAPSHOT_CACHE_MAX_ENTRIES {
                 cache.clear();
@@ -796,6 +807,41 @@ impl AppState {
         None
     }
 
+    pub(crate) async fn read_provider_transport_snapshot_arc(
+        &self,
+        provider_id: &str,
+        endpoint_id: &str,
+        key_id: &str,
+    ) -> Result<
+        Option<Arc<crate::provider_transport::GatewayProviderTransportSnapshot>>,
+        GatewayError,
+    > {
+        let Some(cache_key) =
+            ProviderTransportSnapshotCacheKey::new(provider_id, endpoint_id, key_id)
+        else {
+            return Ok(self
+                .read_provider_transport_snapshot_uncached(provider_id, endpoint_id, key_id)
+                .await?
+                .map(Arc::new));
+        };
+        if let Some(snapshot) = self.get_cached_provider_transport_snapshot_arc(&cache_key) {
+            return Ok(Some(snapshot));
+        }
+
+        let snapshot = self
+            .read_provider_transport_snapshot_uncached(provider_id, endpoint_id, key_id)
+            .await?;
+        match snapshot {
+            Some(snapshot) => {
+                let snapshot = self.apply_global_format_conversion_override(snapshot).await;
+                let snapshot = Arc::new(snapshot);
+                self.put_cached_provider_transport_snapshot(cache_key, Arc::clone(&snapshot));
+                Ok(Some(snapshot))
+            }
+            None => Ok(None),
+        }
+    }
+
     pub(crate) async fn read_provider_transport_snapshot(
         &self,
         provider_id: &str,
@@ -803,31 +849,10 @@ impl AppState {
         key_id: &str,
     ) -> Result<Option<crate::provider_transport::GatewayProviderTransportSnapshot>, GatewayError>
     {
-        let Some(cache_key) =
-            ProviderTransportSnapshotCacheKey::new(provider_id, endpoint_id, key_id)
-        else {
-            return self
-                .read_provider_transport_snapshot_uncached(provider_id, endpoint_id, key_id)
-                .await;
-        };
-        if let Some(snapshot) = self.get_cached_provider_transport_snapshot(&cache_key) {
-            return Ok(Some(
-                self.apply_global_format_conversion_override(snapshot).await,
-            ));
-        }
-
-        let snapshot = self
-            .read_provider_transport_snapshot_uncached(provider_id, endpoint_id, key_id)
-            .await?;
-        if let Some(snapshot) = snapshot.as_ref() {
-            self.put_cached_provider_transport_snapshot(cache_key, snapshot.clone());
-        }
-        match snapshot {
-            Some(snapshot) => Ok(Some(
-                self.apply_global_format_conversion_override(snapshot).await,
-            )),
-            None => Ok(None),
-        }
+        Ok(self
+            .read_provider_transport_snapshot_arc(provider_id, endpoint_id, key_id)
+            .await?
+            .map(|snapshot| (*snapshot).clone()))
     }
 
     pub(crate) async fn update_provider_catalog_key_oauth_credentials(

apps/aether-gateway/src/state/testing.rs

@@ -21,6 +21,11 @@ impl AppState {
         self
     }
 
+    pub(crate) fn without_request_candidate_queue_for_tests(mut self) -> Self {
+        self.request_candidate_queue = None;
+        self
+    }
+
     pub(crate) fn with_turnstile_siteverify_url_for_tests(mut self, url: &str) -> Self {
         self.turnstile_siteverify_url_override = Some(url.trim().to_string());
         self