perf: reduce gateway db pressure under load

apps/aether-gateway/src/data/state/core.rs

@@ -1,11 +1,21 @@
 use aether_data::{DataBackends, DataLayerError, DatabaseDriver};
 use aether_data_contracts::repository::candidate_selection::MinimalCandidateSelectionReadRepository;
+use aether_data_contracts::repository::candidates::RequestCandidateReadRepository;
 use aether_data_contracts::repository::provider_catalog::ProviderCatalogReadRepository;
 use aether_runtime_state::RuntimeQueueStore;
-use std::sync::Arc;
+use std::sync::{Arc, OnceLock};
+use std::time::{Duration, Instant};
+use tokio::sync::Mutex;
 
 use super::{GatewayDataConfig, GatewayDataState, StoredSystemConfigEntry};
 
+const SYSTEM_CONFIG_VALUE_CACHE_TTL: Duration = Duration::from_secs(30);
+
+fn system_config_value_load_guard() -> &'static Mutex<()> {
+    static GUARD: OnceLock<Mutex<()>> = OnceLock::new();
+    GUARD.get_or_init(|| Mutex::new(()))
+}
+
 fn current_system_config_updated_at_unix_secs() -> u64 {
     std::time::SystemTime::now()
         .duration_since(std::time::UNIX_EPOCH)
@@ -64,11 +74,16 @@ impl GatewayDataState {
                 wallet_writer: None,
                 settlement_writer: None,
                 system_config_values: None,
+                system_config_value_cache: Default::default(),
+                billing_model_context_cache: Default::default(),
             });
         }
 
         let backends = DataBackends::from_config(config.to_data_layer_config())?;
-        let auth_api_key_reader = backends.read().auth_api_keys();
+        let auth_api_key_reader = backends.read().auth_api_keys().map(|repository| {
+            Arc::new(super::auth_api_key_cache::CachedAuthApiKeyReadRepository::new(repository))
+                as Arc<dyn aether_data::repository::auth::AuthApiKeyReadRepository>
+        });
         let auth_api_key_writer = backends.write().auth_api_keys();
         let auth_module_reader = backends.read().auth_modules();
         let auth_module_writer = backends.write().auth_modules();
@@ -97,7 +112,13 @@ impl GatewayDataState {
                         ),
                     ) as Arc<dyn MinimalCandidateSelectionReadRepository>
                 });
-        let request_candidate_reader = backends.read().request_candidates();
+        let request_candidate_reader = backends.read().request_candidates().map(|repository| {
+            Arc::new(
+                super::request_candidate_cache::CachedRequestCandidateReadRepository::new(
+                    repository,
+                ),
+            ) as Arc<dyn RequestCandidateReadRepository>
+        });
         let request_candidate_writer = backends.write().request_candidates();
         let gemini_file_mapping_writer = backends.write().gemini_file_mappings();
         let provider_catalog_reader = backends.read().provider_catalog().map(|repository| {
@@ -110,7 +131,11 @@ impl GatewayDataState {
         let pool_score_writer = backends.write().pool_scores();
         let provider_quota_reader = backends.read().provider_quotas();
         let provider_quota_writer = backends.write().provider_quotas();
-        let routing_group_reader = backends.read().routing_groups();
+        let routing_group_reader = backends.read().routing_groups().map(|repository| {
+            Arc::new(super::routing_group_cache::CachedRoutingGroupReadRepository::new(
+                repository,
+            )) as Arc<dyn aether_data_contracts::repository::routing_profiles::RoutingGroupReadRepository>
+        });
         let routing_group_writer = backends.write().routing_groups();
         let usage_reader = backends.read().usage();
         let usage_writer = backends.write().usage();
@@ -166,6 +191,8 @@ impl GatewayDataState {
             wallet_writer,
             settlement_writer,
             system_config_values: None,
+            system_config_value_cache: Default::default(),
+            billing_model_context_cache: Default::default(),
         })
     }
 
@@ -401,10 +428,38 @@ impl GatewayDataState {
                 .get(key)
                 .map(|entry| entry.value.clone()));
         }
+        let cached_value = self
+            .system_config_value_cache
+            .read()
+            .expect("system config value cache lock")
+            .get(key)
+            .cloned();
+        if let Some((cached_at, value)) = cached_value {
+            if cached_at.elapsed() <= SYSTEM_CONFIG_VALUE_CACHE_TTL {
+                return Ok(value);
+            }
+        }
+        let _guard = system_config_value_load_guard().lock().await;
+        let cached_value = self
+            .system_config_value_cache
+            .read()
+            .expect("system config value cache lock")
+            .get(key)
+            .cloned();
+        if let Some((cached_at, value)) = cached_value {
+            if cached_at.elapsed() <= SYSTEM_CONFIG_VALUE_CACHE_TTL {
+                return Ok(value);
+            }
+        }
         let Some(backends) = self.backends.as_ref() else {
             return Ok(None);
         };
-        backends.find_system_config_value(key).await
+        let value = backends.find_system_config_value(key).await?;
+        self.system_config_value_cache
+            .write()
+            .expect("system config value cache lock")
+            .insert(key.to_string(), (Instant::now(), value.clone()));
+        Ok(value)
     }
 
     pub(crate) async fn upsert_system_config_value(
@@ -454,6 +509,7 @@ impl GatewayDataState {
                 updated_at_unix_secs: Some(current_system_config_updated_at_unix_secs()),
             };
             values.insert(key.to_string(), entry.clone());
+            self.clear_cached_system_config_value(key);
             return Ok(entry);
         }
         if let Some(backends) = self.backends.as_ref() {
@@ -461,9 +517,11 @@ impl GatewayDataState {
                 .upsert_system_config_entry(key, value, description)
                 .await?
             {
+                self.clear_cached_system_config_value(key);
                 return Ok(entry);
             }
         }
+        self.clear_cached_system_config_value(key);
         Ok(StoredSystemConfigEntry {
             key: key.to_string(),
             value: value.clone(),
@@ -477,16 +535,27 @@ impl GatewayDataState {
         key: &str,
     ) -> Result<bool, DataLayerError> {
         if let Some(values) = &self.system_config_values {
-            return Ok(values
+            let deleted = values
                 .write()
                 .expect("system config values lock")
                 .remove(key)
-                .is_some());
+                .is_some();
+            self.clear_cached_system_config_value(key);
+            return Ok(deleted);
         }
         let Some(backends) = self.backends.as_ref() else {
             return Ok(false);
         };
-        backends.delete_system_config_value(key).await
+        let deleted = backends.delete_system_config_value(key).await?;
+        self.clear_cached_system_config_value(key);
+        Ok(deleted)
+    }
+
+    fn clear_cached_system_config_value(&self, key: &str) {
+        self.system_config_value_cache
+            .write()
+            .expect("system config value cache lock")
+            .remove(key);
     }
 
     pub(crate) async fn read_admin_system_stats(