Improve gateway scheduling and runtime admission

apps/aether-gateway/src/cache/auth_context.rs

@@ -1,12 +1,54 @@
+use std::collections::HashSet;
 use std::time::Duration;
 
 use aether_cache::ExpiringMap;
+use tokio::sync::Notify;
 
 use crate::control::GatewayControlAuthContext;
 
-#[derive(Debug, Default)]
+#[derive(Debug)]
 pub(crate) struct AuthContextCache {
     entries: ExpiringMap<String, GatewayControlAuthContext>,
+    inflight: std::sync::Mutex<HashSet<String>>,
+    notify: Notify,
+}
+
+impl Default for AuthContextCache {
+    fn default() -> Self {
+        Self {
+            entries: ExpiringMap::default(),
+            inflight: std::sync::Mutex::new(HashSet::new()),
+            notify: Notify::new(),
+        }
+    }
+}
+
+pub(crate) enum AuthContextInflightRegistration<'a> {
+    Leader(AuthContextInflightGuard<'a>),
+    Follower,
+    Bypass,
+}
+
+pub(crate) struct AuthContextInflightGuard<'a> {
+    cache: &'a AuthContextCache,
+    cache_key: Option<String>,
+}
+
+impl Drop for AuthContextInflightGuard<'_> {
+    fn drop(&mut self) {
+        let Some(cache_key) = self.cache_key.take() else {
+            return;
+        };
+        let removed = self
+            .cache
+            .inflight
+            .lock()
+            .map(|mut inflight| inflight.remove(&cache_key))
+            .unwrap_or(false);
+        if removed {
+            self.cache.notify.notify_waiters();
+        }
+    }
 }
 
 impl AuthContextCache {
@@ -29,7 +71,39 @@ impl AuthContextCache {
             .insert(cache_key, auth_context, ttl, max_entries);
     }
 
+    pub(crate) fn notified(&self) -> tokio::sync::futures::Notified<'_> {
+        self.notify.notified()
+    }
+
+    pub(crate) fn register_inflight(&self, cache_key: &str) -> AuthContextInflightRegistration<'_> {
+        let cache_key = cache_key.trim();
+        if cache_key.is_empty() {
+            return AuthContextInflightRegistration::Bypass;
+        }
+        match self.inflight.lock() {
+            Ok(mut inflight) => {
+                if inflight.contains(cache_key) {
+                    AuthContextInflightRegistration::Follower
+                } else {
+                    inflight.insert(cache_key.to_string());
+                    AuthContextInflightRegistration::Leader(AuthContextInflightGuard {
+                        cache: self,
+                        cache_key: Some(cache_key.to_string()),
+                    })
+                }
+            }
+            Err(_) => AuthContextInflightRegistration::Bypass,
+        }
+    }
+
     pub(crate) fn clear(&self) {
         self.entries.clear();
+        if let Ok(mut inflight) = self.inflight.lock() {
+            let had_inflight = !inflight.is_empty();
+            inflight.clear();
+            if had_inflight {
+                self.notify.notify_waiters();
+            }
+        }
     }
 }

apps/aether-gateway/src/cache/auth_runtime.rs

@@ -198,10 +198,10 @@ impl AuthSnapshotCache {
         &self,
         key: AuthSnapshotCacheKey,
         ttl: Duration,
-        load: F,
+        mut load: F,
     ) -> Result<Option<GatewayAuthApiKeySnapshot>, E>
     where
-        F: Fn() -> Fut,
+        F: FnMut() -> Fut,
         Fut: Future<Output = Result<Option<GatewayAuthApiKeySnapshot>, E>>,
     {
         if let Some(value) = self.get(&key, ttl) {
@@ -269,10 +269,10 @@ where
         &self,
         key: K,
         ttl: Duration,
-        load: F,
+        mut load: F,
     ) -> Result<Option<Value>, E>
     where
-        F: Fn() -> Fut,
+        F: FnMut() -> Fut,
         Fut: Future<Output = Result<Option<Value>, E>>,
     {
         if let Some(value) = self.get(&key, ttl) {
@@ -341,10 +341,10 @@ where
         &self,
         key: K,
         ttl: Duration,
-        load: F,
+        mut load: F,
     ) -> Result<Option<V>, E>
     where
-        F: Fn() -> Fut,
+        F: FnMut() -> Fut,
         Fut: Future<Output = Result<Option<V>, E>>,
     {
         if let Some(value) = self.get(&key, ttl) {
@@ -374,15 +374,200 @@ where
         }
     }
 
+    pub(crate) async fn get_or_load_once<E, F, Fut>(
+        &self,
+        key: K,
+        ttl: Duration,
+        load: F,
+    ) -> Result<Option<V>, E>
+    where
+        F: FnOnce() -> Fut,
+        Fut: Future<Output = Result<Option<V>, E>>,
+    {
+        self.get_or_load_once_with_observer(key, ttl, load, CacheLoadObserver::default())
+            .await
+    }
+
+    pub(crate) async fn get_or_load_once_with_observer<E, F, Fut>(
+        &self,
+        key: K,
+        ttl: Duration,
+        load: F,
+        observer: CacheLoadObserver,
+    ) -> Result<Option<V>, E>
+    where
+        F: FnOnce() -> Fut,
+        Fut: Future<Output = Result<Option<V>, E>>,
+    {
+        if let Some(value) = self.get(&key, ttl) {
+            observer.hit();
+            return Ok(value);
+        }
+        observer.miss();
+
+        let mut load = Some(load);
+        loop {
+            let notified = self.singleflight.notified();
+            match self.singleflight.register(&key) {
+                CacheInflightRegistration::Bypass => {
+                    observer.load();
+                    let value =
+                        load.take().expect("cache load closure should be available")().await?;
+                    self.insert(key, value.clone(), ttl);
+                    return Ok(value);
+                }
+                CacheInflightRegistration::Follower => {
+                    observer.follower_wait();
+                    notified.await;
+                    if let Some(value) = self.get(&key, ttl) {
+                        observer.hit();
+                        return Ok(value);
+                    }
+                }
+                CacheInflightRegistration::Leader(_guard) => {
+                    observer.load();
+                    let value =
+                        load.take().expect("cache load closure should be available")().await?;
+                    self.insert(key, value.clone(), ttl);
+                    return Ok(value);
+                }
+            }
+        }
+    }
+
+    pub(crate) async fn get_or_load_once_stale_while_refreshing<E, F, Fut>(
+        &self,
+        key: K,
+        ttl: Duration,
+        stale_ttl: Duration,
+        load: F,
+        observer: CacheLoadObserver,
+    ) -> Result<Option<V>, E>
+    where
+        F: FnOnce() -> Fut,
+        Fut: Future<Output = Result<Option<V>, E>>,
+    {
+        if let Some((value, age)) = self.entries.get_with_age(&key, stale_ttl) {
+            if age <= ttl {
+                observer.hit();
+                return Ok(value);
+            }
+
+            // Keep stale snapshots off the request critical path. The caller's
+            // invalidation path clears entries when provider/catalog/routing
+            // state changes, and the bounded stale TTL limits passive drift.
+            observer.hit();
+            return Ok(value);
+        }
+
+        observer.miss();
+        let mut load = Some(load);
+        loop {
+            let notified = self.singleflight.notified();
+            match self.singleflight.register(&key) {
+                CacheInflightRegistration::Bypass => {
+                    observer.load();
+                    let value =
+                        load.take().expect("cache load closure should be available")().await?;
+                    self.entries.insert(
+                        key,
+                        value.clone(),
+                        stale_ttl,
+                        AUTH_RUNTIME_CACHE_MAX_ENTRIES,
+                    );
+                    return Ok(value);
+                }
+                CacheInflightRegistration::Follower => {
+                    observer.follower_wait();
+                    notified.await;
+                    if let Some((value, _age)) = self.entries.get_with_age(&key, stale_ttl) {
+                        observer.hit();
+                        return Ok(value);
+                    }
+                }
+                CacheInflightRegistration::Leader(_guard) => {
+                    observer.load();
+                    let value =
+                        load.take().expect("cache load closure should be available")().await?;
+                    self.entries.insert(
+                        key,
+                        value.clone(),
+                        stale_ttl,
+                        AUTH_RUNTIME_CACHE_MAX_ENTRIES,
+                    );
+                    return Ok(value);
+                }
+            }
+        }
+    }
+
     pub(crate) fn clear(&self) {
         self.entries.clear();
         self.singleflight.clear();
     }
 }
 
+#[derive(Clone, Copy, Default)]
+pub(crate) struct CacheLoadObserver {
+    on_hit: Option<fn()>,
+    on_miss: Option<fn()>,
+    on_load: Option<fn()>,
+    on_follower_wait: Option<fn()>,
+}
+
+impl CacheLoadObserver {
+    pub(crate) fn new() -> Self {
+        Self::default()
+    }
+
+    pub(crate) fn on_hit(mut self, callback: fn()) -> Self {
+        self.on_hit = Some(callback);
+        self
+    }
+
+    pub(crate) fn on_miss(mut self, callback: fn()) -> Self {
+        self.on_miss = Some(callback);
+        self
+    }
+
+    pub(crate) fn on_load(mut self, callback: fn()) -> Self {
+        self.on_load = Some(callback);
+        self
+    }
+
+    pub(crate) fn on_follower_wait(mut self, callback: fn()) -> Self {
+        self.on_follower_wait = Some(callback);
+        self
+    }
+
+    fn hit(self) {
+        if let Some(callback) = self.on_hit {
+            callback();
+        }
+    }
+
+    fn miss(self) {
+        if let Some(callback) = self.on_miss {
+            callback();
+        }
+    }
+
+    fn load(self) {
+        if let Some(callback) = self.on_load {
+            callback();
+        }
+    }
+
+    fn follower_wait(self) {
+        if let Some(callback) = self.on_follower_wait {
+            callback();
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
-    use super::ValueCache;
+    use super::{CacheLoadObserver, ValueCache};
     use std::sync::atomic::{AtomicUsize, Ordering};
     use std::sync::Arc;
     use std::time::{Duration, Instant};
@@ -472,6 +657,117 @@ mod tests {
         assert_eq!(max_active.load(Ordering::Acquire), 2);
     }
 
+    #[tokio::test]
+    async fn value_cache_returns_stale_without_refreshing_on_request_path() {
+        let cache = Arc::new(ValueCache::<String, u64>::default());
+        let key = "hot-key".to_string();
+        let calls = Arc::new(AtomicUsize::new(0));
+        cache.insert(key.clone(), Some(1), Duration::from_millis(10));
+        tokio::time::sleep(Duration::from_millis(20)).await;
+
+        let first_cache = Arc::clone(&cache);
+        let first_key = key.clone();
+        let first_calls = Arc::clone(&calls);
+        let first_started = Instant::now();
+        let first = tokio::spawn(async move {
+            first_cache
+                .get_or_load_once_stale_while_refreshing::<(), _, _>(
+                    first_key,
+                    Duration::from_millis(10),
+                    Duration::from_secs(1),
+                    || async move {
+                        first_calls.fetch_add(1, Ordering::AcqRel);
+                        tokio::time::sleep(Duration::from_millis(100)).await;
+                        Ok(Some(2))
+                    },
+                    CacheLoadObserver::default(),
+                )
+                .await
+        });
+
+        let follower_cache = Arc::clone(&cache);
+        let follower_started = Instant::now();
+        let follower_calls = Arc::clone(&calls);
+        let follower = tokio::spawn(async move {
+            follower_cache
+                .get_or_load_once_stale_while_refreshing::<(), _, _>(
+                    key,
+                    Duration::from_millis(10),
+                    Duration::from_secs(1),
+                    || async move {
+                        follower_calls.fetch_add(1, Ordering::AcqRel);
+                        Ok(Some(3))
+                    },
+                    CacheLoadObserver::default(),
+                )
+                .await
+        });
+
+        assert_eq!(first.await.unwrap().unwrap(), Some(1));
+        assert!(
+            first_started.elapsed() < Duration::from_millis(80),
+            "stale value should not wait for request-path refresh"
+        );
+        assert_eq!(follower.await.unwrap().unwrap(), Some(1));
+        assert!(
+            follower_started.elapsed() < Duration::from_millis(80),
+            "follower should return stale value without waiting for refresh"
+        );
+        assert_eq!(calls.load(Ordering::Acquire), 0);
+    }
+
+    #[tokio::test]
+    async fn value_cache_cold_stale_followers_do_not_reload_after_fresh_ttl() {
+        let cache = Arc::new(ValueCache::<String, u64>::default());
+        let key = "cold-hot-key".to_string();
+        let calls = Arc::new(AtomicUsize::new(0));
+
+        let leader_cache = Arc::clone(&cache);
+        let leader_key = key.clone();
+        let leader_calls = Arc::clone(&calls);
+        let leader = tokio::spawn(async move {
+            leader_cache
+                .get_or_load_once_stale_while_refreshing::<(), _, _>(
+                    leader_key,
+                    Duration::from_millis(10),
+                    Duration::from_secs(1),
+                    || async move {
+                        leader_calls.fetch_add(1, Ordering::AcqRel);
+                        Ok(Some(1))
+                    },
+                    CacheLoadObserver::default(),
+                )
+                .await
+        });
+        assert_eq!(leader.await.unwrap().unwrap(), Some(1));
+        tokio::time::sleep(Duration::from_millis(25)).await;
+
+        let follower_started = Instant::now();
+        let follower_cache = Arc::clone(&cache);
+        let follower_calls = Arc::clone(&calls);
+        let follower = tokio::spawn(async move {
+            follower_cache
+                .get_or_load_once_stale_while_refreshing::<(), _, _>(
+                    key,
+                    Duration::from_millis(10),
+                    Duration::from_secs(1),
+                    || async move {
+                        follower_calls.fetch_add(1, Ordering::AcqRel);
+                        Ok(Some(2))
+                    },
+                    CacheLoadObserver::default(),
+                )
+                .await
+        });
+
+        assert_eq!(follower.await.unwrap().unwrap(), Some(1));
+        assert_eq!(calls.load(Ordering::Acquire), 1);
+        assert!(
+            follower_started.elapsed() < Duration::from_millis(50),
+            "follower should reuse cold-loaded stale value without reloading"
+        );
+    }
+
     #[tokio::test]
     async fn value_cache_clear_releases_same_key_followers() {
         let cache = Arc::new(ValueCache::<String, u64>::default());

apps/aether-gateway/src/cache/candidate_page.rs

@@ -0,0 +1,479 @@
+use std::collections::hash_map::DefaultHasher;
+use std::hash::{Hash, Hasher};
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, LazyLock};
+use std::time::Duration;
+
+use aether_ai_serving::AiCandidatePreselectionOutcome;
+use aether_ai_serving::AiCandidateResolutionMode;
+use aether_routing_core::ResolvedRoutingPolicy;
+use aether_runtime::{MetricKind, MetricSample};
+use aether_scheduler_core::{
+    normalize_api_format, ClientSessionAffinity, SchedulerMinimalCandidateSelectionCandidate,
+};
+use serde_json::Value;
+use sha2::Digest as _;
+
+use crate::ai_serving::{
+    EligibleLocalExecutionCandidate, GatewayAuthApiKeySnapshot, SkippedLocalExecutionCandidate,
+};
+
+const DEFAULT_CANDIDATE_PAGE_CACHE_TTL_MS: u64 = 250;
+const MIN_CANDIDATE_PAGE_CACHE_TTL_MS: u64 = 50;
+const MAX_CANDIDATE_PAGE_CACHE_TTL_MS: u64 = 1_000;
+const CANDIDATE_PAGE_CACHE_TTL_ENV: &str = "AETHER_GATEWAY_CANDIDATE_PAGE_CACHE_TTL_MS";
+
+pub(crate) type CandidatePageSnapshot = AiCandidatePreselectionOutcome<
+    SchedulerMinimalCandidateSelectionCandidate,
+    SkippedLocalExecutionCandidate,
+>;
+
+pub(crate) type CandidatePageCache =
+    super::ValueCache<CandidatePageCacheKey, Arc<CandidatePageSnapshot>>;
+
+#[derive(Debug, Clone)]
+pub(crate) struct CandidateResolvedPageSnapshot {
+    pub(crate) candidates: Vec<EligibleLocalExecutionCandidate>,
+    pub(crate) resolved_skipped: Vec<SkippedLocalExecutionCandidate>,
+}
+
+pub(crate) type CandidateResolvedPageCache =
+    super::ValueCache<CandidateResolvedPageCacheKey, Arc<CandidateResolvedPageSnapshot>>;
+
+static CANDIDATE_PAGE_CACHE_METRICS: LazyLock<CandidatePageCacheMetrics> =
+    LazyLock::new(CandidatePageCacheMetrics::default);
+
+#[derive(Debug, Default)]
+struct CandidatePageCacheMetrics {
+    hit_total: AtomicU64,
+    load_total: AtomicU64,
+    follower_wait_total: AtomicU64,
+    miss_total: AtomicU64,
+    none_total: AtomicU64,
+    resolve_hit_total: AtomicU64,
+    resolve_load_total: AtomicU64,
+    resolve_follower_wait_total: AtomicU64,
+    resolve_miss_total: AtomicU64,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub(crate) struct CandidatePageCacheKey {
+    requested_model: String,
+    client_api_format: String,
+    auth_identity: CandidatePageAuthIdentity,
+    require_streaming: bool,
+    required_capabilities_hash: String,
+    routing_policy_hash: String,
+    request_auth_channel: String,
+    scheduler_affinity_epoch: u64,
+    preselection_mode: &'static str,
+    use_api_format_alias_match: bool,
+    client_session_affinity_hash: String,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+enum CandidatePageAuthIdentity {
+    Standalone { api_key_id: String },
+    UserApiKey { user_id: String, api_key_id: String },
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub(crate) struct CandidateResolvedPageCacheKey {
+    page_key: CandidatePageCacheKey,
+    resolution_mode: &'static str,
+}
+
+impl CandidatePageCacheKey {
+    #[allow(clippy::too_many_arguments)]
+    pub(crate) fn new(
+        requested_model: &str,
+        client_api_format: &str,
+        require_streaming: bool,
+        auth_snapshot: &GatewayAuthApiKeySnapshot,
+        required_capabilities: Option<&Value>,
+        routing_policy: Option<&ResolvedRoutingPolicy>,
+        request_auth_channel: Option<&str>,
+        scheduler_affinity_epoch: u64,
+        preselection_mode: &'static str,
+        use_api_format_alias_match: bool,
+        client_session_affinity: Option<&ClientSessionAffinity>,
+    ) -> Self {
+        Self {
+            requested_model: normalize_text_key(requested_model),
+            client_api_format: normalize_api_format(client_api_format),
+            auth_identity: CandidatePageAuthIdentity::from_auth_snapshot(auth_snapshot),
+            require_streaming,
+            required_capabilities_hash: stable_json_hash(required_capabilities),
+            routing_policy_hash: stable_json_hash(routing_policy),
+            request_auth_channel: normalize_text_key(request_auth_channel.unwrap_or_default()),
+            scheduler_affinity_epoch,
+            preselection_mode,
+            use_api_format_alias_match,
+            client_session_affinity_hash: client_session_affinity_key(client_session_affinity),
+        }
+    }
+}
+
+impl CandidateResolvedPageCacheKey {
+    #[allow(clippy::too_many_arguments)]
+    pub(crate) fn new(
+        requested_model: &str,
+        client_api_format: &str,
+        require_streaming: bool,
+        auth_snapshot: &GatewayAuthApiKeySnapshot,
+        required_capabilities: Option<&Value>,
+        routing_policy: Option<&ResolvedRoutingPolicy>,
+        request_auth_channel: Option<&str>,
+        scheduler_affinity_epoch: u64,
+        preselection_mode: &'static str,
+        use_api_format_alias_match: bool,
+        client_session_affinity: Option<&ClientSessionAffinity>,
+        resolution_mode: AiCandidateResolutionMode,
+    ) -> Self {
+        Self {
+            page_key: CandidatePageCacheKey::new(
+                requested_model,
+                client_api_format,
+                require_streaming,
+                auth_snapshot,
+                required_capabilities,
+                routing_policy,
+                request_auth_channel,
+                scheduler_affinity_epoch,
+                preselection_mode,
+                use_api_format_alias_match,
+                client_session_affinity,
+            ),
+            resolution_mode: resolution_mode_name(resolution_mode),
+        }
+    }
+}
+
+impl CandidatePageAuthIdentity {
+    fn from_auth_snapshot(auth_snapshot: &GatewayAuthApiKeySnapshot) -> Self {
+        let api_key_id = normalize_text_key(&auth_snapshot.api_key_id);
+        if auth_snapshot.api_key_is_standalone {
+            Self::Standalone { api_key_id }
+        } else {
+            Self::UserApiKey {
+                user_id: normalize_text_key(&auth_snapshot.user_id),
+                api_key_id,
+            }
+        }
+    }
+}
+
+pub(crate) fn candidate_page_cache_ttl_from_env() -> Duration {
+    let ttl_ms = std::env::var(CANDIDATE_PAGE_CACHE_TTL_ENV)
+        .ok()
+        .and_then(|value| value.trim().parse::<u64>().ok())
+        .filter(|value| *value > 0)
+        .unwrap_or(DEFAULT_CANDIDATE_PAGE_CACHE_TTL_MS)
+        .clamp(
+            MIN_CANDIDATE_PAGE_CACHE_TTL_MS,
+            MAX_CANDIDATE_PAGE_CACHE_TTL_MS,
+        );
+    Duration::from_millis(ttl_ms)
+}
+
+pub(crate) fn candidate_page_cache_stale_ttl(ttl: Duration) -> Duration {
+    let stale_ttl = ttl.saturating_mul(8);
+    stale_ttl.min(Duration::from_secs(2)).max(ttl)
+}
+
+pub(crate) fn record_candidate_page_cache_hit() {
+    CANDIDATE_PAGE_CACHE_METRICS
+        .hit_total
+        .fetch_add(1, Ordering::Relaxed);
+}
+
+pub(crate) fn record_candidate_page_cache_miss() {
+    CANDIDATE_PAGE_CACHE_METRICS
+        .miss_total
+        .fetch_add(1, Ordering::Relaxed);
+}
+
+pub(crate) fn record_candidate_page_cache_load() {
+    CANDIDATE_PAGE_CACHE_METRICS
+        .load_total
+        .fetch_add(1, Ordering::Relaxed);
+}
+
+pub(crate) fn record_candidate_page_cache_follower_wait() {
+    CANDIDATE_PAGE_CACHE_METRICS
+        .follower_wait_total
+        .fetch_add(1, Ordering::Relaxed);
+}
+
+pub(crate) fn record_candidate_page_cache_none() {
+    CANDIDATE_PAGE_CACHE_METRICS
+        .none_total
+        .fetch_add(1, Ordering::Relaxed);
+}
+
+pub(crate) fn record_candidate_page_resolve_cache_hit() {
+    CANDIDATE_PAGE_CACHE_METRICS
+        .resolve_hit_total
+        .fetch_add(1, Ordering::Relaxed);
+}
+
+pub(crate) fn record_candidate_page_resolve_cache_miss() {
+    CANDIDATE_PAGE_CACHE_METRICS
+        .resolve_miss_total
+        .fetch_add(1, Ordering::Relaxed);
+}
+
+pub(crate) fn record_candidate_page_resolve_cache_load() {
+    CANDIDATE_PAGE_CACHE_METRICS
+        .resolve_load_total
+        .fetch_add(1, Ordering::Relaxed);
+}
+
+pub(crate) fn record_candidate_page_resolve_cache_follower_wait() {
+    CANDIDATE_PAGE_CACHE_METRICS
+        .resolve_follower_wait_total
+        .fetch_add(1, Ordering::Relaxed);
+}
+
+pub(crate) fn candidate_page_cache_metric_samples() -> Vec<MetricSample> {
+    vec![
+        MetricSample::new(
+            "candidate_page_cache_hit_total",
+            "Total candidate page cache hits.",
+            MetricKind::Counter,
+            CANDIDATE_PAGE_CACHE_METRICS
+                .hit_total
+                .load(Ordering::Relaxed),
+        ),
+        MetricSample::new(
+            "candidate_page_cache_miss_total",
+            "Total candidate page cache misses before singleflight registration.",
+            MetricKind::Counter,
+            CANDIDATE_PAGE_CACHE_METRICS
+                .miss_total
+                .load(Ordering::Relaxed),
+        ),
+        MetricSample::new(
+            "candidate_page_cache_load_total",
+            "Total candidate page cache loader executions.",
+            MetricKind::Counter,
+            CANDIDATE_PAGE_CACHE_METRICS
+                .load_total
+                .load(Ordering::Relaxed),
+        ),
+        MetricSample::new(
+            "candidate_page_cache_follower_wait_total",
+            "Total candidate page cache requests that waited for another loader.",
+            MetricKind::Counter,
+            CANDIDATE_PAGE_CACHE_METRICS
+                .follower_wait_total
+                .load(Ordering::Relaxed),
+        ),
+        MetricSample::new(
+            "candidate_page_cache_none_total",
+            "Total candidate page cache lookups that resolved to no page.",
+            MetricKind::Counter,
+            CANDIDATE_PAGE_CACHE_METRICS
+                .none_total
+                .load(Ordering::Relaxed),
+        ),
+        MetricSample::new(
+            "candidate_page_resolve_cache_hit_total",
+            "Total resolved candidate page cache hits.",
+            MetricKind::Counter,
+            CANDIDATE_PAGE_CACHE_METRICS
+                .resolve_hit_total
+                .load(Ordering::Relaxed),
+        ),
+        MetricSample::new(
+            "candidate_page_resolve_cache_miss_total",
+            "Total resolved candidate page cache misses before singleflight registration.",
+            MetricKind::Counter,
+            CANDIDATE_PAGE_CACHE_METRICS
+                .resolve_miss_total
+                .load(Ordering::Relaxed),
+        ),
+        MetricSample::new(
+            "candidate_page_resolve_cache_load_total",
+            "Total resolved candidate page cache loader executions.",
+            MetricKind::Counter,
+            CANDIDATE_PAGE_CACHE_METRICS
+                .resolve_load_total
+                .load(Ordering::Relaxed),
+        ),
+        MetricSample::new(
+            "candidate_page_resolve_cache_follower_wait_total",
+            "Total resolved candidate page cache requests that waited for another loader.",
+            MetricKind::Counter,
+            CANDIDATE_PAGE_CACHE_METRICS
+                .resolve_follower_wait_total
+                .load(Ordering::Relaxed),
+        ),
+    ]
+}
+
+fn normalize_text_key(value: &str) -> String {
+    value.trim().to_string()
+}
+
+fn client_session_affinity_key(affinity: Option<&ClientSessionAffinity>) -> String {
+    let Some(affinity) = affinity else {
+        return String::new();
+    };
+    let family = affinity
+        .client_family
+        .as_deref()
+        .map(str::trim)
+        .filter(|value| !value.is_empty())
+        .map(str::to_ascii_lowercase)
+        .unwrap_or_default();
+    let session = affinity
+        .session_key
+        .as_deref()
+        .map(str::trim)
+        .filter(|value| !value.is_empty())
+        .map(sha256_hex)
+        .unwrap_or_default();
+    format!("{family}:{session}")
+}
+
+fn stable_json_hash<T>(value: Option<&T>) -> String
+where
+    T: serde::Serialize,
+{
+    let Some(value) = value else {
+        return String::new();
+    };
+    match serde_json::to_vec(value) {
+        Ok(serialized) => sha256_hex(&serialized),
+        Err(_) => {
+            let mut hasher = DefaultHasher::new();
+            std::any::type_name::<T>().hash(&mut hasher);
+            format!("fallback:{:016x}", hasher.finish())
+        }
+    }
+}
+
+fn sha256_hex(value: impl AsRef<[u8]>) -> String {
+    let digest = sha2::Sha256::digest(value.as_ref());
+    digest.iter().map(|byte| format!("{byte:02x}")).collect()
+}
+
+fn resolution_mode_name(mode: AiCandidateResolutionMode) -> &'static str {
+    match mode {
+        AiCandidateResolutionMode::Standard => "standard",
+        AiCandidateResolutionMode::WithoutTransportPairGate => "without_transport_pair_gate",
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use aether_data::repository::auth::ResolvedAuthApiKeySnapshot;
+    use serde_json::json;
+
+    fn auth_snapshot(user_id: &str, api_key_id: &str) -> ResolvedAuthApiKeySnapshot {
+        ResolvedAuthApiKeySnapshot {
+            user_id: user_id.to_string(),
+            username: "user".to_string(),
+            email: None,
+            user_role: "user".to_string(),
+            user_auth_source: "local".to_string(),
+            user_is_active: true,
+            user_is_deleted: false,
+            user_rate_limit: None,
+            user_allowed_providers: None,
+            user_allowed_api_formats: None,
+            user_allowed_models: None,
+            api_key_id: api_key_id.to_string(),
+            api_key_name: None,
+            api_key_is_active: true,
+            api_key_is_locked: false,
+            api_key_is_standalone: false,
+            api_key_rate_limit: None,
+            api_key_concurrent_limit: None,
+            api_key_expires_at_unix_secs: None,
+            api_key_allowed_providers: None,
+            api_key_allowed_api_formats: None,
+            api_key_allowed_models: None,
+            api_key_ip_rules: None,
+            currently_usable: true,
+        }
+    }
+
+    #[test]
+    fn candidate_page_cache_key_isolates_auth_model_format_and_capabilities() {
+        let auth_a = auth_snapshot("user-a", "key-a");
+        let auth_b = auth_snapshot("user-b", "key-a");
+        let base = CandidatePageCacheKey::new(
+            "gpt-4o",
+            "openai:chat",
+            true,
+            &auth_a,
+            Some(&json!({"vision": true})),
+            None,
+            Some("bearer"),
+            7,
+            "provider_endpoint_key_model",
+            true,
+            None,
+        );
+        let different_user = CandidatePageCacheKey::new(
+            "gpt-4o",
+            "openai:chat",
+            true,
+            &auth_b,
+            Some(&json!({"vision": true})),
+            None,
+            Some("bearer"),
+            7,
+            "provider_endpoint_key_model",
+            true,
+            None,
+        );
+        let different_model = CandidatePageCacheKey::new(
+            "gpt-4.1",
+            "openai:chat",
+            true,
+            &auth_a,
+            Some(&json!({"vision": true})),
+            None,
+            Some("bearer"),
+            7,
+            "provider_endpoint_key_model",
+            true,
+            None,
+        );
+        let different_format = CandidatePageCacheKey::new(
+            "gpt-4o",
+            "openai:responses",
+            true,
+            &auth_a,
+            Some(&json!({"vision": true})),
+            None,
+            Some("bearer"),
+            7,
+            "provider_endpoint_key_model",
+            true,
+            None,
+        );
+        let different_capabilities = CandidatePageCacheKey::new(
+            "gpt-4o",
+            "openai:chat",
+            true,
+            &auth_a,
+            Some(&json!({"vision": false})),
+            None,
+            Some("bearer"),
+            7,
+            "provider_endpoint_key_model",
+            true,
+            None,
+        );
+
+        assert_ne!(base, different_user);
+        assert_ne!(base, different_model);
+        assert_ne!(base, different_format);
+        assert_ne!(base, different_capabilities);
+    }
+}

apps/aether-gateway/src/cache/mod.rs

@@ -1,20 +1,31 @@
 mod auth_api_key_last_used;
 mod auth_context;
 mod auth_runtime;
+mod candidate_page;
 mod dashboard_response;
 mod direct_plan_bypass;
 mod scheduler_affinity;
 mod system_config;
 
 pub(crate) use auth_api_key_last_used::AuthApiKeyLastUsedCache;
-pub(crate) use auth_context::AuthContextCache;
+pub(crate) use auth_context::{AuthContextCache, AuthContextInflightRegistration};
 pub(crate) use auth_runtime::{
     AuthApiKeyFeatureCacheKey, AuthApiKeyIdentityCacheKey, AuthSnapshotCache, AuthSnapshotCacheKey,
-    JsonValueCache, ValueCache,
+    CacheLoadObserver, JsonValueCache, ValueCache,
+};
+pub(crate) use candidate_page::{
+    candidate_page_cache_metric_samples, candidate_page_cache_stale_ttl,
+    candidate_page_cache_ttl_from_env, record_candidate_page_cache_follower_wait,
+    record_candidate_page_cache_hit, record_candidate_page_cache_load,
+    record_candidate_page_cache_miss, record_candidate_page_cache_none,
+    record_candidate_page_resolve_cache_follower_wait, record_candidate_page_resolve_cache_hit,
+    record_candidate_page_resolve_cache_load, record_candidate_page_resolve_cache_miss,
+    CandidatePageCache, CandidatePageCacheKey, CandidatePageSnapshot, CandidateResolvedPageCache,
+    CandidateResolvedPageCacheKey, CandidateResolvedPageSnapshot,
 };
 pub(crate) use dashboard_response::DashboardResponseCache;
 pub(crate) use direct_plan_bypass::DirectPlanBypassCache;
 pub(crate) use scheduler_affinity::{
     SchedulerAffinityCache, SchedulerAffinitySnapshotEntry, SchedulerAffinityTarget,
 };
-pub(crate) use system_config::SystemConfigCache;
+pub(crate) use system_config::{SystemConfigCache, SystemConfigInflightRegistration};

apps/aether-gateway/src/cache/system_config.rs

@@ -1,21 +1,43 @@
+use std::collections::HashSet;
 use std::time::Duration;
 
 use aether_cache::ExpiringMap;
-use tokio::sync::{Mutex, MutexGuard};
+use tokio::sync::Notify;
 
 const MAX_ENTRIES: usize = 512;
 
 #[derive(Debug)]
 pub(crate) struct SystemConfigCache {
     entries: ExpiringMap<String, Option<serde_json::Value>>,
-    load_guard: Mutex<()>,
+    inflight: std::sync::Mutex<HashSet<String>>,
+    notify: Notify,
 }
 
 impl Default for SystemConfigCache {
     fn default() -> Self {
         Self {
             entries: ExpiringMap::new(),
-            load_guard: Mutex::new(()),
+            inflight: std::sync::Mutex::new(HashSet::new()),
+            notify: Notify::new(),
+        }
+    }
+}
+
+pub(crate) enum SystemConfigInflightRegistration<'a> {
+    Leader(SystemConfigInflightGuard<'a>),
+    Follower,
+    Bypass,
+}
+
+pub(crate) struct SystemConfigInflightGuard<'a> {
+    cache: &'a SystemConfigCache,
+    key: Option<String>,
+}
+
+impl Drop for SystemConfigInflightGuard<'_> {
+    fn drop(&mut self) {
+        if let Some(key) = self.key.take() {
+            self.cache.finish_load(&key);
         }
     }
 }
@@ -29,11 +51,51 @@ impl SystemConfigCache {
         self.entries.insert(key, value, ttl, MAX_ENTRIES);
     }
 
-    pub(crate) async fn load_guard(&self) -> MutexGuard<'_, ()> {
-        self.load_guard.lock().await
+    pub(crate) fn register_load(&self, key: &str) -> SystemConfigInflightRegistration<'_> {
+        match self.inflight.lock() {
+            Ok(mut inflight) => {
+                if inflight.contains(key) {
+                    SystemConfigInflightRegistration::Follower
+                } else {
+                    inflight.insert(key.to_string());
+                    SystemConfigInflightRegistration::Leader(SystemConfigInflightGuard {
+                        cache: self,
+                        key: Some(key.to_string()),
+                    })
+                }
+            }
+            Err(_) => SystemConfigInflightRegistration::Bypass,
+        }
+    }
+
+    pub(crate) fn notified(&self) -> tokio::sync::futures::Notified<'_> {
+        self.notify.notified()
+    }
+
+    fn finish_load(&self, key: &str) {
+        let removed = self
+            .inflight
+            .lock()
+            .map(|mut inflight| inflight.remove(key))
+            .unwrap_or(false);
+        if removed {
+            self.notify.notify_waiters();
+        }
     }
 
     pub(crate) fn clear(&self) {
         self.entries.clear();
+        let cleared = self
+            .inflight
+            .lock()
+            .map(|mut inflight| {
+                let had_entries = !inflight.is_empty();
+                inflight.clear();
+                had_entries
+            })
+            .unwrap_or(false);
+        if cleared {
+            self.notify.notify_waiters();
+        }
     }
 }