Merge remote-tracking branch 'origin/main' into codex/pool-key-bulk-management-20260714

# Conflicts:
#	apps/aether-gateway/src/handlers/admin/request/provider/tasks.rs
#	frontend/src/api/endpoints/pool.ts

apps/aether-gateway/src/state/app.rs

@@ -2,15 +2,16 @@ use std::collections::HashMap;
 use std::sync::atomic::AtomicU64;
 use std::sync::Arc;
 use std::sync::Mutex as StdMutex;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
 use aether_data::repository::users::StoredUserGroup;
 use aether_data_contracts::repository::billing::UserDailyQuotaAvailabilityRecord;
 use aether_data_contracts::repository::quota::StoredProviderQuotaSnapshot;
+use aether_data_contracts::repository::usage::UsageCounterHealthSnapshot;
 use aether_runtime::ConcurrencyGate;
 use aether_runtime_state::{RuntimeSemaphore, RuntimeState};
 use dashmap::DashMap;
-use tokio::sync::{Mutex as TokioMutex, Semaphore};
+use tokio::sync::{Mutex as TokioMutex, RwLock as TokioRwLock, Semaphore};
 
 use super::super::async_task::{VideoTaskPollerConfig, VideoTaskService};
 use super::super::cache::{
@@ -106,6 +107,8 @@ pub(crate) struct FrontdoorRuntimeGuardConfig {
     pub(crate) upstream_target_gate_limit: Option<usize>,
 }
 
+pub(crate) const METRIC_SNAPSHOT_TTL: Duration = Duration::from_secs(2);
+
 impl FrontdoorRuntimeGuardConfig {
     pub(crate) fn from_env() -> Self {
         Self {
@@ -364,6 +367,8 @@ pub struct AppState {
     #[cfg(test)]
     pub(crate) execution_runtime_sync_override: Option<TestExecutionRuntimeSyncOverride>,
     pub(crate) data: Arc<GatewayDataState>,
+    pub(crate) background_data: Arc<GatewayDataState>,
+    pub(crate) background_data_isolated: bool,
     pub(crate) runtime_state: Arc<RuntimeState>,
     pub(crate) usage_runtime: Arc<usage::UsageRuntime>,
     pub(crate) video_tasks: Arc<VideoTaskService>,
@@ -377,6 +382,7 @@ pub struct AppState {
     pub(crate) upstream_target_admission: Arc<crate::upstream_admission::UpstreamTargetAdmission>,
     pub(crate) distributed_request_gate: Option<Arc<RuntimeSemaphore>>,
     pub(crate) client: reqwest::Client,
+    pub(crate) owner_forward_client: reqwest::Client,
     pub(crate) auth_context_cache: Arc<AuthContextCache>,
     pub(crate) auth_snapshot_cache: Arc<AuthSnapshotCache>,
     pub(crate) admin_security_blacklist_cache: Arc<ValueCache<String, bool>>,
@@ -411,6 +417,13 @@ pub struct AppState {
     pub(crate) usage_counter_flush_metrics: Arc<UsageCounterFlushRuntimeMetrics>,
     pub(crate) task_supervisor_metrics: TaskSupervisorMetrics,
     pub(crate) process_resource_monitor: Arc<crate::process_metrics::GatewayProcessResourceMonitor>,
+    pub(crate) metric_snapshot:
+        Arc<TokioRwLock<Option<(Instant, Vec<aether_runtime::MetricSample>)>>>,
+    pub(crate) metric_snapshot_refresh: Arc<TokioMutex<()>>,
+    pub(crate) usage_counter_exact_health_metric_snapshot:
+        Arc<TokioRwLock<Option<(Instant, UsageCounterHealthSnapshot)>>>,
+    pub(crate) usage_counter_exact_health_metric_last_attempt: Arc<StdMutex<Option<Instant>>>,
+    pub(crate) usage_counter_exact_health_metric_refresh: Arc<TokioMutex<()>>,
     pub(crate) request_candidate_queue: Option<Arc<RequestCandidateQueueRuntime>>,
     pub(crate) frontdoor_cors: Option<Arc<FrontdoorCorsConfig>>,
     pub(crate) frontdoor_user_rpm: Arc<FrontdoorUserRpmLimiter>,

apps/aether-gateway/src/state/catalog.rs

@@ -736,6 +736,79 @@ impl AppState {
         Ok(updated)
     }
 
+    pub(crate) async fn upsert_provider_catalog_key_upstream_metadata_namespace(
+        &self,
+        key_id: &str,
+        namespace: &str,
+        value: &serde_json::Value,
+        updated_at_unix_secs: Option<u64>,
+    ) -> Result<bool, GatewayError> {
+        let updated = self
+            .data
+            .upsert_provider_catalog_key_upstream_metadata_namespace(
+                key_id,
+                namespace,
+                value,
+                updated_at_unix_secs,
+            )
+            .await
+            .map_err(|err| GatewayError::Internal(err.to_string()))?;
+        if updated {
+            self.invalidate_provider_routing_caches();
+        }
+        Ok(updated)
+    }
+
+    pub(crate) async fn update_provider_catalog_key_model_fetch_state(
+        &self,
+        key_id: &str,
+        allowed_models: Option<&serde_json::Value>,
+        last_models_fetch_at_unix_secs: Option<u64>,
+        last_models_fetch_error: Option<&str>,
+        updated_at_unix_secs: Option<u64>,
+    ) -> Result<bool, GatewayError> {
+        let updated = self
+            .data
+            .update_provider_catalog_key_model_fetch_state(
+                key_id,
+                allowed_models,
+                last_models_fetch_at_unix_secs,
+                last_models_fetch_error,
+                updated_at_unix_secs,
+            )
+            .await
+            .map_err(|err| GatewayError::Internal(err.to_string()))?;
+        if updated {
+            self.invalidate_provider_routing_caches();
+        }
+        Ok(updated)
+    }
+
+    pub(crate) async fn update_provider_catalog_key_model_fetch_success(
+        &self,
+        key_id: &str,
+        allowed_models: Option<&serde_json::Value>,
+        last_models_fetch_at_unix_secs: u64,
+        upstream_metadata_updates: &[provider_catalog::ProviderCatalogUpstreamMetadataNamespaceUpdate],
+        updated_at_unix_secs: Option<u64>,
+    ) -> Result<bool, GatewayError> {
+        let updated = self
+            .data
+            .update_provider_catalog_key_model_fetch_success(
+                key_id,
+                allowed_models,
+                last_models_fetch_at_unix_secs,
+                upstream_metadata_updates,
+                updated_at_unix_secs,
+            )
+            .await
+            .map_err(|err| GatewayError::Internal(err.to_string()))?;
+        if updated {
+            self.invalidate_provider_routing_caches();
+        }
+        Ok(updated)
+    }
+
     pub(crate) async fn delete_provider_catalog_key(
         &self,
         key_id: &str,
@@ -1346,6 +1419,7 @@ mod tests {
         let ttl = Duration::from_secs(300);
         let cache_key = CandidatePageCacheKey::new(
             "gpt-5",
+            None,
             "openai:chat",
             true,
             &sample_auth_snapshot(),
@@ -1356,6 +1430,7 @@ mod tests {
             "fixed_order",
             true,
             None,
+            "",
         );
         state.candidate_page_cache.insert(
             cache_key.clone(),

apps/aether-gateway/src/state/integrations.rs

@@ -9,7 +9,8 @@ use aether_data_contracts::repository::global_models::{
     StoredAdminProviderModel, UpsertAdminProviderModelRecord,
 };
 use aether_data_contracts::repository::provider_catalog::{
-    StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
+    ProviderCatalogUpstreamMetadataNamespaceUpdate, StoredProviderCatalogEndpoint,
+    StoredProviderCatalogKey, StoredProviderCatalogProvider,
 };
 use aether_data_contracts::repository::quota::StoredProviderQuotaSnapshot;
 use aether_model_fetch::{
@@ -319,11 +320,43 @@ impl ModelFetchRuntimeState for AppState {
         execution_runtime::execute_execution_runtime_sync_plan(self, None, plan).await
     }
 
-    async fn update_provider_catalog_key(
+    async fn update_provider_catalog_key_model_fetch_state(
         &self,
-        key: &StoredProviderCatalogKey,
+        key_id: &str,
+        allowed_models: Option<&Value>,
+        last_models_fetch_at_unix_secs: Option<u64>,
+        last_models_fetch_error: Option<&str>,
+        updated_at_unix_secs: Option<u64>,
     ) -> Result<(), GatewayError> {
-        AppState::update_provider_catalog_key(self, key).await?;
+        AppState::update_provider_catalog_key_model_fetch_state(
+            self,
+            key_id,
+            allowed_models,
+            last_models_fetch_at_unix_secs,
+            last_models_fetch_error,
+            updated_at_unix_secs,
+        )
+        .await?;
+        Ok(())
+    }
+
+    async fn update_provider_catalog_key_model_fetch_success(
+        &self,
+        key_id: &str,
+        allowed_models: Option<&Value>,
+        last_models_fetch_at_unix_secs: u64,
+        upstream_metadata_updates: &[ProviderCatalogUpstreamMetadataNamespaceUpdate],
+        updated_at_unix_secs: Option<u64>,
+    ) -> Result<(), GatewayError> {
+        AppState::update_provider_catalog_key_model_fetch_success(
+            self,
+            key_id,
+            allowed_models,
+            last_models_fetch_at_unix_secs,
+            upstream_metadata_updates,
+            updated_at_unix_secs,
+        )
+        .await?;
         Ok(())
     }
 

apps/aether-gateway/src/state/runtime/auth/api_keys.rs

@@ -13,7 +13,7 @@ const AUTH_API_KEY_SNAPSHOT_RUNTIME_CACHE_TTL: Duration = Duration::from_secs(30
 use super::super::super::{AUTH_API_KEY_LAST_USED_MAX_ENTRIES, AUTH_API_KEY_LAST_USED_TTL};
 
 impl AppState {
-    async fn acquire_auth_snapshot_load_gate(
+    pub(crate) async fn acquire_auth_snapshot_load_gate(
         &self,
     ) -> Result<Option<aether_runtime::ConcurrencyPermit>, GatewayError> {
         let Some(gate) = self.auth_snapshot_load_gate.as_ref() else {

apps/aether-gateway/src/state/runtime/billing/admin.rs

@@ -525,12 +525,27 @@ impl AppState {
         }
         let ttl = self.frontdoor_runtime_guards.auth_capacity_cache_ttl;
         if ttl.is_zero() {
-            return self.find_user_daily_quota_availability(user_id).await;
+            return self
+                .find_user_daily_quota_availability_for_auth_uncached(user_id)
+                .await;
         }
         self.auth_daily_quota_availability_cache
             .get_or_load(user_id.to_string(), ttl, || async move {
+                let _permit = self.acquire_auth_snapshot_load_gate().await?;
                 self.find_user_daily_quota_availability(user_id).await
             })
             .await
     }
+
+    pub(crate) async fn find_user_daily_quota_availability_for_auth_uncached(
+        &self,
+        user_id: &str,
+    ) -> Result<Option<UserDailyQuotaAvailabilityRecord>, GatewayError> {
+        let user_id = user_id.trim();
+        if user_id.is_empty() {
+            return Ok(None);
+        }
+        let _permit = self.acquire_auth_snapshot_load_gate().await?;
+        self.find_user_daily_quota_availability(user_id).await
+    }
 }

apps/aether-gateway/src/state/runtime/usage_queries.rs

@@ -181,6 +181,16 @@ impl AppState {
             .map_err(|err| GatewayError::Internal(err.to_string()))
     }
 
+    pub(crate) async fn summarize_dashboard_stats(
+        &self,
+        query: &usage::UsageDashboardSummaryQuery,
+    ) -> Result<usage::StoredUsageDashboardStatsSummary, GatewayError> {
+        self.data
+            .summarize_dashboard_stats(query)
+            .await
+            .map_err(|err| GatewayError::Internal(err.to_string()))
+    }
+
     pub(crate) async fn list_dashboard_daily_breakdown(
         &self,
         query: &usage::UsageDashboardDailyBreakdownQuery,

apps/aether-gateway/src/state/runtime/wallet/reads.rs

@@ -73,15 +73,16 @@ impl AppState {
 
         let ttl = self.frontdoor_runtime_guards.auth_capacity_cache_ttl;
         if ttl.is_zero() {
-            return self.find_wallet(lookup).await;
+            return self
+                .read_wallet_snapshot_for_auth_uncached(user_id, api_key_id, api_key_is_standalone)
+                .await;
         }
 
         self.auth_wallet_snapshot_cache
-            .get_or_load(
-                cache_key,
-                ttl,
-                || async move { self.find_wallet(lookup).await },
-            )
+            .get_or_load(cache_key, ttl, || async move {
+                let _permit = self.acquire_auth_snapshot_load_gate().await?;
+                self.find_wallet(lookup).await
+            })
             .await
     }
 
@@ -117,6 +118,7 @@ impl AppState {
             return Ok(None);
         };
 
+        let _permit = self.acquire_auth_snapshot_load_gate().await?;
         self.find_wallet(lookup).await
     }