perf(gateway): scale request hot paths for 20k streams

Shard and singleflight hot-path caches, batch and prioritize candidate and usage lifecycle persistence, and extend database and pressure-test instrumentation for 20k concurrent streams.

crates/aether-data/contracts/src/error.rs

@@ -1,4 +1,4 @@
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, Clone, thiserror::Error)]
 pub enum DataLayerError {
     #[error("invalid configuration: {0}")]
     InvalidConfiguration(String),

crates/aether-data/contracts/src/repository/auth.rs

@@ -581,6 +581,10 @@ pub enum AuthApiKeyLookupKey<'a> {
 
 #[async_trait]
 pub trait AuthApiKeyReadRepository: Send + Sync {
+    /// Clears any local read-through cache maintained by a repository wrapper.
+    /// Concrete database readers normally use the default no-op implementation.
+    fn clear_cache(&self) {}
+
     async fn find_api_key_snapshot(
         &self,
         key: AuthApiKeyLookupKey<'_>,

crates/aether-data/contracts/src/repository/candidates/types.rs

@@ -583,15 +583,25 @@ pub fn request_candidate_lifecycle_would_regress(
             | RequestCandidateStatus::Unused
             | RequestCandidateStatus::Pending
             | RequestCandidateStatus::Streaming
-    ) || existing == RequestCandidateStatus::Streaming
-        && incoming == RequestCandidateStatus::Pending
+    ) || existing == RequestCandidateStatus::Pending
+        && matches!(
+            incoming,
+            RequestCandidateStatus::Available | RequestCandidateStatus::Unused
+        )
+        || existing == RequestCandidateStatus::Streaming
+            && matches!(
+                incoming,
+                RequestCandidateStatus::Available
+                    | RequestCandidateStatus::Unused
+                    | RequestCandidateStatus::Pending
+            )
 }
 
 #[cfg(test)]
 mod tests {
     use super::{
-        derive_request_candidate_final_status, RequestCandidateFinalStatus, RequestCandidateStatus,
-        StoredRequestCandidate,
+        derive_request_candidate_final_status, request_candidate_lifecycle_would_regress,
+        RequestCandidateFinalStatus, RequestCandidateStatus, StoredRequestCandidate,
     };
 
     fn candidate(
@@ -654,4 +664,44 @@ mod tests {
             RequestCandidateFinalStatus::Success
         );
     }
+
+    #[test]
+    fn streaming_candidate_cannot_regress_to_an_earlier_planning_state() {
+        for incoming in [
+            RequestCandidateStatus::Available,
+            RequestCandidateStatus::Unused,
+            RequestCandidateStatus::Pending,
+        ] {
+            assert!(request_candidate_lifecycle_would_regress(
+                RequestCandidateStatus::Streaming,
+                incoming,
+            ));
+        }
+        assert!(!request_candidate_lifecycle_would_regress(
+            RequestCandidateStatus::Streaming,
+            RequestCandidateStatus::Success,
+        ));
+    }
+
+    #[test]
+    fn pending_candidate_cannot_regress_to_an_earlier_planning_state() {
+        for incoming in [
+            RequestCandidateStatus::Available,
+            RequestCandidateStatus::Unused,
+        ] {
+            assert!(request_candidate_lifecycle_would_regress(
+                RequestCandidateStatus::Pending,
+                incoming,
+            ));
+        }
+        for incoming in [
+            RequestCandidateStatus::Streaming,
+            RequestCandidateStatus::Success,
+        ] {
+            assert!(!request_candidate_lifecycle_would_regress(
+                RequestCandidateStatus::Pending,
+                incoming,
+            ));
+        }
+    }
 }

crates/aether-data/contracts/src/repository/pool_scores/mod.rs

@@ -9,8 +9,8 @@ pub use types::{
     GetPoolMemberScoresByIdsQuery, ListPoolMemberProbeCandidatesQuery, ListPoolMemberScoresQuery,
     ListRankedPoolMembersQuery, PoolMemberHardState, PoolMemberIdentity, PoolMemberProbeAttempt,
     PoolMemberProbeResult, PoolMemberProbeStatus, PoolMemberScheduleFeedback,
-    PoolMemberScoreRepository, PoolMemberScoreWriteRepository, PoolScoreReadRepository,
-    PoolScoreScope, StoredPoolMemberScore, UpsertPoolMemberScore, POOL_KIND_PROVIDER_KEY_POOL,
-    POOL_MEMBER_KIND_PROVIDER_API_KEY, POOL_SCORE_CAPABILITY_ACCOUNT,
+    PoolMemberScoreRepository, PoolMemberScoreUpsertMode, PoolMemberScoreWriteRepository,
+    PoolScoreReadRepository, PoolScoreScope, StoredPoolMemberScore, UpsertPoolMemberScore,
+    POOL_KIND_PROVIDER_KEY_POOL, POOL_MEMBER_KIND_PROVIDER_API_KEY, POOL_SCORE_CAPABILITY_ACCOUNT,
     POOL_SCORE_CAPABILITY_API_FORMAT, POOL_SCORE_SCOPE_KIND_ACCOUNT, POOL_SCORE_SCOPE_KIND_MODEL,
 };

crates/aether-data/contracts/src/repository/pool_scores/types.rs

@@ -212,6 +212,15 @@ impl UpsertPoolMemberScore {
     }
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum PoolMemberScoreUpsertMode {
+    #[default]
+    PreserveExistingNullableTimestamps,
+    /// Reset failures observed at or before `updated_at` while retaining successful and newer
+    /// scheduling/probe observations atomically with the upsert.
+    OAuthRecovery,
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 pub struct ListRankedPoolMembersQuery {
     pub pool_kind: String,
@@ -310,6 +319,18 @@ pub trait PoolMemberScoreWriteRepository: Send + Sync {
     async fn upsert_pool_member_score(
         &self,
         score: UpsertPoolMemberScore,
+    ) -> Result<StoredPoolMemberScore, crate::DataLayerError> {
+        self.upsert_pool_member_score_with_mode(
+            score,
+            PoolMemberScoreUpsertMode::PreserveExistingNullableTimestamps,
+        )
+        .await
+    }
+
+    async fn upsert_pool_member_score_with_mode(
+        &self,
+        score: UpsertPoolMemberScore,
+        mode: PoolMemberScoreUpsertMode,
     ) -> Result<StoredPoolMemberScore, crate::DataLayerError>;
 
     async fn mark_pool_member_probe_in_progress(

crates/aether-data/contracts/src/repository/provider_catalog/mod.rs

@@ -3,9 +3,11 @@ mod types;
 
 pub use snapshot::ProviderCatalogSnapshot;
 pub use types::{
-    ProviderCatalogKeyListOrder, ProviderCatalogKeyListQuery, ProviderCatalogReadRepository,
-    ProviderCatalogUpstreamMetadataNamespaceUpdate, ProviderCatalogWriteRepository,
-    StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
+    ProviderCatalogKeyAdaptiveState, ProviderCatalogKeyAdaptiveStateUpdate,
+    ProviderCatalogKeyHealthStateUpdate, ProviderCatalogKeyListOrder, ProviderCatalogKeyListQuery,
+    ProviderCatalogKeyRuntimeMetadataUpdate, ProviderCatalogKeyStatusSnapshotUpdate,
+    ProviderCatalogReadRepository, ProviderCatalogUpstreamMetadataNamespaceUpdate,
+    ProviderCatalogWriteRepository, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
     StoredProviderCatalogKeyMaintenanceSummary, StoredProviderCatalogKeyPage,
     StoredProviderCatalogKeyStats, StoredProviderCatalogProvider,
 };

crates/aether-data/contracts/src/repository/provider_catalog/types.rs

@@ -6,6 +6,73 @@ pub struct ProviderCatalogUpstreamMetadataNamespaceUpdate {
     pub value: serde_json::Value,
 }
 
+#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
+pub struct ProviderCatalogKeyAdaptiveState {
+    pub learned_rpm_limit: Option<u32>,
+    pub concurrent_429_count: Option<u32>,
+    pub rpm_429_count: Option<u32>,
+    pub last_429_at_unix_secs: Option<u64>,
+    pub last_429_type: Option<String>,
+    pub adjustment_history: Option<serde_json::Value>,
+    pub utilization_samples: Option<serde_json::Value>,
+    pub last_probe_increase_at_unix_secs: Option<u64>,
+    pub last_rpm_peak: Option<u32>,
+}
+
+impl ProviderCatalogKeyAdaptiveState {
+    pub fn canonicalized(&self) -> Self {
+        let mut state = self.clone();
+        state.concurrent_429_count = Some(state.concurrent_429_count.unwrap_or(0));
+        state.rpm_429_count = Some(state.rpm_429_count.unwrap_or(0));
+        state
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
+pub struct ProviderCatalogKeyAdaptiveStateUpdate {
+    pub key_id: String,
+    pub expected: ProviderCatalogKeyAdaptiveState,
+    pub next: ProviderCatalogKeyAdaptiveState,
+    /// Top-level status fields owned by adaptive rate-limit learning.
+    pub status_snapshot_patch: serde_json::Value,
+    pub updated_at_unix_secs: Option<u64>,
+}
+
+#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
+pub struct ProviderCatalogKeyRuntimeMetadataUpdate {
+    pub key_id: String,
+    pub namespace: String,
+    /// Value observed for `namespace` immediately before calculating the update.
+    ///
+    /// `None` means that the namespace was absent.  The repository must compare
+    /// this value atomically with the stored namespace and return `false` when a
+    /// concurrent writer changed it.  This makes read/modify/write metadata
+    /// producers safe across gateway instances without replacing the whole
+    /// `upstream_metadata` document.
+    pub expected_upstream_metadata_value: Option<serde_json::Value>,
+    pub upstream_metadata_value: serde_json::Value,
+    /// Top-level status fields owned by the metadata producer, normally `quota`.
+    pub status_snapshot_patch: serde_json::Value,
+    pub updated_at_unix_secs: Option<u64>,
+}
+
+#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
+pub struct ProviderCatalogKeyStatusSnapshotUpdate {
+    pub key_id: String,
+    /// Top-level status fields owned by the caller.
+    pub status_snapshot_patch: serde_json::Value,
+    pub updated_at_unix_secs: Option<u64>,
+}
+
+#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
+pub struct ProviderCatalogKeyHealthStateUpdate {
+    pub key_id: String,
+    pub expected_health_by_format: Option<serde_json::Value>,
+    pub expected_circuit_breaker_by_format: Option<serde_json::Value>,
+    pub health_by_format: Option<serde_json::Value>,
+    pub circuit_breaker_by_format: Option<serde_json::Value>,
+}
+
 #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
 pub struct StoredProviderCatalogProvider {
     pub id: String,
@@ -478,6 +545,22 @@ impl StoredProviderCatalogKey {
     }
 }
 
+impl From<&StoredProviderCatalogKey> for ProviderCatalogKeyAdaptiveState {
+    fn from(key: &StoredProviderCatalogKey) -> Self {
+        Self {
+            learned_rpm_limit: key.learned_rpm_limit,
+            concurrent_429_count: Some(key.concurrent_429_count.unwrap_or(0)),
+            rpm_429_count: Some(key.rpm_429_count.unwrap_or(0)),
+            last_429_at_unix_secs: key.last_429_at_unix_secs,
+            last_429_type: key.last_429_type.clone(),
+            adjustment_history: key.adjustment_history.clone(),
+            utilization_samples: key.utilization_samples.clone(),
+            last_probe_increase_at_unix_secs: key.last_probe_increase_at_unix_secs,
+            last_rpm_peak: key.last_rpm_peak,
+        }
+    }
+}
+
 #[cfg(test)]
 mod transport_tests {
     use super::StoredProviderCatalogKey;
@@ -737,6 +820,15 @@ pub trait ProviderCatalogWriteRepository: Send + Sync {
         expires_at_unix_secs: Option<u64>,
     ) -> Result<bool, crate::DataLayerError>;
 
+    async fn update_key_oauth_runtime_state(
+        &self,
+        key_id: &str,
+        oauth_invalid_at_unix_secs: Option<u64>,
+        oauth_invalid_reason: Option<&str>,
+        encrypted_auth_config_update: Option<&str>,
+        updated_at_unix_secs: Option<u64>,
+    ) -> Result<bool, crate::DataLayerError>;
+
     async fn update_key_health_state(
         &self,
         key_id: &str,
@@ -744,6 +836,53 @@ pub trait ProviderCatalogWriteRepository: Send + Sync {
         health_by_format: Option<&serde_json::Value>,
         circuit_breaker_by_format: Option<&serde_json::Value>,
     ) -> Result<bool, crate::DataLayerError>;
+
+    /// Explicit administrator recovery action; does not replace other usage counters.
+    async fn reset_key_error_count(&self, key_id: &str) -> Result<bool, crate::DataLayerError>;
+
+    /// Compare-and-swap adaptive fields without replacing unrelated key columns.
+    async fn compare_and_update_key_adaptive_state(
+        &self,
+        _update: &ProviderCatalogKeyAdaptiveStateUpdate,
+    ) -> Result<bool, crate::DataLayerError> {
+        Err(crate::DataLayerError::InvalidConfiguration(
+            "provider catalog adaptive state updates are not supported by this repository"
+                .to_string(),
+        ))
+    }
+
+    /// Atomically replaces one upstream metadata namespace and merges owned status fields.
+    async fn update_key_runtime_metadata(
+        &self,
+        _update: &ProviderCatalogKeyRuntimeMetadataUpdate,
+    ) -> Result<bool, crate::DataLayerError> {
+        Err(crate::DataLayerError::InvalidConfiguration(
+            "provider catalog runtime metadata updates are not supported by this repository"
+                .to_string(),
+        ))
+    }
+
+    /// Atomically merges caller-owned top-level status fields.
+    async fn update_key_status_snapshot(
+        &self,
+        _update: &ProviderCatalogKeyStatusSnapshotUpdate,
+    ) -> Result<bool, crate::DataLayerError> {
+        Err(crate::DataLayerError::InvalidConfiguration(
+            "provider catalog status snapshot patches are not supported by this repository"
+                .to_string(),
+        ))
+    }
+
+    /// Compare-and-swap health JSON without changing administrator-owned activation state.
+    async fn compare_and_update_key_health_state(
+        &self,
+        _update: &ProviderCatalogKeyHealthStateUpdate,
+    ) -> Result<bool, crate::DataLayerError> {
+        Err(crate::DataLayerError::InvalidConfiguration(
+            "provider catalog runtime health updates are not supported by this repository"
+                .to_string(),
+        ))
+    }
 }
 
 #[cfg(test)]

crates/aether-data/contracts/src/repository/routing_profiles/types.rs

@@ -181,6 +181,19 @@ pub trait RoutingGroupReadRepository: Send + Sync {
         query: &RoutingGroupBindingQuery,
     ) -> Result<Vec<StoredRoutingGroupBinding>, crate::DataLayerError>;
 
+    /// Return whether at least one routing-group binding exists.
+    ///
+    /// Implementations backed by a database should override this with an
+    /// existence query so callers do not have to materialize the full binding
+    /// table just to choose a fast path. The default keeps third-party and test
+    /// repositories source-compatible.
+    async fn has_any_routing_group_binding(&self) -> Result<bool, crate::DataLayerError> {
+        Ok(!self
+            .list_routing_group_bindings(&RoutingGroupBindingQuery::default())
+            .await?
+            .is_empty())
+    }
+
     async fn list_routing_group_versions(
         &self,
         group_id: &str,

crates/aether-data/contracts/src/repository/usage/types.rs

@@ -2021,11 +2021,61 @@ impl UpsertUsageRecord {
 
 #[async_trait]
 pub trait UsageWriteRepository: Send + Sync {
+    /// Whether lightweight first-byte upserts avoid backend-wide counter rebuilds.
+    fn supports_first_byte_usage_fast_path(&self) -> bool {
+        false
+    }
+
+    /// Whether this backend can persist several first-byte transitions in one statement.
+    ///
+    /// The default deliberately remains disabled so adapters that do not implement a native
+    /// batch write retain the existing single-row behavior through `upsert_first_byte_many`.
+    fn supports_first_byte_usage_batch(&self) -> bool {
+        false
+    }
+
+    /// Whether this backend can persist new pending lifecycle rows and their auxiliary snapshots
+    /// in native batches.
+    ///
+    /// Implementations must preserve the full [`Self::upsert`] persistence contract. In
+    /// particular, opting into this capability must not drop HTTP audit/body data, routing or
+    /// settlement snapshots, counter deltas, or the non-regression rules for existing rows.
+    fn supports_pending_usage_batch(&self) -> bool {
+        false
+    }
+
     async fn upsert(
         &self,
         usage: UpsertUsageRecord,
     ) -> Result<StoredRequestUsageAudit, crate::DataLayerError>;
 
+    async fn upsert_first_byte(
+        &self,
+        usage: UpsertUsageRecord,
+    ) -> Result<(), crate::DataLayerError> {
+        self.upsert(usage).await.map(|_| ())
+    }
+
+    async fn upsert_first_byte_many(
+        &self,
+        usages: Vec<UpsertUsageRecord>,
+    ) -> Result<(), crate::DataLayerError> {
+        for usage in usages {
+            self.upsert_first_byte(usage).await?;
+        }
+        Ok(())
+    }
+
+    async fn upsert_pending_many(
+        &self,
+        usages: Vec<UpsertUsageRecord>,
+    ) -> Result<(), crate::DataLayerError> {
+        for usage in usages {
+            self.upsert(usage).await?;
+        }
+        Ok(())
+    }
+
     async fn rebuild_api_key_usage_stats(&self) -> Result<u64, crate::DataLayerError>;
 
     async fn rebuild_provider_api_key_usage_stats(&self) -> Result<u64, crate::DataLayerError>;