Implement generic pool member scoring and probing

apps/aether-gateway/src/state/catalog.rs

@@ -1,7 +1,8 @@
 use super::{AppState, GatewayError, LocalMutationOutcome, LocalProviderDeleteTaskState};
 use crate::handlers::shared::sync_provider_key_oauth_status_snapshot;
-use aether_data_contracts::repository::{candidates, global_models, provider_catalog};
+use aether_data_contracts::repository::{candidates, global_models, pool_scores, provider_catalog};
 use std::time::{SystemTime, UNIX_EPOCH};
+use tracing::warn;
 
 impl AppState {
     pub fn has_provider_catalog_data_reader(&self) -> bool {
@@ -529,6 +530,25 @@ impl AppState {
             .cleanup_deleted_provider_catalog_refs(provider_id, endpoint_ids, key_ids)
             .await
             .map_err(|err| GatewayError::Internal(err.to_string()))?;
+        for key_id in key_ids {
+            if let Err(err) = self
+                .data
+                .delete_pool_member_scores_for_member(
+                    &pool_scores::PoolMemberIdentity::provider_api_key(
+                        provider_id.to_string(),
+                        key_id.to_string(),
+                    ),
+                )
+                .await
+            {
+                warn!(
+                    provider_id,
+                    key_id,
+                    error = ?err,
+                    "gateway provider catalog cleanup: failed to delete pool member scores"
+                );
+            }
+        }
         if !endpoint_ids.is_empty() || !key_ids.is_empty() {
             self.clear_provider_transport_snapshot_cache();
         }
@@ -620,12 +640,38 @@ impl AppState {
         &self,
         key_id: &str,
     ) -> Result<bool, GatewayError> {
+        let existing_key = self
+            .data
+            .list_provider_catalog_keys_by_ids(&[key_id.to_string()])
+            .await
+            .map_err(|err| GatewayError::Internal(err.to_string()))?
+            .into_iter()
+            .next();
         let deleted = self
             .data
             .delete_provider_catalog_key(key_id)
             .await
             .map_err(|err| GatewayError::Internal(err.to_string()))?;
         if deleted {
+            if let Some(key) = existing_key {
+                if let Err(err) = self
+                    .data
+                    .delete_pool_member_scores_for_member(
+                        &pool_scores::PoolMemberIdentity::provider_api_key(
+                            key.provider_id.clone(),
+                            key.id.clone(),
+                        ),
+                    )
+                    .await
+                {
+                    warn!(
+                        provider_id = %key.provider_id,
+                        key_id = %key.id,
+                        error = ?err,
+                        "gateway provider catalog key delete: failed to delete pool member scores"
+                    );
+                }
+            }
             self.clear_provider_transport_snapshot_cache();
         }
         Ok(deleted)

apps/aether-gateway/src/state/core.rs

@@ -47,6 +47,7 @@ use crate::maintenance::spawn_oauth_token_refresh_worker;
 use crate::maintenance::spawn_pending_cleanup_worker;
 use crate::maintenance::spawn_pool_monitor_worker;
 use crate::maintenance::spawn_pool_quota_probe_worker;
+use crate::maintenance::spawn_pool_score_rebuild_worker;
 use crate::maintenance::spawn_provider_checkin_worker;
 use crate::maintenance::spawn_proxy_node_metrics_cleanup_worker;
 use crate::maintenance::spawn_proxy_node_stale_cleanup_worker;
@@ -1113,6 +1114,10 @@ impl AppState {
             crate::task_runtime::TASK_KEY_POOL_QUOTA_PROBE,
             spawn_pool_quota_probe_worker(self.clone()),
         );
+        supervise_worker(
+            crate::task_runtime::TASK_KEY_POOL_SCORE_REBUILD,
+            spawn_pool_score_rebuild_worker(self.clone()),
+        );
         supervise_worker(
             crate::task_runtime::TASK_KEY_STATS_HOURLY_AGG,
             spawn_stats_hourly_aggregation_worker(self.data.clone()),

apps/aether-gateway/src/state/runtime/candidate_queries.rs

@@ -57,6 +57,16 @@ impl AppState {
             .map_err(|err| GatewayError::Internal(err.to_string()))
     }
 
+    pub(crate) async fn list_pool_key_candidate_rows_for_group_key_ids(
+        &self,
+        query: &candidate_selection::StoredPoolKeyCandidateRowsByKeyIdsQuery,
+    ) -> Result<Vec<candidate_selection::StoredMinimalCandidateSelectionRow>, GatewayError> {
+        self.data
+            .list_pool_key_candidate_rows_for_group_key_ids(query)
+            .await
+            .map_err(|err| GatewayError::Internal(err.to_string()))
+    }
+
     pub(crate) async fn read_provider_quota_snapshot(
         &self,
         provider_id: &str,