feat: add adaptive pool metrics and self-check

apps/aether-gateway/src/handlers/admin/provider/pool_admin/read_routes/overview.rs

@@ -5,7 +5,12 @@ use super::{
     read_admin_provider_pool_cooldown_counts,
     ADMIN_POOL_PROVIDER_CATALOG_READER_UNAVAILABLE_DETAIL,
 };
+use crate::handlers::admin::provider::shared::support::admin_provider_pool_quota_probe_active_members_key;
 use crate::handlers::admin::request::AdminAppState;
+use crate::maintenance::PoolQuotaProbeWorkerConfig;
+use crate::provider_pool_demand::{
+    provider_pool_burst_pending, read_provider_pool_demand_snapshot,
+};
 use crate::GatewayError;
 use aether_admin::provider::pool as admin_provider_pool_pure;
 use axum::{
@@ -14,6 +19,7 @@ use axum::{
     response::{IntoResponse, Response},
     Json,
 };
+use serde_json::{json, Value};
 
 pub(super) async fn build_admin_pool_overview_response(
     state: &AdminAppState<'_>,
@@ -60,17 +66,77 @@ pub(super) async fn build_admin_pool_overview_response(
         .map(|item| (item.provider_id.clone(), item))
         .collect::<BTreeMap<_, _>>();
 
+    let probe_config = PoolQuotaProbeWorkerConfig::from_env();
+    let mut runtime_metrics_by_provider = BTreeMap::new();
+    for (provider, pool_config) in &pool_enabled_providers {
+        let active_keys = key_stats_by_provider
+            .get(&provider.id)
+            .map(|item| item.active_keys as usize)
+            .unwrap_or(0);
+        let hot_count = if pool_config.probing_enabled {
+            state
+                .runtime_state()
+                .set_len(&admin_provider_pool_quota_probe_active_members_key(
+                    &provider.id,
+                ))
+                .await
+                .unwrap_or(0)
+        } else {
+            0
+        };
+        let demand_snapshot = read_provider_pool_demand_snapshot(
+            state.runtime_state(),
+            &provider.id,
+            active_keys,
+            probe_config.max_keys_per_provider,
+        )
+        .await;
+        let burst_pending = pool_config.probing_enabled
+            && provider_pool_burst_pending(state.runtime_state(), &provider.id).await;
+        runtime_metrics_by_provider.insert(
+            provider.id.clone(),
+            json!({
+                "provider_hot_count": hot_count,
+                "provider_desired_hot": if pool_config.probing_enabled {
+                    demand_snapshot.desired_hot
+                } else {
+                    0
+                },
+                "provider_in_flight": demand_snapshot.in_flight,
+                "provider_ema_in_flight": demand_snapshot.ema_in_flight,
+                "provider_burst_pending": burst_pending,
+            }),
+        );
+    }
+
     let providers = pool_enabled_providers
         .into_iter()
         .map(|(provider, _)| provider)
         .collect::<Vec<_>>();
 
-    Ok(
-        Json(admin_provider_pool_pure::build_admin_pool_overview_payload(
-            &providers,
-            &key_stats_by_provider,
-            &cooldown_counts_by_provider,
-        ))
-        .into_response(),
-    )
+    let mut payload = admin_provider_pool_pure::build_admin_pool_overview_payload(
+        &providers,
+        &key_stats_by_provider,
+        &cooldown_counts_by_provider,
+    );
+    if let Some(items) = payload.get_mut("items").and_then(Value::as_array_mut) {
+        for item in items {
+            let Some(provider_id) = item.get("provider_id").and_then(Value::as_str) else {
+                continue;
+            };
+            let Some(metrics) = runtime_metrics_by_provider.get(provider_id) else {
+                continue;
+            };
+            let Some(item_object) = item.as_object_mut() else {
+                continue;
+            };
+            if let Some(metrics_object) = metrics.as_object() {
+                for (key, value) in metrics_object {
+                    item_object.insert(key.clone(), value.clone());
+                }
+            }
+        }
+    }
+
+    Ok(Json(payload).into_response())
 }