Fix active probe pool fallback tracing

apps/aether-gateway/src/ai_serving/planner/candidate_materialization.rs

@@ -7,7 +7,13 @@ use aether_ai_serving::{
     AiCandidatePreselectionOutcome, AiSkippedCandidatePersistencePort,
 };
 use aether_dispatch_core::{DispatchSequence, DispatchSequenceItem};
-use aether_scheduler_core::{ClientSessionAffinity, SchedulerMinimalCandidateSelectionCandidate};
+use aether_routing_core::{
+    rank_vector_for_candidate, CandidateKind, ResolvedRoutingPolicy, RoutingCandidateFacts,
+    RoutingCandidateTrace, RoutingDecisionTrace,
+};
+use aether_scheduler_core::{
+    ClientSessionAffinity, SchedulerMinimalCandidateSelectionCandidate, SchedulerRankingOutcome,
+};
 use async_trait::async_trait;
 use serde_json::Value;
 use std::collections::VecDeque;
@@ -19,6 +25,7 @@ use tracing::warn;
 use uuid::Uuid;
 
 use crate::ai_serving::planner::candidate_affinity_cache::remember_scheduler_affinity_for_candidate_at_epoch;
+use crate::ai_serving::planner::candidate_ranking::scheduler_ordering_config_for_routing_policy;
 use crate::ai_serving::planner::candidate_resolution::{
     resolve_and_rank_logical_local_execution_candidates, EligibleLocalExecutionCandidate,
     LocalExecutionCandidateKind, SkippedLocalExecutionCandidate,
@@ -36,7 +43,7 @@ use crate::dispatch::refs::dispatch_ref_for_local_candidate;
 use crate::handlers::shared::provider_pool::admin_provider_pool_config_from_config_value;
 use crate::orchestration::{local_attempt_slot_count, ExecutionAttemptIdentity};
 use crate::scheduler::candidate::API_KEY_CONCURRENCY_LIMIT_SKIP_REASON;
-use crate::scheduler::config::{read_scheduler_ordering_config, SchedulerSchedulingMode};
+use crate::scheduler::config::SchedulerSchedulingMode;
 use crate::{AppState, GatewayError};
 
 const POOL_KEY_RETRY_INDEX_STRIDE: u32 = 100;
@@ -74,6 +81,7 @@ enum LocalExecutionCandidateAttemptSourceItem<'a> {
         cursor: PoolKeyCursor<'a>,
         candidate_index: u32,
         pending_attempts: DispatchSequence<LocalExecutionCandidateAttempt>,
+        pool_exhaustion_persistence: Option<PoolGroupExhaustionPersistenceContext>,
     },
     RequestedModelPage {
         cursor: Box<RequestedModelAttemptPageCursor<'a>>,
@@ -110,11 +118,20 @@ impl<'a> LocalExecutionCandidateAttemptSource<'a> {
                     cursor,
                     candidate_index,
                     pending_attempts,
+                    pool_exhaustion_persistence,
                 } => {
                     if let Some(attempt) = next_attempt_from_dispatch_sequence(pending_attempts) {
                         return Some(attempt);
                     }
                     let Some(candidate) = cursor.next_key().await else {
+                        if let Some(skipped) = cursor.exhausted_group_skipped_candidate() {
+                            persist_pool_group_exhaustion_skipped_candidate(
+                                pool_exhaustion_persistence.as_ref(),
+                                *candidate_index,
+                                skipped,
+                            )
+                            .await;
+                        }
                         cursor.log_exhausted();
                         let _ = cursor.take_skipped_candidates();
                         self.items.pop_front();
@@ -179,6 +196,39 @@ pub(crate) struct LocalSkippedCandidatePersistenceContext<'a> {
     pub(crate) record_runtime_miss_diagnostic: bool,
 }
 
+#[derive(Clone)]
+struct PoolGroupExhaustionPersistenceContext {
+    app: AppState,
+    trace_id: String,
+    user_id: String,
+    api_key_id: String,
+    required_capabilities: Option<Value>,
+    error_context: &'static str,
+    client_api_format: String,
+    routing_policy: Option<ResolvedRoutingPolicy>,
+}
+
+impl PoolGroupExhaustionPersistenceContext {
+    fn new(
+        app: AppState,
+        trace_id: &str,
+        context: LocalSkippedCandidatePersistenceContext<'_>,
+        client_api_format: &str,
+        routing_policy: Option<&ResolvedRoutingPolicy>,
+    ) -> Self {
+        Self {
+            app,
+            trace_id: trace_id.to_string(),
+            user_id: context.user_id.to_string(),
+            api_key_id: context.api_key_id.to_string(),
+            required_capabilities: context.required_capabilities.cloned(),
+            error_context: context.error_context,
+            client_api_format: client_api_format.to_string(),
+            routing_policy: routing_policy.cloned(),
+        }
+    }
+}
+
 pub(crate) use aether_ai_serving::AiCandidateResolutionMode as LocalCandidateResolutionMode;
 
 struct GatewayLocalCandidateMaterializationPort<'a, F, G> {
@@ -189,6 +239,7 @@ struct GatewayLocalCandidateMaterializationPort<'a, F, G> {
     auth_snapshot: Option<&'a GatewayAuthApiKeySnapshot>,
     client_session_affinity: Option<&'a ClientSessionAffinity>,
     required_capabilities: Option<&'a Value>,
+    routing_policy: Option<&'a ResolvedRoutingPolicy>,
     sticky_session_token: Option<&'a str>,
     request_auth_channel: Option<&'a str>,
     persistence_policy: LocalCandidatePersistencePolicy<'a>,
@@ -243,6 +294,7 @@ where
             self.auth_snapshot,
             self.client_session_affinity,
             self.required_capabilities,
+            self.routing_policy,
             self.sticky_session_token,
             self.request_auth_channel,
             self.resolution_mode,
@@ -281,6 +333,8 @@ where
                 .skipped
                 .record_runtime_miss_diagnostic,
             candidates,
+            self.routing_policy,
+            self.client_api_format,
             self.sticky_session_token,
             self.requested_model,
             self.request_auth_channel,
@@ -294,6 +348,12 @@ where
         starting_candidate_index: u32,
         skipped_candidates: Vec<Self::Skipped>,
     ) -> Result<(), Self::Error> {
+        let skipped_candidates = attach_routing_trace_to_skipped_candidates(
+            self.routing_policy,
+            self.client_api_format,
+            starting_candidate_index,
+            skipped_candidates,
+        );
         persist_skipped_local_execution_candidates_with_context(
             self.state.app(),
             self.trace_id,
@@ -432,6 +492,7 @@ pub(crate) async fn materialize_local_execution_candidates_with_serving<F, G>(
     auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
     client_session_affinity: Option<&ClientSessionAffinity>,
     required_capabilities: Option<&Value>,
+    routing_policy: Option<&ResolvedRoutingPolicy>,
     sticky_session_token: Option<&str>,
     request_auth_channel: Option<&str>,
     persistence_policy: LocalCandidatePersistencePolicy<'_>,
@@ -445,7 +506,8 @@ where
     F: Fn(&EligibleLocalExecutionCandidate) -> Option<Value> + Send + Sync,
     G: Fn(SkippedLocalExecutionCandidate) -> SkippedLocalExecutionCandidate + Send + Sync,
 {
-    let scheduler_cache_affinity_enabled = scheduler_cache_affinity_enabled(state).await;
+    let scheduler_cache_affinity_enabled =
+        scheduler_cache_affinity_enabled(state, routing_policy).await;
     let port = GatewayLocalCandidateMaterializationPort {
         state,
         trace_id,
@@ -454,6 +516,7 @@ where
         auth_snapshot,
         client_session_affinity,
         required_capabilities,
+        routing_policy,
         sticky_session_token,
         request_auth_channel,
         persistence_policy,
@@ -478,6 +541,7 @@ pub(crate) async fn build_local_execution_candidate_attempt_source_with_serving<
     auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
     client_session_affinity: Option<&ClientSessionAffinity>,
     required_capabilities: Option<&Value>,
+    routing_policy: Option<&ResolvedRoutingPolicy>,
     sticky_session_token: Option<&str>,
     request_auth_channel: Option<&str>,
     persistence_policy: LocalCandidatePersistencePolicy<'_>,
@@ -491,7 +555,8 @@ where
     F: Fn(&EligibleLocalExecutionCandidate) -> Option<Value> + Send + Sync,
     G: Fn(SkippedLocalExecutionCandidate) -> SkippedLocalExecutionCandidate + Send + Sync,
 {
-    let scheduler_cache_affinity_enabled = scheduler_cache_affinity_enabled(state).await;
+    let scheduler_cache_affinity_enabled =
+        scheduler_cache_affinity_enabled(state, routing_policy).await;
     let _ = build_available_extra_data;
     let (candidates, resolved_skipped) = resolve_and_rank_logical_local_execution_candidates(
         state,
@@ -501,6 +566,7 @@ where
         auth_snapshot,
         client_session_affinity,
         required_capabilities,
+        routing_policy,
         sticky_session_token,
         request_auth_channel,
         resolution_mode,
@@ -529,7 +595,12 @@ where
         trace_id,
         persistence_policy.skipped,
         u32::try_from(candidates.len()).unwrap_or(u32::MAX),
-        skipped_candidates,
+        attach_routing_trace_to_skipped_candidates(
+            routing_policy,
+            client_api_format,
+            u32::try_from(candidates.len()).unwrap_or(u32::MAX),
+            skipped_candidates,
+        ),
     )
     .await;
 
@@ -542,6 +613,14 @@ where
         sticky_session_token,
         requested_model,
         request_auth_channel,
+        routing_policy,
+        Some(PoolGroupExhaustionPersistenceContext::new(
+            state.app().clone(),
+            trace_id,
+            persistence_policy.skipped,
+            client_api_format,
+            routing_policy,
+        )),
     );
 
     (
@@ -559,6 +638,8 @@ fn build_logical_candidate_items<'a>(
     sticky_session_token: Option<&str>,
     requested_model: Option<&str>,
     request_auth_channel: Option<&str>,
+    routing_policy: Option<&ResolvedRoutingPolicy>,
+    pool_exhaustion_persistence: Option<PoolGroupExhaustionPersistenceContext>,
 ) -> (VecDeque<LocalExecutionCandidateAttemptSourceItem<'a>>, u32) {
     let mut items = VecDeque::new();
     let mut next_candidate_index = starting_candidate_index;
@@ -578,12 +659,13 @@ fn build_logical_candidate_items<'a>(
                 }
             }
             LocalExecutionCandidateKind::PoolGroup => {
-                let cursor = PoolKeyCursor::new(
+                let cursor = PoolKeyCursor::new_with_routing_policy(
                     state,
                     candidate,
                     sticky_session_token,
                     requested_model,
                     request_auth_channel,
+                    routing_policy,
                 );
                 let cursor = if let Some(trace_id) = trace_id {
                     cursor.with_runtime_miss_diagnostic(trace_id, record_runtime_miss_diagnostic)
@@ -594,6 +676,7 @@ fn build_logical_candidate_items<'a>(
                     cursor,
                     candidate_index,
                     pending_attempts: DispatchSequence::new(Vec::new()),
+                    pool_exhaustion_persistence: pool_exhaustion_persistence.clone(),
                 });
             }
         }
@@ -615,6 +698,7 @@ pub(crate) async fn build_lazy_requested_model_execution_candidate_attempt_sourc
     auth_snapshot: &GatewayAuthApiKeySnapshot,
     client_session_affinity: Option<&ClientSessionAffinity>,
     required_capabilities: Option<&Value>,
+    routing_policy: Option<&ResolvedRoutingPolicy>,
     sticky_session_token: Option<&str>,
     request_auth_channel: Option<&str>,
     persistence_policy: LocalCandidatePersistencePolicy<'_>,
@@ -628,7 +712,8 @@ where
     F: Fn(&EligibleLocalExecutionCandidate) -> Option<Value> + Send + Sync + 'a,
     G: Fn(SkippedLocalExecutionCandidate) -> SkippedLocalExecutionCandidate + Send + Sync + 'a,
 {
-    let scheduler_cache_affinity_enabled = scheduler_cache_affinity_enabled(state).await;
+    let scheduler_cache_affinity_enabled =
+        scheduler_cache_affinity_enabled(state, routing_policy).await;
     let _ = build_available_extra_data;
     let decorate_skipped_candidate = Arc::new(decorate_skipped_candidate);
     let record_runtime_miss_diagnostic = persistence_policy.skipped.record_runtime_miss_diagnostic;
@@ -639,6 +724,7 @@ where
         require_streaming,
         required_capabilities,
         auth_snapshot,
+        routing_policy,
         client_session_affinity,
         use_api_format_alias_match,
         key_mode,
@@ -652,6 +738,7 @@ where
         auth_snapshot: auth_snapshot.clone(),
         client_session_affinity: client_session_affinity.cloned(),
         required_capabilities: required_capabilities.cloned(),
+        routing_policy: routing_policy.cloned(),
         sticky_session_token: sticky_session_token.map(str::to_string),
         request_auth_channel: request_auth_channel.map(str::to_string),
         skipped_user_id: persistence_policy.skipped.user_id.to_string(),
@@ -693,6 +780,7 @@ struct RequestedModelAttemptPageCursor<'a> {
     auth_snapshot: GatewayAuthApiKeySnapshot,
     client_session_affinity: Option<ClientSessionAffinity>,
     required_capabilities: Option<Value>,
+    routing_policy: Option<ResolvedRoutingPolicy>,
     sticky_session_token: Option<String>,
     request_auth_channel: Option<String>,
     skipped_user_id: String,
@@ -756,6 +844,7 @@ impl<'a> RequestedModelAttemptPageCursor<'a> {
                     Some(&self.auth_snapshot),
                     self.client_session_affinity.as_ref(),
                     self.required_capabilities.as_ref(),
+                    self.routing_policy.as_ref(),
                     self.sticky_session_token.as_deref(),
                     self.request_auth_channel.as_deref(),
                     self.resolution_mode,
@@ -794,6 +883,17 @@ impl<'a> RequestedModelAttemptPageCursor<'a> {
                 self.sticky_session_token.as_deref(),
                 Some(&self.requested_model),
                 self.request_auth_channel.as_deref(),
+                self.routing_policy.as_ref(),
+                Some(PoolGroupExhaustionPersistenceContext {
+                    app: self.state.app().clone(),
+                    trace_id: self.trace_id.clone(),
+                    user_id: self.skipped_user_id.clone(),
+                    api_key_id: self.skipped_api_key_id.clone(),
+                    required_capabilities: self.skipped_required_capabilities.clone(),
+                    error_context: self.skipped_error_context,
+                    client_api_format: self.client_api_format.clone(),
+                    routing_policy: self.routing_policy.clone(),
+                }),
             );
             self.next_candidate_index = next_candidate_index
                 .saturating_add(u32::try_from(skipped_candidate_count).unwrap_or(u32::MAX));
@@ -814,7 +914,12 @@ impl<'a> RequestedModelAttemptPageCursor<'a> {
                 &self.trace_id,
                 skipped_persistence,
                 skipped_starting_candidate_index,
-                skipped_candidates,
+                attach_routing_trace_to_skipped_candidates(
+                    self.routing_policy.as_ref(),
+                    &self.client_api_format,
+                    skipped_starting_candidate_index,
+                    skipped_candidates,
+                ),
             )
             .await;
         }
@@ -858,7 +963,12 @@ impl<'a> RequestedModelAttemptPageCursor<'a> {
             &self.trace_id,
             skipped_persistence,
             self.next_candidate_index,
-            skipped_candidates,
+            attach_routing_trace_to_skipped_candidates(
+                self.routing_policy.as_ref(),
+                &self.client_api_format,
+                self.next_candidate_index,
+                skipped_candidates,
+            ),
         )
         .await;
         self.next_candidate_index = self
@@ -900,11 +1010,20 @@ async fn pop_attempt_from_items(
                 cursor,
                 candidate_index,
                 pending_attempts,
+                pool_exhaustion_persistence,
             } => {
                 if let Some(attempt) = next_attempt_from_dispatch_sequence(pending_attempts) {
                     return Some(attempt);
                 }
                 let Some(candidate) = cursor.next_key().await else {
+                    if let Some(skipped) = cursor.exhausted_group_skipped_candidate() {
+                        persist_pool_group_exhaustion_skipped_candidate(
+                            pool_exhaustion_persistence.as_ref(),
+                            *candidate_index,
+                            skipped,
+                        )
+                        .await;
+                    }
                     cursor.log_exhausted();
                     let _ = cursor.take_skipped_candidates();
                     items.pop_front();
@@ -925,19 +1044,14 @@ async fn pop_attempt_from_items(
     }
 }
 
-async fn scheduler_cache_affinity_enabled(state: PlannerAppState<'_>) -> bool {
-    match read_scheduler_ordering_config(state.app()).await {
-        Ok(config) => config.scheduling_mode == SchedulerSchedulingMode::CacheAffinity,
-        Err(error) => {
-            warn!(
-                event_name = "planner_scheduler_affinity_config_load_failed",
-                log_type = "event",
-                error = ?error,
-                "failed to load scheduler config while checking cache affinity mode"
-            );
-            SchedulerSchedulingMode::default() == SchedulerSchedulingMode::CacheAffinity
-        }
-    }
+async fn scheduler_cache_affinity_enabled(
+    state: PlannerAppState<'_>,
+    routing_policy: Option<&ResolvedRoutingPolicy>,
+) -> bool {
+    scheduler_ordering_config_for_routing_policy(state, routing_policy)
+        .await
+        .scheduling_mode
+        == SchedulerSchedulingMode::CacheAffinity
 }
 
 pub(crate) fn remember_first_local_candidate_affinity(
@@ -1038,6 +1152,8 @@ async fn materialize_logical_local_execution_candidate_attempts<F>(
     context: LocalAvailableCandidatePersistenceContext<'_>,
     record_runtime_miss_diagnostic: bool,
     candidates: Vec<EligibleLocalExecutionCandidate>,
+    routing_policy: Option<&ResolvedRoutingPolicy>,
+    client_api_format: &str,
     sticky_session_token: Option<&str>,
     requested_model: Option<&str>,
     request_auth_channel: Option<&str>,
@@ -1059,18 +1175,21 @@ where
                         context,
                         candidate,
                         candidate_index,
+                        routing_policy,
+                        client_api_format,
                         build_extra_data,
                     )
                     .await,
                 );
             }
             LocalExecutionCandidateKind::PoolGroup => {
-                let mut cursor = PoolKeyCursor::new(
+                let mut cursor = PoolKeyCursor::new_with_routing_policy(
                     state,
                     candidate,
                     sticky_session_token,
                     requested_model,
                     request_auth_channel,
+                    routing_policy,
                 )
                 .with_runtime_miss_diagnostic(trace_id, record_runtime_miss_diagnostic);
                 let attempt_count_before_pool = attempts.len();
@@ -1082,7 +1201,29 @@ where
                 }
                 let _ = cursor.take_skipped_candidates();
                 if attempts.len() == attempt_count_before_pool {
+                    let skipped = cursor.exhausted_group_skipped_candidate();
                     cursor.log_exhausted();
+                    if let Some(skipped) = skipped {
+                        let pool_exhaustion_context = PoolGroupExhaustionPersistenceContext::new(
+                            state.app().clone(),
+                            trace_id,
+                            LocalSkippedCandidatePersistenceContext {
+                                user_id: context.user_id,
+                                api_key_id: context.api_key_id,
+                                required_capabilities: context.required_capabilities,
+                                error_context: context.error_context,
+                                record_runtime_miss_diagnostic: false,
+                            },
+                            client_api_format,
+                            routing_policy,
+                        );
+                        persist_pool_group_exhaustion_skipped_candidate(
+                            Some(&pool_exhaustion_context),
+                            candidate_index,
+                            skipped,
+                        )
+                        .await;
+                    }
                 }
             }
         }
@@ -1097,6 +1238,8 @@ async fn persist_available_local_execution_candidate_at_index<F>(
     context: LocalAvailableCandidatePersistenceContext<'_>,
     candidate: EligibleLocalExecutionCandidate,
     candidate_index: u32,
+    routing_policy: Option<&ResolvedRoutingPolicy>,
+    client_api_format: &str,
     build_extra_data: &F,
 ) -> Vec<LocalExecutionCandidateAttempt>
 where
@@ -1107,6 +1250,16 @@ where
         available_candidate_base_extra_data_with_dispatch_ref(&candidate, build_extra_data),
         candidate.ranking.as_ref(),
     );
+    let extra_data = attach_routing_trace_to_extra_data(
+        routing_policy,
+        client_api_format,
+        &candidate.candidate,
+        candidate.kind,
+        candidate.ranking.as_ref(),
+        None,
+        Some(candidate_index),
+        extra_data,
+    );
     let should_persist = should_persist_available_local_candidate(&candidate);
     let mut attempts = Vec::with_capacity(attempt_slots as usize);
     let mut owned_candidate = Some(candidate);
@@ -1190,6 +1343,160 @@ where
     Some(Value::Object(object))
 }
 
+fn attach_routing_trace_to_skipped_candidates(
+    routing_policy: Option<&ResolvedRoutingPolicy>,
+    client_api_format: &str,
+    starting_candidate_index: u32,
+    skipped_candidates: Vec<SkippedLocalExecutionCandidate>,
+) -> Vec<SkippedLocalExecutionCandidate> {
+    skipped_candidates
+        .into_iter()
+        .enumerate()
+        .map(|(offset, skipped)| {
+            let selected_order =
+                starting_candidate_index.saturating_add(u32::try_from(offset).unwrap_or(u32::MAX));
+            attach_routing_trace_to_skipped_candidate(
+                routing_policy,
+                client_api_format,
+                selected_order,
+                skipped,
+            )
+        })
+        .collect()
+}
+
+fn attach_routing_trace_to_skipped_candidate(
+    routing_policy: Option<&ResolvedRoutingPolicy>,
+    client_api_format: &str,
+    selected_order: u32,
+    mut skipped_candidate: SkippedLocalExecutionCandidate,
+) -> SkippedLocalExecutionCandidate {
+    let kind = if skipped_candidate
+        .transport
+        .as_ref()
+        .is_some_and(|transport| {
+            admin_provider_pool_config_from_config_value(transport.provider.config.as_ref())
+                .is_some()
+        }) {
+        LocalExecutionCandidateKind::PoolGroup
+    } else {
+        LocalExecutionCandidateKind::SingleKey
+    };
+    skipped_candidate.extra_data = attach_routing_trace_to_extra_data(
+        routing_policy,
+        client_api_format,
+        &skipped_candidate.candidate,
+        kind,
+        skipped_candidate.ranking.as_ref(),
+        Some(skipped_candidate.skip_reason),
+        Some(selected_order),
+        skipped_candidate.extra_data,
+    );
+    skipped_candidate
+}
+
+#[allow(clippy::too_many_arguments)]
+fn attach_routing_trace_to_extra_data(
+    routing_policy: Option<&ResolvedRoutingPolicy>,
+    client_api_format: &str,
+    candidate: &SchedulerMinimalCandidateSelectionCandidate,
+    kind: LocalExecutionCandidateKind,
+    ranking: Option<&SchedulerRankingOutcome>,
+    skip_reason: Option<&'static str>,
+    selected_order: Option<u32>,
+    extra_data: Option<Value>,
+) -> Option<Value> {
+    let Some(policy) = routing_policy else {
+        return extra_data;
+    };
+    let routing_trace = routing_trace_for_candidate(
+        policy,
+        client_api_format,
+        candidate,
+        kind,
+        ranking,
+        skip_reason,
+        selected_order,
+    );
+    Some(merge_routing_trace_into_extra_data(
+        extra_data,
+        routing_trace,
+    ))
+}
+
+fn merge_routing_trace_into_extra_data(
+    extra_data: Option<Value>,
+    routing_trace: RoutingDecisionTrace,
+) -> Value {
+    let mut object = match extra_data {
+        Some(Value::Object(object)) => object,
+        Some(value) => {
+            let mut object = serde_json::Map::new();
+            object.insert("extra".to_string(), value);
+            object
+        }
+        None => serde_json::Map::new(),
+    };
+    object.insert(
+        "routing_trace".to_string(),
+        serde_json::json!(routing_trace),
+    );
+    Value::Object(object)
+}
+
+fn routing_trace_for_candidate(
+    policy: &ResolvedRoutingPolicy,
+    client_api_format: &str,
+    candidate: &SchedulerMinimalCandidateSelectionCandidate,
+    kind: LocalExecutionCandidateKind,
+    ranking: Option<&SchedulerRankingOutcome>,
+    skip_reason: Option<&'static str>,
+    selected_order: Option<u32>,
+) -> RoutingDecisionTrace {
+    let candidate_kind = routing_candidate_kind(kind);
+    let mut trace = crate::routing::build_routing_trace_seed(policy, client_api_format);
+    trace.global_candidates.push(RoutingCandidateTrace {
+        candidate_kind,
+        provider_id: candidate.provider_id.clone(),
+        endpoint_id: candidate.endpoint_id.clone(),
+        model_id: candidate.model_id.clone(),
+        key_id: match candidate_kind {
+            CandidateKind::Provider => Some(candidate.key_id.clone()),
+            CandidateKind::PoolGroup => None,
+        },
+        ranking_vector: rank_vector_for_candidate(
+            &policy.ranking_overlay,
+            &RoutingCandidateFacts {
+                candidate_kind,
+                provider_id: candidate.provider_id.clone(),
+                endpoint_id: candidate.endpoint_id.clone(),
+                model_id: candidate.model_id.clone(),
+                key_id: match candidate_kind {
+                    CandidateKind::Provider => Some(candidate.key_id.clone()),
+                    CandidateKind::PoolGroup => None,
+                },
+                provider_priority: candidate.provider_priority,
+                key_priority: candidate
+                    .key_global_priority_for_format
+                    .unwrap_or(candidate.key_internal_priority),
+            },
+        ),
+        skip_reason: skip_reason.map(str::to_string),
+        selected_order,
+    });
+    if let Some(ranking) = ranking {
+        trace.runtime_facts.cache_affinity_hit = ranking.promoted_by == Some("cached_affinity");
+    }
+    trace
+}
+
+fn routing_candidate_kind(kind: LocalExecutionCandidateKind) -> CandidateKind {
+    match kind {
+        LocalExecutionCandidateKind::SingleKey => CandidateKind::Provider,
+        LocalExecutionCandidateKind::PoolGroup => CandidateKind::PoolGroup,
+    }
+}
+
 fn dispatch_sequence_from_attempts(
     attempts: Vec<LocalExecutionCandidateAttempt>,
 ) -> DispatchSequence<LocalExecutionCandidateAttempt> {
@@ -1252,6 +1559,40 @@ fn build_unpersisted_local_execution_candidate_attempts(
     attempts
 }
 
+async fn persist_pool_group_exhaustion_skipped_candidate(
+    context: Option<&PoolGroupExhaustionPersistenceContext>,
+    candidate_index: u32,
+    skipped: SkippedLocalExecutionCandidate,
+) {
+    let Some(context) = context else {
+        return;
+    };
+    let skipped = attach_routing_trace_to_skipped_candidate(
+        context.routing_policy.as_ref(),
+        &context.client_api_format,
+        candidate_index,
+        skipped,
+    );
+    let extra_data =
+        ai_candidate_extra_data_with_ranking(skipped.extra_data.clone(), skipped.ranking.as_ref());
+    let candidate_id = Uuid::new_v4().to_string();
+    persist_skipped_local_execution_candidate(
+        &context.app,
+        &context.trace_id,
+        &context.user_id,
+        &context.api_key_id,
+        &skipped.candidate,
+        candidate_index,
+        candidate_id.as_str(),
+        context.required_capabilities.as_ref(),
+        skipped.skip_reason,
+        extra_data,
+        context.error_context,
+        false,
+    )
+    .await;
+}
+
 #[allow(clippy::too_many_arguments)]
 pub(crate) async fn persist_skipped_local_execution_candidate(
     state: &AppState,
@@ -1426,6 +1767,7 @@ mod tests {
     use aether_data::repository::candidate_selection::InMemoryMinimalCandidateSelectionReadRepository;
     use aether_data::repository::candidates::InMemoryRequestCandidateRepository;
     use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
+    use aether_data_contracts::repository::candidates::RequestCandidateStatus;
     use aether_provider_transport::snapshot::{
         GatewayProviderTransportEndpoint, GatewayProviderTransportKey,
         GatewayProviderTransportProvider,
@@ -1644,6 +1986,7 @@ mod tests {
             auth_snapshot: Some(&auth_snapshot),
             client_session_affinity: None,
             required_capabilities: None,
+            routing_policy: None,
             sticky_session_token: None,
             request_auth_channel: None,
             persistence_policy: LocalCandidatePersistencePolicy {
@@ -1718,6 +2061,8 @@ mod tests {
             false,
             vec![pool_group, sample_eligible("normal-key", None)],
             None,
+            "openai:chat",
+            None,
             Some("gpt-5"),
             None,
             &|_| None,
@@ -1732,11 +2077,26 @@ mod tests {
             .read_request_candidates_by_request_id("trace-logical-pool")
             .await
             .expect("request candidates should read");
-        assert_eq!(stored.len(), 1);
-        assert_eq!(stored[0].key_id.as_deref(), Some("normal-key"));
-        assert_eq!(stored[0].candidate_index, 1);
+        assert_eq!(stored.len(), 2);
+        assert_eq!(stored[0].key_id.as_deref(), Some("pool-group"));
+        assert_eq!(stored[0].status, RequestCandidateStatus::Skipped);
+        assert_eq!(
+            stored[0].skip_reason.as_deref(),
+            Some("pool_group_exhausted")
+        );
+        assert_eq!(stored[0].candidate_index, 0);
         assert_eq!(
             stored[0]
+                .extra_data
+                .as_ref()
+                .and_then(|value| value.get("pool_group_exhaustion"))
+                .and_then(|value| value.get("scanned_keys")),
+            Some(&json!(0))
+        );
+        assert_eq!(stored[1].key_id.as_deref(), Some("normal-key"));
+        assert_eq!(stored[1].candidate_index, 1);
+        assert_eq!(
+            stored[1]
                 .extra_data
                 .as_ref()
                 .and_then(|value| value.get("dispatch_ref"))
@@ -1861,6 +2221,83 @@ mod tests {
         assert!(source.next_attempt().await.is_none());
     }
 
+    #[tokio::test]
+    async fn dynamic_pool_exhaustion_persists_group_skip_summary() {
+        let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
+        let app = AppState::new()
+            .expect("state should build")
+            .with_data_state_for_tests(
+                GatewayDataState::with_auth_candidate_selection_provider_catalog_and_request_candidate_repository_for_tests(
+                    Arc::new(InMemoryAuthApiKeySnapshotRepository::default()),
+                    Arc::new(InMemoryMinimalCandidateSelectionReadRepository::default()),
+                    Arc::new(InMemoryProviderCatalogReadRepository::seed(
+                        Vec::new(),
+                        Vec::new(),
+                        Vec::new(),
+                    )),
+                    Arc::clone(&request_candidate_repository),
+                    "test-encryption-key",
+                ),
+            );
+        let mut pool_group = sample_eligible("pool-group", None);
+        pool_group.kind = LocalExecutionCandidateKind::PoolGroup;
+        pool_group.transport = sample_transport(
+            "pool-group",
+            Some(json!({ "pool_advanced": { "scheduling_presets": [] } })),
+        );
+        let cursor = PoolKeyCursor::new(
+            PlannerAppState::new(&app),
+            pool_group,
+            None,
+            Some("gpt-5"),
+            None,
+        );
+        let pool_exhaustion_persistence = PoolGroupExhaustionPersistenceContext::new(
+            app.clone(),
+            "trace-dynamic-pool",
+            LocalSkippedCandidatePersistenceContext {
+                user_id: "user-1",
+                api_key_id: "api-key-1",
+                required_capabilities: None,
+                error_context: "persist should not fail",
+                record_runtime_miss_diagnostic: false,
+            },
+            "openai:chat",
+            None,
+        );
+        let mut source = LocalExecutionCandidateAttemptSource {
+            items: VecDeque::from([LocalExecutionCandidateAttemptSourceItem::Pool {
+                cursor,
+                candidate_index: 0,
+                pending_attempts: DispatchSequence::new(Vec::new()),
+                pool_exhaustion_persistence: Some(pool_exhaustion_persistence),
+            }]),
+        };
+
+        assert!(source.next_attempt().await.is_none());
+
+        let stored = app
+            .read_request_candidates_by_request_id("trace-dynamic-pool")
+            .await
+            .expect("request candidates should read");
+        assert_eq!(stored.len(), 1);
+        assert_eq!(stored[0].status, RequestCandidateStatus::Skipped);
+        assert_eq!(
+            stored[0].skip_reason.as_deref(),
+            Some("pool_group_exhausted")
+        );
+        assert_eq!(stored[0].candidate_index, 0);
+        assert_eq!(stored[0].key_id.as_deref(), Some("pool-group"));
+        assert_eq!(
+            stored[0]
+                .extra_data
+                .as_ref()
+                .and_then(|value| value.get("pool_group_exhaustion"))
+                .and_then(|value| value.get("scanned_keys")),
+            Some(&json!(0))
+        );
+    }
+
     #[tokio::test]
     async fn pool_internal_skipped_candidates_are_not_persisted() {
         let repository = Arc::new(InMemoryRequestCandidateRepository::default());