feat(providers): add provider transfer limits

apps/aether-gateway/src/ai_serving/planner/candidate_materialization.rs

@@ -16,7 +16,7 @@ use aether_scheduler_core::{
 };
 use async_trait::async_trait;
 use serde_json::Value;
-use std::collections::VecDeque;
+use std::collections::{BTreeSet, VecDeque};
 use std::convert::Infallible;
 use std::sync::Arc;
 use std::time::Duration;
@@ -67,6 +67,7 @@ pub(crate) struct LocalExecutionCandidateAttempt {
 
 pub(crate) struct LocalExecutionCandidateAttemptSource<'a> {
     items: VecDeque<LocalExecutionCandidateAttemptSourceItem<'a>>,
+    skipped_provider_ids: BTreeSet<String>,
 }
 
 type DecorateSkippedCandidateFn<'a> = Arc<
@@ -78,6 +79,8 @@ pub(crate) trait LocalExecutionAttemptSource<T>: Send {
     async fn next_execution_attempt(&mut self) -> Result<Option<T>, GatewayError>;
 
     async fn drain_execution_attempts(&mut self) -> Result<Vec<T>, GatewayError>;
+
+    async fn skip_provider(&mut self, provider_id: &str) -> Result<(), GatewayError>;
 }
 
 enum LocalExecutionCandidateAttemptSourceItem<'a> {
@@ -105,7 +108,10 @@ impl<'a> LocalExecutionCandidateAttemptSource<'a> {
                 attempts: dispatch_sequence_from_attempts(attempts),
             });
         }
-        Self { items }
+        Self {
+            items,
+            skipped_provider_ids: BTreeSet::new(),
+        }
     }
 
     pub(crate) async fn next_attempt(
@@ -117,6 +123,10 @@ impl<'a> LocalExecutionCandidateAttemptSource<'a> {
             };
             match front {
                 LocalExecutionCandidateAttemptSourceItem::Static { attempts } => {
+                    if dispatch_sequence_provider_is_skipped(attempts, &self.skipped_provider_ids) {
+                        self.items.pop_front();
+                        continue;
+                    }
                     if let Some(attempt) = next_attempt_from_dispatch_sequence(attempts) {
                         if dispatch_sequence_exhausted(attempts) {
                             self.items.pop_front();
@@ -131,6 +141,10 @@ impl<'a> LocalExecutionCandidateAttemptSource<'a> {
                     pending_attempts,
                     pool_exhaustion_persistence,
                 } => {
+                    if self.skipped_provider_ids.contains(cursor.provider_id()) {
+                        self.items.pop_front();
+                        continue;
+                    }
                     if let Some(attempt) = next_attempt_from_dispatch_sequence(pending_attempts) {
                         return Ok(Some(attempt));
                     }
@@ -157,6 +171,9 @@ impl<'a> LocalExecutionCandidateAttemptSource<'a> {
                     );
                 }
                 LocalExecutionCandidateAttemptSourceItem::RequestedModelPage { cursor } => {
+                    for provider_id in &self.skipped_provider_ids {
+                        cursor.skip_provider(provider_id);
+                    }
                     let Some(attempt) = cursor.next_attempt().await? else {
                         self.items.pop_front();
                         continue;
@@ -171,6 +188,19 @@ impl<'a> LocalExecutionCandidateAttemptSource<'a> {
         self.items.clear();
         Vec::new()
     }
+
+    pub(crate) fn skip_provider(&mut self, provider_id: &str) {
+        let provider_id = provider_id.trim();
+        if provider_id.is_empty() {
+            return;
+        }
+        self.skipped_provider_ids.insert(provider_id.to_string());
+        for item in &mut self.items {
+            if let LocalExecutionCandidateAttemptSourceItem::RequestedModelPage { cursor } = item {
+                cursor.skip_provider(provider_id);
+            }
+        }
+    }
 }
 
 impl LocalExecutionCandidateAttempt {
@@ -635,7 +665,10 @@ where
     );
 
     (
-        LocalExecutionCandidateAttemptSource { items },
+        LocalExecutionCandidateAttemptSource {
+            items,
+            skipped_provider_ids: BTreeSet::new(),
+        },
         candidate_count,
     )
 }
@@ -768,6 +801,7 @@ where
         decorate_skipped_candidate,
         page_cursor,
         pending_items: VecDeque::new(),
+        skipped_provider_ids: BTreeSet::new(),
         candidate_count: 0,
         next_candidate_index: 0,
         remembered_affinity: false,
@@ -788,7 +822,10 @@ where
         );
     }
     (
-        LocalExecutionCandidateAttemptSource { items },
+        LocalExecutionCandidateAttemptSource {
+            items,
+            skipped_provider_ids: BTreeSet::new(),
+        },
         candidate_count,
     )
 }
@@ -813,6 +850,7 @@ struct RequestedModelAttemptPageCursor<'a> {
     decorate_skipped_candidate: DecorateSkippedCandidateFn<'a>,
     page_cursor: LocalCandidatePreselectionPageCursor<'a>,
     pending_items: VecDeque<LocalExecutionCandidateAttemptSourceItem<'a>>,
+    skipped_provider_ids: BTreeSet<String>,
     candidate_count: usize,
     next_candidate_index: u32,
     remembered_affinity: bool,
@@ -822,6 +860,10 @@ struct RequestedModelAttemptPageCursor<'a> {
 }
 
 impl<'a> RequestedModelAttemptPageCursor<'a> {
+    fn skip_provider(&mut self, provider_id: &str) {
+        self.skipped_provider_ids.insert(provider_id.to_string());
+    }
+
     async fn next_attempt(
         &mut self,
     ) -> Result<Option<LocalExecutionCandidateAttempt>, GatewayError> {
@@ -829,7 +871,9 @@ impl<'a> RequestedModelAttemptPageCursor<'a> {
             return Err(error);
         }
         loop {
-            if let Some(attempt) = pop_attempt_from_items(&mut self.pending_items).await {
+            if let Some(attempt) =
+                pop_attempt_from_items(&mut self.pending_items, &self.skipped_provider_ids).await
+            {
                 return Ok(Some(attempt));
             }
             if !self.load_next_page().await? {
@@ -1030,11 +1074,16 @@ fn page_is_exact_auth_api_key_concurrency_limited(
 
 async fn pop_attempt_from_items(
     items: &mut VecDeque<LocalExecutionCandidateAttemptSourceItem<'_>>,
+    skipped_provider_ids: &BTreeSet<String>,
 ) -> Option<LocalExecutionCandidateAttempt> {
     loop {
         let front = items.front_mut()?;
         match front {
             LocalExecutionCandidateAttemptSourceItem::Static { attempts } => {
+                if dispatch_sequence_provider_is_skipped(attempts, skipped_provider_ids) {
+                    items.pop_front();
+                    continue;
+                }
                 if let Some(attempt) = next_attempt_from_dispatch_sequence(attempts) {
                     if dispatch_sequence_exhausted(attempts) {
                         items.pop_front();
@@ -1049,6 +1098,10 @@ async fn pop_attempt_from_items(
                 pending_attempts,
                 pool_exhaustion_persistence,
             } => {
+                if skipped_provider_ids.contains(cursor.provider_id()) {
+                    items.pop_front();
+                    continue;
+                }
                 if let Some(attempt) = next_attempt_from_dispatch_sequence(pending_attempts) {
                     return Some(attempt);
                 }
@@ -1727,6 +1780,15 @@ fn next_attempt_from_dispatch_sequence(
     Some(attempt)
 }
 
+fn dispatch_sequence_provider_is_skipped(
+    sequence: &DispatchSequence<LocalExecutionCandidateAttempt>,
+    skipped_provider_ids: &BTreeSet<String>,
+) -> bool {
+    sequence.peek_current().is_some_and(|item| {
+        skipped_provider_ids.contains(&item.candidate.eligible.candidate.provider_id)
+    })
+}
+
 fn dispatch_sequence_exhausted(
     sequence: &mut DispatchSequence<LocalExecutionCandidateAttempt>,
 ) -> bool {
@@ -2278,6 +2340,7 @@ mod tests {
             decorate_skipped_candidate: Arc::new(identity_skipped_candidate),
             page_cursor,
             pending_items: VecDeque::new(),
+            skipped_provider_ids: BTreeSet::new(),
             candidate_count: 0,
             next_candidate_index: 0,
             remembered_affinity: false,
@@ -2372,6 +2435,7 @@ mod tests {
             decorate_skipped_candidate: Arc::new(identity_skipped_candidate),
             page_cursor,
             pending_items: VecDeque::new(),
+            skipped_provider_ids: BTreeSet::new(),
             candidate_count: 0,
             next_candidate_index: 0,
             remembered_affinity: false,
@@ -2569,6 +2633,7 @@ mod tests {
                     .into(),
                 ),
             }]),
+            skipped_provider_ids: BTreeSet::new(),
         };
 
         let first = source
@@ -2587,6 +2652,53 @@ mod tests {
             .is_none());
     }
 
+    #[tokio::test]
+    async fn skipped_provider_discards_pool_cursor_and_continues_with_next_provider() {
+        let app = AppState::new().expect("state should build");
+        let mut pool_group = sample_eligible("pool-group", None);
+        pool_group.kind = LocalExecutionCandidateKind::PoolGroup;
+        pool_group.transport = sample_transport("pool-group", Some(json!({ "pool_advanced": {} })));
+        let pool_cursor = PoolKeyCursor::new(
+            PlannerAppState::new(&app),
+            pool_group,
+            None,
+            Some("gpt-5"),
+            None,
+        );
+
+        let mut fallback = sample_eligible("fallback-key", None);
+        fallback.candidate.provider_id = "provider-b".to_string();
+        Arc::make_mut(&mut fallback.transport).provider.id = "provider-b".to_string();
+        Arc::make_mut(&mut fallback.transport).key.provider_id = "provider-b".to_string();
+        let fallback_attempts = dispatch_sequence_from_attempts(
+            build_unpersisted_local_execution_candidate_attempts(fallback, 1).into(),
+        );
+        let mut source = LocalExecutionCandidateAttemptSource {
+            items: VecDeque::from([
+                LocalExecutionCandidateAttemptSourceItem::Pool {
+                    cursor: pool_cursor,
+                    candidate_index: 0,
+                    pending_attempts: DispatchSequence::new(Vec::new()),
+                    pool_exhaustion_persistence: None,
+                },
+                LocalExecutionCandidateAttemptSourceItem::Static {
+                    attempts: fallback_attempts,
+                },
+            ]),
+            skipped_provider_ids: BTreeSet::new(),
+        };
+
+        source.skip_provider("provider-1");
+        let attempt = source
+            .next_attempt()
+            .await
+            .expect("candidate source should succeed")
+            .expect("fallback provider should remain");
+
+        assert_eq!(attempt.eligible.candidate.provider_id, "provider-b");
+        assert_eq!(attempt.eligible.candidate.key_id, "fallback-key");
+    }
+
     #[tokio::test]
     async fn dynamic_pool_exhaustion_persists_group_skip_summary() {
         let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
@@ -2639,6 +2751,7 @@ mod tests {
                 pending_attempts: DispatchSequence::new(Vec::new()),
                 pool_exhaustion_persistence: Some(pool_exhaustion_persistence),
             }]),
+            skipped_provider_ids: BTreeSet::new(),
         };
 
         assert!(source

apps/aether-gateway/src/ai_serving/planner/passthrough/provider/plans.rs

@@ -213,6 +213,11 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalSameFormatProviderSyncA
         }
         Ok(drained)
     }
+
+    async fn skip_provider(&mut self, provider_id: &str) -> Result<(), GatewayError> {
+        self.candidates.skip_provider(provider_id);
+        Ok(())
+    }
 }
 
 #[async_trait]
@@ -243,6 +248,11 @@ impl LocalExecutionAttemptSource<AiStreamAttempt>
         }
         Ok(drained)
     }
+
+    async fn skip_provider(&mut self, provider_id: &str) -> Result<(), GatewayError> {
+        self.candidates.skip_provider(provider_id);
+        Ok(())
+    }
 }
 
 impl LocalSameFormatProviderSyncAttemptSource<'_> {

apps/aether-gateway/src/ai_serving/planner/specialized/files.rs

@@ -193,6 +193,11 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalGeminiFilesSyncAttemptS
         }
         Ok(drained)
     }
+
+    async fn skip_provider(&mut self, provider_id: &str) -> Result<(), GatewayError> {
+        self.candidates.skip_provider(provider_id);
+        Ok(())
+    }
 }
 
 #[async_trait]
@@ -216,6 +221,11 @@ impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalGeminiFilesStreamAtte
         }
         Ok(drained)
     }
+
+    async fn skip_provider(&mut self, provider_id: &str) -> Result<(), GatewayError> {
+        self.candidates.skip_provider(provider_id);
+        Ok(())
+    }
 }
 
 impl LocalGeminiFilesSyncAttemptSource<'_> {

apps/aether-gateway/src/ai_serving/planner/specialized/files/decision.rs

@@ -14,7 +14,7 @@ use crate::ai_serving::transport::{
     resolve_transport_execution_timeouts, resolve_transport_profile,
 };
 use crate::ai_serving::{ai_local_execution_contract_for_formats, PlannerAppState};
-use crate::{AiExecutionDecision, AppState, GatewayError};
+use crate::{append_local_failover_policy_to_value, AiExecutionDecision, AppState, GatewayError};
 
 use super::request::resolve_local_gemini_files_candidate_payload_parts;
 use super::support::{
@@ -114,6 +114,7 @@ pub(super) async fn maybe_build_local_gemini_files_decision_payload_for_candidat
         needs_conversion: false,
         extra_fields,
     });
+    let report_context = append_local_failover_policy_to_value(report_context, &transport);
     let super::request::LocalGeminiFilesCandidatePayloadParts {
         transport: _,
         auth_header,

apps/aether-gateway/src/ai_serving/planner/specialized/image.rs

@@ -271,6 +271,11 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiImageSyncAttemptS
         }
         Ok(drained)
     }
+
+    async fn skip_provider(&mut self, provider_id: &str) -> Result<(), GatewayError> {
+        self.candidates.skip_provider(provider_id);
+        Ok(())
+    }
 }
 
 #[async_trait]
@@ -294,6 +299,11 @@ impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiImageStreamAtte
         }
         Ok(drained)
     }
+
+    async fn skip_provider(&mut self, provider_id: &str) -> Result<(), GatewayError> {
+        self.candidates.skip_provider(provider_id);
+        Ok(())
+    }
 }
 
 impl LocalOpenAiImageSyncAttemptSource<'_> {

apps/aether-gateway/src/ai_serving/planner/specialized/image/decision.rs

@@ -13,7 +13,8 @@ use crate::ai_serving::transport::{
 };
 use crate::ai_serving::{ai_local_execution_contract_for_formats, PlannerAppState};
 use crate::{
-    append_execution_contract_fields_to_value, AiExecutionDecision, AppState, GatewayError,
+    append_execution_contract_fields_to_value, append_local_failover_policy_to_value,
+    AiExecutionDecision, AppState, GatewayError,
 };
 
 use super::request::resolve_local_openai_image_candidate_payload_parts;
@@ -131,6 +132,7 @@ pub(super) async fn maybe_build_local_openai_image_decision_payload_for_candidat
         spec_metadata.api_format,
         provider_api_format.as_str(),
     );
+    let report_context = append_local_failover_policy_to_value(report_context, &transport);
     let request_encoding = resolve_transport_request_encoding_policy(&transport);
 
     let mut decision = build_ai_execution_decision_response(AiExecutionDecisionResponseParts {

apps/aether-gateway/src/ai_serving/planner/specialized/video.rs

@@ -123,6 +123,11 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalVideoCreateSyncAttemptS
         }
         Ok(drained)
     }
+
+    async fn skip_provider(&mut self, provider_id: &str) -> Result<(), GatewayError> {
+        self.candidates.skip_provider(provider_id);
+        Ok(())
+    }
 }
 
 impl LocalVideoCreateSyncAttemptSource<'_> {

apps/aether-gateway/src/ai_serving/planner/specialized/video/decision.rs

@@ -12,7 +12,7 @@ use crate::ai_serving::transport::{
     resolve_transport_execution_timeouts, resolve_transport_profile,
 };
 use crate::ai_serving::{ai_local_execution_contract_for_formats, PlannerAppState};
-use crate::{AiExecutionDecision, AppState, GatewayError};
+use crate::{append_local_failover_policy_to_value, AiExecutionDecision, AppState, GatewayError};
 
 use super::request::resolve_local_video_create_candidate_payload_parts;
 use super::support::{LocalVideoCreateCandidateAttempt, LocalVideoCreateDecisionInput};
@@ -95,6 +95,7 @@ pub(super) async fn maybe_build_local_video_create_decision_payload_for_candidat
         needs_conversion: false,
         extra_fields,
     });
+    let report_context = append_local_failover_policy_to_value(report_context, &transport);
     let super::request::LocalVideoCreateCandidatePayloadParts {
         transport: _,
         auth_header,

apps/aether-gateway/src/ai_serving/planner/standard/family/build.rs

@@ -201,6 +201,11 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalStandardSyncAttemptSour
         }
         Ok(drained)
     }
+
+    async fn skip_provider(&mut self, provider_id: &str) -> Result<(), GatewayError> {
+        self.candidates.skip_provider(provider_id);
+        Ok(())
+    }
 }
 
 #[async_trait]
@@ -229,6 +234,11 @@ impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalStandardStreamAttempt
         }
         Ok(drained)
     }
+
+    async fn skip_provider(&mut self, provider_id: &str) -> Result<(), GatewayError> {
+        self.candidates.skip_provider(provider_id);
+        Ok(())
+    }
 }
 
 impl LocalStandardSyncAttemptSource<'_> {

apps/aether-gateway/src/ai_serving/planner/standard/openai/chat/plans/stream.rs

@@ -138,6 +138,13 @@ impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiChatStreamAttem
         }
         Ok(drained)
     }
+
+    async fn skip_provider(&mut self, provider_id: &str) -> Result<(), GatewayError> {
+        self.prefetched_attempts
+            .retain(|attempt| attempt.eligible.candidate.provider_id != provider_id);
+        self.candidates.skip_provider(provider_id);
+        Ok(())
+    }
 }
 
 impl LocalOpenAiChatStreamAttemptSource<'_> {

apps/aether-gateway/src/ai_serving/planner/standard/openai/chat/plans/sync.rs

@@ -116,6 +116,11 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiChatSyncAttemptSo
         }
         Ok(drained)
     }
+
+    async fn skip_provider(&mut self, provider_id: &str) -> Result<(), GatewayError> {
+        self.candidates.skip_provider(provider_id);
+        Ok(())
+    }
 }
 
 impl LocalOpenAiChatSyncAttemptSource<'_> {

apps/aether-gateway/src/ai_serving/planner/standard/openai/responses/plans.rs

@@ -185,6 +185,11 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiResponsesSyncAtte
         }
         Ok(drained)
     }
+
+    async fn skip_provider(&mut self, provider_id: &str) -> Result<(), GatewayError> {
+        self.candidates.skip_provider(provider_id);
+        Ok(())
+    }
 }
 
 #[async_trait]
@@ -213,6 +218,11 @@ impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiResponsesStream
         }
         Ok(drained)
     }
+
+    async fn skip_provider(&mut self, provider_id: &str) -> Result<(), GatewayError> {
+        self.candidates.skip_provider(provider_id);
+        Ok(())
+    }
 }
 
 impl LocalOpenAiResponsesSyncAttemptSource<'_> {