feat(gateway): 增强候选路由策略与可观测信息

apps/aether-gateway/src/scheduler/candidate/mod.rs

@@ -1,5 +1,7 @@
 use self::affinity::candidate_affinity_hash;
-use self::selection::collect_selectable_candidates;
+use self::selection::{
+    collect_selectable_candidates, collect_selectable_candidates_with_skip_reasons,
+};
 use super::state::SchedulerRuntimeState;
 
 mod affinity;
@@ -26,6 +28,8 @@ use regex::Regex;
 use sha2::{Digest, Sha256};
 use std::collections::BTreeMap;
 
+pub(crate) use self::selection::SchedulerSkippedCandidate;
+
 use crate::data::auth::GatewayAuthApiKeySnapshot;
 use crate::data::candidate_selection::{
     read_global_model_names_for_api_format, read_global_model_names_for_required_capability,
@@ -65,6 +69,35 @@ pub(crate) async fn list_selectable_candidates(
     .await
 }
 
+pub(crate) async fn list_selectable_candidates_with_skip_reasons(
+    selection_row_source: &(impl MinimalCandidateSelectionRowSource + Sync),
+    runtime_state: &impl SchedulerRuntimeState,
+    api_format: &str,
+    global_model_name: &str,
+    require_streaming: bool,
+    required_capabilities: Option<&serde_json::Value>,
+    auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
+    now_unix_secs: u64,
+) -> Result<
+    (
+        Vec<SchedulerMinimalCandidateSelectionCandidate>,
+        Vec<SchedulerSkippedCandidate>,
+    ),
+    GatewayError,
+> {
+    collect_selectable_candidates_with_skip_reasons(
+        selection_row_source,
+        runtime_state,
+        api_format,
+        global_model_name,
+        require_streaming,
+        required_capabilities,
+        auth_snapshot,
+        now_unix_secs,
+    )
+    .await
+}
+
 pub(crate) async fn list_selectable_candidates_for_required_capability_without_requested_model(
     selection_row_source: &(impl MinimalCandidateSelectionRowSource + Sync),
     runtime_state: &impl SchedulerRuntimeState,

apps/aether-gateway/src/scheduler/candidate/runtime.rs

@@ -4,8 +4,8 @@ use aether_data_contracts::repository::candidates::StoredRequestCandidate;
 use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
 use aether_scheduler_core::{
     auth_api_key_concurrency_limit_reached, build_provider_concurrent_limit_map,
-    candidate_is_selectable_with_runtime_state, CandidateRuntimeSelectabilityInput,
-    SchedulerAffinityTarget,
+    candidate_is_selectable_with_runtime_state, candidate_runtime_skip_reason_with_state,
+    CandidateRuntimeSelectabilityInput, SchedulerAffinityTarget,
 };
 
 use crate::data::auth::GatewayAuthApiKeySnapshot;
@@ -77,6 +77,32 @@ pub(super) fn is_candidate_selectable(
     now_unix_secs: u64,
     cached_affinity_target: Option<&SchedulerAffinityTarget>,
 ) -> bool {
+    candidate_is_selectable_with_runtime_state(CandidateRuntimeSelectabilityInput {
+        candidate,
+        recent_candidates: &snapshot.recent_candidates,
+        provider_concurrent_limits: &snapshot.provider_concurrent_limits,
+        provider_key_rpm_states: &snapshot.provider_key_rpm_states,
+        now_unix_secs,
+        cached_affinity_target,
+        provider_quota_blocks_requests: snapshot
+            .provider_quota_blocks_requests
+            .get(candidate.provider_id.as_str())
+            .copied()
+            .unwrap_or(false),
+        rpm_reset_at: snapshot
+            .provider_key_rpm_reset_ats
+            .get(candidate.key_id.as_str())
+            .copied()
+            .flatten(),
+    })
+}
+
+pub(super) fn current_candidate_runtime_skip_reason(
+    candidate: &SchedulerMinimalCandidateSelectionCandidate,
+    snapshot: &CandidateRuntimeSelectionSnapshot,
+    now_unix_secs: u64,
+    cached_affinity_target: Option<&SchedulerAffinityTarget>,
+) -> Option<&'static str> {
     let provider_quota_blocks_requests = snapshot
         .provider_quota_blocks_requests
         .get(candidate.provider_id.as_str())
@@ -88,7 +114,7 @@ pub(super) fn is_candidate_selectable(
         .copied()
         .flatten();
 
-    candidate_is_selectable_with_runtime_state(CandidateRuntimeSelectabilityInput {
+    candidate_runtime_skip_reason_with_state(CandidateRuntimeSelectabilityInput {
         candidate,
         recent_candidates: &snapshot.recent_candidates,
         provider_concurrent_limits: &snapshot.provider_concurrent_limits,

apps/aether-gateway/src/scheduler/candidate/selection.rs

@@ -20,11 +20,17 @@ use super::affinity::{
     build_scheduler_affinity_cache_key, candidate_key, remember_scheduler_affinity,
 };
 use super::runtime::{
-    auth_snapshot_concurrency_limit_reached, is_candidate_selectable,
+    auth_snapshot_concurrency_limit_reached, current_candidate_runtime_skip_reason,
     read_candidate_runtime_selection_snapshot,
 };
 use super::{SchedulerMinimalCandidateSelectionCandidate, SchedulerRuntimeState};
 
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) struct SchedulerSkippedCandidate {
+    pub(crate) candidate: SchedulerMinimalCandidateSelectionCandidate,
+    pub(crate) skip_reason: &'static str,
+}
+
 pub(super) fn reorder_candidates_by_scheduler_health(
     candidates: &mut [SchedulerMinimalCandidateSelectionCandidate],
     provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
@@ -130,6 +136,36 @@ pub(super) async fn collect_selectable_candidates(
     auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
     now_unix_secs: u64,
 ) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
+    Ok(collect_selectable_candidates_with_skip_reasons(
+        selection_row_source,
+        runtime_state,
+        api_format,
+        global_model_name,
+        require_streaming,
+        required_capabilities,
+        auth_snapshot,
+        now_unix_secs,
+    )
+    .await?
+    .0)
+}
+
+pub(super) async fn collect_selectable_candidates_with_skip_reasons(
+    selection_row_source: &(impl MinimalCandidateSelectionRowSource + Sync),
+    runtime_state: &impl SchedulerRuntimeState,
+    api_format: &str,
+    global_model_name: &str,
+    require_streaming: bool,
+    required_capabilities: Option<&serde_json::Value>,
+    auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
+    now_unix_secs: u64,
+) -> Result<
+    (
+        Vec<SchedulerMinimalCandidateSelectionCandidate>,
+        Vec<SchedulerSkippedCandidate>,
+    ),
+    GatewayError,
+> {
     let ordering_config = runtime_state.read_scheduler_ordering_config().await?;
     let priority_affinity_key =
         scheduling_priority_affinity_key(auth_snapshot, ordering_config.scheduling_mode);
@@ -176,27 +212,48 @@ pub(super) async fn collect_selectable_candidates(
     };
 
     if auth_snapshot_concurrency_limit_reached(auth_snapshot, &runtime_snapshot, now_unix_secs) {
-        return Ok(Vec::new());
+        return Ok((
+            Vec::new(),
+            candidates
+                .into_iter()
+                .map(|candidate| SchedulerSkippedCandidate {
+                    candidate,
+                    skip_reason: "api_key_concurrency_limit_reached",
+                })
+                .collect(),
+        ));
     }
 
     let mut selected_keys = BTreeSet::new();
+    let mut skipped = Vec::new();
+    let mut emitted_skipped_keys = BTreeSet::new();
 
     for candidate in &candidates {
-        if !is_candidate_selectable(
+        let key = candidate_key(candidate);
+        if let Some(skip_reason) = current_candidate_runtime_skip_reason(
             candidate,
             &runtime_snapshot,
             now_unix_secs,
             cached_affinity_target.as_ref(),
         ) {
+            if emitted_skipped_keys.insert(key) {
+                skipped.push(SchedulerSkippedCandidate {
+                    candidate: candidate.clone(),
+                    skip_reason,
+                });
+            }
             continue;
         }
-        selected_keys.insert(candidate_key(candidate));
+        selected_keys.insert(key);
     }
 
-    Ok(collect_selectable_candidates_from_keys(
-        candidates,
-        &selected_keys,
-        cached_affinity_target.as_ref(),
+    Ok((
+        collect_selectable_candidates_from_keys(
+            candidates,
+            &selected_keys,
+            cached_affinity_target.as_ref(),
+        ),
+        skipped,
     ))
 }
 

apps/aether-gateway/src/scheduler/candidate/tests/selection.rs

@@ -22,6 +22,7 @@ use crate::{AppState, GatewayError};
 use super::super::runtime::should_skip_provider_quota;
 use super::super::selection::{
     collect_selectable_candidates as collect_selectable_candidates_impl,
+    collect_selectable_candidates_with_skip_reasons as collect_selectable_candidates_with_skip_reasons_impl,
     select_minimal_candidate as select_candidate_impl,
 };
 use super::support::{sample_auth_snapshot, sample_key, sample_provider, sample_row};
@@ -70,6 +71,34 @@ async fn collect_selectable_candidates(
     .await
 }
 
+async fn collect_selectable_candidates_with_skip_reasons(
+    selection_row_source: &(impl MinimalCandidateSelectionRowSource + Sync),
+    runtime_state: &AppState,
+    api_format: &str,
+    global_model_name: &str,
+    require_streaming: bool,
+    auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
+    now_unix_secs: u64,
+) -> Result<
+    (
+        Vec<SchedulerMinimalCandidateSelectionCandidate>,
+        Vec<super::super::SchedulerSkippedCandidate>,
+    ),
+    GatewayError,
+> {
+    collect_selectable_candidates_with_skip_reasons_impl(
+        selection_row_source,
+        runtime_state,
+        api_format,
+        global_model_name,
+        require_streaming,
+        None,
+        auth_snapshot,
+        now_unix_secs,
+    )
+    .await
+}
+
 #[test]
 fn skips_inactive_or_exhausted_monthly_quota_provider() {
     let inactive = StoredProviderQuotaSnapshot::new(
@@ -1044,6 +1073,73 @@ async fn selects_next_candidate_when_first_provider_key_circuit_is_open() {
     assert_eq!(selected.key_id, "key-b");
 }
 
+#[tokio::test]
+async fn exposes_runtime_skipped_candidates_with_skip_reasons() {
+    let mut first = sample_row();
+    first.provider_id = "provider-a".to_string();
+    first.provider_name = "openai-a".to_string();
+    first.endpoint_id = "endpoint-a".to_string();
+    first.key_id = "key-a".to_string();
+    first.key_name = "alpha".to_string();
+    first.key_global_priority_by_format = Some(serde_json::json!({"openai:chat": 1}));
+
+    let mut second = sample_row();
+    second.provider_id = "provider-b".to_string();
+    second.provider_name = "openai-b".to_string();
+    second.endpoint_id = "endpoint-b".to_string();
+    second.key_id = "key-b".to_string();
+    second.key_name = "beta".to_string();
+    second.key_global_priority_by_format = Some(serde_json::json!({"openai:chat": 2}));
+
+    let candidates = Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
+        first, second,
+    ]));
+    let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
+        vec![
+            sample_provider("provider-a", None),
+            sample_provider("provider-b", None),
+        ],
+        Vec::new(),
+        vec![
+            sample_key("key-a", "provider-a", Some(10)).with_health_fields(
+                Some(serde_json::json!({"openai:chat": {"health_score": 0.2}})),
+                Some(serde_json::json!({"openai:chat": {"open": true}})),
+            ),
+            sample_key("key-b", "provider-b", Some(10)),
+        ],
+    ));
+    let quotas = Arc::new(InMemoryProviderQuotaRepository::seed(vec![]));
+    let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![]));
+    let state = AppState::new()
+        .expect("state should build")
+        .with_data_state_for_tests(
+            GatewayDataState::with_candidate_selection_provider_catalog_quota_and_request_candidates_for_tests(
+                candidates,
+                provider_catalog,
+                quotas,
+                request_candidates,
+            ),
+        );
+
+    let (selected, skipped) = collect_selectable_candidates_with_skip_reasons(
+        state.data.as_ref(),
+        &state,
+        "openai:chat",
+        "gpt-4.1",
+        false,
+        None,
+        100,
+    )
+    .await
+    .expect("selection should succeed");
+
+    assert_eq!(selected.len(), 1);
+    assert_eq!(selected[0].provider_id, "provider-b");
+    assert_eq!(skipped.len(), 1);
+    assert_eq!(skipped[0].candidate.provider_id, "provider-a");
+    assert_eq!(skipped[0].skip_reason, "key_circuit_open");
+}
+
 #[tokio::test]
 async fn same_priority_candidates_prefer_healthier_provider_key_before_id_order() {
     let mut first = sample_row();