Fix/api key concurrency runtime miss (#309)

* test(cli): 覆盖 API key 并发等待与超时路径 * feat(scheduler): API key 并发饱和时等待可用槽位 * fix(proxy): 区分 API key 并发受限与真正的 runtime miss * fix(outcome): runtime miss 仅归因真实执行候选 * feat(api-keys): 统一 concurrent_limit 默认值与校验辅助 * feat(admin): 独立 Key 接口支持 concurrent_limit * feat(admin): 用户 API Key 路由支持 concurrent_limit * feat(public): 自助 API Key 路由支持 concurrent_limit * feat(import): 导入与存储层持久化 concurrent_limit * feat(frontend): 同步 API Key concurrent_limit 类型定义 * feat(frontend): 独立 Key 表单支持 concurrent_limit * feat(frontend): 管理员用户 API Key 表单支持 concurrent_limit * feat(frontend): 自助 API Key 页面支持 concurrent_limit * chore(fmt): 统一 runtime 归因相关 Rust 格式 * chore(fmt): 统一 admin API key 路由 Rust 格式 * chore(fmt): 统一 public 路由与相关测试 Rust 格式 * fix(test): 对齐 no-execution usage 归因断言 * test(middleware): 固定 access log tracing 用例线程模型 * fix(frontend): 提取用户 API Key payload 默认并发辅助 * fix(frontend): 保留用户 Key 的 concurrent_limit 默认值 * fix(api-keys): remove hardcoded concurrent limit default --------- Co-authored-by: fawney19 <[email protected]>
2026-10-09 18:59:50 +08:00 · 2026-04-17 14:21:43 +08:00
parent e5d3722adf
commit b8702ae124
39 changed files with 1739 additions and 128 deletions
@@ -101,6 +101,16 @@ pub(crate) fn apply_local_candidate_terminal_plan_reason(
    let skipped_candidate_count = diagnostic.skipped_candidate_count.unwrap_or(0);
    diagnostic.reason = if candidate_count == 0 {
        "candidate_list_empty".to_string()
+    } else if skipped_candidate_count >= candidate_count
+        && diagnostic.skip_reasons.len() == 1
+        && diagnostic
+            .skip_reasons
+            .get("api_key_concurrency_limit_reached")
+            .copied()
+            .unwrap_or(0)
+            > 0
+    {
+        "api_key_concurrency_limit_reached".to_string()
    } else if skipped_candidate_count >= candidate_count {
        "all_candidates_skipped".to_string()
    } else {
@@ -222,7 +232,15 @@ mod tests {
        apply_local_candidate_terminal_plan_reason(&mut diagnostic, "no_local_sync_plans");
        assert_eq!(diagnostic.reason, "all_candidates_skipped");

+        diagnostic.skip_reasons = std::collections::BTreeMap::from([(
+            "api_key_concurrency_limit_reached".to_string(),
+            2,
+        )]);
+        apply_local_candidate_terminal_plan_reason(&mut diagnostic, "no_local_sync_plans");
+        assert_eq!(diagnostic.reason, "api_key_concurrency_limit_reached");
+
        diagnostic.skipped_candidate_count = Some(1);
+        diagnostic.skip_reasons.clear();
        apply_local_candidate_terminal_plan_reason(&mut diagnostic, "no_local_sync_plans");
        assert_eq!(diagnostic.reason, "no_local_sync_plans");
    }
@@ -47,7 +47,7 @@ pub(crate) fn build_local_candidate_persistence_policy<'a>(
            LocalCandidatePersistencePolicyKind::OpenAiCliDecision => (
                "gateway local openai cli decision request candidate upsert failed",
                "gateway local openai cli decision failed to persist skipped candidate",
-                false,
+                true,
            ),
            LocalCandidatePersistencePolicyKind::GeminiFilesDecision => (
                "gateway local gemini files request candidate upsert failed",
@@ -1,6 +1,12 @@
 use aether_scheduler_core::SchedulerMinimalCandidateSelectionCandidate;
+use std::time::Duration;
+use tokio::time::Instant;

 use super::{GatewayAuthApiKeySnapshot, PlannerAppState};
+use crate::clock::current_unix_secs;
+use crate::constants::{
+    API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS, API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS,
+};
 use crate::scheduler::candidate::SchedulerSkippedCandidate;
 use crate::GatewayError;

@@ -42,17 +48,39 @@ impl<'a> PlannerAppState<'a> {
        ),
        GatewayError,
    > {
-        crate::scheduler::candidate::list_selectable_candidates_with_skip_reasons(
-            self.app().data.as_ref(),
-            self.app(),
-            api_format,
-            global_model_name,
-            require_streaming,
-            required_capabilities,
-            auth_snapshot,
-            now_unix_secs,
-        )
-        .await
+        let wait_timeout = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS);
+        let wait_interval = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS.max(1));
+        let wait_deadline = Instant::now() + wait_timeout;
+        let mut attempt_now_unix_secs = now_unix_secs;
+
+        loop {
+            let result = crate::scheduler::candidate::list_selectable_candidates_with_skip_reasons(
+                self.app().data.as_ref(),
+                self.app(),
+                api_format,
+                global_model_name,
+                require_streaming,
+                required_capabilities,
+                auth_snapshot,
+                attempt_now_unix_secs,
+            )
+            .await?;
+
+            if !crate::scheduler::candidate::is_exact_all_skipped_by_auth_limit(
+                &result.0, &result.1,
+            ) {
+                return Ok(result);
+            }
+
+            let now = Instant::now();
+            if now >= wait_deadline {
+                return Ok(result);
+            }
+
+            let remaining = wait_deadline.duration_since(now);
+            tokio::time::sleep(wait_interval.min(remaining)).await;
+            attempt_now_unix_secs = current_unix_secs();
+        }
    }

    pub(crate) async fn list_selectable_candidates_for_required_capability_without_requested_model(
@@ -63,15 +91,35 @@ impl<'a> PlannerAppState<'a> {
        auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
        now_unix_secs: u64,
    ) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
-        crate::scheduler::candidate::list_selectable_candidates_for_required_capability_without_requested_model(
-            self.app().data.as_ref(),
-            self.app(),
-            candidate_api_format,
-            required_capability,
-            require_streaming,
-            auth_snapshot,
-            now_unix_secs,
-        )
-        .await
+        let wait_timeout = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS);
+        let wait_interval = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS.max(1));
+        let wait_deadline = Instant::now() + wait_timeout;
+        let mut attempt_now_unix_secs = now_unix_secs;
+
+        loop {
+            let (result, auth_limit_blocked) = crate::scheduler::candidate::list_selectable_candidates_for_required_capability_without_requested_model_with_auth_limit_signal(
+                self.app().data.as_ref(),
+                self.app(),
+                candidate_api_format,
+                required_capability,
+                require_streaming,
+                auth_snapshot,
+                attempt_now_unix_secs,
+            )
+            .await?;
+
+            if !auth_limit_blocked {
+                return Ok(result);
+            }
+
+            let now = Instant::now();
+            if now >= wait_deadline {
+                return Ok(result);
+            }
+
+            let remaining = wait_deadline.duration_since(now);
+            tokio::time::sleep(wait_interval.min(remaining)).await;
+            attempt_now_unix_secs = current_unix_secs();
+        }
    }
 }