feat(test,quota,failover): 模型并发测试、统一配额读取器与故障转移取消支持

- 新增 QuotaReader 抽象层，统一 Codex/Kiro/Antigravity 配额解析逻辑，
  替换 pool/routes.py 中分散的配额构建函数
- 模型测试支持并发执行多候选，前端新增 useModelTest composable 统一
  ModelsTab 和 ModelMappingTab 的测试逻辑
- ModelTestDialog 增加结果概览摘要、超长结果折叠、端点列和新状态支持，
  删除已合并的 TestResultDialog
- FailoverEngine 新增客户端断开检测，支持取消剩余候选并标记记录
- 刷新配额改为分批执行，直连测试候选按可用性排序
- 修复 error 判断从 "error" in dict 改为 dict.get("error") 避免误判

aether-hub/README.md

@@ -2,21 +2,9 @@
 
 `aether-hub` 是 Tunnel Hub 服务，负责在 proxy 与 worker 之间路由帧。
 
-## 快速命令
+已集成在Docker镜像中, 无需单独部署。
 
-### 1) 构建并上传 Hub 二进制（推荐生产）
+## 部署端指定 Hub 版本并构建
-
-```bash
-cd aether-hub
-./build.sh --upload hub-v0.1.0
-```
-
-说明：
-
-- 默认会构建 `amd64 + arm64` 两个二进制并上传到 GitHub Release。
-- 如只需单架构，可先 `./build.sh amd64` 或 `./build.sh arm64`。
-
-### 2) 部署端指定 Hub 版本并构建
 
 ```bash
 cd /path/to/Aether
@@ -25,15 +13,6 @@ cd /path/to/Aether
 
 不指定 `--hub-tag` 时，`./deploy.sh` 会自动解析最新 `hub-v*` release，并在构建 app 镜像时从 GitHub Release 下载对应架构的 Hub 二进制。
 
-### 3) 镜像模式（可选，调试/实验用）
-
-仅本地加载镜像（单平台）：
-
-```bash
-cd aether-hub
-BUILDKIT_PROGRESS=plain ./build.sh --image --tag local-test --platforms linux/amd64 --load
-```
-
 ## build.sh 模式说明
 
 - 默认是 `binary` 模式（`cross` 构建二进制）。

frontend/src/api/endpoints/keys.ts

@@ -202,7 +202,11 @@ export async function refreshProviderQuota(
   keyIds?: string[],
 ): Promise<RefreshQuotaResult> {
   const body = keyIds && keyIds.length > 0 ? { key_ids: keyIds } : undefined
-  const response = await client.post(`/api/admin/endpoints/providers/${providerId}/refresh-quota`, body)
+  const response = await client.post(
+    `/api/admin/endpoints/providers/${providerId}/refresh-quota`,
+    body,
+    { timeout: 5 * 60 * 1000 },
+  )
   return response.data
 }
 

frontend/src/api/endpoints/providers.ts

@@ -148,6 +148,7 @@ export interface TestModelFailoverRequest {
   endpoint_id?: string
   message?: string
   request_id?: string
+  concurrency?: number
 }
 
 export interface TestAttemptDetail {
@@ -159,7 +160,7 @@ export interface TestAttemptDetail {
   key_id: string
   auth_type: string
   effective_model?: string | null
-  status: 'success' | 'failed' | 'skipped'
+  status: 'success' | 'failed' | 'skipped' | 'cancelled' | 'pending' | 'streaming' | 'stream_interrupted' | 'available' | 'unused'
   skip_reason?: string | null
   error_message?: string | null
   status_code?: number | null
@@ -177,9 +178,13 @@ export interface TestModelFailoverResponse {
   error?: string | null
 }
 
-export async function testModelFailover(data: TestModelFailoverRequest): Promise<TestModelFailoverResponse> {
+export async function testModelFailover(
+  data: TestModelFailoverRequest,
+  options: { signal?: AbortSignal } = {}
+): Promise<TestModelFailoverResponse> {
   const response = await client.post('/api/admin/provider-query/test-model-failover', data, {
     timeout: 10 * 60 * 1000,
+    signal: options.signal,
   })
   return response.data
 }

frontend/src/composables/useModelTest.ts

@@ -0,0 +1,183 @@
+import { ref, onBeforeUnmount } from 'vue'
+import { isAxiosError } from 'axios'
+import { useToast } from './useToast'
+import {
+  testModelFailover,
+  type TestModelFailoverResponse,
+} from '@/api/endpoints/providers'
+import { requestTraceApi, type RequestTrace } from '@/api/requestTrace'
+import { parseApiError } from '@/utils/errorParser'
+
+export interface StartTestParams {
+  mode: 'global' | 'direct'
+  modelName: string
+  displayLabel: string
+  apiFormat?: string
+  endpointId?: string
+  message?: string
+  concurrency?: number
+  onSuccess?: (result: TestModelFailoverResponse) => void
+  /** Return `true` to indicate the failure has been handled; otherwise the composable sets `testResult`. */
+  onFailure?: (result: TestModelFailoverResponse) => boolean | void
+  /** Return `true` to indicate the error has been handled; otherwise a toast is shown and state is reset. */
+  onError?: (err: unknown) => boolean | void
+}
+
+export interface UseModelTestOptions {
+  providerId: () => string
+  pollInterval?: number
+}
+
+export function useModelTest(options: UseModelTestOptions) {
+  const { providerId, pollInterval = 800 } = options
+  const { success: showSuccess, error: showError } = useToast()
+
+  const testing = ref(false)
+  const testMode = ref<'global' | 'direct'>('global')
+  const testResult = ref<TestModelFailoverResponse | null>(null)
+  const testTrace = ref<RequestTrace | null>(null)
+  const requestId = ref<string | null>(null)
+  const dialogOpen = ref(false)
+
+  let tracePollTimer: ReturnType<typeof setInterval> | null = null
+  let tracePollToken = 0
+  let activeAbortController: AbortController | null = null
+
+  function buildTestRequestId(): string {
+    const randomUUID = globalThis.crypto?.randomUUID?.bind(globalThis.crypto)
+    if (randomUUID) {
+      return `provider-test-${randomUUID().replace(/-/g, '').slice(0, 20)}`
+    }
+    return `provider-test-${Date.now().toString(36)}${Math.random().toString(36).slice(2, 10)}`
+  }
+
+  async function pollTestTrace(reqId: string, token: number) {
+    try {
+      const trace = await requestTraceApi.getRequestTrace(reqId, { attemptedOnly: false })
+      if (tracePollToken !== token || requestId.value !== reqId) return
+      testTrace.value = trace
+    } catch (err: unknown) {
+      if (isAxiosError(err) && err.response?.status === 404) return
+    }
+  }
+
+  function stopPolling(opts: { clearState?: boolean } = {}) {
+    tracePollToken += 1
+    if (tracePollTimer) {
+      clearInterval(tracePollTimer)
+      tracePollTimer = null
+    }
+    if (opts.clearState !== false) {
+      requestId.value = null
+      testTrace.value = null
+    }
+  }
+
+  function startPolling(reqId: string) {
+    stopPolling()
+    requestId.value = reqId
+    testTrace.value = null
+    const token = ++tracePollToken
+    void pollTestTrace(reqId, token)
+    tracePollTimer = setInterval(() => {
+      void pollTestTrace(reqId, token)
+    }, pollInterval)
+  }
+
+  function abortActiveRequest() {
+    if (!activeAbortController) return
+    activeAbortController.abort()
+    activeAbortController = null
+  }
+
+  function isRequestCancelled(err: unknown): boolean {
+    if (isAxiosError(err)) {
+      return err.code === 'ERR_CANCELED'
+    }
+    return err instanceof DOMException && err.name === 'AbortError'
+  }
+
+  function resetState() {
+    abortActiveRequest()
+    stopPolling()
+    dialogOpen.value = false
+    testResult.value = null
+  }
+
+  async function startTest(params: StartTestParams) {
+    abortActiveRequest()
+    testing.value = true
+    testMode.value = params.mode
+    dialogOpen.value = true
+    testResult.value = null
+
+    const abortController = new AbortController()
+    activeAbortController = abortController
+    const reqId = buildTestRequestId()
+    startPolling(reqId)
+
+    try {
+      const result = await testModelFailover({
+        provider_id: providerId(),
+        mode: params.mode,
+        model_name: params.modelName,
+        api_format: params.apiFormat,
+        endpoint_id: params.endpointId,
+        message: params.message ?? 'hello',
+        request_id: reqId,
+        concurrency: params.concurrency,
+      }, {
+        signal: abortController.signal,
+      })
+
+      if (result.success) {
+        const successAttempt = result.attempts.find(a => a.status === 'success')
+        const latency = successAttempt?.latency_ms != null ? ` (${successAttempt.latency_ms}ms)` : ''
+        const mapped = successAttempt?.effective_model && successAttempt.effective_model !== params.modelName
+          ? ` -> ${successAttempt.effective_model}`
+          : ''
+        params.onSuccess?.(result)
+        showSuccess(`${params.displayLabel}${mapped} 测试成功${latency}`)
+        resetState()
+        return
+      }
+
+      stopPolling({ clearState: false })
+      const handled = params.onFailure?.(result)
+      if (!handled) {
+        testResult.value = result
+      }
+    } catch (err: unknown) {
+      if (isRequestCancelled(err)) {
+        return
+      }
+      stopPolling()
+      const handled = params.onError?.(err)
+      if (!handled) {
+        showError(`模型测试失败: ${parseApiError(err, '测试请求失败')}`)
+        resetState()
+      }
+    } finally {
+      if (activeAbortController === abortController) {
+        activeAbortController = null
+      }
+      testing.value = false
+    }
+  }
+
+  onBeforeUnmount(() => {
+    resetState()
+  })
+
+  return {
+    testing,
+    testMode,
+    testResult,
+    testTrace,
+    requestId,
+    dialogOpen,
+    startTest,
+    resetState,
+    stopPolling,
+  }
+}

frontend/src/features/pool/components/PoolAccountBatchDialog.vue

@@ -600,11 +600,25 @@ async function executeAction(): Promise<void> {
   try {
     if (selectedAction.value === 'refresh_quota') {
       const targetIds = selectedKeys.map((key) => key.key_id)
-      const result = await refreshProviderQuota(props.providerId, targetIds)
+      const BATCH_SIZE = 20
-      successCount = Number(result.success || 0)
+      const totalBatches = Math.ceil(targetIds.length / BATCH_SIZE)
-      failedCount = Number(result.failed || 0)
+
-      skippedCount = Math.max(0, targetIds.length - Number(result.total || 0))
+      for (let i = 0; i < targetIds.length; i += BATCH_SIZE) {
-      progressDone.value = targetIds.length
+        const batchIndex = Math.floor(i / BATCH_SIZE) + 1
+        const batch = targetIds.slice(i, i + BATCH_SIZE)
+        progressLabel.value = `正在${actionLabel}...（第 ${batchIndex}/${totalBatches} 批）`
+
+        try {
+          const result = await refreshProviderQuota(props.providerId, batch)
+          successCount += Number(result.success || 0)
+          failedCount += Number(result.failed || 0)
+          skippedCount += Math.max(0, batch.length - Number(result.total || 0))
+        } catch {
+          failedCount += batch.length
+        }
+
+        progressDone.value = Math.min(i + BATCH_SIZE, targetIds.length)
+      }
     } else {
       const CONCURRENCY = props.batchConcurrency || 8
       const taskForKey = (key: PoolKeyDetail): (() => Promise<'success' | 'skip'>) | null => {

frontend/src/features/providers/components/provider-tabs/ModelMappingTab.vue

@@ -308,31 +308,35 @@
     @cancel="deleteConfirmOpen = false"
   />
 
-  <!-- 测试结果对话框（仅失败时显示） -->
+  <!-- 模型测试对话框 -->
-  <TestResultDialog
+  <ModelTestDialog
-    :result="testResult"
+    :open="modelTest.dialogOpen.value"
+    :result="modelTest.testResult.value"
     mode="direct"
-    @close="testResult = null"
+    :selecting-model-name="testingModelName"
+    :testing="modelTest.testing.value"
+    :trace="modelTest.testTrace.value"
+    :request-id="modelTest.requestId.value"
+    @close="handleTestDialogClose"
   />
 </template>
 
 <script setup lang="ts">
 import { ref, computed } from 'vue'
 import { useSmartPagination } from '@/composables/useSmartPagination'
+import { useModelTest } from '@/composables/useModelTest'
 import { Tag, Plus, Edit, Trash2, ChevronRight, Loader2, Play } from 'lucide-vue-next'
 import {
   Card, Button, Badge,
 } from '@/components/ui'
 import AlertDialog from '@/components/common/AlertDialog.vue'
 import ModelMappingDialog, { type AliasGroup } from '../ModelMappingDialog.vue'
-import TestResultDialog from './TestResultDialog.vue'
+import ModelTestDialog from './ModelTestDialog.vue'
 import { useToast } from '@/composables/useToast'
 import {
-  testModelFailover,
   type Model,
   type ProviderModelAlias,
   type ProviderMappingPreviewResponse,
-  type TestModelFailoverResponse
 } from '@/api/endpoints'
 import { type EndpointAPIKey } from '@/api/endpoints/keys'
 import { updateModel } from '@/api/endpoints/models'
@@ -376,6 +380,9 @@ const emit = defineEmits<{
 
 const { error: showError, success: showSuccess } = useToast()
 
+// 模型测试 composable
+const modelTest = useModelTest({ providerId: () => props.provider.id })
+
 // 状态
 const loading = ref(false)
 const dialogOpen = ref(false)
@@ -383,7 +390,7 @@ const deleteConfirmOpen = ref(false)
 const editingGroup = ref<AliasGroup | null>(null)
 const deletingGroup = ref<AliasGroup | null>(null)
 const testingMapping = ref<string | null>(null)
-const testResult = ref<TestModelFailoverResponse | null>(null)
+const testingModelName = ref<string | null>(null)
 const preselectedModelId = ref<string | null>(null)
 
 // 使用 props 传入的数据
@@ -617,30 +624,25 @@ async function onDialogSaved() {
   emit('refresh')
 }
 
-// 测试映射（直连测试，带故障转移）
+function handleTestDialogClose() {
+  modelTest.resetState()
+  testingModelName.value = null
+}
+
+// 测试映射（直连测试，带故障转移和实时进度）
 async function runMappingTest(testingKey: string, modelName: string) {
   testingMapping.value = testingKey
-
+  testingModelName.value = modelName
-  try {
+  await modelTest.startTest({
-    const result = await testModelFailover({
+    mode: 'direct',
-      provider_id: props.provider.id,
+    modelName,
-      mode: 'direct',
+    displayLabel: `映射 "${modelName}"`,
-      model_name: modelName,
+    message: 'hello',
-      message: "hello",
+    onSuccess: () => {
-    })
+      testingModelName.value = null
-
+    },
-    if (result.success) {
+  })
-      const successAttempt = result.attempts.find(a => a.status === 'success')
+  testingMapping.value = null
-      const latency = successAttempt?.latency_ms != null ? ` (${successAttempt.latency_ms}ms)` : ''
-      showSuccess(`映射 "${modelName}" 测试成功${latency}`)
-    } else {
-      testResult.value = result
-    }
-  } catch (err: unknown) {
-    showError(`映射测试失败: ${parseApiError(err, '测试请求失败')}`)
-  } finally {
-    testingMapping.value = null
-  }
 }
 
 // 测试精确映射

frontend/src/features/providers/components/provider-tabs/ModelTestDialog.vue

@@ -159,7 +159,7 @@
                     :variant="statusVariant(candidate.status)"
                     class="text-[10px] px-1.5 py-0 shrink-0"
                   >
-                    {{ candidate.status_code || statusLabel(candidate.status) }}
+                    {{ statusDisplay(candidate) }}
                   </Badge>
                   <span class="truncate font-medium">{{ formatTraceCandidateAccount(candidate) }}</span>
                 </div>
@@ -221,13 +221,51 @@
         {{ result.error }}
       </div>
 
+      <div
+        v-if="attemptSummaryItems.length > 0"
+        class="rounded-md border border-border/60 bg-muted/20 p-3 space-y-2"
+      >
+        <div class="text-xs font-medium text-muted-foreground">
+          结果概览
+        </div>
+        <div class="flex flex-wrap gap-2">
+          <div
+            v-for="item in attemptSummaryItems"
+            :key="item.key"
+            class="flex items-center gap-2 rounded-md border border-border/60 bg-background/80 px-2.5 py-1.5 text-xs"
+          >
+            <Badge
+              :variant="item.variant"
+              class="text-[10px] px-1.5 py-0"
+            >
+              {{ item.count }}x
+            </Badge>
+            <span class="text-muted-foreground break-all">{{ item.label }}</span>
+          </div>
+        </div>
+      </div>
+
+      <div
+        v-if="shouldCollapseAttempts"
+        class="flex items-center justify-between gap-3 text-xs text-muted-foreground"
+      >
+        <span>仅展示前 {{ visibleAttempts.length }} 条，共 {{ resultAttempts.length }} 条</span>
+        <Button
+          variant="ghost"
+          size="sm"
+          @click="showAllAttempts = !showAllAttempts"
+        >
+          {{ showAllAttempts ? '收起详情' : `展开全部 ${resultAttempts.length} 条` }}
+        </Button>
+      </div>
+
       <!-- mobile: list layout -->
       <div
-        v-if="result.attempts.length > 0"
+        v-if="resultAttempts.length > 0"
         class="space-y-2 sm:hidden"
       >
         <div
-          v-for="(attempt, idx) in result.attempts"
+          v-for="(attempt, idx) in visibleAttempts"
           :key="'m' + idx"
           class="rounded-md border px-3 py-2 text-xs"
           :class="attemptRowClass(attempt.status)"
@@ -239,7 +277,7 @@
                 :variant="statusVariant(attempt.status)"
                 class="text-[10px] px-1.5 py-0 shrink-0"
               >
-                {{ attempt.status_code || statusLabel(attempt.status) }}
+                {{ statusDisplay(attempt) }}
               </Badge>
               <span
                 v-if="attempt.latency_ms != null"
@@ -248,6 +286,10 @@
                 {{ attempt.latency_ms }}ms
               </span>
             </div>
+            <code
+              v-if="showEndpointColumn"
+              class="text-[11px] bg-muted px-1 py-0.5 rounded shrink-0"
+            >{{ attempt.endpoint_api_format }}</code>
           </div>
           <div class="mt-1.5 space-y-0.5">
             <div
@@ -277,16 +319,20 @@
 
       <!-- desktop: table layout -->
       <div
-        v-if="result.attempts.length > 0"
+        v-if="resultAttempts.length > 0"
         class="border rounded-md overflow-hidden hidden sm:block"
       >
         <table class="w-full text-xs table-fixed">
           <colgroup>
             <col class="w-8">
             <col class="w-[22%]">
+            <col
+              v-if="showEndpointColumn"
+              class="w-20"
+            >
             <col
               v-if="hasEffectiveModel"
-              class="w-[18%]"
+              class="w-[16%]"
             >
             <col class="w-16">
             <col class="w-16">
@@ -300,6 +346,12 @@
               <th class="px-3 py-2 text-left font-medium">
                 Key
               </th>
+              <th
+                v-if="showEndpointColumn"
+                class="px-3 py-2 text-left font-medium"
+              >
+                端点
+              </th>
               <th
                 v-if="hasEffectiveModel"
                 class="px-3 py-2 text-left font-medium"
@@ -319,7 +371,7 @@
           </thead>
           <tbody>
             <tr
-              v-for="(attempt, idx) in result.attempts"
+              v-for="(attempt, idx) in visibleAttempts"
               :key="idx"
               class="border-b last:border-b-0 align-top"
               :class="attemptRowClass(attempt.status)"
@@ -342,6 +394,12 @@
                   {{ maskKey(attempt.key_id) }}
                 </div>
               </td>
+              <td
+                v-if="showEndpointColumn"
+                class="px-3 py-2"
+              >
+                <code class="text-[11px] bg-muted px-1 py-0.5 rounded">{{ attempt.endpoint_api_format }}</code>
+              </td>
               <td
                 v-if="hasEffectiveModel"
                 class="px-3 py-2 truncate"
@@ -354,7 +412,7 @@
                   :variant="statusVariant(attempt.status)"
                   class="text-[10px] px-1.5 py-0"
                 >
-                  {{ attempt.status_code || statusLabel(attempt.status) }}
+                  {{ statusDisplay(attempt) }}
                 </Badge>
               </td>
               <td class="px-3 py-2 text-right text-muted-foreground tabular-nums">
@@ -402,7 +460,7 @@
 </template>
 
 <script setup lang="ts">
-import { computed } from 'vue'
+import { computed, ref, watch } from 'vue'
 import { Loader2 } from 'lucide-vue-next'
 import { Dialog, Badge } from '@/components/ui'
 import Button from '@/components/ui/button.vue'
@@ -474,6 +532,67 @@ const hasEffectiveModel = computed(() => {
   return props.result.attempts.some(a => a.effective_model && a.effective_model !== props.result?.model)
 })
 
+const showEndpointColumn = computed(() => {
+  if (!props.result) return false
+  if (props.mode === 'direct') return true
+  const formats = new Set(props.result.attempts.map(a => a.endpoint_api_format))
+  return formats.size > 1
+})
+
+const resultAttempts = computed(() => props.result?.attempts ?? [])
+const showAllAttempts = ref(false)
+
+watch(() => props.result, () => {
+  showAllAttempts.value = false
+})
+
+const shouldCollapseAttempts = computed(() => resultAttempts.value.length > 20)
+
+const visibleAttempts = computed(() => {
+  if (!shouldCollapseAttempts.value || showAllAttempts.value) {
+    return resultAttempts.value
+  }
+  return resultAttempts.value.slice(0, 20)
+})
+
+type AttemptSummaryItem = {
+  key: string
+  label: string
+  count: number
+  variant: 'success' | 'destructive' | 'secondary'
+}
+
+const attemptSummaryItems = computed<AttemptSummaryItem[]>(() => {
+  const groups = new Map<string, AttemptSummaryItem>()
+
+  for (const attempt of resultAttempts.value) {
+    const label = summarizeAttempt(attempt)
+    const key = `${attempt.status}:${label}`
+    const existing = groups.get(key)
+    if (existing) {
+      existing.count += 1
+      continue
+    }
+    groups.set(key, {
+      key,
+      label,
+      count: 1,
+      variant: statusVariant(attempt.status),
+    })
+  }
+
+  const variantRank: Record<AttemptSummaryItem['variant'], number> = {
+    destructive: 0,
+    secondary: 1,
+    success: 2,
+  }
+
+  return [...groups.values()].sort((left, right) => {
+    if (right.count !== left.count) return right.count - left.count
+    return variantRank[left.variant] - variantRank[right.variant]
+  })
+})
+
 const liveTraceSummary = computed(() => {
   const summary = {
     total: traceCandidates.value.length,
@@ -552,7 +671,7 @@ const liveRecentCandidates = computed(() => {
 
 function statusVariant(status: string) {
   if (status === 'success') return 'success' as const
-  if (status === 'failed') return 'destructive' as const
+  if (status === 'failed' || status === 'stream_interrupted') return 'destructive' as const
   return 'secondary' as const
 }
 
@@ -568,9 +687,44 @@ function statusLabel(status: string) {
   return status
 }
 
+function statusDisplay(item: { status: string; status_code?: number | null }): string {
+  const code = item.status_code
+  const status = item.status
+  if (!code) return statusLabel(status)
+  // 失败但 HTTP 状态码是 2xx：显示 "200 体内错误" 以区分
+  if (status === 'failed' && code >= 200 && code < 300) {
+    return `${code} 体内错误`
+  }
+  return String(code)
+}
+
+function compactDetail(value: string | null | undefined, maxLength = 64): string | null {
+  if (!value) return null
+  const compact = value.replace(/\s+/g, ' ').trim()
+  if (!compact) return null
+  return compact.length > maxLength ? `${compact.slice(0, maxLength)}…` : compact
+}
+
+function summarizeAttempt(attempt: TestAttemptDetail): string {
+  if (attempt.status === 'skipped') return '跳过'
+  if (attempt.status === 'cancelled') return '已取消'
+  if (attempt.status === 'success') return '成功'
+
+  const detail = compactDetail(attempt.error_message || attempt.skip_reason)
+  if (attempt.status_code != null) {
+    if (detail) return `${attempt.status_code} ${detail}`
+    if (attempt.status === 'failed' && attempt.status_code >= 200 && attempt.status_code < 300) {
+      return `${attempt.status_code} 体内错误`
+    }
+    return `${attempt.status_code} ${statusLabel(attempt.status)}`
+  }
+  return detail || statusLabel(attempt.status)
+}
+
 function attemptRowClass(status: string) {
   if (status === 'success') return 'bg-green-500/5'
   if (status === 'failed') return 'bg-red-500/5'
+  if (status === 'cancelled') return 'bg-amber-500/5'
   if (status === 'skipped') return 'bg-muted/20'
   return ''
 }
@@ -612,6 +766,7 @@ function traceCandidateDetail(candidate: CandidateRecord): string {
 }
 
 function attemptDetail(attempt: TestAttemptDetail): string {
+  if (attempt.status === 'cancelled') return '测试已取消'
   if (attempt.skip_reason) return attempt.skip_reason
   if (attempt.error_message) return attempt.error_message
   if (attempt.status === 'success') return attempt.endpoint_base_url

frontend/src/features/providers/components/provider-tabs/ModelsTab.vue

@@ -130,11 +130,11 @@
                   size="icon"
                   class="h-8 w-8"
                   title="测试模型"
-                  :disabled="testingModelId === model.id"
+                  :disabled="modelTest.testing.value && pendingTestModel?.id === model.id"
                   @click="testModelConnection(model)"
                 >
                   <Loader2
-                    v-if="testingModelId === model.id"
+                    v-if="modelTest.testing.value && pendingTestModel?.id === model.id"
                     class="w-3.5 h-3.5 animate-spin"
                   />
                   <Play
@@ -212,15 +212,15 @@
   </Card>
 
   <ModelTestDialog
-    :open="testDialogOpen"
+    :open="modelTest.dialogOpen.value"
-    :result="testResult"
+    :result="modelTest.testResult.value"
-    :mode="testResultMode"
+    :mode="modelTest.testMode.value"
     :selecting-model-name="pendingTestModel ? (pendingTestModel.global_model_display_name || pendingTestModel.provider_model_name) : null"
     :endpoints="activeEndpoints"
     :selected-endpoint="selectedTestEndpoint"
-    :testing="!!pendingTestModel && testingModelId === pendingTestModel.id"
+    :testing="modelTest.testing.value"
-    :trace="testTrace"
+    :trace="modelTest.testTrace.value"
-    :request-id="currentTestRequestId"
+    :request-id="modelTest.requestId.value"
     :show-endpoint-selector="activeEndpoints.length > 1"
     @close="handleTestDialogClose"
     @back="handleTestDialogBack"
@@ -229,9 +229,9 @@
 </template>
 
 <script setup lang="ts">
-import { ref, computed, onBeforeUnmount } from 'vue'
+import { ref, computed } from 'vue'
-import { isAxiosError } from 'axios'
 import { useSmartPagination } from '@/composables/useSmartPagination'
+import { useModelTest } from '@/composables/useModelTest'
 import { Box, Edit, Layers, Power, Copy, Loader2, Play } from 'lucide-vue-next'
 import Card from '@/components/ui/card.vue'
 import Button from '@/components/ui/button.vue'
@@ -239,13 +239,10 @@ import { useToast } from '@/composables/useToast'
 import { useClipboard } from '@/composables/useClipboard'
 import { sortResolutionEntries } from '@/utils/form'
 import {
-  testModelFailover,
   type Model,
   type ProviderEndpoint,
-  type TestModelFailoverResponse,
 } from '@/api/endpoints'
 import { updateModel } from '@/api/endpoints/models'
-import { requestTraceApi, type RequestTrace } from '@/api/requestTrace'
 import { parseApiError } from '@/utils/errorParser'
 import { formatApiFormat } from '@/api/endpoints/types/api-format'
 import type { ProviderWithEndpointsSummary } from '@/api/endpoints'
@@ -266,23 +263,16 @@ const emit = defineEmits<{
 const { error: showError, success: showSuccess } = useToast()
 const { copyToClipboard } = useClipboard()
 
+// 模型测试 composable
+const modelTest = useModelTest({ providerId: () => props.provider.id })
+
 // 状态
 const loading = ref(false)
 const localModels = ref<Model[]>([])
 const togglingModelId = ref<string | null>(null)
-const testingModelId = ref<string | null>(null)
-const testResult = ref<TestModelFailoverResponse | null>(null)
-const testResultMode = ref<'global' | 'direct'>('global')
-const testDialogOpen = ref(false)
 const pendingTestModel = ref<Model | null>(null)
 const selectedTestEndpoint = ref<ProviderEndpoint | null>(null)
-const currentTestRequestId = ref<string | null>(null)
-const testTrace = ref<RequestTrace | null>(null)
-let tracePollTimer: ReturnType<typeof setInterval> | null = null
-let tracePollToken = 0
-// 使用 props 传入的数据，或使用本地数据
 const activeEndpoints = computed(() => (props.endpoints ?? []).filter(endpoint => endpoint.is_active))
-// 使用 props 传入的数据，或使用本地数据
 const models = computed(() => props.models ?? localModels.value)
 // 按名称排序的模型列表
 const sortedModels = computed(() => {
@@ -312,47 +302,6 @@ function refresh() {
   emit('refresh')
 }
 
-function buildTestRequestId(): string {
-  const randomUUID = globalThis.crypto?.randomUUID?.bind(globalThis.crypto)
-  if (randomUUID) {
-    return `provider-test-${randomUUID().replace(/-/g, '').slice(0, 20)}`
-  }
-  return `provider-test-${Date.now().toString(36)}${Math.random().toString(36).slice(2, 10)}`
-}
-
-async function pollTestTrace(requestId: string, token: number) {
-  try {
-    const trace = await requestTraceApi.getRequestTrace(requestId, { attemptedOnly: false })
-    if (tracePollToken !== token || currentTestRequestId.value !== requestId) return
-    testTrace.value = trace
-  } catch (err: unknown) {
-    if (isAxiosError(err) && err.response?.status === 404) return
-  }
-}
-
-function stopTestTracePolling(options: { clearState?: boolean } = {}) {
-  tracePollToken += 1
-  if (tracePollTimer) {
-    clearInterval(tracePollTimer)
-    tracePollTimer = null
-  }
-  if (options.clearState !== false) {
-    currentTestRequestId.value = null
-    testTrace.value = null
-  }
-}
-
-function startTestTracePolling(requestId: string) {
-  stopTestTracePolling()
-  currentTestRequestId.value = requestId
-  testTrace.value = null
-  const token = ++tracePollToken
-  void pollTestTrace(requestId, token)
-  tracePollTimer = setInterval(() => {
-    void pollTestTrace(requestId, token)
-  }, 800)
-}
-
 // 格式化价格显示
 function formatPrice(price: number | null | undefined): string {
   if (price === null || price === undefined) return '-'
@@ -485,21 +434,15 @@ async function toggleModelActive(model: Model) {
   }
 }
 
-function resetTestDialogState() {
+function handleTestDialogClose() {
-  stopTestTracePolling()
+  modelTest.resetState()
-  testDialogOpen.value = false
   pendingTestModel.value = null
   selectedTestEndpoint.value = null
-  testResult.value = null
-}
-
-function handleTestDialogClose() {
-  resetTestDialogState()
 }
 
 function handleTestDialogBack() {
-  if (testingModelId.value) return
+  if (modelTest.testing.value) return
-  testResult.value = null
+  modelTest.testResult.value = null
   selectedTestEndpoint.value = null
 }
 
@@ -507,60 +450,33 @@ async function handleSelectTestEndpoint(endpointId: string) {
   if (!pendingTestModel.value) return
   const endpoint = activeEndpoints.value.find(item => item.id === endpointId)
   if (!endpoint) return
-  await runModelTest(pendingTestModel.value, endpoint)
+  selectedTestEndpoint.value = endpoint
+  const model = pendingTestModel.value
+  const modelName = model.global_model_name || model.provider_model_name
+  const endpointPrefix = `[${formatApiFormat(endpoint.api_format)}] `
+  await modelTest.startTest({
+    mode: 'global',
+    modelName,
+    displayLabel: `${endpointPrefix}${modelName}`,
+    apiFormat: endpoint.api_format,
+    endpointId: endpoint.id,
+    message: 'hello',
+    concurrency: 5,
+    onSuccess: () => {
+      pendingTestModel.value = null
+      selectedTestEndpoint.value = null
+    },
+    onError: () => {
+      if (activeEndpoints.value.length > 1) {
+        selectedTestEndpoint.value = null
+        return true
+      }
+    },
+  })
 }
 
-async function runModelTest(model: Model, endpoint?: ProviderEndpoint) {
-  if (testingModelId.value) return
-
-  testingModelId.value = model.id
-  testDialogOpen.value = true
-  selectedTestEndpoint.value = endpoint ?? null
-  const requestId = buildTestRequestId()
-  startTestTracePolling(requestId)
-  try {
-    const modelName = model.global_model_name || model.provider_model_name
-
-    const result = await testModelFailover({
-      provider_id: props.provider.id,
-      mode: 'global',
-      model_name: modelName,
-      api_format: endpoint?.api_format,
-      endpoint_id: endpoint?.id,
-      message: 'hello',
-      request_id: requestId,
-    })
-
-    if (result.success) {
-      const successAttempt = result.attempts.find(a => a.status === 'success')
-      const latency = successAttempt?.latency_ms != null ? ` (${successAttempt.latency_ms}ms)` : ''
-      const mapped = successAttempt?.effective_model && successAttempt.effective_model !== modelName
-        ? ` -> ${successAttempt.effective_model}`
-        : ''
-      const endpointPrefix = endpoint ? `[${formatApiFormat(endpoint.api_format)}] ` : ''
-      showSuccess(`${endpointPrefix}${modelName}${mapped} 测试成功${latency}`)
-      resetTestDialogState()
-      return
-    }
-    stopTestTracePolling({ clearState: false })
-    testResultMode.value = 'global'
-    testResult.value = result
-  } catch (err: unknown) {
-    stopTestTracePolling()
-    showError(`模型测试失败: ${parseApiError(err, '测试请求失败')}`)
-    if (activeEndpoints.value.length <= 1) {
-      resetTestDialogState()
-      return
-    }
-    selectedTestEndpoint.value = null
-  } finally {
-    testingModelId.value = null
-  }
-}
-
-// 测试模型连接性（模拟外部请求，带故障转移）
 async function testModelConnection(model: Model) {
-  if (testingModelId.value) return
+  if (modelTest.testing.value) return
 
   if (activeEndpoints.value.length === 0) {
     showError('暂无可用于测试的活跃端点')
@@ -569,11 +485,11 @@ async function testModelConnection(model: Model) {
 
   pendingTestModel.value = model
   selectedTestEndpoint.value = null
-  testResult.value = null
+  modelTest.testResult.value = null
-  testDialogOpen.value = true
+  modelTest.dialogOpen.value = true
 
   if (activeEndpoints.value.length === 1) {
-    await runModelTest(model, activeEndpoints.value[0])
+    await handleSelectTestEndpoint(activeEndpoints.value[0].id)
   }
 }
 
@@ -581,8 +497,4 @@ async function testModelConnection(model: Model) {
 defineExpose({
   reload: refresh
 })
-
-onBeforeUnmount(() => {
-  stopTestTracePolling()
-})
 </script>

frontend/src/features/providers/components/provider-tabs/TestResultDialog.vue

@@ -1,301 +0,0 @@
-<template>
-  <Dialog
-    :open="!!result"
-    size="2xl"
-    title="模型测试结果"
-    @update:open="(val: boolean) => { if (!val) $emit('close') }"
-  >
-    <div
-      v-if="result"
-      class="space-y-4"
-    >
-      <div class="flex items-center justify-between">
-        <div class="flex items-center gap-2">
-          <Badge :variant="result.success ? 'success' : 'destructive'">
-            {{ result.success ? '成功' : '失败' }}
-          </Badge>
-          <span class="text-sm text-muted-foreground">
-            {{ modeLabel }}
-          </span>
-        </div>
-        <div class="text-xs text-muted-foreground">
-          候选 {{ result.total_candidates }} / 尝试 {{ result.total_attempts }}
-        </div>
-      </div>
-
-      <div class="text-sm space-y-1">
-        <div>
-          <span class="text-muted-foreground">请求模型: </span>
-          <span class="font-medium">{{ result.model }}</span>
-        </div>
-        <div v-if="successEffectiveModel">
-          <span class="text-muted-foreground">发送模型: </span>
-          <span class="font-medium text-primary">{{ successEffectiveModel }}</span>
-          <span
-            v-if="successEffectiveModel !== result.model"
-            class="text-xs text-muted-foreground ml-1"
-          >(已映射)</span>
-        </div>
-      </div>
-
-      <div
-        v-if="result.error && !result.success"
-        class="rounded-md bg-destructive/10 border border-destructive/20 px-3 py-2 text-xs text-destructive"
-      >
-        {{ result.error }}
-      </div>
-
-      <!-- mobile: list layout -->
-      <div
-        v-if="result.attempts.length > 0"
-        class="space-y-2 sm:hidden"
-      >
-        <div
-          v-for="(attempt, idx) in result.attempts"
-          :key="'m' + idx"
-          class="rounded-md border px-3 py-2 text-xs"
-          :class="attemptRowClass(attempt.status)"
-        >
-          <div class="flex items-center justify-between gap-2">
-            <div class="flex items-center gap-1.5 min-w-0">
-              <span class="text-muted-foreground shrink-0">{{ formatAttemptIndex(attempt) }}</span>
-              <Badge
-                :variant="statusVariant(attempt.status)"
-                class="text-[10px] px-1.5 py-0 shrink-0"
-              >
-                {{ attempt.status_code || statusLabel(attempt.status) }}
-              </Badge>
-              <span
-                v-if="attempt.latency_ms != null"
-                class="text-muted-foreground shrink-0 tabular-nums"
-              >
-                {{ attempt.latency_ms }}ms
-              </span>
-            </div>
-            <code class="text-[11px] bg-muted px-1 py-0.5 rounded shrink-0">{{ attempt.endpoint_api_format }}</code>
-          </div>
-          <div class="mt-1.5 space-y-0.5">
-            <div
-              v-if="attempt.key_name"
-              class="font-medium truncate"
-            >
-              {{ attempt.key_name }}
-            </div>
-            <div class="text-muted-foreground">
-              {{ maskKey(attempt.key_id) }}
-            </div>
-            <div
-              v-if="hasEffectiveModel && attempt.effective_model"
-              class="text-muted-foreground"
-            >
-              模型: {{ attempt.effective_model }}
-            </div>
-            <div
-              v-if="attemptDetail(attempt) !== '-'"
-              class="text-muted-foreground break-all mt-1"
-            >
-              {{ attemptDetail(attempt) }}
-            </div>
-          </div>
-        </div>
-      </div>
-
-      <!-- desktop: table layout -->
-      <div
-        v-if="result.attempts.length > 0"
-        class="border rounded-md overflow-hidden hidden sm:block"
-      >
-        <table class="w-full text-xs table-fixed">
-          <colgroup>
-            <col class="w-8">
-            <col class="w-[22%]">
-            <col class="w-20">
-            <col
-              v-if="hasEffectiveModel"
-              class="w-[16%]"
-            >
-            <col class="w-16">
-            <col class="w-16">
-            <col>
-          </colgroup>
-          <thead>
-            <tr class="border-b bg-muted/30">
-              <th class="pl-3 pr-1 py-2 text-left font-medium">
-                #
-              </th>
-              <th class="px-3 py-2 text-left font-medium">
-                Key
-              </th>
-              <th class="px-3 py-2 text-left font-medium">
-                端点
-              </th>
-              <th
-                v-if="hasEffectiveModel"
-                class="px-3 py-2 text-left font-medium"
-              >
-                发送模型
-              </th>
-              <th class="px-3 py-2 text-left font-medium">
-                状态
-              </th>
-              <th class="px-3 py-2 text-right font-medium">
-                延迟
-              </th>
-              <th class="px-3 py-2 text-left font-medium">
-                详情
-              </th>
-            </tr>
-          </thead>
-          <tbody>
-            <tr
-              v-for="(attempt, idx) in result.attempts"
-              :key="idx"
-              class="border-b last:border-b-0 align-top"
-              :class="attemptRowClass(attempt.status)"
-            >
-              <td class="pl-3 pr-1 py-2 text-muted-foreground">
-                {{ formatAttemptIndex(attempt) }}
-              </td>
-              <td class="px-3 py-2">
-                <div
-                  v-if="attempt.key_name"
-                  class="font-medium truncate"
-                  :title="attempt.key_name"
-                >
-                  {{ attempt.key_name }}
-                </div>
-                <div
-                  class="text-muted-foreground truncate"
-                  :title="attempt.key_id"
-                >
-                  {{ maskKey(attempt.key_id) }}
-                </div>
-              </td>
-              <td class="px-3 py-2">
-                <code class="text-[11px] bg-muted px-1 py-0.5 rounded">{{ attempt.endpoint_api_format }}</code>
-              </td>
-              <td
-                v-if="hasEffectiveModel"
-                class="px-3 py-2 truncate"
-                :title="attempt.effective_model || '-'"
-              >
-                {{ attempt.effective_model || '-' }}
-              </td>
-              <td class="px-3 py-2">
-                <Badge
-                  :variant="statusVariant(attempt.status)"
-                  class="text-[10px] px-1.5 py-0"
-                >
-                  {{ attempt.status_code || statusLabel(attempt.status) }}
-                </Badge>
-              </td>
-              <td class="px-3 py-2 text-right text-muted-foreground tabular-nums">
-                {{ attempt.latency_ms != null ? attempt.latency_ms + 'ms' : '-' }}
-              </td>
-              <td class="px-3 py-2 text-muted-foreground">
-                <div
-                  class="break-all line-clamp-2"
-                  :title="attemptDetail(attempt)"
-                >
-                  {{ attemptDetail(attempt) }}
-                </div>
-              </td>
-            </tr>
-          </tbody>
-        </table>
-      </div>
-
-      <div
-        v-else
-        class="text-center text-sm text-muted-foreground py-4"
-      >
-        没有可用的候选进行测试
-      </div>
-    </div>
-
-    <template #footer>
-      <Button
-        variant="outline"
-        size="sm"
-        @click="$emit('close')"
-      >
-        关闭
-      </Button>
-    </template>
-  </Dialog>
-</template>
-
-<script setup lang="ts">
-import { computed } from 'vue'
-import { Dialog, Badge } from '@/components/ui'
-import Button from '@/components/ui/button.vue'
-import type { TestModelFailoverResponse, TestAttemptDetail } from '@/api/endpoints/providers'
-
-const props = defineProps<{
-  result: TestModelFailoverResponse | null
-  mode?: 'global' | 'direct'
-}>()
-
-defineEmits<{
-  close: []
-}>()
-
-const modeLabel = computed(() => {
-  if (props.mode === 'global') return '模拟外部请求'
-  if (props.mode === 'direct') return '直接测试'
-  return ''
-})
-
-const successEffectiveModel = computed(() => {
-  if (!props.result) return null
-  const successAttempt = props.result.attempts.find(a => a.status === 'success')
-  return successAttempt?.effective_model || null
-})
-
-const hasEffectiveModel = computed(() => {
-  if (!props.result) return false
-  return props.result.attempts.some(a => a.effective_model && a.effective_model !== props.result?.model)
-})
-
-function statusVariant(status: string) {
-  if (status === 'success') return 'success' as const
-  if (status === 'failed') return 'destructive' as const
-  return 'secondary' as const
-}
-
-function statusLabel(status: string) {
-  if (status === 'success') return '成功'
-  if (status === 'failed') return '失败'
-  if (status === 'skipped') return '跳过'
-  if (status === 'pending') return '等待中'
-  if (status === 'streaming') return '测试中'
-  if (status === 'cancelled') return '已取消'
-  if (status === 'stream_interrupted') return '流中断'
-  if (status === 'available') return '待执行'
-  return status
-}
-
-function attemptRowClass(status: string) {
-  if (status === 'success') return 'bg-green-500/5'
-  if (status === 'failed') return 'bg-red-500/5'
-  if (status === 'skipped') return 'bg-muted/20'
-  return ''
-}
-
-function maskKey(key: string): string {
-  if (key.length <= 8) return key
-  return `${key.slice(0, 4)}...${key.slice(-4)}`
-}
-
-function formatAttemptIndex(attempt: TestAttemptDetail): string {
-  const retryIndex = attempt.retry_index ?? 0
-  return retryIndex > 0 ? `#${attempt.candidate_index}.${retryIndex}` : `#${attempt.candidate_index}`
-}
-
-function attemptDetail(attempt: TestAttemptDetail): string {
-  if (attempt.skip_reason) return attempt.skip_reason
-  if (attempt.error_message) return attempt.error_message
-  if (attempt.status === 'success') return attempt.endpoint_base_url
-  return '-'
-}
-</script>

src/api/admin/pool/routes.py

@@ -37,6 +37,7 @@ from src.services.provider.pool.scheduling_dimensions import (
     evaluate_pool_scheduling_dimensions,
     summarize_pool_scheduling_dimensions,
 )
+from src.services.provider_keys.quota_reader import get_quota_reader
 
 from .schemas import (
     BatchActionRequest,
@@ -197,180 +198,12 @@ def _is_known_banned_key(key: ProviderAPIKey, provider_type: str) -> bool:
     return state.blocked
 
 
-def _format_percent(value: float) -> str:
-    clamped = max(0.0, min(value, 100.0))
-    return f"{clamped:.1f}%"
-
-
-def _format_quota_value(value: float) -> str:
-    rounded = round(value)
-    if abs(value - rounded) < 1e-6:
-        return str(rounded)
-    return f"{value:.1f}"
-
-
-def _format_reset_after(seconds_raw: Any) -> str | None:
-    seconds = _to_float(seconds_raw)
-    if seconds is None:
-        return None
-
-    total_seconds = int(seconds)
-    if total_seconds <= 0:
-        return "已重置"
-
-    days = total_seconds // 86400
-    hours = (total_seconds % 86400) // 3600
-    minutes = (total_seconds % 3600) // 60
-
-    if days > 0:
-        return f"{days}天{hours}小时后重置"
-    if hours > 0:
-        return f"{hours}小时{minutes}分钟后重置"
-    if minutes > 0:
-        return f"{minutes}分钟后重置"
-    return "即将重置"
-
-
-def _build_codex_account_quota(upstream_metadata: dict[str, Any]) -> str | None:
-    codex = upstream_metadata.get("codex")
-    if not isinstance(codex, dict):
-        return None
-
-    parts: list[str] = []
-
-    primary_used = _to_float(codex.get("primary_used_percent"))
-    if primary_used is not None:
-        part = f"周剩余 {_format_percent(100.0 - primary_used)}"
-        reset_text = _format_reset_after(codex.get("primary_reset_seconds"))
-        if reset_text:
-            part = f"{part} ({reset_text})"
-        parts.append(part)
-
-    secondary_used = _to_float(codex.get("secondary_used_percent"))
-    if secondary_used is not None:
-        part = f"5H剩余 {_format_percent(100.0 - secondary_used)}"
-        reset_text = _format_reset_after(codex.get("secondary_reset_seconds"))
-        if reset_text:
-            part = f"{part} ({reset_text})"
-        parts.append(part)
-
-    if parts:
-        return " | ".join(parts)
-
-    has_credits = codex.get("has_credits")
-    credits_balance = _to_float(codex.get("credits_balance"))
-    if has_credits is True and credits_balance is not None:
-        return f"积分 {credits_balance:.2f}"
-    if has_credits is True:
-        return "有积分"
-    return None
-
-
-def _build_kiro_account_quota(upstream_metadata: dict[str, Any]) -> str | None:
-    kiro = upstream_metadata.get("kiro")
-    if not isinstance(kiro, dict):
-        return None
-
-    if kiro.get("is_banned") is True:
-        return "账号已封禁"
-
-    usage_percentage = _to_float(kiro.get("usage_percentage"))
-    if usage_percentage is not None:
-        remaining = 100.0 - usage_percentage
-        current_usage = _to_float(kiro.get("current_usage"))
-        usage_limit = _to_float(kiro.get("usage_limit"))
-        if current_usage is not None and usage_limit is not None and usage_limit > 0:
-            return (
-                f"剩余 {_format_percent(remaining)} "
-                f"({_format_quota_value(current_usage)}/{_format_quota_value(usage_limit)})"
-            )
-        return f"剩余 {_format_percent(remaining)}"
-
-    remaining = _to_float(kiro.get("remaining"))
-    usage_limit = _to_float(kiro.get("usage_limit"))
-    if remaining is not None and usage_limit is not None and usage_limit > 0:
-        return f"剩余 {_format_quota_value(remaining)}/{_format_quota_value(usage_limit)}"
-    return None
-
-
-def _build_antigravity_account_quota(upstream_metadata: dict[str, Any]) -> str | None:
-    antigravity = upstream_metadata.get("antigravity")
-    if not isinstance(antigravity, dict):
-        return None
-
-    if antigravity.get("is_forbidden") is True:
-        return "访问受限"
-
-    quota_by_model = antigravity.get("quota_by_model")
-    if not isinstance(quota_by_model, dict) or not quota_by_model:
-        return None
-
-    remaining_list: list[float] = []
-    for raw_info in quota_by_model.values():
-        if not isinstance(raw_info, dict):
-            continue
-
-        used_percent = _to_float(raw_info.get("used_percent"))
-        if used_percent is None:
-            remaining_fraction = _to_float(raw_info.get("remaining_fraction"))
-            if remaining_fraction is not None:
-                used_percent = (1.0 - remaining_fraction) * 100.0
-
-        if used_percent is None:
-            continue
-
-        remaining = max(0.0, min(100.0 - used_percent, 100.0))
-        remaining_list.append(remaining)
-
-    if not remaining_list:
-        return None
-
-    min_remaining = min(remaining_list)
-    if len(remaining_list) == 1:
-        return f"剩余 {_format_percent(min_remaining)}"
-    return f"最低剩余 {_format_percent(min_remaining)} ({len(remaining_list)} 模型)"
-
-
 def _build_account_quota(provider_type: str, upstream_metadata: Any) -> str | None:
-    if not isinstance(upstream_metadata, dict):
+    return get_quota_reader(provider_type, upstream_metadata).display_summary()
-        return None
-
-    normalized_type = provider_type.strip().lower()
-    if normalized_type == "codex":
-        return _build_codex_account_quota(upstream_metadata)
-    if normalized_type == "kiro":
-        return _build_kiro_account_quota(upstream_metadata)
-    if normalized_type == "antigravity":
-        return _build_antigravity_account_quota(upstream_metadata)
-    return None
 
 
 def _extract_quota_updated_at(provider_type: str, upstream_metadata: Any) -> int | None:
-    if not isinstance(upstream_metadata, dict):
+    return get_quota_reader(provider_type, upstream_metadata).updated_at()
-        return None
-
-    normalized_type = provider_type.strip().lower()
-    if normalized_type == "codex":
-        source = upstream_metadata.get("codex")
-    elif normalized_type == "antigravity":
-        source = upstream_metadata.get("antigravity")
-    elif normalized_type == "kiro":
-        source = upstream_metadata.get("kiro")
-    else:
-        return None
-
-    if not isinstance(source, dict):
-        return None
-
-    updated_at = _to_float(source.get("updated_at"))
-    if updated_at is None or updated_at <= 0:
-        return None
-
-    # 部分上游可能返回毫秒时间戳，统一转换为秒
-    if updated_at > 1_000_000_000_000:
-        updated_at /= 1000
-
-    return int(updated_at)
 
 
 def _normalize_oauth_plan_type(plan_type: Any, provider_type: str) -> str | None:

src/api/admin/provider_query.py

@@ -7,13 +7,17 @@ from __future__ import annotations
 
 import asyncio
 import json
+import time
+from collections.abc import Awaitable, Callable
+from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Any
 from uuid import uuid4
 
 import httpx
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends, HTTPException, Request
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
-from sqlalchemy.orm import Session, joinedload
+from sqlalchemy import update
+from sqlalchemy.orm import Session, joinedload, make_transient
 
 from src.config.constants import TimeoutDefaults
 from src.core.api_format import get_extra_headers_from_endpoint
@@ -21,8 +25,9 @@ from src.core.cache_service import CacheService
 from src.core.crypto import crypto_service
 from src.core.logger import logger
 from src.core.provider_types import ProviderType
+from src.database import create_session
 from src.database.database import get_db
-from src.models.database import Provider, ProviderEndpoint, User
+from src.models.database import Provider, ProviderAPIKey, ProviderEndpoint, RequestCandidate, User
 from src.services.model.fetch_scheduler import (
     MODEL_FETCH_HTTP_TIMEOUT,
     UPSTREAM_MODELS_CACHE_TTL_SECONDS,
@@ -39,6 +44,7 @@ from src.services.model.upstream_fetcher import (
 )
 from src.services.provider.oauth_token import resolve_oauth_access_token
 from src.services.proxy_node.resolver import resolve_effective_proxy
+from src.services.request.candidate import RequestCandidateService
 from src.utils.auth_utils import get_current_user
 
 if TYPE_CHECKING:
@@ -209,6 +215,7 @@ class TestModelFailoverRequest(BaseModel):
     endpoint_id: str | None = None  # 指定仅使用该端点测试
     message: str | None = "Hello"
     request_id: str | None = None
+    concurrency: int = Field(default=1, ge=1, le=20)
 
 
 class TestAttemptDetail(BaseModel):
@@ -893,7 +900,7 @@ async def test_model(
 
         def _response_has_error(resp: dict) -> bool:
             """快速判断响应是否包含错误"""
-            if "error" in resp:
+            if resp.get("error"):
                 return True
             if resp.get("status_code", 0) != 200:
                 return True
@@ -905,7 +912,7 @@ async def test_model(
                     parsed = json.loads(resp_body)
                 except (json.JSONDecodeError, ValueError):
                     pass
-            if isinstance(parsed, dict) and "error" in parsed:
+            if isinstance(parsed, dict) and parsed.get("error"):
                 return True
             return False
 
@@ -987,7 +994,7 @@ async def test_model(
             except json.JSONDecodeError:
                 pass
 
-        if isinstance(parsed_body, dict) and "error" in parsed_body:
+        if isinstance(parsed_body, dict) and parsed_body.get("error"):
             error_obj = parsed_body["error"]
             # 兼容 error 可能是字典或字符串的情况
             if isinstance(error_obj, dict):
@@ -1055,7 +1062,7 @@ async def test_model(
                     status_code=upstream_status,
                     detail=str(error_obj)[:500] if error_obj else "Provider error",
                 )
-        elif "error" in response:
+        elif response.get("error"):
             logger.debug(f"[test-model] Error: {response['error']}")
             upstream_status = int(response.get("status_code", 0) or 500)
             if not (400 <= upstream_status <= 599):
@@ -1134,6 +1141,7 @@ def _build_direct_test_candidates(
     为直接测试模式构建候选列表。
 
     遍历 Provider 的活跃 Endpoint 和 Key，不经过 GlobalModel 解析。
+    按可用性排序：熔断器关闭 > 健康度高 > 连续失败少 > Key 优先级。
     """
     from src.services.scheduling.schemas import ProviderCandidate
 
@@ -1165,9 +1173,50 @@ def _build_direct_test_candidates(
                     provider_api_format=ep_format,
                 )
             )
+
+    candidates.sort(key=lambda c: _direct_candidate_sort_key(c))
     return candidates
 
 
+def _direct_candidate_sort_key(candidate: ProviderCandidate) -> tuple[int, float, int, int]:
+    """
+    按可用性排序候选：
+    1. 熔断器状态：关闭(0) > 打开(2)
+    2. 健康度评分：越高越好（取负值以升序排列）
+    3. 连续失败次数：越少越好
+    4. Key 优先级：数字越小越优先
+    """
+    key = candidate.key
+    ep_format = candidate.provider_api_format
+
+    # 熔断器状态
+    circuit_breaker_order = 0
+    cb_data = getattr(key, "circuit_breaker_by_format", None) or {}
+    cb_entry = cb_data.get(ep_format, {}) if isinstance(cb_data, dict) else {}
+    if isinstance(cb_entry, dict) and cb_entry.get("open"):
+        circuit_breaker_order = 2
+
+    # 健康度评分（默认 1.0 表示完全健康）
+    health_score = 1.0
+    consecutive_failures = 0
+    health_data = getattr(key, "health_by_format", None) or {}
+    health_entry = health_data.get(ep_format, {}) if isinstance(health_data, dict) else {}
+    if isinstance(health_entry, dict):
+        health_score = health_entry.get("health_score", 1.0)
+        consecutive_failures = health_entry.get("consecutive_failures", 0)
+
+    # Key 优先级
+    internal_priority_raw = getattr(key, "internal_priority", None)
+    try:
+        internal_priority = (
+            int(internal_priority_raw) if internal_priority_raw is not None else 999999
+        )
+    except (TypeError, ValueError):
+        internal_priority = 999999
+
+    return (circuit_breaker_order, -health_score, consecutive_failures, internal_priority)
+
+
 def _filter_test_candidates_by_endpoint(
     candidates: list[ProviderCandidate],
     endpoint_id: str | None,
@@ -1278,6 +1327,457 @@ def _build_test_candidate_meta(
     return by_pair, by_candidate
 
 
+def _flatten_test_candidates_for_concurrency(
+    candidates: list[ProviderCandidate],
+) -> list[ProviderCandidate]:
+    from src.services.scheduling.schemas import (
+        PoolCandidate,
+    )
+    from src.services.scheduling.schemas import ProviderCandidate as SchedulerCandidate
+
+    flattened: list[ProviderCandidate] = []
+    for candidate in candidates:
+        if not isinstance(candidate, PoolCandidate):
+            flattened.append(candidate)
+            continue
+
+        for pool_key in candidate.pool_keys or []:
+            key_skipped = candidate.is_skipped or bool(getattr(pool_key, "_pool_skipped", False))
+            key_skip_reason_raw = (
+                getattr(pool_key, "_pool_skip_reason", None) if key_skipped else None
+            )
+            key_skip_reason = (
+                str(key_skip_reason_raw)
+                if key_skip_reason_raw
+                else (str(candidate.skip_reason) if candidate.skip_reason else None)
+            )
+            flattened.append(
+                SchedulerCandidate(
+                    provider=candidate.provider,
+                    endpoint=candidate.endpoint,
+                    key=pool_key,
+                    is_cached=bool(getattr(candidate, "is_cached", False)),
+                    is_skipped=key_skipped,
+                    skip_reason=key_skip_reason,
+                    mapping_matched_model=(
+                        getattr(pool_key, "_pool_mapping_matched_model", None)
+                        or getattr(candidate, "mapping_matched_model", None)
+                    ),
+                    needs_conversion=bool(getattr(candidate, "needs_conversion", False)),
+                    provider_api_format=(
+                        getattr(candidate, "provider_api_format", "")
+                        or str(getattr(candidate.endpoint, "api_format", "") or "")
+                    ),
+                    output_limit=getattr(candidate, "output_limit", None),
+                    capability_miss_count=int(getattr(candidate, "capability_miss_count", 0) or 0),
+                )
+            )
+
+    return flattened
+
+
+def _build_test_candidate_extra_data(candidate: ProviderCandidate) -> dict[str, Any]:
+    extra_data: dict[str, Any] = {
+        "needs_conversion": bool(getattr(candidate, "needs_conversion", False)),
+        "provider_api_format": (
+            getattr(candidate, "provider_api_format", None)
+            or getattr(getattr(candidate, "endpoint", None), "api_format", None)
+        ),
+        "mapping_matched_model": (
+            getattr(candidate, "mapping_matched_model", None)
+            or getattr(getattr(candidate, "key", None), "_pool_mapping_matched_model", None)
+        ),
+    }
+    key_extra = getattr(getattr(candidate, "key", None), "_pool_extra_data", None)
+    if isinstance(key_extra, dict):
+        extra_data.update(key_extra)
+    return extra_data
+
+
+def _precreate_concurrent_test_records(
+    *,
+    db: Session,
+    request_id: str,
+    candidates: list[ProviderCandidate],
+    user: User | None,
+) -> dict[int, str]:
+    record_map: dict[int, str] = {}
+    rows: list[dict[str, Any]] = []
+
+    user_id = str(getattr(user, "id", "") or "") or None
+    now = datetime.now(timezone.utc)
+    for candidate_index, candidate in enumerate(candidates):
+        record_id = str(uuid4())
+        record_map[candidate_index] = record_id
+        rows.append(
+            {
+                "id": record_id,
+                "request_id": request_id,
+                "candidate_index": candidate_index,
+                "retry_index": 0,
+                "user_id": user_id,
+                "api_key_id": None,
+                "provider_id": str(getattr(candidate.provider, "id", "") or "") or None,
+                "endpoint_id": str(getattr(candidate.endpoint, "id", "") or "") or None,
+                "key_id": str(getattr(candidate.key, "id", "") or "") or None,
+                "status": (
+                    "skipped" if bool(getattr(candidate, "is_skipped", False)) else "available"
+                ),
+                "skip_reason": getattr(candidate, "skip_reason", None),
+                "is_cached": bool(getattr(candidate, "is_cached", False)),
+                "extra_data": _build_test_candidate_extra_data(candidate),
+                "required_capabilities": None,
+                "created_at": now,
+            }
+        )
+
+    if rows:
+        db.bulk_insert_mappings(RequestCandidate, rows)  # type: ignore[arg-type]
+        db.commit()
+
+    return record_map
+
+
+def _mark_concurrent_test_record_cancelled(record_id: str) -> None:
+    if not record_id:
+        return
+    with create_session() as local_db:
+        RequestCandidateService.mark_candidate_cancelled(
+            db=local_db,
+            candidate_id=record_id,
+            status_code=499,
+        )
+
+
+def _cancel_remaining_concurrent_test_records(request_id: str) -> None:
+    if not request_id:
+        return
+    with create_session() as local_db:
+        local_db.execute(
+            update(RequestCandidate)
+            .where(RequestCandidate.request_id == request_id)
+            .where(RequestCandidate.status.in_(["available", "pending"]))
+            .values(
+                status="cancelled",
+                status_code=499,
+                finished_at=datetime.now(timezone.utc),
+            )
+        )
+        local_db.commit()
+
+
+async def _execute_test_check(
+    *,
+    provider_obj: Any,
+    endpoint: Any,
+    key: Any,
+    effective_model: str,
+    request_payload: dict[str, Any],
+    request_timeout: float,
+    provider_type: str,
+    user: User | None,
+    db: Session | None,
+) -> tuple[dict[str, Any], str]:
+    effective_proxy = resolve_effective_proxy(
+        getattr(provider_obj, "proxy", None), getattr(key, "proxy", None)
+    )
+    try:
+        api_key_value, auth_config = await _resolve_key_auth(
+            key,
+            provider_obj,
+            provider_proxy_config=effective_proxy,
+        )
+    except _KeyAuthError as e:
+        raise RuntimeError(e.message) from e
+
+    auth_type = str(getattr(key, "auth_type", "api_key") or "api_key").lower()
+    extra_headers = get_extra_headers_from_endpoint(endpoint) or {}
+    if auth_type == "oauth":
+        account_id = (auth_config or {}).get("account_id")
+        if account_id:
+            extra_headers["chatgpt-account-id"] = str(account_id)
+
+    adapter_class = get_adapter_for_format(endpoint.api_format)
+    if not adapter_class:
+        raise ValueError(f"Unknown API format: {endpoint.api_format}")
+
+    response = await adapter_class.check_endpoint(
+        None,
+        endpoint.base_url,
+        api_key_value,
+        {
+            **request_payload,
+            "model": effective_model,
+        },
+        extra_headers if extra_headers else None,
+        body_rules=getattr(endpoint, "body_rules", None),
+        header_rules=getattr(endpoint, "header_rules", None),
+        db=db,
+        user=user,
+        provider_name=provider_obj.name,
+        provider_id=str(provider_obj.id),
+        api_key_id=str(key.id),
+        model_name=effective_model,
+        auth_type=auth_type,
+        provider_type=provider_type if provider_type else None,
+        decrypted_auth_config=auth_config if auth_config else None,
+        provider_endpoint=endpoint,
+        provider_api_key=key,
+        proxy_config=effective_proxy,
+        timeout_seconds=request_timeout,
+    )
+    return response, auth_type
+
+
+async def _run_concurrent_test(
+    *,
+    candidates: list[ProviderCandidate],
+    concurrency: int,
+    is_cancelled: Callable[[], Awaitable[bool]],
+    request_id: str,
+    request_payload: dict[str, Any],
+    effective_model_by_candidate_index: dict[int, str],
+    request_timeout: float,
+    provider_type: str,
+    user: User | None,
+    db: Session,
+) -> dict[str, Any]:
+    from src.core.exceptions import EmbeddedErrorException
+    from src.services.candidate.recorder import CandidateRecorder
+    from src.services.task.service import pool_on_error
+
+    semaphore = asyncio.Semaphore(max(1, concurrency))
+    record_map = _precreate_concurrent_test_records(
+        db=db,
+        request_id=request_id,
+        candidates=candidates,
+        user=user,
+    )
+
+    # 预加载所有候选的 provider/endpoint/key，避免每个 worker 重复查询
+    _preloaded: dict[int, tuple[Provider, ProviderEndpoint, ProviderAPIKey]] = {}
+    with create_session() as preload_db:
+        provider_ids = {str(getattr(c.provider, "id", "") or "") for c in candidates}
+        endpoint_ids = {str(getattr(c.endpoint, "id", "") or "") for c in candidates}
+        key_ids = {str(getattr(c.key, "id", "") or "") for c in candidates}
+        providers_by_id = {
+            str(p.id): p
+            for p in preload_db.query(Provider).filter(Provider.id.in_(provider_ids)).all()
+        }
+        endpoints_by_id = {
+            str(e.id): e
+            for e in preload_db.query(ProviderEndpoint)
+            .filter(ProviderEndpoint.id.in_(endpoint_ids))
+            .all()
+        }
+        keys_by_id = {
+            str(k.id): k
+            for k in preload_db.query(ProviderAPIKey).filter(ProviderAPIKey.id.in_(key_ids)).all()
+        }
+        _already_detached: set[int] = set()
+        for idx, cand in enumerate(candidates):
+            p = providers_by_id.get(str(getattr(cand.provider, "id", "") or ""))
+            e = endpoints_by_id.get(str(getattr(cand.endpoint, "id", "") or ""))
+            k = keys_by_id.get(str(getattr(cand.key, "id", "") or ""))
+            if p is not None and e is not None and k is not None:
+                # make_transient 将对象脱离 session 并保留已加载属性，
+                # 避免 expired 状态导致跨协程访问时触发 lazy load 报错。
+                # 同一个对象（多个 candidate 可能共享同一 provider/endpoint）
+                # 只需处理一次。
+                for obj in (p, e, k):
+                    obj_id = id(obj)
+                    if obj_id not in _already_detached:
+                        make_transient(obj)
+                        _already_detached.add(obj_id)
+                _preloaded[idx] = (p, e, k)
+
+    success_payload: dict[str, Any] = {}
+    success_event = asyncio.Event()
+    candidate_recorder = CandidateRecorder(db)
+    last_error: Exception | None = None
+
+    async def _worker(candidate_index: int) -> dict[str, Any]:
+        nonlocal last_error
+        record_id = record_map[candidate_index]
+
+        started = False
+        started_at = 0.0
+
+        try:
+            preloaded = _preloaded.get(candidate_index)
+            if preloaded is None:
+                raise RuntimeError("并发测试目标不存在或已被删除")
+            local_provider, local_endpoint, local_key = preloaded
+            if success_event.is_set() or await is_cancelled():
+                _mark_concurrent_test_record_cancelled(record_id)
+                return {"status": "cancelled"}
+
+            async with semaphore:
+                if success_event.is_set() or await is_cancelled():
+                    _mark_concurrent_test_record_cancelled(record_id)
+                    return {"status": "cancelled"}
+
+                with create_session() as update_db:
+                    RequestCandidateService.mark_candidate_started(update_db, record_id)
+
+                started = True
+                started_at = time.perf_counter()
+                response, auth_type = await _execute_test_check(
+                    provider_obj=local_provider,
+                    endpoint=local_endpoint,
+                    key=local_key,
+                    effective_model=effective_model_by_candidate_index.get(
+                        candidate_index,
+                        str(request_payload.get("model", "") or ""),
+                    ),
+                    request_payload=request_payload,
+                    request_timeout=request_timeout,
+                    provider_type=provider_type,
+                    user=user,
+                    db=None,
+                )
+                elapsed_ms = max(0, int((time.perf_counter() - started_at) * 1000))
+
+                with create_session() as parse_db:
+                    parse_key = (
+                        parse_db.query(ProviderAPIKey)
+                        .filter(ProviderAPIKey.id == str(getattr(local_key, "id", "") or ""))
+                        .first()
+                    )
+                    parsed = _extract_test_response_or_raise(
+                        response=response,
+                        endpoint=local_endpoint,
+                        provider_name=str(local_provider.name),
+                        auth_type=auth_type,
+                        api_key=parse_key or local_key,
+                        db=parse_db,
+                    )
+
+                with create_session() as update_db:
+                    RequestCandidateService.mark_candidate_success(
+                        db=update_db,
+                        candidate_id=record_id,
+                        status_code=200,
+                        latency_ms=elapsed_ms,
+                    )
+
+                if not success_event.is_set():
+                    success_payload.update(
+                        {
+                            "response": parsed,
+                            "candidate_index": candidate_index,
+                            "key_id": str(getattr(local_key, "id", "") or "") or None,
+                        }
+                    )
+                    success_event.set()
+                return {"status": "success"}
+        except asyncio.CancelledError:
+            if started or not success_event.is_set():
+                _mark_concurrent_test_record_cancelled(record_id)
+            return {"status": "cancelled"}
+        except Exception as exc:
+            last_error = exc
+            elapsed_ms = max(0, int((time.perf_counter() - started_at) * 1000)) if started else None
+            status_code = None
+            if isinstance(exc, httpx.HTTPStatusError):
+                status_code = int(exc.response.status_code)
+            elif isinstance(exc, httpx.TimeoutException):
+                status_code = 408
+            elif isinstance(exc, EmbeddedErrorException):
+                status_code = int(exc.error_code or 200)
+
+            loaded = _preloaded.get(candidate_index)
+            if loaded is not None and status_code is not None:
+                await pool_on_error(loaded[0], loaded[2], status_code, exc)
+
+            with create_session() as update_db:
+                RequestCandidateService.mark_candidate_failed(
+                    db=update_db,
+                    candidate_id=record_id,
+                    error_type=type(exc).__name__,
+                    error_message=str(
+                        getattr(exc, "error_message", None)
+                        or getattr(exc, "upstream_response", None)
+                        or exc
+                    ),
+                    status_code=status_code,
+                    latency_ms=elapsed_ms,
+                )
+            return {"status": "failed", "error": exc}
+
+    async def _watch_disconnect() -> bool:
+        while not success_event.is_set():
+            if await is_cancelled():
+                return True
+            await asyncio.sleep(0.1)
+        return False
+
+    tasks = [
+        asyncio.create_task(_worker(candidate_index))
+        for candidate_index, candidate in enumerate(candidates)
+        if not bool(getattr(candidate, "is_skipped", False))
+    ]
+    disconnect_task = asyncio.create_task(_watch_disconnect())
+    pending: set[asyncio.Task[Any]] = set(tasks)
+    pending.add(disconnect_task)
+
+    try:
+        while pending:
+            if pending == {disconnect_task}:
+                disconnect_task.cancel()
+                pending.clear()
+                break
+
+            done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
+            if disconnect_task in done and disconnect_task.result() is True:
+                for task in pending:
+                    task.cancel()
+                break
+
+            for finished in done:
+                if finished is disconnect_task:
+                    continue
+                result = finished.result()
+                if result.get("status") == "success":
+                    for task in pending:
+                        task.cancel()
+                    pending.discard(disconnect_task)
+                    disconnect_task.cancel()
+                    break
+            if success_event.is_set():
+                break
+    finally:
+        await asyncio.gather(*pending, return_exceptions=True)
+        if not disconnect_task.done():
+            disconnect_task.cancel()
+        await asyncio.gather(disconnect_task, return_exceptions=True)
+
+    if success_event.is_set():
+        _cancel_remaining_concurrent_test_records(request_id)
+    elif await is_cancelled():
+        _cancel_remaining_concurrent_test_records(request_id)
+
+    try:
+        db.expire_all()
+        candidate_keys = candidate_recorder.get_candidate_keys(request_id)
+    except Exception:
+        candidate_keys = []
+
+    attempt_count = sum(
+        1
+        for item in candidate_keys
+        if str(getattr(item, "status", "") or "")
+        not in {"skipped", "cancelled", "available", "unused"}
+    )
+    return {
+        "success": success_event.is_set(),
+        "candidate_keys": candidate_keys,
+        "attempt_count": attempt_count,
+        "run_error": last_error,
+        "response": success_payload.get("response"),
+    }
+
+
 def _maybe_mark_test_oauth_key_invalid(
     *,
     db: Session,
@@ -1326,7 +1826,7 @@ def _extract_test_response_or_raise(
     if isinstance(parsed_payload, dict) and "response_body" in parsed_payload:
         parsed_payload = _parse_jsonish(parsed_payload.get("response_body"))
 
-    if isinstance(parsed_payload, dict) and "error" in parsed_payload:
+    if isinstance(parsed_payload, dict) and parsed_payload.get("error"):
         _maybe_mark_test_oauth_key_invalid(
             db=db,
             key=api_key,
@@ -1444,6 +1944,7 @@ def _build_test_attempts_from_candidate_keys(
 @router.post("/test-model-failover")
 async def test_model_failover(
     request: TestModelFailoverRequest,
+    http_request: Request,
     db: Session = Depends(get_db),
     current_user: User = Depends(get_current_user),
 ) -> Any:
@@ -1571,25 +2072,6 @@ async def test_model_failover(
     provider_type = str(getattr(provider, "provider_type", "") or "").lower()
 
     async def _request_func(provider_obj: Any, endpoint: Any, key: Any, candidate: Any) -> Any:
-        effective_proxy = resolve_effective_proxy(
-            getattr(provider_obj, "proxy", None), getattr(key, "proxy", None)
-        )
-        try:
-            api_key_value, auth_config = await _resolve_key_auth(
-                key,
-                provider_obj,
-                provider_proxy_config=effective_proxy,
-            )
-        except _KeyAuthError as e:
-            raise RuntimeError(e.message) from e
-
-        auth_type = str(getattr(key, "auth_type", "api_key") or "api_key").lower()
-        extra_headers = get_extra_headers_from_endpoint(endpoint) or {}
-        if auth_type == "oauth":
-            account_id = (auth_config or {}).get("account_id")
-            if account_id:
-                extra_headers["chatgpt-account-id"] = str(account_id)
-
         effective_model = _resolve_test_effective_model(
             provider=provider,
             candidate=candidate,
@@ -1597,34 +2079,16 @@ async def test_model_failover(
             gm_obj=gm_obj,
             key=key,
         )
-        adapter_class = get_adapter_for_format(endpoint.api_format)
+        response, auth_type = await _execute_test_check(
-        if not adapter_class:
+            provider_obj=provider_obj,
-            raise ValueError(f"Unknown API format: {endpoint.api_format}")
+            endpoint=endpoint,
-
+            key=key,
-        response = await adapter_class.check_endpoint(
+            effective_model=effective_model,
-            None,
+            request_payload=request_payload,
-            endpoint.base_url,
+            request_timeout=request_timeout,
-            api_key_value,
+            provider_type=provider_type,
-            {
-                **request_payload,
-                "model": effective_model,
-            },
-            extra_headers if extra_headers else None,
-            body_rules=getattr(endpoint, "body_rules", None),
-            header_rules=getattr(endpoint, "header_rules", None),
-            db=db,
             user=current_user,
-            provider_name=provider_obj.name,
+            db=db,
-            provider_id=str(provider_obj.id),
-            api_key_id=str(key.id),
-            model_name=effective_model,
-            auth_type=auth_type,
-            provider_type=provider_type if provider_type else None,
-            decrypted_auth_config=auth_config if auth_config else None,
-            provider_endpoint=endpoint,
-            provider_api_key=key,
-            proxy_config=effective_proxy,
-            timeout_seconds=request_timeout,
         )
         return _extract_test_response_or_raise(
             response=response,
@@ -1639,46 +2103,97 @@ async def test_model_failover(
     task_service = TaskService(db)
     exec_result = None
     run_error: Exception | None = None
+    concurrent_result: dict[str, Any] | None = None
+    result_candidates = candidates
+
+    if request.concurrency > 1:
+        result_candidates = _flatten_test_candidates_for_concurrency(candidates)
+
+    candidate_meta_by_pair, candidate_meta_by_index = _build_test_candidate_meta(
+        candidates=result_candidates,
+        provider=provider,
+        request=request,
+        gm_obj=gm_obj,
+    )
+    effective_model_by_candidate_index = {
+        index: str(meta.get("effective_model") or request.model_name)
+        for index, meta in candidate_meta_by_index.items()
+    }
 
     try:
-        exec_result = await task_service.execute_sync_candidates(
+        if request.concurrency > 1:
-            api_format=client_format or "openai:chat",
+            concurrent_result = await _run_concurrent_test(
-            model_name=request.model_name,
+                candidates=result_candidates,
-            candidates=candidates,
+                concurrency=request.concurrency,
-            request_func=_request_func,
+                is_cancelled=http_request.is_disconnected,
-            request_id=request_id,
+                request_id=request_id,
-            current_user=current_user,
+                request_payload=dict(request_payload),
-            user_api_key=None,
+                effective_model_by_candidate_index=effective_model_by_candidate_index,
-            is_stream=False,
+                request_timeout=request_timeout,
-            capability_requirements=None,
+                provider_type=provider_type,
-            request_body_ref={"body": dict(request_payload)},
+                user=current_user,
-            request_headers=None,
+                db=db,
-            request_body=dict(request_payload),
+            )
-            affinity_key=f"provider-test:{provider.id}",
+        else:
-            create_pending_usage=False,
+            exec_result = await task_service.execute_sync_candidates(
-            enable_cache_affinity=False,
+                api_format=client_format or "openai:chat",
-        )
+                model_name=request.model_name,
+                candidates=result_candidates,
+                request_func=_request_func,
+                request_id=request_id,
+                current_user=current_user,
+                user_api_key=None,
+                is_stream=False,
+                capability_requirements=None,
+                request_body_ref={"body": dict(request_payload)},
+                request_headers=None,
+                request_body=dict(request_payload),
+                affinity_key=f"provider-test:{provider.id}",
+                create_pending_usage=False,
+                enable_cache_affinity=False,
+                is_cancelled=http_request.is_disconnected,
+            )
     except Exception as exc:
         run_error = exc
         logger.error("[test-model-failover] Error: {}", exc)
 
     try:
-        candidate_keys = candidate_recorder.get_candidate_keys(request_id)
+        candidate_keys = (
+            list(concurrent_result.get("candidate_keys", []))
+            if concurrent_result is not None
+            else candidate_recorder.get_candidate_keys(request_id)
+        )
     except Exception:
         candidate_keys = list(exec_result.candidate_keys) if exec_result else []
-
-    candidate_meta_by_pair, candidate_meta_by_index = _build_test_candidate_meta(
-        candidates=candidates,
-        provider=provider,
-        request=request,
-        gm_obj=gm_obj,
-    )
     attempts = _build_test_attempts_from_candidate_keys(
         candidate_keys=candidate_keys,
         candidate_meta_by_pair=candidate_meta_by_pair,
         candidate_meta_by_index=candidate_meta_by_index,
     )
-    total_attempts = sum(1 for attempt in attempts if attempt.status != "skipped")
+    total_attempts = (
+        int(exec_result.attempt_count)
+        if exec_result is not None
+        else (
+            int(concurrent_result.get("attempt_count", 0))
+            if concurrent_result is not None
+            else sum(1 for attempt in attempts if attempt.status not in {"skipped", "cancelled"})
+        )
+    )
+
+    if concurrent_result is not None and concurrent_result.get("success"):
+        return TestModelFailoverResponse(
+            success=True,
+            model=request.model_name,
+            provider={"id": str(provider.id), "name": provider.name},
+            attempts=attempts,
+            total_candidates=len(result_candidates),
+            total_attempts=total_attempts,
+            data={
+                "stream": True,
+                "response": concurrent_result.get("response"),
+            },
+            error=None,
+        ).model_dump()
 
     if exec_result and exec_result.success:
         return TestModelFailoverResponse(
@@ -1686,7 +2201,7 @@ async def test_model_failover(
             model=request.model_name,
             provider={"id": str(provider.id), "name": provider.name},
             attempts=attempts,
-            total_candidates=len(candidates),
+            total_candidates=len(result_candidates),
             total_attempts=exec_result.attempt_count,
             data={
                 "stream": True,
@@ -1703,6 +2218,10 @@ async def test_model_failover(
             error_message = str(run_error.upstream_response)[:500]
         if not error_message:
             error_message = str(run_error)
+    if not error_message and concurrent_result is not None and concurrent_result.get("run_error"):
+        error_message = str(concurrent_result.get("run_error"))
+    if not error_message and exec_result is not None and exec_result.error_message:
+        error_message = str(exec_result.error_message)
     if not error_message:
         failed_attempt = next(
             (attempt for attempt in reversed(attempts) if attempt.error_message),
@@ -1717,7 +2236,7 @@ async def test_model_failover(
         model=request.model_name,
         provider={"id": str(provider.id), "name": provider.name},
         attempts=attempts,
-        total_candidates=len(candidates),
+        total_candidates=len(result_candidates),
         total_attempts=total_attempts,
         error=str(error_message)[:500],
     ).model_dump()

src/services/candidate/failover.py

@@ -181,6 +181,147 @@ class FailoverEngine:
             )
             await asyncio.sleep(backoff_seconds)
 
+    async def _check_cancellation(
+        self,
+        is_cancelled: Callable[[], Awaitable[bool]] | None,
+    ) -> bool:
+        if is_cancelled is None:
+            return False
+        try:
+            return bool(await is_cancelled())
+        except Exception:
+            return False
+
+    def _mark_remaining_cancelled(
+        self,
+        *,
+        candidate_record_map: dict[tuple[int, int], str] | None,
+        candidates: list[ProviderCandidate],
+        from_candidate_idx: int,
+        from_retry_idx: int,
+        retry_policy: RetryPolicy,
+    ) -> None:
+        if not candidate_record_map:
+            return
+
+        now = datetime.now(timezone.utc)
+        updated = False
+        for candidate_idx, cand in enumerate(candidates):
+            if candidate_idx < from_candidate_idx:
+                continue
+            max_retries = self._get_max_retries(cand, retry_policy)
+            for retry_idx in range(max_retries):
+                if candidate_idx == from_candidate_idx and retry_idx < from_retry_idx:
+                    continue
+                record_id = candidate_record_map.get((candidate_idx, retry_idx))
+                if not record_id:
+                    continue
+                self.db.execute(
+                    update(RequestCandidate)
+                    .where(RequestCandidate.id == record_id)
+                    .where(RequestCandidate.status.in_(["available", "pending"]))
+                    .values(
+                        status="cancelled",
+                        status_code=499,
+                        error_message="cancelled_by_client",
+                        finished_at=now,
+                    )
+                )
+                updated = True
+
+        if updated:
+            self.db.commit()
+
+    def _append_cancelled_fallback_candidate_keys(
+        self,
+        *,
+        fallback: list[CandidateKey],
+        candidates: list[ProviderCandidate],
+        from_candidate_idx: int,
+        from_retry_idx: int,
+        retry_policy: RetryPolicy,
+    ) -> None:
+        existing = {(item.candidate_index, item.retry_index) for item in fallback}
+        for candidate_idx, cand in enumerate(candidates):
+            if candidate_idx < from_candidate_idx:
+                continue
+            max_retries = self._get_max_retries(cand, retry_policy)
+            for retry_idx in range(max_retries):
+                if candidate_idx == from_candidate_idx and retry_idx < from_retry_idx:
+                    continue
+                key = (candidate_idx, retry_idx)
+                if key in existing:
+                    continue
+                original_key = getattr(cand, "key", None)
+                original_pool_key_index = getattr(cand, "_pool_key_index", 0)
+                if isinstance(cand, PoolCandidate) and cand.pool_keys:
+                    retry_slots_per_key = self._get_pool_key_max_retries(cand, retry_policy)
+                    pool_key_index = min(retry_idx // retry_slots_per_key, len(cand.pool_keys) - 1)
+                    cand.key = cand.pool_keys[pool_key_index]
+                    cand._pool_key_index = pool_key_index
+                fallback.append(
+                    self._make_candidate_key(
+                        candidate=cand,
+                        candidate_index=candidate_idx,
+                        retry_index=retry_idx,
+                        status="cancelled",
+                        error_message="cancelled_by_client",
+                        status_code=499,
+                    )
+                )
+                if isinstance(cand, PoolCandidate):
+                    cand.key = original_key
+                    cand._pool_key_index = original_pool_key_index
+                existing.add(key)
+
+    async def _maybe_cancel_execution(
+        self,
+        *,
+        is_cancelled: Callable[[], Awaitable[bool]] | None,
+        candidate_record_map: dict[tuple[int, int], str] | None,
+        candidate_keys_fallback: list[CandidateKey],
+        candidates: list[ProviderCandidate],
+        from_candidate_idx: int,
+        from_retry_idx: int,
+        retry_policy: RetryPolicy,
+        request_id: str | None,
+        attempt_count: int,
+    ) -> ExecutionResult | None:
+        if not await self._check_cancellation(is_cancelled):
+            return None
+
+        logger.info(
+            "[FailoverEngine] Request cancelled by client at candidate_index={}, retry_index={}",
+            from_candidate_idx,
+            from_retry_idx,
+        )
+        self._mark_remaining_cancelled(
+            candidate_record_map=candidate_record_map,
+            candidates=candidates,
+            from_candidate_idx=from_candidate_idx,
+            from_retry_idx=from_retry_idx,
+            retry_policy=retry_policy,
+        )
+        self._append_cancelled_fallback_candidate_keys(
+            fallback=candidate_keys_fallback,
+            candidates=candidates,
+            from_candidate_idx=from_candidate_idx,
+            from_retry_idx=from_retry_idx,
+            retry_policy=retry_policy,
+        )
+        return ExecutionResult(
+            success=False,
+            error_type="cancelled",
+            error_message="cancelled_by_client",
+            last_status_code=499,
+            candidate_keys=self._get_candidate_keys(
+                request_id=request_id,
+                fallback=candidate_keys_fallback,
+                candidates=candidates,
+            ),
+            attempt_count=attempt_count,
+        )
+
     async def execute(
         self,
         *,
@@ -201,6 +342,7 @@ class FailoverEngine:
             ]
             | None
         ) = None,
+        is_cancelled: Callable[[], Awaitable[bool]] | None = None,
     ) -> ExecutionResult:
         """
         Execute candidate traversal + retry + failover.
@@ -229,6 +371,20 @@ class FailoverEngine:
             max_attempts = computed
 
         for candidate_index, candidate in enumerate(candidates):
+            cancelled_result = await self._maybe_cancel_execution(
+                is_cancelled=is_cancelled,
+                candidate_record_map=candidate_record_map,
+                candidate_keys_fallback=candidate_keys_fallback,
+                candidates=candidates,
+                from_candidate_idx=candidate_index,
+                from_retry_idx=0,
+                retry_policy=retry_policy,
+                request_id=request_id,
+                attempt_count=attempt_count,
+            )
+            if cancelled_result is not None:
+                return cancelled_result
+
             should_skip, skip_reason = self._should_skip(candidate, skip_policy)
             if should_skip:
                 # PRE_EXPAND: mark all retry slots skipped.
@@ -279,6 +435,7 @@ class FailoverEngine:
                         max_attempts=max_attempts,
                         execution_error_handler=execution_error_handler,
                         consecutive_failures=consecutive_failures,
+                        is_cancelled=is_cancelled,
                     )
                 )
                 if pool_result is not None:
@@ -288,6 +445,20 @@ class FailoverEngine:
             max_retries = self._get_max_retries(candidate, retry_policy)
             retry_index = 0
             while retry_index < max_retries:
+                cancelled_result = await self._maybe_cancel_execution(
+                    is_cancelled=is_cancelled,
+                    candidate_record_map=candidate_record_map,
+                    candidate_keys_fallback=candidate_keys_fallback,
+                    candidates=candidates,
+                    from_candidate_idx=candidate_index,
+                    from_retry_idx=retry_index,
+                    retry_policy=retry_policy,
+                    request_id=request_id,
+                    attempt_count=attempt_count,
+                )
+                if cancelled_result is not None:
+                    return cancelled_result
+
                 attempt_count += 1
 
                 # Resolve/create record_id
@@ -456,6 +627,7 @@ class FailoverEngine:
         consecutive_failures: int,
         max_attempts: int | None,
         execution_error_handler: Any,
+        is_cancelled: Callable[[], Awaitable[bool]] | None,
     ) -> tuple[ExecutionResult | None, int, int, int | None]:
         """Execute a PoolCandidate with in-pool key failover."""
         last_status_code: int | None = None
@@ -463,6 +635,20 @@ class FailoverEngine:
 
         for key_index, pool_key in enumerate(candidate.pool_keys or []):
             base_retry_index = key_index * retry_slots_per_key
+            cancelled_result = await self._maybe_cancel_execution(
+                is_cancelled=is_cancelled,
+                candidate_record_map=candidate_record_map,
+                candidate_keys_fallback=candidate_keys_fallback,
+                candidates=candidates,
+                from_candidate_idx=candidate_index,
+                from_retry_idx=base_retry_index,
+                retry_policy=retry_policy,
+                request_id=request_id,
+                attempt_count=attempt_count,
+            )
+            if cancelled_result is not None:
+                return cancelled_result, attempt_count, consecutive_failures, last_status_code
+
             candidate.key = pool_key
             candidate._pool_key_index = key_index
             candidate.mapping_matched_model = getattr(pool_key, "_pool_mapping_matched_model", None)
@@ -507,8 +693,22 @@ class FailoverEngine:
             max_retries_for_key = retry_slots_per_key
             retry_index = 0
             while retry_index < max_retries_for_key:
-                attempt_count += 1
                 composite_retry_index = base_retry_index + retry_index
+                cancelled_result = await self._maybe_cancel_execution(
+                    is_cancelled=is_cancelled,
+                    candidate_record_map=candidate_record_map,
+                    candidate_keys_fallback=candidate_keys_fallback,
+                    candidates=candidates,
+                    from_candidate_idx=candidate_index,
+                    from_retry_idx=composite_retry_index,
+                    retry_policy=retry_policy,
+                    request_id=request_id,
+                    attempt_count=attempt_count,
+                )
+                if cancelled_result is not None:
+                    return cancelled_result, attempt_count, consecutive_failures, last_status_code
+
+                attempt_count += 1
 
                 record_id = None
                 if candidate_record_map:

src/services/provider/pool/account_state.py

@@ -9,6 +9,8 @@ from __future__ import annotations
 from dataclasses import dataclass
 from typing import Any
 
+from src.services.provider_keys.quota_reader import get_quota_reader
+
 OAUTH_ACCOUNT_BLOCK_PREFIX = "[ACCOUNT_BLOCK] "
 OAUTH_REFRESH_FAILED_PREFIX = "[REFRESH_FAILED] "
 OAUTH_EXPIRED_PREFIX = "[OAUTH_EXPIRED] "
@@ -69,7 +71,7 @@ def _classify_block_reason(text: str) -> tuple[str, str]:
         return "oauth_expired", "Token 失效"
     if any(kw in lowered for kw in _KEYWORDS_VERIFICATION):
         return "account_verification", "需要验证"
-    if 'deactivated_workspace' in lowered:
+    if "deactivated_workspace" in lowered:
         return "workspace_deactivated", "工作区停用"
     if any(kw in lowered for kw in _KEYWORDS_DISABLED):
         return "account_disabled", "账号停用"
@@ -130,30 +132,13 @@ def _resolve_from_metadata(
         if isinstance(maybe_bucket, dict):
             provider_bucket = maybe_bucket
 
-    if (
+    quota_block = get_quota_reader(normalized_provider, upstream_metadata).account_block()
-        normalized_provider == "kiro"
+    if quota_block.blocked:
-        and provider_bucket
-        and _is_truthy_flag(provider_bucket.get("is_banned"))
-    ):
-        reason = _extract_reason(provider_bucket, "ban_reason", "reason", "message")
         return PoolAccountState(
             blocked=True,
-            code="account_banned",
+            code=quota_block.code,
-            label="账号封禁",
+            label=quota_block.label,
-            reason=reason or "Kiro 账号已封禁",
+            reason=quota_block.reason,
-        )
-
-    if (
-        normalized_provider == "antigravity"
-        and provider_bucket
-        and _is_truthy_flag(provider_bucket.get("is_forbidden"))
-    ):
-        reason = _extract_reason(provider_bucket, "forbidden_reason", "reason", "message")
-        return PoolAccountState(
-            blocked=True,
-            code="account_forbidden",
-            label="访问受限",
-            reason=reason or "Antigravity 账户访问受限",
         )
 
     for source in (provider_bucket, upstream_metadata):

src/services/provider/pool/dimensions/_helpers.py

@@ -3,9 +3,11 @@
 from __future__ import annotations
 
 import math
-import time
 from typing import Any
 
+from src.core.provider_types import ProviderType
+from src.services.provider_keys.quota_reader import get_quota_reader
+
 
 def safe_float(value: Any) -> float | None:
     try:
@@ -98,26 +100,10 @@ def extract_plan_type(key_obj: Any) -> str | None:
         return direct
 
     metadata = safe_metadata(key_obj)
-    codex = metadata.get("codex")
+    for provider_type in (ProviderType.CODEX, ProviderType.KIRO, ProviderType.ANTIGRAVITY):
-    if isinstance(codex, dict):
+        plan_type = get_quota_reader(provider_type, metadata).plan_type()
-        codex_plan = normalize_plan(codex.get("plan_type"))
+        if plan_type:
-        if codex_plan:
+            return plan_type
-            return codex_plan
-
-    kiro = metadata.get("kiro")
-    if isinstance(kiro, dict):
-        subscription_title = normalize_plan(kiro.get("subscription_title"))
-        if subscription_title:
-            # Normalize common Kiro labels into free/team buckets used by free_team_first.
-            if "team" in subscription_title:
-                return "team"
-            if "free" in subscription_title:
-                return "free"
-            if "pro" in subscription_title:
-                return "pro"
-            if "plus" in subscription_title:
-                return "plus"
-            return subscription_title
 
     return None
 
@@ -126,19 +112,11 @@ def extract_reset_seconds(key_obj: Any) -> float | None:
     metadata = safe_metadata(key_obj)
     candidates: list[float] = []
 
-    codex = metadata.get("codex")
+    for provider_type in (ProviderType.CODEX, ProviderType.KIRO, ProviderType.ANTIGRAVITY):
-    if isinstance(codex, dict):
+        reset_seconds = get_quota_reader(provider_type, metadata).reset_seconds()
-        for field in ("secondary_reset_seconds", "primary_reset_seconds"):
+        if reset_seconds is None:
-            parsed = safe_float(codex.get(field))
+            continue
-            if parsed is None or parsed < 0:
+        candidates.append(reset_seconds)
-                continue
-            candidates.append(parsed)
-
-    kiro = metadata.get("kiro")
-    if isinstance(kiro, dict):
-        next_reset_at = safe_float(kiro.get("next_reset_at"))
-        if next_reset_at is not None and next_reset_at > 0:
-            candidates.append(max(0.0, next_reset_at - time.time()))
 
     if not candidates:
         return None
@@ -148,41 +126,10 @@ def extract_reset_seconds(key_obj: Any) -> float | None:
 def extract_usage_ratio(key_obj: Any) -> float | None:
     metadata = safe_metadata(key_obj)
 
-    codex = metadata.get("codex")
+    for provider_type in (ProviderType.CODEX, ProviderType.KIRO, ProviderType.ANTIGRAVITY):
-    if isinstance(codex, dict):
+        usage_ratio = get_quota_reader(provider_type, metadata).usage_ratio()
-        codex_values: list[float] = []
+        if usage_ratio is not None:
-        for field in ("primary_used_percent", "secondary_used_percent"):
+            return usage_ratio
-            parsed = safe_float(codex.get(field))
-            if parsed is None:
-                continue
-            codex_values.append(max(0.0, min(parsed, 100.0)) / 100.0)
-        if codex_values:
-            return sum(codex_values) / len(codex_values)
-
-    kiro = metadata.get("kiro")
-    if isinstance(kiro, dict):
-        parsed = safe_float(kiro.get("usage_percentage"))
-        if parsed is not None:
-            return max(0.0, min(parsed, 100.0)) / 100.0
-
-    antigravity = metadata.get("antigravity")
-    if isinstance(antigravity, dict):
-        quota_by_model = antigravity.get("quota_by_model")
-        if isinstance(quota_by_model, dict):
-            usage_values: list[float] = []
-            for model_info in quota_by_model.values():
-                if not isinstance(model_info, dict):
-                    continue
-                used_percent = safe_float(model_info.get("used_percent"))
-                if used_percent is None:
-                    remaining_fraction = safe_float(model_info.get("remaining_fraction"))
-                    if remaining_fraction is not None:
-                        used_percent = (1.0 - remaining_fraction) * 100.0
-                if used_percent is None:
-                    continue
-                usage_values.append(max(0.0, min(used_percent, 100.0)) / 100.0)
-            if usage_values:
-                return sum(usage_values) / len(usage_values)
 
     return None
 

src/services/provider_keys/key_quota_service.py

@@ -26,6 +26,13 @@ from src.services.provider_keys.quota_refresh import (
 QuotaRefreshHandler = Callable[..., Awaitable[dict]]
 
 
+_QUOTA_REFRESH_HANDLERS: dict[str, QuotaRefreshHandler] = {
+    ProviderType.CODEX: refresh_codex_key_quota,
+    ProviderType.ANTIGRAVITY: refresh_antigravity_key_quota,
+    ProviderType.KIRO: refresh_kiro_key_quota,
+}
+
+
 def _normalize_api_format(api_format: Any) -> str:
     """规范化 api_format，兼容大小写和首尾空白。"""
     if not isinstance(api_format, str):
@@ -55,12 +62,9 @@ def _select_refresh_endpoint(provider: Provider, provider_type: str) -> Provider
 
 def _resolve_quota_refresh_handler(provider_type: str) -> QuotaRefreshHandler:
     """按 provider 类型返回刷新策略。"""
-    if provider_type == ProviderType.CODEX:
+    handler = _QUOTA_REFRESH_HANDLERS.get(provider_type)
-        return refresh_codex_key_quota
+    if handler is not None:
-    if provider_type == ProviderType.ANTIGRAVITY:
+        return handler
-        return refresh_antigravity_key_quota
-    if provider_type == ProviderType.KIRO:
-        return refresh_kiro_key_quota
     raise InvalidRequestException("仅支持 Codex / Antigravity / Kiro 类型的 Provider 刷新限额")
 
 

src/services/provider_keys/quota_reader.py

@@ -0,0 +1,439 @@
+"""Unified quota readers for provider key upstream metadata."""
+
+from __future__ import annotations
+
+import math
+import time
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any
+
+from src.core.provider_types import ProviderType, normalize_provider_type
+
+
+def _to_float(value: Any) -> float | None:
+    try:
+        parsed = float(value)
+    except (TypeError, ValueError):
+        return None
+    if math.isnan(parsed) or math.isinf(parsed):
+        return None
+    return parsed
+
+
+def _normalize_plan(value: Any) -> str | None:
+    if not isinstance(value, str):
+        return None
+    normalized = value.strip().lower()
+    return normalized or None
+
+
+def _is_truthy_flag(value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, (int, float)):
+        return value != 0
+    if isinstance(value, str):
+        normalized = value.strip().lower()
+        return normalized in {"1", "true", "yes", "y"}
+    return False
+
+
+def _extract_reason(source: dict[str, Any], *fields: str) -> str | None:
+    for field in fields:
+        value = source.get(field)
+        if not isinstance(value, str):
+            continue
+        text = value.strip()
+        if text:
+            return text
+    return None
+
+
+def _pct_is_exhausted(value: Any) -> bool:
+    pct = _to_float(value)
+    if pct is None:
+        return False
+    return pct >= 100.0 - 1e-6
+
+
+def _format_percent(value: float) -> str:
+    clamped = max(0.0, min(value, 100.0))
+    return f"{clamped:.1f}%"
+
+
+def _format_quota_value(value: float) -> str:
+    rounded = round(value)
+    if abs(value - rounded) < 1e-6:
+        return str(rounded)
+    return f"{value:.1f}"
+
+
+def _format_reset_after(seconds_raw: Any) -> str | None:
+    seconds = _to_float(seconds_raw)
+    if seconds is None:
+        return None
+
+    total_seconds = int(seconds)
+    if total_seconds <= 0:
+        return "已重置"
+
+    days = total_seconds // 86400
+    hours = (total_seconds % 86400) // 3600
+    minutes = (total_seconds % 3600) // 60
+
+    if days > 0:
+        return f"{days}天{hours}小时后重置"
+    if hours > 0:
+        return f"{hours}小时{minutes}分钟后重置"
+    if minutes > 0:
+        return f"{minutes}分钟后重置"
+    return "即将重置"
+
+
+@dataclass(frozen=True, slots=True)
+class QuotaExhaustedResult:
+    exhausted: bool
+    reason: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class AccountBlockResult:
+    blocked: bool
+    code: str | None = None
+    label: str | None = None
+    reason: str | None = None
+
+
+class PoolQuotaReader(ABC):
+    """Read-only view over one provider namespace in upstream_metadata."""
+
+    namespace: str | None = None
+
+    def __init__(self, data: dict[str, Any] | None) -> None:
+        self._data: dict[str, Any] = data if isinstance(data, dict) else {}
+
+    @abstractmethod
+    def is_exhausted(self, model_name: str | None = None) -> QuotaExhaustedResult:
+        """Return whether this key/model should be skipped for quota exhaustion."""
+
+    @abstractmethod
+    def usage_ratio(self) -> float | None:
+        """Return usage ratio within [0, 1], when available."""
+
+    @abstractmethod
+    def plan_type(self) -> str | None:
+        """Return normalized plan type, when available."""
+
+    @abstractmethod
+    def reset_seconds(self) -> float | None:
+        """Return seconds until next reset, when available."""
+
+    @abstractmethod
+    def account_block(self) -> AccountBlockResult:
+        """Return account-level block state derived from metadata."""
+
+    @abstractmethod
+    def display_summary(self) -> str | None:
+        """Return admin-facing quota summary string."""
+
+    def updated_at(self) -> int | None:
+        updated_at = _to_float(self._data.get("updated_at"))
+        if updated_at is None or updated_at <= 0:
+            return None
+        if updated_at > 1_000_000_000_000:
+            updated_at /= 1000
+        return int(updated_at)
+
+
+class NullQuotaReader(PoolQuotaReader):
+    def is_exhausted(self, model_name: str | None = None) -> QuotaExhaustedResult:
+        _ = model_name
+        return QuotaExhaustedResult(exhausted=False)
+
+    def usage_ratio(self) -> float | None:
+        return None
+
+    def plan_type(self) -> str | None:
+        return None
+
+    def reset_seconds(self) -> float | None:
+        return None
+
+    def account_block(self) -> AccountBlockResult:
+        return AccountBlockResult(blocked=False)
+
+    def display_summary(self) -> str | None:
+        return None
+
+
+class CodexQuotaReader(PoolQuotaReader):
+    namespace = "codex"
+
+    def is_exhausted(self, model_name: str | None = None) -> QuotaExhaustedResult:
+        _ = model_name
+        exhausted_parts: list[str] = []
+        if _pct_is_exhausted(self._data.get("primary_used_percent")):
+            exhausted_parts.append("周限额剩余 0%")
+        if _pct_is_exhausted(self._data.get("secondary_used_percent")):
+            exhausted_parts.append("5H 限额剩余 0%")
+        if exhausted_parts:
+            return QuotaExhaustedResult(True, "Codex " + "，".join(exhausted_parts))
+        return QuotaExhaustedResult(False)
+
+    def usage_ratio(self) -> float | None:
+        values: list[float] = []
+        for field in ("primary_used_percent", "secondary_used_percent"):
+            parsed = _to_float(self._data.get(field))
+            if parsed is None:
+                continue
+            values.append(max(0.0, min(parsed, 100.0)) / 100.0)
+        if not values:
+            return None
+        return sum(values) / len(values)
+
+    def plan_type(self) -> str | None:
+        return _normalize_plan(self._data.get("plan_type"))
+
+    def reset_seconds(self) -> float | None:
+        candidates: list[float] = []
+        for field in ("secondary_reset_seconds", "primary_reset_seconds"):
+            parsed = _to_float(self._data.get(field))
+            if parsed is None or parsed < 0:
+                continue
+            candidates.append(parsed)
+        if not candidates:
+            return None
+        return min(candidates)
+
+    def account_block(self) -> AccountBlockResult:
+        if not _is_truthy_flag(self._data.get("account_disabled")):
+            return AccountBlockResult(blocked=False)
+        reason = _extract_reason(self._data, "forbidden_reason", "ban_reason", "reason", "message")
+        return AccountBlockResult(
+            blocked=True,
+            code="account_forbidden",
+            label="访问受限",
+            reason=reason or "账号访问受限",
+        )
+
+    def display_summary(self) -> str | None:
+        parts: list[str] = []
+
+        primary_used = _to_float(self._data.get("primary_used_percent"))
+        if primary_used is not None:
+            part = f"周剩余 {_format_percent(100.0 - primary_used)}"
+            reset_text = _format_reset_after(self._data.get("primary_reset_seconds"))
+            if reset_text:
+                part = f"{part} ({reset_text})"
+            parts.append(part)
+
+        secondary_used = _to_float(self._data.get("secondary_used_percent"))
+        if secondary_used is not None:
+            part = f"5H剩余 {_format_percent(100.0 - secondary_used)}"
+            reset_text = _format_reset_after(self._data.get("secondary_reset_seconds"))
+            if reset_text:
+                part = f"{part} ({reset_text})"
+            parts.append(part)
+
+        if parts:
+            return " | ".join(parts)
+
+        has_credits = self._data.get("has_credits")
+        credits_balance = _to_float(self._data.get("credits_balance"))
+        if has_credits is True and credits_balance is not None:
+            return f"积分 {credits_balance:.2f}"
+        if has_credits is True:
+            return "有积分"
+        return None
+
+
+class KiroQuotaReader(PoolQuotaReader):
+    namespace = "kiro"
+
+    def is_exhausted(self, model_name: str | None = None) -> QuotaExhaustedResult:
+        _ = model_name
+        remaining = _to_float(self._data.get("remaining"))
+        if remaining is not None and remaining <= 0.0:
+            return QuotaExhaustedResult(True, "Kiro 账号配额剩余 0")
+        return QuotaExhaustedResult(False)
+
+    def usage_ratio(self) -> float | None:
+        parsed = _to_float(self._data.get("usage_percentage"))
+        if parsed is None:
+            return None
+        return max(0.0, min(parsed, 100.0)) / 100.0
+
+    def plan_type(self) -> str | None:
+        subscription_title = _normalize_plan(self._data.get("subscription_title"))
+        if not subscription_title:
+            return None
+        if "team" in subscription_title:
+            return "team"
+        if "free" in subscription_title:
+            return "free"
+        if "pro" in subscription_title:
+            return "pro"
+        if "plus" in subscription_title:
+            return "plus"
+        return subscription_title
+
+    def reset_seconds(self) -> float | None:
+        next_reset_at = _to_float(self._data.get("next_reset_at"))
+        if next_reset_at is None or next_reset_at <= 0:
+            return None
+        return max(0.0, next_reset_at - time.time())
+
+    def account_block(self) -> AccountBlockResult:
+        if not _is_truthy_flag(self._data.get("is_banned")):
+            return AccountBlockResult(blocked=False)
+        reason = _extract_reason(self._data, "ban_reason", "reason", "message")
+        return AccountBlockResult(
+            blocked=True,
+            code="account_banned",
+            label="账号封禁",
+            reason=reason or "Kiro 账号已封禁",
+        )
+
+    def display_summary(self) -> str | None:
+        if self._data.get("is_banned") is True:
+            return "账号已封禁"
+
+        usage_percentage = _to_float(self._data.get("usage_percentage"))
+        if usage_percentage is not None:
+            remaining = 100.0 - usage_percentage
+            current_usage = _to_float(self._data.get("current_usage"))
+            usage_limit = _to_float(self._data.get("usage_limit"))
+            if current_usage is not None and usage_limit is not None and usage_limit > 0:
+                return (
+                    f"剩余 {_format_percent(remaining)} "
+                    f"({_format_quota_value(current_usage)}/{_format_quota_value(usage_limit)})"
+                )
+            return f"剩余 {_format_percent(remaining)}"
+
+        remaining = _to_float(self._data.get("remaining"))
+        usage_limit = _to_float(self._data.get("usage_limit"))
+        if remaining is not None and usage_limit is not None and usage_limit > 0:
+            return f"剩余 {_format_quota_value(remaining)}/{_format_quota_value(usage_limit)}"
+        return None
+
+
+class AntigravityQuotaReader(PoolQuotaReader):
+    namespace = "antigravity"
+
+    def _quota_by_model(self) -> dict[str, Any]:
+        quota_by_model = self._data.get("quota_by_model")
+        if not isinstance(quota_by_model, dict):
+            return {}
+        return quota_by_model
+
+    def _used_percent(self, model_info: dict[str, Any]) -> float | None:
+        used_percent = _to_float(model_info.get("used_percent"))
+        if used_percent is not None:
+            return max(0.0, min(used_percent, 100.0))
+        remaining_fraction = _to_float(model_info.get("remaining_fraction"))
+        if remaining_fraction is None:
+            return None
+        return max(0.0, min((1.0 - remaining_fraction) * 100.0, 100.0))
+
+    def is_exhausted(self, model_name: str | None = None) -> QuotaExhaustedResult:
+        if not model_name:
+            return QuotaExhaustedResult(False)
+        model_quota = self._quota_by_model().get(model_name)
+        if not isinstance(model_quota, dict):
+            return QuotaExhaustedResult(False)
+
+        remaining_fraction = _to_float(model_quota.get("remaining_fraction"))
+        if remaining_fraction is not None and remaining_fraction <= 0.0:
+            return QuotaExhaustedResult(True, f"Antigravity 模型 {model_name} 配额剩余 0%")
+        if _pct_is_exhausted(model_quota.get("used_percent")):
+            return QuotaExhaustedResult(True, f"Antigravity 模型 {model_name} 配额剩余 0%")
+        return QuotaExhaustedResult(False)
+
+    def usage_ratio(self) -> float | None:
+        usage_values: list[float] = []
+        for model_info in self._quota_by_model().values():
+            if not isinstance(model_info, dict):
+                continue
+            used_percent = self._used_percent(model_info)
+            if used_percent is None:
+                continue
+            usage_values.append(used_percent / 100.0)
+        if not usage_values:
+            return None
+        return sum(usage_values) / len(usage_values)
+
+    def plan_type(self) -> str | None:
+        return None
+
+    def reset_seconds(self) -> float | None:
+        return None
+
+    def account_block(self) -> AccountBlockResult:
+        if not _is_truthy_flag(self._data.get("is_forbidden")):
+            return AccountBlockResult(blocked=False)
+        reason = _extract_reason(self._data, "forbidden_reason", "reason", "message")
+        return AccountBlockResult(
+            blocked=True,
+            code="account_forbidden",
+            label="访问受限",
+            reason=reason or "Antigravity 账户访问受限",
+        )
+
+    def display_summary(self) -> str | None:
+        if self._data.get("is_forbidden") is True:
+            return "访问受限"
+
+        remaining_list: list[float] = []
+        for raw_info in self._quota_by_model().values():
+            if not isinstance(raw_info, dict):
+                continue
+            used_percent = self._used_percent(raw_info)
+            if used_percent is None:
+                continue
+            remaining_list.append(max(0.0, min(100.0 - used_percent, 100.0)))
+
+        if not remaining_list:
+            return None
+
+        min_remaining = min(remaining_list)
+        if len(remaining_list) == 1:
+            return f"剩余 {_format_percent(min_remaining)}"
+        return f"最低剩余 {_format_percent(min_remaining)} ({len(remaining_list)} 模型)"
+
+
+_READER_CLASSES: dict[str, type[PoolQuotaReader]] = {
+    ProviderType.CODEX: CodexQuotaReader,
+    ProviderType.KIRO: KiroQuotaReader,
+    ProviderType.ANTIGRAVITY: AntigravityQuotaReader,
+}
+
+
+def get_quota_reader(provider_type: str | None, upstream_metadata: Any) -> PoolQuotaReader:
+    """Return a quota reader for one provider namespace in upstream_metadata."""
+
+    normalized_type = normalize_provider_type(provider_type)
+    reader_cls = _READER_CLASSES.get(normalized_type)
+    if reader_cls is None or not isinstance(upstream_metadata, dict):
+        return NullQuotaReader(None)
+
+    namespace = reader_cls.namespace
+    if not namespace:
+        return NullQuotaReader(None)
+
+    data = upstream_metadata.get(namespace)
+    if not isinstance(data, dict):
+        return NullQuotaReader(None)
+    return reader_cls(data)
+
+
+__all__ = [
+    "AccountBlockResult",
+    "AntigravityQuotaReader",
+    "CodexQuotaReader",
+    "KiroQuotaReader",
+    "NullQuotaReader",
+    "PoolQuotaReader",
+    "QuotaExhaustedResult",
+    "get_quota_reader",
+]

src/services/scheduling/quota_skipper.py

@@ -1,26 +1,7 @@
 from __future__ import annotations
 
-from src.core.provider_types import ProviderType, normalize_provider_type
 from src.models.database import ProviderAPIKey
-
+from src.services.provider_keys.quota_reader import get_quota_reader
-
-def _pct_is_exhausted(value: object) -> bool:
-    """Return True when used_percent indicates 0% remaining."""
-    try:
-        pct = float(value)  # type: ignore[arg-type]
-    except (TypeError, ValueError):
-        return False
-    # Some upstreams may return values slightly above 100 due to rounding.
-    return pct >= 100.0 - 1e-6
-
-
-def _float_or_none(value: object) -> float | None:
-    try:
-        if value is None:
-            return None
-        return float(value)  # type: ignore[arg-type]
-    except (TypeError, ValueError):
-        return None
 
 
 def is_key_quota_exhausted(
@@ -29,72 +10,8 @@ def is_key_quota_exhausted(
     *,
     model_name: str,
 ) -> tuple[bool, str | None]:
-    """Check ProviderAPIKey.upstream_metadata quota and decide whether to skip.
+    """Check ProviderAPIKey.upstream_metadata quota and decide whether to skip."""
 
-    Requirements:
+    reader = get_quota_reader(provider_type, getattr(key, "upstream_metadata", None))
-    - Kiro: account-level quota. When remaining == 0, skip this key; allow again when remaining > 0.
+    result = reader.is_exhausted(model_name)
-    - Codex: only consider weekly quota + 5H quota.
+    return result.exhausted, result.reason
-             If either remaining is 0%, skip this key.
-    - Antigravity: quota is per-model; do not disable the account.
-                   When the requested model's quota is 0%, skip this key.
-    """
-
-    pt = normalize_provider_type(provider_type)
-
-    upstream = getattr(key, "upstream_metadata", None) or {}
-    if not isinstance(upstream, dict):
-        return False, None
-
-    if pt == ProviderType.KIRO:
-        kiro_meta = upstream.get("kiro")
-        if not isinstance(kiro_meta, dict):
-            return False, None
-
-        remaining = _float_or_none(kiro_meta.get("remaining"))
-
-        if remaining is not None and remaining <= 0.0:
-            return True, "Kiro 账号配额剩余 0"
-
-        return False, None
-
-    if pt == ProviderType.CODEX:
-        codex_meta = upstream.get("codex")
-        if not isinstance(codex_meta, dict):
-            return False, None
-
-        weekly_used = codex_meta.get("primary_used_percent")
-        five_hour_used = codex_meta.get("secondary_used_percent")
-
-        exhausted_parts: list[str] = []
-        if _pct_is_exhausted(weekly_used):
-            exhausted_parts.append("周限额剩余 0%")
-        if _pct_is_exhausted(five_hour_used):
-            exhausted_parts.append("5H 限额剩余 0%")
-
-        if exhausted_parts:
-            return True, "Codex " + "，".join(exhausted_parts)
-
-        return False, None
-
-    if pt == ProviderType.ANTIGRAVITY:
-        ag_meta = upstream.get("antigravity")
-        if not isinstance(ag_meta, dict):
-            return False, None
-
-        quota_by_model = ag_meta.get("quota_by_model")
-        if not isinstance(quota_by_model, dict):
-            return False, None
-
-        model_quota = quota_by_model.get(model_name)
-        if not isinstance(model_quota, dict):
-            return False, None
-
-        remaining_fraction = _float_or_none(model_quota.get("remaining_fraction"))
-        if remaining_fraction is not None and remaining_fraction <= 0.0:
-            return True, f"Antigravity 模型 {model_name} 配额剩余 0%"
-        if _pct_is_exhausted(model_quota.get("used_percent")):
-            return True, f"Antigravity 模型 {model_name} 配额剩余 0%"
-
-        return False, None
-
-    return False, None

src/services/task/service.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import re
-from collections.abc import Callable
+from collections.abc import Awaitable, Callable
 from typing import Any
 from uuid import uuid4
 
@@ -56,6 +56,47 @@ _SENSITIVE_PATTERN = re.compile(
 )
 
 
+async def pool_on_error(
+    provider: Any,
+    key: Any,
+    status_code: int,
+    cause: Any,
+) -> None:
+    """Notify the pool manager about an upstream error (health policy)."""
+    try:
+        from src.services.provider.pool.config import parse_pool_config
+        from src.services.provider.pool.health_policy import apply_health_policy
+
+        pool_cfg = parse_pool_config(getattr(provider, "config", None))
+        if pool_cfg is None:
+            return
+
+        error_text = ""
+        resp_headers: dict[str, str] = {}
+        if getattr(cause, "response", None) is not None:
+            try:
+                error_text = (cause.response.text or "")[:4000]
+            except Exception:
+                pass
+            try:
+                resp_headers = dict(cause.response.headers)
+            except Exception:
+                pass
+        elif isinstance(getattr(cause, "error_message", None), str):
+            error_text = str(getattr(cause, "error_message", "") or "")[:4000]
+
+        await apply_health_policy(
+            provider_id=str(provider.id),
+            key_id=str(key.id),
+            status_code=status_code,
+            error_body=error_text,
+            response_headers=resp_headers,
+            config=pool_cfg,
+        )
+    except Exception:
+        pass
+
+
 class TaskService:
     """
     Unified task service facade (Phase 3).
@@ -205,6 +246,7 @@ class TaskService:
         affinity_key: str | None = None,
         create_pending_usage: bool = False,
         enable_cache_affinity: bool = False,
+        is_cancelled: Callable[[], Awaitable[bool]] | None = None,
     ) -> ExecutionResult:
         """Execute a pre-built candidate set through the unified SYNC runtime."""
         from src.services.rate_limit.adaptive_rpm import get_adaptive_rpm_manager
@@ -478,6 +520,7 @@ class TaskService:
             candidate_record_map=candidate_record_map,
             max_attempts=max_attempts,
             execution_error_handler=_handle_exec_err,
+            is_cancelled=is_cancelled,
         )
 
         if result.success:
@@ -687,44 +730,7 @@ class TaskService:
         except Exception:
             logger.opt(exception=True).debug("Pool on_request_success failed (non-blocking)")
 
-    @staticmethod
+    _pool_on_error = staticmethod(pool_on_error)
-    async def _pool_on_error(
-        provider: Any,
-        key: Any,
-        status_code: int,
-        cause: Any,
-    ) -> None:
-        """Notify the pool manager about an upstream error (health policy)."""
-        try:
-            from src.services.provider.pool.config import parse_pool_config
-            from src.services.provider.pool.health_policy import apply_health_policy
-
-            pool_cfg = parse_pool_config(getattr(provider, "config", None))
-            if pool_cfg is None:
-                return
-
-            error_text = ""
-            resp_headers: dict[str, str] = {}
-            if getattr(cause, "response", None) is not None:
-                try:
-                    error_text = (cause.response.text or "")[:4000]
-                except Exception:
-                    pass
-                try:
-                    resp_headers = dict(cause.response.headers)
-                except Exception:
-                    pass
-
-            await apply_health_policy(
-                provider_id=str(provider.id),
-                key_id=str(key.id),
-                status_code=status_code,
-                error_body=error_text,
-                response_headers=resp_headers,
-                config=pool_cfg,
-            )
-        except Exception:
-            pass
 
     async def _execute_sync_unified(
         self,
@@ -1472,11 +1478,13 @@ class TaskService:
         if isinstance(cause, EmbeddedErrorException):
             error_message = cause.error_message or ""
             embedded_status = cause.error_code or 200
+            embedded_detail = error_message[:200] or cause.error_status or f"code={embedded_status}"
             if error_classifier.is_client_error(error_message):
                 logger.warning(
-                    "  [{}] 嵌入式客户端错误，继续转移: {}",
+                    "  [{}] 嵌入式客户端错误 (HTTP 200, status={}), 继续转移: {}",
                     request_id,
-                    error_message[:200],
+                    cause.error_status or embedded_status,
+                    embedded_detail,
                 )
                 RequestCandidateService.mark_candidate_failed(
                     db=self.db,
@@ -1488,12 +1496,14 @@ class TaskService:
                     concurrent_requests=captured_key_concurrent,
                     extra_data=_proxy_extra,
                 )
+                await self._pool_on_error(provider, key, embedded_status, cause)
                 return "break"
 
             logger.warning(
-                "  [{}] 嵌入式服务端错误，尝试重试: {}",
+                "  [{}] 嵌入式服务端错误 (HTTP 200, status={}), 尝试重试: {}",
                 request_id,
-                error_message[:200],
+                cause.error_status or embedded_status,
+                embedded_detail,
             )
             RequestCandidateService.mark_candidate_failed(
                 db=self.db,
@@ -1505,6 +1515,7 @@ class TaskService:
                 concurrent_requests=captured_key_concurrent,
                 extra_data=_proxy_extra,
             )
+            await self._pool_on_error(provider, key, embedded_status, cause)
             return "continue" if has_retry_left else "break"
 
         if isinstance(cause, httpx.HTTPStatusError):

tests/services/test_failover_engine.py

@@ -9,6 +9,7 @@ import pytest
 from src.services.candidate.failover import FailoverEngine
 from src.services.candidate.policy import RetryMode, RetryPolicy, SkipPolicy
 from src.services.orchestration.error_classifier import ErrorAction
+from src.services.scheduling.schemas import PoolCandidate
 from src.services.task.protocol import AttemptKind, AttemptResult
 
 
@@ -502,3 +503,69 @@ async def test_failover_engine_rotates_client_on_stream_capacity_error(
 
     rotate_mock.assert_awaited_once()
     sleep_mock.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_failover_engine_stops_when_cancelled_before_attempt() -> None:
+    db = MagicMock()
+    engine = FailoverEngine(db, error_classifier=_StubErrorClassifier(action=ErrorAction.BREAK))
+
+    candidates = [_make_candidate(provider_id="p1"), _make_candidate(provider_id="p2")]
+    attempt = AsyncMock()
+
+    async def _cancelled() -> bool:
+        return True
+
+    result = await engine.execute(
+        candidates=candidates,
+        attempt_func=attempt,
+        retry_policy=RetryPolicy(mode=RetryMode.DISABLED, max_retries=1),
+        skip_policy=SkipPolicy(),
+        request_id=None,
+        is_cancelled=_cancelled,
+    )
+
+    assert result.success is False
+    assert result.error_type == "cancelled"
+    assert result.last_status_code == 499
+    assert result.attempt_count == 0
+    assert attempt.await_count == 0
+    assert [item.status for item in result.candidate_keys] == ["cancelled", "cancelled"]
+
+
+@pytest.mark.asyncio
+async def test_failover_engine_stops_before_next_pool_key_when_cancelled() -> None:
+    db = MagicMock()
+    engine = FailoverEngine(db, error_classifier=_StubErrorClassifier(action=ErrorAction.BREAK))
+
+    provider = SimpleNamespace(id="p1", name="prov", max_retries=1, config={})
+    endpoint = SimpleNamespace(id="e1")
+    key1 = SimpleNamespace(id="k1", name="key-1", auth_type="api_key", priority=0)
+    key2 = SimpleNamespace(id="k2", name="key-2", auth_type="api_key", priority=0)
+    pool_candidate = PoolCandidate(
+        provider=provider,  # type: ignore[arg-type]
+        endpoint=endpoint,  # type: ignore[arg-type]
+        key=key1,  # type: ignore[arg-type]
+        pool_keys=[key1, key2],  # type: ignore[list-item]
+    )
+
+    attempt = AsyncMock(side_effect=RuntimeError("boom"))
+    cancel_checks = {"count": 0}
+
+    async def _cancelled() -> bool:
+        cancel_checks["count"] += 1
+        return cancel_checks["count"] >= 4
+
+    result = await engine.execute(
+        candidates=[pool_candidate],
+        attempt_func=attempt,
+        retry_policy=RetryPolicy(mode=RetryMode.DISABLED, max_retries=1),
+        skip_policy=SkipPolicy(),
+        request_id=None,
+        is_cancelled=_cancelled,
+    )
+
+    assert result.success is False
+    assert result.error_type == "cancelled"
+    assert attempt.await_count == 1
+    assert any(item.key_id == "k2" and item.status == "cancelled" for item in result.candidate_keys)

tests/services/test_quota_reader.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from src.services.provider.pool.account_state import resolve_pool_account_state
+from src.services.provider.pool.dimensions._helpers import (
+    extract_plan_type,
+    extract_reset_seconds,
+    extract_usage_ratio,
+)
+from src.services.provider_keys import quota_reader
+from src.services.provider_keys.quota_reader import get_quota_reader
+from src.services.scheduling.quota_skipper import is_key_quota_exhausted
+
+
+def test_codex_reader_preserves_summary_formats() -> None:
+    reader = get_quota_reader(
+        "codex",
+        {
+            "codex": {
+                "primary_used_percent": 14.8,
+                "primary_reset_seconds": 266400,
+                "secondary_used_percent": 27.9,
+            }
+        },
+    )
+
+    assert reader.display_summary() == "周剩余 85.2% (3天2小时后重置) | 5H剩余 72.1%"
+
+    credits_reader = get_quota_reader(
+        "codex",
+        {"codex": {"has_credits": True, "credits_balance": 12.345}},
+    )
+    assert credits_reader.display_summary() == "积分 12.35"
+
+
+def test_antigravity_reader_keeps_used_percent_fallbacks() -> None:
+    reader = get_quota_reader(
+        "antigravity",
+        {
+            "antigravity": {
+                "quota_by_model": {
+                    "claude-sonnet-4": {"used_percent": 100.0},
+                    "gemini-2.5-pro": {"remaining_fraction": 0.6},
+                }
+            }
+        },
+    )
+
+    exhausted = reader.is_exhausted("claude-sonnet-4")
+    assert exhausted.exhausted is True
+    assert exhausted.reason == "Antigravity 模型 claude-sonnet-4 配额剩余 0%"
+    assert reader.display_summary() == "最低剩余 0.0% (2 模型)"
+    assert reader.usage_ratio() == pytest.approx(0.7)
+
+
+def test_dimension_helpers_delegate_to_unified_reader(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(quota_reader.time, "time", lambda: 100.0)
+    key_obj = SimpleNamespace(
+        upstream_metadata={
+            "kiro": {
+                "next_reset_at": 160.0,
+                "usage_percentage": 45.0,
+                "subscription_title": "Kiro Team",
+            }
+        },
+        oauth_plan_type=None,
+    )
+
+    assert extract_plan_type(key_obj) == "team"
+    assert extract_reset_seconds(key_obj) == pytest.approx(60.0)
+    assert extract_usage_ratio(key_obj) == pytest.approx(0.45)
+
+
+def test_resolve_pool_account_state_keeps_codex_metadata_block() -> None:
+    state = resolve_pool_account_state(
+        provider_type="codex",
+        upstream_metadata={"codex": {"account_disabled": True, "message": "deactivated_workspace"}},
+        oauth_invalid_reason=None,
+    )
+
+    assert state.blocked is True
+    assert state.code == "account_forbidden"
+    assert state.label == "访问受限"
+    assert state.reason == "deactivated_workspace"
+
+
+def test_quota_skipper_uses_unified_reader() -> None:
+    codex_key = SimpleNamespace(
+        upstream_metadata={"codex": {"primary_used_percent": 100.0, "secondary_used_percent": 20.0}}
+    )
+    exhausted, reason = is_key_quota_exhausted("codex", codex_key, model_name="")  # type: ignore[arg-type]
+    assert exhausted is True
+    assert reason == "Codex 周限额剩余 0%"
+
+    antigravity_key = SimpleNamespace(
+        upstream_metadata={
+            "antigravity": {"quota_by_model": {"gemini-2.5-pro": {"remaining_fraction": 0.0}}}
+        }
+    )
+    exhausted, reason = is_key_quota_exhausted(
+        "antigravity",
+        antigravity_key,  # type: ignore[arg-type]
+        model_name="gemini-2.5-pro",
+    )
+    assert exhausted is True
+    assert reason == "Antigravity 模型 gemini-2.5-pro 配额剩余 0%"

tests/services/test_task_service_async_execute.py

@@ -5,8 +5,11 @@ from unittest.mock import AsyncMock, MagicMock
 
 import pytest
 
+from src.core.exceptions import EmbeddedErrorException
 from src.services.candidate.schema import CandidateKey
 from src.services.candidate.submit import SubmitOutcome
+from src.services.request.executor import ExecutionContext, ExecutionError
+from src.services.task import service as task_service_module
 from src.services.task.context import TaskMode
 from src.services.task.protocol import AttemptKind
 from src.services.task.service import TaskService
@@ -107,3 +110,115 @@ async def test_task_service_execute_sync_passes_request_headers_and_body() -> No
     assert kwargs["request_headers"] == request_headers
     assert kwargs["request_body"] == request_body
     assert kwargs["request_body_ref"] == request_body_ref
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("is_client_error", "retry_index", "max_retries_for_candidate", "expected_action"),
+    [
+        (True, 0, 1, "break"),
+        (False, 0, 2, "continue"),
+    ],
+)
+async def test_task_service_embedded_error_branch_applies_pool_health_policy(
+    monkeypatch: pytest.MonkeyPatch,
+    is_client_error: bool,
+    retry_index: int,
+    max_retries_for_candidate: int,
+    expected_action: str,
+) -> None:
+    db = MagicMock()
+    svc = TaskService(db)
+
+    monkeypatch.setattr(
+        task_service_module.RequestCandidateService, "mark_candidate_failed", MagicMock()
+    )
+    monkeypatch.setattr(
+        "src.services.proxy_node.resolver.resolve_effective_proxy",
+        lambda provider_proxy, key_proxy: provider_proxy or key_proxy,
+    )
+    monkeypatch.setattr("src.services.proxy_node.resolver.resolve_proxy_info", lambda _proxy: None)
+
+    pool_on_error = AsyncMock()
+    monkeypatch.setattr(svc, "_pool_on_error", pool_on_error)
+
+    candidate = SimpleNamespace(
+        provider=SimpleNamespace(id="p1", name="prov", proxy=None),
+        endpoint=SimpleNamespace(id="e1"),
+        key=SimpleNamespace(id="k1", proxy=None),
+    )
+    cause = EmbeddedErrorException(
+        provider_name="prov",
+        error_code=429,
+        error_message="usage_limit_reached",
+        error_status="RESOURCE_EXHAUSTED",
+    )
+    context = ExecutionContext(
+        candidate_id="cid-1",
+        candidate_index=0,
+        provider_id="p1",
+        endpoint_id="e1",
+        key_id="k1",
+        user_id=None,
+        api_key_id=None,
+        is_cached_user=False,
+        elapsed_ms=12,
+        concurrent_requests=3,
+    )
+    exec_err = ExecutionError(cause, context)
+    classifier = SimpleNamespace(is_client_error=lambda _text: is_client_error)
+
+    action = await svc._handle_candidate_error(
+        exec_err=exec_err,
+        candidate=candidate,
+        candidate_record_id="cand-1",
+        retry_index=retry_index,
+        max_retries_for_candidate=max_retries_for_candidate,
+        affinity_key="provider-test:p1",
+        api_format="openai:chat",
+        global_model_id="gpt-4o-mini",
+        request_id="req-1",
+        attempt=1,
+        max_attempts=3,
+        error_classifier=classifier,
+    )
+
+    assert action == expected_action
+    pool_on_error.assert_awaited_once_with(candidate.provider, candidate.key, 429, cause)
+
+
+@pytest.mark.asyncio
+async def test_task_service_pool_on_error_uses_embedded_error_message_fallback(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    db = MagicMock()
+    svc = TaskService(db)
+
+    parsed_pool_cfg = object()
+    apply_health_policy = AsyncMock()
+    monkeypatch.setattr(
+        "src.services.provider.pool.config.parse_pool_config", lambda _cfg: parsed_pool_cfg
+    )
+    monkeypatch.setattr(
+        "src.services.provider.pool.health_policy.apply_health_policy",
+        apply_health_policy,
+    )
+
+    provider = SimpleNamespace(id="p1", config={})
+    key = SimpleNamespace(id="k1")
+    cause = EmbeddedErrorException(
+        provider_name="prov",
+        error_code=429,
+        error_message="usage_limit_reached",
+    )
+
+    await svc._pool_on_error(provider, key, 429, cause)
+
+    apply_health_policy.assert_awaited_once()
+    kwargs = apply_health_policy.await_args.kwargs
+    assert kwargs["provider_id"] == "p1"
+    assert kwargs["key_id"] == "k1"
+    assert kwargs["status_code"] == 429
+    assert kwargs["error_body"] == "usage_limit_reached"
+    assert kwargs["response_headers"] == {}
+    assert kwargs["config"] is parsed_pool_cfg

tests/unit/test_provider_query_failover_endpoint.py

@@ -1,11 +1,17 @@
 from types import SimpleNamespace
 
+import pytest
+from pydantic import ValidationError
+
+from src.api.admin.provider_query import TestModelFailoverRequest as FailoverRequestModel
 from src.api.admin.provider_query import (
     _build_direct_test_candidates,
     _build_test_attempts_from_candidate_keys,
     _filter_test_candidates_by_endpoint,
+    _flatten_test_candidates_for_concurrency,
     _resolve_test_effective_model,
 )
+from src.services.scheduling.schemas import PoolCandidate
 
 
 def _build_provider() -> tuple[SimpleNamespace, SimpleNamespace, SimpleNamespace]:
@@ -89,3 +95,56 @@ def test_build_test_attempts_from_candidate_keys_includes_retry_index() -> None:
     assert attempts[0].retry_index == 1
     assert attempts[0].effective_model == "mapped-model"
     assert attempts[0].endpoint_api_format == "openai:chat"
+
+
+def test_flatten_test_candidates_for_concurrency_expands_pool_keys() -> None:
+    provider, endpoint_a, _endpoint_b = _build_provider()
+    pool_key_a = SimpleNamespace(
+        id="pool-a",
+        name="Pool A",
+        auth_type="oauth",
+        _pool_mapping_matched_model="mapped-a",
+    )
+    pool_key_b = SimpleNamespace(
+        id="pool-b",
+        name="Pool B",
+        auth_type="oauth",
+        _pool_skipped=True,
+        _pool_skip_reason="quota_exhausted",
+    )
+    candidate = PoolCandidate(
+        provider=provider,  # type: ignore[arg-type]
+        endpoint=endpoint_a,  # type: ignore[arg-type]
+        key=pool_key_a,  # type: ignore[arg-type]
+        pool_keys=[pool_key_a, pool_key_b],  # type: ignore[list-item]
+        is_cached=True,
+        provider_api_format="openai:chat",
+    )
+
+    flattened = _flatten_test_candidates_for_concurrency([candidate])
+
+    assert len(flattened) == 2
+    assert flattened[0].key.id == "pool-a"
+    assert flattened[0].mapping_matched_model == "mapped-a"
+    assert flattened[0].is_skipped is False
+    assert flattened[1].key.id == "pool-b"
+    assert flattened[1].is_skipped is True
+    assert flattened[1].skip_reason == "quota_exhausted"
+
+
+def test_test_model_failover_request_validates_concurrency_range() -> None:
+    ok = FailoverRequestModel(
+        provider_id="p1",
+        mode="global",
+        model_name="gpt-4o-mini",
+        concurrency=5,
+    )
+    assert ok.concurrency == 5
+
+    with pytest.raises(ValidationError):
+        FailoverRequestModel(
+            provider_id="p1",
+            mode="global",
+            model_name="gpt-4o-mini",
+            concurrency=0,
+        )