feat(test,quota,failover): 模型并发测试、统一配额读取器与故障转移取消支持

- 新增 QuotaReader 抽象层，统一 Codex/Kiro/Antigravity 配额解析逻辑，
  替换 pool/routes.py 中分散的配额构建函数
- 模型测试支持并发执行多候选，前端新增 useModelTest composable 统一
  ModelsTab 和 ModelMappingTab 的测试逻辑
- ModelTestDialog 增加结果概览摘要、超长结果折叠、端点列和新状态支持，
  删除已合并的 TestResultDialog
- FailoverEngine 新增客户端断开检测，支持取消剩余候选并标记记录
- 刷新配额改为分批执行，直连测试候选按可用性排序
- 修复 error 判断从 "error" in dict 改为 dict.get("error") 避免误判

src/services/candidate/failover.py

@@ -181,6 +181,147 @@ class FailoverEngine:
             )
             await asyncio.sleep(backoff_seconds)
 
+    async def _check_cancellation(
+        self,
+        is_cancelled: Callable[[], Awaitable[bool]] | None,
+    ) -> bool:
+        if is_cancelled is None:
+            return False
+        try:
+            return bool(await is_cancelled())
+        except Exception:
+            return False
+
+    def _mark_remaining_cancelled(
+        self,
+        *,
+        candidate_record_map: dict[tuple[int, int], str] | None,
+        candidates: list[ProviderCandidate],
+        from_candidate_idx: int,
+        from_retry_idx: int,
+        retry_policy: RetryPolicy,
+    ) -> None:
+        if not candidate_record_map:
+            return
+
+        now = datetime.now(timezone.utc)
+        updated = False
+        for candidate_idx, cand in enumerate(candidates):
+            if candidate_idx < from_candidate_idx:
+                continue
+            max_retries = self._get_max_retries(cand, retry_policy)
+            for retry_idx in range(max_retries):
+                if candidate_idx == from_candidate_idx and retry_idx < from_retry_idx:
+                    continue
+                record_id = candidate_record_map.get((candidate_idx, retry_idx))
+                if not record_id:
+                    continue
+                self.db.execute(
+                    update(RequestCandidate)
+                    .where(RequestCandidate.id == record_id)
+                    .where(RequestCandidate.status.in_(["available", "pending"]))
+                    .values(
+                        status="cancelled",
+                        status_code=499,
+                        error_message="cancelled_by_client",
+                        finished_at=now,
+                    )
+                )
+                updated = True
+
+        if updated:
+            self.db.commit()
+
+    def _append_cancelled_fallback_candidate_keys(
+        self,
+        *,
+        fallback: list[CandidateKey],
+        candidates: list[ProviderCandidate],
+        from_candidate_idx: int,
+        from_retry_idx: int,
+        retry_policy: RetryPolicy,
+    ) -> None:
+        existing = {(item.candidate_index, item.retry_index) for item in fallback}
+        for candidate_idx, cand in enumerate(candidates):
+            if candidate_idx < from_candidate_idx:
+                continue
+            max_retries = self._get_max_retries(cand, retry_policy)
+            for retry_idx in range(max_retries):
+                if candidate_idx == from_candidate_idx and retry_idx < from_retry_idx:
+                    continue
+                key = (candidate_idx, retry_idx)
+                if key in existing:
+                    continue
+                original_key = getattr(cand, "key", None)
+                original_pool_key_index = getattr(cand, "_pool_key_index", 0)
+                if isinstance(cand, PoolCandidate) and cand.pool_keys:
+                    retry_slots_per_key = self._get_pool_key_max_retries(cand, retry_policy)
+                    pool_key_index = min(retry_idx // retry_slots_per_key, len(cand.pool_keys) - 1)
+                    cand.key = cand.pool_keys[pool_key_index]
+                    cand._pool_key_index = pool_key_index
+                fallback.append(
+                    self._make_candidate_key(
+                        candidate=cand,
+                        candidate_index=candidate_idx,
+                        retry_index=retry_idx,
+                        status="cancelled",
+                        error_message="cancelled_by_client",
+                        status_code=499,
+                    )
+                )
+                if isinstance(cand, PoolCandidate):
+                    cand.key = original_key
+                    cand._pool_key_index = original_pool_key_index
+                existing.add(key)
+
+    async def _maybe_cancel_execution(
+        self,
+        *,
+        is_cancelled: Callable[[], Awaitable[bool]] | None,
+        candidate_record_map: dict[tuple[int, int], str] | None,
+        candidate_keys_fallback: list[CandidateKey],
+        candidates: list[ProviderCandidate],
+        from_candidate_idx: int,
+        from_retry_idx: int,
+        retry_policy: RetryPolicy,
+        request_id: str | None,
+        attempt_count: int,
+    ) -> ExecutionResult | None:
+        if not await self._check_cancellation(is_cancelled):
+            return None
+
+        logger.info(
+            "[FailoverEngine] Request cancelled by client at candidate_index={}, retry_index={}",
+            from_candidate_idx,
+            from_retry_idx,
+        )
+        self._mark_remaining_cancelled(
+            candidate_record_map=candidate_record_map,
+            candidates=candidates,
+            from_candidate_idx=from_candidate_idx,
+            from_retry_idx=from_retry_idx,
+            retry_policy=retry_policy,
+        )
+        self._append_cancelled_fallback_candidate_keys(
+            fallback=candidate_keys_fallback,
+            candidates=candidates,
+            from_candidate_idx=from_candidate_idx,
+            from_retry_idx=from_retry_idx,
+            retry_policy=retry_policy,
+        )
+        return ExecutionResult(
+            success=False,
+            error_type="cancelled",
+            error_message="cancelled_by_client",
+            last_status_code=499,
+            candidate_keys=self._get_candidate_keys(
+                request_id=request_id,
+                fallback=candidate_keys_fallback,
+                candidates=candidates,
+            ),
+            attempt_count=attempt_count,
+        )
+
     async def execute(
         self,
         *,
@@ -201,6 +342,7 @@ class FailoverEngine:
             ]
             | None
         ) = None,
+        is_cancelled: Callable[[], Awaitable[bool]] | None = None,
     ) -> ExecutionResult:
         """
         Execute candidate traversal + retry + failover.
@@ -229,6 +371,20 @@ class FailoverEngine:
             max_attempts = computed
 
         for candidate_index, candidate in enumerate(candidates):
+            cancelled_result = await self._maybe_cancel_execution(
+                is_cancelled=is_cancelled,
+                candidate_record_map=candidate_record_map,
+                candidate_keys_fallback=candidate_keys_fallback,
+                candidates=candidates,
+                from_candidate_idx=candidate_index,
+                from_retry_idx=0,
+                retry_policy=retry_policy,
+                request_id=request_id,
+                attempt_count=attempt_count,
+            )
+            if cancelled_result is not None:
+                return cancelled_result
+
             should_skip, skip_reason = self._should_skip(candidate, skip_policy)
             if should_skip:
                 # PRE_EXPAND: mark all retry slots skipped.
@@ -279,6 +435,7 @@ class FailoverEngine:
                         max_attempts=max_attempts,
                         execution_error_handler=execution_error_handler,
                         consecutive_failures=consecutive_failures,
+                        is_cancelled=is_cancelled,
                     )
                 )
                 if pool_result is not None:
@@ -288,6 +445,20 @@ class FailoverEngine:
             max_retries = self._get_max_retries(candidate, retry_policy)
             retry_index = 0
             while retry_index < max_retries:
+                cancelled_result = await self._maybe_cancel_execution(
+                    is_cancelled=is_cancelled,
+                    candidate_record_map=candidate_record_map,
+                    candidate_keys_fallback=candidate_keys_fallback,
+                    candidates=candidates,
+                    from_candidate_idx=candidate_index,
+                    from_retry_idx=retry_index,
+                    retry_policy=retry_policy,
+                    request_id=request_id,
+                    attempt_count=attempt_count,
+                )
+                if cancelled_result is not None:
+                    return cancelled_result
+
                 attempt_count += 1
 
                 # Resolve/create record_id
@@ -456,6 +627,7 @@ class FailoverEngine:
         consecutive_failures: int,
         max_attempts: int | None,
         execution_error_handler: Any,
+        is_cancelled: Callable[[], Awaitable[bool]] | None,
     ) -> tuple[ExecutionResult | None, int, int, int | None]:
         """Execute a PoolCandidate with in-pool key failover."""
         last_status_code: int | None = None
@@ -463,6 +635,20 @@ class FailoverEngine:
 
         for key_index, pool_key in enumerate(candidate.pool_keys or []):
             base_retry_index = key_index * retry_slots_per_key
+            cancelled_result = await self._maybe_cancel_execution(
+                is_cancelled=is_cancelled,
+                candidate_record_map=candidate_record_map,
+                candidate_keys_fallback=candidate_keys_fallback,
+                candidates=candidates,
+                from_candidate_idx=candidate_index,
+                from_retry_idx=base_retry_index,
+                retry_policy=retry_policy,
+                request_id=request_id,
+                attempt_count=attempt_count,
+            )
+            if cancelled_result is not None:
+                return cancelled_result, attempt_count, consecutive_failures, last_status_code
+
             candidate.key = pool_key
             candidate._pool_key_index = key_index
             candidate.mapping_matched_model = getattr(pool_key, "_pool_mapping_matched_model", None)
@@ -507,8 +693,22 @@ class FailoverEngine:
             max_retries_for_key = retry_slots_per_key
             retry_index = 0
             while retry_index < max_retries_for_key:
-                attempt_count += 1
                 composite_retry_index = base_retry_index + retry_index
+                cancelled_result = await self._maybe_cancel_execution(
+                    is_cancelled=is_cancelled,
+                    candidate_record_map=candidate_record_map,
+                    candidate_keys_fallback=candidate_keys_fallback,
+                    candidates=candidates,
+                    from_candidate_idx=candidate_index,
+                    from_retry_idx=composite_retry_index,
+                    retry_policy=retry_policy,
+                    request_id=request_id,
+                    attempt_count=attempt_count,
+                )
+                if cancelled_result is not None:
+                    return cancelled_result, attempt_count, consecutive_failures, last_status_code
+
+                attempt_count += 1
 
                 record_id = None
                 if candidate_record_map:

src/services/provider/pool/account_state.py

@@ -9,6 +9,8 @@ from __future__ import annotations
 from dataclasses import dataclass
 from typing import Any
 
+from src.services.provider_keys.quota_reader import get_quota_reader
+
 OAUTH_ACCOUNT_BLOCK_PREFIX = "[ACCOUNT_BLOCK] "
 OAUTH_REFRESH_FAILED_PREFIX = "[REFRESH_FAILED] "
 OAUTH_EXPIRED_PREFIX = "[OAUTH_EXPIRED] "
@@ -69,7 +71,7 @@ def _classify_block_reason(text: str) -> tuple[str, str]:
         return "oauth_expired", "Token 失效"
     if any(kw in lowered for kw in _KEYWORDS_VERIFICATION):
         return "account_verification", "需要验证"
-    if 'deactivated_workspace' in lowered:
+    if "deactivated_workspace" in lowered:
         return "workspace_deactivated", "工作区停用"
     if any(kw in lowered for kw in _KEYWORDS_DISABLED):
         return "account_disabled", "账号停用"
@@ -130,30 +132,13 @@ def _resolve_from_metadata(
         if isinstance(maybe_bucket, dict):
             provider_bucket = maybe_bucket
 
-    if (
-        normalized_provider == "kiro"
-        and provider_bucket
-        and _is_truthy_flag(provider_bucket.get("is_banned"))
-    ):
-        reason = _extract_reason(provider_bucket, "ban_reason", "reason", "message")
+    quota_block = get_quota_reader(normalized_provider, upstream_metadata).account_block()
+    if quota_block.blocked:
         return PoolAccountState(
             blocked=True,
-            code="account_banned",
-            label="账号封禁",
-            reason=reason or "Kiro 账号已封禁",
-        )
-
-    if (
-        normalized_provider == "antigravity"
-        and provider_bucket
-        and _is_truthy_flag(provider_bucket.get("is_forbidden"))
-    ):
-        reason = _extract_reason(provider_bucket, "forbidden_reason", "reason", "message")
-        return PoolAccountState(
-            blocked=True,
-            code="account_forbidden",
-            label="访问受限",
-            reason=reason or "Antigravity 账户访问受限",
+            code=quota_block.code,
+            label=quota_block.label,
+            reason=quota_block.reason,
         )
 
     for source in (provider_bucket, upstream_metadata):

src/services/provider/pool/dimensions/_helpers.py

@@ -3,9 +3,11 @@
 from __future__ import annotations
 
 import math
-import time
 from typing import Any
 
+from src.core.provider_types import ProviderType
+from src.services.provider_keys.quota_reader import get_quota_reader
+
 
 def safe_float(value: Any) -> float | None:
     try:
@@ -98,26 +100,10 @@ def extract_plan_type(key_obj: Any) -> str | None:
         return direct
 
     metadata = safe_metadata(key_obj)
-    codex = metadata.get("codex")
-    if isinstance(codex, dict):
-        codex_plan = normalize_plan(codex.get("plan_type"))
-        if codex_plan:
-            return codex_plan
-
-    kiro = metadata.get("kiro")
-    if isinstance(kiro, dict):
-        subscription_title = normalize_plan(kiro.get("subscription_title"))
-        if subscription_title:
-            # Normalize common Kiro labels into free/team buckets used by free_team_first.
-            if "team" in subscription_title:
-                return "team"
-            if "free" in subscription_title:
-                return "free"
-            if "pro" in subscription_title:
-                return "pro"
-            if "plus" in subscription_title:
-                return "plus"
-            return subscription_title
+    for provider_type in (ProviderType.CODEX, ProviderType.KIRO, ProviderType.ANTIGRAVITY):
+        plan_type = get_quota_reader(provider_type, metadata).plan_type()
+        if plan_type:
+            return plan_type
 
     return None
 
@@ -126,19 +112,11 @@ def extract_reset_seconds(key_obj: Any) -> float | None:
     metadata = safe_metadata(key_obj)
     candidates: list[float] = []
 
-    codex = metadata.get("codex")
-    if isinstance(codex, dict):
-        for field in ("secondary_reset_seconds", "primary_reset_seconds"):
-            parsed = safe_float(codex.get(field))
-            if parsed is None or parsed < 0:
-                continue
-            candidates.append(parsed)
-
-    kiro = metadata.get("kiro")
-    if isinstance(kiro, dict):
-        next_reset_at = safe_float(kiro.get("next_reset_at"))
-        if next_reset_at is not None and next_reset_at > 0:
-            candidates.append(max(0.0, next_reset_at - time.time()))
+    for provider_type in (ProviderType.CODEX, ProviderType.KIRO, ProviderType.ANTIGRAVITY):
+        reset_seconds = get_quota_reader(provider_type, metadata).reset_seconds()
+        if reset_seconds is None:
+            continue
+        candidates.append(reset_seconds)
 
     if not candidates:
         return None
@@ -148,41 +126,10 @@ def extract_reset_seconds(key_obj: Any) -> float | None:
 def extract_usage_ratio(key_obj: Any) -> float | None:
     metadata = safe_metadata(key_obj)
 
-    codex = metadata.get("codex")
-    if isinstance(codex, dict):
-        codex_values: list[float] = []
-        for field in ("primary_used_percent", "secondary_used_percent"):
-            parsed = safe_float(codex.get(field))
-            if parsed is None:
-                continue
-            codex_values.append(max(0.0, min(parsed, 100.0)) / 100.0)
-        if codex_values:
-            return sum(codex_values) / len(codex_values)
-
-    kiro = metadata.get("kiro")
-    if isinstance(kiro, dict):
-        parsed = safe_float(kiro.get("usage_percentage"))
-        if parsed is not None:
-            return max(0.0, min(parsed, 100.0)) / 100.0
-
-    antigravity = metadata.get("antigravity")
-    if isinstance(antigravity, dict):
-        quota_by_model = antigravity.get("quota_by_model")
-        if isinstance(quota_by_model, dict):
-            usage_values: list[float] = []
-            for model_info in quota_by_model.values():
-                if not isinstance(model_info, dict):
-                    continue
-                used_percent = safe_float(model_info.get("used_percent"))
-                if used_percent is None:
-                    remaining_fraction = safe_float(model_info.get("remaining_fraction"))
-                    if remaining_fraction is not None:
-                        used_percent = (1.0 - remaining_fraction) * 100.0
-                if used_percent is None:
-                    continue
-                usage_values.append(max(0.0, min(used_percent, 100.0)) / 100.0)
-            if usage_values:
-                return sum(usage_values) / len(usage_values)
+    for provider_type in (ProviderType.CODEX, ProviderType.KIRO, ProviderType.ANTIGRAVITY):
+        usage_ratio = get_quota_reader(provider_type, metadata).usage_ratio()
+        if usage_ratio is not None:
+            return usage_ratio
 
     return None
 

src/services/provider_keys/key_quota_service.py

@@ -26,6 +26,13 @@ from src.services.provider_keys.quota_refresh import (
 QuotaRefreshHandler = Callable[..., Awaitable[dict]]
 
 
+_QUOTA_REFRESH_HANDLERS: dict[str, QuotaRefreshHandler] = {
+    ProviderType.CODEX: refresh_codex_key_quota,
+    ProviderType.ANTIGRAVITY: refresh_antigravity_key_quota,
+    ProviderType.KIRO: refresh_kiro_key_quota,
+}
+
+
 def _normalize_api_format(api_format: Any) -> str:
     """规范化 api_format，兼容大小写和首尾空白。"""
     if not isinstance(api_format, str):
@@ -55,12 +62,9 @@ def _select_refresh_endpoint(provider: Provider, provider_type: str) -> Provider
 
 def _resolve_quota_refresh_handler(provider_type: str) -> QuotaRefreshHandler:
     """按 provider 类型返回刷新策略。"""
-    if provider_type == ProviderType.CODEX:
-        return refresh_codex_key_quota
-    if provider_type == ProviderType.ANTIGRAVITY:
-        return refresh_antigravity_key_quota
-    if provider_type == ProviderType.KIRO:
-        return refresh_kiro_key_quota
+    handler = _QUOTA_REFRESH_HANDLERS.get(provider_type)
+    if handler is not None:
+        return handler
     raise InvalidRequestException("仅支持 Codex / Antigravity / Kiro 类型的 Provider 刷新限额")
 
 

src/services/provider_keys/quota_reader.py

@@ -0,0 +1,439 @@
+"""Unified quota readers for provider key upstream metadata."""
+
+from __future__ import annotations
+
+import math
+import time
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any
+
+from src.core.provider_types import ProviderType, normalize_provider_type
+
+
+def _to_float(value: Any) -> float | None:
+    try:
+        parsed = float(value)
+    except (TypeError, ValueError):
+        return None
+    if math.isnan(parsed) or math.isinf(parsed):
+        return None
+    return parsed
+
+
+def _normalize_plan(value: Any) -> str | None:
+    if not isinstance(value, str):
+        return None
+    normalized = value.strip().lower()
+    return normalized or None
+
+
+def _is_truthy_flag(value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, (int, float)):
+        return value != 0
+    if isinstance(value, str):
+        normalized = value.strip().lower()
+        return normalized in {"1", "true", "yes", "y"}
+    return False
+
+
+def _extract_reason(source: dict[str, Any], *fields: str) -> str | None:
+    for field in fields:
+        value = source.get(field)
+        if not isinstance(value, str):
+            continue
+        text = value.strip()
+        if text:
+            return text
+    return None
+
+
+def _pct_is_exhausted(value: Any) -> bool:
+    pct = _to_float(value)
+    if pct is None:
+        return False
+    return pct >= 100.0 - 1e-6
+
+
+def _format_percent(value: float) -> str:
+    clamped = max(0.0, min(value, 100.0))
+    return f"{clamped:.1f}%"
+
+
+def _format_quota_value(value: float) -> str:
+    rounded = round(value)
+    if abs(value - rounded) < 1e-6:
+        return str(rounded)
+    return f"{value:.1f}"
+
+
+def _format_reset_after(seconds_raw: Any) -> str | None:
+    seconds = _to_float(seconds_raw)
+    if seconds is None:
+        return None
+
+    total_seconds = int(seconds)
+    if total_seconds <= 0:
+        return "已重置"
+
+    days = total_seconds // 86400
+    hours = (total_seconds % 86400) // 3600
+    minutes = (total_seconds % 3600) // 60
+
+    if days > 0:
+        return f"{days}天{hours}小时后重置"
+    if hours > 0:
+        return f"{hours}小时{minutes}分钟后重置"
+    if minutes > 0:
+        return f"{minutes}分钟后重置"
+    return "即将重置"
+
+
+@dataclass(frozen=True, slots=True)
+class QuotaExhaustedResult:
+    exhausted: bool
+    reason: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class AccountBlockResult:
+    blocked: bool
+    code: str | None = None
+    label: str | None = None
+    reason: str | None = None
+
+
+class PoolQuotaReader(ABC):
+    """Read-only view over one provider namespace in upstream_metadata."""
+
+    namespace: str | None = None
+
+    def __init__(self, data: dict[str, Any] | None) -> None:
+        self._data: dict[str, Any] = data if isinstance(data, dict) else {}
+
+    @abstractmethod
+    def is_exhausted(self, model_name: str | None = None) -> QuotaExhaustedResult:
+        """Return whether this key/model should be skipped for quota exhaustion."""
+
+    @abstractmethod
+    def usage_ratio(self) -> float | None:
+        """Return usage ratio within [0, 1], when available."""
+
+    @abstractmethod
+    def plan_type(self) -> str | None:
+        """Return normalized plan type, when available."""
+
+    @abstractmethod
+    def reset_seconds(self) -> float | None:
+        """Return seconds until next reset, when available."""
+
+    @abstractmethod
+    def account_block(self) -> AccountBlockResult:
+        """Return account-level block state derived from metadata."""
+
+    @abstractmethod
+    def display_summary(self) -> str | None:
+        """Return admin-facing quota summary string."""
+
+    def updated_at(self) -> int | None:
+        updated_at = _to_float(self._data.get("updated_at"))
+        if updated_at is None or updated_at <= 0:
+            return None
+        if updated_at > 1_000_000_000_000:
+            updated_at /= 1000
+        return int(updated_at)
+
+
+class NullQuotaReader(PoolQuotaReader):
+    def is_exhausted(self, model_name: str | None = None) -> QuotaExhaustedResult:
+        _ = model_name
+        return QuotaExhaustedResult(exhausted=False)
+
+    def usage_ratio(self) -> float | None:
+        return None
+
+    def plan_type(self) -> str | None:
+        return None
+
+    def reset_seconds(self) -> float | None:
+        return None
+
+    def account_block(self) -> AccountBlockResult:
+        return AccountBlockResult(blocked=False)
+
+    def display_summary(self) -> str | None:
+        return None
+
+
+class CodexQuotaReader(PoolQuotaReader):
+    namespace = "codex"
+
+    def is_exhausted(self, model_name: str | None = None) -> QuotaExhaustedResult:
+        _ = model_name
+        exhausted_parts: list[str] = []
+        if _pct_is_exhausted(self._data.get("primary_used_percent")):
+            exhausted_parts.append("周限额剩余 0%")
+        if _pct_is_exhausted(self._data.get("secondary_used_percent")):
+            exhausted_parts.append("5H 限额剩余 0%")
+        if exhausted_parts:
+            return QuotaExhaustedResult(True, "Codex " + "，".join(exhausted_parts))
+        return QuotaExhaustedResult(False)
+
+    def usage_ratio(self) -> float | None:
+        values: list[float] = []
+        for field in ("primary_used_percent", "secondary_used_percent"):
+            parsed = _to_float(self._data.get(field))
+            if parsed is None:
+                continue
+            values.append(max(0.0, min(parsed, 100.0)) / 100.0)
+        if not values:
+            return None
+        return sum(values) / len(values)
+
+    def plan_type(self) -> str | None:
+        return _normalize_plan(self._data.get("plan_type"))
+
+    def reset_seconds(self) -> float | None:
+        candidates: list[float] = []
+        for field in ("secondary_reset_seconds", "primary_reset_seconds"):
+            parsed = _to_float(self._data.get(field))
+            if parsed is None or parsed < 0:
+                continue
+            candidates.append(parsed)
+        if not candidates:
+            return None
+        return min(candidates)
+
+    def account_block(self) -> AccountBlockResult:
+        if not _is_truthy_flag(self._data.get("account_disabled")):
+            return AccountBlockResult(blocked=False)
+        reason = _extract_reason(self._data, "forbidden_reason", "ban_reason", "reason", "message")
+        return AccountBlockResult(
+            blocked=True,
+            code="account_forbidden",
+            label="访问受限",
+            reason=reason or "账号访问受限",
+        )
+
+    def display_summary(self) -> str | None:
+        parts: list[str] = []
+
+        primary_used = _to_float(self._data.get("primary_used_percent"))
+        if primary_used is not None:
+            part = f"周剩余 {_format_percent(100.0 - primary_used)}"
+            reset_text = _format_reset_after(self._data.get("primary_reset_seconds"))
+            if reset_text:
+                part = f"{part} ({reset_text})"
+            parts.append(part)
+
+        secondary_used = _to_float(self._data.get("secondary_used_percent"))
+        if secondary_used is not None:
+            part = f"5H剩余 {_format_percent(100.0 - secondary_used)}"
+            reset_text = _format_reset_after(self._data.get("secondary_reset_seconds"))
+            if reset_text:
+                part = f"{part} ({reset_text})"
+            parts.append(part)
+
+        if parts:
+            return " | ".join(parts)
+
+        has_credits = self._data.get("has_credits")
+        credits_balance = _to_float(self._data.get("credits_balance"))
+        if has_credits is True and credits_balance is not None:
+            return f"积分 {credits_balance:.2f}"
+        if has_credits is True:
+            return "有积分"
+        return None
+
+
+class KiroQuotaReader(PoolQuotaReader):
+    namespace = "kiro"
+
+    def is_exhausted(self, model_name: str | None = None) -> QuotaExhaustedResult:
+        _ = model_name
+        remaining = _to_float(self._data.get("remaining"))
+        if remaining is not None and remaining <= 0.0:
+            return QuotaExhaustedResult(True, "Kiro 账号配额剩余 0")
+        return QuotaExhaustedResult(False)
+
+    def usage_ratio(self) -> float | None:
+        parsed = _to_float(self._data.get("usage_percentage"))
+        if parsed is None:
+            return None
+        return max(0.0, min(parsed, 100.0)) / 100.0
+
+    def plan_type(self) -> str | None:
+        subscription_title = _normalize_plan(self._data.get("subscription_title"))
+        if not subscription_title:
+            return None
+        if "team" in subscription_title:
+            return "team"
+        if "free" in subscription_title:
+            return "free"
+        if "pro" in subscription_title:
+            return "pro"
+        if "plus" in subscription_title:
+            return "plus"
+        return subscription_title
+
+    def reset_seconds(self) -> float | None:
+        next_reset_at = _to_float(self._data.get("next_reset_at"))
+        if next_reset_at is None or next_reset_at <= 0:
+            return None
+        return max(0.0, next_reset_at - time.time())
+
+    def account_block(self) -> AccountBlockResult:
+        if not _is_truthy_flag(self._data.get("is_banned")):
+            return AccountBlockResult(blocked=False)
+        reason = _extract_reason(self._data, "ban_reason", "reason", "message")
+        return AccountBlockResult(
+            blocked=True,
+            code="account_banned",
+            label="账号封禁",
+            reason=reason or "Kiro 账号已封禁",
+        )
+
+    def display_summary(self) -> str | None:
+        if self._data.get("is_banned") is True:
+            return "账号已封禁"
+
+        usage_percentage = _to_float(self._data.get("usage_percentage"))
+        if usage_percentage is not None:
+            remaining = 100.0 - usage_percentage
+            current_usage = _to_float(self._data.get("current_usage"))
+            usage_limit = _to_float(self._data.get("usage_limit"))
+            if current_usage is not None and usage_limit is not None and usage_limit > 0:
+                return (
+                    f"剩余 {_format_percent(remaining)} "
+                    f"({_format_quota_value(current_usage)}/{_format_quota_value(usage_limit)})"
+                )
+            return f"剩余 {_format_percent(remaining)}"
+
+        remaining = _to_float(self._data.get("remaining"))
+        usage_limit = _to_float(self._data.get("usage_limit"))
+        if remaining is not None and usage_limit is not None and usage_limit > 0:
+            return f"剩余 {_format_quota_value(remaining)}/{_format_quota_value(usage_limit)}"
+        return None
+
+
+class AntigravityQuotaReader(PoolQuotaReader):
+    namespace = "antigravity"
+
+    def _quota_by_model(self) -> dict[str, Any]:
+        quota_by_model = self._data.get("quota_by_model")
+        if not isinstance(quota_by_model, dict):
+            return {}
+        return quota_by_model
+
+    def _used_percent(self, model_info: dict[str, Any]) -> float | None:
+        used_percent = _to_float(model_info.get("used_percent"))
+        if used_percent is not None:
+            return max(0.0, min(used_percent, 100.0))
+        remaining_fraction = _to_float(model_info.get("remaining_fraction"))
+        if remaining_fraction is None:
+            return None
+        return max(0.0, min((1.0 - remaining_fraction) * 100.0, 100.0))
+
+    def is_exhausted(self, model_name: str | None = None) -> QuotaExhaustedResult:
+        if not model_name:
+            return QuotaExhaustedResult(False)
+        model_quota = self._quota_by_model().get(model_name)
+        if not isinstance(model_quota, dict):
+            return QuotaExhaustedResult(False)
+
+        remaining_fraction = _to_float(model_quota.get("remaining_fraction"))
+        if remaining_fraction is not None and remaining_fraction <= 0.0:
+            return QuotaExhaustedResult(True, f"Antigravity 模型 {model_name} 配额剩余 0%")
+        if _pct_is_exhausted(model_quota.get("used_percent")):
+            return QuotaExhaustedResult(True, f"Antigravity 模型 {model_name} 配额剩余 0%")
+        return QuotaExhaustedResult(False)
+
+    def usage_ratio(self) -> float | None:
+        usage_values: list[float] = []
+        for model_info in self._quota_by_model().values():
+            if not isinstance(model_info, dict):
+                continue
+            used_percent = self._used_percent(model_info)
+            if used_percent is None:
+                continue
+            usage_values.append(used_percent / 100.0)
+        if not usage_values:
+            return None
+        return sum(usage_values) / len(usage_values)
+
+    def plan_type(self) -> str | None:
+        return None
+
+    def reset_seconds(self) -> float | None:
+        return None
+
+    def account_block(self) -> AccountBlockResult:
+        if not _is_truthy_flag(self._data.get("is_forbidden")):
+            return AccountBlockResult(blocked=False)
+        reason = _extract_reason(self._data, "forbidden_reason", "reason", "message")
+        return AccountBlockResult(
+            blocked=True,
+            code="account_forbidden",
+            label="访问受限",
+            reason=reason or "Antigravity 账户访问受限",
+        )
+
+    def display_summary(self) -> str | None:
+        if self._data.get("is_forbidden") is True:
+            return "访问受限"
+
+        remaining_list: list[float] = []
+        for raw_info in self._quota_by_model().values():
+            if not isinstance(raw_info, dict):
+                continue
+            used_percent = self._used_percent(raw_info)
+            if used_percent is None:
+                continue
+            remaining_list.append(max(0.0, min(100.0 - used_percent, 100.0)))
+
+        if not remaining_list:
+            return None
+
+        min_remaining = min(remaining_list)
+        if len(remaining_list) == 1:
+            return f"剩余 {_format_percent(min_remaining)}"
+        return f"最低剩余 {_format_percent(min_remaining)} ({len(remaining_list)} 模型)"
+
+
+_READER_CLASSES: dict[str, type[PoolQuotaReader]] = {
+    ProviderType.CODEX: CodexQuotaReader,
+    ProviderType.KIRO: KiroQuotaReader,
+    ProviderType.ANTIGRAVITY: AntigravityQuotaReader,
+}
+
+
+def get_quota_reader(provider_type: str | None, upstream_metadata: Any) -> PoolQuotaReader:
+    """Return a quota reader for one provider namespace in upstream_metadata."""
+
+    normalized_type = normalize_provider_type(provider_type)
+    reader_cls = _READER_CLASSES.get(normalized_type)
+    if reader_cls is None or not isinstance(upstream_metadata, dict):
+        return NullQuotaReader(None)
+
+    namespace = reader_cls.namespace
+    if not namespace:
+        return NullQuotaReader(None)
+
+    data = upstream_metadata.get(namespace)
+    if not isinstance(data, dict):
+        return NullQuotaReader(None)
+    return reader_cls(data)
+
+
+__all__ = [
+    "AccountBlockResult",
+    "AntigravityQuotaReader",
+    "CodexQuotaReader",
+    "KiroQuotaReader",
+    "NullQuotaReader",
+    "PoolQuotaReader",
+    "QuotaExhaustedResult",
+    "get_quota_reader",
+]

src/services/scheduling/quota_skipper.py

@@ -1,26 +1,7 @@
 from __future__ import annotations
 
-from src.core.provider_types import ProviderType, normalize_provider_type
 from src.models.database import ProviderAPIKey
-
-
-def _pct_is_exhausted(value: object) -> bool:
-    """Return True when used_percent indicates 0% remaining."""
-    try:
-        pct = float(value)  # type: ignore[arg-type]
-    except (TypeError, ValueError):
-        return False
-    # Some upstreams may return values slightly above 100 due to rounding.
-    return pct >= 100.0 - 1e-6
-
-
-def _float_or_none(value: object) -> float | None:
-    try:
-        if value is None:
-            return None
-        return float(value)  # type: ignore[arg-type]
-    except (TypeError, ValueError):
-        return None
+from src.services.provider_keys.quota_reader import get_quota_reader
 
 
 def is_key_quota_exhausted(
@@ -29,72 +10,8 @@ def is_key_quota_exhausted(
     *,
     model_name: str,
 ) -> tuple[bool, str | None]:
-    """Check ProviderAPIKey.upstream_metadata quota and decide whether to skip.
+    """Check ProviderAPIKey.upstream_metadata quota and decide whether to skip."""
 
-    Requirements:
-    - Kiro: account-level quota. When remaining == 0, skip this key; allow again when remaining > 0.
-    - Codex: only consider weekly quota + 5H quota.
-             If either remaining is 0%, skip this key.
-    - Antigravity: quota is per-model; do not disable the account.
-                   When the requested model's quota is 0%, skip this key.
-    """
-
-    pt = normalize_provider_type(provider_type)
-
-    upstream = getattr(key, "upstream_metadata", None) or {}
-    if not isinstance(upstream, dict):
-        return False, None
-
-    if pt == ProviderType.KIRO:
-        kiro_meta = upstream.get("kiro")
-        if not isinstance(kiro_meta, dict):
-            return False, None
-
-        remaining = _float_or_none(kiro_meta.get("remaining"))
-
-        if remaining is not None and remaining <= 0.0:
-            return True, "Kiro 账号配额剩余 0"
-
-        return False, None
-
-    if pt == ProviderType.CODEX:
-        codex_meta = upstream.get("codex")
-        if not isinstance(codex_meta, dict):
-            return False, None
-
-        weekly_used = codex_meta.get("primary_used_percent")
-        five_hour_used = codex_meta.get("secondary_used_percent")
-
-        exhausted_parts: list[str] = []
-        if _pct_is_exhausted(weekly_used):
-            exhausted_parts.append("周限额剩余 0%")
-        if _pct_is_exhausted(five_hour_used):
-            exhausted_parts.append("5H 限额剩余 0%")
-
-        if exhausted_parts:
-            return True, "Codex " + "，".join(exhausted_parts)
-
-        return False, None
-
-    if pt == ProviderType.ANTIGRAVITY:
-        ag_meta = upstream.get("antigravity")
-        if not isinstance(ag_meta, dict):
-            return False, None
-
-        quota_by_model = ag_meta.get("quota_by_model")
-        if not isinstance(quota_by_model, dict):
-            return False, None
-
-        model_quota = quota_by_model.get(model_name)
-        if not isinstance(model_quota, dict):
-            return False, None
-
-        remaining_fraction = _float_or_none(model_quota.get("remaining_fraction"))
-        if remaining_fraction is not None and remaining_fraction <= 0.0:
-            return True, f"Antigravity 模型 {model_name} 配额剩余 0%"
-        if _pct_is_exhausted(model_quota.get("used_percent")):
-            return True, f"Antigravity 模型 {model_name} 配额剩余 0%"
-
-        return False, None
-
-    return False, None
+    reader = get_quota_reader(provider_type, getattr(key, "upstream_metadata", None))
+    result = reader.is_exhausted(model_name)
+    return result.exhausted, result.reason

src/services/task/service.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import re
-from collections.abc import Callable
+from collections.abc import Awaitable, Callable
 from typing import Any
 from uuid import uuid4
 
@@ -56,6 +56,47 @@ _SENSITIVE_PATTERN = re.compile(
 )
 
 
+async def pool_on_error(
+    provider: Any,
+    key: Any,
+    status_code: int,
+    cause: Any,
+) -> None:
+    """Notify the pool manager about an upstream error (health policy)."""
+    try:
+        from src.services.provider.pool.config import parse_pool_config
+        from src.services.provider.pool.health_policy import apply_health_policy
+
+        pool_cfg = parse_pool_config(getattr(provider, "config", None))
+        if pool_cfg is None:
+            return
+
+        error_text = ""
+        resp_headers: dict[str, str] = {}
+        if getattr(cause, "response", None) is not None:
+            try:
+                error_text = (cause.response.text or "")[:4000]
+            except Exception:
+                pass
+            try:
+                resp_headers = dict(cause.response.headers)
+            except Exception:
+                pass
+        elif isinstance(getattr(cause, "error_message", None), str):
+            error_text = str(getattr(cause, "error_message", "") or "")[:4000]
+
+        await apply_health_policy(
+            provider_id=str(provider.id),
+            key_id=str(key.id),
+            status_code=status_code,
+            error_body=error_text,
+            response_headers=resp_headers,
+            config=pool_cfg,
+        )
+    except Exception:
+        pass
+
+
 class TaskService:
     """
     Unified task service facade (Phase 3).
@@ -205,6 +246,7 @@ class TaskService:
         affinity_key: str | None = None,
         create_pending_usage: bool = False,
         enable_cache_affinity: bool = False,
+        is_cancelled: Callable[[], Awaitable[bool]] | None = None,
     ) -> ExecutionResult:
         """Execute a pre-built candidate set through the unified SYNC runtime."""
         from src.services.rate_limit.adaptive_rpm import get_adaptive_rpm_manager
@@ -478,6 +520,7 @@ class TaskService:
             candidate_record_map=candidate_record_map,
             max_attempts=max_attempts,
             execution_error_handler=_handle_exec_err,
+            is_cancelled=is_cancelled,
         )
 
         if result.success:
@@ -687,44 +730,7 @@ class TaskService:
         except Exception:
             logger.opt(exception=True).debug("Pool on_request_success failed (non-blocking)")
 
-    @staticmethod
-    async def _pool_on_error(
-        provider: Any,
-        key: Any,
-        status_code: int,
-        cause: Any,
-    ) -> None:
-        """Notify the pool manager about an upstream error (health policy)."""
-        try:
-            from src.services.provider.pool.config import parse_pool_config
-            from src.services.provider.pool.health_policy import apply_health_policy
-
-            pool_cfg = parse_pool_config(getattr(provider, "config", None))
-            if pool_cfg is None:
-                return
-
-            error_text = ""
-            resp_headers: dict[str, str] = {}
-            if getattr(cause, "response", None) is not None:
-                try:
-                    error_text = (cause.response.text or "")[:4000]
-                except Exception:
-                    pass
-                try:
-                    resp_headers = dict(cause.response.headers)
-                except Exception:
-                    pass
-
-            await apply_health_policy(
-                provider_id=str(provider.id),
-                key_id=str(key.id),
-                status_code=status_code,
-                error_body=error_text,
-                response_headers=resp_headers,
-                config=pool_cfg,
-            )
-        except Exception:
-            pass
+    _pool_on_error = staticmethod(pool_on_error)
 
     async def _execute_sync_unified(
         self,
@@ -1472,11 +1478,13 @@ class TaskService:
         if isinstance(cause, EmbeddedErrorException):
             error_message = cause.error_message or ""
             embedded_status = cause.error_code or 200
+            embedded_detail = error_message[:200] or cause.error_status or f"code={embedded_status}"
             if error_classifier.is_client_error(error_message):
                 logger.warning(
-                    "  [{}] 嵌入式客户端错误，继续转移: {}",
+                    "  [{}] 嵌入式客户端错误 (HTTP 200, status={}), 继续转移: {}",
                     request_id,
-                    error_message[:200],
+                    cause.error_status or embedded_status,
+                    embedded_detail,
                 )
                 RequestCandidateService.mark_candidate_failed(
                     db=self.db,
@@ -1488,12 +1496,14 @@ class TaskService:
                     concurrent_requests=captured_key_concurrent,
                     extra_data=_proxy_extra,
                 )
+                await self._pool_on_error(provider, key, embedded_status, cause)
                 return "break"
 
             logger.warning(
-                "  [{}] 嵌入式服务端错误，尝试重试: {}",
+                "  [{}] 嵌入式服务端错误 (HTTP 200, status={}), 尝试重试: {}",
                 request_id,
-                error_message[:200],
+                cause.error_status or embedded_status,
+                embedded_detail,
             )
             RequestCandidateService.mark_candidate_failed(
                 db=self.db,
@@ -1505,6 +1515,7 @@ class TaskService:
                 concurrent_requests=captured_key_concurrent,
                 extra_data=_proxy_extra,
             )
+            await self._pool_on_error(provider, key, embedded_status, cause)
             return "continue" if has_retry_left else "break"
 
         if isinstance(cause, httpx.HTTPStatusError):