feat(test,quota,failover): 模型并发测试、统一配额读取器与故障转移取消支持

- 新增 QuotaReader 抽象层，统一 Codex/Kiro/Antigravity 配额解析逻辑，
  替换 pool/routes.py 中分散的配额构建函数
- 模型测试支持并发执行多候选，前端新增 useModelTest composable 统一
  ModelsTab 和 ModelMappingTab 的测试逻辑
- ModelTestDialog 增加结果概览摘要、超长结果折叠、端点列和新状态支持，
  删除已合并的 TestResultDialog
- FailoverEngine 新增客户端断开检测，支持取消剩余候选并标记记录
- 刷新配额改为分批执行，直连测试候选按可用性排序
- 修复 error 判断从 "error" in dict 改为 dict.get("error") 避免误判

tests/services/test_failover_engine.py

@@ -9,6 +9,7 @@ import pytest
 from src.services.candidate.failover import FailoverEngine
 from src.services.candidate.policy import RetryMode, RetryPolicy, SkipPolicy
 from src.services.orchestration.error_classifier import ErrorAction
+from src.services.scheduling.schemas import PoolCandidate
 from src.services.task.protocol import AttemptKind, AttemptResult
 
 
@@ -502,3 +503,69 @@ async def test_failover_engine_rotates_client_on_stream_capacity_error(
 
     rotate_mock.assert_awaited_once()
     sleep_mock.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_failover_engine_stops_when_cancelled_before_attempt() -> None:
+    db = MagicMock()
+    engine = FailoverEngine(db, error_classifier=_StubErrorClassifier(action=ErrorAction.BREAK))
+
+    candidates = [_make_candidate(provider_id="p1"), _make_candidate(provider_id="p2")]
+    attempt = AsyncMock()
+
+    async def _cancelled() -> bool:
+        return True
+
+    result = await engine.execute(
+        candidates=candidates,
+        attempt_func=attempt,
+        retry_policy=RetryPolicy(mode=RetryMode.DISABLED, max_retries=1),
+        skip_policy=SkipPolicy(),
+        request_id=None,
+        is_cancelled=_cancelled,
+    )
+
+    assert result.success is False
+    assert result.error_type == "cancelled"
+    assert result.last_status_code == 499
+    assert result.attempt_count == 0
+    assert attempt.await_count == 0
+    assert [item.status for item in result.candidate_keys] == ["cancelled", "cancelled"]
+
+
+@pytest.mark.asyncio
+async def test_failover_engine_stops_before_next_pool_key_when_cancelled() -> None:
+    db = MagicMock()
+    engine = FailoverEngine(db, error_classifier=_StubErrorClassifier(action=ErrorAction.BREAK))
+
+    provider = SimpleNamespace(id="p1", name="prov", max_retries=1, config={})
+    endpoint = SimpleNamespace(id="e1")
+    key1 = SimpleNamespace(id="k1", name="key-1", auth_type="api_key", priority=0)
+    key2 = SimpleNamespace(id="k2", name="key-2", auth_type="api_key", priority=0)
+    pool_candidate = PoolCandidate(
+        provider=provider,  # type: ignore[arg-type]
+        endpoint=endpoint,  # type: ignore[arg-type]
+        key=key1,  # type: ignore[arg-type]
+        pool_keys=[key1, key2],  # type: ignore[list-item]
+    )
+
+    attempt = AsyncMock(side_effect=RuntimeError("boom"))
+    cancel_checks = {"count": 0}
+
+    async def _cancelled() -> bool:
+        cancel_checks["count"] += 1
+        return cancel_checks["count"] >= 4
+
+    result = await engine.execute(
+        candidates=[pool_candidate],
+        attempt_func=attempt,
+        retry_policy=RetryPolicy(mode=RetryMode.DISABLED, max_retries=1),
+        skip_policy=SkipPolicy(),
+        request_id=None,
+        is_cancelled=_cancelled,
+    )
+
+    assert result.success is False
+    assert result.error_type == "cancelled"
+    assert attempt.await_count == 1
+    assert any(item.key_id == "k2" and item.status == "cancelled" for item in result.candidate_keys)

tests/services/test_quota_reader.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from src.services.provider.pool.account_state import resolve_pool_account_state
+from src.services.provider.pool.dimensions._helpers import (
+    extract_plan_type,
+    extract_reset_seconds,
+    extract_usage_ratio,
+)
+from src.services.provider_keys import quota_reader
+from src.services.provider_keys.quota_reader import get_quota_reader
+from src.services.scheduling.quota_skipper import is_key_quota_exhausted
+
+
+def test_codex_reader_preserves_summary_formats() -> None:
+    reader = get_quota_reader(
+        "codex",
+        {
+            "codex": {
+                "primary_used_percent": 14.8,
+                "primary_reset_seconds": 266400,
+                "secondary_used_percent": 27.9,
+            }
+        },
+    )
+
+    assert reader.display_summary() == "周剩余 85.2% (3天2小时后重置) | 5H剩余 72.1%"
+
+    credits_reader = get_quota_reader(
+        "codex",
+        {"codex": {"has_credits": True, "credits_balance": 12.345}},
+    )
+    assert credits_reader.display_summary() == "积分 12.35"
+
+
+def test_antigravity_reader_keeps_used_percent_fallbacks() -> None:
+    reader = get_quota_reader(
+        "antigravity",
+        {
+            "antigravity": {
+                "quota_by_model": {
+                    "claude-sonnet-4": {"used_percent": 100.0},
+                    "gemini-2.5-pro": {"remaining_fraction": 0.6},
+                }
+            }
+        },
+    )
+
+    exhausted = reader.is_exhausted("claude-sonnet-4")
+    assert exhausted.exhausted is True
+    assert exhausted.reason == "Antigravity 模型 claude-sonnet-4 配额剩余 0%"
+    assert reader.display_summary() == "最低剩余 0.0% (2 模型)"
+    assert reader.usage_ratio() == pytest.approx(0.7)
+
+
+def test_dimension_helpers_delegate_to_unified_reader(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(quota_reader.time, "time", lambda: 100.0)
+    key_obj = SimpleNamespace(
+        upstream_metadata={
+            "kiro": {
+                "next_reset_at": 160.0,
+                "usage_percentage": 45.0,
+                "subscription_title": "Kiro Team",
+            }
+        },
+        oauth_plan_type=None,
+    )
+
+    assert extract_plan_type(key_obj) == "team"
+    assert extract_reset_seconds(key_obj) == pytest.approx(60.0)
+    assert extract_usage_ratio(key_obj) == pytest.approx(0.45)
+
+
+def test_resolve_pool_account_state_keeps_codex_metadata_block() -> None:
+    state = resolve_pool_account_state(
+        provider_type="codex",
+        upstream_metadata={"codex": {"account_disabled": True, "message": "deactivated_workspace"}},
+        oauth_invalid_reason=None,
+    )
+
+    assert state.blocked is True
+    assert state.code == "account_forbidden"
+    assert state.label == "访问受限"
+    assert state.reason == "deactivated_workspace"
+
+
+def test_quota_skipper_uses_unified_reader() -> None:
+    codex_key = SimpleNamespace(
+        upstream_metadata={"codex": {"primary_used_percent": 100.0, "secondary_used_percent": 20.0}}
+    )
+    exhausted, reason = is_key_quota_exhausted("codex", codex_key, model_name="")  # type: ignore[arg-type]
+    assert exhausted is True
+    assert reason == "Codex 周限额剩余 0%"
+
+    antigravity_key = SimpleNamespace(
+        upstream_metadata={
+            "antigravity": {"quota_by_model": {"gemini-2.5-pro": {"remaining_fraction": 0.0}}}
+        }
+    )
+    exhausted, reason = is_key_quota_exhausted(
+        "antigravity",
+        antigravity_key,  # type: ignore[arg-type]
+        model_name="gemini-2.5-pro",
+    )
+    assert exhausted is True
+    assert reason == "Antigravity 模型 gemini-2.5-pro 配额剩余 0%"

tests/services/test_task_service_async_execute.py

@@ -5,8 +5,11 @@ from unittest.mock import AsyncMock, MagicMock
 
 import pytest
 
+from src.core.exceptions import EmbeddedErrorException
 from src.services.candidate.schema import CandidateKey
 from src.services.candidate.submit import SubmitOutcome
+from src.services.request.executor import ExecutionContext, ExecutionError
+from src.services.task import service as task_service_module
 from src.services.task.context import TaskMode
 from src.services.task.protocol import AttemptKind
 from src.services.task.service import TaskService
@@ -107,3 +110,115 @@ async def test_task_service_execute_sync_passes_request_headers_and_body() -> No
     assert kwargs["request_headers"] == request_headers
     assert kwargs["request_body"] == request_body
     assert kwargs["request_body_ref"] == request_body_ref
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("is_client_error", "retry_index", "max_retries_for_candidate", "expected_action"),
+    [
+        (True, 0, 1, "break"),
+        (False, 0, 2, "continue"),
+    ],
+)
+async def test_task_service_embedded_error_branch_applies_pool_health_policy(
+    monkeypatch: pytest.MonkeyPatch,
+    is_client_error: bool,
+    retry_index: int,
+    max_retries_for_candidate: int,
+    expected_action: str,
+) -> None:
+    db = MagicMock()
+    svc = TaskService(db)
+
+    monkeypatch.setattr(
+        task_service_module.RequestCandidateService, "mark_candidate_failed", MagicMock()
+    )
+    monkeypatch.setattr(
+        "src.services.proxy_node.resolver.resolve_effective_proxy",
+        lambda provider_proxy, key_proxy: provider_proxy or key_proxy,
+    )
+    monkeypatch.setattr("src.services.proxy_node.resolver.resolve_proxy_info", lambda _proxy: None)
+
+    pool_on_error = AsyncMock()
+    monkeypatch.setattr(svc, "_pool_on_error", pool_on_error)
+
+    candidate = SimpleNamespace(
+        provider=SimpleNamespace(id="p1", name="prov", proxy=None),
+        endpoint=SimpleNamespace(id="e1"),
+        key=SimpleNamespace(id="k1", proxy=None),
+    )
+    cause = EmbeddedErrorException(
+        provider_name="prov",
+        error_code=429,
+        error_message="usage_limit_reached",
+        error_status="RESOURCE_EXHAUSTED",
+    )
+    context = ExecutionContext(
+        candidate_id="cid-1",
+        candidate_index=0,
+        provider_id="p1",
+        endpoint_id="e1",
+        key_id="k1",
+        user_id=None,
+        api_key_id=None,
+        is_cached_user=False,
+        elapsed_ms=12,
+        concurrent_requests=3,
+    )
+    exec_err = ExecutionError(cause, context)
+    classifier = SimpleNamespace(is_client_error=lambda _text: is_client_error)
+
+    action = await svc._handle_candidate_error(
+        exec_err=exec_err,
+        candidate=candidate,
+        candidate_record_id="cand-1",
+        retry_index=retry_index,
+        max_retries_for_candidate=max_retries_for_candidate,
+        affinity_key="provider-test:p1",
+        api_format="openai:chat",
+        global_model_id="gpt-4o-mini",
+        request_id="req-1",
+        attempt=1,
+        max_attempts=3,
+        error_classifier=classifier,
+    )
+
+    assert action == expected_action
+    pool_on_error.assert_awaited_once_with(candidate.provider, candidate.key, 429, cause)
+
+
+@pytest.mark.asyncio
+async def test_task_service_pool_on_error_uses_embedded_error_message_fallback(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    db = MagicMock()
+    svc = TaskService(db)
+
+    parsed_pool_cfg = object()
+    apply_health_policy = AsyncMock()
+    monkeypatch.setattr(
+        "src.services.provider.pool.config.parse_pool_config", lambda _cfg: parsed_pool_cfg
+    )
+    monkeypatch.setattr(
+        "src.services.provider.pool.health_policy.apply_health_policy",
+        apply_health_policy,
+    )
+
+    provider = SimpleNamespace(id="p1", config={})
+    key = SimpleNamespace(id="k1")
+    cause = EmbeddedErrorException(
+        provider_name="prov",
+        error_code=429,
+        error_message="usage_limit_reached",
+    )
+
+    await svc._pool_on_error(provider, key, 429, cause)
+
+    apply_health_policy.assert_awaited_once()
+    kwargs = apply_health_policy.await_args.kwargs
+    assert kwargs["provider_id"] == "p1"
+    assert kwargs["key_id"] == "k1"
+    assert kwargs["status_code"] == 429
+    assert kwargs["error_body"] == "usage_limit_reached"
+    assert kwargs["response_headers"] == {}
+    assert kwargs["config"] is parsed_pool_cfg