feat(pool,scheduling): 号池调度维度、配额冷却机制与管理后台重构

- 新增 scheduling_dimensions 模块,为每个 Key 计算多维调度状态(手动/冷却/熔断/成本/健康)
- 新增 quota_cooldown 模块,统一判定 Key 的有效冷却原因
- Pool 管理后台 API 扩展 Key 详情字段(调度状态/维度/配额/OAuth 信息)
- 前端 Pool 管理页面重写,支持调度状态展示、批量清理封禁 Key
- Handler 基类增加请求调度元数据采集,stream telemetry 增强
- 请求时间线组件增强,支持 attempted 候选展示
- Kiro OAuth 凭证导入解析改进
- 新增 usage 表 provider_key 索引迁移
- 补充调度维度、配额冷却、候选枚举等单元测试

Closes #197

Co-authored-by: AAEE86 <33052466+AAEE86@users.noreply.github.com>
This commit is contained in:
fawney19
2026-03-03 09:22:20 +08:00
parent f787b1b02a
commit 11997c024e
40 changed files with 4419 additions and 823 deletions

View File

@@ -0,0 +1,82 @@
from __future__ import annotations
import asyncio
from datetime import datetime, timezone
from types import SimpleNamespace
from unittest.mock import MagicMock
from src.api.admin.monitoring.trace import AdminGetRequestTraceAdapter
from src.services.request.candidate import RequestCandidateService
def _candidate(*, status: str, latency_ms: int | None = None) -> SimpleNamespace:
now = datetime.now(timezone.utc)
return SimpleNamespace(
id=f"cand-{status}",
request_id="req-1",
candidate_index=0,
retry_index=0,
provider_id=None,
endpoint_id=None,
key_id=None,
required_capabilities=None,
status=status,
skip_reason=None,
is_cached=False,
status_code=None,
error_type=None,
error_message=None,
latency_ms=latency_ms,
concurrent_requests=None,
extra_data=None,
created_at=now,
started_at=None,
finished_at=None,
)
def _context() -> SimpleNamespace:
return SimpleNamespace(
db=MagicMock(),
add_audit_metadata=lambda **_: None,
)
def test_trace_prefers_attempted_subset(monkeypatch: object) -> None:
candidates = [
_candidate(status="available"),
_candidate(status="unused"),
_candidate(status="failed", latency_ms=123),
]
monkeypatch.setattr(
RequestCandidateService,
"get_candidates_by_request_id",
lambda _db, _request_id: candidates,
)
adapter = AdminGetRequestTraceAdapter(request_id="req-1")
response = asyncio.run(adapter.handle(_context()))
assert response.total_candidates == 1
assert len(response.candidates) == 1
assert response.candidates[0].status == "failed"
assert response.total_latency_ms == 123
def test_trace_falls_back_to_all_when_no_attempted(monkeypatch: object) -> None:
candidates = [
_candidate(status="available"),
_candidate(status="unused"),
]
monkeypatch.setattr(
RequestCandidateService,
"get_candidates_by_request_id",
lambda _db, _request_id: candidates,
)
adapter = AdminGetRequestTraceAdapter(request_id="req-1")
response = asyncio.run(adapter.handle(_context()))
assert response.total_candidates == 2
assert len(response.candidates) == 2
assert {c.status for c in response.candidates} == {"available", "unused"}

View File

@@ -0,0 +1,128 @@
"""Tests for pool management scheduling state assembly."""
from __future__ import annotations
from types import SimpleNamespace
from src.api.admin.pool.routes import (
_build_pool_scheduling_state,
_is_known_banned_key,
_is_known_banned_reason,
)
def test_pool_scheduling_state_manual_disabled_is_blocked() -> None:
(
status,
reason,
_label,
reasons,
score,
candidate_eligible,
blocked_count,
_degraded_count,
dimensions,
) = _build_pool_scheduling_state(
is_active=False,
cooldown_reason=None,
cooldown_ttl_seconds=None,
circuit_breaker_open=False,
cost_window_usage=0,
cost_limit=None,
cost_soft_threshold_percent=80,
health_score=1.0,
)
assert status == "blocked"
assert reason == "manual_disabled"
assert candidate_eligible is False
assert blocked_count >= 1
assert score < 100
assert any(item.code == "manual_disabled" for item in reasons)
assert any(item.code == "manual_disabled" for item in dimensions)
def test_pool_scheduling_state_cooldown_detail_is_mapped() -> None:
(
_status,
reason,
_label,
reasons,
_score,
_candidate_eligible,
_blocked_count,
_degraded_count,
dimensions,
) = _build_pool_scheduling_state(
is_active=True,
cooldown_reason="rate_limited_429",
cooldown_ttl_seconds=180,
circuit_breaker_open=False,
cost_window_usage=0,
cost_limit=None,
cost_soft_threshold_percent=80,
health_score=1.0,
)
assert reason == "cooldown"
cooldown_reason = next(item for item in reasons if item.code == "cooldown")
cooldown_dimension = next(item for item in dimensions if item.code == "cooldown")
assert cooldown_reason.detail == "429 限流"
assert cooldown_dimension.detail == "429 限流"
def test_pool_scheduling_state_cost_soft_is_degraded() -> None:
(
status,
reason,
_label,
_reasons,
_score,
candidate_eligible,
blocked_count,
degraded_count,
_dimensions,
) = _build_pool_scheduling_state(
is_active=True,
cooldown_reason=None,
cooldown_ttl_seconds=None,
circuit_breaker_open=False,
cost_window_usage=85,
cost_limit=100,
cost_soft_threshold_percent=80,
health_score=1.0,
)
assert status == "degraded"
assert reason == "cost_soft"
assert candidate_eligible is True
assert blocked_count == 0
assert degraded_count >= 1
def test_known_banned_reason_account_block_prefix() -> None:
assert _is_known_banned_reason("[ACCOUNT_BLOCK] Google 要求验证账号") is True
def test_known_banned_key_detects_kiro_banned_metadata() -> None:
key = SimpleNamespace(
upstream_metadata={"kiro": {"is_banned": True}},
oauth_invalid_reason=None,
)
assert _is_known_banned_key(key, "kiro") is True
def test_known_banned_key_detects_reason_keywords() -> None:
key = SimpleNamespace(
upstream_metadata={},
oauth_invalid_reason="AWS account temporarily suspended",
)
assert _is_known_banned_key(key, "antigravity") is True
def test_known_banned_key_does_not_treat_token_expired_as_banned() -> None:
key = SimpleNamespace(
upstream_metadata={"kiro": {"is_banned": False}},
oauth_invalid_reason="access token expired",
)
assert _is_known_banned_key(key, "kiro") is False

View File

@@ -0,0 +1,154 @@
from src.api.handlers.base.base_handler import BaseMessageHandler
from src.services.candidate.schema import CandidateKey
def _handler() -> BaseMessageHandler:
return BaseMessageHandler.__new__(BaseMessageHandler)
def test_scheduling_audit_detects_internal_failover() -> None:
handler = _handler()
metadata = handler._merge_scheduling_metadata(
{},
candidate_keys=[
{
"candidate_index": 0,
"retry_index": 0,
"provider_id": "p1",
"provider_name": "provider-a",
"key_id": "k1",
"key_name": "account-a",
"status": "failed",
"status_code": 429,
},
{
"candidate_index": 1,
"retry_index": 0,
"provider_id": "p1",
"provider_name": "provider-a",
"key_id": "k2",
"key_name": "account-b",
"status": "success",
"status_code": 200,
},
],
selected_key_id="k2",
fallback_from_request=False,
)
assert metadata is not None
audit = metadata.get("scheduling_audit")
assert isinstance(audit, dict)
assert audit.get("attempted_count") == 2
assert audit.get("account_count") == 2
assert audit.get("retry_occurred") is True
assert audit.get("failover_occurred") is True
assert audit.get("selected_key_id") == "k2"
accounts = audit.get("accounts")
assert isinstance(accounts, list)
assert any(a.get("key_id") == "k2" and a.get("selected") for a in accounts)
def test_scheduling_audit_distinguishes_retry_from_failover() -> None:
handler = _handler()
metadata = handler._merge_scheduling_metadata(
{},
candidate_keys=[
{
"candidate_index": 0,
"retry_index": 0,
"provider_id": "p1",
"provider_name": "provider-a",
"key_id": "k1",
"key_name": "account-a",
"status": "failed",
"status_code": 503,
},
{
"candidate_index": 0,
"retry_index": 1,
"provider_id": "p1",
"provider_name": "provider-a",
"key_id": "k1",
"key_name": "account-a",
"status": "success",
"status_code": 200,
},
],
selected_key_id="k1",
fallback_from_request=False,
)
assert metadata is not None
audit = metadata.get("scheduling_audit")
assert isinstance(audit, dict)
assert audit.get("attempted_count") == 2
assert audit.get("account_count") == 1
assert audit.get("retry_occurred") is True
assert audit.get("failover_occurred") is False
def test_scheduling_metadata_supports_candidate_key_dataclass() -> None:
handler = _handler()
metadata = handler._merge_scheduling_metadata(
{},
candidate_keys=[
CandidateKey(
candidate_index=0,
retry_index=0,
provider_id="p1",
provider_name="provider-a",
endpoint_id="e1",
key_id="k1",
key_name="account-a",
status="success",
status_code=200,
)
],
selected_key_id="k1",
fallback_from_request=False,
)
assert metadata is not None
snapshots = metadata.get("candidate_keys")
assert isinstance(snapshots, list)
assert snapshots[0]["status"] == "success"
assert snapshots[0]["key_id"] == "k1"
def test_scheduling_audit_excludes_unused_candidates() -> None:
handler = _handler()
metadata = handler._merge_scheduling_metadata(
{},
candidate_keys=[
{
"candidate_index": 0,
"retry_index": 0,
"provider_id": "p1",
"provider_name": "provider-a",
"key_id": "k1",
"key_name": "account-a",
"status": "success",
"status_code": 200,
},
{
"candidate_index": 1,
"retry_index": 0,
"provider_id": "p1",
"provider_name": "provider-a",
"key_id": "k2",
"key_name": "account-b",
"status": "unused",
},
],
selected_key_id="k1",
fallback_from_request=False,
)
assert metadata is not None
audit = metadata.get("scheduling_audit")
assert isinstance(audit, dict)
assert audit.get("attempted_count") == 1
assert audit.get("account_count") == 1
attempts = audit.get("attempts")
assert isinstance(attempts, list)
assert len(attempts) == 1
assert attempts[0].get("key_id") == "k1"