mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-01 17:00:21 +08:00
feat(pool,scheduling): 号池调度维度、配额冷却机制与管理后台重构
- 新增 scheduling_dimensions 模块,为每个 Key 计算多维调度状态(手动/冷却/熔断/成本/健康) - 新增 quota_cooldown 模块,统一判定 Key 的有效冷却原因 - Pool 管理后台 API 扩展 Key 详情字段(调度状态/维度/配额/OAuth 信息) - 前端 Pool 管理页面重写,支持调度状态展示、批量清理封禁 Key - Handler 基类增加请求调度元数据采集,stream telemetry 增强 - 请求时间线组件增强,支持 attempted 候选展示 - Kiro OAuth 凭证导入解析改进 - 新增 usage 表 provider_key 索引迁移 - 补充调度维度、配额冷却、候选枚举等单元测试 Closes #197 Co-authored-by: AAEE86 <33052466+AAEE86@users.noreply.github.com>
This commit is contained in:
82
tests/unit/test_monitoring_trace_attempted_subset.py
Normal file
82
tests/unit/test_monitoring_trace_attempted_subset.py
Normal file
@@ -0,0 +1,82 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from src.api.admin.monitoring.trace import AdminGetRequestTraceAdapter
|
||||
from src.services.request.candidate import RequestCandidateService
|
||||
|
||||
|
||||
def _candidate(*, status: str, latency_ms: int | None = None) -> SimpleNamespace:
|
||||
now = datetime.now(timezone.utc)
|
||||
return SimpleNamespace(
|
||||
id=f"cand-{status}",
|
||||
request_id="req-1",
|
||||
candidate_index=0,
|
||||
retry_index=0,
|
||||
provider_id=None,
|
||||
endpoint_id=None,
|
||||
key_id=None,
|
||||
required_capabilities=None,
|
||||
status=status,
|
||||
skip_reason=None,
|
||||
is_cached=False,
|
||||
status_code=None,
|
||||
error_type=None,
|
||||
error_message=None,
|
||||
latency_ms=latency_ms,
|
||||
concurrent_requests=None,
|
||||
extra_data=None,
|
||||
created_at=now,
|
||||
started_at=None,
|
||||
finished_at=None,
|
||||
)
|
||||
|
||||
|
||||
def _context() -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
db=MagicMock(),
|
||||
add_audit_metadata=lambda **_: None,
|
||||
)
|
||||
|
||||
|
||||
def test_trace_prefers_attempted_subset(monkeypatch: object) -> None:
|
||||
candidates = [
|
||||
_candidate(status="available"),
|
||||
_candidate(status="unused"),
|
||||
_candidate(status="failed", latency_ms=123),
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
RequestCandidateService,
|
||||
"get_candidates_by_request_id",
|
||||
lambda _db, _request_id: candidates,
|
||||
)
|
||||
|
||||
adapter = AdminGetRequestTraceAdapter(request_id="req-1")
|
||||
response = asyncio.run(adapter.handle(_context()))
|
||||
|
||||
assert response.total_candidates == 1
|
||||
assert len(response.candidates) == 1
|
||||
assert response.candidates[0].status == "failed"
|
||||
assert response.total_latency_ms == 123
|
||||
|
||||
|
||||
def test_trace_falls_back_to_all_when_no_attempted(monkeypatch: object) -> None:
|
||||
candidates = [
|
||||
_candidate(status="available"),
|
||||
_candidate(status="unused"),
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
RequestCandidateService,
|
||||
"get_candidates_by_request_id",
|
||||
lambda _db, _request_id: candidates,
|
||||
)
|
||||
|
||||
adapter = AdminGetRequestTraceAdapter(request_id="req-1")
|
||||
response = asyncio.run(adapter.handle(_context()))
|
||||
|
||||
assert response.total_candidates == 2
|
||||
assert len(response.candidates) == 2
|
||||
assert {c.status for c in response.candidates} == {"available", "unused"}
|
||||
128
tests/unit/test_pool_management_scheduling_state.py
Normal file
128
tests/unit/test_pool_management_scheduling_state.py
Normal file
@@ -0,0 +1,128 @@
|
||||
"""Tests for pool management scheduling state assembly."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from src.api.admin.pool.routes import (
|
||||
_build_pool_scheduling_state,
|
||||
_is_known_banned_key,
|
||||
_is_known_banned_reason,
|
||||
)
|
||||
|
||||
|
||||
def test_pool_scheduling_state_manual_disabled_is_blocked() -> None:
|
||||
(
|
||||
status,
|
||||
reason,
|
||||
_label,
|
||||
reasons,
|
||||
score,
|
||||
candidate_eligible,
|
||||
blocked_count,
|
||||
_degraded_count,
|
||||
dimensions,
|
||||
) = _build_pool_scheduling_state(
|
||||
is_active=False,
|
||||
cooldown_reason=None,
|
||||
cooldown_ttl_seconds=None,
|
||||
circuit_breaker_open=False,
|
||||
cost_window_usage=0,
|
||||
cost_limit=None,
|
||||
cost_soft_threshold_percent=80,
|
||||
health_score=1.0,
|
||||
)
|
||||
|
||||
assert status == "blocked"
|
||||
assert reason == "manual_disabled"
|
||||
assert candidate_eligible is False
|
||||
assert blocked_count >= 1
|
||||
assert score < 100
|
||||
assert any(item.code == "manual_disabled" for item in reasons)
|
||||
assert any(item.code == "manual_disabled" for item in dimensions)
|
||||
|
||||
|
||||
def test_pool_scheduling_state_cooldown_detail_is_mapped() -> None:
|
||||
(
|
||||
_status,
|
||||
reason,
|
||||
_label,
|
||||
reasons,
|
||||
_score,
|
||||
_candidate_eligible,
|
||||
_blocked_count,
|
||||
_degraded_count,
|
||||
dimensions,
|
||||
) = _build_pool_scheduling_state(
|
||||
is_active=True,
|
||||
cooldown_reason="rate_limited_429",
|
||||
cooldown_ttl_seconds=180,
|
||||
circuit_breaker_open=False,
|
||||
cost_window_usage=0,
|
||||
cost_limit=None,
|
||||
cost_soft_threshold_percent=80,
|
||||
health_score=1.0,
|
||||
)
|
||||
|
||||
assert reason == "cooldown"
|
||||
cooldown_reason = next(item for item in reasons if item.code == "cooldown")
|
||||
cooldown_dimension = next(item for item in dimensions if item.code == "cooldown")
|
||||
assert cooldown_reason.detail == "429 限流"
|
||||
assert cooldown_dimension.detail == "429 限流"
|
||||
|
||||
|
||||
def test_pool_scheduling_state_cost_soft_is_degraded() -> None:
|
||||
(
|
||||
status,
|
||||
reason,
|
||||
_label,
|
||||
_reasons,
|
||||
_score,
|
||||
candidate_eligible,
|
||||
blocked_count,
|
||||
degraded_count,
|
||||
_dimensions,
|
||||
) = _build_pool_scheduling_state(
|
||||
is_active=True,
|
||||
cooldown_reason=None,
|
||||
cooldown_ttl_seconds=None,
|
||||
circuit_breaker_open=False,
|
||||
cost_window_usage=85,
|
||||
cost_limit=100,
|
||||
cost_soft_threshold_percent=80,
|
||||
health_score=1.0,
|
||||
)
|
||||
|
||||
assert status == "degraded"
|
||||
assert reason == "cost_soft"
|
||||
assert candidate_eligible is True
|
||||
assert blocked_count == 0
|
||||
assert degraded_count >= 1
|
||||
|
||||
|
||||
def test_known_banned_reason_account_block_prefix() -> None:
|
||||
assert _is_known_banned_reason("[ACCOUNT_BLOCK] Google 要求验证账号") is True
|
||||
|
||||
|
||||
def test_known_banned_key_detects_kiro_banned_metadata() -> None:
|
||||
key = SimpleNamespace(
|
||||
upstream_metadata={"kiro": {"is_banned": True}},
|
||||
oauth_invalid_reason=None,
|
||||
)
|
||||
assert _is_known_banned_key(key, "kiro") is True
|
||||
|
||||
|
||||
def test_known_banned_key_detects_reason_keywords() -> None:
|
||||
key = SimpleNamespace(
|
||||
upstream_metadata={},
|
||||
oauth_invalid_reason="AWS account temporarily suspended",
|
||||
)
|
||||
assert _is_known_banned_key(key, "antigravity") is True
|
||||
|
||||
|
||||
def test_known_banned_key_does_not_treat_token_expired_as_banned() -> None:
|
||||
key = SimpleNamespace(
|
||||
upstream_metadata={"kiro": {"is_banned": False}},
|
||||
oauth_invalid_reason="access token expired",
|
||||
)
|
||||
assert _is_known_banned_key(key, "kiro") is False
|
||||
154
tests/unit/test_request_scheduling_metadata.py
Normal file
154
tests/unit/test_request_scheduling_metadata.py
Normal file
@@ -0,0 +1,154 @@
|
||||
from src.api.handlers.base.base_handler import BaseMessageHandler
|
||||
from src.services.candidate.schema import CandidateKey
|
||||
|
||||
|
||||
def _handler() -> BaseMessageHandler:
|
||||
return BaseMessageHandler.__new__(BaseMessageHandler)
|
||||
|
||||
|
||||
def test_scheduling_audit_detects_internal_failover() -> None:
|
||||
handler = _handler()
|
||||
metadata = handler._merge_scheduling_metadata(
|
||||
{},
|
||||
candidate_keys=[
|
||||
{
|
||||
"candidate_index": 0,
|
||||
"retry_index": 0,
|
||||
"provider_id": "p1",
|
||||
"provider_name": "provider-a",
|
||||
"key_id": "k1",
|
||||
"key_name": "account-a",
|
||||
"status": "failed",
|
||||
"status_code": 429,
|
||||
},
|
||||
{
|
||||
"candidate_index": 1,
|
||||
"retry_index": 0,
|
||||
"provider_id": "p1",
|
||||
"provider_name": "provider-a",
|
||||
"key_id": "k2",
|
||||
"key_name": "account-b",
|
||||
"status": "success",
|
||||
"status_code": 200,
|
||||
},
|
||||
],
|
||||
selected_key_id="k2",
|
||||
fallback_from_request=False,
|
||||
)
|
||||
assert metadata is not None
|
||||
audit = metadata.get("scheduling_audit")
|
||||
assert isinstance(audit, dict)
|
||||
assert audit.get("attempted_count") == 2
|
||||
assert audit.get("account_count") == 2
|
||||
assert audit.get("retry_occurred") is True
|
||||
assert audit.get("failover_occurred") is True
|
||||
assert audit.get("selected_key_id") == "k2"
|
||||
|
||||
accounts = audit.get("accounts")
|
||||
assert isinstance(accounts, list)
|
||||
assert any(a.get("key_id") == "k2" and a.get("selected") for a in accounts)
|
||||
|
||||
|
||||
def test_scheduling_audit_distinguishes_retry_from_failover() -> None:
|
||||
handler = _handler()
|
||||
metadata = handler._merge_scheduling_metadata(
|
||||
{},
|
||||
candidate_keys=[
|
||||
{
|
||||
"candidate_index": 0,
|
||||
"retry_index": 0,
|
||||
"provider_id": "p1",
|
||||
"provider_name": "provider-a",
|
||||
"key_id": "k1",
|
||||
"key_name": "account-a",
|
||||
"status": "failed",
|
||||
"status_code": 503,
|
||||
},
|
||||
{
|
||||
"candidate_index": 0,
|
||||
"retry_index": 1,
|
||||
"provider_id": "p1",
|
||||
"provider_name": "provider-a",
|
||||
"key_id": "k1",
|
||||
"key_name": "account-a",
|
||||
"status": "success",
|
||||
"status_code": 200,
|
||||
},
|
||||
],
|
||||
selected_key_id="k1",
|
||||
fallback_from_request=False,
|
||||
)
|
||||
assert metadata is not None
|
||||
audit = metadata.get("scheduling_audit")
|
||||
assert isinstance(audit, dict)
|
||||
assert audit.get("attempted_count") == 2
|
||||
assert audit.get("account_count") == 1
|
||||
assert audit.get("retry_occurred") is True
|
||||
assert audit.get("failover_occurred") is False
|
||||
|
||||
|
||||
def test_scheduling_metadata_supports_candidate_key_dataclass() -> None:
|
||||
handler = _handler()
|
||||
metadata = handler._merge_scheduling_metadata(
|
||||
{},
|
||||
candidate_keys=[
|
||||
CandidateKey(
|
||||
candidate_index=0,
|
||||
retry_index=0,
|
||||
provider_id="p1",
|
||||
provider_name="provider-a",
|
||||
endpoint_id="e1",
|
||||
key_id="k1",
|
||||
key_name="account-a",
|
||||
status="success",
|
||||
status_code=200,
|
||||
)
|
||||
],
|
||||
selected_key_id="k1",
|
||||
fallback_from_request=False,
|
||||
)
|
||||
assert metadata is not None
|
||||
snapshots = metadata.get("candidate_keys")
|
||||
assert isinstance(snapshots, list)
|
||||
assert snapshots[0]["status"] == "success"
|
||||
assert snapshots[0]["key_id"] == "k1"
|
||||
|
||||
|
||||
def test_scheduling_audit_excludes_unused_candidates() -> None:
|
||||
handler = _handler()
|
||||
metadata = handler._merge_scheduling_metadata(
|
||||
{},
|
||||
candidate_keys=[
|
||||
{
|
||||
"candidate_index": 0,
|
||||
"retry_index": 0,
|
||||
"provider_id": "p1",
|
||||
"provider_name": "provider-a",
|
||||
"key_id": "k1",
|
||||
"key_name": "account-a",
|
||||
"status": "success",
|
||||
"status_code": 200,
|
||||
},
|
||||
{
|
||||
"candidate_index": 1,
|
||||
"retry_index": 0,
|
||||
"provider_id": "p1",
|
||||
"provider_name": "provider-a",
|
||||
"key_id": "k2",
|
||||
"key_name": "account-b",
|
||||
"status": "unused",
|
||||
},
|
||||
],
|
||||
selected_key_id="k1",
|
||||
fallback_from_request=False,
|
||||
)
|
||||
|
||||
assert metadata is not None
|
||||
audit = metadata.get("scheduling_audit")
|
||||
assert isinstance(audit, dict)
|
||||
assert audit.get("attempted_count") == 1
|
||||
assert audit.get("account_count") == 1
|
||||
attempts = audit.get("attempts")
|
||||
assert isinstance(attempts, list)
|
||||
assert len(attempts) == 1
|
||||
assert attempts[0].get("key_id") == "k1"
|
||||
Reference in New Issue
Block a user