Files
Aether/tests/api/test_health_monitor_api_formats.py
fawney19 1d72a8f9c1 feat: 流式空闲超时、健康监控查询优化、限流桶内存上限与维护清理修复
Close #233

Co-authored-by: AAEE86 <ppk0227@hotmail.com>

- cli_monitor_mixin: 引入 STREAM_IDLE_TIMEOUT_SECONDS(可通过环境变量配置),
  流传输开始后若超出空闲窗口无新 chunk 则提前取消并返回 504,避免长时间挂起
- stream_context: 新增 managed_recorded_bodies 上下文管理器,确保 chunks 在
  telemetry 完成后及时释放;stream_telemetry 使用该接口统一管理 response body 构建
- health endpoint: 将状态聚合改为 GROUP BY 直接统计,事件列表按 api_format
  单独查询,避免单次 limit 拉取大量记录导致的遗漏与性能问题;同时过滤不活跃
  provider/endpoint,与公开健康接口保持一致
- endpoint health service: 修正时间线数据按 endpoint_id 而非 key_id 聚合
- token_bucket: 引入 max_buckets/bucket_expiry 上限与定时清理,防止内存无限增长;
  修复 refill_rate=0 时 get_reset_time 除零异常;新增 _is_unlimited_rate_limit 判断
- maintenance_scheduler: 调整清理顺序(先删整行再按窗口清理),新增 newer_than
  边界参数,避免同一行在同一轮中被重复改写
- sync_execute: 新增 create_pending_usage 开关,允许已预创建记录的调用方跳过重复创建
- quota_reader / provider_ops balance: 小幅修复与健壮性提升
- Dockerfile: 添加 MALLOC_ARENA_MAX=2 环境变量以降低 gunicorn worker RSS
- 补充相关测试覆盖
2026-03-18 23:38:26 +08:00

147 lines
4.7 KiB
Python

from __future__ import annotations
from datetime import datetime, timezone
from types import SimpleNamespace
from typing import Any, cast
from unittest.mock import MagicMock
import pytest
from src.api.admin.endpoints.health import AdminApiFormatHealthMonitorAdapter
from src.api.public.catalog import PublicApiFormatHealthMonitorAdapter
def _build_query(result: object) -> MagicMock:
query = MagicMock()
query.join.return_value = query
query.distinct.return_value = query
query.filter.return_value = query
query.group_by.return_value = query
query.order_by.return_value = query
query.limit.return_value = query
query.all.return_value = result
return query
def _expr_texts(query: MagicMock) -> list[str]:
return [str(arg) for arg in query.filter.call_args.args]
@pytest.mark.asyncio
async def test_admin_api_format_health_monitor_filters_inactive_sources(
monkeypatch: pytest.MonkeyPatch,
) -> None:
endpoint_query = _build_query([("openai:compact", "ep-active", "provider-active")])
key_query = _build_query([("provider-active", ["openai:compact"])])
status_query = _build_query([("openai:compact", "success", 3)])
rows_query = _build_query([])
db = MagicMock()
db.query.side_effect = [endpoint_query, key_query, status_query, rows_query]
monkeypatch.setattr(
"src.api.admin.endpoints.health.EndpointHealthService._generate_timeline_from_usage",
lambda **_: {
"timeline": ["healthy"] * 100,
"time_range_start": None,
"time_range_end": None,
},
)
context = SimpleNamespace(
db=db,
request=SimpleNamespace(state=SimpleNamespace()),
add_audit_metadata=lambda **kwargs: None,
)
adapter = AdminApiFormatHealthMonitorAdapter(lookback_hours=6, per_format_limit=20)
await adapter.handle(cast(Any, context))
status_filters = _expr_texts(status_query)
rows_filters = _expr_texts(rows_query)
assert any("provider_endpoints.is_active" in expr for expr in status_filters)
assert any("providers.is_active" in expr for expr in status_filters)
assert any("provider_endpoints.is_active" in expr for expr in rows_filters)
assert any("providers.is_active" in expr for expr in rows_filters)
@pytest.mark.asyncio
async def test_public_api_format_health_monitor_uses_real_counts_not_sampled_events(
monkeypatch: pytest.MonkeyPatch,
) -> None:
now = datetime.now(timezone.utc)
active_formats_query = _build_query([("openai:compact",)])
endpoint_rows_query = _build_query([("openai:compact", "ep-active")])
status_query = _build_query(
[
("openai:compact", "success", 7),
("openai:compact", "failed", 3),
("openai:compact", "skipped", 5),
]
)
rows_query = _build_query(
[
SimpleNamespace(
status="failed",
status_code=500,
latency_ms=321,
error_type="provider_error",
finished_at=now,
started_at=None,
created_at=now,
),
SimpleNamespace(
status="success",
status_code=200,
latency_ms=123,
error_type=None,
finished_at=now,
started_at=None,
created_at=now,
),
]
)
db = MagicMock()
db.query.side_effect = [
active_formats_query,
endpoint_rows_query,
status_query,
rows_query,
]
monkeypatch.setattr(
"src.api.public.catalog.EndpointHealthService._generate_timeline_from_usage",
lambda **_: {
"timeline": ["healthy"] * 100,
"time_range_start": None,
"time_range_end": now,
},
)
monkeypatch.setattr(
"src.core.api_format.get_local_path_for_endpoint",
lambda api_format: f"/{api_format}",
)
context = SimpleNamespace(
db=db,
request=SimpleNamespace(state=SimpleNamespace()),
)
adapter = PublicApiFormatHealthMonitorAdapter(lookback_hours=6, per_format_limit=20)
result = await adapter.handle(cast(Any, context))
monitor = result["formats"][0]
assert monitor["api_format"] == "openai:compact"
assert monitor["total_attempts"] == 15
assert monitor["success_count"] == 7
assert monitor["failed_count"] == 3
assert monitor["skipped_count"] == 5
assert monitor["success_rate"] == pytest.approx(0.7)
assert len(monitor["events"]) == 2
rows_filters = _expr_texts(rows_query)
assert any("provider_endpoints.is_active" in expr for expr in rows_filters)
assert any("providers.is_active" in expr for expr in rows_filters)