Files
Aether/tests/unit/test_endpoint_health_timeline.py
fawney19 1d72a8f9c1 feat: 流式空闲超时、健康监控查询优化、限流桶内存上限与维护清理修复
Close #233

Co-authored-by: AAEE86 <ppk0227@hotmail.com>

- cli_monitor_mixin: 引入 STREAM_IDLE_TIMEOUT_SECONDS(可通过环境变量配置),
  流传输开始后若超出空闲窗口无新 chunk 则提前取消并返回 504,避免长时间挂起
- stream_context: 新增 managed_recorded_bodies 上下文管理器,确保 chunks 在
  telemetry 完成后及时释放;stream_telemetry 使用该接口统一管理 response body 构建
- health endpoint: 将状态聚合改为 GROUP BY 直接统计,事件列表按 api_format
  单独查询,避免单次 limit 拉取大量记录导致的遗漏与性能问题;同时过滤不活跃
  provider/endpoint,与公开健康接口保持一致
- endpoint health service: 修正时间线数据按 endpoint_id 而非 key_id 聚合
- token_bucket: 引入 max_buckets/bucket_expiry 上限与定时清理,防止内存无限增长;
  修复 refill_rate=0 时 get_reset_time 除零异常;新增 _is_unlimited_rate_limit 判断
- maintenance_scheduler: 调整清理顺序(先删整行再按窗口清理),新增 newer_than
  边界参数,避免同一行在同一轮中被重复改写
- sync_execute: 新增 create_pending_usage 开关,允许已预创建记录的调用方跳过重复创建
- quota_reader / provider_ops balance: 小幅修复与健壮性提升
- Dockerfile: 添加 MALLOC_ARENA_MAX=2 环境变量以降低 gunicorn worker RSS
- 补充相关测试覆盖
2026-03-18 23:38:26 +08:00

174 lines
4.9 KiB
Python

from __future__ import annotations
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from typing import Any, cast
from unittest.mock import MagicMock
import pytest
from src.services.health.endpoint import EndpointHealthService
class _FakeQuery:
def __init__(self, rows: list[SimpleNamespace]) -> None:
self._rows = rows
def filter(self, *args: Any, **kwargs: Any) -> _FakeQuery:
return self
def group_by(self, *args: Any, **kwargs: Any) -> _FakeQuery:
return self
def all(self) -> list[SimpleNamespace]:
return self._rows
class _FakeDb:
def __init__(self, rows: list[SimpleNamespace]) -> None:
self._rows = rows
def query(self, *args: Any, **kwargs: Any) -> _FakeQuery:
return _FakeQuery(self._rows)
def _expr_texts(query: MagicMock) -> list[str]:
return [str(arg) for arg in query.filter.call_args.args]
def test_generate_timeline_batch_keeps_compact_and_cli_isolated() -> None:
now = datetime(2026, 3, 18, 12, 0, tzinfo=timezone.utc)
db = _FakeDb(
[
SimpleNamespace(
endpoint_id="endpoint-compact",
segment_idx=0,
total_count=2,
success_count=2,
failed_count=0,
min_time=now - timedelta(minutes=55),
max_time=now - timedelta(minutes=40),
),
SimpleNamespace(
endpoint_id="endpoint-cli",
segment_idx=0,
total_count=3,
success_count=0,
failed_count=3,
min_time=now - timedelta(minutes=54),
max_time=now - timedelta(minutes=39),
),
]
)
result = EndpointHealthService._generate_timeline_batch(
db=cast(Any, db),
format_endpoint_mapping={
"openai:compact": ["endpoint-compact"],
"openai:cli": ["endpoint-cli"],
},
now=now,
lookback_hours=1,
segments=4,
)
assert result["openai:compact"]["timeline"][0] == "healthy"
assert result["openai:cli"]["timeline"][0] == "unhealthy"
def test_generate_timeline_from_usage_uses_endpoint_ids_directly(
monkeypatch: pytest.MonkeyPatch,
) -> None:
expected = {
"timeline": ["healthy", "warning"],
"time_range_start": "start",
"time_range_end": "end",
}
captured: dict[str, object] = {}
def _fake_generate_timeline_batch(
db: Any,
format_endpoint_mapping: dict[str, list[str]],
now: datetime,
lookback_hours: int,
segments: int,
) -> dict[str, dict[str, Any]]:
captured["db"] = db
captured["mapping"] = format_endpoint_mapping
captured["lookback_hours"] = lookback_hours
captured["segments"] = segments
return {"_single": expected}
monkeypatch.setattr(
EndpointHealthService,
"_generate_timeline_batch",
staticmethod(_fake_generate_timeline_batch),
)
db = cast(Any, object())
now = datetime(2026, 3, 18, 12, 0, tzinfo=timezone.utc)
result = EndpointHealthService._generate_timeline_from_usage(
db=db,
endpoint_ids=["endpoint-compact"],
now=now,
lookback_hours=6,
segments=2,
)
assert result == expected
assert captured["db"] is db
assert captured["mapping"] == {"_single": ["endpoint-compact"]}
assert captured["lookback_hours"] == 6
assert captured["segments"] == 2
def test_get_endpoint_health_by_format_filters_inactive_endpoints(
monkeypatch: pytest.MonkeyPatch,
) -> None:
endpoint_query = MagicMock()
endpoint_query.join.return_value = endpoint_query
endpoint_query.filter.return_value = endpoint_query
endpoint_query.all.return_value = [
SimpleNamespace(
id="endpoint-compact",
provider_id="provider-1",
api_format="openai:compact",
is_active=True,
)
]
key_query = MagicMock()
key_query.filter.return_value = key_query
key_query.options.return_value = key_query
key_query.all.return_value = []
db = MagicMock()
db.query.side_effect = [endpoint_query, key_query]
monkeypatch.setattr(
EndpointHealthService,
"_generate_timeline_batch",
staticmethod(
lambda db, format_endpoint_mapping, now, lookback_hours: {
"openai:compact": {
"timeline": ["unknown"] * 100,
"time_range_start": None,
"time_range_end": None,
}
}
),
)
EndpointHealthService.get_endpoint_health_by_format(
db=cast(Any, db),
lookback_hours=6,
include_admin_fields=False,
use_cache=False,
)
filters = _expr_texts(endpoint_query)
assert any("provider_endpoints.is_active" in expr for expr in filters)
assert any("providers.is_active" in expr for expr in filters)