feat: 流式空闲超时、健康监控查询优化、限流桶内存上限与维护清理修复

Close #233

Co-authored-by: AAEE86 <ppk0227@hotmail.com>

- cli_monitor_mixin: 引入 STREAM_IDLE_TIMEOUT_SECONDS(可通过环境变量配置),
  流传输开始后若超出空闲窗口无新 chunk 则提前取消并返回 504,避免长时间挂起
- stream_context: 新增 managed_recorded_bodies 上下文管理器,确保 chunks 在
  telemetry 完成后及时释放;stream_telemetry 使用该接口统一管理 response body 构建
- health endpoint: 将状态聚合改为 GROUP BY 直接统计,事件列表按 api_format
  单独查询,避免单次 limit 拉取大量记录导致的遗漏与性能问题;同时过滤不活跃
  provider/endpoint,与公开健康接口保持一致
- endpoint health service: 修正时间线数据按 endpoint_id 而非 key_id 聚合
- token_bucket: 引入 max_buckets/bucket_expiry 上限与定时清理,防止内存无限增长;
  修复 refill_rate=0 时 get_reset_time 除零异常;新增 _is_unlimited_rate_limit 判断
- maintenance_scheduler: 调整清理顺序(先删整行再按窗口清理),新增 newer_than
  边界参数,避免同一行在同一轮中被重复改写
- sync_execute: 新增 create_pending_usage 开关,允许已预创建记录的调用方跳过重复创建
- quota_reader / provider_ops balance: 小幅修复与健壮性提升
- Dockerfile: 添加 MALLOC_ARENA_MAX=2 环境变量以降低 gunicorn worker RSS
- 补充相关测试覆盖
This commit is contained in:
fawney19
2026-03-18 23:38:26 +08:00
parent 3d5b6141a5
commit 1d72a8f9c1
37 changed files with 1787 additions and 607 deletions

View File

@@ -1,3 +1,9 @@
import os
import pytest
os.environ.setdefault("JWT_SECRET_KEY", "test-secret-key")
from src.api.handlers.base import stream_context
from src.api.handlers.base.stream_context import StreamContext
@@ -50,7 +56,36 @@ def test_reset_for_retry_clears_state() -> None:
assert ctx.error_message is None
def test_record_first_byte_time(monkeypatch) -> None:
def test_release_recorded_chunks_clears_both_chunk_lists() -> None:
ctx = StreamContext(model="test-model", api_format="openai:chat")
ctx.parsed_chunks.append({"type": "client"})
ctx.provider_parsed_chunks.append({"type": "provider"})
ctx.release_recorded_chunks()
assert ctx.parsed_chunks == []
assert ctx.provider_parsed_chunks == []
def test_managed_recorded_bodies_builds_then_releases_chunks() -> None:
ctx = StreamContext(model="test-model", api_format="openai:chat")
ctx.parsed_chunks.append({"type": "client"})
ctx.provider_parsed_chunks.append({"type": "provider"})
ctx.data_count = 1
with ctx.managed_recorded_bodies(123) as recorded_bodies:
assert recorded_bodies.response_body is not None
assert recorded_bodies.response_body["chunks"] == [{"type": "provider"}]
assert recorded_bodies.client_response_body is not None
assert recorded_bodies.client_response_body["chunks"] == [{"type": "client"}]
assert ctx.parsed_chunks == []
assert ctx.provider_parsed_chunks == []
assert recorded_bodies.response_body is None
assert recorded_bodies.client_response_body is None
def test_record_first_byte_time(monkeypatch: pytest.MonkeyPatch) -> None:
"""测试记录首字时间"""
ctx = StreamContext(model="claude-3", api_format="claude_messages")
start_time = 100.0
@@ -63,7 +98,7 @@ def test_record_first_byte_time(monkeypatch) -> None:
assert ctx.first_byte_time_ms == 12
def test_record_first_byte_time_idempotent(monkeypatch) -> None:
def test_record_first_byte_time_idempotent(monkeypatch: pytest.MonkeyPatch) -> None:
"""测试首字时间只记录一次"""
ctx = StreamContext(model="claude-3", api_format="claude_messages")
start_time = 100.0
@@ -82,7 +117,7 @@ def test_record_first_byte_time_idempotent(monkeypatch) -> None:
assert first_value == second_value
def test_reset_for_retry_clears_first_byte_time(monkeypatch) -> None:
def test_reset_for_retry_clears_first_byte_time(monkeypatch: pytest.MonkeyPatch) -> None:
"""测试重试时清除首字时间"""
ctx = StreamContext(model="claude-3", api_format="claude_messages")
start_time = 100.0