mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 01:10:23 +08:00
feat(hub,stability): bounded outbound queue、worker liveness 检测、事件循环 watchdog 及 DB 操作异步化
- aether-hub: unbounded channel 改为 bounded channel (BoundedOutbound),队列满时标记拥塞并主动关闭连接,防止内存无限增长 - aether-hub: worker idle timeout 从命令行参数改为基于心跳的 liveness 检测,默认 60 秒 - aether-hub: 新增 ConnConfig 统一管理连接配置,新增 outbound_queue_capacity 参数 - hub_transport: 新增事件循环 watchdog,检测 lag 超过阈值时临时降级暂停新流 - gunicorn_conf: 启用 faulthandler,worker abort 时自动 dump 全部线程栈用于诊断 - health/endpoint_checker/recording: 同步 DB 操作移至 asyncio.to_thread,避免阻塞事件循环 - Dockerfile: 移除 --worker-idle-timeout 0 命令行参数,改由环境变量和默认值控制
This commit is contained in:
66
tests/unit/test_hub_transport_watchdog.py
Normal file
66
tests/unit/test_hub_transport_watchdog.py
Normal file
@@ -0,0 +1,66 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from src.services.proxy_node.hub_config import HubConfig
|
||||
from src.services.proxy_node.hub_transport import HubConnectionManager
|
||||
from src.services.proxy_node.tunnel_manager import TunnelStreamError
|
||||
|
||||
|
||||
def _build_manager() -> HubConnectionManager:
|
||||
return HubConnectionManager(
|
||||
HubConfig(
|
||||
enabled=True,
|
||||
url="ws://127.0.0.1:8085",
|
||||
connect_timeout_seconds=1.0,
|
||||
ping_interval_seconds=1.0,
|
||||
send_timeout_seconds=1.0,
|
||||
max_streams=16,
|
||||
max_frame_size=1024 * 1024,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_record_loop_lag_warning_does_not_degrade() -> None:
|
||||
manager = _build_manager()
|
||||
|
||||
manager._record_loop_lag(1.5)
|
||||
|
||||
assert manager._degraded_until == 0.0
|
||||
|
||||
|
||||
def test_record_loop_lag_degrades_manager(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
manager = _build_manager()
|
||||
now = 1234.0
|
||||
monkeypatch.setattr("src.services.proxy_node.hub_transport._time.monotonic", lambda: now)
|
||||
|
||||
manager._record_loop_lag(4.0)
|
||||
|
||||
assert manager._degraded_until == pytest.approx(now + 12.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_request_rejects_while_manager_degraded(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
manager = _build_manager()
|
||||
manager._degraded_until = time.monotonic() + 5.0
|
||||
|
||||
async def _fake_ensure_connected() -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(manager, "ensure_connected", _fake_ensure_connected)
|
||||
|
||||
with pytest.raises(TunnelStreamError, match="event loop degraded"):
|
||||
await manager.send_request(
|
||||
"node-1",
|
||||
method="POST",
|
||||
url="https://example.com/v1/chat/completions",
|
||||
headers={"content-type": "application/json"},
|
||||
body=b"{}",
|
||||
timeout=5.0,
|
||||
)
|
||||
|
||||
assert manager._pending_streams == {}
|
||||
Reference in New Issue
Block a user