feat(hub,stability): bounded outbound queue、worker liveness 检测、事件循环 watchdog 及 DB 操作异步化

- aether-hub: unbounded channel 改为 bounded channel (BoundedOutbound),队列满时标记拥塞并主动关闭连接,防止内存无限增长
- aether-hub: worker idle timeout 从命令行参数改为基于心跳的 liveness 检测,默认 60 秒
- aether-hub: 新增 ConnConfig 统一管理连接配置,新增 outbound_queue_capacity 参数
- hub_transport: 新增事件循环 watchdog,检测 lag 超过阈值时临时降级暂停新流
- gunicorn_conf: 启用 faulthandler,worker abort 时自动 dump 全部线程栈用于诊断
- health/endpoint_checker/recording: 同步 DB 操作移至 asyncio.to_thread,避免阻塞事件循环
- Dockerfile: 移除 --worker-idle-timeout 0 命令行参数,改由环境变量和默认值控制
This commit is contained in:
fawney19
2026-03-12 12:19:04 +08:00
parent 4d338ebd3d
commit 71ae1a2307
14 changed files with 676 additions and 280 deletions

View File

@@ -0,0 +1,66 @@
from __future__ import annotations
import time
import pytest
from src.services.proxy_node.hub_config import HubConfig
from src.services.proxy_node.hub_transport import HubConnectionManager
from src.services.proxy_node.tunnel_manager import TunnelStreamError
def _build_manager() -> HubConnectionManager:
return HubConnectionManager(
HubConfig(
enabled=True,
url="ws://127.0.0.1:8085",
connect_timeout_seconds=1.0,
ping_interval_seconds=1.0,
send_timeout_seconds=1.0,
max_streams=16,
max_frame_size=1024 * 1024,
)
)
def test_record_loop_lag_warning_does_not_degrade() -> None:
manager = _build_manager()
manager._record_loop_lag(1.5)
assert manager._degraded_until == 0.0
def test_record_loop_lag_degrades_manager(monkeypatch: pytest.MonkeyPatch) -> None:
manager = _build_manager()
now = 1234.0
monkeypatch.setattr("src.services.proxy_node.hub_transport._time.monotonic", lambda: now)
manager._record_loop_lag(4.0)
assert manager._degraded_until == pytest.approx(now + 12.0)
@pytest.mark.asyncio
async def test_send_request_rejects_while_manager_degraded(
monkeypatch: pytest.MonkeyPatch,
) -> None:
manager = _build_manager()
manager._degraded_until = time.monotonic() + 5.0
async def _fake_ensure_connected() -> None:
return None
monkeypatch.setattr(manager, "ensure_connected", _fake_ensure_connected)
with pytest.raises(TunnelStreamError, match="event loop degraded"):
await manager.send_request(
"node-1",
method="POST",
url="https://example.com/v1/chat/completions",
headers={"content-type": "application/json"},
body=b"{}",
timeout=5.0,
)
assert manager._pending_streams == {}