feat(proxy,failover,transport): Hyper 上游客户端精细计时、连续失败退避与连接泄漏修复

Proxy:
- 将上游 HTTP 客户端从 reqwest 替换为 hyper,新增 InstrumentedConnector
  实现 TCP 连接/TLS 握手级别的独立计时,上报 connection_reused 等指标
- 前端展示细粒度代理计时(连接复用、等待响应头等)

Failover:
- 引入连续失败退避机制,每 10 次失败递增退避间隔
- 检测 H2 max outbound streams 错误并触发上游客户端重建
- 新增 HTTPClientPool.reset_upstream_client 支持按需重建缓存客户端

连接泄漏修复:
- Handler 异常路径确保 response_ctx 被正确关闭
- HubResponseStream 迭代结束后在 finally 块中清理 stream_id
- HubTunnelTransport.handle_request 捕获所有异常并清理流状态
This commit is contained in:
fawney19
2026-03-06 17:55:14 +08:00
parent 2269617a9f
commit 050cba9563
15 changed files with 902 additions and 72 deletions

View File

@@ -456,3 +456,49 @@ async def test_error_stop_pattern_without_status_codes_matches_any() -> None:
# No status_codes filter, pattern matches -> stop
assert result.success is False
assert attempt.await_count == 1
@pytest.mark.asyncio
async def test_failover_engine_applies_backoff_every_tenth_failure(
monkeypatch: pytest.MonkeyPatch,
) -> None:
db = MagicMock()
engine = FailoverEngine(db, error_classifier=_StubErrorClassifier(action=ErrorAction.BREAK))
sleep_mock = AsyncMock()
rotate_mock = AsyncMock(return_value=False)
monkeypatch.setattr("src.services.candidate.failover.asyncio.sleep", sleep_mock)
monkeypatch.setattr(engine, "_rotate_upstream_client", rotate_mock)
await engine._apply_retry_pacing(
candidate=_make_candidate(),
consecutive_failures=10,
error=RuntimeError("boom"),
request_id="req-1",
)
sleep_mock.assert_awaited_once()
rotate_mock.assert_not_awaited()
@pytest.mark.asyncio
async def test_failover_engine_rotates_client_on_stream_capacity_error(
monkeypatch: pytest.MonkeyPatch,
) -> None:
db = MagicMock()
engine = FailoverEngine(db, error_classifier=_StubErrorClassifier(action=ErrorAction.BREAK))
sleep_mock = AsyncMock()
rotate_mock = AsyncMock(return_value=True)
monkeypatch.setattr("src.services.candidate.failover.asyncio.sleep", sleep_mock)
monkeypatch.setattr(engine, "_rotate_upstream_client", rotate_mock)
await engine._apply_retry_pacing(
candidate=_make_candidate(),
consecutive_failures=3,
error=RuntimeError("LocalProtocolError: Max outbound streams is 100, 100 open"),
request_id="req-2",
)
rotate_mock.assert_awaited_once()
sleep_mock.assert_awaited_once()

View File

@@ -0,0 +1,47 @@
from __future__ import annotations
import pytest
from src.services.proxy_node.hub_transport import HubResponseStream
from src.services.proxy_node.tunnel_manager import _StreamState
class _Manager:
def __init__(self) -> None:
self.removed: list[int] = []
def remove_stream(self, stream_id: int) -> None:
self.removed.append(stream_id)
@pytest.mark.asyncio
async def test_hub_response_stream_removes_stream_after_normal_iteration() -> None:
manager = _Manager()
state = _StreamState(7)
state.set_response_headers(200, {})
state.push_body_chunk(b"hello")
state.set_done()
stream = HubResponseStream(manager, state, timeout=0.1)
chunks = []
async for chunk in stream:
chunks.append(chunk)
assert chunks == [b"hello"]
assert manager.removed == [7]
@pytest.mark.asyncio
async def test_hub_response_stream_removes_stream_after_body_error() -> None:
manager = _Manager()
state = _StreamState(9)
state.set_response_headers(200, {})
state.set_error("boom")
stream = HubResponseStream(manager, state, timeout=0.1)
with pytest.raises(Exception):
async for _chunk in stream:
pass
assert manager.removed == [9]