feat(proxy-tunnel): 增强隧道连接稳定性与恢复速度

- writer 增加 WebSocket Ping keepalive,防止中间代理空闲超时断开
- 服务端增加应用层 PING 循环(30s 间隔),空闲超时延长至 180s
- 重连基础延迟从 1000ms 降低到 500ms
- 节点缓存 TTL 缩短至 15s,不可用节点 TTL 缩短至 5s 加速恢复感知
- 心跳检测间隔从 30s 缩短到 15s
This commit is contained in:
fawney19
2026-02-27 19:51:38 +08:00
parent 934723f5f5
commit a174cf1b02
6 changed files with 69 additions and 18 deletions

View File

@@ -16,15 +16,18 @@ from src.services.proxy_node.tunnel_manager import (
TunnelConnection,
get_tunnel_manager,
)
from src.services.proxy_node.tunnel_protocol import Frame
from src.services.proxy_node.tunnel_protocol import Frame, MsgType
router = APIRouter()
# 单帧最大 64 MB -- AI API 请求体可能包含多张 base64 图片,需要足够余量
_MAX_FRAME_SIZE = 64 * 1024 * 1024
# WebSocket 空闲超时(秒)-- proxy 端 ping 间隔默认 15s3 倍余量
_IDLE_TIMEOUT = 90.0
# WebSocket 空闲超时(秒)-- proxy 端 WebSocket ping 间隔 15s + 心跳 30s180s 提供充足余量
_IDLE_TIMEOUT = 180.0
# 服务端应用层 ping 间隔(秒)-- 确保即使 proxy 端心跳延迟,连接也不会因中间代理空闲超时而断开
_SERVER_PING_INTERVAL = 30.0
async def _authenticate(ws: WebSocket) -> tuple[str, str] | None:
@@ -99,6 +102,9 @@ async def proxy_tunnel_ws(ws: WebSocket) -> None:
# 更新 DB: tunnel_connected = True
await _update_tunnel_status(node_id, connected=True)
# 启动服务端 ping 任务,防止中间代理因空闲超时关闭连接
ping_task = asyncio.create_task(_ping_loop(conn))
try:
oversized_count = 0
while True:
@@ -130,10 +136,27 @@ async def proxy_tunnel_ws(ws: WebSocket) -> None:
except Exception as e:
logger.error("tunnel WebSocket error for node_id={}: {}", node_id, e)
finally:
ping_task.cancel()
manager.unregister(node_id)
await _update_tunnel_status(node_id, connected=False)
async def _ping_loop(conn: TunnelConnection) -> None:
"""定期发送应用层 PING 帧,保持连接活跃"""
try:
while True:
await asyncio.sleep(_SERVER_PING_INTERVAL)
if not conn.is_alive:
break
try:
await conn.send_frame(Frame(0, MsgType.PING, 0, b""))
except Exception as e:
logger.debug("ping loop send failed for node_id={}: {}", conn.node_id, e)
break
except asyncio.CancelledError:
pass
async def _update_tunnel_status(node_id: str, *, connected: bool) -> None:
"""更新 ProxyNode 的 tunnel 连接状态(在线程池中执行,避免阻塞 event loop"""

View File

@@ -35,7 +35,7 @@ class ProxyNodeHealthScheduler:
scheduler = get_scheduler()
scheduler.add_interval_job(
self._scheduled_check,
seconds=30,
seconds=15,
job_id="proxy_node_health_check",
name="代理节点心跳检测",
)

View File

@@ -21,7 +21,8 @@ from src.core.logger import logger
# ProxyNode 信息缓存(降低高频 DB 查询开销)
# ---------------------------------------------------------------------------
_proxy_node_cache: dict[str, tuple[dict[str, Any] | None, float]] = {}
_PROXY_NODE_CACHE_TTL_SECONDS = 60.0
_PROXY_NODE_CACHE_TTL_SECONDS = 15.0
_PROXY_NODE_CACHE_NEGATIVE_TTL_SECONDS = 5.0 # 不可用节点使用更短的 TTL加速恢复感知
_PROXY_NODE_CACHE_MAX_SIZE = 256
@@ -60,12 +61,12 @@ def _get_proxy_node_info(node_id: str) -> dict[str, Any] | None:
try:
node = db.query(ProxyNode).filter(ProxyNode.id == node_id).first()
if not node or node.status != ProxyNodeStatus.ONLINE:
_proxy_node_cache[node_id] = (None, now + _PROXY_NODE_CACHE_TTL_SECONDS)
_proxy_node_cache[node_id] = (None, now + _PROXY_NODE_CACHE_NEGATIVE_TTL_SECONDS)
return None
# tunnel 模式节点必须 tunnel 已连接才可用
if node.tunnel_mode and not node.tunnel_connected:
_proxy_node_cache[node_id] = (None, now + _PROXY_NODE_CACHE_TTL_SECONDS)
_proxy_node_cache[node_id] = (None, now + _PROXY_NODE_CACHE_NEGATIVE_TTL_SECONDS)
return None
if node.is_manual: