feat(heartbeat): 心跳可靠性增强,原子计数与去重优化

- Rust proxy: 引入 snapshot+ACK 确认机制,心跳未确认时保留快照重发,
  避免指标丢失;添加 heartbeat_session_id 防跨进程去重误判
- Hub transport: Redis SETNX 心跳去重,避免多 worker 重复写库;
  ACK 回显 heartbeat_id 供 Rust 端匹配
- ProxyNodeService.heartbeat: 改用 SQLAlchemy atomic update 原子累加
  指标,避免 ORM read-modify-write 的并发覆盖问题
- 启动顺序修正: tunnel 状态重置移到 Hub 连接建立之前,避免竞态
- OAuth 批量导入: 动态超时(默认30s,走代理60s),Kiro 适配器透传
- 提取 normalize_heartbeat_id 到 tunnel_protocol 共享模块,消除重复
This commit is contained in:
fawney19
2026-03-02 12:27:51 +08:00
parent f3b9f42202
commit f978888759
13 changed files with 507 additions and 71 deletions

View File

@@ -295,11 +295,20 @@ async def refresh_access_token(
cfg: KiroAuthConfig,
*,
proxy_config: dict[str, Any] | None,
timeout_seconds: float = 30.0,
) -> tuple[str, KiroAuthConfig]:
method = (cfg.auth_method or "social").strip().lower()
if method == "idc":
return await refresh_idc_token(cfg, proxy_config=proxy_config)
return await refresh_social_token(cfg, proxy_config=proxy_config)
return await refresh_idc_token(
cfg,
proxy_config=proxy_config,
timeout_seconds=timeout_seconds,
)
return await refresh_social_token(
cfg,
proxy_config=proxy_config,
timeout_seconds=timeout_seconds,
)
__all__ = [

View File

@@ -20,7 +20,7 @@ from src.core.logger import logger
from .hub_config import HubConfig, get_hub_config
from .tunnel_manager import TunnelStreamError, _StreamState
from .tunnel_protocol import Frame, FrameFlags, MsgType
from .tunnel_protocol import Frame, FrameFlags, MsgType, normalize_heartbeat_id
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Coroutine
@@ -28,6 +28,7 @@ if TYPE_CHECKING:
_TUNNEL_COMPRESS_MIN_SIZE = 512
_RECONNECT_DELAYS_SECONDS: tuple[float, ...] = (0.0, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0)
_HEARTBEAT_DEDUP_TTL_SECONDS = 600
_HOP_BY_HOP_HEADERS = frozenset(
{
@@ -317,6 +318,36 @@ class HubConnectionManager:
data = {}
node_id = str(data.get("node_id") or "").strip()
heartbeat_session_id = str(data.get("heartbeat_session_id") or "").strip()
if len(heartbeat_session_id) > 128:
heartbeat_session_id = heartbeat_session_id[:128]
heartbeat_id = normalize_heartbeat_id(data.get("heartbeat_id"))
ack: dict[str, object] = {}
if heartbeat_id is not None:
ack["heartbeat_id"] = heartbeat_id
should_process = True
if node_id and heartbeat_id is not None:
if heartbeat_session_id:
dedup_key = f"hub:heartbeat:{node_id}:{heartbeat_session_id}:{heartbeat_id}"
else:
dedup_key = f"hub:heartbeat:{node_id}:{heartbeat_id}"
try:
from src.clients import get_redis_client
redis = await get_redis_client()
if redis:
acquired = await redis.set(
dedup_key,
"1",
ex=_HEARTBEAT_DEDUP_TTL_SECONDS,
nx=True,
)
if not acquired:
should_process = False
except Exception:
# Redis 不可用时降级为不去重,避免心跳链路阻塞
pass
def _sync_heartbeat() -> dict[str, object]:
from src.database import create_session
@@ -345,11 +376,11 @@ class HubConnectionManager:
finally:
db.close()
try:
ack = await asyncio.to_thread(_sync_heartbeat)
except Exception as e:
logger.warning("hub heartbeat DB update failed: {}", e)
ack = {}
if should_process:
try:
ack.update(await asyncio.to_thread(_sync_heartbeat))
except Exception as e:
logger.warning("hub heartbeat DB update failed: {}", e)
try:
await self._send_frame(

View File

@@ -14,6 +14,7 @@ from typing import Any
from urllib.parse import urlparse
import httpx
from sqlalchemy import update
from sqlalchemy.orm import Session
from src.core.exceptions import InvalidRequestException, NotFoundException
@@ -338,36 +339,43 @@ class ProxyNodeService:
)
now = datetime.now(timezone.utc)
values: dict[str, Any] = {"last_heartbeat_at": now}
# 心跳通过 tunnel 连接传输,能收到心跳说明 tunnel 一定连通。
# 如果状态不是 ONLINE 或 tunnel_connected 不一致(例如并发写入覆盖),修正状态
# 状态不一致(例如并发写入覆盖),修正为 ONLINE
if node.status != ProxyNodeStatus.ONLINE or not node.tunnel_connected:
node.status = ProxyNodeStatus.ONLINE
node.tunnel_connected = True
node.tunnel_connected_at = now
node.updated_at = now
node.last_heartbeat_at = now
values["status"] = ProxyNodeStatus.ONLINE
values["tunnel_connected"] = True
values["tunnel_connected_at"] = now
values["updated_at"] = now
if heartbeat_interval is not None:
node.heartbeat_interval = heartbeat_interval
values["heartbeat_interval"] = heartbeat_interval
# 实时快照指标 -- 直接覆盖
if active_connections is not None:
node.active_connections = active_connections
values["active_connections"] = active_connections
if avg_latency_ms is not None:
node.avg_latency_ms = avg_latency_ms
values["avg_latency_ms"] = avg_latency_ms
# 区间增量指标 -- 累加到累计值
# 区间增量指标 -- 使用数据库原子自增,避免并发心跳读改写丢增量
if total_requests is not None and total_requests > 0:
node.total_requests = (node.total_requests or 0) + total_requests
values["total_requests"] = ProxyNode.total_requests + int(total_requests)
if failed_requests is not None and failed_requests > 0:
node.failed_requests = (node.failed_requests or 0) + failed_requests
values["failed_requests"] = ProxyNode.failed_requests + int(failed_requests)
if dns_failures is not None and dns_failures > 0:
node.dns_failures = (node.dns_failures or 0) + dns_failures
values["dns_failures"] = ProxyNode.dns_failures + int(dns_failures)
if stream_errors is not None and stream_errors > 0:
node.stream_errors = (node.stream_errors or 0) + stream_errors
values["stream_errors"] = ProxyNode.stream_errors + int(stream_errors)
db.execute(update(ProxyNode).where(ProxyNode.id == node_id).values(**values))
db.commit()
db.refresh(node)
return node
db.expire_all()
refreshed = db.query(ProxyNode).filter(ProxyNode.id == node_id).first()
if not refreshed:
raise NotFoundException(f"ProxyNode {node_id} 不存在", "proxy_node")
return refreshed
@staticmethod
def unregister_node(db: Session, *, node_id: str) -> ProxyNode:

View File

@@ -20,7 +20,7 @@ from starlette.websockets import WebSocket, WebSocketState
from src.core.logger import logger
from .tunnel_protocol import Frame, FrameFlags, MsgType
from .tunnel_protocol import Frame, FrameFlags, MsgType, normalize_heartbeat_id
# 隧道帧压缩的最小 payload 大小(字节)
# 小于此值的帧压缩收益不大,反而增加 CPU 开销
@@ -463,6 +463,10 @@ class TunnelManager:
data = json.loads(frame.payload) if frame.payload else {}
except Exception:
data = {}
heartbeat_id = normalize_heartbeat_id(data.get("heartbeat_id"))
ack: dict[str, Any] = {}
if heartbeat_id is not None:
ack["heartbeat_id"] = heartbeat_id
def _sync_heartbeat() -> dict[str, Any]:
from src.database import create_session
@@ -489,10 +493,9 @@ class TunnelManager:
db.close()
try:
ack = await asyncio.to_thread(_sync_heartbeat)
ack.update(await asyncio.to_thread(_sync_heartbeat))
except Exception as e:
logger.warning("tunnel heartbeat DB update failed: {}", e)
ack = {}
try:
await conn.send_frame(Frame(0, MsgType.HEARTBEAT_ACK, 0, json.dumps(ack).encode()))

View File

@@ -11,6 +11,8 @@ import struct
from enum import IntEnum
from typing import Self
_U64_MAX = (1 << 64) - 1
HEADER_SIZE = 10 # 4 + 1 + 1 + 4 bytes
@@ -98,3 +100,25 @@ class Frame:
f"Frame(stream={self.stream_id}, type={self.msg_type.name}, "
f"flags=0x{self.flags:02x}, payload_len={len(self.payload)})"
)
def normalize_heartbeat_id(value: object) -> int | None:
"""Normalize heartbeat_id to u64-compatible int for Rust ACK parsing."""
if isinstance(value, bool) or value is None:
return None
parsed: int | None = None
if isinstance(value, int):
parsed = value
elif isinstance(value, float):
if value.is_integer():
parsed = int(value)
elif isinstance(value, str):
stripped = value.strip()
if stripped.isdigit():
try:
parsed = int(stripped)
except ValueError:
parsed = None
if parsed is None or parsed < 0 or parsed > _U64_MAX:
return None
return parsed