mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 01:10:23 +08:00
feat(proxy): 节点状态简化、连接事件记录、可靠性指标与批量删除
- 移除 UNHEALTHY 中间状态,节点状态简化为 ONLINE/OFFLINE - 新增 proxy_node_events 表记录 tunnel 连接/断开/错误事件 - 新增 failed_requests/dns_failures/stream_errors 可靠性指标(增量累加) - tunnel 重连改为固定 1s 延迟,移除指数退避逻辑 - resolver/service 改为以 TunnelManager 内存状态判断节点可用性,避免 DB 竞态 - 修正 Claude cache_control 字段格式,使用 ttl 字段控制缓存时长 - 移除前端手动勾选 capability 的 UI,改为从价格配置自动推断 - 新增全局模型批量删除 API,替换前端并行单个删除
This commit is contained in:
@@ -9,7 +9,7 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, Request, Response
|
||||
from fastapi import APIRouter, Body, Depends, Query, Request, Response
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from src.api.base.admin_adapter import AdminApiAdapter
|
||||
@@ -181,6 +181,21 @@ async def delete_global_model(
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@router.post("/batch-delete")
|
||||
async def batch_delete_global_models(
|
||||
request: Request,
|
||||
ids: list[str] = Body(..., embed=True, max_length=100),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""
|
||||
批量删除 GlobalModel
|
||||
|
||||
顺序删除多个 GlobalModel(每个独立提交),避免并行删除导致的锁竞争。
|
||||
"""
|
||||
adapter = AdminBatchDeleteGlobalModelsAdapter(ids=ids)
|
||||
return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/{global_model_id}/assign-to-providers", response_model=BatchAssignToProvidersResponse
|
||||
)
|
||||
@@ -517,6 +532,50 @@ class AdminDeleteGlobalModelAdapter(AdminApiAdapter):
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class AdminBatchDeleteGlobalModelsAdapter(AdminApiAdapter):
|
||||
"""批量删除多个 GlobalModel(顺序执行,每个删除独立提交)"""
|
||||
|
||||
ids: list[str]
|
||||
|
||||
async def handle(self, context: ApiRequestContext) -> Any: # type: ignore[override]
|
||||
from src.core.exceptions import NotFoundException
|
||||
from src.models.database import GlobalModel
|
||||
|
||||
success_count = 0
|
||||
failed: list[dict] = []
|
||||
deleted_names: list[tuple[str, str]] = [] # (name, id)
|
||||
|
||||
for gm_id in self.ids:
|
||||
try:
|
||||
gm = context.db.query(GlobalModel).filter(GlobalModel.id == gm_id).first()
|
||||
if gm:
|
||||
name = gm.name
|
||||
mid = gm.id
|
||||
GlobalModelService.delete_global_model(context.db, gm_id)
|
||||
deleted_names.append((name, mid))
|
||||
success_count += 1
|
||||
else:
|
||||
failed.append({"id": gm_id, "error": "not found"})
|
||||
except NotFoundException:
|
||||
failed.append({"id": gm_id, "error": "not found"})
|
||||
except Exception as e:
|
||||
context.db.rollback()
|
||||
failed.append({"id": gm_id, "error": str(e)})
|
||||
|
||||
# 批量失效缓存
|
||||
if deleted_names:
|
||||
from src.services.cache.invalidation import get_cache_invalidation_service
|
||||
|
||||
cache_service = get_cache_invalidation_service()
|
||||
for name, mid in deleted_names:
|
||||
await cache_service.on_global_model_changed(name, mid)
|
||||
|
||||
logger.info("批量删除 GlobalModel: success={}, failed={}", success_count, len(failed))
|
||||
|
||||
return {"success_count": success_count, "failed": failed}
|
||||
|
||||
|
||||
@dataclass
|
||||
class AdminBatchAssignToProvidersAdapter(AdminApiAdapter):
|
||||
"""批量为 Provider 添加 GlobalModel 实现"""
|
||||
|
||||
@@ -221,6 +221,17 @@ async def update_proxy_node_config(
|
||||
return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode)
|
||||
|
||||
|
||||
@router.get("/{node_id}/events")
|
||||
async def list_proxy_node_events(
|
||||
node_id: str,
|
||||
request: Request,
|
||||
limit: int = Query(50, ge=1, le=200),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
adapter = AdminListProxyNodeEventsAdapter(node_id=node_id, limit=limit)
|
||||
return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 辅助函数
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -517,3 +528,34 @@ class AdminTestProxyUrlAdapter(AdminApiAdapter):
|
||||
username=req.username,
|
||||
password=req.password,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AdminListProxyNodeEventsAdapter(AdminApiAdapter):
|
||||
"""查询代理节点连接事件(连接/断开/错误历史)"""
|
||||
|
||||
name: str = "admin_list_proxy_node_events"
|
||||
node_id: str = ""
|
||||
limit: int = 50
|
||||
|
||||
async def handle(self, context: ApiRequestContext) -> Any:
|
||||
from src.models.database import ProxyNodeEvent
|
||||
|
||||
events = (
|
||||
context.db.query(ProxyNodeEvent)
|
||||
.filter(ProxyNodeEvent.node_id == self.node_id)
|
||||
.order_by(ProxyNodeEvent.created_at.desc())
|
||||
.limit(self.limit)
|
||||
.all()
|
||||
)
|
||||
return {
|
||||
"items": [
|
||||
{
|
||||
"id": e.id,
|
||||
"event_type": e.event_type,
|
||||
"detail": e.detail,
|
||||
"created_at": e.created_at,
|
||||
}
|
||||
for e in events
|
||||
],
|
||||
}
|
||||
|
||||
@@ -119,6 +119,7 @@ async def proxy_tunnel_ws(ws: WebSocket) -> None:
|
||||
# 启动服务端 ping 任务,防止中间代理因空闲超时关闭连接
|
||||
ping_task = asyncio.create_task(_ping_loop(conn))
|
||||
|
||||
disconnect_reason: str | None = None
|
||||
try:
|
||||
oversized_count = 0
|
||||
while True:
|
||||
@@ -126,6 +127,7 @@ async def proxy_tunnel_ws(ws: WebSocket) -> None:
|
||||
data = await asyncio.wait_for(ws.receive_bytes(), timeout=_IDLE_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning("tunnel idle timeout for node_id={}", node_id)
|
||||
disconnect_reason = "idle timeout"
|
||||
await ws.close(code=4004, reason="idle timeout")
|
||||
break
|
||||
if len(data) > _MAX_FRAME_SIZE:
|
||||
@@ -133,6 +135,7 @@ async def proxy_tunnel_ws(ws: WebSocket) -> None:
|
||||
logger.warning("tunnel frame too large from {}: {} bytes", node_id, len(data))
|
||||
if oversized_count >= 5:
|
||||
logger.warning("too many oversized frames from {}, closing", node_id)
|
||||
disconnect_reason = "too many oversized frames"
|
||||
await ws.close(code=4003, reason="too many oversized frames")
|
||||
break
|
||||
continue
|
||||
@@ -146,14 +149,16 @@ async def proxy_tunnel_ws(ws: WebSocket) -> None:
|
||||
await manager.handle_incoming_frame(conn, frame)
|
||||
|
||||
except WebSocketDisconnect:
|
||||
disconnect_reason = "WebSocket disconnected"
|
||||
logger.info("tunnel WebSocket disconnected: node_id={}", node_id)
|
||||
except Exception as e:
|
||||
disconnect_reason = f"error: {e}"
|
||||
logger.error("tunnel WebSocket error for node_id={}: {}", node_id, e)
|
||||
finally:
|
||||
ping_task.cancel()
|
||||
manager.unregister(conn)
|
||||
if not manager.has_tunnel(node_id):
|
||||
await _update_tunnel_status(node_id, connected=False)
|
||||
await _update_tunnel_status(node_id, connected=False, detail=disconnect_reason)
|
||||
else:
|
||||
logger.info("tunnel connection closed but pool still active: node_id={}", node_id)
|
||||
|
||||
@@ -174,14 +179,16 @@ async def _ping_loop(conn: TunnelConnection) -> None:
|
||||
pass
|
||||
|
||||
|
||||
async def _update_tunnel_status(node_id: str, *, connected: bool) -> None:
|
||||
"""更新 ProxyNode 的 tunnel 连接状态(在线程池中执行,避免阻塞 event loop)"""
|
||||
async def _update_tunnel_status(
|
||||
node_id: str, *, connected: bool, detail: str | None = None
|
||||
) -> None:
|
||||
"""更新 ProxyNode 的 tunnel 连接状态并记录事件(在线程池中执行)"""
|
||||
|
||||
def _sync_update() -> None:
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from src.database import create_session
|
||||
from src.models.database import ProxyNode, ProxyNodeStatus
|
||||
from src.models.database import ProxyNode, ProxyNodeEvent, ProxyNodeStatus
|
||||
|
||||
db = create_session()
|
||||
try:
|
||||
@@ -193,9 +200,16 @@ async def _update_tunnel_status(node_id: str, *, connected: bool) -> None:
|
||||
node.tunnel_connected_at = now
|
||||
node.status = ProxyNodeStatus.ONLINE
|
||||
else:
|
||||
# 记录断开时刻,供 health_scheduler 计算 UNHEALTHY 缓冲期
|
||||
node.tunnel_connected_at = now
|
||||
node.status = ProxyNodeStatus.UNHEALTHY
|
||||
node.status = ProxyNodeStatus.OFFLINE
|
||||
|
||||
# 记录连接事件
|
||||
event = ProxyNodeEvent(
|
||||
node_id=node_id,
|
||||
event_type="connected" if connected else "disconnected",
|
||||
detail=detail,
|
||||
)
|
||||
db.add(event)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -131,7 +131,7 @@ class ClaudeChatAdapter(ChatAdapterBase):
|
||||
request_body: dict[str, Any] | None = None,
|
||||
) -> dict[str, bool]:
|
||||
"""检测 Claude 请求中隐含的能力需求"""
|
||||
return ClaudeCapabilityDetector.detect_from_headers(headers)
|
||||
return ClaudeCapabilityDetector.detect_from_headers(headers, request_body)
|
||||
|
||||
# =========================================================================
|
||||
# Claude 特定的计费逻辑
|
||||
|
||||
@@ -874,7 +874,6 @@ class ProxyNodeStatus(PyEnum):
|
||||
"""代理节点状态"""
|
||||
|
||||
ONLINE = "online"
|
||||
UNHEALTHY = "unhealthy"
|
||||
OFFLINE = "offline"
|
||||
|
||||
|
||||
@@ -919,6 +918,9 @@ class ProxyNode(Base):
|
||||
active_connections = Column(Integer, default=0, nullable=False)
|
||||
total_requests = Column(BigInteger, default=0, nullable=False)
|
||||
avg_latency_ms = Column(Float, nullable=True)
|
||||
failed_requests = Column(BigInteger, default=0, nullable=False, comment="累计失败请求数")
|
||||
dns_failures = Column(BigInteger, default=0, nullable=False, comment="累计 DNS 失败数")
|
||||
stream_errors = Column(BigInteger, default=0, nullable=False, comment="累计流错误数")
|
||||
|
||||
# 硬件信息(注册时上报,JSON 可扩展)
|
||||
hardware_info = Column(
|
||||
@@ -962,6 +964,33 @@ class ProxyNode(Base):
|
||||
__table_args__ = (UniqueConstraint("ip", "port", name="uq_proxy_node_ip_port"),)
|
||||
|
||||
|
||||
class ProxyNodeEvent(Base):
|
||||
"""代理节点连接事件表 -- 记录 tunnel 连接/断开/错误事件,用于连接稳定性分析"""
|
||||
|
||||
__tablename__ = "proxy_node_events"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
node_id = Column(
|
||||
String(36),
|
||||
ForeignKey("proxy_nodes.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
event_type = Column(
|
||||
String(20),
|
||||
nullable=False,
|
||||
comment="事件类型: connected, disconnected, error",
|
||||
)
|
||||
detail = Column(String(500), nullable=True, comment="事件详情(如断开原因)")
|
||||
created_at = Column(
|
||||
DateTime(timezone=True),
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
__table_args__ = (Index("idx_proxy_node_events_node_created", "node_id", "created_at"),)
|
||||
|
||||
|
||||
class GlobalModel(ExportMixin, Base):
|
||||
"""全局统一模型定义 - 包含价格和能力配置
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
def _reset_tunnel_connected_on_startup() -> None:
|
||||
"""服务端启动时将所有 tunnel_connected=True 的节点重置为 False/UNHEALTHY。
|
||||
"""服务端启动时将所有 tunnel_connected=True 的节点重置为 False/OFFLINE。
|
||||
|
||||
服务端重启后 TunnelManager 内存状态丢失,但 DB 中可能残留
|
||||
tunnel_connected=True 的记录。如果不重置,health_scheduler 会错误地
|
||||
@@ -48,7 +48,7 @@ def _reset_tunnel_connected_on_startup() -> None:
|
||||
for node in stale_nodes:
|
||||
node.tunnel_connected = False
|
||||
node.tunnel_connected_at = now
|
||||
node.status = ProxyNodeStatus.UNHEALTHY
|
||||
node.status = ProxyNodeStatus.OFFLINE
|
||||
node.updated_at = now
|
||||
db.commit()
|
||||
logger.info(
|
||||
|
||||
@@ -224,15 +224,32 @@ _VALID_CACHE_TTL_TARGETS = {"ephemeral", "1h"}
|
||||
|
||||
|
||||
def _override_cache_control_in_blocks(blocks: list[Any], target: str) -> int:
|
||||
"""Override cache_control in a list of content blocks. Returns count of overrides."""
|
||||
"""Override cache_control TTL in a list of content blocks. Returns count of overrides.
|
||||
|
||||
According to Anthropic API docs, cache_control format is:
|
||||
{"type": "ephemeral", "ttl": "5m" | "1h"}
|
||||
``type`` is always "ephemeral"; the ``ttl`` field controls the actual duration.
|
||||
When ttl is absent, the default is 5m (ephemeral).
|
||||
"""
|
||||
count = 0
|
||||
for block in blocks:
|
||||
if not isinstance(block, dict):
|
||||
continue
|
||||
cc = block.get("cache_control")
|
||||
if isinstance(cc, dict):
|
||||
if cc.get("type") != target:
|
||||
cc["type"] = target
|
||||
if not isinstance(cc, dict):
|
||||
continue
|
||||
# Ensure type is always "ephemeral"
|
||||
if cc.get("type") != "ephemeral":
|
||||
cc["type"] = "ephemeral"
|
||||
if target == "ephemeral":
|
||||
# Target is 5m (default) -- remove explicit ttl so it falls back to default
|
||||
if "ttl" in cc:
|
||||
del cc["ttl"]
|
||||
count += 1
|
||||
else:
|
||||
# Target is "1h" -- set ttl explicitly
|
||||
if cc.get("ttl") != target:
|
||||
cc["ttl"] = target
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
ProxyNode 心跳检测调度器
|
||||
|
||||
定期检查 proxy_nodes 的 tunnel 连接状态,更新节点状态:
|
||||
- tunnel 实际连接中 -> ONLINE(包括从 OFFLINE 恢复的情况)
|
||||
- tunnel 刚断开 (<60s) -> UNHEALTHY(缓冲期,避免正在进行的请求被立即切走)
|
||||
- tunnel 断开超过 60s -> OFFLINE
|
||||
- tunnel 实际连接中 -> ONLINE
|
||||
- tunnel 未连接 -> OFFLINE
|
||||
以 TunnelManager 内存中的实际连接状态为准。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -17,12 +17,18 @@ from src.database import create_session
|
||||
from src.models.database import ProxyNode, ProxyNodeStatus
|
||||
from src.services.system.scheduler import get_scheduler
|
||||
|
||||
# 事件保留天数
|
||||
_EVENT_RETENTION_DAYS = 30
|
||||
# 每隔多少次心跳检测执行一次事件清理(15s * 240 = 1h)
|
||||
_EVENT_CLEANUP_INTERVAL = 240
|
||||
|
||||
|
||||
class ProxyNodeHealthScheduler:
|
||||
"""代理节点心跳检测调度器"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.running = False
|
||||
self._check_count = 0
|
||||
|
||||
async def start(self) -> Any:
|
||||
if self.running:
|
||||
@@ -51,6 +57,9 @@ class ProxyNodeHealthScheduler:
|
||||
|
||||
async def _scheduled_check(self) -> None:
|
||||
await self._check_heartbeats()
|
||||
self._check_count = (self._check_count + 1) % _EVENT_CLEANUP_INTERVAL
|
||||
if self._check_count == 0:
|
||||
await self._cleanup_old_events()
|
||||
|
||||
async def _check_heartbeats(self) -> None:
|
||||
from src.services.proxy_node.tunnel_manager import get_tunnel_manager
|
||||
@@ -87,16 +96,9 @@ class ProxyNodeHealthScheduler:
|
||||
node.tunnel_connected_at = now
|
||||
changed += 1
|
||||
|
||||
if actually_connected:
|
||||
new_status = ProxyNodeStatus.ONLINE
|
||||
elif node.tunnel_connected_at:
|
||||
# tunnel 刚断开:给 60s 缓冲期标记为 UNHEALTHY
|
||||
elapsed = (now - node.tunnel_connected_at).total_seconds()
|
||||
new_status = (
|
||||
ProxyNodeStatus.UNHEALTHY if elapsed < 60 else ProxyNodeStatus.OFFLINE
|
||||
)
|
||||
else:
|
||||
new_status = ProxyNodeStatus.OFFLINE
|
||||
new_status = (
|
||||
ProxyNodeStatus.ONLINE if actually_connected else ProxyNodeStatus.OFFLINE
|
||||
)
|
||||
|
||||
if node.status != new_status:
|
||||
node.status = new_status
|
||||
@@ -115,6 +117,42 @@ class ProxyNodeHealthScheduler:
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
async def _cleanup_old_events(self) -> None:
|
||||
"""清理超过保留期的连接事件记录(在线程池中执行,避免阻塞事件循环)"""
|
||||
import asyncio
|
||||
|
||||
def _sync_cleanup() -> None:
|
||||
from datetime import timedelta
|
||||
|
||||
from src.models.database import ProxyNodeEvent
|
||||
|
||||
db = create_session()
|
||||
try:
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=_EVENT_RETENTION_DAYS)
|
||||
deleted = (
|
||||
db.query(ProxyNodeEvent)
|
||||
.filter(ProxyNodeEvent.created_at < cutoff)
|
||||
.delete(synchronize_session=False)
|
||||
)
|
||||
if deleted:
|
||||
db.commit()
|
||||
logger.info(
|
||||
"清理 {} 条过期代理节点事件 (>{} 天)", deleted, _EVENT_RETENTION_DAYS
|
||||
)
|
||||
except Exception as e:
|
||||
try:
|
||||
db.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
logger.warning("清理代理节点事件失败: {}", e)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(_sync_cleanup)
|
||||
except Exception as e:
|
||||
logger.warning("清理代理节点事件线程执行失败: {}", e)
|
||||
|
||||
|
||||
_proxy_node_health_scheduler: ProxyNodeHealthScheduler | None = None
|
||||
|
||||
|
||||
@@ -60,17 +60,39 @@ def _get_proxy_node_info(node_id: str) -> dict[str, Any] | None:
|
||||
db = create_session()
|
||||
try:
|
||||
node = db.query(ProxyNode).filter(ProxyNode.id == node_id).first()
|
||||
if not node or node.status != ProxyNodeStatus.ONLINE:
|
||||
if not node:
|
||||
_proxy_node_cache[node_id] = (None, now + _PROXY_NODE_CACHE_NEGATIVE_TTL_SECONDS)
|
||||
return None
|
||||
|
||||
# tunnel 模式节点必须 tunnel 已连接才可用
|
||||
if node.tunnel_mode and not node.tunnel_connected:
|
||||
# tunnel 模式节点:以 TunnelManager 内存中的实际连接状态为准,
|
||||
# 而非依赖 DB 的 status/tunnel_connected 字段(可能因竞态不同步)。
|
||||
if node.tunnel_mode and not node.is_manual:
|
||||
from src.services.proxy_node.tunnel_manager import get_tunnel_manager
|
||||
|
||||
manager = get_tunnel_manager()
|
||||
if not manager.has_tunnel(node_id):
|
||||
_proxy_node_cache[node_id] = (
|
||||
None,
|
||||
now + _PROXY_NODE_CACHE_NEGATIVE_TTL_SECONDS,
|
||||
)
|
||||
return None
|
||||
value: dict[str, Any] = {
|
||||
"name": node.name,
|
||||
"ip": node.ip,
|
||||
"port": node.port,
|
||||
"tunnel_mode": True,
|
||||
"tunnel_connected": True,
|
||||
}
|
||||
_proxy_node_cache[node_id] = (value, now + _PROXY_NODE_CACHE_TTL_SECONDS)
|
||||
return value
|
||||
|
||||
# 手动节点 / 非 tunnel 节点:仍依赖 DB status
|
||||
if node.status != ProxyNodeStatus.ONLINE:
|
||||
_proxy_node_cache[node_id] = (None, now + _PROXY_NODE_CACHE_NEGATIVE_TTL_SECONDS)
|
||||
return None
|
||||
|
||||
if node.is_manual:
|
||||
value: dict[str, Any] = {
|
||||
value = {
|
||||
"is_manual": True,
|
||||
"name": node.name,
|
||||
"proxy_url": node.proxy_url,
|
||||
|
||||
@@ -58,6 +58,9 @@ def node_to_dict(node: ProxyNode) -> dict[str, Any]:
|
||||
"active_connections": node.active_connections,
|
||||
"total_requests": node.total_requests,
|
||||
"avg_latency_ms": node.avg_latency_ms,
|
||||
"failed_requests": node.failed_requests,
|
||||
"dns_failures": node.dns_failures,
|
||||
"stream_errors": node.stream_errors,
|
||||
"hardware_info": node.hardware_info,
|
||||
"estimated_max_concurrency": node.estimated_max_concurrency,
|
||||
"remote_config": node.remote_config,
|
||||
@@ -283,7 +286,7 @@ class ProxyNodeService:
|
||||
port=port,
|
||||
region=region,
|
||||
# 新节点:等 tunnel 连接后才上线
|
||||
status=ProxyNodeStatus.UNHEALTHY,
|
||||
status=ProxyNodeStatus.OFFLINE,
|
||||
registered_by=registered_by,
|
||||
last_heartbeat_at=now,
|
||||
heartbeat_interval=heartbeat_interval,
|
||||
@@ -311,8 +314,16 @@ class ProxyNodeService:
|
||||
active_connections: int | None = None,
|
||||
total_requests: int | None = None,
|
||||
avg_latency_ms: float | None = None,
|
||||
failed_requests: int | None = None,
|
||||
dns_failures: int | None = None,
|
||||
stream_errors: int | None = None,
|
||||
) -> ProxyNode:
|
||||
"""处理节点心跳(仅 tunnel 模式节点,更新指标并修正状态不一致)"""
|
||||
"""处理节点心跳(仅 tunnel 模式节点,更新指标并修正状态不一致)
|
||||
|
||||
注意: total_requests / failed_requests / dns_failures / stream_errors
|
||||
来自 Rust 端的区间增量(swap(0) 后上报),需要累加到 DB 而非覆盖。
|
||||
active_connections 和 avg_latency_ms 是实时快照,直接覆盖。
|
||||
"""
|
||||
node = db.query(ProxyNode).filter(ProxyNode.id == node_id).first()
|
||||
if not node:
|
||||
raise NotFoundException(f"ProxyNode {node_id} 不存在", "proxy_node")
|
||||
@@ -332,13 +343,23 @@ class ProxyNodeService:
|
||||
node.last_heartbeat_at = now
|
||||
if heartbeat_interval is not None:
|
||||
node.heartbeat_interval = heartbeat_interval
|
||||
|
||||
# 实时快照指标 -- 直接覆盖
|
||||
if active_connections is not None:
|
||||
node.active_connections = active_connections
|
||||
if total_requests is not None:
|
||||
node.total_requests = total_requests
|
||||
if avg_latency_ms is not None:
|
||||
node.avg_latency_ms = avg_latency_ms
|
||||
|
||||
# 区间增量指标 -- 累加到累计值
|
||||
if total_requests is not None and total_requests > 0:
|
||||
node.total_requests = (node.total_requests or 0) + total_requests
|
||||
if failed_requests is not None and failed_requests > 0:
|
||||
node.failed_requests = (node.failed_requests or 0) + failed_requests
|
||||
if dns_failures is not None and dns_failures > 0:
|
||||
node.dns_failures = (node.dns_failures or 0) + dns_failures
|
||||
if stream_errors is not None and stream_errors > 0:
|
||||
node.stream_errors = (node.stream_errors or 0) + stream_errors
|
||||
|
||||
db.commit()
|
||||
db.refresh(node)
|
||||
return node
|
||||
@@ -530,7 +551,12 @@ class ProxyNodeService:
|
||||
|
||||
# tunnel 节点:通过 WebSocket tunnel 测试
|
||||
if not node.is_manual:
|
||||
if not node.tunnel_connected:
|
||||
# 以 TunnelManager 内存中的实际连接状态为准(与 health_scheduler 一致),
|
||||
# 而非仅依赖 DB 的 tunnel_connected 字段,避免竞态导致误判。
|
||||
from src.services.proxy_node.tunnel_manager import get_tunnel_manager
|
||||
|
||||
manager = get_tunnel_manager()
|
||||
if not manager.has_tunnel(node.id):
|
||||
return {
|
||||
"success": False,
|
||||
"latency_ms": None,
|
||||
|
||||
@@ -402,6 +402,9 @@ class TunnelManager:
|
||||
active_connections=data.get("active_connections"),
|
||||
total_requests=data.get("total_requests"),
|
||||
avg_latency_ms=data.get("avg_latency_ms"),
|
||||
failed_requests=data.get("failed_requests"),
|
||||
dns_failures=data.get("dns_failures"),
|
||||
stream_errors=data.get("stream_errors"),
|
||||
)
|
||||
result: dict[str, Any] = {}
|
||||
if node.remote_config:
|
||||
|
||||
Reference in New Issue
Block a user