feat(proxy): 节点状态简化、连接事件记录、可靠性指标与批量删除

- 移除 UNHEALTHY 中间状态,节点状态简化为 ONLINE/OFFLINE
- 新增 proxy_node_events 表记录 tunnel 连接/断开/错误事件
- 新增 failed_requests/dns_failures/stream_errors 可靠性指标(增量累加)
- tunnel 重连改为固定 1s 延迟,移除指数退避逻辑
- resolver/service 改为以 TunnelManager 内存状态判断节点可用性,避免 DB 竞态
- 修正 Claude cache_control 字段格式,使用 ttl 字段控制缓存时长
- 移除前端手动勾选 capability 的 UI,改为从价格配置自动推断
- 新增全局模型批量删除 API,替换前端并行单个删除
This commit is contained in:
fawney19
2026-02-28 13:52:32 +08:00
parent ecb16d345a
commit 54530faf03
25 changed files with 586 additions and 225 deletions

View File

@@ -9,7 +9,7 @@ from __future__ import annotations
from dataclasses import dataclass
from typing import Any
from fastapi import APIRouter, Depends, Query, Request, Response
from fastapi import APIRouter, Body, Depends, Query, Request, Response
from sqlalchemy.orm import Session
from src.api.base.admin_adapter import AdminApiAdapter
@@ -181,6 +181,21 @@ async def delete_global_model(
return Response(status_code=204)
@router.post("/batch-delete")
async def batch_delete_global_models(
request: Request,
ids: list[str] = Body(..., embed=True, max_length=100),
db: Session = Depends(get_db),
) -> dict:
"""
批量删除 GlobalModel
顺序删除多个 GlobalModel每个独立提交避免并行删除导致的锁竞争。
"""
adapter = AdminBatchDeleteGlobalModelsAdapter(ids=ids)
return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode)
@router.post(
"/{global_model_id}/assign-to-providers", response_model=BatchAssignToProvidersResponse
)
@@ -517,6 +532,50 @@ class AdminDeleteGlobalModelAdapter(AdminApiAdapter):
return None
@dataclass
class AdminBatchDeleteGlobalModelsAdapter(AdminApiAdapter):
"""批量删除多个 GlobalModel顺序执行每个删除独立提交"""
ids: list[str]
async def handle(self, context: ApiRequestContext) -> Any: # type: ignore[override]
from src.core.exceptions import NotFoundException
from src.models.database import GlobalModel
success_count = 0
failed: list[dict] = []
deleted_names: list[tuple[str, str]] = [] # (name, id)
for gm_id in self.ids:
try:
gm = context.db.query(GlobalModel).filter(GlobalModel.id == gm_id).first()
if gm:
name = gm.name
mid = gm.id
GlobalModelService.delete_global_model(context.db, gm_id)
deleted_names.append((name, mid))
success_count += 1
else:
failed.append({"id": gm_id, "error": "not found"})
except NotFoundException:
failed.append({"id": gm_id, "error": "not found"})
except Exception as e:
context.db.rollback()
failed.append({"id": gm_id, "error": str(e)})
# 批量失效缓存
if deleted_names:
from src.services.cache.invalidation import get_cache_invalidation_service
cache_service = get_cache_invalidation_service()
for name, mid in deleted_names:
await cache_service.on_global_model_changed(name, mid)
logger.info("批量删除 GlobalModel: success={}, failed={}", success_count, len(failed))
return {"success_count": success_count, "failed": failed}
@dataclass
class AdminBatchAssignToProvidersAdapter(AdminApiAdapter):
"""批量为 Provider 添加 GlobalModel 实现"""

View File

@@ -221,6 +221,17 @@ async def update_proxy_node_config(
return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode)
@router.get("/{node_id}/events")
async def list_proxy_node_events(
node_id: str,
request: Request,
limit: int = Query(50, ge=1, le=200),
db: Session = Depends(get_db),
) -> Any:
adapter = AdminListProxyNodeEventsAdapter(node_id=node_id, limit=limit)
return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode)
# ---------------------------------------------------------------------------
# 辅助函数
# ---------------------------------------------------------------------------
@@ -517,3 +528,34 @@ class AdminTestProxyUrlAdapter(AdminApiAdapter):
username=req.username,
password=req.password,
)
@dataclass
class AdminListProxyNodeEventsAdapter(AdminApiAdapter):
"""查询代理节点连接事件(连接/断开/错误历史)"""
name: str = "admin_list_proxy_node_events"
node_id: str = ""
limit: int = 50
async def handle(self, context: ApiRequestContext) -> Any:
from src.models.database import ProxyNodeEvent
events = (
context.db.query(ProxyNodeEvent)
.filter(ProxyNodeEvent.node_id == self.node_id)
.order_by(ProxyNodeEvent.created_at.desc())
.limit(self.limit)
.all()
)
return {
"items": [
{
"id": e.id,
"event_type": e.event_type,
"detail": e.detail,
"created_at": e.created_at,
}
for e in events
],
}

View File

@@ -119,6 +119,7 @@ async def proxy_tunnel_ws(ws: WebSocket) -> None:
# 启动服务端 ping 任务,防止中间代理因空闲超时关闭连接
ping_task = asyncio.create_task(_ping_loop(conn))
disconnect_reason: str | None = None
try:
oversized_count = 0
while True:
@@ -126,6 +127,7 @@ async def proxy_tunnel_ws(ws: WebSocket) -> None:
data = await asyncio.wait_for(ws.receive_bytes(), timeout=_IDLE_TIMEOUT)
except asyncio.TimeoutError:
logger.warning("tunnel idle timeout for node_id={}", node_id)
disconnect_reason = "idle timeout"
await ws.close(code=4004, reason="idle timeout")
break
if len(data) > _MAX_FRAME_SIZE:
@@ -133,6 +135,7 @@ async def proxy_tunnel_ws(ws: WebSocket) -> None:
logger.warning("tunnel frame too large from {}: {} bytes", node_id, len(data))
if oversized_count >= 5:
logger.warning("too many oversized frames from {}, closing", node_id)
disconnect_reason = "too many oversized frames"
await ws.close(code=4003, reason="too many oversized frames")
break
continue
@@ -146,14 +149,16 @@ async def proxy_tunnel_ws(ws: WebSocket) -> None:
await manager.handle_incoming_frame(conn, frame)
except WebSocketDisconnect:
disconnect_reason = "WebSocket disconnected"
logger.info("tunnel WebSocket disconnected: node_id={}", node_id)
except Exception as e:
disconnect_reason = f"error: {e}"
logger.error("tunnel WebSocket error for node_id={}: {}", node_id, e)
finally:
ping_task.cancel()
manager.unregister(conn)
if not manager.has_tunnel(node_id):
await _update_tunnel_status(node_id, connected=False)
await _update_tunnel_status(node_id, connected=False, detail=disconnect_reason)
else:
logger.info("tunnel connection closed but pool still active: node_id={}", node_id)
@@ -174,14 +179,16 @@ async def _ping_loop(conn: TunnelConnection) -> None:
pass
async def _update_tunnel_status(node_id: str, *, connected: bool) -> None:
"""更新 ProxyNode 的 tunnel 连接状态(在线程池中执行,避免阻塞 event loop"""
async def _update_tunnel_status(
node_id: str, *, connected: bool, detail: str | None = None
) -> None:
"""更新 ProxyNode 的 tunnel 连接状态并记录事件(在线程池中执行)"""
def _sync_update() -> None:
from datetime import datetime, timezone
from src.database import create_session
from src.models.database import ProxyNode, ProxyNodeStatus
from src.models.database import ProxyNode, ProxyNodeEvent, ProxyNodeStatus
db = create_session()
try:
@@ -193,9 +200,16 @@ async def _update_tunnel_status(node_id: str, *, connected: bool) -> None:
node.tunnel_connected_at = now
node.status = ProxyNodeStatus.ONLINE
else:
# 记录断开时刻,供 health_scheduler 计算 UNHEALTHY 缓冲期
node.tunnel_connected_at = now
node.status = ProxyNodeStatus.UNHEALTHY
node.status = ProxyNodeStatus.OFFLINE
# 记录连接事件
event = ProxyNodeEvent(
node_id=node_id,
event_type="connected" if connected else "disconnected",
detail=detail,
)
db.add(event)
db.commit()
finally:
db.close()

View File

@@ -131,7 +131,7 @@ class ClaudeChatAdapter(ChatAdapterBase):
request_body: dict[str, Any] | None = None,
) -> dict[str, bool]:
"""检测 Claude 请求中隐含的能力需求"""
return ClaudeCapabilityDetector.detect_from_headers(headers)
return ClaudeCapabilityDetector.detect_from_headers(headers, request_body)
# =========================================================================
# Claude 特定的计费逻辑

View File

@@ -874,7 +874,6 @@ class ProxyNodeStatus(PyEnum):
"""代理节点状态"""
ONLINE = "online"
UNHEALTHY = "unhealthy"
OFFLINE = "offline"
@@ -919,6 +918,9 @@ class ProxyNode(Base):
active_connections = Column(Integer, default=0, nullable=False)
total_requests = Column(BigInteger, default=0, nullable=False)
avg_latency_ms = Column(Float, nullable=True)
failed_requests = Column(BigInteger, default=0, nullable=False, comment="累计失败请求数")
dns_failures = Column(BigInteger, default=0, nullable=False, comment="累计 DNS 失败数")
stream_errors = Column(BigInteger, default=0, nullable=False, comment="累计流错误数")
# 硬件信息注册时上报JSON 可扩展)
hardware_info = Column(
@@ -962,6 +964,33 @@ class ProxyNode(Base):
__table_args__ = (UniqueConstraint("ip", "port", name="uq_proxy_node_ip_port"),)
class ProxyNodeEvent(Base):
"""代理节点连接事件表 -- 记录 tunnel 连接/断开/错误事件,用于连接稳定性分析"""
__tablename__ = "proxy_node_events"
id = Column(BigInteger, primary_key=True, autoincrement=True)
node_id = Column(
String(36),
ForeignKey("proxy_nodes.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
event_type = Column(
String(20),
nullable=False,
comment="事件类型: connected, disconnected, error",
)
detail = Column(String(500), nullable=True, comment="事件详情(如断开原因)")
created_at = Column(
DateTime(timezone=True),
default=lambda: datetime.now(timezone.utc),
nullable=False,
)
__table_args__ = (Index("idx_proxy_node_events_node_created", "node_id", "created_at"),)
class GlobalModel(ExportMixin, Base):
"""全局统一模型定义 - 包含价格和能力配置

View File

@@ -21,7 +21,7 @@ if TYPE_CHECKING:
def _reset_tunnel_connected_on_startup() -> None:
"""服务端启动时将所有 tunnel_connected=True 的节点重置为 False/UNHEALTHY
"""服务端启动时将所有 tunnel_connected=True 的节点重置为 False/OFFLINE
服务端重启后 TunnelManager 内存状态丢失,但 DB 中可能残留
tunnel_connected=True 的记录。如果不重置health_scheduler 会错误地
@@ -48,7 +48,7 @@ def _reset_tunnel_connected_on_startup() -> None:
for node in stale_nodes:
node.tunnel_connected = False
node.tunnel_connected_at = now
node.status = ProxyNodeStatus.UNHEALTHY
node.status = ProxyNodeStatus.OFFLINE
node.updated_at = now
db.commit()
logger.info(

View File

@@ -224,15 +224,32 @@ _VALID_CACHE_TTL_TARGETS = {"ephemeral", "1h"}
def _override_cache_control_in_blocks(blocks: list[Any], target: str) -> int:
"""Override cache_control in a list of content blocks. Returns count of overrides."""
"""Override cache_control TTL in a list of content blocks. Returns count of overrides.
According to Anthropic API docs, cache_control format is:
{"type": "ephemeral", "ttl": "5m" | "1h"}
``type`` is always "ephemeral"; the ``ttl`` field controls the actual duration.
When ttl is absent, the default is 5m (ephemeral).
"""
count = 0
for block in blocks:
if not isinstance(block, dict):
continue
cc = block.get("cache_control")
if isinstance(cc, dict):
if cc.get("type") != target:
cc["type"] = target
if not isinstance(cc, dict):
continue
# Ensure type is always "ephemeral"
if cc.get("type") != "ephemeral":
cc["type"] = "ephemeral"
if target == "ephemeral":
# Target is 5m (default) -- remove explicit ttl so it falls back to default
if "ttl" in cc:
del cc["ttl"]
count += 1
else:
# Target is "1h" -- set ttl explicitly
if cc.get("ttl") != target:
cc["ttl"] = target
count += 1
return count

View File

@@ -2,9 +2,9 @@
ProxyNode 心跳检测调度器
定期检查 proxy_nodes 的 tunnel 连接状态,更新节点状态:
- tunnel 实际连接中 -> ONLINE(包括从 OFFLINE 恢复的情况)
- tunnel 刚断开 (<60s) -> UNHEALTHY缓冲期避免正在进行的请求被立即切走
- tunnel 断开超过 60s -> OFFLINE
- tunnel 实际连接中 -> ONLINE
- tunnel 未连接 -> OFFLINE
以 TunnelManager 内存中的实际连接状态为准。
"""
from __future__ import annotations
@@ -17,12 +17,18 @@ from src.database import create_session
from src.models.database import ProxyNode, ProxyNodeStatus
from src.services.system.scheduler import get_scheduler
# 事件保留天数
_EVENT_RETENTION_DAYS = 30
# 每隔多少次心跳检测执行一次事件清理15s * 240 = 1h
_EVENT_CLEANUP_INTERVAL = 240
class ProxyNodeHealthScheduler:
"""代理节点心跳检测调度器"""
def __init__(self) -> None:
self.running = False
self._check_count = 0
async def start(self) -> Any:
if self.running:
@@ -51,6 +57,9 @@ class ProxyNodeHealthScheduler:
async def _scheduled_check(self) -> None:
await self._check_heartbeats()
self._check_count = (self._check_count + 1) % _EVENT_CLEANUP_INTERVAL
if self._check_count == 0:
await self._cleanup_old_events()
async def _check_heartbeats(self) -> None:
from src.services.proxy_node.tunnel_manager import get_tunnel_manager
@@ -87,16 +96,9 @@ class ProxyNodeHealthScheduler:
node.tunnel_connected_at = now
changed += 1
if actually_connected:
new_status = ProxyNodeStatus.ONLINE
elif node.tunnel_connected_at:
# tunnel 刚断开:给 60s 缓冲期标记为 UNHEALTHY
elapsed = (now - node.tunnel_connected_at).total_seconds()
new_status = (
ProxyNodeStatus.UNHEALTHY if elapsed < 60 else ProxyNodeStatus.OFFLINE
)
else:
new_status = ProxyNodeStatus.OFFLINE
new_status = (
ProxyNodeStatus.ONLINE if actually_connected else ProxyNodeStatus.OFFLINE
)
if node.status != new_status:
node.status = new_status
@@ -115,6 +117,42 @@ class ProxyNodeHealthScheduler:
finally:
db.close()
async def _cleanup_old_events(self) -> None:
"""清理超过保留期的连接事件记录(在线程池中执行,避免阻塞事件循环)"""
import asyncio
def _sync_cleanup() -> None:
from datetime import timedelta
from src.models.database import ProxyNodeEvent
db = create_session()
try:
cutoff = datetime.now(timezone.utc) - timedelta(days=_EVENT_RETENTION_DAYS)
deleted = (
db.query(ProxyNodeEvent)
.filter(ProxyNodeEvent.created_at < cutoff)
.delete(synchronize_session=False)
)
if deleted:
db.commit()
logger.info(
"清理 {} 条过期代理节点事件 (>{} 天)", deleted, _EVENT_RETENTION_DAYS
)
except Exception as e:
try:
db.rollback()
except Exception:
pass
logger.warning("清理代理节点事件失败: {}", e)
finally:
db.close()
try:
await asyncio.to_thread(_sync_cleanup)
except Exception as e:
logger.warning("清理代理节点事件线程执行失败: {}", e)
_proxy_node_health_scheduler: ProxyNodeHealthScheduler | None = None

View File

@@ -60,17 +60,39 @@ def _get_proxy_node_info(node_id: str) -> dict[str, Any] | None:
db = create_session()
try:
node = db.query(ProxyNode).filter(ProxyNode.id == node_id).first()
if not node or node.status != ProxyNodeStatus.ONLINE:
if not node:
_proxy_node_cache[node_id] = (None, now + _PROXY_NODE_CACHE_NEGATIVE_TTL_SECONDS)
return None
# tunnel 模式节点必须 tunnel 已连接才可用
if node.tunnel_mode and not node.tunnel_connected:
# tunnel 模式节点:以 TunnelManager 内存中的实际连接状态为准,
# 而非依赖 DB 的 status/tunnel_connected 字段(可能因竞态不同步)。
if node.tunnel_mode and not node.is_manual:
from src.services.proxy_node.tunnel_manager import get_tunnel_manager
manager = get_tunnel_manager()
if not manager.has_tunnel(node_id):
_proxy_node_cache[node_id] = (
None,
now + _PROXY_NODE_CACHE_NEGATIVE_TTL_SECONDS,
)
return None
value: dict[str, Any] = {
"name": node.name,
"ip": node.ip,
"port": node.port,
"tunnel_mode": True,
"tunnel_connected": True,
}
_proxy_node_cache[node_id] = (value, now + _PROXY_NODE_CACHE_TTL_SECONDS)
return value
# 手动节点 / 非 tunnel 节点:仍依赖 DB status
if node.status != ProxyNodeStatus.ONLINE:
_proxy_node_cache[node_id] = (None, now + _PROXY_NODE_CACHE_NEGATIVE_TTL_SECONDS)
return None
if node.is_manual:
value: dict[str, Any] = {
value = {
"is_manual": True,
"name": node.name,
"proxy_url": node.proxy_url,

View File

@@ -58,6 +58,9 @@ def node_to_dict(node: ProxyNode) -> dict[str, Any]:
"active_connections": node.active_connections,
"total_requests": node.total_requests,
"avg_latency_ms": node.avg_latency_ms,
"failed_requests": node.failed_requests,
"dns_failures": node.dns_failures,
"stream_errors": node.stream_errors,
"hardware_info": node.hardware_info,
"estimated_max_concurrency": node.estimated_max_concurrency,
"remote_config": node.remote_config,
@@ -283,7 +286,7 @@ class ProxyNodeService:
port=port,
region=region,
# 新节点:等 tunnel 连接后才上线
status=ProxyNodeStatus.UNHEALTHY,
status=ProxyNodeStatus.OFFLINE,
registered_by=registered_by,
last_heartbeat_at=now,
heartbeat_interval=heartbeat_interval,
@@ -311,8 +314,16 @@ class ProxyNodeService:
active_connections: int | None = None,
total_requests: int | None = None,
avg_latency_ms: float | None = None,
failed_requests: int | None = None,
dns_failures: int | None = None,
stream_errors: int | None = None,
) -> ProxyNode:
"""处理节点心跳(仅 tunnel 模式节点,更新指标并修正状态不一致)"""
"""处理节点心跳(仅 tunnel 模式节点,更新指标并修正状态不一致)
注意: total_requests / failed_requests / dns_failures / stream_errors
来自 Rust 端的区间增量swap(0) 后上报),需要累加到 DB 而非覆盖。
active_connections 和 avg_latency_ms 是实时快照,直接覆盖。
"""
node = db.query(ProxyNode).filter(ProxyNode.id == node_id).first()
if not node:
raise NotFoundException(f"ProxyNode {node_id} 不存在", "proxy_node")
@@ -332,13 +343,23 @@ class ProxyNodeService:
node.last_heartbeat_at = now
if heartbeat_interval is not None:
node.heartbeat_interval = heartbeat_interval
# 实时快照指标 -- 直接覆盖
if active_connections is not None:
node.active_connections = active_connections
if total_requests is not None:
node.total_requests = total_requests
if avg_latency_ms is not None:
node.avg_latency_ms = avg_latency_ms
# 区间增量指标 -- 累加到累计值
if total_requests is not None and total_requests > 0:
node.total_requests = (node.total_requests or 0) + total_requests
if failed_requests is not None and failed_requests > 0:
node.failed_requests = (node.failed_requests or 0) + failed_requests
if dns_failures is not None and dns_failures > 0:
node.dns_failures = (node.dns_failures or 0) + dns_failures
if stream_errors is not None and stream_errors > 0:
node.stream_errors = (node.stream_errors or 0) + stream_errors
db.commit()
db.refresh(node)
return node
@@ -530,7 +551,12 @@ class ProxyNodeService:
# tunnel 节点:通过 WebSocket tunnel 测试
if not node.is_manual:
if not node.tunnel_connected:
# 以 TunnelManager 内存中的实际连接状态为准(与 health_scheduler 一致),
# 而非仅依赖 DB 的 tunnel_connected 字段,避免竞态导致误判。
from src.services.proxy_node.tunnel_manager import get_tunnel_manager
manager = get_tunnel_manager()
if not manager.has_tunnel(node.id):
return {
"success": False,
"latency_ms": None,

View File

@@ -402,6 +402,9 @@ class TunnelManager:
active_connections=data.get("active_connections"),
total_requests=data.get("total_requests"),
avg_latency_ms=data.get("avg_latency_ms"),
failed_requests=data.get("failed_requests"),
dns_failures=data.get("dns_failures"),
stream_errors=data.get("stream_errors"),
)
result: dict[str, Any] = {}
if node.remote_config: