i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 09:20:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(tunnel): 引入 aether-hub 帧路由器，支持多 worker 共享 tunnel 连接

Browse Source 
新增 Rust 实现的 aether-hub 服务，作为 Docker 容器内部 WebSocket 帧路由器，
解决多 Gunicorn worker 进程间 tunnel 连接隔离问题。

主要改动：
- 新增 aether-hub Rust 项目，实现 proxy/worker 双向帧路由与 stream_id 重映射
- 新增 HubConnectionManager/HubTunnelTransport，worker 通过 Hub 转发 tunnel 帧
- 新增 create_tunnel_transport 工厂函数，按运行环境自动选择 Hub 或直连模式
- 新增 NODE_STATUS 广播机制，Hub 实时通知所有 worker 节点连接状态变化
- CI/CD 新增 build-hub job，Dockerfile 集成 Hub 二进制，deploy.sh 适配 Hub 构建
- 默认 GUNICORN_WORKERS 从 4 降为 2
This commit is contained in:
fawney19
2026-03-02 02:43:14 +08:00
parent 97d42703da
commit 039a18c243
30 changed files with 3728 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/clients/http_client.py
View File

@@ -495,7 +495,7 @@ class HTTPClientPool:
调用方不应关闭此 client，其生命周期由 HTTPClientPool 管理。
当 timeout 非 None 时（流式请求），每次创建新 client，由调用方负责关闭。
"""
from src.services.proxy_node.tunnel_transport import TunnelTransport
from src.services.proxy_node.tunnel_transport import create_tunnel_transport
t = timeout or httpx.Timeout(
connect=config.http_connect_timeout,
@@ -507,7 +507,7 @@ class HTTPClientPool:
# 流式请求：每次创建新 client（调用方负责关闭）
if timeout is not None:
transport = TunnelTransport(node_id, timeout=timeout_secs or 60.0)
transport = create_tunnel_transport(node_id, timeout=timeout_secs or 60.0)
return httpx.AsyncClient(transport=transport, timeout=t)
# 非流式请求：复用缓存的 client（加锁与 proxy_clients 保持一致）
@@ -517,7 +517,7 @@ class HTTPClientPool:
if existing and not existing.is_closed:
return existing
transport = TunnelTransport(node_id, timeout=timeout_secs or 60.0)
transport = create_tunnel_transport(node_id, timeout=timeout_secs or 60.0)
client = httpx.AsyncClient(transport=transport, timeout=t)
cls._tunnel_clients[node_id] = client
return client

28
src/modules/proxy_nodes/__init__.py
View File

@@ -78,14 +78,22 @@ async def _on_startup() -> None:
from src.config import config
from src.services.proxy_node.health_scheduler import get_proxy_node_health_scheduler
from src.services.proxy_node.hub_config import get_hub_config
from src.utils.task_coordinator import StartupTaskCoordinator
logger = logging.getLogger("aether.modules.proxy_nodes")
hub_enabled = get_hub_config().enabled
if config.worker_processes > 1:
if config.worker_processes > 1 and not hub_enabled:
logger.warning(
"检测到 WEB_CONCURRENCY={}。Proxy tunnel 连接是进程内资源，"
"多 worker 场景可能出现节点显示 ONLINE 但当前 worker 无可用 tunnel 的情况。",
"多 worker 场景可能出现节点显示 ONLINE 但当前 worker 无可用 tunnel 的情况。"
"建议设置 GUNICORN_WORKERS/WEB_CONCURRENCY=1。",
config.worker_processes,
)
elif config.worker_processes > 1 and hub_enabled:
logger.info(
"检测到 WEB_CONCURRENCY={}，Hub 模式已启用，允许多 worker 共享 tunnel。",
config.worker_processes,
)
@@ -111,14 +119,22 @@ async def _on_shutdown() -> None:
import logging
from src.services.proxy_node.health_scheduler import get_proxy_node_health_scheduler
from src.services.proxy_node.tunnel_manager import get_tunnel_manager
from src.services.proxy_node.hub_config import get_hub_config
from src.utils.task_coordinator import StartupTaskCoordinator
logger = logging.getLogger("aether.modules.proxy_nodes")
# 先向所有 tunnel 连接发送 GoAway，让 proxy 端立即重连到其他 worker
manager = get_tunnel_manager()
await manager.shutdown_all()
hub_enabled = get_hub_config().enabled
if hub_enabled:
from src.services.proxy_node.hub_transport import shutdown_hub_connection_manager
await shutdown_hub_connection_manager()
else:
# 先向所有 tunnel 连接发送 GoAway，让 proxy 端立即重连到其他 worker
from src.services.proxy_node.tunnel_manager import get_tunnel_manager
manager = get_tunnel_manager()
await manager.shutdown_all()
from src.clients import get_redis_client

4
src/services/provider_ops/architectures/anyrouter.py
View File

@@ -214,9 +214,9 @@ async def _get_acw_cookie(
"verify": get_ssl_context(),
}
if tunnel_node_id:
from src.services.proxy_node.tunnel_transport import TunnelTransport
from src.services.proxy_node.tunnel_transport import create_tunnel_transport
client_kwargs["transport"] = TunnelTransport(tunnel_node_id, timeout=timeout)
client_kwargs["transport"] = create_tunnel_transport(tunnel_node_id, timeout=timeout)
elif proxy:
client_kwargs["proxy"] = proxy
logger.debug(f"获取 acw_sc__v2 Cookie 使用代理: {proxy}")

4
src/services/provider_ops/architectures/base.py
View File

@@ -127,9 +127,9 @@ class ProviderConnector(ABC):
"""
transport = None
if self._tunnel_node_id:
from src.services.proxy_node.tunnel_transport import TunnelTransport
from src.services.proxy_node.tunnel_transport import create_tunnel_transport
transport = TunnelTransport(self._tunnel_node_id, timeout=self._timeout)
transport = create_tunnel_transport(self._tunnel_node_id, timeout=self._timeout)
elif self._proxy:
transport = httpx.AsyncHTTPTransport(proxy=self._proxy)

4
src/services/provider_ops/architectures/nekocode.py
View File

@@ -197,9 +197,9 @@ class NekoCodeArchitecture(ProviderArchitecture):
proxy, tunnel_node_id = resolve_ops_proxy_config(config)
if tunnel_node_id:
from src.services.proxy_node.tunnel_transport import TunnelTransport
from src.services.proxy_node.tunnel_transport import create_tunnel_transport
client_kwargs["transport"] = TunnelTransport(tunnel_node_id, timeout=10.0)
client_kwargs["transport"] = create_tunnel_transport(tunnel_node_id, timeout=10.0)
elif proxy:
client_kwargs["proxy"] = proxy

8
src/services/provider_ops/architectures/sub2api.py
View File

@@ -106,9 +106,9 @@ class _Sub2ApiTokenMixin:
"""获取不带 auth hook 的裸 HTTP 客户端（用于登录/刷新 token）"""
transport = None
if self._tunnel_node_id:
from src.services.proxy_node.tunnel_transport import TunnelTransport
from src.services.proxy_node.tunnel_transport import create_tunnel_transport
transport = TunnelTransport(self._tunnel_node_id, timeout=self._timeout)
transport = create_tunnel_transport(self._tunnel_node_id, timeout=self._timeout)
elif self._proxy:
transport = httpx.AsyncHTTPTransport(proxy=self._proxy)
async with httpx.AsyncClient(
@@ -488,9 +488,9 @@ class Sub2ApiArchitecture(ProviderArchitecture):
"verify": get_ssl_context(),
}
if tunnel_node_id:
from src.services.proxy_node.tunnel_transport import TunnelTransport
from src.services.proxy_node.tunnel_transport import create_tunnel_transport
client_kwargs["transport"] = TunnelTransport(tunnel_node_id, timeout=30.0)
client_kwargs["transport"] = create_tunnel_transport(tunnel_node_id, timeout=30.0)
elif proxy:
client_kwargs["proxy"] = proxy

4
src/services/provider_ops/architectures/yescode.py
View File

@@ -245,9 +245,9 @@ class YesCodeArchitecture(ProviderArchitecture):
"verify": get_ssl_context(),
}
if tunnel_node_id:
from src.services.proxy_node.tunnel_transport import TunnelTransport
from src.services.proxy_node.tunnel_transport import create_tunnel_transport
client_kwargs["transport"] = TunnelTransport(tunnel_node_id, timeout=10.0)
client_kwargs["transport"] = create_tunnel_transport(tunnel_node_id, timeout=10.0)
elif proxy:
client_kwargs["proxy"] = proxy

4
src/services/provider_ops/service.py
View File

@@ -1045,9 +1045,9 @@ class ProviderOpsService:
"verify": get_ssl_context(),
}
if tunnel_node_id:
from src.services.proxy_node.tunnel_transport import TunnelTransport
from src.services.proxy_node.tunnel_transport import create_tunnel_transport
client_kwargs["transport"] = TunnelTransport(tunnel_node_id, timeout=30.0)
client_kwargs["transport"] = create_tunnel_transport(tunnel_node_id, timeout=30.0)
logger.debug("使用 tunnel 代理: node_id={}", tunnel_node_id)
elif proxy:
client_kwargs["proxy"] = proxy

70
src/services/proxy_node/hub_config.py Normal file
View File

@@ -0,0 +1,70 @@
"""
Tunnel Hub 配置
控制 worker 是否通过 aether-hub 转发 tunnel 帧。
设计约束：
- Hub 作为 Docker 内部固定服务运行
- 不对外暴露运行时配置项（不依赖 TUNNEL_HUB_* 环境变量）
"""
from __future__ import annotations
import os
from dataclasses import dataclass
_DOCKER_HUB_URL = "ws://127.0.0.1:8085"
_DOCKER_HUB_CONNECT_TIMEOUT_SECONDS = 5.0
_DOCKER_HUB_PING_INTERVAL_SECONDS = 15.0
_DOCKER_HUB_SEND_TIMEOUT_SECONDS = 10.0
_DOCKER_HUB_MAX_STREAMS = 2048
_DOCKER_HUB_MAX_FRAME_SIZE = 64 * 1024 * 1024
@dataclass(frozen=True)
class HubConfig:
enabled: bool
url: str
connect_timeout_seconds: float
ping_interval_seconds: float
send_timeout_seconds: float
max_streams: int
max_frame_size: int
@property
def worker_ws_url(self) -> str:
return f"{self.url.rstrip('/')}/worker"
_hub_config: HubConfig | None = None
def _is_docker_runtime() -> bool:
if os.getenv("DOCKER_CONTAINER", "").strip().lower() == "true":
return True
return os.path.exists("/.dockerenv")
def get_hub_config() -> HubConfig:
"""读取 Hub 配置（进程内缓存）。"""
global _hub_config
if _hub_config is not None:
return _hub_config
docker_runtime = _is_docker_runtime()
_hub_config = HubConfig(
enabled=docker_runtime,
url=_DOCKER_HUB_URL,
connect_timeout_seconds=_DOCKER_HUB_CONNECT_TIMEOUT_SECONDS,
ping_interval_seconds=_DOCKER_HUB_PING_INTERVAL_SECONDS,
send_timeout_seconds=_DOCKER_HUB_SEND_TIMEOUT_SECONDS,
max_streams=_DOCKER_HUB_MAX_STREAMS,
max_frame_size=_DOCKER_HUB_MAX_FRAME_SIZE,
)
return _hub_config
def reset_hub_config_cache() -> None:
"""测试或热更新场景下清理配置缓存。"""
global _hub_config
_hub_config = None

639
src/services/proxy_node/hub_transport.py Normal file
View File

@@ -0,0 +1,639 @@
"""
Hub 模式 tunnel transport
Worker 通过单条到 aether-hub 的 WebSocket 长连接转发 tunnel 帧。
"""
from __future__ import annotations
import asyncio
import gzip
import json
from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any
import aiohttp
import httpx
from aiohttp import WSMsgType
from src.core.logger import logger
from .hub_config import HubConfig, get_hub_config
from .tunnel_manager import TunnelStreamError, _StreamState
from .tunnel_protocol import Frame, FrameFlags, MsgType
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Coroutine
_TUNNEL_COMPRESS_MIN_SIZE = 512
_RECONNECT_DELAYS_SECONDS: tuple[float, ...] = (0.0, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0)
_HOP_BY_HOP_HEADERS = frozenset(
{
"host",
"transfer-encoding",
"content-length",
"connection",
"upgrade",
"keep-alive",
"proxy-authorization",
"proxy-connection",
"te",
"trailer",
}
)
_HOP_BY_HOP_HEADERS_BYTES = frozenset(h.encode("ascii") for h in _HOP_BY_HOP_HEADERS)
class HubConnectionManager:
"""Worker 进程级 Hub 连接管理器（单例）。"""
def __init__(self, config: HubConfig | None = None) -> None:
self._config = config or get_hub_config()
self._session: aiohttp.ClientSession | None = None
self._ws: aiohttp.ClientWebSocketResponse | None = None
self._connect_lock = asyncio.Lock()
self._write_lock = asyncio.Lock()
self._next_stream_id = 2
self._pending_streams: dict[int, _StreamState] = {}
self._reader_task: asyncio.Task[None] | None = None
self._ping_task: asyncio.Task[None] | None = None
self._reconnect_task: asyncio.Task[None] | None = None
self._background_tasks: set[asyncio.Task[None]] = set()
self._closing = False
@property
def is_connected(self) -> bool:
ws = self._ws
return ws is not None and not ws.closed
def _background(self, coro: Coroutine[Any, Any, None]) -> None:
task = asyncio.create_task(coro)
self._background_tasks.add(task)
task.add_done_callback(self._background_tasks.discard)
async def ensure_connected(self) -> None:
if self._closing:
raise TunnelStreamError("hub connection manager is shutting down")
if self.is_connected:
return
async with self._connect_lock:
if self._closing:
raise TunnelStreamError("hub connection manager is shutting down")
if self.is_connected:
return
try:
await self._connect_once()
except Exception as e:
self._start_reconnect_loop()
raise TunnelStreamError(f"failed to connect hub worker channel: {e}") from e
async def _ensure_session(self) -> aiohttp.ClientSession:
if self._session is None or self._session.closed:
self._session = aiohttp.ClientSession()
return self._session
async def _connect_once(self) -> None:
session = await self._ensure_session()
ws = await session.ws_connect(
self._config.worker_ws_url,
timeout=self._config.connect_timeout_seconds,
autoping=False,
heartbeat=None,
max_msg_size=self._config.max_frame_size,
)
old_ws = self._ws
self._ws = ws
if old_ws is not None and not old_ws.closed:
try:
await old_ws.close()
except Exception:
pass
if self._reader_task is not None:
self._reader_task.cancel()
if self._ping_task is not None:
self._ping_task.cancel()
self._reader_task = asyncio.create_task(self._reader_loop(ws))
self._ping_task = asyncio.create_task(self._ping_loop(ws))
logger.info("Hub worker channel connected: {}", self._config.worker_ws_url)
def _start_reconnect_loop(self) -> None:
if self._closing:
return
if self._reconnect_task is not None and not self._reconnect_task.done():
return
self._reconnect_task = asyncio.create_task(self._reconnect_loop())
async def _reconnect_loop(self) -> None:
attempt = 0
while not self._closing and not self.is_connected:
delay = _RECONNECT_DELAYS_SECONDS[min(attempt, len(_RECONNECT_DELAYS_SECONDS) - 1)]
if delay > 0:
await asyncio.sleep(delay)
try:
async with self._connect_lock:
if self._closing or self.is_connected:
break
await self._connect_once()
if self.is_connected:
logger.info("Hub worker channel reconnected")
break
except Exception as e:
attempt += 1
logger.debug("Hub reconnect attempt {} failed: {}", attempt, e)
async def _handle_disconnect(
self,
reason: str,
*,
ws: aiohttp.ClientWebSocketResponse | None = None,
) -> None:
current: aiohttp.ClientWebSocketResponse | None = None
async with self._connect_lock:
if self._ws is None:
return
if ws is not None and self._ws is not ws:
return
current = self._ws
self._ws = None
if current is not None and not current.closed:
try:
await current.close()
except Exception:
pass
if self._pending_streams:
for state in self._pending_streams.values():
state.set_error("hub disconnected")
self._pending_streams.clear()
if not self._closing:
logger.warning("Hub worker channel disconnected: {}", reason)
self._start_reconnect_loop()
async def _send_frame(self, frame: Frame) -> None:
ws = self._ws
if ws is None or ws.closed:
raise TunnelStreamError("hub not connected")
try:
async with asyncio.timeout(self._config.send_timeout_seconds):
async with self._write_lock:
await ws.send_bytes(frame.encode())
except TimeoutError as e:
await self._handle_disconnect("send timeout", ws=ws)
raise TunnelStreamError("hub frame send timeout") from e
except Exception as e:
await self._handle_disconnect(f"send failed: {e}", ws=ws)
raise TunnelStreamError(f"hub frame send failed: {e}") from e
async def _reader_loop(self, ws: aiohttp.ClientWebSocketResponse) -> None:
try:
while not self._closing:
msg = await ws.receive()
if msg.type == WSMsgType.BINARY:
raw = msg.data
if isinstance(raw, memoryview):
raw = raw.tobytes()
elif isinstance(raw, bytearray):
raw = bytes(raw)
if not isinstance(raw, bytes):
continue
try:
frame = Frame.decode(raw)
except Exception as e:
logger.debug("invalid frame from hub: {}", e)
continue
await self._handle_incoming_frame(frame)
continue
if msg.type == WSMsgType.CLOSE or msg.type == WSMsgType.CLOSED:
break
if msg.type == WSMsgType.ERROR:
logger.debug("hub ws reader error: {}", ws.exception())
break
if msg.type == WSMsgType.PING:
payload = msg.data if isinstance(msg.data, bytes) else b""
self._background(self._send_pong(payload))
continue
# TEXT / PONG / 其他类型直接忽略
except asyncio.CancelledError:
return
except Exception as e:
logger.debug("hub reader loop aborted: {}", e)
finally:
await self._handle_disconnect("reader ended", ws=ws)
async def _ping_loop(self, ws: aiohttp.ClientWebSocketResponse) -> None:
try:
while not self._closing:
await asyncio.sleep(self._config.ping_interval_seconds)
if self._ws is not ws or ws.closed:
break
try:
await self._send_frame(Frame(0, MsgType.PING, 0, b""))
except TunnelStreamError:
break
except asyncio.CancelledError:
return
async def _send_pong(self, payload: bytes) -> None:
try:
await self._send_frame(Frame(0, MsgType.PONG, 0, payload))
except TunnelStreamError:
pass
async def _handle_incoming_frame(self, frame: Frame) -> None:
match frame.msg_type:
# -- stream-level frames --
case MsgType.RESPONSE_HEADERS:
stream = self._pending_streams.get(frame.stream_id)
if not stream:
return
try:
payload = _decompress_frame_payload(frame)
meta = json.loads(payload)
stream.set_response_headers(meta["status"], meta.get("headers", []))
except Exception as e:
stream.set_error(f"invalid response headers: {e}")
self._pending_streams.pop(frame.stream_id, None)
case MsgType.RESPONSE_BODY:
stream = self._pending_streams.get(frame.stream_id)
if stream:
stream.push_body_chunk(_decompress_frame_payload(frame))
case MsgType.STREAM_END:
stream = self._pending_streams.pop(frame.stream_id, None)
if stream:
stream.set_done()
case MsgType.STREAM_ERROR:
stream = self._pending_streams.pop(frame.stream_id, None)
if stream:
message = (
frame.payload.decode(errors="replace") if frame.payload else "stream error"
)
stream.set_error(message)
# -- connection-level frames --
case MsgType.PING:
self._background(self._send_pong(frame.payload))
case MsgType.PONG:
pass
case MsgType.GOAWAY:
await self._handle_disconnect("received GOAWAY")
case MsgType.HEARTBEAT_DATA:
self._background(self._handle_heartbeat(frame))
case MsgType.HEARTBEAT_ACK:
pass
case MsgType.NODE_STATUS:
self._background(self._handle_node_status(frame.payload))
async def _handle_heartbeat(self, frame: Frame) -> None:
try:
data = json.loads(frame.payload) if frame.payload else {}
except Exception:
data = {}
node_id = str(data.get("node_id") or "").strip()
def _sync_heartbeat() -> dict[str, object]:
from src.database import create_session
from src.services.proxy_node.service import ProxyNodeService
if not node_id:
return {}
db = create_session()
try:
node = ProxyNodeService.heartbeat(
db,
node_id=node_id,
active_connections=data.get("active_connections"),
total_requests=data.get("total_requests"),
avg_latency_ms=data.get("avg_latency_ms"),
failed_requests=data.get("failed_requests"),
dns_failures=data.get("dns_failures"),
stream_errors=data.get("stream_errors"),
)
result: dict[str, object] = {}
if node.remote_config:
result["remote_config"] = node.remote_config
result["config_version"] = node.config_version or 0
return result
finally:
db.close()
try:
ack = await asyncio.to_thread(_sync_heartbeat)
except Exception as e:
logger.warning("hub heartbeat DB update failed: {}", e)
ack = {}
try:
await self._send_frame(
Frame(
frame.stream_id,
MsgType.HEARTBEAT_ACK,
0,
json.dumps(ack, ensure_ascii=False).encode("utf-8"),
)
)
except TunnelStreamError:
logger.debug("hub heartbeat ACK send failed")
async def _handle_node_status(self, payload: bytes) -> None:
try:
data = json.loads(payload) if payload else {}
except Exception:
return
node_id = str(data.get("node_id") or "").strip()
if not node_id:
return
connected = bool(data.get("connected"))
conn_count = int(data.get("conn_count") or 0)
# 所有 worker 都需要立即失效本地缓存，保证请求路由正确
try:
from src.services.proxy_node.resolver import invalidate_proxy_node_cache
invalidate_proxy_node_cache(node_id)
except Exception:
pass
# 使用 Redis SETNX 去重：同一次 NODE_STATUS 广播只有一个 worker 执行 DB 写入，
# 避免 N 个 worker 并发写同一行并产生 N 条重复事件记录。
dedup_key = f"hub:node_status:{node_id}:{connected}:{conn_count}"
try:
from src.clients import get_redis_client
redis = await get_redis_client()
if redis:
acquired = await redis.set(dedup_key, "1", ex=10, nx=True)
if not acquired:
return
except Exception:
# Redis 不可用时不去重，允许重复写入（幂等）
pass
def _sync_update() -> None:
from src.database import create_session
from src.models.database import ProxyNode, ProxyNodeEvent, ProxyNodeStatus
db = create_session()
try:
node = db.query(ProxyNode).filter(ProxyNode.id == node_id).first()
if not node:
return
now = datetime.now(timezone.utc)
node.tunnel_connected = connected
if connected:
node.tunnel_connected_at = now
node.status = ProxyNodeStatus.ONLINE if connected else ProxyNodeStatus.OFFLINE
node.updated_at = now
event = ProxyNodeEvent(
node_id=node_id,
event_type="connected" if connected else "disconnected",
detail=f"[hub_node_status] conn_count={conn_count}",
)
db.add(event)
db.commit()
except Exception:
try:
db.rollback()
except Exception:
pass
raise
finally:
db.close()
try:
await asyncio.to_thread(_sync_update)
except Exception as e:
logger.warning("hub NODE_STATUS DB update failed: node_id={}, error={}", node_id, e)
async def send_request(
self,
node_id: str,
*,
method: str,
url: str,
headers: dict[str, str],
body: bytes | None = None,
timeout: float = 60.0,
) -> _StreamState:
await self.ensure_connected()
if len(self._pending_streams) >= self._config.max_streams:
raise TunnelStreamError(
f"hub stream limit reached ({self._config.max_streams}) for node {node_id}"
)
stream_id = self._alloc_stream_id()
stream_state = _StreamState(stream_id)
self._pending_streams[stream_id] = stream_state
try:
meta = json.dumps(
{
"node_id": node_id,
"method": method,
"url": url,
"headers": headers,
"timeout": timeout,
},
ensure_ascii=False,
separators=(",", ":"),
).encode("utf-8")
meta_payload, meta_flags = _compress_frame_payload(meta)
await self._send_frame(
Frame(stream_id, MsgType.REQUEST_HEADERS, meta_flags, meta_payload)
)
body_data = body or b""
if body_data:
body_payload, body_flags = _compress_frame_payload(body_data)
else:
body_payload, body_flags = body_data, 0
body_flags |= FrameFlags.END_STREAM
await self._send_frame(Frame(stream_id, MsgType.REQUEST_BODY, body_flags, body_payload))
except Exception:
self._pending_streams.pop(stream_id, None)
raise
return stream_state
def remove_stream(self, stream_id: int) -> None:
self._pending_streams.pop(stream_id, None)
def _alloc_stream_id(self) -> int:
sid = self._next_stream_id
self._next_stream_id = sid + 2 if sid < 0xFFFF_FFFE else 2
return sid
async def shutdown(self) -> None:
self._closing = True
if self._reconnect_task is not None:
self._reconnect_task.cancel()
if self._reader_task is not None:
self._reader_task.cancel()
if self._ping_task is not None:
self._ping_task.cancel()
tasks = list(self._background_tasks)
for task in tasks:
task.cancel()
if tasks:
await asyncio.gather(*tasks, return_exceptions=True)
if self._ws is not None and not self._ws.closed:
try:
await self._ws.close()
except Exception:
pass
self._ws = None
if self._session is not None and not self._session.closed:
try:
await self._session.close()
except Exception:
pass
self._session = None
if self._pending_streams:
for state in self._pending_streams.values():
state.set_error("hub connection manager shutdown")
self._pending_streams.clear()
logger.info("Hub connection manager shutdown completed")
class HubTunnelTransport(httpx.AsyncBaseTransport):
"""通过 aether-hub 转发请求的 httpx transport。"""
def __init__(self, node_id: str, timeout: float = 60.0) -> None:
self._node_id = node_id
self._timeout = timeout
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
manager = get_hub_connection_manager()
headers: dict[str, str] = {}
for key, value in request.headers.raw:
if key not in _HOP_BY_HOP_HEADERS_BYTES:
headers[key.decode("latin-1")] = value.decode("latin-1")
body = request.content or await request.aread() or None
stream_state: _StreamState | None = None
try:
stream_state = await manager.send_request(
self._node_id,
method=request.method,
url=str(request.url),
headers=headers,
body=body,
timeout=self._timeout,
)
await stream_state.wait_headers(timeout=self._timeout)
return httpx.Response(
status_code=stream_state.status,
headers=httpx.Headers(stream_state.headers),
stream=HubResponseStream(manager, stream_state, timeout=self._timeout),
)
except TunnelStreamError as e:
self._cleanup_stream(manager, stream_state)
if stream_state and stream_state.status > 0:
raise httpx.ReadError(str(e)) from e
raise httpx.ConnectError(str(e)) from e
except asyncio.TimeoutError:
self._cleanup_stream(manager, stream_state)
raise httpx.ReadTimeout("hub tunnel request timeout") from None
def _cleanup_stream(
self,
manager: HubConnectionManager,
stream_state: _StreamState | None,
) -> None:
if stream_state is None:
return
manager.remove_stream(stream_state.stream_id)
class HubResponseStream(httpx.AsyncByteStream):
def __init__(
self,
manager: HubConnectionManager,
stream_state: _StreamState,
timeout: float = 60.0,
) -> None:
self._manager = manager
self._stream_state = stream_state
self._timeout = timeout
async def __aiter__(self) -> AsyncGenerator[bytes, None]:
async for chunk in self._stream_state.iter_body(chunk_timeout=self._timeout):
yield chunk
async def aclose(self) -> None:
self._manager.remove_stream(self._stream_state.stream_id)
def _compress_frame_payload(data: bytes) -> tuple[bytes, int]:
if len(data) >= _TUNNEL_COMPRESS_MIN_SIZE:
compressed = gzip.compress(data, compresslevel=6)
if len(compressed) < len(data):
return compressed, FrameFlags.GZIP_COMPRESSED
return data, 0
def _decompress_frame_payload(frame: Frame) -> bytes:
if frame.is_gzip:
return gzip.decompress(frame.payload)
return frame.payload
_hub_connection_manager: HubConnectionManager | None = None
def get_hub_connection_manager() -> HubConnectionManager:
global _hub_connection_manager
if _hub_connection_manager is None:
_hub_connection_manager = HubConnectionManager()
return _hub_connection_manager
async def shutdown_hub_connection_manager() -> None:
global _hub_connection_manager
if _hub_connection_manager is None:
return
await _hub_connection_manager.shutdown()
_hub_connection_manager = None

113
src/services/proxy_node/resolver.py
View File

@@ -33,6 +33,51 @@ _TUNNEL_LOCAL_MISS_LOG_COOLDOWN_SECONDS = 30.0
_tunnel_local_miss_log_next_at: dict[str, float] = {}
def _is_hub_mode_enabled() -> bool:
from src.services.proxy_node.hub_config import get_hub_config
return get_hub_config().enabled
def _build_tunnel_local_miss_message(node_id: str) -> str:
"""构建“当前 worker 无本地 tunnel”的用户可读错误信息。"""
return (
f"代理节点 {node_id} 当前 worker 无 tunnel 连接（pid={os.getpid()}）。"
"这通常发生在多 worker 部署（WEB_CONCURRENCY/GUNICORN_WORKERS > 1）时。"
"请设置 GUNICORN_WORKERS=1（或 WEB_CONCURRENCY=1），"
"或确保每个 worker 都建立该节点的 tunnel 连接。"
)
def _is_tunnel_local_miss(node_id: str) -> bool:
"""判断节点是否“全局在线但当前 worker 无本地 tunnel 连接”。
仅在错误路径调用（build_proxy_url 解析失败时），用于提供更准确的报错信息。
"""
if _is_hub_mode_enabled():
return False
from src.database import create_session
from src.models.database import ProxyNode, ProxyNodeStatus
from src.services.proxy_node.tunnel_manager import get_tunnel_manager
db = create_session()
try:
node = db.query(ProxyNode).filter(ProxyNode.id == node_id).first()
if not node:
return False
if node.is_manual or not node.tunnel_mode:
return False
if node.status != ProxyNodeStatus.ONLINE:
return False
manager = get_tunnel_manager()
return not manager.has_tunnel(node_id)
except Exception:
return False
finally:
db.close()
def _get_proxy_node_info(node_id: str) -> dict[str, Any] | None:
"""
读取 ProxyNode 信息（带内存 TTL 缓存）
@@ -71,32 +116,41 @@ def _get_proxy_node_info(node_id: str) -> dict[str, Any] | None:
_proxy_node_cache[node_id] = (None, now + _PROXY_NODE_CACHE_NEGATIVE_TTL_SECONDS)
return None
# tunnel 模式节点：以 TunnelManager 内存中的实际连接状态为准，
# 而非依赖 DB 的 status/tunnel_connected 字段（可能因竞态不同步）。
# tunnel 模式节点：
# - Hub 模式：信任 DB 的 tunnel_connected/status（由 Hub 广播统一维护）
# - 非 Hub 模式：以当前 worker 本地 TunnelManager 状态为准
if node.tunnel_mode and not node.is_manual:
from src.services.proxy_node.tunnel_manager import get_tunnel_manager
if _is_hub_mode_enabled():
if node.status != ProxyNodeStatus.ONLINE or not bool(node.tunnel_connected):
_proxy_node_cache[node_id] = (
None,
now + _PROXY_NODE_CACHE_NEGATIVE_TTL_SECONDS,
)
return None
else:
from src.services.proxy_node.tunnel_manager import get_tunnel_manager
manager = get_tunnel_manager()
if not manager.has_tunnel(node_id):
# 多 worker 部署时，DB 可能显示 ONLINE（其他 worker 有 tunnel），
# 但当前 worker 无本地连接，请求仍不可用。记录限频告警便于定位。
if now >= _tunnel_local_miss_log_next_at.get(node_id, 0.0):
_tunnel_local_miss_log_next_at[node_id] = (
now + _TUNNEL_LOCAL_MISS_LOG_COOLDOWN_SECONDS
manager = get_tunnel_manager()
if not manager.has_tunnel(node_id):
# 多 worker 部署时，DB 可能显示 ONLINE（其他 worker 有 tunnel），
# 但当前 worker 无本地连接，请求仍不可用。记录限频告警便于定位。
if now >= _tunnel_local_miss_log_next_at.get(node_id, 0.0):
_tunnel_local_miss_log_next_at[node_id] = (
now + _TUNNEL_LOCAL_MISS_LOG_COOLDOWN_SECONDS
)
logger.warning(
"tunnel node {} has no local connection on pid={} "
"(db_status={}, db_tunnel_connected={}), request may fail on this worker",
node_id,
os.getpid(),
str(getattr(node, "status", "unknown")),
bool(getattr(node, "tunnel_connected", False)),
)
_proxy_node_cache[node_id] = (
None,
now + _PROXY_NODE_CACHE_TUNNEL_LOCAL_MISS_TTL_SECONDS,
)
logger.warning(
"tunnel node {} has no local connection on pid={} "
"(db_status={}, db_tunnel_connected={}), request may fail on this worker",
node_id,
os.getpid(),
str(getattr(node, "status", "unknown")),
bool(getattr(node, "tunnel_connected", False)),
)
_proxy_node_cache[node_id] = (
None,
now + _PROXY_NODE_CACHE_TUNNEL_LOCAL_MISS_TTL_SECONDS,
)
return None
return None
value: dict[str, Any] = {
"name": node.name,
"ip": node.ip,
@@ -408,13 +462,13 @@ def build_proxy_client_kwargs(
kwargs: dict[str, Any] = {"timeout": timeout, "verify": verify, **extra}
# tunnel 模式优先：当代理节点为 tunnel 模式时，使用 TunnelTransport
# tunnel 模式优先：当代理节点为 tunnel 模式时，使用 tunnel transport 工厂
delegate_cfg = resolve_delegate_config(proxy_config)
if delegate_cfg and delegate_cfg.get("tunnel"):
from src.services.proxy_node.tunnel_transport import TunnelTransport
from src.services.proxy_node.tunnel_transport import create_tunnel_transport
timeout_secs = timeout if isinstance(timeout, (int, float)) else 60.0
kwargs["transport"] = TunnelTransport(delegate_cfg["node_id"], timeout=timeout_secs)
kwargs["transport"] = create_tunnel_transport(delegate_cfg["node_id"], timeout=timeout_secs)
return kwargs
proxy_param = resolve_proxy_param(proxy_config)
@@ -455,7 +509,12 @@ def build_proxy_url(proxy_config: dict[str, Any]) -> str | None:
node_info = _get_proxy_node_info(node_id)
if not node_info:
logger.warning("代理节点不可用（离线或不存在）: node_id={}", node_id)
raise ProxyNodeUnavailableError(f"代理节点 {node_id} 不可用", node_id=node_id)
message = (
_build_tunnel_local_miss_message(node_id)
if _is_tunnel_local_miss(node_id)
else f"代理节点 {node_id} 不可用"
)
raise ProxyNodeUnavailableError(message, node_id=node_id)
# 手动节点：直接使用存储的代理 URL（含认证信息）
if node_info.get("is_manual"):

37
src/services/proxy_node/service.py
View File

@@ -177,10 +177,10 @@ async def _test_tunnel_connectivity(node_id: str) -> dict[str, Any]:
"""通过 WebSocket tunnel 测试连通性，返回标准化结果 dict"""
import time as _time
from .tunnel_transport import TunnelTransport
from .tunnel_transport import create_tunnel_transport
test_url = "https://1.1.1.1/cdn-cgi/trace"
transport = TunnelTransport(node_id, timeout=15.0)
transport = create_tunnel_transport(node_id, timeout=15.0)
start = _time.monotonic()
try:
@@ -556,17 +556,38 @@ class ProxyNodeService:
# tunnel 节点：通过 WebSocket tunnel 测试
if not node.is_manual:
# 以 TunnelManager 内存中的实际连接状态为准（与 health_scheduler 一致），
# 而非仅依赖 DB 的 tunnel_connected 字段，避免竞态导致误判。
from src.services.proxy_node.tunnel_manager import get_tunnel_manager
from src.services.proxy_node.hub_config import get_hub_config
manager = get_tunnel_manager()
if not manager.has_tunnel(node.id):
hub_enabled = get_hub_config().enabled
connected = False
if hub_enabled:
connected = bool(node.tunnel_connected) and node.status == ProxyNodeStatus.ONLINE
else:
# 非 Hub 模式：以当前 worker 本地 TunnelManager 状态为准，
# 避免多 worker 场景的跨进程状态误判。
from src.services.proxy_node.tunnel_manager import get_tunnel_manager
manager = get_tunnel_manager()
connected = manager.has_tunnel(node.id)
if not connected:
hint = ""
try:
from src.config import config
if config.worker_processes > 1 and not hub_enabled:
hint = (
"（当前 worker 无 tunnel 连接；检测到多 worker 部署，"
"建议设置 GUNICORN_WORKERS/WEB_CONCURRENCY=1）"
)
except Exception:
# 配置读取失败时保持原始错误，避免影响主流程
hint = ""
return {
"success": False,
"latency_ms": None,
"exit_ip": None,
"error": "tunnel 未连接",
"error": f"tunnel 未连接{hint}",
}
result = await _test_tunnel_connectivity(node.id)

1
src/services/proxy_node/tunnel_protocol.py
View File

@@ -29,6 +29,7 @@ class MsgType(IntEnum):
GOAWAY = 0x12 # \u53cc\u5411: \u4f18\u96c5\u5173\u95ed (stream_id=0)
HEARTBEAT_DATA = 0x13 # Proxy -> Aether: \u6307\u6807\u4e0a\u62a5
HEARTBEAT_ACK = 0x14 # Aether -> Proxy: \u5fc3\u8df3\u786e\u8ba4 + \u8fdc\u7a0b\u914d\u7f6e
NODE_STATUS = 0x15 # Hub -> Worker: \u8282\u70b9\u8fde\u63a5\u72b6\u6001\u5e7f\u64ad
class FrameFlags:

12
src/services/proxy_node/tunnel_transport.py
View File

@@ -136,3 +136,15 @@ def is_tunnel_node(node_info: dict[str, Any] | None) -> bool:
if not node_info:
return False
return bool(node_info.get("tunnel_mode")) and bool(node_info.get("tunnel_connected"))
def create_tunnel_transport(node_id: str, timeout: float = 60.0) -> httpx.AsyncBaseTransport:
"""根据配置创建 tunnel transport（Hub 模式或直连 tunnel 模式）。"""
from .hub_config import get_hub_config
hub_cfg = get_hub_config()
if hub_cfg.enabled:
from .hub_transport import HubTunnelTransport
return HubTunnelTransport(node_id, timeout=timeout)
return TunnelTransport(node_id, timeout=timeout)