i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-03 01:40:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(tunnel): 重连指数退避、多worker兼容与手动节点请求计数

Browse Source 
- Proxy tunnel 重连策略从固定1s改为指数退避+jitter，首次重试立即执行，
  稳定连接30s后重置退避计数，上限3s保证快速恢复
- 多连接启动时增加错峰延迟，避免同时发起连接风暴
- 服务端 tunnel ping间隔和空闲超时支持环境变量配置
- 修复多worker启动时tunnel状态重置逻辑，仅leader执行重置避免覆盖其他worker连接
- Resolver增加本地tunnel缺失的限频告警和更短缓存TTL，加速多worker场景恢复
- 用量记录中统计手动代理节点的请求数和失败数
This commit is contained in:
fawney19
2026-03-01 17:37:11 +08:00
parent 2c1e51a490
commit d4840df447
6 changed files with 340 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
src/services/proxy_node/resolver.py
View File

@@ -10,6 +10,7 @@ from __future__ import annotations
import gzip
import hashlib
import json
import os
import time
from typing import Any
from urllib.parse import quote, urlparse
@@ -26,7 +27,10 @@ from src.core.logger import logger
_proxy_node_cache: dict[str, tuple[dict[str, Any] | None, float]] = {}
_PROXY_NODE_CACHE_TTL_SECONDS = 15.0
_PROXY_NODE_CACHE_NEGATIVE_TTL_SECONDS = 5.0 # 不可用节点使用更短的 TTL，加速恢复感知
_PROXY_NODE_CACHE_TUNNEL_LOCAL_MISS_TTL_SECONDS = 0.5 # 本地 worker 无 tunnel 时，快速重试
_PROXY_NODE_CACHE_MAX_SIZE = 256
_TUNNEL_LOCAL_MISS_LOG_COOLDOWN_SECONDS = 30.0
_tunnel_local_miss_log_next_at: dict[str, float] = {}
def _get_proxy_node_info(node_id: str) -> dict[str, Any] | None:
@@ -74,9 +78,23 @@ def _get_proxy_node_info(node_id: str) -> dict[str, Any] | None:
manager = get_tunnel_manager()
if not manager.has_tunnel(node_id):
# 多 worker 部署时，DB 可能显示 ONLINE（其他 worker 有 tunnel），
# 但当前 worker 无本地连接，请求仍不可用。记录限频告警便于定位。
if now >= _tunnel_local_miss_log_next_at.get(node_id, 0.0):
_tunnel_local_miss_log_next_at[node_id] = (
now + _TUNNEL_LOCAL_MISS_LOG_COOLDOWN_SECONDS
)
logger.warning(
"tunnel node {} has no local connection on pid={} "
"(db_status={}, db_tunnel_connected={}), request may fail on this worker",
node_id,
os.getpid(),
str(getattr(node, "status", "unknown")),
bool(getattr(node, "tunnel_connected", False)),
)
_proxy_node_cache[node_id] = (
None,
now + _PROXY_NODE_CACHE_NEGATIVE_TTL_SECONDS,
now + _PROXY_NODE_CACHE_TUNNEL_LOCAL_MISS_TTL_SECONDS,
)
return None
value: dict[str, Any] = {
@@ -127,6 +145,7 @@ _SYSTEM_PROXY_CACHE_TTL = 60.0
def invalidate_proxy_node_cache(node_id: str) -> None:
"""主动清除指定节点的信息缓存（tunnel 断开时调用，避免使用过期的连接状态）"""
_proxy_node_cache.pop(node_id, None)
_tunnel_local_miss_log_next_at.pop(node_id, None)
def invalidate_system_proxy_cache() -> None:
@@ -509,7 +528,10 @@ def resolve_proxy_info(proxy_config: dict[str, Any] | None) -> dict[str, Any] |
node_id = node_id.strip()
node_info = _get_proxy_node_info(node_id)
node_name = node_info.get("name", "unknown") if node_info else "offline"
return {"node_id": node_id, "node_name": node_name, "source": source}
info: dict[str, Any] = {"node_id": node_id, "node_name": node_name, "source": source}
if node_info and node_info.get("is_manual"):
info["is_manual"] = True
return info
# 旧格式 URL 模式
proxy_url = effective_config.get("url")

69
src/services/usage/recording.py
View File

@@ -7,7 +7,7 @@ from typing import Any
from sqlalchemy.orm import Session
from src.core.logger import logger
from src.models.database import ApiKey, Provider, Usage, User, UserModelUsageCount
from src.models.database import ApiKey, Provider, ProxyNode, Usage, User, UserModelUsageCount
from src.services.provider_keys.codex_quota_sync_dispatcher import (
dispatch_codex_quota_sync_from_response_headers,
)
@@ -22,6 +22,50 @@ from src.services.usage._recording_helpers import (
from src.services.usage._types import UsageCostInfo, UsageRecordParams
def _extract_manual_proxy_node_id(metadata: dict[str, Any] | None) -> str | None:
"""从 request_metadata 中提取手动代理节点 ID（仅 is_manual 节点）。
Tunnel 节点的统计由 aether-proxy 心跳上报，此处只处理手动节点以避免重复计数。
"""
if not metadata:
return None
proxy = metadata.get("proxy")
if not isinstance(proxy, dict):
return None
if not proxy.get("is_manual"):
return None
node_id = proxy.get("node_id")
return node_id if isinstance(node_id, str) and node_id.strip() else None
def _increment_proxy_node_requests(
db: Session,
node_counts: dict[str, int],
failed_counts: dict[str, int] | None = None,
) -> None:
"""批量递增手动代理节点的 total_requests 和 failed_requests（原子 SQL UPDATE）。"""
if not node_counts and not failed_counts:
return
from sqlalchemy import update
# 合并所有涉及的 node_id
all_ids = set(node_counts) | set(failed_counts or {})
for node_id in all_ids:
total = node_counts.get(node_id, 0)
failed = (failed_counts or {}).get(node_id, 0)
values: dict[str, Any] = {}
if total > 0:
values["total_requests"] = ProxyNode.total_requests + total
if failed > 0:
values["failed_requests"] = ProxyNode.failed_requests + failed
if values:
db.execute(
update(ProxyNode)
.where(ProxyNode.id == node_id, ProxyNode.is_manual == True) # noqa: E712
.values(**values)
)
class UsageRecordingMixin(UsageBillingIntegrationMixin):
"""记录用量相关方法"""
@@ -404,6 +448,12 @@ class UsageRecordingMixin(UsageBillingIntegrationMixin):
.values(monthly_used_usd=Provider.monthly_used_usd + actual_total_cost)
)
# 更新手动代理节点请求计数（tunnel 节点由心跳上报，不在此处统计）
manual_node_id = _extract_manual_proxy_node_id(metadata)
if manual_node_id:
failed = {manual_node_id: 1} if status == "failed" else None
_increment_proxy_node_requests(db, {manual_node_id: 1}, failed)
# 结算标记：终态请求写入 settled + finalized_at
if status not in ("pending", "streaming"):
usage.billing_status = "settled"
@@ -785,6 +835,8 @@ class UsageRecordingMixin(UsageBillingIntegrationMixin):
int
) # (user_id, model) -> count
provider_costs: dict[str, float] = defaultdict(float) # provider_id -> cost
proxy_node_counts: dict[str, int] = defaultdict(int) # node_id -> request count
proxy_node_failed: dict[str, int] = defaultdict(int) # node_id -> failed count
quota_update_candidates: dict[str, dict[str, Any]] = {}
# 合并所有需要处理的记录（用于预取 user/api_key）
@@ -944,6 +996,12 @@ class UsageRecordingMixin(UsageBillingIntegrationMixin):
apikey_stats[key_id]["cost"] += total_cost
apikey_stats[key_id]["is_standalone"] = api_key.is_standalone
manual_nid = _extract_manual_proxy_node_id(record.get("metadata"))
if manual_nid:
proxy_node_counts[manual_nid] += 1
if record.get("status") == "failed":
proxy_node_failed[manual_nid] += 1
provider_api_key_id = record.get("provider_api_key_id")
response_headers = record.get("response_headers")
if (
@@ -1005,6 +1063,12 @@ class UsageRecordingMixin(UsageBillingIntegrationMixin):
apikey_stats[key_id]["cost"] += total_cost
apikey_stats[key_id]["is_standalone"] = api_key.is_standalone
manual_nid = _extract_manual_proxy_node_id(record.get("metadata"))
if manual_nid:
proxy_node_counts[manual_nid] += 1
if record.get("status") == "failed":
proxy_node_failed[manual_nid] += 1
provider_api_key_id = record.get("provider_api_key_id")
response_headers = record.get("response_headers")
if (
@@ -1130,6 +1194,9 @@ class UsageRecordingMixin(UsageBillingIntegrationMixin):
)
)
# 批量更新手动代理节点请求计数
_increment_proxy_node_requests(db, proxy_node_counts, proxy_node_failed)
# 配额头实时同步：同一 key 仅取本批次最后一组响应头并执行一次对比更新。
for provider_api_key_id, response_headers in quota_update_candidates.items():
dispatch_codex_quota_sync_from_response_headers(