i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-03 01:40:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 共享请求管道、按需懒加载、流式内存护栏与连接池治理

Browse Source 
- 抽取 ApiRequestPipeline 单例，44 个路由文件共享同一实例
- Handler/Adapter 模块级 __getattr__ 延迟导入，减少启动时间
- 新增 ensure_stream_buffer_limit() 流式内存护栏（16MB 单行 / 32MB 总量）
- HTTP 空闲连接清理与 curl_cffi LRU 会话池
- ensure_providers_bootstrapped 按需引导指定 provider_types
- Usage 事件序列化迁移至 msgpack，Redis codec 隔离
- 启动预热任务（/readyz 就绪门控）与优雅关闭
- 通知邮件模块独立开关与 SMTP 配置校验
- CryptoService DCL 线程安全修复
- 通知模块开关 DB 查询 30s 内存缓存
- /readyz 对 unknown 状态返回 503
- 预热关闭 5s 超时保护
- 预热适配器逐个 try-except 容错
- FormatConversionRegistry 哨兵模式防并发重复物化
- 流式缓冲检查无条件执行

Closes #230

Co-authored-by: AAEE86 <ppk0227@hotmail.com>
This commit is contained in:
fawney19
2026-03-14 11:59:07 +08:00
parent 45985f1c04
commit e0286aebe3
111 changed files with 2775 additions and 1102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
src/clients/curl_cffi_transport.py
View File

@@ -20,6 +20,8 @@ Design notes:
from __future__ import annotations
import asyncio
import os
from collections import OrderedDict
from typing import Any
import httpx
@@ -46,10 +48,21 @@ except ImportError:
DEFAULT_IMPERSONATE = "chrome120"
def _get_max_sessions() -> int:
raw = os.getenv("CURL_CFFI_MAX_SESSIONS", "20")
try:
value = int(raw)
except ValueError:
logger.warning("环境变量 CURL_CFFI_MAX_SESSIONS 非法: {}, 使用默认值 20", raw)
return 20
return max(1, value)
# ---------------------------------------------------------------------------
# Session pool (module-level, async-safe)
# ---------------------------------------------------------------------------
_session_pool: dict[str, AsyncSession] = {}
_MAX_SESSIONS = _get_max_sessions()
_session_pool: OrderedDict[str, AsyncSession] = OrderedDict()
_pool_lock = asyncio.Lock()
@@ -63,9 +76,11 @@ async def _get_or_create_session(
) -> AsyncSession:
"""Get or create a cached curl_cffi AsyncSession."""
key = _session_key(impersonate, proxy)
evicted_sessions: list[tuple[str, AsyncSession]] = []
async with _pool_lock:
session = _session_pool.get(key)
if session is not None:
_session_pool.move_to_end(key)
return session
kwargs: dict[str, Any] = {
@@ -77,12 +92,22 @@ async def _get_or_create_session(
session = AsyncSession(**kwargs)
_session_pool[key] = session
_session_pool.move_to_end(key)
while len(_session_pool) > _MAX_SESSIONS:
old_key, old_session = _session_pool.popitem(last=False)
evicted_sessions.append((old_key, old_session))
logger.info(
"curl_cffi session created: impersonate={}, proxy={}",
impersonate,
proxy or "direct",
)
return session
for old_key, old_session in evicted_sessions:
try:
await old_session.close()
logger.debug("curl_cffi session evicted: {}", old_key)
except Exception as exc:
logger.warning("curl_cffi session close failed during eviction ({}): {}", old_key, exc)
return session
async def close_all_sessions() -> None:

89
src/clients/http_client.py
View File

@@ -11,6 +11,7 @@
from __future__ import annotations
import asyncio
import os
import time
from contextlib import asynccontextmanager
from typing import Any
@@ -19,7 +20,6 @@ import httpx
from src.config import config
from src.core.logger import logger
from src.services.provider.fingerprint import KNOWN_IMPERSONATE_PROFILES
from src.services.proxy_node.resolver import (
build_proxy_url_async,
compute_proxy_cache_key,
@@ -34,6 +34,19 @@ _proxy_clients_lock = asyncio.Lock()
_default_client_lock = asyncio.Lock()
def _get_int_env(name: str, default: int, minimum: int) -> int:
"""Read positive integer env value with bounds and fallback."""
raw = os.getenv(name)
if raw is None:
return default
try:
value = int(raw)
except ValueError:
logger.warning("环境变量 {} 不是有效整数: {}, 使用默认值 {}", name, raw, default)
return default
return max(minimum, value)
class HTTPClientPool:
"""
全局HTTP客户端池单例
@@ -268,6 +281,8 @@ class HTTPClientPool:
# - direct chrome impersonate profile names (e.g. "chrome124")
use_curl_cffi_tls = False
if tls_profile_key:
from src.services.provider.fingerprint import KNOWN_IMPERSONATE_PROFILES
use_curl_cffi_tls = (
tls_profile_key == "claude_code_nodejs"
or tls_profile_key in KNOWN_IMPERSONATE_PROFILES
@@ -392,6 +407,78 @@ class HTTPClientPool:
logger.info("所有HTTP客户端已关闭")
@classmethod
async def cleanup_idle_clients(
cls,
max_idle_seconds: int | None = None,
) -> dict[str, int]:
"""清理空闲的代理/Tunnel 客户端并关闭连接池资源。"""
idle_seconds = max_idle_seconds
if idle_seconds is None:
idle_seconds = _get_int_env("HTTP_CLIENT_IDLE_CLEANUP_MAX_SECONDS", 600, minimum=60)
now = time.time()
stale_proxy_clients: list[tuple[str, httpx.AsyncClient]] = []
stale_tunnel_clients: list[tuple[str, httpx.AsyncClient]] = []
removed_closed_proxy = 0
removed_closed_tunnel = 0
lock = cls._get_proxy_clients_lock()
async with lock:
for cache_key, (client, last_used) in list(cls._proxy_clients.items()):
if client.is_closed:
cls._proxy_clients.pop(cache_key, None)
removed_closed_proxy += 1
continue
if now - last_used > idle_seconds:
entry = cls._proxy_clients.pop(cache_key, None)
if entry is not None:
stale_proxy_clients.append((cache_key, entry[0]))
for node_id, (client, last_used) in list(cls._tunnel_clients.items()):
if client.is_closed:
cls._tunnel_clients.pop(node_id, None)
removed_closed_tunnel += 1
continue
if now - last_used > idle_seconds:
entry = cls._tunnel_clients.pop(node_id, None)
if entry is not None:
stale_tunnel_clients.append((node_id, entry[0]))
proxy_closed = 0
tunnel_closed = 0
for cache_key, client in stale_proxy_clients:
try:
await client.aclose()
proxy_closed += 1
except Exception as e:
logger.warning("关闭空闲代理客户端失败(key={}): {}", cache_key, e)
for node_id, client in stale_tunnel_clients:
try:
await client.aclose()
tunnel_closed += 1
except Exception as e:
logger.warning("关闭空闲 Tunnel 客户端失败(node_id={}): {}", node_id, e)
if proxy_closed or tunnel_closed or removed_closed_proxy or removed_closed_tunnel:
logger.info(
"HTTP 客户端空闲清理完成: proxy_closed={}, tunnel_closed={}, "
"proxy_already_closed={}, tunnel_already_closed={}, idle_seconds={}",
proxy_closed,
tunnel_closed,
removed_closed_proxy,
removed_closed_tunnel,
idle_seconds,
)
return {
"proxy_closed": proxy_closed,
"tunnel_closed": tunnel_closed,
"proxy_already_closed": removed_closed_proxy,
"tunnel_already_closed": removed_closed_tunnel,
}
@classmethod
@asynccontextmanager
async def get_temp_client(cls, **kwargs: Any) -> Any:

137
src/clients/redis_client.py
View File

@@ -32,26 +32,22 @@ class RedisState(Enum):
class RedisClientManager:
"""
Redis客户端管理器（单例）
Redis 客户端管理器
提供 Redis 连接管理、熔断器保护和状态监控。
"""
_instance: RedisClientManager | None = None
_redis: aioredis.Redis | None = None
def __new__(cls) -> "RedisClientManager":
"""单例模式"""
if cls._instance is None:
cls._instance = super().__new__(cls)
return cls._instance
def __init__(self) -> None:
# 避免重复初始化
if getattr(self, "_initialized", False):
return
self._initialized = True
def __init__(
self,
*,
client_name: str,
encoding_errors: str = "strict",
degraded_warning: str | None = None,
) -> None:
self._redis: aioredis.Redis | None = None
self._client_name = client_name
self._encoding_errors = encoding_errors
self._degraded_warning = degraded_warning
self._circuit_open_until: float | None = None
self._consecutive_failures: int = 0
self._circuit_threshold = int(os.getenv("REDIS_CIRCUIT_BREAKER_THRESHOLD", "3"))
@@ -97,7 +93,7 @@ class RedisClientManager:
"""
手动重置熔断器（用于管理后台紧急恢复）
"""
logger.info("Redis 熔断器手动重置")
logger.info("{} 熔断器手动重置", self._client_name)
self._circuit_open_until = None
self._consecutive_failures = 0
self._last_error = None
@@ -122,7 +118,8 @@ class RedisClientManager:
if self._circuit_open_until and time.time() < self._circuit_open_until:
remaining = self._circuit_open_until - time.time()
logger.warning(
"Redis 客户端处于熔断状态，跳过初始化，剩余 {:.1f} 秒 (last_error: {})",
"{} 处于熔断状态，跳过初始化，剩余 {:.1f} 秒 (last_error: {})",
self._client_name,
remaining,
self._last_error,
)
@@ -173,6 +170,7 @@ class RedisClientManager:
service_name=sentinel_service,
max_connections=redis_max_conn,
decode_responses=True,
encoding_errors=self._encoding_errors,
socket_connect_timeout=5.0,
health_check_interval=30, # 每 30 秒检查连接健康状态
)
@@ -182,6 +180,7 @@ class RedisClientManager:
redis_url,
encoding="utf-8",
decode_responses=True,
encoding_errors=self._encoding_errors,
socket_timeout=5.0,
socket_connect_timeout=5.0,
max_connections=redis_max_conn,
@@ -191,22 +190,22 @@ class RedisClientManager:
# 测试连接
await self._redis.ping()
logger.info(f"[OK] 全局Redis客户端初始化成功: {safe_url}")
logger.info("[OK] {} 初始化成功: {}", self._client_name, safe_url)
self._consecutive_failures = 0
self._circuit_open_until = None
return self._redis
except Exception as e:
error_msg = str(e)
self._last_error = error_msg
logger.error(f"[ERROR] Redis连接失败: {error_msg}")
logger.error("[ERROR] {} 连接失败: {}", self._client_name, error_msg)
self._consecutive_failures += 1
if self._consecutive_failures >= self._circuit_threshold:
self._circuit_open_until = time.time() + self._circuit_reset_seconds
logger.warning(
"Redis 初始化连续失败 {} 次，开启熔断 {} 秒。"
"熔断期间以下功能将降级: 缓存亲和性、分布式并发控制、RPM限流。"
"{} 初始化连续失败 {} 次，开启熔断 {} 秒。"
"可通过管理 API /api/admin/system/redis/reset-circuit 手动重置。",
self._client_name,
self._consecutive_failures,
self._circuit_reset_seconds,
)
@@ -222,12 +221,8 @@ class RedisClientManager:
"3. Redis端口（默认6379）是否可访问"
) from e
logger.warning(
"[WARN] Redis 不可用，以下功能将降级运行（仅在单实例环境下安全）:\n"
" - 缓存亲和性: 禁用（每次请求随机选择 Endpoint）\n"
" - 分布式并发控制: 降级为本地计数\n"
" - RPM 限流: 降级为本地限流"
)
if self._degraded_warning:
logger.warning(self._degraded_warning)
self._redis = None
return None
@@ -236,7 +231,7 @@ class RedisClientManager:
if self._redis:
await self._redis.close()
self._redis = None
logger.info("全局Redis客户端已关闭")
logger.info("{} 已关闭", self._client_name)
def get_client(self) -> aioredis.Redis | None:
"""
@@ -250,8 +245,42 @@ class RedisClientManager:
return self._redis
# 全局单例
_GLOBAL_REDIS_DEGRADED_WARNING = (
"[WARN] Redis 不可用，以下功能将降级运行（仅在单实例环境下安全）:\n"
" - 缓存亲和性: 禁用（每次请求随机选择 Endpoint）\n"
" - 分布式并发控制: 降级为本地计数\n"
" - RPM 限流: 降级为本地限流"
)
_USAGE_QUEUE_REDIS_DEGRADED_WARNING = (
"[WARN] Usage Queue Redis 不可用，usage queue 写入与消费将暂时不可用"
)
_redis_manager: RedisClientManager | None = None
_usage_queue_redis_manager: RedisClientManager | None = None
def _get_global_redis_manager() -> RedisClientManager:
global _redis_manager
if _redis_manager is None:
_redis_manager = RedisClientManager(
client_name="全局Redis客户端",
encoding_errors="strict",
degraded_warning=_GLOBAL_REDIS_DEGRADED_WARNING,
)
return _redis_manager
def _get_usage_queue_redis_manager() -> RedisClientManager:
global _usage_queue_redis_manager
if _usage_queue_redis_manager is None:
_usage_queue_redis_manager = RedisClientManager(
client_name="Usage Queue Redis客户端",
encoding_errors="surrogateescape",
degraded_warning=_USAGE_QUEUE_REDIS_DEGRADED_WARNING,
)
return _usage_queue_redis_manager
async def get_redis_client(require_redis: bool = False) -> aioredis.Redis | None:
@@ -267,16 +296,26 @@ async def get_redis_client(require_redis: bool = False) -> aioredis.Redis | None
Raises:
RuntimeError: 当require_redis=True且连接失败时
"""
global _redis_manager
if _redis_manager is None:
_redis_manager = RedisClientManager()
manager = _get_global_redis_manager()
# 如果尚未连接（例如启动时降级、或 close() 后），尝试重新初始化。
# initialize() 内部包含熔断器逻辑，避免频繁重试导致抖动。
if _redis_manager.get_client() is None:
await _redis_manager.initialize(require_redis=require_redis)
if manager.get_client() is None:
await manager.initialize(require_redis=require_redis)
return _redis_manager.get_client()
return manager.get_client()
async def get_usage_queue_redis_client(require_redis: bool = False) -> aioredis.Redis | None:
"""
获取 Usage Queue 专用 Redis 客户端。
与全局 Redis 客户端隔离，专门用于 usage queue 的 msgpack/surrogateescape 编解码链路。
"""
manager = _get_usage_queue_redis_manager()
if manager.get_client() is None:
await manager.initialize(require_redis=require_redis)
return manager.get_client()
def get_redis_client_sync() -> aioredis.Redis | None:
@@ -295,11 +334,11 @@ def get_redis_client_sync() -> aioredis.Redis | None:
async def close_redis_client() -> None:
"""关闭全局Redis客户端"""
global _redis_manager
"""关闭 Redis 客户端（包含全局客户端和 Usage Queue 专用客户端）"""
if _redis_manager:
await _redis_manager.close()
if _usage_queue_redis_manager:
await _usage_queue_redis_manager.close()
def get_redis_state() -> RedisState:
@@ -341,13 +380,19 @@ def reset_redis_circuit_breaker() -> bool:
"""
手动重置 Redis 熔断器（同步方法）
同时重置全局 Redis 客户端与 Usage Queue 专用客户端，
避免其中一方仍停留在熔断状态导致功能未恢复。
Returns:
是否成功重置
是否至少重置了一个客户端
"""
global _redis_manager
reset_any = False
if _redis_manager is None:
return False
if _redis_manager is not None:
_redis_manager.reset_circuit_breaker()
reset_any = True
if _usage_queue_redis_manager is not None:
_usage_queue_redis_manager.reset_circuit_breaker()
reset_any = True
_redis_manager.reset_circuit_breaker()
return True
return reset_any