mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-03 01:40:21 +08:00
fix: 治理 Prometheus 指标基数爆炸和内存缓存无界增长
- 删除高基数标签 key_id/model,移除未使用的指标 concurrency_slots_in_use/streaming_request_duration_seconds - HTTP 客户端池:命名客户端添加 LRU 淘汰上限,tunnel 客户端添加 LRU 淘汰和 last_used_time 追踪 - ResilienceManager:last_errors 改用 deque(maxlen),error_stats/circuit_breakers 添加上限淘汰 - HealthMonitor:_circuit_history 改用 deque(maxlen) - ProviderHealthTracker:清理已无记录的过期 key - PrometheusPlugin:动态指标数量添加上限,超限后拒绝创建 - 中间件使用路由模板替代实际路径,防止动态路径段导致标签爆炸
This commit is contained in:
@@ -48,12 +48,14 @@ class HTTPClientPool:
|
||||
_instance: HTTPClientPool | None = None
|
||||
_default_client: httpx.AsyncClient | None = None
|
||||
_clients: dict[str, httpx.AsyncClient] = {}
|
||||
_max_named_clients: int = 20
|
||||
# 代理客户端缓存:{cache_key: (client, last_used_time)}
|
||||
_proxy_clients: dict[str, tuple[httpx.AsyncClient, float]] = {}
|
||||
# 代理客户端缓存上限(避免内存泄漏)
|
||||
_max_proxy_clients: int = 50
|
||||
# Tunnel 客户端缓存:{node_id: client}
|
||||
_tunnel_clients: dict[str, httpx.AsyncClient] = {}
|
||||
# Tunnel 客户端缓存:{node_id: (client, last_used_time)}
|
||||
_tunnel_clients: dict[str, tuple[httpx.AsyncClient, float]] = {}
|
||||
_max_tunnel_clients: int = 30
|
||||
# 后台清理任务引用集合(防止被 GC 回收)
|
||||
_background_tasks: set[asyncio.Task[None]] = set()
|
||||
|
||||
@@ -143,23 +145,37 @@ class HTTPClientPool:
|
||||
name: 客户端标识符
|
||||
**kwargs: httpx.AsyncClient的配置参数
|
||||
"""
|
||||
if name not in cls._clients:
|
||||
# 合并默认配置和自定义配置
|
||||
default_config = {
|
||||
"http2": config.enable_http2,
|
||||
"verify": get_ssl_context(),
|
||||
"timeout": httpx.Timeout(
|
||||
connect=config.http_connect_timeout,
|
||||
read=config.http_read_timeout,
|
||||
write=config.http_write_timeout,
|
||||
pool=config.http_pool_timeout,
|
||||
),
|
||||
"follow_redirects": True,
|
||||
}
|
||||
default_config.update(kwargs)
|
||||
if name in cls._clients:
|
||||
# 命中缓存:移到末尾以维护 LRU 顺序
|
||||
cls._clients[name] = cls._clients.pop(name)
|
||||
return cls._clients[name]
|
||||
|
||||
cls._clients[name] = httpx.AsyncClient(**default_config) # type: ignore[arg-type]
|
||||
logger.debug("创建命名HTTP客户端: {}", name)
|
||||
# 淘汰最久未使用的客户端(dict 头部即 LRU)
|
||||
if len(cls._clients) >= cls._max_named_clients:
|
||||
oldest_name = next(iter(cls._clients))
|
||||
old_client = cls._clients.pop(oldest_name)
|
||||
try:
|
||||
asyncio.get_running_loop().create_task(old_client.aclose())
|
||||
except RuntimeError:
|
||||
pass
|
||||
logger.debug("淘汰命名HTTP客户端: {}", oldest_name)
|
||||
|
||||
# 合并默认配置和自定义配置
|
||||
default_config = {
|
||||
"http2": config.enable_http2,
|
||||
"verify": get_ssl_context(),
|
||||
"timeout": httpx.Timeout(
|
||||
connect=config.http_connect_timeout,
|
||||
read=config.http_read_timeout,
|
||||
write=config.http_write_timeout,
|
||||
pool=config.http_pool_timeout,
|
||||
),
|
||||
"follow_redirects": True,
|
||||
}
|
||||
default_config.update(kwargs)
|
||||
|
||||
cls._clients[name] = httpx.AsyncClient(**default_config) # type: ignore[arg-type]
|
||||
logger.debug("创建命名HTTP客户端: {}", name)
|
||||
|
||||
return cls._clients[name]
|
||||
|
||||
@@ -357,7 +373,7 @@ class HTTPClientPool:
|
||||
cls._proxy_clients.clear()
|
||||
|
||||
# 关闭 tunnel 客户端缓存
|
||||
for nid, client in cls._tunnel_clients.items():
|
||||
for nid, (client, _) in cls._tunnel_clients.items():
|
||||
try:
|
||||
await client.aclose()
|
||||
logger.debug("tunnel 客户端已关闭: {}", nid)
|
||||
@@ -524,9 +540,10 @@ class HTTPClientPool:
|
||||
return False
|
||||
lock = cls._get_proxy_clients_lock()
|
||||
async with lock:
|
||||
client = cls._tunnel_clients.pop(node_id, None)
|
||||
if client is None:
|
||||
entry = cls._tunnel_clients.pop(node_id, None)
|
||||
if entry is None:
|
||||
return False
|
||||
client, _ = entry
|
||||
try:
|
||||
await client.aclose()
|
||||
except Exception as exc:
|
||||
@@ -635,13 +652,27 @@ class HTTPClientPool:
|
||||
# 非流式请求:复用缓存的 client(加锁与 proxy_clients 保持一致)
|
||||
lock = cls._get_proxy_clients_lock()
|
||||
async with lock:
|
||||
existing = cls._tunnel_clients.get(node_id)
|
||||
if existing and not existing.is_closed:
|
||||
return existing
|
||||
entry = cls._tunnel_clients.get(node_id)
|
||||
if entry is not None:
|
||||
existing, _ = entry
|
||||
if not existing.is_closed:
|
||||
cls._tunnel_clients[node_id] = (existing, time.time())
|
||||
return existing
|
||||
del cls._tunnel_clients[node_id]
|
||||
|
||||
# 淘汰最久未使用的 tunnel 客户端
|
||||
if len(cls._tunnel_clients) >= cls._max_tunnel_clients:
|
||||
oldest_nid = min(cls._tunnel_clients, key=lambda k: cls._tunnel_clients[k][1])
|
||||
old_client, _ = cls._tunnel_clients.pop(oldest_nid)
|
||||
try:
|
||||
await old_client.aclose()
|
||||
except Exception:
|
||||
pass
|
||||
logger.debug("淘汰 tunnel 客户端: {}", oldest_nid)
|
||||
|
||||
transport = create_tunnel_transport(node_id, timeout=timeout_secs or 60.0)
|
||||
client = httpx.AsyncClient(transport=transport, timeout=t)
|
||||
cls._tunnel_clients[node_id] = client
|
||||
cls._tunnel_clients[node_id] = (client, time.time())
|
||||
return client
|
||||
|
||||
@classmethod
|
||||
|
||||
Reference in New Issue
Block a user