fix: 治理 Prometheus 指标基数爆炸和内存缓存无界增长

- 删除高基数标签 key_id/model,移除未使用的指标 concurrency_slots_in_use/streaming_request_duration_seconds
- HTTP 客户端池:命名客户端添加 LRU 淘汰上限,tunnel 客户端添加 LRU 淘汰和 last_used_time 追踪
- ResilienceManager:last_errors 改用 deque(maxlen),error_stats/circuit_breakers 添加上限淘汰
- HealthMonitor:_circuit_history 改用 deque(maxlen)
- ProviderHealthTracker:清理已无记录的过期 key
- PrometheusPlugin:动态指标数量添加上限,超限后拒绝创建
- 中间件使用路由模板替代实际路径,防止动态路径段导致标签爆炸
This commit is contained in:
fawney19
2026-03-10 09:52:21 +08:00
parent afd0dcf2ff
commit c9f0685b40
9 changed files with 116 additions and 62 deletions

View File

@@ -48,12 +48,14 @@ class HTTPClientPool:
_instance: HTTPClientPool | None = None
_default_client: httpx.AsyncClient | None = None
_clients: dict[str, httpx.AsyncClient] = {}
_max_named_clients: int = 20
# 代理客户端缓存:{cache_key: (client, last_used_time)}
_proxy_clients: dict[str, tuple[httpx.AsyncClient, float]] = {}
# 代理客户端缓存上限(避免内存泄漏)
_max_proxy_clients: int = 50
# Tunnel 客户端缓存:{node_id: client}
_tunnel_clients: dict[str, httpx.AsyncClient] = {}
# Tunnel 客户端缓存:{node_id: (client, last_used_time)}
_tunnel_clients: dict[str, tuple[httpx.AsyncClient, float]] = {}
_max_tunnel_clients: int = 30
# 后台清理任务引用集合(防止被 GC 回收)
_background_tasks: set[asyncio.Task[None]] = set()
@@ -143,23 +145,37 @@ class HTTPClientPool:
name: 客户端标识符
**kwargs: httpx.AsyncClient的配置参数
"""
if name not in cls._clients:
# 合并默认配置和自定义配置
default_config = {
"http2": config.enable_http2,
"verify": get_ssl_context(),
"timeout": httpx.Timeout(
connect=config.http_connect_timeout,
read=config.http_read_timeout,
write=config.http_write_timeout,
pool=config.http_pool_timeout,
),
"follow_redirects": True,
}
default_config.update(kwargs)
if name in cls._clients:
# 命中缓存:移到末尾以维护 LRU 顺序
cls._clients[name] = cls._clients.pop(name)
return cls._clients[name]
cls._clients[name] = httpx.AsyncClient(**default_config) # type: ignore[arg-type]
logger.debug("创建命名HTTP客户端: {}", name)
# 淘汰最久未使用的客户端dict 头部即 LRU
if len(cls._clients) >= cls._max_named_clients:
oldest_name = next(iter(cls._clients))
old_client = cls._clients.pop(oldest_name)
try:
asyncio.get_running_loop().create_task(old_client.aclose())
except RuntimeError:
pass
logger.debug("淘汰命名HTTP客户端: {}", oldest_name)
# 合并默认配置和自定义配置
default_config = {
"http2": config.enable_http2,
"verify": get_ssl_context(),
"timeout": httpx.Timeout(
connect=config.http_connect_timeout,
read=config.http_read_timeout,
write=config.http_write_timeout,
pool=config.http_pool_timeout,
),
"follow_redirects": True,
}
default_config.update(kwargs)
cls._clients[name] = httpx.AsyncClient(**default_config) # type: ignore[arg-type]
logger.debug("创建命名HTTP客户端: {}", name)
return cls._clients[name]
@@ -357,7 +373,7 @@ class HTTPClientPool:
cls._proxy_clients.clear()
# 关闭 tunnel 客户端缓存
for nid, client in cls._tunnel_clients.items():
for nid, (client, _) in cls._tunnel_clients.items():
try:
await client.aclose()
logger.debug("tunnel 客户端已关闭: {}", nid)
@@ -524,9 +540,10 @@ class HTTPClientPool:
return False
lock = cls._get_proxy_clients_lock()
async with lock:
client = cls._tunnel_clients.pop(node_id, None)
if client is None:
entry = cls._tunnel_clients.pop(node_id, None)
if entry is None:
return False
client, _ = entry
try:
await client.aclose()
except Exception as exc:
@@ -635,13 +652,27 @@ class HTTPClientPool:
# 非流式请求:复用缓存的 client加锁与 proxy_clients 保持一致)
lock = cls._get_proxy_clients_lock()
async with lock:
existing = cls._tunnel_clients.get(node_id)
if existing and not existing.is_closed:
return existing
entry = cls._tunnel_clients.get(node_id)
if entry is not None:
existing, _ = entry
if not existing.is_closed:
cls._tunnel_clients[node_id] = (existing, time.time())
return existing
del cls._tunnel_clients[node_id]
# 淘汰最久未使用的 tunnel 客户端
if len(cls._tunnel_clients) >= cls._max_tunnel_clients:
oldest_nid = min(cls._tunnel_clients, key=lambda k: cls._tunnel_clients[k][1])
old_client, _ = cls._tunnel_clients.pop(oldest_nid)
try:
await old_client.aclose()
except Exception:
pass
logger.debug("淘汰 tunnel 客户端: {}", oldest_nid)
transport = create_tunnel_transport(node_id, timeout=timeout_secs or 60.0)
client = httpx.AsyncClient(transport=transport, timeout=t)
cls._tunnel_clients[node_id] = client
cls._tunnel_clients[node_id] = (client, time.time())
return client
@classmethod