fix: 治理 Prometheus 指标基数爆炸和内存缓存无界增长

- 删除高基数标签 key_id/model,移除未使用的指标 concurrency_slots_in_use/streaming_request_duration_seconds
- HTTP 客户端池:命名客户端添加 LRU 淘汰上限,tunnel 客户端添加 LRU 淘汰和 last_used_time 追踪
- ResilienceManager:last_errors 改用 deque(maxlen),error_stats/circuit_breakers 添加上限淘汰
- HealthMonitor:_circuit_history 改用 deque(maxlen)
- ProviderHealthTracker:清理已无记录的过期 key
- PrometheusPlugin:动态指标数量添加上限,超限后拒绝创建
- 中间件使用路由模板替代实际路径,防止动态路径段导致标签爆炸
This commit is contained in:
fawney19
2026-03-10 09:52:21 +08:00
parent afd0dcf2ff
commit c9f0685b40
9 changed files with 116 additions and 62 deletions

View File

@@ -4,11 +4,11 @@ Prometheus metrics for monitoring
from prometheus_client import Counter, Gauge, Histogram
# 并发槽位占用时长分布
# 并发槽位占用时长分布(按异常类型聚合,不按 key_id 拆分)
concurrency_slot_duration_seconds = Histogram(
"concurrency_slot_duration_seconds",
"Duration of concurrency slot occupation in seconds",
["key_id", "exception"],
["exception"],
buckets=[0.1, 0.5, 1, 5, 10, 30, 60, 120, 300, 600], # 0.1s 到 10 分钟
)
@@ -16,20 +16,7 @@ concurrency_slot_duration_seconds = Histogram(
concurrency_slot_release_total = Counter(
"concurrency_slot_release_total",
"Total number of concurrency slot releases",
["key_id", "exception"],
)
# 当前并发槽位使用数
concurrency_slots_in_use = Gauge(
"concurrency_slots_in_use", "Current number of concurrency slots in use", ["key_id"]
)
# 流式请求时长分布
streaming_request_duration_seconds = Histogram(
"streaming_request_duration_seconds",
"Duration of streaming requests in seconds",
["key_id", "status"],
buckets=[1, 5, 10, 30, 60, 120, 300, 600, 1800], # 1s 到 30 分钟
["exception"],
)
# 请求总数(按类型)
@@ -118,5 +105,5 @@ billing_invariant_violation_total = Counter(
antigravity_degradation_total = Counter(
"aether_antigravity_degradation_total",
"Count of Antigravity signature degradation (rectification) events",
["stage", "model"],
["stage"],
)

View File

@@ -110,6 +110,16 @@ class ProviderHealthTracker:
if all(current_time - t > self.recovery_time for t in self.successes[provider_name]):
self.priority_adjustments[provider_name] = 0
# 清理已无记录且无优先级调整的 key防止 dict 无限增长
if (
not self.failures[provider_name]
and not self.successes[provider_name]
and self.priority_adjustments.get(provider_name, 0) == 0
):
self.failures.pop(provider_name, None)
self.successes.pop(provider_name, None)
self.priority_adjustments.pop(provider_name, None)
def _get_status_label(self, failure_rate: float, recent_failures: int) -> str:
"""根据失败率返回状态标签"""
if recent_failures >= self.failure_threshold:

View File

@@ -11,6 +11,7 @@ import threading
import time
import traceback
import uuid
from collections import deque
from collections.abc import Callable
from contextlib import asynccontextmanager
from datetime import datetime, timezone
@@ -115,11 +116,15 @@ class CircuitBreaker:
class ResilienceManager:
"""系统韧性管理器"""
_MAX_ERROR_STATS = 500
_MAX_CIRCUIT_BREAKERS = 200
_MAX_LAST_ERRORS = 100
def __init__(self) -> None:
self.error_patterns: list[ErrorPattern] = []
self.circuit_breakers: dict[str, CircuitBreaker] = {}
self.error_stats: dict[str, int] = {}
self.last_errors: list[dict[str, Any]] = []
self.last_errors: deque[dict[str, Any]] = deque(maxlen=self._MAX_LAST_ERRORS)
self._setup_default_patterns()
def _setup_default_patterns(self) -> None:
@@ -203,6 +208,11 @@ class ResilienceManager:
def get_circuit_breaker(self, key: str) -> CircuitBreaker:
"""获取或创建熔断器"""
if key not in self.circuit_breakers:
# 淘汰已恢复的旧熔断器,防止无界增长
if len(self.circuit_breakers) >= self._MAX_CIRCUIT_BREAKERS:
closed_keys = [k for k, cb in self.circuit_breakers.items() if cb.state == "closed"]
for k in closed_keys:
del self.circuit_breakers[k]
self.circuit_breakers[key] = CircuitBreaker()
return self.circuit_breakers[key]
@@ -226,13 +236,18 @@ class ResilienceManager:
}
self.last_errors.append(error_info)
# 只保留最近100个错误
if len(self.last_errors) > 100:
self.last_errors.pop(0)
# 更新错误统计
# 更新错误统计(超上限时淘汰计数最低的条目)
error_key = f"{type(error).__name__}:{operation}"
self.error_stats[error_key] = self.error_stats.get(error_key, 0) + 1
if len(self.error_stats) > self._MAX_ERROR_STATS:
min_key = min(
(k for k in self.error_stats if k != error_key),
key=lambda k: self.error_stats[k],
default=None,
)
if min_key is not None:
del self.error_stats[min_key]
# 查找匹配的错误处理模式
pattern = self._find_matching_pattern(error)