fix: 治理 Prometheus 指标基数爆炸和内存缓存无界增长

- 删除高基数标签 key_id/model,移除未使用的指标 concurrency_slots_in_use/streaming_request_duration_seconds
- HTTP 客户端池:命名客户端添加 LRU 淘汰上限,tunnel 客户端添加 LRU 淘汰和 last_used_time 追踪
- ResilienceManager:last_errors 改用 deque(maxlen),error_stats/circuit_breakers 添加上限淘汰
- HealthMonitor:_circuit_history 改用 deque(maxlen)
- ProviderHealthTracker:清理已无记录的过期 key
- PrometheusPlugin:动态指标数量添加上限,超限后拒绝创建
- 中间件使用路由模板替代实际路径,防止动态路径段导致标签爆炸
This commit is contained in:
fawney19
2026-03-10 09:52:21 +08:00
parent afd0dcf2ff
commit c9f0685b40
9 changed files with 116 additions and 62 deletions

View File

@@ -39,6 +39,7 @@ class PrometheusPlugin(MonitorPlugin):
# 指标注册表
self._metrics: dict[str, Any] = {}
self._max_dynamic_metrics: int = 100 # 动态创建的指标数上限
self._buffer: list[Metric] = []
self._lock = asyncio.Lock()
self._flush_task: asyncio.Task | None = None # 跟踪后台任务
@@ -140,9 +141,12 @@ class PrometheusPlugin(MonitorPlugin):
def _get_or_create_metric(
self, name: str, metric_type: MetricType, labels: list[str] | None = None
) -> Any:
"""获取或创建指标"""
) -> Any | None:
"""获取或创建指标,超过上限后拒绝创建新指标"""
if name not in self._metrics:
if len(self._metrics) >= self._max_dynamic_metrics:
logger.warning("Prometheus dynamic metrics limit reached, dropping: {}", name)
return None
labels = labels or []
if metric_type == MetricType.COUNTER:
self._metrics[name] = Counter(name, f"Auto-created counter {name}", labels)
@@ -153,7 +157,7 @@ class PrometheusPlugin(MonitorPlugin):
elif metric_type == MetricType.SUMMARY:
self._metrics[name] = Summary(name, f"Auto-created summary {name}", labels)
return self._metrics[name]
return self._metrics.get(name)
async def record_metric(self, metric: Metric) -> None:
"""记录单个指标"""
@@ -190,6 +194,8 @@ class PrometheusPlugin(MonitorPlugin):
# 创建新的计数器
label_names = list(labels.keys()) if labels else []
metric = self._get_or_create_metric(name, MetricType.COUNTER, label_names)
if metric is None:
return
if labels:
metric.labels(**labels).inc(value)
else:
@@ -212,6 +218,8 @@ class PrometheusPlugin(MonitorPlugin):
# 创建新的仪表
label_names = list(labels.keys()) if labels else []
metric = self._get_or_create_metric(name, MetricType.GAUGE, label_names)
if metric is None:
return
if labels:
metric.labels(**labels).set(value)
else:
@@ -239,11 +247,15 @@ class PrometheusPlugin(MonitorPlugin):
# 创建新的直方图
label_names = list(labels.keys()) if labels else []
if buckets:
if len(self._metrics) >= self._max_dynamic_metrics:
return
metric = Histogram(
name, f"Auto-created histogram {name}", label_names, buckets=buckets
)
else:
metric = self._get_or_create_metric(name, MetricType.HISTOGRAM, label_names)
if metric is None:
return
self._metrics[name] = metric
if labels: