mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 09:20:22 +08:00
fix: 治理 Prometheus 指标基数爆炸和内存缓存无界增长
- 删除高基数标签 key_id/model,移除未使用的指标 concurrency_slots_in_use/streaming_request_duration_seconds - HTTP 客户端池:命名客户端添加 LRU 淘汰上限,tunnel 客户端添加 LRU 淘汰和 last_used_time 追踪 - ResilienceManager:last_errors 改用 deque(maxlen),error_stats/circuit_breakers 添加上限淘汰 - HealthMonitor:_circuit_history 改用 deque(maxlen) - ProviderHealthTracker:清理已无记录的过期 key - PrometheusPlugin:动态指标数量添加上限,超限后拒绝创建 - 中间件使用路由模板替代实际路径,防止动态路径段导致标签爆炸
This commit is contained in:
@@ -39,6 +39,7 @@ class PrometheusPlugin(MonitorPlugin):
|
||||
|
||||
# 指标注册表
|
||||
self._metrics: dict[str, Any] = {}
|
||||
self._max_dynamic_metrics: int = 100 # 动态创建的指标数上限
|
||||
self._buffer: list[Metric] = []
|
||||
self._lock = asyncio.Lock()
|
||||
self._flush_task: asyncio.Task | None = None # 跟踪后台任务
|
||||
@@ -140,9 +141,12 @@ class PrometheusPlugin(MonitorPlugin):
|
||||
|
||||
def _get_or_create_metric(
|
||||
self, name: str, metric_type: MetricType, labels: list[str] | None = None
|
||||
) -> Any:
|
||||
"""获取或创建指标"""
|
||||
) -> Any | None:
|
||||
"""获取或创建指标,超过上限后拒绝创建新指标"""
|
||||
if name not in self._metrics:
|
||||
if len(self._metrics) >= self._max_dynamic_metrics:
|
||||
logger.warning("Prometheus dynamic metrics limit reached, dropping: {}", name)
|
||||
return None
|
||||
labels = labels or []
|
||||
if metric_type == MetricType.COUNTER:
|
||||
self._metrics[name] = Counter(name, f"Auto-created counter {name}", labels)
|
||||
@@ -153,7 +157,7 @@ class PrometheusPlugin(MonitorPlugin):
|
||||
elif metric_type == MetricType.SUMMARY:
|
||||
self._metrics[name] = Summary(name, f"Auto-created summary {name}", labels)
|
||||
|
||||
return self._metrics[name]
|
||||
return self._metrics.get(name)
|
||||
|
||||
async def record_metric(self, metric: Metric) -> None:
|
||||
"""记录单个指标"""
|
||||
@@ -190,6 +194,8 @@ class PrometheusPlugin(MonitorPlugin):
|
||||
# 创建新的计数器
|
||||
label_names = list(labels.keys()) if labels else []
|
||||
metric = self._get_or_create_metric(name, MetricType.COUNTER, label_names)
|
||||
if metric is None:
|
||||
return
|
||||
if labels:
|
||||
metric.labels(**labels).inc(value)
|
||||
else:
|
||||
@@ -212,6 +218,8 @@ class PrometheusPlugin(MonitorPlugin):
|
||||
# 创建新的仪表
|
||||
label_names = list(labels.keys()) if labels else []
|
||||
metric = self._get_or_create_metric(name, MetricType.GAUGE, label_names)
|
||||
if metric is None:
|
||||
return
|
||||
if labels:
|
||||
metric.labels(**labels).set(value)
|
||||
else:
|
||||
@@ -239,11 +247,15 @@ class PrometheusPlugin(MonitorPlugin):
|
||||
# 创建新的直方图
|
||||
label_names = list(labels.keys()) if labels else []
|
||||
if buckets:
|
||||
if len(self._metrics) >= self._max_dynamic_metrics:
|
||||
return
|
||||
metric = Histogram(
|
||||
name, f"Auto-created histogram {name}", label_names, buckets=buckets
|
||||
)
|
||||
else:
|
||||
metric = self._get_or_create_metric(name, MetricType.HISTOGRAM, label_names)
|
||||
if metric is None:
|
||||
return
|
||||
self._metrics[name] = metric
|
||||
|
||||
if labels:
|
||||
|
||||
Reference in New Issue
Block a user