mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 17:30:23 +08:00
1d9c77522a
- 删除全部 Python 源码 (src/) 及 Alembic 迁移脚本，归档至 _deprecated_py_src/ - 重构 Rust gateway ai_pipeline: 拆分 planner/finalize 模块，新增 contracts/adaptation 层 - 重组 handlers 模块为 admin/public/proxy/internal/shared 子模块结构 - 新增 executor 模块，引入 Rust 原生数据库迁移 (aether-data/migrations) - 简化 CI/Docker 构建流程，移除 base image 二级构建，统一为单一 app image - 移除 Python 相关基础设施文件 (entrypoint.sh, gunicorn_conf.py, Dockerfile.base)
335 lines
12 KiB
Python
335 lines
12 KiB
Python
"""
|
||
Prometheus监控插件
|
||
支持将指标导出到Prometheus
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
from typing import Any
|
||
|
||
try:
|
||
from prometheus_client import REGISTRY, Counter, Gauge, Histogram, Summary, generate_latest
|
||
|
||
PROMETHEUS_AVAILABLE = True
|
||
except ImportError:
|
||
# Prometheus client not installed, plugin will be disabled
|
||
PROMETHEUS_AVAILABLE = False
|
||
Counter = Gauge = Histogram = Summary = REGISTRY = generate_latest = None
|
||
|
||
from src.core.logger import logger
|
||
|
||
from .base import Metric, MetricType, MonitorPlugin
|
||
|
||
|
||
class PrometheusPlugin(MonitorPlugin):
|
||
"""
|
||
Prometheus监控插件
|
||
使用prometheus_client库导出指标
|
||
"""
|
||
|
||
def __init__(self, name: str = "prometheus", config: dict[str, Any] | None = None):
|
||
super().__init__(name, config)
|
||
|
||
# Check if prometheus_client is available
|
||
if not PROMETHEUS_AVAILABLE:
|
||
self.enabled = False
|
||
logger.warning("Prometheus client not installed, plugin disabled")
|
||
return
|
||
|
||
# 指标注册表
|
||
self._metrics: dict[str, Any] = {}
|
||
self._max_dynamic_metrics: int = 100 # 动态创建的指标数上限
|
||
self._buffer: list[Metric] = []
|
||
self._lock = asyncio.Lock()
|
||
self._flush_task: asyncio.Task | None = None # 跟踪后台任务
|
||
|
||
# 预定义常用指标
|
||
self._init_default_metrics()
|
||
|
||
# 启动刷新任务
|
||
self._start_flush_task()
|
||
|
||
def _init_default_metrics(self) -> None:
|
||
"""初始化默认指标"""
|
||
# HTTP请求指标
|
||
http_label_names = ["method", "endpoint", "status", "status_class"]
|
||
|
||
self._metrics["http_requests_total"] = Counter(
|
||
"http_requests_total",
|
||
"Total HTTP requests",
|
||
http_label_names,
|
||
)
|
||
|
||
self._metrics["http_request_duration_seconds"] = Histogram(
|
||
"http_request_duration_seconds",
|
||
"HTTP request duration in seconds",
|
||
http_label_names,
|
||
buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10),
|
||
)
|
||
|
||
self._metrics["http_errors_total"] = Counter(
|
||
"http_errors_total",
|
||
"Total HTTP errors",
|
||
http_label_names,
|
||
)
|
||
|
||
# Token使用指标（仅按 provider 维度聚合，避免 provider x model 笛卡尔积导致基数爆炸）
|
||
self._metrics["tokens_input_total"] = Counter(
|
||
"tokens_input_total", "Total input tokens", ["provider"]
|
||
)
|
||
|
||
self._metrics["tokens_output_total"] = Counter(
|
||
"tokens_output_total", "Total output tokens", ["provider"]
|
||
)
|
||
|
||
self._metrics["tokens_total"] = Counter("tokens_total", "Total tokens", ["provider"])
|
||
|
||
self._metrics["usage_cost_total"] = Counter(
|
||
"usage_cost_total", "Total usage cost in USD", ["provider"]
|
||
)
|
||
|
||
# 系统指标
|
||
self._metrics["active_connections"] = Gauge(
|
||
"active_connections", "Number of active connections"
|
||
)
|
||
|
||
self._metrics["cache_hits_total"] = Counter(
|
||
"cache_hits_total", "Total cache hits", ["cache_type"]
|
||
)
|
||
|
||
self._metrics["cache_misses_total"] = Counter(
|
||
"cache_misses_total", "Total cache misses", ["cache_type"]
|
||
)
|
||
|
||
# 提供商健康指标
|
||
self._metrics["provider_health"] = Gauge(
|
||
"provider_health", "Provider health status (1=healthy, 0=unhealthy)", ["provider"]
|
||
)
|
||
|
||
self._metrics["provider_latency_seconds"] = Histogram(
|
||
"provider_latency_seconds",
|
||
"Provider response latency in seconds",
|
||
["provider"],
|
||
buckets=(0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30),
|
||
)
|
||
|
||
def _start_flush_task(self) -> None:
|
||
"""启动定期刷新任务"""
|
||
|
||
async def flush_loop() -> Any:
|
||
try:
|
||
while self.enabled:
|
||
await asyncio.sleep(self.flush_interval)
|
||
if self.enabled: # 再次检查，避免关闭时执行
|
||
await self.flush()
|
||
except asyncio.CancelledError:
|
||
# 任务被取消，正常关闭
|
||
logger.debug("Prometheus flush task cancelled")
|
||
except Exception as e:
|
||
logger.error(f"Prometheus flush loop error: {e}")
|
||
|
||
# 保存任务句柄以便后续取消
|
||
try:
|
||
loop = asyncio.get_running_loop()
|
||
self._flush_task = loop.create_task(flush_loop())
|
||
except RuntimeError:
|
||
# 如果没有运行的事件循环，任务将在后续创建
|
||
logger.warning("No event loop available for Prometheus flush task")
|
||
|
||
def _get_or_create_metric(
|
||
self, name: str, metric_type: MetricType, labels: list[str] | None = None
|
||
) -> Any | None:
|
||
"""获取或创建指标，超过上限后拒绝创建新指标"""
|
||
if name not in self._metrics:
|
||
if len(self._metrics) >= self._max_dynamic_metrics:
|
||
logger.warning("Prometheus dynamic metrics limit reached, dropping: {}", name)
|
||
return None
|
||
labels = labels or []
|
||
if metric_type == MetricType.COUNTER:
|
||
self._metrics[name] = Counter(name, f"Auto-created counter {name}", labels)
|
||
elif metric_type == MetricType.GAUGE:
|
||
self._metrics[name] = Gauge(name, f"Auto-created gauge {name}", labels)
|
||
elif metric_type == MetricType.HISTOGRAM:
|
||
self._metrics[name] = Histogram(name, f"Auto-created histogram {name}", labels)
|
||
elif metric_type == MetricType.SUMMARY:
|
||
self._metrics[name] = Summary(name, f"Auto-created summary {name}", labels)
|
||
|
||
return self._metrics.get(name)
|
||
|
||
async def record_metric(self, metric: Metric) -> None:
|
||
"""记录单个指标"""
|
||
async with self._lock:
|
||
self._buffer.append(metric)
|
||
|
||
# 如果缓冲区满，自动刷新
|
||
if len(self._buffer) >= self.batch_size:
|
||
await self.flush()
|
||
|
||
async def record_batch(self, metrics: list[Metric]) -> None:
|
||
"""批量记录指标"""
|
||
async with self._lock:
|
||
self._buffer.extend(metrics)
|
||
|
||
# 如果缓冲区满，自动刷新
|
||
if len(self._buffer) >= self.batch_size:
|
||
await self.flush()
|
||
|
||
async def increment(
|
||
self, name: str, value: float = 1, labels: dict[str, str] | None = None
|
||
) -> Any:
|
||
"""增加计数器"""
|
||
try:
|
||
if name in self._metrics:
|
||
metric = self._metrics[name]
|
||
if labels:
|
||
# 过滤掉不存在的标签
|
||
filtered_labels = {k: v for k, v in labels.items() if k in metric._labelnames}
|
||
metric.labels(**filtered_labels).inc(value)
|
||
else:
|
||
metric.inc(value)
|
||
else:
|
||
# 创建新的计数器
|
||
label_names = list(labels.keys()) if labels else []
|
||
metric = self._get_or_create_metric(name, MetricType.COUNTER, label_names)
|
||
if metric is None:
|
||
return
|
||
if labels:
|
||
metric.labels(**labels).inc(value)
|
||
else:
|
||
metric.inc(value)
|
||
except Exception as e:
|
||
# 记录错误但不中断
|
||
logger.warning(f"Error recording metric {name}: {e}")
|
||
|
||
async def gauge(self, name: str, value: float, labels: dict[str, str] | None = None) -> Any:
|
||
"""设置仪表值"""
|
||
try:
|
||
if name in self._metrics:
|
||
metric = self._metrics[name]
|
||
if labels:
|
||
filtered_labels = {k: v for k, v in labels.items() if k in metric._labelnames}
|
||
metric.labels(**filtered_labels).set(value)
|
||
else:
|
||
metric.set(value)
|
||
else:
|
||
# 创建新的仪表
|
||
label_names = list(labels.keys()) if labels else []
|
||
metric = self._get_or_create_metric(name, MetricType.GAUGE, label_names)
|
||
if metric is None:
|
||
return
|
||
if labels:
|
||
metric.labels(**labels).set(value)
|
||
else:
|
||
metric.set(value)
|
||
except Exception as e:
|
||
logger.warning(f"Error recording gauge {name}: {e}")
|
||
|
||
async def histogram(
|
||
self,
|
||
name: str,
|
||
value: float,
|
||
labels: dict[str, str] | None = None,
|
||
buckets: list[float] | None = None,
|
||
) -> Any:
|
||
"""记录直方图数据"""
|
||
try:
|
||
if name in self._metrics:
|
||
metric = self._metrics[name]
|
||
if labels:
|
||
filtered_labels = {k: v for k, v in labels.items() if k in metric._labelnames}
|
||
metric.labels(**filtered_labels).observe(value)
|
||
else:
|
||
metric.observe(value)
|
||
else:
|
||
# 创建新的直方图
|
||
label_names = list(labels.keys()) if labels else []
|
||
if buckets:
|
||
if len(self._metrics) >= self._max_dynamic_metrics:
|
||
return
|
||
metric = Histogram(
|
||
name, f"Auto-created histogram {name}", label_names, buckets=buckets
|
||
)
|
||
else:
|
||
metric = self._get_or_create_metric(name, MetricType.HISTOGRAM, label_names)
|
||
if metric is None:
|
||
return
|
||
self._metrics[name] = metric
|
||
|
||
if labels:
|
||
metric.labels(**labels).observe(value)
|
||
else:
|
||
metric.observe(value)
|
||
except Exception as e:
|
||
logger.warning(f"Error recording histogram {name}: {e}")
|
||
|
||
async def timing(self, name: str, duration: float, labels: dict[str, str] | None = None) -> Any:
|
||
"""记录时间指标"""
|
||
# 使用直方图记录时间
|
||
await self.histogram(f"{name}_seconds", duration, labels)
|
||
|
||
async def flush(self) -> Any:
|
||
"""刷新缓冲的指标到Prometheus"""
|
||
async with self._lock:
|
||
if not self._buffer:
|
||
return
|
||
|
||
# 处理缓冲区中的指标
|
||
for metric in self._buffer:
|
||
if metric.metric_type == MetricType.COUNTER:
|
||
await self.increment(metric.name, metric.value, metric.labels)
|
||
elif metric.metric_type == MetricType.GAUGE:
|
||
await self.gauge(metric.name, metric.value, metric.labels)
|
||
elif metric.metric_type == MetricType.HISTOGRAM:
|
||
await self.histogram(metric.name, metric.value, metric.labels)
|
||
|
||
# 清空缓冲区
|
||
self._buffer.clear()
|
||
|
||
async def get_stats(self) -> dict[str, Any]:
|
||
"""获取插件统计信息"""
|
||
return {
|
||
"type": "prometheus",
|
||
"metrics_count": len(self._metrics),
|
||
"buffer_size": len(self._buffer),
|
||
"flush_interval": self.flush_interval,
|
||
"batch_size": self.batch_size,
|
||
}
|
||
|
||
def get_metrics(self) -> bytes:
|
||
"""
|
||
获取Prometheus格式的指标数据
|
||
|
||
Returns:
|
||
Prometheus文本格式的指标
|
||
"""
|
||
return generate_latest(REGISTRY)
|
||
|
||
async def shutdown(self) -> Any:
|
||
"""
|
||
关闭插件，取消后台任务
|
||
|
||
这个方法应该在应用关闭时调用
|
||
"""
|
||
# 禁用插件
|
||
self.enabled = False
|
||
|
||
# 取消并等待后台任务完成
|
||
if self._flush_task and not self._flush_task.done():
|
||
self._flush_task.cancel()
|
||
try:
|
||
await self._flush_task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
|
||
# 最后一次刷新缓冲区
|
||
await self.flush()
|
||
|
||
logger.info("Prometheus plugin shutdown complete")
|
||
|
||
async def cleanup(self) -> Any:
|
||
"""
|
||
清理资源（别名方法）
|
||
"""
|
||
await self.shutdown()
|