mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 01:10:23 +08:00
refactor: 移除 Python 后端源码，全面迁移至 Rust gateway 架构
- 删除全部 Python 源码 (src/) 及 Alembic 迁移脚本，归档至 _deprecated_py_src/ - 重构 Rust gateway ai_pipeline: 拆分 planner/finalize 模块，新增 contracts/adaptation 层 - 重组 handlers 模块为 admin/public/proxy/internal/shared 子模块结构 - 新增 executor 模块，引入 Rust 原生数据库迁移 (aether-data/migrations) - 简化 CI/Docker 构建流程，移除 base image 二级构建，统一为单一 app image - 移除 Python 相关基础设施文件 (entrypoint.sh, gunicorn_conf.py, Dockerfile.base)
This commit is contained in:
fawney19
5
_deprecated_py_src/plugins/monitor/__init__.py
Normal file
5
_deprecated_py_src/plugins/monitor/__init__.py
Normal file
@@ -0,0 +1,5 @@
|
||||
"""监控插件包"""
|
||||
|
||||
from .base import Metric, MetricType, MonitorPlugin
|
||||
|
||||
__all__ = ["MonitorPlugin", "Metric", "MetricType"]
|
||||
250
_deprecated_py_src/plugins/monitor/base.py
Normal file
250
_deprecated_py_src/plugins/monitor/base.py
Normal file
@@ -0,0 +1,250 @@
|
||||
"""
|
||||
监控插件基类
|
||||
定义监控和指标收集的接口
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import abstractmethod
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
from src.plugins.common import BasePlugin
|
||||
|
||||
|
||||
class MetricType(Enum):
|
||||
"""指标类型"""
|
||||
|
||||
COUNTER = "counter" # 计数器（只增不减）
|
||||
GAUGE = "gauge" # 仪表（可增可减）
|
||||
HISTOGRAM = "histogram" # 直方图（分布）
|
||||
SUMMARY = "summary" # 摘要（分位数）
|
||||
|
||||
|
||||
class Metric:
|
||||
"""指标数据"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
value: float,
|
||||
metric_type: MetricType,
|
||||
labels: dict[str, str] | None = None,
|
||||
timestamp: datetime | None = None,
|
||||
description: str | None = None,
|
||||
):
|
||||
self.name = name
|
||||
self.value = value
|
||||
self.metric_type = metric_type
|
||||
self.labels = labels or {}
|
||||
self.timestamp = timestamp or datetime.now(timezone.utc)
|
||||
self.description = description
|
||||
|
||||
|
||||
class MonitorPlugin(BasePlugin):
|
||||
"""
|
||||
监控插件基类
|
||||
所有监控插件必须继承此类并实现相关方法
|
||||
"""
|
||||
|
||||
def __init__(self, name: str, config: dict[str, Any] | None = None):
|
||||
"""
|
||||
初始化监控插件
|
||||
|
||||
Args:
|
||||
name: 插件名称
|
||||
config: 配置字典
|
||||
"""
|
||||
# 调用父类初始化，设置metadata
|
||||
super().__init__(name=name, config=config, description="Monitor Plugin", version="1.0.0")
|
||||
|
||||
self.flush_interval = self.config.get("flush_interval", 60)
|
||||
self.batch_size = self.config.get("batch_size", 100)
|
||||
|
||||
@abstractmethod
|
||||
async def record_metric(self, metric: Metric) -> None:
|
||||
"""
|
||||
记录单个指标
|
||||
|
||||
Args:
|
||||
metric: 指标数据
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def record_batch(self, metrics: list[Metric]) -> None:
|
||||
"""
|
||||
批量记录指标
|
||||
|
||||
Args:
|
||||
metrics: 指标列表
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def increment(
|
||||
self, name: str, value: float = 1, labels: dict[str, str] | None = None
|
||||
) -> Any:
|
||||
"""
|
||||
增加计数器
|
||||
|
||||
Args:
|
||||
name: 指标名称
|
||||
value: 增加的值
|
||||
labels: 标签字典
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def gauge(self, name: str, value: float, labels: dict[str, str] | None = None) -> Any:
|
||||
"""
|
||||
设置仪表值
|
||||
|
||||
Args:
|
||||
name: 指标名称
|
||||
value: 仪表值
|
||||
labels: 标签字典
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def histogram(
|
||||
self,
|
||||
name: str,
|
||||
value: float,
|
||||
labels: dict[str, str] | None = None,
|
||||
buckets: list[float] | None = None,
|
||||
) -> Any:
|
||||
"""
|
||||
记录直方图数据
|
||||
|
||||
Args:
|
||||
name: 指标名称
|
||||
value: 观测值
|
||||
labels: 标签字典
|
||||
buckets: 桶边界
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def timing(self, name: str, duration: float, labels: dict[str, str] | None = None) -> Any:
|
||||
"""
|
||||
记录时间指标
|
||||
|
||||
Args:
|
||||
name: 指标名称
|
||||
duration: 持续时间（秒）
|
||||
labels: 标签字典
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def flush(self) -> Any:
|
||||
"""
|
||||
刷新缓冲的指标到后端
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def get_stats(self) -> dict[str, Any]:
|
||||
"""
|
||||
获取插件统计信息
|
||||
|
||||
Returns:
|
||||
统计信息字典
|
||||
"""
|
||||
pass
|
||||
|
||||
def record_request(
|
||||
self,
|
||||
method: str,
|
||||
endpoint: str,
|
||||
status_code: int,
|
||||
duration: float,
|
||||
provider: str | None = None,
|
||||
model: str | None = None,
|
||||
) -> Any:
|
||||
"""
|
||||
记录API请求指标（便捷方法）
|
||||
|
||||
Args:
|
||||
method: HTTP方法
|
||||
endpoint: 端点路径
|
||||
status_code: 状态码
|
||||
duration: 请求时长
|
||||
provider: 提供商名称
|
||||
model: 模型名称
|
||||
"""
|
||||
labels = {
|
||||
"method": method,
|
||||
"endpoint": endpoint,
|
||||
"status": str(status_code),
|
||||
"status_class": f"{status_code // 100}xx",
|
||||
}
|
||||
|
||||
if provider:
|
||||
labels["provider"] = provider
|
||||
if model:
|
||||
labels["model"] = model
|
||||
|
||||
# 异步记录指标
|
||||
from src.utils.async_utils import safe_create_task
|
||||
|
||||
# 请求计数
|
||||
safe_create_task(self.increment("http_requests_total", labels=labels))
|
||||
|
||||
# 请求延迟
|
||||
safe_create_task(self.histogram("http_request_duration_seconds", duration, labels=labels))
|
||||
|
||||
# 错误计数
|
||||
if status_code >= 400:
|
||||
safe_create_task(self.increment("http_errors_total", labels=labels))
|
||||
|
||||
def record_token_usage(
|
||||
self,
|
||||
provider: str,
|
||||
model: str, # noqa: ARG002 - 保留签名兼容性，不再用于 Prometheus 标签
|
||||
input_tokens: int,
|
||||
output_tokens: int,
|
||||
cost: float | None = None,
|
||||
) -> Any:
|
||||
"""
|
||||
记录Token使用指标（便捷方法）
|
||||
|
||||
Args:
|
||||
provider: 提供商名称
|
||||
model: 模型名称（保留签名兼容性，不再作为 Prometheus 标签）
|
||||
input_tokens: 输入token数
|
||||
output_tokens: 输出token数
|
||||
cost: 费用
|
||||
"""
|
||||
labels = {"provider": provider}
|
||||
|
||||
from src.utils.async_utils import safe_create_task
|
||||
|
||||
# Token计数
|
||||
safe_create_task(self.increment("tokens_input_total", input_tokens, labels=labels))
|
||||
safe_create_task(self.increment("tokens_output_total", output_tokens, labels=labels))
|
||||
safe_create_task(
|
||||
self.increment("tokens_total", input_tokens + output_tokens, labels=labels)
|
||||
)
|
||||
|
||||
# 费用
|
||||
if cost is not None:
|
||||
safe_create_task(self.increment("usage_cost_total", cost, labels=labels))
|
||||
|
||||
def configure(self, config: dict[str, Any]) -> Any:
|
||||
"""
|
||||
配置插件
|
||||
|
||||
Args:
|
||||
config: 配置字典
|
||||
"""
|
||||
self.config.update(config)
|
||||
self.enabled = config.get("enabled", True)
|
||||
self.flush_interval = config.get("flush_interval", self.flush_interval)
|
||||
self.batch_size = config.get("batch_size", self.batch_size)
|
||||
|
||||
def __repr__(self) -> None:
|
||||
return f"<{self.__class__.__name__}(name={self.name}, enabled={self.enabled})>"
|
||||
334
_deprecated_py_src/plugins/monitor/prometheus.py
Normal file
334
_deprecated_py_src/plugins/monitor/prometheus.py
Normal file
@@ -0,0 +1,334 @@
|
||||
"""
|
||||
Prometheus监控插件
|
||||
支持将指标导出到Prometheus
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
from prometheus_client import REGISTRY, Counter, Gauge, Histogram, Summary, generate_latest
|
||||
|
||||
PROMETHEUS_AVAILABLE = True
|
||||
except ImportError:
|
||||
# Prometheus client not installed, plugin will be disabled
|
||||
PROMETHEUS_AVAILABLE = False
|
||||
Counter = Gauge = Histogram = Summary = REGISTRY = generate_latest = None
|
||||
|
||||
from src.core.logger import logger
|
||||
|
||||
from .base import Metric, MetricType, MonitorPlugin
|
||||
|
||||
|
||||
class PrometheusPlugin(MonitorPlugin):
|
||||
"""
|
||||
Prometheus监控插件
|
||||
使用prometheus_client库导出指标
|
||||
"""
|
||||
|
||||
def __init__(self, name: str = "prometheus", config: dict[str, Any] | None = None):
|
||||
super().__init__(name, config)
|
||||
|
||||
# Check if prometheus_client is available
|
||||
if not PROMETHEUS_AVAILABLE:
|
||||
self.enabled = False
|
||||
logger.warning("Prometheus client not installed, plugin disabled")
|
||||
return
|
||||
|
||||
# 指标注册表
|
||||
self._metrics: dict[str, Any] = {}
|
||||
self._max_dynamic_metrics: int = 100 # 动态创建的指标数上限
|
||||
self._buffer: list[Metric] = []
|
||||
self._lock = asyncio.Lock()
|
||||
self._flush_task: asyncio.Task | None = None # 跟踪后台任务
|
||||
|
||||
# 预定义常用指标
|
||||
self._init_default_metrics()
|
||||
|
||||
# 启动刷新任务
|
||||
self._start_flush_task()
|
||||
|
||||
def _init_default_metrics(self) -> None:
|
||||
"""初始化默认指标"""
|
||||
# HTTP请求指标
|
||||
http_label_names = ["method", "endpoint", "status", "status_class"]
|
||||
|
||||
self._metrics["http_requests_total"] = Counter(
|
||||
"http_requests_total",
|
||||
"Total HTTP requests",
|
||||
http_label_names,
|
||||
)
|
||||
|
||||
self._metrics["http_request_duration_seconds"] = Histogram(
|
||||
"http_request_duration_seconds",
|
||||
"HTTP request duration in seconds",
|
||||
http_label_names,
|
||||
buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10),
|
||||
)
|
||||
|
||||
self._metrics["http_errors_total"] = Counter(
|
||||
"http_errors_total",
|
||||
"Total HTTP errors",
|
||||
http_label_names,
|
||||
)
|
||||
|
||||
# Token使用指标（仅按 provider 维度聚合，避免 provider x model 笛卡尔积导致基数爆炸）
|
||||
self._metrics["tokens_input_total"] = Counter(
|
||||
"tokens_input_total", "Total input tokens", ["provider"]
|
||||
)
|
||||
|
||||
self._metrics["tokens_output_total"] = Counter(
|
||||
"tokens_output_total", "Total output tokens", ["provider"]
|
||||
)
|
||||
|
||||
self._metrics["tokens_total"] = Counter("tokens_total", "Total tokens", ["provider"])
|
||||
|
||||
self._metrics["usage_cost_total"] = Counter(
|
||||
"usage_cost_total", "Total usage cost in USD", ["provider"]
|
||||
)
|
||||
|
||||
# 系统指标
|
||||
self._metrics["active_connections"] = Gauge(
|
||||
"active_connections", "Number of active connections"
|
||||
)
|
||||
|
||||
self._metrics["cache_hits_total"] = Counter(
|
||||
"cache_hits_total", "Total cache hits", ["cache_type"]
|
||||
)
|
||||
|
||||
self._metrics["cache_misses_total"] = Counter(
|
||||
"cache_misses_total", "Total cache misses", ["cache_type"]
|
||||
)
|
||||
|
||||
# 提供商健康指标
|
||||
self._metrics["provider_health"] = Gauge(
|
||||
"provider_health", "Provider health status (1=healthy, 0=unhealthy)", ["provider"]
|
||||
)
|
||||
|
||||
self._metrics["provider_latency_seconds"] = Histogram(
|
||||
"provider_latency_seconds",
|
||||
"Provider response latency in seconds",
|
||||
["provider"],
|
||||
buckets=(0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30),
|
||||
)
|
||||
|
||||
def _start_flush_task(self) -> None:
|
||||
"""启动定期刷新任务"""
|
||||
|
||||
async def flush_loop() -> Any:
|
||||
try:
|
||||
while self.enabled:
|
||||
await asyncio.sleep(self.flush_interval)
|
||||
if self.enabled: # 再次检查，避免关闭时执行
|
||||
await self.flush()
|
||||
except asyncio.CancelledError:
|
||||
# 任务被取消，正常关闭
|
||||
logger.debug("Prometheus flush task cancelled")
|
||||
except Exception as e:
|
||||
logger.error(f"Prometheus flush loop error: {e}")
|
||||
|
||||
# 保存任务句柄以便后续取消
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
self._flush_task = loop.create_task(flush_loop())
|
||||
except RuntimeError:
|
||||
# 如果没有运行的事件循环，任务将在后续创建
|
||||
logger.warning("No event loop available for Prometheus flush task")
|
||||
|
||||
def _get_or_create_metric(
|
||||
self, name: str, metric_type: MetricType, labels: list[str] | None = None
|
||||
) -> Any | None:
|
||||
"""获取或创建指标，超过上限后拒绝创建新指标"""
|
||||
if name not in self._metrics:
|
||||
if len(self._metrics) >= self._max_dynamic_metrics:
|
||||
logger.warning("Prometheus dynamic metrics limit reached, dropping: {}", name)
|
||||
return None
|
||||
labels = labels or []
|
||||
if metric_type == MetricType.COUNTER:
|
||||
self._metrics[name] = Counter(name, f"Auto-created counter {name}", labels)
|
||||
elif metric_type == MetricType.GAUGE:
|
||||
self._metrics[name] = Gauge(name, f"Auto-created gauge {name}", labels)
|
||||
elif metric_type == MetricType.HISTOGRAM:
|
||||
self._metrics[name] = Histogram(name, f"Auto-created histogram {name}", labels)
|
||||
elif metric_type == MetricType.SUMMARY:
|
||||
self._metrics[name] = Summary(name, f"Auto-created summary {name}", labels)
|
||||
|
||||
return self._metrics.get(name)
|
||||
|
||||
async def record_metric(self, metric: Metric) -> None:
|
||||
"""记录单个指标"""
|
||||
async with self._lock:
|
||||
self._buffer.append(metric)
|
||||
|
||||
# 如果缓冲区满，自动刷新
|
||||
if len(self._buffer) >= self.batch_size:
|
||||
await self.flush()
|
||||
|
||||
async def record_batch(self, metrics: list[Metric]) -> None:
|
||||
"""批量记录指标"""
|
||||
async with self._lock:
|
||||
self._buffer.extend(metrics)
|
||||
|
||||
# 如果缓冲区满，自动刷新
|
||||
if len(self._buffer) >= self.batch_size:
|
||||
await self.flush()
|
||||
|
||||
async def increment(
|
||||
self, name: str, value: float = 1, labels: dict[str, str] | None = None
|
||||
) -> Any:
|
||||
"""增加计数器"""
|
||||
try:
|
||||
if name in self._metrics:
|
||||
metric = self._metrics[name]
|
||||
if labels:
|
||||
# 过滤掉不存在的标签
|
||||
filtered_labels = {k: v for k, v in labels.items() if k in metric._labelnames}
|
||||
metric.labels(**filtered_labels).inc(value)
|
||||
else:
|
||||
metric.inc(value)
|
||||
else:
|
||||
# 创建新的计数器
|
||||
label_names = list(labels.keys()) if labels else []
|
||||
metric = self._get_or_create_metric(name, MetricType.COUNTER, label_names)
|
||||
if metric is None:
|
||||
return
|
||||
if labels:
|
||||
metric.labels(**labels).inc(value)
|
||||
else:
|
||||
metric.inc(value)
|
||||
except Exception as e:
|
||||
# 记录错误但不中断
|
||||
logger.warning(f"Error recording metric {name}: {e}")
|
||||
|
||||
async def gauge(self, name: str, value: float, labels: dict[str, str] | None = None) -> Any:
|
||||
"""设置仪表值"""
|
||||
try:
|
||||
if name in self._metrics:
|
||||
metric = self._metrics[name]
|
||||
if labels:
|
||||
filtered_labels = {k: v for k, v in labels.items() if k in metric._labelnames}
|
||||
metric.labels(**filtered_labels).set(value)
|
||||
else:
|
||||
metric.set(value)
|
||||
else:
|
||||
# 创建新的仪表
|
||||
label_names = list(labels.keys()) if labels else []
|
||||
metric = self._get_or_create_metric(name, MetricType.GAUGE, label_names)
|
||||
if metric is None:
|
||||
return
|
||||
if labels:
|
||||
metric.labels(**labels).set(value)
|
||||
else:
|
||||
metric.set(value)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error recording gauge {name}: {e}")
|
||||
|
||||
async def histogram(
|
||||
self,
|
||||
name: str,
|
||||
value: float,
|
||||
labels: dict[str, str] | None = None,
|
||||
buckets: list[float] | None = None,
|
||||
) -> Any:
|
||||
"""记录直方图数据"""
|
||||
try:
|
||||
if name in self._metrics:
|
||||
metric = self._metrics[name]
|
||||
if labels:
|
||||
filtered_labels = {k: v for k, v in labels.items() if k in metric._labelnames}
|
||||
metric.labels(**filtered_labels).observe(value)
|
||||
else:
|
||||
metric.observe(value)
|
||||
else:
|
||||
# 创建新的直方图
|
||||
label_names = list(labels.keys()) if labels else []
|
||||
if buckets:
|
||||
if len(self._metrics) >= self._max_dynamic_metrics:
|
||||
return
|
||||
metric = Histogram(
|
||||
name, f"Auto-created histogram {name}", label_names, buckets=buckets
|
||||
)
|
||||
else:
|
||||
metric = self._get_or_create_metric(name, MetricType.HISTOGRAM, label_names)
|
||||
if metric is None:
|
||||
return
|
||||
self._metrics[name] = metric
|
||||
|
||||
if labels:
|
||||
metric.labels(**labels).observe(value)
|
||||
else:
|
||||
metric.observe(value)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error recording histogram {name}: {e}")
|
||||
|
||||
async def timing(self, name: str, duration: float, labels: dict[str, str] | None = None) -> Any:
|
||||
"""记录时间指标"""
|
||||
# 使用直方图记录时间
|
||||
await self.histogram(f"{name}_seconds", duration, labels)
|
||||
|
||||
async def flush(self) -> Any:
|
||||
"""刷新缓冲的指标到Prometheus"""
|
||||
async with self._lock:
|
||||
if not self._buffer:
|
||||
return
|
||||
|
||||
# 处理缓冲区中的指标
|
||||
for metric in self._buffer:
|
||||
if metric.metric_type == MetricType.COUNTER:
|
||||
await self.increment(metric.name, metric.value, metric.labels)
|
||||
elif metric.metric_type == MetricType.GAUGE:
|
||||
await self.gauge(metric.name, metric.value, metric.labels)
|
||||
elif metric.metric_type == MetricType.HISTOGRAM:
|
||||
await self.histogram(metric.name, metric.value, metric.labels)
|
||||
|
||||
# 清空缓冲区
|
||||
self._buffer.clear()
|
||||
|
||||
async def get_stats(self) -> dict[str, Any]:
|
||||
"""获取插件统计信息"""
|
||||
return {
|
||||
"type": "prometheus",
|
||||
"metrics_count": len(self._metrics),
|
||||
"buffer_size": len(self._buffer),
|
||||
"flush_interval": self.flush_interval,
|
||||
"batch_size": self.batch_size,
|
||||
}
|
||||
|
||||
def get_metrics(self) -> bytes:
|
||||
"""
|
||||
获取Prometheus格式的指标数据
|
||||
|
||||
Returns:
|
||||
Prometheus文本格式的指标
|
||||
"""
|
||||
return generate_latest(REGISTRY)
|
||||
|
||||
async def shutdown(self) -> Any:
|
||||
"""
|
||||
关闭插件，取消后台任务
|
||||
|
||||
这个方法应该在应用关闭时调用
|
||||
"""
|
||||
# 禁用插件
|
||||
self.enabled = False
|
||||
|
||||
# 取消并等待后台任务完成
|
||||
if self._flush_task and not self._flush_task.done():
|
||||
self._flush_task.cancel()
|
||||
try:
|
||||
await self._flush_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
# 最后一次刷新缓冲区
|
||||
await self.flush()
|
||||
|
||||
logger.info("Prometheus plugin shutdown complete")
|
||||
|
||||
async def cleanup(self) -> Any:
|
||||
"""
|
||||
清理资源（别名方法）
|
||||
"""
|
||||
await self.shutdown()
|
||||
Reference in New Issue
Block a user