mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 01:10:23 +08:00
1d9c77522a
- 删除全部 Python 源码 (src/) 及 Alembic 迁移脚本，归档至 _deprecated_py_src/ - 重构 Rust gateway ai_pipeline: 拆分 planner/finalize 模块，新增 contracts/adaptation 层 - 重组 handlers 模块为 admin/public/proxy/internal/shared 子模块结构 - 新增 executor 模块，引入 Rust 原生数据库迁移 (aether-data/migrations) - 简化 CI/Docker 构建流程，移除 base image 二级构建，统一为单一 app image - 移除 Python 相关基础设施文件 (entrypoint.sh, gunicorn_conf.py, Dockerfile.base)
493 lines
18 KiB
Python
493 lines
18 KiB
Python
"""
|
||
端点健康状态服务
|
||
|
||
提供统一的端点健康监控功能，支持：
|
||
1. 按 API 格式聚合的健康状态
|
||
2. 基于时间窗口的状态追踪
|
||
3. 管理员和普通用户的差异化视图
|
||
4. Redis 缓存优化
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
from collections import defaultdict
|
||
from datetime import datetime, timedelta, timezone
|
||
from typing import Any
|
||
|
||
from sqlalchemy import case, func
|
||
from sqlalchemy.orm import Session, load_only
|
||
|
||
from src.core.logger import logger
|
||
from src.models.database import Provider, ProviderAPIKey, ProviderEndpoint, RequestCandidate
|
||
|
||
# 缓存配置
|
||
CACHE_TTL_SECONDS = 30 # 缓存 30 秒
|
||
CACHE_KEY_PREFIX = "health:endpoint:"
|
||
|
||
|
||
def _get_redis_client() -> Any:
|
||
"""获取 Redis 客户端，失败返回 None"""
|
||
try:
|
||
from src.clients.redis_client import redis_client
|
||
|
||
return redis_client
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
class EndpointHealthService:
|
||
"""端点健康状态服务"""
|
||
|
||
@staticmethod
|
||
def get_endpoint_health_by_format(
|
||
db: Session,
|
||
lookback_hours: int = 6,
|
||
include_admin_fields: bool = False,
|
||
use_cache: bool = True,
|
||
) -> list[dict[str, Any]]:
|
||
"""
|
||
获取按 API 格式聚合的端点健康状态
|
||
|
||
Args:
|
||
db: 数据库会话
|
||
lookback_hours: 回溯小时数
|
||
include_admin_fields: 是否包含管理员字段（provider_count, key_count等）
|
||
use_cache: 是否使用缓存（仅对普通用户视图有效）
|
||
|
||
Returns:
|
||
按 API 格式聚合的健康状态列表
|
||
"""
|
||
# 尝试从缓存获取
|
||
cache_key = f"{CACHE_KEY_PREFIX}format:{lookback_hours}:{include_admin_fields}"
|
||
if use_cache:
|
||
cached = EndpointHealthService._get_from_cache(cache_key)
|
||
if cached is not None:
|
||
return cached
|
||
|
||
now = datetime.now(timezone.utc)
|
||
|
||
# 查询所有活跃的端点（一次性获取所有需要的数据）
|
||
endpoints = (
|
||
db.query(ProviderEndpoint)
|
||
.join(Provider)
|
||
.filter(
|
||
ProviderEndpoint.is_active.is_(True),
|
||
Provider.is_active.is_(True),
|
||
)
|
||
.all()
|
||
)
|
||
|
||
# 收集所有 provider_ids
|
||
all_provider_ids = list({ep.provider_id for ep in endpoints})
|
||
|
||
# 批量查询所有密钥（通过 provider_id 关联，只加载分组和统计所需列）
|
||
all_keys = (
|
||
(
|
||
db.query(ProviderAPIKey)
|
||
.filter(ProviderAPIKey.provider_id.in_(all_provider_ids))
|
||
.options(
|
||
load_only(
|
||
ProviderAPIKey.id,
|
||
ProviderAPIKey.provider_id,
|
||
ProviderAPIKey.is_active,
|
||
ProviderAPIKey.api_formats,
|
||
ProviderAPIKey.health_by_format,
|
||
ProviderAPIKey.circuit_breaker_by_format,
|
||
)
|
||
)
|
||
.all()
|
||
)
|
||
if all_provider_ids
|
||
else []
|
||
)
|
||
|
||
# 按 api_format 分组密钥（通过 api_formats 字段）
|
||
keys_by_format: dict[str, list[ProviderAPIKey]] = defaultdict(list)
|
||
for key in all_keys:
|
||
for fmt in key.api_formats or []:
|
||
keys_by_format[fmt].append(key)
|
||
|
||
# 按 API 格式聚合
|
||
format_stats = defaultdict(
|
||
lambda: {
|
||
"total_endpoints": 0,
|
||
"total_keys": 0,
|
||
"active_keys": 0,
|
||
"health_scores": [],
|
||
"endpoint_ids": [],
|
||
"provider_ids": set(),
|
||
"key_ids": [],
|
||
}
|
||
)
|
||
|
||
for ep in endpoints:
|
||
api_format = ep.api_format if ep.api_format else "UNKNOWN"
|
||
|
||
# 统计端点数
|
||
format_stats[api_format]["total_endpoints"] += 1
|
||
format_stats[api_format]["endpoint_ids"].append(ep.id)
|
||
format_stats[api_format]["provider_ids"].add(ep.provider_id)
|
||
|
||
# 统计每个格式的密钥（直接从 keys_by_format 获取）
|
||
for api_format, keys in keys_by_format.items():
|
||
if api_format not in format_stats:
|
||
# 如果有 Key 但没有对应的 Endpoint，跳过
|
||
continue
|
||
|
||
# 去重（同一个 Key 可能支持多个格式）
|
||
seen_key_ids = set()
|
||
unique_keys = []
|
||
for key in keys:
|
||
if key.id not in seen_key_ids:
|
||
seen_key_ids.add(key.id)
|
||
unique_keys.append(key)
|
||
|
||
format_stats[api_format]["total_keys"] = len(unique_keys)
|
||
|
||
for key in unique_keys:
|
||
format_stats[api_format]["key_ids"].append(key.id)
|
||
# 检查该格式的熔断器状态
|
||
circuit_by_format = key.circuit_breaker_by_format or {}
|
||
format_circuit = circuit_by_format.get(api_format, {})
|
||
is_circuit_open = format_circuit.get("open", False)
|
||
|
||
if key.is_active and not is_circuit_open:
|
||
format_stats[api_format]["active_keys"] += 1
|
||
# 获取该格式的健康度
|
||
health_by_format = key.health_by_format or {}
|
||
format_health = health_by_format.get(api_format, {})
|
||
health_score = float(format_health.get("health_score") or 1.0)
|
||
format_stats[api_format]["health_scores"].append(health_score)
|
||
|
||
# 批量生成所有格式的时间线数据
|
||
format_endpoint_mapping: dict[str, list[str]] = {}
|
||
for api_format, stats in format_stats.items():
|
||
format_endpoint_mapping[api_format] = stats["endpoint_ids"]
|
||
|
||
# 一次性查询所有时间线数据
|
||
timeline_data_map = EndpointHealthService._generate_timeline_batch(
|
||
db, format_endpoint_mapping, now, lookback_hours
|
||
)
|
||
|
||
# 生成结果
|
||
result = []
|
||
|
||
for api_format, stats in format_stats.items():
|
||
timeline_data = timeline_data_map.get(
|
||
api_format,
|
||
{
|
||
"timeline": ["unknown"] * 100,
|
||
"time_range_start": None,
|
||
"time_range_end": None,
|
||
},
|
||
)
|
||
timeline = timeline_data["timeline"]
|
||
time_range_start = timeline_data.get("time_range_start")
|
||
time_range_end = timeline_data.get("time_range_end")
|
||
|
||
# 基于时间线计算实际健康度
|
||
if timeline:
|
||
healthy_count = sum(1 for status in timeline if status == "healthy")
|
||
warning_count = sum(1 for status in timeline if status == "warning")
|
||
unhealthy_count = sum(1 for status in timeline if status == "unhealthy")
|
||
known_count = healthy_count + warning_count + unhealthy_count
|
||
|
||
if known_count > 0:
|
||
avg_health = (healthy_count * 1.0 + warning_count * 0.8) / known_count
|
||
else:
|
||
if stats["health_scores"]:
|
||
avg_health = sum(stats["health_scores"]) / len(stats["health_scores"])
|
||
elif stats["total_keys"] == 0:
|
||
avg_health = 0.0
|
||
else:
|
||
avg_health = 0.1
|
||
else:
|
||
avg_health = 0.0
|
||
|
||
item = {
|
||
"api_format": api_format,
|
||
"display_name": EndpointHealthService._format_display_name(api_format),
|
||
"health_score": avg_health,
|
||
"timeline": timeline,
|
||
"time_range_start": time_range_start.isoformat() if time_range_start else None,
|
||
"time_range_end": time_range_end.isoformat() if time_range_end else None,
|
||
}
|
||
|
||
if include_admin_fields:
|
||
item.update(
|
||
{
|
||
"total_endpoints": stats["total_endpoints"],
|
||
"total_keys": stats["total_keys"],
|
||
"active_keys": stats["active_keys"],
|
||
"provider_count": len(stats["provider_ids"]),
|
||
}
|
||
)
|
||
|
||
result.append(item)
|
||
|
||
result.sort(key=lambda x: x["health_score"], reverse=True)
|
||
|
||
# 写入缓存
|
||
if use_cache:
|
||
EndpointHealthService._set_to_cache(cache_key, result)
|
||
|
||
return result
|
||
|
||
@staticmethod
|
||
def _generate_timeline_batch(
|
||
db: Session,
|
||
format_endpoint_mapping: dict[str, list[str]],
|
||
now: datetime,
|
||
lookback_hours: int,
|
||
segments: int = 100,
|
||
) -> dict[str, dict[str, Any]]:
|
||
"""
|
||
批量生成多个 API 格式的时间线数据（基于 RequestCandidate 表）
|
||
|
||
使用 RequestCandidate 表可以：
|
||
1. 记录所有尝试（包括 fallback 中失败的尝试）
|
||
2. 准确反映每个 Provider/Key 的真实健康状态
|
||
3. 失败的请求会显示为红色节点
|
||
|
||
Args:
|
||
db: 数据库会话
|
||
format_endpoint_mapping: API格式 -> endpoint_ids 的映射
|
||
now: 当前时间
|
||
lookback_hours: 回溯小时数
|
||
segments: 时间段数量
|
||
|
||
Returns:
|
||
API格式 -> 时间线数据的映射
|
||
"""
|
||
# 收集所有 endpoint_ids
|
||
all_endpoint_ids = []
|
||
for endpoint_ids in format_endpoint_mapping.values():
|
||
all_endpoint_ids.extend(endpoint_ids)
|
||
|
||
if not all_endpoint_ids:
|
||
return {
|
||
api_format: {
|
||
"timeline": ["unknown"] * 100,
|
||
"time_range_start": None,
|
||
"time_range_end": None,
|
||
}
|
||
for api_format in format_endpoint_mapping.keys()
|
||
}
|
||
|
||
# 参数校验（API 层已通过 Query(ge=1) 保证，这里做防御性检查）
|
||
if lookback_hours <= 0 or segments <= 0:
|
||
raise ValueError(
|
||
f"lookback_hours and segments must be positive, "
|
||
f"got lookback_hours={lookback_hours}, segments={segments}"
|
||
)
|
||
|
||
# 计算时间范围
|
||
segment_seconds = (lookback_hours * 3600) / segments
|
||
start_time = now - timedelta(hours=lookback_hours)
|
||
|
||
# 使用 RequestCandidate 表查询所有尝试记录
|
||
# 只统计最终状态：success, failed, skipped
|
||
final_statuses = ["success", "failed", "skipped"]
|
||
|
||
segment_expr = func.floor(
|
||
func.extract("epoch", RequestCandidate.created_at - start_time) / segment_seconds
|
||
).label("segment_idx")
|
||
|
||
candidate_stats = (
|
||
db.query(
|
||
RequestCandidate.endpoint_id,
|
||
segment_expr,
|
||
func.count(RequestCandidate.id).label("total_count"),
|
||
func.sum(case((RequestCandidate.status == "success", 1), else_=0)).label(
|
||
"success_count"
|
||
),
|
||
func.sum(case((RequestCandidate.status == "failed", 1), else_=0)).label(
|
||
"failed_count"
|
||
),
|
||
func.min(RequestCandidate.created_at).label("min_time"),
|
||
func.max(RequestCandidate.created_at).label("max_time"),
|
||
)
|
||
.filter(
|
||
RequestCandidate.endpoint_id.in_(all_endpoint_ids),
|
||
RequestCandidate.created_at >= start_time,
|
||
RequestCandidate.created_at <= now,
|
||
RequestCandidate.status.in_(final_statuses),
|
||
)
|
||
.group_by(RequestCandidate.endpoint_id, segment_expr)
|
||
.all()
|
||
)
|
||
|
||
# 构建 endpoint_id -> api_format 的反向映射
|
||
endpoint_to_format: dict[str, str] = {}
|
||
for api_format, endpoint_ids in format_endpoint_mapping.items():
|
||
for endpoint_id in endpoint_ids:
|
||
endpoint_to_format[endpoint_id] = api_format
|
||
|
||
# 按 api_format 和 segment 聚合数据
|
||
format_segment_data: dict[str, dict[int, dict]] = defaultdict(
|
||
lambda: defaultdict(
|
||
lambda: {
|
||
"total": 0,
|
||
"success": 0,
|
||
"failed": 0,
|
||
"min_time": None,
|
||
"max_time": None,
|
||
}
|
||
)
|
||
)
|
||
|
||
for row in candidate_stats:
|
||
endpoint_id = row.endpoint_id
|
||
segment_idx = int(row.segment_idx) if row.segment_idx is not None else 0
|
||
api_format = endpoint_to_format.get(endpoint_id)
|
||
|
||
if api_format and 0 <= segment_idx < segments:
|
||
seg_data = format_segment_data[api_format][segment_idx]
|
||
seg_data["total"] += row.total_count or 0
|
||
seg_data["success"] += row.success_count or 0
|
||
seg_data["failed"] += row.failed_count or 0
|
||
|
||
if row.min_time:
|
||
if seg_data["min_time"] is None or row.min_time < seg_data["min_time"]:
|
||
seg_data["min_time"] = row.min_time
|
||
if row.max_time:
|
||
if seg_data["max_time"] is None or row.max_time > seg_data["max_time"]:
|
||
seg_data["max_time"] = row.max_time
|
||
|
||
# 生成各格式的时间线
|
||
result: dict[str, dict[str, Any]] = {}
|
||
|
||
for api_format in format_endpoint_mapping.keys():
|
||
timeline = []
|
||
earliest_time = None
|
||
latest_time = None
|
||
|
||
segment_data = format_segment_data.get(api_format, {})
|
||
|
||
for i in range(segments):
|
||
seg = segment_data.get(i)
|
||
if not seg or seg["total"] == 0:
|
||
timeline.append("unknown")
|
||
else:
|
||
# 更新时间范围
|
||
if seg["min_time"]:
|
||
if earliest_time is None or seg["min_time"] < earliest_time:
|
||
earliest_time = seg["min_time"]
|
||
if seg["max_time"]:
|
||
if latest_time is None or seg["max_time"] > latest_time:
|
||
latest_time = seg["max_time"]
|
||
|
||
# 计算成功率 = success / (success + failed)
|
||
# skipped 不算失败，不影响成功率
|
||
actual_completed = seg["success"] + seg["failed"]
|
||
if actual_completed > 0:
|
||
success_rate = seg["success"] / actual_completed
|
||
else:
|
||
# 只有 skipped，视为健康
|
||
success_rate = 1.0
|
||
|
||
if success_rate >= 0.95:
|
||
timeline.append("healthy")
|
||
elif success_rate >= 0.7:
|
||
timeline.append("warning")
|
||
else:
|
||
timeline.append("unhealthy")
|
||
|
||
result[api_format] = {
|
||
"timeline": timeline,
|
||
"time_range_start": earliest_time,
|
||
"time_range_end": latest_time if latest_time else now,
|
||
}
|
||
|
||
return result
|
||
|
||
@staticmethod
|
||
def _generate_timeline_from_usage(
|
||
db: Session,
|
||
endpoint_ids: list[str],
|
||
now: datetime,
|
||
lookback_hours: int,
|
||
segments: int = 100,
|
||
) -> dict[str, Any]:
|
||
"""
|
||
从真实使用记录生成时间线数据（使用批量查询优化）
|
||
|
||
Args:
|
||
db: 数据库会话
|
||
endpoint_ids: 端点ID列表
|
||
now: 当前时间
|
||
lookback_hours: 回溯小时数
|
||
segments: 时间段数量
|
||
|
||
Returns:
|
||
包含时间线和时间范围的字典
|
||
"""
|
||
if not endpoint_ids:
|
||
return {
|
||
"timeline": ["unknown"] * 100,
|
||
"time_range_start": None,
|
||
"time_range_end": None,
|
||
}
|
||
|
||
# 直接按 endpoint_id 聚合，避免共享 Key 时把不同 API 格式串桶。
|
||
format_endpoint_mapping = {"_single": endpoint_ids}
|
||
result = EndpointHealthService._generate_timeline_batch(
|
||
db, format_endpoint_mapping, now, lookback_hours, segments
|
||
)
|
||
|
||
return result.get(
|
||
"_single",
|
||
{
|
||
"timeline": ["unknown"] * 100,
|
||
"time_range_start": None,
|
||
"time_range_end": None,
|
||
},
|
||
)
|
||
|
||
@staticmethod
|
||
def _format_display_name(api_format: str) -> str:
|
||
"""格式化 API 格式的显示名称"""
|
||
raw = str(api_format or "").strip()
|
||
normalized = raw.lower()
|
||
if ":" not in normalized:
|
||
return raw or api_format
|
||
|
||
fam, kind = normalized.split(":", 1)
|
||
fam_label = {"claude": "Claude", "openai": "OpenAI", "gemini": "Gemini"}.get(fam, fam)
|
||
kind_label = {
|
||
"chat": "Chat",
|
||
"cli": "CLI",
|
||
"compact": "Compact",
|
||
"video": "Video",
|
||
"image": "Image",
|
||
}.get(kind, kind)
|
||
return f"{fam_label} {kind_label}"
|
||
|
||
@staticmethod
|
||
def _get_from_cache(key: str) -> list[dict[str, Any]] | None:
|
||
"""从 Redis 缓存获取数据"""
|
||
redis_client = _get_redis_client()
|
||
if not redis_client:
|
||
return None
|
||
|
||
try:
|
||
data = redis_client.get(key)
|
||
if data:
|
||
return json.loads(data)
|
||
except Exception as e:
|
||
logger.warning(f"Failed to get from cache: {e}")
|
||
return None
|
||
|
||
@staticmethod
|
||
def _set_to_cache(key: str, data: list[dict[str, Any]]) -> None:
|
||
"""写入 Redis 缓存"""
|
||
redis_client = _get_redis_client()
|
||
if not redis_client:
|
||
return
|
||
|
||
try:
|
||
redis_client.setex(key, CACHE_TTL_SECONDS, json.dumps(data, default=str))
|
||
except Exception as e:
|
||
logger.warning(f"Failed to set cache: {e}")
|