Aether/_deprecated_py_src/services/health/endpoint.py

"""
端点健康状态服务

提供统一的端点健康监控功能，支持：
1. 按 API 格式聚合的健康状态
2. 基于时间窗口的状态追踪
3. 管理员和普通用户的差异化视图
4. Redis 缓存优化
"""

from __future__ import annotations

import json
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from typing import Any

from sqlalchemy import case, func
from sqlalchemy.orm import Session, load_only

from src.core.logger import logger
from src.models.database import Provider, ProviderAPIKey, ProviderEndpoint, RequestCandidate

# 缓存配置
CACHE_TTL_SECONDS = 30  # 缓存 30 秒
CACHE_KEY_PREFIX = "health:endpoint:"


def _get_redis_client() -> Any:
    """获取 Redis 客户端，失败返回 None"""
    try:
        from src.clients.redis_client import redis_client

        return redis_client
    except Exception:
        return None


class EndpointHealthService:
    """端点健康状态服务"""

    @staticmethod
    def get_endpoint_health_by_format(
        db: Session,
        lookback_hours: int = 6,
        include_admin_fields: bool = False,
        use_cache: bool = True,
    ) -> list[dict[str, Any]]:
        """
        获取按 API 格式聚合的端点健康状态

        Args:
            db: 数据库会话
            lookback_hours: 回溯小时数
            include_admin_fields: 是否包含管理员字段（provider_count, key_count等）
            use_cache: 是否使用缓存（仅对普通用户视图有效）

        Returns:
            按 API 格式聚合的健康状态列表
        """
        # 尝试从缓存获取
        cache_key = f"{CACHE_KEY_PREFIX}format:{lookback_hours}:{include_admin_fields}"
        if use_cache:
            cached = EndpointHealthService._get_from_cache(cache_key)
            if cached is not None:
                return cached

        now = datetime.now(timezone.utc)

        # 查询所有活跃的端点（一次性获取所有需要的数据）
        endpoints = (
            db.query(ProviderEndpoint)
            .join(Provider)
            .filter(
                ProviderEndpoint.is_active.is_(True),
                Provider.is_active.is_(True),
            )
            .all()
        )

        # 收集所有 provider_ids
        all_provider_ids = list({ep.provider_id for ep in endpoints})

        # 批量查询所有密钥（通过 provider_id 关联，只加载分组和统计所需列）
        all_keys = (
            (
                db.query(ProviderAPIKey)
                .filter(ProviderAPIKey.provider_id.in_(all_provider_ids))
                .options(
                    load_only(
                        ProviderAPIKey.id,
                        ProviderAPIKey.provider_id,
                        ProviderAPIKey.is_active,
                        ProviderAPIKey.api_formats,
                        ProviderAPIKey.health_by_format,
                        ProviderAPIKey.circuit_breaker_by_format,
                    )
                )
                .all()
            )
            if all_provider_ids
            else []
        )

        # 按 api_format 分组密钥（通过 api_formats 字段）
        keys_by_format: dict[str, list[ProviderAPIKey]] = defaultdict(list)
        for key in all_keys:
            for fmt in key.api_formats or []:
                keys_by_format[fmt].append(key)

        # 按 API 格式聚合
        format_stats = defaultdict(
            lambda: {
                "total_endpoints": 0,
                "total_keys": 0,
                "active_keys": 0,
                "health_scores": [],
                "endpoint_ids": [],
                "provider_ids": set(),
                "key_ids": [],
            }
        )

        for ep in endpoints:
            api_format = ep.api_format if ep.api_format else "UNKNOWN"

            # 统计端点数
            format_stats[api_format]["total_endpoints"] += 1
            format_stats[api_format]["endpoint_ids"].append(ep.id)
            format_stats[api_format]["provider_ids"].add(ep.provider_id)

        # 统计每个格式的密钥（直接从 keys_by_format 获取）
        for api_format, keys in keys_by_format.items():
            if api_format not in format_stats:
                # 如果有 Key 但没有对应的 Endpoint，跳过
                continue

            # 去重（同一个 Key 可能支持多个格式）
            seen_key_ids = set()
            unique_keys = []
            for key in keys:
                if key.id not in seen_key_ids:
                    seen_key_ids.add(key.id)
                    unique_keys.append(key)

            format_stats[api_format]["total_keys"] = len(unique_keys)

            for key in unique_keys:
                format_stats[api_format]["key_ids"].append(key.id)
                # 检查该格式的熔断器状态
                circuit_by_format = key.circuit_breaker_by_format or {}
                format_circuit = circuit_by_format.get(api_format, {})
                is_circuit_open = format_circuit.get("open", False)

                if key.is_active and not is_circuit_open:
                    format_stats[api_format]["active_keys"] += 1
                    # 获取该格式的健康度
                    health_by_format = key.health_by_format or {}
                    format_health = health_by_format.get(api_format, {})
                    health_score = float(format_health.get("health_score") or 1.0)
                    format_stats[api_format]["health_scores"].append(health_score)

        # 批量生成所有格式的时间线数据
        format_endpoint_mapping: dict[str, list[str]] = {}
        for api_format, stats in format_stats.items():
            format_endpoint_mapping[api_format] = stats["endpoint_ids"]

        # 一次性查询所有时间线数据
        timeline_data_map = EndpointHealthService._generate_timeline_batch(
            db, format_endpoint_mapping, now, lookback_hours
        )

        # 生成结果
        result = []

        for api_format, stats in format_stats.items():
            timeline_data = timeline_data_map.get(
                api_format,
                {
                    "timeline": ["unknown"] * 100,
                    "time_range_start": None,
                    "time_range_end": None,
                },
            )
            timeline = timeline_data["timeline"]
            time_range_start = timeline_data.get("time_range_start")
            time_range_end = timeline_data.get("time_range_end")

            # 基于时间线计算实际健康度
            if timeline:
                healthy_count = sum(1 for status in timeline if status == "healthy")
                warning_count = sum(1 for status in timeline if status == "warning")
                unhealthy_count = sum(1 for status in timeline if status == "unhealthy")
                known_count = healthy_count + warning_count + unhealthy_count

                if known_count > 0:
                    avg_health = (healthy_count * 1.0 + warning_count * 0.8) / known_count
                else:
                    if stats["health_scores"]:
                        avg_health = sum(stats["health_scores"]) / len(stats["health_scores"])
                    elif stats["total_keys"] == 0:
                        avg_health = 0.0
                    else:
                        avg_health = 0.1
            else:
                avg_health = 0.0

            item = {
                "api_format": api_format,
                "display_name": EndpointHealthService._format_display_name(api_format),
                "health_score": avg_health,
                "timeline": timeline,
                "time_range_start": time_range_start.isoformat() if time_range_start else None,
                "time_range_end": time_range_end.isoformat() if time_range_end else None,
            }

            if include_admin_fields:
                item.update(
                    {
                        "total_endpoints": stats["total_endpoints"],
                        "total_keys": stats["total_keys"],
                        "active_keys": stats["active_keys"],
                        "provider_count": len(stats["provider_ids"]),
                    }
                )

            result.append(item)

        result.sort(key=lambda x: x["health_score"], reverse=True)

        # 写入缓存
        if use_cache:
            EndpointHealthService._set_to_cache(cache_key, result)

        return result

    @staticmethod
    def _generate_timeline_batch(
        db: Session,
        format_endpoint_mapping: dict[str, list[str]],
        now: datetime,
        lookback_hours: int,
        segments: int = 100,
    ) -> dict[str, dict[str, Any]]:
        """
        批量生成多个 API 格式的时间线数据（基于 RequestCandidate 表）

        使用 RequestCandidate 表可以：
        1. 记录所有尝试（包括 fallback 中失败的尝试）
        2. 准确反映每个 Provider/Key 的真实健康状态
        3. 失败的请求会显示为红色节点

        Args:
            db: 数据库会话
            format_endpoint_mapping: API格式 -> endpoint_ids 的映射
            now: 当前时间
            lookback_hours: 回溯小时数
            segments: 时间段数量

        Returns:
            API格式 -> 时间线数据的映射
        """
        # 收集所有 endpoint_ids
        all_endpoint_ids = []
        for endpoint_ids in format_endpoint_mapping.values():
            all_endpoint_ids.extend(endpoint_ids)

        if not all_endpoint_ids:
            return {
                api_format: {
                    "timeline": ["unknown"] * 100,
                    "time_range_start": None,
                    "time_range_end": None,
                }
                for api_format in format_endpoint_mapping.keys()
            }

        # 参数校验（API 层已通过 Query(ge=1) 保证，这里做防御性检查）
        if lookback_hours <= 0 or segments <= 0:
            raise ValueError(
                f"lookback_hours and segments must be positive, "
                f"got lookback_hours={lookback_hours}, segments={segments}"
            )

        # 计算时间范围
        segment_seconds = (lookback_hours * 3600) / segments
        start_time = now - timedelta(hours=lookback_hours)

        # 使用 RequestCandidate 表查询所有尝试记录
        # 只统计最终状态：success, failed, skipped
        final_statuses = ["success", "failed", "skipped"]

        segment_expr = func.floor(
            func.extract("epoch", RequestCandidate.created_at - start_time) / segment_seconds
        ).label("segment_idx")

        candidate_stats = (
            db.query(
                RequestCandidate.endpoint_id,
                segment_expr,
                func.count(RequestCandidate.id).label("total_count"),
                func.sum(case((RequestCandidate.status == "success", 1), else_=0)).label(
                    "success_count"
                ),
                func.sum(case((RequestCandidate.status == "failed", 1), else_=0)).label(
                    "failed_count"
                ),
                func.min(RequestCandidate.created_at).label("min_time"),
                func.max(RequestCandidate.created_at).label("max_time"),
            )
            .filter(
                RequestCandidate.endpoint_id.in_(all_endpoint_ids),
                RequestCandidate.created_at >= start_time,
                RequestCandidate.created_at <= now,
                RequestCandidate.status.in_(final_statuses),
            )
            .group_by(RequestCandidate.endpoint_id, segment_expr)
            .all()
        )

        # 构建 endpoint_id -> api_format 的反向映射
        endpoint_to_format: dict[str, str] = {}
        for api_format, endpoint_ids in format_endpoint_mapping.items():
            for endpoint_id in endpoint_ids:
                endpoint_to_format[endpoint_id] = api_format

        # 按 api_format 和 segment 聚合数据
        format_segment_data: dict[str, dict[int, dict]] = defaultdict(
            lambda: defaultdict(
                lambda: {
                    "total": 0,
                    "success": 0,
                    "failed": 0,
                    "min_time": None,
                    "max_time": None,
                }
            )
        )

        for row in candidate_stats:
            endpoint_id = row.endpoint_id
            segment_idx = int(row.segment_idx) if row.segment_idx is not None else 0
            api_format = endpoint_to_format.get(endpoint_id)

            if api_format and 0 <= segment_idx < segments:
                seg_data = format_segment_data[api_format][segment_idx]
                seg_data["total"] += row.total_count or 0
                seg_data["success"] += row.success_count or 0
                seg_data["failed"] += row.failed_count or 0

                if row.min_time:
                    if seg_data["min_time"] is None or row.min_time < seg_data["min_time"]:
                        seg_data["min_time"] = row.min_time
                if row.max_time:
                    if seg_data["max_time"] is None or row.max_time > seg_data["max_time"]:
                        seg_data["max_time"] = row.max_time

        # 生成各格式的时间线
        result: dict[str, dict[str, Any]] = {}

        for api_format in format_endpoint_mapping.keys():
            timeline = []
            earliest_time = None
            latest_time = None

            segment_data = format_segment_data.get(api_format, {})

            for i in range(segments):
                seg = segment_data.get(i)
                if not seg or seg["total"] == 0:
                    timeline.append("unknown")
                else:
                    # 更新时间范围
                    if seg["min_time"]:
                        if earliest_time is None or seg["min_time"] < earliest_time:
                            earliest_time = seg["min_time"]
                    if seg["max_time"]:
                        if latest_time is None or seg["max_time"] > latest_time:
                            latest_time = seg["max_time"]

                    # 计算成功率 = success / (success + failed)
                    # skipped 不算失败，不影响成功率
                    actual_completed = seg["success"] + seg["failed"]
                    if actual_completed > 0:
                        success_rate = seg["success"] / actual_completed
                    else:
                        # 只有 skipped，视为健康
                        success_rate = 1.0

                    if success_rate >= 0.95:
                        timeline.append("healthy")
                    elif success_rate >= 0.7:
                        timeline.append("warning")
                    else:
                        timeline.append("unhealthy")

            result[api_format] = {
                "timeline": timeline,
                "time_range_start": earliest_time,
                "time_range_end": latest_time if latest_time else now,
            }

        return result

    @staticmethod
    def _generate_timeline_from_usage(
        db: Session,
        endpoint_ids: list[str],
        now: datetime,
        lookback_hours: int,
        segments: int = 100,
    ) -> dict[str, Any]:
        """
        从真实使用记录生成时间线数据（使用批量查询优化）

        Args:
            db: 数据库会话
            endpoint_ids: 端点ID列表
            now: 当前时间
            lookback_hours: 回溯小时数
            segments: 时间段数量

        Returns:
            包含时间线和时间范围的字典
        """
        if not endpoint_ids:
            return {
                "timeline": ["unknown"] * 100,
                "time_range_start": None,
                "time_range_end": None,
            }

        # 直接按 endpoint_id 聚合，避免共享 Key 时把不同 API 格式串桶。
        format_endpoint_mapping = {"_single": endpoint_ids}
        result = EndpointHealthService._generate_timeline_batch(
            db, format_endpoint_mapping, now, lookback_hours, segments
        )

        return result.get(
            "_single",
            {
                "timeline": ["unknown"] * 100,
                "time_range_start": None,
                "time_range_end": None,
            },
        )

    @staticmethod
    def _format_display_name(api_format: str) -> str:
        """格式化 API 格式的显示名称"""
        raw = str(api_format or "").strip()
        normalized = raw.lower()
        if ":" not in normalized:
            return raw or api_format

        fam, kind = normalized.split(":", 1)
        fam_label = {"claude": "Claude", "openai": "OpenAI", "gemini": "Gemini"}.get(fam, fam)
        kind_label = {
            "chat": "Chat",
            "cli": "CLI",
            "compact": "Compact",
            "video": "Video",
            "image": "Image",
        }.get(kind, kind)
        return f"{fam_label} {kind_label}"

    @staticmethod
    def _get_from_cache(key: str) -> list[dict[str, Any]] | None:
        """从 Redis 缓存获取数据"""
        redis_client = _get_redis_client()
        if not redis_client:
            return None

        try:
            data = redis_client.get(key)
            if data:
                return json.loads(data)
        except Exception as e:
            logger.warning(f"Failed to get from cache: {e}")
        return None

    @staticmethod
    def _set_to_cache(key: str, data: list[dict[str, Any]]) -> None:
        """写入 Redis 缓存"""
        redis_client = _get_redis_client()
        if not redis_client:
            return

        try:
            redis_client.setex(key, CACHE_TTL_SECONDS, json.dumps(data, default=str))
        except Exception as e:
            logger.warning(f"Failed to set cache: {e}")