Initial commit

src/clients/__init__.py

@@ -0,0 +1,11 @@
+from .http_client import HTTPClientPool, close_http_clients, get_http_client
+from .redis_client import close_redis_client, get_redis_client, get_redis_client_sync
+
+__all__ = [
+    "HTTPClientPool",
+    "get_http_client",
+    "close_http_clients",
+    "get_redis_client",
+    "get_redis_client_sync",
+    "close_redis_client",
+]

src/clients/http_client.py

@@ -0,0 +1,133 @@
+"""
+全局HTTP客户端池管理
+避免每次请求都创建新的AsyncClient,提高性能
+"""
+
+from contextlib import asynccontextmanager
+from typing import Any, Dict, Optional
+
+import httpx
+
+from src.core.logger import logger
+
+
+
+class HTTPClientPool:
+    """
+    全局HTTP客户端池单例
+
+    管理可重用的httpx.AsyncClient实例,避免频繁创建/销毁连接
+    """
+
+    _instance: Optional["HTTPClientPool"] = None
+    _default_client: Optional[httpx.AsyncClient] = None
+    _clients: Dict[str, httpx.AsyncClient] = {}
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    @classmethod
+    def get_default_client(cls) -> httpx.AsyncClient:
+        """
+        获取默认的HTTP客户端
+
+        用于大多数HTTP请求,具有合理的默认配置
+        """
+        if cls._default_client is None:
+            cls._default_client = httpx.AsyncClient(
+                http2=False,  # 暂时禁用HTTP/2以提高兼容性
+                verify=True,  # 启用SSL验证
+                timeout=httpx.Timeout(
+                    connect=10.0,  # 连接超时
+                    read=300.0,  # 读取超时(5分钟,适合流式响应)
+                    write=10.0,  # 写入超时
+                    pool=5.0,  # 连接池超时
+                ),
+                limits=httpx.Limits(
+                    max_connections=100,  # 最大连接数
+                    max_keepalive_connections=20,  # 最大保活连接数
+                    keepalive_expiry=30.0,  # 保活过期时间(秒)
+                ),
+                follow_redirects=True,  # 跟随重定向
+            )
+            logger.info("全局HTTP客户端池已初始化")
+        return cls._default_client
+
+    @classmethod
+    def get_client(cls, name: str, **kwargs: Any) -> httpx.AsyncClient:
+        """
+        获取或创建命名的HTTP客户端
+
+        用于需要特定配置的场景(如不同的超时设置、代理等)
+
+        Args:
+            name: 客户端标识符
+            **kwargs: httpx.AsyncClient的配置参数
+        """
+        if name not in cls._clients:
+            # 合并默认配置和自定义配置
+            config = {
+                "http2": False,
+                "verify": True,
+                "timeout": httpx.Timeout(10.0, read=300.0),
+                "follow_redirects": True,
+            }
+            config.update(kwargs)
+
+            cls._clients[name] = httpx.AsyncClient(**config)
+            logger.debug(f"创建命名HTTP客户端: {name}")
+
+        return cls._clients[name]
+
+    @classmethod
+    async def close_all(cls):
+        """关闭所有HTTP客户端"""
+        if cls._default_client is not None:
+            await cls._default_client.aclose()
+            cls._default_client = None
+            logger.info("默认HTTP客户端已关闭")
+
+        for name, client in cls._clients.items():
+            await client.aclose()
+            logger.debug(f"命名HTTP客户端已关闭: {name}")
+
+        cls._clients.clear()
+        logger.info("所有HTTP客户端已关闭")
+
+    @classmethod
+    @asynccontextmanager
+    async def get_temp_client(cls, **kwargs: Any):
+        """
+        获取临时HTTP客户端(上下文管理器)
+
+        用于一次性请求,使用后自动关闭
+
+        用法:
+            async with HTTPClientPool.get_temp_client() as client:
+                response = await client.get('https://example.com')
+        """
+        config = {
+            "http2": False,
+            "verify": True,
+            "timeout": httpx.Timeout(10.0),
+        }
+        config.update(kwargs)
+
+        client = httpx.AsyncClient(**config)
+        try:
+            yield client
+        finally:
+            await client.aclose()
+
+
+# 便捷访问函数
+def get_http_client() -> httpx.AsyncClient:
+    """获取默认HTTP客户端的便捷函数"""
+    return HTTPClientPool.get_default_client()
+
+
+async def close_http_clients():
+    """关闭所有HTTP客户端的便捷函数"""
+    await HTTPClientPool.close_all()

src/clients/redis_client.py

@@ -0,0 +1,346 @@
+"""
+全局Redis客户端管理
+
+提供统一的Redis客户端访问，确保所有服务使用同一个连接池
+
+熔断器说明:
+- 连续失败达到阈值后开启熔断
+- 熔断期间返回明确的状态而非静默失败
+- 调用方可以根据状态决定降级策略
+"""
+
+import os
+import time
+from enum import Enum
+from typing import Optional
+
+import redis.asyncio as aioredis
+from src.core.logger import logger
+from redis.asyncio import sentinel as redis_sentinel
+
+
+class RedisState(Enum):
+    """Redis 连接状态"""
+
+    NOT_INITIALIZED = "not_initialized"  # 未初始化
+    CONNECTED = "connected"  # 已连接
+    CIRCUIT_OPEN = "circuit_open"  # 熔断中
+    DISCONNECTED = "disconnected"  # 断开连接
+
+
+class RedisClientManager:
+    """
+    Redis客户端管理器（单例）
+
+    提供 Redis 连接管理、熔断器保护和状态监控。
+    """
+
+    _instance: Optional["RedisClientManager"] = None
+    _redis: Optional[aioredis.Redis] = None
+
+    def __new__(cls):
+        """单例模式"""
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def __init__(self):
+        # 避免重复初始化
+        if getattr(self, "_initialized", False):
+            return
+
+        self._initialized = True
+        self._circuit_open_until: Optional[float] = None
+        self._consecutive_failures: int = 0
+        self._circuit_threshold = int(os.getenv("REDIS_CIRCUIT_BREAKER_THRESHOLD", "3"))
+        self._circuit_reset_seconds = int(os.getenv("REDIS_CIRCUIT_BREAKER_RESET_SECONDS", "60"))
+        self._last_error: Optional[str] = None  # 记录最后一次错误
+
+    def get_state(self) -> RedisState:
+        """
+        获取 Redis 连接状态
+
+        Returns:
+            当前连接状态枚举值
+        """
+        if self._redis is not None:
+            return RedisState.CONNECTED
+        if self._circuit_open_until and time.time() < self._circuit_open_until:
+            return RedisState.CIRCUIT_OPEN
+        if self._last_error:
+            return RedisState.DISCONNECTED
+        return RedisState.NOT_INITIALIZED
+
+    def get_circuit_info(self) -> dict:
+        """
+        获取熔断器详细信息
+
+        Returns:
+            包含熔断器状态的字典
+        """
+        state = self.get_state()
+        info = {
+            "state": state.value,
+            "consecutive_failures": self._consecutive_failures,
+            "circuit_threshold": self._circuit_threshold,
+            "last_error": self._last_error,
+        }
+
+        if state == RedisState.CIRCUIT_OPEN and self._circuit_open_until:
+            info["circuit_remaining_seconds"] = max(0, self._circuit_open_until - time.time())
+
+        return info
+
+    def reset_circuit_breaker(self) -> None:
+        """
+        手动重置熔断器（用于管理后台紧急恢复）
+        """
+        logger.info("Redis 熔断器手动重置")
+        self._circuit_open_until = None
+        self._consecutive_failures = 0
+        self._last_error = None
+
+    async def initialize(self, require_redis: bool = False) -> Optional[aioredis.Redis]:
+        """
+        初始化Redis连接
+
+        Args:
+            require_redis: 是否强制要求Redis连接成功，如果为True则连接失败时抛出异常
+
+        Returns:
+            Redis客户端实例，如果连接失败返回None（当require_redis=False时）
+
+        Raises:
+            RuntimeError: 当require_redis=True且连接失败时
+        """
+        if self._redis is not None:
+            return self._redis
+
+        # 检查熔断状态
+        if self._circuit_open_until and time.time() < self._circuit_open_until:
+            remaining = self._circuit_open_until - time.time()
+            logger.warning(
+                "Redis 客户端处于熔断状态，跳过初始化，剩余 %.1f 秒 (last_error: %s)",
+                remaining,
+                self._last_error,
+            )
+            if require_redis:
+                raise RuntimeError(
+                    f"Redis 处于熔断状态，剩余 {remaining:.1f} 秒。"
+                    f"最后错误: {self._last_error}。"
+                    "使用管理 API 重置熔断器或等待自动恢复。"
+                )
+            return None
+
+        # 优先使用 REDIS_URL，如果没有则根据密码构建 URL
+        redis_url = os.getenv("REDIS_URL")
+        redis_max_conn = int(os.getenv("REDIS_MAX_CONNECTIONS", "50"))
+        sentinel_hosts = os.getenv("REDIS_SENTINEL_HOSTS")
+        sentinel_service = os.getenv("REDIS_SENTINEL_SERVICE_NAME", "mymaster")
+        redis_password = os.getenv("REDIS_PASSWORD")
+
+        if not redis_url and not sentinel_hosts:
+            # 本地开发模式：从 REDIS_PASSWORD 构建 URL
+            if redis_password:
+                redis_url = f"redis://:{redis_password}@localhost:6379/0"
+            else:
+                redis_url = "redis://localhost:6379/0"
+
+        try:
+            if sentinel_hosts:
+                sentinel_list = []
+                for host in sentinel_hosts.split(","):
+                    host = host.strip()
+                    if not host:
+                        continue
+                    if ":" in host:
+                        hostname, port = host.split(":", 1)
+                        sentinel_list.append((hostname, int(port)))
+                    else:
+                        sentinel_list.append((host, 26379))
+
+                sentinel_kwargs = {
+                    "password": redis_password,
+                    "socket_timeout": 5.0,
+                }
+                sentinel = redis_sentinel.Sentinel(
+                    sentinel_list,
+                    **sentinel_kwargs,
+                )
+                self._redis = sentinel.master_for(
+                    service_name=sentinel_service,
+                    max_connections=redis_max_conn,
+                    decode_responses=True,
+                    socket_connect_timeout=5.0,
+                )
+                safe_url = f"sentinel://{sentinel_service}"
+            else:
+                self._redis = await aioredis.from_url(
+                    redis_url,
+                    encoding="utf-8",
+                    decode_responses=True,
+                    socket_timeout=5.0,
+                    socket_connect_timeout=5.0,
+                    max_connections=redis_max_conn,
+                )
+                safe_url = redis_url.split("@")[-1] if "@" in redis_url else redis_url
+
+            # 测试连接
+            await self._redis.ping()
+            logger.info(f"[OK] 全局Redis客户端初始化成功: {safe_url}")
+            self._consecutive_failures = 0
+            self._circuit_open_until = None
+            return self._redis
+        except Exception as e:
+            error_msg = str(e)
+            self._last_error = error_msg
+            logger.error(f"[ERROR] Redis连接失败: {error_msg}")
+
+            self._consecutive_failures += 1
+            if self._consecutive_failures >= self._circuit_threshold:
+                self._circuit_open_until = time.time() + self._circuit_reset_seconds
+                logger.warning(
+                    "Redis 初始化连续失败 %s 次，开启熔断 %s 秒。"
+                    "熔断期间以下功能将降级: 缓存亲和性、分布式并发控制、RPM限流。"
+                    "可通过管理 API /api/admin/system/redis/reset-circuit 手动重置。",
+                    self._consecutive_failures,
+                    self._circuit_reset_seconds,
+                )
+
+            if require_redis:
+                # 强制要求Redis时，抛出异常拒绝启动
+                raise RuntimeError(
+                    f"Redis连接失败: {error_msg}\n"
+                    "缓存亲和性功能需要Redis支持，请确保Redis服务正常运行。\n"
+                    "检查事项：\n"
+                    "1. Redis服务是否已启动（docker-compose up -d redis）\n"
+                    "2. 环境变量 REDIS_URL 或 REDIS_PASSWORD 是否配置正确\n"
+                    "3. Redis端口（默认6379）是否可访问"
+                ) from e
+
+            logger.warning(
+                "[WARN] Redis 不可用，以下功能将降级运行（仅在单实例环境下安全）:\n"
+                "  - 缓存亲和性: 禁用（每次请求随机选择 Endpoint）\n"
+                "  - 分布式并发控制: 降级为本地计数\n"
+                "  - RPM 限流: 降级为本地限流"
+            )
+            self._redis = None
+            return None
+
+    async def close(self) -> None:
+        """关闭Redis连接"""
+        if self._redis:
+            await self._redis.close()
+            self._redis = None
+            logger.info("全局Redis客户端已关闭")
+
+    def get_client(self) -> Optional[aioredis.Redis]:
+        """
+        获取Redis客户端（非异步）
+
+        注意：必须先调用initialize()初始化
+
+        Returns:
+            Redis客户端实例或None
+        """
+        return self._redis
+
+
+# 全局单例
+_redis_manager: Optional[RedisClientManager] = None
+
+
+async def get_redis_client(require_redis: bool = False) -> Optional[aioredis.Redis]:
+    """
+    获取全局Redis客户端
+
+    Args:
+        require_redis: 是否强制要求Redis连接成功，如果为True则连接失败时抛出异常
+
+    Returns:
+        Redis客户端实例，如果未初始化或连接失败返回None（当require_redis=False时）
+
+    Raises:
+        RuntimeError: 当require_redis=True且连接失败时
+    """
+    global _redis_manager
+
+    if _redis_manager is None:
+        _redis_manager = RedisClientManager()
+        await _redis_manager.initialize(require_redis=require_redis)
+
+    return _redis_manager.get_client()
+
+
+def get_redis_client_sync() -> Optional[aioredis.Redis]:
+    """
+    同步获取Redis客户端（不会初始化）
+
+    Returns:
+        Redis客户端实例或None
+    """
+    global _redis_manager
+
+    if _redis_manager is None:
+        return None
+
+    return _redis_manager.get_client()
+
+
+async def close_redis_client() -> None:
+    """关闭全局Redis客户端"""
+    global _redis_manager
+
+    if _redis_manager:
+        await _redis_manager.close()
+
+
+def get_redis_state() -> RedisState:
+    """
+    获取 Redis 连接状态（同步方法）
+
+    Returns:
+        Redis 连接状态枚举
+    """
+    global _redis_manager
+
+    if _redis_manager is None:
+        return RedisState.NOT_INITIALIZED
+
+    return _redis_manager.get_state()
+
+
+def get_redis_circuit_info() -> dict:
+    """
+    获取 Redis 熔断器详细信息（同步方法）
+
+    Returns:
+        熔断器状态字典
+    """
+    global _redis_manager
+
+    if _redis_manager is None:
+        return {
+            "state": RedisState.NOT_INITIALIZED.value,
+            "consecutive_failures": 0,
+            "circuit_threshold": 3,
+            "last_error": None,
+        }
+
+    return _redis_manager.get_circuit_info()
+
+
+def reset_redis_circuit_breaker() -> bool:
+    """
+    手动重置 Redis 熔断器（同步方法）
+
+    Returns:
+        是否成功重置
+    """
+    global _redis_manager
+
+    if _redis_manager is None:
+        return False
+
+    _redis_manager.reset_circuit_breaker()
+    return True