mirror of
https://github.com/fawney19/Aether.git
synced 2026-01-12 04:28:28 +08:00
refactor: 重构限流系统和健康监控,支持按 API 格式区分
- 将 adaptive_concurrency 重命名为 adaptive_rpm,从并发控制改为 RPM 控制 - 健康监控器支持按 API 格式独立管理健康度和熔断器状态 - 新增 model_permissions 模块,支持按格式配置允许的模型 - 重构前端提供商相关表单组件,新增 Collapsible UI 组件 - 新增数据库迁移脚本支持新的数据结构
This commit is contained in:
@@ -1,11 +1,15 @@
|
||||
"""
|
||||
健康监控器 - Endpoint 和 Key 的健康度追踪
|
||||
健康监控器 - Endpoint 和 Key 的健康度追踪(按 API 格式区分)
|
||||
|
||||
功能:
|
||||
1. 基于滑动窗口的错误率计算
|
||||
2. 三态熔断器:关闭 -> 打开 -> 半开 -> 关闭
|
||||
1. 基于滑动窗口的错误率计算(按 API 格式独立)
|
||||
2. 三态熔断器:关闭 -> 打开 -> 半开 -> 关闭(按 API 格式独立)
|
||||
3. 半开状态允许少量请求验证服务恢复
|
||||
4. 提供健康度查询和管理 API
|
||||
|
||||
数据结构:
|
||||
- health_by_format: {"CLAUDE": {"health_score": 1.0, "consecutive_failures": 0, ...}, ...}
|
||||
- circuit_breaker_by_format: {"CLAUDE": {"open": false, "open_at": null, ...}, ...}
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -30,8 +34,30 @@ class CircuitState:
|
||||
HALF_OPEN = "half_open" # 半开(验证恢复)
|
||||
|
||||
|
||||
# 默认健康度数据结构
|
||||
def _default_health_data() -> Dict[str, Any]:
|
||||
return {
|
||||
"health_score": 1.0,
|
||||
"consecutive_failures": 0,
|
||||
"last_failure_at": None,
|
||||
"request_results_window": [],
|
||||
}
|
||||
|
||||
|
||||
# 默认熔断器数据结构
|
||||
def _default_circuit_data() -> Dict[str, Any]:
|
||||
return {
|
||||
"open": False,
|
||||
"open_at": None,
|
||||
"next_probe_at": None,
|
||||
"half_open_until": None,
|
||||
"half_open_successes": 0,
|
||||
"half_open_failures": 0,
|
||||
}
|
||||
|
||||
|
||||
class HealthMonitor:
|
||||
"""健康监控器(滑动窗口 + 半开状态模式)"""
|
||||
"""健康监控器(滑动窗口 + 半开状态模式,按 API 格式区分)"""
|
||||
|
||||
# === 滑动窗口配置 ===
|
||||
WINDOW_SIZE = int(os.getenv("HEALTH_WINDOW_SIZE", str(CircuitBreakerDefaults.WINDOW_SIZE)))
|
||||
@@ -96,6 +122,38 @@ class HealthMonitor:
|
||||
_circuit_history: List[Dict[str, Any]] = []
|
||||
_open_circuit_keys: int = 0
|
||||
|
||||
# ==================== 数据访问辅助方法 ====================
|
||||
|
||||
@classmethod
|
||||
def _get_health_data(cls, key: ProviderAPIKey, api_format: str) -> Dict[str, Any]:
|
||||
"""获取指定格式的健康度数据,不存在则返回默认值"""
|
||||
health_by_format = key.health_by_format or {}
|
||||
if api_format not in health_by_format:
|
||||
return _default_health_data()
|
||||
return health_by_format[api_format]
|
||||
|
||||
@classmethod
|
||||
def _set_health_data(cls, key: ProviderAPIKey, api_format: str, data: Dict[str, Any]) -> None:
|
||||
"""设置指定格式的健康度数据"""
|
||||
health_by_format = dict(key.health_by_format or {})
|
||||
health_by_format[api_format] = data
|
||||
key.health_by_format = health_by_format # type: ignore[assignment]
|
||||
|
||||
@classmethod
|
||||
def _get_circuit_data(cls, key: ProviderAPIKey, api_format: str) -> Dict[str, Any]:
|
||||
"""获取指定格式的熔断器数据,不存在则返回默认值"""
|
||||
circuit_by_format = key.circuit_breaker_by_format or {}
|
||||
if api_format not in circuit_by_format:
|
||||
return _default_circuit_data()
|
||||
return circuit_by_format[api_format]
|
||||
|
||||
@classmethod
|
||||
def _set_circuit_data(cls, key: ProviderAPIKey, api_format: str, data: Dict[str, Any]) -> None:
|
||||
"""设置指定格式的熔断器数据"""
|
||||
circuit_by_format = dict(key.circuit_breaker_by_format or {})
|
||||
circuit_by_format[api_format] = data
|
||||
key.circuit_breaker_by_format = circuit_by_format # type: ignore[assignment]
|
||||
|
||||
# ==================== 核心方法 ====================
|
||||
|
||||
@classmethod
|
||||
@@ -103,9 +161,21 @@ class HealthMonitor:
|
||||
cls,
|
||||
db: Session,
|
||||
key_id: Optional[str] = None,
|
||||
api_format: Optional[str] = None,
|
||||
response_time_ms: Optional[int] = None,
|
||||
) -> None:
|
||||
"""记录成功请求"""
|
||||
"""记录成功请求(按 API 格式)
|
||||
|
||||
Args:
|
||||
db: 数据库会话
|
||||
key_id: Key ID(必需)
|
||||
api_format: API 格式(必需,用于区分不同格式的健康度)
|
||||
response_time_ms: 响应时间(可选)
|
||||
|
||||
Note:
|
||||
api_format 在逻辑上是必需的,但为了向后兼容保持 Optional 签名。
|
||||
如果未提供,会尝试从 Key 的 api_formats 中获取第一个格式作为 fallback。
|
||||
"""
|
||||
try:
|
||||
if not key_id:
|
||||
return
|
||||
@@ -114,39 +184,96 @@ class HealthMonitor:
|
||||
if not key:
|
||||
return
|
||||
|
||||
# api_format 兼容处理:如果未提供,尝试使用 Key 的第一个格式
|
||||
effective_api_format = api_format
|
||||
if not effective_api_format:
|
||||
if key.api_formats and len(key.api_formats) > 0:
|
||||
effective_api_format = key.api_formats[0]
|
||||
logger.debug(
|
||||
f"record_success: api_format 未提供,使用默认格式 {effective_api_format}"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"record_success: api_format 未提供且 Key 无可用格式: key_id={key_id[:8]}..."
|
||||
)
|
||||
return
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
now_ts = now.timestamp()
|
||||
|
||||
# 获取当前格式的健康度数据
|
||||
health_data = cls._get_health_data(key, effective_api_format)
|
||||
circuit_data = cls._get_circuit_data(key, effective_api_format)
|
||||
|
||||
# 1. 更新滑动窗口
|
||||
cls._add_to_window(key, now_ts, success=True)
|
||||
window = health_data.get("request_results_window") or []
|
||||
window.append({"ts": now_ts, "ok": True})
|
||||
cutoff_ts = now_ts - cls.WINDOW_SECONDS
|
||||
window = [r for r in window if r["ts"] > cutoff_ts]
|
||||
if len(window) > cls.WINDOW_SIZE:
|
||||
window = window[-cls.WINDOW_SIZE :]
|
||||
health_data["request_results_window"] = window
|
||||
|
||||
# 2. 更新健康度(用于展示)
|
||||
new_score = min(float(key.health_score or 0) + cls.SUCCESS_INCREMENT, 1.0)
|
||||
key.health_score = new_score # type: ignore[assignment]
|
||||
current_score = float(health_data.get("health_score") or 0)
|
||||
new_score = min(current_score + cls.SUCCESS_INCREMENT, 1.0)
|
||||
health_data["health_score"] = new_score
|
||||
|
||||
# 3. 更新统计
|
||||
key.consecutive_failures = 0 # type: ignore[assignment]
|
||||
key.last_failure_at = None # type: ignore[assignment]
|
||||
health_data["consecutive_failures"] = 0
|
||||
health_data["last_failure_at"] = None
|
||||
|
||||
# 4. 处理熔断器状态
|
||||
state = cls._get_circuit_state_from_data(circuit_data, now)
|
||||
|
||||
if state == CircuitState.HALF_OPEN:
|
||||
# 半开状态:记录成功
|
||||
circuit_data["half_open_successes"] = int(
|
||||
circuit_data.get("half_open_successes") or 0
|
||||
) + 1
|
||||
|
||||
if circuit_data["half_open_successes"] >= cls.HALF_OPEN_SUCCESS_THRESHOLD:
|
||||
# 达到成功阈值,关闭熔断器
|
||||
cls._close_circuit_data(circuit_data, health_data, reason="半开状态验证成功")
|
||||
cls._push_circuit_event(
|
||||
{
|
||||
"event": "closed",
|
||||
"key_id": key.id,
|
||||
"api_format": effective_api_format,
|
||||
"reason": "半开状态验证成功",
|
||||
"timestamp": now.isoformat(),
|
||||
}
|
||||
)
|
||||
logger.info(
|
||||
f"[CLOSED] Key 熔断器关闭: {key.id[:8]}.../{effective_api_format} | 原因: 半开状态验证成功"
|
||||
)
|
||||
|
||||
elif state == CircuitState.OPEN:
|
||||
# 打开状态下的成功(探测成功),进入半开状态
|
||||
cls._enter_half_open_data(circuit_data, now)
|
||||
cls._push_circuit_event(
|
||||
{
|
||||
"event": "half_open",
|
||||
"key_id": key.id,
|
||||
"api_format": effective_api_format,
|
||||
"timestamp": now.isoformat(),
|
||||
}
|
||||
)
|
||||
logger.info(
|
||||
f"[HALF-OPEN] Key 进入半开状态: {key.id[:8]}.../{effective_api_format} | "
|
||||
f"需要 {cls.HALF_OPEN_SUCCESS_THRESHOLD} 次成功关闭熔断器"
|
||||
)
|
||||
|
||||
# 保存数据
|
||||
cls._set_health_data(key, effective_api_format, health_data)
|
||||
cls._set_circuit_data(key, effective_api_format, circuit_data)
|
||||
|
||||
# 更新全局统计
|
||||
key.success_count = int(key.success_count or 0) + 1 # type: ignore[assignment]
|
||||
key.request_count = int(key.request_count or 0) + 1 # type: ignore[assignment]
|
||||
if response_time_ms:
|
||||
key.total_response_time_ms = int(key.total_response_time_ms or 0) + response_time_ms # type: ignore[assignment]
|
||||
|
||||
# 4. 处理熔断器状态
|
||||
state = cls._get_circuit_state(key, now)
|
||||
|
||||
if state == CircuitState.HALF_OPEN:
|
||||
# 半开状态:记录成功
|
||||
key.half_open_successes = int(key.half_open_successes or 0) + 1 # type: ignore[assignment]
|
||||
|
||||
if int(key.half_open_successes or 0) >= cls.HALF_OPEN_SUCCESS_THRESHOLD:
|
||||
# 达到成功阈值,关闭熔断器
|
||||
cls._close_circuit(key, now, reason="半开状态验证成功")
|
||||
|
||||
elif state == CircuitState.OPEN:
|
||||
# 打开状态下的成功(探测成功),进入半开状态
|
||||
cls._enter_half_open(key, now)
|
||||
|
||||
db.flush()
|
||||
get_batch_committer().mark_dirty(db)
|
||||
|
||||
@@ -159,9 +286,21 @@ class HealthMonitor:
|
||||
cls,
|
||||
db: Session,
|
||||
key_id: Optional[str] = None,
|
||||
api_format: Optional[str] = None,
|
||||
error_type: Optional[str] = None,
|
||||
) -> None:
|
||||
"""记录失败请求"""
|
||||
"""记录失败请求(按 API 格式)
|
||||
|
||||
Args:
|
||||
db: 数据库会话
|
||||
key_id: Key ID(必需)
|
||||
api_format: API 格式(必需,用于区分不同格式的健康度)
|
||||
error_type: 错误类型(可选)
|
||||
|
||||
Note:
|
||||
api_format 在逻辑上是必需的,但为了向后兼容保持 Optional 签名。
|
||||
如果未提供,会尝试从 Key 的 api_formats 中获取第一个格式作为 fallback。
|
||||
"""
|
||||
try:
|
||||
if not key_id:
|
||||
return
|
||||
@@ -170,46 +309,117 @@ class HealthMonitor:
|
||||
if not key:
|
||||
return
|
||||
|
||||
# api_format 兼容处理:如果未提供,尝试使用 Key 的第一个格式
|
||||
effective_api_format = api_format
|
||||
if not effective_api_format:
|
||||
if key.api_formats and len(key.api_formats) > 0:
|
||||
effective_api_format = key.api_formats[0]
|
||||
logger.debug(
|
||||
f"record_failure: api_format 未提供,使用默认格式 {effective_api_format}"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"record_failure: api_format 未提供且 Key 无可用格式: key_id={key_id[:8]}..."
|
||||
)
|
||||
return
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
now_ts = now.timestamp()
|
||||
|
||||
# 获取当前格式的健康度数据
|
||||
health_data = cls._get_health_data(key, effective_api_format)
|
||||
circuit_data = cls._get_circuit_data(key, effective_api_format)
|
||||
|
||||
# 1. 更新滑动窗口
|
||||
cls._add_to_window(key, now_ts, success=False)
|
||||
window = health_data.get("request_results_window") or []
|
||||
window.append({"ts": now_ts, "ok": False})
|
||||
cutoff_ts = now_ts - cls.WINDOW_SECONDS
|
||||
window = [r for r in window if r["ts"] > cutoff_ts]
|
||||
if len(window) > cls.WINDOW_SIZE:
|
||||
window = window[-cls.WINDOW_SIZE :]
|
||||
health_data["request_results_window"] = window
|
||||
|
||||
# 2. 更新健康度(用于展示)
|
||||
new_score = max(float(key.health_score or 1) - cls.FAILURE_DECREMENT, 0.0)
|
||||
key.health_score = new_score # type: ignore[assignment]
|
||||
current_score = float(health_data.get("health_score") or 1)
|
||||
new_score = max(current_score - cls.FAILURE_DECREMENT, 0.0)
|
||||
health_data["health_score"] = new_score
|
||||
|
||||
# 3. 更新统计
|
||||
key.consecutive_failures = int(key.consecutive_failures or 0) + 1 # type: ignore[assignment]
|
||||
key.last_failure_at = now # type: ignore[assignment]
|
||||
key.error_count = int(key.error_count or 0) + 1 # type: ignore[assignment]
|
||||
key.request_count = int(key.request_count or 0) + 1 # type: ignore[assignment]
|
||||
health_data["consecutive_failures"] = (
|
||||
int(health_data.get("consecutive_failures") or 0) + 1
|
||||
)
|
||||
health_data["last_failure_at"] = now.isoformat()
|
||||
|
||||
# 4. 处理熔断器状态
|
||||
state = cls._get_circuit_state(key, now)
|
||||
state = cls._get_circuit_state_from_data(circuit_data, now)
|
||||
|
||||
if state == CircuitState.HALF_OPEN:
|
||||
# 半开状态:记录失败
|
||||
key.half_open_failures = int(key.half_open_failures or 0) + 1 # type: ignore[assignment]
|
||||
circuit_data["half_open_failures"] = int(
|
||||
circuit_data.get("half_open_failures") or 0
|
||||
) + 1
|
||||
|
||||
if int(key.half_open_failures or 0) >= cls.HALF_OPEN_FAILURE_THRESHOLD:
|
||||
if circuit_data["half_open_failures"] >= cls.HALF_OPEN_FAILURE_THRESHOLD:
|
||||
# 达到失败阈值,重新打开熔断器
|
||||
cls._open_circuit(key, now, reason="半开状态验证失败")
|
||||
# 注意:半开状态本身就是打开状态的子状态,不需要增加计数
|
||||
consecutive = int(health_data.get("consecutive_failures") or 0)
|
||||
recovery_seconds = cls._calculate_recovery_seconds(consecutive)
|
||||
cls._open_circuit_data(
|
||||
circuit_data, now, recovery_seconds, reason="半开状态验证失败"
|
||||
)
|
||||
cls._push_circuit_event(
|
||||
{
|
||||
"event": "opened",
|
||||
"key_id": key.id,
|
||||
"api_format": effective_api_format,
|
||||
"reason": "半开状态验证失败",
|
||||
"recovery_seconds": recovery_seconds,
|
||||
"timestamp": now.isoformat(),
|
||||
}
|
||||
)
|
||||
logger.warning(
|
||||
f"[OPEN] Key 熔断器打开: {key.id[:8]}.../{effective_api_format} | 原因: 半开状态验证失败 | "
|
||||
f"{recovery_seconds}秒后进入半开状态"
|
||||
)
|
||||
|
||||
elif state == CircuitState.CLOSED:
|
||||
# 关闭状态:检查是否需要打开熔断器
|
||||
error_rate = cls._calculate_error_rate(key, now_ts)
|
||||
window = key.request_results_window or []
|
||||
error_rate = cls._calculate_error_rate_from_window(window, now_ts)
|
||||
|
||||
if len(window) >= cls.MIN_REQUESTS and error_rate >= cls.ERROR_RATE_THRESHOLD:
|
||||
cls._open_circuit(
|
||||
key, now, reason=f"错误率 {error_rate:.0%} 超过阈值 {cls.ERROR_RATE_THRESHOLD:.0%}"
|
||||
consecutive = int(health_data.get("consecutive_failures") or 0)
|
||||
recovery_seconds = cls._calculate_recovery_seconds(consecutive)
|
||||
reason = f"错误率 {error_rate:.0%} 超过阈值 {cls.ERROR_RATE_THRESHOLD:.0%}"
|
||||
cls._open_circuit_data(circuit_data, now, recovery_seconds, reason=reason)
|
||||
cls._open_circuit_keys += 1
|
||||
health_open_circuits.set(cls._open_circuit_keys)
|
||||
cls._push_circuit_event(
|
||||
{
|
||||
"event": "opened",
|
||||
"key_id": key.id,
|
||||
"api_format": effective_api_format,
|
||||
"reason": reason,
|
||||
"recovery_seconds": recovery_seconds,
|
||||
"timestamp": now.isoformat(),
|
||||
}
|
||||
)
|
||||
logger.warning(
|
||||
f"[OPEN] Key 熔断器打开: {key.id[:8]}.../{effective_api_format} | 原因: {reason} | "
|
||||
f"{recovery_seconds}秒后进入半开状态"
|
||||
)
|
||||
|
||||
# 保存数据
|
||||
cls._set_health_data(key, effective_api_format, health_data)
|
||||
cls._set_circuit_data(key, effective_api_format, circuit_data)
|
||||
|
||||
# 更新全局统计
|
||||
key.error_count = int(key.error_count or 0) + 1 # type: ignore[assignment]
|
||||
key.request_count = int(key.request_count or 0) + 1 # type: ignore[assignment]
|
||||
key.last_error_at = now # type: ignore[assignment]
|
||||
|
||||
logger.debug(
|
||||
f"[WARN] Key 健康度下降: {key_id[:8]}... -> {new_score:.2f} "
|
||||
f"(连续失败 {key.consecutive_failures} 次, error_type={error_type})"
|
||||
f"[WARN] Key 健康度下降: {key_id[:8]}.../{effective_api_format} -> {new_score:.2f} "
|
||||
f"(连续失败 {health_data['consecutive_failures']} 次, error_type={error_type})"
|
||||
)
|
||||
|
||||
db.flush()
|
||||
@@ -222,31 +432,13 @@ class HealthMonitor:
|
||||
# ==================== 滑动窗口方法 ====================
|
||||
|
||||
@classmethod
|
||||
def _add_to_window(cls, key: ProviderAPIKey, now_ts: float, success: bool) -> None:
|
||||
"""添加请求结果到滑动窗口"""
|
||||
window: List[Dict[str, Any]] = key.request_results_window or []
|
||||
|
||||
# 添加新记录
|
||||
window.append({"ts": now_ts, "ok": success})
|
||||
|
||||
# 清理过期记录
|
||||
cutoff_ts = now_ts - cls.WINDOW_SECONDS
|
||||
window = [r for r in window if r["ts"] > cutoff_ts]
|
||||
|
||||
# 限制窗口大小
|
||||
if len(window) > cls.WINDOW_SIZE:
|
||||
window = window[-cls.WINDOW_SIZE :]
|
||||
|
||||
key.request_results_window = window # type: ignore[assignment]
|
||||
|
||||
@classmethod
|
||||
def _calculate_error_rate(cls, key: ProviderAPIKey, now_ts: float) -> float:
|
||||
"""计算滑动窗口内的错误率"""
|
||||
window: List[Dict[str, Any]] = key.request_results_window or []
|
||||
def _calculate_error_rate_from_window(
|
||||
cls, window: List[Dict[str, Any]], now_ts: float
|
||||
) -> float:
|
||||
"""从窗口数据计算错误率"""
|
||||
if not window:
|
||||
return 0.0
|
||||
|
||||
# 过滤过期记录
|
||||
cutoff_ts = now_ts - cls.WINDOW_SECONDS
|
||||
valid_records = [r for r in window if r["ts"] > cutoff_ts]
|
||||
|
||||
@@ -256,157 +448,158 @@ class HealthMonitor:
|
||||
failures = sum(1 for r in valid_records if not r["ok"])
|
||||
return failures / len(valid_records)
|
||||
|
||||
# ==================== 熔断器状态方法 ====================
|
||||
# ==================== 熔断器状态方法(操作数据字典)====================
|
||||
|
||||
@classmethod
|
||||
def _get_circuit_state(cls, key: ProviderAPIKey, now: datetime) -> str:
|
||||
"""获取当前熔断器状态"""
|
||||
if not key.circuit_breaker_open:
|
||||
def _get_circuit_state_from_data(cls, circuit_data: Dict[str, Any], now: datetime) -> str:
|
||||
"""从数据字典获取当前熔断器状态"""
|
||||
if not circuit_data.get("open"):
|
||||
return CircuitState.CLOSED
|
||||
|
||||
# 检查是否在半开状态
|
||||
if key.half_open_until and now < key.half_open_until:
|
||||
return CircuitState.HALF_OPEN
|
||||
half_open_until_str = circuit_data.get("half_open_until")
|
||||
if half_open_until_str:
|
||||
half_open_until = datetime.fromisoformat(half_open_until_str)
|
||||
if now < half_open_until:
|
||||
return CircuitState.HALF_OPEN
|
||||
|
||||
# 检查是否到了探测时间(进入半开)
|
||||
if key.next_probe_at and now >= key.next_probe_at:
|
||||
return CircuitState.HALF_OPEN
|
||||
next_probe_str = circuit_data.get("next_probe_at")
|
||||
if next_probe_str:
|
||||
next_probe_at = datetime.fromisoformat(next_probe_str)
|
||||
if now >= next_probe_at:
|
||||
return CircuitState.HALF_OPEN
|
||||
|
||||
return CircuitState.OPEN
|
||||
|
||||
@classmethod
|
||||
def _open_circuit(cls, key: ProviderAPIKey, now: datetime, reason: str) -> None:
|
||||
"""打开熔断器"""
|
||||
was_open = key.circuit_breaker_open
|
||||
|
||||
key.circuit_breaker_open = True # type: ignore[assignment]
|
||||
key.circuit_breaker_open_at = now # type: ignore[assignment]
|
||||
key.half_open_until = None # type: ignore[assignment]
|
||||
key.half_open_successes = 0 # type: ignore[assignment]
|
||||
key.half_open_failures = 0 # type: ignore[assignment]
|
||||
|
||||
# 计算下次探测时间(进入半开状态的时间)
|
||||
consecutive = int(key.consecutive_failures or 0)
|
||||
recovery_seconds = cls._calculate_recovery_seconds(consecutive)
|
||||
key.next_probe_at = now + timedelta(seconds=recovery_seconds) # type: ignore[assignment]
|
||||
|
||||
if not was_open:
|
||||
cls._open_circuit_keys += 1
|
||||
health_open_circuits.set(cls._open_circuit_keys)
|
||||
|
||||
logger.warning(
|
||||
f"[OPEN] Key 熔断器打开: {key.id[:8]}... | 原因: {reason} | "
|
||||
f"{recovery_seconds}秒后进入半开状态"
|
||||
)
|
||||
|
||||
cls._push_circuit_event(
|
||||
{
|
||||
"event": "opened",
|
||||
"key_id": key.id,
|
||||
"reason": reason,
|
||||
"recovery_seconds": recovery_seconds,
|
||||
"timestamp": now.isoformat(),
|
||||
}
|
||||
)
|
||||
def _open_circuit_data(
|
||||
cls,
|
||||
circuit_data: Dict[str, Any],
|
||||
now: datetime,
|
||||
recovery_seconds: int,
|
||||
reason: str,
|
||||
) -> None:
|
||||
"""打开熔断器(操作数据字典)"""
|
||||
circuit_data["open"] = True
|
||||
circuit_data["open_at"] = now.isoformat()
|
||||
circuit_data["half_open_until"] = None
|
||||
circuit_data["half_open_successes"] = 0
|
||||
circuit_data["half_open_failures"] = 0
|
||||
circuit_data["next_probe_at"] = (now + timedelta(seconds=recovery_seconds)).isoformat()
|
||||
|
||||
@classmethod
|
||||
def _enter_half_open(cls, key: ProviderAPIKey, now: datetime) -> None:
|
||||
"""进入半开状态"""
|
||||
key.half_open_until = now + timedelta(seconds=cls.HALF_OPEN_DURATION) # type: ignore[assignment]
|
||||
key.half_open_successes = 0 # type: ignore[assignment]
|
||||
key.half_open_failures = 0 # type: ignore[assignment]
|
||||
|
||||
logger.info(
|
||||
f"[HALF-OPEN] Key 进入半开状态: {key.id[:8]}... | "
|
||||
f"需要 {cls.HALF_OPEN_SUCCESS_THRESHOLD} 次成功关闭熔断器"
|
||||
)
|
||||
|
||||
cls._push_circuit_event(
|
||||
{
|
||||
"event": "half_open",
|
||||
"key_id": key.id,
|
||||
"timestamp": now.isoformat(),
|
||||
}
|
||||
)
|
||||
def _enter_half_open_data(cls, circuit_data: Dict[str, Any], now: datetime) -> None:
|
||||
"""进入半开状态(操作数据字典)"""
|
||||
circuit_data["half_open_until"] = (
|
||||
now + timedelta(seconds=cls.HALF_OPEN_DURATION)
|
||||
).isoformat()
|
||||
circuit_data["half_open_successes"] = 0
|
||||
circuit_data["half_open_failures"] = 0
|
||||
|
||||
@classmethod
|
||||
def _close_circuit(cls, key: ProviderAPIKey, now: datetime, reason: str) -> None:
|
||||
"""关闭熔断器"""
|
||||
key.circuit_breaker_open = False # type: ignore[assignment]
|
||||
key.circuit_breaker_open_at = None # type: ignore[assignment]
|
||||
key.next_probe_at = None # type: ignore[assignment]
|
||||
key.half_open_until = None # type: ignore[assignment]
|
||||
key.half_open_successes = 0 # type: ignore[assignment]
|
||||
key.half_open_failures = 0 # type: ignore[assignment]
|
||||
def _close_circuit_data(
|
||||
cls, circuit_data: Dict[str, Any], health_data: Dict[str, Any], reason: str
|
||||
) -> None:
|
||||
"""关闭熔断器(操作数据字典)"""
|
||||
circuit_data["open"] = False
|
||||
circuit_data["open_at"] = None
|
||||
circuit_data["next_probe_at"] = None
|
||||
circuit_data["half_open_until"] = None
|
||||
circuit_data["half_open_successes"] = 0
|
||||
circuit_data["half_open_failures"] = 0
|
||||
|
||||
# 快速恢复健康度
|
||||
key.health_score = max(float(key.health_score or 0), cls.PROBE_RECOVERY_SCORE) # type: ignore[assignment]
|
||||
current_score = float(health_data.get("health_score") or 0)
|
||||
health_data["health_score"] = max(current_score, cls.PROBE_RECOVERY_SCORE)
|
||||
|
||||
cls._open_circuit_keys = max(0, cls._open_circuit_keys - 1)
|
||||
health_open_circuits.set(cls._open_circuit_keys)
|
||||
|
||||
logger.info(f"[CLOSED] Key 熔断器关闭: {key.id[:8]}... | 原因: {reason}")
|
||||
|
||||
cls._push_circuit_event(
|
||||
{
|
||||
"event": "closed",
|
||||
"key_id": key.id,
|
||||
"reason": reason,
|
||||
"timestamp": now.isoformat(),
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _calculate_recovery_seconds(cls, consecutive_failures: int) -> int:
|
||||
"""计算恢复等待时间(指数退避)"""
|
||||
# 指数退避:30s -> 60s -> 120s -> 240s -> 300s(上限)
|
||||
exponent = min(consecutive_failures // 5, 4) # 每5次失败增加一级
|
||||
exponent = min(consecutive_failures // 5, 4)
|
||||
seconds = cls.INITIAL_RECOVERY_SECONDS * (cls.RECOVERY_BACKOFF**exponent)
|
||||
return min(int(seconds), cls.MAX_RECOVERY_SECONDS)
|
||||
|
||||
# ==================== 状态查询方法 ====================
|
||||
|
||||
@classmethod
|
||||
def is_circuit_breaker_closed(cls, resource: ProviderAPIKey) -> bool:
|
||||
"""检查熔断器是否允许请求通过"""
|
||||
if not resource.circuit_breaker_open:
|
||||
def is_circuit_breaker_closed(
|
||||
cls, resource: ProviderAPIKey, api_format: Optional[str] = None
|
||||
) -> bool:
|
||||
"""检查熔断器是否允许请求通过(按 API 格式)"""
|
||||
if not api_format:
|
||||
# 兼容旧调用:检查是否有任何格式的熔断器开启
|
||||
circuit_by_format = resource.circuit_breaker_by_format or {}
|
||||
for fmt, circuit_data in circuit_by_format.items():
|
||||
if circuit_data.get("open"):
|
||||
return False
|
||||
return True
|
||||
|
||||
circuit_data = cls._get_circuit_data(resource, api_format)
|
||||
|
||||
if not circuit_data.get("open"):
|
||||
return True
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
state = cls._get_circuit_state(resource, now)
|
||||
state = cls._get_circuit_state_from_data(circuit_data, now)
|
||||
|
||||
# 半开状态允许请求通过
|
||||
if state == CircuitState.HALF_OPEN:
|
||||
return True
|
||||
|
||||
# 检查是否到了探测时间
|
||||
if resource.next_probe_at and now >= resource.next_probe_at:
|
||||
# 自动进入半开状态
|
||||
cls._enter_half_open(resource, now)
|
||||
return True
|
||||
next_probe_str = circuit_data.get("next_probe_at")
|
||||
if next_probe_str:
|
||||
next_probe_at = datetime.fromisoformat(next_probe_str)
|
||||
if now >= next_probe_at:
|
||||
# 自动进入半开状态
|
||||
cls._enter_half_open_data(circuit_data, now)
|
||||
cls._set_circuit_data(resource, api_format, circuit_data)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def get_circuit_breaker_status(
|
||||
cls, resource: ProviderAPIKey
|
||||
cls, resource: ProviderAPIKey, api_format: Optional[str] = None
|
||||
) -> Tuple[bool, Optional[str]]:
|
||||
"""获取熔断器详细状态"""
|
||||
if not resource.circuit_breaker_open:
|
||||
"""获取熔断器详细状态(按 API 格式)"""
|
||||
if not api_format:
|
||||
# 兼容旧调用:返回第一个开启的熔断器状态
|
||||
circuit_by_format = resource.circuit_breaker_by_format or {}
|
||||
for fmt, circuit_data in circuit_by_format.items():
|
||||
if circuit_data.get("open"):
|
||||
return cls._get_status_from_circuit_data(circuit_data)
|
||||
return True, None
|
||||
|
||||
circuit_data = cls._get_circuit_data(resource, api_format)
|
||||
return cls._get_status_from_circuit_data(circuit_data)
|
||||
|
||||
@classmethod
|
||||
def _get_status_from_circuit_data(
|
||||
cls, circuit_data: Dict[str, Any]
|
||||
) -> Tuple[bool, Optional[str]]:
|
||||
"""从熔断器数据获取状态描述"""
|
||||
if not circuit_data.get("open"):
|
||||
return True, None
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
state = cls._get_circuit_state(resource, now)
|
||||
state = cls._get_circuit_state_from_data(circuit_data, now)
|
||||
|
||||
if state == CircuitState.HALF_OPEN:
|
||||
successes = int(resource.half_open_successes or 0)
|
||||
successes = int(circuit_data.get("half_open_successes") or 0)
|
||||
return True, f"半开状态({successes}/{cls.HALF_OPEN_SUCCESS_THRESHOLD}成功)"
|
||||
|
||||
if resource.next_probe_at:
|
||||
if now >= resource.next_probe_at:
|
||||
next_probe_str = circuit_data.get("next_probe_at")
|
||||
if next_probe_str:
|
||||
next_probe_at = datetime.fromisoformat(next_probe_str)
|
||||
if now >= next_probe_at:
|
||||
return True, None
|
||||
|
||||
remaining = resource.next_probe_at - now
|
||||
remaining = next_probe_at - now
|
||||
remaining_seconds = int(remaining.total_seconds())
|
||||
if remaining_seconds >= 60:
|
||||
time_str = f"{remaining_seconds // 60}min{remaining_seconds % 60}s"
|
||||
@@ -417,8 +610,10 @@ class HealthMonitor:
|
||||
return False, "熔断中"
|
||||
|
||||
@classmethod
|
||||
def get_key_health(cls, db: Session, key_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""获取 Key 健康状态"""
|
||||
def get_key_health(
|
||||
cls, db: Session, key_id: str, api_format: Optional[str] = None
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""获取 Key 健康状态(支持按格式查询)"""
|
||||
try:
|
||||
key = db.query(ProviderAPIKey).filter(ProviderAPIKey.id == key_id).first()
|
||||
if not key:
|
||||
@@ -427,24 +622,15 @@ class HealthMonitor:
|
||||
now = datetime.now(timezone.utc)
|
||||
now_ts = now.timestamp()
|
||||
|
||||
# 计算当前错误率
|
||||
error_rate = cls._calculate_error_rate(key, now_ts)
|
||||
window = key.request_results_window or []
|
||||
valid_window = [r for r in window if r["ts"] > now_ts - cls.WINDOW_SECONDS]
|
||||
|
||||
avg_response_time_ms = (
|
||||
int(key.total_response_time_ms or 0) / int(key.success_count or 1)
|
||||
if key.success_count
|
||||
else 0
|
||||
)
|
||||
|
||||
return {
|
||||
# 全局统计
|
||||
result = {
|
||||
"key_id": key.id,
|
||||
"health_score": float(key.health_score or 1.0),
|
||||
"error_rate": error_rate,
|
||||
"window_size": len(valid_window),
|
||||
"consecutive_failures": int(key.consecutive_failures or 0),
|
||||
"last_failure_at": key.last_failure_at.isoformat() if key.last_failure_at else None,
|
||||
"is_active": key.is_active,
|
||||
"statistics": {
|
||||
"request_count": int(key.request_count or 0),
|
||||
@@ -457,25 +643,84 @@ class HealthMonitor:
|
||||
),
|
||||
"avg_response_time_ms": round(avg_response_time_ms, 2),
|
||||
},
|
||||
"circuit_breaker": {
|
||||
"state": cls._get_circuit_state(key, now),
|
||||
"open": key.circuit_breaker_open,
|
||||
"open_at": (
|
||||
key.circuit_breaker_open_at.isoformat()
|
||||
if key.circuit_breaker_open_at
|
||||
else None
|
||||
),
|
||||
"next_probe_at": (
|
||||
key.next_probe_at.isoformat() if key.next_probe_at else None
|
||||
),
|
||||
"half_open_until": (
|
||||
key.half_open_until.isoformat() if key.half_open_until else None
|
||||
),
|
||||
"half_open_successes": int(key.half_open_successes or 0),
|
||||
"half_open_failures": int(key.half_open_failures or 0),
|
||||
},
|
||||
}
|
||||
|
||||
# 按格式的健康度数据
|
||||
health_by_format = key.health_by_format or {}
|
||||
circuit_by_format = key.circuit_breaker_by_format or {}
|
||||
|
||||
if api_format:
|
||||
# 查询单个格式
|
||||
health_data = cls._get_health_data(key, api_format)
|
||||
circuit_data = cls._get_circuit_data(key, api_format)
|
||||
window = health_data.get("request_results_window") or []
|
||||
valid_window = [r for r in window if r["ts"] > now_ts - cls.WINDOW_SECONDS]
|
||||
|
||||
result["api_format"] = api_format
|
||||
result["health_score"] = float(health_data.get("health_score") or 1.0)
|
||||
result["error_rate"] = cls._calculate_error_rate_from_window(window, now_ts)
|
||||
result["window_size"] = len(valid_window)
|
||||
result["consecutive_failures"] = int(
|
||||
health_data.get("consecutive_failures") or 0
|
||||
)
|
||||
result["last_failure_at"] = health_data.get("last_failure_at")
|
||||
result["circuit_breaker"] = {
|
||||
"state": cls._get_circuit_state_from_data(circuit_data, now),
|
||||
"open": circuit_data.get("open", False),
|
||||
"open_at": circuit_data.get("open_at"),
|
||||
"next_probe_at": circuit_data.get("next_probe_at"),
|
||||
"half_open_until": circuit_data.get("half_open_until"),
|
||||
"half_open_successes": int(circuit_data.get("half_open_successes") or 0),
|
||||
"half_open_failures": int(circuit_data.get("half_open_failures") or 0),
|
||||
}
|
||||
else:
|
||||
# 返回所有格式的健康度数据
|
||||
formats_health = {}
|
||||
for fmt in (key.api_formats or []):
|
||||
health_data = health_by_format.get(fmt, _default_health_data())
|
||||
circuit_data = circuit_by_format.get(fmt, _default_circuit_data())
|
||||
window = health_data.get("request_results_window") or []
|
||||
valid_window = [r for r in window if r["ts"] > now_ts - cls.WINDOW_SECONDS]
|
||||
|
||||
formats_health[fmt] = {
|
||||
"health_score": float(health_data.get("health_score") or 1.0),
|
||||
"error_rate": cls._calculate_error_rate_from_window(window, now_ts),
|
||||
"window_size": len(valid_window),
|
||||
"consecutive_failures": int(
|
||||
health_data.get("consecutive_failures") or 0
|
||||
),
|
||||
"last_failure_at": health_data.get("last_failure_at"),
|
||||
"circuit_breaker": {
|
||||
"state": cls._get_circuit_state_from_data(circuit_data, now),
|
||||
"open": circuit_data.get("open", False),
|
||||
"open_at": circuit_data.get("open_at"),
|
||||
"next_probe_at": circuit_data.get("next_probe_at"),
|
||||
"half_open_until": circuit_data.get("half_open_until"),
|
||||
"half_open_successes": int(
|
||||
circuit_data.get("half_open_successes") or 0
|
||||
),
|
||||
"half_open_failures": int(
|
||||
circuit_data.get("half_open_failures") or 0
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
result["health_by_format"] = formats_health
|
||||
|
||||
# 计算整体健康度(取最低值)
|
||||
if formats_health:
|
||||
result["health_score"] = min(
|
||||
h["health_score"] for h in formats_health.values()
|
||||
)
|
||||
result["any_circuit_open"] = any(
|
||||
h["circuit_breaker"]["open"] for h in formats_health.values()
|
||||
)
|
||||
else:
|
||||
result["health_score"] = 1.0
|
||||
result["any_circuit_open"] = False
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"获取 Key 健康状态失败: {e}")
|
||||
return None
|
||||
@@ -507,23 +752,24 @@ class HealthMonitor:
|
||||
# ==================== 管理方法 ====================
|
||||
|
||||
@classmethod
|
||||
def reset_health(cls, db: Session, key_id: Optional[str] = None) -> bool:
|
||||
"""重置健康度"""
|
||||
def reset_health(
|
||||
cls, db: Session, key_id: Optional[str] = None, api_format: Optional[str] = None
|
||||
) -> bool:
|
||||
"""重置健康度(支持按格式重置)"""
|
||||
try:
|
||||
if key_id:
|
||||
key = db.query(ProviderAPIKey).filter(ProviderAPIKey.id == key_id).first()
|
||||
if key:
|
||||
key.health_score = 1.0 # type: ignore[assignment]
|
||||
key.consecutive_failures = 0 # type: ignore[assignment]
|
||||
key.last_failure_at = None # type: ignore[assignment]
|
||||
key.request_results_window = [] # type: ignore[assignment]
|
||||
key.circuit_breaker_open = False # type: ignore[assignment]
|
||||
key.circuit_breaker_open_at = None # type: ignore[assignment]
|
||||
key.next_probe_at = None # type: ignore[assignment]
|
||||
key.half_open_until = None # type: ignore[assignment]
|
||||
key.half_open_successes = 0 # type: ignore[assignment]
|
||||
key.half_open_failures = 0 # type: ignore[assignment]
|
||||
logger.info(f"[RESET] 重置 Key 健康度: {key_id}")
|
||||
if api_format:
|
||||
# 重置单个格式
|
||||
cls._set_health_data(key, api_format, _default_health_data())
|
||||
cls._set_circuit_data(key, api_format, _default_circuit_data())
|
||||
logger.info(f"[RESET] 重置 Key 健康度: {key_id}/{api_format}")
|
||||
else:
|
||||
# 重置所有格式
|
||||
key.health_by_format = {} # type: ignore[assignment]
|
||||
key.circuit_breaker_by_format = {} # type: ignore[assignment]
|
||||
logger.info(f"[RESET] 重置 Key 所有格式健康度: {key_id}")
|
||||
|
||||
db.flush()
|
||||
get_batch_committer().mark_dirty(db)
|
||||
@@ -542,7 +788,9 @@ class HealthMonitor:
|
||||
key = db.query(ProviderAPIKey).filter(ProviderAPIKey.id == key_id).first()
|
||||
if key and not key.is_active:
|
||||
key.is_active = True # type: ignore[assignment]
|
||||
key.consecutive_failures = 0 # type: ignore[assignment]
|
||||
# 重置所有格式的健康度
|
||||
key.health_by_format = {} # type: ignore[assignment]
|
||||
key.circuit_breaker_by_format = {} # type: ignore[assignment]
|
||||
logger.info(f"[OK] 手动启用 Key: {key_id}")
|
||||
|
||||
db.flush()
|
||||
@@ -566,14 +814,28 @@ class HealthMonitor:
|
||||
),
|
||||
).first()
|
||||
|
||||
key_stats = db.query(
|
||||
func.count(ProviderAPIKey.id).label("total"),
|
||||
func.sum(case((ProviderAPIKey.is_active == True, 1), else_=0)).label("active"),
|
||||
func.sum(case((ProviderAPIKey.health_score < 0.5, 1), else_=0)).label("unhealthy"),
|
||||
func.sum(case((ProviderAPIKey.circuit_breaker_open == True, 1), else_=0)).label(
|
||||
"circuit_open"
|
||||
),
|
||||
).first()
|
||||
# 统计 Key(需要遍历 JSON 字段计算熔断状态)
|
||||
keys = db.query(ProviderAPIKey).all()
|
||||
total_keys = len(keys)
|
||||
active_keys = sum(1 for k in keys if k.is_active)
|
||||
unhealthy_keys = 0
|
||||
circuit_open_keys = 0
|
||||
|
||||
for key in keys:
|
||||
health_by_format = key.health_by_format or {}
|
||||
circuit_by_format = key.circuit_breaker_by_format or {}
|
||||
|
||||
# 检查是否有任何格式健康度低于 0.5
|
||||
for fmt, health_data in health_by_format.items():
|
||||
if float(health_data.get("health_score") or 1.0) < 0.5:
|
||||
unhealthy_keys += 1
|
||||
break
|
||||
|
||||
# 检查是否有任何格式熔断器开启
|
||||
for fmt, circuit_data in circuit_by_format.items():
|
||||
if circuit_data.get("open"):
|
||||
circuit_open_keys += 1
|
||||
break
|
||||
|
||||
return {
|
||||
"endpoints": {
|
||||
@@ -582,10 +844,10 @@ class HealthMonitor:
|
||||
"unhealthy": int(endpoint_stats.unhealthy or 0) if endpoint_stats else 0,
|
||||
},
|
||||
"keys": {
|
||||
"total": key_stats.total or 0 if key_stats else 0,
|
||||
"active": int(key_stats.active or 0) if key_stats else 0,
|
||||
"unhealthy": int(key_stats.unhealthy or 0) if key_stats else 0,
|
||||
"circuit_open": int(key_stats.circuit_open or 0) if key_stats else 0,
|
||||
"total": total_keys,
|
||||
"active": active_keys,
|
||||
"unhealthy": unhealthy_keys,
|
||||
"circuit_open": circuit_open_keys,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -618,8 +880,9 @@ class HealthMonitor:
|
||||
db: Session,
|
||||
endpoint_id: Optional[str] = None,
|
||||
key_id: Optional[str] = None,
|
||||
api_format: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""检查是否有资格进行探测(兼容旧接口)"""
|
||||
"""检查是否有资格进行探测(按 API 格式)"""
|
||||
if not cls.ALLOW_AUTO_RECOVER:
|
||||
return False
|
||||
|
||||
@@ -628,13 +891,53 @@ class HealthMonitor:
|
||||
|
||||
if key_id:
|
||||
key = db.query(ProviderAPIKey).filter(ProviderAPIKey.id == key_id).first()
|
||||
if key and key.circuit_breaker_open:
|
||||
now = datetime.now(timezone.utc)
|
||||
state = cls._get_circuit_state(key, now)
|
||||
return state == CircuitState.HALF_OPEN
|
||||
if key:
|
||||
if api_format:
|
||||
circuit_data = cls._get_circuit_data(key, api_format)
|
||||
if circuit_data.get("open"):
|
||||
now = datetime.now(timezone.utc)
|
||||
state = cls._get_circuit_state_from_data(circuit_data, now)
|
||||
return state == CircuitState.HALF_OPEN
|
||||
else:
|
||||
# 兼容旧调用:检查是否有任何格式处于半开状态
|
||||
circuit_by_format = key.circuit_breaker_by_format or {}
|
||||
now = datetime.now(timezone.utc)
|
||||
for fmt, circuit_data in circuit_by_format.items():
|
||||
if circuit_data.get("open"):
|
||||
state = cls._get_circuit_state_from_data(circuit_data, now)
|
||||
if state == CircuitState.HALF_OPEN:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
# ==================== 便捷方法 ====================
|
||||
|
||||
@classmethod
|
||||
def get_health_score(
|
||||
cls, key: ProviderAPIKey, api_format: Optional[str] = None
|
||||
) -> float:
|
||||
"""获取指定格式的健康度分数"""
|
||||
if not api_format:
|
||||
# 返回所有格式中的最低健康度
|
||||
health_by_format = key.health_by_format or {}
|
||||
if not health_by_format:
|
||||
return 1.0
|
||||
return min(
|
||||
float(h.get("health_score") or 1.0) for h in health_by_format.values()
|
||||
)
|
||||
|
||||
health_data = cls._get_health_data(key, api_format)
|
||||
return float(health_data.get("health_score") or 1.0)
|
||||
|
||||
@classmethod
|
||||
def is_any_circuit_open(cls, key: ProviderAPIKey) -> bool:
|
||||
"""检查是否有任何格式的熔断器开启"""
|
||||
circuit_by_format = key.circuit_breaker_by_format or {}
|
||||
for circuit_data in circuit_by_format.values():
|
||||
if circuit_data.get("open"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# 全局健康监控器实例
|
||||
health_monitor = HealthMonitor()
|
||||
|
||||
Reference in New Issue
Block a user