refactor: 重构限流系统和健康监控,支持按 API 格式区分

- 将 adaptive_concurrency 重命名为 adaptive_rpm,从并发控制改为 RPM 控制
- 健康监控器支持按 API 格式独立管理健康度和熔断器状态
- 新增 model_permissions 模块,支持按格式配置允许的模型
- 重构前端提供商相关表单组件,新增 Collapsible UI 组件
- 新增数据库迁移脚本支持新的数据结构
This commit is contained in:
fawney19
2026-01-10 18:43:53 +08:00
parent dd2fbf4424
commit 09e0f594ff
97 changed files with 6642 additions and 4169 deletions

View File

@@ -1,11 +1,15 @@
"""
健康监控器 - Endpoint 和 Key 的健康度追踪
健康监控器 - Endpoint 和 Key 的健康度追踪(按 API 格式区分)
功能:
1. 基于滑动窗口的错误率计算
2. 三态熔断器:关闭 -> 打开 -> 半开 -> 关闭
1. 基于滑动窗口的错误率计算(按 API 格式独立)
2. 三态熔断器:关闭 -> 打开 -> 半开 -> 关闭(按 API 格式独立)
3. 半开状态允许少量请求验证服务恢复
4. 提供健康度查询和管理 API
数据结构:
- health_by_format: {"CLAUDE": {"health_score": 1.0, "consecutive_failures": 0, ...}, ...}
- circuit_breaker_by_format: {"CLAUDE": {"open": false, "open_at": null, ...}, ...}
"""
import os
@@ -30,8 +34,30 @@ class CircuitState:
HALF_OPEN = "half_open" # 半开(验证恢复)
# 默认健康度数据结构
def _default_health_data() -> Dict[str, Any]:
return {
"health_score": 1.0,
"consecutive_failures": 0,
"last_failure_at": None,
"request_results_window": [],
}
# 默认熔断器数据结构
def _default_circuit_data() -> Dict[str, Any]:
return {
"open": False,
"open_at": None,
"next_probe_at": None,
"half_open_until": None,
"half_open_successes": 0,
"half_open_failures": 0,
}
class HealthMonitor:
"""健康监控器(滑动窗口 + 半开状态模式)"""
"""健康监控器(滑动窗口 + 半开状态模式,按 API 格式区分"""
# === 滑动窗口配置 ===
WINDOW_SIZE = int(os.getenv("HEALTH_WINDOW_SIZE", str(CircuitBreakerDefaults.WINDOW_SIZE)))
@@ -96,6 +122,38 @@ class HealthMonitor:
_circuit_history: List[Dict[str, Any]] = []
_open_circuit_keys: int = 0
# ==================== 数据访问辅助方法 ====================
@classmethod
def _get_health_data(cls, key: ProviderAPIKey, api_format: str) -> Dict[str, Any]:
"""获取指定格式的健康度数据,不存在则返回默认值"""
health_by_format = key.health_by_format or {}
if api_format not in health_by_format:
return _default_health_data()
return health_by_format[api_format]
@classmethod
def _set_health_data(cls, key: ProviderAPIKey, api_format: str, data: Dict[str, Any]) -> None:
"""设置指定格式的健康度数据"""
health_by_format = dict(key.health_by_format or {})
health_by_format[api_format] = data
key.health_by_format = health_by_format # type: ignore[assignment]
@classmethod
def _get_circuit_data(cls, key: ProviderAPIKey, api_format: str) -> Dict[str, Any]:
"""获取指定格式的熔断器数据,不存在则返回默认值"""
circuit_by_format = key.circuit_breaker_by_format or {}
if api_format not in circuit_by_format:
return _default_circuit_data()
return circuit_by_format[api_format]
@classmethod
def _set_circuit_data(cls, key: ProviderAPIKey, api_format: str, data: Dict[str, Any]) -> None:
"""设置指定格式的熔断器数据"""
circuit_by_format = dict(key.circuit_breaker_by_format or {})
circuit_by_format[api_format] = data
key.circuit_breaker_by_format = circuit_by_format # type: ignore[assignment]
# ==================== 核心方法 ====================
@classmethod
@@ -103,9 +161,21 @@ class HealthMonitor:
cls,
db: Session,
key_id: Optional[str] = None,
api_format: Optional[str] = None,
response_time_ms: Optional[int] = None,
) -> None:
"""记录成功请求"""
"""记录成功请求(按 API 格式)
Args:
db: 数据库会话
key_id: Key ID必需
api_format: API 格式(必需,用于区分不同格式的健康度)
response_time_ms: 响应时间(可选)
Note:
api_format 在逻辑上是必需的,但为了向后兼容保持 Optional 签名。
如果未提供,会尝试从 Key 的 api_formats 中获取第一个格式作为 fallback。
"""
try:
if not key_id:
return
@@ -114,39 +184,96 @@ class HealthMonitor:
if not key:
return
# api_format 兼容处理:如果未提供,尝试使用 Key 的第一个格式
effective_api_format = api_format
if not effective_api_format:
if key.api_formats and len(key.api_formats) > 0:
effective_api_format = key.api_formats[0]
logger.debug(
f"record_success: api_format 未提供,使用默认格式 {effective_api_format}"
)
else:
logger.warning(
f"record_success: api_format 未提供且 Key 无可用格式: key_id={key_id[:8]}..."
)
return
now = datetime.now(timezone.utc)
now_ts = now.timestamp()
# 获取当前格式的健康度数据
health_data = cls._get_health_data(key, effective_api_format)
circuit_data = cls._get_circuit_data(key, effective_api_format)
# 1. 更新滑动窗口
cls._add_to_window(key, now_ts, success=True)
window = health_data.get("request_results_window") or []
window.append({"ts": now_ts, "ok": True})
cutoff_ts = now_ts - cls.WINDOW_SECONDS
window = [r for r in window if r["ts"] > cutoff_ts]
if len(window) > cls.WINDOW_SIZE:
window = window[-cls.WINDOW_SIZE :]
health_data["request_results_window"] = window
# 2. 更新健康度(用于展示)
new_score = min(float(key.health_score or 0) + cls.SUCCESS_INCREMENT, 1.0)
key.health_score = new_score # type: ignore[assignment]
current_score = float(health_data.get("health_score") or 0)
new_score = min(current_score + cls.SUCCESS_INCREMENT, 1.0)
health_data["health_score"] = new_score
# 3. 更新统计
key.consecutive_failures = 0 # type: ignore[assignment]
key.last_failure_at = None # type: ignore[assignment]
health_data["consecutive_failures"] = 0
health_data["last_failure_at"] = None
# 4. 处理熔断器状态
state = cls._get_circuit_state_from_data(circuit_data, now)
if state == CircuitState.HALF_OPEN:
# 半开状态:记录成功
circuit_data["half_open_successes"] = int(
circuit_data.get("half_open_successes") or 0
) + 1
if circuit_data["half_open_successes"] >= cls.HALF_OPEN_SUCCESS_THRESHOLD:
# 达到成功阈值,关闭熔断器
cls._close_circuit_data(circuit_data, health_data, reason="半开状态验证成功")
cls._push_circuit_event(
{
"event": "closed",
"key_id": key.id,
"api_format": effective_api_format,
"reason": "半开状态验证成功",
"timestamp": now.isoformat(),
}
)
logger.info(
f"[CLOSED] Key 熔断器关闭: {key.id[:8]}.../{effective_api_format} | 原因: 半开状态验证成功"
)
elif state == CircuitState.OPEN:
# 打开状态下的成功(探测成功),进入半开状态
cls._enter_half_open_data(circuit_data, now)
cls._push_circuit_event(
{
"event": "half_open",
"key_id": key.id,
"api_format": effective_api_format,
"timestamp": now.isoformat(),
}
)
logger.info(
f"[HALF-OPEN] Key 进入半开状态: {key.id[:8]}.../{effective_api_format} | "
f"需要 {cls.HALF_OPEN_SUCCESS_THRESHOLD} 次成功关闭熔断器"
)
# 保存数据
cls._set_health_data(key, effective_api_format, health_data)
cls._set_circuit_data(key, effective_api_format, circuit_data)
# 更新全局统计
key.success_count = int(key.success_count or 0) + 1 # type: ignore[assignment]
key.request_count = int(key.request_count or 0) + 1 # type: ignore[assignment]
if response_time_ms:
key.total_response_time_ms = int(key.total_response_time_ms or 0) + response_time_ms # type: ignore[assignment]
# 4. 处理熔断器状态
state = cls._get_circuit_state(key, now)
if state == CircuitState.HALF_OPEN:
# 半开状态:记录成功
key.half_open_successes = int(key.half_open_successes or 0) + 1 # type: ignore[assignment]
if int(key.half_open_successes or 0) >= cls.HALF_OPEN_SUCCESS_THRESHOLD:
# 达到成功阈值,关闭熔断器
cls._close_circuit(key, now, reason="半开状态验证成功")
elif state == CircuitState.OPEN:
# 打开状态下的成功(探测成功),进入半开状态
cls._enter_half_open(key, now)
db.flush()
get_batch_committer().mark_dirty(db)
@@ -159,9 +286,21 @@ class HealthMonitor:
cls,
db: Session,
key_id: Optional[str] = None,
api_format: Optional[str] = None,
error_type: Optional[str] = None,
) -> None:
"""记录失败请求"""
"""记录失败请求(按 API 格式)
Args:
db: 数据库会话
key_id: Key ID必需
api_format: API 格式(必需,用于区分不同格式的健康度)
error_type: 错误类型(可选)
Note:
api_format 在逻辑上是必需的,但为了向后兼容保持 Optional 签名。
如果未提供,会尝试从 Key 的 api_formats 中获取第一个格式作为 fallback。
"""
try:
if not key_id:
return
@@ -170,46 +309,117 @@ class HealthMonitor:
if not key:
return
# api_format 兼容处理:如果未提供,尝试使用 Key 的第一个格式
effective_api_format = api_format
if not effective_api_format:
if key.api_formats and len(key.api_formats) > 0:
effective_api_format = key.api_formats[0]
logger.debug(
f"record_failure: api_format 未提供,使用默认格式 {effective_api_format}"
)
else:
logger.warning(
f"record_failure: api_format 未提供且 Key 无可用格式: key_id={key_id[:8]}..."
)
return
now = datetime.now(timezone.utc)
now_ts = now.timestamp()
# 获取当前格式的健康度数据
health_data = cls._get_health_data(key, effective_api_format)
circuit_data = cls._get_circuit_data(key, effective_api_format)
# 1. 更新滑动窗口
cls._add_to_window(key, now_ts, success=False)
window = health_data.get("request_results_window") or []
window.append({"ts": now_ts, "ok": False})
cutoff_ts = now_ts - cls.WINDOW_SECONDS
window = [r for r in window if r["ts"] > cutoff_ts]
if len(window) > cls.WINDOW_SIZE:
window = window[-cls.WINDOW_SIZE :]
health_data["request_results_window"] = window
# 2. 更新健康度(用于展示)
new_score = max(float(key.health_score or 1) - cls.FAILURE_DECREMENT, 0.0)
key.health_score = new_score # type: ignore[assignment]
current_score = float(health_data.get("health_score") or 1)
new_score = max(current_score - cls.FAILURE_DECREMENT, 0.0)
health_data["health_score"] = new_score
# 3. 更新统计
key.consecutive_failures = int(key.consecutive_failures or 0) + 1 # type: ignore[assignment]
key.last_failure_at = now # type: ignore[assignment]
key.error_count = int(key.error_count or 0) + 1 # type: ignore[assignment]
key.request_count = int(key.request_count or 0) + 1 # type: ignore[assignment]
health_data["consecutive_failures"] = (
int(health_data.get("consecutive_failures") or 0) + 1
)
health_data["last_failure_at"] = now.isoformat()
# 4. 处理熔断器状态
state = cls._get_circuit_state(key, now)
state = cls._get_circuit_state_from_data(circuit_data, now)
if state == CircuitState.HALF_OPEN:
# 半开状态:记录失败
key.half_open_failures = int(key.half_open_failures or 0) + 1 # type: ignore[assignment]
circuit_data["half_open_failures"] = int(
circuit_data.get("half_open_failures") or 0
) + 1
if int(key.half_open_failures or 0) >= cls.HALF_OPEN_FAILURE_THRESHOLD:
if circuit_data["half_open_failures"] >= cls.HALF_OPEN_FAILURE_THRESHOLD:
# 达到失败阈值,重新打开熔断器
cls._open_circuit(key, now, reason="半开状态验证失败")
# 注意:半开状态本身就是打开状态的子状态,不需要增加计数
consecutive = int(health_data.get("consecutive_failures") or 0)
recovery_seconds = cls._calculate_recovery_seconds(consecutive)
cls._open_circuit_data(
circuit_data, now, recovery_seconds, reason="半开状态验证失败"
)
cls._push_circuit_event(
{
"event": "opened",
"key_id": key.id,
"api_format": effective_api_format,
"reason": "半开状态验证失败",
"recovery_seconds": recovery_seconds,
"timestamp": now.isoformat(),
}
)
logger.warning(
f"[OPEN] Key 熔断器打开: {key.id[:8]}.../{effective_api_format} | 原因: 半开状态验证失败 | "
f"{recovery_seconds}秒后进入半开状态"
)
elif state == CircuitState.CLOSED:
# 关闭状态:检查是否需要打开熔断器
error_rate = cls._calculate_error_rate(key, now_ts)
window = key.request_results_window or []
error_rate = cls._calculate_error_rate_from_window(window, now_ts)
if len(window) >= cls.MIN_REQUESTS and error_rate >= cls.ERROR_RATE_THRESHOLD:
cls._open_circuit(
key, now, reason=f"错误率 {error_rate:.0%} 超过阈值 {cls.ERROR_RATE_THRESHOLD:.0%}"
consecutive = int(health_data.get("consecutive_failures") or 0)
recovery_seconds = cls._calculate_recovery_seconds(consecutive)
reason = f"错误率 {error_rate:.0%} 超过阈值 {cls.ERROR_RATE_THRESHOLD:.0%}"
cls._open_circuit_data(circuit_data, now, recovery_seconds, reason=reason)
cls._open_circuit_keys += 1
health_open_circuits.set(cls._open_circuit_keys)
cls._push_circuit_event(
{
"event": "opened",
"key_id": key.id,
"api_format": effective_api_format,
"reason": reason,
"recovery_seconds": recovery_seconds,
"timestamp": now.isoformat(),
}
)
logger.warning(
f"[OPEN] Key 熔断器打开: {key.id[:8]}.../{effective_api_format} | 原因: {reason} | "
f"{recovery_seconds}秒后进入半开状态"
)
# 保存数据
cls._set_health_data(key, effective_api_format, health_data)
cls._set_circuit_data(key, effective_api_format, circuit_data)
# 更新全局统计
key.error_count = int(key.error_count or 0) + 1 # type: ignore[assignment]
key.request_count = int(key.request_count or 0) + 1 # type: ignore[assignment]
key.last_error_at = now # type: ignore[assignment]
logger.debug(
f"[WARN] Key 健康度下降: {key_id[:8]}... -> {new_score:.2f} "
f"(连续失败 {key.consecutive_failures} 次, error_type={error_type})"
f"[WARN] Key 健康度下降: {key_id[:8]}.../{effective_api_format} -> {new_score:.2f} "
f"(连续失败 {health_data['consecutive_failures']} 次, error_type={error_type})"
)
db.flush()
@@ -222,31 +432,13 @@ class HealthMonitor:
# ==================== 滑动窗口方法 ====================
@classmethod
def _add_to_window(cls, key: ProviderAPIKey, now_ts: float, success: bool) -> None:
"""添加请求结果到滑动窗口"""
window: List[Dict[str, Any]] = key.request_results_window or []
# 添加新记录
window.append({"ts": now_ts, "ok": success})
# 清理过期记录
cutoff_ts = now_ts - cls.WINDOW_SECONDS
window = [r for r in window if r["ts"] > cutoff_ts]
# 限制窗口大小
if len(window) > cls.WINDOW_SIZE:
window = window[-cls.WINDOW_SIZE :]
key.request_results_window = window # type: ignore[assignment]
@classmethod
def _calculate_error_rate(cls, key: ProviderAPIKey, now_ts: float) -> float:
"""计算滑动窗口内的错误率"""
window: List[Dict[str, Any]] = key.request_results_window or []
def _calculate_error_rate_from_window(
cls, window: List[Dict[str, Any]], now_ts: float
) -> float:
"""从窗口数据计算错误率"""
if not window:
return 0.0
# 过滤过期记录
cutoff_ts = now_ts - cls.WINDOW_SECONDS
valid_records = [r for r in window if r["ts"] > cutoff_ts]
@@ -256,157 +448,158 @@ class HealthMonitor:
failures = sum(1 for r in valid_records if not r["ok"])
return failures / len(valid_records)
# ==================== 熔断器状态方法 ====================
# ==================== 熔断器状态方法(操作数据字典)====================
@classmethod
def _get_circuit_state(cls, key: ProviderAPIKey, now: datetime) -> str:
"""获取当前熔断器状态"""
if not key.circuit_breaker_open:
def _get_circuit_state_from_data(cls, circuit_data: Dict[str, Any], now: datetime) -> str:
"""从数据字典获取当前熔断器状态"""
if not circuit_data.get("open"):
return CircuitState.CLOSED
# 检查是否在半开状态
if key.half_open_until and now < key.half_open_until:
return CircuitState.HALF_OPEN
half_open_until_str = circuit_data.get("half_open_until")
if half_open_until_str:
half_open_until = datetime.fromisoformat(half_open_until_str)
if now < half_open_until:
return CircuitState.HALF_OPEN
# 检查是否到了探测时间(进入半开)
if key.next_probe_at and now >= key.next_probe_at:
return CircuitState.HALF_OPEN
next_probe_str = circuit_data.get("next_probe_at")
if next_probe_str:
next_probe_at = datetime.fromisoformat(next_probe_str)
if now >= next_probe_at:
return CircuitState.HALF_OPEN
return CircuitState.OPEN
@classmethod
def _open_circuit(cls, key: ProviderAPIKey, now: datetime, reason: str) -> None:
"""打开熔断器"""
was_open = key.circuit_breaker_open
key.circuit_breaker_open = True # type: ignore[assignment]
key.circuit_breaker_open_at = now # type: ignore[assignment]
key.half_open_until = None # type: ignore[assignment]
key.half_open_successes = 0 # type: ignore[assignment]
key.half_open_failures = 0 # type: ignore[assignment]
# 计算下次探测时间(进入半开状态的时间)
consecutive = int(key.consecutive_failures or 0)
recovery_seconds = cls._calculate_recovery_seconds(consecutive)
key.next_probe_at = now + timedelta(seconds=recovery_seconds) # type: ignore[assignment]
if not was_open:
cls._open_circuit_keys += 1
health_open_circuits.set(cls._open_circuit_keys)
logger.warning(
f"[OPEN] Key 熔断器打开: {key.id[:8]}... | 原因: {reason} | "
f"{recovery_seconds}秒后进入半开状态"
)
cls._push_circuit_event(
{
"event": "opened",
"key_id": key.id,
"reason": reason,
"recovery_seconds": recovery_seconds,
"timestamp": now.isoformat(),
}
)
def _open_circuit_data(
cls,
circuit_data: Dict[str, Any],
now: datetime,
recovery_seconds: int,
reason: str,
) -> None:
"""打开熔断器(操作数据字典)"""
circuit_data["open"] = True
circuit_data["open_at"] = now.isoformat()
circuit_data["half_open_until"] = None
circuit_data["half_open_successes"] = 0
circuit_data["half_open_failures"] = 0
circuit_data["next_probe_at"] = (now + timedelta(seconds=recovery_seconds)).isoformat()
@classmethod
def _enter_half_open(cls, key: ProviderAPIKey, now: datetime) -> None:
"""进入半开状态"""
key.half_open_until = now + timedelta(seconds=cls.HALF_OPEN_DURATION) # type: ignore[assignment]
key.half_open_successes = 0 # type: ignore[assignment]
key.half_open_failures = 0 # type: ignore[assignment]
logger.info(
f"[HALF-OPEN] Key 进入半开状态: {key.id[:8]}... | "
f"需要 {cls.HALF_OPEN_SUCCESS_THRESHOLD} 次成功关闭熔断器"
)
cls._push_circuit_event(
{
"event": "half_open",
"key_id": key.id,
"timestamp": now.isoformat(),
}
)
def _enter_half_open_data(cls, circuit_data: Dict[str, Any], now: datetime) -> None:
"""进入半开状态(操作数据字典)"""
circuit_data["half_open_until"] = (
now + timedelta(seconds=cls.HALF_OPEN_DURATION)
).isoformat()
circuit_data["half_open_successes"] = 0
circuit_data["half_open_failures"] = 0
@classmethod
def _close_circuit(cls, key: ProviderAPIKey, now: datetime, reason: str) -> None:
"""关闭熔断器"""
key.circuit_breaker_open = False # type: ignore[assignment]
key.circuit_breaker_open_at = None # type: ignore[assignment]
key.next_probe_at = None # type: ignore[assignment]
key.half_open_until = None # type: ignore[assignment]
key.half_open_successes = 0 # type: ignore[assignment]
key.half_open_failures = 0 # type: ignore[assignment]
def _close_circuit_data(
cls, circuit_data: Dict[str, Any], health_data: Dict[str, Any], reason: str
) -> None:
"""关闭熔断器(操作数据字典)"""
circuit_data["open"] = False
circuit_data["open_at"] = None
circuit_data["next_probe_at"] = None
circuit_data["half_open_until"] = None
circuit_data["half_open_successes"] = 0
circuit_data["half_open_failures"] = 0
# 快速恢复健康度
key.health_score = max(float(key.health_score or 0), cls.PROBE_RECOVERY_SCORE) # type: ignore[assignment]
current_score = float(health_data.get("health_score") or 0)
health_data["health_score"] = max(current_score, cls.PROBE_RECOVERY_SCORE)
cls._open_circuit_keys = max(0, cls._open_circuit_keys - 1)
health_open_circuits.set(cls._open_circuit_keys)
logger.info(f"[CLOSED] Key 熔断器关闭: {key.id[:8]}... | 原因: {reason}")
cls._push_circuit_event(
{
"event": "closed",
"key_id": key.id,
"reason": reason,
"timestamp": now.isoformat(),
}
)
@classmethod
def _calculate_recovery_seconds(cls, consecutive_failures: int) -> int:
"""计算恢复等待时间(指数退避)"""
# 指数退避30s -> 60s -> 120s -> 240s -> 300s上限
exponent = min(consecutive_failures // 5, 4) # 每5次失败增加一级
exponent = min(consecutive_failures // 5, 4)
seconds = cls.INITIAL_RECOVERY_SECONDS * (cls.RECOVERY_BACKOFF**exponent)
return min(int(seconds), cls.MAX_RECOVERY_SECONDS)
# ==================== 状态查询方法 ====================
@classmethod
def is_circuit_breaker_closed(cls, resource: ProviderAPIKey) -> bool:
"""检查熔断器是否允许请求通过"""
if not resource.circuit_breaker_open:
def is_circuit_breaker_closed(
cls, resource: ProviderAPIKey, api_format: Optional[str] = None
) -> bool:
"""检查熔断器是否允许请求通过(按 API 格式)"""
if not api_format:
# 兼容旧调用:检查是否有任何格式的熔断器开启
circuit_by_format = resource.circuit_breaker_by_format or {}
for fmt, circuit_data in circuit_by_format.items():
if circuit_data.get("open"):
return False
return True
circuit_data = cls._get_circuit_data(resource, api_format)
if not circuit_data.get("open"):
return True
now = datetime.now(timezone.utc)
state = cls._get_circuit_state(resource, now)
state = cls._get_circuit_state_from_data(circuit_data, now)
# 半开状态允许请求通过
if state == CircuitState.HALF_OPEN:
return True
# 检查是否到了探测时间
if resource.next_probe_at and now >= resource.next_probe_at:
# 自动进入半开状态
cls._enter_half_open(resource, now)
return True
next_probe_str = circuit_data.get("next_probe_at")
if next_probe_str:
next_probe_at = datetime.fromisoformat(next_probe_str)
if now >= next_probe_at:
# 自动进入半开状态
cls._enter_half_open_data(circuit_data, now)
cls._set_circuit_data(resource, api_format, circuit_data)
return True
return False
@classmethod
def get_circuit_breaker_status(
cls, resource: ProviderAPIKey
cls, resource: ProviderAPIKey, api_format: Optional[str] = None
) -> Tuple[bool, Optional[str]]:
"""获取熔断器详细状态"""
if not resource.circuit_breaker_open:
"""获取熔断器详细状态(按 API 格式)"""
if not api_format:
# 兼容旧调用:返回第一个开启的熔断器状态
circuit_by_format = resource.circuit_breaker_by_format or {}
for fmt, circuit_data in circuit_by_format.items():
if circuit_data.get("open"):
return cls._get_status_from_circuit_data(circuit_data)
return True, None
circuit_data = cls._get_circuit_data(resource, api_format)
return cls._get_status_from_circuit_data(circuit_data)
@classmethod
def _get_status_from_circuit_data(
cls, circuit_data: Dict[str, Any]
) -> Tuple[bool, Optional[str]]:
"""从熔断器数据获取状态描述"""
if not circuit_data.get("open"):
return True, None
now = datetime.now(timezone.utc)
state = cls._get_circuit_state(resource, now)
state = cls._get_circuit_state_from_data(circuit_data, now)
if state == CircuitState.HALF_OPEN:
successes = int(resource.half_open_successes or 0)
successes = int(circuit_data.get("half_open_successes") or 0)
return True, f"半开状态({successes}/{cls.HALF_OPEN_SUCCESS_THRESHOLD}成功)"
if resource.next_probe_at:
if now >= resource.next_probe_at:
next_probe_str = circuit_data.get("next_probe_at")
if next_probe_str:
next_probe_at = datetime.fromisoformat(next_probe_str)
if now >= next_probe_at:
return True, None
remaining = resource.next_probe_at - now
remaining = next_probe_at - now
remaining_seconds = int(remaining.total_seconds())
if remaining_seconds >= 60:
time_str = f"{remaining_seconds // 60}min{remaining_seconds % 60}s"
@@ -417,8 +610,10 @@ class HealthMonitor:
return False, "熔断中"
@classmethod
def get_key_health(cls, db: Session, key_id: str) -> Optional[Dict[str, Any]]:
"""获取 Key 健康状态"""
def get_key_health(
cls, db: Session, key_id: str, api_format: Optional[str] = None
) -> Optional[Dict[str, Any]]:
"""获取 Key 健康状态(支持按格式查询)"""
try:
key = db.query(ProviderAPIKey).filter(ProviderAPIKey.id == key_id).first()
if not key:
@@ -427,24 +622,15 @@ class HealthMonitor:
now = datetime.now(timezone.utc)
now_ts = now.timestamp()
# 计算当前错误率
error_rate = cls._calculate_error_rate(key, now_ts)
window = key.request_results_window or []
valid_window = [r for r in window if r["ts"] > now_ts - cls.WINDOW_SECONDS]
avg_response_time_ms = (
int(key.total_response_time_ms or 0) / int(key.success_count or 1)
if key.success_count
else 0
)
return {
# 全局统计
result = {
"key_id": key.id,
"health_score": float(key.health_score or 1.0),
"error_rate": error_rate,
"window_size": len(valid_window),
"consecutive_failures": int(key.consecutive_failures or 0),
"last_failure_at": key.last_failure_at.isoformat() if key.last_failure_at else None,
"is_active": key.is_active,
"statistics": {
"request_count": int(key.request_count or 0),
@@ -457,25 +643,84 @@ class HealthMonitor:
),
"avg_response_time_ms": round(avg_response_time_ms, 2),
},
"circuit_breaker": {
"state": cls._get_circuit_state(key, now),
"open": key.circuit_breaker_open,
"open_at": (
key.circuit_breaker_open_at.isoformat()
if key.circuit_breaker_open_at
else None
),
"next_probe_at": (
key.next_probe_at.isoformat() if key.next_probe_at else None
),
"half_open_until": (
key.half_open_until.isoformat() if key.half_open_until else None
),
"half_open_successes": int(key.half_open_successes or 0),
"half_open_failures": int(key.half_open_failures or 0),
},
}
# 按格式的健康度数据
health_by_format = key.health_by_format or {}
circuit_by_format = key.circuit_breaker_by_format or {}
if api_format:
# 查询单个格式
health_data = cls._get_health_data(key, api_format)
circuit_data = cls._get_circuit_data(key, api_format)
window = health_data.get("request_results_window") or []
valid_window = [r for r in window if r["ts"] > now_ts - cls.WINDOW_SECONDS]
result["api_format"] = api_format
result["health_score"] = float(health_data.get("health_score") or 1.0)
result["error_rate"] = cls._calculate_error_rate_from_window(window, now_ts)
result["window_size"] = len(valid_window)
result["consecutive_failures"] = int(
health_data.get("consecutive_failures") or 0
)
result["last_failure_at"] = health_data.get("last_failure_at")
result["circuit_breaker"] = {
"state": cls._get_circuit_state_from_data(circuit_data, now),
"open": circuit_data.get("open", False),
"open_at": circuit_data.get("open_at"),
"next_probe_at": circuit_data.get("next_probe_at"),
"half_open_until": circuit_data.get("half_open_until"),
"half_open_successes": int(circuit_data.get("half_open_successes") or 0),
"half_open_failures": int(circuit_data.get("half_open_failures") or 0),
}
else:
# 返回所有格式的健康度数据
formats_health = {}
for fmt in (key.api_formats or []):
health_data = health_by_format.get(fmt, _default_health_data())
circuit_data = circuit_by_format.get(fmt, _default_circuit_data())
window = health_data.get("request_results_window") or []
valid_window = [r for r in window if r["ts"] > now_ts - cls.WINDOW_SECONDS]
formats_health[fmt] = {
"health_score": float(health_data.get("health_score") or 1.0),
"error_rate": cls._calculate_error_rate_from_window(window, now_ts),
"window_size": len(valid_window),
"consecutive_failures": int(
health_data.get("consecutive_failures") or 0
),
"last_failure_at": health_data.get("last_failure_at"),
"circuit_breaker": {
"state": cls._get_circuit_state_from_data(circuit_data, now),
"open": circuit_data.get("open", False),
"open_at": circuit_data.get("open_at"),
"next_probe_at": circuit_data.get("next_probe_at"),
"half_open_until": circuit_data.get("half_open_until"),
"half_open_successes": int(
circuit_data.get("half_open_successes") or 0
),
"half_open_failures": int(
circuit_data.get("half_open_failures") or 0
),
},
}
result["health_by_format"] = formats_health
# 计算整体健康度(取最低值)
if formats_health:
result["health_score"] = min(
h["health_score"] for h in formats_health.values()
)
result["any_circuit_open"] = any(
h["circuit_breaker"]["open"] for h in formats_health.values()
)
else:
result["health_score"] = 1.0
result["any_circuit_open"] = False
return result
except Exception as e:
logger.error(f"获取 Key 健康状态失败: {e}")
return None
@@ -507,23 +752,24 @@ class HealthMonitor:
# ==================== 管理方法 ====================
@classmethod
def reset_health(cls, db: Session, key_id: Optional[str] = None) -> bool:
"""重置健康度"""
def reset_health(
cls, db: Session, key_id: Optional[str] = None, api_format: Optional[str] = None
) -> bool:
"""重置健康度(支持按格式重置)"""
try:
if key_id:
key = db.query(ProviderAPIKey).filter(ProviderAPIKey.id == key_id).first()
if key:
key.health_score = 1.0 # type: ignore[assignment]
key.consecutive_failures = 0 # type: ignore[assignment]
key.last_failure_at = None # type: ignore[assignment]
key.request_results_window = [] # type: ignore[assignment]
key.circuit_breaker_open = False # type: ignore[assignment]
key.circuit_breaker_open_at = None # type: ignore[assignment]
key.next_probe_at = None # type: ignore[assignment]
key.half_open_until = None # type: ignore[assignment]
key.half_open_successes = 0 # type: ignore[assignment]
key.half_open_failures = 0 # type: ignore[assignment]
logger.info(f"[RESET] 重置 Key 健康度: {key_id}")
if api_format:
# 重置单个格式
cls._set_health_data(key, api_format, _default_health_data())
cls._set_circuit_data(key, api_format, _default_circuit_data())
logger.info(f"[RESET] 重置 Key 健康度: {key_id}/{api_format}")
else:
# 重置所有格式
key.health_by_format = {} # type: ignore[assignment]
key.circuit_breaker_by_format = {} # type: ignore[assignment]
logger.info(f"[RESET] 重置 Key 所有格式健康度: {key_id}")
db.flush()
get_batch_committer().mark_dirty(db)
@@ -542,7 +788,9 @@ class HealthMonitor:
key = db.query(ProviderAPIKey).filter(ProviderAPIKey.id == key_id).first()
if key and not key.is_active:
key.is_active = True # type: ignore[assignment]
key.consecutive_failures = 0 # type: ignore[assignment]
# 重置所有格式的健康度
key.health_by_format = {} # type: ignore[assignment]
key.circuit_breaker_by_format = {} # type: ignore[assignment]
logger.info(f"[OK] 手动启用 Key: {key_id}")
db.flush()
@@ -566,14 +814,28 @@ class HealthMonitor:
),
).first()
key_stats = db.query(
func.count(ProviderAPIKey.id).label("total"),
func.sum(case((ProviderAPIKey.is_active == True, 1), else_=0)).label("active"),
func.sum(case((ProviderAPIKey.health_score < 0.5, 1), else_=0)).label("unhealthy"),
func.sum(case((ProviderAPIKey.circuit_breaker_open == True, 1), else_=0)).label(
"circuit_open"
),
).first()
# 统计 Key需要遍历 JSON 字段计算熔断状态)
keys = db.query(ProviderAPIKey).all()
total_keys = len(keys)
active_keys = sum(1 for k in keys if k.is_active)
unhealthy_keys = 0
circuit_open_keys = 0
for key in keys:
health_by_format = key.health_by_format or {}
circuit_by_format = key.circuit_breaker_by_format or {}
# 检查是否有任何格式健康度低于 0.5
for fmt, health_data in health_by_format.items():
if float(health_data.get("health_score") or 1.0) < 0.5:
unhealthy_keys += 1
break
# 检查是否有任何格式熔断器开启
for fmt, circuit_data in circuit_by_format.items():
if circuit_data.get("open"):
circuit_open_keys += 1
break
return {
"endpoints": {
@@ -582,10 +844,10 @@ class HealthMonitor:
"unhealthy": int(endpoint_stats.unhealthy or 0) if endpoint_stats else 0,
},
"keys": {
"total": key_stats.total or 0 if key_stats else 0,
"active": int(key_stats.active or 0) if key_stats else 0,
"unhealthy": int(key_stats.unhealthy or 0) if key_stats else 0,
"circuit_open": int(key_stats.circuit_open or 0) if key_stats else 0,
"total": total_keys,
"active": active_keys,
"unhealthy": unhealthy_keys,
"circuit_open": circuit_open_keys,
},
}
@@ -618,8 +880,9 @@ class HealthMonitor:
db: Session,
endpoint_id: Optional[str] = None,
key_id: Optional[str] = None,
api_format: Optional[str] = None,
) -> bool:
"""检查是否有资格进行探测(兼容旧接口"""
"""检查是否有资格进行探测(按 API 格式"""
if not cls.ALLOW_AUTO_RECOVER:
return False
@@ -628,13 +891,53 @@ class HealthMonitor:
if key_id:
key = db.query(ProviderAPIKey).filter(ProviderAPIKey.id == key_id).first()
if key and key.circuit_breaker_open:
now = datetime.now(timezone.utc)
state = cls._get_circuit_state(key, now)
return state == CircuitState.HALF_OPEN
if key:
if api_format:
circuit_data = cls._get_circuit_data(key, api_format)
if circuit_data.get("open"):
now = datetime.now(timezone.utc)
state = cls._get_circuit_state_from_data(circuit_data, now)
return state == CircuitState.HALF_OPEN
else:
# 兼容旧调用:检查是否有任何格式处于半开状态
circuit_by_format = key.circuit_breaker_by_format or {}
now = datetime.now(timezone.utc)
for fmt, circuit_data in circuit_by_format.items():
if circuit_data.get("open"):
state = cls._get_circuit_state_from_data(circuit_data, now)
if state == CircuitState.HALF_OPEN:
return True
return False
# ==================== 便捷方法 ====================
@classmethod
def get_health_score(
cls, key: ProviderAPIKey, api_format: Optional[str] = None
) -> float:
"""获取指定格式的健康度分数"""
if not api_format:
# 返回所有格式中的最低健康度
health_by_format = key.health_by_format or {}
if not health_by_format:
return 1.0
return min(
float(h.get("health_score") or 1.0) for h in health_by_format.values()
)
health_data = cls._get_health_data(key, api_format)
return float(health_data.get("health_score") or 1.0)
@classmethod
def is_any_circuit_open(cls, key: ProviderAPIKey) -> bool:
"""检查是否有任何格式的熔断器开启"""
circuit_by_format = key.circuit_breaker_by_format or {}
for circuit_data in circuit_by_format.values():
if circuit_data.get("open"):
return True
return False
# 全局健康监控器实例
health_monitor = HealthMonitor()