feat(failover): 支持 Provider 级别故障转移规则,默认全部转移策略

- 新增 failover_rules 配置:支持 success_failover_patterns(成功响应匹配时转移)
  和 error_stop_patterns(错误响应匹配时终止),支持按 status_code 过滤
- 修改默认转移策略:ErrorClassifier 不再返回 RAISE,所有错误默认继续转移
- TaskService 中客户端错误不再直接抛出,改为 break 继续尝试下一个候选
- 修复 proxy tunnel 连接/断连竞态:引入 per-node 锁和事件时间戳排序
- 优化 ProxyNode 状态判定:OFFLINE 统一由心跳超时判定,兼容多 worker 场景
- has_tunnel 改为纯检查方法,避免在 finally 块中误清理新注册连接
- Redis stream NOGROUP 异常自愈处理
- OAuthAccountDialog 输入框焦点样式补全
This commit is contained in:
fawney19
2026-03-01 00:21:16 +08:00
parent fbcb54a8a5
commit 005cc3e388
18 changed files with 941 additions and 116 deletions

View File

@@ -82,6 +82,32 @@ def _merge_pool_advanced_config(
return merged_config or None, config_changed
def _merge_failover_rules_config(
*,
provider_config: dict[str, Any] | None,
failover_rules: dict[str, Any] | None,
failover_rules_in_payload: bool,
) -> tuple[dict[str, Any] | None, bool]:
"""合并 failover_rules 到 provider.config。"""
merged_config = dict(provider_config or {})
config_changed = False
if not failover_rules_in_payload:
return merged_config or None, config_changed
if failover_rules is None:
if "failover_rules" in merged_config:
merged_config.pop("failover_rules", None)
config_changed = True
else:
next_value = dict(failover_rules)
if merged_config.get("failover_rules") != next_value:
merged_config["failover_rules"] = next_value
config_changed = True
return merged_config or None, config_changed
def _merge_claude_code_advanced_config(
*,
provider_type: str | None,
@@ -394,6 +420,15 @@ class AdminCreateProviderAdapter(AdminApiAdapter):
),
pool_advanced_in_payload=validated_data.pool_advanced is not None,
)
provider_config, _ = _merge_failover_rules_config(
provider_config=provider_config,
failover_rules=(
validated_data.failover_rules.model_dump()
if validated_data.failover_rules is not None
else None
),
failover_rules_in_payload=validated_data.failover_rules is not None,
)
# 创建 Provider 对象
provider = Provider(
@@ -509,6 +544,7 @@ class AdminUpdateProviderAdapter(AdminApiAdapter):
config_in_payload = "config" in update_data
claude_advanced_in_payload = "claude_code_advanced" in update_data
pool_advanced_in_payload = "pool_advanced" in update_data
failover_rules_in_payload = "failover_rules" in update_data
provider_config = (
dict(update_data.pop("config") or {})
if config_in_payload
@@ -518,6 +554,9 @@ class AdminUpdateProviderAdapter(AdminApiAdapter):
update_data.pop("claude_code_advanced") if claude_advanced_in_payload else None
)
pool_advanced = update_data.pop("pool_advanced") if pool_advanced_in_payload else None
failover_rules = (
update_data.pop("failover_rules") if failover_rules_in_payload else None
)
target_provider_type = (
update_data.get("provider_type")
or getattr(provider, "provider_type", None)
@@ -535,6 +574,11 @@ class AdminUpdateProviderAdapter(AdminApiAdapter):
pool_advanced=pool_advanced,
pool_advanced_in_payload=pool_advanced_in_payload,
)
provider_config, config_changed_by_failover = _merge_failover_rules_config(
provider_config=provider_config,
failover_rules=failover_rules,
failover_rules_in_payload=failover_rules_in_payload,
)
config_touched = (
config_in_payload
@@ -542,6 +586,8 @@ class AdminUpdateProviderAdapter(AdminApiAdapter):
or config_changed_by_claude
or pool_advanced_in_payload
or config_changed_by_pool
or failover_rules_in_payload
or config_changed_by_failover
)
if config_touched:
update_data["config"] = provider_config

View File

@@ -18,7 +18,11 @@ from src.core.enums import ProviderBillingType
from src.core.exceptions import InvalidRequestException, NotFoundException
from src.core.logger import logger
from src.database import get_db
from src.models.admin_requests import ClaudeCodeAdvancedConfig, PoolAdvancedConfig
from src.models.admin_requests import (
ClaudeCodeAdvancedConfig,
FailoverRulesConfig,
PoolAdvancedConfig,
)
from src.models.database import (
Model,
Provider,
@@ -282,6 +286,38 @@ def _extract_claude_code_advanced_from_config(
return None
def _extract_failover_rules_from_config(
provider_config: dict[str, Any] | None,
*,
provider_id: str,
) -> FailoverRulesConfig | None:
"""从 Provider.config 中安全提取故障转移规则配置。"""
raw = (provider_config or {}).get("failover_rules")
if raw is None:
return None
if isinstance(raw, FailoverRulesConfig):
return raw
if not isinstance(raw, dict):
logger.warning(
"Provider {} 的 failover_rules 类型无效: {},已忽略",
provider_id,
type(raw).__name__,
)
return None
try:
return FailoverRulesConfig.model_validate(raw)
except Exception as exc:
logger.warning(
"Provider {} 的 failover_rules 配置无效,已忽略: {}",
provider_id,
str(exc),
)
return None
def _build_provider_summary(db: Session, provider: Provider) -> ProviderWithEndpointsSummary:
endpoints = db.query(ProviderEndpoint).filter(ProviderEndpoint.provider_id == provider.id).all()
@@ -402,6 +438,10 @@ def _build_provider_summary(db: Session, provider: Provider) -> ProviderWithEndp
provider_config,
provider_id=str(provider.id),
)
failover_rules = _extract_failover_rules_from_config(
provider_config,
provider_id=str(provider.id),
)
return ProviderWithEndpointsSummary(
id=provider.id,
@@ -425,6 +465,7 @@ def _build_provider_summary(db: Session, provider: Provider) -> ProviderWithEndp
request_timeout=provider.request_timeout,
claude_code_advanced=claude_code_advanced,
pool_advanced=pool_advanced,
failover_rules=failover_rules,
total_endpoints=total_endpoints,
active_endpoints=active_endpoints,
total_keys=total_keys,