feat(failover): 支持 Provider 级别故障转移规则,默认全部转移策略

- 新增 failover_rules 配置:支持 success_failover_patterns(成功响应匹配时转移)
  和 error_stop_patterns(错误响应匹配时终止),支持按 status_code 过滤
- 修改默认转移策略:ErrorClassifier 不再返回 RAISE,所有错误默认继续转移
- TaskService 中客户端错误不再直接抛出,改为 break 继续尝试下一个候选
- 修复 proxy tunnel 连接/断连竞态:引入 per-node 锁和事件时间戳排序
- 优化 ProxyNode 状态判定:OFFLINE 统一由心跳超时判定,兼容多 worker 场景
- has_tunnel 改为纯检查方法,避免在 finally 块中误清理新注册连接
- Redis stream NOGROUP 异常自愈处理
- OAuthAccountDialog 输入框焦点样式补全
This commit is contained in:
fawney19
2026-03-01 00:21:16 +08:00
parent fbcb54a8a5
commit 005cc3e388
18 changed files with 941 additions and 116 deletions

View File

@@ -26,8 +26,14 @@ def _make_candidate(
skip_reason: str | None = None,
needs_conversion: bool = False,
provider_max_retries: int | None = None,
provider_config: dict[str, Any] | None = None,
) -> SimpleNamespace:
provider = SimpleNamespace(id=provider_id, name=provider_name, max_retries=provider_max_retries)
provider = SimpleNamespace(
id=provider_id,
name=provider_name,
max_retries=provider_max_retries,
config=provider_config,
)
endpoint = SimpleNamespace(id=endpoint_id)
key = SimpleNamespace(id=key_id, name=key_name, auth_type=auth_type, priority=priority)
return SimpleNamespace(
@@ -156,7 +162,9 @@ async def test_failover_engine_retry_same_candidate_when_classifier_says_continu
@pytest.mark.asyncio
async def test_failover_engine_stop_when_classifier_raises() -> None:
async def test_failover_engine_continues_when_classifier_raises() -> None:
"""After the 'default failover' change, RAISE no longer stops failover.
All candidates should be attempted."""
db = MagicMock()
engine = FailoverEngine(db, error_classifier=_StubErrorClassifier(action=ErrorAction.RAISE))
@@ -172,10 +180,9 @@ async def test_failover_engine_stop_when_classifier_raises() -> None:
)
assert result.success is False
assert result.error_type == "RuntimeError"
assert result.attempt_count == 1
# should not try candidate 2
assert attempt.await_count == 1
assert result.error_type == "AllCandidatesFailed"
# both candidates should be tried
assert attempt.await_count == 2
async def _stream_two_chunks() -> AsyncIterator[bytes]:
@@ -308,3 +315,144 @@ async def test_failover_engine_pre_expand_marks_unused_slots_on_success(
if call.kwargs.get("status") == "unused"
}
assert unused_record_ids == {"r01", "r10"}
# ========== error_stop_patterns with status_codes ==========
class _HttpError(Exception):
"""Stub exception with status_code and response text."""
def __init__(self, status_code: int, text: str) -> None:
super().__init__(text)
self.status_code = status_code
@pytest.mark.asyncio
async def test_error_stop_pattern_with_matching_status_code_stops_failover() -> None:
"""When status_codes is set and matches, failover should stop."""
db = MagicMock()
engine = FailoverEngine(db, error_classifier=_StubErrorClassifier(action=ErrorAction.BREAK))
config = {
"failover_rules": {
"error_stop_patterns": [
{"pattern": "content_policy", "status_codes": [403]},
],
},
}
candidates = [
_make_candidate(provider_id="p1", provider_config=config),
_make_candidate(provider_id="p2"),
]
attempt = AsyncMock(
side_effect=[
_HttpError(403, "content_policy_violation"),
AttemptResult(
kind=AttemptKind.SYNC_RESPONSE,
http_status=200,
http_headers={},
response_body={"ok": True},
),
]
)
result = await engine.execute(
candidates=candidates,
attempt_func=attempt,
retry_policy=RetryPolicy(mode=RetryMode.DISABLED, max_retries=1),
skip_policy=SkipPolicy(),
request_id=None,
)
# Should stop at first candidate, not try second
assert result.success is False
assert attempt.await_count == 1
@pytest.mark.asyncio
async def test_error_stop_pattern_with_non_matching_status_code_continues() -> None:
"""When status_codes is set but doesn't match, the rule is skipped and failover continues."""
db = MagicMock()
engine = FailoverEngine(db, error_classifier=_StubErrorClassifier(action=ErrorAction.BREAK))
config = {
"failover_rules": {
"error_stop_patterns": [
{"pattern": "content_policy", "status_codes": [403]},
],
},
}
candidates = [
_make_candidate(provider_id="p1", provider_config=config),
_make_candidate(provider_id="p2"),
]
attempt = AsyncMock(
side_effect=[
_HttpError(500, "content_policy_violation"),
AttemptResult(
kind=AttemptKind.SYNC_RESPONSE,
http_status=200,
http_headers={},
response_body={"ok": True},
),
]
)
result = await engine.execute(
candidates=candidates,
attempt_func=attempt,
retry_policy=RetryPolicy(mode=RetryMode.DISABLED, max_retries=1),
skip_policy=SkipPolicy(),
request_id=None,
)
# status_code 500 doesn't match [403], so rule is skipped; failover continues to p2
assert result.success is True
assert result.provider_id == "p2"
assert attempt.await_count == 2
@pytest.mark.asyncio
async def test_error_stop_pattern_without_status_codes_matches_any() -> None:
"""When status_codes is not set, the rule matches any status code (existing behavior)."""
db = MagicMock()
engine = FailoverEngine(db, error_classifier=_StubErrorClassifier(action=ErrorAction.BREAK))
config = {
"failover_rules": {
"error_stop_patterns": [
{"pattern": "content_policy"},
],
},
}
candidates = [
_make_candidate(provider_id="p1", provider_config=config),
_make_candidate(provider_id="p2"),
]
attempt = AsyncMock(
side_effect=[
_HttpError(500, "content_policy_violation"),
AttemptResult(
kind=AttemptKind.SYNC_RESPONSE,
http_status=200,
http_headers={},
response_body={"ok": True},
),
]
)
result = await engine.execute(
candidates=candidates,
attempt_func=attempt,
retry_policy=RetryPolicy(mode=RetryMode.DISABLED, max_retries=1),
skip_policy=SkipPolicy(),
request_id=None,
)
# No status_codes filter, pattern matches -> stop
assert result.success is False
assert attempt.await_count == 1