feat(failover): 支持 Provider 级别故障转移规则，默认全部转移策略

- 新增 failover_rules 配置：支持 success_failover_patterns（成功响应匹配时转移）
  和 error_stop_patterns（错误响应匹配时终止），支持按 status_code 过滤
- 修改默认转移策略：ErrorClassifier 不再返回 RAISE，所有错误默认继续转移
- TaskService 中客户端错误不再直接抛出，改为 break 继续尝试下一个候选
- 修复 proxy tunnel 连接/断连竞态：引入 per-node 锁和事件时间戳排序
- 优化 ProxyNode 状态判定：OFFLINE 统一由心跳超时判定，兼容多 worker 场景
- has_tunnel 改为纯检查方法，避免在 finally 块中误清理新注册连接
- Redis stream NOGROUP 异常自愈处理
- OAuthAccountDialog 输入框焦点样式补全

src/services/candidate/failover.py

@@ -1,13 +1,13 @@
 from __future__ import annotations
 
 import asyncio
+import json
 import re
 from collections.abc import Awaitable, Callable
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from typing import Any, AsyncIterator
 
-import httpx
 from sqlalchemy import update
 from sqlalchemy.orm import Session
 
@@ -195,8 +195,33 @@ class FailoverEngine:
                         attempt_result = await self._probe_stream_first_chunk(
                             attempt_result=attempt_result,
                             record_id=record_id,
+                            candidate=candidate,
                         )
 
+                    # Sync: check success_failover_patterns on response body
+                    if attempt_result.kind == AttemptKind.SYNC_RESPONSE:
+                        body = getattr(attempt_result, "response_body", None)
+                        if body:
+                            if isinstance(body, bytes):
+                                body_text = body.decode("utf-8", errors="replace")
+                            elif isinstance(body, (dict, list)):
+                                body_text = json.dumps(body, ensure_ascii=False)
+                            else:
+                                body_text = str(body)
+                            rule_action = self._check_provider_failover_rules(
+                                candidate, is_success=True, response_text=body_text
+                            )
+                            if rule_action == FailoverAction.CONTINUE:
+                                self._record_attempt_failure(
+                                    record_id,
+                                    Exception("success_failover_pattern matched"),
+                                    200,
+                                )
+                                raise StreamProbeError(
+                                    "Success failover pattern matched",
+                                    http_status=200,
+                                )
+
                     self._record_attempt_success(record_id, attempt_result)
 
                     # PRE_EXPAND: mark unused slots after request ends (success)
@@ -598,14 +623,6 @@ class FailoverEngine:
             value = int(retry_policy.max_retries or 1)
         return max(1, value)
 
-    def _should_stop_on_http_error(self, *, status_code: int, error_text: str) -> bool:
-        # follow CandidateService rules
-        if status_code in (401, 403, 429):
-            return False
-        if 400 <= status_code < 500:
-            return self._error_classifier.is_client_error(error_text)
-        return False
-
     async def _handle_error(
         self,
         error: Exception,
@@ -613,29 +630,139 @@ class FailoverEngine:
         candidate: ProviderCandidate,
         has_retry_left: bool,
     ) -> FailoverAction:
-        # Special: HTTP client errors should stop failover.
-        if isinstance(error, httpx.HTTPStatusError):
-            status_code = int(getattr(error.response, "status_code", 0) or 0)
-            try:
-                error_text = error.response.text or ""
-            except Exception:
-                error_text = ""
-            if self._should_stop_on_http_error(status_code=status_code, error_text=error_text):
-                return FailoverAction.STOP
+        # 检查提供商级别的错误终止规则
+        error_text = self._extract_error_text(error)
+        status_code = int(getattr(error, "status_code", 0) or 0) or int(
+            getattr(error, "http_status", 0) or 0
+        )
+        # ExecutionError wrapping: check cause for status_code
+        if not status_code:
+            cause = getattr(error, "cause", None)
+            if cause is not None:
+                status_code = int(getattr(cause, "status_code", 0) or 0) or int(
+                    getattr(cause, "http_status", 0) or 0
+                )
+        if error_text:
+            rule_action = self._check_provider_failover_rules(
+                candidate,
+                is_success=False,
+                response_text=error_text,
+                status_code=status_code or None,
+            )
+            if rule_action is not None:
+                return rule_action
 
-        # Default: reuse legacy ErrorClassifier decision and map to FailoverAction.
+        # 默认全部转移: ErrorClassifier 结果统一映射为 CONTINUE/RETRY，不再 STOP
         action = self._error_classifier.classify(error, has_retry_left=has_retry_left)
-        if action == ErrorAction.RAISE:
-            return FailoverAction.STOP
-        if action == ErrorAction.BREAK:
-            return FailoverAction.CONTINUE
-        return FailoverAction.RETRY
+        if action == ErrorAction.CONTINUE:
+            return FailoverAction.RETRY
+        return FailoverAction.CONTINUE
+
+    def _check_provider_failover_rules(
+        self,
+        candidate: ProviderCandidate,
+        *,
+        is_success: bool,
+        response_text: str,
+        status_code: int | None = None,
+    ) -> FailoverAction | None:
+        """检查提供商级别的故障转移规则。返回 None 表示无规则命中，使用默认行为。"""
+        config = getattr(candidate.provider, "config", None) or {}
+        rules = config.get("failover_rules")
+        if not rules or not isinstance(rules, dict):
+            return None
+
+        compiled = self._get_compiled_patterns(rules)
+
+        if is_success:
+            for regex, rule in compiled.get("success", []):
+                if regex.search(response_text):
+                    logger.info(
+                        "[FailoverEngine] 成功转移规则命中: pattern={}, provider={}",
+                        rule.get("pattern", ""),
+                        candidate.provider.name,
+                    )
+                    return FailoverAction.CONTINUE
+        else:
+            for regex, rule in compiled.get("error", []):
+                # 检查状态码过滤
+                rule_status_codes = rule.get("status_codes")
+                if rule_status_codes and status_code not in rule_status_codes:
+                    continue
+                if regex.search(response_text):
+                    logger.info(
+                        "[FailoverEngine] 错误终止规则命中: pattern={}, status_code={}, provider={}",
+                        rule.get("pattern", ""),
+                        status_code,
+                        candidate.provider.name,
+                    )
+                    return FailoverAction.STOP
+
+        return None
+
+    @staticmethod
+    def _get_compiled_patterns(
+        rules: dict[str, Any],
+    ) -> dict[str, list[tuple[re.Pattern[str], dict[str, Any]]]]:
+        """编译 failover_rules 中的正则模式。
+
+        编译结果缓存在 rules dict 的 _compiled 键上，避免每次请求都重复编译。
+        """
+        cached = rules.get("_compiled")
+        if cached is not None:
+            return cached
+
+        result: dict[str, list[tuple[re.Pattern[str], dict[str, Any]]]] = {
+            "success": [],
+            "error": [],
+        }
+        for rule in rules.get("success_failover_patterns", []):
+            pattern = rule.get("pattern", "")
+            if pattern:
+                try:
+                    result["success"].append((re.compile(pattern), rule))
+                except re.error:
+                    pass
+        for rule in rules.get("error_stop_patterns", []):
+            pattern = rule.get("pattern", "")
+            if pattern:
+                try:
+                    result["error"].append((re.compile(pattern), rule))
+                except re.error:
+                    pass
+        rules["_compiled"] = result
+        return result
+
+    @staticmethod
+    def _extract_error_text(error: Exception) -> str:
+        """从异常中提取错误响应文本。"""
+        # ExecutionError wrapping
+        cause = getattr(error, "cause", None)
+        if cause is not None:
+            error = cause
+
+        # httpx.HTTPStatusError
+        response = getattr(error, "response", None)
+        if response is not None:
+            try:
+                return response.text or ""
+            except Exception:
+                pass
+
+        # upstream_response / upstream_error attribute
+        for attr in ("upstream_response", "upstream_error", "error_message"):
+            val = getattr(error, attr, None)
+            if val:
+                return str(val)
+
+        return str(error)
 
     async def _probe_stream_first_chunk(
         self,
         *,
         attempt_result: AttemptResult,
         record_id: str | None,
+        candidate: ProviderCandidate | None = None,
     ) -> AttemptResult:
         """
         Probe first chunk for a streaming response.
@@ -672,6 +799,22 @@ class FailoverEngine:
                 original_exception=exc,
             ) from exc
 
+        # Check success_failover_patterns on first chunk
+        if candidate is not None and first_chunk:
+            chunk_text = (
+                first_chunk.decode("utf-8", errors="replace")
+                if isinstance(first_chunk, bytes)
+                else str(first_chunk)
+            )
+            rule_action = self._check_provider_failover_rules(
+                candidate, is_success=True, response_text=chunk_text
+            )
+            if rule_action == FailoverAction.CONTINUE:
+                raise StreamProbeError(
+                    "Success failover pattern matched in first chunk",
+                    http_status=attempt_result.http_status,
+                )
+
         wrapped = self._wrap_stream_with_finalizer(
             first_chunk=first_chunk,
             original_iterator=original_iterator,

src/services/orchestration/error_classifier.py

@@ -384,6 +384,8 @@ class ErrorClassifier:
         """
         分类错误，返回处理动作
 
+        默认全部转移策略: 不再返回 RAISE，所有错误都允许故障转移
+
         Args:
             error: 异常对象
             has_retry_left: 当前候选是否还有重试次数
@@ -404,11 +406,8 @@ class ErrorClassifier:
         if isinstance(error, self.RETRIABLE_ERRORS):
             return ErrorAction.CONTINUE if has_retry_left else ErrorAction.BREAK
 
-        if isinstance(error, self.NON_RETRIABLE_ERRORS):
-            return ErrorAction.RAISE
-
-        # 未知错误，直接抛出
-        return ErrorAction.RAISE
+        # 所有其他错误: 不再 RAISE，改为 BREAK（跳到下一个候选继续转移）
+        return ErrorAction.BREAK
 
     async def handle_rate_limit(
         self,

src/services/proxy_node/health_scheduler.py

@@ -1,10 +1,9 @@
 """
 ProxyNode 心跳检测调度器
 
-定期检查 proxy_nodes 的 tunnel 连接状态，更新节点状态：
-- tunnel 实际连接中  -> ONLINE
-- tunnel 未连接      -> OFFLINE
-以 TunnelManager 内存中的实际连接状态为准。
+定期检查 proxy_nodes 的连接健康状态，更新节点状态：
+- 本地 TunnelManager 观测到连接 -> ONLINE（自愈）
+- 心跳超时（跨 worker 共享信号） -> OFFLINE
 """
 
 from __future__ import annotations
@@ -21,6 +20,28 @@ from src.services.system.scheduler import get_scheduler
 _EVENT_RETENTION_DAYS = 30
 # 每隔多少次心跳检测执行一次事件清理（15s * 240 = 1h）
 _EVENT_CLEANUP_INTERVAL = 240
+# 心跳超时判定：max(90s, heartbeat_interval * 3)
+HEARTBEAT_STALE_MIN_SECONDS = 90
+HEARTBEAT_STALE_MULTIPLIER = 3
+
+
+def heartbeat_is_stale(node: object, now: datetime) -> bool:
+    """根据 last_heartbeat_at 判定节点心跳是否超时。
+
+    接受任意具有 last_heartbeat_at / heartbeat_interval 属性的对象，
+    兼容 ProxyNode ORM 实例和在 asyncio.to_thread 中使用的场景。
+    """
+    last_heartbeat = getattr(node, "last_heartbeat_at", None)
+    if not last_heartbeat:
+        return True
+
+    # 兼容 DB 中可能出现的 naive datetime
+    if last_heartbeat.tzinfo is None:
+        last_heartbeat = last_heartbeat.replace(tzinfo=timezone.utc)
+
+    interval = max(int(getattr(node, "heartbeat_interval", None) or 30), 5)
+    stale_seconds = max(HEARTBEAT_STALE_MIN_SECONDS, interval * HEARTBEAT_STALE_MULTIPLIER)
+    return (now - last_heartbeat).total_seconds() > stale_seconds
 
 
 class ProxyNodeHealthScheduler:
@@ -83,27 +104,32 @@ class ProxyNodeHealthScheduler:
 
             changed = 0
             for node in nodes:
-                # 以 TunnelManager 内存中的实际连接状态为准，
-                # 而非仅依赖 DB 的 tunnel_connected 字段。
-                # 服务端重启后 DB 可能残留 tunnel_connected=True，
-                # 但 TunnelManager 内存中已无连接。
-                actually_connected = manager.has_tunnel(node.id)
+                # 注意：TunnelManager 仅是当前 worker 的进程内状态，跨 worker 不共享。
+                # 因此“本地无 tunnel”不能直接判定 OFFLINE（可能连接在其他 worker）。
+                # OFFLINE 统一由心跳超时判定，避免多进程误判。
+                actually_connected_local = manager.has_tunnel(node.id)
 
-                # 同步修正 DB 中不一致的 tunnel_connected 字段
-                if node.tunnel_connected != actually_connected:
-                    node.tunnel_connected = actually_connected
-                    if not actually_connected:
+                if actually_connected_local:
+                    if not node.tunnel_connected:
+                        node.tunnel_connected = True
                         node.tunnel_connected_at = now
-                    changed += 1
+                        changed += 1
+                    if node.status != ProxyNodeStatus.ONLINE:
+                        node.status = ProxyNodeStatus.ONLINE
+                        node.updated_at = now
+                        changed += 1
+                    continue
 
-                new_status = (
-                    ProxyNodeStatus.ONLINE if actually_connected else ProxyNodeStatus.OFFLINE
-                )
-
-                if node.status != new_status:
-                    node.status = new_status
-                    node.updated_at = now
-                    changed += 1
+                # 本地无 tunnel：仅在心跳超时时标记 OFFLINE
+                if heartbeat_is_stale(node, now):
+                    if node.tunnel_connected:
+                        node.tunnel_connected = False
+                        node.tunnel_connected_at = now
+                        changed += 1
+                    if node.status != ProxyNodeStatus.OFFLINE:
+                        node.status = ProxyNodeStatus.OFFLINE
+                        node.updated_at = now
+                        changed += 1
 
             if changed:
                 db.commit()

src/services/proxy_node/service.py

@@ -339,10 +339,11 @@ class ProxyNodeService:
 
         now = datetime.now(timezone.utc)
         # 心跳通过 tunnel 连接传输，能收到心跳说明 tunnel 一定连通。
-        # 如果状态不是 ONLINE（例如 _update_tunnel_status 执行失败），修正状态。
-        if node.status != ProxyNodeStatus.ONLINE:
+        # 如果状态不是 ONLINE 或 tunnel_connected 不一致（例如并发写入覆盖），修正状态。
+        if node.status != ProxyNodeStatus.ONLINE or not node.tunnel_connected:
             node.status = ProxyNodeStatus.ONLINE
             node.tunnel_connected = True
+            node.tunnel_connected_at = now
             node.updated_at = now
         node.last_heartbeat_at = now
         if heartbeat_interval is not None:
@@ -567,7 +568,23 @@ class ProxyNodeService:
                     "exit_ip": None,
                     "error": "tunnel 未连接",
                 }
-            return await _test_tunnel_connectivity(node.id)
+            result = await _test_tunnel_connectivity(node.id)
+
+            # 连通性测试成功但 DB 状态不一致时，修正为 ONLINE
+            if result.get("success") and (
+                node.status != ProxyNodeStatus.ONLINE or not node.tunnel_connected
+            ):
+                node.status = ProxyNodeStatus.ONLINE
+                node.tunnel_connected = True
+                node.tunnel_connected_at = datetime.now(timezone.utc)
+                node.updated_at = node.tunnel_connected_at
+                db.commit()
+
+                from .resolver import invalidate_proxy_node_cache
+
+                invalidate_proxy_node_cache(node.id)
+
+            return result
 
         # 手动节点：通过代理 URL 测试
         try:

src/services/proxy_node/tunnel_manager.py

@@ -278,8 +278,15 @@ class TunnelManager:
         return True
 
     def has_tunnel(self, node_id: str) -> bool:
-        conn = self.get_connection(node_id)
-        return conn is not None
+        """检查指定 node 是否有存活的 tunnel 连接（纯检查，无副作用）
+
+        与 get_connection 不同，此方法不会清理 dead 连接，
+        避免在 finally 块或 health_scheduler 中误清理刚注册的连接。
+        """
+        conns = self._connections.get(node_id)
+        if not conns:
+            return False
+        return any(c.is_alive for c in conns)
 
     def connection_count(self, node_id: str) -> int:
         """返回指定 node 当前存活的连接数"""

src/services/task/service.py

@@ -898,7 +898,7 @@ class TaskService:
                 error_message=str(exec_err),
                 extra_data=_proxy_extra,
             )
-            return "raise"
+            return "break"
 
         provider = candidate.provider
         endpoint = candidate.endpoint
@@ -1034,16 +1034,10 @@ class TaskService:
             embedded_status = cause.error_code or 200
             if error_classifier.is_client_error(error_message):
                 logger.warning(
-                    "  [{}] 嵌入式客户端错误，停止重试: {}",
+                    "  [{}] 嵌入式客户端错误，继续转移: {}",
                     request_id,
                     error_message[:200],
                 )
-                client_error = UpstreamClientException(
-                    message=error_message or "请求无效",
-                    provider_name=str(provider.name),
-                    status_code=embedded_status,
-                    upstream_error=error_message,
-                )
                 RequestCandidateService.mark_candidate_failed(
                     db=self.db,
                     candidate_id=candidate_record_id,
@@ -1054,14 +1048,7 @@ class TaskService:
                     concurrent_requests=captured_key_concurrent,
                     extra_data=_proxy_extra,
                 )
-                client_error.request_metadata = {
-                    "provider": provider.name,
-                    "provider_id": str(provider.id),
-                    "provider_endpoint_id": str(endpoint.id),
-                    "provider_api_key_id": str(key.id),
-                    "api_format": str(api_format),
-                }
-                raise client_error
+                return "break"
 
             logger.warning(
                 "  [{}] 嵌入式服务端错误，尝试重试: {}",
@@ -1124,7 +1111,7 @@ class TaskService:
 
             if isinstance(converted_error, UpstreamClientException):
                 logger.warning(
-                    "  [{}] 客户端请求错误，停止重试: {}",
+                    "  [{}] 客户端请求错误，继续转移: {}",
                     request_id,
                     str(converted_error.message),
                 )
@@ -1138,14 +1125,7 @@ class TaskService:
                     concurrent_requests=captured_key_concurrent,
                     extra_data=serializable_extra_data,
                 )
-                converted_error.request_metadata = {
-                    "provider": provider.name,
-                    "provider_id": str(provider.id),
-                    "provider_endpoint_id": str(endpoint.id),
-                    "provider_api_key_id": str(key.id),
-                    "api_format": str(api_format),
-                }
-                raise converted_error
+                return "break"
 
             RequestCandidateService.mark_candidate_failed(
                 db=self.db,
@@ -1207,7 +1187,7 @@ class TaskService:
             concurrent_requests=captured_key_concurrent,
             extra_data=_proxy_extra,
         )
-        return "raise"
+        return "break"
 
     async def submit_with_failover(
         self,

src/services/usage/consumer_streams.py

@@ -211,13 +211,19 @@ class UsageQueueConsumer:
         await self._process_messages(redis_client, messages)
 
     async def _read_new(self, redis_client: Any) -> None:
-        result = await redis_client.xreadgroup(
-            groupname=self._stream_group,
-            consumername=self._consumer,
-            streams={self._stream_key: ">"},
-            count=self._batch_size,
-            block=self._block_ms,
-        )
+        try:
+            result = await redis_client.xreadgroup(
+                groupname=self._stream_group,
+                consumername=self._consumer,
+                streams={self._stream_key: ">"},
+                count=self._batch_size,
+                block=self._block_ms,
+            )
+        except ResponseError as exc:
+            if "NOGROUP" in str(exc):
+                await ensure_usage_stream_group()
+                return
+            raise
         if not result:
             return
         for _stream, messages in result: