mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-03 01:40:21 +08:00
feat: 新增 Thinking 整流器处理跨 Provider 签名错误 (#115)
当 Provider A 生成的 thinking 块被发送到 Provider B 时,签名验证会失败。 本次更新实现了自动整流机制,在遇到签名错误时自动清洗 thinking 块后重试。 主要更改: - 新增 ThinkingRectifier 整流器,移除 thinking 块和 signature 字段 - 新增 ThinkingSignatureException 异常类型 - ErrorClassifier 新增 Thinking 错误模式检测 - FallbackOrchestrator 支持整流后在当前候选重试 - Handler 层传递 request_body_ref 容器支持请求体动态修改 - Usage API 新增 has_rectified 字段标识整流过的请求 - 新增 THINKING_RECTIFIER_ENABLED 配置项控制功能开关 其他改进: - CacheAwareScheduler 支持 exact/convertible 候选分组排序 - StreamProcessor 预读阶段新增格式转换试验 - ProviderAPIKey.api_formats 改为可空(None 表示支持所有格式) - Dockerfile 修复 entrypoint.sh 换行符问题 Closes #115 Co-Authored-By: FredericMN <FredericMN@users.noreply.github.com>
This commit is contained in:
34
src/services/cache/aware_scheduler.py
vendored
34
src/services/cache/aware_scheduler.py
vendored
@@ -1084,10 +1084,12 @@ class CacheAwareScheduler:
|
||||
continue
|
||||
|
||||
# Key 直属 Provider,通过 api_formats 按端点格式筛选
|
||||
# api_formats=None 视为"全支持"(兼容历史数据)
|
||||
active_keys = [
|
||||
key
|
||||
for key in provider.api_keys
|
||||
if key.is_active and endpoint_format_str in (key.api_formats or [])
|
||||
if key.is_active
|
||||
and (key.api_formats is None or endpoint_format_str in key.api_formats)
|
||||
]
|
||||
if not active_keys:
|
||||
continue
|
||||
@@ -1264,21 +1266,31 @@ class CacheAwareScheduler:
|
||||
"""
|
||||
根据优先级模式对候选列表排序(数字越小越优先)
|
||||
|
||||
- provider: 提供商优先模式,保持原有顺序(按 Provider.provider_priority -> Key.internal_priority 排序,已由查询保证)
|
||||
Key.internal_priority 表示 Endpoint 内部优先级,同优先级内通过哈希分散负载均衡
|
||||
- global_key: 全局 Key 优先模式,按 Key.global_priority_by_format 升序排序(数字小的优先)
|
||||
有优先级的优先,NULL 的排后面
|
||||
同优先级内通过哈希分散实现负载均衡
|
||||
排序规则:
|
||||
1. exact 候选(needs_conversion=False)优先于 convertible 候选
|
||||
2. 在同一类型内,按优先级模式排序:
|
||||
- provider: 提供商优先模式,按 Provider.provider_priority -> Key.internal_priority 排序
|
||||
- global_key: 全局 Key 优先模式,按 Key.global_priority_by_format 排序
|
||||
"""
|
||||
if not candidates:
|
||||
return candidates
|
||||
|
||||
if self.priority_mode == self.PRIORITY_MODE_GLOBAL_KEY:
|
||||
# 全局 Key 优先模式:按 global_priority 分组,同组内哈希分散负载均衡
|
||||
return self._sort_by_global_priority_with_hash(candidates, affinity_key, api_format)
|
||||
# 按 needs_conversion 分组:exact 优先
|
||||
exact_candidates = [c for c in candidates if not c.needs_conversion]
|
||||
convertible_candidates = [c for c in candidates if c.needs_conversion]
|
||||
|
||||
# 提供商优先模式:保持原有顺序(provider_priority 排序已经由查询保证)
|
||||
return candidates
|
||||
if self.priority_mode == self.PRIORITY_MODE_GLOBAL_KEY:
|
||||
# 全局 Key 优先模式:分别对两组排序后合并
|
||||
sorted_exact = self._sort_by_global_priority_with_hash(
|
||||
exact_candidates, affinity_key, api_format
|
||||
)
|
||||
sorted_convertible = self._sort_by_global_priority_with_hash(
|
||||
convertible_candidates, affinity_key, api_format
|
||||
)
|
||||
return sorted_exact + sorted_convertible
|
||||
|
||||
# 提供商优先模式:exact 在前,convertible 在后(各组内部顺序已由构建时保证)
|
||||
return exact_candidates + convertible_candidates
|
||||
|
||||
def _sort_by_global_priority_with_hash(
|
||||
self,
|
||||
|
||||
7
src/services/message/__init__.py
Normal file
7
src/services/message/__init__.py
Normal file
@@ -0,0 +1,7 @@
|
||||
"""
|
||||
消息处理服务模块
|
||||
"""
|
||||
|
||||
from .thinking_rectifier import ThinkingRectifier
|
||||
|
||||
__all__ = ["ThinkingRectifier"]
|
||||
216
src/services/message/thinking_rectifier.py
Normal file
216
src/services/message/thinking_rectifier.py
Normal file
@@ -0,0 +1,216 @@
|
||||
"""
|
||||
Thinking 整流器(Rectifier)
|
||||
|
||||
采用 cc-switch 的"错误触发"模式,在遇到 Thinking 签名/结构错误时触发整流。
|
||||
|
||||
核心功能:
|
||||
1. 移除所有 thinking 和 redacted_thinking 块
|
||||
2. 移除非 thinking 块上的 signature 字段
|
||||
3. 条件删除顶层 thinking 参数
|
||||
|
||||
使用场景:
|
||||
当遇到 ThinkingSignatureException 时,调用 rectify() 整流请求体后重试一次。
|
||||
"""
|
||||
|
||||
import copy
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
from src.core.logger import logger
|
||||
|
||||
|
||||
class ThinkingRectifier:
|
||||
"""
|
||||
Thinking 整流器
|
||||
|
||||
在遇到 Thinking 签名/结构错误时,整流请求体以便重试。
|
||||
采用"彻底清洗 + 条件禁用 thinking"策略。
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def rectify(request_body: Dict[str, Any]) -> Tuple[Dict[str, Any], bool]:
|
||||
"""
|
||||
整流请求体
|
||||
|
||||
执行以下操作:
|
||||
1. 移除所有 thinking 和 redacted_thinking 块
|
||||
2. 移除非 thinking 块上的 signature 字段
|
||||
3. 条件删除顶层 thinking 参数
|
||||
|
||||
Args:
|
||||
request_body: 原始请求体
|
||||
|
||||
Returns:
|
||||
Tuple[整流后的请求体, 是否有修改]
|
||||
"""
|
||||
if not request_body:
|
||||
return request_body, False
|
||||
|
||||
# 深拷贝以避免修改原始数据
|
||||
rectified_body = copy.deepcopy(request_body)
|
||||
modified = False
|
||||
|
||||
# 1. 整流 messages
|
||||
messages = rectified_body.get("messages", [])
|
||||
if messages:
|
||||
rectified_messages, messages_modified = ThinkingRectifier._rectify_messages(messages)
|
||||
if messages_modified:
|
||||
rectified_body["messages"] = rectified_messages
|
||||
modified = True
|
||||
|
||||
# 2. 条件删除顶层 thinking 参数(使用整流后的 messages 判断)
|
||||
# 与 cc-switch 行为一致:在整流 messages 之后获取快照进行判断
|
||||
if ThinkingRectifier._should_remove_top_level_thinking(rectified_body):
|
||||
if "thinking" in rectified_body:
|
||||
del rectified_body["thinking"]
|
||||
modified = True
|
||||
logger.info("ThinkingRectifier: 已移除顶层 thinking 参数")
|
||||
|
||||
return rectified_body, modified
|
||||
|
||||
@staticmethod
|
||||
def _rectify_messages(messages: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], bool]:
|
||||
"""
|
||||
整流消息列表
|
||||
|
||||
移除所有 thinking/redacted_thinking 块和 signature 字段
|
||||
|
||||
Args:
|
||||
messages: 原始消息列表
|
||||
|
||||
Returns:
|
||||
Tuple[整流后的消息列表, 是否有修改]
|
||||
"""
|
||||
if not messages:
|
||||
return messages, False
|
||||
|
||||
modified = False
|
||||
result_messages: List[Dict[str, Any]] = []
|
||||
thinking_removed = 0
|
||||
signature_removed = 0
|
||||
|
||||
for message in messages:
|
||||
# 类型保护:跳过非 dict 消息
|
||||
if not isinstance(message, dict):
|
||||
result_messages.append(message)
|
||||
continue
|
||||
|
||||
# 消息级浅拷贝:外层 rectify() 已深拷贝整个 request_body
|
||||
# content 会被重建为新列表,不会影响原始数据
|
||||
new_message = dict(message)
|
||||
content = message.get("content")
|
||||
|
||||
if isinstance(content, list):
|
||||
new_content = []
|
||||
for block in content:
|
||||
if isinstance(block, dict):
|
||||
block_type = block.get("type")
|
||||
|
||||
# 移除 thinking 和 redacted_thinking 块
|
||||
if block_type in ("thinking", "redacted_thinking"):
|
||||
thinking_removed += 1
|
||||
modified = True
|
||||
continue
|
||||
|
||||
# 移除非 thinking 块上的 signature 字段
|
||||
if "signature" in block:
|
||||
new_block = {k: v for k, v in block.items() if k != "signature"}
|
||||
new_content.append(new_block)
|
||||
signature_removed += 1
|
||||
modified = True
|
||||
continue
|
||||
|
||||
new_content.append(block)
|
||||
else:
|
||||
new_content.append(block)
|
||||
|
||||
# 更新 content
|
||||
new_message["content"] = new_content
|
||||
|
||||
# 如果整流后 assistant 消息的 content 为空,记录警告
|
||||
# (空 content 本身不是"修改",只是检测到的状态,不设置 modified)
|
||||
# 保留消息是必要的:跳过会破坏对话结构(后续 tool_result 消息需要前置 assistant 消息)
|
||||
if new_message.get("role") == "assistant":
|
||||
effective_content = new_message.get("content")
|
||||
is_empty = not effective_content or (
|
||||
isinstance(effective_content, list) and len(effective_content) == 0
|
||||
)
|
||||
if is_empty:
|
||||
msg_idx = len(result_messages)
|
||||
logger.warning(
|
||||
f"ThinkingRectifier: assistant 消息整流后 content 为空 (message_index={msg_idx})"
|
||||
)
|
||||
|
||||
result_messages.append(new_message)
|
||||
|
||||
if thinking_removed > 0 or signature_removed > 0:
|
||||
logger.info(
|
||||
f"ThinkingRectifier: 移除了 {thinking_removed} 个 thinking 块, "
|
||||
f"{signature_removed} 个 signature 字段"
|
||||
)
|
||||
|
||||
return result_messages, modified
|
||||
|
||||
@staticmethod
|
||||
def _should_remove_top_level_thinking(body: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
判断是否应该删除顶层 thinking 参数
|
||||
|
||||
与 cc-switch 行为一致:只检查最后一条 assistant 消息
|
||||
|
||||
设计思路:
|
||||
- body 中的 messages 是整流后的状态,thinking 块已被移除
|
||||
- Claude API 只校验最后一条 assistant 消息的结构
|
||||
- 如果最后一条有 tool_use 但首块不是 thinking,需要禁用 thinking 参数
|
||||
|
||||
Args:
|
||||
body: 整流后的请求体
|
||||
|
||||
Returns:
|
||||
是否应该删除顶层 thinking 参数
|
||||
"""
|
||||
# 条件 1: thinking 参数存在且已启用
|
||||
thinking_param = body.get("thinking")
|
||||
if not isinstance(thinking_param, dict) or thinking_param.get("type") != "enabled":
|
||||
return False
|
||||
|
||||
# 从 body 中获取 messages
|
||||
messages = body.get("messages", [])
|
||||
|
||||
# 类型保护:确保 messages 是 list
|
||||
if not isinstance(messages, list) or not messages:
|
||||
return False
|
||||
|
||||
# 条件 2: 找到最后一条 assistant 消息
|
||||
last_assistant = None
|
||||
for message in reversed(messages):
|
||||
if isinstance(message, dict) and message.get("role") == "assistant":
|
||||
last_assistant = message
|
||||
break
|
||||
|
||||
if not last_assistant:
|
||||
return False
|
||||
|
||||
content = last_assistant.get("content")
|
||||
if not isinstance(content, list) or not content:
|
||||
return False
|
||||
|
||||
# 注意:传入的 messages 是整流后的状态,thinking 块已被移除
|
||||
# 因此只需检查是否有 tool_use,如果有则需要禁用 thinking 参数
|
||||
# (因为整流后的 assistant 消息不再以 thinking 块开头)
|
||||
|
||||
# 检查是否有 tool_use
|
||||
has_tool_use = any(
|
||||
isinstance(block, dict) and block.get("type") == "tool_use" for block in content
|
||||
)
|
||||
|
||||
# 整流后 assistant 消息不再以 thinking 块开头,如果有 tool_use 则需要禁用 thinking 参数
|
||||
# (Claude API 要求:启用 thinking 时,有 tool_use 的 assistant 消息必须以 thinking 块开头)
|
||||
if has_tool_use:
|
||||
logger.info(
|
||||
"ThinkingRectifier: 整流后 assistant 消息有 tool_use 但无 thinking 前缀,"
|
||||
"禁用 thinking 参数以通过 API 校验"
|
||||
)
|
||||
return True
|
||||
|
||||
logger.debug("ThinkingRectifier: 整流后 assistant 消息无 tool_use,保留 thinking 参数")
|
||||
return False
|
||||
@@ -19,6 +19,7 @@ from src.core.exceptions import (
|
||||
ProviderException,
|
||||
ProviderNotAvailableException,
|
||||
ProviderRateLimitException,
|
||||
ThinkingSignatureException,
|
||||
UpstreamClientException,
|
||||
)
|
||||
from src.core.logger import logger
|
||||
@@ -151,6 +152,30 @@ class ErrorClassifier:
|
||||
"not available for this model", # 此模型不可用
|
||||
)
|
||||
|
||||
# Thinking 块相关错误模式 - 这类错误需要清洗 thinking 块或调整请求
|
||||
# 场景:多供应商环境下,Provider A 生成的 thinking 块被发送到 Provider B 时签名验证失败
|
||||
THINKING_ERROR_PATTERNS: Tuple[str, ...] = (
|
||||
# 签名错误:跨 Provider 发送 thinking 块时,签名无法被目标 Provider 验证
|
||||
# 例: "invalid `signature` in `thinking` block: signature is for a different request"
|
||||
"invalid `signature` in `thinking` block",
|
||||
"invalid signature in thinking block",
|
||||
# 签名字段缺失或格式错误
|
||||
# 例: "messages.0.content.0.thinking.signature: field required"
|
||||
"thinking.signature: field required",
|
||||
"thinking.signature:", # 匹配路径模式 messages.X.content.X.thinking.signature: xxx
|
||||
"signature verification failed",
|
||||
# 结构错误:启用 thinking 时,有 tool_use 的 assistant 消息必须以 thinking 块开头
|
||||
# 例: "when `thinking` is enabled, the first content block ... must start with a `thinking` block"
|
||||
"must start with a thinking block",
|
||||
# 例: "expected thinking or redacted_thinking, found tool_use"
|
||||
"expected thinking or redacted_thinking",
|
||||
"expected `thinking`",
|
||||
"expected thinking, found", # 统一匹配 "found tool_use/text" 等变体
|
||||
"expected `thinking`, found", # 带反引号变体
|
||||
"expected redacted_thinking, found",
|
||||
"expected `redacted_thinking`, found",
|
||||
)
|
||||
|
||||
def _parse_error_response(self, error_text: Optional[str]) -> Dict[str, Any]:
|
||||
"""
|
||||
解析错误响应为结构化数据
|
||||
@@ -295,6 +320,25 @@ class ErrorClassifier:
|
||||
search_text = error_text.lower()
|
||||
return any(pattern.lower() in search_text for pattern in self.COMPATIBILITY_ERROR_PATTERNS)
|
||||
|
||||
def _is_thinking_error(self, error_text: Optional[str]) -> bool:
|
||||
"""
|
||||
检测错误响应是否为 Thinking 块相关错误(签名错误或结构错误)
|
||||
|
||||
这类错误通常发生在:
|
||||
1. 多供应商场景下,当一个供应商生成的 thinking 块被发送到另一个供应商时,签名验证会失败
|
||||
2. 请求体中有 tool_use 但没有以 thinking 块开头时,Claude 会报结构错误
|
||||
|
||||
Args:
|
||||
error_text: 错误响应文本
|
||||
|
||||
Returns:
|
||||
是否为 Thinking 相关错误
|
||||
"""
|
||||
if not error_text:
|
||||
return False
|
||||
search_text = error_text.lower()
|
||||
return any(p.lower() in search_text for p in self.THINKING_ERROR_PATTERNS)
|
||||
|
||||
def _extract_error_message(self, error_text: Optional[str]) -> Optional[str]:
|
||||
"""
|
||||
从错误响应中提取错误消息
|
||||
@@ -463,6 +507,15 @@ class ErrorClassifier:
|
||||
),
|
||||
)
|
||||
|
||||
# 400 错误:检查是否为 Thinking 块签名错误
|
||||
if status == 400 and self._is_thinking_error(error_response_text):
|
||||
logger.info(f"检测到 Thinking 块错误: {extracted_message}")
|
||||
return ThinkingSignatureException(
|
||||
message=extracted_message or "Thinking block signature validation failed",
|
||||
provider_name=provider_name,
|
||||
upstream_error=error_response_text,
|
||||
)
|
||||
|
||||
# 400 错误:先检查是否为 Provider 兼容性错误(应触发故障转移)
|
||||
if status == 400 and self._is_compatibility_error(error_response_text):
|
||||
logger.info(f"检测到 Provider 兼容性错误,将触发故障转移: {extracted_message}")
|
||||
|
||||
@@ -29,12 +29,14 @@ import httpx
|
||||
from redis import Redis
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from src.config.settings import config
|
||||
from src.core.api_format import APIFormat, FormatConversionError
|
||||
from src.core.error_utils import extract_error_message
|
||||
from src.core.exceptions import (
|
||||
ConcurrencyLimitError,
|
||||
EmbeddedErrorException,
|
||||
ProviderNotAvailableException,
|
||||
ThinkingSignatureException,
|
||||
UpstreamClientException,
|
||||
)
|
||||
from src.core.logger import logger
|
||||
@@ -44,6 +46,7 @@ from src.services.cache.aware_scheduler import (
|
||||
ProviderCandidate,
|
||||
get_cache_aware_scheduler,
|
||||
)
|
||||
from src.services.message.thinking_rectifier import ThinkingRectifier
|
||||
from src.services.provider.format import normalize_api_format
|
||||
from src.services.rate_limit.adaptive_rpm import get_adaptive_rpm_manager
|
||||
from src.services.rate_limit.concurrency_manager import get_concurrency_manager
|
||||
@@ -298,6 +301,114 @@ class FallbackOrchestrator:
|
||||
is_stream=is_stream,
|
||||
)
|
||||
|
||||
def _handle_thinking_signature_error(
|
||||
self,
|
||||
converted_error: ThinkingSignatureException,
|
||||
request_id: Optional[str],
|
||||
candidate_record_id: str,
|
||||
elapsed_ms: int,
|
||||
captured_key_concurrent: Optional[int],
|
||||
serializable_extra_data: Dict[str, Any],
|
||||
request_body_ref: Optional[Dict[str, Any]],
|
||||
) -> str:
|
||||
"""
|
||||
处理 ThinkingSignatureException 错误
|
||||
|
||||
尝试整流请求体后重试。如果无法整流或整流后仍失败,则抛出异常。
|
||||
|
||||
Args:
|
||||
converted_error: Thinking 签名异常
|
||||
request_id: 请求 ID
|
||||
candidate_record_id: 候选记录 ID
|
||||
elapsed_ms: 耗时(毫秒)
|
||||
captured_key_concurrent: 捕获的并发数
|
||||
serializable_extra_data: 可序列化的额外数据
|
||||
request_body_ref: 请求体引用容器
|
||||
|
||||
Returns:
|
||||
"continue" 表示整流成功应继续重试
|
||||
|
||||
Raises:
|
||||
ThinkingSignatureException: 无法整流或整流后仍失败时
|
||||
"""
|
||||
# 检查整流器是否启用
|
||||
if not config.thinking_rectifier_enabled:
|
||||
logger.info(f" [{request_id}] Thinking 错误:整流器已禁用,终止重试")
|
||||
self._mark_thinking_error_failed(
|
||||
candidate_record_id, converted_error, elapsed_ms,
|
||||
captured_key_concurrent, serializable_extra_data
|
||||
)
|
||||
raise converted_error
|
||||
|
||||
# 检查是否有请求体引用(由 Handler 层传入)
|
||||
if request_body_ref is None:
|
||||
logger.warning(f" [{request_id}] Thinking 错误:无法获取请求体引用,终止重试")
|
||||
self._mark_thinking_error_failed(
|
||||
candidate_record_id, converted_error, elapsed_ms,
|
||||
captured_key_concurrent, serializable_extra_data
|
||||
)
|
||||
raise converted_error
|
||||
|
||||
# 检查是否已整流过(避免无限循环,单次重试)
|
||||
if request_body_ref.get("_rectified", False):
|
||||
logger.warning(f" [{request_id}] Thinking 错误:已整流仍失败,终止重试")
|
||||
self._mark_thinking_error_failed(
|
||||
candidate_record_id, converted_error, elapsed_ms,
|
||||
captured_key_concurrent, {**serializable_extra_data, "rectified": True}
|
||||
)
|
||||
raise converted_error
|
||||
|
||||
# 使用整流器
|
||||
request_body = request_body_ref.get("body", {})
|
||||
rectified_body, modified = ThinkingRectifier.rectify(request_body)
|
||||
|
||||
if modified:
|
||||
# 更新容器中的请求体
|
||||
request_body_ref["body"] = rectified_body
|
||||
# _rectified: 全局标记,防止重复整流(整流只执行一次)
|
||||
request_body_ref["_rectified"] = True
|
||||
# _rectified_this_turn: 单轮标记,用于在当前 candidate 扩展重试次数
|
||||
request_body_ref["_rectified_this_turn"] = True
|
||||
|
||||
logger.info(f" [{request_id}] 请求已整流,在当前候选上重试")
|
||||
|
||||
# 标记当前尝试为失败(整流前的状态)
|
||||
# 注意:整流后重试会复用此记录 ID,成功时会覆盖为 success 状态
|
||||
self._mark_thinking_error_failed(
|
||||
candidate_record_id, converted_error, elapsed_ms,
|
||||
captured_key_concurrent, {**serializable_extra_data, "rectified": True}
|
||||
)
|
||||
|
||||
# 返回 continue:在当前候选的重试循环中继续,使用整流后的请求体重试
|
||||
return "continue"
|
||||
else:
|
||||
logger.warning(f" [{request_id}] Thinking 错误:无可整流内容")
|
||||
self._mark_thinking_error_failed(
|
||||
candidate_record_id, converted_error, elapsed_ms,
|
||||
captured_key_concurrent, serializable_extra_data
|
||||
)
|
||||
raise converted_error
|
||||
|
||||
def _mark_thinking_error_failed(
|
||||
self,
|
||||
candidate_record_id: str,
|
||||
error: ThinkingSignatureException,
|
||||
elapsed_ms: int,
|
||||
captured_key_concurrent: Optional[int],
|
||||
extra_data: Dict[str, Any],
|
||||
) -> None:
|
||||
"""标记 Thinking 签名错误导致的候选失败"""
|
||||
RequestCandidateService.mark_candidate_failed(
|
||||
db=self.db,
|
||||
candidate_id=candidate_record_id,
|
||||
error_type="ThinkingSignatureException",
|
||||
error_message=str(error),
|
||||
status_code=400,
|
||||
latency_ms=elapsed_ms,
|
||||
concurrent_requests=captured_key_concurrent,
|
||||
extra_data=extra_data,
|
||||
)
|
||||
|
||||
async def _handle_candidate_error(
|
||||
self,
|
||||
exec_err: ExecutionError,
|
||||
@@ -311,6 +422,7 @@ class FallbackOrchestrator:
|
||||
request_id: Optional[str],
|
||||
attempt: int,
|
||||
max_attempts: int,
|
||||
request_body_ref: Optional[Dict[str, Any]] = None,
|
||||
) -> str:
|
||||
"""
|
||||
处理候选执行错误
|
||||
@@ -327,6 +439,7 @@ class FallbackOrchestrator:
|
||||
request_id: 请求 ID
|
||||
attempt: 当前尝试次数
|
||||
max_attempts: 最大尝试次数
|
||||
request_body_ref: 请求体引用容器(用于 Thinking 签名错误重试)
|
||||
|
||||
Returns:
|
||||
action: "continue" (继续重试), "break" (跳到下一个候选), "raise" (抛出异常)
|
||||
@@ -431,6 +544,22 @@ class FallbackOrchestrator:
|
||||
k: v for k, v in extra_data.items() if k != "converted_error"
|
||||
}
|
||||
|
||||
# 先检查 ThinkingSignatureException(它继承自 UpstreamClientException)
|
||||
# 签名/结构错误需要特殊处理:整流请求体后重试一次
|
||||
if isinstance(converted_error, ThinkingSignatureException):
|
||||
action = self._handle_thinking_signature_error(
|
||||
converted_error=converted_error,
|
||||
request_id=request_id,
|
||||
candidate_record_id=candidate_record_id,
|
||||
elapsed_ms=elapsed_ms,
|
||||
captured_key_concurrent=captured_key_concurrent,
|
||||
serializable_extra_data=serializable_extra_data,
|
||||
request_body_ref=request_body_ref,
|
||||
)
|
||||
if action == "continue":
|
||||
return "continue"
|
||||
# action == "raise" 时已在方法内部 raise
|
||||
|
||||
if isinstance(converted_error, UpstreamClientException):
|
||||
logger.warning(
|
||||
f" [{request_id}] 客户端请求错误,停止重试: {converted_error.message}"
|
||||
@@ -562,6 +691,7 @@ class FallbackOrchestrator:
|
||||
affinity_key: str,
|
||||
global_model_id: str,
|
||||
is_stream: bool = False,
|
||||
request_body_ref: Optional[Dict[str, Any]] = None,
|
||||
) -> Tuple[Any, str, Optional[str], Optional[str], Optional[str], Optional[str]]:
|
||||
"""遍历所有候选执行请求,返回第一个成功的结果或抛出异常"""
|
||||
attempt_counter = 0
|
||||
@@ -593,6 +723,7 @@ class FallbackOrchestrator:
|
||||
attempt_counter=attempt_counter,
|
||||
max_attempts=max_attempts,
|
||||
is_stream=is_stream,
|
||||
request_body_ref=request_body_ref,
|
||||
)
|
||||
|
||||
if result["success"]:
|
||||
@@ -632,6 +763,7 @@ class FallbackOrchestrator:
|
||||
attempt_counter: int,
|
||||
max_attempts: int,
|
||||
is_stream: bool = False,
|
||||
request_body_ref: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""尝试单个候选(含重试逻辑),返回执行结果"""
|
||||
provider = candidate.provider
|
||||
@@ -640,7 +772,8 @@ class FallbackOrchestrator:
|
||||
max_retries_for_candidate = int(provider.max_retries or 2) if candidate.is_cached else 1
|
||||
last_error: Optional[Exception] = None
|
||||
|
||||
for retry_index in range(max_retries_for_candidate):
|
||||
retry_index = 0
|
||||
while retry_index < max_retries_for_candidate:
|
||||
attempt_counter += 1
|
||||
max_attempts = max(max_attempts, attempt_counter)
|
||||
|
||||
@@ -655,7 +788,20 @@ class FallbackOrchestrator:
|
||||
f" [{request_id[:8] if request_id else 'N/A'}] -> {provider.name} (retry {retry_index})"
|
||||
)
|
||||
|
||||
candidate_record_id = candidate_record_map[(candidate_index, retry_index)]
|
||||
# 获取候选记录 ID
|
||||
# 正常情况下 record_key = (candidate_index, retry_index)
|
||||
# 整流重试时 retry_index 可能超出预创建范围,复用最后一个有效记录
|
||||
record_key = (candidate_index, retry_index)
|
||||
if record_key not in candidate_record_map:
|
||||
# 整流重试:复用该候选的最后一个有效记录(通常是 retry_index=0)
|
||||
# 这样整流后的成功/失败会覆盖之前的记录状态
|
||||
fallback_key = (candidate_index, 0)
|
||||
candidate_record_id = candidate_record_map.get(fallback_key, "")
|
||||
logger.debug(
|
||||
f" [{request_id}] 整流重试:复用记录 {fallback_key} -> {candidate_record_id[:8] if candidate_record_id else 'N/A'}"
|
||||
)
|
||||
else:
|
||||
candidate_record_id = candidate_record_map[record_key]
|
||||
|
||||
try:
|
||||
response = await self._try_single_candidate(
|
||||
@@ -690,9 +836,20 @@ class FallbackOrchestrator:
|
||||
request_id=request_id,
|
||||
attempt=attempt_counter,
|
||||
max_attempts=max_attempts,
|
||||
request_body_ref=request_body_ref,
|
||||
)
|
||||
|
||||
if action == "continue":
|
||||
# 检查是否刚完成整流,需要额外重试一次
|
||||
if request_body_ref and request_body_ref.get("_rectified_this_turn", False):
|
||||
# 清除标记,扩展重试上限以允许整流后的请求发出
|
||||
# 使用 max() 确保不会减少已有的重试次数
|
||||
request_body_ref["_rectified_this_turn"] = False
|
||||
max_retries_for_candidate = max(max_retries_for_candidate, retry_index + 2)
|
||||
logger.debug(
|
||||
f" [{request_id}] 整流后扩展重试次数至 {max_retries_for_candidate}"
|
||||
)
|
||||
retry_index += 1
|
||||
continue
|
||||
elif action == "break":
|
||||
break
|
||||
@@ -704,6 +861,10 @@ class FallbackOrchestrator:
|
||||
"attempt_counter": attempt_counter,
|
||||
"max_attempts": max_attempts,
|
||||
}
|
||||
else:
|
||||
# 未知 action,安全起见跳出循环
|
||||
logger.warning(f" [{request_id}] 未知 action: {action},跳出重试")
|
||||
break
|
||||
|
||||
return {
|
||||
"success": False,
|
||||
@@ -826,6 +987,7 @@ class FallbackOrchestrator:
|
||||
request_id: Optional[str] = None,
|
||||
is_stream: bool = False,
|
||||
capability_requirements: Optional[Dict[str, bool]] = None,
|
||||
request_body_ref: Optional[Dict[str, Any]] = None,
|
||||
) -> Tuple[Any, str, Optional[str], Optional[str], Optional[str], Optional[str]]:
|
||||
"""
|
||||
执行请求,并在失败时自动故障转移(缓存感知)
|
||||
@@ -838,6 +1000,7 @@ class FallbackOrchestrator:
|
||||
request_id: 请求 ID(用于日志)
|
||||
is_stream: 是否是流式请求,如果为 True 则过滤不支持流式的 Provider
|
||||
capability_requirements: 能力需求(用于过滤不满足能力要求的 Key)
|
||||
request_body_ref: 请求体引用容器(用于 Thinking 签名错误重试)
|
||||
|
||||
Returns:
|
||||
(请求响应, 实际Provider名称, RequestTraceAttempt ID, provider_id, endpoint_id, key_id)
|
||||
@@ -895,4 +1058,5 @@ class FallbackOrchestrator:
|
||||
affinity_key=affinity_key,
|
||||
global_model_id=global_model_id,
|
||||
is_stream=is_stream,
|
||||
request_body_ref=request_body_ref,
|
||||
)
|
||||
|
||||
@@ -165,6 +165,9 @@ class RequestCandidateService:
|
||||
candidate.latency_ms = latency_ms
|
||||
candidate.concurrent_requests = concurrent_requests
|
||||
candidate.finished_at = datetime.now(timezone.utc)
|
||||
# 成功时清空错误字段(可能是整流重试后成功,之前记录过错误)
|
||||
candidate.error_type = None
|
||||
candidate.error_message = None
|
||||
if extra_data:
|
||||
candidate.extra_data = {**(candidate.extra_data or {}), **extra_data}
|
||||
# 关键状态更新:立即提交,不使用批量提交
|
||||
|
||||
Reference in New Issue
Block a user