refactor: 拆分职责、引入 dataclass 封装并增强缓存健壮性

- ErrorClassifier 副作用操作分离为 ErrorHandlerService(缓存失效、健康记录、RPM 调整)
- chat_handler_base 提取 ProviderRequestResult dataclass 和 _prepare_provider_request 方法
- failover 提取 AttemptErrorOutcome dataclass 和辅助方法
- formula_engine 拆分 _resolve_mapping 为子方法,增加求值异常日志
- usage recording 引入 UsageCostInfo dataclass 封装成本参数
- 前端 types.ts 拆分为 types/ 子模块
- cache backend 工厂函数加锁防止并发重复创建,LocalCache 容量检查修正
- CacheSync 监听增加断线重连机制,publish 增加重试
- guide 页面修正 useSiteInfo() 调用顺序
This commit is contained in:
fawney19
2026-02-14 16:34:52 +08:00
parent 26ede849e2
commit 6ea33c6bb8
24 changed files with 2005 additions and 1767 deletions

View File

@@ -4,16 +4,19 @@ Orchestration 模块
提供请求编排相关的组件:
- CandidateResolver: 候选解析器,负责获取和排序可用的 Provider 组合
- RequestDispatcher: 请求分发器,负责执行单个候选请求
- ErrorClassifier: 错误分类器,负责错误分类和处理策略
- ErrorClassifier: 错误分类器,负责错误分类(纯逻辑,无副作用)
- ErrorHandlerService: 错误处理服务,负责错误后的副作用(缓存失效、健康记录等)
"""
from .candidate_resolver import CandidateResolver
from .error_classifier import ErrorAction, ErrorClassifier
from .error_handler import ErrorHandlerService
from .request_dispatcher import RequestDispatcher
__all__ = [
"CandidateResolver",
"RequestDispatcher",
"ErrorClassifier",
"ErrorHandlerService",
"ErrorAction",
]

View File

@@ -13,8 +13,6 @@ from typing import Any
import httpx
from sqlalchemy.orm import Session
from src.core.api_format.signature import make_signature_key
from src.core.crypto import CryptoService
from src.core.exceptions import (
ConcurrencyLimitError,
ProviderAuthException,
@@ -28,10 +26,8 @@ from src.core.exceptions import (
from src.core.logger import logger
from src.models.database import Provider, ProviderAPIKey, ProviderEndpoint
from src.services.cache.aware_scheduler import CacheAwareScheduler
from src.services.health.monitor import health_monitor
from src.services.provider.format import normalize_endpoint_signature
from src.services.orchestration.error_handler import ErrorHandlerService
from src.services.rate_limit.adaptive_rpm import get_adaptive_rpm_manager
from src.services.rate_limit.detector import RateLimitType, detect_rate_limit_type
class ErrorAction(Enum):
@@ -117,37 +113,11 @@ class ErrorClassifier:
self.db = db
self.adaptive_manager = adaptive_manager or get_adaptive_rpm_manager()
self.cache_scheduler = cache_scheduler
def _extract_oauth_email(self, key: ProviderAPIKey | None) -> str | None:
if not key or str(getattr(key, "auth_type", "") or "").lower() != "oauth":
return None
encrypted_auth_config = getattr(key, "auth_config", None)
if not encrypted_auth_config:
return None
try:
decrypted = CryptoService().decrypt(encrypted_auth_config, silent=True)
auth_config = json.loads(decrypted) if decrypted else {}
except Exception:
return None
email = auth_config.get("email")
if isinstance(email, str):
email = email.strip()
if email:
return email
return None
def _format_key_display(self, key: ProviderAPIKey | None) -> str:
if not key:
return "key=unknown"
key_id = str(getattr(key, "id", "") or "")[:8] or "unknown"
name = str(getattr(key, "name", "") or "").strip()
email = self._extract_oauth_email(key)
parts = [f"key={key_id}"]
if email:
parts.append(f"email={email}")
if name and name != email:
parts.append(f"name={name}")
return " ".join(parts)
self._error_handler = ErrorHandlerService(
db=db,
adaptive_manager=self.adaptive_manager,
cache_scheduler=cache_scheduler,
)
# 表示客户端错误的 error type不区分大小写
# 这些 type 表明是请求本身的问题,不应重试
@@ -376,38 +346,6 @@ class ErrorClassifier:
search_text = error_text.lower()
return any(p.lower() in search_text for p in self.THINKING_ERROR_PATTERNS)
def _is_account_validation_required(self, error_text: str | None) -> bool:
"""
检测 403 错误是否为 Google 账号验证要求 (VALIDATION_REQUIRED)
Google 会在某些情况下要求账号所有者手动完成人机验证,
此时所有 API 请求都会返回 403 + VALIDATION_REQUIRED。
这是账号级别的永久性错误,重试无法修复,需要人工干预。
匹配条件(满足任一即可):
- error.details 中包含 reason=VALIDATION_REQUIRED
- error.status 为 PERMISSION_DENIED 且 message 包含 "verify your account"
- error.message 包含 "verify your account"
Args:
error_text: 错误响应文本
Returns:
是否为账号验证要求错误
"""
if not error_text:
return False
search_text = error_text.lower()
# 快速路径:关键词匹配
if "validation_required" in search_text:
return True
if "verify your account" in search_text and "permission_denied" in search_text:
return True
return False
def _extract_error_message(self, error_text: str | None) -> str | None:
"""
从错误响应中提取错误消息
@@ -480,66 +418,14 @@ class ErrorClassifier:
exception: ProviderRateLimitException,
request_id: str | None = None,
) -> str:
"""
处理 429 速率限制错误的自适应调整
Args:
key: API Key 对象
provider_name: 提供商名称
current_rpm: 当前分钟内的请求数
exception: 速率限制异常
request_id: 请求 ID用于日志
Returns:
限制类型: "concurrent""rpm""unknown"
"""
try:
# 提取响应头(如果有)
response_headers = {}
if hasattr(exception, "response_headers"):
response_headers = exception.response_headers or {}
# 检测速率限制类型
rate_limit_info = detect_rate_limit_type(
headers=response_headers,
provider_name=provider_name,
current_usage=current_rpm,
)
logger.info(
f" [{request_id}] 429错误分析: "
f"类型={rate_limit_info.limit_type}, "
f"retry_after={rate_limit_info.retry_after}s, "
f"当前RPM={current_rpm}"
)
# 调用自适应管理器处理
new_limit = self.adaptive_manager.handle_429_error(
db=self.db,
key=key,
rate_limit_info=rate_limit_info,
current_rpm=current_rpm,
)
if rate_limit_info.limit_type == RateLimitType.CONCURRENT:
logger.warning(f" [{request_id}] 并发限制触发不调整RPM")
return "concurrent"
elif rate_limit_info.limit_type == RateLimitType.RPM:
if new_limit is not None:
logger.warning(
f" [{request_id}] 自适应调整: Key {key.id[:8]}... RPM限制 -> {new_limit}"
)
else:
logger.info(
f" [{request_id}] 学习中: Key {key.id[:8]}... 观察已记录,暂不设限"
)
return "rpm"
else:
return "unknown"
except Exception as e:
logger.exception(f" [{request_id}] 处理429错误时异常: {e}")
return "unknown"
"""委托给 ErrorHandlerService"""
return await self._error_handler.handle_rate_limit(
key=key,
provider_name=provider_name,
current_rpm=current_rpm,
exception=exception,
request_id=request_id,
)
def convert_http_error(
self,
@@ -650,32 +536,10 @@ class ErrorClassifier:
attempt: int,
max_attempts: int,
) -> dict[str, Any]:
"""
处理 HTTP 错误,返回 extra_data
Args:
http_error: HTTP 状态错误
provider: Provider 对象
endpoint: Endpoint 对象
key: API Key 对象
affinity_key: 亲和性标识符(通常为 API Key ID
api_format: API 格式
global_model_id: GlobalModel ID规范化的模型标识
request_id: 请求 ID
captured_key_concurrent: 捕获的并发数
elapsed_ms: 耗时(毫秒)
attempt: 当前尝试次数
max_attempts: 最大尝试次数
Returns:
Dict[str, Any]: 额外数据,包含:
- error_response: 错误响应文本(如有)
- converted_error: 转换后的异常对象(用于判断是否应该重试)
"""
"""处理 HTTP 错误,返回 extra_data分类 + 委托副作用给 ErrorHandlerService"""
provider_name = str(provider.name)
# 尝试读取错误响应内容
# 优先使用 handler 附加的 upstream_response 属性(流式请求中 response.text 可能为空)
error_response_text = getattr(http_error, "upstream_response", None)
if not error_response_text:
try:
@@ -689,111 +553,35 @@ class ErrorClassifier:
f"{http_error.response.status_code if http_error.response else 'unknown'}"
)
# 分类(纯逻辑)
converted_error = self.convert_http_error(http_error, provider_name, error_response_text)
# 构建 extra_data包含转换后的异常
extra_data: dict[str, Any] = {
"converted_error": converted_error,
}
if error_response_text:
extra_data["error_response"] = error_response_text
# client_format用于缓存亲和性/缓存失效(用户视角)
client_format_str = normalize_endpoint_signature(api_format)
# provider_format用于健康度/熔断 bucketProvider 真实端点格式)
fam = str(getattr(endpoint, "api_family", "")).strip().lower()
kind = str(getattr(endpoint, "endpoint_kind", "")).strip().lower()
provider_format_str = make_signature_key(fam, kind) if fam and kind else client_format_str
# 处理客户端请求错误(不应重试,不失效缓存,不记录健康失败)
if isinstance(converted_error, UpstreamClientException):
logger.warning(
f" [{request_id}] 客户端请求错误,不进行重试: {converted_error.message}"
)
return extra_data
# 处理认证错误
if isinstance(converted_error, ProviderAuthException):
if endpoint and key and self.cache_scheduler is not None:
await self.cache_scheduler.invalidate_cache(
affinity_key=affinity_key,
api_format=client_format_str,
global_model_id=global_model_id,
endpoint_id=str(endpoint.id),
key_id=str(key.id),
)
if key:
health_monitor.record_failure(
db=self.db,
key_id=str(key.id),
api_format=provider_format_str,
error_type="ProviderAuthException",
)
# 403 VALIDATION_REQUIRED → 标记 OAuth key 为账号级别封禁
# 这与 test-model 端点的行为对齐provider_query.py 第 669-690 行)
status_code = http_error.response.status_code if http_error.response else None
if (
status_code == 403
and key
and str(getattr(key, "auth_type", "") or "").lower() == "oauth"
and self._is_account_validation_required(error_response_text)
):
try:
from datetime import datetime, timezone
from src.services.provider.oauth_token import (
OAUTH_ACCOUNT_BLOCK_PREFIX,
)
key.oauth_invalid_at = datetime.now(timezone.utc)
key.oauth_invalid_reason = f"{OAUTH_ACCOUNT_BLOCK_PREFIX}Google 要求验证账号"
key.is_active = False
self.db.commit()
logger.warning(
" [{}] {} 因 403 VALIDATION_REQUIRED 已标记为账号异常并自动停用",
request_id,
self._format_key_display(key),
)
except Exception as mark_exc:
logger.debug(" [{}] 标记 oauth_invalid 失败: {}", request_id, mark_exc)
return extra_data
# 处理限流错误
if isinstance(converted_error, ProviderRateLimitException) and key:
await self.handle_rate_limit(
key=key,
provider_name=provider_name,
current_rpm=captured_key_concurrent,
exception=converted_error,
request_id=request_id,
)
if endpoint and self.cache_scheduler is not None:
await self.cache_scheduler.invalidate_cache(
affinity_key=affinity_key,
api_format=client_format_str,
global_model_id=global_model_id,
endpoint_id=str(endpoint.id),
key_id=str(key.id),
)
else:
# 其他错误也失效缓存
if endpoint and key and self.cache_scheduler is not None:
await self.cache_scheduler.invalidate_cache(
affinity_key=affinity_key,
api_format=client_format_str,
global_model_id=global_model_id,
endpoint_id=str(endpoint.id),
key_id=str(key.id),
)
# 记录健康失败
if key:
health_monitor.record_failure(
db=self.db,
key_id=str(key.id),
api_format=provider_format_str,
error_type=type(converted_error).__name__,
)
# 副作用(委托给 ErrorHandlerService
await self._error_handler.handle_http_error(
http_error,
converted_error,
error_response_text,
provider=provider,
endpoint=endpoint,
key=key,
affinity_key=affinity_key,
api_format=api_format,
global_model_id=global_model_id,
request_id=request_id,
captured_key_concurrent=captured_key_concurrent,
)
return extra_data
@@ -813,69 +601,20 @@ class ErrorClassifier:
attempt: int,
max_attempts: int,
) -> None:
"""
处理可重试错误
Args:
error: 异常对象
provider: Provider 对象
endpoint: Endpoint 对象
key: API Key 对象
affinity_key: 亲和性标识符(通常为 API Key ID
api_format: API 格式
global_model_id: GlobalModel ID规范化的模型标识用于缓存亲和性
captured_key_concurrent: 捕获的并发数
elapsed_ms: 耗时(毫秒)
request_id: 请求 ID
attempt: 当前尝试次数
max_attempts: 最大尝试次数
"""
provider_name = str(provider.name)
"""委托给 ErrorHandlerService"""
logger.warning(
f" [{request_id}] 请求失败 (attempt={attempt}/{max_attempts}): "
f"{type(error).__name__}: {str(error)}"
)
# client_format用于缓存亲和性/缓存失效(用户视角)
client_format_str = normalize_endpoint_signature(api_format)
# provider_format用于健康度/熔断 bucketProvider 真实端点格式)
fam = str(getattr(endpoint, "api_family", "")).strip().lower()
kind = str(getattr(endpoint, "endpoint_kind", "")).strip().lower()
provider_format_str = make_signature_key(fam, kind) if fam and kind else client_format_str
# 处理限流错误
if isinstance(error, ProviderRateLimitException) and key:
await self.handle_rate_limit(
key=key,
provider_name=provider_name,
current_rpm=captured_key_concurrent,
exception=error,
request_id=request_id,
)
if endpoint and self.cache_scheduler is not None:
await self.cache_scheduler.invalidate_cache(
affinity_key=affinity_key,
api_format=client_format_str,
global_model_id=global_model_id,
endpoint_id=str(endpoint.id),
key_id=str(key.id),
)
elif endpoint and key and self.cache_scheduler is not None:
# 其他错误也失效缓存
await self.cache_scheduler.invalidate_cache(
affinity_key=affinity_key,
api_format=client_format_str,
global_model_id=global_model_id,
endpoint_id=str(endpoint.id),
key_id=str(key.id),
)
# 记录健康失败
if key:
health_monitor.record_failure(
db=self.db,
key_id=str(key.id),
api_format=provider_format_str,
error_type=type(error).__name__,
)
await self._error_handler.handle_retriable_error(
error,
provider=provider,
endpoint=endpoint,
key=key,
affinity_key=affinity_key,
api_format=api_format,
global_model_id=global_model_id,
captured_key_concurrent=captured_key_concurrent,
request_id=request_id,
)

View File

@@ -0,0 +1,330 @@
"""
错误处理服务
负责错误发生后的副作用操作缓存失效、健康记录、RPM 调整、OAuth Key 标记等)。
与 ErrorClassifier纯分类无副作用分离遵循单一职责原则。
"""
from __future__ import annotations
import json
from typing import Any
import httpx
from sqlalchemy.orm import Session
from src.core.api_format.signature import make_signature_key
from src.core.crypto import CryptoService
from src.core.exceptions import (
ProviderAuthException,
ProviderRateLimitException,
UpstreamClientException,
)
from src.core.logger import logger
from src.models.database import Provider, ProviderAPIKey, ProviderEndpoint
from src.services.cache.aware_scheduler import CacheAwareScheduler
from src.services.health.monitor import health_monitor
from src.services.provider.format import normalize_endpoint_signature
from src.services.rate_limit.adaptive_rpm import get_adaptive_rpm_manager
from src.services.rate_limit.detector import RateLimitType, detect_rate_limit_type
class ErrorHandlerService:
"""
错误处理服务 - 负责错误发生后的副作用操作
职责:
1. 缓存亲和性失效
2. 健康监控记录
3. 429 自适应 RPM 调整
4. OAuth Key 状态标记
"""
def __init__(
self,
db: Session,
adaptive_manager: Any | None = None,
cache_scheduler: CacheAwareScheduler | None = None,
) -> None:
self.db = db
self.adaptive_manager = adaptive_manager or get_adaptive_rpm_manager()
self.cache_scheduler = cache_scheduler
async def handle_rate_limit(
self,
key: ProviderAPIKey,
provider_name: str,
current_rpm: int | None,
exception: ProviderRateLimitException,
request_id: str | None = None,
) -> str:
"""
处理 429 速率限制错误的自适应调整
Returns:
限制类型: "concurrent""rpm""unknown"
"""
try:
response_headers = {}
if hasattr(exception, "response_headers"):
response_headers = exception.response_headers or {}
rate_limit_info = detect_rate_limit_type(
headers=response_headers,
provider_name=provider_name,
current_usage=current_rpm,
)
logger.info(
" [{}] 429错误分析: 类型={}, retry_after={}s, 当前RPM={}",
request_id,
rate_limit_info.limit_type,
rate_limit_info.retry_after,
current_rpm,
)
new_limit = self.adaptive_manager.handle_429_error(
db=self.db,
key=key,
rate_limit_info=rate_limit_info,
current_rpm=current_rpm,
)
if rate_limit_info.limit_type == RateLimitType.CONCURRENT:
logger.warning(" [{}] 并发限制触发不调整RPM", request_id)
return "concurrent"
elif rate_limit_info.limit_type == RateLimitType.RPM:
if new_limit is not None:
logger.warning(
" [{}] 自适应调整: Key {}... RPM限制 -> {}",
request_id,
str(key.id)[:8],
new_limit,
)
else:
logger.info(
" [{}] 学习中: Key {}... 观察已记录,暂不设限",
request_id,
str(key.id)[:8],
)
return "rpm"
else:
return "unknown"
except Exception as e:
logger.exception(" [{}] 处理429错误时异常: {}", request_id, e)
return "unknown"
async def handle_http_error(
self,
http_error: httpx.HTTPStatusError,
converted_error: Exception,
error_response_text: str | None,
*,
provider: Provider,
endpoint: ProviderEndpoint,
key: ProviderAPIKey,
affinity_key: str,
api_format: str,
global_model_id: str,
request_id: str | None,
captured_key_concurrent: int | None,
) -> None:
"""
处理 HTTP 错误的副作用缓存失效、健康记录、OAuth 标记)。
纯副作用方法:不返回分类结果,不做错误转换。
"""
client_format_str = normalize_endpoint_signature(api_format)
fam = str(getattr(endpoint, "api_family", "")).strip().lower()
kind = str(getattr(endpoint, "endpoint_kind", "")).strip().lower()
provider_format_str = make_signature_key(fam, kind) if fam and kind else client_format_str
# 客户端请求错误:不失效缓存,不记录健康失败
if isinstance(converted_error, UpstreamClientException):
return
can_invalidate = bool(endpoint and key and self.cache_scheduler is not None)
# 认证错误
if isinstance(converted_error, ProviderAuthException):
if can_invalidate:
await self._invalidate_cache(
affinity_key, client_format_str, global_model_id, endpoint, key
)
if key:
health_monitor.record_failure(
db=self.db,
key_id=str(key.id),
api_format=provider_format_str,
error_type="ProviderAuthException",
)
# 403 VALIDATION_REQUIRED -> 标记 OAuth key 为账号级别封禁
status_code = http_error.response.status_code if http_error.response else None
if (
status_code == 403
and key
and str(getattr(key, "auth_type", "") or "").lower() == "oauth"
and self._is_account_validation_required(error_response_text)
):
self._mark_oauth_key_blocked(key, request_id)
return
# 限流错误
if isinstance(converted_error, ProviderRateLimitException) and key:
await self.handle_rate_limit(
key=key,
provider_name=str(provider.name),
current_rpm=captured_key_concurrent,
exception=converted_error,
request_id=request_id,
)
# 所有非客户端错误均失效缓存
if can_invalidate:
await self._invalidate_cache(
affinity_key, client_format_str, global_model_id, endpoint, key
)
# 记录健康失败
if key:
health_monitor.record_failure(
db=self.db,
key_id=str(key.id),
api_format=provider_format_str,
error_type=type(converted_error).__name__,
)
async def handle_retriable_error(
self,
error: Exception,
*,
provider: Provider,
endpoint: ProviderEndpoint,
key: ProviderAPIKey,
affinity_key: str,
api_format: str,
global_model_id: str,
captured_key_concurrent: int | None,
request_id: str | None,
) -> None:
"""处理可重试错误的副作用(缓存失效、健康记录)"""
client_format_str = normalize_endpoint_signature(api_format)
fam = str(getattr(endpoint, "api_family", "")).strip().lower()
kind = str(getattr(endpoint, "endpoint_kind", "")).strip().lower()
provider_format_str = make_signature_key(fam, kind) if fam and kind else client_format_str
# 限流错误
if isinstance(error, ProviderRateLimitException) and key:
await self.handle_rate_limit(
key=key,
provider_name=str(provider.name),
current_rpm=captured_key_concurrent,
exception=error,
request_id=request_id,
)
# 失效缓存
if endpoint and key and self.cache_scheduler is not None:
await self._invalidate_cache(
affinity_key, client_format_str, global_model_id, endpoint, key
)
# 记录健康失败
if key:
health_monitor.record_failure(
db=self.db,
key_id=str(key.id),
api_format=provider_format_str,
error_type=type(error).__name__,
)
async def _invalidate_cache(
self,
affinity_key: str,
api_format: str,
global_model_id: str,
endpoint: ProviderEndpoint,
key: ProviderAPIKey,
) -> None:
"""失效缓存亲和性(调用方需确保 cache_scheduler 可用)"""
assert self.cache_scheduler is not None # noqa: S101
await self.cache_scheduler.invalidate_cache(
affinity_key=affinity_key,
api_format=api_format,
global_model_id=global_model_id,
endpoint_id=str(endpoint.id),
key_id=str(key.id),
)
@staticmethod
def _extract_oauth_email(key: ProviderAPIKey | None) -> str | None:
"""从 OAuth Key 的加密 auth_config 中提取邮箱"""
if not key or str(getattr(key, "auth_type", "") or "").lower() != "oauth":
return None
encrypted_auth_config = getattr(key, "auth_config", None)
if not encrypted_auth_config:
return None
try:
decrypted = CryptoService().decrypt(encrypted_auth_config, silent=True)
auth_config = json.loads(decrypted) if decrypted else {}
except Exception:
return None
email = auth_config.get("email")
if isinstance(email, str):
email = email.strip()
if email:
return email
return None
@classmethod
def _format_key_display(cls, key: ProviderAPIKey | None) -> str:
"""格式化 Key 显示信息(用于日志)"""
if not key:
return "key=unknown"
key_id = str(getattr(key, "id", "") or "")[:8] or "unknown"
name = str(getattr(key, "name", "") or "").strip()
email = cls._extract_oauth_email(key)
parts = [f"key={key_id}"]
if email:
parts.append(f"email={email}")
if name and name != email:
parts.append(f"name={name}")
return " ".join(parts)
@staticmethod
def _is_account_validation_required(error_text: str | None) -> bool:
"""
检测 403 错误是否为 Google 账号验证要求 (VALIDATION_REQUIRED)
匹配条件(满足任一即可):
- error.details 中包含 reason=VALIDATION_REQUIRED
- error.status 为 PERMISSION_DENIED 且 message 包含 "verify your account"
"""
if not error_text:
return False
search_text = error_text.lower()
if "validation_required" in search_text:
return True
if "verify your account" in search_text and "permission_denied" in search_text:
return True
return False
def _mark_oauth_key_blocked(self, key: ProviderAPIKey, request_id: str | None) -> None:
"""标记 OAuth key 为账号级别封禁"""
try:
from datetime import datetime, timezone
from src.services.provider.oauth_token import OAUTH_ACCOUNT_BLOCK_PREFIX
key.oauth_invalid_at = datetime.now(timezone.utc)
key.oauth_invalid_reason = f"{OAUTH_ACCOUNT_BLOCK_PREFIX}Google 要求验证账号"
key.is_active = False
self.db.commit()
logger.warning(
" [{}] {} 因 403 VALIDATION_REQUIRED 已标记为账号异常并自动停用",
request_id,
self._format_key_display(key),
)
except Exception as mark_exc:
logger.debug(" [{}] 标记 oauth_invalid 失败: {}", request_id, mark_exc)