mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 17:30:23 +08:00
refactor: 拆分职责、引入 dataclass 封装并增强缓存健壮性
- ErrorClassifier 副作用操作分离为 ErrorHandlerService(缓存失效、健康记录、RPM 调整) - chat_handler_base 提取 ProviderRequestResult dataclass 和 _prepare_provider_request 方法 - failover 提取 AttemptErrorOutcome dataclass 和辅助方法 - formula_engine 拆分 _resolve_mapping 为子方法,增加求值异常日志 - usage recording 引入 UsageCostInfo dataclass 封装成本参数 - 前端 types.ts 拆分为 types/ 子模块 - cache backend 工厂函数加锁防止并发重复创建,LocalCache 容量检查修正 - CacheSync 监听增加断线重连机制,publish 增加重试 - guide 页面修正 useSiteInfo() 调用顺序
This commit is contained in:
@@ -4,16 +4,19 @@ Orchestration 模块
|
||||
提供请求编排相关的组件:
|
||||
- CandidateResolver: 候选解析器,负责获取和排序可用的 Provider 组合
|
||||
- RequestDispatcher: 请求分发器,负责执行单个候选请求
|
||||
- ErrorClassifier: 错误分类器,负责错误分类和处理策略
|
||||
- ErrorClassifier: 错误分类器,负责错误分类(纯逻辑,无副作用)
|
||||
- ErrorHandlerService: 错误处理服务,负责错误后的副作用(缓存失效、健康记录等)
|
||||
"""
|
||||
|
||||
from .candidate_resolver import CandidateResolver
|
||||
from .error_classifier import ErrorAction, ErrorClassifier
|
||||
from .error_handler import ErrorHandlerService
|
||||
from .request_dispatcher import RequestDispatcher
|
||||
|
||||
__all__ = [
|
||||
"CandidateResolver",
|
||||
"RequestDispatcher",
|
||||
"ErrorClassifier",
|
||||
"ErrorHandlerService",
|
||||
"ErrorAction",
|
||||
]
|
||||
|
||||
@@ -13,8 +13,6 @@ from typing import Any
|
||||
import httpx
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from src.core.api_format.signature import make_signature_key
|
||||
from src.core.crypto import CryptoService
|
||||
from src.core.exceptions import (
|
||||
ConcurrencyLimitError,
|
||||
ProviderAuthException,
|
||||
@@ -28,10 +26,8 @@ from src.core.exceptions import (
|
||||
from src.core.logger import logger
|
||||
from src.models.database import Provider, ProviderAPIKey, ProviderEndpoint
|
||||
from src.services.cache.aware_scheduler import CacheAwareScheduler
|
||||
from src.services.health.monitor import health_monitor
|
||||
from src.services.provider.format import normalize_endpoint_signature
|
||||
from src.services.orchestration.error_handler import ErrorHandlerService
|
||||
from src.services.rate_limit.adaptive_rpm import get_adaptive_rpm_manager
|
||||
from src.services.rate_limit.detector import RateLimitType, detect_rate_limit_type
|
||||
|
||||
|
||||
class ErrorAction(Enum):
|
||||
@@ -117,37 +113,11 @@ class ErrorClassifier:
|
||||
self.db = db
|
||||
self.adaptive_manager = adaptive_manager or get_adaptive_rpm_manager()
|
||||
self.cache_scheduler = cache_scheduler
|
||||
|
||||
def _extract_oauth_email(self, key: ProviderAPIKey | None) -> str | None:
|
||||
if not key or str(getattr(key, "auth_type", "") or "").lower() != "oauth":
|
||||
return None
|
||||
encrypted_auth_config = getattr(key, "auth_config", None)
|
||||
if not encrypted_auth_config:
|
||||
return None
|
||||
try:
|
||||
decrypted = CryptoService().decrypt(encrypted_auth_config, silent=True)
|
||||
auth_config = json.loads(decrypted) if decrypted else {}
|
||||
except Exception:
|
||||
return None
|
||||
email = auth_config.get("email")
|
||||
if isinstance(email, str):
|
||||
email = email.strip()
|
||||
if email:
|
||||
return email
|
||||
return None
|
||||
|
||||
def _format_key_display(self, key: ProviderAPIKey | None) -> str:
|
||||
if not key:
|
||||
return "key=unknown"
|
||||
key_id = str(getattr(key, "id", "") or "")[:8] or "unknown"
|
||||
name = str(getattr(key, "name", "") or "").strip()
|
||||
email = self._extract_oauth_email(key)
|
||||
parts = [f"key={key_id}"]
|
||||
if email:
|
||||
parts.append(f"email={email}")
|
||||
if name and name != email:
|
||||
parts.append(f"name={name}")
|
||||
return " ".join(parts)
|
||||
self._error_handler = ErrorHandlerService(
|
||||
db=db,
|
||||
adaptive_manager=self.adaptive_manager,
|
||||
cache_scheduler=cache_scheduler,
|
||||
)
|
||||
|
||||
# 表示客户端错误的 error type(不区分大小写)
|
||||
# 这些 type 表明是请求本身的问题,不应重试
|
||||
@@ -376,38 +346,6 @@ class ErrorClassifier:
|
||||
search_text = error_text.lower()
|
||||
return any(p.lower() in search_text for p in self.THINKING_ERROR_PATTERNS)
|
||||
|
||||
def _is_account_validation_required(self, error_text: str | None) -> bool:
|
||||
"""
|
||||
检测 403 错误是否为 Google 账号验证要求 (VALIDATION_REQUIRED)
|
||||
|
||||
Google 会在某些情况下要求账号所有者手动完成人机验证,
|
||||
此时所有 API 请求都会返回 403 + VALIDATION_REQUIRED。
|
||||
这是账号级别的永久性错误,重试无法修复,需要人工干预。
|
||||
|
||||
匹配条件(满足任一即可):
|
||||
- error.details 中包含 reason=VALIDATION_REQUIRED
|
||||
- error.status 为 PERMISSION_DENIED 且 message 包含 "verify your account"
|
||||
- error.message 包含 "verify your account"
|
||||
|
||||
Args:
|
||||
error_text: 错误响应文本
|
||||
|
||||
Returns:
|
||||
是否为账号验证要求错误
|
||||
"""
|
||||
if not error_text:
|
||||
return False
|
||||
|
||||
search_text = error_text.lower()
|
||||
|
||||
# 快速路径:关键词匹配
|
||||
if "validation_required" in search_text:
|
||||
return True
|
||||
if "verify your account" in search_text and "permission_denied" in search_text:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _extract_error_message(self, error_text: str | None) -> str | None:
|
||||
"""
|
||||
从错误响应中提取错误消息
|
||||
@@ -480,66 +418,14 @@ class ErrorClassifier:
|
||||
exception: ProviderRateLimitException,
|
||||
request_id: str | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
处理 429 速率限制错误的自适应调整
|
||||
|
||||
Args:
|
||||
key: API Key 对象
|
||||
provider_name: 提供商名称
|
||||
current_rpm: 当前分钟内的请求数
|
||||
exception: 速率限制异常
|
||||
request_id: 请求 ID(用于日志)
|
||||
|
||||
Returns:
|
||||
限制类型: "concurrent" 或 "rpm" 或 "unknown"
|
||||
"""
|
||||
try:
|
||||
# 提取响应头(如果有)
|
||||
response_headers = {}
|
||||
if hasattr(exception, "response_headers"):
|
||||
response_headers = exception.response_headers or {}
|
||||
|
||||
# 检测速率限制类型
|
||||
rate_limit_info = detect_rate_limit_type(
|
||||
headers=response_headers,
|
||||
provider_name=provider_name,
|
||||
current_usage=current_rpm,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f" [{request_id}] 429错误分析: "
|
||||
f"类型={rate_limit_info.limit_type}, "
|
||||
f"retry_after={rate_limit_info.retry_after}s, "
|
||||
f"当前RPM={current_rpm}"
|
||||
)
|
||||
|
||||
# 调用自适应管理器处理
|
||||
new_limit = self.adaptive_manager.handle_429_error(
|
||||
db=self.db,
|
||||
key=key,
|
||||
rate_limit_info=rate_limit_info,
|
||||
current_rpm=current_rpm,
|
||||
)
|
||||
|
||||
if rate_limit_info.limit_type == RateLimitType.CONCURRENT:
|
||||
logger.warning(f" [{request_id}] 并发限制触发(不调整RPM)")
|
||||
return "concurrent"
|
||||
elif rate_limit_info.limit_type == RateLimitType.RPM:
|
||||
if new_limit is not None:
|
||||
logger.warning(
|
||||
f" [{request_id}] 自适应调整: Key {key.id[:8]}... RPM限制 -> {new_limit}"
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f" [{request_id}] 学习中: Key {key.id[:8]}... 观察已记录,暂不设限"
|
||||
)
|
||||
return "rpm"
|
||||
else:
|
||||
return "unknown"
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f" [{request_id}] 处理429错误时异常: {e}")
|
||||
return "unknown"
|
||||
"""委托给 ErrorHandlerService"""
|
||||
return await self._error_handler.handle_rate_limit(
|
||||
key=key,
|
||||
provider_name=provider_name,
|
||||
current_rpm=current_rpm,
|
||||
exception=exception,
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
def convert_http_error(
|
||||
self,
|
||||
@@ -650,32 +536,10 @@ class ErrorClassifier:
|
||||
attempt: int,
|
||||
max_attempts: int,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
处理 HTTP 错误,返回 extra_data
|
||||
|
||||
Args:
|
||||
http_error: HTTP 状态错误
|
||||
provider: Provider 对象
|
||||
endpoint: Endpoint 对象
|
||||
key: API Key 对象
|
||||
affinity_key: 亲和性标识符(通常为 API Key ID)
|
||||
api_format: API 格式
|
||||
global_model_id: GlobalModel ID(规范化的模型标识)
|
||||
request_id: 请求 ID
|
||||
captured_key_concurrent: 捕获的并发数
|
||||
elapsed_ms: 耗时(毫秒)
|
||||
attempt: 当前尝试次数
|
||||
max_attempts: 最大尝试次数
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 额外数据,包含:
|
||||
- error_response: 错误响应文本(如有)
|
||||
- converted_error: 转换后的异常对象(用于判断是否应该重试)
|
||||
"""
|
||||
"""处理 HTTP 错误,返回 extra_data(分类 + 委托副作用给 ErrorHandlerService)"""
|
||||
provider_name = str(provider.name)
|
||||
|
||||
# 尝试读取错误响应内容
|
||||
# 优先使用 handler 附加的 upstream_response 属性(流式请求中 response.text 可能为空)
|
||||
error_response_text = getattr(http_error, "upstream_response", None)
|
||||
if not error_response_text:
|
||||
try:
|
||||
@@ -689,111 +553,35 @@ class ErrorClassifier:
|
||||
f"{http_error.response.status_code if http_error.response else 'unknown'}"
|
||||
)
|
||||
|
||||
# 分类(纯逻辑)
|
||||
converted_error = self.convert_http_error(http_error, provider_name, error_response_text)
|
||||
|
||||
# 构建 extra_data,包含转换后的异常
|
||||
extra_data: dict[str, Any] = {
|
||||
"converted_error": converted_error,
|
||||
}
|
||||
if error_response_text:
|
||||
extra_data["error_response"] = error_response_text
|
||||
|
||||
# client_format:用于缓存亲和性/缓存失效(用户视角)
|
||||
client_format_str = normalize_endpoint_signature(api_format)
|
||||
# provider_format:用于健康度/熔断 bucket(Provider 真实端点格式)
|
||||
fam = str(getattr(endpoint, "api_family", "")).strip().lower()
|
||||
kind = str(getattr(endpoint, "endpoint_kind", "")).strip().lower()
|
||||
provider_format_str = make_signature_key(fam, kind) if fam and kind else client_format_str
|
||||
|
||||
# 处理客户端请求错误(不应重试,不失效缓存,不记录健康失败)
|
||||
if isinstance(converted_error, UpstreamClientException):
|
||||
logger.warning(
|
||||
f" [{request_id}] 客户端请求错误,不进行重试: {converted_error.message}"
|
||||
)
|
||||
return extra_data
|
||||
|
||||
# 处理认证错误
|
||||
if isinstance(converted_error, ProviderAuthException):
|
||||
if endpoint and key and self.cache_scheduler is not None:
|
||||
await self.cache_scheduler.invalidate_cache(
|
||||
affinity_key=affinity_key,
|
||||
api_format=client_format_str,
|
||||
global_model_id=global_model_id,
|
||||
endpoint_id=str(endpoint.id),
|
||||
key_id=str(key.id),
|
||||
)
|
||||
if key:
|
||||
health_monitor.record_failure(
|
||||
db=self.db,
|
||||
key_id=str(key.id),
|
||||
api_format=provider_format_str,
|
||||
error_type="ProviderAuthException",
|
||||
)
|
||||
# 403 VALIDATION_REQUIRED → 标记 OAuth key 为账号级别封禁
|
||||
# 这与 test-model 端点的行为对齐(provider_query.py 第 669-690 行)
|
||||
status_code = http_error.response.status_code if http_error.response else None
|
||||
if (
|
||||
status_code == 403
|
||||
and key
|
||||
and str(getattr(key, "auth_type", "") or "").lower() == "oauth"
|
||||
and self._is_account_validation_required(error_response_text)
|
||||
):
|
||||
try:
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from src.services.provider.oauth_token import (
|
||||
OAUTH_ACCOUNT_BLOCK_PREFIX,
|
||||
)
|
||||
|
||||
key.oauth_invalid_at = datetime.now(timezone.utc)
|
||||
key.oauth_invalid_reason = f"{OAUTH_ACCOUNT_BLOCK_PREFIX}Google 要求验证账号"
|
||||
key.is_active = False
|
||||
self.db.commit()
|
||||
logger.warning(
|
||||
" [{}] {} 因 403 VALIDATION_REQUIRED 已标记为账号异常并自动停用",
|
||||
request_id,
|
||||
self._format_key_display(key),
|
||||
)
|
||||
except Exception as mark_exc:
|
||||
logger.debug(" [{}] 标记 oauth_invalid 失败: {}", request_id, mark_exc)
|
||||
return extra_data
|
||||
|
||||
# 处理限流错误
|
||||
if isinstance(converted_error, ProviderRateLimitException) and key:
|
||||
await self.handle_rate_limit(
|
||||
key=key,
|
||||
provider_name=provider_name,
|
||||
current_rpm=captured_key_concurrent,
|
||||
exception=converted_error,
|
||||
request_id=request_id,
|
||||
)
|
||||
if endpoint and self.cache_scheduler is not None:
|
||||
await self.cache_scheduler.invalidate_cache(
|
||||
affinity_key=affinity_key,
|
||||
api_format=client_format_str,
|
||||
global_model_id=global_model_id,
|
||||
endpoint_id=str(endpoint.id),
|
||||
key_id=str(key.id),
|
||||
)
|
||||
else:
|
||||
# 其他错误也失效缓存
|
||||
if endpoint and key and self.cache_scheduler is not None:
|
||||
await self.cache_scheduler.invalidate_cache(
|
||||
affinity_key=affinity_key,
|
||||
api_format=client_format_str,
|
||||
global_model_id=global_model_id,
|
||||
endpoint_id=str(endpoint.id),
|
||||
key_id=str(key.id),
|
||||
)
|
||||
|
||||
# 记录健康失败
|
||||
if key:
|
||||
health_monitor.record_failure(
|
||||
db=self.db,
|
||||
key_id=str(key.id),
|
||||
api_format=provider_format_str,
|
||||
error_type=type(converted_error).__name__,
|
||||
)
|
||||
# 副作用(委托给 ErrorHandlerService)
|
||||
await self._error_handler.handle_http_error(
|
||||
http_error,
|
||||
converted_error,
|
||||
error_response_text,
|
||||
provider=provider,
|
||||
endpoint=endpoint,
|
||||
key=key,
|
||||
affinity_key=affinity_key,
|
||||
api_format=api_format,
|
||||
global_model_id=global_model_id,
|
||||
request_id=request_id,
|
||||
captured_key_concurrent=captured_key_concurrent,
|
||||
)
|
||||
|
||||
return extra_data
|
||||
|
||||
@@ -813,69 +601,20 @@ class ErrorClassifier:
|
||||
attempt: int,
|
||||
max_attempts: int,
|
||||
) -> None:
|
||||
"""
|
||||
处理可重试错误
|
||||
|
||||
Args:
|
||||
error: 异常对象
|
||||
provider: Provider 对象
|
||||
endpoint: Endpoint 对象
|
||||
key: API Key 对象
|
||||
affinity_key: 亲和性标识符(通常为 API Key ID)
|
||||
api_format: API 格式
|
||||
global_model_id: GlobalModel ID(规范化的模型标识,用于缓存亲和性)
|
||||
captured_key_concurrent: 捕获的并发数
|
||||
elapsed_ms: 耗时(毫秒)
|
||||
request_id: 请求 ID
|
||||
attempt: 当前尝试次数
|
||||
max_attempts: 最大尝试次数
|
||||
"""
|
||||
provider_name = str(provider.name)
|
||||
|
||||
"""委托给 ErrorHandlerService"""
|
||||
logger.warning(
|
||||
f" [{request_id}] 请求失败 (attempt={attempt}/{max_attempts}): "
|
||||
f"{type(error).__name__}: {str(error)}"
|
||||
)
|
||||
|
||||
# client_format:用于缓存亲和性/缓存失效(用户视角)
|
||||
client_format_str = normalize_endpoint_signature(api_format)
|
||||
# provider_format:用于健康度/熔断 bucket(Provider 真实端点格式)
|
||||
fam = str(getattr(endpoint, "api_family", "")).strip().lower()
|
||||
kind = str(getattr(endpoint, "endpoint_kind", "")).strip().lower()
|
||||
provider_format_str = make_signature_key(fam, kind) if fam and kind else client_format_str
|
||||
|
||||
# 处理限流错误
|
||||
if isinstance(error, ProviderRateLimitException) and key:
|
||||
await self.handle_rate_limit(
|
||||
key=key,
|
||||
provider_name=provider_name,
|
||||
current_rpm=captured_key_concurrent,
|
||||
exception=error,
|
||||
request_id=request_id,
|
||||
)
|
||||
if endpoint and self.cache_scheduler is not None:
|
||||
await self.cache_scheduler.invalidate_cache(
|
||||
affinity_key=affinity_key,
|
||||
api_format=client_format_str,
|
||||
global_model_id=global_model_id,
|
||||
endpoint_id=str(endpoint.id),
|
||||
key_id=str(key.id),
|
||||
)
|
||||
elif endpoint and key and self.cache_scheduler is not None:
|
||||
# 其他错误也失效缓存
|
||||
await self.cache_scheduler.invalidate_cache(
|
||||
affinity_key=affinity_key,
|
||||
api_format=client_format_str,
|
||||
global_model_id=global_model_id,
|
||||
endpoint_id=str(endpoint.id),
|
||||
key_id=str(key.id),
|
||||
)
|
||||
|
||||
# 记录健康失败
|
||||
if key:
|
||||
health_monitor.record_failure(
|
||||
db=self.db,
|
||||
key_id=str(key.id),
|
||||
api_format=provider_format_str,
|
||||
error_type=type(error).__name__,
|
||||
)
|
||||
await self._error_handler.handle_retriable_error(
|
||||
error,
|
||||
provider=provider,
|
||||
endpoint=endpoint,
|
||||
key=key,
|
||||
affinity_key=affinity_key,
|
||||
api_format=api_format,
|
||||
global_model_id=global_model_id,
|
||||
captured_key_concurrent=captured_key_concurrent,
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
330
src/services/orchestration/error_handler.py
Normal file
330
src/services/orchestration/error_handler.py
Normal file
@@ -0,0 +1,330 @@
|
||||
"""
|
||||
错误处理服务
|
||||
|
||||
负责错误发生后的副作用操作(缓存失效、健康记录、RPM 调整、OAuth Key 标记等)。
|
||||
与 ErrorClassifier(纯分类,无副作用)分离,遵循单一职责原则。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from src.core.api_format.signature import make_signature_key
|
||||
from src.core.crypto import CryptoService
|
||||
from src.core.exceptions import (
|
||||
ProviderAuthException,
|
||||
ProviderRateLimitException,
|
||||
UpstreamClientException,
|
||||
)
|
||||
from src.core.logger import logger
|
||||
from src.models.database import Provider, ProviderAPIKey, ProviderEndpoint
|
||||
from src.services.cache.aware_scheduler import CacheAwareScheduler
|
||||
from src.services.health.monitor import health_monitor
|
||||
from src.services.provider.format import normalize_endpoint_signature
|
||||
from src.services.rate_limit.adaptive_rpm import get_adaptive_rpm_manager
|
||||
from src.services.rate_limit.detector import RateLimitType, detect_rate_limit_type
|
||||
|
||||
|
||||
class ErrorHandlerService:
|
||||
"""
|
||||
错误处理服务 - 负责错误发生后的副作用操作
|
||||
|
||||
职责:
|
||||
1. 缓存亲和性失效
|
||||
2. 健康监控记录
|
||||
3. 429 自适应 RPM 调整
|
||||
4. OAuth Key 状态标记
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db: Session,
|
||||
adaptive_manager: Any | None = None,
|
||||
cache_scheduler: CacheAwareScheduler | None = None,
|
||||
) -> None:
|
||||
self.db = db
|
||||
self.adaptive_manager = adaptive_manager or get_adaptive_rpm_manager()
|
||||
self.cache_scheduler = cache_scheduler
|
||||
|
||||
async def handle_rate_limit(
|
||||
self,
|
||||
key: ProviderAPIKey,
|
||||
provider_name: str,
|
||||
current_rpm: int | None,
|
||||
exception: ProviderRateLimitException,
|
||||
request_id: str | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
处理 429 速率限制错误的自适应调整
|
||||
|
||||
Returns:
|
||||
限制类型: "concurrent" 或 "rpm" 或 "unknown"
|
||||
"""
|
||||
try:
|
||||
response_headers = {}
|
||||
if hasattr(exception, "response_headers"):
|
||||
response_headers = exception.response_headers or {}
|
||||
|
||||
rate_limit_info = detect_rate_limit_type(
|
||||
headers=response_headers,
|
||||
provider_name=provider_name,
|
||||
current_usage=current_rpm,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
" [{}] 429错误分析: 类型={}, retry_after={}s, 当前RPM={}",
|
||||
request_id,
|
||||
rate_limit_info.limit_type,
|
||||
rate_limit_info.retry_after,
|
||||
current_rpm,
|
||||
)
|
||||
|
||||
new_limit = self.adaptive_manager.handle_429_error(
|
||||
db=self.db,
|
||||
key=key,
|
||||
rate_limit_info=rate_limit_info,
|
||||
current_rpm=current_rpm,
|
||||
)
|
||||
|
||||
if rate_limit_info.limit_type == RateLimitType.CONCURRENT:
|
||||
logger.warning(" [{}] 并发限制触发(不调整RPM)", request_id)
|
||||
return "concurrent"
|
||||
elif rate_limit_info.limit_type == RateLimitType.RPM:
|
||||
if new_limit is not None:
|
||||
logger.warning(
|
||||
" [{}] 自适应调整: Key {}... RPM限制 -> {}",
|
||||
request_id,
|
||||
str(key.id)[:8],
|
||||
new_limit,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
" [{}] 学习中: Key {}... 观察已记录,暂不设限",
|
||||
request_id,
|
||||
str(key.id)[:8],
|
||||
)
|
||||
return "rpm"
|
||||
else:
|
||||
return "unknown"
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(" [{}] 处理429错误时异常: {}", request_id, e)
|
||||
return "unknown"
|
||||
|
||||
async def handle_http_error(
|
||||
self,
|
||||
http_error: httpx.HTTPStatusError,
|
||||
converted_error: Exception,
|
||||
error_response_text: str | None,
|
||||
*,
|
||||
provider: Provider,
|
||||
endpoint: ProviderEndpoint,
|
||||
key: ProviderAPIKey,
|
||||
affinity_key: str,
|
||||
api_format: str,
|
||||
global_model_id: str,
|
||||
request_id: str | None,
|
||||
captured_key_concurrent: int | None,
|
||||
) -> None:
|
||||
"""
|
||||
处理 HTTP 错误的副作用(缓存失效、健康记录、OAuth 标记)。
|
||||
|
||||
纯副作用方法:不返回分类结果,不做错误转换。
|
||||
"""
|
||||
client_format_str = normalize_endpoint_signature(api_format)
|
||||
fam = str(getattr(endpoint, "api_family", "")).strip().lower()
|
||||
kind = str(getattr(endpoint, "endpoint_kind", "")).strip().lower()
|
||||
provider_format_str = make_signature_key(fam, kind) if fam and kind else client_format_str
|
||||
|
||||
# 客户端请求错误:不失效缓存,不记录健康失败
|
||||
if isinstance(converted_error, UpstreamClientException):
|
||||
return
|
||||
|
||||
can_invalidate = bool(endpoint and key and self.cache_scheduler is not None)
|
||||
|
||||
# 认证错误
|
||||
if isinstance(converted_error, ProviderAuthException):
|
||||
if can_invalidate:
|
||||
await self._invalidate_cache(
|
||||
affinity_key, client_format_str, global_model_id, endpoint, key
|
||||
)
|
||||
if key:
|
||||
health_monitor.record_failure(
|
||||
db=self.db,
|
||||
key_id=str(key.id),
|
||||
api_format=provider_format_str,
|
||||
error_type="ProviderAuthException",
|
||||
)
|
||||
# 403 VALIDATION_REQUIRED -> 标记 OAuth key 为账号级别封禁
|
||||
status_code = http_error.response.status_code if http_error.response else None
|
||||
if (
|
||||
status_code == 403
|
||||
and key
|
||||
and str(getattr(key, "auth_type", "") or "").lower() == "oauth"
|
||||
and self._is_account_validation_required(error_response_text)
|
||||
):
|
||||
self._mark_oauth_key_blocked(key, request_id)
|
||||
return
|
||||
|
||||
# 限流错误
|
||||
if isinstance(converted_error, ProviderRateLimitException) and key:
|
||||
await self.handle_rate_limit(
|
||||
key=key,
|
||||
provider_name=str(provider.name),
|
||||
current_rpm=captured_key_concurrent,
|
||||
exception=converted_error,
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
# 所有非客户端错误均失效缓存
|
||||
if can_invalidate:
|
||||
await self._invalidate_cache(
|
||||
affinity_key, client_format_str, global_model_id, endpoint, key
|
||||
)
|
||||
|
||||
# 记录健康失败
|
||||
if key:
|
||||
health_monitor.record_failure(
|
||||
db=self.db,
|
||||
key_id=str(key.id),
|
||||
api_format=provider_format_str,
|
||||
error_type=type(converted_error).__name__,
|
||||
)
|
||||
|
||||
async def handle_retriable_error(
|
||||
self,
|
||||
error: Exception,
|
||||
*,
|
||||
provider: Provider,
|
||||
endpoint: ProviderEndpoint,
|
||||
key: ProviderAPIKey,
|
||||
affinity_key: str,
|
||||
api_format: str,
|
||||
global_model_id: str,
|
||||
captured_key_concurrent: int | None,
|
||||
request_id: str | None,
|
||||
) -> None:
|
||||
"""处理可重试错误的副作用(缓存失效、健康记录)"""
|
||||
client_format_str = normalize_endpoint_signature(api_format)
|
||||
fam = str(getattr(endpoint, "api_family", "")).strip().lower()
|
||||
kind = str(getattr(endpoint, "endpoint_kind", "")).strip().lower()
|
||||
provider_format_str = make_signature_key(fam, kind) if fam and kind else client_format_str
|
||||
|
||||
# 限流错误
|
||||
if isinstance(error, ProviderRateLimitException) and key:
|
||||
await self.handle_rate_limit(
|
||||
key=key,
|
||||
provider_name=str(provider.name),
|
||||
current_rpm=captured_key_concurrent,
|
||||
exception=error,
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
# 失效缓存
|
||||
if endpoint and key and self.cache_scheduler is not None:
|
||||
await self._invalidate_cache(
|
||||
affinity_key, client_format_str, global_model_id, endpoint, key
|
||||
)
|
||||
|
||||
# 记录健康失败
|
||||
if key:
|
||||
health_monitor.record_failure(
|
||||
db=self.db,
|
||||
key_id=str(key.id),
|
||||
api_format=provider_format_str,
|
||||
error_type=type(error).__name__,
|
||||
)
|
||||
|
||||
async def _invalidate_cache(
|
||||
self,
|
||||
affinity_key: str,
|
||||
api_format: str,
|
||||
global_model_id: str,
|
||||
endpoint: ProviderEndpoint,
|
||||
key: ProviderAPIKey,
|
||||
) -> None:
|
||||
"""失效缓存亲和性(调用方需确保 cache_scheduler 可用)"""
|
||||
assert self.cache_scheduler is not None # noqa: S101
|
||||
await self.cache_scheduler.invalidate_cache(
|
||||
affinity_key=affinity_key,
|
||||
api_format=api_format,
|
||||
global_model_id=global_model_id,
|
||||
endpoint_id=str(endpoint.id),
|
||||
key_id=str(key.id),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_oauth_email(key: ProviderAPIKey | None) -> str | None:
|
||||
"""从 OAuth Key 的加密 auth_config 中提取邮箱"""
|
||||
if not key or str(getattr(key, "auth_type", "") or "").lower() != "oauth":
|
||||
return None
|
||||
encrypted_auth_config = getattr(key, "auth_config", None)
|
||||
if not encrypted_auth_config:
|
||||
return None
|
||||
try:
|
||||
decrypted = CryptoService().decrypt(encrypted_auth_config, silent=True)
|
||||
auth_config = json.loads(decrypted) if decrypted else {}
|
||||
except Exception:
|
||||
return None
|
||||
email = auth_config.get("email")
|
||||
if isinstance(email, str):
|
||||
email = email.strip()
|
||||
if email:
|
||||
return email
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _format_key_display(cls, key: ProviderAPIKey | None) -> str:
|
||||
"""格式化 Key 显示信息(用于日志)"""
|
||||
if not key:
|
||||
return "key=unknown"
|
||||
key_id = str(getattr(key, "id", "") or "")[:8] or "unknown"
|
||||
name = str(getattr(key, "name", "") or "").strip()
|
||||
email = cls._extract_oauth_email(key)
|
||||
parts = [f"key={key_id}"]
|
||||
if email:
|
||||
parts.append(f"email={email}")
|
||||
if name and name != email:
|
||||
parts.append(f"name={name}")
|
||||
return " ".join(parts)
|
||||
|
||||
@staticmethod
|
||||
def _is_account_validation_required(error_text: str | None) -> bool:
|
||||
"""
|
||||
检测 403 错误是否为 Google 账号验证要求 (VALIDATION_REQUIRED)
|
||||
|
||||
匹配条件(满足任一即可):
|
||||
- error.details 中包含 reason=VALIDATION_REQUIRED
|
||||
- error.status 为 PERMISSION_DENIED 且 message 包含 "verify your account"
|
||||
"""
|
||||
if not error_text:
|
||||
return False
|
||||
search_text = error_text.lower()
|
||||
if "validation_required" in search_text:
|
||||
return True
|
||||
if "verify your account" in search_text and "permission_denied" in search_text:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _mark_oauth_key_blocked(self, key: ProviderAPIKey, request_id: str | None) -> None:
|
||||
"""标记 OAuth key 为账号级别封禁"""
|
||||
try:
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from src.services.provider.oauth_token import OAUTH_ACCOUNT_BLOCK_PREFIX
|
||||
|
||||
key.oauth_invalid_at = datetime.now(timezone.utc)
|
||||
key.oauth_invalid_reason = f"{OAUTH_ACCOUNT_BLOCK_PREFIX}Google 要求验证账号"
|
||||
key.is_active = False
|
||||
self.db.commit()
|
||||
logger.warning(
|
||||
" [{}] {} 因 403 VALIDATION_REQUIRED 已标记为账号异常并自动停用",
|
||||
request_id,
|
||||
self._format_key_display(key),
|
||||
)
|
||||
except Exception as mark_exc:
|
||||
logger.debug(" [{}] 标记 oauth_invalid 失败: {}", request_id, mark_exc)
|
||||
Reference in New Issue
Block a user