refactor: 重构 API 格式相关代码到统一的 api_format 模块

- 新增 src/core/api_format/ 模块,整合所有 API 格式相关功能
  - enums.py: APIFormat 枚举定义
  - metadata.py: API 格式元数据注册
  - headers.py: 请求头处理逻辑
  - detection.py: API 格式检测
  - conversion/: 格式转换器(Claude/OpenAI/Gemini 互转)
- 删除分散在各处的旧实现
  - src/core/api_format_metadata.py
  - src/core/headers.py
  - src/api/handlers/*/converter.py
  - src/api/handlers/base/format_converter_registry.py
- 更新所有引用路径,使用新的统一模块
- 新增 cli_handler_base 中的格式转换支持
This commit is contained in:
fawney19
2026-01-21 18:07:10 +08:00
parent 856077a46c
commit 99388bfa33
57 changed files with 2405 additions and 450 deletions

View File

@@ -35,8 +35,7 @@ from fastapi.responses import JSONResponse, StreamingResponse
from sqlalchemy.orm import Session
from src.clients.redis_client import get_redis_client_sync
from src.core.api_format_metadata import resolve_api_format
from src.core.enums import APIFormat
from src.core.api_format import APIFormat, resolve_api_format
from src.core.logger import logger
from src.services.orchestration.fallback_orchestrator import FallbackOrchestrator
from src.services.provider.format import normalize_api_format

View File

@@ -28,7 +28,7 @@ from fastapi.responses import JSONResponse
from src.api.base.adapter import ApiAdapter, ApiMode
from src.api.base.context import ApiRequestContext
from src.api.handlers.base.chat_handler_base import ChatHandlerBase
from src.core.enums import APIFormat
from src.core.api_format import APIFormat
from src.core.exceptions import (
InvalidRequestException,
ModelNotSupportedException,
@@ -40,7 +40,7 @@ from src.core.exceptions import (
QuotaExceededException,
UpstreamClientException,
)
from src.core.headers import (
from src.core.api_format import (
build_adapter_base_headers,
build_adapter_headers,
extract_client_api_key,

View File

@@ -26,7 +26,7 @@ from fastapi.responses import JSONResponse
from src.api.base.adapter import ApiAdapter, ApiMode
from src.api.base.context import ApiRequestContext
from src.api.handlers.base.cli_handler_base import CliMessageHandlerBase
from src.core.enums import APIFormat
from src.core.api_format import APIFormat
from src.core.exceptions import (
InvalidRequestException,
ModelNotSupportedException,
@@ -38,7 +38,7 @@ from src.core.exceptions import (
QuotaExceededException,
UpstreamClientException,
)
from src.core.headers import (
from src.core.api_format import (
build_adapter_base_headers,
build_adapter_headers,
extract_client_api_key,

View File

@@ -19,6 +19,7 @@ from typing import (
AsyncGenerator,
Callable,
Dict,
List,
Optional,
)
@@ -671,10 +672,11 @@ class CliMessageHandlerBase(BaseMessageHandler):
# 格式转换或直接透传
if needs_conversion:
converted_line = self._convert_sse_line(ctx, line, events)
if converted_line:
self._mark_first_output(ctx, output_state)
yield (converted_line + "\n").encode("utf-8")
converted_lines = self._convert_sse_line(ctx, line, events)
for converted_line in converted_lines:
if converted_line:
self._mark_first_output(ctx, output_state)
yield (converted_line + "\n").encode("utf-8")
else:
self._mark_first_output(ctx, output_state)
yield (line + "\n").encode("utf-8")
@@ -999,10 +1001,11 @@ class CliMessageHandlerBase(BaseMessageHandler):
# 格式转换或直接透传
if needs_conversion:
converted_line = self._convert_sse_line(ctx, line, events)
if converted_line:
self._mark_first_output(ctx, output_state)
yield (converted_line + "\n").encode("utf-8")
converted_lines = self._convert_sse_line(ctx, line, events)
for converted_line in converted_lines:
if converted_line:
self._mark_first_output(ctx, output_state)
yield (converted_line + "\n").encode("utf-8")
else:
self._mark_first_output(ctx, output_state)
yield (line + "\n").encode("utf-8")
@@ -1073,10 +1076,11 @@ class CliMessageHandlerBase(BaseMessageHandler):
# 格式转换或直接透传
if needs_conversion:
converted_line = self._convert_sse_line(ctx, line, events)
if converted_line:
self._mark_first_output(ctx, output_state)
yield (converted_line + "\n").encode("utf-8")
converted_lines = self._convert_sse_line(ctx, line, events)
for converted_line in converted_lines:
if converted_line:
self._mark_first_output(ctx, output_state)
yield (converted_line + "\n").encode("utf-8")
else:
self._mark_first_output(ctx, output_state)
yield (line + "\n").encode("utf-8")
@@ -1760,7 +1764,7 @@ class CliMessageHandlerBase(BaseMessageHandler):
and api_format
and provider_api_format.upper() != api_format.upper()
):
from src.api.handlers.base.format_converter_registry import converter_registry
from src.core.api_format import converter_registry
try:
response_json = converter_registry.convert_response(
@@ -1923,28 +1927,32 @@ class CliMessageHandlerBase(BaseMessageHandler):
self,
ctx: StreamContext,
line: str,
events: list,
) -> Optional[str]:
events: list, # noqa: ARG002 - 预留给上下文感知转换
) -> List[str]:
"""
将 SSE 行从 Provider 格式转换为客户端格式
Args:
ctx: 流上下文
line: 原始 SSE 行
events: 解析后的事件列表
events: 当前累积的事件列表(预留参数,用于未来上下文感知转换如合并相邻事件)
Returns:
转换后的 SSE 行,如果无法转换则返回 None
转换后的 SSE 行列表(一入多出),空列表表示跳过该行
"""
from src.api.handlers.base.format_converter_registry import converter_registry
from src.core.api_format import (
GeminiStreamConversionState,
StreamConversionState,
converter_registry,
)
# 如果是空行或特殊控制行,直接返回
if not line or line.strip() == "" or line == "data: [DONE]":
return line
return [line] if line else []
# 如果不是 data 行,直接透传
if not line.startswith("data:"):
return line
return [line]
# 提取 data 内容
data_content = line[5:].strip() # 去掉 "data:" 前缀
@@ -1954,17 +1962,40 @@ class CliMessageHandlerBase(BaseMessageHandler):
data_obj = json.loads(data_content)
except json.JSONDecodeError:
# 无法解析,直接透传
return line
return [line]
# 使用注册表进行格式转换
# 类型断言:当 needs_conversion=True 调用此方法时,格式字段必定有值
provider_format = ctx.provider_api_format or ""
client_format = ctx.client_api_format or ""
# 初始化流式转换状态(首次调用时,根据 Provider 格式选择状态类)
if ctx.stream_conversion_state is None:
if provider_format.upper() == "GEMINI":
ctx.stream_conversion_state = GeminiStreamConversionState(
model=ctx.mapped_model or ctx.model or "",
message_id=ctx.response_id or ctx.request_id or "",
)
else:
ctx.stream_conversion_state = StreamConversionState(
model=ctx.mapped_model or ctx.model or "",
message_id=ctx.response_id or ctx.request_id or "",
)
# 使用注册表进行格式转换(严格模式,返回 List[Dict]
try:
converted_obj = converter_registry.convert_stream_chunk(
converted_events = converter_registry.convert_stream_chunk_strict(
data_obj,
ctx.provider_api_format,
ctx.client_api_format,
provider_format,
client_format,
state=ctx.stream_conversion_state,
)
# 重新构建 SSE 行
return f"data: {json.dumps(converted_obj, ensure_ascii=False)}"
# 转换为 SSE 行列表
result = []
for evt in converted_events:
result.append(f"data: {json.dumps(evt, ensure_ascii=False)}")
return result
except Exception as e:
logger.warning(f"格式转换失败,透传原始数据: {e}")
return line
return [line]

View File

@@ -27,7 +27,7 @@ from collections import defaultdict
import httpx
from src.core.logger import logger
from src.core.headers import CORE_REDACT_HEADERS, merge_headers_with_protection, redact_headers_for_log
from src.core.api_format import CORE_REDACT_HEADERS, merge_headers_with_protection, redact_headers_for_log
from src.utils.ssl_utils import get_ssl_context

View File

@@ -1,279 +0,0 @@
"""
格式转换器注册表
自动管理不同 API 格式之间的转换器,支持:
- 请求转换:客户端格式 → Provider 格式
- 响应转换Provider 格式 → 客户端格式
使用方法:
1. 实现 Converter 类(需要有 convert_request 和/或 convert_response 方法)
2. 调用 registry.register() 注册转换器
3. 在 Handler 中调用 registry.convert_request/convert_response
示例:
from src.api.handlers.base.format_converter_registry import converter_registry
# 注册转换器
converter_registry.register("CLAUDE", "GEMINI", ClaudeToGeminiConverter())
converter_registry.register("GEMINI", "CLAUDE", GeminiToClaudeConverter())
# 使用转换器
gemini_request = converter_registry.convert_request(claude_request, "CLAUDE", "GEMINI")
claude_response = converter_registry.convert_response(gemini_response, "GEMINI", "CLAUDE")
"""
from typing import Any, Dict, Optional, Protocol, Tuple
from src.core.logger import logger
class RequestConverter(Protocol):
"""请求转换器协议"""
def convert_request(self, request: Dict[str, Any]) -> Dict[str, Any]: ...
class ResponseConverter(Protocol):
"""响应转换器协议"""
def convert_response(self, response: Dict[str, Any]) -> Dict[str, Any]: ...
class StreamChunkConverter(Protocol):
"""流式响应块转换器协议"""
def convert_stream_chunk(self, chunk: Dict[str, Any]) -> Dict[str, Any]: ...
class FormatConverterRegistry:
"""
格式转换器注册表
管理不同 API 格式之间的双向转换器
"""
def __init__(self):
# key: (source_format, target_format), value: converter instance
self._converters: Dict[Tuple[str, str], Any] = {}
def register(
self,
source_format: str,
target_format: str,
converter: Any,
) -> None:
"""
注册格式转换器
Args:
source_format: 源格式(如 "CLAUDE", "OPENAI", "GEMINI"
target_format: 目标格式
converter: 转换器实例(需要有 convert_request/convert_response 方法)
"""
key = (source_format.upper(), target_format.upper())
self._converters[key] = converter
logger.info(f"[ConverterRegistry] 注册转换器: {source_format} -> {target_format}")
def get_converter(
self,
source_format: str,
target_format: str,
) -> Optional[Any]:
"""
获取转换器
Args:
source_format: 源格式
target_format: 目标格式
Returns:
转换器实例,如果不存在返回 None
"""
key = (source_format.upper(), target_format.upper())
return self._converters.get(key)
def has_converter(
self,
source_format: str,
target_format: str,
) -> bool:
"""检查是否存在转换器"""
key = (source_format.upper(), target_format.upper())
return key in self._converters
def convert_request(
self,
request: Dict[str, Any],
source_format: str,
target_format: str,
) -> Dict[str, Any]:
"""
转换请求
Args:
request: 原始请求字典
source_format: 源格式(客户端格式)
target_format: 目标格式Provider 格式)
Returns:
转换后的请求字典,如果无需转换或没有转换器则返回原始请求
"""
# 同格式无需转换
if source_format.upper() == target_format.upper():
return request
converter = self.get_converter(source_format, target_format)
if converter is None:
logger.warning(f"[ConverterRegistry] 未找到请求转换器: {source_format} -> {target_format},返回原始请求")
return request
if not hasattr(converter, "convert_request"):
logger.warning(f"[ConverterRegistry] 转换器缺少 convert_request 方法: {source_format} -> {target_format}")
return request
try:
converted = converter.convert_request(request)
logger.debug(f"[ConverterRegistry] 请求转换成功: {source_format} -> {target_format}")
return converted
except Exception as e:
logger.error(f"[ConverterRegistry] 请求转换失败: {source_format} -> {target_format}: {e}")
return request
def convert_response(
self,
response: Dict[str, Any],
source_format: str,
target_format: str,
) -> Dict[str, Any]:
"""
转换响应
Args:
response: 原始响应字典
source_format: 源格式Provider 格式)
target_format: 目标格式(客户端格式)
Returns:
转换后的响应字典,如果无需转换或没有转换器则返回原始响应
"""
# 同格式无需转换
if source_format.upper() == target_format.upper():
return response
converter = self.get_converter(source_format, target_format)
if converter is None:
logger.warning(f"[ConverterRegistry] 未找到响应转换器: {source_format} -> {target_format},返回原始响应")
return response
if not hasattr(converter, "convert_response"):
logger.warning(f"[ConverterRegistry] 转换器缺少 convert_response 方法: {source_format} -> {target_format}")
return response
try:
converted = converter.convert_response(response)
logger.debug(f"[ConverterRegistry] 响应转换成功: {source_format} -> {target_format}")
return converted
except Exception as e:
logger.error(f"[ConverterRegistry] 响应转换失败: {source_format} -> {target_format}: {e}")
return response
def convert_stream_chunk(
self,
chunk: Dict[str, Any],
source_format: str,
target_format: str,
) -> Dict[str, Any]:
"""
转换流式响应块
Args:
chunk: 原始流式响应块
source_format: 源格式Provider 格式)
target_format: 目标格式(客户端格式)
Returns:
转换后的流式响应块
"""
# 同格式无需转换
if source_format.upper() == target_format.upper():
return chunk
converter = self.get_converter(source_format, target_format)
if converter is None:
return chunk
# 优先使用专门的流式转换方法
if hasattr(converter, "convert_stream_chunk"):
try:
return converter.convert_stream_chunk(chunk)
except Exception as e:
logger.error(f"[ConverterRegistry] 流式块转换失败: {source_format} -> {target_format}: {e}")
return chunk
# 降级到普通响应转换
if hasattr(converter, "convert_response"):
try:
return converter.convert_response(chunk)
except Exception:
return chunk
return chunk
def list_converters(self) -> list[Tuple[str, str]]:
"""列出所有已注册的转换器"""
return list(self._converters.keys())
# 全局单例
converter_registry = FormatConverterRegistry()
def register_all_converters():
"""
注册所有内置的格式转换器
在应用启动时调用此函数
"""
# Claude <-> OpenAI
try:
from src.api.handlers.claude.converter import OpenAIToClaudeConverter
from src.api.handlers.openai.converter import ClaudeToOpenAIConverter
converter_registry.register("OPENAI", "CLAUDE", OpenAIToClaudeConverter())
converter_registry.register("CLAUDE", "OPENAI", ClaudeToOpenAIConverter())
except ImportError as e:
logger.warning(f"[ConverterRegistry] 无法加载 Claude/OpenAI 转换器: {e}")
# Claude <-> Gemini
try:
from src.api.handlers.gemini.converter import (
ClaudeToGeminiConverter,
GeminiToClaudeConverter,
)
converter_registry.register("CLAUDE", "GEMINI", ClaudeToGeminiConverter())
converter_registry.register("GEMINI", "CLAUDE", GeminiToClaudeConverter())
except ImportError as e:
logger.warning(f"[ConverterRegistry] 无法加载 Claude/Gemini 转换器: {e}")
# OpenAI <-> Gemini
try:
from src.api.handlers.gemini.converter import (
GeminiToOpenAIConverter,
OpenAIToGeminiConverter,
)
converter_registry.register("OPENAI", "GEMINI", OpenAIToGeminiConverter())
converter_registry.register("GEMINI", "OPENAI", GeminiToOpenAIConverter())
except ImportError as e:
logger.warning(f"[ConverterRegistry] 无法加载 OpenAI/Gemini 转换器: {e}")
logger.info(f"[ConverterRegistry] 已注册 {len(converter_registry.list_converters())} 个格式转换器")
__all__ = [
"FormatConverterRegistry",
"converter_registry",
"register_all_converters",
]

View File

@@ -16,6 +16,9 @@ from src.api.handlers.base.response_parser import (
)
from src.api.handlers.base.utils import extract_cache_creation_tokens
# is_cli_format 权威定义在 core 层
from src.core.api_format import is_cli_format
def _check_nested_error(response: Dict[str, Any]) -> Tuple[bool, Optional[Dict[str, Any]]]:
"""
@@ -571,11 +574,6 @@ def get_parser_for_format(format_id: str) -> ResponseParser:
return _PARSERS[format_id]()
def is_cli_format(format_id: str) -> bool:
"""判断是否为 CLI 格式"""
return format_id.upper().endswith("_CLI")
__all__ = [
"OpenAIResponseParser",
"OpenAICliResponseParser",

View File

@@ -17,7 +17,7 @@ from abc import ABC, abstractmethod
from typing import Any, Dict, FrozenSet, Optional, Tuple
from src.core.crypto import crypto_service
from src.core.headers import HeaderBuilder, UPSTREAM_DROP_HEADERS
from src.core.api_format import HeaderBuilder, UPSTREAM_DROP_HEADERS
# ==============================================================================
# 统一的头部配置常量
@@ -125,7 +125,7 @@ class PassthroughRequestBuilder(RequestBuilder):
"""
透传请求头 - 清理敏感头部(黑名单),透传其他所有头部
"""
from src.core.api_format_metadata import get_auth_config, resolve_api_format
from src.core.api_format import get_auth_config, resolve_api_format
# 1. 根据 API 格式自动设置认证头
decrypted_key = crypto_service.decrypt(key.api_key)

View File

@@ -10,7 +10,10 @@
import time
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
if TYPE_CHECKING:
from src.core.api_format import GeminiStreamConversionState, StreamConversionState
@dataclass
@@ -82,6 +85,11 @@ class StreamContext:
chunk_count: int = 0
parsed_chunks: List[Dict[str, Any]] = field(default_factory=list)
# 流式格式转换状态(跨 chunk 追踪)
stream_conversion_state: Optional[
Union["StreamConversionState", "GeminiStreamConversionState"]
] = None
def reset_for_retry(self) -> None:
"""
重试时重置状态
@@ -109,6 +117,7 @@ class StreamContext:
self.response_id = None
self.final_usage = None
self.final_response = None
self.stream_conversion_state = None
@property
def collected_text(self) -> str:

View File

@@ -6,7 +6,7 @@ import json
from typing import Any, Dict, Optional
from src.core.exceptions import EmbeddedErrorException, ProviderNotAvailableException
from src.core.headers import filter_response_headers
from src.core.api_format import filter_response_headers
from src.core.logger import logger