refactor: 重构 API 格式相关代码到统一的 api_format 模块

- 新增 src/core/api_format/ 模块,整合所有 API 格式相关功能
  - enums.py: APIFormat 枚举定义
  - metadata.py: API 格式元数据注册
  - headers.py: 请求头处理逻辑
  - detection.py: API 格式检测
  - conversion/: 格式转换器(Claude/OpenAI/Gemini 互转)
- 删除分散在各处的旧实现
  - src/core/api_format_metadata.py
  - src/core/headers.py
  - src/api/handlers/*/converter.py
  - src/api/handlers/base/format_converter_registry.py
- 更新所有引用路径,使用新的统一模块
- 新增 cli_handler_base 中的格式转换支持
This commit is contained in:
fawney19
2026-01-21 18:07:10 +08:00
parent 856077a46c
commit 99388bfa33
57 changed files with 2405 additions and 450 deletions

View File

@@ -0,0 +1,281 @@
"""
CliMessageHandlerBase._convert_sse_line 单元测试
测试覆盖:
1. 基本转换(空行、非 data 行、JSON 解析失败)
2. 一入多出场景
3. 状态追踪
4. 错误处理
"""
import json
from typing import Any, Dict, List, Optional
from unittest.mock import MagicMock, patch
import pytest
from src.api.handlers.base.stream_context import StreamContext
from src.core.api_format import StreamConversionState
# Mock CliMessageHandlerBase 用于测试
class MockCliHandler:
"""Mock handler for testing _convert_sse_line"""
def _convert_sse_line(
self,
ctx: StreamContext,
line: str,
events: list,
) -> List[str]:
"""复制自 CliMessageHandlerBase._convert_sse_line"""
from src.core.api_format import converter_registry
# 如果是空行或特殊控制行,直接返回
if not line or line.strip() == "" or line == "data: [DONE]":
return [line] if line else []
# 如果不是 data 行,直接透传
if not line.startswith("data:"):
return [line]
# 提取 data 内容
data_content = line[5:].strip()
# 尝试解析 JSON
try:
data_obj = json.loads(data_content)
except json.JSONDecodeError:
return [line]
# 初始化流式转换状态
if ctx.stream_conversion_state is None:
ctx.stream_conversion_state = StreamConversionState(
model=ctx.mapped_model or ctx.model,
message_id=ctx.response_id or ctx.request_id,
)
provider_format = ctx.provider_api_format or ""
client_format = ctx.client_api_format or ""
try:
converted_events = converter_registry.convert_stream_chunk_strict(
data_obj,
provider_format,
client_format,
state=ctx.stream_conversion_state,
)
if converted_events and not ctx.stream_conversion_state.message_started:
for evt in converted_events:
if evt.get("type") == "message_start" or evt.get("choices"):
ctx.stream_conversion_state.message_started = True
break
result = []
for evt in converted_events:
result.append(f"data: {json.dumps(evt, ensure_ascii=False)}")
return result
except Exception:
return [line]
class TestConvertSseLineBasic:
"""基本转换测试"""
def test_empty_line_returns_empty_list(self) -> None:
handler = MockCliHandler()
ctx = StreamContext(model="test", api_format="OPENAI")
result = handler._convert_sse_line(ctx, "", [])
assert result == []
def test_whitespace_line_returns_line(self) -> None:
handler = MockCliHandler()
ctx = StreamContext(model="test", api_format="OPENAI")
result = handler._convert_sse_line(ctx, " ", [])
assert result == [" "]
def test_done_marker_returns_as_is(self) -> None:
handler = MockCliHandler()
ctx = StreamContext(model="test", api_format="OPENAI")
result = handler._convert_sse_line(ctx, "data: [DONE]", [])
assert result == ["data: [DONE]"]
def test_non_data_line_passthrough(self) -> None:
handler = MockCliHandler()
ctx = StreamContext(model="test", api_format="OPENAI")
result = handler._convert_sse_line(ctx, "event: message_start", [])
assert result == ["event: message_start"]
def test_invalid_json_passthrough(self) -> None:
handler = MockCliHandler()
ctx = StreamContext(model="test", api_format="OPENAI")
result = handler._convert_sse_line(ctx, "data: {invalid json}", [])
assert result == ["data: {invalid json}"]
class TestConvertSseLineWithMockConverter:
"""使用 Mock 转换器的测试"""
def test_same_format_returns_original(self) -> None:
"""同格式无需转换"""
handler = MockCliHandler()
ctx = StreamContext(model="test", api_format="OPENAI")
ctx.provider_api_format = "OPENAI"
ctx.client_api_format = "OPENAI"
chunk = {"choices": [{"delta": {"content": "hello"}}]}
line = f"data: {json.dumps(chunk)}"
result = handler._convert_sse_line(ctx, line, [])
assert len(result) == 1
assert json.loads(result[0][6:]) == chunk
def test_state_initialization(self) -> None:
"""测试状态自动初始化"""
handler = MockCliHandler()
ctx = StreamContext(model="gpt-4", api_format="OPENAI")
ctx.provider_api_format = "OPENAI"
ctx.client_api_format = "OPENAI"
ctx.mapped_model = "claude-3-5-sonnet"
ctx.request_id = "req_123"
chunk = {"choices": [{"delta": {"content": "test"}}]}
line = f"data: {json.dumps(chunk)}"
handler._convert_sse_line(ctx, line, [])
# 验证状态已初始化
assert ctx.stream_conversion_state is not None
assert ctx.stream_conversion_state.model == "claude-3-5-sonnet"
assert ctx.stream_conversion_state.message_id == "req_123"
class TestConvertSseLineOneInManyOut:
"""一入多出测试(需要注册转换器)"""
@pytest.fixture(autouse=True)
def setup_converters(self):
"""注册测试用转换器"""
from src.core.api_format import (
ClaudeToOpenAIConverter,
OpenAIToClaudeConverter,
converter_registry,
)
# 保存原始状态
original_converters = converter_registry._converters.copy()
# 注册转换器
converter_registry.register("OPENAI", "CLAUDE", OpenAIToClaudeConverter())
converter_registry.register("CLAUDE", "OPENAI", ClaudeToOpenAIConverter())
yield
# 恢复原始状态
converter_registry._converters = original_converters
def test_openai_to_claude_conversion(self) -> None:
"""测试 OpenAI -> Claude 流式转换"""
handler = MockCliHandler()
ctx = StreamContext(model="gpt-4", api_format="OPENAI")
ctx.provider_api_format = "OPENAI"
ctx.client_api_format = "CLAUDE"
ctx.mapped_model = "claude-3-5-sonnet"
ctx.request_id = "req_test"
# 第一个 chunk带 role
chunk1 = {
"id": "chatcmpl-123",
"object": "chat.completion.chunk",
"choices": [{"index": 0, "delta": {"role": "assistant"}, "finish_reason": None}],
}
line1 = f"data: {json.dumps(chunk1)}"
result1 = handler._convert_sse_line(ctx, line1, [])
# 应返回 message_start 事件
assert len(result1) >= 1
first_event = json.loads(result1[0][6:])
assert first_event.get("type") == "message_start"
# 状态应更新
assert ctx.stream_conversion_state.message_started is True
def test_claude_to_openai_conversion(self) -> None:
"""测试 Claude -> OpenAI 流式转换"""
handler = MockCliHandler()
ctx = StreamContext(model="claude-3-5-sonnet", api_format="CLAUDE")
ctx.provider_api_format = "CLAUDE"
ctx.client_api_format = "OPENAI"
ctx.mapped_model = "gpt-4"
ctx.request_id = "msg_test"
# Claude message_start 事件
event = {"type": "message_start", "message": {"id": "msg_123", "role": "assistant"}}
line = f"data: {json.dumps(event)}"
result = handler._convert_sse_line(ctx, line, [])
# 应返回 OpenAI 格式的 chunk
assert len(result) >= 1
chunk = json.loads(result[0][6:])
assert "choices" in chunk
def test_multiple_chunks_state_persistence(self) -> None:
"""测试多个 chunk 之间状态持久化"""
handler = MockCliHandler()
ctx = StreamContext(model="gpt-4", api_format="OPENAI")
ctx.provider_api_format = "OPENAI"
ctx.client_api_format = "CLAUDE"
ctx.mapped_model = "claude-3-5-sonnet"
# 第一个 chunk
chunk1 = {"choices": [{"delta": {"role": "assistant"}}]}
handler._convert_sse_line(ctx, f"data: {json.dumps(chunk1)}", [])
state_after_first = ctx.stream_conversion_state
message_started_after_first = state_after_first.message_started
# 第二个 chunk
chunk2 = {"choices": [{"delta": {"content": "hello"}}]}
handler._convert_sse_line(ctx, f"data: {json.dumps(chunk2)}", [])
# 状态应该是同一个对象
assert ctx.stream_conversion_state is state_after_first
# message_started 应该保持 True
assert ctx.stream_conversion_state.message_started is True
class TestStreamContextIntegration:
"""StreamContext 集成测试"""
def test_stream_conversion_state_reset_on_retry(self) -> None:
"""测试重试时重置流式转换状态"""
ctx = StreamContext(model="test", api_format="OPENAI")
ctx.stream_conversion_state = StreamConversionState(
model="test", message_id="123", message_started=True
)
ctx.reset_for_retry()
assert ctx.stream_conversion_state is None
def test_stream_conversion_state_field_exists(self) -> None:
"""测试 StreamContext 有 stream_conversion_state 字段"""
ctx = StreamContext(model="test", api_format="OPENAI")
assert hasattr(ctx, "stream_conversion_state")
assert ctx.stream_conversion_state is None

View File

@@ -0,0 +1,406 @@
"""
FormatConverterRegistry 单元测试
测试覆盖:
1. 基本注册和查询
2. 能力查询方法
3. 严格模式转换
4. 流式转换签名适配
"""
from typing import Any, Dict, List
import pytest
from src.core.api_format.conversion.exceptions import FormatConversionError
from src.core.api_format.conversion.registry import FormatConverterRegistry
from src.core.api_format.conversion.state import StreamConversionState
# ==================== Mock 转换器 ====================
class MockRequestResponseConverter:
"""只支持请求/响应转换的 Mock 转换器"""
def convert_request(self, request: Dict[str, Any]) -> Dict[str, Any]:
return {"converted": True, "original": request}
def convert_response(self, response: Dict[str, Any]) -> Dict[str, Any]:
return {"converted": True, "original": response}
class MockNewSignatureStreamConverter:
"""使用新签名 (chunk, state) 的流式转换器"""
def convert_request(self, request: Dict[str, Any]) -> Dict[str, Any]:
return request
def convert_response(self, response: Dict[str, Any]) -> Dict[str, Any]:
return response
def convert_stream_chunk(
self, chunk: Dict[str, Any], state: StreamConversionState
) -> List[Dict[str, Any]]:
"""新签名:返回多个事件"""
events = []
# 模拟一入多出:第一个 chunk 返回 2 个事件
if not state.message_started:
events.append({"type": "message_start", "model": state.model})
state.message_started = True
events.append({"type": "content", "data": chunk})
return events
class MockUnifiedStreamConverter:
"""使用统一签名 (chunk, state) 的流式转换器"""
def convert_request(self, request: Dict[str, Any]) -> Dict[str, Any]:
return request
def convert_response(self, response: Dict[str, Any]) -> Dict[str, Any]:
return response
def convert_stream_chunk(
self,
chunk: Dict[str, Any],
state: StreamConversionState,
) -> List[Dict[str, Any]]:
"""统一签名:返回事件列表"""
events = []
if not state.message_started:
events.append({"type": "message_start", "model": state.model, "id": state.message_id})
state.message_started = True
events.append({"type": "delta", "chunk": chunk})
return events
class MockReturnsNoneConverter:
"""流式转换返回空列表的转换器"""
def convert_request(self, request: Dict[str, Any]) -> Dict[str, Any]:
return request
def convert_response(self, response: Dict[str, Any]) -> Dict[str, Any]:
return response
def convert_stream_chunk(
self,
chunk: Dict[str, Any],
state: StreamConversionState,
) -> List[Dict[str, Any]]:
"""对于不支持的事件类型返回空列表"""
event_type = chunk.get("type")
if event_type == "message_start":
return [{"choices": [{"delta": {"role": "assistant"}}], "model": state.model}]
if event_type == "content_block_delta":
return [{"choices": [{"delta": {"content": chunk.get("text", "")}}]}]
return []
class MockFailingConverter:
"""转换时抛出异常的 Mock 转换器"""
def convert_request(self, request: Dict[str, Any]) -> Dict[str, Any]:
raise ValueError("Request conversion failed")
def convert_response(self, response: Dict[str, Any]) -> Dict[str, Any]:
raise ValueError("Response conversion failed")
def convert_stream_chunk(
self, chunk: Dict[str, Any], state: Any
) -> List[Dict[str, Any]]:
raise ValueError("Stream conversion failed")
# ==================== 测试类 ====================
class TestFormatConverterRegistryBasic:
"""基本注册和查询测试"""
def test_register_and_get_converter(self) -> None:
registry = FormatConverterRegistry()
converter = MockRequestResponseConverter()
registry.register("OPENAI", "CLAUDE", converter)
assert registry.get_converter("OPENAI", "CLAUDE") is converter
assert registry.get_converter("openai", "claude") is converter # 大小写不敏感
def test_get_nonexistent_converter_returns_none(self) -> None:
registry = FormatConverterRegistry()
assert registry.get_converter("OPENAI", "CLAUDE") is None
def test_has_converter(self) -> None:
registry = FormatConverterRegistry()
converter = MockRequestResponseConverter()
registry.register("OPENAI", "CLAUDE", converter)
assert registry.has_converter("OPENAI", "CLAUDE") is True
assert registry.has_converter("CLAUDE", "OPENAI") is False
def test_list_converters(self) -> None:
registry = FormatConverterRegistry()
registry.register("OPENAI", "CLAUDE", MockRequestResponseConverter())
registry.register("CLAUDE", "OPENAI", MockRequestResponseConverter())
converters = registry.list_converters()
assert ("OPENAI", "CLAUDE") in converters
assert ("CLAUDE", "OPENAI") in converters
assert len(converters) == 2
class TestCapabilityQueries:
"""能力查询方法测试"""
def test_can_convert_request(self) -> None:
registry = FormatConverterRegistry()
registry.register("A", "B", MockRequestResponseConverter())
assert registry.can_convert_request("A", "B") is True
assert registry.can_convert_request("B", "A") is False
def test_can_convert_response(self) -> None:
registry = FormatConverterRegistry()
registry.register("A", "B", MockRequestResponseConverter())
assert registry.can_convert_response("A", "B") is True
assert registry.can_convert_response("B", "A") is False
def test_can_convert_stream_with_convert_stream_chunk(self) -> None:
registry = FormatConverterRegistry()
registry.register("A", "B", MockNewSignatureStreamConverter())
assert registry.can_convert_stream("A", "B") is True
assert registry.can_convert_stream("B", "A") is False
def test_can_convert_stream_unified_signature(self) -> None:
registry = FormatConverterRegistry()
registry.register("A", "B", MockUnifiedStreamConverter())
assert registry.can_convert_stream("A", "B") is True
def test_can_convert_full_request_response_only(self) -> None:
"""只有请求/响应转换器,不支持流式"""
registry = FormatConverterRegistry()
registry.register("OPENAI", "CLAUDE", MockRequestResponseConverter())
registry.register("CLAUDE", "OPENAI", MockRequestResponseConverter())
# 不要求流式:通过
assert registry.can_convert_full("OPENAI", "CLAUDE", require_stream=False) is True
# 要求流式:失败
assert registry.can_convert_full("OPENAI", "CLAUDE", require_stream=True) is False
def test_can_convert_full_with_stream(self) -> None:
"""完整双向转换(含流式)"""
registry = FormatConverterRegistry()
registry.register("OPENAI", "CLAUDE", MockNewSignatureStreamConverter())
registry.register("CLAUDE", "OPENAI", MockNewSignatureStreamConverter())
assert registry.can_convert_full("OPENAI", "CLAUDE", require_stream=True) is True
def test_can_convert_full_missing_reverse(self) -> None:
"""缺少反向转换器"""
registry = FormatConverterRegistry()
registry.register("OPENAI", "CLAUDE", MockNewSignatureStreamConverter())
# 没有 CLAUDE -> OPENAI
assert registry.can_convert_full("OPENAI", "CLAUDE", require_stream=False) is False
def test_get_supported_targets(self) -> None:
registry = FormatConverterRegistry()
registry.register("OPENAI", "CLAUDE", MockRequestResponseConverter())
registry.register("OPENAI", "GEMINI", MockRequestResponseConverter())
registry.register("CLAUDE", "OPENAI", MockRequestResponseConverter())
targets = registry.get_supported_targets("OPENAI")
assert "CLAUDE" in targets
assert "GEMINI" in targets
assert len(targets) == 2
class TestStrictModeConversion:
"""严格模式转换测试"""
def test_convert_request_strict_success(self) -> None:
registry = FormatConverterRegistry()
registry.register("A", "B", MockRequestResponseConverter())
result = registry.convert_request_strict({"foo": "bar"}, "A", "B")
assert result["converted"] is True
assert result["original"] == {"foo": "bar"}
def test_convert_request_strict_same_format(self) -> None:
"""同格式返回原始请求"""
registry = FormatConverterRegistry()
result = registry.convert_request_strict({"foo": "bar"}, "A", "A")
assert result == {"foo": "bar"}
def test_convert_request_strict_no_converter(self) -> None:
registry = FormatConverterRegistry()
with pytest.raises(FormatConversionError) as exc_info:
registry.convert_request_strict({"foo": "bar"}, "A", "B")
assert "未找到转换器" in str(exc_info.value)
def test_convert_request_strict_converter_fails(self) -> None:
registry = FormatConverterRegistry()
registry.register("A", "B", MockFailingConverter())
with pytest.raises(FormatConversionError) as exc_info:
registry.convert_request_strict({"foo": "bar"}, "A", "B")
assert "Request conversion failed" in str(exc_info.value)
def test_convert_response_strict_success(self) -> None:
registry = FormatConverterRegistry()
registry.register("A", "B", MockRequestResponseConverter())
result = registry.convert_response_strict({"data": "test"}, "A", "B")
assert result["converted"] is True
def test_convert_response_strict_no_converter(self) -> None:
registry = FormatConverterRegistry()
with pytest.raises(FormatConversionError):
registry.convert_response_strict({"data": "test"}, "A", "B")
class TestStreamChunkStrictConversion:
"""流式转换严格模式测试"""
def test_stream_chunk_strict_same_format(self) -> None:
"""同格式返回原始 chunk 包装在列表中"""
registry = FormatConverterRegistry()
chunk = {"type": "delta", "text": "hello"}
result = registry.convert_stream_chunk_strict(chunk, "A", "A")
assert result == [chunk]
def test_stream_chunk_strict_no_converter(self) -> None:
registry = FormatConverterRegistry()
with pytest.raises(FormatConversionError) as exc_info:
registry.convert_stream_chunk_strict({"type": "delta"}, "A", "B")
assert "未找到转换器" in str(exc_info.value)
def test_stream_chunk_strict_new_signature(self) -> None:
"""测试新签名 (chunk, state) 转换器"""
registry = FormatConverterRegistry()
registry.register("A", "B", MockNewSignatureStreamConverter())
state = StreamConversionState(model="test-model", message_id="msg_123")
chunk = {"content": "hello"}
# 第一次调用:应返回 2 个事件message_start + content
result = registry.convert_stream_chunk_strict(chunk, "A", "B", state)
assert len(result) == 2
assert result[0]["type"] == "message_start"
assert result[1]["type"] == "content"
assert state.message_started is True
# 第二次调用:应返回 1 个事件(只有 content
result2 = registry.convert_stream_chunk_strict({"content": "world"}, "A", "B", state)
assert len(result2) == 1
assert result2[0]["type"] == "content"
def test_stream_chunk_strict_unified_signature(self) -> None:
"""测试统一签名 (chunk, state) 转换器"""
registry = FormatConverterRegistry()
registry.register("A", "B", MockUnifiedStreamConverter())
state = StreamConversionState(
model="gpt-4", message_id="chatcmpl_123", message_started=False
)
chunk = {"text": "hello"}
result = registry.convert_stream_chunk_strict(chunk, "A", "B", state)
assert len(result) == 2
assert result[0]["type"] == "message_start"
assert result[0]["model"] == "gpt-4"
assert result[0]["id"] == "chatcmpl_123"
assert state.message_started is True
def test_stream_chunk_strict_returns_empty_list(self) -> None:
"""测试转换器返回空列表"""
registry = FormatConverterRegistry()
registry.register("CLAUDE", "OPENAI", MockReturnsNoneConverter())
state = StreamConversionState(model="gpt-4", message_id="msg_123")
event = {"type": "message_start"}
result = registry.convert_stream_chunk_strict(event, "CLAUDE", "OPENAI", state)
assert len(result) == 1
assert "choices" in result[0]
assert result[0]["model"] == "gpt-4"
def test_stream_chunk_strict_unknown_event_returns_empty(self) -> None:
"""不支持的事件类型应返回空列表"""
registry = FormatConverterRegistry()
registry.register("CLAUDE", "OPENAI", MockReturnsNoneConverter())
state = StreamConversionState(model="gpt-4", message_id="msg_123")
event = {"type": "unknown_event"} # 不支持的事件类型
result = registry.convert_stream_chunk_strict(event, "CLAUDE", "OPENAI", state)
assert result == []
def test_stream_chunk_strict_converter_fails(self) -> None:
registry = FormatConverterRegistry()
registry.register("A", "B", MockFailingConverter())
state = StreamConversionState(model="test", message_id="123")
with pytest.raises(FormatConversionError) as exc_info:
registry.convert_stream_chunk_strict({"data": "test"}, "A", "B", state)
assert "流式块转换失败" in str(exc_info.value)
class TestNonStrictConversion:
"""非严格模式转换测试(失败时返回原始数据)"""
def test_convert_request_fallback_on_failure(self) -> None:
registry = FormatConverterRegistry()
registry.register("A", "B", MockFailingConverter())
original = {"foo": "bar"}
result = registry.convert_request(original, "A", "B")
# 非严格模式:失败时返回原始请求
assert result == original
def test_convert_response_fallback_on_failure(self) -> None:
registry = FormatConverterRegistry()
registry.register("A", "B", MockFailingConverter())
original = {"data": "test"}
result = registry.convert_response(original, "A", "B")
assert result == original
def test_convert_stream_chunk_fallback_on_failure(self) -> None:
registry = FormatConverterRegistry()
registry.register("A", "B", MockFailingConverter())
original = {"chunk": "data"}
result = registry.convert_stream_chunk(original, "A", "B")
assert result == original

View File

@@ -1,5 +1,5 @@
from src.core.enums import APIFormat
from src.core.headers import (
from src.core.api_format import APIFormat
from src.core.api_format import (
CORE_REDACT_HEADERS,
HeaderBuilder,
build_upstream_headers,