i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 01:10:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 重构格式转换系统为 Hub-and-Spoke Normalizer 架构

Browse Source 
- 移除旧的 converters 模块，改用基于 Internal 中间表示的 Normalizer 模式
- 新增 internal.py 定义统一的内部数据结构（InternalRequest/Response/Error）
- 新增 normalizer.py 定义 FormatNormalizer 基类接口
- 实现 OpenAI/Claude/Gemini 及其 CLI 格式的 Normalizer
- 新增 stream_events.py 和 stream_state.py 支持流式转换
- 重构 registry.py 使用 Normalizer 进行格式转换
- 更新 handlers 适配新的转换架构
- 前端移除 CLI 格式转换按钮的限制
- 添加 golden data 测试确保转换正确性
This commit is contained in:
fawney19
2026-01-27 02:17:18 +08:00
parent 6cf251b19a
commit add477fece
84 changed files with 9169 additions and 3932 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
tests/api/handlers/base/test_cli_handler_convert.py
View File

@@ -15,7 +15,7 @@ from unittest.mock import MagicMock, patch
import pytest
from src.api.handlers.base.stream_context import StreamContext
from src.core.api_format import StreamConversionState
from src.core.api_format.conversion.stream_state import StreamState
# Mock CliMessageHandlerBase 用于测试
@@ -29,7 +29,12 @@ class MockCliHandler:
events: list,
) -> List[str]:
"""复制自 CliMessageHandlerBase._convert_sse_line"""
from src.core.api_format import converter_registry
from src.core.api_format.conversion import (
format_conversion_registry,
register_default_normalizers,
)
register_default_normalizers()
# 如果是空行或特殊控制行，直接返回
if not line or line.strip() == "" or line == "data: [DONE]":
@@ -50,7 +55,7 @@ class MockCliHandler:
# 初始化流式转换状态
if ctx.stream_conversion_state is None:
ctx.stream_conversion_state = StreamConversionState(
ctx.stream_conversion_state = StreamState(
model=ctx.mapped_model or ctx.model,
message_id=ctx.response_id or ctx.request_id,
)
@@ -59,19 +64,13 @@ class MockCliHandler:
client_format = ctx.client_api_format or ""
try:
converted_events = converter_registry.convert_stream_chunk_strict(
converted_events = format_conversion_registry.convert_stream_chunk(
data_obj,
provider_format,
client_format,
state=ctx.stream_conversion_state,
)
if converted_events and not ctx.stream_conversion_state.message_started:
for evt in converted_events:
if evt.get("type") == "message_start" or evt.get("choices"):
ctx.stream_conversion_state.message_started = True
break
result = []
for evt in converted_events:
result.append(f"data: {json.dumps(evt, ensure_ascii=False)}")
@@ -168,25 +167,13 @@ class TestConvertSseLineOneInManyOut:
@pytest.fixture(autouse=True)
def setup_converters(self):
"""注册测试用转换器"""
from src.core.api_format import (
ClaudeToOpenAIConverter,
OpenAIToClaudeConverter,
converter_registry,
)
"""确保 Canonical normalizers 已注册"""
from src.core.api_format.conversion import register_default_normalizers
# 保存原始状态
original_converters = converter_registry._converters.copy()
# 注册转换器
converter_registry.register("OPENAI", "CLAUDE", OpenAIToClaudeConverter())
converter_registry.register("CLAUDE", "OPENAI", ClaudeToOpenAIConverter())
register_default_normalizers()
yield
# 恢复原始状态
converter_registry._converters = original_converters
def test_openai_to_claude_conversion(self) -> None:
"""测试 OpenAI -> Claude 流式转换"""
handler = MockCliHandler()
@@ -211,9 +198,6 @@ class TestConvertSseLineOneInManyOut:
first_event = json.loads(result1[0][6:])
assert first_event.get("type") == "message_start"
# 状态应更新
assert ctx.stream_conversion_state.message_started is True
def test_claude_to_openai_conversion(self) -> None:
"""测试 Claude -> OpenAI 流式转换"""
handler = MockCliHandler()
@@ -247,7 +231,6 @@ class TestConvertSseLineOneInManyOut:
handler._convert_sse_line(ctx, f"data: {json.dumps(chunk1)}", [])
state_after_first = ctx.stream_conversion_state
message_started_after_first = state_after_first.message_started
# 第二个 chunk
chunk2 = {"choices": [{"delta": {"content": "hello"}}]}
@@ -255,8 +238,6 @@ class TestConvertSseLineOneInManyOut:
# 状态应该是同一个对象
assert ctx.stream_conversion_state is state_after_first
# message_started 应该保持 True
assert ctx.stream_conversion_state.message_started is True
class TestStreamContextIntegration:
@@ -265,9 +246,7 @@ class TestStreamContextIntegration:
def test_stream_conversion_state_reset_on_retry(self) -> None:
"""测试重试时重置流式转换状态"""
ctx = StreamContext(model="test", api_format="OPENAI")
ctx.stream_conversion_state = StreamConversionState(
model="test", message_id="123", message_started=True
)
ctx.stream_conversion_state = StreamState(model="test", message_id="123")
ctx.reset_for_retry()

4
tests/api/handlers/base/test_stream_processor_format_conversion.py
View File

@@ -7,7 +7,7 @@ import pytest
from src.api.handlers.base.response_parser import ParsedResponse, ResponseParser, StreamStats
from src.api.handlers.base.stream_context import StreamContext
from src.api.handlers.base.stream_processor import StreamProcessor
from src.core.api_format import register_all_converters
from src.core.api_format.conversion import register_default_normalizers
class DummyParser(ResponseParser):
@@ -31,7 +31,7 @@ async def _empty_async_iter():
@pytest.mark.asyncio
async def test_create_response_stream_converts_claude_to_openai() -> None:
register_all_converters()
register_default_normalizers()
ctx = StreamContext(model="test-model", api_format="OPENAI")
ctx.client_api_format = "OPENAI"

122
tests/api/handlers/base/test_stream_processor_gemini_json_fallback.py
View File

@@ -6,7 +6,7 @@ import pytest
from src.api.handlers.base.parsers import get_parser_for_format
from src.api.handlers.base.stream_context import StreamContext
from src.api.handlers.base.stream_processor import StreamProcessor
from src.core.api_format import GeminiToOpenAIConverter, converter_registry
from src.core.api_format.conversion import register_default_normalizers
class _DummyResponseCtx:
@@ -26,77 +26,71 @@ async def _iter_bytes(chunks: list[bytes]) -> AsyncIterator[bytes]:
@pytest.mark.asyncio
async def test_stream_processor_converts_gemini_json_lines_without_data_prefix() -> None:
# Register only what we need, and restore after test.
original_converters = converter_registry._converters.copy()
try:
converter_registry.register("GEMINI", "OPENAI", GeminiToOpenAIConverter())
register_default_normalizers()
ctx = StreamContext(model="gemini-test", api_format="OPENAI")
ctx.provider_api_format = "GEMINI"
ctx.client_api_format = "OPENAI"
ctx.needs_conversion = True
ctx.request_id = "req_test"
ctx.mapped_model = "gemini-test"
ctx = StreamContext(model="gemini-test", api_format="OPENAI")
ctx.provider_api_format = "GEMINI"
ctx.client_api_format = "OPENAI"
ctx.needs_conversion = True
ctx.request_id = "req_test"
ctx.mapped_model = "gemini-test"
# Simulate Gemini JSON-array/chunks stream: wrapper lines + two JSON objects.
chunk1 = {
"candidates": [
{
"content": {"parts": [{"text": "Hello"}], "role": "model"},
}
]
}
chunk2 = {
"candidates": [
{
"content": {"parts": [{"text": "Hello world"}], "role": "model"},
"finishReason": "STOP",
}
],
"usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 2, "totalTokenCount": 3},
}
upstream_lines = [
b"[\n",
(json.dumps(chunk1) + ",\n").encode("utf-8"),
(json.dumps(chunk2) + "\n").encode("utf-8"),
b"]\n",
# Simulate Gemini JSON-array/chunks stream: wrapper lines + two JSON objects.
chunk1 = {
"candidates": [
{
"content": {"parts": [{"text": "Hello"}], "role": "model"},
}
]
}
chunk2 = {
"candidates": [
{
"content": {"parts": [{"text": "Hello world"}], "role": "model"},
"finishReason": "STOP",
}
],
"usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 2, "totalTokenCount": 3},
}
processor = StreamProcessor(
request_id="req_test",
default_parser=get_parser_for_format("OPENAI"),
)
upstream_lines = [
b"[\n",
(json.dumps(chunk1) + ",\n").encode("utf-8"),
(json.dumps(chunk2) + "\n").encode("utf-8"),
b"]\n",
]
out = b""
async for b in processor.create_response_stream(
ctx=ctx,
byte_iterator=_iter_bytes(upstream_lines),
response_ctx=_DummyResponseCtx(),
http_client=_DummyHTTPClient(), # type: ignore[arg-type]
prefetched_chunks=[],
start_time=None,
):
out += b
processor = StreamProcessor(
request_id="req_test",
default_parser=get_parser_for_format("OPENAI"),
)
text = out.decode("utf-8", errors="replace")
data_lines = [ln for ln in text.splitlines() if ln.startswith("data: ")]
out = b""
async for b in processor.create_response_stream(
ctx=ctx,
byte_iterator=_iter_bytes(upstream_lines),
response_ctx=_DummyResponseCtx(),
http_client=_DummyHTTPClient(), # type: ignore[arg-type]
prefetched_chunks=[],
start_time=None,
):
out += b
# OpenAI termination marker should be present (StreamProcessor will append if upstream doesn't send it).
assert "data: [DONE]" in data_lines
text = out.decode("utf-8", errors="replace")
data_lines = [ln for ln in text.splitlines() if ln.startswith("data: ")]
# Parse JSON events (excluding [DONE]) and validate we have expected deltas.
events = [json.loads(ln[6:]) for ln in data_lines if ln != "data: [DONE]"]
# OpenAI termination marker should be present (StreamProcessor will append if upstream doesn't send it).
assert "data: [DONE]" in data_lines
delta_contents: list[str] = []
for evt in events:
for choice in evt.get("choices", []) or []:
delta = choice.get("delta") or {}
if "content" in delta and delta["content"]:
delta_contents.append(delta["content"])
# Parse JSON events (excluding [DONE]) and validate we have expected deltas.
events = [json.loads(ln[6:]) for ln in data_lines if ln != "data: [DONE]"]
assert "Hello" in "".join(delta_contents)
assert " world" in "".join(delta_contents)
finally:
converter_registry._converters = original_converters
delta_contents: list[str] = []
for evt in events:
for choice in evt.get("choices", []) or []:
delta = choice.get("delta") or {}
if "content" in delta and delta["content"]:
delta_contents.append(delta["content"])
assert "Hello" in "".join(delta_contents)
assert " world" in "".join(delta_contents)