refactor: 完善跨格式 normalizer 转换精度,统一 CLI 流式 buffer flush 逻辑

- OpenAI normalizer: 修正 file/image 内容块的标准格式解析与输出,
  assistant 有 tool_calls 时 content 输出 null,非流式 tool_calls
  不再包含 index 字段,保留 system_fingerprint/service_tier roundtrip
- OpenAI CLI normalizer: 支持 reasoning/ThinkingConfig 双向转换,
  补全 parallel_tool_calls/required tool_choice,input_file 解析,
  function_call_output.output 强制字符串化,流式事件补全 item_id/
  output_index/content_index 字段
- Gemini normalizer: 扩展 finishReason 映射,自动检测 tool_use
  stop_reason,保留 safetySettings/cachedContent/generationConfig
  额外字段 roundtrip,error_from_internal 使用精确 HTTP status code
- Claude normalizer: 修正 tool_choice type="tool" 输出,扩展
  extra 提取白名单
- internal.py: 为所有 dataclass 补充跨格式映射文档和修改须知
- stream_bridge: aggregator 新增 open_count/final_count 诊断属性,
  build() 时 flush 未关闭的 open blocks
- CLI handler: 提取 _flush_buffer_with_conversion 统一 prefetch/
  stream 两条路径的 buffer + SSE parser flush 逻辑
- upstream_stream_bridge: 新增事件类型计数和聚合器状态诊断日志
- 新增 fixtures 和测试: roundtrip/to_internal/cross_format/error/
  stream 等多维度转换测试
This commit is contained in:
fawney19
2026-02-20 19:45:14 +08:00
parent d7f0a555c0
commit 37758c2032
26 changed files with 5441 additions and 125 deletions

View File

@@ -2,7 +2,9 @@
from __future__ import annotations
import codecs
import json
from collections.abc import Iterator
from typing import TYPE_CHECKING, Any
from src.api.handlers.base.parsers import get_parser_for_format
@@ -10,6 +12,7 @@ from src.api.handlers.base.stream_context import StreamContext
from src.api.handlers.base.utils import get_format_converter_registry
from src.core.logger import logger
from src.services.provider.behavior import get_provider_behavior
from src.utils.sse_parser import SSEEventParser
from .cli_sse_helpers import (
_format_converted_events_to_sse,
@@ -281,6 +284,68 @@ class CliEventMixin:
if any([new_input, new_output, new_cached, new_cache_creation]):
ctx.final_usage = usage
def _flush_buffer_with_conversion(
self: CliHandlerProtocol,
ctx: StreamContext,
buffer: bytes,
decoder: codecs.IncrementalDecoder,
sse_parser: SSEEventParser,
needs_conversion: bool,
) -> Iterator[bytes]:
"""flush 字节 buffer 残余数据 + SSE parser 内部缓冲区,并做格式转换。
正常流结束时调用(区别于异常路径的 _flush_remaining_sse_data
当 needs_conversion=True 时 yield 转换后的 SSE 行;
当 needs_conversion=False 时 yield 原始行透传。
"""
# 1) flush 字节 buffer 中的残余数据(最后一个 chunk 可能不以换行结尾)
if buffer:
try:
remaining = decoder.decode(buffer, True)
except Exception:
remaining = ""
for tail_line in remaining.split("\n"):
stripped = tail_line.rstrip("\r")
tail_events = sse_parser.feed_line(stripped)
if stripped == "":
for event in tail_events:
self._handle_sse_event(
ctx,
event.get("event"),
event.get("data") or "",
record_chunk=not needs_conversion,
)
continue
if needs_conversion and stripped:
converted_lines, converted_events = self._convert_sse_line(
ctx, stripped, tail_events
)
self._record_converted_chunks(ctx, converted_events)
for converted_line in converted_lines:
if converted_line:
yield (converted_line + "\n").encode("utf-8")
elif stripped:
# 非 conversion 模式:透传原始行
yield (stripped + "\n").encode("utf-8")
# 2) flush SSE parser 内部缓冲区中的残余事件
for event in sse_parser.flush():
self._handle_sse_event(
ctx,
event.get("event"),
event.get("data") or "",
record_chunk=not needs_conversion,
)
if needs_conversion:
data_str = event.get("data") or ""
if data_str and data_str != "[DONE]":
flush_line = f"data: {data_str}"
converted_lines, converted_events = self._convert_sse_line(ctx, flush_line, [])
self._record_converted_chunks(ctx, converted_events)
for converted_line in converted_lines:
if converted_line:
yield (converted_line + "\n").encode("utf-8")
def _finalize_stream_metadata(self, ctx: StreamContext) -> None:
"""
在记录统计前从 parsed_chunks 中提取额外的元数据 - 子类可覆盖

View File

@@ -515,15 +515,11 @@ class CliPrefetchMixin:
if ctx.data_count > 0:
last_data_time = time.time()
# 处理剩余事件
flushed_events = sse_parser.flush()
for event in flushed_events:
self._handle_sse_event(
ctx,
event.get("event"),
event.get("data") or "",
record_chunk=not needs_conversion,
)
# flush 字节 buffer 残余数据 + SSE parser 内部缓冲区
for chunk in self._flush_buffer_with_conversion(
ctx, buffer, decoder, sse_parser, needs_conversion
):
yield chunk
# 检查是否收到数据
if ctx.data_count == 0:

View File

@@ -203,6 +203,15 @@ class CliHandlerProtocol(Protocol):
converted_events: list[dict[str, Any]],
) -> None: ...
def _flush_buffer_with_conversion(
self,
ctx: StreamContext,
buffer: bytes,
decoder: Any,
sse_parser: Any,
needs_conversion: bool,
) -> Any: ... # Iterator[bytes]
def _finalize_stream_metadata(
self,
ctx: StreamContext,

View File

@@ -909,14 +909,11 @@ class CliStreamMixin:
if ctx.data_count > 0:
last_data_time = time.time()
# 处理剩余事件
for event in sse_parser.flush():
self._handle_sse_event(
ctx,
event.get("event"),
event.get("data") or "",
record_chunk=not needs_conversion,
)
# flush 字节 buffer 残余数据 + SSE parser 内部缓冲区
for chunk in self._flush_buffer_with_conversion(
ctx, buffer, decoder, sse_parser, needs_conversion
):
yield chunk
# 检查是否收到数据
if ctx.data_count == 0:

View File

@@ -12,6 +12,7 @@ from __future__ import annotations
import codecs
import json
from collections import Counter
from collections.abc import AsyncIterator
from typing import Any
@@ -148,7 +149,11 @@ async def aggregate_upstream_stream_to_internal_response(
buffer = b""
decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
_event_type_counts: Counter[str] = Counter()
_total_events = 0
def _feed_line(normalized_line: str) -> None:
nonlocal _total_events
data_obj, st = parse_provider_stream_line_to_json(normalized_line, provider_api_format)
if st != "ok" or data_obj is None:
return
@@ -171,6 +176,9 @@ async def aggregate_upstream_stream_to_internal_response(
error_status=parsed.error_type,
)
etype = str(data_obj.get("type") or "")
_event_type_counts[etype] += 1
_total_events += 1
internal_events = src_norm.stream_chunk_to_internal(data_obj, state)
aggregator.feed(internal_events)
@@ -193,7 +201,35 @@ async def aggregate_upstream_stream_to_internal_response(
if normalized_tail:
_feed_line(normalized_tail)
return aggregator.build()
# 诊断日志:在 build() 之前记录聚合器状态
open_count = aggregator.open_count
final_count = aggregator.final_count
if not final_count and not open_count:
logger.warning(
"[{}] aggregate_upstream_stream: 聚合器无内容, "
"open={}, final={}, usage={}, stop_reason={}, "
"event_types={}, total_events={}",
request_id,
open_count,
final_count,
aggregator.usage,
aggregator.stop_reason,
dict(_event_type_counts),
_total_events,
)
elif not final_count and open_count:
logger.warning(
"[{}] aggregate_upstream_stream: final 为空但 open 有内容, "
"open={}, final={}, event_types={}, total_events={}",
request_id,
open_count,
final_count,
dict(_event_type_counts),
_total_events,
)
result = aggregator.build()
return result
__all__ = [