mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-03 01:40:21 +08:00
feat: 多项功能优化与问题修复
- 添加高频轮询端点日志抑制，减少 debug 日志噪音 - Gemini 格式转换支持 responseModalities、thinkingConfig 透传和图片生成输出 - OpenAI 格式转换支持流式图片内容块，区分 URL 引用和 base64 内嵌图片 - Provider 余额缓存认证失败时使用短 TTL，避免前端无限加载中 - Usage 服务支持更新 pending/streaming 记录，处理重复 request_id 冲突 - Usage 超时检测增强，避免错误标记已完成请求为超时
This commit is contained in:
fawney19
@@ -151,6 +151,22 @@ class GeminiNormalizer(FormatNormalizer):
|
||||
else request.get("toolConfig")
|
||||
)
|
||||
|
||||
# 构建 extra，保留原始 gemini 字段
|
||||
extra: Dict[str, Any] = {"gemini": self._extract_extra(request, {"contents"})}
|
||||
|
||||
# 保留 generationConfig 中的特殊字段（responseModalities, thinkingConfig 等）
|
||||
# 这些字段在 _get_generation_config 中已提取，需要单独存储以便转换时使用
|
||||
if isinstance(generation_config, dict):
|
||||
response_modalities = generation_config.get("response_modalities")
|
||||
thinking_config = generation_config.get("thinking_config")
|
||||
if response_modalities or thinking_config:
|
||||
google_extra: Dict[str, Any] = {}
|
||||
if response_modalities:
|
||||
google_extra["response_modalities"] = response_modalities
|
||||
if thinking_config:
|
||||
google_extra["thinking_config"] = thinking_config
|
||||
extra["google"] = google_extra
|
||||
|
||||
internal = InternalRequest(
|
||||
model=model,
|
||||
messages=messages,
|
||||
@@ -164,7 +180,7 @@ class GeminiNormalizer(FormatNormalizer):
|
||||
stream=bool(request.get("stream") or False),
|
||||
tools=tools,
|
||||
tool_choice=tool_choice,
|
||||
extra={"gemini": self._extract_extra(request, {"contents"})},
|
||||
extra=extra,
|
||||
)
|
||||
|
||||
if dropped:
|
||||
@@ -208,6 +224,47 @@ class GeminiNormalizer(FormatNormalizer):
|
||||
if internal.stop_sequences:
|
||||
generation_config["stop_sequences"] = list(internal.stop_sequences)
|
||||
|
||||
# 从 internal.extra["google"] 读取 OpenAI extra_body.google 透传的配置
|
||||
google_extra = internal.extra.get("google", {})
|
||||
if isinstance(google_extra, dict):
|
||||
# 处理 thinking_config -> thinkingConfig
|
||||
thinking_config = google_extra.get("thinking_config")
|
||||
if isinstance(thinking_config, dict):
|
||||
# snake_case -> camelCase 转换
|
||||
gemini_thinking: Dict[str, Any] = {}
|
||||
if "thinking_budget" in thinking_config:
|
||||
gemini_thinking["thinkingBudget"] = thinking_config["thinking_budget"]
|
||||
if "include_thoughts" in thinking_config:
|
||||
gemini_thinking["includeThoughts"] = thinking_config["include_thoughts"]
|
||||
# 保留其他可能的字段
|
||||
for k, v in thinking_config.items():
|
||||
if k not in ("thinking_budget", "include_thoughts"):
|
||||
gemini_thinking[k] = v
|
||||
if gemini_thinking:
|
||||
generation_config["thinkingConfig"] = gemini_thinking
|
||||
|
||||
# 处理 response_modalities -> responseModalities
|
||||
response_modalities = google_extra.get("response_modalities")
|
||||
if response_modalities:
|
||||
generation_config["responseModalities"] = response_modalities
|
||||
|
||||
# 从 internal.extra["gemini"] 读取原生 Gemini 配置（Gemini -> Gemini 场景）
|
||||
gemini_extra = internal.extra.get("gemini", {})
|
||||
if isinstance(gemini_extra, dict):
|
||||
# 保留原生 Gemini generationConfig 中的额外字段
|
||||
orig_gc = gemini_extra.get("generation_config") or gemini_extra.get("generationConfig")
|
||||
if isinstance(orig_gc, dict):
|
||||
# responseModalities
|
||||
if "responseModalities" in orig_gc and "responseModalities" not in generation_config:
|
||||
generation_config["responseModalities"] = orig_gc["responseModalities"]
|
||||
if "response_modalities" in orig_gc and "responseModalities" not in generation_config:
|
||||
generation_config["responseModalities"] = orig_gc["response_modalities"]
|
||||
# thinkingConfig
|
||||
if "thinkingConfig" in orig_gc and "thinkingConfig" not in generation_config:
|
||||
generation_config["thinkingConfig"] = orig_gc["thinkingConfig"]
|
||||
if "thinking_config" in orig_gc and "thinkingConfig" not in generation_config:
|
||||
generation_config["thinkingConfig"] = orig_gc["thinking_config"]
|
||||
|
||||
contents: List[Dict[str, Any]] = []
|
||||
for msg in internal.messages:
|
||||
contents.append(self._internal_message_to_content(msg))
|
||||
@@ -431,6 +488,35 @@ class GeminiNormalizer(FormatNormalizer):
|
||||
events.append(ContentBlockStopEvent(block_index=block_index))
|
||||
continue
|
||||
|
||||
# inlineData（图片生成等多模态输出）
|
||||
inline_data = part.get("inlineData")
|
||||
if inline_data is None:
|
||||
inline_data = part.get("inline_data")
|
||||
|
||||
if isinstance(inline_data, dict):
|
||||
mime_type = str(inline_data.get("mimeType") or inline_data.get("mime_type") or "").strip()
|
||||
data = str(inline_data.get("data") or "").strip()
|
||||
|
||||
# 确保 mime_type 和 data 都非空
|
||||
if mime_type and data and len(data) > 10: # base64 图片数据至少几十个字符
|
||||
block_index = int(ss.get("next_block_index") or 1)
|
||||
ss["next_block_index"] = block_index + 1
|
||||
|
||||
# 使用 ContentBlockStartEvent 传递图片数据
|
||||
# 图片数据存储在 extra 中，供 target normalizer 处理
|
||||
events.append(
|
||||
ContentBlockStartEvent(
|
||||
block_index=block_index,
|
||||
block_type=ContentType.IMAGE,
|
||||
extra={
|
||||
"image_data": data,
|
||||
"image_media_type": mime_type,
|
||||
},
|
||||
)
|
||||
)
|
||||
events.append(ContentBlockStopEvent(block_index=block_index))
|
||||
continue
|
||||
|
||||
finish_reason = candidate0.get("finishReason")
|
||||
if finish_reason is not None:
|
||||
stop_reason = self._FINISH_REASON_TO_STOP.get(str(finish_reason), StopReason.UNKNOWN)
|
||||
@@ -488,6 +574,25 @@ class GeminiNormalizer(FormatNormalizer):
|
||||
}
|
||||
return out
|
||||
|
||||
# 图片内容块（来自其他格式的图像生成输出）
|
||||
if isinstance(event, ContentBlockStartEvent) and event.block_type == ContentType.IMAGE:
|
||||
image_data = event.extra.get("image_data")
|
||||
image_media_type = event.extra.get("image_media_type")
|
||||
if image_data and image_media_type:
|
||||
out.append(
|
||||
base_chunk(
|
||||
[
|
||||
{
|
||||
"inlineData": {
|
||||
"mimeType": image_media_type,
|
||||
"data": image_data,
|
||||
}
|
||||
}
|
||||
]
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
if isinstance(event, ToolCallDeltaEvent):
|
||||
tool_blocks = ss.get("tool_blocks")
|
||||
if isinstance(tool_blocks, dict):
|
||||
@@ -784,6 +889,17 @@ class GeminiNormalizer(FormatNormalizer):
|
||||
normalized["top_p"] = pick("top_p", "topP")
|
||||
normalized["top_k"] = pick("top_k", "topK")
|
||||
normalized["stop_sequences"] = pick("stop_sequences", "stopSequences")
|
||||
|
||||
# 保留 responseModalities（图像生成等多模态输出必需）
|
||||
response_modalities = pick("response_modalities", "responseModalities")
|
||||
if response_modalities:
|
||||
normalized["response_modalities"] = response_modalities
|
||||
|
||||
# 保留 thinkingConfig（思考模式配置）
|
||||
thinking_config = pick("thinking_config", "thinkingConfig")
|
||||
if thinking_config:
|
||||
normalized["thinking_config"] = thinking_config
|
||||
|
||||
return {k: v for k, v in normalized.items() if v is not None}
|
||||
|
||||
def _gemini_tools_to_internal(self, tools: Any) -> Optional[List[ToolDefinition]]:
|
||||
|
||||
@@ -13,6 +13,7 @@ import json
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
from src.core.logger import logger
|
||||
from src.core.api_format.conversion.field_mappings import (
|
||||
ERROR_TYPE_MAPPINGS,
|
||||
RETRYABLE_ERROR_TYPES,
|
||||
@@ -144,6 +145,16 @@ class OpenAINormalizer(FormatNormalizer):
|
||||
mct if mct is not None else request.get("max_tokens")
|
||||
)
|
||||
|
||||
# 构建 extra，保留未识别字段
|
||||
extra: Dict[str, Any] = {"openai": self._extract_extra(request, {"messages"})}
|
||||
|
||||
# 处理 extra_body.google (用于 Gemini 特定功能透传，如 thinkingConfig, responseModalities)
|
||||
extra_body = request.get("extra_body")
|
||||
if isinstance(extra_body, dict):
|
||||
google_extra = extra_body.get("google")
|
||||
if isinstance(google_extra, dict) and google_extra:
|
||||
extra["google"] = google_extra
|
||||
|
||||
internal = InternalRequest(
|
||||
model=model,
|
||||
messages=messages,
|
||||
@@ -156,7 +167,7 @@ class OpenAINormalizer(FormatNormalizer):
|
||||
stream=bool(request.get("stream") or False),
|
||||
tools=tools,
|
||||
tool_choice=tool_choice,
|
||||
extra={"openai": self._extract_extra(request, {"messages"})},
|
||||
extra=extra,
|
||||
)
|
||||
|
||||
if dropped:
|
||||
@@ -470,6 +481,50 @@ class OpenAINormalizer(FormatNormalizer):
|
||||
)
|
||||
return out
|
||||
|
||||
# 图片内容块（来自 Gemini 图像生成等多模态输出）
|
||||
if isinstance(event, ContentBlockStartEvent) and event.block_type == ContentType.IMAGE:
|
||||
image_data = event.extra.get("image_data")
|
||||
image_media_type = event.extra.get("image_media_type")
|
||||
# 确保图片数据有效（base64 数据至少有一定长度）
|
||||
if image_data and image_media_type and isinstance(image_data, str) and len(image_data) > 10:
|
||||
# 构造 data URL 格式的图片
|
||||
data_url = f"data:{image_media_type};base64,{image_data}"
|
||||
# 存储图片数据，在 ContentBlockStopEvent 时输出
|
||||
image_blocks = ss.get("image_blocks")
|
||||
if not isinstance(image_blocks, dict):
|
||||
image_blocks = {}
|
||||
ss["image_blocks"] = image_blocks
|
||||
image_blocks[int(event.block_index)] = {
|
||||
"url": data_url,
|
||||
"media_type": image_media_type,
|
||||
}
|
||||
return out
|
||||
|
||||
# 图片内容块结束时输出
|
||||
if isinstance(event, ContentBlockStopEvent):
|
||||
image_blocks = ss.get("image_blocks")
|
||||
if isinstance(image_blocks, dict):
|
||||
entry = image_blocks.get(int(event.block_index))
|
||||
if isinstance(entry, dict):
|
||||
url = entry.get("url")
|
||||
# 确保 URL 有效（data URL 至少包含 "data:" 前缀 + 一些数据）
|
||||
if url and isinstance(url, str) and len(url) > 20:
|
||||
# OpenAI 流式响应中 delta.content 必须是字符串
|
||||
# 使用 markdown 图片格式，兼容各种客户端渲染
|
||||
# 注意：base64 data URL 可能很长（几百KB~几MB），客户端需支持长内容
|
||||
# 典型图片大小：100KB 原图 ≈ 130KB base64，1MB 原图 ≈ 1.3MB base64
|
||||
if len(url) > 1_000_000: # > 1MB
|
||||
logger.warning(
|
||||
f"Large image in stream response: {len(url)} bytes, "
|
||||
"client may have rendering issues"
|
||||
)
|
||||
out.append(base_chunk({"content": f""}))
|
||||
# 清理已处理的图片
|
||||
del image_blocks[int(event.block_index)]
|
||||
return out
|
||||
# 其他 ContentBlockStopEvent 不处理
|
||||
return out
|
||||
|
||||
if isinstance(event, ToolCallDeltaEvent):
|
||||
tool_index = self._ensure_tool_call_index(ss, event.tool_id)
|
||||
out.append(
|
||||
@@ -854,20 +909,30 @@ class OpenAINormalizer(FormatNormalizer):
|
||||
text_parts.append(b.text)
|
||||
continue
|
||||
if isinstance(b, ImageBlock):
|
||||
url = b.url
|
||||
if not url and b.data and b.media_type:
|
||||
url = f"data:{b.media_type};base64,{b.data}"
|
||||
if url:
|
||||
parts.append({"type": "image_url", "image_url": {"url": url}})
|
||||
# 区分两种图片来源：
|
||||
# 1. URL 引用（OpenAI 原生格式）-> multipart content
|
||||
# 2. base64 内嵌数据（格式转换来的）-> markdown 格式
|
||||
if b.url and not b.data:
|
||||
# OpenAI 原生格式：URL 引用的图片，使用 multipart content
|
||||
parts.append({"type": "image_url", "image_url": {"url": b.url}})
|
||||
elif b.data and b.media_type:
|
||||
# 格式转换来的图片（base64 内嵌），使用 markdown 格式
|
||||
data_url = f"data:{b.media_type};base64,{b.data}"
|
||||
text_parts.append(f"")
|
||||
elif b.url:
|
||||
# 有 URL 也有 data，优先使用 URL
|
||||
parts.append({"type": "image_url", "image_url": {"url": b.url}})
|
||||
continue
|
||||
|
||||
# Unknown / Tool blocks 不进入 OpenAI content
|
||||
|
||||
# 如果有 OpenAI 原生格式的图片（URL 引用），使用 multipart content
|
||||
if parts:
|
||||
if text_parts:
|
||||
parts = [{"type": "text", "text": "\n".join(text_parts)}] + parts
|
||||
return parts
|
||||
|
||||
# 纯文本或包含 markdown 图片
|
||||
if text_parts:
|
||||
return "\n".join(text_parts)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user