refactor: 重构异步任务系统和计费服务架构

- 重构任务系统:新增 lifecycle (TaskStatus/BillingStatus)、context、application 模块
- 将 video tasks 泛化为 async tasks,支持更通用的异步任务管理
- 新增 Gemini Files 管理模块和管理界面
- 重构 billing 服务:拆分 schema.py 和 service.py
- 新增 candidate 服务模块用于请求候选管理
- 数据库迁移:添加 billing_status、request_id、gemini_file_mappings 表和索引
- 移除废弃的 video_telemetry、task orchestrator 等模块
This commit is contained in:
fawney19
2026-02-02 03:16:52 +08:00
parent feb7484fda
commit 9e31efe26c
75 changed files with 7511 additions and 2068 deletions

View File

@@ -3,11 +3,20 @@
用于候选筛选时判断端点是否可以处理客户端请求格式。
三层开关优先级(从高到低):
1. 全局开关 ON → 强制允许(跳过后续检查)
2. 全局开关 OFF → 看提供商开关
- 提供商开关 ON → 强制允许(跳过端点检查)
- 提供商开关 OFF → 看端点配置
3. 端点配置format_acceptance_config
- enabled=true + 白名单/黑名单检查 → 允许
- enabled=false 或未配置 → 禁止
转换逻辑:
1. 格式完全匹配 -> 透传(无需转换)
2. 格式不同 -> 需要检查全局开关 + 端点开关
- data_format_id 相同 -> 可透传(无需数据转换),但需全局开关 + 端点开关
- data_format_id 不同 -> 需要转换,检查全局开关 + 端点配置 + 转换器能力
2. 格式不同 -> 需要检查三层开关
- data_format_id 相同 -> 可透传(无需数据转换)
- data_format_id 不同 -> 需要转换,检查转换器能力
"""
from __future__ import annotations
@@ -28,8 +37,10 @@ def is_format_compatible(
endpoint_api_format: str,
endpoint_format_acceptance_config: dict | None,
is_stream: bool,
global_conversion_enabled: bool,
effective_conversion_enabled: bool,
registry: FormatConversionRegistry | None = None,
*,
skip_endpoint_check: bool = False,
) -> tuple[bool, bool, str | None]:
"""
检查端点是否兼容客户端格式
@@ -39,8 +50,9 @@ def is_format_compatible(
endpoint_api_format: 端点的 API 格式
endpoint_format_acceptance_config: 端点的格式接受配置
is_stream: 是否是流式请求
global_conversion_enabled: 全局格式转换开关(来自环境变量 FORMAT_CONVERSION_ENABLED默认 True
effective_conversion_enabled: 有效格式转换开关(全局 OR 提供商
registry: 转换器注册表(可选,默认使用全局单例)
skip_endpoint_check: 是否跳过端点配置检查(当全局或提供商开关为 ON 时设为 True
Returns:
(is_compatible, needs_conversion, skip_reason)
@@ -66,30 +78,36 @@ def is_format_compatible(
if provider_key == client_key:
return True, False, None
# 2. 格式不同 -> 需要检查全局格式转换开关
# 即使 data_format_id 相同(如 claude:chat / claude:cli也需要全局开关启用
if not global_conversion_enabled:
return False, False, "全局格式转换未启用(环境变量 FORMAT_CONVERSION_ENABLED=false"
# 2. 格式不同 -> 需要检查格式转换开关
# 如果有效开关为 False全局 OFF 且提供商 OFF直接拒绝
if not effective_conversion_enabled:
return False, False, "格式转换已禁用(全局和提供商开关均为关闭"
# 3. 格式不同时,统一检查端点配置(核心控制)
if endpoint_format_acceptance_config is None:
return False, False, "端点配置格式接受策略"
# 3. 如果全局或提供商开关为 ON跳过端点配置检查
if not skip_endpoint_check:
# 检查端点配置(第三层开关)
if endpoint_format_acceptance_config is None:
return False, False, "端点未配置格式接受策略"
config = endpoint_format_acceptance_config
if not isinstance(config, dict):
return False, False, "端点格式配置无效"
if not config.get("enabled", False):
return False, False, "端点格式接受未启用"
config = endpoint_format_acceptance_config
if not isinstance(config, dict):
return False, False, "端点格式配置无效"
if not config.get("enabled", False):
return False, False, "端点格式接受未启用"
# 检查 reject_formats优先
reject_formats = config.get("reject_formats", [])
if client_key in [f.upper() for f in reject_formats]:
return False, False, f"端点拒绝 {client_format} 格式"
# 检查 reject_formats优先
reject_formats = config.get("reject_formats", [])
if client_key in [f.upper() for f in reject_formats]:
return False, False, f"端点拒绝 {client_format} 格式"
# 检查 accept_formats
accept_formats = config.get("accept_formats", [])
if accept_formats and client_key not in [f.upper() for f in accept_formats]:
return False, False, f"端点不接受 {client_format} 格式"
# 检查 accept_formats
accept_formats = config.get("accept_formats", [])
if accept_formats and client_key not in [f.upper() for f in accept_formats]:
return False, False, f"端点不接受 {client_format} 格式"
# 检查流式转换
if is_stream and not config.get("stream_conversion", True):
return False, False, "端点不支持流式格式转换"
# 4. 检查是否可以透传data_format_id 相同)
# 例如claude:chat / claude:cli 的 data_format_id 都是 "claude",数据格式相同可透传
@@ -99,11 +117,7 @@ def is_format_compatible(
return True, False, None
# 5. 需要数据转换的情况data_format_id 不同)
# 检查流式转换
if is_stream and not config.get("stream_conversion", True):
return False, False, "端点不支持流式格式转换"
# 6. 检查转换器能力
# 检查转换器能力
if not registry.can_convert_full(
client_key,
provider_key,

View File

@@ -736,10 +736,17 @@ class GeminiNormalizer(FormatNormalizer):
instance = instances[0] if isinstance(instances[0], dict) else {}
params = request.get("parameters") or {}
# 解析 image用于 image-to-video 或第一帧)
# 官方格式: {"image": {"inlineData": {"mimeType": "image/png", "data": "base64..."}}}
image = instance.get("image") if isinstance(instance, dict) else None
image_ref = None
if isinstance(image, dict):
image_ref = image.get("bytesBase64Encoded")
inline_data = image.get("inlineData", {})
if isinstance(inline_data, dict):
image_ref = inline_data.get("data")
# 兼容旧格式
if not image_ref:
image_ref = image.get("bytesBase64Encoded")
prompt = instance.get("prompt") if isinstance(instance, dict) else None
prompt_str = str(prompt).strip() if prompt else ""
@@ -747,7 +754,7 @@ class GeminiNormalizer(FormatNormalizer):
raise ValueError("Video prompt is required")
duration_raw = params.get("durationSeconds")
sample_count_raw = params.get("sampleCount")
sample_count_raw = params.get("sampleCount") or params.get("numberOfVideos")
try:
duration_seconds = int(duration_raw) if duration_raw else 8
@@ -760,6 +767,35 @@ class GeminiNormalizer(FormatNormalizer):
except (ValueError, TypeError):
sample_count = 1
# 构建 extra 字段,保留所有 Veo 特有的参数
extra: dict[str, Any] = {
"personGeneration": params.get("personGeneration"),
"sampleCount": sample_count,
}
# negativePrompt - 负面提示词
if params.get("negativePrompt"):
extra["negativePrompt"] = params["negativePrompt"]
# lastFrame - 最后一帧(用于插值)
last_frame = params.get("lastFrame")
if isinstance(last_frame, dict):
extra["lastFrame"] = last_frame
# referenceImages - 参考图像最多3张仅 Veo 3.1
ref_images = params.get("referenceImages")
if isinstance(ref_images, list) and ref_images:
extra["referenceImages"] = ref_images
# video - 视频扩展输入(用于视频续写)
video_input = instance.get("video") if isinstance(instance, dict) else None
if isinstance(video_input, dict):
extra["video"] = video_input
# seed - 种子值Veo 3
if params.get("seed") is not None:
extra["seed"] = params["seed"]
return InternalVideoRequest(
prompt=prompt_str,
model=str(request.get("model") or "veo-3.1-generate-preview"),
@@ -767,10 +803,7 @@ class GeminiNormalizer(FormatNormalizer):
aspect_ratio=str(params.get("aspectRatio") or "16:9"),
resolution=str(params.get("resolution") or "720p"),
reference_image_url=image_ref,
extra={
"personGeneration": params.get("personGeneration"),
"sampleCount": sample_count,
},
extra=extra,
)
def video_request_from_internal(self, internal: InternalVideoRequest) -> dict[str, Any]:
@@ -783,15 +816,31 @@ class GeminiNormalizer(FormatNormalizer):
"resolution": internal.resolution,
"durationSeconds": internal.duration_seconds,
}
for key in ["personGeneration", "sampleCount"]:
for key in ["personGeneration", "sampleCount", "negativePrompt", "seed"]:
if key in internal.extra:
parameters[key] = internal.extra[key]
return {
# lastFrame 和 referenceImages 需要特殊处理
if internal.extra.get("lastFrame"):
parameters["lastFrame"] = internal.extra["lastFrame"]
if internal.extra.get("referenceImages"):
parameters["referenceImages"] = internal.extra["referenceImages"]
# video 输入(视频续写)
if internal.extra.get("video"):
instance["video"] = internal.extra["video"]
result: dict[str, Any] = {
"instances": [instance],
"parameters": parameters,
}
# 模型信息(用于 URL 构建)
if internal.model:
result["model"] = internal.model
return result
def video_task_to_internal(self, response: dict[str, Any]) -> InternalVideoTask:
operation_name = str(response.get("name") or "")
done = bool(response.get("done"))
@@ -824,20 +873,30 @@ class GeminiNormalizer(FormatNormalizer):
)
def video_task_from_internal(self, internal: InternalVideoTask) -> dict[str, Any]:
# 优先使用 external_id(上游返回的 operation name),否则用内部 id
operation_name = internal.external_id or f"operations/{internal.id}"
if not operation_name.startswith("operations/"):
operation_name = f"operations/{operation_name}"
# external_id 中提取 model 名称,用于构建 operation name
# external_id 格式: models/{model}/operations/{gemini_id}
model_name = "unknown"
if internal.external_id:
parts = internal.external_id.split("/")
if len(parts) >= 2 and parts[0] == "models":
model_name = parts[1]
# 使用我们的内部 task_id 构建 operation name不暴露 Gemini 的 operation ID
# 格式: models/{model}/operations/{our_task_id}
operation_name = f"models/{model_name}/operations/{internal.id}"
if internal.status == VideoStatus.COMPLETED:
urls = internal.video_urls or ([internal.video_url] if internal.video_url else [])
# 使用我们的内部 task_id 构建下载 URL不暴露真实的 Gemini file_id
# 使用 aev_ 前缀标识这是视频任务的下载链接
# 格式:/v1beta/files/aev_{task_id}:download?alt=media
proxy_download_url = f"/v1beta/files/aev_{internal.id}:download?alt=media"
return {
"name": operation_name,
"done": True,
"response": {
"generateVideoResponse": {
"generatedSamples": [
{"video": {"uri": url, "mimeType": "video/mp4"}} for url in urls
{"video": {"uri": proxy_download_url, "mimeType": "video/mp4"}}
]
}
},

View File

@@ -752,10 +752,21 @@ class OpenAINormalizer(FormatNormalizer):
"message": internal.error_message,
}
for key in ["model", "size", "seconds"]:
if key in internal.extra:
# 基本字段
for key in ["model", "size", "prompt"]:
if internal.extra.get(key):
payload[key] = internal.extra[key]
# seconds 必须是字符串类型
seconds = internal.extra.get("seconds")
if seconds is not None:
payload["seconds"] = str(seconds)
# remix 相关字段
remixed_from = internal.extra.get("remixed_from_video_id")
if remixed_from:
payload["remixed_from_video_id"] = remixed_from
return payload
def video_poll_to_internal(self, response: dict[str, Any]) -> InternalVideoPollResult:
@@ -764,14 +775,18 @@ class OpenAINormalizer(FormatNormalizer):
if status == "completed":
expires_at = response.get("expires_at")
# 使用任务 ID 构建内容路径,由调用方拼接完整 URL
# 如果 task_id 不存在,说明上游响应异常
video_url = f"videos/{task_id}/content" if task_id else None
# 优先使用上游返回的直接 URL某些代理如 API易 会返回 CDN URL
# 回退到构建相对路径(标准 OpenAI API 通过 /content 端点下载)
direct_url = (
response.get("video_url") or response.get("url") or response.get("result_url")
)
# 使用直接 URL 或回退到相对路径
video_url = direct_url or (f"videos/{task_id}/content" if task_id else None)
if not video_url:
return InternalVideoPollResult(
status=VideoStatus.FAILED,
error_code="missing_task_id",
error_message="Upstream response missing task id",
error_code="missing_video_url",
error_message="Upstream response missing video url",
raw_response=response,
)
return InternalVideoPollResult(

View File

@@ -155,6 +155,110 @@ class FormatConversionRegistry:
except Exception as e:
raise FormatConversionError(source_format, target_format, str(e)) from e
# ==================== 视频格式转换 ====================
def convert_video_request(
self,
request: dict[str, Any],
source_format: str,
target_format: str,
) -> dict[str, Any]:
"""转换视频请求格式OpenAI <-> Gemini
Args:
request: 原始视频请求
source_format: 源格式(如 openai:video, gemini:video
target_format: 目标格式
Returns:
转换后的视频请求
"""
# 统一使用基础格式 ID去掉 :video 后缀)
src_base = self._video_format_to_base(source_format)
tgt_base = self._video_format_to_base(target_format)
if src_base == tgt_base:
return request
src = self._require_normalizer(src_base)
tgt = self._require_normalizer(tgt_base)
with _track_conversion_metrics(
"video_request", str(source_format).upper(), str(target_format).upper()
):
try:
internal = src.video_request_to_internal(request)
return tgt.video_request_from_internal(internal)
except Exception as e:
raise FormatConversionError(source_format, target_format, str(e)) from e
def convert_video_task(
self,
task_response: dict[str, Any],
source_format: str,
target_format: str,
) -> dict[str, Any]:
"""转换视频任务响应格式OpenAI <-> Gemini
Args:
task_response: 原始任务响应
source_format: 源格式
target_format: 目标格式
Returns:
转换后的任务响应
"""
src_base = self._video_format_to_base(source_format)
tgt_base = self._video_format_to_base(target_format)
if src_base == tgt_base:
return task_response
src = self._require_normalizer(src_base)
tgt = self._require_normalizer(tgt_base)
with _track_conversion_metrics(
"video_task", str(source_format).upper(), str(target_format).upper()
):
try:
internal = src.video_task_to_internal(task_response)
return tgt.video_task_from_internal(internal)
except Exception as e:
raise FormatConversionError(source_format, target_format, str(e)) from e
def can_convert_video(self, source_format: str, target_format: str) -> bool:
"""检查是否支持视频格式转换"""
src_base = self._video_format_to_base(source_format)
tgt_base = self._video_format_to_base(target_format)
if src_base == tgt_base:
return True
src = self.get_normalizer(src_base)
tgt = self.get_normalizer(tgt_base)
if src is None or tgt is None:
return False
# 检查是否有视频转换方法
return (
hasattr(src, "video_request_to_internal")
and hasattr(src, "video_task_to_internal")
and hasattr(tgt, "video_request_from_internal")
and hasattr(tgt, "video_task_from_internal")
)
def _video_format_to_base(self, format_id: str) -> str:
"""将视频格式 ID 转换为基础格式 ID
例如: openai:video -> openai:chat, gemini:video -> gemini:chat
"""
upper = str(format_id).upper()
if upper.endswith(":VIDEO"):
base = upper[:-6] # 去掉 :VIDEO
return f"{base}:CHAT"
return upper
# ==================== 流式转换(严格) ====================
def convert_stream_chunk(

View File

@@ -248,10 +248,10 @@ register_capability(
)
register_capability(
name="gemini_files_api",
display_name="Gemini文件上传",
description="支持 Gemini Files API上传、查询、删除),第三方 Key 通常不支持",
match_mode=CapabilityMatchMode.COMPATIBLE, # 需要时选有的,不需要时都可选
config_mode=CapabilityConfigMode.REQUEST_PARAM, # 从请求路径检测
short_name="文件上传",
name="gemini_files",
display_name="Gemini 文件 API",
description="支持 Gemini Files API文件上传/管理),仅 Google 官方 API 支持",
match_mode=CapabilityMatchMode.COMPATIBLE,
config_mode=CapabilityConfigMode.USER_CONFIGURABLE,
short_name="文件API",
)

View File

@@ -236,15 +236,23 @@ class ModuleRegistry:
# 获取启用状态
enabled = self.is_enabled(name, db) if available else False
# 注意:配置验证失败时自动禁用模块
# 自动禁用会在查询方法中产生写操作副作用,违反幂等性原则
# 配置验证状态通过 config_validated/config_error 字段返回,由调用方决定如何处理
# 配置验证失败时自动禁用模块
# 注意:此处故意在 get_status() 中写入,以确保模块状态与配置同步
# 场景:用户删除了模块所依赖的 Provider Key 后,模块应自动关闭
# 权衡:查询方法中的写操作副作用 vs 状态一致性保证
if enabled and not config_validated:
self.set_enabled(name, False, db)
enabled = False
# 计算激活状态available && enabled && config_validated && 依赖模块都激活
is_active = self.is_active(name, db) if available else False
active = is_active and config_validated
return ModuleStatus(
name=name,
available=available,
enabled=enabled,
active=self.is_active(name, db) if available else False,
active=active,
config_validated=config_validated,
config_error=config_error,
display_name=meta.display_name,