mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 09:20:22 +08:00
refactor: 重构异步任务系统和计费服务架构
- 重构任务系统:新增 lifecycle (TaskStatus/BillingStatus)、context、application 模块 - 将 video tasks 泛化为 async tasks,支持更通用的异步任务管理 - 新增 Gemini Files 管理模块和管理界面 - 重构 billing 服务:拆分 schema.py 和 service.py - 新增 candidate 服务模块用于请求候选管理 - 数据库迁移:添加 billing_status、request_id、gemini_file_mappings 表和索引 - 移除废弃的 video_telemetry、task orchestrator 等模块
This commit is contained in:
@@ -3,11 +3,20 @@
|
||||
|
||||
用于候选筛选时判断端点是否可以处理客户端请求格式。
|
||||
|
||||
三层开关优先级(从高到低):
|
||||
1. 全局开关 ON → 强制允许(跳过后续检查)
|
||||
2. 全局开关 OFF → 看提供商开关
|
||||
- 提供商开关 ON → 强制允许(跳过端点检查)
|
||||
- 提供商开关 OFF → 看端点配置
|
||||
3. 端点配置(format_acceptance_config)
|
||||
- enabled=true + 白名单/黑名单检查 → 允许
|
||||
- enabled=false 或未配置 → 禁止
|
||||
|
||||
转换逻辑:
|
||||
1. 格式完全匹配 -> 透传(无需转换)
|
||||
2. 格式不同 -> 需要检查全局开关 + 端点开关
|
||||
- data_format_id 相同 -> 可透传(无需数据转换),但需全局开关 + 端点开关
|
||||
- data_format_id 不同 -> 需要转换,检查全局开关 + 端点配置 + 转换器能力
|
||||
2. 格式不同 -> 需要检查三层开关
|
||||
- data_format_id 相同 -> 可透传(无需数据转换)
|
||||
- data_format_id 不同 -> 需要转换,检查转换器能力
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -28,8 +37,10 @@ def is_format_compatible(
|
||||
endpoint_api_format: str,
|
||||
endpoint_format_acceptance_config: dict | None,
|
||||
is_stream: bool,
|
||||
global_conversion_enabled: bool,
|
||||
effective_conversion_enabled: bool,
|
||||
registry: FormatConversionRegistry | None = None,
|
||||
*,
|
||||
skip_endpoint_check: bool = False,
|
||||
) -> tuple[bool, bool, str | None]:
|
||||
"""
|
||||
检查端点是否兼容客户端格式
|
||||
@@ -39,8 +50,9 @@ def is_format_compatible(
|
||||
endpoint_api_format: 端点的 API 格式
|
||||
endpoint_format_acceptance_config: 端点的格式接受配置
|
||||
is_stream: 是否是流式请求
|
||||
global_conversion_enabled: 全局格式转换开关(来自环境变量 FORMAT_CONVERSION_ENABLED,默认 True)
|
||||
effective_conversion_enabled: 有效格式转换开关(全局 OR 提供商)
|
||||
registry: 转换器注册表(可选,默认使用全局单例)
|
||||
skip_endpoint_check: 是否跳过端点配置检查(当全局或提供商开关为 ON 时设为 True)
|
||||
|
||||
Returns:
|
||||
(is_compatible, needs_conversion, skip_reason)
|
||||
@@ -66,30 +78,36 @@ def is_format_compatible(
|
||||
if provider_key == client_key:
|
||||
return True, False, None
|
||||
|
||||
# 2. 格式不同 -> 需要检查全局格式转换开关
|
||||
# 即使 data_format_id 相同(如 claude:chat / claude:cli),也需要全局开关启用
|
||||
if not global_conversion_enabled:
|
||||
return False, False, "全局格式转换未启用(环境变量 FORMAT_CONVERSION_ENABLED=false)"
|
||||
# 2. 格式不同 -> 需要检查格式转换开关
|
||||
# 如果有效开关为 False(全局 OFF 且提供商 OFF),直接拒绝
|
||||
if not effective_conversion_enabled:
|
||||
return False, False, "格式转换已禁用(全局和提供商开关均为关闭)"
|
||||
|
||||
# 3. 格式不同时,统一检查端点配置(核心控制)
|
||||
if endpoint_format_acceptance_config is None:
|
||||
return False, False, "端点未配置格式接受策略"
|
||||
# 3. 如果全局或提供商开关为 ON,跳过端点配置检查
|
||||
if not skip_endpoint_check:
|
||||
# 检查端点配置(第三层开关)
|
||||
if endpoint_format_acceptance_config is None:
|
||||
return False, False, "端点未配置格式接受策略"
|
||||
|
||||
config = endpoint_format_acceptance_config
|
||||
if not isinstance(config, dict):
|
||||
return False, False, "端点格式配置无效"
|
||||
if not config.get("enabled", False):
|
||||
return False, False, "端点格式接受未启用"
|
||||
config = endpoint_format_acceptance_config
|
||||
if not isinstance(config, dict):
|
||||
return False, False, "端点格式配置无效"
|
||||
if not config.get("enabled", False):
|
||||
return False, False, "端点格式接受未启用"
|
||||
|
||||
# 检查 reject_formats(优先)
|
||||
reject_formats = config.get("reject_formats", [])
|
||||
if client_key in [f.upper() for f in reject_formats]:
|
||||
return False, False, f"端点拒绝 {client_format} 格式"
|
||||
# 检查 reject_formats(优先)
|
||||
reject_formats = config.get("reject_formats", [])
|
||||
if client_key in [f.upper() for f in reject_formats]:
|
||||
return False, False, f"端点拒绝 {client_format} 格式"
|
||||
|
||||
# 检查 accept_formats
|
||||
accept_formats = config.get("accept_formats", [])
|
||||
if accept_formats and client_key not in [f.upper() for f in accept_formats]:
|
||||
return False, False, f"端点不接受 {client_format} 格式"
|
||||
# 检查 accept_formats
|
||||
accept_formats = config.get("accept_formats", [])
|
||||
if accept_formats and client_key not in [f.upper() for f in accept_formats]:
|
||||
return False, False, f"端点不接受 {client_format} 格式"
|
||||
|
||||
# 检查流式转换
|
||||
if is_stream and not config.get("stream_conversion", True):
|
||||
return False, False, "端点不支持流式格式转换"
|
||||
|
||||
# 4. 检查是否可以透传(data_format_id 相同)
|
||||
# 例如:claude:chat / claude:cli 的 data_format_id 都是 "claude",数据格式相同可透传
|
||||
@@ -99,11 +117,7 @@ def is_format_compatible(
|
||||
return True, False, None
|
||||
|
||||
# 5. 需要数据转换的情况(data_format_id 不同)
|
||||
# 检查流式转换
|
||||
if is_stream and not config.get("stream_conversion", True):
|
||||
return False, False, "端点不支持流式格式转换"
|
||||
|
||||
# 6. 检查转换器能力
|
||||
# 检查转换器能力
|
||||
if not registry.can_convert_full(
|
||||
client_key,
|
||||
provider_key,
|
||||
|
||||
@@ -736,10 +736,17 @@ class GeminiNormalizer(FormatNormalizer):
|
||||
instance = instances[0] if isinstance(instances[0], dict) else {}
|
||||
params = request.get("parameters") or {}
|
||||
|
||||
# 解析 image(用于 image-to-video 或第一帧)
|
||||
# 官方格式: {"image": {"inlineData": {"mimeType": "image/png", "data": "base64..."}}}
|
||||
image = instance.get("image") if isinstance(instance, dict) else None
|
||||
image_ref = None
|
||||
if isinstance(image, dict):
|
||||
image_ref = image.get("bytesBase64Encoded")
|
||||
inline_data = image.get("inlineData", {})
|
||||
if isinstance(inline_data, dict):
|
||||
image_ref = inline_data.get("data")
|
||||
# 兼容旧格式
|
||||
if not image_ref:
|
||||
image_ref = image.get("bytesBase64Encoded")
|
||||
|
||||
prompt = instance.get("prompt") if isinstance(instance, dict) else None
|
||||
prompt_str = str(prompt).strip() if prompt else ""
|
||||
@@ -747,7 +754,7 @@ class GeminiNormalizer(FormatNormalizer):
|
||||
raise ValueError("Video prompt is required")
|
||||
|
||||
duration_raw = params.get("durationSeconds")
|
||||
sample_count_raw = params.get("sampleCount")
|
||||
sample_count_raw = params.get("sampleCount") or params.get("numberOfVideos")
|
||||
|
||||
try:
|
||||
duration_seconds = int(duration_raw) if duration_raw else 8
|
||||
@@ -760,6 +767,35 @@ class GeminiNormalizer(FormatNormalizer):
|
||||
except (ValueError, TypeError):
|
||||
sample_count = 1
|
||||
|
||||
# 构建 extra 字段,保留所有 Veo 特有的参数
|
||||
extra: dict[str, Any] = {
|
||||
"personGeneration": params.get("personGeneration"),
|
||||
"sampleCount": sample_count,
|
||||
}
|
||||
|
||||
# negativePrompt - 负面提示词
|
||||
if params.get("negativePrompt"):
|
||||
extra["negativePrompt"] = params["negativePrompt"]
|
||||
|
||||
# lastFrame - 最后一帧(用于插值)
|
||||
last_frame = params.get("lastFrame")
|
||||
if isinstance(last_frame, dict):
|
||||
extra["lastFrame"] = last_frame
|
||||
|
||||
# referenceImages - 参考图像(最多3张,仅 Veo 3.1)
|
||||
ref_images = params.get("referenceImages")
|
||||
if isinstance(ref_images, list) and ref_images:
|
||||
extra["referenceImages"] = ref_images
|
||||
|
||||
# video - 视频扩展输入(用于视频续写)
|
||||
video_input = instance.get("video") if isinstance(instance, dict) else None
|
||||
if isinstance(video_input, dict):
|
||||
extra["video"] = video_input
|
||||
|
||||
# seed - 种子值(Veo 3)
|
||||
if params.get("seed") is not None:
|
||||
extra["seed"] = params["seed"]
|
||||
|
||||
return InternalVideoRequest(
|
||||
prompt=prompt_str,
|
||||
model=str(request.get("model") or "veo-3.1-generate-preview"),
|
||||
@@ -767,10 +803,7 @@ class GeminiNormalizer(FormatNormalizer):
|
||||
aspect_ratio=str(params.get("aspectRatio") or "16:9"),
|
||||
resolution=str(params.get("resolution") or "720p"),
|
||||
reference_image_url=image_ref,
|
||||
extra={
|
||||
"personGeneration": params.get("personGeneration"),
|
||||
"sampleCount": sample_count,
|
||||
},
|
||||
extra=extra,
|
||||
)
|
||||
|
||||
def video_request_from_internal(self, internal: InternalVideoRequest) -> dict[str, Any]:
|
||||
@@ -783,15 +816,31 @@ class GeminiNormalizer(FormatNormalizer):
|
||||
"resolution": internal.resolution,
|
||||
"durationSeconds": internal.duration_seconds,
|
||||
}
|
||||
for key in ["personGeneration", "sampleCount"]:
|
||||
for key in ["personGeneration", "sampleCount", "negativePrompt", "seed"]:
|
||||
if key in internal.extra:
|
||||
parameters[key] = internal.extra[key]
|
||||
|
||||
return {
|
||||
# lastFrame 和 referenceImages 需要特殊处理
|
||||
if internal.extra.get("lastFrame"):
|
||||
parameters["lastFrame"] = internal.extra["lastFrame"]
|
||||
if internal.extra.get("referenceImages"):
|
||||
parameters["referenceImages"] = internal.extra["referenceImages"]
|
||||
|
||||
# video 输入(视频续写)
|
||||
if internal.extra.get("video"):
|
||||
instance["video"] = internal.extra["video"]
|
||||
|
||||
result: dict[str, Any] = {
|
||||
"instances": [instance],
|
||||
"parameters": parameters,
|
||||
}
|
||||
|
||||
# 模型信息(用于 URL 构建)
|
||||
if internal.model:
|
||||
result["model"] = internal.model
|
||||
|
||||
return result
|
||||
|
||||
def video_task_to_internal(self, response: dict[str, Any]) -> InternalVideoTask:
|
||||
operation_name = str(response.get("name") or "")
|
||||
done = bool(response.get("done"))
|
||||
@@ -824,20 +873,30 @@ class GeminiNormalizer(FormatNormalizer):
|
||||
)
|
||||
|
||||
def video_task_from_internal(self, internal: InternalVideoTask) -> dict[str, Any]:
|
||||
# 优先使用 external_id(上游返回的 operation name),否则用内部 id
|
||||
operation_name = internal.external_id or f"operations/{internal.id}"
|
||||
if not operation_name.startswith("operations/"):
|
||||
operation_name = f"operations/{operation_name}"
|
||||
# 从 external_id 中提取 model 名称,用于构建 operation name
|
||||
# external_id 格式: models/{model}/operations/{gemini_id}
|
||||
model_name = "unknown"
|
||||
if internal.external_id:
|
||||
parts = internal.external_id.split("/")
|
||||
if len(parts) >= 2 and parts[0] == "models":
|
||||
model_name = parts[1]
|
||||
|
||||
# 使用我们的内部 task_id 构建 operation name,不暴露 Gemini 的 operation ID
|
||||
# 格式: models/{model}/operations/{our_task_id}
|
||||
operation_name = f"models/{model_name}/operations/{internal.id}"
|
||||
|
||||
if internal.status == VideoStatus.COMPLETED:
|
||||
urls = internal.video_urls or ([internal.video_url] if internal.video_url else [])
|
||||
# 使用我们的内部 task_id 构建下载 URL,不暴露真实的 Gemini file_id
|
||||
# 使用 aev_ 前缀标识这是视频任务的下载链接
|
||||
# 格式:/v1beta/files/aev_{task_id}:download?alt=media
|
||||
proxy_download_url = f"/v1beta/files/aev_{internal.id}:download?alt=media"
|
||||
return {
|
||||
"name": operation_name,
|
||||
"done": True,
|
||||
"response": {
|
||||
"generateVideoResponse": {
|
||||
"generatedSamples": [
|
||||
{"video": {"uri": url, "mimeType": "video/mp4"}} for url in urls
|
||||
{"video": {"uri": proxy_download_url, "mimeType": "video/mp4"}}
|
||||
]
|
||||
}
|
||||
},
|
||||
|
||||
@@ -752,10 +752,21 @@ class OpenAINormalizer(FormatNormalizer):
|
||||
"message": internal.error_message,
|
||||
}
|
||||
|
||||
for key in ["model", "size", "seconds"]:
|
||||
if key in internal.extra:
|
||||
# 基本字段
|
||||
for key in ["model", "size", "prompt"]:
|
||||
if internal.extra.get(key):
|
||||
payload[key] = internal.extra[key]
|
||||
|
||||
# seconds 必须是字符串类型
|
||||
seconds = internal.extra.get("seconds")
|
||||
if seconds is not None:
|
||||
payload["seconds"] = str(seconds)
|
||||
|
||||
# remix 相关字段
|
||||
remixed_from = internal.extra.get("remixed_from_video_id")
|
||||
if remixed_from:
|
||||
payload["remixed_from_video_id"] = remixed_from
|
||||
|
||||
return payload
|
||||
|
||||
def video_poll_to_internal(self, response: dict[str, Any]) -> InternalVideoPollResult:
|
||||
@@ -764,14 +775,18 @@ class OpenAINormalizer(FormatNormalizer):
|
||||
|
||||
if status == "completed":
|
||||
expires_at = response.get("expires_at")
|
||||
# 使用任务 ID 构建内容路径,由调用方拼接完整 URL
|
||||
# 如果 task_id 不存在,说明上游响应异常
|
||||
video_url = f"videos/{task_id}/content" if task_id else None
|
||||
# 优先使用上游返回的直接 URL(某些代理如 API易 会返回 CDN URL)
|
||||
# 回退到构建相对路径(标准 OpenAI API 通过 /content 端点下载)
|
||||
direct_url = (
|
||||
response.get("video_url") or response.get("url") or response.get("result_url")
|
||||
)
|
||||
# 使用直接 URL 或回退到相对路径
|
||||
video_url = direct_url or (f"videos/{task_id}/content" if task_id else None)
|
||||
if not video_url:
|
||||
return InternalVideoPollResult(
|
||||
status=VideoStatus.FAILED,
|
||||
error_code="missing_task_id",
|
||||
error_message="Upstream response missing task id",
|
||||
error_code="missing_video_url",
|
||||
error_message="Upstream response missing video url",
|
||||
raw_response=response,
|
||||
)
|
||||
return InternalVideoPollResult(
|
||||
|
||||
@@ -155,6 +155,110 @@ class FormatConversionRegistry:
|
||||
except Exception as e:
|
||||
raise FormatConversionError(source_format, target_format, str(e)) from e
|
||||
|
||||
# ==================== 视频格式转换 ====================
|
||||
|
||||
def convert_video_request(
|
||||
self,
|
||||
request: dict[str, Any],
|
||||
source_format: str,
|
||||
target_format: str,
|
||||
) -> dict[str, Any]:
|
||||
"""转换视频请求格式(OpenAI <-> Gemini)
|
||||
|
||||
Args:
|
||||
request: 原始视频请求
|
||||
source_format: 源格式(如 openai:video, gemini:video)
|
||||
target_format: 目标格式
|
||||
|
||||
Returns:
|
||||
转换后的视频请求
|
||||
"""
|
||||
# 统一使用基础格式 ID(去掉 :video 后缀)
|
||||
src_base = self._video_format_to_base(source_format)
|
||||
tgt_base = self._video_format_to_base(target_format)
|
||||
|
||||
if src_base == tgt_base:
|
||||
return request
|
||||
|
||||
src = self._require_normalizer(src_base)
|
||||
tgt = self._require_normalizer(tgt_base)
|
||||
|
||||
with _track_conversion_metrics(
|
||||
"video_request", str(source_format).upper(), str(target_format).upper()
|
||||
):
|
||||
try:
|
||||
internal = src.video_request_to_internal(request)
|
||||
return tgt.video_request_from_internal(internal)
|
||||
except Exception as e:
|
||||
raise FormatConversionError(source_format, target_format, str(e)) from e
|
||||
|
||||
def convert_video_task(
|
||||
self,
|
||||
task_response: dict[str, Any],
|
||||
source_format: str,
|
||||
target_format: str,
|
||||
) -> dict[str, Any]:
|
||||
"""转换视频任务响应格式(OpenAI <-> Gemini)
|
||||
|
||||
Args:
|
||||
task_response: 原始任务响应
|
||||
source_format: 源格式
|
||||
target_format: 目标格式
|
||||
|
||||
Returns:
|
||||
转换后的任务响应
|
||||
"""
|
||||
src_base = self._video_format_to_base(source_format)
|
||||
tgt_base = self._video_format_to_base(target_format)
|
||||
|
||||
if src_base == tgt_base:
|
||||
return task_response
|
||||
|
||||
src = self._require_normalizer(src_base)
|
||||
tgt = self._require_normalizer(tgt_base)
|
||||
|
||||
with _track_conversion_metrics(
|
||||
"video_task", str(source_format).upper(), str(target_format).upper()
|
||||
):
|
||||
try:
|
||||
internal = src.video_task_to_internal(task_response)
|
||||
return tgt.video_task_from_internal(internal)
|
||||
except Exception as e:
|
||||
raise FormatConversionError(source_format, target_format, str(e)) from e
|
||||
|
||||
def can_convert_video(self, source_format: str, target_format: str) -> bool:
|
||||
"""检查是否支持视频格式转换"""
|
||||
src_base = self._video_format_to_base(source_format)
|
||||
tgt_base = self._video_format_to_base(target_format)
|
||||
|
||||
if src_base == tgt_base:
|
||||
return True
|
||||
|
||||
src = self.get_normalizer(src_base)
|
||||
tgt = self.get_normalizer(tgt_base)
|
||||
|
||||
if src is None or tgt is None:
|
||||
return False
|
||||
|
||||
# 检查是否有视频转换方法
|
||||
return (
|
||||
hasattr(src, "video_request_to_internal")
|
||||
and hasattr(src, "video_task_to_internal")
|
||||
and hasattr(tgt, "video_request_from_internal")
|
||||
and hasattr(tgt, "video_task_from_internal")
|
||||
)
|
||||
|
||||
def _video_format_to_base(self, format_id: str) -> str:
|
||||
"""将视频格式 ID 转换为基础格式 ID
|
||||
|
||||
例如: openai:video -> openai:chat, gemini:video -> gemini:chat
|
||||
"""
|
||||
upper = str(format_id).upper()
|
||||
if upper.endswith(":VIDEO"):
|
||||
base = upper[:-6] # 去掉 :VIDEO
|
||||
return f"{base}:CHAT"
|
||||
return upper
|
||||
|
||||
# ==================== 流式转换(严格) ====================
|
||||
|
||||
def convert_stream_chunk(
|
||||
|
||||
@@ -248,10 +248,10 @@ register_capability(
|
||||
)
|
||||
|
||||
register_capability(
|
||||
name="gemini_files_api",
|
||||
display_name="Gemini文件上传",
|
||||
description="支持 Gemini Files API(上传、查询、删除),第三方 Key 通常不支持",
|
||||
match_mode=CapabilityMatchMode.COMPATIBLE, # 需要时选有的,不需要时都可选
|
||||
config_mode=CapabilityConfigMode.REQUEST_PARAM, # 从请求路径检测
|
||||
short_name="文件上传",
|
||||
name="gemini_files",
|
||||
display_name="Gemini 文件 API",
|
||||
description="支持 Gemini Files API(文件上传/管理),仅 Google 官方 API 支持",
|
||||
match_mode=CapabilityMatchMode.COMPATIBLE,
|
||||
config_mode=CapabilityConfigMode.USER_CONFIGURABLE,
|
||||
short_name="文件API",
|
||||
)
|
||||
|
||||
@@ -236,15 +236,23 @@ class ModuleRegistry:
|
||||
# 获取启用状态
|
||||
enabled = self.is_enabled(name, db) if available else False
|
||||
|
||||
# 注意:配置验证失败时不自动禁用模块
|
||||
# 自动禁用会在查询方法中产生写操作副作用,违反幂等性原则
|
||||
# 配置验证状态通过 config_validated/config_error 字段返回,由调用方决定如何处理
|
||||
# 配置验证失败时自动禁用模块
|
||||
# 注意:此处故意在 get_status() 中写入,以确保模块状态与配置同步
|
||||
# 场景:用户删除了模块所依赖的 Provider Key 后,模块应自动关闭
|
||||
# 权衡:查询方法中的写操作副作用 vs 状态一致性保证
|
||||
if enabled and not config_validated:
|
||||
self.set_enabled(name, False, db)
|
||||
enabled = False
|
||||
|
||||
# 计算激活状态:available && enabled && config_validated && 依赖模块都激活
|
||||
is_active = self.is_active(name, db) if available else False
|
||||
active = is_active and config_validated
|
||||
|
||||
return ModuleStatus(
|
||||
name=name,
|
||||
available=available,
|
||||
enabled=enabled,
|
||||
active=self.is_active(name, db) if available else False,
|
||||
active=active,
|
||||
config_validated=config_validated,
|
||||
config_error=config_error,
|
||||
display_name=meta.display_name,
|
||||
|
||||
Reference in New Issue
Block a user