i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 01:10:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 重构异步任务系统和计费服务架构

Browse Source 
- 重构任务系统：新增 lifecycle (TaskStatus/BillingStatus)、context、application 模块
- 将 video tasks 泛化为 async tasks，支持更通用的异步任务管理
- 新增 Gemini Files 管理模块和管理界面
- 重构 billing 服务：拆分 schema.py 和 service.py
- 新增 candidate 服务模块用于请求候选管理
- 数据库迁移：添加 billing_status、request_id、gemini_file_mappings 表和索引
- 移除废弃的 video_telemetry、task orchestrator 等模块
This commit is contained in:
fawney19
2026-02-02 03:16:52 +08:00
parent feb7484fda
commit 9e31efe26c
75 changed files with 7511 additions and 2068 deletions
Download Patch File Download Diff File Expand all files Collapse all files

198
tests/core/api_format/conversion/test_video_format_conversion.py Normal file
View File

@@ -0,0 +1,198 @@
"""
视频格式转换单元测试
覆盖重点：
- OpenAI <-> Gemini 视频请求格式转换
- 视频任务响应格式转换
"""
from __future__ import annotations
from src.core.api_format.conversion.normalizers.gemini import GeminiNormalizer
from src.core.api_format.conversion.normalizers.openai import OpenAINormalizer
from src.core.api_format.conversion.registry import FormatConversionRegistry
def _make_registry() -> FormatConversionRegistry:
reg = FormatConversionRegistry()
reg.register(OpenAINormalizer())
reg.register(GeminiNormalizer())
return reg
class TestVideoRequestConversion:
"""视频请求格式转换测试"""
def test_openai_to_gemini_video_request(self) -> None:
"""OpenAI Sora -> Gemini Veo 请求格式转换"""
reg = _make_registry()
openai_request = {
"model": "sora-2",
"prompt": "A cat playing piano",
"size": "1280x720",
"seconds": 8,
}
gemini_request = reg.convert_video_request(openai_request, "openai:video", "gemini:video")
# 验证 Gemini 格式
assert "instances" in gemini_request
assert "parameters" in gemini_request
assert isinstance(gemini_request["instances"], list)
assert len(gemini_request["instances"]) > 0
instance = gemini_request["instances"][0]
assert instance["prompt"] == "A cat playing piano"
params = gemini_request["parameters"]
assert params["aspectRatio"] == "16:9"
assert params["resolution"] == "720p"
assert params["durationSeconds"] == 8
def test_gemini_to_openai_video_request(self) -> None:
"""Gemini Veo -> OpenAI Sora 请求格式转换"""
reg = _make_registry()
gemini_request = {
"model": "veo-3.1-generate-preview",
"instances": [{"prompt": "A beautiful sunset over mountains"}],
"parameters": {
"aspectRatio": "16:9",
"resolution": "1080p",
"durationSeconds": 5,
},
}
openai_request = reg.convert_video_request(gemini_request, "gemini:video", "openai:video")
# 验证 OpenAI 格式
assert openai_request["prompt"] == "A beautiful sunset over mountains"
assert openai_request["model"] == "veo-3.1-generate-preview"
assert openai_request["seconds"] == 5
assert openai_request["size"] == "1920x1080"
def test_same_format_no_conversion(self) -> None:
"""相同格式不进行转换"""
reg = _make_registry()
openai_request = {
"model": "sora-2",
"prompt": "Test",
"size": "720x1280",
"seconds": 4,
}
result = reg.convert_video_request(openai_request, "openai:video", "openai:video")
assert result == openai_request
def test_conversion_with_reference_image(self) -> None:
"""带参考图片的请求转换"""
reg = _make_registry()
openai_request = {
"model": "sora-2",
"prompt": "Animate this image",
"size": "1280x720",
"seconds": 4,
"input_reference": "base64_encoded_image_data",
}
gemini_request = reg.convert_video_request(openai_request, "openai:video", "gemini:video")
instance = gemini_request["instances"][0]
assert "image" in instance
assert instance["image"]["bytesBase64Encoded"] == "base64_encoded_image_data"
class TestVideoTaskConversion:
"""视频任务响应格式转换测试"""
def test_gemini_to_openai_processing_task(self) -> None:
"""Gemini 处理中任务 -> OpenAI 格式"""
reg = _make_registry()
gemini_response = {
"name": "operations/12345",
"done": False,
"metadata": {"progress": 50},
}
openai_response = reg.convert_video_task(gemini_response, "gemini:video", "openai:video")
# OpenAI 格式使用 status 字段
assert openai_response["status"] in ("queued", "processing")
assert "id" in openai_response
def test_gemini_to_openai_completed_task(self) -> None:
"""Gemini 完成任务 -> OpenAI 格式"""
reg = _make_registry()
gemini_response = {
"name": "operations/12345",
"done": True,
"response": {
"generateVideoResponse": {
"generatedSamples": [{"video": {"uri": "https://example.com/video.mp4"}}]
}
},
}
openai_response = reg.convert_video_task(gemini_response, "gemini:video", "openai:video")
assert openai_response["status"] == "completed"
assert openai_response["progress"] == 100
def test_openai_to_gemini_processing_task(self) -> None:
"""OpenAI 处理中任务 -> Gemini 格式"""
reg = _make_registry()
openai_response = {
"id": "task_12345",
"object": "video",
"status": "processing",
"progress": 30,
"created_at": 1700000000,
}
gemini_response = reg.convert_video_task(openai_response, "openai:video", "gemini:video")
# Gemini 格式使用 done 字段
assert gemini_response["done"] is False
assert "name" in gemini_response
def test_openai_to_gemini_completed_task(self) -> None:
"""OpenAI 完成任务 -> Gemini 格式"""
reg = _make_registry()
openai_response = {
"id": "task_12345",
"object": "video",
"status": "completed",
"progress": 100,
"created_at": 1700000000,
"completed_at": 1700000100,
}
gemini_response = reg.convert_video_task(openai_response, "openai:video", "gemini:video")
assert gemini_response["done"] is True
class TestVideoConversionCapabilities:
"""视频格式转换能力检查测试"""
def test_can_convert_video(self) -> None:
"""检查视频格式转换能力"""
reg = _make_registry()
assert reg.can_convert_video("openai:video", "gemini:video") is True
assert reg.can_convert_video("gemini:video", "openai:video") is True
assert reg.can_convert_video("openai:video", "openai:video") is True
def test_cannot_convert_unsupported_format(self) -> None:
"""不支持的格式无法转换"""
reg = _make_registry()
# 没有注册 Claude normalizer 用于视频
assert reg.can_convert_video("openai:video", "claude:video") is False

57
tests/services/test_async_task_orchestrator.py → tests/services/test_candidate_service_submit.py
View File

@@ -6,11 +6,8 @@ import httpx
import pytest
from src.config.settings import config
from src.services.task.orchestrator import (
AllCandidatesFailedError,
AsyncTaskOrchestrator,
UpstreamClientRequestError,
)
from src.services.candidate.service import CandidateService
from src.services.candidate.submit import AllCandidatesFailedError, UpstreamClientRequestError
def _make_candidate(
@@ -46,10 +43,10 @@ async def test_submit_with_failover_skips_http_500_then_succeeds(
monkeypatch: pytest.MonkeyPatch,
) -> None:
db = MagicMock()
orch = AsyncTaskOrchestrator(db)
svc = CandidateService(db)
# bypass init
orch._candidate_resolver = SimpleNamespace(
svc._resolver = SimpleNamespace(
fetch_candidates=AsyncMock(
return_value=(
[
@@ -60,8 +57,8 @@ async def test_submit_with_failover_skips_http_500_then_succeeds(
)
)
)
orch._error_classifier = SimpleNamespace(is_client_error=lambda _text: False)
orch._ensure_initialized = AsyncMock(return_value=None)
svc._error_classifier = SimpleNamespace(is_client_error=lambda _text: False)
svc._ensure_initialized = AsyncMock(return_value=None)
responses = [
httpx.Response(500, text='{"error": {"message": "server"}}'),
@@ -69,12 +66,12 @@ async def test_submit_with_failover_skips_http_500_then_succeeds(
]
submit = AsyncMock(side_effect=responses)
outcome = await orch.submit_with_failover(
outcome = await svc.submit_with_failover(
api_format="openai:video",
model_name="sora",
affinity_key="a1",
user_api_key=MagicMock(),
request_id="req-1",
request_id=None,
task_type="video",
submit_func=submit,
extract_external_task_id=lambda payload: payload.get("id"),
@@ -92,13 +89,13 @@ async def test_submit_with_failover_skips_http_500_then_succeeds(
@pytest.mark.asyncio
async def test_submit_with_failover_stops_on_client_error(monkeypatch: pytest.MonkeyPatch) -> None:
db = MagicMock()
orch = AsyncTaskOrchestrator(db)
svc = CandidateService(db)
orch._candidate_resolver = SimpleNamespace(
svc._resolver = SimpleNamespace(
fetch_candidates=AsyncMock(return_value=([_make_candidate()], "gm1"))
)
orch._error_classifier = SimpleNamespace(is_client_error=lambda _text: True)
orch._ensure_initialized = AsyncMock(return_value=None)
svc._error_classifier = SimpleNamespace(is_client_error=lambda _text: True)
svc._ensure_initialized = AsyncMock(return_value=None)
response = httpx.Response(
400,
@@ -107,12 +104,12 @@ async def test_submit_with_failover_stops_on_client_error(monkeypatch: pytest.Mo
submit = AsyncMock(return_value=response)
with pytest.raises(UpstreamClientRequestError):
await orch.submit_with_failover(
await svc.submit_with_failover(
api_format="openai:video",
model_name="sora",
affinity_key="a1",
user_api_key=MagicMock(),
request_id="req-2",
request_id=None,
task_type="video",
submit_func=submit,
extract_external_task_id=lambda payload: payload.get("id"),
@@ -127,21 +124,21 @@ async def test_submit_with_failover_no_eligible_candidates_due_to_auth_type(
monkeypatch: pytest.MonkeyPatch,
) -> None:
db = MagicMock()
orch = AsyncTaskOrchestrator(db)
svc = CandidateService(db)
orch._candidate_resolver = SimpleNamespace(
svc._resolver = SimpleNamespace(
fetch_candidates=AsyncMock(return_value=([_make_candidate(auth_type="vertex_ai")], "gm1"))
)
orch._error_classifier = SimpleNamespace(is_client_error=lambda _text: False)
orch._ensure_initialized = AsyncMock(return_value=None)
svc._error_classifier = SimpleNamespace(is_client_error=lambda _text: False)
svc._ensure_initialized = AsyncMock(return_value=None)
with pytest.raises(AllCandidatesFailedError) as excinfo:
await orch.submit_with_failover(
await svc.submit_with_failover(
api_format="openai:video",
model_name="sora",
affinity_key="a1",
user_api_key=MagicMock(),
request_id="req-3",
request_id=None,
task_type="video",
submit_func=AsyncMock(),
extract_external_task_id=lambda payload: payload.get("id"),
@@ -158,9 +155,9 @@ async def test_submit_with_failover_filters_missing_billing_rule(
monkeypatch: pytest.MonkeyPatch,
) -> None:
db = MagicMock()
orch = AsyncTaskOrchestrator(db)
svc = CandidateService(db)
orch._candidate_resolver = SimpleNamespace(
svc._resolver = SimpleNamespace(
fetch_candidates=AsyncMock(
return_value=(
[
@@ -171,8 +168,8 @@ async def test_submit_with_failover_filters_missing_billing_rule(
)
)
)
orch._error_classifier = SimpleNamespace(is_client_error=lambda _text: False)
orch._ensure_initialized = AsyncMock(return_value=None)
svc._error_classifier = SimpleNamespace(is_client_error=lambda _text: False)
svc._ensure_initialized = AsyncMock(return_value=None)
# enable require_rule
old = config.billing_require_rule
@@ -185,16 +182,16 @@ async def test_submit_with_failover_filters_missing_billing_rule(
return None if provider_id == "p1" else object()
monkeypatch.setattr(
"src.services.task.orchestrator.BillingRuleService.find_rule", _find_rule
"src.services.candidate.service.BillingRuleService.find_rule", _find_rule
)
submit = AsyncMock(return_value=httpx.Response(200, json={"id": "task-999"}))
outcome = await orch.submit_with_failover(
outcome = await svc.submit_with_failover(
api_format="openai:video",
model_name="sora",
affinity_key="a1",
user_api_key=MagicMock(),
request_id="req-4",
request_id=None,
task_type="video",
submit_func=submit,
extract_external_task_id=lambda payload: payload.get("id"),

115
tests/services/test_video_telemetry.py → tests/services/test_video_finalize.py
View File

@@ -7,12 +7,13 @@ import pytest
from src.config.settings import config
from src.core.api_format.conversion.internal_video import VideoStatus
from src.services.billing.formula_engine import BillingIncompleteError
from src.services.task.impl.video_telemetry import VideoTelemetry
from src.services.task.application import TaskApplicationService
def _make_task(**overrides: Any) -> SimpleNamespace:
task = SimpleNamespace(
id="t1",
request_id="req-1",
user_id="u1",
api_key_id="ak1",
provider_id="p1",
@@ -37,7 +38,7 @@ def _make_task(**overrides: Any) -> SimpleNamespace:
error_code=None,
error_message=None,
status=VideoStatus.COMPLETED.value,
request_metadata={"request_id": "req-1", "poll_raw_response": {"foo": "bar"}},
request_metadata={"poll_raw_response": {"foo": "bar"}},
)
for k, v in overrides.items():
setattr(task, k, v)
@@ -50,7 +51,15 @@ def _make_db() -> MagicMock:
user_obj = SimpleNamespace(id="u1")
api_key_obj = SimpleNamespace(id="ak1")
provider_obj = SimpleNamespace(id="p1", name="prov1")
usage_obj = SimpleNamespace(
id="usage-1",
request_id="req-1",
billing_status="settled", # finalize_submitted already settled
request_metadata=None, # no billing_updated_at yet
)
q_usage = MagicMock()
q_usage.filter.return_value.first.return_value = usage_obj
q_user = MagicMock()
q_user.filter.return_value.first.return_value = user_obj
q_key = MagicMock()
@@ -58,107 +67,117 @@ def _make_db() -> MagicMock:
q_provider = MagicMock()
q_provider.filter.return_value.first.return_value = provider_obj
db.query.side_effect = [q_user, q_key, q_provider]
db.query.side_effect = [q_usage, q_user, q_key, q_provider]
return db
@pytest.mark.asyncio
async def test_video_telemetry_failed_records_cost_zero(monkeypatch: pytest.MonkeyPatch) -> None:
async def test_video_finalize_failed_records_cost_zero(monkeypatch: pytest.MonkeyPatch) -> None:
db = _make_db()
task = _make_task(status=VideoStatus.FAILED.value, error_message="boom")
monkeypatch.setattr(
"src.services.task.impl.video_telemetry.DimensionCollectorService.collect_dimensions",
"src.services.billing.dimension_collector_service.DimensionCollectorService.collect_dimensions",
lambda _self, **_kwargs: {"duration_seconds": 4},
)
record = AsyncMock()
monkeypatch.setattr(
"src.services.task.impl.video_telemetry.UsageService.record_usage_with_custom_cost",
record,
"src.services.task.application.BillingRuleService.find_rule",
lambda *_args, **_kwargs: None,
)
# Mock update_settled_billing (used by finalize_video_task)
update_settled = MagicMock(return_value=True)
monkeypatch.setattr(
"src.services.task.application.UsageService.update_settled_billing",
update_settled,
)
telemetry = VideoTelemetry(db)
await telemetry.record_terminal_usage(task)
app = TaskApplicationService(db)
await app.finalize_video_task(task)
# billing_snapshot should be written back to task.request_metadata
assert task.request_metadata["billing_snapshot"]["billed_reason"] == "task_failed"
assert task.request_metadata["billing_snapshot"]["cost"] == 0.0
assert record.await_count == 1
kwargs = record.await_args.kwargs
assert update_settled.call_count == 1
kwargs = update_settled.call_args.kwargs
assert kwargs["total_cost_usd"] == 0.0
assert kwargs["status"] == "failed"
@pytest.mark.asyncio
async def test_video_telemetry_completed_no_rule(monkeypatch: pytest.MonkeyPatch) -> None:
async def test_video_finalize_completed_no_rule(monkeypatch: pytest.MonkeyPatch) -> None:
db = _make_db()
task = _make_task(status=VideoStatus.COMPLETED.value)
monkeypatch.setattr(
"src.services.task.impl.video_telemetry.DimensionCollectorService.collect_dimensions",
"src.services.billing.dimension_collector_service.DimensionCollectorService.collect_dimensions",
lambda _self, **_kwargs: {"duration_seconds": 4},
)
monkeypatch.setattr(
"src.services.task.impl.video_telemetry.BillingRuleService.find_rule",
"src.services.task.application.BillingRuleService.find_rule",
lambda *_args, **_kwargs: None,
)
record = AsyncMock()
# Mock update_settled_billing (used by finalize_video_task)
update_settled = MagicMock(return_value=True)
monkeypatch.setattr(
"src.services.task.impl.video_telemetry.UsageService.record_usage_with_custom_cost",
record,
"src.services.task.application.UsageService.update_settled_billing",
update_settled,
)
telemetry = VideoTelemetry(db)
await telemetry.record_terminal_usage(task)
app = TaskApplicationService(db)
await app.finalize_video_task(task)
assert task.request_metadata["billing_snapshot"]["status"] == "no_rule"
kwargs = record.await_args.kwargs
kwargs = update_settled.call_args.kwargs
assert kwargs["total_cost_usd"] == 0.0
assert kwargs["status"] == "completed"
@pytest.mark.asyncio
async def test_video_telemetry_strict_mode_missing_required_marks_failed(
async def test_video_finalize_strict_mode_missing_required_marks_failed(
monkeypatch: pytest.MonkeyPatch,
) -> None:
db = _make_db()
task = _make_task(status=VideoStatus.COMPLETED.value)
# provide a billing rule so formula path is taken
rule = SimpleNamespace(
id="r1",
name="video",
expression="duration_seconds",
variables={},
dimension_mappings={},
task = _make_task(
status=VideoStatus.COMPLETED.value,
request_metadata={
"poll_raw_response": {"foo": "bar"},
"billing_rule_snapshot": {
"status": "ok",
"rule_id": "r1",
"rule_name": "video",
"scope": "model",
"expression": "duration_seconds",
"variables": {},
"dimension_mappings": {},
},
},
)
lookup = SimpleNamespace(rule=rule, scope="model", effective_task_type="video")
monkeypatch.setattr(
"src.services.task.impl.video_telemetry.DimensionCollectorService.collect_dimensions",
"src.services.billing.dimension_collector_service.DimensionCollectorService.collect_dimensions",
lambda _self, **_kwargs: {"duration_seconds": None},
)
# Mock update_settled_billing (used by finalize_video_task)
update_settled = MagicMock(return_value=True)
monkeypatch.setattr(
"src.services.task.impl.video_telemetry.BillingRuleService.find_rule",
lambda *_args, **_kwargs: lookup,
)
record = AsyncMock()
monkeypatch.setattr(
"src.services.task.impl.video_telemetry.UsageService.record_usage_with_custom_cost",
record,
"src.services.task.application.UsageService.update_settled_billing",
update_settled,
)
old = config.billing_strict_mode
try:
config.billing_strict_mode = True
telemetry = VideoTelemetry(db)
telemetry._formula_engine.evaluate = MagicMock(
side_effect=BillingIncompleteError(
"Missing required dimensions", missing_required=["duration_seconds"]
)
monkeypatch.setattr(
"src.services.task.application.FormulaEngine.evaluate",
MagicMock(
side_effect=BillingIncompleteError(
"Missing required dimensions", missing_required=["duration_seconds"]
)
),
)
await telemetry.record_terminal_usage(task)
app = TaskApplicationService(db)
await app.finalize_video_task(task)
finally:
config.billing_strict_mode = old
@@ -167,6 +186,6 @@ async def test_video_telemetry_strict_mode_missing_required_marks_failed(
assert task.video_urls is None
assert "billing_incomplete" in (task.error_code or "")
kwargs = record.await_args.kwargs
kwargs = update_settled.call_args.kwargs
assert kwargs["total_cost_usd"] == 0.0
assert kwargs["status"] == "failed"

18
tests/services/test_video_task_poller.py
View File

@@ -4,14 +4,14 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from src.core.api_format.conversion.internal_video import InternalVideoPollResult, VideoStatus
from src.services.video.task_poller import VideoTaskPollerService
from src.services.task.impl.video_poller import VideoTaskPollerAdapter
@pytest.mark.asyncio
async def test_poll_task_status_routes_gemini_video_to_gemini(
monkeypatch: pytest.MonkeyPatch,
) -> None:
poller = VideoTaskPollerService()
adapter = VideoTaskPollerAdapter()
task = SimpleNamespace(
endpoint_id="e1",
@@ -22,24 +22,24 @@ async def test_poll_task_status_routes_gemini_video_to_gemini(
endpoint = SimpleNamespace(id="e1", base_url="https://example.com", api_format="gemini:video")
key = SimpleNamespace(id="k1", api_key="enc")
monkeypatch.setattr(poller, "_get_endpoint", lambda _db, _id: endpoint)
monkeypatch.setattr(poller, "_get_key", lambda _db, _id: key)
monkeypatch.setattr(adapter, "_get_endpoint", lambda _db, _id: endpoint)
monkeypatch.setattr(adapter, "_get_key", lambda _db, _id: key)
monkeypatch.setattr(
"src.services.video.task_poller.crypto_service.decrypt", lambda _v: "decrypted"
"src.services.task.impl.video_poller.crypto_service.decrypt", lambda _v: "decrypted"
)
auth_info = SimpleNamespace(auth_header="authorization", auth_value="Bearer x")
monkeypatch.setattr(
"src.services.video.task_poller.get_provider_auth",
"src.services.task.impl.video_poller.get_provider_auth",
AsyncMock(return_value=auth_info),
)
poll_gemini = AsyncMock(return_value=InternalVideoPollResult(status=VideoStatus.PROCESSING))
poll_openai = AsyncMock(return_value=InternalVideoPollResult(status=VideoStatus.PROCESSING))
monkeypatch.setattr(poller, "_poll_gemini", poll_gemini)
monkeypatch.setattr(poller, "_poll_openai", poll_openai)
monkeypatch.setattr(adapter, "_poll_gemini", poll_gemini)
monkeypatch.setattr(adapter, "_poll_openai", poll_openai)
result = await poller._poll_task_status(MagicMock(), task)
result = await adapter._poll_task_status(MagicMock(), task)
assert result.status == VideoStatus.PROCESSING
assert poll_gemini.await_count == 1
assert poll_openai.await_count == 0