mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 01:10:23 +08:00
refactor: 统一任务框架 Phase 3 - 用 TaskService/FailoverEngine 替代 FallbackOrchestrator
核心重构:
- 移除 FallbackOrchestrator,用 TaskService + FailoverEngine 替代
- TaskService 作为统一入口,支持 SYNC/ASYNC 两种任务模式
- FailoverEngine 实现候选遍历、重试、故障转移逻辑
- 新增 AttemptFunc/AttemptResult 协议,统一尝试结果表示
功能改进:
- 流式响应首字节探测(30s 超时,空流触发故障转移)
- 流式取消归因优化(区分客户端断连 vs 服务端中断)
- 新增 OpenAI Sora 视频取消路由 POST /v1/videos/{task_id}/cancel
- OpenAI 流式请求自动添加 stream_options.include_usage
代码规范:
- 修复 loguru 日志格式(%s → {})
- 新增 FORMAT_CONVERSION_ENABLED 环境变量说明
测试覆盖:
- test_failover_engine.py: FailoverEngine 单元测试
- test_task_service_async_execute.py: TaskService ASYNC 模式测试
- test_video_cancel_e2e.py: 视频取消端到端测试
This commit is contained in:
233
tests/e2e/test_video_cancel_e2e.py
Normal file
233
tests/e2e/test_video_cancel_e2e.py
Normal file
@@ -0,0 +1,233 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from src.api.base.pipeline import ApiRequestPipeline
|
||||
from src.api.handlers.gemini.video_adapter import GeminiVeoAdapter
|
||||
from src.api.handlers.openai.video_adapter import OpenAIVideoAdapter
|
||||
from src.core.api_format.conversion.internal_video import VideoStatus
|
||||
|
||||
|
||||
def _make_request(
|
||||
*,
|
||||
method: str,
|
||||
path: str,
|
||||
headers: dict[str, str],
|
||||
body: bytes,
|
||||
) -> MagicMock:
|
||||
req = MagicMock()
|
||||
req.method = method
|
||||
req.url = SimpleNamespace(path=path)
|
||||
req.headers = headers
|
||||
req.query_params = {}
|
||||
req.client = None
|
||||
req.state = SimpleNamespace()
|
||||
req.body = AsyncMock(return_value=body)
|
||||
return req
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_cancel_openai_route_end_to_end(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""
|
||||
End-to-end-ish test:
|
||||
ApiRequestPipeline -> OpenAIVideoAdapter -> OpenAIVideoHandler -> TaskService.cancel
|
||||
"""
|
||||
pipeline = ApiRequestPipeline()
|
||||
|
||||
# Pipeline auth/quota/audit shortcuts
|
||||
user = SimpleNamespace(id="u1", username="u1", role="user", quota_usd=None, used_usd=0.0)
|
||||
api_key = SimpleNamespace(id="ak1", user_id="u1", is_standalone=False)
|
||||
monkeypatch.setattr(pipeline.auth_service, "authenticate_api_key", lambda _db, _k: (user, api_key))
|
||||
monkeypatch.setattr(pipeline.usage_service, "check_user_quota", lambda *_args, **_kwargs: (True, "ok"))
|
||||
monkeypatch.setattr(pipeline.audit_service, "log_event", MagicMock())
|
||||
|
||||
# DB stubs used by TaskService.cancel
|
||||
task = SimpleNamespace(
|
||||
id="t1",
|
||||
short_id="s1",
|
||||
user_id="u1",
|
||||
request_id="r1",
|
||||
external_task_id="ext-1",
|
||||
endpoint_id="e1",
|
||||
key_id="k1",
|
||||
status=VideoStatus.SUBMITTED.value,
|
||||
updated_at=None,
|
||||
request_metadata={},
|
||||
)
|
||||
endpoint = SimpleNamespace(
|
||||
id="e1",
|
||||
base_url="https://upstream.example.com",
|
||||
api_format="openai:video",
|
||||
api_family="openai",
|
||||
endpoint_kind="video",
|
||||
header_rules=None,
|
||||
)
|
||||
provider_key = SimpleNamespace(
|
||||
id="k1",
|
||||
api_key="encrypted",
|
||||
auth_type="api_key",
|
||||
)
|
||||
|
||||
q_task = MagicMock()
|
||||
q_task.filter.return_value.first.return_value = task
|
||||
q_endpoint = MagicMock()
|
||||
q_endpoint.filter.return_value.first.return_value = endpoint
|
||||
q_key = MagicMock()
|
||||
q_key.filter.return_value.first.return_value = provider_key
|
||||
|
||||
db = MagicMock()
|
||||
|
||||
def _query(model): # noqa: ANN001
|
||||
name = getattr(model, "__name__", "")
|
||||
if name == "VideoTask":
|
||||
return q_task
|
||||
if name == "ProviderEndpoint":
|
||||
return q_endpoint
|
||||
if name == "ProviderAPIKey":
|
||||
return q_key
|
||||
return MagicMock()
|
||||
|
||||
db.query.side_effect = _query
|
||||
|
||||
# Upstream call stubs
|
||||
upstream = SimpleNamespace(
|
||||
delete=AsyncMock(return_value=httpx.Response(200, json={"ok": True})),
|
||||
post=AsyncMock(), # not used for openai cancel
|
||||
)
|
||||
|
||||
with (
|
||||
patch("src.clients.http_client.HTTPClientPool.get_default_client_async", AsyncMock(return_value=upstream)),
|
||||
patch("src.core.crypto.crypto_service.decrypt", lambda _v: "upstream-key"),
|
||||
patch(
|
||||
"src.services.provider.transport.build_provider_url",
|
||||
lambda _endpoint, **_kwargs: "https://upstream.example.com/v1/videos",
|
||||
),
|
||||
patch("src.services.usage.service.UsageService.finalize_void", MagicMock(return_value=True)),
|
||||
patch("src.services.usage.service.UsageService.void_settled", MagicMock()),
|
||||
):
|
||||
request = _make_request(
|
||||
method="POST",
|
||||
path="/v1/videos/t1/cancel",
|
||||
headers={"authorization": "Bearer sk-test", "x-real-ip": "127.0.0.1", "user-agent": "pytest"},
|
||||
body=b"",
|
||||
)
|
||||
adapter = OpenAIVideoAdapter()
|
||||
resp = await pipeline.run(
|
||||
adapter=adapter,
|
||||
http_request=request,
|
||||
db=db,
|
||||
mode=adapter.mode,
|
||||
api_format_hint=adapter.allowed_api_formats[0],
|
||||
path_params={"task_id": "t1"},
|
||||
)
|
||||
|
||||
assert getattr(resp, "status_code", None) == 200
|
||||
assert upstream.delete.await_count == 1
|
||||
assert upstream.delete.call_args.args[0] == "https://upstream.example.com/v1/videos/ext-1"
|
||||
assert task.status == VideoStatus.CANCELLED.value
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_cancel_gemini_route_end_to_end(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""
|
||||
End-to-end-ish test:
|
||||
ApiRequestPipeline -> GeminiVeoAdapter -> GeminiVeoHandler -> TaskService.cancel
|
||||
"""
|
||||
pipeline = ApiRequestPipeline()
|
||||
|
||||
# Pipeline auth/quota/audit shortcuts
|
||||
user = SimpleNamespace(id="u1", username="u1", role="user", quota_usd=None, used_usd=0.0)
|
||||
api_key = SimpleNamespace(id="ak1", user_id="u1", is_standalone=False)
|
||||
monkeypatch.setattr(pipeline.auth_service, "authenticate_api_key", lambda _db, _k: (user, api_key))
|
||||
monkeypatch.setattr(pipeline.usage_service, "check_user_quota", lambda *_args, **_kwargs: (True, "ok"))
|
||||
monkeypatch.setattr(pipeline.audit_service, "log_event", MagicMock())
|
||||
|
||||
# DB stubs used by TaskService.cancel
|
||||
task = SimpleNamespace(
|
||||
id="t1",
|
||||
short_id="op123",
|
||||
user_id="u1",
|
||||
request_id="r1",
|
||||
external_task_id="op123",
|
||||
endpoint_id="e1",
|
||||
key_id="k1",
|
||||
status=VideoStatus.SUBMITTED.value,
|
||||
updated_at=None,
|
||||
request_metadata={},
|
||||
)
|
||||
endpoint = SimpleNamespace(
|
||||
id="e1",
|
||||
base_url="https://generativelanguage.googleapis.com",
|
||||
api_format="gemini:video",
|
||||
api_family="gemini",
|
||||
endpoint_kind="video",
|
||||
header_rules=None,
|
||||
)
|
||||
provider_key = SimpleNamespace(
|
||||
id="k1",
|
||||
api_key="encrypted",
|
||||
auth_type="api_key",
|
||||
)
|
||||
|
||||
q_task_id = MagicMock()
|
||||
q_task_id.filter.return_value.first.return_value = None
|
||||
q_task_short = MagicMock()
|
||||
q_task_short.filter.return_value.first.return_value = task
|
||||
q_endpoint = MagicMock()
|
||||
q_endpoint.filter.return_value.first.return_value = endpoint
|
||||
q_key = MagicMock()
|
||||
q_key.filter.return_value.first.return_value = provider_key
|
||||
|
||||
db = MagicMock()
|
||||
_video_query_count = {"n": 0}
|
||||
|
||||
def _query(model): # noqa: ANN001
|
||||
name = getattr(model, "__name__", "")
|
||||
if name == "VideoTask":
|
||||
_video_query_count["n"] += 1
|
||||
return q_task_id if _video_query_count["n"] == 1 else q_task_short
|
||||
if name == "ProviderEndpoint":
|
||||
return q_endpoint
|
||||
if name == "ProviderAPIKey":
|
||||
return q_key
|
||||
return MagicMock()
|
||||
|
||||
db.query.side_effect = _query
|
||||
|
||||
upstream = SimpleNamespace(
|
||||
post=AsyncMock(return_value=httpx.Response(200, json={"done": True})),
|
||||
delete=AsyncMock(), # not used for gemini cancel
|
||||
)
|
||||
|
||||
with (
|
||||
patch("src.clients.http_client.HTTPClientPool.get_default_client_async", AsyncMock(return_value=upstream)),
|
||||
patch("src.core.crypto.crypto_service.decrypt", lambda _v: "upstream-key"),
|
||||
patch("src.api.handlers.base.request_builder.get_provider_auth", AsyncMock(return_value=None)),
|
||||
patch("src.services.usage.service.UsageService.finalize_void", MagicMock(return_value=True)),
|
||||
patch("src.services.usage.service.UsageService.void_settled", MagicMock()),
|
||||
):
|
||||
request = _make_request(
|
||||
method="POST",
|
||||
path="/v1beta/operations/op123:cancel",
|
||||
headers={"x-goog-api-key": "sk-test", "x-real-ip": "127.0.0.1", "user-agent": "pytest"},
|
||||
body=b"",
|
||||
)
|
||||
adapter = GeminiVeoAdapter()
|
||||
resp = await pipeline.run(
|
||||
adapter=adapter,
|
||||
http_request=request,
|
||||
db=db,
|
||||
mode=adapter.mode,
|
||||
api_format_hint=adapter.allowed_api_formats[0],
|
||||
path_params={"task_id": "op123", "action": "cancel"},
|
||||
)
|
||||
|
||||
assert getattr(resp, "status_code", None) == 200
|
||||
assert upstream.post.await_count == 1
|
||||
assert upstream.post.call_args.args[0].endswith("/v1beta/operations/op123:cancel")
|
||||
assert task.status == VideoStatus.CANCELLED.value
|
||||
|
||||
@@ -6,8 +6,8 @@ import httpx
|
||||
import pytest
|
||||
|
||||
from src.config.settings import config
|
||||
from src.services.candidate.service import CandidateService
|
||||
from src.services.candidate.submit import AllCandidatesFailedError, UpstreamClientRequestError
|
||||
from src.services.task.service import TaskService
|
||||
|
||||
|
||||
def _make_candidate(
|
||||
@@ -43,11 +43,19 @@ async def test_submit_with_failover_skips_http_500_then_succeeds(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
db = MagicMock()
|
||||
svc = CandidateService(db)
|
||||
svc = TaskService(db)
|
||||
|
||||
# bypass init
|
||||
svc._resolver = SimpleNamespace(
|
||||
fetch_candidates=AsyncMock(
|
||||
monkeypatch.setattr(
|
||||
"src.services.system.config.SystemConfigService.get_config",
|
||||
lambda *_args, **_kwargs: "provider",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.services.cache.aware_scheduler.get_cache_aware_scheduler",
|
||||
AsyncMock(return_value=None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.services.candidate.resolver.CandidateResolver.fetch_candidates",
|
||||
AsyncMock(
|
||||
return_value=(
|
||||
[
|
||||
_make_candidate(provider_id="p1", endpoint_id="e1", key_id="k1"),
|
||||
@@ -55,10 +63,12 @@ async def test_submit_with_failover_skips_http_500_then_succeeds(
|
||||
],
|
||||
"gm1",
|
||||
)
|
||||
)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.services.orchestration.error_classifier.ErrorClassifier.is_client_error",
|
||||
lambda _self, _text: False,
|
||||
)
|
||||
svc._error_classifier = SimpleNamespace(is_client_error=lambda _text: False)
|
||||
svc._ensure_initialized = AsyncMock(return_value=None)
|
||||
|
||||
responses = [
|
||||
httpx.Response(500, text='{"error": {"message": "server"}}'),
|
||||
@@ -89,13 +99,24 @@ async def test_submit_with_failover_skips_http_500_then_succeeds(
|
||||
@pytest.mark.asyncio
|
||||
async def test_submit_with_failover_stops_on_client_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
db = MagicMock()
|
||||
svc = CandidateService(db)
|
||||
svc = TaskService(db)
|
||||
|
||||
svc._resolver = SimpleNamespace(
|
||||
fetch_candidates=AsyncMock(return_value=([_make_candidate()], "gm1"))
|
||||
monkeypatch.setattr(
|
||||
"src.services.system.config.SystemConfigService.get_config",
|
||||
lambda *_args, **_kwargs: "provider",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.services.cache.aware_scheduler.get_cache_aware_scheduler",
|
||||
AsyncMock(return_value=None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.services.candidate.resolver.CandidateResolver.fetch_candidates",
|
||||
AsyncMock(return_value=([_make_candidate()], "gm1")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.services.orchestration.error_classifier.ErrorClassifier.is_client_error",
|
||||
lambda _self, _text: True,
|
||||
)
|
||||
svc._error_classifier = SimpleNamespace(is_client_error=lambda _text: True)
|
||||
svc._ensure_initialized = AsyncMock(return_value=None)
|
||||
|
||||
response = httpx.Response(
|
||||
400,
|
||||
@@ -124,13 +145,24 @@ async def test_submit_with_failover_no_eligible_candidates_due_to_auth_type(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
db = MagicMock()
|
||||
svc = CandidateService(db)
|
||||
svc = TaskService(db)
|
||||
|
||||
svc._resolver = SimpleNamespace(
|
||||
fetch_candidates=AsyncMock(return_value=([_make_candidate(auth_type="vertex_ai")], "gm1"))
|
||||
monkeypatch.setattr(
|
||||
"src.services.system.config.SystemConfigService.get_config",
|
||||
lambda *_args, **_kwargs: "provider",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.services.cache.aware_scheduler.get_cache_aware_scheduler",
|
||||
AsyncMock(return_value=None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.services.candidate.resolver.CandidateResolver.fetch_candidates",
|
||||
AsyncMock(return_value=([_make_candidate(auth_type="vertex_ai")], "gm1")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.services.orchestration.error_classifier.ErrorClassifier.is_client_error",
|
||||
lambda _self, _text: False,
|
||||
)
|
||||
svc._error_classifier = SimpleNamespace(is_client_error=lambda _text: False)
|
||||
svc._ensure_initialized = AsyncMock(return_value=None)
|
||||
|
||||
with pytest.raises(AllCandidatesFailedError) as excinfo:
|
||||
await svc.submit_with_failover(
|
||||
@@ -155,10 +187,19 @@ async def test_submit_with_failover_filters_missing_billing_rule(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
db = MagicMock()
|
||||
svc = CandidateService(db)
|
||||
svc = TaskService(db)
|
||||
|
||||
svc._resolver = SimpleNamespace(
|
||||
fetch_candidates=AsyncMock(
|
||||
monkeypatch.setattr(
|
||||
"src.services.system.config.SystemConfigService.get_config",
|
||||
lambda *_args, **_kwargs: "provider",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.services.cache.aware_scheduler.get_cache_aware_scheduler",
|
||||
AsyncMock(return_value=None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.services.candidate.resolver.CandidateResolver.fetch_candidates",
|
||||
AsyncMock(
|
||||
return_value=(
|
||||
[
|
||||
_make_candidate(provider_id="p1", endpoint_id="e1", key_id="k1"),
|
||||
@@ -166,10 +207,12 @@ async def test_submit_with_failover_filters_missing_billing_rule(
|
||||
],
|
||||
"gm1",
|
||||
)
|
||||
)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.services.orchestration.error_classifier.ErrorClassifier.is_client_error",
|
||||
lambda _self, _text: False,
|
||||
)
|
||||
svc._error_classifier = SimpleNamespace(is_client_error=lambda _text: False)
|
||||
svc._ensure_initialized = AsyncMock(return_value=None)
|
||||
|
||||
# enable require_rule
|
||||
old = config.billing_require_rule
|
||||
@@ -182,7 +225,7 @@ async def test_submit_with_failover_filters_missing_billing_rule(
|
||||
return None if provider_id == "p1" else object()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"src.services.candidate.service.BillingRuleService.find_rule", _find_rule
|
||||
"src.services.billing.rule_service.BillingRuleService.find_rule", _find_rule
|
||||
)
|
||||
|
||||
submit = AsyncMock(return_value=httpx.Response(200, json={"id": "task-999"}))
|
||||
|
||||
310
tests/services/test_failover_engine.py
Normal file
310
tests/services/test_failover_engine.py
Normal file
@@ -0,0 +1,310 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, AsyncIterator
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from src.services.candidate.failover import FailoverEngine
|
||||
from src.services.candidate.policy import RetryMode, RetryPolicy, SkipPolicy
|
||||
from src.services.orchestration.error_classifier import ErrorAction
|
||||
from src.services.task.protocol import AttemptKind, AttemptResult
|
||||
|
||||
|
||||
def _make_candidate(
|
||||
*,
|
||||
provider_id: str = "p1",
|
||||
provider_name: str = "prov",
|
||||
endpoint_id: str = "e1",
|
||||
key_id: str = "k1",
|
||||
key_name: str = "key",
|
||||
auth_type: str = "api_key",
|
||||
priority: int = 0,
|
||||
is_cached: bool = False,
|
||||
is_skipped: bool = False,
|
||||
skip_reason: str | None = None,
|
||||
needs_conversion: bool = False,
|
||||
provider_max_retries: int | None = None,
|
||||
) -> SimpleNamespace:
|
||||
provider = SimpleNamespace(id=provider_id, name=provider_name, max_retries=provider_max_retries)
|
||||
endpoint = SimpleNamespace(id=endpoint_id)
|
||||
key = SimpleNamespace(id=key_id, name=key_name, auth_type=auth_type, priority=priority)
|
||||
return SimpleNamespace(
|
||||
provider=provider,
|
||||
endpoint=endpoint,
|
||||
key=key,
|
||||
is_cached=is_cached,
|
||||
is_skipped=is_skipped,
|
||||
skip_reason=skip_reason,
|
||||
needs_conversion=needs_conversion,
|
||||
)
|
||||
|
||||
|
||||
class _StubErrorClassifier:
|
||||
def __init__(self, *, action: ErrorAction, client_error: bool = False) -> None:
|
||||
self._action = action
|
||||
self._client_error = client_error
|
||||
|
||||
def is_client_error(self, _text: str | None) -> bool:
|
||||
return self._client_error
|
||||
|
||||
def classify(
|
||||
self, _error: Exception, *, has_retry_left: bool = False
|
||||
) -> ErrorAction: # noqa: ARG002
|
||||
return self._action
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failover_engine_success_first_candidate() -> None:
|
||||
db = MagicMock()
|
||||
engine = FailoverEngine(db, error_classifier=_StubErrorClassifier(action=ErrorAction.BREAK))
|
||||
|
||||
candidates = [_make_candidate(provider_id="p1"), _make_candidate(provider_id="p2")]
|
||||
|
||||
attempt = AsyncMock(
|
||||
return_value=AttemptResult(
|
||||
kind=AttemptKind.SYNC_RESPONSE,
|
||||
http_status=200,
|
||||
http_headers={},
|
||||
response_body={"ok": True},
|
||||
)
|
||||
)
|
||||
|
||||
result = await engine.execute(
|
||||
candidates=candidates,
|
||||
attempt_func=attempt,
|
||||
retry_policy=RetryPolicy(mode=RetryMode.DISABLED, max_retries=1),
|
||||
skip_policy=SkipPolicy(),
|
||||
request_id=None,
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
assert result.candidate_index == 0
|
||||
assert result.attempt_count == 1
|
||||
assert result.provider_id == "p1"
|
||||
assert result.response == {"ok": True}
|
||||
assert attempt.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failover_engine_continue_to_next_candidate_on_error() -> None:
|
||||
db = MagicMock()
|
||||
engine = FailoverEngine(db, error_classifier=_StubErrorClassifier(action=ErrorAction.BREAK))
|
||||
|
||||
candidates = [_make_candidate(provider_id="p1"), _make_candidate(provider_id="p2")]
|
||||
|
||||
attempt = AsyncMock(
|
||||
side_effect=[
|
||||
RuntimeError("boom"),
|
||||
AttemptResult(
|
||||
kind=AttemptKind.SYNC_RESPONSE,
|
||||
http_status=200,
|
||||
http_headers={},
|
||||
response_body={"ok": True},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
result = await engine.execute(
|
||||
candidates=candidates,
|
||||
attempt_func=attempt,
|
||||
retry_policy=RetryPolicy(mode=RetryMode.DISABLED, max_retries=1),
|
||||
skip_policy=SkipPolicy(),
|
||||
request_id=None,
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
assert result.candidate_index == 1
|
||||
assert result.provider_id == "p2"
|
||||
assert result.attempt_count == 2
|
||||
assert attempt.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failover_engine_retry_same_candidate_when_classifier_says_continue() -> None:
|
||||
db = MagicMock()
|
||||
# ErrorAction.CONTINUE => retry current candidate (mapped to FailoverAction.RETRY)
|
||||
engine = FailoverEngine(db, error_classifier=_StubErrorClassifier(action=ErrorAction.CONTINUE))
|
||||
|
||||
candidates = [_make_candidate(provider_id="p1", is_cached=True, provider_max_retries=2)]
|
||||
|
||||
attempt = AsyncMock(
|
||||
side_effect=[
|
||||
RuntimeError("transient"),
|
||||
AttemptResult(
|
||||
kind=AttemptKind.SYNC_RESPONSE,
|
||||
http_status=200,
|
||||
http_headers={},
|
||||
response_body={"ok": True},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
result = await engine.execute(
|
||||
candidates=candidates,
|
||||
attempt_func=attempt,
|
||||
retry_policy=RetryPolicy(mode=RetryMode.ON_DEMAND, max_retries=2),
|
||||
skip_policy=SkipPolicy(),
|
||||
request_id=None,
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
assert result.candidate_index == 0
|
||||
assert result.attempt_count == 2
|
||||
assert attempt.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failover_engine_stop_when_classifier_raises() -> None:
|
||||
db = MagicMock()
|
||||
engine = FailoverEngine(db, error_classifier=_StubErrorClassifier(action=ErrorAction.RAISE))
|
||||
|
||||
candidates = [_make_candidate(provider_id="p1"), _make_candidate(provider_id="p2")]
|
||||
attempt = AsyncMock(side_effect=RuntimeError("client-ish"))
|
||||
|
||||
result = await engine.execute(
|
||||
candidates=candidates,
|
||||
attempt_func=attempt,
|
||||
retry_policy=RetryPolicy(mode=RetryMode.DISABLED, max_retries=1),
|
||||
skip_policy=SkipPolicy(),
|
||||
request_id=None,
|
||||
)
|
||||
|
||||
assert result.success is False
|
||||
assert result.error_type == "RuntimeError"
|
||||
assert result.attempt_count == 1
|
||||
# should not try candidate 2
|
||||
assert attempt.await_count == 1
|
||||
|
||||
|
||||
async def _stream_two_chunks() -> AsyncIterator[bytes]:
|
||||
yield b"chunk1"
|
||||
yield b"chunk2"
|
||||
|
||||
|
||||
async def _empty_stream() -> AsyncIterator[bytes]:
|
||||
if False: # pragma: no cover
|
||||
yield b""
|
||||
return
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failover_engine_stream_probe_wraps_first_chunk() -> None:
|
||||
db = MagicMock()
|
||||
engine = FailoverEngine(db, error_classifier=_StubErrorClassifier(action=ErrorAction.BREAK))
|
||||
|
||||
candidates = [_make_candidate(provider_id="p1")]
|
||||
attempt = AsyncMock(
|
||||
return_value=AttemptResult(
|
||||
kind=AttemptKind.STREAM,
|
||||
http_status=200,
|
||||
http_headers={},
|
||||
stream_iterator=_stream_two_chunks(),
|
||||
)
|
||||
)
|
||||
|
||||
result = await engine.execute(
|
||||
candidates=candidates,
|
||||
attempt_func=attempt,
|
||||
retry_policy=RetryPolicy(mode=RetryMode.DISABLED, max_retries=1),
|
||||
skip_policy=SkipPolicy(),
|
||||
request_id=None,
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
assert result.attempt_result is not None
|
||||
assert result.attempt_result.kind == AttemptKind.STREAM
|
||||
|
||||
collected: list[bytes] = []
|
||||
assert result.response is not None
|
||||
async for chunk in result.response: # type: ignore[union-attr]
|
||||
collected.append(chunk)
|
||||
assert collected == [b"chunk1", b"chunk2"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failover_engine_stream_probe_empty_triggers_failover() -> None:
|
||||
db = MagicMock()
|
||||
engine = FailoverEngine(db, error_classifier=_StubErrorClassifier(action=ErrorAction.BREAK))
|
||||
|
||||
candidates = [_make_candidate(provider_id="p1"), _make_candidate(provider_id="p2")]
|
||||
|
||||
attempt = AsyncMock(
|
||||
side_effect=[
|
||||
AttemptResult(
|
||||
kind=AttemptKind.STREAM,
|
||||
http_status=200,
|
||||
http_headers={},
|
||||
stream_iterator=_empty_stream(),
|
||||
),
|
||||
AttemptResult(
|
||||
kind=AttemptKind.SYNC_RESPONSE,
|
||||
http_status=200,
|
||||
http_headers={},
|
||||
response_body={"ok": True},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
result = await engine.execute(
|
||||
candidates=candidates,
|
||||
attempt_func=attempt,
|
||||
retry_policy=RetryPolicy(mode=RetryMode.DISABLED, max_retries=1),
|
||||
skip_policy=SkipPolicy(),
|
||||
request_id=None,
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
assert result.candidate_index == 1
|
||||
assert result.attempt_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failover_engine_pre_expand_marks_unused_slots_on_success(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
db = MagicMock()
|
||||
engine = FailoverEngine(db, error_classifier=_StubErrorClassifier(action=ErrorAction.BREAK))
|
||||
|
||||
# patch low-level record updater to observe unused marking
|
||||
engine._update_record = MagicMock() # type: ignore[method-assign]
|
||||
engine.db.commit = MagicMock() # type: ignore[method-assign]
|
||||
engine._commit_before_await = MagicMock() # type: ignore[method-assign]
|
||||
|
||||
c0 = _make_candidate(provider_id="p1", is_cached=True, provider_max_retries=2)
|
||||
c1 = _make_candidate(provider_id="p2", is_cached=False)
|
||||
|
||||
attempt = AsyncMock(
|
||||
return_value=AttemptResult(
|
||||
kind=AttemptKind.SYNC_RESPONSE,
|
||||
http_status=200,
|
||||
http_headers={},
|
||||
response_body={"ok": True},
|
||||
)
|
||||
)
|
||||
|
||||
record_map = {
|
||||
(0, 0): "r00",
|
||||
(0, 1): "r01",
|
||||
(1, 0): "r10",
|
||||
}
|
||||
|
||||
result = await engine.execute(
|
||||
candidates=[c0, c1],
|
||||
attempt_func=attempt,
|
||||
retry_policy=RetryPolicy(mode=RetryMode.PRE_EXPAND, max_retries=2),
|
||||
skip_policy=SkipPolicy(),
|
||||
request_id=None,
|
||||
candidate_record_map=record_map,
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
|
||||
# Ensure we marked the remaining slots unused (r01 + r10)
|
||||
unused_record_ids = {
|
||||
call.args[0]
|
||||
for call in engine._update_record.call_args_list # type: ignore[attr-defined]
|
||||
if call.kwargs.get("status") == "unused"
|
||||
}
|
||||
assert unused_record_ids == {"r01", "r10"}
|
||||
@@ -28,7 +28,9 @@ def _mock_endpoint(api_format: str, config: dict | None = None) -> MagicMock:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_candidates_blocks_cross_format_when_global_switch_off() -> None:
|
||||
async def test_build_candidates_allows_cross_format_when_endpoint_accepts_and_overrides_off() -> (
|
||||
None
|
||||
):
|
||||
register_default_normalizers()
|
||||
|
||||
scheduler = CacheAwareScheduler()
|
||||
@@ -37,6 +39,41 @@ async def test_build_candidates_blocks_cross_format_when_global_switch_off() ->
|
||||
|
||||
provider = MagicMock()
|
||||
provider.name = "p1"
|
||||
provider.enable_format_conversion = False
|
||||
provider.endpoints = [
|
||||
_mock_endpoint(
|
||||
"openai:chat",
|
||||
{"enabled": True, "accept_formats": ["claude:chat"], "stream_conversion": True},
|
||||
)
|
||||
]
|
||||
provider.api_keys = [_mock_key("k1", ["openai:chat"])]
|
||||
|
||||
candidates = await scheduler._build_candidates(
|
||||
db=MagicMock(),
|
||||
providers=[provider],
|
||||
client_format="claude:chat",
|
||||
model_name="dummy-model",
|
||||
affinity_key=None,
|
||||
global_conversion_enabled=False, # DB 全局覆盖关闭
|
||||
master_conversion_enabled=True, # ENV 总闸开启(默认)
|
||||
)
|
||||
|
||||
assert len(candidates) == 1
|
||||
assert candidates[0].needs_conversion is True
|
||||
assert candidates[0].provider_api_format == "openai:chat"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_candidates_blocks_cross_format_when_master_switch_off() -> None:
|
||||
register_default_normalizers()
|
||||
|
||||
scheduler = CacheAwareScheduler()
|
||||
scheduler._check_model_support = AsyncMock(return_value=(True, None, None, {"m"})) # type: ignore[attr-defined]
|
||||
scheduler._check_key_availability = MagicMock(return_value=(True, None, None)) # type: ignore[attr-defined]
|
||||
|
||||
provider = MagicMock()
|
||||
provider.name = "p1"
|
||||
provider.enable_format_conversion = False
|
||||
provider.endpoints = [
|
||||
_mock_endpoint(
|
||||
"openai:chat",
|
||||
@@ -52,6 +89,7 @@ async def test_build_candidates_blocks_cross_format_when_global_switch_off() ->
|
||||
model_name="dummy-model",
|
||||
affinity_key=None,
|
||||
global_conversion_enabled=False,
|
||||
master_conversion_enabled=False,
|
||||
)
|
||||
|
||||
assert candidates == []
|
||||
@@ -67,11 +105,10 @@ async def test_build_candidates_includes_cross_format_when_enabled() -> None:
|
||||
|
||||
provider = MagicMock()
|
||||
provider.name = "p1"
|
||||
provider.enable_format_conversion = False
|
||||
provider.endpoints = [
|
||||
_mock_endpoint(
|
||||
"openai:chat",
|
||||
{"enabled": True, "accept_formats": ["claude:chat"], "stream_conversion": True},
|
||||
)
|
||||
# 端点未配置/未启用格式接受策略,但 DB 全局覆盖开启应强制允许
|
||||
_mock_endpoint("openai:chat", None)
|
||||
]
|
||||
provider.api_keys = [_mock_key("k1", ["openai:chat"])]
|
||||
|
||||
@@ -81,7 +118,8 @@ async def test_build_candidates_includes_cross_format_when_enabled() -> None:
|
||||
client_format="claude:chat",
|
||||
model_name="dummy-model",
|
||||
affinity_key=None,
|
||||
global_conversion_enabled=True,
|
||||
global_conversion_enabled=True, # DB 全局覆盖开启:跳过端点检查
|
||||
master_conversion_enabled=True,
|
||||
)
|
||||
|
||||
assert len(candidates) == 1
|
||||
@@ -99,6 +137,7 @@ async def test_exact_matches_rank_before_convertible() -> None:
|
||||
|
||||
provider = MagicMock()
|
||||
provider.name = "p1"
|
||||
provider.enable_format_conversion = False
|
||||
# 故意把 OPENAI 放在 endpoints[0],验证排序仍然是 CLAUDE(exact)在前
|
||||
provider.endpoints = [
|
||||
_mock_endpoint(
|
||||
@@ -118,7 +157,8 @@ async def test_exact_matches_rank_before_convertible() -> None:
|
||||
client_format="claude:chat",
|
||||
model_name="dummy-model",
|
||||
affinity_key=None,
|
||||
global_conversion_enabled=True,
|
||||
global_conversion_enabled=False,
|
||||
master_conversion_enabled=True,
|
||||
)
|
||||
|
||||
assert len(candidates) == 2
|
||||
|
||||
74
tests/services/test_task_service_async_execute.py
Normal file
74
tests/services/test_task_service_async_execute.py
Normal file
@@ -0,0 +1,74 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from src.services.candidate.schema import CandidateKey
|
||||
from src.services.candidate.submit import SubmitOutcome
|
||||
from src.services.task.context import TaskMode
|
||||
from src.services.task.protocol import AttemptKind
|
||||
from src.services.task.service import TaskService
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_service_execute_async_requires_extract_external_task_id() -> None:
|
||||
svc = TaskService(MagicMock())
|
||||
with pytest.raises(ValueError):
|
||||
await svc.execute(
|
||||
task_type="video",
|
||||
task_mode=TaskMode.ASYNC,
|
||||
api_format="openai:video",
|
||||
model_name="m",
|
||||
user_api_key=MagicMock(id="u", user_id="user"),
|
||||
request_func=AsyncMock(),
|
||||
request_id="rid",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_service_execute_async_returns_execution_result() -> None:
|
||||
db = MagicMock()
|
||||
svc = TaskService(db)
|
||||
|
||||
candidate = SimpleNamespace(
|
||||
provider=SimpleNamespace(id="p1", name="prov"),
|
||||
endpoint=SimpleNamespace(id="e1"),
|
||||
key=SimpleNamespace(id="k1"),
|
||||
)
|
||||
outcome = SubmitOutcome(
|
||||
candidate=candidate,
|
||||
candidate_keys=[{"index": 0, "provider_id": "p1"}],
|
||||
external_task_id="task_123",
|
||||
rule_lookup=None,
|
||||
upstream_payload={"id": "x"},
|
||||
upstream_headers={"x-test": "1"},
|
||||
upstream_status_code=200,
|
||||
)
|
||||
|
||||
svc.submit_with_failover = AsyncMock(return_value=outcome) # type: ignore[method-assign]
|
||||
svc._recorder.get_candidate_keys = MagicMock( # type: ignore[attr-defined]
|
||||
return_value=[
|
||||
CandidateKey(candidate_index=0, retry_index=0, status="success", provider_id="p1")
|
||||
]
|
||||
)
|
||||
|
||||
result = await svc.execute(
|
||||
task_type="video",
|
||||
task_mode=TaskMode.ASYNC,
|
||||
api_format="openai:video",
|
||||
model_name="m",
|
||||
user_api_key=MagicMock(id="u", user_id="user"),
|
||||
request_func=AsyncMock(),
|
||||
request_id="rid",
|
||||
extract_external_task_id=MagicMock(),
|
||||
allow_format_conversion=True,
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
assert result.attempt_result is not None
|
||||
assert result.attempt_result.kind == AttemptKind.ASYNC_SUBMIT
|
||||
assert result.provider_task_id == "task_123"
|
||||
assert result.provider_id == "p1"
|
||||
assert result.candidate_index == 0
|
||||
@@ -7,7 +7,7 @@ import pytest
|
||||
from src.config.settings import config
|
||||
from src.core.api_format.conversion.internal_video import VideoStatus
|
||||
from src.services.billing.formula_engine import BillingIncompleteError
|
||||
from src.services.task.application import TaskApplicationService
|
||||
from src.services.task.service import TaskService
|
||||
|
||||
|
||||
def _make_task(**overrides: Any) -> SimpleNamespace:
|
||||
@@ -81,18 +81,18 @@ async def test_video_finalize_failed_records_cost_zero(monkeypatch: pytest.Monke
|
||||
lambda _self, **_kwargs: {"duration_seconds": 4},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.services.task.application.BillingRuleService.find_rule",
|
||||
"src.services.billing.rule_service.BillingRuleService.find_rule",
|
||||
lambda *_args, **_kwargs: None,
|
||||
)
|
||||
# Mock update_settled_billing (used by finalize_video_task)
|
||||
update_settled = MagicMock(return_value=True)
|
||||
monkeypatch.setattr(
|
||||
"src.services.task.application.UsageService.update_settled_billing",
|
||||
"src.services.usage.service.UsageService.update_settled_billing",
|
||||
update_settled,
|
||||
)
|
||||
|
||||
app = TaskApplicationService(db)
|
||||
await app.finalize_video_task(task)
|
||||
svc = TaskService(db)
|
||||
await svc.finalize_video_task(task)
|
||||
|
||||
# billing_snapshot should be written back to task.request_metadata
|
||||
assert task.request_metadata["billing_snapshot"]["cost"] == 0.0
|
||||
@@ -113,18 +113,18 @@ async def test_video_finalize_completed_no_rule(monkeypatch: pytest.MonkeyPatch)
|
||||
lambda _self, **_kwargs: {"duration_seconds": 4},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.services.task.application.BillingRuleService.find_rule",
|
||||
"src.services.billing.rule_service.BillingRuleService.find_rule",
|
||||
lambda *_args, **_kwargs: None,
|
||||
)
|
||||
# Mock update_settled_billing (used by finalize_video_task)
|
||||
update_settled = MagicMock(return_value=True)
|
||||
monkeypatch.setattr(
|
||||
"src.services.task.application.UsageService.update_settled_billing",
|
||||
"src.services.usage.service.UsageService.update_settled_billing",
|
||||
update_settled,
|
||||
)
|
||||
|
||||
app = TaskApplicationService(db)
|
||||
await app.finalize_video_task(task)
|
||||
svc = TaskService(db)
|
||||
await svc.finalize_video_task(task)
|
||||
|
||||
assert task.request_metadata["billing_snapshot"]["status"] == "no_rule"
|
||||
|
||||
@@ -161,7 +161,7 @@ async def test_video_finalize_strict_mode_missing_required_marks_failed(
|
||||
# Mock update_settled_billing (used by finalize_video_task)
|
||||
update_settled = MagicMock(return_value=True)
|
||||
monkeypatch.setattr(
|
||||
"src.services.task.application.UsageService.update_settled_billing",
|
||||
"src.services.usage.service.UsageService.update_settled_billing",
|
||||
update_settled,
|
||||
)
|
||||
|
||||
@@ -169,15 +169,15 @@ async def test_video_finalize_strict_mode_missing_required_marks_failed(
|
||||
try:
|
||||
config.billing_strict_mode = True
|
||||
monkeypatch.setattr(
|
||||
"src.services.task.application.FormulaEngine.evaluate",
|
||||
"src.services.billing.formula_engine.FormulaEngine.evaluate",
|
||||
MagicMock(
|
||||
side_effect=BillingIncompleteError(
|
||||
"Missing required dimensions", missing_required=["duration_seconds"]
|
||||
)
|
||||
),
|
||||
)
|
||||
app = TaskApplicationService(db)
|
||||
await app.finalize_video_task(task)
|
||||
svc = TaskService(db)
|
||||
await svc.finalize_video_task(task)
|
||||
finally:
|
||||
config.billing_strict_mode = old
|
||||
|
||||
|
||||
Reference in New Issue
Block a user