mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 17:30:23 +08:00
refactor: 统一任务框架 Phase 3 - 用 TaskService/FailoverEngine 替代 FallbackOrchestrator
核心重构：
- 移除 FallbackOrchestrator，用 TaskService + FailoverEngine 替代
- TaskService 作为统一入口，支持 SYNC/ASYNC 两种任务模式
- FailoverEngine 实现候选遍历、重试、故障转移逻辑
- 新增 AttemptFunc/AttemptResult 协议，统一尝试结果表示
功能改进：
- 流式响应首字节探测（30s 超时，空流触发故障转移）
- 流式取消归因优化（区分客户端断连 vs 服务端中断）
- 新增 OpenAI Sora 视频取消路由 POST /v1/videos/{task_id}/cancel
- OpenAI 流式请求自动添加 stream_options.include_usage
代码规范：
- 修复 loguru 日志格式（%s → {}）
- 新增 FORMAT_CONVERSION_ENABLED 环境变量说明
测试覆盖：
- test_failover_engine.py: FailoverEngine 单元测试
- test_task_service_async_execute.py: TaskService ASYNC 模式测试
- test_video_cancel_e2e.py: 视频取消端到端测试
This commit is contained in:
fawney19
@@ -1,28 +1,54 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Protocol
|
||||
import asyncio
|
||||
import re
|
||||
from collections.abc import Awaitable, Callable
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, AsyncIterator
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from src.core.logger import logger
|
||||
from src.models.database import RequestCandidate
|
||||
from src.services.cache.aware_scheduler import ProviderCandidate
|
||||
from src.services.orchestration.error_classifier import ErrorAction, ErrorClassifier
|
||||
from src.services.request.candidate import RequestCandidateService
|
||||
from src.services.task.exceptions import StreamProbeError
|
||||
from src.services.task.protocol import AttemptFunc, AttemptKind, AttemptResult
|
||||
from src.services.task.schema import ExecutionResult
|
||||
|
||||
from .policy import RetryPolicy, SkipPolicy
|
||||
from .schema import CandidateKey, CandidateResult
|
||||
from .policy import FailoverAction, RetryMode, RetryPolicy, SkipPolicy
|
||||
from .recorder import CandidateRecorder
|
||||
from .schema import CandidateKey
|
||||
|
||||
|
||||
class AttemptFunc(Protocol):
|
||||
async def __call__(self, candidate: ProviderCandidate) -> Any: ...
|
||||
_SENSITIVE_PATTERN = re.compile(
|
||||
r"(api[_-]?key|token|bearer|authorization)[=:\s]+\S+",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
class FailoverEngine:
|
||||
"""
|
||||
FailoverEngine executes candidate attempts under policies.
|
||||
|
||||
Phase2 scaffolding: implementation will gradually replace legacy orchestrators.
|
||||
Phase3 core: unified failover loop used by TaskService.
|
||||
"""
|
||||
|
||||
def __init__(self, db: Session):
|
||||
# Hard constraint: streaming first chunk probe timeout
|
||||
STREAM_FIRST_CHUNK_TIMEOUT_SECONDS: int = 30
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db: Session,
|
||||
*,
|
||||
error_classifier: ErrorClassifier | None = None,
|
||||
recorder: CandidateRecorder | None = None,
|
||||
) -> None:
|
||||
self.db = db
|
||||
self._error_classifier = error_classifier or ErrorClassifier(db=db)
|
||||
self._recorder = recorder or CandidateRecorder(db)
|
||||
|
||||
async def execute(
|
||||
self,
|
||||
@@ -32,26 +58,688 @@ class FailoverEngine:
|
||||
retry_policy: RetryPolicy,
|
||||
skip_policy: SkipPolicy,
|
||||
request_id: str | None = None,
|
||||
user_id: str | None = None,
|
||||
api_key_id: str | None = None,
|
||||
candidate_record_map: dict[tuple[int, int], str] | None = None,
|
||||
max_candidates: int | None = None,
|
||||
) -> CandidateResult:
|
||||
# NOTE: intentionally minimal for now; legacy orchestrators still in use.
|
||||
# This will be implemented when migrating video/chat flows to CandidateService.
|
||||
_ = (retry_policy, skip_policy, request_id, max_candidates)
|
||||
candidate_keys: list[CandidateKey] = []
|
||||
max_attempts: int | None = None,
|
||||
execution_error_handler: (
|
||||
Callable[
|
||||
...,
|
||||
Awaitable[tuple[FailoverAction, int | None]],
|
||||
]
|
||||
| None
|
||||
) = None,
|
||||
) -> ExecutionResult:
|
||||
"""
|
||||
Execute candidate traversal + retry + failover.
|
||||
|
||||
Notes:
|
||||
- For PRE_EXPAND: `candidate_record_map` should be provided (created by CandidateResolver).
|
||||
- For ON_DEMAND/DISABLED: records are created when used (and on skip, best-effort).
|
||||
"""
|
||||
candidate_keys_fallback: list[CandidateKey] = []
|
||||
|
||||
if max_candidates is not None and max_candidates > 0:
|
||||
candidates = candidates[:max_candidates]
|
||||
|
||||
attempt_count = 0
|
||||
last_status_code: int | None = None
|
||||
|
||||
# For logging / dispatcher parity only; callers may pass an exact value.
|
||||
if max_attempts is None:
|
||||
computed = 0
|
||||
for cand in candidates:
|
||||
should_skip, _ = self._should_skip(cand, skip_policy)
|
||||
if should_skip:
|
||||
continue
|
||||
computed += self._get_max_retries(cand, retry_policy)
|
||||
max_attempts = computed
|
||||
|
||||
for candidate_index, candidate in enumerate(candidates):
|
||||
should_skip, skip_reason = self._should_skip(candidate, skip_policy)
|
||||
if should_skip:
|
||||
# PRE_EXPAND: mark all retry slots skipped.
|
||||
if retry_policy.mode == RetryMode.PRE_EXPAND and candidate_record_map:
|
||||
self._mark_candidate_skipped(
|
||||
candidate_record_map=candidate_record_map,
|
||||
candidate_index=candidate_index,
|
||||
candidate=candidate,
|
||||
retry_policy=retry_policy,
|
||||
skip_reason=skip_reason,
|
||||
)
|
||||
else:
|
||||
# ON_DEMAND/DISABLED: create a skipped record for audit (best-effort).
|
||||
if request_id:
|
||||
await self._create_skipped_record(
|
||||
request_id=request_id,
|
||||
candidate=candidate,
|
||||
candidate_index=candidate_index,
|
||||
user_id=user_id,
|
||||
api_key_id=api_key_id,
|
||||
skip_reason=skip_reason,
|
||||
)
|
||||
candidate_keys_fallback.append(
|
||||
self._make_candidate_key(
|
||||
candidate=candidate,
|
||||
candidate_index=candidate_index,
|
||||
retry_index=0,
|
||||
status="skipped",
|
||||
skip_reason=skip_reason,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
max_retries = self._get_max_retries(candidate, retry_policy)
|
||||
retry_index = 0
|
||||
while retry_index < max_retries:
|
||||
attempt_count += 1
|
||||
|
||||
# Resolve/create record_id
|
||||
record_id = None
|
||||
if candidate_record_map:
|
||||
record_id = candidate_record_map.get((candidate_index, retry_index))
|
||||
if record_id is None:
|
||||
# Rectify may extend retries beyond pre-created range; reuse retry 0 record.
|
||||
record_id = candidate_record_map.get((candidate_index, 0))
|
||||
if record_id is None and request_id and retry_policy.mode != RetryMode.PRE_EXPAND:
|
||||
record_id = await self._ensure_record_exists(
|
||||
request_id=request_id,
|
||||
candidate=candidate,
|
||||
candidate_index=candidate_index,
|
||||
retry_index=retry_index,
|
||||
user_id=user_id,
|
||||
api_key_id=api_key_id,
|
||||
)
|
||||
|
||||
# Attach per-attempt context onto candidate for attempt_func (keeps AttemptFunc signature stable).
|
||||
try:
|
||||
setattr(candidate, "_utf_candidate_index", candidate_index)
|
||||
setattr(candidate, "_utf_retry_index", retry_index)
|
||||
setattr(candidate, "_utf_candidate_record_id", record_id)
|
||||
setattr(candidate, "_utf_attempt_count", attempt_count)
|
||||
setattr(candidate, "_utf_max_attempts", max_attempts)
|
||||
except Exception:
|
||||
# Best-effort only; attempt_func may not rely on these attributes.
|
||||
pass
|
||||
|
||||
# Mark pending
|
||||
now = datetime.now(timezone.utc)
|
||||
if record_id:
|
||||
self._update_record(
|
||||
record_id,
|
||||
status="pending",
|
||||
started_at=now,
|
||||
)
|
||||
|
||||
# Commit BEFORE await (avoid holding DB connections during slow upstream calls)
|
||||
self._commit_before_await()
|
||||
|
||||
try:
|
||||
attempt_result = await attempt_func(candidate)
|
||||
last_status_code = int(getattr(attempt_result, "http_status", 0) or 0)
|
||||
|
||||
# Stream: probe first chunk, failover only before first chunk
|
||||
if attempt_result.kind == AttemptKind.STREAM:
|
||||
attempt_result = await self._probe_stream_first_chunk(
|
||||
attempt_result=attempt_result,
|
||||
record_id=record_id,
|
||||
)
|
||||
|
||||
# Mark success-like status
|
||||
if record_id:
|
||||
if attempt_result.kind == AttemptKind.STREAM:
|
||||
# For streaming, mark "streaming" (final status is recorded elsewhere).
|
||||
self._update_record(
|
||||
record_id,
|
||||
status="streaming",
|
||||
status_code=attempt_result.http_status,
|
||||
)
|
||||
else:
|
||||
self._update_record(
|
||||
record_id,
|
||||
status="success",
|
||||
status_code=attempt_result.http_status,
|
||||
finished_at=datetime.now(timezone.utc),
|
||||
)
|
||||
self.db.commit()
|
||||
|
||||
# PRE_EXPAND: mark unused slots after request ends (success)
|
||||
if retry_policy.mode == RetryMode.PRE_EXPAND and candidate_record_map:
|
||||
self._mark_remaining_slots_unused(
|
||||
candidate_record_map=candidate_record_map,
|
||||
candidates=candidates,
|
||||
success_candidate_idx=candidate_index,
|
||||
success_retry_idx=retry_index,
|
||||
retry_policy=retry_policy,
|
||||
)
|
||||
|
||||
return ExecutionResult(
|
||||
success=True,
|
||||
attempt_result=attempt_result,
|
||||
candidate=candidate,
|
||||
candidate_index=candidate_index,
|
||||
retry_index=retry_index,
|
||||
provider_id=str(candidate.provider.id),
|
||||
provider_name=str(candidate.provider.name),
|
||||
endpoint_id=str(candidate.endpoint.id),
|
||||
key_id=str(candidate.key.id),
|
||||
candidate_keys=self._get_candidate_keys(
|
||||
request_id=request_id,
|
||||
fallback=candidate_keys_fallback,
|
||||
candidates=candidates,
|
||||
),
|
||||
attempt_count=attempt_count,
|
||||
request_candidate_id=record_id,
|
||||
)
|
||||
|
||||
except StreamProbeError as exc:
|
||||
# Probe failed (before first chunk) => eligible for failover
|
||||
last_status_code = exc.http_status
|
||||
if record_id:
|
||||
self._update_record(
|
||||
record_id,
|
||||
status="failed",
|
||||
status_code=exc.http_status,
|
||||
error_type=type(exc).__name__,
|
||||
error_message=self._sanitize(str(exc)),
|
||||
finished_at=datetime.now(timezone.utc),
|
||||
)
|
||||
self.db.commit()
|
||||
action = FailoverAction.CONTINUE
|
||||
|
||||
except Exception as exc:
|
||||
has_retry_left = retry_index + 1 < max_retries
|
||||
|
||||
# If caller provides an execution_error_handler, prefer it for RequestExecutor's ExecutionError.
|
||||
handler_used = False
|
||||
if execution_error_handler is not None:
|
||||
try:
|
||||
from src.services.request.executor import (
|
||||
ExecutionError as _ExecutionError,
|
||||
)
|
||||
|
||||
if isinstance(exc, _ExecutionError):
|
||||
handler_used = True
|
||||
action, new_max_retries = await execution_error_handler(
|
||||
exec_err=exc,
|
||||
candidate=candidate,
|
||||
candidate_index=candidate_index,
|
||||
retry_index=retry_index,
|
||||
max_retries_for_candidate=max_retries,
|
||||
record_id=record_id,
|
||||
attempt_count=attempt_count,
|
||||
max_attempts=max_attempts,
|
||||
)
|
||||
if new_max_retries is not None:
|
||||
max_retries = max(max_retries, int(new_max_retries))
|
||||
except Exception:
|
||||
# Fall back to internal handler below.
|
||||
handler_used = False
|
||||
|
||||
if not handler_used:
|
||||
action = await self._handle_error(
|
||||
exc,
|
||||
candidate=candidate,
|
||||
has_retry_left=has_retry_left,
|
||||
)
|
||||
|
||||
last_status_code = int(getattr(exc, "status_code", 0) or 0) or int(
|
||||
getattr(exc, "http_status", 0) or 0
|
||||
)
|
||||
|
||||
if record_id:
|
||||
self._update_record(
|
||||
record_id,
|
||||
status="failed",
|
||||
status_code=last_status_code or None,
|
||||
error_type=type(exc).__name__,
|
||||
error_message=self._sanitize(str(exc)),
|
||||
finished_at=datetime.now(timezone.utc),
|
||||
)
|
||||
self.db.commit()
|
||||
|
||||
if action == FailoverAction.STOP:
|
||||
# PRE_EXPAND: STOP ends the request => mark remaining slots unused.
|
||||
if retry_policy.mode == RetryMode.PRE_EXPAND and candidate_record_map:
|
||||
self._mark_remaining_slots_unused(
|
||||
candidate_record_map=candidate_record_map,
|
||||
candidates=candidates,
|
||||
success_candidate_idx=candidate_index,
|
||||
success_retry_idx=retry_index,
|
||||
retry_policy=retry_policy,
|
||||
)
|
||||
return ExecutionResult(
|
||||
success=False,
|
||||
error_type=type(exc).__name__,
|
||||
error_message=self._sanitize(str(exc)),
|
||||
last_status_code=last_status_code or None,
|
||||
candidate_keys=self._get_candidate_keys(
|
||||
request_id=request_id,
|
||||
fallback=candidate_keys_fallback,
|
||||
candidates=candidates,
|
||||
),
|
||||
attempt_count=attempt_count,
|
||||
)
|
||||
|
||||
# action switch: continue/ retry
|
||||
if action == FailoverAction.CONTINUE:
|
||||
# PRE_EXPAND: if we break early, mark remaining retries of this candidate unused.
|
||||
if retry_policy.mode == RetryMode.PRE_EXPAND and candidate_record_map:
|
||||
self._mark_candidate_remaining_retries_unused(
|
||||
candidate_record_map=candidate_record_map,
|
||||
candidate_idx=candidate_index,
|
||||
from_retry_idx=retry_index + 1,
|
||||
retry_policy=retry_policy,
|
||||
)
|
||||
break
|
||||
if action == FailoverAction.RETRY:
|
||||
retry_index += 1
|
||||
continue
|
||||
|
||||
# Safety: unknown action -> stop retrying this candidate.
|
||||
break
|
||||
|
||||
# exhausted: PRE_EXPAND should not leave 'available' records behind
|
||||
if retry_policy.mode == RetryMode.PRE_EXPAND and candidate_record_map:
|
||||
self._mark_all_remaining_available_unused(candidate_record_map)
|
||||
|
||||
return ExecutionResult(
|
||||
success=False,
|
||||
error_type="AllCandidatesFailed",
|
||||
error_message="All candidates exhausted",
|
||||
last_status_code=last_status_code,
|
||||
candidate_keys=self._get_candidate_keys(
|
||||
request_id=request_id,
|
||||
fallback=candidate_keys_fallback,
|
||||
candidates=candidates,
|
||||
),
|
||||
attempt_count=attempt_count,
|
||||
)
|
||||
|
||||
def _sanitize(self, message: str, max_length: int = 200) -> str:
|
||||
if not message:
|
||||
return "request_failed"
|
||||
return _SENSITIVE_PATTERN.sub("[REDACTED]", message)[:max_length]
|
||||
|
||||
def _make_candidate_key(
|
||||
self,
|
||||
*,
|
||||
candidate: ProviderCandidate,
|
||||
candidate_index: int,
|
||||
retry_index: int,
|
||||
status: str,
|
||||
skip_reason: str | None = None,
|
||||
error_type: str | None = None,
|
||||
error_message: str | None = None,
|
||||
status_code: int | None = None,
|
||||
) -> CandidateKey:
|
||||
return CandidateKey(
|
||||
candidate_index=candidate_index,
|
||||
retry_index=retry_index,
|
||||
provider_id=str(candidate.provider.id),
|
||||
provider_name=str(candidate.provider.name),
|
||||
endpoint_id=str(candidate.endpoint.id),
|
||||
key_id=str(candidate.key.id),
|
||||
key_name=str(getattr(candidate.key, "name", "") or ""),
|
||||
auth_type=str(getattr(candidate.key, "auth_type", "") or ""),
|
||||
priority=int(getattr(candidate.key, "priority", 0) or 0),
|
||||
is_cached=bool(getattr(candidate, "is_cached", False)),
|
||||
status=status,
|
||||
skip_reason=skip_reason,
|
||||
error_type=error_type,
|
||||
error_message=error_message,
|
||||
status_code=status_code,
|
||||
)
|
||||
|
||||
def _get_candidate_keys(
|
||||
self,
|
||||
*,
|
||||
request_id: str | None,
|
||||
fallback: list[CandidateKey],
|
||||
candidates: list[ProviderCandidate],
|
||||
) -> list[CandidateKey]:
|
||||
if request_id:
|
||||
try:
|
||||
return self._recorder.get_candidate_keys(request_id)
|
||||
except Exception as exc:
|
||||
# 降级到 fallback 但记录 warning（影响审计追踪可见性）
|
||||
logger.warning(
|
||||
"[FailoverEngine] get_candidate_keys failed, using fallback: {}",
|
||||
self._sanitize(str(exc)),
|
||||
)
|
||||
if fallback:
|
||||
return fallback
|
||||
# fallback snapshot (no DB audit)
|
||||
result: list[CandidateKey] = []
|
||||
for idx, cand in enumerate(candidates):
|
||||
candidate_keys.append(
|
||||
CandidateKey(
|
||||
result.append(
|
||||
self._make_candidate_key(
|
||||
candidate=cand,
|
||||
candidate_index=idx,
|
||||
provider_id=str(cand.provider.id),
|
||||
provider_name=str(cand.provider.name),
|
||||
endpoint_id=str(cand.endpoint.id),
|
||||
key_id=str(cand.key.id),
|
||||
key_name=str(getattr(cand.key, "name", "") or ""),
|
||||
auth_type=str(getattr(cand.key, "auth_type", "") or ""),
|
||||
priority=int(getattr(cand.key, "priority", 0) or 0),
|
||||
is_cached=bool(getattr(cand, "is_cached", False)),
|
||||
retry_index=0,
|
||||
status="available",
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
raise NotImplementedError("FailoverEngine.execute is not implemented yet")
|
||||
def _commit_before_await(self) -> None:
|
||||
if self.db.in_transaction():
|
||||
try:
|
||||
self.db.commit()
|
||||
except Exception:
|
||||
self.db.rollback()
|
||||
raise
|
||||
|
||||
def _update_record(self, record_id: str, /, **values: Any) -> None:
|
||||
self.db.execute(
|
||||
update(RequestCandidate).where(RequestCandidate.id == record_id).values(**values)
|
||||
)
|
||||
|
||||
async def _ensure_record_exists(
|
||||
self,
|
||||
*,
|
||||
request_id: str,
|
||||
candidate: ProviderCandidate,
|
||||
candidate_index: int,
|
||||
retry_index: int,
|
||||
user_id: str | None,
|
||||
api_key_id: str | None,
|
||||
) -> str:
|
||||
# Create "available" record, then caller will mark pending.
|
||||
row = RequestCandidateService.create_candidate(
|
||||
db=self.db,
|
||||
request_id=request_id,
|
||||
candidate_index=candidate_index,
|
||||
retry_index=retry_index,
|
||||
user_id=user_id,
|
||||
api_key_id=api_key_id,
|
||||
provider_id=str(candidate.provider.id),
|
||||
endpoint_id=str(candidate.endpoint.id),
|
||||
key_id=str(candidate.key.id),
|
||||
status="available",
|
||||
is_cached=bool(getattr(candidate, "is_cached", False)),
|
||||
extra_data={},
|
||||
)
|
||||
return str(row.id)
|
||||
|
||||
async def _create_skipped_record(
|
||||
self,
|
||||
*,
|
||||
request_id: str,
|
||||
candidate: ProviderCandidate,
|
||||
candidate_index: int,
|
||||
user_id: str | None,
|
||||
api_key_id: str | None,
|
||||
skip_reason: str | None,
|
||||
) -> str:
|
||||
row = RequestCandidateService.create_candidate(
|
||||
db=self.db,
|
||||
request_id=request_id,
|
||||
candidate_index=candidate_index,
|
||||
retry_index=0,
|
||||
user_id=user_id,
|
||||
api_key_id=api_key_id,
|
||||
provider_id=str(candidate.provider.id),
|
||||
endpoint_id=str(candidate.endpoint.id),
|
||||
key_id=str(candidate.key.id),
|
||||
status="skipped",
|
||||
skip_reason=skip_reason,
|
||||
is_cached=bool(getattr(candidate, "is_cached", False)),
|
||||
extra_data={},
|
||||
)
|
||||
# ensure visible for subsequent recorder reads
|
||||
if self.db.in_transaction():
|
||||
self.db.commit()
|
||||
return str(row.id)
|
||||
|
||||
def _should_skip(
|
||||
self, candidate: ProviderCandidate, skip_policy: SkipPolicy
|
||||
) -> tuple[bool, str | None]:
|
||||
if bool(getattr(candidate, "is_skipped", False)):
|
||||
return True, str(getattr(candidate, "skip_reason", None) or "scheduler_marked")
|
||||
|
||||
auth_type = str(getattr(getattr(candidate, "key", None), "auth_type", "") or "api_key")
|
||||
if (
|
||||
skip_policy.supported_auth_types is not None
|
||||
and auth_type not in skip_policy.supported_auth_types
|
||||
):
|
||||
return True, "unsupported_auth_type"
|
||||
|
||||
needs_conversion = bool(getattr(candidate, "needs_conversion", False))
|
||||
if needs_conversion and not skip_policy.allow_format_conversion:
|
||||
return True, "format_conversion_not_supported"
|
||||
|
||||
return False, None
|
||||
|
||||
def _get_max_retries(self, candidate: ProviderCandidate, retry_policy: RetryPolicy) -> int:
|
||||
if retry_policy.mode == RetryMode.DISABLED:
|
||||
return 1
|
||||
if retry_policy.retry_on_cached_only and not bool(getattr(candidate, "is_cached", False)):
|
||||
return 1
|
||||
provider_max = getattr(getattr(candidate, "provider", None), "max_retries", None)
|
||||
try:
|
||||
value = int(provider_max or retry_policy.max_retries or 1)
|
||||
except Exception:
|
||||
value = int(retry_policy.max_retries or 1)
|
||||
return max(1, value)
|
||||
|
||||
def _should_stop_on_http_error(self, *, status_code: int, error_text: str) -> bool:
|
||||
# follow CandidateService rules
|
||||
if status_code in (401, 403, 429):
|
||||
return False
|
||||
if 400 <= status_code < 500:
|
||||
return self._error_classifier.is_client_error(error_text)
|
||||
return False
|
||||
|
||||
async def _handle_error(
|
||||
self,
|
||||
error: Exception,
|
||||
*,
|
||||
candidate: ProviderCandidate,
|
||||
has_retry_left: bool,
|
||||
) -> FailoverAction:
|
||||
# Special: HTTP client errors should stop failover.
|
||||
if isinstance(error, httpx.HTTPStatusError):
|
||||
status_code = int(getattr(error.response, "status_code", 0) or 0)
|
||||
try:
|
||||
error_text = error.response.text or ""
|
||||
except Exception:
|
||||
error_text = ""
|
||||
if self._should_stop_on_http_error(status_code=status_code, error_text=error_text):
|
||||
return FailoverAction.STOP
|
||||
|
||||
# Default: reuse legacy ErrorClassifier decision and map to FailoverAction.
|
||||
action = self._error_classifier.classify(error, has_retry_left=has_retry_left)
|
||||
if action == ErrorAction.RAISE:
|
||||
return FailoverAction.STOP
|
||||
if action == ErrorAction.BREAK:
|
||||
return FailoverAction.CONTINUE
|
||||
return FailoverAction.RETRY
|
||||
|
||||
async def _probe_stream_first_chunk(
|
||||
self,
|
||||
*,
|
||||
attempt_result: AttemptResult,
|
||||
record_id: str | None,
|
||||
) -> AttemptResult:
|
||||
"""
|
||||
Probe first chunk for a streaming response.
|
||||
|
||||
Strong constraints:
|
||||
- Must have timeout.
|
||||
- Empty stream before first chunk is treated as probe failure (eligible for failover).
|
||||
"""
|
||||
assert attempt_result.kind == AttemptKind.STREAM
|
||||
assert attempt_result.stream_iterator is not None
|
||||
|
||||
original_iterator = attempt_result.stream_iterator
|
||||
try:
|
||||
first_chunk = await asyncio.wait_for(
|
||||
original_iterator.__anext__(),
|
||||
timeout=self.STREAM_FIRST_CHUNK_TIMEOUT_SECONDS,
|
||||
)
|
||||
except asyncio.TimeoutError as exc:
|
||||
raise StreamProbeError(
|
||||
"Timeout waiting for first chunk",
|
||||
http_status=attempt_result.http_status,
|
||||
original_exception=exc,
|
||||
) from exc
|
||||
except StopAsyncIteration as exc:
|
||||
raise StreamProbeError(
|
||||
"Empty stream: no data received before EOF",
|
||||
http_status=attempt_result.http_status,
|
||||
original_exception=exc,
|
||||
) from exc
|
||||
except Exception as exc:
|
||||
raise StreamProbeError(
|
||||
f"Failed to read first chunk: {exc}",
|
||||
http_status=attempt_result.http_status,
|
||||
original_exception=exc,
|
||||
) from exc
|
||||
|
||||
wrapped = self._wrap_stream_with_finalizer(
|
||||
first_chunk=first_chunk,
|
||||
original_iterator=original_iterator,
|
||||
record_id=record_id,
|
||||
)
|
||||
return AttemptResult(
|
||||
kind=AttemptKind.STREAM,
|
||||
http_status=attempt_result.http_status,
|
||||
http_headers=attempt_result.http_headers,
|
||||
stream_iterator=wrapped,
|
||||
raw_response=attempt_result.raw_response,
|
||||
)
|
||||
|
||||
def _wrap_stream_with_finalizer(
|
||||
self,
|
||||
*,
|
||||
first_chunk: bytes,
|
||||
original_iterator: AsyncIterator[bytes],
|
||||
record_id: str | None,
|
||||
) -> AsyncIterator[bytes]:
|
||||
async def _gen() -> AsyncIterator[bytes]:
|
||||
yield first_chunk
|
||||
try:
|
||||
async for chunk in original_iterator:
|
||||
yield chunk
|
||||
except Exception as exc:
|
||||
# Best-effort: mark stream interrupted using a new session (stream may outlive request session).
|
||||
if record_id:
|
||||
self._mark_record_stream_interrupted(record_id, exc)
|
||||
raise
|
||||
|
||||
return _gen()
|
||||
|
||||
def _mark_record_stream_interrupted(self, record_id: str, exc: Exception) -> None:
|
||||
try:
|
||||
from src.database import create_session
|
||||
|
||||
with create_session() as db:
|
||||
db.execute(
|
||||
update(RequestCandidate)
|
||||
.where(RequestCandidate.id == record_id)
|
||||
.values(
|
||||
status="stream_interrupted",
|
||||
error_type=type(exc).__name__,
|
||||
error_message=self._sanitize(str(exc)),
|
||||
finished_at=datetime.now(timezone.utc),
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
except Exception as inner:
|
||||
logger.debug(
|
||||
"[FailoverEngine] Failed to mark stream_interrupted: {}",
|
||||
self._sanitize(str(inner)),
|
||||
)
|
||||
|
||||
def _mark_candidate_skipped(
|
||||
self,
|
||||
*,
|
||||
candidate_record_map: dict[tuple[int, int], str],
|
||||
candidate_index: int,
|
||||
candidate: ProviderCandidate,
|
||||
retry_policy: RetryPolicy,
|
||||
skip_reason: str | None,
|
||||
) -> None:
|
||||
max_retries = self._get_max_retries(candidate, retry_policy)
|
||||
now = datetime.now(timezone.utc)
|
||||
for retry_index in range(max_retries):
|
||||
record_id = candidate_record_map.get((candidate_index, retry_index))
|
||||
if record_id:
|
||||
self._update_record(
|
||||
record_id,
|
||||
status="skipped",
|
||||
skip_reason=skip_reason,
|
||||
finished_at=now,
|
||||
)
|
||||
self.db.commit()
|
||||
|
||||
def _mark_remaining_slots_unused(
|
||||
self,
|
||||
*,
|
||||
candidate_record_map: dict[tuple[int, int], str],
|
||||
candidates: list[ProviderCandidate],
|
||||
success_candidate_idx: int,
|
||||
success_retry_idx: int,
|
||||
retry_policy: RetryPolicy,
|
||||
) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
for candidate_idx, cand in enumerate(candidates):
|
||||
max_retries = self._get_max_retries(cand, retry_policy)
|
||||
for retry_idx in range(max_retries):
|
||||
if candidate_idx < success_candidate_idx:
|
||||
continue
|
||||
if candidate_idx == success_candidate_idx and retry_idx <= success_retry_idx:
|
||||
continue
|
||||
record_id = candidate_record_map.get((candidate_idx, retry_idx))
|
||||
if record_id:
|
||||
self._update_record(
|
||||
record_id,
|
||||
status="unused",
|
||||
finished_at=now,
|
||||
)
|
||||
self.db.commit()
|
||||
|
||||
def _mark_candidate_remaining_retries_unused(
|
||||
self,
|
||||
*,
|
||||
candidate_record_map: dict[tuple[int, int], str],
|
||||
candidate_idx: int,
|
||||
from_retry_idx: int,
|
||||
retry_policy: RetryPolicy,
|
||||
) -> None:
|
||||
# Only meaningful for PRE_EXPAND.
|
||||
# We don't have access to candidate object list here, so infer max_retries from map keys.
|
||||
# Fallback to retry_policy.max_retries.
|
||||
now = datetime.now(timezone.utc)
|
||||
# try best-effort upper bound
|
||||
upper = max(
|
||||
(ri for (ci, ri) in candidate_record_map.keys() if ci == candidate_idx),
|
||||
default=retry_policy.max_retries - 1,
|
||||
)
|
||||
for retry_idx in range(from_retry_idx, upper + 1):
|
||||
record_id = candidate_record_map.get((candidate_idx, retry_idx))
|
||||
if record_id:
|
||||
self._update_record(record_id, status="unused", finished_at=now)
|
||||
self.db.commit()
|
||||
|
||||
def _mark_all_remaining_available_unused(
|
||||
self, candidate_record_map: dict[tuple[int, int], str]
|
||||
) -> None:
|
||||
# As a safety net: do not leave available records behind in PRE_EXPAND mode.
|
||||
try:
|
||||
ids = list(candidate_record_map.values())
|
||||
if not ids:
|
||||
return
|
||||
now = datetime.now(timezone.utc)
|
||||
self.db.execute(
|
||||
update(RequestCandidate)
|
||||
.where(RequestCandidate.id.in_(ids))
|
||||
.where(RequestCandidate.status == "available")
|
||||
.values(status="unused", finished_at=now)
|
||||
)
|
||||
self.db.commit()
|
||||
except Exception:
|
||||
self.db.rollback()
|
||||
raise
|
||||
|
||||
@@ -12,6 +12,14 @@ class RetryMode(str, Enum):
|
||||
DISABLED = "disabled" # no retry
|
||||
|
||||
|
||||
class FailoverAction(str, Enum):
|
||||
"""Failover decision after an attempt error."""
|
||||
|
||||
STOP = "stop" # stop failover (client error / non-retriable)
|
||||
CONTINUE = "continue" # continue with next candidate
|
||||
RETRY = "retry" # retry current candidate
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RetryPolicy:
|
||||
"""Unified retry policy."""
|
||||
|
||||
@@ -1,24 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from src.config.settings import config
|
||||
from src.core.exceptions import ProviderNotAvailableException
|
||||
from src.core.logger import logger
|
||||
from src.models.database import ApiKey, RequestCandidate
|
||||
from src.services.billing.rule_service import BillingRuleLookupResult, BillingRuleService
|
||||
from src.models.database import ApiKey
|
||||
from src.services.cache.aware_scheduler import ProviderCandidate, get_cache_aware_scheduler
|
||||
from src.services.candidate.submit import (
|
||||
AllCandidatesFailedError,
|
||||
SubmitOutcome,
|
||||
UpstreamClientRequestError,
|
||||
)
|
||||
from src.services.orchestration.error_classifier import ErrorClassifier
|
||||
from src.services.system.config import SystemConfigService
|
||||
|
||||
@@ -26,17 +13,6 @@ from .recorder import CandidateRecorder
|
||||
from .resolver import CandidateResolver
|
||||
from .schema import CandidateKey
|
||||
|
||||
_SENSITIVE_PATTERN = re.compile(
|
||||
r"(api[_-]?key|token|bearer|authorization)[=:\s]+\S+",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _sanitize(message: str, max_length: int = 200) -> str:
|
||||
if not message:
|
||||
return "request_failed"
|
||||
return _SENSITIVE_PATTERN.sub("[REDACTED]", message)[:max_length]
|
||||
|
||||
|
||||
class CandidateService:
|
||||
"""
|
||||
@@ -101,396 +77,6 @@ class CandidateService:
|
||||
preferred_key_ids=preferred_key_ids,
|
||||
)
|
||||
|
||||
def _should_stop_on_http_error(self, *, status_code: int, error_text: str) -> bool:
|
||||
"""
|
||||
Decide whether an upstream HTTP error is a client error (no failover).
|
||||
|
||||
Rules:
|
||||
- 401/403/429 are usually key/permission/ratelimit issues -> allow failover
|
||||
- other 4xx: stop only if ErrorClassifier says it's a client error
|
||||
"""
|
||||
if status_code in (401, 403, 429):
|
||||
return False
|
||||
if 400 <= status_code < 500:
|
||||
assert self._error_classifier is not None
|
||||
return self._error_classifier.is_client_error(error_text)
|
||||
return False
|
||||
|
||||
async def submit_with_failover(
|
||||
self,
|
||||
*,
|
||||
api_format: str,
|
||||
model_name: str,
|
||||
affinity_key: str,
|
||||
user_api_key: ApiKey,
|
||||
request_id: str | None,
|
||||
task_type: str,
|
||||
submit_func: Any,
|
||||
extract_external_task_id: Any,
|
||||
supported_auth_types: set[str] | None = None,
|
||||
allow_format_conversion: bool = False,
|
||||
capability_requirements: dict[str, bool] | None = None,
|
||||
max_candidates: int | None = None,
|
||||
) -> SubmitOutcome:
|
||||
"""
|
||||
Submit async task with failover, returning the selected candidate + external_task_id.
|
||||
|
||||
Phase2 submit entrypoint (replaces legacy submit orchestrator).
|
||||
"""
|
||||
# IMPORTANT:
|
||||
# This method awaits upstream HTTP calls. If we have an open DB transaction before awaiting,
|
||||
# the connection can be held for a long time (pool exhaustion under concurrency).
|
||||
#
|
||||
# Also note SQLAlchemy's default expire_on_commit=True would expire ORM objects and may
|
||||
# trigger unexpected lazy DB loads after we commit (potentially during the await).
|
||||
# We disable it temporarily to keep candidate/provider/key objects in-memory.
|
||||
original_expire_on_commit = getattr(self.db, "expire_on_commit", True)
|
||||
self.db.expire_on_commit = False
|
||||
await self._ensure_initialized()
|
||||
assert self._resolver is not None
|
||||
try:
|
||||
candidates, _global_model_id = await self._resolver.fetch_candidates(
|
||||
api_format=api_format,
|
||||
model_name=model_name,
|
||||
affinity_key=affinity_key,
|
||||
user_api_key=user_api_key,
|
||||
request_id=request_id,
|
||||
is_stream=False,
|
||||
capability_requirements=capability_requirements,
|
||||
)
|
||||
|
||||
if not candidates:
|
||||
raise ProviderNotAvailableException("No candidates available")
|
||||
|
||||
if max_candidates is not None and max_candidates > 0:
|
||||
candidates = candidates[:max_candidates]
|
||||
|
||||
# Pre-create RequestCandidate records (no retry expand for async submit stage)
|
||||
record_map: dict[tuple[int, int], str] = {}
|
||||
if request_id:
|
||||
try:
|
||||
record_map = self.create_candidate_records(
|
||||
candidates=candidates,
|
||||
request_id=request_id,
|
||||
user_api_key=user_api_key,
|
||||
required_capabilities=capability_requirements,
|
||||
expand_retries=False,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"[CandidateService] Failed to create candidate records: %s",
|
||||
_sanitize(str(exc)),
|
||||
)
|
||||
record_map = {}
|
||||
|
||||
candidate_keys: list[dict[str, Any]] = []
|
||||
eligible_count = 0
|
||||
last_status_code: int | None = None
|
||||
|
||||
for idx, cand in enumerate(candidates):
|
||||
now = datetime.now(timezone.utc)
|
||||
auth_type = getattr(cand.key, "auth_type", "api_key") or "api_key"
|
||||
|
||||
candidate_info: dict[str, Any] = {
|
||||
"index": idx,
|
||||
"provider_id": cand.provider.id,
|
||||
"provider_name": cand.provider.name,
|
||||
"endpoint_id": cand.endpoint.id,
|
||||
"key_id": cand.key.id,
|
||||
"key_name": getattr(cand.key, "name", None),
|
||||
"auth_type": auth_type,
|
||||
"priority": getattr(cand.key, "priority", 0) or 0,
|
||||
"is_cached": bool(getattr(cand, "is_cached", False)),
|
||||
}
|
||||
candidate_keys.append(candidate_info)
|
||||
|
||||
record_id = record_map.get((idx, 0))
|
||||
|
||||
# Scheduler marked skip
|
||||
if getattr(cand, "is_skipped", False):
|
||||
skip_reason = getattr(cand, "skip_reason", None) or "skipped"
|
||||
candidate_info.update({"skipped": True, "skip_reason": skip_reason})
|
||||
if record_id:
|
||||
# record is usually already skipped, but keep it consistent
|
||||
self.db.execute(
|
||||
update(RequestCandidate)
|
||||
.where(RequestCandidate.id == record_id)
|
||||
.values(status="skipped", skip_reason=skip_reason)
|
||||
)
|
||||
continue
|
||||
|
||||
# Format conversion checks
|
||||
# 优先级：全局开关 ON 强制允许，全局开关 OFF 看提供商开关
|
||||
needs_conversion = bool(getattr(cand, "needs_conversion", False))
|
||||
if needs_conversion:
|
||||
# 1. Check handler-level switch (handler 不支持则直接跳过)
|
||||
if not allow_format_conversion:
|
||||
skip_reason = "format_conversion_not_supported"
|
||||
candidate_info.update({"skipped": True, "skip_reason": skip_reason})
|
||||
if record_id:
|
||||
self.db.execute(
|
||||
update(RequestCandidate)
|
||||
.where(RequestCandidate.id == record_id)
|
||||
.values(status="skipped", skip_reason=skip_reason)
|
||||
)
|
||||
continue
|
||||
|
||||
# 2. Check global + provider switches
|
||||
# 全局 ON → 允许；全局 OFF → 看提供商
|
||||
from src.services.system.config import SystemConfigService
|
||||
|
||||
global_enabled = SystemConfigService.is_format_conversion_enabled(self.db)
|
||||
provider_enabled = getattr(cand.provider, "enable_format_conversion", True)
|
||||
effective_enabled = global_enabled or provider_enabled
|
||||
|
||||
if not effective_enabled:
|
||||
skip_reason = "format_conversion_disabled"
|
||||
candidate_info.update(
|
||||
{
|
||||
"skipped": True,
|
||||
"skip_reason": skip_reason,
|
||||
"global_conversion_enabled": global_enabled,
|
||||
"provider_conversion_enabled": provider_enabled,
|
||||
}
|
||||
)
|
||||
if record_id:
|
||||
self.db.execute(
|
||||
update(RequestCandidate)
|
||||
.where(RequestCandidate.id == record_id)
|
||||
.values(status="skipped", skip_reason=skip_reason)
|
||||
)
|
||||
continue
|
||||
|
||||
# auth_type filter
|
||||
if supported_auth_types is not None and auth_type not in supported_auth_types:
|
||||
skip_reason = f"unsupported_auth_type:{auth_type}"
|
||||
candidate_info.update({"skipped": True, "skip_reason": skip_reason})
|
||||
if record_id:
|
||||
self.db.execute(
|
||||
update(RequestCandidate)
|
||||
.where(RequestCandidate.id == record_id)
|
||||
.values(status="skipped", skip_reason=skip_reason)
|
||||
)
|
||||
continue
|
||||
|
||||
# billing rule filter
|
||||
rule_lookup: BillingRuleLookupResult | None = None
|
||||
has_billing_rule = True
|
||||
if config.billing_require_rule:
|
||||
rule_lookup = BillingRuleService.find_rule(
|
||||
self.db,
|
||||
provider_id=cand.provider.id,
|
||||
model_name=model_name,
|
||||
task_type=task_type,
|
||||
)
|
||||
has_billing_rule = rule_lookup is not None
|
||||
if not has_billing_rule:
|
||||
skip_reason = "billing_rule_missing"
|
||||
candidate_info.update(
|
||||
{"has_billing_rule": False, "skipped": True, "skip_reason": skip_reason}
|
||||
)
|
||||
if record_id:
|
||||
self.db.execute(
|
||||
update(RequestCandidate)
|
||||
.where(RequestCandidate.id == record_id)
|
||||
.values(status="skipped", skip_reason=skip_reason)
|
||||
)
|
||||
continue
|
||||
candidate_info["has_billing_rule"] = has_billing_rule
|
||||
|
||||
eligible_count += 1
|
||||
|
||||
# Mark pending
|
||||
if record_id:
|
||||
self.db.execute(
|
||||
update(RequestCandidate)
|
||||
.where(RequestCandidate.id == record_id)
|
||||
.values(status="pending", started_at=now)
|
||||
)
|
||||
|
||||
# Flush/commit BEFORE awaiting upstream submit to avoid holding DB connections
|
||||
# during potentially slow network operations.
|
||||
if self.db.in_transaction():
|
||||
try:
|
||||
self.db.commit()
|
||||
except Exception:
|
||||
self.db.rollback()
|
||||
raise
|
||||
|
||||
# Attempt submit (upstream HTTP)
|
||||
try:
|
||||
response: httpx.Response = await submit_func(cand)
|
||||
except Exception as exc:
|
||||
finished_at = datetime.now(timezone.utc)
|
||||
error_msg = _sanitize(str(exc))
|
||||
candidate_info.update(
|
||||
{
|
||||
"attempt_status": "exception",
|
||||
"error_type": type(exc).__name__,
|
||||
"error_message": error_msg,
|
||||
}
|
||||
)
|
||||
if record_id:
|
||||
self.db.execute(
|
||||
update(RequestCandidate)
|
||||
.where(RequestCandidate.id == record_id)
|
||||
.values(
|
||||
status="failed",
|
||||
error_type=type(exc).__name__,
|
||||
error_message=error_msg,
|
||||
finished_at=finished_at,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
last_status_code = int(getattr(response, "status_code", 0) or 0)
|
||||
|
||||
if response.status_code >= 400:
|
||||
finished_at = datetime.now(timezone.utc)
|
||||
try:
|
||||
error_text = response.text or ""
|
||||
except Exception:
|
||||
error_text = ""
|
||||
error_msg = _sanitize(error_text)
|
||||
candidate_info.update(
|
||||
{
|
||||
"attempt_status": "http_error",
|
||||
"status_code": response.status_code,
|
||||
"error_message": error_msg,
|
||||
}
|
||||
)
|
||||
if record_id:
|
||||
self.db.execute(
|
||||
update(RequestCandidate)
|
||||
.where(RequestCandidate.id == record_id)
|
||||
.values(
|
||||
status="failed",
|
||||
status_code=response.status_code,
|
||||
error_type="http_error",
|
||||
error_message=error_msg,
|
||||
finished_at=finished_at,
|
||||
)
|
||||
)
|
||||
|
||||
if self._should_stop_on_http_error(
|
||||
status_code=response.status_code, error_text=error_text
|
||||
):
|
||||
try:
|
||||
self.db.commit()
|
||||
except Exception:
|
||||
self.db.rollback()
|
||||
raise UpstreamClientRequestError(
|
||||
response=response,
|
||||
candidate_keys=candidate_keys,
|
||||
)
|
||||
continue
|
||||
|
||||
# Parse JSON
|
||||
payload: dict[str, Any] | None = None
|
||||
try:
|
||||
data = response.json()
|
||||
if isinstance(data, dict):
|
||||
payload = data
|
||||
except Exception as exc:
|
||||
finished_at = datetime.now(timezone.utc)
|
||||
error_msg = _sanitize(str(exc))
|
||||
candidate_info.update(
|
||||
{
|
||||
"attempt_status": "invalid_json",
|
||||
"error_type": type(exc).__name__,
|
||||
"error_message": error_msg,
|
||||
}
|
||||
)
|
||||
if record_id:
|
||||
self.db.execute(
|
||||
update(RequestCandidate)
|
||||
.where(RequestCandidate.id == record_id)
|
||||
.values(
|
||||
status="failed",
|
||||
status_code=response.status_code,
|
||||
error_type="invalid_json",
|
||||
error_message=error_msg,
|
||||
finished_at=finished_at,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
external_task_id = extract_external_task_id(payload or {})
|
||||
if not external_task_id:
|
||||
finished_at = datetime.now(timezone.utc)
|
||||
candidate_info.update(
|
||||
{
|
||||
"attempt_status": "empty_task_id",
|
||||
"error_message": "Upstream returned empty task id",
|
||||
}
|
||||
)
|
||||
if record_id:
|
||||
self.db.execute(
|
||||
update(RequestCandidate)
|
||||
.where(RequestCandidate.id == record_id)
|
||||
.values(
|
||||
status="failed",
|
||||
status_code=response.status_code,
|
||||
error_type="empty_task_id",
|
||||
error_message="Upstream returned empty task id",
|
||||
finished_at=finished_at,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
# Success
|
||||
finished_at = datetime.now(timezone.utc)
|
||||
candidate_info.update({"attempt_status": "success", "selected": True})
|
||||
if record_id:
|
||||
self.db.execute(
|
||||
update(RequestCandidate)
|
||||
.where(RequestCandidate.id == record_id)
|
||||
.values(
|
||||
status="success",
|
||||
status_code=response.status_code,
|
||||
finished_at=finished_at,
|
||||
)
|
||||
)
|
||||
try:
|
||||
self.db.commit()
|
||||
except Exception:
|
||||
self.db.rollback()
|
||||
|
||||
return SubmitOutcome(
|
||||
candidate=cand,
|
||||
candidate_keys=candidate_keys,
|
||||
external_task_id=str(external_task_id),
|
||||
rule_lookup=rule_lookup,
|
||||
upstream_payload=payload,
|
||||
upstream_headers=dict(response.headers),
|
||||
upstream_status_code=response.status_code,
|
||||
)
|
||||
|
||||
# Persist candidate records before raising
|
||||
try:
|
||||
self.db.commit()
|
||||
except Exception:
|
||||
self.db.rollback()
|
||||
|
||||
if eligible_count == 0:
|
||||
reason = "no_eligible_candidates"
|
||||
if config.billing_require_rule:
|
||||
reason = "no_candidate_with_billing_rule"
|
||||
raise AllCandidatesFailedError(
|
||||
reason=reason,
|
||||
candidate_keys=candidate_keys,
|
||||
last_status_code=last_status_code,
|
||||
)
|
||||
|
||||
raise AllCandidatesFailedError(
|
||||
reason="all_candidates_failed",
|
||||
candidate_keys=candidate_keys,
|
||||
last_status_code=last_status_code,
|
||||
)
|
||||
finally:
|
||||
# Restore Session behavior for the rest of the request lifecycle.
|
||||
self.db.expire_on_commit = original_expire_on_commit
|
||||
|
||||
def create_candidate_records(
|
||||
self,
|
||||
*,
|
||||
|
||||
Reference in New Issue
Block a user