i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 01:10:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 移除 Python 后端源码，全面迁移至 Rust gateway 架构

Browse Source 
- 删除全部 Python 源码 (src/) 及 Alembic 迁移脚本，归档至 _deprecated_py_src/
- 重构 Rust gateway ai_pipeline: 拆分 planner/finalize 模块，新增 contracts/adaptation 层
- 重组 handlers 模块为 admin/public/proxy/internal/shared 子模块结构
- 新增 executor 模块，引入 Rust 原生数据库迁移 (aether-data/migrations)
- 简化 CI/Docker 构建流程，移除 base image 二级构建，统一为单一 app image
- 移除 Python 相关基础设施文件 (entrypoint.sh, gunicorn_conf.py, Dockerfile.base)
This commit is contained in:
fawney19
2026-04-03 16:26:16 +08:00
parent 8f26e1a31f
commit 1d9c77522a
868 changed files with 1735 additions and 2433 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
_deprecated_py_src/services/task/submit/__init__.py Normal file
View File

@@ -0,0 +1,12 @@
from __future__ import annotations
from collections.abc import Awaitable, Callable
from typing import Any
ApplyPoolReorderFn = Callable[
[list[Any], dict[str, Any] | None],
Awaitable[tuple[list[Any], list[Any]]],
]
ExpandPoolCandidatesFn = Callable[[list[Any]], list[Any]]
__all__ = ["ApplyPoolReorderFn", "ExpandPoolCandidatesFn"]

75
_deprecated_py_src/services/task/submit/attempt.py Normal file
View File

@@ -0,0 +1,75 @@
from __future__ import annotations
from collections.abc import Callable
from typing import Any
import httpx
from sqlalchemy.orm import Session
from src.services.billing.rule_service import BillingRuleLookupResult
from src.services.candidate.submit import SubmitOutcome
from src.services.task.submit.record import AsyncSubmitRecordService
from src.services.task.submit.response import AsyncSubmitResponseService
class AsyncSubmitAttemptService:
"""异步提交单候选执行编排服务。"""
def __init__(
self,
db: Session,
*,
sanitize: Callable[[str], str],
extract_response_text: Callable[[httpx.Response], str],
match_provider_failover_rule: Callable[..., str | None],
) -> None:
self.db = db
self._record_ops = AsyncSubmitRecordService(db)
self._response_ops = AsyncSubmitResponseService(
db,
record_ops=self._record_ops,
sanitize=sanitize,
extract_response_text=extract_response_text,
match_provider_failover_rule=match_provider_failover_rule,
)
async def submit_candidate(
self,
*,
candidate: Any,
record_id: str | None,
candidate_info: dict[str, Any],
candidate_keys: list[dict[str, Any]],
rule_lookup: BillingRuleLookupResult | None,
submit_func: Any,
extract_external_task_id: Any,
) -> tuple[SubmitOutcome | None, int | None]:
self._record_ops.mark_pending(record_id=record_id)
# Flush/commit BEFORE awaiting upstream submit to avoid holding DB connections.
if self.db.in_transaction():
try:
self.db.commit()
except Exception:
self.db.rollback()
raise
# Attempt submit (upstream HTTP)
try:
response: httpx.Response = await submit_func(candidate)
except Exception as exc:
return self._response_ops.handle_submit_exception(
record_id=record_id,
candidate_info=candidate_info,
exc=exc,
)
return self._response_ops.handle_submit_response(
candidate=candidate,
record_id=record_id,
candidate_info=candidate_info,
candidate_keys=candidate_keys,
rule_lookup=rule_lookup,
response=response,
extract_external_task_id=extract_external_task_id,
)

104
_deprecated_py_src/services/task/submit/execute.py Normal file
View File

@@ -0,0 +1,104 @@
from __future__ import annotations
from collections.abc import Callable
from typing import Any
import httpx
from sqlalchemy.orm import Session
from src.config.settings import config
from src.services.candidate.submit import AllCandidatesFailedError, SubmitOutcome
from src.services.task.submit.attempt import AsyncSubmitAttemptService
from src.services.task.submit.filter import AsyncSubmitFilterService
class AsyncSubmitExecutionService:
"""异步提交候选执行编排服务。"""
def __init__(
self,
db: Session,
*,
sanitize: Callable[[str], str],
extract_response_text: Callable[[httpx.Response], str],
match_provider_failover_rule: Callable[..., str | None],
) -> None:
self.db = db
self._filter_ops = AsyncSubmitFilterService(db)
self._attempt_ops = AsyncSubmitAttemptService(
db,
sanitize=sanitize,
extract_response_text=extract_response_text,
match_provider_failover_rule=match_provider_failover_rule,
)
async def execute_submit_loop(
self,
*,
candidates: list[Any],
record_map: dict[tuple[int, int], str],
task_type: str,
model_name: str,
submit_func: Any,
extract_external_task_id: Any,
supported_auth_types: set[str] | None,
allow_format_conversion: bool,
) -> SubmitOutcome:
candidate_keys: list[dict[str, Any]] = []
eligible_count = 0
last_status_code: int | None = None
for idx, cand in enumerate(candidates):
candidate_info = self._filter_ops.build_candidate_info(idx=idx, candidate=cand)
candidate_keys.append(candidate_info)
attempt_plan = self._filter_ops.prepare_candidate_for_attempt(
idx=idx,
candidate=cand,
record_map=record_map,
candidate_info=candidate_info,
task_type=task_type,
model_name=model_name,
supported_auth_types=supported_auth_types,
allow_format_conversion=allow_format_conversion,
)
if attempt_plan is None:
continue
eligible_count += 1
outcome, status_code = await self._attempt_ops.submit_candidate(
candidate=cand,
record_id=attempt_plan.record_id,
candidate_info=candidate_info,
candidate_keys=candidate_keys,
rule_lookup=attempt_plan.rule_lookup,
submit_func=submit_func,
extract_external_task_id=extract_external_task_id,
)
if status_code is not None:
last_status_code = status_code
if outcome is not None:
return outcome
# Persist candidate records before raising.
try:
self.db.commit()
except Exception:
self.db.rollback()
if eligible_count == 0:
reason = "no_eligible_candidates"
if config.billing_require_rule:
reason = "no_candidate_with_billing_rule"
raise AllCandidatesFailedError(
reason=reason,
candidate_keys=candidate_keys,
last_status_code=last_status_code,
)
raise AllCandidatesFailedError(
reason="all_candidates_failed",
candidate_keys=candidate_keys,
last_status_code=last_status_code,
)

137
_deprecated_py_src/services/task/submit/filter.py Normal file
View File

@@ -0,0 +1,137 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
from sqlalchemy import update
from sqlalchemy.orm import Session
from src.config.settings import config
from src.models.database import RequestCandidate
from src.services.billing.rule_service import BillingRuleLookupResult, BillingRuleService
from src.services.system.config import SystemConfigService
@dataclass(slots=True)
class CandidateAttemptPlan:
"""候选尝试计划（通过过滤后可进入提交阶段）。"""
record_id: str | None
rule_lookup: BillingRuleLookupResult | None
class AsyncSubmitFilterService:
"""异步提交候选过滤服务。"""
def __init__(self, db: Session) -> None:
self.db = db
@staticmethod
def build_candidate_info(*, idx: int, candidate: Any) -> dict[str, Any]:
auth_type = getattr(candidate.key, "auth_type", "api_key") or "api_key"
return {
"index": idx,
"provider_id": candidate.provider.id,
"provider_name": candidate.provider.name,
"endpoint_id": candidate.endpoint.id,
"key_id": candidate.key.id,
"key_name": getattr(candidate.key, "name", None),
"auth_type": auth_type,
"priority": getattr(candidate.key, "priority", 0) or 0,
"is_cached": bool(getattr(candidate, "is_cached", False)),
}
def prepare_candidate_for_attempt(
self,
*,
idx: int,
candidate: Any,
record_map: dict[tuple[int, int], str],
candidate_info: dict[str, Any],
task_type: str,
model_name: str,
supported_auth_types: set[str] | None,
allow_format_conversion: bool,
) -> CandidateAttemptPlan | None:
record_id = record_map.get((idx, 0))
auth_type = candidate_info.get("auth_type", "api_key")
# Scheduler marked skip
if getattr(candidate, "is_skipped", False):
skip_reason = getattr(candidate, "skip_reason", None) or "skipped"
self._mark_skip(
record_id=record_id, candidate_info=candidate_info, skip_reason=skip_reason
)
return None
# Format conversion checks
needs_conversion = bool(getattr(candidate, "needs_conversion", False))
if needs_conversion:
# 1. handler-level switch
if not allow_format_conversion:
self._mark_skip(
record_id=record_id,
candidate_info=candidate_info,
skip_reason="format_conversion_not_supported",
)
return None
# 2. global switch (from database config)
if not SystemConfigService.is_format_conversion_enabled(self.db):
self._mark_skip(
record_id=record_id,
candidate_info=candidate_info,
skip_reason="format_conversion_disabled",
extra_info={"format_conversion_enabled": False},
)
return None
# auth_type filter
if supported_auth_types is not None and auth_type not in supported_auth_types:
self._mark_skip(
record_id=record_id,
candidate_info=candidate_info,
skip_reason=f"unsupported_auth_type:{auth_type}",
)
return None
# billing rule filter
rule_lookup: BillingRuleLookupResult | None = None
has_billing_rule = True
if config.billing_require_rule:
rule_lookup = BillingRuleService.find_rule(
self.db,
provider_id=candidate.provider.id,
model_name=model_name,
task_type=task_type,
)
has_billing_rule = rule_lookup is not None
if not has_billing_rule:
self._mark_skip(
record_id=record_id,
candidate_info=candidate_info,
skip_reason="billing_rule_missing",
extra_info={"has_billing_rule": False},
)
return None
candidate_info["has_billing_rule"] = has_billing_rule
return CandidateAttemptPlan(record_id=record_id, rule_lookup=rule_lookup)
def _mark_skip(
self,
*,
record_id: str | None,
candidate_info: dict[str, Any],
skip_reason: str,
extra_info: dict[str, Any] | None = None,
) -> None:
candidate_info.update({"skipped": True, "skip_reason": skip_reason})
if extra_info:
candidate_info.update(extra_info)
if record_id:
self.db.execute(
update(RequestCandidate)
.where(RequestCandidate.id == record_id)
.values(status="skipped", skip_reason=skip_reason)
)

60
_deprecated_py_src/services/task/submit/outcome_builder.py Normal file
View File

@@ -0,0 +1,60 @@
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
import httpx
from src.services.billing.rule_service import BillingRuleLookupResult
from src.services.candidate.submit import SubmitOutcome
@dataclass(slots=True)
class SubmitPayloadParseResult:
"""提交响应解析结果。"""
payload: dict[str, Any] | None
error_type: str | None = None
error_message: str | None = None
class AsyncSubmitOutcomeBuilderService:
"""异步提交结果构建服务。"""
def __init__(self, *, sanitize: Callable[[str], str]) -> None:
self._sanitize = sanitize
def parse_payload(self, *, response: httpx.Response) -> SubmitPayloadParseResult:
payload: dict[str, Any] | None = None
try:
data = response.json()
if isinstance(data, dict):
payload = data
except Exception as exc:
return SubmitPayloadParseResult(
payload=None,
error_type=type(exc).__name__,
error_message=self._sanitize(str(exc)),
)
return SubmitPayloadParseResult(payload=payload)
@staticmethod
def build_success_outcome(
*,
candidate: Any,
candidate_keys: list[dict[str, Any]],
external_task_id: str,
rule_lookup: BillingRuleLookupResult | None,
payload: dict[str, Any] | None,
response: httpx.Response,
) -> SubmitOutcome:
return SubmitOutcome(
candidate=candidate,
candidate_keys=candidate_keys,
external_task_id=external_task_id,
rule_lookup=rule_lookup,
upstream_payload=payload,
upstream_headers=dict(response.headers),
upstream_status_code=response.status_code,
)

126
_deprecated_py_src/services/task/submit/prepare.py Normal file
View File

@@ -0,0 +1,126 @@
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
from sqlalchemy.orm import Session
from src.core.logger import logger
from src.models.database import ApiKey
from src.services.candidate.resolver import CandidateResolver
from src.services.candidate.submit import AllCandidatesFailedError
from src.services.scheduling.aware_scheduler import get_cache_aware_scheduler
from src.services.system.config import SystemConfigService
from src.services.task.submit import ApplyPoolReorderFn, ExpandPoolCandidatesFn
@dataclass(slots=True)
class PreparedSubmitCandidates:
"""异步提交前的候选准备结果。"""
candidates: list[Any]
record_map: dict[tuple[int, int], str]
class AsyncSubmitPreparationService:
"""异步提交候选准备服务。"""
def __init__(
self,
db: Session,
redis_client: Any | None,
*,
sanitize: Callable[[str], str],
) -> None:
self.db = db
self.redis = redis_client
self._sanitize = sanitize
async def prepare_candidates(
self,
*,
api_format: str,
model_name: str,
affinity_key: str,
user_api_key: ApiKey,
request_id: str | None,
capability_requirements: dict[str, bool] | None,
request_body: dict[str, Any] | None,
max_candidates: int | None,
apply_pool_reorder: ApplyPoolReorderFn,
expand_pool_candidates_for_async_submit: ExpandPoolCandidatesFn,
) -> PreparedSubmitCandidates:
priority_mode = SystemConfigService.get_config(
self.db,
"provider_priority_mode",
"provider",
)
scheduling_mode = SystemConfigService.get_config(
self.db,
"scheduling_mode",
"cache_affinity",
)
cache_scheduler = await get_cache_aware_scheduler(
self.redis,
priority_mode=priority_mode,
scheduling_mode=scheduling_mode,
)
resolver = CandidateResolver(db=self.db, cache_scheduler=cache_scheduler)
candidates, _global_model_id = await resolver.fetch_candidates(
api_format=api_format,
model_name=model_name,
affinity_key=affinity_key,
user_api_key=user_api_key,
request_id=request_id,
is_stream=False,
capability_requirements=capability_requirements,
request_body=request_body,
)
_ = _global_model_id
if not candidates:
raise AllCandidatesFailedError(
reason="no_candidates",
candidate_keys=[],
last_status_code=None,
)
# 号池排序涉及大量 Redis 操作，提前释放 DB 连接避免连接池压力
from src.services.scheduling.utils import release_db_connection_before_await
release_db_connection_before_await(self.db)
# Account Pool: keep internal key failover order/skip behavior
# consistent with the SYNC path.
candidates, _pool_traces = await apply_pool_reorder(
candidates,
request_body=request_body,
)
_ = _pool_traces
candidates = expand_pool_candidates_for_async_submit(candidates)
if max_candidates is not None and max_candidates > 0:
candidates = candidates[:max_candidates]
# Pre-create RequestCandidate records (no retry expand for async submit stage)
record_map: dict[tuple[int, int], str] = {}
if request_id:
try:
record_map = await resolver.create_candidate_records_async(
all_candidates=candidates,
request_id=request_id,
user_id=str(user_api_key.user_id),
user_api_key=user_api_key,
required_capabilities=capability_requirements,
expand_retries=False,
)
except Exception as exc:
logger.warning(
"[TaskService] Failed to create candidate records: {}",
self._sanitize(str(exc)),
)
record_map = {}
return PreparedSubmitCandidates(candidates=candidates, record_map=record_map)

67
_deprecated_py_src/services/task/submit/record.py Normal file
View File

@@ -0,0 +1,67 @@
from __future__ import annotations
from datetime import datetime, timezone
from sqlalchemy import update
from sqlalchemy.orm import Session
from src.models.database import RequestCandidate
class AsyncSubmitRecordService:
"""异步提交阶段的 RequestCandidate 落库服务。"""
def __init__(self, db: Session) -> None:
self.db = db
def mark_pending(self, *, record_id: str | None) -> None:
if not record_id:
return
started_at = datetime.now(timezone.utc)
self.db.execute(
update(RequestCandidate)
.where(RequestCandidate.id == record_id)
.values(status="pending", started_at=started_at)
)
def mark_failed(
self,
*,
record_id: str | None,
error_type: str,
error_message: str,
status_code: int | None = None,
) -> None:
if not record_id:
return
values: dict[str, object] = {
"status": "failed",
"error_type": error_type,
"error_message": error_message,
"finished_at": datetime.now(timezone.utc),
}
if status_code is not None:
values["status_code"] = status_code
self.db.execute(
update(RequestCandidate).where(RequestCandidate.id == record_id).values(**values)
)
def mark_success(
self,
*,
record_id: str | None,
status_code: int,
) -> None:
if not record_id:
return
self.db.execute(
update(RequestCandidate)
.where(RequestCandidate.id == record_id)
.values(
status="success",
status_code=status_code,
finished_at=datetime.now(timezone.utc),
)
)

201
_deprecated_py_src/services/task/submit/response.py Normal file
View File

@@ -0,0 +1,201 @@
from __future__ import annotations
from collections.abc import Callable
from typing import Any
import httpx
from sqlalchemy.orm import Session
from src.core.logger import logger
from src.services.billing.rule_service import BillingRuleLookupResult
from src.services.candidate.submit import SubmitOutcome, UpstreamClientRequestError
from src.services.task.submit.outcome_builder import (
AsyncSubmitOutcomeBuilderService,
)
from src.services.task.submit.record import AsyncSubmitRecordService
from src.services.task.submit.rule_decider import AsyncSubmitRuleDeciderService
class AsyncSubmitResponseService:
"""异步提交响应判定服务。"""
def __init__(
self,
db: Session,
*,
record_ops: AsyncSubmitRecordService,
sanitize: Callable[[str], str],
extract_response_text: Callable[[httpx.Response], str],
match_provider_failover_rule: Callable[..., str | None],
) -> None:
self.db = db
self._record_ops = record_ops
self._sanitize = sanitize
self._extract_response_text = extract_response_text
self._rule_decider = AsyncSubmitRuleDeciderService(
match_provider_failover_rule=match_provider_failover_rule
)
self._outcome_builder = AsyncSubmitOutcomeBuilderService(sanitize=sanitize)
def handle_submit_exception(
self,
*,
record_id: str | None,
candidate_info: dict[str, Any],
exc: Exception,
) -> tuple[None, None]:
error_type = type(exc).__name__
error_msg = self._sanitize(str(exc))
candidate_info.update(
{
"attempt_status": "exception",
"error_type": error_type,
"error_message": error_msg,
}
)
self._record_ops.mark_failed(
record_id=record_id,
error_type=error_type,
error_message=error_msg,
)
return None, None
def handle_submit_response(
self,
*,
candidate: Any,
record_id: str | None,
candidate_info: dict[str, Any],
candidate_keys: list[dict[str, Any]],
rule_lookup: BillingRuleLookupResult | None,
response: httpx.Response,
extract_external_task_id: Any,
) -> tuple[SubmitOutcome | None, int | None]:
last_status_code = int(getattr(response, "status_code", 0) or 0)
if response.status_code >= 400:
error_text = self._extract_response_text(response)
error_msg = self._sanitize(error_text)
candidate_info.update(
{
"attempt_status": "http_error",
"status_code": response.status_code,
"error_message": error_msg,
}
)
self._record_ops.mark_failed(
record_id=record_id,
status_code=response.status_code,
error_type="http_error",
error_message=error_msg,
)
stop_pattern = self._rule_decider.detect_error_stop_pattern(
candidate=candidate,
response_text=error_text,
status_code=response.status_code,
)
if stop_pattern:
logger.info(
"[TaskService] 错误终止规则命中: pattern={}, status_code={}, provider={}",
stop_pattern,
response.status_code,
candidate.provider.name,
)
candidate_info["stop_rule_pattern"] = stop_pattern
try:
self.db.commit()
except Exception:
self.db.rollback()
raise UpstreamClientRequestError(
response=response,
candidate_keys=candidate_keys,
)
return None, last_status_code
success_text = self._extract_response_text(response)
success_continue_pattern = self._rule_decider.detect_success_failover_pattern(
candidate=candidate,
response_text=success_text,
status_code=response.status_code,
)
if success_continue_pattern:
logger.info(
"[TaskService] 成功转移规则命中: pattern={}, status_code={}, provider={}",
success_continue_pattern,
response.status_code,
candidate.provider.name,
)
failover_reason = f"success_failover_rule_matched:{success_continue_pattern}"
candidate_info.update(
{
"attempt_status": "success_failover",
"status_code": response.status_code,
"error_message": failover_reason,
"success_rule_pattern": success_continue_pattern,
}
)
self._record_ops.mark_failed(
record_id=record_id,
status_code=response.status_code,
error_type="success_failover_pattern",
error_message=failover_reason,
)
return None, last_status_code
parse_result = self._outcome_builder.parse_payload(response=response)
if parse_result.error_type:
candidate_info.update(
{
"attempt_status": "invalid_json",
"error_type": parse_result.error_type,
"error_message": parse_result.error_message,
}
)
self._record_ops.mark_failed(
record_id=record_id,
status_code=response.status_code,
error_type="invalid_json",
error_message=parse_result.error_message or "invalid_json",
)
return None, last_status_code
payload = parse_result.payload
external_task_id = extract_external_task_id(payload or {})
if not external_task_id:
candidate_info.update(
{
"attempt_status": "empty_task_id",
"error_message": "Upstream returned empty task id",
}
)
self._record_ops.mark_failed(
record_id=record_id,
status_code=response.status_code,
error_type="empty_task_id",
error_message="Upstream returned empty task id",
)
return None, last_status_code
# Success
candidate_info.update({"attempt_status": "success", "selected": True})
self._record_ops.mark_success(
record_id=record_id,
status_code=response.status_code,
)
try:
self.db.commit()
except Exception:
self.db.rollback()
return (
self._outcome_builder.build_success_outcome(
candidate=candidate,
candidate_keys=candidate_keys,
external_task_id=str(external_task_id),
rule_lookup=rule_lookup,
payload=payload,
response=response,
),
last_status_code,
)

43
_deprecated_py_src/services/task/submit/rule_decider.py Normal file
View File

@@ -0,0 +1,43 @@
from __future__ import annotations
from collections.abc import Callable
from typing import Any
class AsyncSubmitRuleDeciderService:
"""异步提交故障转移规则判定服务。"""
def __init__(
self,
*,
match_provider_failover_rule: Callable[..., str | None],
) -> None:
self._match_provider_failover_rule = match_provider_failover_rule
def detect_error_stop_pattern(
self,
*,
candidate: Any,
response_text: str,
status_code: int,
) -> str | None:
return self._match_provider_failover_rule(
candidate,
is_success=False,
response_text=response_text,
status_code=status_code,
)
def detect_success_failover_pattern(
self,
*,
candidate: Any,
response_text: str,
status_code: int,
) -> str | None:
return self._match_provider_failover_rule(
candidate,
is_success=True,
response_text=response_text,
status_code=status_code,
)

152
_deprecated_py_src/services/task/submit/submit_service.py Normal file
View File

@@ -0,0 +1,152 @@
from __future__ import annotations
import re
from typing import Any
import httpx
from sqlalchemy.orm import Session
from src.models.database import ApiKey
from src.services.candidate.failover import FailoverEngine
from src.services.candidate.submit import SubmitOutcome
from src.services.task.submit import ApplyPoolReorderFn, ExpandPoolCandidatesFn
from src.services.task.submit.execute import AsyncSubmitExecutionService
from src.services.task.submit.prepare import AsyncSubmitPreparationService
_SENSITIVE_PATTERN = re.compile(
r"(api[_-]?key|token|bearer|authorization)[=:\s]+\S+",
re.IGNORECASE,
)
class AsyncTaskSubmitService:
"""异步任务提交应用服务（候选选择 + 故障转移）。"""
def __init__(
self,
db: Session,
redis_client: Any | None,
*,
apply_pool_reorder: ApplyPoolReorderFn,
expand_pool_candidates_for_async_submit: ExpandPoolCandidatesFn,
) -> None:
self.db = db
self.redis = redis_client
self._apply_pool_reorder = apply_pool_reorder
self._expand_pool_candidates_for_async_submit = expand_pool_candidates_for_async_submit
self._prepare_ops = AsyncSubmitPreparationService(
db,
redis_client,
sanitize=self._sanitize,
)
self._execute_ops = AsyncSubmitExecutionService(
db,
sanitize=self._sanitize,
extract_response_text=self._extract_response_text,
match_provider_failover_rule=self._match_provider_failover_rule,
)
@staticmethod
def _sanitize(message: str, max_length: int = 200) -> str:
if not message:
return "request_failed"
return _SENSITIVE_PATTERN.sub("[REDACTED]", message)[:max_length]
@staticmethod
def _extract_response_text(response: httpx.Response) -> str:
try:
return response.text or ""
except Exception:
return ""
@staticmethod
def _match_provider_failover_rule(
candidate: Any,
*,
is_success: bool,
response_text: str,
status_code: int | None = None,
) -> str | None:
provider_config = getattr(candidate.provider, "config", None) or {}
rules = provider_config.get("failover_rules")
if not rules or not isinstance(rules, dict):
return None
compiled = FailoverEngine._get_compiled_patterns(rules)
key = "success" if is_success else "error"
for regex, rule in compiled.get(key, []):
if not is_success:
rule_status_codes = rule.get("status_codes")
if rule_status_codes and status_code not in rule_status_codes:
continue
if regex.search(response_text):
return rule.get("pattern", "")
return None
async def submit_with_failover(
self,
*,
api_format: str,
model_name: str,
affinity_key: str,
user_api_key: ApiKey,
request_id: str | None,
task_type: str,
submit_func: Any,
extract_external_task_id: Any,
supported_auth_types: set[str] | None = None,
allow_format_conversion: bool = False,
capability_requirements: dict[str, bool] | None = None,
max_candidates: int | None = None,
request_body: dict[str, Any] | None = None,
) -> SubmitOutcome:
"""
异步提交入口。
行为保持与原 TaskService.submit_with_failover 一致：
- 按候选顺序依次尝试（提交阶段不做单候选重试）
- 记录 RequestCandidate 审计行
- 命中 error_stop_patterns 时立即停止并抛出上游错误
- 命中 success_failover_patterns 时继续尝试下一个候选
"""
# IMPORTANT:
# This method awaits upstream HTTP calls. If we have an open DB transaction before awaiting,
# the connection can be held for a long time (pool exhaustion under concurrency).
#
# Also note SQLAlchemy's default expire_on_commit=True would expire ORM objects and may
# trigger unexpected lazy DB loads after we commit (potentially during the await).
# We disable it temporarily to keep candidate/provider/key objects in-memory.
original_expire_on_commit = getattr(self.db, "expire_on_commit", True)
self.db.expire_on_commit = False
try:
prepared = await self._prepare_ops.prepare_candidates(
api_format=api_format,
model_name=model_name,
affinity_key=affinity_key,
user_api_key=user_api_key,
request_id=request_id,
capability_requirements=capability_requirements,
request_body=request_body,
max_candidates=max_candidates,
apply_pool_reorder=self._apply_pool_reorder,
expand_pool_candidates_for_async_submit=(
self._expand_pool_candidates_for_async_submit
),
)
return await self._execute_ops.execute_submit_loop(
candidates=prepared.candidates,
record_map=prepared.record_map,
task_type=task_type,
model_name=model_name,
submit_func=submit_func,
extract_external_task_id=extract_external_task_id,
supported_auth_types=supported_auth_types,
allow_format_conversion=allow_format_conversion,
)
finally:
# Restore Session behavior for the rest of the request lifecycle.
self.db.expire_on_commit = original_expire_on_commit