i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-03 01:40:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 统一任务框架 Phase 3 - 用 TaskService/FailoverEngine 替代 FallbackOrchestrator

Browse Source 
核心重构：
- 移除 FallbackOrchestrator，用 TaskService + FailoverEngine 替代
- TaskService 作为统一入口，支持 SYNC/ASYNC 两种任务模式
- FailoverEngine 实现候选遍历、重试、故障转移逻辑
- 新增 AttemptFunc/AttemptResult 协议，统一尝试结果表示

功能改进：
- 流式响应首字节探测（30s 超时，空流触发故障转移）
- 流式取消归因优化（区分客户端断连 vs 服务端中断）
- 新增 OpenAI Sora 视频取消路由 POST /v1/videos/{task_id}/cancel
- OpenAI 流式请求自动添加 stream_options.include_usage

代码规范：
- 修复 loguru 日志格式（%s → {}）
- 新增 FORMAT_CONVERSION_ENABLED 环境变量说明

测试覆盖：
- test_failover_engine.py: FailoverEngine 单元测试
- test_task_service_async_execute.py: TaskService ASYNC 模式测试
- test_video_cancel_e2e.py: 视频取消端到端测试
This commit is contained in:
fawney19
2026-02-02 21:16:28 +08:00
parent ebe1a8d3e3
commit ed68aebfb0
53 changed files with 3966 additions and 2256 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
src/api/public/gemini_files.py
View File

@@ -310,7 +310,7 @@ async def _resolve_upstream_context(
try:
upstream_key = crypto_service.decrypt(candidate.key.api_key)
except Exception as exc:
logger.error("Failed to decrypt provider key for Gemini Files API: %s", exc)
logger.error("Failed to decrypt provider key for Gemini Files API: {}", exc)
raise HTTPException(
status_code=500,
detail={
@@ -451,12 +451,12 @@ async def _proxy_request(
mapped_count += 1
if mapped_count > 0:
logger.debug(
"Gemini list_files 批量映射已存储: %d 个文件 → key_id=%s",
"Gemini list_files 批量映射已存储: {} 个文件 → key_id={}",
mapped_count,
file_key_id,
)
except (ValueError, KeyError) as e:
logger.debug("Failed to store Gemini file mapping: %s", e)
logger.debug("Failed to store Gemini file mapping: {}", e)
return Response(
content=response.content,
@@ -467,7 +467,7 @@ async def _proxy_request(
except Exception as e:
sanitized_error = redact_url_for_log(str(e))
logger.error("Gemini Files API proxy error: %s", sanitized_error)
logger.error("Gemini Files API proxy error: {}", sanitized_error)
return JSONResponse(
status_code=502,
content={
@@ -533,7 +533,7 @@ async def upload_file(
headers = _build_upstream_headers(dict(request.headers), ctx.upstream_key)
logger.debug("Gemini Files upload proxy: POST %s", redact_url_for_log(upstream_url))
logger.debug("Gemini Files upload proxy: POST {}", redact_url_for_log(upstream_url))
return await _proxy_request(
"POST",
@@ -603,7 +603,7 @@ async def list_files(
upstream_url = _build_upstream_url(ctx.base_url, "/v1beta/files", query_params)
headers = _build_upstream_headers(dict(request.headers), ctx.upstream_key)
logger.debug("Gemini Files list proxy: GET %s", redact_url_for_log(upstream_url))
logger.debug("Gemini Files list proxy: GET {}", redact_url_for_log(upstream_url))
return await _proxy_request(
"GET", upstream_url, headers, file_key_id=ctx.file_key_id, user_id=ctx.user_id
@@ -633,7 +633,7 @@ async def _find_video_task_by_id(
from src.models.database import ProviderAPIKey, VideoTask
logger.debug(
"[Files Download] Searching video task: short_id=%s, user_id=%s", short_id, user_id
"[Files Download] Searching video task: short_id={}, user_id={}", short_id, user_id
)
# 通过 short_id 查找，同时验证用户权限
@@ -644,29 +644,29 @@ async def _find_video_task_by_id(
)
if not task:
logger.debug("[Files Download] No video task found: short_id=%s", short_id)
logger.debug("[Files Download] No video task found: short_id={}", short_id)
return None, None
if not task.video_url:
logger.debug("[Files Download] Task found but no video_url: short_id=%s", short_id)
logger.debug("[Files Download] Task found but no video_url: short_id={}", short_id)
return None, None
if not task.key_id:
logger.debug("[Files Download] Task found but no key_id: short_id=%s", short_id)
logger.debug("[Files Download] Task found but no key_id: short_id={}", short_id)
return None, task.video_url
# 获取 provider key
provider_key = db.query(ProviderAPIKey).filter(ProviderAPIKey.id == task.key_id).first()
if not provider_key or not provider_key.api_key:
logger.debug("[Files Download] Provider key not found: key_id=%s", task.key_id)
logger.debug("[Files Download] Provider key not found: key_id={}", task.key_id)
return None, task.video_url
try:
upstream_key = crypto_service.decrypt(provider_key.api_key)
logger.debug("[Files Download] Found key for task: short_id=%s", short_id)
logger.debug("[Files Download] Found key for task: short_id={}", short_id)
return upstream_key, task.video_url
except Exception as e:
logger.error("[Files Download] Failed to decrypt key: %s", e)
logger.error("[Files Download] Failed to decrypt key: {}", e)
return None, task.video_url
@@ -733,7 +733,7 @@ async def download_file(
if file_id.startswith("aev_"):
# 视频任务下载：使用短 ID 查找
short_id = file_id[4:] # 去掉 "aev_" 前缀
logger.debug("[Files Download] Video task: short_id=%s, user_id=%s", short_id, user.id)
logger.debug("[Files Download] Video task: short_id={}, user_id={}", short_id, user.id)
upstream_key, video_url = await _find_video_task_by_id(db, short_id, user.id)
if not upstream_key or not video_url:
raise HTTPException(
@@ -774,14 +774,14 @@ async def download_file(
# ========== 阶段 2：HTTP 下载（不持有数据库连接）==========
headers = _build_upstream_headers(dict(request.headers), upstream_key)
logger.debug("Gemini Files download proxy: GET %s", redact_url_for_log(upstream_url))
logger.debug("Gemini Files download proxy: GET {}", redact_url_for_log(upstream_url))
# 使用 follow_redirects=True 跟随重定向（Gemini 文件下载会重定向）
try:
async with httpx.AsyncClient(follow_redirects=True, timeout=httpx.Timeout(300.0)) as client:
response = await client.get(upstream_url, headers=headers)
except Exception as exc:
logger.error("Gemini Files download failed: %s", exc)
logger.error("Gemini Files download failed: {}", exc)
raise HTTPException(status_code=502, detail="Failed to download file")
if response.status_code >= 400:
@@ -861,7 +861,7 @@ async def get_file(
)
headers = _build_upstream_headers(dict(request.headers), ctx.upstream_key)
logger.debug("Gemini Files get proxy: GET %s", redact_url_for_log(upstream_url))
logger.debug("Gemini Files get proxy: GET {}", redact_url_for_log(upstream_url))
return await _proxy_request(
"GET", upstream_url, headers, file_key_id=ctx.file_key_id, user_id=ctx.user_id
@@ -908,13 +908,13 @@ async def delete_file(
)
headers = _build_upstream_headers(dict(request.headers), ctx.upstream_key)
logger.debug("Gemini Files delete proxy: DELETE %s", redact_url_for_log(upstream_url))
logger.debug("Gemini Files delete proxy: DELETE {}", redact_url_for_log(upstream_url))
response = await _proxy_request("DELETE", upstream_url, headers)
if response.status_code < 300:
await delete_file_key_mapping(file_name)
else:
logger.debug(
"Gemini Files delete failed, skip mapping cleanup: status=%s", response.status_code
"Gemini Files delete failed, skip mapping cleanup: status={}", response.status_code
)
return response

82
src/api/public/system_catalog.py
View File

@@ -15,12 +15,12 @@ from sqlalchemy import func
from sqlalchemy.orm import Session, selectinload
from src.api.handlers.base.request_builder import PassthroughRequestBuilder
from src.clients.redis_client import get_redis_client, get_redis_client_sync
from src.api.handlers.base.request_builder import build_test_request_body, get_provider_auth
from src.clients.redis_client import get_redis_client
from src.core.logger import logger
from src.database import get_db
from src.database.database import get_pool_status
from src.models.database import Model, Provider
from src.services.orchestration.fallback_orchestrator import FallbackOrchestrator
from src.models.database import Model, Provider, ProviderAPIKey, ProviderEndpoint
from src.services.provider.transport import build_provider_url
from src.utils.ssl_utils import get_ssl_context
@@ -256,24 +256,58 @@ async def test_connection(
if not selected_provider:
raise HTTPException(status_code=503, detail="No active provider available")
# 构建测试请求体
payload = {
"model": model,
"messages": [{"role": "user", "content": "Health check"}],
"max_tokens": 5,
}
# Determine endpoint format: prefer explicit api_format; otherwise use the provider's first active endpoint.
active_endpoints: list[ProviderEndpoint] = [
ep for ep in (selected_provider.endpoints or []) if getattr(ep, "is_active", False)
]
if not active_endpoints:
raise HTTPException(status_code=503, detail="Provider has no active endpoints")
# 确定 API 格式
format_value = api_format or "claude:chat"
if api_format:
endpoint = next(
(ep for ep in active_endpoints if (ep.api_format or "") == api_format),
None,
)
if not endpoint:
raise HTTPException(
status_code=400,
detail=f"Provider has no active endpoint for api_format={api_format}",
)
format_value = api_format
else:
endpoint = active_endpoints[0]
format_value = endpoint.api_format or "claude:chat"
# 创建 FallbackOrchestrator
redis_client = get_redis_client_sync()
orchestrator = FallbackOrchestrator(db, redis_client)
# Pick an active ProviderAPIKey that supports this format (best-effort).
active_keys: list[ProviderAPIKey] = [
k for k in (selected_provider.api_keys or []) if getattr(k, "is_active", False)
]
if not active_keys:
raise HTTPException(status_code=503, detail="Provider has no active api keys")
# 定义请求函数
async def test_request_func(_prov: Any, endpoint: Any, key: str, _candidate: Any) -> Any:
from src.api.handlers.base.request_builder import get_provider_auth
def _key_supports_format(k: ProviderAPIKey) -> bool:
formats = getattr(k, "api_formats", None)
# None => supports all formats; [] => supports none
if formats is None:
return True
if isinstance(formats, list):
return str(format_value) in {str(x) for x in formats}
# unexpected type: be permissive
return True
key = next((k for k in active_keys if _key_supports_format(k)), active_keys[0])
# Build a safe test request body in the endpoint's format (via format conversion registry).
payload = build_test_request_body(
format_value,
request_data={
"model": model,
"messages": [{"role": "user", "content": "Health check"}],
"max_tokens": 5,
},
)
try:
# 获取认证信息（处理 Service Account 等异步认证场景）
auth_info = await get_provider_auth(endpoint, key)
@@ -299,19 +333,13 @@ async def test_connection(
async with httpx.AsyncClient(timeout=30.0, verify=get_ssl_context()) as client:
resp = await client.post(url, json=provider_payload, headers=provider_headers)
resp.raise_for_status()
return resp.json()
response = resp.json()
try:
response, actual_provider, *_ = await orchestrator.execute_with_fallback(
api_format=format_value,
model_name=model,
user_api_key=None,
request_func=test_request_func,
request_id=None,
)
return {
"status": "success",
"provider": actual_provider,
"provider": selected_provider.name,
"endpoint_id": getattr(endpoint, "id", None),
"api_format": format_value,
"timestamp": datetime.now(timezone.utc).isoformat(),
"response_id": response.get("id", "unknown"),
}

16
src/api/public/videos.py
View File

@@ -33,6 +33,22 @@ async def create_video_sora(http_request: Request, db: Session = Depends(get_db)
)
@router.post("/v1/videos/{task_id}/cancel")
async def cancel_video_sora(
task_id: str, http_request: Request, db: Session = Depends(get_db)
) -> Any:
"""Cancel video task (OpenAI Sora style)."""
adapter = OpenAIVideoAdapter()
return await pipeline.run(
adapter=adapter,
http_request=http_request,
db=db,
mode=adapter.mode,
api_format_hint=adapter.allowed_api_formats[0],
path_params={"task_id": task_id, "action": "cancel"},
)
@router.get("/v1/videos/{task_id}")
async def get_video_task_sora(
task_id: str, http_request: Request, db: Session = Depends(get_db)