refactor: 统一任务框架 Phase 3 - 用 TaskService/FailoverEngine 替代 FallbackOrchestrator

核心重构： - 移除 FallbackOrchestrator，用 TaskService + FailoverEngine 替代 - TaskService 作为统一入口，支持 SYNC/ASYNC 两种任务模式 - FailoverEngine 实现候选遍历、重试、故障转移逻辑 - 新增 AttemptFunc/AttemptResult 协议，统一尝试结果表示功能改进： - 流式响应首字节探测（30s 超时，空流触发故障转移） - 流式取消归因优化（区分客户端断连 vs 服务端中断） - 新增 OpenAI Sora 视频取消路由 POST /v1/videos/{task_id}/cancel - OpenAI 流式请求自动添加 stream_options.include_usage 代码规范： - 修复 loguru 日志格式（%s → {}） - 新增 FORMAT_CONVERSION_ENABLED 环境变量说明测试覆盖： - test_failover_engine.py: FailoverEngine 单元测试 - test_task_service_async_execute.py: TaskService ASYNC 模式测试 - test_video_cancel_e2e.py: 视频取消端到端测试
2026-09-03 01:40:21 +08:00 · 2026-02-02 21:16:28 +08:00
parent ebe1a8d3e3
commit ed68aebfb0
53 changed files with 3966 additions and 2256 deletions
--- a/src/api/handlers/base/base_handler.py
+++ b/src/api/handlers/base/base_handler.py
@@ -44,7 +44,6 @@ from sqlalchemy.orm import Session

 from src.clients.redis_client import get_redis_client_sync
 from src.core.logger import logger
-from src.services.orchestration.fallback_orchestrator import FallbackOrchestrator
 from src.services.provider.format import normalize_endpoint_signature
 from src.services.system.audit import audit_service
 from src.services.usage.service import UsageService
@@ -397,7 +396,7 @@ class BaseMessageHandler:
        self.adapter_detector = adapter_detector

        redis_client = get_redis_client_sync()
-        self.orchestrator = FallbackOrchestrator(db, redis_client)  # type: ignore[arg-type]
+        self.redis = redis_client
        self.telemetry = MessageTelemetry(db, user, api_key, request_id, client_ip)

    def elapsed_ms(self) -> int:
--- a/src/api/handlers/base/chat_handler_base.py
+++ b/src/api/handlers/base/chat_handler_base.py
@@ -213,7 +213,7 @@ class ChatHandlerBase(BaseMessageHandler, ABC):
    Chat Handler 基类

    主要职责：
-    - 通过 FallbackOrchestrator 选择 Provider/Endpoint/Key
+    - 通过 TaskService/FailoverEngine 选择 Provider/Endpoint/Key
    - 发送请求并处理响应
    - 记录日志、审计、统计
    - 错误处理
@@ -466,6 +466,17 @@ class ChatHandlerBase(BaseMessageHandler, ABC):
            # 保守兜底：目标需要 stream 且当前缺失时写入
            request_body["stream"] = is_stream

+        # OpenAI Chat Completions: request usage in streaming mode.
+        # When the client format doesn't carry a `stream` field (e.g. Gemini streaming endpoint),
+        # the normalizer won't see internal.stream=True, so we need to add this here.
+        provider_fmt = str(provider_api_format or "").strip().lower()
+        if is_stream and provider_fmt == "openai:chat":
+            stream_options = request_body.get("stream_options")
+            if not isinstance(stream_options, dict):
+                stream_options = {}
+            stream_options["include_usage"] = True
+            request_body["stream_options"] = stream_options
+
    async def _get_mapped_model(
        self,
        source_model: str,
@@ -520,7 +531,7 @@ class ChatHandlerBase(BaseMessageHandler, ABC):
        model = getattr(converted_request, "model", original_request_body.get("model", "unknown"))
        api_format = self.allowed_api_formats[0]

-        # 可变请求体容器：允许 orchestrator 在遇到 Thinking 签名错误时整流请求体后重试
+        # 可变请求体容器：允许 TaskService 在遇到 Thinking 签名错误时整流请求体后重试
        # 结构: {"body": 实际请求体, "_rectified": 是否已整流, "_rectified_this_turn": 本轮是否整流}
        request_body_ref: dict[str, Any] = {"body": original_request_body}

@@ -574,15 +585,13 @@ class ChatHandlerBase(BaseMessageHandler, ABC):
                request_body=original_request_body,
            )

-            # 执行请求（通过 FallbackOrchestrator）
-            (
-                stream_generator,
-                provider_name,
-                attempt_id,
-                provider_id,
-                endpoint_id,
-                key_id,
-            ) = await self.orchestrator.execute_with_fallback(
+            # 统一入口：总是通过 TaskService
+            from src.services.task import TaskService
+            from src.services.task.context import TaskMode
+
+            exec_result = await TaskService(self.db, self.redis).execute(
+                task_type="chat",
+                task_mode=TaskMode.SYNC,
                api_format=api_format,
                model_name=model,
                user_api_key=self.api_key,
@@ -591,8 +600,14 @@ class ChatHandlerBase(BaseMessageHandler, ABC):
                is_stream=True,
                capability_requirements=capability_requirements or None,
                preferred_key_ids=preferred_key_ids or None,
-                request_body_ref=request_body_ref,  # 传递容器引用
+                request_body_ref=request_body_ref,
            )
+            stream_generator = exec_result.response
+            provider_name = exec_result.provider_name or "unknown"
+            attempt_id = exec_result.request_candidate_id
+            provider_id = exec_result.provider_id
+            endpoint_id = exec_result.endpoint_id
+            key_id = exec_result.key_id

            # 更新上下文
            ctx.attempt_id = attempt_id
@@ -644,7 +659,7 @@ class ChatHandlerBase(BaseMessageHandler, ABC):
            )

        except (ThinkingSignatureException, UpstreamClientException) as e:
-            # ThinkingSignatureException: orchestrator 层已处理整流重试但仍失败
+            # ThinkingSignatureException: TaskService 层已处理整流重试但仍失败
            # UpstreamClientException: 上游客户端错误（HTTP 4xx），不重试，直接返回给客户端
            error_type = (
                "签名错误" if isinstance(e, ThinkingSignatureException) else "上游客户端错误"
@@ -977,7 +992,7 @@ class ChatHandlerBase(BaseMessageHandler, ABC):
        model = getattr(converted_request, "model", original_request_body.get("model", "unknown"))
        api_format = self.allowed_api_formats[0]

-        # 可变请求体容器：允许 orchestrator 在遇到 Thinking 签名错误时整流请求体后重试
+        # 可变请求体容器：允许 TaskService 在遇到 Thinking 签名错误时整流请求体后重试
        # 结构: {"body": 实际请求体, "_rectified": 是否已整流, "_rectified_this_turn": 本轮是否整流}
        request_body_ref: dict[str, Any] = {"body": original_request_body}

@@ -1121,7 +1136,7 @@ class ChatHandlerBase(BaseMessageHandler, ABC):
            status_code = resp.status_code
            response_headers = dict(resp.headers)

-            # 统一使用 HTTPStatusError，让 orchestrator/error_classifier 负责分类（客户端错误/兼容性错误/限流等）
+            # 统一使用 HTTPStatusError，让 TaskService/error_classifier 负责分类（客户端错误/兼容性错误/限流等）
            try:
                resp.raise_for_status()
            except httpx.HTTPStatusError as e:
@@ -1205,23 +1220,29 @@ class ChatHandlerBase(BaseMessageHandler, ABC):
                request_body=original_request_body,
            )

-            (
-                result,
-                actual_provider_name,
-                attempt_id,
-                provider_id,
-                endpoint_id,
-                key_id,
-            ) = await self.orchestrator.execute_with_fallback(
+            # 统一入口：总是通过 TaskService
+            from src.services.task import TaskService
+            from src.services.task.context import TaskMode
+
+            exec_result = await TaskService(self.db, self.redis).execute(
+                task_type="chat",
+                task_mode=TaskMode.SYNC,
                api_format=api_format,
                model_name=model,
                user_api_key=self.api_key,
                request_func=sync_request_func,
                request_id=self.request_id,
+                is_stream=False,
                capability_requirements=capability_requirements or None,
                preferred_key_ids=preferred_key_ids or None,
-                request_body_ref=request_body_ref,  # 传递容器引用
+                request_body_ref=request_body_ref,
            )
+            result = exec_result.response
+            actual_provider_name = exec_result.provider_name or "unknown"
+            attempt_id = exec_result.request_candidate_id
+            provider_id = exec_result.provider_id
+            endpoint_id = exec_result.endpoint_id
+            key_id = exec_result.key_id

            provider_name = actual_provider_name
            response_time_ms = self.elapsed_ms()
@@ -1290,7 +1311,7 @@ class ChatHandlerBase(BaseMessageHandler, ABC):
            )

        except ThinkingSignatureException as e:
-            # Thinking 签名错误：orchestrator 层已处理整流重试但仍失败
+            # Thinking 签名错误：TaskService 层已处理整流重试但仍失败
            # 记录实际发送给 Provider 的请求体，便于排查问题根因
            response_time_ms = self.elapsed_ms()
            actual_request_body = provider_request_body or original_request_body
--- a/src/api/handlers/base/cli_handler_base.py
+++ b/src/api/handlers/base/cli_handler_base.py
@@ -406,6 +406,15 @@ class CliMessageHandlerBase(BaseMessageHandler):
        else:
            request_body.pop("stream", None)

+        # OpenAI Chat Completions: request usage in streaming mode.
+        provider_fmt = str(provider_api_format or "").strip().lower()
+        if is_stream and provider_fmt == "openai:chat":
+            stream_options = request_body.get("stream_options")
+            if not isinstance(stream_options, dict):
+                stream_options = {}
+            stream_options["include_usage"] = True
+            request_body["stream_options"] = stream_options
+
    def _convert_request_for_cross_format(
        self,
        request_body: dict[str, Any],
@@ -506,7 +515,7 @@ class CliMessageHandlerBase(BaseMessageHandler):

        通用流程：
        1. 创建流上下文
-        2. 定义请求函数（供 FallbackOrchestrator 调用）
+        2. 定义请求函数（供 TaskService/FailoverEngine 调用）
        3. 执行请求并返回 StreamingResponse
        4. 后台任务记录统计信息

@@ -519,7 +528,7 @@ class CliMessageHandlerBase(BaseMessageHandler):
        """
        logger.debug(f"开始流式响应处理 ({self.FORMAT_ID})")

-        # 可变请求体容器：允许 orchestrator 在遇到 Thinking 签名错误时整流请求体后重试
+        # 可变请求体容器：允许 TaskService 在遇到 Thinking 签名错误时整流请求体后重试
        # 结构: {"body": 实际请求体, "_rectified": 是否已整流, "_rectified_this_turn": 本轮是否整流}
        request_body_ref: dict[str, Any] = {"body": original_request_body}

@@ -567,15 +576,13 @@ class CliMessageHandlerBase(BaseMessageHandler):
                request_body=original_request_body,
            )

-            # 执行请求（通过 FallbackOrchestrator）
-            (
-                stream_generator,
-                provider_name,
-                attempt_id,
-                provider_id,
-                endpoint_id,
-                key_id,
-            ) = await self.orchestrator.execute_with_fallback(
+            # 统一入口：总是通过 TaskService
+            from src.services.task import TaskService
+            from src.services.task.context import TaskMode
+
+            exec_result = await TaskService(self.db, self.redis).execute(
+                task_type="cli",
+                task_mode=TaskMode.SYNC,
                api_format=ctx.api_format,
                model_name=ctx.model,
                user_api_key=self.api_key,
@@ -584,8 +591,14 @@ class CliMessageHandlerBase(BaseMessageHandler):
                is_stream=True,
                capability_requirements=capability_requirements or None,
                preferred_key_ids=preferred_key_ids or None,
-                request_body_ref=request_body_ref,  # 传递容器引用
+                request_body_ref=request_body_ref,
            )
+            stream_generator = exec_result.response
+            provider_name = exec_result.provider_name or "unknown"
+            attempt_id = exec_result.request_candidate_id
+            provider_id = exec_result.provider_id
+            endpoint_id = exec_result.endpoint_id
+            key_id = exec_result.key_id

            # 更新上下文（确保 provider 信息已设置，用于 streaming 状态更新）
            ctx.attempt_id = attempt_id
@@ -628,7 +641,7 @@ class CliMessageHandlerBase(BaseMessageHandler):
            )

        except ThinkingSignatureException as e:
-            # Thinking 签名错误：orchestrator 层已处理整流重试但仍失败
+            # Thinking 签名错误：TaskService 层已处理整流重试但仍失败
            # 记录 original_request_body（客户端原始请求），便于排查问题根因
            self._log_request_error("流式请求失败（签名错误）", e)
            await self._record_stream_failure(ctx, e, original_headers, original_request_body)
@@ -1803,19 +1816,50 @@ class CliMessageHandlerBase(BaseMessageHandler):
                    yield chunk

        except asyncio.CancelledError:
-            # 计算距离上次收到 chunk 的时间
+            # 注意：CancelledError 不等于“用户手动取消”，它既可能是客户端断连触发，
+            # 也可能是服务端（重载/关停/内部取消）导致的协程取消。
+            # 这里尽量做一次“断连归因”：仅当能确认客户端已断开时才记为 499 cancelled。
            time_since_last_chunk = time_module.time() - last_chunk_time
-            # 如果响应已完成，不标记为失败
+
+            is_client_disconnected = False
+            if http_request is not None:
+                try:
+                    # shield + timeout: 避免在取消态下二次被 CancelledError 打断，尽力取到断连状态
+                    # 限时 0.5s 防止极端情况下的阻塞
+                    is_client_disconnected = await asyncio.wait_for(
+                        asyncio.shield(http_request.is_disconnected()),
+                        timeout=0.5,
+                    )
+                except (asyncio.CancelledError, asyncio.TimeoutError):
+                    # 无法在取消态/超时下完成断连检查，保守视为未知（不强行归因为客户端）
+                    is_client_disconnected = False
+                except Exception as e:
+                    logger.debug(f"ID:{ctx.request_id} | cancel 断连检测失败: {e}")
+                    is_client_disconnected = False
+
+            # 如果响应已完成，不标记为失败/取消
            if not ctx.has_completion:
-                ctx.status_code = 499
-                ctx.error_message = "Client disconnected"
-                logger.warning(
-                    f"ID:{ctx.request_id} | Stream cancelled: "
-                    f"chunks={chunk_count}, "
-                    f"has_completion={ctx.has_completion}, "
-                    f"time_since_last_chunk={time_since_last_chunk:.2f}s, "
-                    f"output_tokens={ctx.output_tokens}"
-                )
+                if is_client_disconnected:
+                    ctx.status_code = 499
+                    ctx.error_message = "client_disconnected"
+                    logger.warning(
+                        f"ID:{ctx.request_id} | Stream cancelled by client: "
+                        f"chunks={chunk_count}, "
+                        f"has_completion={ctx.has_completion}, "
+                        f"time_since_last_chunk={time_since_last_chunk:.2f}s, "
+                        f"output_tokens={ctx.output_tokens}"
+                    )
+                else:
+                    # 服务端中断（例如重载/关停/内部取消）— 不应伪装成客户端取消
+                    ctx.status_code = 503
+                    ctx.error_message = "server_cancelled"
+                    logger.error(
+                        f"ID:{ctx.request_id} | Stream interrupted by server: "
+                        f"chunks={chunk_count}, "
+                        f"has_completion={ctx.has_completion}, "
+                        f"time_since_last_chunk={time_since_last_chunk:.2f}s, "
+                        f"output_tokens={ctx.output_tokens}"
+                    )
            raise
        except httpx.TimeoutException as e:
            ctx.status_code = 504
@@ -1883,45 +1927,73 @@ class CliMessageHandlerBase(BaseMessageHandler):
                actual_request_body = ctx.provider_request_body or original_request_body

                # 根据状态码决定记录成功还是失败
-                # 499 = 客户端断开连接，503 = 服务不可用（如流中断）
+                # 499 = 客户端取消（不算系统失败）；其他 4xx/5xx 视为失败
                if ctx.status_code and ctx.status_code >= 400:
-                    # 记录失败的 Usage，但使用已收到的预估 token 信息（来自 message_start）
-                    # 这样即使请求中断，也能记录预估成本
-                    # 失败时返回给客户端的是 JSON 错误响应，如果没有设置则使用默认值
                    client_response_headers = ctx.client_response_headers or {
                        "content-type": "application/json"
                    }
-                    await bg_telemetry.record_failure(
-                        provider=ctx.provider_name or "unknown",
-                        model=ctx.model,
-                        response_time_ms=response_time_ms,
-                        status_code=ctx.status_code,
-                        error_message=ctx.error_message or f"HTTP {ctx.status_code}",
-                        request_headers=original_headers,
-                        request_body=actual_request_body,
-                        is_stream=True,
-                        api_format=ctx.api_format,
-                        provider_request_headers=ctx.provider_request_headers,
-                        # 预估 token 信息（来自 message_start 事件）
-                        input_tokens=actual_input_tokens,
-                        output_tokens=ctx.output_tokens,
-                        cache_creation_tokens=ctx.cache_creation_tokens,
-                        cache_read_tokens=ctx.cached_tokens,
-                        response_body=response_body,
-                        response_headers=ctx.response_headers,
-                        client_response_headers=client_response_headers,
-                        # 格式转换追踪
-                        endpoint_api_format=ctx.provider_api_format or None,
-                        has_format_conversion=ctx.needs_conversion,
-                        # 模型映射信息
-                        target_model=ctx.mapped_model,
-                    )
-                    logger.debug(f"{self.FORMAT_ID} 流式响应中断")
-                    # 简洁的请求失败摘要（包含预估 token 信息）
-                    logger.info(
-                        f"[FAIL] {self.request_id[:8]} | {ctx.model} | {ctx.provider_name} | {response_time_ms}ms | "
-                        f"{ctx.status_code} | in:{actual_input_tokens} out:{ctx.output_tokens} cache:{ctx.cached_tokens}"
-                    )
+
+                    if ctx.is_client_disconnected():
+                        # 客户端取消：记录为 cancelled（不算系统失败）
+                        await bg_telemetry.record_cancelled(
+                            provider=ctx.provider_name or "unknown",
+                            model=ctx.model,
+                            response_time_ms=response_time_ms,
+                            first_byte_time_ms=ctx.first_byte_time_ms,
+                            status_code=ctx.status_code,
+                            request_headers=original_headers,
+                            request_body=actual_request_body,
+                            is_stream=True,
+                            api_format=ctx.api_format,
+                            provider_request_headers=ctx.provider_request_headers,
+                            input_tokens=actual_input_tokens,
+                            output_tokens=ctx.output_tokens,
+                            cache_creation_tokens=ctx.cache_creation_tokens,
+                            cache_read_tokens=ctx.cached_tokens,
+                            response_body=response_body,
+                            response_headers=ctx.response_headers,
+                            client_response_headers=client_response_headers,
+                            endpoint_api_format=ctx.provider_api_format or None,
+                            has_format_conversion=ctx.needs_conversion,
+                            target_model=ctx.mapped_model,
+                        )
+                        logger.debug(f"{self.FORMAT_ID} 流式响应被客户端取消")
+                        logger.info(
+                            f"[CANCEL] {self.request_id[:8]} | {ctx.model} | {ctx.provider_name} | {response_time_ms}ms | "
+                            f"{ctx.status_code} | in:{actual_input_tokens} out:{ctx.output_tokens} cache:{ctx.cached_tokens}"
+                        )
+                    else:
+                        # 服务端/上游异常：记录为失败
+                        await bg_telemetry.record_failure(
+                            provider=ctx.provider_name or "unknown",
+                            model=ctx.model,
+                            response_time_ms=response_time_ms,
+                            status_code=ctx.status_code,
+                            error_message=ctx.error_message or f"HTTP {ctx.status_code}",
+                            request_headers=original_headers,
+                            request_body=actual_request_body,
+                            is_stream=True,
+                            api_format=ctx.api_format,
+                            provider_request_headers=ctx.provider_request_headers,
+                            # 预估 token 信息（来自 message_start 事件）
+                            input_tokens=actual_input_tokens,
+                            output_tokens=ctx.output_tokens,
+                            cache_creation_tokens=ctx.cache_creation_tokens,
+                            cache_read_tokens=ctx.cached_tokens,
+                            response_body=response_body,
+                            response_headers=ctx.response_headers,
+                            client_response_headers=client_response_headers,
+                            # 格式转换追踪
+                            endpoint_api_format=ctx.provider_api_format or None,
+                            has_format_conversion=ctx.needs_conversion,
+                            # 模型映射信息
+                            target_model=ctx.mapped_model,
+                        )
+                        logger.debug(f"{self.FORMAT_ID} 流式响应中断")
+                        logger.info(
+                            f"[FAIL] {self.request_id[:8]} | {ctx.model} | {ctx.provider_name} | {response_time_ms}ms | "
+                            f"{ctx.status_code} | in:{actual_input_tokens} out:{ctx.output_tokens} cache:{ctx.cached_tokens}"
+                        )
                else:
                    # 在记录统计前，允许子类从 parsed_chunks 中提取额外的元数据
                    self._finalize_stream_metadata(ctx)
@@ -2016,17 +2088,24 @@ class CliMessageHandlerBase(BaseMessageHandler):
                        }
                        if candidate_first_byte_time_ms is not None:
                            extra_data["first_byte_time_ms"] = candidate_first_byte_time_ms
-                        RequestCandidateService.mark_candidate_failed(
-                            db=bg_db,
-                            candidate_id=ctx.attempt_id,
-                            error_type=(
-                                "client_disconnected" if ctx.status_code == 499 else "stream_error"
-                            ),
-                            error_message=trace_error_message,
-                            status_code=ctx.status_code,
-                            latency_ms=response_time_ms,
-                            extra_data=extra_data,
-                        )
+                        if ctx.is_client_disconnected():
+                            RequestCandidateService.mark_candidate_cancelled(
+                                db=bg_db,
+                                candidate_id=ctx.attempt_id,
+                                status_code=ctx.status_code,
+                                latency_ms=response_time_ms,
+                                extra_data=extra_data,
+                            )
+                        else:
+                            RequestCandidateService.mark_candidate_failed(
+                                db=bg_db,
+                                candidate_id=ctx.attempt_id,
+                                error_type="stream_error",
+                                error_message=trace_error_message,
+                                status_code=ctx.status_code,
+                                latency_ms=response_time_ms,
+                                extra_data=extra_data,
+                            )
                    else:
                        extra_data = {
                            "stream_completed": True,
@@ -2115,7 +2194,7 @@ class CliMessageHandlerBase(BaseMessageHandler):

        通用流程：
        1. 构建请求
-        2. 通过 FallbackOrchestrator 执行
+        2. 通过 TaskService/FailoverEngine 执行
        3. 解析响应并记录统计
        """
        logger.debug(f"开始非流式响应处理 ({self.FORMAT_ID})")
@@ -2139,7 +2218,7 @@ class CliMessageHandlerBase(BaseMessageHandler):
        response_metadata_result: dict[str, Any] = {}  # Provider 响应元数据
        needs_conversion = False  # 是否需要格式转换（由 candidate 决定）

-        # 可变请求体容器：允许 orchestrator 在遇到 Thinking 签名错误时整流请求体后重试
+        # 可变请求体容器：允许 TaskService 在遇到 Thinking 签名错误时整流请求体后重试
        # 结构: {"body": 实际请求体, "_rectified": 是否已整流, "_rectified_this_turn": 本轮是否整流}
        request_body_ref: dict[str, Any] = {"body": original_request_body}

@@ -2333,23 +2412,29 @@ class CliMessageHandlerBase(BaseMessageHandler):
                request_body=original_request_body,
            )

-            (
-                result,
-                actual_provider_name,
-                attempt_id,
-                provider_id,
-                endpoint_id,
-                key_id,
-            ) = await self.orchestrator.execute_with_fallback(
+            # 统一入口：总是通过 TaskService
+            from src.services.task import TaskService
+            from src.services.task.context import TaskMode
+
+            exec_result = await TaskService(self.db, self.redis).execute(
+                task_type="cli",
+                task_mode=TaskMode.SYNC,
                api_format=api_format,
                model_name=model,
                user_api_key=self.api_key,
                request_func=sync_request_func,
                request_id=self.request_id,
+                is_stream=False,
                capability_requirements=capability_requirements or None,
                preferred_key_ids=preferred_key_ids or None,
-                request_body_ref=request_body_ref,  # 传递容器引用
+                request_body_ref=request_body_ref,
            )
+            result = exec_result.response
+            actual_provider_name = exec_result.provider_name or "unknown"
+            attempt_id = exec_result.request_candidate_id
+            provider_id = exec_result.provider_id
+            endpoint_id = exec_result.endpoint_id
+            key_id = exec_result.key_id

            provider_name = actual_provider_name
            response_time_ms = int((time.time() - sync_start_time) * 1000)
@@ -2434,7 +2519,7 @@ class CliMessageHandlerBase(BaseMessageHandler):
            )

        except ThinkingSignatureException as e:
-            # Thinking 签名错误：orchestrator 层已处理整流重试但仍失败
+            # Thinking 签名错误：TaskService 层已处理整流重试但仍失败
            # 记录实际发送给 Provider 的请求体，便于排查问题根因
            response_time_ms = int((time.time() - sync_start_time) * 1000)
            actual_request_body = provider_request_body or original_request_body
--- a/src/api/handlers/base/stream_context.py
+++ b/src/api/handlers/base/stream_context.py
@@ -256,7 +256,12 @@ class StreamContext:
        用于请求完成/失败时的日志输出。
        包含首字时间 (TTFB) 和总响应时间,分两行显示。
        """
-        status = "OK" if self.is_success() else "FAIL"
+        if self.is_success():
+            status = "OK"
+        elif self.is_client_disconnected():
+            status = "CANCEL"
+        else:
+            status = "FAIL"

        # 第一行:基本信息 + 首字时间
        line1 = (
--- a/src/api/handlers/base/video_adapter_base.py
+++ b/src/api/handlers/base/video_adapter_base.py
@@ -57,13 +57,11 @@ class VideoAdapterBase(ApiAdapter):
        path = http_request.url.path.lower()
        task_id = path_params.get("task_id")

-        if method in {"POST", "PUT", "PATCH"}:
-            original_request_body = context.ensure_json_body()
-        else:
-            original_request_body = {}
+        # Note: not every POST endpoint requires a body (e.g. cancel).
+        original_request_body: dict[str, Any] = {}

        logger.debug(
-            "[VideoAdapter] dispatch method=%s path=%s task_id=%s",
+            "[VideoAdapter] dispatch method={} path={} task_id={}",
            method,
            path,
            task_id,
@@ -105,6 +103,7 @@ class VideoAdapterBase(ApiAdapter):

        # Remix task
        if method == "POST" and path.endswith("/remix") and task_id:
+            original_request_body = context.ensure_json_body()
            return await handler.handle_remix_task(
                task_id=task_id,
                http_request=http_request,
@@ -134,6 +133,8 @@ class VideoAdapterBase(ApiAdapter):
            )

        # Create task (default)
+        if method in {"POST", "PUT", "PATCH"}:
+            original_request_body = context.ensure_json_body()
        return await handler.handle_create_task(
            http_request=http_request,
            original_headers=context.original_headers,
--- a/src/api/handlers/base/video_handler_base.py
+++ b/src/api/handlers/base/video_handler_base.py
@@ -315,7 +315,7 @@ class VideoHandlerBase(ABC):
            )
        except Exception as exc:
            logger.warning(
-                "Failed to finalize usage on submit failure: request_id=%s, error=%s",
+                "Failed to finalize usage on submit failure: request_id={}, error={}",
                self.request_id,
                sanitize_error_message(str(exc)),
            )
@@ -367,16 +367,20 @@ class VideoHandlerBase(ABC):
        - 无可用候选 / 全部失败：抛 HTTPException(503)
        """
        # 延迟导入，避免 handler 基类层引入过多依赖导致循环
-        from src.services.candidate.service import CandidateService
        from src.services.candidate.submit import (
            AllCandidatesFailedError,
            SubmitOutcome,
            UpstreamClientRequestError,
        )

-        candidate_service = CandidateService(self.db)
+        # 统一入口：总是通过 TaskService（内部可继续委托 CandidateService，便于逐步内核统一）
+        from src.services.task import TaskService
+
+        submitter: Any = TaskService(self.db)
+        submit_call = submitter.submit_with_failover
+
        try:
-            return await candidate_service.submit_with_failover(
+            return await submit_call(
                api_format=api_format,
                model_name=model_name,
                affinity_key=str(self.api_key.id),
@@ -402,7 +406,7 @@ class VideoHandlerBase(ABC):
                detail = "No available provider with billing rule for video generation"
            # 记录候选信息到日志
            logger.warning(
-                "[VideoHandler] All candidates failed: reason=%s, candidate_keys=%s",
+                "[VideoHandler] All candidates failed: reason={}, candidate_keys={}",
                exc.reason,
                exc.candidate_keys,
            )
--- a/src/api/handlers/claude/handler.py
+++ b/src/api/handlers/claude/handler.py
@@ -72,7 +72,7 @@ class ClaudeChatHandler(ChatHandlerBase):
        将请求转换为 Claude 格式的 Pydantic 对象

        注意：此方法只做类型转换（dict → Pydantic），不做跨格式转换。
-        跨格式转换由 FallbackOrchestrator 在选中候选后、发送请求前执行，
+        跨格式转换由调度/执行层（TaskService + RequestDispatcher）在选中候选后、发送请求前执行，
        并受全局开关和端点配置控制。

        Args:
--- a/src/api/handlers/claude_cli/handler.py
+++ b/src/api/handlers/claude_cli/handler.py
@@ -19,7 +19,7 @@ class ClaudeCliMessageHandler(CliMessageHandlerBase):
    Claude CLI Message Handler - 处理 Claude CLI API 格式

    使用新三层架构 (Provider -> ProviderEndpoint -> ProviderAPIKey)
-    通过 FallbackOrchestrator 实现自动故障转移、健康监控和并发控制
+    通过 TaskService/FailoverEngine 实现自动故障转移、健康监控和并发控制

    响应格式特点：
    - 使用 content[] 数组
--- a/src/api/handlers/gemini/handler.py
+++ b/src/api/handlers/gemini/handler.py
@@ -111,7 +111,7 @@ class GeminiChatHandler(ChatHandlerBase):
        将请求转换为 Gemini 格式的 Pydantic 对象

        注意：此方法只做类型转换（dict → Pydantic），不做跨格式转换。
-        跨格式转换由 FallbackOrchestrator 在选中候选后、发送请求前执行，
+        跨格式转换由调度/执行层（TaskService + RequestDispatcher）在选中候选后、发送请求前执行，
        并受全局开关和端点配置控制。

        Args:
--- a/src/api/handlers/gemini/video_handler.py
+++ b/src/api/handlers/gemini/video_handler.py
@@ -111,7 +111,7 @@ class GeminiVeoHandler(VideoHandlerBase):
            )
        except Exception as exc:
            logger.warning(
-                "Failed to create pending usage for video request_id=%s: %s",
+                "Failed to create pending usage for video request_id={}: {}",
                self.request_id,
                sanitize_error_message(str(exc)),
            )
@@ -171,7 +171,7 @@ class GeminiVeoHandler(VideoHandlerBase):
            if "name" in payload:
                value = payload.get("name")
                logger.debug(
-                    "[GeminiVeoHandler] Upstream response name=%s, keys=%s",
+                    "[GeminiVeoHandler] Upstream response name={}, keys={}",
                    value,
                    list(payload.keys()) if isinstance(payload, dict) else type(payload),
                )
@@ -222,7 +222,7 @@ class GeminiVeoHandler(VideoHandlerBase):
                )
            except Exception as e:
                logger.warning(
-                    "[GeminiVeoHandler] Failed to record converted request: %s",
+                    "[GeminiVeoHandler] Failed to record converted request: {}",
                    sanitize_error_message(str(e)),
                )

@@ -243,7 +243,7 @@ class GeminiVeoHandler(VideoHandlerBase):
            self.db.commit()
            self.db.refresh(task)
            logger.debug(
-                "[GeminiVeoHandler] Task created: id=%s, external_task_id=%s",
+                "[GeminiVeoHandler] Task created: id={}, external_task_id={}",
                task.id,
                task.external_task_id,
            )
@@ -292,7 +292,7 @@ class GeminiVeoHandler(VideoHandlerBase):
            self.db.commit()
        except Exception as exc:
            logger.warning(
-                "Failed to finalize submitted usage for video request_id=%s: %s",
+                "Failed to finalize submitted usage for video request_id={}: {}",
                self.request_id,
                sanitize_error_message(str(exc)),
            )
@@ -345,51 +345,16 @@ class GeminiVeoHandler(VideoHandlerBase):
        query_params: dict[str, str] | None = None,
        path_params: dict[str, Any] | None = None,
    ) -> JSONResponse:
-        task = self._get_task_by_external_id(task_id)
-        if not task.external_task_id:
-            raise HTTPException(status_code=500, detail="Task missing external_task_id")
-        endpoint, key = self._get_endpoint_and_key(task)
-        if not key.api_key:
-            raise HTTPException(status_code=500, detail="Provider key not configured")
-        upstream_key = crypto_service.decrypt(key.api_key)
+        from src.services.task.service import TaskService

-        operation_name = task.external_task_id
-        if not operation_name.startswith("operations/"):
-            operation_name = f"operations/{operation_name}"
-        upstream_url = self._build_cancel_url(endpoint.base_url, operation_name)
-        auth_info = await get_provider_auth(endpoint, key)
-        headers = self._build_upstream_headers(original_headers, upstream_key, endpoint, auth_info)
-
-        client = await HTTPClientPool.get_default_client_async()
-        response = await client.post(upstream_url, headers=headers, json={})
-        if response.status_code >= 400:
-            return self._build_error_response(response)
-
-        task.status = VideoStatus.CANCELLED.value
-        task.updated_at = datetime.now(timezone.utc)
-
-        # 将 Usage 作废（不收费）
-        # 尝试 finalize_void（处理 pending）和 void_settled（处理已 settled）
-        try:
-            voided = UsageService.finalize_void(
-                self.db,
-                request_id=task.request_id,
-                reason="cancelled_by_user",
-            )
-            if not voided:
-                # pending 状态未找到，尝试处理已 settled 的记录
-                UsageService.void_settled(
-                    self.db,
-                    request_id=task.request_id,
-                    reason="cancelled_by_user",
-                )
-        except Exception as exc:
-            logger.warning(
-                "Failed to void usage for cancelled task=%s: %s",
-                task.id,
-                sanitize_error_message(str(exc)),
-            )
-        self.db.commit()
+        _ = (http_request, query_params, path_params)  # reserved for future extensions
+        err_resp = await TaskService(self.db).cancel(
+            task_id,
+            user_id=str(self.user.id),
+            original_headers=original_headers,
+        )
+        if err_resp is not None:
+            return self._build_error_response(err_resp)
        return JSONResponse({})

    async def handle_download_content(
@@ -446,7 +411,7 @@ class GeminiVeoHandler(VideoHandlerBase):
                    download_headers[auth_info.auth_header] = auth_info.auth_value
            except Exception as exc:
                logger.warning(
-                    "[VideoDownload] Failed to get auth for download task=%s: %s",
+                    "[VideoDownload] Failed to get auth for download task={}: {}",
                    task.id,
                    sanitize_error_message(str(exc)),
                )
@@ -464,7 +429,7 @@ class GeminiVeoHandler(VideoHandlerBase):
                response = await client.get(task.video_url, headers=download_headers)
        except Exception as exc:
            logger.error(
-                "[VideoDownload] Upstream fetch failed user=%s task=%s: %s",
+                "[VideoDownload] Upstream fetch failed user={} task={}: {}",
                self.user.id,
                task.id,
                sanitize_error_message(str(exc)),
@@ -496,7 +461,7 @@ class GeminiVeoHandler(VideoHandlerBase):
            upstream_key = crypto_service.decrypt(candidate.key.api_key)
        except Exception as exc:
            logger.error(
-                "Failed to decrypt provider key id=%s: %s",
+                "Failed to decrypt provider key id={}: {}",
                candidate.key.id,
                sanitize_error_message(str(exc)),
            )
@@ -684,7 +649,7 @@ class GeminiVeoHandler(VideoHandlerBase):
            .first()
        )
        if not task:
-            logger.debug("[GeminiVeoHandler] Task not found: short_id=%s", short_id)
+            logger.debug("[GeminiVeoHandler] Task not found: short_id={}", short_id)
            raise HTTPException(status_code=404, detail="Video task not found")
        return task

--- a/src/api/handlers/gemini_cli/handler.py
+++ b/src/api/handlers/gemini_cli/handler.py
@@ -18,7 +18,7 @@ class GeminiCliMessageHandler(CliMessageHandlerBase):
    Gemini CLI Message Handler - 处理 Gemini CLI API 格式

    使用新三层架构 (Provider -> ProviderEndpoint -> ProviderAPIKey)
-    通过 FallbackOrchestrator 实现自动故障转移、健康监控和并发控制
+    通过 TaskService/FailoverEngine 实现自动故障转移、健康监控和并发控制

    响应格式特点：
    - Gemini 使用 JSON 数组格式流式响应（非 SSE）
--- a/src/api/handlers/openai/handler.py
+++ b/src/api/handlers/openai/handler.py
@@ -75,7 +75,7 @@ class OpenAIChatHandler(ChatHandlerBase):
        将请求转换为 OpenAI 格式的 Pydantic 对象

        注意：此方法只做类型转换（dict → Pydantic），不做跨格式转换。
-        跨格式转换由 FallbackOrchestrator 在选中候选后、发送请求前执行，
+        跨格式转换由调度/执行层（TaskService + RequestDispatcher）在选中候选后、发送请求前执行，
        并受全局开关和端点配置控制。

        Args:
--- a/src/api/handlers/openai/video_handler.py
+++ b/src/api/handlers/openai/video_handler.py
@@ -110,7 +110,7 @@ class OpenAIVideoHandler(VideoHandlerBase):
            )
        except Exception as exc:
            logger.warning(
-                "Failed to create pending usage for video request_id=%s: %s",
+                "Failed to create pending usage for video request_id={}: {}",
                self.request_id,
                sanitize_error_message(str(exc)),
            )
@@ -245,7 +245,7 @@ class OpenAIVideoHandler(VideoHandlerBase):
                )
            except Exception as e:
                logger.warning(
-                    "[OpenAIVideoHandler] Failed to record converted request: %s",
+                    "[OpenAIVideoHandler] Failed to record converted request: {}",
                    sanitize_error_message(str(e)),
                )

@@ -309,7 +309,7 @@ class OpenAIVideoHandler(VideoHandlerBase):
            self.db.commit()
        except Exception as exc:
            logger.warning(
-                "Failed to finalize submitted usage for video request_id=%s: %s",
+                "Failed to finalize submitted usage for video request_id={}: {}",
                self.request_id,
                sanitize_error_message(str(exc)),
            )
@@ -401,47 +401,16 @@ class OpenAIVideoHandler(VideoHandlerBase):
        query_params: dict[str, str] | None = None,
        path_params: dict[str, Any] | None = None,
    ) -> JSONResponse:
-        task = self._get_task(task_id)
-        if not task.external_task_id:
-            raise HTTPException(status_code=500, detail="Task missing external_task_id")
-        endpoint, key = self._get_endpoint_and_key(task)
-        if not key.api_key:
-            raise HTTPException(status_code=500, detail="Provider key not configured")
-        upstream_key = crypto_service.decrypt(key.api_key)
+        from src.services.task.service import TaskService

-        upstream_url = self._build_upstream_url(endpoint.base_url, task.external_task_id)
-        headers = self._build_upstream_headers(original_headers, upstream_key, endpoint)
-
-        client = await HTTPClientPool.get_default_client_async()
-        response = await client.delete(upstream_url, headers=headers)
-        if response.status_code >= 400:
-            return self._build_error_response(response)
-
-        task.status = VideoStatus.CANCELLED.value
-        task.updated_at = datetime.now(timezone.utc)
-
-        # 将 Usage 作废（不收费）
-        # 尝试 finalize_void（处理 pending）和 void_settled（处理已 settled）
-        try:
-            voided = UsageService.finalize_void(
-                self.db,
-                request_id=task.request_id,
-                reason="cancelled_by_user",
-            )
-            if not voided:
-                # pending 状态未找到，尝试处理已 settled 的记录
-                UsageService.void_settled(
-                    self.db,
-                    request_id=task.request_id,
-                    reason="cancelled_by_user",
-                )
-        except Exception as exc:
-            logger.warning(
-                "Failed to void usage for cancelled task=%s: %s",
-                task.id,
-                sanitize_error_message(str(exc)),
-            )
-        self.db.commit()
+        _ = (http_request, query_params, path_params)  # reserved for future extensions
+        err_resp = await TaskService(self.db).cancel(
+            task_id,
+            user_id=str(self.user.id),
+            original_headers=original_headers,
+        )
+        if err_resp is not None:
+            return self._build_error_response(err_resp)
        return JSONResponse({})

    async def handle_delete_task(
@@ -481,7 +450,7 @@ class OpenAIVideoHandler(VideoHandlerBase):
                        return self._build_error_response(response)
            except Exception as exc:
                logger.warning(
-                    "Failed to delete video from upstream task=%s: %s",
+                    "Failed to delete video from upstream task={}: {}",
                    task.id,
                    sanitize_error_message(str(exc)),
                )
@@ -646,7 +615,7 @@ class OpenAIVideoHandler(VideoHandlerBase):
        # 保持流式代理而非重定向，确保客户端行为与官方 OpenAI 一致
        if variant == "video" and task.video_url and task.video_url.startswith("http"):
            logger.debug(
-                "[VideoDownload] Proxying direct URL task=%s url=%s",
+                "[VideoDownload] Proxying direct URL task={} url={}",
                task_id,
                task.video_url,
            )
@@ -673,7 +642,7 @@ class OpenAIVideoHandler(VideoHandlerBase):

        client = await HTTPClientPool.get_default_client_async()
        logger.debug(
-            "[VideoDownload] Requesting upstream url=%s task=%s external_task_id=%s",
+            "[VideoDownload] Requesting upstream url={} task={} external_task_id={}",
            upstream_url,
            task_id,
            task.external_task_id,
@@ -685,7 +654,7 @@ class OpenAIVideoHandler(VideoHandlerBase):
            response = await client.send(request, stream=True, timeout=300.0)
        except Exception as exc:
            logger.warning(
-                "[VideoDownload] Upstream connection failed task=%s url=%s: %s",
+                "[VideoDownload] Upstream connection failed task={} url={}: {}",
                task_id,
                upstream_url,
                sanitize_error_message(str(exc)),
@@ -743,7 +712,7 @@ class OpenAIVideoHandler(VideoHandlerBase):
            upstream_key = crypto_service.decrypt(candidate.key.api_key)
        except Exception as exc:
            logger.error(
-                "Failed to decrypt provider key id=%s: %s",
+                "Failed to decrypt provider key id={}: {}",
                candidate.key.id,
                sanitize_error_message(str(exc)),
            )
@@ -758,7 +727,7 @@ class OpenAIVideoHandler(VideoHandlerBase):
            response = await client.send(request, stream=True, timeout=300.0)
        except Exception as exc:
            logger.warning(
-                "[VideoDownload] Direct URL connection failed task=%s url=%s: %s",
+                "[VideoDownload] Direct URL connection failed task={} url={}: {}",
                task_id,
                url,
                sanitize_error_message(str(exc)),
@@ -1016,7 +985,7 @@ class OpenAIVideoHandler(VideoHandlerBase):
                target_model=None,
            )
        except Exception as exc:
-            logger.warning("Failed to record failed usage: %s", sanitize_error_message(str(exc)))
+            logger.warning("Failed to record failed usage: {}", sanitize_error_message(str(exc)))

    async def _create_failed_task_and_usage(
        self,
@@ -1089,7 +1058,7 @@ class OpenAIVideoHandler(VideoHandlerBase):
        except Exception as exc:
            self.db.rollback()
            logger.warning(
-                "Failed to create failed task record: %s", sanitize_error_message(str(exc))
+                "Failed to create failed task record: {}", sanitize_error_message(str(exc))
            )
            # 即使任务记录失败，仍然尝试记录使用记录
            task = None
@@ -1136,7 +1105,7 @@ class OpenAIVideoHandler(VideoHandlerBase):
                target_model=None,
            )
        except Exception as exc:
-            logger.warning("Failed to record failed usage: %s", sanitize_error_message(str(exc)))
+            logger.warning("Failed to record failed usage: {}", sanitize_error_message(str(exc)))


 __all__ = ["OpenAIVideoHandler"]
--- a/src/api/handlers/openai_cli/handler.py
+++ b/src/api/handlers/openai_cli/handler.py
@@ -19,7 +19,7 @@ class OpenAICliMessageHandler(CliMessageHandlerBase):
    OpenAI CLI Message Handler - 处理 OpenAI CLI Responses API 格式

    使用新三层架构 (Provider -> ProviderEndpoint -> ProviderAPIKey)
-    通过 FallbackOrchestrator 实现自动故障转移、健康监控和并发控制
+    通过 TaskService/FailoverEngine 实现自动故障转移、健康监控和并发控制

    响应格式特点：
    - 使用 output[] 数组而非 content[]