feat(usage): 改进请求追踪与缓存预热

- 区分请求重试与故障转移：新增 has_retry 标识亲和缓存重试
- 修正候选 TTFB 计算：记录候选自身的首字节时间而非全局时间
- Streaming 状态同步 rate_multiplier，支持按 API 格式配置
- 新增缓存预热服务：启动时预热仪表盘统计、热力图、每日统计
- 优化缓存 TTL 配置：仪表盘统计 2 分钟、热力图和每日统计 10 分钟
- 自动刷新间隔从 10 秒调整为 5 秒

src/api/admin/usage/routes.py

@@ -1,5 +1,6 @@
 """管理员使用情况统计路由。"""
 
+from collections import defaultdict
 from dataclasses import dataclass
 from datetime import datetime
 from typing import Optional
@@ -717,19 +718,42 @@ class AdminUsageRecordsAdapter(AdminApiAdapter):
 
         request_ids = [usage.request_id for usage, _, _, _, _ in records if usage.request_id]
         fallback_map = {}
+        retry_map = {}
         if request_ids:
+            # 查询每个请求的候选执行情况
             # 只统计实际执行的候选（success 或 failed），不包括 skipped/pending/available
-            executed_counts = (
-                db.query(RequestCandidate.request_id, func.count(RequestCandidate.id))
+            executed_candidates = (
+                db.query(
+                    RequestCandidate.request_id,
+                    RequestCandidate.candidate_index,
+                    RequestCandidate.retry_index,
+                )
                 .filter(
                     RequestCandidate.request_id.in_(request_ids),
                     RequestCandidate.status.in_(["success", "failed"]),
                 )
-                .group_by(RequestCandidate.request_id)
                 .all()
             )
-            # 如果实际执行的候选数 > 1，说明发生了 Provider 切换
-            fallback_map = {req_id: count > 1 for req_id, count in executed_counts}
+
+            # 按 request_id 分组分析
+            request_candidates: dict[str, list[tuple[int, int]]] = defaultdict(list)
+            for req_id, candidate_idx, retry_idx in executed_candidates:
+                request_candidates[req_id].append((candidate_idx, retry_idx))
+
+            for req_id, candidates in request_candidates.items():
+                # 提取所有不同的 candidate_index
+                unique_candidates = set(c[0] for c in candidates)
+                # 如果有多个不同的 candidate_index，说明发生了 Fallback（Provider 切换）
+                fallback_map[req_id] = len(unique_candidates) > 1
+
+                # 检查是否有重试：同一个 candidate_index 有多个 retry_index
+                has_retry = False
+                for candidate_idx in unique_candidates:
+                    retry_indices = [c[1] for c in candidates if c[0] == candidate_idx]
+                    if len(retry_indices) > 1 or (retry_indices and max(retry_indices) > 0):
+                        has_retry = True
+                        break
+                retry_map[req_id] = has_retry
 
         context.add_audit_metadata(
             action="usage_records",
@@ -809,6 +833,7 @@ class AdminUsageRecordsAdapter(AdminApiAdapter):
                     "error_message": usage.error_message,
                     "status": usage.status,  # 请求状态: pending, streaming, completed, failed
                     "has_fallback": fallback_map.get(usage.request_id, False),
+                    "has_retry": retry_map.get(usage.request_id, False),
                     "api_format": usage.api_format
                     or (endpoint.api_format if endpoint and endpoint.api_format else None),
                     "api_key_name": provider_api_key.name if provider_api_key else None,

src/api/dashboard/routes.py

@@ -11,6 +11,7 @@ from sqlalchemy.orm import Session
 from src.api.base.adapter import ApiAdapter, ApiMode
 from src.api.base.admin_adapter import AdminApiAdapter
 from src.api.base.pipeline import ApiRequestPipeline
+from src.config.constants import CacheTTL
 from src.core.enums import UserRole
 from src.database import get_db
 from src.models.database import ApiKey, Provider, RequestCandidate, StatsDaily, StatsDailyModel, Usage
@@ -177,7 +178,7 @@ class DashboardStatsAdapter(DashboardAdapter):
 
 
 class AdminDashboardStatsAdapter(AdminApiAdapter):
-    @cache_result(key_prefix="dashboard:admin:stats", ttl=60, user_specific=False)
+    @cache_result(key_prefix="dashboard:admin:stats", ttl=CacheTTL.DASHBOARD_STATS, user_specific=False)
     async def handle(self, context):  # type: ignore[override]
         """管理员仪表盘统计 - 使用预聚合数据优化性能"""
         from zoneinfo import ZoneInfo
@@ -786,7 +787,7 @@ class DashboardProviderStatusAdapter(DashboardAdapter):
 class DashboardDailyStatsAdapter(DashboardAdapter):
     days: int
 
-    @cache_result(key_prefix="dashboard:daily:stats", ttl=300, user_specific=True)
+    @cache_result(key_prefix="dashboard:daily:stats", ttl=CacheTTL.DASHBOARD_DAILY, user_specific=True)
     async def handle(self, context):  # type: ignore[override]
         from zoneinfo import ZoneInfo
         from src.services.system.stats_aggregator import APP_TIMEZONE

src/api/handlers/base/base_handler.py

@@ -432,6 +432,7 @@ class BaseMessageHandler:
         endpoint_id = ctx.endpoint_id
         key_id = ctx.key_id
         first_byte_time_ms = ctx.first_byte_time_ms
+        api_format = ctx.api_format
 
         # 如果 provider 为空，记录警告（不应该发生，但用于调试）
         if not provider:
@@ -455,6 +456,7 @@ class BaseMessageHandler:
                         provider_endpoint_id=endpoint_id,
                         provider_api_key_id=key_id,
                         first_byte_time_ms=first_byte_time_ms,
+                        api_format=api_format,
                     )
                 finally:
                     db.close()

src/api/handlers/base/cli_handler_base.py

@@ -1432,6 +1432,16 @@ class CliMessageHandlerBase(BaseMessageHandler):
                 if ctx.attempt_id:
                     from src.services.request.candidate import RequestCandidateService
 
+                    # 计算候选自身的 TTFB
+                    candidate_first_byte_time_ms: Optional[int] = None
+                    if ctx.first_byte_time_ms is not None:
+                        candidate_first_byte_time_ms = RequestCandidateService.calculate_candidate_ttfb(
+                            db=bg_db,
+                            candidate_id=ctx.attempt_id,
+                            request_start_time=self.start_time,
+                            global_first_byte_time_ms=ctx.first_byte_time_ms,
+                        )
+
                     # 根据状态码决定是成功还是失败
                     # 499 = 客户端断开连接，应标记为失败
                     # 503 = 服务不可用（如流中断），应标记为失败
@@ -1443,8 +1453,8 @@ class CliMessageHandlerBase(BaseMessageHandler):
                             "chunk_count": ctx.chunk_count,
                             "data_count": ctx.data_count,
                         }
-                        if ctx.first_byte_time_ms is not None:
-                            extra_data["first_byte_time_ms"] = ctx.first_byte_time_ms
+                        if candidate_first_byte_time_ms is not None:
+                            extra_data["first_byte_time_ms"] = candidate_first_byte_time_ms
                         RequestCandidateService.mark_candidate_failed(
                             db=bg_db,
                             candidate_id=ctx.attempt_id,
@@ -1462,8 +1472,8 @@ class CliMessageHandlerBase(BaseMessageHandler):
                             "chunk_count": ctx.chunk_count,
                             "data_count": ctx.data_count,
                         }
-                        if ctx.first_byte_time_ms is not None:
-                            extra_data["first_byte_time_ms"] = ctx.first_byte_time_ms
+                        if candidate_first_byte_time_ms is not None:
+                            extra_data["first_byte_time_ms"] = candidate_first_byte_time_ms
                         RequestCandidateService.mark_candidate_success(
                             db=bg_db,
                             candidate_id=ctx.attempt_id,

src/api/handlers/base/stream_telemetry.py

@@ -132,7 +132,7 @@ class StreamTelemetryRecorder:
                     )
 
                 # 更新候选记录状态
-                await self._update_candidate_status(bg_db, ctx, response_time_ms)
+                await self._update_candidate_status(bg_db, ctx, response_time_ms, start_time)
 
             finally:
                 if bg_db:
@@ -234,6 +234,7 @@ class StreamTelemetryRecorder:
         db: Session,
         ctx: StreamContext,
         response_time_ms: int,
+        request_start_time: float,
     ) -> None:
         """更新候选记录状态"""
         if not ctx.attempt_id:
@@ -246,7 +247,14 @@ class StreamTelemetryRecorder:
             "data_count": ctx.data_count,
         }
         if ctx.first_byte_time_ms is not None:
-            extra_data["first_byte_time_ms"] = ctx.first_byte_time_ms
+            # 计算候选自身的 TTFB
+            first_byte_time_ms = RequestCandidateService.calculate_candidate_ttfb(
+                db=db,
+                candidate_id=ctx.attempt_id,
+                request_start_time=request_start_time,
+                global_first_byte_time_ms=ctx.first_byte_time_ms,
+            )
+            extra_data["first_byte_time_ms"] = first_byte_time_ms
 
         if ctx.is_success():
             RequestCandidateService.mark_candidate_success(

src/config/constants.py

@@ -21,8 +21,12 @@ class CacheTTL:
     # L1 本地缓存（用于减少 Redis 访问）
     L1_LOCAL = 3  # 3秒
 
-    # 活跃度热力图缓存 - 历史数据变化不频繁
-    ACTIVITY_HEATMAP = 300  # 5分钟
+    # 活跃度热力图缓存 - 历史数据变化不频繁，查询成本高
+    ACTIVITY_HEATMAP = 600  # 10分钟
+
+    # 仪表盘统计缓存
+    DASHBOARD_STATS = 120  # 2分钟（管理员）
+    DASHBOARD_DAILY = 600  # 10分钟（每日统计）
 
     # 并发锁 TTL - 防止死锁
     CONCURRENCY_LOCK = 600  # 10分钟

src/main.py

@@ -206,6 +206,11 @@ async def lifespan(app: FastAPI):
     task_scheduler = get_scheduler()
     task_scheduler.start()
 
+    # 启动缓存预热（后台任务，不阻塞启动）
+    from src.services.system.cache_warmup import start_cache_warmup
+
+    await start_cache_warmup()
+
     yield  # 应用运行期间
 
     # 关闭时执行

src/services/cache/provider_cache.py

@@ -25,6 +25,41 @@ class ProviderCacheService:
 
     CACHE_TTL = CacheTTL.PROVIDER  # 5 分钟
 
+    @staticmethod
+    def compute_rate_multiplier(
+        rate_multiplier: Optional[float],
+        rate_multipliers: Optional[dict],
+        api_format: Optional[str] = None,
+    ) -> float:
+        """
+        计算 rate_multiplier 的纯函数（无数据库/缓存依赖）
+
+        优先返回指定 API 格式的倍率，如果没有则返回默认倍率。
+        规则：
+        - 如果指定了 api_format 且 rate_multipliers 存在：
+          - 如果 rate_multipliers[api_format] 存在，返回它
+          - 否则返回 1.0（rate_multipliers 存在但该格式未配置）
+        - 否则返回 rate_multiplier 或 1.0
+
+        Args:
+            rate_multiplier: 默认倍率
+            rate_multipliers: 按 API 格式的倍率配置字典
+            api_format: API 格式（可选），如 "CLAUDE"、"OPENAI"
+
+        Returns:
+            计算后的 rate_multiplier
+        """
+        if api_format and rate_multipliers:
+            format_upper = api_format.upper()
+            if format_upper in rate_multipliers:
+                return float(rate_multipliers[format_upper])
+            else:
+                # rate_multipliers 存在但该格式未配置，使用默认值 1.0
+                return 1.0
+        else:
+            # rate_multipliers 不存在或未指定 api_format，回退到默认倍率
+            return rate_multiplier or 1.0
+
     @staticmethod
     async def get_provider_api_key_rate_multiplier(
         db: Session, provider_api_key_id: str, api_format: Optional[str] = None
@@ -64,19 +99,9 @@ class ProviderCacheService:
 
         # 3. 计算倍率并写入缓存
         if provider_key:
-            # 优先使用 rate_multipliers[api_format]
-            # 如果 rate_multipliers 存在但未配置该格式，默认为 1.0
-            # 只有当 rate_multipliers 完全不存在时，才回退到 rate_multiplier
-            if api_format and provider_key.rate_multipliers:
-                format_upper = api_format.upper()
-                if format_upper in provider_key.rate_multipliers:
-                    rate_multiplier = provider_key.rate_multipliers[format_upper]
-                else:
-                    # rate_multipliers 存在但该格式未配置，使用默认值 1.0
-                    rate_multiplier = 1.0
-            else:
-                # rate_multipliers 不存在或未指定 api_format，回退到默认倍率
-                rate_multiplier = provider_key.rate_multiplier or 1.0
+            rate_multiplier = ProviderCacheService.compute_rate_multiplier(
+                provider_key.rate_multiplier, provider_key.rate_multipliers, api_format
+            )
 
             await CacheService.set(
                 cache_key, rate_multiplier, ttl_seconds=ProviderCacheService.CACHE_TTL

src/services/request/candidate.py

@@ -9,6 +9,7 @@ from typing import List, Optional
 from sqlalchemy.orm import Session
 
 from src.core.batch_committer import get_batch_committer
+from src.core.logger import logger
 from src.models.database import RequestCandidate
 
 
@@ -289,3 +290,40 @@ class RequestCandidateService:
             "available_count": available_count,  # 新增：尚未被调度的候选数
             "failure_rate": round(failure_rate, 2),
         }
+
+    @staticmethod
+    def calculate_candidate_ttfb(
+        db: Session,
+        candidate_id: str,
+        request_start_time: float,
+        global_first_byte_time_ms: int,
+    ) -> int:
+        """
+        计算候选自身的首字节时间 (TTFB)
+
+        请求链路追踪中的 TTFB 应该是"该候选自身"的首字时间，
+        而不是整个请求从开始到收到首字节的时间。
+
+        Args:
+            db: 数据库会话
+            candidate_id: 候选 ID
+            request_start_time: 请求开始时间（Unix timestamp，秒）
+            global_first_byte_time_ms: 全局首字节时间（相对于 request_start_time 的毫秒数）
+
+        Returns:
+            候选自身的 TTFB（毫秒），如果计算失败则返回 global_first_byte_time_ms
+        """
+        try:
+            candidate = db.query(RequestCandidate).filter(RequestCandidate.id == candidate_id).first()
+            if candidate and candidate.started_at:
+                started_at = candidate.started_at
+                if started_at.tzinfo is None:
+                    started_at = started_at.replace(tzinfo=timezone.utc)
+                # 使用整数毫秒计算，避免浮点精度问题
+                request_start_epoch_ms = round(request_start_time * 1000)
+                started_at_epoch_ms = round(started_at.timestamp() * 1000)
+                first_byte_epoch_ms = request_start_epoch_ms + global_first_byte_time_ms
+                return max(0, int(first_byte_epoch_ms - started_at_epoch_ms))
+        except Exception as e:
+            logger.debug(f"计算候选 TTFB 失败: {e}")
+        return global_first_byte_time_ms

src/services/system/cache_warmup.py

@@ -0,0 +1,159 @@
+"""
+缓存预热服务
+
+在应用启动时预热关键缓存，避免用户首次访问时的长时间等待。
+
+预热的缓存包括：
+- 管理员仪表盘统计数据
+- 管理员热力图数据
+- 每日统计数据
+"""
+
+import asyncio
+import time
+from dataclasses import dataclass, field
+from typing import Any, Dict
+
+from sqlalchemy.orm import Session
+
+from src.core.logger import logger
+from src.database import create_session
+
+
+@dataclass
+class WarmupContext:
+    """缓存预热专用的简化 Context"""
+
+    db: Session
+    user: Any  # User model
+    audit_metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def add_audit_metadata(self, **kwargs: Any) -> None:
+        """兼容 ApiRequestContext 接口"""
+        self.audit_metadata.update(kwargs)
+
+
+class CacheWarmupService:
+    """缓存预热服务"""
+
+    @classmethod
+    async def warmup_all(cls, delay_seconds: float = 3.0) -> None:
+        """
+        预热所有关键缓存
+
+        Args:
+            delay_seconds: 启动后延迟执行的秒数，确保系统完全就绪
+        """
+        await asyncio.sleep(delay_seconds)
+
+        logger.info("开始预热关键缓存...")
+        start_time = time.time()
+
+        results = await asyncio.gather(
+            cls._warmup_admin_dashboard_stats(),
+            cls._warmup_admin_heatmap(),
+            cls._warmup_daily_stats(),
+            return_exceptions=True,
+        )
+
+        success_count = sum(1 for r in results if r is True)
+        error_count = sum(1 for r in results if isinstance(r, Exception))
+        elapsed = time.time() - start_time
+
+        if error_count > 0:
+            logger.warning(f"缓存预热完成: {success_count}/3 成功, {error_count} 失败, 耗时 {elapsed:.2f}s")
+        else:
+            logger.info(f"缓存预热完成: {success_count}/3 成功, 耗时 {elapsed:.2f}s")
+
+    @classmethod
+    async def _warmup_admin_dashboard_stats(cls) -> bool:
+        """预热管理员仪表盘统计缓存"""
+        db = None
+        try:
+            from src.api.dashboard.routes import AdminDashboardStatsAdapter
+            from src.models.database import User as DBUser
+
+            db = create_session()
+
+            # 获取一个管理员用户用于构造 context
+            admin_user = db.query(DBUser).filter(DBUser.role == "admin").first()
+            if not admin_user:
+                logger.info("缓存预热: 无管理员用户，跳过仪表盘统计预热")
+                return True
+
+            context = WarmupContext(db=db, user=admin_user)
+            adapter = AdminDashboardStatsAdapter()
+            await adapter.handle(context)
+
+            logger.debug("缓存预热: 管理员仪表盘统计已预热")
+            return True
+
+        except Exception as e:
+            logger.warning(f"缓存预热失败 (仪表盘统计): {e}")
+            return False
+        finally:
+            if db:
+                db.close()
+
+    @classmethod
+    async def _warmup_admin_heatmap(cls) -> bool:
+        """预热管理员热力图缓存"""
+        db = None
+        try:
+            from src.services.usage.service import UsageService
+
+            db = create_session()
+
+            # 预热全局热力图（管理员视角）
+            await UsageService.get_cached_heatmap(
+                db=db,
+                user_id=None,
+                include_actual_cost=True,
+            )
+
+            logger.debug("缓存预热: 管理员热力图已预热")
+            return True
+
+        except Exception as e:
+            logger.warning(f"缓存预热失败 (热力图): {e}")
+            return False
+        finally:
+            if db:
+                db.close()
+
+    @classmethod
+    async def _warmup_daily_stats(cls) -> bool:
+        """预热每日统计缓存"""
+        db = None
+        try:
+            from src.api.dashboard.routes import DashboardDailyStatsAdapter
+            from src.models.database import User as DBUser
+
+            db = create_session()
+
+            # 获取一个管理员用户
+            admin_user = db.query(DBUser).filter(DBUser.role == "admin").first()
+            if not admin_user:
+                logger.info("缓存预热: 无管理员用户，跳过每日统计预热")
+                return True
+
+            context = WarmupContext(db=db, user=admin_user)
+
+            # 预热 7 天的每日统计
+            adapter = DashboardDailyStatsAdapter(days=7)
+            await adapter.handle(context)
+
+            logger.debug("缓存预热: 每日统计已预热")
+            return True
+
+        except Exception as e:
+            logger.warning(f"缓存预热失败 (每日统计): {e}")
+            return False
+        finally:
+            if db:
+                db.close()
+
+
+async def start_cache_warmup() -> None:
+    """启动缓存预热（作为后台任务）"""
+    asyncio.create_task(CacheWarmupService.warmup_all())

src/services/usage/service.py

@@ -1485,6 +1485,7 @@ class UsageService:
         provider_id: Optional[str] = None,
         provider_endpoint_id: Optional[str] = None,
         provider_api_key_id: Optional[str] = None,
+        api_format: Optional[str] = None,
     ) -> Optional[Usage]:
         """
         快速更新使用记录状态
@@ -1500,6 +1501,7 @@ class UsageService:
             provider_id: Provider ID（可选，streaming 状态时更新）
             provider_endpoint_id: Endpoint ID（可选，streaming 状态时更新）
             provider_api_key_id: Provider API Key ID（可选，streaming 状态时更新）
+            api_format: API 格式（可选，用于获取按格式配置的倍率）
 
         Returns:
             更新后的 Usage 记录，如果未找到则返回 None
@@ -1531,6 +1533,13 @@ class UsageService:
             usage.provider_endpoint_id = provider_endpoint_id
         if provider_api_key_id is not None:
             usage.provider_api_key_id = provider_api_key_id
+            # 当设置 provider_api_key_id 时，同步获取并更新 rate_multiplier
+            # 这样前端在 streaming 状态就能显示倍率
+            rate_multiplier = cls._get_rate_multiplier_sync(
+                db, provider_api_key_id, api_format or usage.api_format
+            )
+            if rate_multiplier is not None:
+                usage.rate_multiplier = rate_multiplier
 
         db.commit()
 
@@ -1538,6 +1547,38 @@ class UsageService:
 
         return usage
 
+    @staticmethod
+    def _get_rate_multiplier_sync(
+        db: Session,
+        provider_api_key_id: str,
+        api_format: Optional[str] = None,
+    ) -> Optional[float]:
+        """
+        同步获取 ProviderAPIKey 的 rate_multiplier
+
+        Args:
+            db: 数据库会话
+            provider_api_key_id: ProviderAPIKey ID
+            api_format: API 格式（可选），如 "CLAUDE"、"OPENAI"
+
+        Returns:
+            rate_multiplier 或 None
+        """
+        from src.services.cache.provider_cache import ProviderCacheService
+
+        provider_key = (
+            db.query(ProviderAPIKey.rate_multiplier, ProviderAPIKey.rate_multipliers)
+            .filter(ProviderAPIKey.id == provider_api_key_id)
+            .first()
+        )
+
+        if not provider_key:
+            return None
+
+        return ProviderCacheService.compute_rate_multiplier(
+            provider_key.rate_multiplier, provider_key.rate_multipliers, api_format
+        )
+
     @classmethod
     def get_active_requests(
         cls,

src/services/usage/stream.py

@@ -487,6 +487,7 @@ class StreamUsageTracker:
                             provider_id=self.provider_id,
                             provider_endpoint_id=self.provider_endpoint_id,
                             provider_api_key_id=self.provider_api_key_id,
+                            api_format=self.api_format,
                         )
                     except Exception as e:
                         logger.warning(f"更新使用记录状态为 streaming 失败: {e}")
@@ -948,6 +949,7 @@ class EnhancedStreamUsageTracker(StreamUsageTracker):
                             provider_id=self.provider_id,
                             provider_endpoint_id=self.provider_endpoint_id,
                             provider_api_key_id=self.provider_api_key_id,
+                            api_format=self.api_format,
                         )
                     except Exception as e:
                         logger.warning(f"更新使用记录状态为 streaming 失败: {e}")