""" 请求链路追踪 API 端点 """ from __future__ import annotations import json from dataclasses import dataclass from datetime import datetime from typing import Any from fastapi import APIRouter, Depends, HTTPException, Query, Request from pydantic import BaseModel, ConfigDict from sqlalchemy.orm import Session from src.api.base.admin_adapter import AdminApiAdapter from src.api.base.context import ApiRequestContext from src.api.base.pipeline import get_pipeline from src.core.crypto import crypto_service from src.database import get_db from src.models.database import Provider, ProviderAPIKey, ProviderEndpoint from src.services.request.candidate import RequestCandidateService router = APIRouter(prefix="/api/admin/monitoring/trace", tags=["Admin - Monitoring: Trace"]) pipeline = get_pipeline() class CandidateResponse(BaseModel): """候选记录响应""" id: str request_id: str candidate_index: int retry_index: int = 0 # 重试序号（从0开始） provider_id: str | None = None provider_name: str | None = None provider_website: str | None = None # Provider 官网 endpoint_id: str | None = None endpoint_name: str | None = None # 端点显示名称（api_format） key_id: str | None = None key_name: str | None = None # 密钥名称 key_account_label: str | None = None # 更适合展示的测试账号标签（优先 OAuth 邮箱） key_preview: str | None = None # 密钥脱敏预览（如 sk-***abc），OAuth 类型不返回 key_auth_type: str | None = None # 密钥认证类型（api_key, service_account, oauth） key_oauth_plan_type: str | None = None # OAuth 账号套餐类型（free/plus/team/enterprise） key_capabilities: dict | None = None # Key 支持的能力 required_capabilities: dict | None = None # 请求实际需要的能力标签 status: str # 'pending', 'success', 'failed', 'skipped' skip_reason: str | None = None is_cached: bool = False # 执行结果字段 status_code: int | None = None error_type: str | None = None error_message: str | None = None latency_ms: int | None = None concurrent_requests: int | None = None extra_data: dict | None = None created_at: datetime started_at: datetime | None = None finished_at: datetime | None = None model_config = ConfigDict(from_attributes=True) class RequestTraceResponse(BaseModel): """请求追踪完整响应""" request_id: str total_candidates: int final_status: str # 'success', 'failed', 'cancelled', 'streaming', 'pending' total_latency_ms: int candidates: list[CandidateResponse] @router.get("/{request_id}", response_model=RequestTraceResponse) async def get_request_trace( request_id: str, request: Request, attempted_only: bool = Query(False, description="仅返回实际尝试过的候选"), db: Session = Depends(get_db), ) -> Any: """ 获取请求的完整追踪信息 获取指定请求的完整链路追踪信息，包括所有候选（candidates）的执行情况。 **路径参数**: - `request_id`: 请求 ID **返回字段**: - `request_id`: 请求 ID - `total_candidates`: 候选总数 - `final_status`: 最终状态（success: 成功，failed: 失败，streaming: 流式传输中，pending: 等待中） - `total_latency_ms`: 总延迟（毫秒） - `candidates`: 候选列表，每个候选包含： - `id`: 候选 ID - `request_id`: 请求 ID - `candidate_index`: 候选索引 - `retry_index`: 重试序号 - `provider_id`: 提供商 ID - `provider_name`: 提供商名称 - `provider_website`: 提供商官网 - `endpoint_id`: 端点 ID - `endpoint_name`: 端点名称（API 格式） - `key_id`: 密钥 ID - `key_name`: 密钥名称 - `key_preview`: 密钥脱敏预览 - `key_capabilities`: 密钥支持的能力 - `required_capabilities`: 请求需要的能力标签 - `status`: 状态（pending, success, failed, skipped） - `skip_reason`: 跳过原因 - `is_cached`: 是否缓存命中 - `status_code`: HTTP 状态码 - `error_type`: 错误类型 - `error_message`: 错误信息 - `latency_ms`: 延迟（毫秒） - `concurrent_requests`: 并发请求数 - `extra_data`: 额外数据 - `created_at`: 创建时间 - `started_at`: 开始时间 - `finished_at`: 完成时间 """ adapter = AdminGetRequestTraceAdapter(request_id=request_id, attempted_only=attempted_only) return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode) @router.get("/stats/provider/{provider_id}") async def get_provider_failure_rate( provider_id: str, request: Request, limit: int = Query(100, ge=1, le=1000, description="统计最近的尝试数量"), db: Session = Depends(get_db), ) -> Any: """ 获取提供商的失败率统计 获取指定提供商最近的失败率统计信息。需要管理员权限。 **路径参数**: - `provider_id`: 提供商 ID **查询参数**: - `limit`: 统计最近的尝试数量，默认 100，最大 1000 **返回字段**: - `provider_id`: 提供商 ID - `total_attempts`: 总尝试次数 - `success_count`: 成功次数 - `failed_count`: 失败次数 - `failure_rate`: 失败率（百分比） - `avg_latency_ms`: 平均延迟（毫秒） """ adapter = AdminProviderFailureRateAdapter(provider_id=provider_id, limit=limit) return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode) # -------- 请求追踪适配器 -------- @dataclass class AdminGetRequestTraceAdapter(AdminApiAdapter): request_id: str attempted_only: bool = False @staticmethod def _is_attempted_candidate(candidate: Any) -> bool: status = str(getattr(candidate, "status", "") or "").strip().lower() # pre-created / never executed rows should not be considered as attempted if status in {"", "available", "unused", "skipped"}: return False # pending must have started_at to be considered truly entered execution if status == "pending" and getattr(candidate, "started_at", None) is None: return False return True async def handle(self, context: ApiRequestContext) -> Any: # type: ignore[override] db = context.db # 查询并展示该请求的全量候选： # - 包含 available/unused（未执行） # - 包含 skipped（跳过） # - 包含 pending/streaming/success/failed/cancelled（执行中/结果） all_candidates = RequestCandidateService.get_candidates_by_request_id(db, self.request_id) # 如果没有数据，返回 404 if not all_candidates: raise HTTPException(status_code=404, detail="Request not found") candidates = ( [c for c in all_candidates if self._is_attempted_candidate(c)] if self.attempted_only else all_candidates ) # 计算总延迟（只统计已完成的候选：success, failed, cancelled） # 使用显式的 is not None 检查，避免过滤掉 0ms 的快速响应 total_latency = sum( c.latency_ms for c in candidates if c.status in ("success", "failed", "cancelled") and c.latency_ms is not None ) # 判断最终状态： # 1. status="success" 即视为成功（无论 status_code 是什么） # - 流式请求即使客户端断开（499），只要 Provider 成功返回数据，也算成功 # 2. 同时检查 status_code 在 200-299 范围，作为额外的成功判断条件 # - 用于兼容非流式请求或未正确设置 status 的旧数据 # 3. status="streaming" 表示流式请求正在进行中 # 4. status="pending" 表示请求尚未开始执行 # 5. status="cancelled" 表示客户端主动断开连接（不算失败） final_status_source = all_candidates if self.attempted_only else candidates has_success = any( c.status == "success" or (c.status_code is not None and 200 <= c.status_code < 300) for c in final_status_source ) has_streaming = any(c.status == "streaming" for c in final_status_source) has_pending = any(c.status == "pending" for c in final_status_source) has_cancelled = any(c.status == "cancelled" for c in final_status_source) has_failed = any(c.status == "failed" for c in final_status_source) if has_success: final_status = "success" elif has_streaming: # 有候选正在流式传输中 final_status = "streaming" elif has_pending: # 有候选正在等待执行 final_status = "pending" elif has_cancelled and not has_failed: # 只有取消没有失败，算作取消 final_status = "cancelled" else: final_status = "failed" # 批量加载 provider 信息，避免 N+1 查询 provider_ids = {c.provider_id for c in candidates if c.provider_id} provider_map: dict[str, str] = {} provider_website_map: dict[str, str | None] = {} provider_type_map: dict[str, str] = {} if provider_ids: providers = db.query(Provider).filter(Provider.id.in_(provider_ids)).all() for p in providers: provider_map[p.id] = p.name provider_website_map[p.id] = p.website provider_type_map[p.id] = getattr(p, "provider_type", "custom") or "custom" # 批量加载 endpoint 信息 endpoint_ids = {c.endpoint_id for c in candidates if c.endpoint_id} endpoint_map = {} if endpoint_ids: endpoints = ( db.query(ProviderEndpoint).filter(ProviderEndpoint.id.in_(endpoint_ids)).all() ) endpoint_map = {e.id: e.api_format for e in endpoints} # 批量加载 key 信息 key_ids = {c.key_id for c in candidates if c.key_id} key_map: dict[str, str] = {} key_preview_map: dict[str, str] = {} key_account_label_map: dict[str, str | None] = {} key_capabilities_map: dict[str, dict | None] = {} key_auth_type_map: dict[str, str] = {} key_oauth_plan_map: dict[str, str | None] = {} # 建立 key_id -> provider_id 的映射（用于获取 provider_type） key_provider_map: dict[str, str | None] = { c.key_id: c.provider_id for c in candidates if c.key_id } if key_ids: keys = db.query(ProviderAPIKey).filter(ProviderAPIKey.id.in_(key_ids)).all() for k in keys: key_map[k.id] = k.name key_account_label_map[k.id] = k.name key_capabilities_map[k.id] = k.capabilities is_oauth = k.auth_type == "oauth" if is_oauth: # OAuth: auth_type 使用具体的 provider_type（如 kiro/codex/antigravity） pid = key_provider_map.get(k.id) key_auth_type_map[k.id] = ( provider_type_map.get(pid, "oauth") if pid else "oauth" ) # 提取 plan_type（不同 provider 存储位置不同） oauth_plan_type = None # 1. Codex: auth_config.plan_type # 2. Antigravity: auth_config.tier if k.auth_config: try: decrypted_config = crypto_service.decrypt(k.auth_config) auth_config = json.loads(decrypted_config) email = auth_config.get("email") if isinstance(email, str) and email.strip(): key_account_label_map[k.id] = email.strip() oauth_plan_type = auth_config.get("plan_type") if not oauth_plan_type: ag_tier = auth_config.get("tier") if ag_tier and isinstance(ag_tier, str): oauth_plan_type = ag_tier.lower() except Exception: pass # 3. Kiro: upstream_metadata.kiro.subscription_title # subscription_title 通常为 "KIRO FREE" / "KIRO PRO+" 等， # 去掉 provider 名称前缀，只保留等级部分 if not oauth_plan_type: um = getattr(k, "upstream_metadata", None) or {} kiro_meta = um.get("kiro") if isinstance(um, dict) else None if isinstance(kiro_meta, dict): sub_title = kiro_meta.get("subscription_title") if sub_title and isinstance(sub_title, str): # "KIRO FREE" -> "Free", "KIRO PRO+" -> "Pro+" ptype = provider_type_map.get(pid, "") if pid else "" if ptype and sub_title.upper().startswith(ptype.upper()): sub_title = sub_title[len(ptype) :].strip() oauth_plan_type = sub_title key_oauth_plan_map[k.id] = oauth_plan_type continue else: key_auth_type_map[k.id] = k.auth_type or "api_key" # 非 OAuth：生成脱敏预览 try: decrypted_key = crypto_service.decrypt(k.api_key) if len(decrypted_key) > 8: # 检测常见前缀模式 prefix_end = 0 for prefix in ["sk-", "key-", "api-", "ak-"]: if decrypted_key.lower().startswith(prefix): prefix_end = len(prefix) break if prefix_end > 0: key_preview_map[k.id] = ( f"{decrypted_key[:prefix_end]}***{decrypted_key[-4:]}" ) else: key_preview_map[k.id] = f"{decrypted_key[:4]}***{decrypted_key[-4:]}" elif len(decrypted_key) > 4: key_preview_map[k.id] = f"***{decrypted_key[-4:]}" else: key_preview_map[k.id] = "***" except Exception: key_preview_map[k.id] = "***" # 构建 candidate 响应列表 candidate_responses: list[CandidateResponse] = [] for candidate in candidates: provider_name = ( provider_map.get(candidate.provider_id) if candidate.provider_id else None ) provider_website = ( provider_website_map.get(candidate.provider_id) if candidate.provider_id else None ) endpoint_name = ( endpoint_map.get(candidate.endpoint_id) if candidate.endpoint_id else None ) key_name = key_map.get(candidate.key_id) if candidate.key_id else None key_account_label = ( key_account_label_map.get(candidate.key_id) if candidate.key_id else None ) key_preview = key_preview_map.get(candidate.key_id) if candidate.key_id else None key_auth_type = key_auth_type_map.get(candidate.key_id) if candidate.key_id else None key_oauth_plan_type = ( key_oauth_plan_map.get(candidate.key_id) if candidate.key_id else None ) key_capabilities = ( key_capabilities_map.get(candidate.key_id) if candidate.key_id else None ) candidate_responses.append( CandidateResponse( id=candidate.id, request_id=candidate.request_id, candidate_index=candidate.candidate_index, retry_index=candidate.retry_index, provider_id=candidate.provider_id, provider_name=provider_name, provider_website=provider_website, endpoint_id=candidate.endpoint_id, endpoint_name=endpoint_name, key_id=candidate.key_id, key_name=key_name, key_account_label=key_account_label, key_preview=key_preview, key_auth_type=key_auth_type, key_oauth_plan_type=key_oauth_plan_type, key_capabilities=key_capabilities, required_capabilities=candidate.required_capabilities, status=candidate.status, skip_reason=candidate.skip_reason, is_cached=candidate.is_cached, status_code=candidate.status_code, error_type=candidate.error_type, error_message=candidate.error_message, latency_ms=candidate.latency_ms, concurrent_requests=candidate.concurrent_requests, extra_data=candidate.extra_data, created_at=candidate.created_at, started_at=candidate.started_at, finished_at=candidate.finished_at, ) ) response = RequestTraceResponse( request_id=self.request_id, total_candidates=len(candidates), final_status=final_status, total_latency_ms=total_latency, candidates=candidate_responses, ) context.add_audit_metadata( action="trace_request_detail", request_id=self.request_id, total_candidates=len(candidates), final_status=final_status, total_latency_ms=total_latency, attempted_only=self.attempted_only, ) return response @dataclass class AdminProviderFailureRateAdapter(AdminApiAdapter): provider_id: str limit: int async def handle(self, context: ApiRequestContext) -> Any: # type: ignore[override] result = RequestCandidateService.get_candidate_stats_by_provider( db=context.db, provider_id=self.provider_id, limit=self.limit, ) context.add_audit_metadata( action="trace_provider_failure_rate", provider_id=self.provider_id, limit=self.limit, total_attempts=result.get("total_attempts"), failure_rate=result.get("failure_rate"), ) return result