mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 09:20:22 +08:00
refactor: 移除 Python 后端源码，全面迁移至 Rust gateway 架构
- 删除全部 Python 源码 (src/) 及 Alembic 迁移脚本，归档至 _deprecated_py_src/ - 重构 Rust gateway ai_pipeline: 拆分 planner/finalize 模块，新增 contracts/adaptation 层 - 重组 handlers 模块为 admin/public/proxy/internal/shared 子模块结构 - 新增 executor 模块，引入 Rust 原生数据库迁移 (aether-data/migrations) - 简化 CI/Docker 构建流程，移除 base image 二级构建，统一为单一 app image - 移除 Python 相关基础设施文件 (entrypoint.sh, gunicorn_conf.py, Dockerfile.base)
This commit is contained in:
fawney19
14
_deprecated_py_src/api/admin/monitoring/__init__.py
Normal file
14
_deprecated_py_src/api/admin/monitoring/__init__.py
Normal file
@@ -0,0 +1,14 @@
|
||||
"""Admin monitoring router合集。"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from .audit import router as audit_router
|
||||
from .cache import router as cache_router
|
||||
from .trace import router as trace_router
|
||||
|
||||
router = APIRouter()
|
||||
router.include_router(audit_router)
|
||||
router.include_router(cache_router)
|
||||
router.include_router(trace_router)
|
||||
|
||||
__all__ = ["router"]
|
||||
551
_deprecated_py_src/api/admin/monitoring/audit.py
Normal file
551
_deprecated_py_src/api/admin/monitoring/audit.py
Normal file
@@ -0,0 +1,551 @@
|
||||
"""管理员监控与审计端点。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from sqlalchemy import case, func
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from src.api.base.admin_adapter import AdminApiAdapter
|
||||
from src.api.base.context import ApiRequestContext
|
||||
from src.api.base.pagination import PaginationMeta, build_pagination_payload
|
||||
from src.api.base.pipeline import get_pipeline
|
||||
from src.config.constants import CacheTTL
|
||||
from src.core.logger import logger
|
||||
from src.database import get_db
|
||||
from src.models.database import (
|
||||
ApiKey,
|
||||
AuditEventType,
|
||||
AuditLog,
|
||||
Provider,
|
||||
Usage,
|
||||
)
|
||||
from src.models.database import User as DBUser
|
||||
from src.services.health.monitor import HealthMonitor
|
||||
from src.services.system.audit import audit_service
|
||||
from src.utils.cache_decorator import cache_result
|
||||
from src.utils.database_helpers import escape_like_pattern
|
||||
|
||||
router = APIRouter(prefix="/api/admin/monitoring", tags=["Admin - Monitoring"])
|
||||
pipeline = get_pipeline()
|
||||
|
||||
|
||||
@router.get("/audit-logs")
|
||||
async def get_audit_logs(
|
||||
request: Request,
|
||||
username: str | None = Query(None, description="用户名筛选 (模糊匹配)"),
|
||||
event_type: str | None = Query(None, description="事件类型筛选"),
|
||||
days: int = Query(7, description="查询天数"),
|
||||
limit: int = Query(100, description="返回数量限制"),
|
||||
offset: int = Query(0, description="偏移量"),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
获取审计日志
|
||||
|
||||
获取系统审计日志列表，支持按用户名、事件类型、时间范围筛选。需要管理员权限。
|
||||
|
||||
**查询参数**:
|
||||
- `username`: 可选，用户名筛选（模糊匹配）
|
||||
- `event_type`: 可选，事件类型筛选
|
||||
- `days`: 查询最近多少天的日志，默认 7 天
|
||||
- `limit`: 返回数量限制，默认 100
|
||||
- `offset`: 分页偏移量，默认 0
|
||||
|
||||
**返回字段**:
|
||||
- `items`: 审计日志列表，每条日志包含：
|
||||
- `id`: 日志 ID
|
||||
- `event_type`: 事件类型
|
||||
- `user_id`: 用户 ID
|
||||
- `user_email`: 用户邮箱
|
||||
- `user_username`: 用户名
|
||||
- `description`: 事件描述
|
||||
- `ip_address`: IP 地址
|
||||
- `status_code`: HTTP 状态码
|
||||
- `error_message`: 错误信息
|
||||
- `metadata`: 事件元数据
|
||||
- `created_at`: 创建时间
|
||||
- `meta`: 分页元数据（total, limit, offset, count）
|
||||
- `filters`: 筛选条件
|
||||
"""
|
||||
adapter = AdminGetAuditLogsAdapter(
|
||||
username=username,
|
||||
event_type=event_type,
|
||||
days=days,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode)
|
||||
|
||||
|
||||
@router.get("/system-status")
|
||||
async def get_system_status(request: Request, db: Session = Depends(get_db)) -> Any:
|
||||
"""
|
||||
获取系统状态
|
||||
|
||||
获取系统当前的运行状态和关键指标。需要管理员权限。
|
||||
|
||||
**返回字段**:
|
||||
- `timestamp`: 当前时间戳
|
||||
- `users`: 用户统计（total: 总用户数, active: 活跃用户数）
|
||||
- `providers`: 提供商统计（total: 总提供商数, active: 活跃提供商数）
|
||||
- `api_keys`: API Key 统计（total: 总数, active: 活跃数）
|
||||
- `today_stats`: 今日统计（requests: 请求数, tokens: token 数, cost_usd: 成本）
|
||||
- `recent_errors`: 最近 1 小时内的错误数
|
||||
"""
|
||||
adapter = AdminSystemStatusAdapter()
|
||||
return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode)
|
||||
|
||||
|
||||
@router.get("/suspicious-activities")
|
||||
async def get_suspicious_activities(
|
||||
request: Request,
|
||||
hours: int = Query(24, description="时间范围（小时）"),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
获取可疑活动记录
|
||||
|
||||
获取系统检测到的可疑活动记录。需要管理员权限。
|
||||
|
||||
**查询参数**:
|
||||
- `hours`: 时间范围（小时），默认 24 小时
|
||||
|
||||
**返回字段**:
|
||||
- `activities`: 可疑活动列表，每条记录包含：
|
||||
- `id`: 记录 ID
|
||||
- `event_type`: 事件类型
|
||||
- `user_id`: 用户 ID
|
||||
- `description`: 事件描述
|
||||
- `ip_address`: IP 地址
|
||||
- `metadata`: 事件元数据
|
||||
- `created_at`: 创建时间
|
||||
- `count`: 活动总数
|
||||
- `time_range_hours`: 查询的时间范围（小时）
|
||||
"""
|
||||
adapter = AdminSuspiciousActivitiesAdapter(hours=hours)
|
||||
return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode)
|
||||
|
||||
|
||||
@router.get("/user-behavior/{user_id}")
|
||||
async def analyze_user_behavior(
|
||||
user_id: str,
|
||||
request: Request,
|
||||
days: int = Query(30, description="分析天数"),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
分析用户行为
|
||||
|
||||
分析指定用户的行为模式和使用情况。需要管理员权限。
|
||||
|
||||
**路径参数**:
|
||||
- `user_id`: 用户 ID
|
||||
|
||||
**查询参数**:
|
||||
- `days`: 分析最近多少天的数据，默认 30 天
|
||||
|
||||
**返回字段**:
|
||||
- 用户行为分析结果，包括活动频率、使用模式、异常行为等
|
||||
"""
|
||||
adapter = AdminUserBehaviorAdapter(user_id=user_id, days=days)
|
||||
return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode)
|
||||
|
||||
|
||||
@router.get("/resilience-status")
|
||||
async def get_resilience_status(request: Request, db: Session = Depends(get_db)) -> Any:
|
||||
"""
|
||||
获取韧性系统状态
|
||||
|
||||
获取系统韧性管理的当前状态，包括错误统计、熔断器状态等。需要管理员权限。
|
||||
|
||||
**返回字段**:
|
||||
- `timestamp`: 当前时间戳
|
||||
- `health_score`: 健康评分（0-100）
|
||||
- `status`: 系统状态（healthy: 健康，degraded: 降级，critical: 严重）
|
||||
- `error_statistics`: 错误统计信息
|
||||
- `recent_errors`: 最近的错误列表（最多 10 条）
|
||||
- `recommendations`: 系统建议
|
||||
"""
|
||||
adapter = AdminResilienceStatusAdapter()
|
||||
return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode)
|
||||
|
||||
|
||||
@router.delete("/resilience/error-stats")
|
||||
async def reset_error_stats(request: Request, db: Session = Depends(get_db)) -> None:
|
||||
"""
|
||||
重置错误统计
|
||||
|
||||
重置韧性系统的错误统计数据。需要管理员权限。
|
||||
|
||||
**返回字段**:
|
||||
- `message`: 操作结果信息
|
||||
- `previous_stats`: 重置前的统计数据
|
||||
- `reset_by`: 执行重置的管理员邮箱
|
||||
- `reset_at`: 重置时间
|
||||
"""
|
||||
adapter = AdminResetErrorStatsAdapter()
|
||||
return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode)
|
||||
|
||||
|
||||
@router.get("/resilience/circuit-history")
|
||||
async def get_circuit_history(
|
||||
request: Request,
|
||||
limit: int = Query(50, ge=1, le=200),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
获取熔断器历史记录
|
||||
|
||||
获取熔断器的状态变更历史记录。需要管理员权限。
|
||||
|
||||
**查询参数**:
|
||||
- `limit`: 返回数量限制，默认 50，最大 200
|
||||
|
||||
**返回字段**:
|
||||
- `items`: 熔断器历史记录列表
|
||||
- `count`: 记录总数
|
||||
"""
|
||||
adapter = AdminCircuitHistoryAdapter(limit=limit)
|
||||
return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AdminGetAuditLogsAdapter(AdminApiAdapter):
|
||||
username: str | None
|
||||
event_type: str | None
|
||||
days: int
|
||||
limit: int
|
||||
offset: int
|
||||
|
||||
# 查看审计日志本身不应该产生审计记录，避免刷新页面时产生大量无意义的日志
|
||||
audit_log_enabled: bool = False
|
||||
|
||||
@cache_result(
|
||||
key_prefix="admin:monitoring:audit-logs",
|
||||
ttl=CacheTTL.ADMIN_USAGE_RECORDS,
|
||||
user_specific=False,
|
||||
vary_by=["username", "event_type", "days", "limit", "offset"],
|
||||
)
|
||||
async def handle(self, context: ApiRequestContext) -> Any: # type: ignore[override]
|
||||
db = context.db
|
||||
cutoff_time = datetime.now(timezone.utc) - timedelta(days=self.days)
|
||||
|
||||
count_query = db.query(func.count(AuditLog.id)).filter(AuditLog.created_at >= cutoff_time)
|
||||
if self.username:
|
||||
escaped = escape_like_pattern(self.username)
|
||||
count_query = count_query.outerjoin(DBUser, AuditLog.user_id == DBUser.id).filter(
|
||||
DBUser.username.ilike(f"%{escaped}%", escape="\\")
|
||||
)
|
||||
if self.event_type:
|
||||
count_query = count_query.filter(AuditLog.event_type == self.event_type)
|
||||
total = int(count_query.scalar() or 0)
|
||||
|
||||
base_query = (
|
||||
db.query(AuditLog, DBUser)
|
||||
.outerjoin(DBUser, AuditLog.user_id == DBUser.id)
|
||||
.filter(AuditLog.created_at >= cutoff_time)
|
||||
)
|
||||
if self.username:
|
||||
escaped = escape_like_pattern(self.username)
|
||||
base_query = base_query.filter(DBUser.username.ilike(f"%{escaped}%", escape="\\"))
|
||||
if self.event_type:
|
||||
base_query = base_query.filter(AuditLog.event_type == self.event_type)
|
||||
|
||||
logs_with_users = (
|
||||
base_query.order_by(AuditLog.created_at.desc())
|
||||
.offset(self.offset)
|
||||
.limit(self.limit)
|
||||
.all()
|
||||
)
|
||||
|
||||
items = [
|
||||
{
|
||||
"id": log.id,
|
||||
"event_type": log.event_type,
|
||||
"user_id": log.user_id,
|
||||
"user_email": user.email if user else None,
|
||||
"user_username": user.username if user else None,
|
||||
"description": log.description,
|
||||
"ip_address": log.ip_address,
|
||||
"status_code": log.status_code,
|
||||
"error_message": log.error_message,
|
||||
"metadata": log.event_metadata,
|
||||
"created_at": log.created_at.isoformat() if log.created_at else None,
|
||||
}
|
||||
for log, user in logs_with_users
|
||||
]
|
||||
meta = PaginationMeta(
|
||||
total=total,
|
||||
limit=self.limit,
|
||||
offset=self.offset,
|
||||
count=len(items),
|
||||
)
|
||||
|
||||
payload = build_pagination_payload(
|
||||
items,
|
||||
meta,
|
||||
filters={
|
||||
"username": self.username,
|
||||
"event_type": self.event_type,
|
||||
"days": self.days,
|
||||
},
|
||||
)
|
||||
context.add_audit_metadata(
|
||||
action="monitor_audit_logs",
|
||||
filter_username=self.username,
|
||||
filter_event_type=self.event_type,
|
||||
days=self.days,
|
||||
limit=self.limit,
|
||||
offset=self.offset,
|
||||
total=total,
|
||||
result_count=meta.count,
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
class AdminSystemStatusAdapter(AdminApiAdapter):
|
||||
@cache_result(
|
||||
key_prefix="admin:monitoring:system-status",
|
||||
ttl=CacheTTL.ADMIN_USAGE_RECORDS,
|
||||
user_specific=False,
|
||||
)
|
||||
async def handle(self, context: ApiRequestContext) -> Any: # type: ignore[override]
|
||||
db = context.db
|
||||
|
||||
user_stats = db.query(
|
||||
func.count(DBUser.id).label("total"),
|
||||
func.sum(case((DBUser.is_active.is_(True), 1), else_=0)).label("active"),
|
||||
).first()
|
||||
total_users = int((user_stats.total if user_stats else 0) or 0)
|
||||
active_users = int((user_stats.active if user_stats else 0) or 0)
|
||||
|
||||
provider_stats = db.query(
|
||||
func.count(Provider.id).label("total"),
|
||||
func.sum(case((Provider.is_active.is_(True), 1), else_=0)).label("active"),
|
||||
).first()
|
||||
total_providers = int((provider_stats.total if provider_stats else 0) or 0)
|
||||
active_providers = int((provider_stats.active if provider_stats else 0) or 0)
|
||||
|
||||
api_key_stats = db.query(
|
||||
func.count(ApiKey.id).label("total"),
|
||||
func.sum(case((ApiKey.is_active.is_(True), 1), else_=0)).label("active"),
|
||||
).first()
|
||||
total_api_keys = int((api_key_stats.total if api_key_stats else 0) or 0)
|
||||
active_api_keys = int((api_key_stats.active if api_key_stats else 0) or 0)
|
||||
|
||||
today_start = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
today_stats = (
|
||||
db.query(
|
||||
func.count(Usage.id).label("requests"),
|
||||
func.coalesce(func.sum(Usage.total_tokens), 0).label("tokens"),
|
||||
func.coalesce(func.sum(Usage.total_cost_usd), 0.0).label("cost"),
|
||||
)
|
||||
.filter(Usage.created_at >= today_start)
|
||||
.first()
|
||||
)
|
||||
today_requests = int((today_stats.requests if today_stats else 0) or 0)
|
||||
today_tokens = int((today_stats.tokens if today_stats else 0) or 0)
|
||||
today_cost = float((today_stats.cost if today_stats else 0.0) or 0.0)
|
||||
|
||||
recent_errors = (
|
||||
db.query(func.count(AuditLog.id))
|
||||
.filter(
|
||||
AuditLog.event_type.in_(
|
||||
[
|
||||
AuditEventType.REQUEST_FAILED.value,
|
||||
AuditEventType.SUSPICIOUS_ACTIVITY.value,
|
||||
]
|
||||
),
|
||||
AuditLog.created_at >= datetime.now(timezone.utc) - timedelta(hours=1),
|
||||
)
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
context.add_audit_metadata(
|
||||
action="system_status_snapshot",
|
||||
total_users=total_users,
|
||||
active_users=active_users,
|
||||
total_providers=total_providers,
|
||||
active_providers=active_providers,
|
||||
total_api_keys=total_api_keys,
|
||||
active_api_keys=active_api_keys,
|
||||
today_requests=today_requests,
|
||||
today_tokens=today_tokens,
|
||||
today_cost=today_cost,
|
||||
recent_errors=int(recent_errors or 0),
|
||||
)
|
||||
|
||||
return {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"users": {"total": total_users, "active": active_users},
|
||||
"providers": {"total": total_providers, "active": active_providers},
|
||||
"api_keys": {"total": total_api_keys, "active": active_api_keys},
|
||||
"today_stats": {
|
||||
"requests": today_requests,
|
||||
"tokens": today_tokens,
|
||||
"cost_usd": f"${today_cost:.4f}",
|
||||
},
|
||||
"recent_errors": recent_errors,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class AdminSuspiciousActivitiesAdapter(AdminApiAdapter):
|
||||
hours: int
|
||||
|
||||
async def handle(self, context: ApiRequestContext) -> Any: # type: ignore[override]
|
||||
db = context.db
|
||||
activities = audit_service.get_suspicious_activities(db=db, hours=self.hours, limit=100)
|
||||
response = {
|
||||
"activities": [
|
||||
{
|
||||
"id": activity.id,
|
||||
"event_type": activity.event_type,
|
||||
"user_id": activity.user_id,
|
||||
"description": activity.description,
|
||||
"ip_address": activity.ip_address,
|
||||
"metadata": activity.event_metadata,
|
||||
"created_at": activity.created_at.isoformat() if activity.created_at else None,
|
||||
}
|
||||
for activity in activities
|
||||
],
|
||||
"count": len(activities),
|
||||
"time_range_hours": self.hours,
|
||||
}
|
||||
context.add_audit_metadata(
|
||||
action="monitor_suspicious_activity",
|
||||
hours=self.hours,
|
||||
result_count=len(activities),
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
@dataclass
|
||||
class AdminUserBehaviorAdapter(AdminApiAdapter):
|
||||
user_id: str
|
||||
days: int
|
||||
|
||||
async def handle(self, context: ApiRequestContext) -> Any: # type: ignore[override]
|
||||
result = audit_service.analyze_user_behavior(
|
||||
db=context.db,
|
||||
user_id=self.user_id,
|
||||
days=self.days,
|
||||
)
|
||||
context.add_audit_metadata(
|
||||
action="monitor_user_behavior",
|
||||
target_user_id=self.user_id,
|
||||
days=self.days,
|
||||
contains_summary=bool(result),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
class AdminResilienceStatusAdapter(AdminApiAdapter):
|
||||
async def handle(self, context: ApiRequestContext) -> Any: # type: ignore[override]
|
||||
try:
|
||||
from src.core.resilience import resilience_manager
|
||||
except ImportError as exc:
|
||||
raise HTTPException(status_code=503, detail="韧性管理系统未启用") from exc
|
||||
|
||||
error_stats = resilience_manager.get_error_stats()
|
||||
recent_errors = [
|
||||
{
|
||||
"error_id": info["error_id"],
|
||||
"error_type": info["error_type"],
|
||||
"operation": info["operation"],
|
||||
"timestamp": info["timestamp"].isoformat(),
|
||||
"context": info.get("context", {}),
|
||||
}
|
||||
for info in resilience_manager.last_errors[-10:]
|
||||
]
|
||||
|
||||
total_errors = error_stats.get("total_errors", 0)
|
||||
circuit_breakers = error_stats.get("circuit_breakers", {})
|
||||
circuit_breakers_open = sum(
|
||||
1 for status in circuit_breakers.values() if status.get("state") == "open"
|
||||
)
|
||||
health_score = max(0, 100 - (total_errors * 2) - (circuit_breakers_open * 20))
|
||||
|
||||
response = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"health_score": health_score,
|
||||
"status": (
|
||||
"healthy" if health_score > 80 else "degraded" if health_score > 50 else "critical"
|
||||
),
|
||||
"error_statistics": error_stats,
|
||||
"recent_errors": recent_errors,
|
||||
"recommendations": _get_health_recommendations(error_stats, health_score),
|
||||
}
|
||||
context.add_audit_metadata(
|
||||
action="resilience_status",
|
||||
health_score=health_score,
|
||||
error_total=error_stats.get("total_errors") if isinstance(error_stats, dict) else None,
|
||||
open_circuit_breakers=circuit_breakers_open,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
class AdminResetErrorStatsAdapter(AdminApiAdapter):
|
||||
async def handle(self, context: ApiRequestContext) -> Any: # type: ignore[override]
|
||||
try:
|
||||
from src.core.resilience import resilience_manager
|
||||
except ImportError as exc:
|
||||
raise HTTPException(status_code=503, detail="韧性管理系统未启用") from exc
|
||||
|
||||
old_stats = resilience_manager.get_error_stats()
|
||||
resilience_manager.error_stats.clear()
|
||||
resilience_manager.last_errors.clear()
|
||||
|
||||
logger.info(f"管理员 {context.user.email if context.user else 'unknown'} 重置了错误统计")
|
||||
|
||||
context.add_audit_metadata(
|
||||
action="reset_error_stats",
|
||||
previous_total_errors=(
|
||||
old_stats.get("total_errors") if isinstance(old_stats, dict) else None
|
||||
),
|
||||
)
|
||||
|
||||
return {
|
||||
"message": "错误统计已重置",
|
||||
"previous_stats": old_stats,
|
||||
"reset_by": context.user.email if context.user else None,
|
||||
"reset_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
class AdminCircuitHistoryAdapter(AdminApiAdapter):
|
||||
def __init__(self, limit: int = 50):
|
||||
super().__init__()
|
||||
self.limit = limit
|
||||
|
||||
async def handle(self, context: ApiRequestContext) -> Any: # type: ignore[override]
|
||||
history = HealthMonitor.get_circuit_history(self.limit)
|
||||
context.add_audit_metadata(
|
||||
action="circuit_history",
|
||||
limit=self.limit,
|
||||
result_count=len(history),
|
||||
)
|
||||
return {"items": history, "count": len(history)}
|
||||
|
||||
|
||||
def _get_health_recommendations(error_stats: dict, health_score: int) -> list[str]:
|
||||
recommendations: list[str] = []
|
||||
if health_score < 50:
|
||||
recommendations.append("系统健康状况严重，请立即检查错误日志")
|
||||
if error_stats.get("total_errors", 0) > 100:
|
||||
recommendations.append("错误频率过高，建议检查系统配置和外部依赖")
|
||||
|
||||
circuit_breakers = error_stats.get("circuit_breakers", {})
|
||||
open_breakers = [k for k, v in circuit_breakers.items() if v.get("state") == "open"]
|
||||
if open_breakers:
|
||||
recommendations.append(f"以下服务熔断器已打开：{', '.join(open_breakers)}")
|
||||
|
||||
if health_score > 90:
|
||||
recommendations.append("系统运行良好")
|
||||
return recommendations
|
||||
1889
_deprecated_py_src/api/admin/monitoring/cache.py
Normal file
1889
_deprecated_py_src/api/admin/monitoring/cache.py
Normal file
File diff suppressed because it is too large
Load Diff
439
_deprecated_py_src/api/admin/monitoring/trace.py
Normal file
439
_deprecated_py_src/api/admin/monitoring/trace.py
Normal file
@@ -0,0 +1,439 @@
|
||||
"""
|
||||
请求链路追踪 API 端点
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from src.api.base.admin_adapter import AdminApiAdapter
|
||||
from src.api.base.context import ApiRequestContext
|
||||
from src.api.base.pipeline import get_pipeline
|
||||
from src.core.crypto import crypto_service
|
||||
from src.database import get_db
|
||||
from src.models.database import Provider, ProviderAPIKey, ProviderEndpoint
|
||||
from src.services.request.candidate import RequestCandidateService
|
||||
|
||||
router = APIRouter(prefix="/api/admin/monitoring/trace", tags=["Admin - Monitoring: Trace"])
|
||||
pipeline = get_pipeline()
|
||||
|
||||
|
||||
class CandidateResponse(BaseModel):
|
||||
"""候选记录响应"""
|
||||
|
||||
id: str
|
||||
request_id: str
|
||||
candidate_index: int
|
||||
retry_index: int = 0 # 重试序号（从0开始）
|
||||
provider_id: str | None = None
|
||||
provider_name: str | None = None
|
||||
provider_website: str | None = None # Provider 官网
|
||||
endpoint_id: str | None = None
|
||||
endpoint_name: str | None = None # 端点显示名称（api_format）
|
||||
key_id: str | None = None
|
||||
key_name: str | None = None # 密钥名称
|
||||
key_account_label: str | None = None # 更适合展示的测试账号标签（优先 OAuth 邮箱）
|
||||
key_preview: str | None = None # 密钥脱敏预览（如 sk-***abc），OAuth 类型不返回
|
||||
key_auth_type: str | None = None # 密钥认证类型（api_key, service_account, oauth）
|
||||
key_oauth_plan_type: str | None = None # OAuth 账号套餐类型（free/plus/team/enterprise）
|
||||
key_capabilities: dict | None = None # Key 支持的能力
|
||||
required_capabilities: dict | None = None # 请求实际需要的能力标签
|
||||
status: str # 'pending', 'success', 'failed', 'skipped'
|
||||
skip_reason: str | None = None
|
||||
is_cached: bool = False
|
||||
# 执行结果字段
|
||||
status_code: int | None = None
|
||||
error_type: str | None = None
|
||||
error_message: str | None = None
|
||||
latency_ms: int | None = None
|
||||
concurrent_requests: int | None = None
|
||||
extra_data: dict | None = None
|
||||
created_at: datetime
|
||||
started_at: datetime | None = None
|
||||
finished_at: datetime | None = None
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
||||
class RequestTraceResponse(BaseModel):
|
||||
"""请求追踪完整响应"""
|
||||
|
||||
request_id: str
|
||||
total_candidates: int
|
||||
final_status: str # 'success', 'failed', 'cancelled', 'streaming', 'pending'
|
||||
total_latency_ms: int
|
||||
candidates: list[CandidateResponse]
|
||||
|
||||
|
||||
@router.get("/{request_id}", response_model=RequestTraceResponse)
|
||||
async def get_request_trace(
|
||||
request_id: str,
|
||||
request: Request,
|
||||
attempted_only: bool = Query(False, description="仅返回实际尝试过的候选"),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
获取请求的完整追踪信息
|
||||
|
||||
获取指定请求的完整链路追踪信息，包括所有候选（candidates）的执行情况。
|
||||
|
||||
**路径参数**:
|
||||
- `request_id`: 请求 ID
|
||||
|
||||
**返回字段**:
|
||||
- `request_id`: 请求 ID
|
||||
- `total_candidates`: 候选总数
|
||||
- `final_status`: 最终状态（success: 成功，failed: 失败，streaming: 流式传输中，pending: 等待中）
|
||||
- `total_latency_ms`: 总延迟（毫秒）
|
||||
- `candidates`: 候选列表，每个候选包含：
|
||||
- `id`: 候选 ID
|
||||
- `request_id`: 请求 ID
|
||||
- `candidate_index`: 候选索引
|
||||
- `retry_index`: 重试序号
|
||||
- `provider_id`: 提供商 ID
|
||||
- `provider_name`: 提供商名称
|
||||
- `provider_website`: 提供商官网
|
||||
- `endpoint_id`: 端点 ID
|
||||
- `endpoint_name`: 端点名称（API 格式）
|
||||
- `key_id`: 密钥 ID
|
||||
- `key_name`: 密钥名称
|
||||
- `key_preview`: 密钥脱敏预览
|
||||
- `key_capabilities`: 密钥支持的能力
|
||||
- `required_capabilities`: 请求需要的能力标签
|
||||
- `status`: 状态（pending, success, failed, skipped）
|
||||
- `skip_reason`: 跳过原因
|
||||
- `is_cached`: 是否缓存命中
|
||||
- `status_code`: HTTP 状态码
|
||||
- `error_type`: 错误类型
|
||||
- `error_message`: 错误信息
|
||||
- `latency_ms`: 延迟（毫秒）
|
||||
- `concurrent_requests`: 并发请求数
|
||||
- `extra_data`: 额外数据
|
||||
- `created_at`: 创建时间
|
||||
- `started_at`: 开始时间
|
||||
- `finished_at`: 完成时间
|
||||
"""
|
||||
|
||||
adapter = AdminGetRequestTraceAdapter(request_id=request_id, attempted_only=attempted_only)
|
||||
return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode)
|
||||
|
||||
|
||||
@router.get("/stats/provider/{provider_id}")
|
||||
async def get_provider_failure_rate(
|
||||
provider_id: str,
|
||||
request: Request,
|
||||
limit: int = Query(100, ge=1, le=1000, description="统计最近的尝试数量"),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
获取提供商的失败率统计
|
||||
|
||||
获取指定提供商最近的失败率统计信息。需要管理员权限。
|
||||
|
||||
**路径参数**:
|
||||
- `provider_id`: 提供商 ID
|
||||
|
||||
**查询参数**:
|
||||
- `limit`: 统计最近的尝试数量，默认 100，最大 1000
|
||||
|
||||
**返回字段**:
|
||||
- `provider_id`: 提供商 ID
|
||||
- `total_attempts`: 总尝试次数
|
||||
- `success_count`: 成功次数
|
||||
- `failed_count`: 失败次数
|
||||
- `failure_rate`: 失败率（百分比）
|
||||
- `avg_latency_ms`: 平均延迟（毫秒）
|
||||
"""
|
||||
adapter = AdminProviderFailureRateAdapter(provider_id=provider_id, limit=limit)
|
||||
return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode)
|
||||
|
||||
|
||||
# -------- 请求追踪适配器 --------
|
||||
|
||||
|
||||
@dataclass
|
||||
class AdminGetRequestTraceAdapter(AdminApiAdapter):
|
||||
request_id: str
|
||||
attempted_only: bool = False
|
||||
|
||||
@staticmethod
|
||||
def _is_attempted_candidate(candidate: Any) -> bool:
|
||||
status = str(getattr(candidate, "status", "") or "").strip().lower()
|
||||
# pre-created / never executed rows should not be considered as attempted
|
||||
if status in {"", "available", "unused", "skipped"}:
|
||||
return False
|
||||
# pending must have started_at to be considered truly entered execution
|
||||
if status == "pending" and getattr(candidate, "started_at", None) is None:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def handle(self, context: ApiRequestContext) -> Any: # type: ignore[override]
|
||||
db = context.db
|
||||
|
||||
# 查询并展示该请求的全量候选：
|
||||
# - 包含 available/unused（未执行）
|
||||
# - 包含 skipped（跳过）
|
||||
# - 包含 pending/streaming/success/failed/cancelled（执行中/结果）
|
||||
all_candidates = RequestCandidateService.get_candidates_by_request_id(db, self.request_id)
|
||||
|
||||
# 如果没有数据，返回 404
|
||||
if not all_candidates:
|
||||
raise HTTPException(status_code=404, detail="Request not found")
|
||||
|
||||
candidates = (
|
||||
[c for c in all_candidates if self._is_attempted_candidate(c)]
|
||||
if self.attempted_only
|
||||
else all_candidates
|
||||
)
|
||||
|
||||
# 计算总延迟（只统计已完成的候选：success, failed, cancelled）
|
||||
# 使用显式的 is not None 检查，避免过滤掉 0ms 的快速响应
|
||||
total_latency = sum(
|
||||
c.latency_ms
|
||||
for c in candidates
|
||||
if c.status in ("success", "failed", "cancelled") and c.latency_ms is not None
|
||||
)
|
||||
|
||||
# 判断最终状态：
|
||||
# 1. status="success" 即视为成功（无论 status_code 是什么）
|
||||
# - 流式请求即使客户端断开（499），只要 Provider 成功返回数据，也算成功
|
||||
# 2. 同时检查 status_code 在 200-299 范围，作为额外的成功判断条件
|
||||
# - 用于兼容非流式请求或未正确设置 status 的旧数据
|
||||
# 3. status="streaming" 表示流式请求正在进行中
|
||||
# 4. status="pending" 表示请求尚未开始执行
|
||||
# 5. status="cancelled" 表示客户端主动断开连接（不算失败）
|
||||
final_status_source = all_candidates if self.attempted_only else candidates
|
||||
has_success = any(
|
||||
c.status == "success" or (c.status_code is not None and 200 <= c.status_code < 300)
|
||||
for c in final_status_source
|
||||
)
|
||||
has_streaming = any(c.status == "streaming" for c in final_status_source)
|
||||
has_pending = any(c.status == "pending" for c in final_status_source)
|
||||
has_cancelled = any(c.status == "cancelled" for c in final_status_source)
|
||||
has_failed = any(c.status == "failed" for c in final_status_source)
|
||||
|
||||
if has_success:
|
||||
final_status = "success"
|
||||
elif has_streaming:
|
||||
# 有候选正在流式传输中
|
||||
final_status = "streaming"
|
||||
elif has_pending:
|
||||
# 有候选正在等待执行
|
||||
final_status = "pending"
|
||||
elif has_cancelled and not has_failed:
|
||||
# 只有取消没有失败，算作取消
|
||||
final_status = "cancelled"
|
||||
else:
|
||||
final_status = "failed"
|
||||
|
||||
# 批量加载 provider 信息，避免 N+1 查询
|
||||
provider_ids = {c.provider_id for c in candidates if c.provider_id}
|
||||
provider_map: dict[str, str] = {}
|
||||
provider_website_map: dict[str, str | None] = {}
|
||||
provider_type_map: dict[str, str] = {}
|
||||
if provider_ids:
|
||||
providers = db.query(Provider).filter(Provider.id.in_(provider_ids)).all()
|
||||
for p in providers:
|
||||
provider_map[p.id] = p.name
|
||||
provider_website_map[p.id] = p.website
|
||||
provider_type_map[p.id] = getattr(p, "provider_type", "custom") or "custom"
|
||||
|
||||
# 批量加载 endpoint 信息
|
||||
endpoint_ids = {c.endpoint_id for c in candidates if c.endpoint_id}
|
||||
endpoint_map = {}
|
||||
if endpoint_ids:
|
||||
endpoints = (
|
||||
db.query(ProviderEndpoint).filter(ProviderEndpoint.id.in_(endpoint_ids)).all()
|
||||
)
|
||||
endpoint_map = {e.id: e.api_format for e in endpoints}
|
||||
|
||||
# 批量加载 key 信息
|
||||
key_ids = {c.key_id for c in candidates if c.key_id}
|
||||
key_map: dict[str, str] = {}
|
||||
key_preview_map: dict[str, str] = {}
|
||||
key_account_label_map: dict[str, str | None] = {}
|
||||
key_capabilities_map: dict[str, dict | None] = {}
|
||||
key_auth_type_map: dict[str, str] = {}
|
||||
key_oauth_plan_map: dict[str, str | None] = {}
|
||||
# 建立 key_id -> provider_id 的映射（用于获取 provider_type）
|
||||
key_provider_map: dict[str, str | None] = {
|
||||
c.key_id: c.provider_id for c in candidates if c.key_id
|
||||
}
|
||||
if key_ids:
|
||||
keys = db.query(ProviderAPIKey).filter(ProviderAPIKey.id.in_(key_ids)).all()
|
||||
for k in keys:
|
||||
key_map[k.id] = k.name
|
||||
key_account_label_map[k.id] = k.name
|
||||
key_capabilities_map[k.id] = k.capabilities
|
||||
|
||||
is_oauth = k.auth_type == "oauth"
|
||||
|
||||
if is_oauth:
|
||||
# OAuth: auth_type 使用具体的 provider_type（如 kiro/codex/antigravity）
|
||||
pid = key_provider_map.get(k.id)
|
||||
key_auth_type_map[k.id] = (
|
||||
provider_type_map.get(pid, "oauth") if pid else "oauth"
|
||||
)
|
||||
# 提取 plan_type（不同 provider 存储位置不同）
|
||||
oauth_plan_type = None
|
||||
# 1. Codex: auth_config.plan_type
|
||||
# 2. Antigravity: auth_config.tier
|
||||
if k.auth_config:
|
||||
try:
|
||||
decrypted_config = crypto_service.decrypt(k.auth_config)
|
||||
auth_config = json.loads(decrypted_config)
|
||||
email = auth_config.get("email")
|
||||
if isinstance(email, str) and email.strip():
|
||||
key_account_label_map[k.id] = email.strip()
|
||||
oauth_plan_type = auth_config.get("plan_type")
|
||||
if not oauth_plan_type:
|
||||
ag_tier = auth_config.get("tier")
|
||||
if ag_tier and isinstance(ag_tier, str):
|
||||
oauth_plan_type = ag_tier.lower()
|
||||
except Exception:
|
||||
pass
|
||||
# 3. Kiro: upstream_metadata.kiro.subscription_title
|
||||
# subscription_title 通常为 "KIRO FREE" / "KIRO PRO+" 等，
|
||||
# 去掉 provider 名称前缀，只保留等级部分
|
||||
if not oauth_plan_type:
|
||||
um = getattr(k, "upstream_metadata", None) or {}
|
||||
kiro_meta = um.get("kiro") if isinstance(um, dict) else None
|
||||
if isinstance(kiro_meta, dict):
|
||||
sub_title = kiro_meta.get("subscription_title")
|
||||
if sub_title and isinstance(sub_title, str):
|
||||
# "KIRO FREE" -> "Free", "KIRO PRO+" -> "Pro+"
|
||||
ptype = provider_type_map.get(pid, "") if pid else ""
|
||||
if ptype and sub_title.upper().startswith(ptype.upper()):
|
||||
sub_title = sub_title[len(ptype) :].strip()
|
||||
oauth_plan_type = sub_title
|
||||
key_oauth_plan_map[k.id] = oauth_plan_type
|
||||
continue
|
||||
else:
|
||||
key_auth_type_map[k.id] = k.auth_type or "api_key"
|
||||
|
||||
# 非 OAuth：生成脱敏预览
|
||||
try:
|
||||
decrypted_key = crypto_service.decrypt(k.api_key)
|
||||
if len(decrypted_key) > 8:
|
||||
# 检测常见前缀模式
|
||||
prefix_end = 0
|
||||
for prefix in ["sk-", "key-", "api-", "ak-"]:
|
||||
if decrypted_key.lower().startswith(prefix):
|
||||
prefix_end = len(prefix)
|
||||
break
|
||||
if prefix_end > 0:
|
||||
key_preview_map[k.id] = (
|
||||
f"{decrypted_key[:prefix_end]}***{decrypted_key[-4:]}"
|
||||
)
|
||||
else:
|
||||
key_preview_map[k.id] = f"{decrypted_key[:4]}***{decrypted_key[-4:]}"
|
||||
elif len(decrypted_key) > 4:
|
||||
key_preview_map[k.id] = f"***{decrypted_key[-4:]}"
|
||||
else:
|
||||
key_preview_map[k.id] = "***"
|
||||
except Exception:
|
||||
key_preview_map[k.id] = "***"
|
||||
|
||||
# 构建 candidate 响应列表
|
||||
candidate_responses: list[CandidateResponse] = []
|
||||
for candidate in candidates:
|
||||
provider_name = (
|
||||
provider_map.get(candidate.provider_id) if candidate.provider_id else None
|
||||
)
|
||||
provider_website = (
|
||||
provider_website_map.get(candidate.provider_id) if candidate.provider_id else None
|
||||
)
|
||||
endpoint_name = (
|
||||
endpoint_map.get(candidate.endpoint_id) if candidate.endpoint_id else None
|
||||
)
|
||||
key_name = key_map.get(candidate.key_id) if candidate.key_id else None
|
||||
key_account_label = (
|
||||
key_account_label_map.get(candidate.key_id) if candidate.key_id else None
|
||||
)
|
||||
key_preview = key_preview_map.get(candidate.key_id) if candidate.key_id else None
|
||||
key_auth_type = key_auth_type_map.get(candidate.key_id) if candidate.key_id else None
|
||||
key_oauth_plan_type = (
|
||||
key_oauth_plan_map.get(candidate.key_id) if candidate.key_id else None
|
||||
)
|
||||
key_capabilities = (
|
||||
key_capabilities_map.get(candidate.key_id) if candidate.key_id else None
|
||||
)
|
||||
|
||||
candidate_responses.append(
|
||||
CandidateResponse(
|
||||
id=candidate.id,
|
||||
request_id=candidate.request_id,
|
||||
candidate_index=candidate.candidate_index,
|
||||
retry_index=candidate.retry_index,
|
||||
provider_id=candidate.provider_id,
|
||||
provider_name=provider_name,
|
||||
provider_website=provider_website,
|
||||
endpoint_id=candidate.endpoint_id,
|
||||
endpoint_name=endpoint_name,
|
||||
key_id=candidate.key_id,
|
||||
key_name=key_name,
|
||||
key_account_label=key_account_label,
|
||||
key_preview=key_preview,
|
||||
key_auth_type=key_auth_type,
|
||||
key_oauth_plan_type=key_oauth_plan_type,
|
||||
key_capabilities=key_capabilities,
|
||||
required_capabilities=candidate.required_capabilities,
|
||||
status=candidate.status,
|
||||
skip_reason=candidate.skip_reason,
|
||||
is_cached=candidate.is_cached,
|
||||
status_code=candidate.status_code,
|
||||
error_type=candidate.error_type,
|
||||
error_message=candidate.error_message,
|
||||
latency_ms=candidate.latency_ms,
|
||||
concurrent_requests=candidate.concurrent_requests,
|
||||
extra_data=candidate.extra_data,
|
||||
created_at=candidate.created_at,
|
||||
started_at=candidate.started_at,
|
||||
finished_at=candidate.finished_at,
|
||||
)
|
||||
)
|
||||
|
||||
response = RequestTraceResponse(
|
||||
request_id=self.request_id,
|
||||
total_candidates=len(candidates),
|
||||
final_status=final_status,
|
||||
total_latency_ms=total_latency,
|
||||
candidates=candidate_responses,
|
||||
)
|
||||
context.add_audit_metadata(
|
||||
action="trace_request_detail",
|
||||
request_id=self.request_id,
|
||||
total_candidates=len(candidates),
|
||||
final_status=final_status,
|
||||
total_latency_ms=total_latency,
|
||||
attempted_only=self.attempted_only,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
@dataclass
|
||||
class AdminProviderFailureRateAdapter(AdminApiAdapter):
|
||||
provider_id: str
|
||||
limit: int
|
||||
|
||||
async def handle(self, context: ApiRequestContext) -> Any: # type: ignore[override]
|
||||
result = RequestCandidateService.get_candidate_stats_by_provider(
|
||||
db=context.db,
|
||||
provider_id=self.provider_id,
|
||||
limit=self.limit,
|
||||
)
|
||||
context.add_audit_metadata(
|
||||
action="trace_provider_failure_rate",
|
||||
provider_id=self.provider_id,
|
||||
limit=self.limit,
|
||||
total_attempts=result.get("total_attempts"),
|
||||
failure_rate=result.get("failure_rate"),
|
||||
)
|
||||
return result
|
||||
Reference in New Issue
Block a user