perf(usage): 优化 admin usage records 查询性能

- count 查询按需 JOIN,避免不必要的表关联
- 前端 onMounted 将 stats/heatmap/records/users 全部并行加载
- 调整缓存 TTL(聚合 30s->60s,列表 10s->15s)
- 为 request_candidates 添加复合索引优化 fallback/retry 查询
This commit is contained in:
fawney19
2026-02-28 15:09:21 +08:00
parent 11832edf49
commit 4fdebfc78e
5 changed files with 127 additions and 47 deletions

View File

@@ -0,0 +1,49 @@
"""add_request_candidates_composite_indexes
Revision ID: 00b9161b8729
Revises: 48afe197cc15
Create Date: 2026-02-28 14:48:00.000000+00:00
"""
from sqlalchemy import inspect
from alembic import op
# revision identifiers, used by Alembic.
revision = "00b9161b8729"
down_revision = "48afe197cc15"
branch_labels = None
depends_on = None
def _index_exists(index_name: str) -> bool:
bind = op.get_bind()
insp = inspect(bind)
indexes = insp.get_indexes("request_candidates")
return any(idx["name"] == index_name for idx in indexes)
def upgrade() -> None:
# (request_id, status) - fallback/retry 查询优化
if not _index_exists("idx_rc_request_id_status"):
op.create_index(
"idx_rc_request_id_status",
"request_candidates",
["request_id", "status"],
)
# (provider_id, status, created_at) - provider 聚合统计优化
if not _index_exists("idx_rc_provider_status_created"):
op.create_index(
"idx_rc_provider_status_created",
"request_candidates",
["provider_id", "status", "created_at"],
)
def downgrade() -> None:
if _index_exists("idx_rc_provider_status_created"):
op.drop_index("idx_rc_provider_status_created", table_name="request_candidates")
if _index_exists("idx_rc_request_id_status"):
op.drop_index("idx_rc_request_id_status", table_name="request_candidates")

View File

@@ -411,34 +411,30 @@ const selectedRequestId = ref<string | null>(null)
// 初始化加载 // 初始化加载
onMounted(async () => { onMounted(async () => {
// 并行加载统计数据和热力图(使用 allSettled 避免其中一个失败影响另一个 // 所有数据源并行加载stats/heatmap/records/users 之间没有数据依赖
const [statsResult, heatmapResult] = await Promise.allSettled([ const statsTask = loadStats(timeRange.value).catch(err => {
loadStats(timeRange.value), log.error('加载统计数据失败:', err)
loadHeatmapData()
])
// 检查加载结果并通知用户
if (statsResult.status === 'rejected') {
log.error('加载统计数据失败:', statsResult.reason)
warning('统计数据加载失败,请刷新重试') warning('统计数据加载失败,请刷新重试')
} })
if (heatmapResult.status === 'rejected') { const heatmapTask = loadHeatmapData().catch(err => {
log.error('加载热力图数据失败:', heatmapResult.reason) log.error('加载热力图数据失败:', err)
// 热力图加载失败不提示,因为 UI 已显示占位符 })
const recordsTask = loadRecords(
{ page: currentPage.value, pageSize: pageSize.value },
getCurrentFilters()
)
const tasks: Promise<unknown>[] = [statsTask, heatmapTask, recordsTask]
if (isAdminPage.value) {
tasks.push(
usersApi.getAllUsers().then(users => {
availableUsers.value = users.map(u => ({ id: u.id, username: u.username, email: u.email }))
})
)
} }
// 加载记录和用户列表 await Promise.allSettled(tasks)
if (isAdminPage.value) {
// 管理员页面:并行加载用户列表和记录
const [users] = await Promise.all([
usersApi.getAllUsers(),
loadRecords({ page: currentPage.value, pageSize: pageSize.value }, getCurrentFilters())
])
availableUsers.value = users.map(u => ({ id: u.id, username: u.username, email: u.email }))
} else {
// 用户页面:加载记录
await loadRecords({ page: currentPage.value, pageSize: pageSize.value }, getCurrentFilters())
}
}) })
// 处理时间范围变化 // 处理时间范围变化

View File

@@ -869,6 +869,21 @@ class AdminUsageRecordsAdapter(AdminApiAdapter):
from src.utils.database_helpers import escape_like_pattern, safe_truncate_escaped from src.utils.database_helpers import escape_like_pattern, safe_truncate_escaped
db = context.db db = context.db
# -- 构建轻量 count 查询(仅按需 JOIN --
needs_user_join = bool(self.search or self.username)
needs_provider_join = bool(self.search or self.provider)
needs_apikey_join = bool(self.search)
count_query = db.query(func.count(Usage.id))
if needs_user_join:
count_query = count_query.outerjoin(User, Usage.user_id == User.id)
if needs_provider_join:
count_query = count_query.outerjoin(Provider, Usage.provider_id == Provider.id)
if needs_apikey_join:
count_query = count_query.outerjoin(ApiKey, Usage.api_key_id == ApiKey.id)
# -- 构建数据查询(完整 JOIN --
query = ( query = (
db.query(Usage, User, ProviderEndpoint, ProviderAPIKey, ApiKey) db.query(Usage, User, ProviderEndpoint, ProviderAPIKey, ApiKey)
.outerjoin(User, Usage.user_id == User.id) .outerjoin(User, Usage.user_id == User.id)
@@ -889,57 +904,65 @@ class AdminUsageRecordsAdapter(AdminApiAdapter):
for keyword in keywords: for keyword in keywords:
escaped = safe_truncate_escaped(escape_like_pattern(keyword), 100) escaped = safe_truncate_escaped(escape_like_pattern(keyword), 100)
search_pattern = f"%{escaped}%" search_pattern = f"%{escaped}%"
query = query.filter( search_filter = or_(
or_( User.username.ilike(search_pattern, escape="\\"),
User.username.ilike(search_pattern, escape="\\"), ApiKey.name.ilike(search_pattern, escape="\\"),
ApiKey.name.ilike(search_pattern, escape="\\"), Usage.model.ilike(search_pattern, escape="\\"),
Usage.model.ilike(search_pattern, escape="\\"), Provider.name.ilike(search_pattern, escape="\\"),
Provider.name.ilike(search_pattern, escape="\\"),
)
) )
query = query.filter(search_filter)
count_query = count_query.filter(search_filter)
if self.user_id: if self.user_id:
query = query.filter(Usage.user_id == self.user_id) query = query.filter(Usage.user_id == self.user_id)
count_query = count_query.filter(Usage.user_id == self.user_id)
if self.username: if self.username:
# 支持用户名模糊搜索 # 支持用户名模糊搜索
escaped = escape_like_pattern(self.username) escaped = escape_like_pattern(self.username)
query = query.filter(User.username.ilike(f"%{escaped}%", escape="\\")) username_filter = User.username.ilike(f"%{escaped}%", escape="\\")
query = query.filter(username_filter)
count_query = count_query.filter(username_filter)
if self.model: if self.model:
# 模型筛选:前端为下拉框精确值,使用精确匹配以启用索引 # 模型筛选:前端为下拉框精确值,使用精确匹配以启用索引
# 如需模糊搜索,请使用 search 参数。 # 如需模糊搜索,请使用 search 参数。
query = query.filter(Usage.model == self.model) query = query.filter(Usage.model == self.model)
count_query = count_query.filter(Usage.model == self.model)
if self.provider: if self.provider:
# 提供商筛选:前端为下拉框精确值,使用精确匹配以启用索引 # 提供商筛选:前端为下拉框精确值,使用精确匹配以启用索引
# 如需模糊搜索,请使用 search 参数。 # 如需模糊搜索,请使用 search 参数。
query = query.filter(Provider.name == self.provider) query = query.filter(Provider.name == self.provider)
count_query = count_query.filter(Provider.name == self.provider)
if self.api_format: if self.api_format:
# API 格式筛选:精确匹配(大小写不敏感) # API 格式筛选:精确匹配(大小写不敏感)
query = query.filter(func.lower(Usage.api_format) == self.api_format.lower()) api_format_filter = func.lower(Usage.api_format) == self.api_format.lower()
query = query.filter(api_format_filter)
count_query = count_query.filter(api_format_filter)
if self.status: if self.status:
# 状态筛选 # 状态筛选
# 旧的筛选值(基于 is_stream 和 status_codestream, standard, error # 旧的筛选值(基于 is_stream 和 status_codestream, standard, error
# 新的筛选值(基于 status 字段pending, streaming, completed, failed, active # 新的筛选值(基于 status 字段pending, streaming, completed, failed, active
status_filter = None
if self.status == "stream": if self.status == "stream":
query = query.filter(Usage.is_stream == True) # noqa: E712 status_filter = Usage.is_stream == True # noqa: E712
elif self.status == "standard": elif self.status == "standard":
query = query.filter(Usage.is_stream == False) # noqa: E712 status_filter = Usage.is_stream == False # noqa: E712
elif self.status == "error": elif self.status == "error":
query = query.filter((Usage.status_code >= 400) | (Usage.error_message.isnot(None))) status_filter = (Usage.status_code >= 400) | (Usage.error_message.isnot(None))
elif self.status in ("pending", "streaming", "completed", "cancelled"): elif self.status in ("pending", "streaming", "completed", "cancelled"):
# 新的状态筛选:直接按 status 字段过滤 # 新的状态筛选:直接按 status 字段过滤
query = query.filter(Usage.status == self.status) status_filter = Usage.status == self.status
elif self.status == "failed": elif self.status == "failed":
# 失败请求需要同时考虑新旧两种判断方式: # 失败请求需要同时考虑新旧两种判断方式:
# 1. 新方式status = "failed" # 1. 新方式status = "failed"
# 2. 旧方式status_code >= 400 或 error_message 不为空 # 2. 旧方式status_code >= 400 或 error_message 不为空
query = query.filter( status_filter = (
(Usage.status == "failed") (Usage.status == "failed")
| (Usage.status_code >= 400) | (Usage.status_code >= 400)
| (Usage.error_message.isnot(None)) | (Usage.error_message.isnot(None))
) )
elif self.status == "active": elif self.status == "active":
# 活跃请求pending 或 streaming 状态 # 活跃请求pending 或 streaming 状态
query = query.filter(Usage.status.in_(["pending", "streaming"])) status_filter = Usage.status.in_(["pending", "streaming"])
elif self.status == "has_retry": elif self.status == "has_retry":
# 发生重试:存在 retry_index > 0 的已执行候选 # 发生重试:存在 retry_index > 0 的已执行候选
retry_subq = ( retry_subq = (
@@ -951,7 +974,7 @@ class AdminUsageRecordsAdapter(AdminApiAdapter):
.distinct() .distinct()
.subquery() .subquery()
) )
query = query.filter(Usage.request_id.in_(retry_subq)) status_filter = Usage.request_id.in_(retry_subq)
elif self.status == "has_fallback": elif self.status == "has_fallback":
# 发生转移:同一请求有多个不同 candidate_index 的已执行候选 # 发生转移:同一请求有多个不同 candidate_index 的已执行候选
fallback_subq = ( fallback_subq = (
@@ -961,13 +984,21 @@ class AdminUsageRecordsAdapter(AdminApiAdapter):
.having(func.count(func.distinct(RequestCandidate.candidate_index)) > 1) .having(func.count(func.distinct(RequestCandidate.candidate_index)) > 1)
.subquery() .subquery()
) )
query = query.filter(Usage.request_id.in_(fallback_subq)) status_filter = Usage.request_id.in_(fallback_subq)
if status_filter is not None:
query = query.filter(status_filter)
count_query = count_query.filter(status_filter)
if self.time_range: if self.time_range:
start_utc, end_utc = self.time_range.to_utc_datetime_range() start_utc, end_utc = self.time_range.to_utc_datetime_range()
query = query.filter(Usage.created_at >= start_utc, Usage.created_at < end_utc) time_filter_start = Usage.created_at >= start_utc
time_filter_end = Usage.created_at < end_utc
query = query.filter(time_filter_start, time_filter_end)
count_query = count_query.filter(time_filter_start, time_filter_end)
# Perf: avoid Query.count() building a subquery selecting many columns # Perf: count query uses fewer JOINs than the data query
total = int(query.with_entities(func.count(Usage.id)).scalar() or 0) total = int(count_query.scalar() or 0)
# Perf: do not load large request/response columns for list view # Perf: do not load large request/response columns for list view
query = query.options( query = query.options(

View File

@@ -29,8 +29,8 @@ class CacheTTL:
DASHBOARD_DAILY = 600 # 10分钟每日统计 DASHBOARD_DAILY = 600 # 10分钟每日统计
# Admin usage pages (heavy DB aggregations / list queries) # Admin usage pages (heavy DB aggregations / list queries)
ADMIN_USAGE_AGGREGATION = 30 # 30秒 ADMIN_USAGE_AGGREGATION = 60 # 60秒(聚合统计变化不频繁,适当延长减少 DB 压力)
ADMIN_USAGE_RECORDS = 10 # 10秒(列表页短缓存,避免轮询/重复刷新打爆 DB ADMIN_USAGE_RECORDS = 15 # 15秒(列表页短缓存,活跃请求通过轮询接口实时更新
# Admin leaderboard (heavier, slower moving) # Admin leaderboard (heavier, slower moving)
ADMIN_LEADERBOARD = 300 # 5分钟 ADMIN_LEADERBOARD = 300 # 5分钟

View File

@@ -2203,6 +2203,10 @@ class RequestCandidate(Base):
Index("idx_request_candidates_status", "status"), Index("idx_request_candidates_status", "status"),
Index("idx_request_candidates_provider_id", "provider_id"), Index("idx_request_candidates_provider_id", "provider_id"),
Index("idx_request_candidates_created_at", "created_at"), Index("idx_request_candidates_created_at", "created_at"),
# 复合索引: 按 request_id + status 查询 fallback/retry 场景
Index("idx_rc_request_id_status", "request_id", "status"),
# 复合索引: 按 provider 聚合统计provider_id + status + created_at
Index("idx_rc_provider_status_created", "provider_id", "status", "created_at"),
) )
# 关系 # 关系