mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-01 17:00:21 +08:00
perf(usage): 优化 admin usage records 查询性能
- count 查询按需 JOIN,避免不必要的表关联 - 前端 onMounted 将 stats/heatmap/records/users 全部并行加载 - 调整缓存 TTL(聚合 30s->60s,列表 10s->15s) - 为 request_candidates 添加复合索引优化 fallback/retry 查询
This commit is contained in:
@@ -0,0 +1,49 @@
|
||||
"""add_request_candidates_composite_indexes
|
||||
|
||||
Revision ID: 00b9161b8729
|
||||
Revises: 48afe197cc15
|
||||
Create Date: 2026-02-28 14:48:00.000000+00:00
|
||||
|
||||
"""
|
||||
|
||||
from sqlalchemy import inspect
|
||||
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "00b9161b8729"
|
||||
down_revision = "48afe197cc15"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _index_exists(index_name: str) -> bool:
|
||||
bind = op.get_bind()
|
||||
insp = inspect(bind)
|
||||
indexes = insp.get_indexes("request_candidates")
|
||||
return any(idx["name"] == index_name for idx in indexes)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# (request_id, status) - fallback/retry 查询优化
|
||||
if not _index_exists("idx_rc_request_id_status"):
|
||||
op.create_index(
|
||||
"idx_rc_request_id_status",
|
||||
"request_candidates",
|
||||
["request_id", "status"],
|
||||
)
|
||||
|
||||
# (provider_id, status, created_at) - provider 聚合统计优化
|
||||
if not _index_exists("idx_rc_provider_status_created"):
|
||||
op.create_index(
|
||||
"idx_rc_provider_status_created",
|
||||
"request_candidates",
|
||||
["provider_id", "status", "created_at"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _index_exists("idx_rc_provider_status_created"):
|
||||
op.drop_index("idx_rc_provider_status_created", table_name="request_candidates")
|
||||
if _index_exists("idx_rc_request_id_status"):
|
||||
op.drop_index("idx_rc_request_id_status", table_name="request_candidates")
|
||||
@@ -411,34 +411,30 @@ const selectedRequestId = ref<string | null>(null)
|
||||
|
||||
// 初始化加载
|
||||
onMounted(async () => {
|
||||
// 并行加载统计数据和热力图(使用 allSettled 避免其中一个失败影响另一个)
|
||||
const [statsResult, heatmapResult] = await Promise.allSettled([
|
||||
loadStats(timeRange.value),
|
||||
loadHeatmapData()
|
||||
])
|
||||
|
||||
// 检查加载结果并通知用户
|
||||
if (statsResult.status === 'rejected') {
|
||||
log.error('加载统计数据失败:', statsResult.reason)
|
||||
// 所有数据源并行加载(stats/heatmap/records/users 之间没有数据依赖)
|
||||
const statsTask = loadStats(timeRange.value).catch(err => {
|
||||
log.error('加载统计数据失败:', err)
|
||||
warning('统计数据加载失败,请刷新重试')
|
||||
}
|
||||
if (heatmapResult.status === 'rejected') {
|
||||
log.error('加载热力图数据失败:', heatmapResult.reason)
|
||||
// 热力图加载失败不提示,因为 UI 已显示占位符
|
||||
})
|
||||
const heatmapTask = loadHeatmapData().catch(err => {
|
||||
log.error('加载热力图数据失败:', err)
|
||||
})
|
||||
const recordsTask = loadRecords(
|
||||
{ page: currentPage.value, pageSize: pageSize.value },
|
||||
getCurrentFilters()
|
||||
)
|
||||
|
||||
const tasks: Promise<unknown>[] = [statsTask, heatmapTask, recordsTask]
|
||||
|
||||
if (isAdminPage.value) {
|
||||
tasks.push(
|
||||
usersApi.getAllUsers().then(users => {
|
||||
availableUsers.value = users.map(u => ({ id: u.id, username: u.username, email: u.email }))
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// 加载记录和用户列表
|
||||
if (isAdminPage.value) {
|
||||
// 管理员页面:并行加载用户列表和记录
|
||||
const [users] = await Promise.all([
|
||||
usersApi.getAllUsers(),
|
||||
loadRecords({ page: currentPage.value, pageSize: pageSize.value }, getCurrentFilters())
|
||||
])
|
||||
availableUsers.value = users.map(u => ({ id: u.id, username: u.username, email: u.email }))
|
||||
} else {
|
||||
// 用户页面:加载记录
|
||||
await loadRecords({ page: currentPage.value, pageSize: pageSize.value }, getCurrentFilters())
|
||||
}
|
||||
await Promise.allSettled(tasks)
|
||||
})
|
||||
|
||||
// 处理时间范围变化
|
||||
|
||||
@@ -869,6 +869,21 @@ class AdminUsageRecordsAdapter(AdminApiAdapter):
|
||||
from src.utils.database_helpers import escape_like_pattern, safe_truncate_escaped
|
||||
|
||||
db = context.db
|
||||
|
||||
# -- 构建轻量 count 查询(仅按需 JOIN) --
|
||||
needs_user_join = bool(self.search or self.username)
|
||||
needs_provider_join = bool(self.search or self.provider)
|
||||
needs_apikey_join = bool(self.search)
|
||||
|
||||
count_query = db.query(func.count(Usage.id))
|
||||
if needs_user_join:
|
||||
count_query = count_query.outerjoin(User, Usage.user_id == User.id)
|
||||
if needs_provider_join:
|
||||
count_query = count_query.outerjoin(Provider, Usage.provider_id == Provider.id)
|
||||
if needs_apikey_join:
|
||||
count_query = count_query.outerjoin(ApiKey, Usage.api_key_id == ApiKey.id)
|
||||
|
||||
# -- 构建数据查询(完整 JOIN) --
|
||||
query = (
|
||||
db.query(Usage, User, ProviderEndpoint, ProviderAPIKey, ApiKey)
|
||||
.outerjoin(User, Usage.user_id == User.id)
|
||||
@@ -889,57 +904,65 @@ class AdminUsageRecordsAdapter(AdminApiAdapter):
|
||||
for keyword in keywords:
|
||||
escaped = safe_truncate_escaped(escape_like_pattern(keyword), 100)
|
||||
search_pattern = f"%{escaped}%"
|
||||
query = query.filter(
|
||||
or_(
|
||||
User.username.ilike(search_pattern, escape="\\"),
|
||||
ApiKey.name.ilike(search_pattern, escape="\\"),
|
||||
Usage.model.ilike(search_pattern, escape="\\"),
|
||||
Provider.name.ilike(search_pattern, escape="\\"),
|
||||
)
|
||||
search_filter = or_(
|
||||
User.username.ilike(search_pattern, escape="\\"),
|
||||
ApiKey.name.ilike(search_pattern, escape="\\"),
|
||||
Usage.model.ilike(search_pattern, escape="\\"),
|
||||
Provider.name.ilike(search_pattern, escape="\\"),
|
||||
)
|
||||
query = query.filter(search_filter)
|
||||
count_query = count_query.filter(search_filter)
|
||||
|
||||
if self.user_id:
|
||||
query = query.filter(Usage.user_id == self.user_id)
|
||||
count_query = count_query.filter(Usage.user_id == self.user_id)
|
||||
if self.username:
|
||||
# 支持用户名模糊搜索
|
||||
escaped = escape_like_pattern(self.username)
|
||||
query = query.filter(User.username.ilike(f"%{escaped}%", escape="\\"))
|
||||
username_filter = User.username.ilike(f"%{escaped}%", escape="\\")
|
||||
query = query.filter(username_filter)
|
||||
count_query = count_query.filter(username_filter)
|
||||
if self.model:
|
||||
# 模型筛选:前端为下拉框精确值,使用精确匹配以启用索引
|
||||
# 如需模糊搜索,请使用 search 参数。
|
||||
query = query.filter(Usage.model == self.model)
|
||||
count_query = count_query.filter(Usage.model == self.model)
|
||||
if self.provider:
|
||||
# 提供商筛选:前端为下拉框精确值,使用精确匹配以启用索引
|
||||
# 如需模糊搜索,请使用 search 参数。
|
||||
query = query.filter(Provider.name == self.provider)
|
||||
count_query = count_query.filter(Provider.name == self.provider)
|
||||
if self.api_format:
|
||||
# API 格式筛选:精确匹配(大小写不敏感)
|
||||
query = query.filter(func.lower(Usage.api_format) == self.api_format.lower())
|
||||
api_format_filter = func.lower(Usage.api_format) == self.api_format.lower()
|
||||
query = query.filter(api_format_filter)
|
||||
count_query = count_query.filter(api_format_filter)
|
||||
if self.status:
|
||||
# 状态筛选
|
||||
# 旧的筛选值(基于 is_stream 和 status_code):stream, standard, error
|
||||
# 新的筛选值(基于 status 字段):pending, streaming, completed, failed, active
|
||||
status_filter = None
|
||||
if self.status == "stream":
|
||||
query = query.filter(Usage.is_stream == True) # noqa: E712
|
||||
status_filter = Usage.is_stream == True # noqa: E712
|
||||
elif self.status == "standard":
|
||||
query = query.filter(Usage.is_stream == False) # noqa: E712
|
||||
status_filter = Usage.is_stream == False # noqa: E712
|
||||
elif self.status == "error":
|
||||
query = query.filter((Usage.status_code >= 400) | (Usage.error_message.isnot(None)))
|
||||
status_filter = (Usage.status_code >= 400) | (Usage.error_message.isnot(None))
|
||||
elif self.status in ("pending", "streaming", "completed", "cancelled"):
|
||||
# 新的状态筛选:直接按 status 字段过滤
|
||||
query = query.filter(Usage.status == self.status)
|
||||
status_filter = Usage.status == self.status
|
||||
elif self.status == "failed":
|
||||
# 失败请求需要同时考虑新旧两种判断方式:
|
||||
# 1. 新方式:status = "failed"
|
||||
# 2. 旧方式:status_code >= 400 或 error_message 不为空
|
||||
query = query.filter(
|
||||
status_filter = (
|
||||
(Usage.status == "failed")
|
||||
| (Usage.status_code >= 400)
|
||||
| (Usage.error_message.isnot(None))
|
||||
)
|
||||
elif self.status == "active":
|
||||
# 活跃请求:pending 或 streaming 状态
|
||||
query = query.filter(Usage.status.in_(["pending", "streaming"]))
|
||||
status_filter = Usage.status.in_(["pending", "streaming"])
|
||||
elif self.status == "has_retry":
|
||||
# 发生重试:存在 retry_index > 0 的已执行候选
|
||||
retry_subq = (
|
||||
@@ -951,7 +974,7 @@ class AdminUsageRecordsAdapter(AdminApiAdapter):
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
query = query.filter(Usage.request_id.in_(retry_subq))
|
||||
status_filter = Usage.request_id.in_(retry_subq)
|
||||
elif self.status == "has_fallback":
|
||||
# 发生转移:同一请求有多个不同 candidate_index 的已执行候选
|
||||
fallback_subq = (
|
||||
@@ -961,13 +984,21 @@ class AdminUsageRecordsAdapter(AdminApiAdapter):
|
||||
.having(func.count(func.distinct(RequestCandidate.candidate_index)) > 1)
|
||||
.subquery()
|
||||
)
|
||||
query = query.filter(Usage.request_id.in_(fallback_subq))
|
||||
status_filter = Usage.request_id.in_(fallback_subq)
|
||||
|
||||
if status_filter is not None:
|
||||
query = query.filter(status_filter)
|
||||
count_query = count_query.filter(status_filter)
|
||||
|
||||
if self.time_range:
|
||||
start_utc, end_utc = self.time_range.to_utc_datetime_range()
|
||||
query = query.filter(Usage.created_at >= start_utc, Usage.created_at < end_utc)
|
||||
time_filter_start = Usage.created_at >= start_utc
|
||||
time_filter_end = Usage.created_at < end_utc
|
||||
query = query.filter(time_filter_start, time_filter_end)
|
||||
count_query = count_query.filter(time_filter_start, time_filter_end)
|
||||
|
||||
# Perf: avoid Query.count() building a subquery selecting many columns
|
||||
total = int(query.with_entities(func.count(Usage.id)).scalar() or 0)
|
||||
# Perf: count query uses fewer JOINs than the data query
|
||||
total = int(count_query.scalar() or 0)
|
||||
|
||||
# Perf: do not load large request/response columns for list view
|
||||
query = query.options(
|
||||
|
||||
@@ -29,8 +29,8 @@ class CacheTTL:
|
||||
DASHBOARD_DAILY = 600 # 10分钟(每日统计)
|
||||
|
||||
# Admin usage pages (heavy DB aggregations / list queries)
|
||||
ADMIN_USAGE_AGGREGATION = 30 # 30秒
|
||||
ADMIN_USAGE_RECORDS = 10 # 10秒(列表页短缓存,避免轮询/重复刷新打爆 DB)
|
||||
ADMIN_USAGE_AGGREGATION = 60 # 60秒(聚合统计变化不频繁,适当延长减少 DB 压力)
|
||||
ADMIN_USAGE_RECORDS = 15 # 15秒(列表页短缓存,活跃请求通过轮询接口实时更新)
|
||||
|
||||
# Admin leaderboard (heavier, slower moving)
|
||||
ADMIN_LEADERBOARD = 300 # 5分钟
|
||||
|
||||
@@ -2203,6 +2203,10 @@ class RequestCandidate(Base):
|
||||
Index("idx_request_candidates_status", "status"),
|
||||
Index("idx_request_candidates_provider_id", "provider_id"),
|
||||
Index("idx_request_candidates_created_at", "created_at"),
|
||||
# 复合索引: 按 request_id + status 查询 fallback/retry 场景
|
||||
Index("idx_rc_request_id_status", "request_id", "status"),
|
||||
# 复合索引: 按 provider 聚合统计(provider_id + status + created_at)
|
||||
Index("idx_rc_provider_status_created", "provider_id", "status", "created_at"),
|
||||
)
|
||||
|
||||
# 关系
|
||||
|
||||
Reference in New Issue
Block a user