perf: Pool 批量删除改为异步任务模式,避免大批量删除阻塞请求

- 新增 batch_delete_task 模块,提交删除后立即返回 task_id,后台线程分批执行
- 新增查询任务进度的 API 端点,前端轮询展示实时进度
- RequestCandidate 大表清理改为按行数分批删除,防止单条语句超时
This commit is contained in:
fawney19
2026-03-09 01:35:01 +08:00
parent 0379f01ce8
commit e4476d0bc6
6 changed files with 386 additions and 80 deletions

View File

@@ -213,7 +213,7 @@ export async function listPoolKeys(
export async function batchActionPoolKeys(
providerId: string,
body: PoolBatchAction,
): Promise<{ affected: number; message: string }> {
): Promise<{ affected: number; message: string; task_id?: string }> {
const response = await client.post(
`/api/admin/pool/${providerId}/keys/batch-action`,
body,
@@ -222,6 +222,24 @@ export async function batchActionPoolKeys(
return response.data
}
export interface BatchDeleteTaskStatus {
task_id: string
status: 'pending' | 'running' | 'completed' | 'failed'
total: number
deleted: number
message: string
}
export async function getPoolBatchDeleteTask(
providerId: string,
taskId: string,
): Promise<BatchDeleteTaskStatus> {
const response = await client.get<BatchDeleteTaskStatus>(
`/api/admin/pool/${providerId}/keys/batch-delete-task/${taskId}`,
)
return response.data
}
export async function cleanupBannedPoolKeys(
providerId: string,
): Promise<{ affected: number; message: string }> {

View File

@@ -266,7 +266,7 @@ import { RefreshCw, Play, ChevronLeft, ChevronRight, ChevronsLeft, ChevronsRight
import { useToast } from '@/composables/useToast'
import { useConfirm } from '@/composables/useConfirm'
import { parseApiError } from '@/utils/errorParser'
import { listPoolKeys, batchActionPoolKeys, type PoolKeyDetail } from '@/api/endpoints/pool'
import { listPoolKeys, batchActionPoolKeys, getPoolBatchDeleteTask, type PoolKeyDetail } from '@/api/endpoints/pool'
import { refreshProviderQuota } from '@/api/endpoints/keys'
import { refreshProviderOAuth } from '@/api/endpoints/provider_oauth'
import { useProxyNodesStore } from '@/stores/proxy-nodes'
@@ -573,6 +573,36 @@ async function loadAllKeys(): Promise<void> {
}
}
const DELETE_POLL_INTERVAL_MS = 2000
const DELETE_POLL_MAX_MS = 10 * 60 * 1000
const DELETE_POLL_MAX_FAILURES = 3
async function pollDeleteTask(
providerId: string,
taskId: string,
progressOffset: number,
): Promise<{ status: string; deleted: number }> {
const deadline = Date.now() + DELETE_POLL_MAX_MS
let consecutiveFailures = 0
while (Date.now() < deadline) {
await new Promise((r) => setTimeout(r, DELETE_POLL_INTERVAL_MS))
try {
const task = await getPoolBatchDeleteTask(providerId, taskId)
consecutiveFailures = 0
progressDone.value = progressOffset + task.deleted
if (task.status === 'completed' || task.status === 'failed') {
return { status: task.status, deleted: task.deleted }
}
} catch {
consecutiveFailures++
if (consecutiveFailures >= DELETE_POLL_MAX_FAILURES) {
return { status: 'failed', deleted: 0 }
}
}
}
return { status: 'failed', deleted: 0 }
}
async function executeAction(): Promise<void> {
if (executing.value) return
if (selectedKeyIds.value.length === 0) {
@@ -638,7 +668,45 @@ async function executeAction(): Promise<void> {
progressDone.value = Math.min(i + BATCH_SIZE, targetIds.length)
}
} else if (['delete', 'enable', 'disable', 'clear_proxy', 'set_proxy'].includes(selectedAction.value)) {
} else if (selectedAction.value === 'delete') {
// 删除走异步任务模式:提交后轮询进度
const targetIds = selectedKeys.map((key) => key.key_id)
const BATCH_SIZE = 2000
const totalBatches = Math.ceil(targetIds.length / BATCH_SIZE)
for (let i = 0; i < targetIds.length; i += BATCH_SIZE) {
const batchIndex = Math.floor(i / BATCH_SIZE) + 1
const batch = targetIds.slice(i, i + BATCH_SIZE)
if (totalBatches > 1) {
progressLabel.value = `正在${actionLabel}...(第 ${batchIndex}/${totalBatches} 批)`
}
try {
const result = await batchActionPoolKeys(props.providerId, {
key_ids: batch,
action: 'delete',
})
if (result.task_id) {
// 异步任务:轮询进度
progressLabel.value = `正在${actionLabel}...(后台执行中)`
const taskResult = await pollDeleteTask(props.providerId, result.task_id, i)
successCount += taskResult.deleted
if (taskResult.status === 'failed') {
failedCount += batch.length - taskResult.deleted
}
} else {
successCount += result.affected
}
} catch (err) {
// eslint-disable-next-line no-console
console.error(`batch delete failed (batch ${batchIndex}/${totalBatches}):`, err)
failedCount += batch.length
}
progressDone.value = Math.min(i + BATCH_SIZE, targetIds.length)
}
} else if (['enable', 'disable', 'clear_proxy', 'set_proxy'].includes(selectedAction.value)) {
const targetIds = selectedKeys.map((key) => key.key_id)
const BATCH_SIZE = 2000
const totalBatches = Math.ceil(targetIds.length / BATCH_SIZE)
@@ -657,7 +725,7 @@ async function executeAction(): Promise<void> {
try {
const result = await batchActionPoolKeys(props.providerId, {
key_ids: batch,
action: selectedAction.value,
action: selectedAction.value as 'enable' | 'disable' | 'clear_proxy' | 'set_proxy',
...(payload ? { payload } : {}),
})
successCount += result.affected

View File

@@ -46,6 +46,7 @@ from src.services.provider_keys.quota_reader import get_quota_reader
from .schemas import (
BatchActionRequest,
BatchActionResponse,
BatchDeleteTaskResponse,
BatchImportError,
BatchImportRequest,
BatchImportResponse,
@@ -449,6 +450,21 @@ async def batch_action_keys(
return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode)
@router.get(
"/{provider_id}/keys/batch-delete-task/{task_id}",
response_model=BatchDeleteTaskResponse,
)
async def get_batch_delete_task_status(
provider_id: str,
task_id: str,
request: Request,
db: Session = Depends(get_db),
) -> BatchDeleteTaskResponse:
"""Query the progress of an async batch-delete task."""
adapter = AdminBatchDeleteTaskStatusAdapter(provider_id=provider_id, task_id=task_id)
return await pipeline.run(adapter=adapter, http_request=request, db=db, mode=adapter.mode)
@router.post("/{provider_id}/keys/cleanup-banned", response_model=BatchActionResponse)
async def cleanup_banned_keys(
provider_id: str,
@@ -465,6 +481,28 @@ async def cleanup_banned_keys(
# ---------------------------------------------------------------------------
@dataclass
class AdminBatchDeleteTaskStatusAdapter(AdminApiAdapter):
provider_id: str = ""
task_id: str = ""
async def handle(self, context: ApiRequestContext) -> Any: # type: ignore[override]
from fastapi import HTTPException
from src.services.provider_keys.batch_delete_task import get_batch_delete_task
task = get_batch_delete_task(self.task_id)
if task is None or task.provider_id != self.provider_id:
raise HTTPException(status_code=404, detail="Task not found")
return BatchDeleteTaskResponse(
task_id=task.task_id,
status=task.status,
total=task.total,
deleted=task.deleted,
message=task.message,
)
class AdminListSchedulingPresetsAdapter(AdminApiAdapter):
async def handle(self, context: ApiRequestContext) -> Any: # type: ignore[override]
items: list[PresetDimensionMetaResponse] = [
@@ -1070,86 +1108,23 @@ class AdminBatchActionKeysAdapter(AdminApiAdapter):
affected = 0
if self.body.action == "delete":
delete_started_at = time.perf_counter()
sql_delete_ms = 0.0
cleanup_ms = 0.0
commit_ms = 0.0
side_effects_ms = 0.0
from src.services.provider_keys.batch_delete_task import submit_batch_delete
key_ids = list(dict.fromkeys(self.body.key_ids))
delete_batch_size = _resolve_delete_batch_size(db)
delete_batch_count = 0
try:
# 先清理关联表,避免 CASCADE 级联删除导致超时
cleanup_started_at = time.perf_counter()
cleanup_key_references(db, key_ids)
cleanup_ms = (time.perf_counter() - cleanup_started_at) * 1000.0
for batch in _iter_batches(key_ids, delete_batch_size):
batch_started_at = time.perf_counter()
result = db.execute(
sa_delete(ProviderAPIKey).where(
ProviderAPIKey.provider_id == pid,
ProviderAPIKey.id.in_(batch),
)
)
sql_delete_ms += (time.perf_counter() - batch_started_at) * 1000.0
delete_batch_count += 1
rowcount = getattr(result, "rowcount", 0) or 0
if rowcount > 0:
affected += int(rowcount)
commit_started_at = time.perf_counter()
db.commit()
commit_ms = (time.perf_counter() - commit_started_at) * 1000.0
except Exception as exc:
db.rollback()
total_ms = (time.perf_counter() - delete_started_at) * 1000.0
logger.error(
"batch delete commit failed: {} | provider={} requested={} batches={} sql_ms={:.2f} cleanup_ms={:.2f} commit_ms={:.2f} total_ms={:.2f}",
exc,
pid[:8],
len(key_ids),
delete_batch_count,
sql_delete_ms,
cleanup_ms,
commit_ms,
total_ms,
)
return BatchActionResponse(affected=0, message=f"commit failed: {exc}")
if affected > 0:
from src.services.provider_keys.key_side_effects import (
run_delete_key_side_effects,
)
try:
side_effects_started_at = time.perf_counter()
await run_delete_key_side_effects(
db=db,
provider_id=pid,
deleted_key_allowed_models=None,
)
side_effects_ms = (time.perf_counter() - side_effects_started_at) * 1000.0
except Exception as exc:
side_effects_ms = (time.perf_counter() - side_effects_started_at) * 1000.0
logger.error("batch delete side effects failed: {}", exc)
total_ms = (time.perf_counter() - delete_started_at) * 1000.0
task_id = submit_batch_delete(pid, key_ids)
admin_name = context.user.username if context.user else "admin"
logger.info(
"[POOL_BATCH_DELETE_TIMING] provider={} requested={} affected={} batches={} batch_size={} cleanup_ms={:.2f} sql_ms={:.2f} commit_ms={:.2f} side_effects_ms={:.2f} total_ms={:.2f}",
"Pool batch delete submitted by {}: provider={}, keys={}, task_id={}",
admin_name,
pid[:8],
len(key_ids),
affected,
delete_batch_count,
delete_batch_size,
cleanup_ms,
sql_delete_ms,
commit_ms,
side_effects_ms,
total_ms,
task_id,
)
return BatchActionResponse(
affected=0,
message=f"delete task submitted ({len(key_ids)} keys)",
task_id=task_id,
)
admin_name = context.user.username if context.user else "admin"
affected_ids = [kid[:8] for kid in key_ids[:20]]
else:
keys = (
db.query(ProviderAPIKey)

View File

@@ -172,3 +172,12 @@ class BatchActionRequest(BaseModel):
class BatchActionResponse(BaseModel):
affected: int = 0
message: str = ""
task_id: str | None = None
class BatchDeleteTaskResponse(BaseModel):
task_id: str
status: str # pending / running / completed / failed
total: int = 0
deleted: int = 0
message: str = ""

View File

@@ -0,0 +1,208 @@
"""Pool Key 批量删除异步任务。
接口立即返回 task_id后台执行删除前端轮询进度。
任务状态存内存字典,重启丢失无影响。
"""
from __future__ import annotations
import asyncio
import time
import uuid
from dataclasses import dataclass, field
from sqlalchemy import delete as sa_delete
from src.core.logger import logger
# 任务状态
STATUS_PENDING = "pending"
STATUS_RUNNING = "running"
STATUS_COMPLETED = "completed"
STATUS_FAILED = "failed"
# 任务完成后保留时长(秒)
_TASK_RETAIN_SECONDS = 600
# 删除 ProviderAPIKey 时的分批大小
_DELETE_BATCH_SIZE = 500
# 同时存在的最大任务数(含已完成但未过期的)
_MAX_TASKS = 100
@dataclass
class BatchDeleteTask:
task_id: str
provider_id: str
status: str = STATUS_PENDING
total: int = 0
deleted: int = 0
message: str = ""
created_at: float = field(default_factory=time.monotonic)
finished_at: float | None = None
# 模块级任务注册表
_tasks: dict[str, BatchDeleteTask] = {}
# 持有后台 asyncio.Task 引用,防止 GC 回收
_running_tasks: set[asyncio.Task[None]] = set()
def _evict_finished_tasks() -> None:
"""清除已过期的已完成任务,为新任务腾出空间。"""
now = time.monotonic()
expired = [
tid
for tid, t in _tasks.items()
if t.finished_at is not None and (now - t.finished_at) > _TASK_RETAIN_SECONDS
]
for tid in expired:
_tasks.pop(tid, None)
def submit_batch_delete(provider_id: str, key_ids: list[str]) -> str:
"""提交批量删除任务,返回 task_id。
Raises:
RuntimeError: 并发任务数超过 _MAX_TASKS 上限。
"""
_evict_finished_tasks()
if len(_tasks) >= _MAX_TASKS:
raise RuntimeError(f"too many batch-delete tasks ({len(_tasks)}), try again later")
task_id = uuid.uuid4().hex[:16]
task = BatchDeleteTask(
task_id=task_id,
provider_id=provider_id,
total=len(key_ids),
)
_tasks[task_id] = task
bg = asyncio.create_task(
_run_batch_delete(task_id, provider_id, key_ids),
name=f"batch-delete-{task_id}",
)
_running_tasks.add(bg)
bg.add_done_callback(_running_tasks.discard)
return task_id
def get_batch_delete_task(task_id: str) -> BatchDeleteTask | None:
return _tasks.get(task_id)
def _sync_delete(
provider_id: str,
key_ids: list[str],
task: BatchDeleteTask,
) -> int:
"""在线程中执行的同步删除逻辑,避免阻塞事件循环。
每批 cleanup + DELETE + commit 作为独立事务,减少锁持有时间。
"""
from src.database import create_session
from src.models.database import ProviderAPIKey
from src.services.provider_keys.key_side_effects import cleanup_key_references
db = create_session()
try:
affected = 0
for i in range(0, len(key_ids), _DELETE_BATCH_SIZE):
batch = key_ids[i : i + _DELETE_BATCH_SIZE]
try:
cleanup_key_references(db, batch)
result = db.execute(
sa_delete(ProviderAPIKey).where(
ProviderAPIKey.provider_id == provider_id,
ProviderAPIKey.id.in_(batch),
)
)
rowcount = getattr(result, "rowcount", 0) or 0
affected += int(rowcount)
db.commit()
except Exception:
try:
db.rollback()
except Exception:
pass
raise
# NOTE: task.deleted is written from the worker thread and read
# from the event-loop thread. Safe on CPython (GIL protects simple
# attribute assignment) but not guaranteed by the language spec.
task.deleted = affected
return affected
finally:
try:
db.close()
except Exception:
pass
async def _run_batch_delete(
task_id: str,
provider_id: str,
key_ids: list[str],
) -> None:
task = _tasks.get(task_id)
if task is None:
return
task.status = STATUS_RUNNING
try:
affected = await asyncio.to_thread(_sync_delete, provider_id, key_ids, task)
# 副作用(缓存失效等)是异步操作,在事件循环中执行
if affected > 0:
try:
from src.database import get_db_context
from src.services.provider_keys.key_side_effects import (
run_delete_key_side_effects,
)
with get_db_context() as db:
await run_delete_key_side_effects(
db=db,
provider_id=provider_id,
deleted_key_allowed_models=None,
)
except Exception as exc:
logger.error("batch delete side effects failed: {}", exc)
task.status = STATUS_COMPLETED
task.deleted = affected
task.message = f"{affected} keys deleted"
logger.info(
"[BATCH_DELETE_TASK] completed task={} provider={} total={} deleted={}",
task_id,
provider_id[:8],
len(key_ids),
affected,
)
except asyncio.CancelledError:
task.status = STATUS_FAILED
task.message = "task cancelled (shutdown)"
logger.warning(
"[BATCH_DELETE_TASK] cancelled task={} provider={} deleted={}",
task_id,
provider_id[:8],
task.deleted,
)
except Exception as exc:
task.status = STATUS_FAILED
task.message = str(exc)
logger.error(
"[BATCH_DELETE_TASK] failed task={} provider={}: {}",
task_id,
provider_id[:8],
exc,
)
finally:
task.finished_at = time.monotonic()
# 延迟清除已完成的任务
try:
await asyncio.sleep(_TASK_RETAIN_SECONDS)
except asyncio.CancelledError:
pass
_tasks.pop(task_id, None)

View File

@@ -4,7 +4,10 @@ Provider Key 写操作后的副作用处理。
from __future__ import annotations
from typing import Any
from sqlalchemy import delete as sa_delete
from sqlalchemy import select
from sqlalchemy.orm import Session
from src.api.base.models_service import invalidate_models_list_cache
@@ -15,6 +18,10 @@ from src.services.cache.provider_cache import ProviderCacheService
_SQLITE_BATCH_SIZE = 900
_DEFAULT_BATCH_SIZE = 2000
# RequestCandidate 等大表每轮 DELETE 的行数上限,
# 避免单条 DELETE 语句扫描过多行导致超时。
_ROW_DELETE_BATCH = 5000
def cleanup_key_references(db: Session, key_ids: list[str]) -> None:
"""在删除 ProviderAPIKey 前,先清理关联表记录,避免 CASCADE 级联删除超时。"""
@@ -22,11 +29,32 @@ def cleanup_key_references(db: Session, key_ids: list[str]) -> None:
return
batch_size = _resolve_batch_size(db)
for batch in _iter_batches(key_ids, batch_size):
db.execute(sa_delete(RequestCandidate).where(RequestCandidate.key_id.in_(batch)))
# RequestCandidate 数据量通常最大,分批按行数删除
_delete_in_row_batches(db, RequestCandidate, RequestCandidate.key_id, batch)
db.execute(sa_delete(GeminiFileMapping).where(GeminiFileMapping.key_id.in_(batch)))
db.execute(sa_delete(VideoTask).where(VideoTask.key_id.in_(batch)))
def _delete_in_row_batches(db: Session, model: Any, fk_col: Any, key_id_batch: list[str]) -> int:
"""分批删除大表记录,每次最多删除 _ROW_DELETE_BATCH 行,避免单条语句超时。"""
total = 0
while True:
row_ids = [
r[0]
for r in db.execute(
select(model.id).where(fk_col.in_(key_id_batch)).limit(_ROW_DELETE_BATCH)
)
]
if not row_ids:
break
result = db.execute(sa_delete(model).where(model.id.in_(row_ids)))
deleted = getattr(result, "rowcount", 0) or 0
total += deleted
if len(row_ids) < _ROW_DELETE_BATCH:
break
return total
def _resolve_batch_size(db: Session) -> int:
try:
bind = db.get_bind()