perf: Redis 操作优化,移除号池 key 列表的 Usage 聚合查询

- 号池 key 列表移除 Usage 表聚合查询(total_tokens/total_cost_usd),消除慢 SQL
- cooldown 计数从 SCAN 改为 SCARD (O(1)),通过 cooldown_idx SET 维护索引
- 读路径 Lua 脚本移除 ZREMRANGEBYSCORE,清理移至写路径减少开销
- affinity 清理从 KEYS 改为 SCAN 分批删除,invalidate_all_for_provider 改用 pipeline 批量 MGET+UNLINK
- 缓存监控页添加清除按钮 loading 状态,Redis SCAN 并发限制为 4
- 提取 redis_utils 模块统一 SCAN+批量删除逻辑
- 调度热路径跳过 cooldown TTL 查询,account_state 预计算移出循环
This commit is contained in:
fawney19
2026-03-09 14:54:45 +08:00
parent 9516619b92
commit afbb1b9a5d
11 changed files with 352 additions and 187 deletions

54
src/core/redis_utils.py Normal file
View File

@@ -0,0 +1,54 @@
"""Redis batch-delete utilities.
Provides SCAN-based pattern deletion with UNLINK preference to minimise
blocking on the Redis server. Used by cache monitoring and affinity
manager modules.
"""
from __future__ import annotations
from typing import Any
from src.core.logger import logger
# Defaults matching existing usage across the codebase.
DEFAULT_SCAN_BATCH_SIZE = 200
DEFAULT_DELETE_BATCH_SIZE = 500
async def delete_redis_keys(redis: Any, keys: list[str]) -> int:
"""Batch-delete Redis keys, preferring UNLINK over DELETE."""
if not keys:
return 0
try:
unlink = getattr(redis, "unlink", None)
if callable(unlink):
return int(await unlink(*keys))
except Exception as exc:
logger.debug("Redis UNLINK failed, falling back to DELETE: {}", exc)
return int(await redis.delete(*keys))
async def scan_delete_pattern(
redis: Any,
pattern: str,
*,
scan_batch_size: int = DEFAULT_SCAN_BATCH_SIZE,
delete_batch_size: int = DEFAULT_DELETE_BATCH_SIZE,
) -> int:
"""SCAN + batch-delete keys matching *pattern* without blocking Redis."""
deleted_count = 0
cursor: int | str = 0
while True:
cursor, keys = await redis.scan(cursor=cursor, match=pattern, count=scan_batch_size)
if keys:
for i in range(0, len(keys), delete_batch_size):
batch = keys[i : i + delete_batch_size]
deleted_count += await delete_redis_keys(redis, batch)
if int(cursor) == 0:
break
return deleted_count