perf: 依赖数据库 CASCADE/SET NULL 替代手动清理关联表,缩短删除事务

- 批量删除移除 cleanup_key_references 手动清理,改为依赖 FK CASCADE/SET NULL
- video_tasks.key_id FK 增加 ondelete="SET NULL",附带幂等迁移脚本
- _sync_delete 增加 statement_timeout 和任务级超时保护
- 批量导入在每次 await 前释放闲置 DB 连接,按批次提交写入避免长事务
- 前端轮询改为先查后等,首次查询不再多等一个间隔
This commit is contained in:
fawney19
2026-03-09 03:48:20 +08:00
parent 654ce89541
commit fa69287449
8 changed files with 468 additions and 18 deletions

View File

@@ -0,0 +1,81 @@
"""video_tasks.key_id: add ondelete SET NULL
Revision ID: a1b2c3d4e5f6
Revises: 2053ab8ed764
Create Date: 2026-03-09 01:00:00.000000+00:00
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "a1b2c3d4e5f6"
down_revision = "2053ab8ed764"
branch_labels = None
depends_on = None
_TABLE = "video_tasks"
_FK_NAME = "video_tasks_key_id_fkey"
def _fk_ondelete(table_name: str, constraint_name: str) -> str | None:
bind = op.get_bind()
result = bind.execute(
sa.text(
"SELECT rc.delete_rule "
"FROM information_schema.referential_constraints rc "
"JOIN information_schema.table_constraints tc "
" ON rc.constraint_name = tc.constraint_name "
"WHERE tc.table_name = :table AND tc.constraint_name = :name"
),
{"table": table_name, "name": constraint_name},
)
row = result.first()
return row[0] if row else None
def _replace_fk_if_needed(
constraint_name: str,
table_name: str,
ref_table: str,
local_cols: list[str],
remote_cols: list[str],
desired_ondelete: str,
) -> None:
current = _fk_ondelete(table_name, constraint_name)
if current and current.upper() == desired_ondelete.upper():
return
if current:
op.drop_constraint(constraint_name, table_name, type_="foreignkey")
op.create_foreign_key(
constraint_name,
table_name,
ref_table,
local_cols,
remote_cols,
ondelete=desired_ondelete,
)
def upgrade() -> None:
_replace_fk_if_needed(
_FK_NAME,
_TABLE,
"provider_api_keys",
["key_id"],
["id"],
"SET NULL",
)
def downgrade() -> None:
_replace_fk_if_needed(
_FK_NAME,
_TABLE,
"provider_api_keys",
["key_id"],
["id"],
"NO ACTION",
)

View File

@@ -585,7 +585,6 @@ async function pollDeleteTask(
const deadline = Date.now() + DELETE_POLL_MAX_MS const deadline = Date.now() + DELETE_POLL_MAX_MS
let consecutiveFailures = 0 let consecutiveFailures = 0
while (Date.now() < deadline) { while (Date.now() < deadline) {
await new Promise((r) => setTimeout(r, DELETE_POLL_INTERVAL_MS))
try { try {
const task = await getPoolBatchDeleteTask(providerId, taskId) const task = await getPoolBatchDeleteTask(providerId, taskId)
consecutiveFailures = 0 consecutiveFailures = 0
@@ -599,6 +598,7 @@ async function pollDeleteTask(
return { status: 'failed', deleted: 0 } return { status: 'failed', deleted: 0 }
} }
} }
await new Promise((r) => setTimeout(r, DELETE_POLL_INTERVAL_MS))
} }
return { status: 'failed', deleted: 0 } return { status: 'failed', deleted: 0 }
} }

View File

@@ -42,6 +42,7 @@ from src.database.database import get_db
from src.models.database import Provider, ProviderAPIKey, User from src.models.database import Provider, ProviderAPIKey, User
from src.services.provider.pool.config import parse_pool_config from src.services.provider.pool.config import parse_pool_config
from src.services.provider_keys.auth_type import OAUTH_AUTH_TYPES from src.services.provider_keys.auth_type import OAUTH_AUTH_TYPES
from src.services.scheduling.utils import release_db_connection_before_await
from src.utils.auth_utils import require_admin from src.utils.auth_utils import require_admin
router = APIRouter(prefix="/api/admin/provider-oauth", tags=["Provider OAuth"]) router = APIRouter(prefix="/api/admin/provider-oauth", tags=["Provider OAuth"])
@@ -129,6 +130,7 @@ _PROVIDER_OAUTH_BATCH_TASK_TTL_SECONDS = 24 * 3600
_PROVIDER_OAUTH_BATCH_TASK_MAX_ERROR_SAMPLES = 20 _PROVIDER_OAUTH_BATCH_TASK_MAX_ERROR_SAMPLES = 20
_PROVIDER_OAUTH_DEFAULT_TIMEOUT_SECONDS = 30.0 _PROVIDER_OAUTH_DEFAULT_TIMEOUT_SECONDS = 30.0
_PROVIDER_OAUTH_BATCH_IMPORT_PROXY_TIMEOUT_SECONDS = 60.0 _PROVIDER_OAUTH_BATCH_IMPORT_PROXY_TIMEOUT_SECONDS = 60.0
_PROVIDER_OAUTH_BATCH_IMPORT_COMMIT_BATCH_SIZE = 25
_PROVIDER_OAUTH_BATCH_TASK_ALLOWED_STATUSES = { _PROVIDER_OAUTH_BATCH_TASK_ALLOWED_STATUSES = {
"submitted", "submitted",
"processing", "processing",
@@ -1639,6 +1641,27 @@ def _estimate_batch_import_total(provider_type: str, raw_credentials: str) -> in
return len(_parse_standard_oauth_import_entries(raw_credentials)) return len(_parse_standard_oauth_import_entries(raw_credentials))
def _release_batch_import_db_connection_before_await(db: Session) -> None:
"""Best-effort 释放批量导入任务的只读 DB 连接。
批量导入会在单个后台任务里执行大量 await上游 token 校验、邮箱探测、Redis 进度更新)。
如果前面做过 Provider 查询而 Session 一直保持事务打开,连接会长时间占着不放,
在大批量导入且多数条目最终失败时尤其容易把连接池拖满。
这里复用调度器已有的 helper仅在 Session 没有挂起写入时才提前结束事务,
避免影响 flush 后尚未提交的数据。
"""
release_db_connection_before_await(db)
def _commit_batch_import_writes_if_needed(db: Session, pending_writes: int) -> int:
"""按固定批次提交导入写入,避免长事务持续占用连接。"""
if pending_writes < _PROVIDER_OAUTH_BATCH_IMPORT_COMMIT_BATCH_SIZE:
return pending_writes
db.commit()
return 0
def _apply_codex_import_hints(auth_config: dict[str, Any], import_entry: dict[str, str]) -> None: def _apply_codex_import_hints(auth_config: dict[str, Any], import_entry: dict[str, str]) -> None:
"""将导入文件中可用的 Codex 账号信息作为兜底补全(不覆盖已有值)。""" """将导入文件中可用的 Codex 账号信息作为兜底补全(不覆盖已有值)。"""
for field in ("account_id", "plan_type", "user_id", "email"): for field in ("account_id", "plan_type", "user_id", "email"):
@@ -1901,11 +1924,14 @@ async def _batch_import_standard_oauth_internal(
success_count = 0 success_count = 0
failed_count = 0 failed_count = 0
processed_count = 0 processed_count = 0
pending_success_writes = 0
db_lock = asyncio.Lock() db_lock = asyncio.Lock()
sem = asyncio.Semaphore(max(concurrency, 1)) sem = asyncio.Semaphore(max(concurrency, 1))
_release_batch_import_db_connection_before_await(db)
async def _process_entry(idx: int, import_entry: dict[str, Any]) -> None: async def _process_entry(idx: int, import_entry: dict[str, Any]) -> None:
nonlocal success_count, failed_count, processed_count nonlocal success_count, failed_count, processed_count, pending_success_writes
result_item: BatchImportResultItem result_item: BatchImportResultItem
async with sem: async with sem:
@@ -1951,6 +1977,7 @@ async def _batch_import_standard_oauth_internal(
json_body = None json_body = None
try: try:
_release_batch_import_db_connection_before_await(db)
resp = await post_oauth_token( resp = await post_oauth_token(
provider_type=provider_type, provider_type=provider_type,
token_url=token_url, token_url=token_url,
@@ -1970,6 +1997,7 @@ async def _batch_import_standard_oauth_internal(
processed_count += 1 processed_count += 1
results[idx] = result_item results[idx] = result_item
if progress_hook is not None: if progress_hook is not None:
_release_batch_import_db_connection_before_await(db)
await progress_hook( await progress_hook(
total, processed_count, success_count, failed_count, result_item total, processed_count, success_count, failed_count, result_item
) )
@@ -1997,6 +2025,7 @@ async def _batch_import_standard_oauth_internal(
processed_count += 1 processed_count += 1
results[idx] = result_item results[idx] = result_item
if progress_hook is not None: if progress_hook is not None:
_release_batch_import_db_connection_before_await(db)
await progress_hook( await progress_hook(
total, processed_count, success_count, failed_count, result_item total, processed_count, success_count, failed_count, result_item
) )
@@ -2016,6 +2045,7 @@ async def _batch_import_standard_oauth_internal(
processed_count += 1 processed_count += 1
results[idx] = result_item results[idx] = result_item
if progress_hook is not None: if progress_hook is not None:
_release_batch_import_db_connection_before_await(db)
await progress_hook( await progress_hook(
total, processed_count, success_count, failed_count, result_item total, processed_count, success_count, failed_count, result_item
) )
@@ -2039,6 +2069,7 @@ async def _batch_import_standard_oauth_internal(
} }
try: try:
_release_batch_import_db_connection_before_await(db)
auth_config = await enrich_auth_config( auth_config = await enrich_auth_config(
provider_type=provider_type, provider_type=provider_type,
auth_config=auth_config, auth_config=auth_config,
@@ -2067,6 +2098,7 @@ async def _batch_import_standard_oauth_internal(
processed_count += 1 processed_count += 1
results[idx] = result_item results[idx] = result_item
if progress_hook is not None: if progress_hook is not None:
_release_batch_import_db_connection_before_await(db)
await progress_hook( await progress_hook(
total, total,
processed_count, processed_count,
@@ -2108,6 +2140,11 @@ async def _batch_import_standard_oauth_internal(
proxy=key_proxy, proxy=key_proxy,
) )
pending_success_writes += 1
pending_success_writes = _commit_batch_import_writes_if_needed(
db, pending_success_writes
)
result_item = BatchImportResultItem( result_item = BatchImportResultItem(
index=idx, index=idx,
status="success", status="success",
@@ -2129,6 +2166,7 @@ async def _batch_import_standard_oauth_internal(
processed_count += 1 processed_count += 1
results[idx] = result_item results[idx] = result_item
if progress_hook is not None: if progress_hook is not None:
_release_batch_import_db_connection_before_await(db)
await progress_hook(total, processed_count, success_count, failed_count, result_item) await progress_hook(total, processed_count, success_count, failed_count, result_item)
await asyncio.gather( await asyncio.gather(
@@ -2136,7 +2174,7 @@ async def _batch_import_standard_oauth_internal(
return_exceptions=True, return_exceptions=True,
) )
if success_count > 0: if success_count > 0 and pending_success_writes > 0:
db.commit() db.commit()
final_results = [r for r in results if r is not None] final_results = [r for r in results if r is not None]
@@ -2446,11 +2484,14 @@ async def _batch_import_kiro_internal(
success_count = 0 success_count = 0
failed_count = 0 failed_count = 0
processed_count = 0 processed_count = 0
pending_success_writes = 0
db_lock = asyncio.Lock() db_lock = asyncio.Lock()
sem = asyncio.Semaphore(max(concurrency, 1)) sem = asyncio.Semaphore(max(concurrency, 1))
_release_batch_import_db_connection_before_await(db)
async def _process_entry(idx: int, cred: dict[str, Any]) -> None: async def _process_entry(idx: int, cred: dict[str, Any]) -> None:
nonlocal success_count, failed_count, processed_count nonlocal success_count, failed_count, processed_count, pending_success_writes
result_item: BatchImportResultItem result_item: BatchImportResultItem
async with sem: async with sem:
@@ -2466,6 +2507,7 @@ async def _batch_import_kiro_internal(
processed_count += 1 processed_count += 1
results[idx] = result_item results[idx] = result_item
if progress_hook is not None: if progress_hook is not None:
_release_batch_import_db_connection_before_await(db)
await progress_hook( await progress_hook(
total, processed_count, success_count, failed_count, result_item total, processed_count, success_count, failed_count, result_item
) )
@@ -2475,6 +2517,7 @@ async def _batch_import_kiro_internal(
cfg.provider_type = ProviderType.KIRO.value cfg.provider_type = ProviderType.KIRO.value
try: try:
_release_batch_import_db_connection_before_await(db)
access_token, new_cfg = await refresh_access_token( access_token, new_cfg = await refresh_access_token(
cfg, cfg,
proxy_config=proxy_config, proxy_config=proxy_config,
@@ -2490,11 +2533,13 @@ async def _batch_import_kiro_internal(
processed_count += 1 processed_count += 1
results[idx] = result_item results[idx] = result_item
if progress_hook is not None: if progress_hook is not None:
_release_batch_import_db_connection_before_await(db)
await progress_hook( await progress_hook(
total, processed_count, success_count, failed_count, result_item total, processed_count, success_count, failed_count, result_item
) )
return return
_release_batch_import_db_connection_before_await(db)
email = await _fetch_kiro_email(new_cfg.to_dict(), proxy_config=proxy_config) email = await _fetch_kiro_email(new_cfg.to_dict(), proxy_config=proxy_config)
if email and not new_cfg.email: if email and not new_cfg.email:
new_cfg.email = email new_cfg.email = email
@@ -2514,6 +2559,7 @@ async def _batch_import_kiro_internal(
processed_count += 1 processed_count += 1
results[idx] = result_item results[idx] = result_item
if progress_hook is not None: if progress_hook is not None:
_release_batch_import_db_connection_before_await(db)
await progress_hook( await progress_hook(
total, total,
processed_count, processed_count,
@@ -2550,6 +2596,11 @@ async def _batch_import_kiro_internal(
proxy=key_proxy, proxy=key_proxy,
) )
pending_success_writes += 1
pending_success_writes = _commit_batch_import_writes_if_needed(
db, pending_success_writes
)
result_item = BatchImportResultItem( result_item = BatchImportResultItem(
index=idx, index=idx,
status="success", status="success",
@@ -2572,6 +2623,7 @@ async def _batch_import_kiro_internal(
processed_count += 1 processed_count += 1
results[idx] = result_item results[idx] = result_item
if progress_hook is not None: if progress_hook is not None:
_release_batch_import_db_connection_before_await(db)
await progress_hook(total, processed_count, success_count, failed_count, result_item) await progress_hook(total, processed_count, success_count, failed_count, result_item)
await asyncio.gather( await asyncio.gather(
@@ -2580,7 +2632,7 @@ async def _batch_import_kiro_internal(
) )
# 提交所有成功的记录 # 提交所有成功的记录
if success_count > 0: if success_count > 0 and pending_success_writes > 0:
db.commit() db.commit()
final_results = [r for r in results if r is not None] final_results = [r for r in results if r is not None]

View File

@@ -1950,7 +1950,7 @@ class VideoTask(Base):
api_key_name = Column(String(200), nullable=True, comment="API Key 名称快照") api_key_name = Column(String(200), nullable=True, comment="API Key 名称快照")
provider_id = Column(String(36), ForeignKey("providers.id"), index=True) provider_id = Column(String(36), ForeignKey("providers.id"), index=True)
endpoint_id = Column(String(36), ForeignKey("provider_endpoints.id"), index=True) endpoint_id = Column(String(36), ForeignKey("provider_endpoints.id"), index=True)
key_id = Column(String(36), ForeignKey("provider_api_keys.id"), index=True) key_id = Column(String(36), ForeignKey("provider_api_keys.id", ondelete="SET NULL"), index=True)
# 格式转换追踪 # 格式转换追踪
client_api_format = Column(String(50), nullable=False) client_api_format = Column(String(50), nullable=False)

View File

@@ -8,12 +8,14 @@ from __future__ import annotations
import asyncio import asyncio
import json import json
import time
import uuid import uuid
from collections.abc import Callable from collections.abc import Callable
from concurrent.futures import Future from concurrent.futures import Future
import redis.asyncio as aioredis import redis.asyncio as aioredis
from sqlalchemy import delete as sa_delete from sqlalchemy import delete as sa_delete
from sqlalchemy import text
from src.clients.redis_client import get_redis_client from src.clients.redis_client import get_redis_client
from src.core.logger import logger from src.core.logger import logger
@@ -30,6 +32,12 @@ _TASK_RETAIN_SECONDS = 600
# 批量删除时每个独立事务处理的 key 数量 # 批量删除时每个独立事务处理的 key 数量
_CLEANUP_BATCH_SIZE = 50 _CLEANUP_BATCH_SIZE = 50
# 单个批次的数据库 statement 超时(秒)
_BATCH_STATEMENT_TIMEOUT_S = 30
# 整个任务的最大执行时间(秒)
_TASK_TIMEOUT_S = 600
# Redis key 前缀 # Redis key 前缀
_REDIS_KEY_PREFIX = "batch_delete_task" _REDIS_KEY_PREFIX = "batch_delete_task"
@@ -175,23 +183,35 @@ def _sync_delete(
) -> int: ) -> int:
"""在线程中执行的同步删除逻辑,避免阻塞事件循环。 """在线程中执行的同步删除逻辑,避免阻塞事件循环。
按小批次_CLEANUP_BATCH_SIZE清理关联表并删除 key 按小批次_CLEANUP_BATCH_SIZE直接删除 key依赖数据库 CASCADE/SET NULL 自动清理关联表。
每个批次独立事务,单批失败跳过并继续,确保进度条持续推进 每个批次独立事务,单批失败跳过并继续。
""" """
from src.database import create_session from src.database import create_session
from src.models.database import ProviderAPIKey from src.models.database import ProviderAPIKey
from src.services.provider_keys.key_side_effects import cleanup_key_references
db = create_session() db = create_session()
try: try:
affected = 0 affected = 0
total_batches = (len(key_ids) + _CLEANUP_BATCH_SIZE - 1) // _CLEANUP_BATCH_SIZE total_batches = (len(key_ids) + _CLEANUP_BATCH_SIZE - 1) // _CLEANUP_BATCH_SIZE
batch_idx = 0 batch_idx = 0
task_start = time.monotonic()
for i in range(0, len(key_ids), _CLEANUP_BATCH_SIZE): for i in range(0, len(key_ids), _CLEANUP_BATCH_SIZE):
# 整体超时保护
if time.monotonic() - task_start > _TASK_TIMEOUT_S:
logger.warning(
"[BATCH_DELETE] task timeout after {}s, deleted {}/{}",
_TASK_TIMEOUT_S,
affected,
len(key_ids),
)
break
batch = key_ids[i : i + _CLEANUP_BATCH_SIZE] batch = key_ids[i : i + _CLEANUP_BATCH_SIZE]
batch_idx += 1 batch_idx += 1
try: try:
cleanup_key_references(db, batch) # 设置 statement_timeout防止单条 SQL 无限等锁
timeout_ms = _BATCH_STATEMENT_TIMEOUT_S * 1000
db.execute(text(f"SET LOCAL statement_timeout = '{timeout_ms}'"))
result = db.execute( result = db.execute(
sa_delete(ProviderAPIKey).where( sa_delete(ProviderAPIKey).where(
ProviderAPIKey.provider_id == provider_id, ProviderAPIKey.provider_id == provider_id,
@@ -203,7 +223,9 @@ def _sync_delete(
db.commit() db.commit()
except Exception as exc: except Exception as exc:
logger.warning( logger.warning(
"[BATCH_DELETE] batch failed (keys {}-{}): {}", "[BATCH_DELETE] batch {}/{} failed (keys {}-{}): {}",
batch_idx,
total_batches,
i, i,
i + len(batch), i + len(batch),
exc, exc,
@@ -212,9 +234,16 @@ def _sync_delete(
db.rollback() db.rollback()
except Exception: except Exception:
pass pass
# 每 5 个批次或最后一个批次上报进度,避免过于频繁写 Redis # 每个批次都上报一次进度
if progress_callback is not None and (batch_idx % 5 == 0 or batch_idx == total_batches): if progress_callback is not None:
progress_callback(affected) progress_callback(affected)
logger.info(
"[BATCH_DELETE] sync delete finished: deleted={}/{} batches={} elapsed={:.1f}s",
affected,
len(key_ids),
total_batches,
time.monotonic() - task_start,
)
return affected return affected
finally: finally:
try: try:
@@ -230,6 +259,12 @@ async def _run_batch_delete(
) -> None: ) -> None:
r = await get_redis_client(require_redis=False) r = await get_redis_client(require_redis=False)
await _update_task_field(task_id, r=r, status=STATUS_RUNNING) await _update_task_field(task_id, r=r, status=STATUS_RUNNING)
logger.info(
"[BATCH_DELETE_TASK] started task={} provider={} total={}",
task_id,
provider_id[:8],
len(key_ids),
)
# 从工作线程安全地触发 Redis 进度更新 # 从工作线程安全地触发 Redis 进度更新
loop = asyncio.get_running_loop() loop = asyncio.get_running_loop()
@@ -245,7 +280,10 @@ async def _run_batch_delete(
pass pass
try: try:
affected = await asyncio.to_thread(_sync_delete, provider_id, key_ids, on_progress) affected = await asyncio.wait_for(
asyncio.to_thread(_sync_delete, provider_id, key_ids, on_progress),
timeout=_TASK_TIMEOUT_S + 30, # 留一点余量给 _sync_delete 内部超时
)
if progress_futures: if progress_futures:
results = await asyncio.gather( results = await asyncio.gather(
@@ -292,6 +330,15 @@ async def _run_batch_delete(
affected, affected,
) )
except asyncio.TimeoutError:
msg = f"task timeout after {_TASK_TIMEOUT_S + 30}s"
await _update_task_field(task_id, r=r, status=STATUS_FAILED, message=msg)
logger.error(
"[BATCH_DELETE_TASK] {} task={} provider={}",
msg,
task_id,
provider_id[:8],
)
except asyncio.CancelledError: except asyncio.CancelledError:
await _update_task_field( await _update_task_field(
task_id, task_id,

View File

@@ -31,7 +31,6 @@ from src.services.provider.fingerprint import generate_fingerprint, normalize_fi
from src.services.provider_keys.auth_type import normalize_auth_type from src.services.provider_keys.auth_type import normalize_auth_type
from src.services.provider_keys.duplicate_check import check_duplicate_key from src.services.provider_keys.duplicate_check import check_duplicate_key
from src.services.provider_keys.key_side_effects import ( from src.services.provider_keys.key_side_effects import (
cleanup_key_references,
run_create_key_side_effects, run_create_key_side_effects,
run_delete_key_side_effects, run_delete_key_side_effects,
run_update_key_side_effects, run_update_key_side_effects,
@@ -519,12 +518,10 @@ async def batch_delete_endpoint_keys_response(db: Session, key_ids: list[str]) -
# 收集受影响的 provider_id # 收集受影响的 provider_id
affected_provider_ids = {key.provider_id for key in keys if key.provider_id} affected_provider_ids = {key.provider_id for key in keys if key.provider_id}
# 批量 SQL DELETE一次提交 # 批量 SQL DELETE依赖数据库 CASCADE/SET NULL 自动清理关联表
success_count = 0 success_count = 0
try: try:
found_id_list = list(found_ids) found_id_list = list(found_ids)
# 先清理关联表,避免 CASCADE 级联删除超时
cleanup_key_references(db, found_id_list)
db.execute(sa_delete(ProviderAPIKey).where(ProviderAPIKey.id.in_(found_id_list))) db.execute(sa_delete(ProviderAPIKey).where(ProviderAPIKey.id.in_(found_id_list)))
db.commit() db.commit()
success_count = len(found_ids) success_count = len(found_ids)

View File

@@ -0,0 +1,210 @@
from __future__ import annotations
from itertools import count
from types import SimpleNamespace
from unittest.mock import MagicMock
import httpx
import pytest
from src.api.admin import provider_oauth as oauthmod
@pytest.mark.asyncio
async def test_standard_batch_import_releases_db_connection_before_network_await(
monkeypatch: pytest.MonkeyPatch,
) -> None:
release_calls: list[str] = []
monkeypatch.setattr(
oauthmod,
"_require_oauth_template",
lambda _provider_type: SimpleNamespace(
oauth=SimpleNamespace(
token_url="https://example.com/oauth/token",
client_id="client-id",
client_secret=None,
scopes=[],
)
),
)
monkeypatch.setattr(
oauthmod,
"_parse_standard_oauth_import_entries",
lambda _raw: [{"refresh_token": "r" * 120}],
)
monkeypatch.setattr(oauthmod, "_get_provider_api_formats", lambda _provider: [])
monkeypatch.setattr(
oauthmod,
"_release_batch_import_db_connection_before_await",
lambda _db: release_calls.append("release"),
)
async def _fake_post_oauth_token(**_kwargs: object) -> httpx.Response:
raise RuntimeError("upstream unavailable")
monkeypatch.setattr(oauthmod, "post_oauth_token", _fake_post_oauth_token)
db = MagicMock()
result = await oauthmod._batch_import_standard_oauth_internal(
provider_id="provider-1",
provider_type="codex",
provider=SimpleNamespace(endpoints=[]), # type: ignore[arg-type]
raw_credentials="ignored",
db=db,
concurrency=1,
)
assert result.total == 1
assert result.success == 0
assert result.failed == 1
assert release_calls
db.commit.assert_not_called()
@pytest.mark.asyncio
async def test_standard_batch_import_commits_successes_in_chunks(
monkeypatch: pytest.MonkeyPatch,
) -> None:
key_ids = count(1)
monkeypatch.setattr(
oauthmod,
"_PROVIDER_OAUTH_BATCH_IMPORT_COMMIT_BATCH_SIZE",
2,
)
monkeypatch.setattr(
oauthmod,
"_require_oauth_template",
lambda _provider_type: SimpleNamespace(
oauth=SimpleNamespace(
token_url="https://example.com/oauth/token",
client_id="client-id",
client_secret=None,
scopes=[],
)
),
)
monkeypatch.setattr(
oauthmod,
"_parse_standard_oauth_import_entries",
lambda _raw: [{"refresh_token": f"r-{idx}" + ("x" * 120)} for idx in range(3)],
)
monkeypatch.setattr(oauthmod, "_get_provider_api_formats", lambda _provider: ["responses"])
monkeypatch.setattr(
oauthmod,
"_release_batch_import_db_connection_before_await",
lambda _db: None,
)
async def _fake_post_oauth_token(**_kwargs: object) -> httpx.Response:
idx = next(key_ids)
return httpx.Response(
200,
json={
"access_token": f"access-{idx}",
"refresh_token": f"refresh-{idx}",
"expires_in": 3600,
},
request=httpx.Request("POST", "https://example.com/oauth/token"),
)
async def _fake_enrich_auth_config(**kwargs: object) -> dict[str, object]:
auth_config = dict(kwargs["auth_config"]) # type: ignore[call-overload]
auth_config["email"] = f"user-{next(key_ids)}@example.com"
return auth_config
created_ids = count(1)
monkeypatch.setattr(oauthmod, "post_oauth_token", _fake_post_oauth_token)
monkeypatch.setattr(oauthmod, "enrich_auth_config", _fake_enrich_auth_config)
monkeypatch.setattr(oauthmod, "_check_duplicate_oauth_account", lambda *_args, **_kwargs: None)
monkeypatch.setattr(
oauthmod,
"_create_oauth_key",
lambda *_args, **_kwargs: SimpleNamespace(id=f"key-{next(created_ids)}"),
)
db = MagicMock()
result = await oauthmod._batch_import_standard_oauth_internal(
provider_id="provider-1",
provider_type="example",
provider=SimpleNamespace(endpoints=[]), # type: ignore[arg-type]
raw_credentials="ignored",
db=db,
concurrency=1,
)
assert result.total == 3
assert result.success == 3
assert result.failed == 0
assert db.commit.call_count == 2
@pytest.mark.asyncio
async def test_kiro_batch_import_releases_db_connection_before_refresh(
monkeypatch: pytest.MonkeyPatch,
) -> None:
release_calls: list[str] = []
class FakeKiroAuthConfig:
def __init__(self, data: dict[str, object]) -> None:
self._data = dict(data)
self.provider_type = str(data.get("provider_type") or "")
self.email = data.get("email") if isinstance(data.get("email"), str) else None
self.auth_method = (
data.get("auth_method") if isinstance(data.get("auth_method"), str) else "social"
)
self.refresh_token = str(data.get("refresh_token") or "")
@staticmethod
def validate_required_fields(_cred: dict[str, object]) -> tuple[bool, str | None]:
return True, None
@classmethod
def from_dict(cls, data: dict[str, object]) -> "FakeKiroAuthConfig":
return cls(data)
def to_dict(self) -> dict[str, object]:
return dict(self._data)
monkeypatch.setattr(
oauthmod,
"_parse_kiro_import_input",
lambda _raw: [{"refresh_token": "r" * 120, "auth_method": "social"}],
)
monkeypatch.setattr(oauthmod, "_get_provider_api_formats", lambda _provider: [])
monkeypatch.setattr(
oauthmod,
"_release_batch_import_db_connection_before_await",
lambda _db: release_calls.append("release"),
)
monkeypatch.setattr(
"src.services.provider.adapters.kiro.models.credentials.KiroAuthConfig",
FakeKiroAuthConfig,
)
async def _fake_refresh_access_token(*_args: object, **_kwargs: object) -> tuple[str, object]:
raise RuntimeError("refresh token reused")
monkeypatch.setattr(
"src.services.provider.adapters.kiro.token_manager.refresh_access_token",
_fake_refresh_access_token,
)
db = MagicMock()
result = await oauthmod._batch_import_kiro_internal(
provider_id="provider-1",
provider=SimpleNamespace(endpoints=[]), # type: ignore[arg-type]
raw_credentials="ignored",
db=db,
concurrency=1,
)
assert result.total == 1
assert result.success == 0
assert result.failed == 1
assert release_calls
db.commit.assert_not_called()

View File

@@ -4,6 +4,7 @@ import asyncio
from collections.abc import Coroutine, Generator from collections.abc import Coroutine, Generator
from concurrent.futures import Future from concurrent.futures import Future
from contextlib import contextmanager from contextlib import contextmanager
from types import SimpleNamespace
import pytest import pytest
@@ -96,3 +97,65 @@ async def test_run_batch_delete_waits_for_progress_updates_before_completion(
"message": "1 keys deleted", "message": "1 keys deleted",
}, },
] ]
def test_sync_delete_reports_progress_after_each_batch(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class _FakeColumn:
def __eq__(self, other: object) -> tuple[str, object]: # type: ignore[override]
return ("eq", other)
def in_(self, values: list[str]) -> tuple[str, tuple[str, ...]]:
return ("in", tuple(values))
class _FakeProviderAPIKey:
provider_id = _FakeColumn()
id = _FakeColumn()
class _FakeDeleteStatement:
def where(self, *_conditions: object) -> "_FakeDeleteStatement":
return self
class _FakeSession:
def __init__(self) -> None:
self.rowcounts = [2, 1]
self.commits = 0
self.closed = False
def execute(self, _statement: object) -> SimpleNamespace:
# SET LOCAL statement_timeout 不消耗 rowcount
if hasattr(_statement, "text"):
return SimpleNamespace(rowcount=0)
return SimpleNamespace(rowcount=self.rowcounts.pop(0))
def commit(self) -> None:
self.commits += 1
def rollback(self) -> None:
raise AssertionError("rollback should not be called")
def close(self) -> None:
self.closed = True
session = _FakeSession()
progress_updates: list[int] = []
monkeypatch.setattr(taskmod, "_CLEANUP_BATCH_SIZE", 2)
monkeypatch.setattr("src.database.create_session", lambda: session)
monkeypatch.setattr(
"src.models.database.ProviderAPIKey",
_FakeProviderAPIKey,
)
monkeypatch.setattr(taskmod, "sa_delete", lambda _model: _FakeDeleteStatement())
affected = taskmod._sync_delete(
"provider-1",
["key-1", "key-2", "key-3"],
progress_updates.append,
)
assert affected == 3
assert progress_updates == [2, 3]
assert session.commits == 2
assert session.closed is True