mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-01 17:00:21 +08:00
feat(cleanup): 解耦 request_candidates 与 provider_api_keys 生命周期
- 移除 request_candidates.key_id 对 provider_api_keys 的外键约束(含迁移脚本) - 删除 Key 时不再级联删除候选记录,改为独立按保留天数定时清理 - 新增 request_candidates_retention_days / request_candidates_cleanup_batch_size 配置项 - batch_delete_task 增加 lock_timeout 及超时自动降批重试机制 - cleanup_key_references 提取阶段化清理流程,移除 RequestCandidate 联动删除 - 前端 CleanupPolicySection 新增候选记录保留天数和清理批次配置 Closes #227 Co-authored-by: Entropy-Xu <entropy.xu@cloudhabitatsh.com>
This commit is contained in:
@@ -122,10 +122,12 @@ def test_sync_delete_reports_progress_after_each_batch(
|
||||
self.rowcounts = [2, 1]
|
||||
self.commits = 0
|
||||
self.closed = False
|
||||
self.text_statements: list[str] = []
|
||||
|
||||
def execute(self, _statement: object) -> SimpleNamespace:
|
||||
# SET LOCAL statement_timeout 不消耗 rowcount
|
||||
# SET LOCAL statement_timeout / lock_timeout 不消耗 rowcount
|
||||
if hasattr(_statement, "text"):
|
||||
self.text_statements.append(_statement.text)
|
||||
return SimpleNamespace(rowcount=0)
|
||||
return SimpleNamespace(rowcount=self.rowcounts.pop(0))
|
||||
|
||||
@@ -146,7 +148,7 @@ def test_sync_delete_reports_progress_after_each_batch(
|
||||
monkeypatch.setattr(
|
||||
taskmod,
|
||||
"cleanup_key_references",
|
||||
lambda _db, batch: cleanup_batches.append(list(batch)),
|
||||
lambda _db, batch, **_kwargs: cleanup_batches.append(list(batch)),
|
||||
)
|
||||
monkeypatch.setattr("src.database.create_session", lambda: session)
|
||||
monkeypatch.setattr(
|
||||
@@ -165,4 +167,85 @@ def test_sync_delete_reports_progress_after_each_batch(
|
||||
assert progress_updates == [2, 3]
|
||||
assert cleanup_batches == [["key-1", "key-2"], ["key-3"]]
|
||||
assert session.commits == 2
|
||||
assert session.text_statements == [
|
||||
"SET LOCAL statement_timeout = '30000'",
|
||||
"SET LOCAL lock_timeout = '5000'",
|
||||
"SET LOCAL statement_timeout = '30000'",
|
||||
"SET LOCAL lock_timeout = '5000'",
|
||||
]
|
||||
assert session.closed is True
|
||||
|
||||
|
||||
def test_sync_delete_retries_with_smaller_batches_after_timeout(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
class _FakeColumn:
|
||||
def __eq__(self, other: object) -> tuple[str, object]: # type: ignore[override]
|
||||
return ("eq", other)
|
||||
|
||||
def in_(self, values: list[str]) -> tuple[str, tuple[str, ...]]:
|
||||
return ("in", tuple(values))
|
||||
|
||||
class _FakeProviderAPIKey:
|
||||
provider_id = _FakeColumn()
|
||||
id = _FakeColumn()
|
||||
|
||||
class _FakeDeleteStatement:
|
||||
def where(self, *_conditions: object) -> "_FakeDeleteStatement":
|
||||
return self
|
||||
|
||||
class _FakeSession:
|
||||
def __init__(self) -> None:
|
||||
self.rowcounts = [2, 2]
|
||||
self.commits = 0
|
||||
self.rollbacks = 0
|
||||
self.closed = False
|
||||
|
||||
def execute(self, _statement: object) -> SimpleNamespace:
|
||||
if hasattr(_statement, "text"):
|
||||
return SimpleNamespace(rowcount=0)
|
||||
return SimpleNamespace(rowcount=self.rowcounts.pop(0))
|
||||
|
||||
def commit(self) -> None:
|
||||
self.commits += 1
|
||||
|
||||
def rollback(self) -> None:
|
||||
self.rollbacks += 1
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
session = _FakeSession()
|
||||
progress_updates: list[int] = []
|
||||
cleanup_batches: list[list[str]] = []
|
||||
|
||||
def fake_cleanup(_db: object, batch: list[str], **_kwargs: object) -> None:
|
||||
cleanup_batches.append(list(batch))
|
||||
if len(batch) >= 4:
|
||||
raise RuntimeError(
|
||||
"(psycopg2.errors.QueryCanceled) canceling statement due to statement timeout"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(taskmod, "_CLEANUP_BATCH_SIZE", 4)
|
||||
monkeypatch.setattr(taskmod, "_MIN_RETRY_BATCH_SIZE", 1)
|
||||
monkeypatch.setattr(taskmod, "cleanup_key_references", fake_cleanup)
|
||||
monkeypatch.setattr("src.database.create_session", lambda: session)
|
||||
monkeypatch.setattr("src.models.database.ProviderAPIKey", _FakeProviderAPIKey)
|
||||
monkeypatch.setattr(taskmod, "sa_delete", lambda _model: _FakeDeleteStatement())
|
||||
|
||||
affected = taskmod._sync_delete(
|
||||
"provider-1",
|
||||
["key-1", "key-2", "key-3", "key-4"],
|
||||
progress_updates.append,
|
||||
)
|
||||
|
||||
assert affected == 4
|
||||
assert progress_updates == [4]
|
||||
assert cleanup_batches == [
|
||||
["key-1", "key-2", "key-3", "key-4"],
|
||||
["key-1", "key-2"],
|
||||
["key-3", "key-4"],
|
||||
]
|
||||
assert session.commits == 2
|
||||
assert session.rollbacks == 1
|
||||
assert session.closed is True
|
||||
|
||||
@@ -287,13 +287,12 @@ def test_cleanup_key_references_preserves_usage_and_video_tasks(
|
||||
side_effects_module.cleanup_key_references(cast(Any, db), ["key-1", "key-2"])
|
||||
|
||||
assert [(stmt.kind, stmt.model.__name__) for stmt in db.statements] == [
|
||||
("delete", "RequestCandidate"),
|
||||
("delete", "GeminiFileMapping"),
|
||||
("update", "Usage"),
|
||||
("update", "VideoTask"),
|
||||
]
|
||||
assert db.statements[2].values_dict == {"provider_api_key_id": None}
|
||||
assert db.statements[3].values_dict == {"key_id": None}
|
||||
assert db.statements[1].values_dict == {"provider_api_key_id": None}
|
||||
assert db.statements[2].values_dict == {"key_id": None}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -41,3 +42,76 @@ async def test_maintenance_scheduler_start_skips_startup_task_when_disabled(
|
||||
await scheduler.start()
|
||||
|
||||
assert created is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_candidate_cleanup_uses_dedicated_retention_and_batch_settings(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
scheduler = MaintenanceScheduler()
|
||||
|
||||
class _FakeLoop:
|
||||
async def run_in_executor(self, _executor, func): # type: ignore[no-untyped-def]
|
||||
return func()
|
||||
|
||||
class _ConfigSession:
|
||||
def close(self) -> None:
|
||||
return None
|
||||
|
||||
class _BatchSession:
|
||||
def __init__(self, ids: list[str]) -> None:
|
||||
self.ids = ids
|
||||
self.closed = False
|
||||
self.committed = False
|
||||
self.query_obj = MagicMock()
|
||||
filtered = self.query_obj.filter.return_value
|
||||
filtered.order_by.return_value.limit.return_value.all.return_value = [
|
||||
SimpleNamespace(id=value) for value in ids
|
||||
]
|
||||
|
||||
def query(self, _model): # type: ignore[no-untyped-def]
|
||||
return self.query_obj
|
||||
|
||||
def execute(self, _statement): # type: ignore[no-untyped-def]
|
||||
return SimpleNamespace(rowcount=len(self.ids))
|
||||
|
||||
def commit(self) -> None:
|
||||
self.committed = True
|
||||
|
||||
def rollback(self) -> None:
|
||||
raise AssertionError("rollback should not be called")
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
config_session = _ConfigSession()
|
||||
batch_one = _BatchSession(["candidate-1", "candidate-2"])
|
||||
batch_two = _BatchSession([])
|
||||
sessions = iter([config_session, batch_one, batch_two])
|
||||
|
||||
def fake_create_session(): # type: ignore[no-untyped-def]
|
||||
return next(sessions)
|
||||
|
||||
config_values = {
|
||||
"enable_auto_cleanup": True,
|
||||
"request_candidates_retention_days": 21,
|
||||
"request_candidates_cleanup_batch_size": 2,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(maintenance_scheduler_module, "create_session", fake_create_session)
|
||||
monkeypatch.setattr(
|
||||
maintenance_scheduler_module.SystemConfigService,
|
||||
"get_config",
|
||||
lambda _db, key, default=None: config_values.get(key, default),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
maintenance_scheduler_module.asyncio, "get_running_loop", lambda: _FakeLoop()
|
||||
)
|
||||
|
||||
await scheduler._perform_candidate_cleanup()
|
||||
|
||||
batch_one.query_obj.filter.return_value.order_by.return_value.limit.return_value.all.assert_called_once()
|
||||
batch_one.query_obj.filter.return_value.order_by.return_value.limit.assert_called_once_with(2)
|
||||
assert batch_one.committed is True
|
||||
assert batch_one.closed is True
|
||||
assert batch_two.closed is True
|
||||
|
||||
Reference in New Issue
Block a user