fix(startup): 收口 leader 失锁后的后台任务

- 为后台调度器注册失锁回调并只在 stop 成功后清空生命周期引用
- 停止调度器时移除定时 job，补充启动与任务协调器回归测试
- 降低多 worker 下重复调度风险，保持停机收口与统计聚合回归一致

tests/services/test_startup_toggles.py

@@ -1,12 +1,16 @@
 from __future__ import annotations
 
+import asyncio
 import inspect
+from collections.abc import Callable
 from datetime import datetime, timedelta, timezone
 from types import SimpleNamespace
+from typing import Any, cast
 from unittest.mock import MagicMock
 
 import pytest
 
+import src.main as main_module
 import src.services.system.maintenance_scheduler as maintenance_scheduler_module
 from src.config.settings import config
 from src.services.system.maintenance_scheduler import MaintenanceScheduler
@@ -45,6 +49,85 @@ async def test_maintenance_scheduler_start_skips_startup_task_when_disabled(
     assert created is False
 
 
+@pytest.mark.asyncio
+async def test_maintenance_scheduler_stop_cancels_startup_task_and_removes_jobs(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    scheduler = MaintenanceScheduler()
+    scheduler.running = True
+    scheduler._startup_task = asyncio.create_task(asyncio.sleep(3600))
+    removed_jobs: list[str] = []
+
+    monkeypatch.setattr(
+        maintenance_scheduler_module,
+        "get_scheduler",
+        lambda: SimpleNamespace(remove_job=lambda job_id: removed_jobs.append(job_id)),
+    )
+
+    await scheduler.stop()
+
+    assert scheduler.running is False
+    assert scheduler._startup_task is None
+    assert set(removed_jobs) == {
+        "stats_aggregation",
+        "stats_hourly_aggregation",
+        "wallet_daily_usage_aggregation",
+        "usage_cleanup",
+        "pool_monitor",
+        "http_client_idle_cleanup",
+        "pending_cleanup",
+        "audit_cleanup",
+        "gemini_file_mapping_cleanup",
+        "candidate_cleanup",
+        "db_maintenance",
+        "antigravity_ua_refresh",
+        scheduler.CHECKIN_JOB_ID,
+    }
+
+
+@pytest.mark.asyncio
+async def test_stop_service_on_lock_lost_keeps_state_when_stop_fails() -> None:
+    state = main_module.LifecycleState()
+    service = SimpleNamespace()
+    state.quota_scheduler = cast(Any, service)
+
+    async def fail_stop() -> None:
+        raise RuntimeError("boom")
+
+    await main_module._stop_service_on_lock_lost(
+        state,
+        lock_name="quota_scheduler",
+        service_name="月卡额度重置调度器",
+        state_attr="quota_scheduler",
+        stop=fail_stop,
+    )
+
+    assert state.quota_scheduler is service
+
+
+@pytest.mark.asyncio
+async def test_stop_service_on_lock_lost_clears_state_after_success() -> None:
+    state = main_module.LifecycleState()
+    service = SimpleNamespace()
+    state.quota_scheduler = cast(Any, service)
+    stopped = False
+
+    async def stop() -> None:
+        nonlocal stopped
+        stopped = True
+
+    await main_module._stop_service_on_lock_lost(
+        state,
+        lock_name="quota_scheduler",
+        service_name="月卡额度重置调度器",
+        state_attr="quota_scheduler",
+        stop=stop,
+    )
+
+    assert stopped is True
+    assert state.quota_scheduler is None
+
+
 def test_http_client_idle_cleanup_interval_env_invalid(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
@@ -276,7 +359,7 @@ async def test_perform_cleanup_deletes_first_and_uses_non_overlapping_windows(
 
     class _FakeDateTime(datetime):
         @classmethod
-        def now(cls, tz=None):  # type: ignore[override]
+        def now(cls, tz: timezone | None = None) -> datetime:  # type: ignore[override]
             if tz is None:
                 return fixed_now.replace(tzinfo=None)
             return fixed_now.astimezone(tz)
@@ -291,7 +374,7 @@ async def test_perform_cleanup_deletes_first_and_uses_non_overlapping_windows(
 
     calls: list[tuple[str, datetime, int, datetime | None]] = []
 
-    def _record(name: str, count: int):
+    def _record(name: str, count: int) -> Callable[..., int]:
         def _inner(
             cutoff_time: datetime,
             batch_size: int,
@@ -313,6 +396,10 @@ async def test_perform_cleanup_deletes_first_and_uses_non_overlapping_windows(
         "auto_delete_expired_keys": False,
     }
 
+    def _delete_old_records(cutoff_time: datetime, batch_size: int) -> int:
+        calls.append(("delete", cutoff_time, batch_size, None))
+        return 5
+
     monkeypatch.setattr(maintenance_scheduler_module, "datetime", _FakeDateTime)
     monkeypatch.setattr(
         maintenance_scheduler_module.asyncio, "get_running_loop", lambda: _FakeLoop()
@@ -330,8 +417,7 @@ async def test_perform_cleanup_deletes_first_and_uses_non_overlapping_windows(
     monkeypatch.setattr(
         scheduler,
         "_delete_old_records",
-        lambda cutoff_time, batch_size: calls.append(("delete", cutoff_time, batch_size, None))
-        or 5,
+        _delete_old_records,
     )
     monkeypatch.setattr(
         scheduler,

tests/services/test_stats_aggregator_optimization.py

@@ -5,6 +5,7 @@ from types import SimpleNamespace
 from typing import Any, cast
 
 import pytest
+from sqlalchemy.exc import IntegrityError
 
 from src.models.database import StatsDaily, StatsUserDaily
 from src.services.system.stats_aggregator import (
@@ -61,6 +62,20 @@ class _BatchUserStatsSession:
         self.commit_count += 1
 
 
+class _RetryCommitSession:
+    def __init__(self) -> None:
+        self.commit_count = 0
+        self.rollback_count = 0
+
+    def commit(self) -> None:
+        self.commit_count += 1
+        if self.commit_count == 1:
+            raise IntegrityError("insert", {}, Exception("duplicate key"))
+
+    def rollback(self) -> None:
+        self.rollback_count += 1
+
+
 def test_query_stats_hybrid_batches_statsdaily_lookup_and_merges_realtime_ranges(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
@@ -167,6 +182,79 @@ def test_aggregate_user_daily_stats_batch_updates_all_users_in_two_queries() ->
     assert user_two.total_cost == 0.0
 
 
+def test_aggregate_daily_stats_bundle_retries_after_integrity_error(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    target_day = datetime(2026, 3, 1, tzinfo=timezone.utc)
+    db = _RetryCommitSession()
+    stats_calls: list[SimpleNamespace] = []
+    stage_calls: list[str] = []
+
+    def fake_aggregate_daily_stats(*_args: object, **_kwargs: object) -> SimpleNamespace:
+        stats = SimpleNamespace(is_complete=False, aggregated_at=None)
+        stats_calls.append(stats)
+        stage_calls.append("daily")
+        return stats
+
+    def fake_model_stats(*_args: object, **_kwargs: object) -> list[object]:
+        stage_calls.append("model")
+        return []
+
+    def fake_provider_stats(*_args: object, **_kwargs: object) -> list[object]:
+        stage_calls.append("provider")
+        return []
+
+    def fake_api_key_stats(*_args: object, **_kwargs: object) -> list[object]:
+        stage_calls.append("api_key")
+        return []
+
+    def fake_error_stats(*_args: object, **_kwargs: object) -> list[object]:
+        stage_calls.append("error")
+        return []
+
+    def fake_user_daily_stats(*_args: object, **_kwargs: object) -> list[object]:
+        stage_calls.append("user")
+        return []
+
+    monkeypatch.setattr(StatsAggregatorService, "aggregate_daily_stats", fake_aggregate_daily_stats)
+    monkeypatch.setattr(StatsAggregatorService, "aggregate_daily_model_stats", fake_model_stats)
+    monkeypatch.setattr(
+        StatsAggregatorService, "aggregate_daily_provider_stats", fake_provider_stats
+    )
+    monkeypatch.setattr(StatsAggregatorService, "aggregate_daily_api_key_stats", fake_api_key_stats)
+    monkeypatch.setattr(StatsAggregatorService, "aggregate_daily_error_stats", fake_error_stats)
+    monkeypatch.setattr(
+        StatsAggregatorService, "aggregate_user_daily_stats_batch", fake_user_daily_stats
+    )
+
+    result = StatsAggregatorService.aggregate_daily_stats_bundle(
+        cast(Any, db),
+        target_day,
+        user_ids=["user-1"],
+    )
+
+    assert db.commit_count == 2
+    assert db.rollback_count == 1
+    assert len(stats_calls) == 2
+    assert stage_calls == [
+        "daily",
+        "model",
+        "provider",
+        "api_key",
+        "error",
+        "user",
+        "daily",
+        "model",
+        "provider",
+        "api_key",
+        "error",
+        "user",
+    ]
+    assert result is stats_calls[-1]
+    assert result.is_complete is True
+    assert result.aggregated_at is not None
+
+
 def test_compute_percentiles_by_local_day_returns_sqlite_fallback_without_queries() -> None:
     db = SimpleNamespace(bind=SimpleNamespace(dialect=SimpleNamespace(name="sqlite")))
     time_range = TimeRangeParams(

tests/unit/test_task_coordinator.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+
+from src.utils.task_coordinator import StartupTaskCoordinator
+
+
+class _FakeRedis:
+    def __init__(self, eval_results: list[int] | None = None, set_result: bool = True) -> None:
+        self.eval_results = list(eval_results or [])
+        self.eval_calls: list[tuple[Any, ...]] = []
+        self.set_result = set_result
+        self.set_calls: list[tuple[Any, ...]] = []
+
+    async def eval(self, script: str, numkeys: int, *args: Any) -> int:
+        self.eval_calls.append((script, numkeys, *args))
+        if self.eval_results:
+            return self.eval_results.pop(0)
+        return 1
+
+    async def set(self, key: str, value: str, *, nx: bool, ex: int) -> bool:
+        self.set_calls.append((key, value, nx, ex))
+        return self.set_result
+
+
+class _FakeTask:
+    def __init__(self) -> None:
+        self.cancelled = False
+
+    def cancel(self) -> None:
+        self.cancelled = True
+
+
+@pytest.mark.asyncio
+async def test_startup_task_coordinator_acquire_starts_refresh_for_redis_lock(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    redis = _FakeRedis(eval_results=[1])
+    coordinator = StartupTaskCoordinator(redis)
+    started: list[tuple[str, int]] = []
+
+    monkeypatch.setattr(
+        coordinator,
+        "_start_refresh_task",
+        lambda name, ttl: started.append((name, ttl)),
+    )
+
+    acquired = await coordinator.acquire("maintenance_scheduler", ttl=120)
+
+    assert acquired is True
+    assert "maintenance_scheduler" in coordinator._tokens
+    assert started == [("maintenance_scheduler", 120)]
+
+
+@pytest.mark.asyncio
+async def test_startup_task_coordinator_refresh_lock_extends_matching_token() -> None:
+    redis = _FakeRedis(eval_results=[1])
+    coordinator = StartupTaskCoordinator(redis)
+    coordinator._tokens["maintenance_scheduler"] = "token-1"
+
+    refreshed = await coordinator._refresh_lock("maintenance_scheduler", ttl=180)
+
+    assert refreshed is True
+    assert len(redis.eval_calls) == 1
+    _script, numkeys, key, token, ttl = redis.eval_calls[0]
+    assert numkeys == 1
+    assert key == "task_lock:maintenance_scheduler"
+    assert token == "token-1"
+    assert ttl == 180
+
+
+@pytest.mark.asyncio
+async def test_startup_task_coordinator_release_cancels_refresh_task() -> None:
+    redis = _FakeRedis(eval_results=[1])
+    coordinator = StartupTaskCoordinator(redis)
+    coordinator._tokens["maintenance_scheduler"] = "token-1"
+    refresh_task = _FakeTask()
+    coordinator._refresh_tasks["maintenance_scheduler"] = refresh_task  # type: ignore[assignment]
+
+    await coordinator.release("maintenance_scheduler")
+
+    assert refresh_task.cancelled is True
+    assert "maintenance_scheduler" not in coordinator._refresh_tasks
+    assert "maintenance_scheduler" not in coordinator._tokens
+    assert len(redis.eval_calls) == 1
+    _script, numkeys, key, token = redis.eval_calls[0]
+    assert numkeys == 1
+    assert key == "task_lock:maintenance_scheduler"
+    assert token == "token-1"
+
+
+@pytest.mark.asyncio
+async def test_startup_task_coordinator_notify_lock_lost_runs_registered_callback() -> None:
+    coordinator = StartupTaskCoordinator()
+    called: list[str] = []
+
+    async def on_lock_lost(name: str) -> None:
+        called.append(name)
+
+    coordinator.register_lock_lost_callback("maintenance_scheduler", on_lock_lost)
+
+    await coordinator._notify_lock_lost("maintenance_scheduler")
+
+    assert called == ["maintenance_scheduler"]