fix(provider_keys): 提升 Codex 配额异步同步的可靠性与可观测性

- 为 flush 引入 FlushResult，统一返回更新数与重试批次
- 增加指数退避与失败日志限流，成功后重置 backoff
- 批量提交失败时回退到单条提交，降低整批失败风险
- 补充测试，覆盖 flush 重试与提交失败回退场景

src/services/provider_keys/codex_quota_sync_dispatcher.py

@@ -10,6 +10,8 @@ Codex 配额实时同步调度器（异步去重版）。
 from __future__ import annotations
 
 import asyncio
+import time
+from dataclasses import dataclass
 from threading import Lock
 from typing import Any
 
@@ -20,16 +22,33 @@ from src.database.database import create_session
 from src.services.provider_keys.codex_realtime_quota import sync_codex_quota_from_response_headers
 
 
+@dataclass(slots=True)
+class FlushResult:
+    queued_count: int
+    updated_count: int
+    retry_batch: dict[str, dict[str, Any]]
+
+
 class CodexQuotaSyncDispatcher:
     """Codex 配额同步异步调度器。"""
 
-    def __init__(self, flush_interval_seconds: float = 0.5) -> None:
+    def __init__(
-        self.flush_interval_seconds = flush_interval_seconds
+        self,
+        flush_interval_seconds: float = 0.5,
+        *,
+        max_backoff_seconds: float = 8.0,
+        error_log_interval_seconds: float = 30.0,
+    ) -> None:
+        self.flush_interval_seconds = max(float(flush_interval_seconds), 0.001)
+        self.max_backoff_seconds = max(float(max_backoff_seconds), self.flush_interval_seconds)
+        self.error_log_interval_seconds = max(float(error_log_interval_seconds), 0.0)
         self._pending: dict[str, dict[str, Any]] = {}
         self._pending_lock = Lock()
         self._event: asyncio.Event | None = None
         self._loop: asyncio.AbstractEventLoop | None = None
         self._task: asyncio.Task[None] | None = None
+        self._current_flush_delay_seconds = self.flush_interval_seconds
+        self._last_flush_error_log_at: float | None = None
         self._running = False
 
     async def start(self) -> None:
@@ -38,10 +57,13 @@ class CodexQuotaSyncDispatcher:
         self._loop = asyncio.get_running_loop()
         self._event = asyncio.Event()
         self._task = asyncio.create_task(self._run(), name="codex-quota-sync-dispatcher")
+        self._current_flush_delay_seconds = self.flush_interval_seconds
+        self._last_flush_error_log_at = None
         self._running = True
         logger.info(
-            "Codex 配额异步同步器已启动，flush_interval={}s",
+            "Codex 配额异步同步器已启动，flush_interval={}s, max_backoff={}s",
             self.flush_interval_seconds,
+            self.max_backoff_seconds,
         )
 
     async def stop(self) -> None:
@@ -110,14 +132,35 @@ class CodexQuotaSyncDispatcher:
         try:
             while True:
                 await event.wait()
-                await asyncio.sleep(self.flush_interval_seconds)
+                await asyncio.sleep(self._current_flush_delay_seconds)
                 batch = self._drain_pending()
                 if batch:
                     try:
-                        await asyncio.to_thread(self._flush_batch_sync, batch)
+                        result = await asyncio.to_thread(self._flush_batch_sync, batch)
                     except Exception as exc:
-                        logger.warning("Codex 配额异步同步器 flush 失败，将在下一轮重试: {}", exc)
                         self._merge_back_pending(batch)
+                        self._increase_backoff()
+                        self._log_flush_failure_with_rate_limit(
+                            queued_count=len(batch),
+                            retry_count=len(batch),
+                            error=exc,
+                        )
+                    else:
+                        if result.updated_count > 0:
+                            logger.debug(
+                                "异步同步 Codex 配额完成: queued_keys={}, updated_keys={}",
+                                result.queued_count,
+                                result.updated_count,
+                            )
+                        if result.retry_batch:
+                            self._merge_back_pending(result.retry_batch)
+                            self._increase_backoff()
+                            self._log_flush_failure_with_rate_limit(
+                                queued_count=result.queued_count,
+                                retry_count=len(result.retry_batch),
+                            )
+                        else:
+                            self._reset_backoff()
                 with self._pending_lock:
                     if not self._pending:
                         event.clear()
@@ -125,7 +168,12 @@ class CodexQuotaSyncDispatcher:
             batch = self._drain_pending()
             if batch:
                 try:
-                    await asyncio.to_thread(self._flush_batch_sync, batch)
+                    result = await asyncio.to_thread(self._flush_batch_sync, batch)
+                    if result.retry_batch:
+                        logger.warning(
+                            "Codex 配额异步同步器停止时仍有未同步事件: retry_keys={}",
+                            len(result.retry_batch),
+                        )
                 except Exception as exc:
                     logger.warning("Codex 配额异步同步器停止时 flush 失败: {}", exc)
             raise
@@ -144,38 +192,114 @@ class CodexQuotaSyncDispatcher:
         with self._pending_lock:
             self._pending.update(batch)
 
-    def _flush_batch_sync(self, batch: dict[str, dict[str, Any]]) -> None:
+    def _reset_backoff(self) -> None:
-        if not batch:
+        self._current_flush_delay_seconds = self.flush_interval_seconds
+
+    def _increase_backoff(self) -> None:
+        self._current_flush_delay_seconds = min(
+            self.max_backoff_seconds,
+            max(self.flush_interval_seconds, self._current_flush_delay_seconds * 2),
+        )
+
+    def _log_flush_failure_with_rate_limit(
+        self,
+        *,
+        queued_count: int,
+        retry_count: int,
+        error: Exception | None = None,
+    ) -> None:
+        now = time.monotonic()
+        if (
+            self._last_flush_error_log_at is not None
+            and now - self._last_flush_error_log_at < self.error_log_interval_seconds
+        ):
             return
+        self._last_flush_error_log_at = now
+        if error is not None:
+            logger.warning(
+                "Codex 配额异步同步器 flush 失败，将重试: queued_keys={}, retry_keys={}, backoff={}s, error={}",
+                queued_count,
+                retry_count,
+                round(self._current_flush_delay_seconds, 3),
+                error,
+            )
+            return
+        logger.warning(
+            "Codex 配额异步同步器 flush 部分失败，将重试: queued_keys={}, retry_keys={}, backoff={}s",
+            queued_count,
+            retry_count,
+            round(self._current_flush_delay_seconds, 3),
+        )
+
+    def _flush_batch_fallback(
+        self,
+        entries: list[tuple[str, dict[str, Any]]],
+    ) -> tuple[int, dict[str, dict[str, Any]]]:
+        updated_count = 0
+        retry_batch: dict[str, dict[str, Any]] = {}
+        for provider_api_key_id, response_headers in entries:
+            db: Session = create_session()
+            try:
+                updated = sync_codex_quota_from_response_headers(
+                    db=db,
+                    provider_api_key_id=provider_api_key_id,
+                    response_headers=response_headers,
+                )
+                if updated:
+                    db.commit()
+                    updated_count += 1
+                else:
+                    db.rollback()
+            except Exception:
+                db.rollback()
+                retry_batch[provider_api_key_id] = response_headers
+            finally:
+                db.close()
+        return updated_count, retry_batch
+
+    def _flush_batch_sync(self, batch: dict[str, dict[str, Any]]) -> FlushResult:
+        if not batch:
+            return FlushResult(
+                queued_count=0,
+                updated_count=0,
+                retry_batch={},
+            )
 
         db: Session = create_session()
-        updated_count = 0
+        updated_entries: list[tuple[str, dict[str, Any]]] = []
+        retry_batch: dict[str, dict[str, Any]] = {}
         try:
             for provider_api_key_id, response_headers in batch.items():
                 try:
-                    updated = sync_codex_quota_from_response_headers(
+                    with db.begin_nested():
-                        db=db,
+                        updated = sync_codex_quota_from_response_headers(
-                        provider_api_key_id=provider_api_key_id,
+                            db=db,
-                        response_headers=response_headers,
+                            provider_api_key_id=provider_api_key_id,
-                    )
+                            response_headers=response_headers,
+                        )
                     if updated:
-                        db.commit()
+                        updated_entries.append((provider_api_key_id, response_headers))
-                        updated_count += 1
+                except Exception:
-                    else:
+                    retry_batch[provider_api_key_id] = response_headers
-                        db.rollback()
+
-                except Exception as exc:
+            updated_count = 0
+            if updated_entries:
+                try:
+                    db.commit()
+                    updated_count = len(updated_entries)
+                except Exception:
                     db.rollback()
-                    logger.warning(
+                    fallback_updated, fallback_retry_batch = self._flush_batch_fallback(
-                        "异步同步 Codex 配额失败，已跳过: provider_api_key_id={}, error={}",
+                        updated_entries
-                        provider_api_key_id,
-                        exc,
                     )
-            if updated_count > 0:
+                    updated_count = fallback_updated
-                logger.debug(
+                    retry_batch.update(fallback_retry_batch)
-                    "异步同步 Codex 配额完成: queued_keys={}, updated_keys={}",
+
-                    len(batch),
+            return FlushResult(
-                    updated_count,
+                queued_count=len(batch),
-                )
+                updated_count=updated_count,
+                retry_batch=retry_batch,
+            )
         finally:
             db.close()
 

tests/services/test_provider_keys_codex_quota_sync_dispatcher.py

@@ -8,6 +8,45 @@ import pytest
 from src.services.provider_keys import codex_quota_sync_dispatcher as dispatcher_module
 
 
+def _flush_ok(batch: dict[str, dict[str, Any]]) -> dispatcher_module.FlushResult:
+    return dispatcher_module.FlushResult(
+        queued_count=len(batch),
+        updated_count=len(batch),
+        retry_batch={},
+    )
+
+
+class _FakeNestedTxn:
+    def __enter__(self) -> "_FakeNestedTxn":
+        return self
+
+    def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
+        _ = exc_type, exc, tb
+        return None
+
+
+class _FakeSession:
+    def __init__(self, *, fail_commit: bool = False) -> None:
+        self.fail_commit = fail_commit
+        self.commit_calls = 0
+        self.rollback_calls = 0
+        self.closed = False
+
+    def begin_nested(self) -> _FakeNestedTxn:
+        return _FakeNestedTxn()
+
+    def commit(self) -> None:
+        self.commit_calls += 1
+        if self.fail_commit and self.commit_calls == 1:
+            raise RuntimeError("commit failed once")
+
+    def rollback(self) -> None:
+        self.rollback_calls += 1
+
+    def close(self) -> None:
+        self.closed = True
+
+
 def test_dispatch_falls_back_to_sync_when_dispatcher_not_running(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
@@ -46,8 +85,9 @@ async def test_dispatcher_deduplicates_headers_by_provider_api_key_id() -> None:
     dispatcher = dispatcher_module.CodexQuotaSyncDispatcher(flush_interval_seconds=0.01)
     flushed_batches: list[dict[str, dict[str, Any]]] = []
 
-    def _fake_flush(batch: dict[str, dict[str, Any]]) -> None:
+    def _fake_flush(batch: dict[str, dict[str, Any]]) -> dispatcher_module.FlushResult:
         flushed_batches.append(batch)
+        return _flush_ok(batch)
 
     dispatcher._flush_batch_sync = _fake_flush  # type: ignore[method-assign]
 
@@ -83,12 +123,13 @@ async def test_dispatcher_retries_batch_after_flush_error() -> None:
     flushed_batches: list[dict[str, dict[str, Any]]] = []
     flush_attempts = 0
 
-    def _flaky_flush(batch: dict[str, dict[str, Any]]) -> None:
+    def _flaky_flush(batch: dict[str, dict[str, Any]]) -> dispatcher_module.FlushResult:
         nonlocal flush_attempts
         flush_attempts += 1
         if flush_attempts == 1:
             raise RuntimeError("temporary flush error")
         flushed_batches.append(batch)
+        return _flush_ok(batch)
 
     dispatcher._flush_batch_sync = _flaky_flush  # type: ignore[method-assign]
 
@@ -110,14 +151,54 @@ async def test_dispatcher_retries_batch_after_flush_error() -> None:
     assert merged["retry-key"] == {"x-codex-primary-used-percent": "7"}
 
 
+def test_flush_batch_falls_back_to_single_item_commit(monkeypatch: pytest.MonkeyPatch) -> None:
+    dispatcher = dispatcher_module.CodexQuotaSyncDispatcher(flush_interval_seconds=0.01)
+    main_session = _FakeSession(fail_commit=True)
+    fallback_session_a = _FakeSession()
+    fallback_session_b = _FakeSession()
+    sessions: list[_FakeSession] = [main_session, fallback_session_a, fallback_session_b]
+
+    def _fake_create_session() -> _FakeSession:
+        return sessions.pop(0)
+
+    def _fake_sync(
+        *,
+        db: Any,
+        provider_api_key_id: str | None,
+        response_headers: dict[str, Any] | None,
+    ) -> bool:
+        _ = db, provider_api_key_id, response_headers
+        return True
+
+    monkeypatch.setattr(dispatcher_module, "create_session", _fake_create_session)
+    monkeypatch.setattr(dispatcher_module, "sync_codex_quota_from_response_headers", _fake_sync)
+
+    result = dispatcher._flush_batch_sync(
+        {
+            "key-a": {"x-codex-primary-used-percent": "1"},
+            "key-b": {"x-codex-primary-used-percent": "2"},
+        }
+    )
+
+    assert result.updated_count == 2
+    assert result.retry_batch == {}
+    assert main_session.rollback_calls == 1
+    assert fallback_session_a.commit_calls == 1
+    assert fallback_session_b.commit_calls == 1
+    assert main_session.closed is True
+    assert fallback_session_a.closed is True
+    assert fallback_session_b.closed is True
+
+
 @pytest.mark.asyncio
 async def test_dispatch_uses_async_queue_when_dispatcher_running() -> None:
     dispatcher = dispatcher_module.CodexQuotaSyncDispatcher(flush_interval_seconds=0.01)
     flushed_batches: list[dict[str, dict[str, Any]]] = []
     dispatcher_module._dispatcher_instance = dispatcher
 
-    def _fake_flush(batch: dict[str, dict[str, Any]]) -> None:
+    def _fake_flush(batch: dict[str, dict[str, Any]]) -> dispatcher_module.FlushResult:
         flushed_batches.append(batch)
+        return _flush_ok(batch)
 
     dispatcher._flush_batch_sync = _fake_flush  # type: ignore[method-assign]