feat(hub,stability): bounded outbound queue、worker liveness 检测、事件循环 watchdog 及 DB 操作异步化

- aether-hub: unbounded channel 改为 bounded channel (BoundedOutbound)，队列满时标记拥塞并主动关闭连接，防止内存无限增长
- aether-hub: worker idle timeout 从命令行参数改为基于心跳的 liveness 检测，默认 60 秒
- aether-hub: 新增 ConnConfig 统一管理连接配置，新增 outbound_queue_capacity 参数
- hub_transport: 新增事件循环 watchdog，检测 lag 超过阈值时临时降级暂停新流
- gunicorn_conf: 启用 faulthandler，worker abort 时自动 dump 全部线程栈用于诊断
- health/endpoint_checker/recording: 同步 DB 操作移至 asyncio.to_thread，避免阻塞事件循环
- Dockerfile: 移除 --worker-idle-timeout 0 命令行参数，改由环境变量和默认值控制

src/api/admin/endpoints/health.py

@@ -4,6 +4,7 @@ Endpoint 健康监控 API
 
 from __future__ import annotations
 
+import asyncio
 from collections import defaultdict
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
@@ -33,6 +34,60 @@ from src.services.health.monitor import HealthMonitor, health_monitor
 router = APIRouter(tags=["Endpoint Health"])
 
 
+def _recover_key_health_sync(db: Session, key_id: str, api_format: str | None) -> dict[str, Any]:
+    key = db.query(ProviderAPIKey).filter(ProviderAPIKey.id == key_id).first()
+    if not key:
+        raise NotFoundException(f"Key {key_id} 不存在")
+
+    success = health_monitor.reset_health(db, key_id=key_id, api_format=api_format)
+    if not success:
+        raise Exception("重置健康度失败")
+
+    if not key.is_active:
+        key.is_active = True  # type: ignore[assignment]
+
+    db.commit()
+    return {
+        "is_active": bool(key.is_active),
+        "api_format": api_format,
+    }
+
+
+def _recover_all_keys_health_sync(db: Session) -> list[dict[str, Any]]:
+    candidates = (
+        db.query(ProviderAPIKey)
+        .filter(
+            ProviderAPIKey.circuit_breaker_by_format.isnot(None),
+            ProviderAPIKey.circuit_breaker_by_format != "{}",
+        )
+        .all()
+    )
+
+    circuit_open_keys = [
+        key
+        for key in candidates
+        if any(cb.get("open") for cb in (key.circuit_breaker_by_format or {}).values())
+    ]
+
+    recovered_keys: list[dict[str, Any]] = []
+    for key in circuit_open_keys:
+        key.health_by_format = {}  # type: ignore[assignment]
+        key.circuit_breaker_by_format = {}  # type: ignore[assignment]
+        recovered_keys.append(
+            {
+                "key_id": key.id,
+                "key_name": key.name,
+                "provider_id": key.provider_id,
+                "api_formats": key.api_formats,
+            }
+        )
+
+    if recovered_keys:
+        db.commit()
+
+    return recovered_keys
+
+
 def _format_str(api_format_enum: Any) -> str:
     """将 DB 查询返回的 api_format（可能是 enum 或 str）统一转为 str。"""
     return api_format_enum.value if hasattr(api_format_enum, "value") else str(api_format_enum)
@@ -491,20 +546,7 @@ class AdminRecoverKeyHealthAdapter(AdminApiAdapter):
 
     async def handle(self, context: ApiRequestContext) -> Any:  # type: ignore[override]
         db = context.db
-        key = db.query(ProviderAPIKey).filter(ProviderAPIKey.id == self.key_id).first()
-        if not key:
-            raise NotFoundException(f"Key {self.key_id} 不存在")
-
-        # 使用 health_monitor.reset_health 重置健康度
-        success = health_monitor.reset_health(db, key_id=self.key_id, api_format=self.api_format)
-        if not success:
-            raise Exception("重置健康度失败")
-
-        # 如果 Key 被禁用，重新启用
-        if not key.is_active:
-            key.is_active = True  # type: ignore[assignment]
-
-        db.commit()
+        await asyncio.to_thread(_recover_key_health_sync, db, self.key_id, self.api_format)
 
         if self.api_format:
             logger.info(f"管理员恢复Key健康状态: {self.key_id}/{self.api_format}")
@@ -534,47 +576,15 @@ class AdminRecoverAllKeysHealthAdapter(AdminApiAdapter):
 
     async def handle(self, context: ApiRequestContext) -> Any:  # type: ignore[override]
         db = context.db
+        recovered_keys = await asyncio.to_thread(_recover_all_keys_health_sync, db)
 
-        # 粗过滤：仅加载 circuit_breaker_by_format 非空的 Key，避免全表扫描
-        candidates = (
-            db.query(ProviderAPIKey)
-            .filter(
-                ProviderAPIKey.circuit_breaker_by_format.isnot(None),
-                ProviderAPIKey.circuit_breaker_by_format != "{}",
-            )
-            .all()
-        )
-
-        # 精确筛选有任何格式熔断的 Key
-        circuit_open_keys = [
-            key
-            for key in candidates
-            if any(cb.get("open") for cb in (key.circuit_breaker_by_format or {}).values())
-        ]
-
-        if not circuit_open_keys:
+        if not recovered_keys:
             return {
                 "message": "没有需要恢复的 Key",
                 "recovered_count": 0,
                 "recovered_keys": [],
             }
 
-        recovered_keys = []
-        for key in circuit_open_keys:
-            # 重置所有格式的健康度
-            key.health_by_format = {}  # type: ignore[assignment]
-            key.circuit_breaker_by_format = {}  # type: ignore[assignment]
-            recovered_keys.append(
-                {
-                    "key_id": key.id,
-                    "key_name": key.name,
-                    "provider_id": key.provider_id,
-                    "api_formats": key.api_formats,
-                }
-            )
-
-        db.commit()
-
         # 重置健康监控器的熔断计数
         HealthMonitor.reset_open_circuit_count()
 

src/api/handlers/base/endpoint_checker.py

@@ -161,29 +161,43 @@ async def _calculate_and_record_usage(
     from src.services.request.candidate import RequestCandidateService
     from src.services.usage.service import UsageService
 
-    # 获取Provider API Key对象（不是用户API Key）
-    provider_api_key = db.query(ProviderAPIKey).filter(ProviderAPIKey.id == api_key_id).first()
+    def _load_usage_context() -> tuple[Any, Any, Any]:
+        # 获取Provider API Key对象（不是用户API Key）
+        provider_api_key_local = (
+            db.query(ProviderAPIKey).filter(ProviderAPIKey.id == api_key_id).first()
+        )
+        if not provider_api_key_local:
+            return None, None, None
+
+        provider_endpoint_local = None
+        if api_format and provider_api_key_local.provider_id:
+            from src.models.database import Provider
+
+            provider = (
+                db.query(Provider).filter(Provider.id == provider_api_key_local.provider_id).first()
+            )
+            if provider:
+                for ep in provider.endpoints:
+                    if ep.api_format == api_format:
+                        provider_endpoint_local = ep
+                        break
+
+        user_api_key_local = None
+        if user:
+            try:
+                user_api_key_local = db.query(ApiKey).filter(ApiKey.user_id == user.id).first()
+            except Exception:
+                user_api_key_local = None
+
+        return provider_api_key_local, provider_endpoint_local, user_api_key_local
+
+    provider_api_key, provider_endpoint, user_api_key = await asyncio.to_thread(_load_usage_context)
     if not provider_api_key:
         logger.warning(f"Provider API Key not found for usage calculation: {api_key_id}")
         return {"error": "Provider API Key not found"}
 
-    # 获取Provider Endpoint信息（通过 api_format 查找）
-    provider_endpoint = None
-    if api_format and provider_api_key.provider_id:
-        from src.models.database import Provider
-
-        provider = db.query(Provider).filter(Provider.id == provider_api_key.provider_id).first()
-        if provider:
-            for ep in provider.endpoints:
-                if ep.api_format == api_format:
-                    provider_endpoint = ep
-                    break
-
-    # 获取用户的API Key（用于记录关联，即使实际使用的是Provider API Key）
-    user_api_key = None
     if user:
         try:
-            user_api_key = db.query(ApiKey).filter(ApiKey.user_id == user.id).first()
             logger.info(
                 f"[endpoint_check] User API Key found: {user_api_key.id if user_api_key else None}"
             )
@@ -317,45 +331,46 @@ async def _calculate_and_record_usage(
 
         # 创建RequestCandidate记录，用于监控追踪API
         try:
-            # 首先创建候选记录
-            candidate = RequestCandidateService.create_candidate(
-                db=db,
-                request_id=f"test_{request_id}",
-                candidate_index=0,  # 测试请求只有一个候选
-                user_id=user.id if user else None,
-                api_key_id=user_api_key.id if user_api_key else None,
-                provider_id=provider_id,
-                endpoint_id=provider_endpoint.id if provider_endpoint else None,
-                key_id=api_key_id,
-                status="available",
-                extra_data={"model_name": model_name, "request_type": "endpoint_test"},
-            )
 
-            # 立即标记为开始执行
-            RequestCandidateService.mark_candidate_started(db, candidate.id)
-
-            # 根据结果标记为成功或失败
-            if status_code == 200:
-                RequestCandidateService.mark_candidate_success(
+            def _record_candidate_sync() -> str:
+                candidate = RequestCandidateService.create_candidate(
                     db=db,
-                    candidate_id=candidate.id,
-                    status_code=status_code,
-                    latency_ms=response_time_ms,
-                    extra_data={"model_name": model_name, "api_format": api_format},
-                )
-            else:
-                RequestCandidateService.mark_candidate_failed(
-                    db=db,
-                    candidate_id=candidate.id,
-                    error_type="http_error" if status_code > 0 else "network_error",
-                    error_message=error_message or "Unknown error",
-                    status_code=status_code,
-                    latency_ms=response_time_ms,
-                    extra_data={"model_name": model_name, "api_format": api_format},
+                    request_id=f"test_{request_id}",
+                    candidate_index=0,  # 测试请求只有一个候选
+                    user_id=user.id if user else None,
+                    api_key_id=user_api_key.id if user_api_key else None,
+                    provider_id=provider_id,
+                    endpoint_id=provider_endpoint.id if provider_endpoint else None,
+                    key_id=api_key_id,
+                    status="available",
+                    extra_data={"model_name": model_name, "request_type": "endpoint_test"},
                 )
 
+                RequestCandidateService.mark_candidate_started(db, candidate.id)
+
+                if status_code == 200:
+                    RequestCandidateService.mark_candidate_success(
+                        db=db,
+                        candidate_id=candidate.id,
+                        status_code=status_code,
+                        latency_ms=response_time_ms,
+                        extra_data={"model_name": model_name, "api_format": api_format},
+                    )
+                else:
+                    RequestCandidateService.mark_candidate_failed(
+                        db=db,
+                        candidate_id=candidate.id,
+                        error_type="http_error" if status_code > 0 else "network_error",
+                        error_message=error_message or "Unknown error",
+                        status_code=status_code,
+                        latency_ms=response_time_ms,
+                        extra_data={"model_name": model_name, "api_format": api_format},
+                    )
+                return str(candidate.id)
+
+            candidate_id = await asyncio.to_thread(_record_candidate_sync)
             logger.info(
-                f"[endpoint_check] RequestCandidate created | request_id=test_{request_id}, candidate_id={candidate.id}"
+                f"[endpoint_check] RequestCandidate created | request_id=test_{request_id}, candidate_id={candidate_id}"
             )
         except Exception as e:
             logger.warning(f"[endpoint_check] Failed to create RequestCandidate: {e}")

src/services/proxy_node/hub_transport.py

@@ -30,6 +30,12 @@ if TYPE_CHECKING:
 _TUNNEL_COMPRESS_MIN_SIZE = 512
 _RECONNECT_DELAYS_SECONDS: tuple[float, ...] = (0.0, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0)
 _HEARTBEAT_DEDUP_TTL_SECONDS = 600
+_LOOP_WATCHDOG_INTERVAL_SECONDS = 1.0
+_LOOP_LAG_WARNING_SECONDS = 1.0
+_LOOP_LAG_DEGRADE_SECONDS = 3.0
+_LOOP_LAG_DEGRADE_MIN_COOLDOWN_SECONDS = 10.0
+_LOOP_LAG_DEGRADE_MAX_COOLDOWN_SECONDS = 30.0
+_LOOP_LAG_WARNING_LOG_INTERVAL_SECONDS = 10.0
 
 _HOP_BY_HOP_HEADERS = frozenset(
     {
@@ -65,9 +71,12 @@ class HubConnectionManager:
         self._reader_task: asyncio.Task[None] | None = None
         self._ping_task: asyncio.Task[None] | None = None
         self._reconnect_task: asyncio.Task[None] | None = None
+        self._watchdog_task: asyncio.Task[None] | None = None
         self._background_tasks: set[asyncio.Task[None]] = set()
 
         self._closing = False
+        self._degraded_until: float = 0.0
+        self._last_loop_lag_warning_ts: float = 0.0
 
         self._disconnect_count = 0  # 连续断开计数，用于抑制重复日志
         # 连续快速断开退避：防止 Hub 端持续发送 GOAWAY 时产生重连风暴
@@ -85,6 +94,7 @@ class HubConnectionManager:
         task.add_done_callback(self._background_tasks.discard)
 
     async def ensure_connected(self) -> None:
+        self._ensure_watchdog_running()
         if self._closing:
             raise TunnelStreamError("hub connection manager is shutting down")
         if self.is_connected:
@@ -142,6 +152,55 @@ class HubConnectionManager:
             )
         self._disconnect_count = 0
 
+    def _ensure_watchdog_running(self) -> None:
+        if self._closing:
+            return
+        if self._watchdog_task is not None and not self._watchdog_task.done():
+            return
+        self._watchdog_task = asyncio.create_task(self._loop_watchdog())
+
+    def _record_loop_lag(self, lag_seconds: float) -> None:
+        if lag_seconds < _LOOP_LAG_WARNING_SECONDS:
+            return
+
+        now = _time.monotonic()
+        if lag_seconds >= _LOOP_LAG_DEGRADE_SECONDS:
+            cooldown = min(
+                _LOOP_LAG_DEGRADE_MAX_COOLDOWN_SECONDS,
+                max(_LOOP_LAG_DEGRADE_MIN_COOLDOWN_SECONDS, lag_seconds * 3.0),
+            )
+            degraded_until = now + cooldown
+            self._degraded_until = max(self._degraded_until, degraded_until)
+            logger.warning(
+                "Hub worker event loop lag detected: lag={:.2f}s, pausing new streams for {:.1f}s",
+                lag_seconds,
+                cooldown,
+            )
+            return
+
+        if now - self._last_loop_lag_warning_ts >= _LOOP_LAG_WARNING_LOG_INTERVAL_SECONDS:
+            self._last_loop_lag_warning_ts = now
+            logger.warning("Hub worker event loop lag observed: lag={:.2f}s", lag_seconds)
+
+    def _raise_if_degraded(self) -> None:
+        remaining = self._degraded_until - _time.monotonic()
+        if remaining <= 0:
+            return
+        raise TunnelStreamError(f"hub worker event loop degraded, retry in {remaining:.1f}s")
+
+    async def _loop_watchdog(self) -> None:
+        interval = _LOOP_WATCHDOG_INTERVAL_SECONDS
+        expected_at = _time.monotonic() + interval
+        try:
+            while not self._closing:
+                await asyncio.sleep(interval)
+                now = _time.monotonic()
+                lag_seconds = max(0.0, now - expected_at)
+                expected_at = now + interval
+                self._record_loop_lag(lag_seconds)
+        except asyncio.CancelledError:
+            return
+
     def _start_reconnect_loop(self) -> None:
         if self._closing:
             return
@@ -530,6 +589,7 @@ class HubConnectionManager:
         timeout: float = 60.0,
     ) -> _StreamState:
         await self.ensure_connected()
+        self._raise_if_degraded()
 
         if len(self._pending_streams) >= self._config.max_streams:
             raise TunnelStreamError(
@@ -587,6 +647,8 @@ class HubConnectionManager:
             self._reader_task.cancel()
         if self._ping_task is not None:
             self._ping_task.cancel()
+        if self._watchdog_task is not None:
+            self._watchdog_task.cancel()
 
         tasks = list(self._background_tasks)
         for task in tasks:

src/services/usage/recording.py

@@ -102,6 +102,7 @@ def _increment_provider_api_key_totals(
         return
 
     from sqlalchemy import func as sql_func
+
     token_increment = int(total_tokens or 0)
     cost_increment = to_money_decimal(total_cost)
 
@@ -329,57 +330,60 @@ class UsageRecordingMixin(UsageBillingIntegrationMixin):
         usage_params, total_cost = await cls._prepare_usage_record(params)
         total_cost = to_money_decimal(total_cost)
 
-        # 创建 Usage 记录
-        usage = Usage(**usage_params)
-        db.add(usage)
+        def _sync_record() -> Usage:
+            # 创建 Usage 记录与相关统计；同步 SQLAlchemy 操作统一移到线程池，避免阻塞事件循环。
+            usage = Usage(**usage_params)
+            db.add(usage)
 
-        # 更新 GlobalModel 使用计数（原子操作）
-        from sqlalchemy import update
+            # 更新 GlobalModel 使用计数（原子操作）
+            from sqlalchemy import update
 
-        from src.models.database import GlobalModel
+            from src.models.database import GlobalModel
 
-        db.execute(
-            update(GlobalModel)
-            .where(GlobalModel.name == model)
-            .values(usage_count=GlobalModel.usage_count + 1)
-        )
-
-        # 更新用户-模型调用次数计数器
-        cls._increment_user_model_usage(db, user, model)
-
-        # 更新 Provider 月度使用量（原子操作）
-        if provider_id:
-            actual_total_cost = Decimal(str(usage_params["actual_total_cost_usd"]))
             db.execute(
-                update(Provider)
-                .where(Provider.id == provider_id)
-                .values(monthly_used_usd=Provider.monthly_used_usd + actual_total_cost)
+                update(GlobalModel)
+                .where(GlobalModel.name == model)
+                .values(usage_count=GlobalModel.usage_count + 1)
             )
 
-        accounted, _charge_applied = cls._finalize_usage_billing(
-            db,
-            usage=usage,
-            total_cost=total_cost,
-            status=status,
-            finalized_at=finalized_at,
-        )
+            # 更新用户-模型调用次数计数器
+            cls._increment_user_model_usage(db, user, model)
 
-        if accounted:
-            _increment_provider_api_key_totals(
+            # 更新 Provider 月度使用量（原子操作）
+            if provider_id:
+                actual_total_cost = Decimal(str(usage_params["actual_total_cost_usd"]))
+                db.execute(
+                    update(Provider)
+                    .where(Provider.id == provider_id)
+                    .values(monthly_used_usd=Provider.monthly_used_usd + actual_total_cost)
+                )
+
+            accounted, _charge_applied = cls._finalize_usage_billing(
                 db,
-                provider_api_key_id,
-                total_tokens=int(usage_params.get("total_tokens") or 0),
-                total_cost=_get_actual_total_cost_usd(usage_params),
+                usage=usage,
+                total_cost=total_cost,
+                status=status,
+                finalized_at=finalized_at,
             )
 
-        dispatch_codex_quota_sync_from_response_headers(
-            provider_api_key_id=provider_api_key_id,
-            response_headers=response_headers,
-            db=db,
-        )
+            if accounted:
+                _increment_provider_api_key_totals(
+                    db,
+                    provider_api_key_id,
+                    total_tokens=int(usage_params.get("total_tokens") or 0),
+                    total_cost=_get_actual_total_cost_usd(usage_params),
+                )
 
-        db.commit()  # 立即提交事务，释放数据库锁
-        return usage
+            dispatch_codex_quota_sync_from_response_headers(
+                provider_api_key_id=provider_api_key_id,
+                response_headers=response_headers,
+                db=db,
+            )
+
+            db.commit()  # 立即提交事务，释放数据库锁
+            return usage
+
+        return await asyncio.to_thread(_sync_record)
 
     @classmethod
     async def record_usage(