""" 系统维护定时任务调度器 包含以下任务： - 统计聚合：每天凌晨聚合前一天的统计数据 - Provider 签到：每天凌晨执行所有已配置 Provider 的签到 - 使用记录清理：分级清理策略（压缩、清空、删除） - 审计日志清理：定期清理过期的审计日志 - 连接池监控：定期检查数据库连接池状态 - Pending 状态清理：清理异常的 Pending 状态记录 - Gemini 文件映射清理：清理过期的 Gemini 文件→Key 映射 - 请求候选记录清理：定期清理过期的 request_candidates 记录 - 数据库表维护：定期 VACUUM ANALYZE 防止表和索引膨胀 使用 APScheduler 进行任务调度，支持时区配置。 """ from __future__ import annotations import asyncio import os from datetime import date, datetime, timedelta, timezone from typing import Any from sqlalchemy import delete, literal_column, text from src.clients.http_client import HTTPClientPool from src.config.settings import config from src.core.logger import logger from src.database import create_session from src.models.database import AuditLog, Provider, RequestCandidate, Usage from src.services.provider_ops.service import ProviderOpsService from src.services.system.config import SystemConfigService from src.services.system.scheduler import get_scheduler from src.services.system.stats_aggregator import StatsAggregatorService from src.services.user.apikey import ApiKeyService from src.services.wallet import WalletDailyUsageLedgerService from src.utils.compression import compress_json class MaintenanceScheduler: """系统维护任务调度器""" # 签到任务的 job_id CHECKIN_JOB_ID = "provider_checkin" def __init__(self) -> None: self.running = False self._interval_tasks = [] self._stats_aggregation_lock = asyncio.Lock() self._wallet_daily_usage_lock = asyncio.Lock() self._startup_task: asyncio.Task[Any] | None = None self._registered_job_ids: list[str] = [] @staticmethod def _get_http_client_idle_cleanup_interval_minutes() -> int: """获取 HTTP 客户端空闲清理调度间隔（分钟）。""" raw = os.getenv("HTTP_CLIENT_IDLE_CLEANUP_INTERVAL_MINUTES", "5") try: minutes = int(raw) except ValueError: logger.warning( "环境变量 HTTP_CLIENT_IDLE_CLEANUP_INTERVAL_MINUTES 非法: {}, 使用默认值 5", raw, ) return 5 return max(1, minutes) @staticmethod def _has_python_startup_tasks() -> bool: return bool( config.pending_cleanup_python_enabled or config.antigravity_ua_refresh_python_enabled ) def _get_checkin_time(self) -> tuple[int, int]: """获取签到任务的执行时间 Returns: (hour, minute) 元组 """ db = create_session() try: time_str = SystemConfigService.get_config(db, "provider_checkin_time", "01:05") return self._parse_time_string(time_str) finally: db.close() @staticmethod def _parse_time_string(time_str: str) -> tuple[int, int]: """解析时间字符串为 (hour, minute) 元组 Args: time_str: HH:MM 格式的时间字符串 Returns: (hour, minute) 元组，解析失败返回默认值 (1, 5) """ try: if not time_str or ":" not in time_str: return (1, 5) parts = time_str.split(":") hour = int(parts[0]) minute = int(parts[1]) # 验证范围 if 0 <= hour <= 23 and 0 <= minute <= 59: return (hour, minute) return (1, 5) except (ValueError, IndexError): return (1, 5) def update_checkin_time(self, time_str: str) -> bool: """更新签到任务的执行时间 Args: time_str: HH:MM 格式的时间字符串 Returns: 是否成功更新 """ hour, minute = self._parse_time_string(time_str) scheduler = get_scheduler() success = scheduler.reschedule_cron_job( self.CHECKIN_JOB_ID, hour=hour, minute=minute, ) if success: logger.info(f"Provider 签到任务时间已更新为: {hour:02d}:{minute:02d}") return success def get_checkin_job_info(self) -> dict | None: """获取签到任务的信息 Returns: 任务信息字典 """ scheduler = get_scheduler() return scheduler.get_job_info(self.CHECKIN_JOB_ID) async def start(self) -> Any: """启动调度器""" if self.running: logger.warning("Maintenance scheduler already running") return self.running = True self._registered_job_ids.clear() logger.info("系统维护调度器已启动") scheduler = get_scheduler() def _add_cron(job_id: str, **kwargs: Any) -> None: scheduler.add_cron_job(job_id=job_id, **kwargs) self._registered_job_ids.append(job_id) def _add_interval(job_id: str, **kwargs: Any) -> None: scheduler.add_interval_job(job_id=job_id, **kwargs) self._registered_job_ids.append(job_id) # 注册定时任务 if config.stats_aggregation_python_enabled: # 统计聚合任务 - UTC 00:05 执行 _add_cron( "stats_aggregation", func=self._scheduled_stats_aggregation, hour=0, minute=5, name="统计数据聚合", timezone="UTC", ) else: logger.info("每日统计聚合已切到 Rust gateway，Python 调度器跳过注册") if config.stats_hourly_aggregation_python_enabled: # 小时统计聚合任务 - 每小时 05 分执行（UTC） _add_cron( "stats_hourly_aggregation", func=self._scheduled_hourly_stats_aggregation, hour="*", minute=5, name="统计小时数据聚合", timezone="UTC", ) else: logger.info("小时统计聚合已切到 Rust gateway，Python 调度器跳过注册") if config.wallet_daily_usage_aggregation_python_enabled: _add_cron( "wallet_daily_usage_aggregation", func=self._scheduled_wallet_daily_usage_aggregation, hour=0, minute=10, name="钱包每日消费汇总", ) else: logger.info("钱包每日消费汇总已切到 Rust gateway，Python 调度器跳过注册") if config.usage_cleanup_python_enabled: # 清理任务 - 凌晨 3 点执行 _add_cron( "usage_cleanup", func=self._scheduled_cleanup, hour=3, minute=0, name="使用记录清理", ) else: logger.info("使用记录清理已切到 Rust gateway，Python 调度器跳过注册") if config.pool_monitor_python_enabled: # 连接池监控 - 每 5 分钟 _add_interval( "pool_monitor", func=self._scheduled_monitor, minutes=5, name="连接池监控", ) else: logger.info("连接池监控已切到 Rust gateway，Python 调度器跳过注册") if config.http_client_idle_cleanup_python_enabled: # HTTP 代理/Tunnel 客户端空闲清理 - 默认每 5 分钟 _add_interval( "http_client_idle_cleanup", func=self._scheduled_http_client_idle_cleanup, minutes=self._get_http_client_idle_cleanup_interval_minutes(), name="HTTP客户端空闲清理", ) else: logger.info("HTTP 客户端空闲清理默认跳过 Python 调度注册") if config.pending_cleanup_python_enabled: # Pending 状态清理 - 每 5 分钟 _add_interval( "pending_cleanup", func=self._scheduled_pending_cleanup, minutes=5, name="Pending状态清理", ) else: logger.info("Pending 状态清理已切到 Rust gateway，Python 调度器跳过注册") if config.audit_cleanup_python_enabled: # 审计日志清理 - 凌晨 4 点执行 _add_cron( "audit_cleanup", func=self._scheduled_audit_cleanup, hour=4, minute=0, name="审计日志清理", ) else: logger.info("审计日志清理已切到 Rust gateway，Python 调度器跳过注册") if config.gemini_file_mapping_cleanup_python_enabled: # Gemini 文件映射清理 - 每小时执行 _add_interval( "gemini_file_mapping_cleanup", func=self._scheduled_gemini_file_mapping_cleanup, hours=1, name="Gemini文件映射清理", ) else: logger.info("Gemini 文件映射清理已切到 Rust gateway，Python 调度器跳过注册") if config.request_candidate_cleanup_python_enabled: # 请求候选记录清理 - 凌晨 3:30 执行 _add_cron( "candidate_cleanup", func=self._scheduled_candidate_cleanup, hour=3, minute=30, name="请求候选记录清理", ) else: logger.info("请求候选记录清理已切到 Rust gateway，Python 调度器跳过注册") if config.db_maintenance_python_enabled: # 数据库表维护 - 每周日凌晨 5 点执行 VACUUM ANALYZE _add_cron( "db_maintenance", func=self._scheduled_db_maintenance, day_of_week="sun", hour=5, minute=0, name="数据库表维护", ) else: logger.info("数据库维护已切到 Rust gateway，Python 调度器跳过注册") if config.antigravity_ua_refresh_python_enabled: # Antigravity User-Agent 版本刷新 - 每 6 小时 _add_interval( "antigravity_ua_refresh", func=self._scheduled_antigravity_ua_refresh, hours=6, name="Antigravity UA版本刷新", ) else: logger.info("Antigravity UA 版本刷新默认跳过 Python 调度注册") if config.provider_checkin_python_enabled: # Provider 签到任务 - 根据配置时间执行 checkin_hour, checkin_minute = self._get_checkin_time() _add_cron( self.CHECKIN_JOB_ID, func=self._scheduled_provider_checkin, hour=checkin_hour, minute=checkin_minute, name="Provider签到", ) else: logger.info("Provider 签到已切到 Rust gateway，Python 调度器跳过注册") # 启动时执行一次初始化任务 if config.maintenance_startup_tasks_enabled and self._has_python_startup_tasks(): from src.utils.async_utils import safe_create_task self._startup_task = safe_create_task(self._run_startup_tasks()) elif config.maintenance_startup_tasks_enabled: self._startup_task = None logger.info("维护调度器启动任务已无剩余 Python owner，跳过创建") else: self._startup_task = None logger.info("维护调度器启动任务已禁用（MAINTENANCE_STARTUP_TASKS_ENABLED=false）") async def _run_startup_tasks(self) -> None: """启动时执行的初始化任务""" # 延迟执行，等待系统完全启动（Redis 连接、其他后台任务稳定） # 增加延迟时间避免与 UsageQueueConsumer 等后台任务竞争数据库连接 await asyncio.sleep(10) if config.antigravity_ua_refresh_python_enabled: # 刷新 Antigravity User-Agent 版本号（不阻塞其他启动任务） try: from src.services.provider.adapters.antigravity.client import refresh_user_agent await refresh_user_agent() except Exception as e: logger.debug("启动时刷新 Antigravity UA 版本失败（不影响运行）: {}", e) else: logger.info("启动时 Antigravity UA 刷新默认跳过") if config.pending_cleanup_python_enabled: try: logger.info("启动时清理残留的 pending/streaming 请求...") await self._perform_pending_cleanup() except Exception as e: logger.exception(f"启动时 pending 清理执行出错: {e}") else: logger.info("启动时 pending 清理已切到 Rust gateway，Python 宿主跳过执行") async def stop(self) -> Any: """停止调度器""" if not self.running: return self.running = False if self._startup_task and not self._startup_task.done(): self._startup_task.cancel() try: await self._startup_task except asyncio.CancelledError: pass self._startup_task = None scheduler = get_scheduler() for job_id in self._registered_job_ids: scheduler.remove_job(job_id) self._registered_job_ids.clear() logger.info("系统维护调度器已停止") # ========== 任务函数（APScheduler 直接调用异步函数） ========== async def _scheduled_stats_aggregation(self, backfill: bool = False) -> None: """统计聚合任务（定时调用）""" if not self.running: return await self._perform_stats_aggregation(backfill=backfill) async def _scheduled_wallet_daily_usage_aggregation(self) -> None: """钱包每日消费汇总任务（定时调用）""" if not self.running: return await self._perform_wallet_daily_usage_aggregation() async def _scheduled_hourly_stats_aggregation(self) -> None: """小时统计聚合任务（定时调用）""" if not self.running: return await self._perform_hourly_stats_aggregation() async def _scheduled_cleanup(self) -> None: """清理任务（定时调用）""" if not self.running: return await self._perform_cleanup() async def _scheduled_monitor(self) -> None: """监控任务（定时调用）""" if not self.running: return try: from src.database import log_pool_status log_pool_status() except Exception as e: logger.exception("连接池监控任务出错: {}", e) async def _scheduled_http_client_idle_cleanup(self) -> None: """HTTP 客户端空闲清理任务（定时调用）。""" if not self.running: return try: stats = await HTTPClientPool.cleanup_idle_clients() if stats.get("proxy_closed", 0) or stats.get("tunnel_closed", 0): logger.info( "HTTP 客户端空闲清理释放连接: proxy={}, tunnel={}", stats.get("proxy_closed", 0), stats.get("tunnel_closed", 0), ) except Exception as e: logger.exception("HTTP 客户端空闲清理任务出错: {}", e) async def _scheduled_pending_cleanup(self) -> None: """Pending 清理任务（定时调用）""" if not self.running: return await self._perform_pending_cleanup() async def _scheduled_audit_cleanup(self) -> None: """审计日志清理任务（定时调用）""" if not self.running: return await self._perform_audit_cleanup() async def _scheduled_candidate_cleanup(self) -> None: """请求候选记录清理任务（定时调用）""" if not self.running: return await self._perform_candidate_cleanup() async def _scheduled_db_maintenance(self) -> None: """数据库表维护任务（定时调用）""" if not self.running: return await self._perform_db_maintenance() async def _scheduled_gemini_file_mapping_cleanup(self) -> None: """Gemini 文件映射清理任务（定时调用）""" if not self.running: return await self._perform_gemini_file_mapping_cleanup() async def _scheduled_antigravity_ua_refresh(self) -> None: """Antigravity User-Agent 版本刷新（定时调用）""" if not self.running: return try: from src.services.provider.adapters.antigravity.client import refresh_user_agent await refresh_user_agent() except Exception as e: logger.debug("定时刷新 Antigravity UA 版本失败: {}", e) async def _scheduled_provider_checkin(self) -> None: """Provider 签到任务（定时调用）""" if not self.running: return await self._perform_provider_checkin() # ========== 实际任务实现 ========== async def _perform_stats_aggregation(self, backfill: bool = False) -> None: """执行统计聚合任务 Args: backfill: 是否回填历史数据（启动时检查缺失的日期） """ if self._stats_aggregation_lock.locked(): logger.info("统计聚合任务正在运行，跳过本次触发") return async with self._stats_aggregation_lock: db = create_session() try: # 检查是否启用统计聚合 if not SystemConfigService.get_config(db, "enable_stats_aggregation", True): logger.info("统计聚合已禁用，跳过聚合任务") return logger.info("开始执行统计数据聚合...") from src.models.database import StatsDaily from src.models.database import User as DBUser # 使用 UTC 日期，定时任务在 UTC 00:05 触发，聚合 UTC 昨天 now_utc = datetime.now(timezone.utc) today_utc = now_utc.replace(hour=0, minute=0, second=0, microsecond=0) if backfill: # 启动时检查并回填缺失的日期 from src.models.database import StatsSummary summary = db.query(StatsSummary).first() if not summary: # 首次运行，回填所有历史数据 logger.info("检测到首次运行，开始回填历史统计数据...") days_to_backfill = SystemConfigService.get_config( db, "stats_backfill_days", 365 ) count = StatsAggregatorService.backfill_historical_data( db, days=days_to_backfill ) logger.info(f"历史数据回填完成，共 {count} 天") return # 非首次运行，检查最近是否有缺失的日期需要回填 from src.models.database import StatsDailyModel, StatsDailyProvider yesterday_utc_date = today_utc.date() - timedelta(days=1) max_backfill_days: int = ( SystemConfigService.get_config(db, "max_stats_backfill_days", 30) or 30 ) # 计算回填检查的起始日期 check_start_date = yesterday_utc_date - timedelta(days=max_backfill_days - 1) check_start_dt = datetime.combine( check_start_date, datetime.min.time(), tzinfo=timezone.utc ) # 单次查询获取三张统计表中已有数据的日期（UNION ALL 合并） existing_daily_dates: set[date] = set() existing_model_dates: set[date] = set() existing_provider_dates: set[date] = set() q_daily = db.query( StatsDaily.date.label("dt"), literal_column("'daily'").label("src"), ).filter(StatsDaily.date >= check_start_dt) q_model = ( db.query( StatsDailyModel.date.label("dt"), literal_column("'model'").label("src"), ) .filter(StatsDailyModel.date >= check_start_dt) .distinct() ) q_provider = ( db.query( StatsDailyProvider.date.label("dt"), literal_column("'provider'").label("src"), ) .filter(StatsDailyProvider.date >= check_start_dt) .distinct() ) combined = q_daily.union_all(q_model).union_all(q_provider).all() for stat_date, src in combined: if stat_date.tzinfo is None: stat_date = stat_date.replace(tzinfo=timezone.utc) d = stat_date.date() if src == "daily": existing_daily_dates.add(d) elif src == "model": existing_model_dates.add(d) else: existing_provider_dates.add(d) # 找出需要回填的日期 all_dates = set() current = check_start_date while current <= yesterday_utc_date: all_dates.add(current) current += timedelta(days=1) # 需要回填 StatsDaily 的日期 missing_daily_dates = all_dates - existing_daily_dates # 需要回填 StatsDailyModel 的日期 missing_model_dates = all_dates - existing_model_dates # 需要回填 StatsDailyProvider 的日期 missing_provider_dates = all_dates - existing_provider_dates # 合并所有需要处理的日期 dates_to_process = ( missing_daily_dates | missing_model_dates | missing_provider_dates ) if dates_to_process: sorted_dates = sorted(dates_to_process) logger.info( f"检测到 {len(dates_to_process)} 天的统计数据需要回填 " f"(StatsDaily 缺失 {len(missing_daily_dates)} 天, " f"StatsDailyModel 缺失 {len(missing_model_dates)} 天, " f"StatsDailyProvider 缺失 {len(missing_provider_dates)} 天)" ) users = db.query(DBUser.id).filter(DBUser.is_active.is_(True)).all() user_ids = [user_id for (user_id,) in users] failed_dates = 0 for current_date in sorted_dates: try: current_date_utc = datetime.combine( current_date, datetime.min.time(), tzinfo=timezone.utc ) StatsAggregatorService.aggregate_daily_stats_bundle( db, current_date_utc, user_ids=user_ids ) db.expunge_all() except Exception as e: failed_dates += 1 logger.warning(f"回填日期 {current_date} 失败: {e}") try: db.rollback() except Exception as rollback_err: logger.error(f"回滚失败: {rollback_err}") StatsAggregatorService.update_summary(db) if failed_dates > 0: logger.warning( f"回填完成，共处理 {len(dates_to_process)} 天，" f"失败: {failed_dates} 天" ) else: logger.info(f"缺失数据回填完成，共处理 {len(dates_to_process)} 天") else: logger.info("统计数据已是最新，无需回填") return # 定时任务：聚合昨天 (UTC) 的数据 yesterday_utc = today_utc - timedelta(days=1) users = db.query(DBUser.id).filter(DBUser.is_active.is_(True)).all() user_ids = [user_id for (user_id,) in users] StatsAggregatorService.aggregate_daily_stats_bundle( db, yesterday_utc, user_ids=user_ids ) StatsAggregatorService.update_summary(db) logger.info("统计数据聚合完成") except Exception as e: logger.exception(f"统计聚合任务执行失败: {e}") try: db.rollback() except Exception: pass finally: db.close() async def _perform_wallet_daily_usage_aggregation(self) -> None: if self._wallet_daily_usage_lock.locked(): logger.info("钱包每日消费汇总任务正在运行，跳过本次触发") return async with self._wallet_daily_usage_lock: def _do() -> None: db = create_session() try: logger.info("开始执行钱包每日消费汇总...") billing_today = WalletDailyUsageLedgerService.get_today_billing_date() billing_yesterday = billing_today - timedelta(days=1) affected = WalletDailyUsageLedgerService.aggregate_day(db, billing_yesterday) logger.info( "钱包每日消费汇总完成: date={}, wallets={}", billing_yesterday.isoformat(), affected, ) except Exception as e: logger.exception("钱包每日消费汇总任务执行失败: {}", e) try: db.rollback() except Exception: pass finally: db.close() await asyncio.to_thread(_do) async def _perform_hourly_stats_aggregation(self) -> None: """执行小时统计聚合任务""" def _do() -> None: db = create_session() try: if not SystemConfigService.get_config(db, "enable_stats_aggregation", True): logger.info("统计聚合已禁用，跳过小时聚合任务") return now_utc = datetime.now(timezone.utc) last_hour = now_utc.replace(minute=0, second=0, microsecond=0) - timedelta(hours=1) StatsAggregatorService.aggregate_hourly_stats_bundle(db, last_hour) logger.info("小时统计聚合完成: {}", last_hour.isoformat()) except Exception as e: logger.exception("小时统计聚合任务执行失败: {}", e) try: db.rollback() except Exception: pass finally: db.close() await asyncio.to_thread(_do) async def _perform_pending_cleanup(self) -> None: """执行 pending 状态清理""" def _do_pending_cleanup() -> int: db = create_session() try: from src.services.usage.service import UsageService timeout_minutes = SystemConfigService.get_config( db, "pending_request_timeout_minutes", 10 ) # pending 清理涉及 candidate 表关联查询，限制批次大小以控制内存 batch_size = min( max(SystemConfigService.get_config(db, "cleanup_batch_size", 1000), 1), 200, ) return UsageService.cleanup_stale_pending_requests( db, timeout_minutes=timeout_minutes, batch_size=batch_size, ) except Exception as e: logger.exception(f"清理 pending 请求失败: {e}") db.rollback() return 0 finally: db.close() loop = asyncio.get_running_loop() cleaned_count = await loop.run_in_executor(None, _do_pending_cleanup) if cleaned_count > 0: logger.info(f"清理了 {cleaned_count} 条超时的 pending/streaming 请求") async def _perform_audit_cleanup(self) -> None: """执行审计日志清理任务""" def _do_audit_cleanup() -> int: db = create_session() try: if not SystemConfigService.get_config(db, "enable_auto_cleanup", True): logger.info("自动清理已禁用，跳过审计日志清理") return 0 audit_retention_days = max( SystemConfigService.get_config(db, "audit_log_retention_days", 30), 7, ) batch_size = SystemConfigService.get_config(db, "cleanup_batch_size", 1000) cutoff_time = datetime.now(timezone.utc) - timedelta(days=audit_retention_days) logger.info(f"开始清理 {audit_retention_days} 天前的审计日志...") total_deleted = 0 while True: batch_db = create_session() try: records_to_delete = ( batch_db.query(AuditLog.id) .filter(AuditLog.created_at < cutoff_time) .limit(batch_size) .all() ) if not records_to_delete: break record_ids = [r.id for r in records_to_delete] result = batch_db.execute( delete(AuditLog) .where(AuditLog.id.in_(record_ids)) .execution_options(synchronize_session=False) ) rows_deleted = result.rowcount batch_db.commit() total_deleted += rows_deleted logger.debug(f"已删除 {rows_deleted} 条审计日志，累计 {total_deleted} 条") except Exception as e: logger.exception(f"删除审计日志批次失败: {e}") try: batch_db.rollback() except Exception: pass break finally: batch_db.close() return total_deleted except Exception as e: logger.exception(f"审计日志清理失败: {e}") return 0 finally: db.close() loop = asyncio.get_running_loop() total_deleted = await loop.run_in_executor(None, _do_audit_cleanup) if total_deleted > 0: logger.info(f"审计日志清理完成，共删除 {total_deleted} 条记录") else: logger.info("无需清理的审计日志") async def _perform_gemini_file_mapping_cleanup(self) -> None: """清理过期的 Gemini 文件映射记录""" def _do() -> None: db = create_session() try: from src.services.gemini_files_mapping import cleanup_expired_mappings deleted_count = cleanup_expired_mappings(db) if deleted_count > 0: logger.info(f"清理了 {deleted_count} 条过期的 Gemini 文件映射") except Exception as e: logger.exception(f"Gemini 文件映射清理失败: {e}") try: db.rollback() except Exception: pass finally: db.close() await asyncio.to_thread(_do) async def _perform_provider_checkin(self) -> None: """执行 Provider 签到任务 遍历所有已配置 provider_ops 的 Provider，触发签到。 签到会在余额查询时一起执行（先签到再查询余额）。 """ def _load_provider_ids() -> list[str]: db = create_session() try: if not SystemConfigService.get_config(db, "enable_provider_checkin", True): return [] providers = ( db.query(Provider.id, Provider.config) .filter(Provider.is_active.is_(True)) .all() ) return [p.id for p in providers if p.config and p.config.get("provider_ops")] finally: db.close() try: provider_ids = await asyncio.to_thread(_load_provider_ids) if not provider_ids: logger.info("无已配置的 Provider，跳过签到任务") return logger.info(f"开始执行 Provider 签到，共 {len(provider_ids)} 个...") # 使用信号量限制并发，避免同时发起过多请求 concurrency = 3 # 签到任务并发数 semaphore = asyncio.Semaphore(concurrency) async def _checkin_provider(provider_id: str) -> tuple[str, bool, str]: """执行单个 Provider 的签到""" async with semaphore: task_db = create_session() try: service = ProviderOpsService(task_db) # 触发余额查询（会先执行签到） result = await service.query_balance(provider_id) # 检查签到结果 checkin_success = None checkin_message = "" if result.data and hasattr(result.data, "extra") and result.data.extra: checkin_success = result.data.extra.get("checkin_success") checkin_message = result.data.extra.get("checkin_message", "") if checkin_success is True: return provider_id, True, checkin_message elif checkin_success is False: return provider_id, False, checkin_message else: # None 表示未执行签到（可能没配置 Cookie） return provider_id, False, "未执行签到" except Exception as e: logger.warning(f"Provider {provider_id} 签到失败: {e}") return provider_id, False, str(e) finally: try: task_db.close() except Exception: pass # 并行执行签到 tasks = [_checkin_provider(pid) for pid in provider_ids] results = await asyncio.gather(*tasks) # 统计结果 success_count = sum(1 for _, success, _ in results if success) logger.info(f"Provider 签到完成: {success_count}/{len(provider_ids)} 成功") # 记录详细结果 for provider_id, success, message in results: if success: logger.debug(f" - {provider_id}: 签到成功 - {message}") elif message != "未执行签到": logger.debug(f" - {provider_id}: 签到失败 - {message}") except Exception as e: logger.exception(f"Provider 签到任务执行失败: {e}") async def _perform_candidate_cleanup(self) -> None: """清理过期的 request_candidates 记录""" def _do_candidate_cleanup() -> int: db = create_session() try: if not SystemConfigService.get_config(db, "enable_auto_cleanup", True): logger.info("自动清理已禁用，跳过候选记录清理") return 0 retention_days = max( SystemConfigService.get_config( db, "request_candidates_retention_days", SystemConfigService.get_config(db, "detail_log_retention_days", 7), ), 3, ) batch_size = max( SystemConfigService.get_config( db, "request_candidates_cleanup_batch_size", SystemConfigService.get_config(db, "cleanup_batch_size", 1000), ), 1, ) cutoff_time = datetime.now(timezone.utc) - timedelta(days=retention_days) logger.info( "开始清理 {} 天前的请求候选记录，batch_size={}", retention_days, batch_size, ) except Exception as e: logger.exception(f"候选记录清理配置读取失败: {e}") return 0 finally: db.close() total_deleted = 0 while True: batch_db = create_session() try: records_to_delete = ( batch_db.query(RequestCandidate.id) .filter(RequestCandidate.created_at < cutoff_time) .order_by(RequestCandidate.created_at.asc(), RequestCandidate.id.asc()) .limit(batch_size) .all() ) if not records_to_delete: break record_ids = [r.id for r in records_to_delete] result = batch_db.execute( delete(RequestCandidate) .where(RequestCandidate.id.in_(record_ids)) .execution_options(synchronize_session=False) ) rows_deleted = result.rowcount batch_db.commit() total_deleted += rows_deleted logger.debug(f"已删除 {rows_deleted} 条候选记录，累计 {total_deleted} 条") except Exception as e: logger.exception(f"删除候选记录批次失败: {e}") try: batch_db.rollback() except Exception: pass break finally: batch_db.close() return total_deleted loop = asyncio.get_running_loop() total_deleted = await loop.run_in_executor(None, _do_candidate_cleanup) if total_deleted > 0: logger.info(f"请求候选记录清理完成，共删除 {total_deleted} 条记录") else: logger.info("无需清理的候选记录") async def _perform_db_maintenance(self) -> None: """执行数据库表维护（VACUUM ANALYZE） 对大表执行 VACUUM ANALYZE，防止表和索引膨胀。 VACUUM 不能在事务内执行，需要使用 autocommit 连接。 使用线程池执行，避免阻塞事件循环。 """ db = create_session() try: if not SystemConfigService.get_config(db, "enable_db_maintenance", True): logger.info("数据库维护已禁用，跳过") return except Exception as e: logger.exception(f"读取数据库维护配置失败: {e}") return finally: db.close() tables = ["usage", "request_candidates", "audit_logs"] logger.info(f"开始数据库表维护（VACUUM ANALYZE），目标表: {', '.join(tables)}") from src.database.database import _ensure_engine engine = _ensure_engine() def _vacuum_table(table_name: str) -> tuple[str, bool, str]: """在线程池中执行 VACUUM ANALYZE（同步阻塞操作）""" try: with engine.connect() as raw_conn: conn = raw_conn.execution_options(isolation_level="AUTOCOMMIT") conn.execute(text(f"VACUUM ANALYZE {table_name}")) return table_name, True, "" except Exception as e: return table_name, False, str(e) loop = asyncio.get_running_loop() for table in tables: table_name, success, error = await loop.run_in_executor(None, _vacuum_table, table) if success: logger.info(f"VACUUM ANALYZE {table_name} 完成") else: logger.warning(f"VACUUM ANALYZE {table_name} 失败: {error}") logger.info("数据库表维护完成") async def _perform_cleanup(self) -> None: """执行清理任务（在线程池中运行，避免阻塞事件循环）""" def _do_cleanup() -> None: db = create_session() try: if not SystemConfigService.get_config(db, "enable_auto_cleanup", True): logger.info("自动清理已禁用，跳过清理任务") return logger.info("开始执行使用记录分级清理...") detail_retention = SystemConfigService.get_config( db, "detail_log_retention_days", 7 ) compressed_retention = SystemConfigService.get_config( db, "compressed_log_retention_days", 30 ) header_retention = SystemConfigService.get_config(db, "header_retention_days", 90) log_retention = SystemConfigService.get_config(db, "log_retention_days", 365) batch_size = SystemConfigService.get_config(db, "cleanup_batch_size", 1000) auto_delete = SystemConfigService.get_config(db, "auto_delete_expired_keys", False) except Exception as e: logger.exception(f"清理任务配置读取失败: {e}") return finally: db.close() try: now = datetime.now(timezone.utc) detail_cutoff = now - timedelta(days=detail_retention) compressed_cutoff = now - timedelta(days=compressed_retention) header_cutoff = now - timedelta(days=header_retention) log_cutoff = now - timedelta(days=log_retention) # 先删最老的整行，再按窗口处理剩余记录，避免同一行在一轮里被重复改写。 records_deleted = self._delete_old_records(log_cutoff, batch_size) header_cleaned = self._cleanup_header_fields( header_cutoff, batch_size, newer_than=log_cutoff ) body_cleaned = self._cleanup_stale_body_fields( compressed_cutoff, batch_size, newer_than=log_cutoff ) # 仅压缩 7-30 天窗口内的 body；更老记录直接清空 body，不再先压缩再清理。 body_compressed = self._cleanup_body_fields( detail_cutoff, batch_size, newer_than=compressed_cutoff ) # 5. 清理过期的API Keys keys_db = create_session() try: keys_cleaned = ApiKeyService.cleanup_expired_keys( keys_db, auto_delete=auto_delete ) except Exception as e: logger.exception(f"清理过期 Keys 失败: {e}") keys_cleaned = 0 finally: keys_db.close() logger.info( f"清理完成: 压缩 {body_compressed} 条, " f"清理body {body_cleaned} 条, " f"清理header {header_cleaned} 条, " f"删除记录 {records_deleted} 条, " f"清理过期Keys {keys_cleaned} 条" ) except Exception as e: logger.exception(f"清理任务执行失败: {e}") loop = asyncio.get_running_loop() await loop.run_in_executor(None, _do_cleanup) def _cleanup_body_fields( self, cutoff_time: datetime, batch_size: int, *, newer_than: datetime | None = None, ) -> int: """压缩 request_body 和 response_body 字段到压缩字段 仅处理指定时间窗口内仍保留原始 body 的记录，避免对更老记录重复写放大。 """ from sqlalchemy import null, update total_compressed = 0 no_progress_count = 0 memory_safe_batch_size = max(1, min(batch_size, 25)) if newer_than is not None and newer_than >= cutoff_time: logger.warning( "压缩 body 字段跳过: 无效时间窗口 newer_than={} cutoff_time={}", newer_than, cutoff_time, ) return 0 while True: batch_db = create_session() try: query = batch_db.query( Usage.id, Usage.request_body, Usage.response_body, Usage.provider_request_body, Usage.client_response_body, ).filter(Usage.created_at < cutoff_time) if newer_than is not None: query = query.filter(Usage.created_at >= newer_than) records = ( query.filter( (Usage.request_body.isnot(None)) | (Usage.response_body.isnot(None)) | (Usage.provider_request_body.isnot(None)) | (Usage.client_response_body.isnot(None)) ) .order_by(Usage.created_at.asc(), Usage.id.asc()) .limit(memory_safe_batch_size) .all() ) if not records: break batch_success = 0 batch_progress = False for record in records: result = batch_db.execute( update(Usage) .where(Usage.id == record.id) .values( request_body=null(), response_body=null(), provider_request_body=null(), client_response_body=null(), request_body_compressed=( compress_json(record.request_body) if record.request_body else None ), response_body_compressed=( compress_json(record.response_body) if record.response_body else None ), provider_request_body_compressed=( compress_json(record.provider_request_body) if record.provider_request_body else None ), client_response_body_compressed=( compress_json(record.client_response_body) if record.client_response_body else None ), ) .execution_options(synchronize_session=False) ) if result.rowcount > 0: batch_success += 1 batch_progress = True batch_db.commit() except Exception as e: logger.warning("压缩 body 批次失败: {}", e) try: batch_db.rollback() except Exception: pass break finally: batch_db.close() if not batch_progress: no_progress_count += 1 if no_progress_count >= 3: logger.error( f"压缩 body 字段连续 {no_progress_count} 批无进展，" "终止循环以避免死循环" ) break else: no_progress_count = 0 total_compressed += batch_success if batch_success > 0: logger.debug( f"已压缩 {batch_success} 条记录的 body 字段，累计 {total_compressed} 条" ) return total_compressed def _cleanup_stale_body_fields( self, cutoff_time: datetime, batch_size: int, *, newer_than: datetime | None = None, ) -> int: """清理已超过压缩保留期的 body 字段 直接清空 raw/compressed body，避免更老记录先压缩再马上被清掉。 """ from sqlalchemy import null, update total_cleaned = 0 if newer_than is not None and newer_than >= cutoff_time: logger.warning( "清理 body 字段跳过: 无效时间窗口 newer_than={} cutoff_time={}", newer_than, cutoff_time, ) return 0 while True: batch_db = create_session() try: query = batch_db.query(Usage.id).filter(Usage.created_at < cutoff_time) if newer_than is not None: query = query.filter(Usage.created_at >= newer_than) records_to_clean = ( query.filter( (Usage.request_body.isnot(None)) | (Usage.response_body.isnot(None)) | (Usage.provider_request_body.isnot(None)) | (Usage.client_response_body.isnot(None)) | (Usage.request_body_compressed.isnot(None)) | (Usage.response_body_compressed.isnot(None)) | (Usage.provider_request_body_compressed.isnot(None)) | (Usage.client_response_body_compressed.isnot(None)) ) .order_by(Usage.created_at.asc(), Usage.id.asc()) .limit(batch_size) .all() ) if not records_to_clean: break record_ids = [r.id for r in records_to_clean] result = batch_db.execute( update(Usage) .where(Usage.id.in_(record_ids)) .values( request_body=null(), response_body=null(), provider_request_body=null(), client_response_body=null(), request_body_compressed=null(), response_body_compressed=null(), provider_request_body_compressed=null(), client_response_body_compressed=null(), ) ) rows_updated = result.rowcount batch_db.commit() if rows_updated == 0: logger.warning("清理 body 字段: rowcount=0，可能存在问题") break total_cleaned += rows_updated logger.debug(f"已清理 {rows_updated} 条记录的 body 字段，累计 {total_cleaned} 条") except Exception as e: logger.exception(f"清理 body 字段失败: {e}") try: batch_db.rollback() except Exception: pass break finally: batch_db.close() return total_cleaned def _cleanup_header_fields( self, cutoff_time: datetime, batch_size: int, *, newer_than: datetime | None = None, ) -> int: """清理 request_headers, response_headers 和 provider_request_headers 字段 每批使用短生命周期 session（同步方法，在线程池中调用） """ from sqlalchemy import null, update total_cleaned = 0 if newer_than is not None and newer_than >= cutoff_time: logger.warning( "清理 header 字段跳过: 无效时间窗口 newer_than={} cutoff_time={}", newer_than, cutoff_time, ) return 0 while True: batch_db = create_session() try: query = batch_db.query(Usage.id).filter(Usage.created_at < cutoff_time) if newer_than is not None: query = query.filter(Usage.created_at >= newer_than) records_to_clean = ( query.filter( (Usage.request_headers.isnot(None)) | (Usage.response_headers.isnot(None)) | (Usage.provider_request_headers.isnot(None)) | (Usage.client_response_headers.isnot(None)) ) .order_by(Usage.created_at.asc(), Usage.id.asc()) .limit(batch_size) .all() ) if not records_to_clean: break record_ids = [r.id for r in records_to_clean] result = batch_db.execute( update(Usage) .where(Usage.id.in_(record_ids)) .values( request_headers=null(), response_headers=null(), provider_request_headers=null(), client_response_headers=null(), ) ) rows_updated = result.rowcount batch_db.commit() if rows_updated == 0: logger.warning("清理 header 字段: rowcount=0，可能存在问题") break total_cleaned += rows_updated logger.debug(f"已清理 {rows_updated} 条记录的 header 字段，累计 {total_cleaned} 条") except Exception as e: logger.exception(f"清理 header 字段失败: {e}") try: batch_db.rollback() except Exception: pass break finally: batch_db.close() return total_cleaned def _delete_old_records(self, cutoff_time: datetime, batch_size: int) -> int: """删除过期的完整记录（同步方法，在线程池中调用）""" total_deleted = 0 while True: batch_db = create_session() try: records_to_delete = ( batch_db.query(Usage.id) .filter(Usage.created_at < cutoff_time) .order_by(Usage.created_at.asc(), Usage.id.asc()) .limit(batch_size) .all() ) if not records_to_delete: break record_ids = [r.id for r in records_to_delete] result = batch_db.execute( delete(Usage) .where(Usage.id.in_(record_ids)) .execution_options(synchronize_session=False) ) rows_deleted = result.rowcount batch_db.commit() total_deleted += rows_deleted logger.debug(f"已删除 {rows_deleted} 条过期记录，累计 {total_deleted} 条") except Exception as e: logger.exception(f"删除过期记录失败: {e}") try: batch_db.rollback() except Exception: pass break finally: batch_db.close() return total_deleted def should_start_python_maintenance_scheduler() -> bool: return bool( config.audit_cleanup_python_enabled or config.antigravity_ua_refresh_python_enabled or config.db_maintenance_python_enabled or config.gemini_file_mapping_cleanup_python_enabled or config.http_client_idle_cleanup_python_enabled or config.pending_cleanup_python_enabled or config.pool_monitor_python_enabled or config.provider_checkin_python_enabled or config.request_candidate_cleanup_python_enabled or config.stats_aggregation_python_enabled or config.stats_hourly_aggregation_python_enabled or config.usage_cleanup_python_enabled or config.wallet_daily_usage_aggregation_python_enabled ) # 全局单例 _maintenance_scheduler = None def get_maintenance_scheduler() -> MaintenanceScheduler: """获取维护调度器单例""" global _maintenance_scheduler if _maintenance_scheduler is None: _maintenance_scheduler = MaintenanceScheduler() return _maintenance_scheduler