mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 01:10:23 +08:00
feat: 添加启动任务开关,修复统计聚合内存泄漏
- 新增 CACHE_WARMUP_ENABLED 和 MAINTENANCE_STARTUP_TASKS_ENABLED 环境变量, 允许禁用缓存预热和维护调度器启动任务 - 在统计聚合批量处理循环中添加 db.expunge_all(),释放 Session identity map, 防止 ORM 对象累积导致内存暴涨 - 添加启动任务开关的单元测试
This commit is contained in:
@@ -312,6 +312,14 @@ class Config:
|
|||||||
# 每个用户最多可创建的 Management Token 数量
|
# 每个用户最多可创建的 Management Token 数量
|
||||||
self.management_token_max_per_user = int(os.getenv("MANAGEMENT_TOKEN_MAX_PER_USER", "20"))
|
self.management_token_max_per_user = int(os.getenv("MANAGEMENT_TOKEN_MAX_PER_USER", "20"))
|
||||||
|
|
||||||
|
# 启动任务开关
|
||||||
|
# CACHE_WARMUP_ENABLED: 是否在启动时执行缓存预热
|
||||||
|
# MAINTENANCE_STARTUP_TASKS_ENABLED: 是否在启动时执行维护调度器初始化任务(清理、统计回填等)
|
||||||
|
self.cache_warmup_enabled = os.getenv("CACHE_WARMUP_ENABLED", "true").lower() == "true"
|
||||||
|
self.maintenance_startup_tasks_enabled = (
|
||||||
|
os.getenv("MAINTENANCE_STARTUP_TASKS_ENABLED", "true").lower() == "true"
|
||||||
|
)
|
||||||
|
|
||||||
# API 文档配置
|
# API 文档配置
|
||||||
# DOCS_ENABLED: 是否启用 API 文档(/docs, /redoc, /openapi.json)
|
# DOCS_ENABLED: 是否启用 API 文档(/docs, /redoc, /openapi.json)
|
||||||
# - 未设置: 开发环境启用,生产环境禁用
|
# - 未设置: 开发环境启用,生产环境禁用
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from typing import Any
|
|||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from src.config.settings import config
|
||||||
from src.core.enums import UserRole
|
from src.core.enums import UserRole
|
||||||
from src.core.logger import logger
|
from src.core.logger import logger
|
||||||
from src.database import create_session
|
from src.database import create_session
|
||||||
@@ -172,4 +173,8 @@ class CacheWarmupService:
|
|||||||
|
|
||||||
async def start_cache_warmup() -> None:
|
async def start_cache_warmup() -> None:
|
||||||
"""启动缓存预热(作为后台任务)"""
|
"""启动缓存预热(作为后台任务)"""
|
||||||
|
if not config.cache_warmup_enabled:
|
||||||
|
logger.info("缓存预热已禁用(CACHE_WARMUP_ENABLED=false)")
|
||||||
|
return
|
||||||
|
|
||||||
asyncio.create_task(CacheWarmupService.warmup_all())
|
asyncio.create_task(CacheWarmupService.warmup_all())
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ from typing import Any
|
|||||||
|
|
||||||
from sqlalchemy import delete, literal_column, text
|
from sqlalchemy import delete, literal_column, text
|
||||||
|
|
||||||
|
from src.config.settings import config
|
||||||
from src.core.logger import logger
|
from src.core.logger import logger
|
||||||
from src.database import create_session
|
from src.database import create_session
|
||||||
from src.models.database import AuditLog, Provider, RequestCandidate, Usage
|
from src.models.database import AuditLog, Provider, RequestCandidate, Usage
|
||||||
@@ -232,7 +233,10 @@ class MaintenanceScheduler:
|
|||||||
)
|
)
|
||||||
|
|
||||||
# 启动时执行一次初始化任务
|
# 启动时执行一次初始化任务
|
||||||
asyncio.create_task(self._run_startup_tasks())
|
if config.maintenance_startup_tasks_enabled:
|
||||||
|
asyncio.create_task(self._run_startup_tasks())
|
||||||
|
else:
|
||||||
|
logger.info("维护调度器启动任务已禁用(MAINTENANCE_STARTUP_TASKS_ENABLED=false)")
|
||||||
|
|
||||||
async def _run_startup_tasks(self) -> None:
|
async def _run_startup_tasks(self) -> None:
|
||||||
"""启动时执行的初始化任务"""
|
"""启动时执行的初始化任务"""
|
||||||
@@ -470,6 +474,7 @@ class MaintenanceScheduler:
|
|||||||
StatsAggregatorService.aggregate_daily_stats_bundle(
|
StatsAggregatorService.aggregate_daily_stats_bundle(
|
||||||
db, current_date_utc, user_ids=user_ids
|
db, current_date_utc, user_ids=user_ids
|
||||||
)
|
)
|
||||||
|
db.expunge_all()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
failed_dates += 1
|
failed_dates += 1
|
||||||
logger.warning(f"回填日期 {current_date} 失败: {e}")
|
logger.warning(f"回填日期 {current_date} 失败: {e}")
|
||||||
|
|||||||
@@ -1096,6 +1096,7 @@ class StatsAggregatorService:
|
|||||||
current_date = start_date
|
current_date = start_date
|
||||||
while current_date < today_utc:
|
while current_date < today_utc:
|
||||||
StatsAggregatorService.aggregate_daily_stats_bundle(db, current_date, user_ids=user_ids)
|
StatsAggregatorService.aggregate_daily_stats_bundle(db, current_date, user_ids=user_ids)
|
||||||
|
db.expunge_all() # 释放 Session identity map,防止 ORM 对象累积导致内存暴涨
|
||||||
count += 1
|
count += 1
|
||||||
current_date += timedelta(days=1)
|
current_date += timedelta(days=1)
|
||||||
|
|
||||||
@@ -1114,6 +1115,7 @@ class StatsAggregatorService:
|
|||||||
count = 0
|
count = 0
|
||||||
while current <= end_dt:
|
while current <= end_dt:
|
||||||
StatsAggregatorService.aggregate_daily_api_key_stats(db, current, commit=True)
|
StatsAggregatorService.aggregate_daily_api_key_stats(db, current, commit=True)
|
||||||
|
db.expunge_all()
|
||||||
count += 1
|
count += 1
|
||||||
current += timedelta(days=1)
|
current += timedelta(days=1)
|
||||||
return count
|
return count
|
||||||
@@ -1145,9 +1147,11 @@ class StatsAggregatorService:
|
|||||||
processed += 1
|
processed += 1
|
||||||
if processed % 30 == 0:
|
if processed % 30 == 0:
|
||||||
db.commit()
|
db.commit()
|
||||||
|
db.expunge_all()
|
||||||
current += timedelta(days=1)
|
current += timedelta(days=1)
|
||||||
|
|
||||||
db.commit()
|
db.commit()
|
||||||
|
db.expunge_all()
|
||||||
return processed
|
return processed
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -1183,6 +1187,7 @@ class StatsAggregatorService:
|
|||||||
db.commit()
|
db.commit()
|
||||||
last_id = records[-1].id
|
last_id = records[-1].id
|
||||||
total_processed += len(records)
|
total_processed += len(records)
|
||||||
|
db.expunge_all()
|
||||||
|
|
||||||
return total_processed
|
return total_processed
|
||||||
|
|
||||||
@@ -1194,6 +1199,7 @@ class StatsAggregatorService:
|
|||||||
count = 0
|
count = 0
|
||||||
while current <= end_dt:
|
while current <= end_dt:
|
||||||
StatsAggregatorService.aggregate_daily_error_stats(db, current, commit=True)
|
StatsAggregatorService.aggregate_daily_error_stats(db, current, commit=True)
|
||||||
|
db.expunge_all()
|
||||||
count += 1
|
count += 1
|
||||||
current += timedelta(days=1)
|
current += timedelta(days=1)
|
||||||
return count
|
return count
|
||||||
|
|||||||
86
tests/services/test_startup_toggles.py
Normal file
86
tests/services/test_startup_toggles.py
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import inspect
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import src.services.system.cache_warmup as cache_warmup_module
|
||||||
|
import src.services.system.maintenance_scheduler as maintenance_scheduler_module
|
||||||
|
from src.config.settings import config
|
||||||
|
from src.services.system.maintenance_scheduler import MaintenanceScheduler
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_start_cache_warmup_skips_task_when_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(config, "cache_warmup_enabled", False)
|
||||||
|
|
||||||
|
created = False
|
||||||
|
|
||||||
|
def fake_create_task(coro): # type: ignore[no-untyped-def]
|
||||||
|
nonlocal created
|
||||||
|
created = True
|
||||||
|
if inspect.iscoroutine(coro):
|
||||||
|
coro.close()
|
||||||
|
return object()
|
||||||
|
|
||||||
|
monkeypatch.setattr(cache_warmup_module.asyncio, "create_task", fake_create_task)
|
||||||
|
|
||||||
|
await cache_warmup_module.start_cache_warmup()
|
||||||
|
|
||||||
|
assert created is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_start_cache_warmup_creates_task_when_enabled(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(config, "cache_warmup_enabled", True)
|
||||||
|
|
||||||
|
created = False
|
||||||
|
|
||||||
|
def fake_create_task(coro): # type: ignore[no-untyped-def]
|
||||||
|
nonlocal created
|
||||||
|
created = True
|
||||||
|
if inspect.iscoroutine(coro):
|
||||||
|
coro.close()
|
||||||
|
return object()
|
||||||
|
|
||||||
|
monkeypatch.setattr(cache_warmup_module.asyncio, "create_task", fake_create_task)
|
||||||
|
|
||||||
|
await cache_warmup_module.start_cache_warmup()
|
||||||
|
|
||||||
|
assert created is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_maintenance_scheduler_start_skips_startup_task_when_disabled(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(config, "maintenance_startup_tasks_enabled", False)
|
||||||
|
|
||||||
|
scheduler = MaintenanceScheduler()
|
||||||
|
|
||||||
|
created = False
|
||||||
|
|
||||||
|
def fake_create_task(coro): # type: ignore[no-untyped-def]
|
||||||
|
nonlocal created
|
||||||
|
created = True
|
||||||
|
if inspect.iscoroutine(coro):
|
||||||
|
coro.close()
|
||||||
|
return object()
|
||||||
|
|
||||||
|
monkeypatch.setattr(maintenance_scheduler_module.asyncio, "create_task", fake_create_task)
|
||||||
|
monkeypatch.setattr(scheduler, "_get_checkin_time", lambda: (1, 5))
|
||||||
|
monkeypatch.setattr(
|
||||||
|
maintenance_scheduler_module,
|
||||||
|
"get_scheduler",
|
||||||
|
lambda: SimpleNamespace(
|
||||||
|
add_cron_job=lambda *args, **kwargs: None,
|
||||||
|
add_interval_job=lambda *args, **kwargs: None,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
await scheduler.start()
|
||||||
|
|
||||||
|
assert created is False
|
||||||
Reference in New Issue
Block a user