refactor: 移除 Python 后端源码,全面迁移至 Rust gateway 架构

- 删除全部 Python 源码 (src/) 及 Alembic 迁移脚本,归档至 _deprecated_py_src/
- 重构 Rust gateway ai_pipeline: 拆分 planner/finalize 模块,新增 contracts/adaptation 层
- 重组 handlers 模块为 admin/public/proxy/internal/shared 子模块结构
- 新增 executor 模块,引入 Rust 原生数据库迁移 (aether-data/migrations)
- 简化 CI/Docker 构建流程,移除 base image 二级构建,统一为单一 app image
- 移除 Python 相关基础设施文件 (entrypoint.sh, gunicorn_conf.py, Dockerfile.base)
This commit is contained in:
fawney19
2026-04-03 16:26:16 +08:00
parent 8f26e1a31f
commit 1d9c77522a
868 changed files with 1735 additions and 2433 deletions

View File

@@ -0,0 +1,47 @@
"""
模块化系统核心
提供可扩展的功能模块管理,支持:
- 声明式模块注册
- available/enabled 双层状态控制
- 延迟导入避免重依赖加载
- 前后端状态同步
"""
from src.core.modules.base import (
ModuleCategory,
ModuleDefinition,
ModuleMetadata,
ModuleStatus,
)
from src.core.modules.hooks import (
AUTH_AUTHENTICATE,
AUTH_CHECK_EXCLUSIVE_MODE,
AUTH_CHECK_REGISTRATION,
AUTH_GET_METHODS,
AUTH_TOKEN_PREFIX_AUTHENTICATORS,
HookDispatcher,
HookSpec,
HookStrategy,
get_hook_dispatcher,
)
from src.core.modules.registry import ModuleRegistry, get_module_registry
__all__ = [
"ModuleCategory",
"ModuleMetadata",
"ModuleDefinition",
"ModuleStatus",
"ModuleRegistry",
"get_module_registry",
# Hook system
"HookDispatcher",
"HookSpec",
"HookStrategy",
"get_hook_dispatcher",
"AUTH_GET_METHODS",
"AUTH_AUTHENTICATE",
"AUTH_CHECK_REGISTRATION",
"AUTH_CHECK_EXCLUSIVE_MODE",
"AUTH_TOKEN_PREFIX_AUTHENTICATORS",
]

View File

@@ -0,0 +1,127 @@
"""
模块基础定义
包含模块元数据、定义和状态的数据结构
"""
from __future__ import annotations
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
from enum import Enum
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from fastapi import APIRouter
from sqlalchemy.orm import Session
class ModuleCategory(str, Enum):
"""模块分类"""
AUTH = "auth" # 认证相关
MONITORING = "monitoring" # 监控相关
SECURITY = "security" # 安全相关
INTEGRATION = "integration" # 第三方集成
class ModuleHealth(str, Enum):
"""模块健康状态"""
HEALTHY = "healthy"
DEGRADED = "degraded"
UNHEALTHY = "unhealthy"
UNKNOWN = "unknown"
@dataclass
class ModuleMetadata:
"""
模块元数据 - 纯数据描述,无重依赖
用于声明式定义模块的基本信息和配置
"""
# 基本信息
name: str # 唯一标识: ldap, audit_log
display_name: str # 显示名称: "LDAP 认证"
description: str # 模块描述
# 分类
category: ModuleCategory
# 可用性控制(部署级)
env_key: str # 环境变量名: LDAP_AVAILABLE
default_available: bool = False # 默认是否可用
required_packages: list[str] = field(default_factory=list) # 依赖的 Python 包
dependencies: list[str] = field(default_factory=list) # 依赖的其他模块
# 路由配置 - 模块自定义前缀
api_prefix: str | None = None # 如 "/api/admin/ldap"
# 前端配置
admin_route: str | None = None # 管理页面路由: "/admin/ldap"
admin_menu_icon: str | None = None # 菜单图标
admin_menu_group: str | None = None # 菜单分组: "system", "security"
admin_menu_order: int = 100 # 菜单排序(越小越靠前)
@dataclass
class ModuleDefinition:
"""
完整模块定义
包含元数据和生命周期钩子,钩子函数内部延迟导入重依赖
"""
metadata: ModuleMetadata
# 工厂函数 - 内部再 import 重依赖
router_factory: Callable[[], APIRouter] | None = None
service_factory: Callable[[], Any] | None = None
# 生命周期钩子
on_startup: Callable[[], Awaitable[None]] | None = None
on_shutdown: Callable[[], Awaitable[None]] | None = None
health_check: Callable[[], Awaitable[ModuleHealth]] | None = None
# 自定义依赖检测(可选,用于检测 ldap3 等库是否安装)
check_dependencies: Callable[[], bool] | None = None
# 配置验证(可选,启用模块时调用,返回 (success, error_message)
validate_config: Callable[[Session], tuple[bool, str]] | None = None
# 钩子实现(可选)
# {hook_name: handler_callable}
# 模块通过此字段声明自己对核心扩展点的实现
hooks: dict[str, Callable[..., Any]] = field(default_factory=dict)
@dataclass
class ModuleStatus:
"""
模块运行状态
用于 API 返回,供前端使用
"""
name: str
available: bool # 部署级可用(环境变量 + 依赖库)
enabled: bool # 运行级启用(数据库配置)
active: bool # 最终激活状态 (available && enabled && dependencies_ok)
config_validated: bool # 配置验证通过(只有验证通过才允许启用)
config_error: str | None # 配置验证失败的错误信息
# 显示信息
display_name: str
description: str
category: ModuleCategory
# 前端配置
admin_route: str | None
admin_menu_icon: str | None
admin_menu_group: str | None
admin_menu_order: int
# 健康状态
health: ModuleHealth = ModuleHealth.UNKNOWN

View File

@@ -0,0 +1,245 @@
"""
模块钩子系统
提供模块与核心代码之间的动态扩展点。
模块通过 ModuleDefinition.hooks 声明钩子实现,
核心代码通过 HookDispatcher 调用所有活跃模块的钩子。
"""
from __future__ import annotations
from collections import defaultdict
from dataclasses import dataclass
from enum import Enum
from inspect import isawaitable
from typing import TYPE_CHECKING, Any
from src.core.logger import logger
if TYPE_CHECKING:
from sqlalchemy.orm import Session
# 钩子处理器类型: 可以是同步或异步函数
HookHandler = Any # Callable[..., Any]
class HookStrategy(str, Enum):
"""钩子执行策略"""
FIRST_RESULT = "first_result" # 返回第一个非 None 结果
COLLECT_ALL = "collect_all" # 收集所有结果到列表
@dataclass(frozen=True)
class HookSpec:
"""钩子规格定义"""
name: str # 如 "auth.authenticate"
strategy: HookStrategy = HookStrategy.FIRST_RESULT
requires_active_check: bool = True # 是否过滤非活跃模块
# ==================== 预定义钩子规格 ====================
AUTH_GET_METHODS = HookSpec(
name="auth.get_methods",
strategy=HookStrategy.COLLECT_ALL,
)
"""查询所有可用认证方法。返回 list[dict],每个 dict 包含认证方式信息。"""
AUTH_AUTHENTICATE = HookSpec(
name="auth.authenticate",
strategy=HookStrategy.FIRST_RESULT,
)
"""模块参与认证流程。kwargs: db, email, password, auth_type。返回 User 或 None。"""
AUTH_CHECK_REGISTRATION = HookSpec(
name="auth.check_registration",
strategy=HookStrategy.FIRST_RESULT,
)
"""模块检查是否允许本地注册。返回 {"blocked": True, "reason": "..."} 或 None。"""
AUTH_CHECK_EXCLUSIVE_MODE = HookSpec(
name="auth.check_exclusive_mode",
strategy=HookStrategy.FIRST_RESULT,
)
"""检查是否有模块开启了排他登录模式。返回 True 或 None。"""
AUTH_TOKEN_PREFIX_AUTHENTICATORS = HookSpec(
name="auth.token_prefix_authenticators",
strategy=HookStrategy.COLLECT_ALL,
requires_active_check=False, # token 前缀认证是核心鉴权路径,只要模块已注册即可
)
"""获取 token 前缀认证器列表。返回 list[{"prefix": "ae_", "module": "..."}]。"""
class HookDispatcher:
"""
钩子分发器 -- 单例
职责:
- 注册模块的钩子实现
- 在核心代码调用时,只执行活跃模块的钩子
- 支持 FIRST_RESULT 和 COLLECT_ALL 两种执行策略
"""
_instance: HookDispatcher | None = None
def __init__(self) -> None:
# {hook_name: [(module_name, handler), ...]}
self._handlers: defaultdict[str, list[tuple[str, HookHandler]]] = defaultdict(list)
@classmethod
def get_instance(cls) -> HookDispatcher:
if cls._instance is None:
cls._instance = cls()
return cls._instance
@classmethod
def reset_instance(cls) -> None:
"""重置单例(仅用于测试)"""
cls._instance = None
def register(self, hook_name: str, module_name: str, handler: HookHandler) -> None:
"""注册钩子处理器"""
self._handlers[hook_name].append((module_name, handler))
logger.debug("Hook [{}] registered handler from module [{}]", hook_name, module_name)
def has_handlers(self, hook_name: str) -> bool:
"""检查是否有注册的处理器"""
return bool(self._handlers.get(hook_name))
def _get_active_handlers(
self, spec: HookSpec, db: Session | None
) -> list[tuple[str, HookHandler]]:
"""获取活跃模块的处理器列表"""
handlers = self._handlers.get(spec.name, [])
if not handlers:
return []
if not spec.requires_active_check or db is None:
return handlers
from src.core.modules.registry import get_module_registry
registry = get_module_registry()
return [(name, handler) for name, handler in handlers if registry.is_active(name, db)]
# ==================== 异步分发 ====================
async def dispatch(
self,
spec: HookSpec,
**kwargs: Any,
) -> Any:
"""
异步分发钩子调用
从 kwargs 中提取 db 参数用于活跃性检查,所有 kwargs 原样传递给处理器。
Args:
spec: 钩子规格
**kwargs: 传递给处理器的参数(其中 db 同时用于活跃性检查)
Returns:
FIRST_RESULT: 第一个非 None 结果,或 None
COLLECT_ALL: 结果列表
"""
db = kwargs.get("db")
active_handlers = self._get_active_handlers(spec, db)
if not active_handlers:
return [] if spec.strategy == HookStrategy.COLLECT_ALL else None
if spec.strategy == HookStrategy.FIRST_RESULT:
return await self._dispatch_first_result(spec.name, active_handlers, **kwargs)
elif spec.strategy == HookStrategy.COLLECT_ALL:
return await self._dispatch_collect_all(spec.name, active_handlers, **kwargs)
return None
async def _call_handler(self, handler: HookHandler, **kwargs: Any) -> Any:
"""调用处理器(支持同步和异步)"""
result = handler(**kwargs)
if isawaitable(result):
return await result
return result
async def _dispatch_first_result(
self, hook_name: str, handlers: list[tuple[str, HookHandler]], **kwargs: Any
) -> Any:
for module_name, handler in handlers:
try:
result = await self._call_handler(handler, **kwargs)
if result is not None:
return result
except Exception as e:
logger.error("Hook [{}] handler from [{}] failed: {}", hook_name, module_name, e)
return None
async def _dispatch_collect_all(
self, hook_name: str, handlers: list[tuple[str, HookHandler]], **kwargs: Any
) -> list[Any]:
results: list[Any] = []
for module_name, handler in handlers:
try:
result = await self._call_handler(handler, **kwargs)
if result is not None:
if isinstance(result, list):
results.extend(result)
else:
results.append(result)
except Exception as e:
logger.error("Hook [{}] handler from [{}] failed: {}", hook_name, module_name, e)
return results
# ==================== 同步分发 ====================
def dispatch_sync(
self,
spec: HookSpec,
**kwargs: Any,
) -> Any:
"""
同步版本的 dispatch仅适用于同步钩子处理器
从 kwargs 中提取 db 参数用于活跃性检查,所有 kwargs 原样传递给处理器。
用于无法使用 await 的同步上下文(如 OAuthService 的某些方法)。
"""
db = kwargs.get("db")
active_handlers = self._get_active_handlers(spec, db)
if not active_handlers:
return [] if spec.strategy == HookStrategy.COLLECT_ALL else None
if spec.strategy == HookStrategy.FIRST_RESULT:
for module_name, handler in active_handlers:
try:
result = handler(**kwargs)
if result is not None:
return result
except Exception as e:
logger.error(
"Hook [{}] sync handler from [{}] failed: {}", spec.name, module_name, e
)
return None
elif spec.strategy == HookStrategy.COLLECT_ALL:
results: list[Any] = []
for module_name, handler in active_handlers:
try:
result = handler(**kwargs)
if result is not None:
if isinstance(result, list):
results.extend(result)
else:
results.append(result)
except Exception as e:
logger.error(
"Hook [{}] sync handler from [{}] failed: {}", spec.name, module_name, e
)
return results
return None
def get_hook_dispatcher() -> HookDispatcher:
"""获取钩子分发器实例"""
return HookDispatcher.get_instance()

View File

@@ -0,0 +1,416 @@
"""
模块注册中心
负责模块的注册、状态管理和生命周期控制
"""
from __future__ import annotations
import importlib
import importlib.util
import os
from typing import TYPE_CHECKING, Any, Protocol
from src.core.logger import logger
from src.core.modules.base import (
ModuleCategory,
ModuleDefinition,
ModuleHealth,
ModuleStatus,
)
if TYPE_CHECKING:
from sqlalchemy.orm import Session
class ConfigBackend(Protocol):
"""模块配置读写后端协议。
通过 ``ModuleRegistry.set_config_backend()`` 在应用启动时注入实现,
使 core 层无需在运行时 import services 层。
"""
def get_config(self, db: Any, key: str, default: Any = None) -> Any: ...
def set_config(self, db: Any, key: str, value: Any, description: Any = None) -> Any: ...
class _DefaultConfigBackend:
"""默认配置后端:始终返回 default用于独立脚本/极简测试场景)。"""
def get_config(self, _db: Any, _key: str, default: Any = None) -> Any:
return default
def set_config(self, _db: Any, _key: str, _value: Any, _description: Any = None) -> Any:
return None
_DEFAULT_CONFIG_BACKEND: ConfigBackend = _DefaultConfigBackend()
class ModuleRegistry:
"""
模块注册中心 - 单例模式
职责:
- 注册模块定义(仅元数据,不加载重依赖)
- 检查模块可用性(环境变量 + 依赖库)
- 管理模块启用状态(数据库配置)
- 提供模块状态查询
"""
_instance: ModuleRegistry | None = None
_config_backend: ConfigBackend | None = None
def __init__(self) -> None:
self._modules: dict[str, ModuleDefinition] = {}
self._initialized: set[str] = set()
@classmethod
def set_config_backend(cls, backend: ConfigBackend) -> None:
"""注入配置读写后端,消除 core→services 的运行时依赖"""
cls._config_backend = backend
@classmethod
def get_instance(cls) -> ModuleRegistry:
"""获取单例实例"""
if cls._instance is None:
cls._instance = cls()
return cls._instance
@classmethod
def reset_instance(cls) -> None:
"""重置单例(仅用于测试)"""
cls._instance = None
cls._config_backend = None
def register(self, module: ModuleDefinition) -> None:
"""
注册模块
仅注册元数据,不加载重依赖
"""
name = module.metadata.name
if name in self._modules:
logger.warning(f"Module [{name}] already registered, skipping")
return
self._modules[name] = module
logger.debug(f"Module [{name}] registered")
def get_module(self, name: str) -> ModuleDefinition | None:
"""获取模块定义"""
return self._modules.get(name)
def get_all_modules(self) -> list[ModuleDefinition]:
"""获取所有已注册模块"""
return list(self._modules.values())
# ========== 可用性检查(部署级)==========
def is_available(self, name: str) -> bool:
"""
检查模块是否部署可用
检查顺序:
1. 模块是否已注册
2. 环境变量是否启用
3. 依赖的 Python 包是否安装
4. 自定义依赖检测(如果有)
"""
if name not in self._modules:
return False
module = self._modules[name]
meta = module.metadata
# 1. 检查环境变量
env_value = os.getenv(meta.env_key)
if env_value is not None:
if env_value.lower() not in ("true", "1", "yes"):
return False
elif not meta.default_available:
return False
# 2. 检查依赖的 Python 包
for pkg in meta.required_packages:
if importlib.util.find_spec(pkg) is None:
logger.debug(f"Module [{name}] unavailable: package '{pkg}' not installed")
return False
# 3. 自定义依赖检测
if module.check_dependencies:
try:
if not module.check_dependencies():
logger.debug(f"Module [{name}] unavailable: custom dependency check failed")
return False
except Exception as e:
logger.warning(f"Module [{name}] dependency check error: {e}")
return False
return True
def get_available_modules(self) -> list[ModuleDefinition]:
"""获取所有部署可用的模块"""
return [m for m in self._modules.values() if self.is_available(m.metadata.name)]
# ========== 启用状态检查(运行级)==========
def _get_config_backend(self) -> ConfigBackend:
"""获取配置后端(优先使用已注入的)。"""
if self._config_backend is not None:
return self._config_backend
# 兜底best-effort 动态加载(避免 core→services 的静态依赖)。
try:
module = importlib.import_module("src.services.system.config")
backend = getattr(module, "SystemConfigService", None)
if backend is not None:
return backend # type: ignore[return-value]
except Exception:
pass
# 最终兜底:未注入且无法动态加载时,使用默认后端(始终返回 default
return _DEFAULT_CONFIG_BACKEND
def is_enabled(self, name: str, db: Session) -> bool:
"""
检查模块是否运行启用(数据库配置)
Args:
name: 模块名称
db: 数据库会话
"""
config_key = f"module.{name}.enabled"
value = self._get_config_backend().get_config(db, config_key, default=False)
return bool(value)
def set_enabled(self, name: str, enabled: bool, db: Session) -> None:
"""
设置模块启用状态
Args:
name: 模块名称
enabled: 是否启用
db: 数据库会话
"""
if name not in self._modules:
raise ValueError(f"Module [{name}] not registered")
config_key = f"module.{name}.enabled"
module = self._modules[name]
description = f"模块 [{module.metadata.display_name}] 启用状态"
self._get_config_backend().set_config(db, config_key, enabled, description)
# ========== 激活状态检查 ==========
def is_active(self, name: str, db: Session, _visited: set[str] | None = None) -> bool:
"""
检查模块是否最终激活
激活条件available && enabled && 依赖模块都激活
Args:
name: 模块名称
db: 数据库会话
_visited: 内部递归防御,防止循环依赖导致无限递归
"""
if _visited is None:
_visited = set()
if name in _visited:
logger.warning(f"Circular dependency detected in module activation chain: {name}")
return False
_visited.add(name)
if not self.is_available(name):
return False
if not self.is_enabled(name, db):
return False
# 检查依赖模块
module = self._modules[name]
for dep in module.metadata.dependencies:
if not self.is_active(dep, db, _visited):
return False
return True
# ========== 配置验证 ==========
def validate_config(self, name: str, db: Session) -> tuple[bool, str]:
"""
验证模块配置是否有效
Args:
name: 模块名称
db: 数据库会话
Returns:
(validated, error_message) - validated 为 True 表示配置有效
"""
if name not in self._modules:
return False, "模块不存在"
module = self._modules[name]
# 没有配置验证函数的模块,默认配置有效
if not module.validate_config:
return True, ""
try:
return module.validate_config(db)
except Exception as e:
logger.warning(f"Module [{name}] config validation error: {e}")
return False, f"配置验证出错: {str(e)}"
def reconcile_module_state(self, name: str, db: Session) -> None:
"""
修复模块启用状态与配置的一致性
如果模块已启用但配置验证失败(例如依赖的 Provider Key 被删除),
则自动禁用该模块以保证状态一致性。
此方法是显式的写操作,应在需要状态修复的场景中调用,
而非在纯查询方法中隐式执行。
"""
if name not in self._modules:
return
if not self.is_available(name):
return
config_validated, config_error = self.validate_config(name, db)
if self.is_enabled(name, db) and not config_validated:
self.set_enabled(name, False, db)
logger.info(
f"Module [{name}] auto-disabled: config validation failed" f" ({config_error})"
)
# ========== 状态查询 ==========
def get_module_status(
self, name: str, db: Session, health: ModuleHealth | None = None
) -> ModuleStatus | None:
"""
获取单个模块状态
Args:
name: 模块名称
db: 数据库会话
health: 预先获取的健康状态(可选,用于异步场景)
"""
if name not in self._modules:
return None
module = self._modules[name]
meta = module.metadata
available = self.is_available(name)
# 获取配置验证状态
config_validated = False
config_error: str | None = None
if available:
config_validated, config_error = self.validate_config(name, db)
if config_validated:
config_error = None # 验证通过时清空错误信息
# 获取启用状态
enabled = self.is_enabled(name, db) if available else False
# 计算激活状态available && enabled && config_validated && 依赖模块都激活
is_active = self.is_active(name, db) if available else False
active = is_active and config_validated
return ModuleStatus(
name=name,
available=available,
enabled=enabled,
active=active,
config_validated=config_validated,
config_error=config_error,
display_name=meta.display_name,
description=meta.description,
category=meta.category,
admin_route=meta.admin_route if available else None,
admin_menu_icon=meta.admin_menu_icon,
admin_menu_group=meta.admin_menu_group,
admin_menu_order=meta.admin_menu_order,
health=health if health else ModuleHealth.UNKNOWN,
)
async def check_health(self, name: str) -> ModuleHealth:
"""
执行模块健康检查
Args:
name: 模块名称
Returns:
健康状态
"""
if name not in self._modules:
return ModuleHealth.UNKNOWN
module = self._modules[name]
if not module.health_check:
return ModuleHealth.UNKNOWN
try:
return await module.health_check()
except Exception as e:
logger.warning(f"Module [{name}] health check failed: {e}")
return ModuleHealth.UNHEALTHY
async def get_module_status_async(self, name: str, db: Session) -> ModuleStatus | None:
"""异步获取模块状态(包含健康检查)"""
if name not in self._modules:
return None
self.reconcile_module_state(name, db)
health = await self.check_health(name) if self.is_available(name) else ModuleHealth.UNKNOWN
return self.get_module_status(name, db, health=health)
async def get_all_status_async(self, db: Session) -> dict[str, ModuleStatus]:
"""异步获取所有模块状态(包含健康检查)"""
result = {}
for name in self._modules:
status = await self.get_module_status_async(name, db)
if status:
result[name] = status
return result
def get_all_status(self, db: Session) -> dict[str, ModuleStatus]:
"""获取所有模块状态(同步版本,不含健康检查)"""
result = {}
for name in self._modules:
self.reconcile_module_state(name, db)
status = self.get_module_status(name, db)
if status:
result[name] = status
return result
def get_available_status(self, db: Session) -> dict[str, ModuleStatus]:
"""获取所有可用模块的状态"""
result = {}
for name, module in self._modules.items():
if self.is_available(name):
status = self.get_module_status(name, db)
if status:
result[name] = status
return result
def get_auth_modules_status(self, db: Session) -> list[ModuleStatus]:
"""获取认证模块状态(供登录页使用)"""
result = []
for name, module in self._modules.items():
if module.metadata.category == ModuleCategory.AUTH:
if self.is_available(name):
status = self.get_module_status(name, db)
if status:
result.append(status)
return result
def get_module_registry() -> ModuleRegistry:
"""获取模块注册中心实例"""
return ModuleRegistry.get_instance()