refactor: 移除 Python 后端源码,全面迁移至 Rust gateway 架构

- 删除全部 Python 源码 (src/) 及 Alembic 迁移脚本,归档至 _deprecated_py_src/
- 重构 Rust gateway ai_pipeline: 拆分 planner/finalize 模块,新增 contracts/adaptation 层
- 重组 handlers 模块为 admin/public/proxy/internal/shared 子模块结构
- 新增 executor 模块,引入 Rust 原生数据库迁移 (aether-data/migrations)
- 简化 CI/Docker 构建流程,移除 base image 二级构建,统一为单一 app image
- 移除 Python 相关基础设施文件 (entrypoint.sh, gunicorn_conf.py, Dockerfile.base)
This commit is contained in:
fawney19
2026-04-03 16:26:16 +08:00
parent 8f26e1a31f
commit 1d9c77522a
868 changed files with 1735 additions and 2433 deletions

View File

@@ -0,0 +1,57 @@
"""
功能模块注册 -- 自动发现
扫描 src/modules/ 下的子目录,自动查找 ModuleDefinition 实例。
新增模块只需创建 src/modules/<name>/__init__.py 并导出 ModuleDefinition无需修改此文件。
"""
import importlib
from pathlib import Path
from src.core.logger import logger
from src.core.modules.base import ModuleDefinition
def discover_modules() -> list[ModuleDefinition]:
"""
自动发现所有模块定义
扫描 src/modules/ 下的每个子目录,导入其 __init__.py
查找所有 ModuleDefinition 实例并返回。
"""
modules_dir = Path(__file__).parent
discovered: list[ModuleDefinition] = []
seen_names: set[str] = set()
for child in sorted(modules_dir.iterdir()):
if not child.is_dir():
continue
if child.name.startswith("_"):
continue
if not (child / "__init__.py").exists():
continue
module_path = f"src.modules.{child.name}"
try:
mod = importlib.import_module(module_path)
except Exception as e:
logger.error("Failed to import module {}: {}", module_path, e)
continue
# 扫描模块顶层属性,查找 ModuleDefinition 实例
for obj in vars(mod).values():
if isinstance(obj, ModuleDefinition):
name = obj.metadata.name
if name in seen_names:
logger.warning("Duplicate module name '{}' in {}, skipping", name, module_path)
continue
seen_names.add(name)
discovered.append(obj)
logger.debug("Discovered module: {} from {}", name, module_path)
return discovered
ALL_MODULES: list[ModuleDefinition] = discover_modules()
__all__ = ["ALL_MODULES", "discover_modules"]

View File

@@ -0,0 +1,96 @@
"""
Gemini Files 文件管理模块
提供 Gemini Files API 文件上传和管理功能:
- 文件上传到 Google Gemini Files API
- 文件映射管理file_id → key_id
- 文件列表查看和删除
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from src.core.modules.base import (
ModuleCategory,
ModuleDefinition,
ModuleHealth,
ModuleMetadata,
)
if TYPE_CHECKING:
from sqlalchemy.orm import Session
def _get_router() -> Any:
"""延迟导入路由"""
from src.api.admin.gemini_files import router
return router
async def _health_check() -> ModuleHealth:
"""健康检查"""
# 检查是否有可用的 gemini_files 能力的 Key
from src.database import create_session
from src.models.database import ProviderAPIKey
db = create_session()
try:
# 查找有 gemini_files 能力的 Key
keys = db.query(ProviderAPIKey).filter(ProviderAPIKey.is_active.is_(True)).all()
has_capable_key = any(
key.capabilities and key.capabilities.get("gemini_files", False) for key in keys
)
if has_capable_key:
return ModuleHealth.HEALTHY
return ModuleHealth.DEGRADED
except Exception:
return ModuleHealth.UNKNOWN
finally:
db.close()
def _validate_config(db: Session) -> tuple[bool, str]:
"""
验证 Gemini Files 模块配置
检查项:
1. 至少有一个有 gemini_files 能力的 Provider Key
"""
from src.models.database import ProviderAPIKey
# 查找有 gemini_files 能力的 Key
keys = db.query(ProviderAPIKey).filter(ProviderAPIKey.is_active.is_(True)).all()
capable_keys = [
key for key in keys if key.capabilities and key.capabilities.get("gemini_files", False)
]
if not capable_keys:
return (
False,
"至少启用一个具有「Gemini 文件 API」能力的 Key",
)
return True, ""
gemini_files_module = ModuleDefinition(
metadata=ModuleMetadata(
name="gemini_files",
display_name="文件缓存",
description="管理 Gemini Files API 上传的文件,支持文件上传、查看和删除",
category=ModuleCategory.INTEGRATION,
env_key="GEMINI_FILES_AVAILABLE",
default_available=True,
required_packages=[],
api_prefix="/api/admin/gemini-files",
admin_route="/admin/gemini-files",
admin_menu_icon="FileUp",
admin_menu_group="system",
admin_menu_order=60,
),
router_factory=_get_router,
health_check=_health_check,
validate_config=_validate_config,
)

View File

@@ -0,0 +1,193 @@
"""
LDAP 认证模块
提供 LDAP/Active Directory 用户认证支持
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from src.core.modules.base import (
ModuleCategory,
ModuleDefinition,
ModuleHealth,
ModuleMetadata,
)
if TYPE_CHECKING:
from sqlalchemy.orm import Session
def _get_router() -> Any:
"""延迟导入路由(避免启动时加载重依赖)"""
from src.api.admin.ldap import router
return router
async def _health_check() -> ModuleHealth:
"""健康检查 - 简化版,不依赖数据库连接"""
# 健康检查在启动时调用,此时可能没有数据库会话
# 返回 UNKNOWN 表示需要进一步检查
return ModuleHealth.UNKNOWN
def _validate_config(db: Session) -> tuple[bool, str]:
"""
验证 LDAP 配置是否可以启用模块
检查项:
1. 配置是否存在
2. 必填字段是否完整
3. 绑定密码是否可解密
注意:不在此处执行连接测试,因为 validate_config 会在每次查询模块状态时调用,
同步阻塞等待 LDAP 服务器响应会严重影响性能。连接测试应在专门的测试接口中进行。
"""
from src.core.crypto import crypto_service
from src.models.database import LDAPConfig
config = db.query(LDAPConfig).first()
if not config:
return False, "请先配置 LDAP 连接信息"
# 检查必填字段
if not config.server_url:
return False, "请配置 LDAP 服务器地址"
if not config.bind_dn:
return False, "请配置绑定 DN"
if not config.base_dn:
return False, "请配置搜索基准 DN"
if not config.bind_password_encrypted:
return False, "请配置绑定密码"
# 尝试解密密码(仅验证可解密,不执行连接测试)
try:
bind_password = crypto_service.decrypt(config.bind_password_encrypted)
if not bind_password:
return False, "绑定密码为空,请重新设置"
except Exception:
return False, "绑定密码解密失败,请重新设置"
return True, ""
# ==================== 钩子实现 ====================
def _hook_get_auth_methods(db: Session) -> list[dict[str, Any]]:
"""auth.get_methods: 返回 LDAP 认证方法信息"""
from src.services.auth.ldap import LDAPService
if not LDAPService.is_ldap_enabled(db):
return []
is_exclusive = LDAPService.is_ldap_exclusive(db)
return [
{
"type": "ldap",
"enabled": True,
"exclusive": is_exclusive,
}
]
async def _hook_authenticate(db: Session, email: str, password: str, auth_type: str) -> Any:
"""auth.authenticate: LDAP 认证
仅当 auth_type == "ldap" 时处理,否则返回 None 让其他模块尝试。
"""
if auth_type != "ldap":
return None
import asyncio
from starlette.concurrency import run_in_threadpool
from src.core.logger import logger
from src.services.auth.ldap import LDAPService
# 预取配置,避免将 Session 传递到线程池
config_data = LDAPService.get_config_data(db)
if not config_data:
logger.warning("登录失败 - LDAP 未启用或配置无效")
return None
# 计算总体超时
single_timeout = config_data.get("connect_timeout", 10)
total_timeout = max(20, min(int(single_timeout * 4 * 1.1), 60))
try:
ldap_user = await asyncio.wait_for(
run_in_threadpool(LDAPService.authenticate_with_config, config_data, email, password),
timeout=total_timeout,
)
except TimeoutError:
logger.error("LDAP 认证总体超时({}秒): {}", total_timeout, email)
return None
if not ldap_user:
return None
# 获取或创建本地用户
from src.services.auth.service import AuthService
user = await AuthService.get_or_create_ldap_user(db, ldap_user)
if not user:
return None
if user.is_deleted:
logger.warning("登录失败 - 用户已删除: {}", email)
return None
if not user.is_active:
logger.warning("登录失败 - 用户已禁用: {}", email)
return None
return user
def _hook_check_exclusive_mode(db: Session) -> bool | None:
"""auth.check_exclusive_mode: 检查 LDAP 排他登录模式"""
from src.services.auth.ldap import LDAPService
if LDAPService.is_ldap_exclusive(db):
return True
return None
def _hook_check_registration(db: Session) -> dict[str, Any] | None:
"""auth.check_registration: LDAP 排他模式下阻止本地注册"""
from src.services.auth.ldap import LDAPService
if LDAPService.is_ldap_exclusive(db):
return {"blocked": True, "reason": "系统已启用 LDAP 专属登录,禁止本地注册"}
return None
# LDAP 模块定义
ldap_module = ModuleDefinition(
metadata=ModuleMetadata(
name="ldap",
display_name="LDAP 认证",
description="支持通过 LDAP/Active Directory 进行用户认证",
category=ModuleCategory.AUTH,
# 可用性控制
env_key="LDAP_AVAILABLE",
default_available=True,
required_packages=["ldap3"],
# 路由配置(使用现有路由,不改变路径)
api_prefix="/api/admin/ldap",
# 前端配置
admin_route="/admin/ldap",
admin_menu_icon="Users",
admin_menu_group="system",
admin_menu_order=50,
),
router_factory=_get_router,
health_check=_health_check,
validate_config=_validate_config,
hooks={
"auth.get_methods": _hook_get_auth_methods,
"auth.authenticate": _hook_authenticate,
"auth.check_exclusive_mode": _hook_check_exclusive_mode,
"auth.check_registration": _hook_check_registration,
},
)

View File

@@ -0,0 +1,104 @@
"""
访问令牌模块
提供管理 API 访问令牌的功能
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from fastapi import APIRouter
from src.core.modules.base import (
ModuleCategory,
ModuleDefinition,
ModuleHealth,
ModuleMetadata,
)
if TYPE_CHECKING:
from sqlalchemy.orm import Session
def _get_router() -> Any:
"""延迟导入路由(避免启动时加载重依赖)
返回一个合并了管理员和用户两个路由的 APIRouter
- /api/admin/management-tokens: 管理员管理所有用户的令牌
- /api/me/management-tokens: 用户管理自己的令牌
"""
from src.api.admin.management_tokens import router as admin_router
from src.api.user_me.management_tokens import router as user_router
# 创建一个组合路由器
combined_router = APIRouter()
combined_router.include_router(admin_router)
combined_router.include_router(user_router)
return combined_router
async def _health_check() -> ModuleHealth:
"""健康检查 - 简化版,不依赖数据库连接"""
return ModuleHealth.HEALTHY
def _validate_config(db: Session) -> tuple[bool, str]:
"""
验证配置是否可以启用模块
访问令牌模块没有特殊配置要求,始终可用
"""
return True, ""
# ==================== 钩子实现 ====================
async def _authenticate_by_token(db: Any, token: str, client_ip: str) -> tuple[Any, Any] | None:
"""执行 Management Token 认证"""
from src.services.auth.service import AuthService
return await AuthService.authenticate_management_token(db, token, client_ip)
def _hook_token_prefix_authenticators(**_kwargs: Any) -> list[dict[str, Any]]:
"""auth.token_prefix_authenticators: 声明 ae_ 前缀认证器"""
from src.models.database import ManagementToken
return [
{
"prefix": ManagementToken.TOKEN_PREFIX,
"module": "management_tokens",
"authenticate": _authenticate_by_token,
}
]
# 访问令牌模块定义
management_tokens_module = ModuleDefinition(
metadata=ModuleMetadata(
name="management_tokens",
display_name="访问令牌",
description="管理 API 访问令牌,支持细粒度权限控制和 IP 白名单",
category=ModuleCategory.SECURITY,
# 可用性控制
env_key="MANAGEMENT_TOKENS_AVAILABLE",
default_available=True,
required_packages=[],
# 路由配置
api_prefix="/api/admin/management-tokens, /api/me/management-tokens",
# 前端配置 - 不在导航栏显示,但保留路由用于直接访问
admin_route="/admin/management-tokens",
admin_menu_icon=None,
admin_menu_group=None, # 不显示在导航菜单中
admin_menu_order=0,
),
router_factory=_get_router,
health_check=_health_check,
validate_config=_validate_config,
hooks={
"auth.token_prefix_authenticators": _hook_token_prefix_authenticators,
},
)

View File

@@ -0,0 +1,46 @@
"""
通知邮件模块
提供错误通知邮件发送开关,并复用 SMTP 配置页。
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from src.core.modules.base import ModuleCategory, ModuleDefinition, ModuleMetadata
if TYPE_CHECKING:
from sqlalchemy.orm import Session
def _validate_config(db: Session) -> tuple[bool, str]:
"""
验证通知邮件模块配置。
启用要求SMTP 基础配置有效(至少 host + from_email
"""
from src.services.email.email_sender import EmailSenderService
if not EmailSenderService.is_smtp_configured(db):
return False, "请先完成邮件配置SMTP"
return True, ""
notification_email_module = ModuleDefinition(
metadata=ModuleMetadata(
name="notification_email",
display_name="异常通知",
description="为 5xx 异常发送邮件通知,可在模块管理中启用或禁用",
category=ModuleCategory.INTEGRATION,
env_key="NOTIFICATION_EMAIL_AVAILABLE",
default_available=True,
required_packages=[],
# 通知邮件与 SMTP 配置复用同一页面,不在模块卡片展示独立“配置”入口。
admin_route=None,
admin_menu_icon="Mail",
admin_menu_group="system",
admin_menu_order=58,
),
validate_config=_validate_config,
)

View File

@@ -0,0 +1,85 @@
"""
OAuth 认证模块
提供可配置的 OAuth 登录/绑定能力。
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from src.core.modules.base import (
ModuleCategory,
ModuleDefinition,
ModuleHealth,
ModuleMetadata,
)
if TYPE_CHECKING:
from sqlalchemy.orm import Session
def _get_router() -> Any:
"""延迟导入路由(避免启动时加载重依赖/副作用)。"""
# 延迟 discover避免 alembic/mypy 等场景导入时触发 entry_points 解析
from src.services.auth.oauth.registry import get_oauth_provider_registry
get_oauth_provider_registry().discover_providers()
from src.api.oauth import router
return router
async def _health_check() -> ModuleHealth:
# v1不做外部网络探测避免启动时阻塞
return ModuleHealth.UNKNOWN
def _validate_config(db: Session) -> tuple[bool, str]:
"""
验证 OAuth 配置是否可以启用模块
检查项:
1. 至少有一个已启用的 Provider 配置
2. 已启用的 Provider 必须有 client_id 和 client_secret
"""
from src.models.database import OAuthProvider
# 查找所有已启用的 Provider
enabled_providers = db.query(OAuthProvider).filter(OAuthProvider.is_enabled.is_(True)).all()
if not enabled_providers:
return False, "请先配置并启用至少一个 OAuth Provider"
# 检查每个已启用的 Provider 配置完整性
for provider in enabled_providers:
if not provider.client_id:
return False, f"Provider [{provider.display_name}] 未配置 Client ID"
if not provider.client_secret_encrypted:
return False, f"Provider [{provider.display_name}] 未配置 Client Secret"
if not provider.redirect_uri:
return False, f"Provider [{provider.display_name}] 未配置回调地址"
return True, ""
oauth_module = ModuleDefinition(
metadata=ModuleMetadata(
name="oauth",
display_name="OAuth 登录",
description="支持通过第三方 OAuth Provider 登录/绑定账号",
category=ModuleCategory.AUTH,
env_key="OAUTH_AVAILABLE",
default_available=True,
required_packages=["httpx", "redis"],
api_prefix="/api/oauth",
admin_route="/admin/oauth",
admin_menu_icon="Key",
admin_menu_group="system",
admin_menu_order=55,
),
router_factory=_get_router,
health_check=_health_check,
validate_config=_validate_config,
)

View File

@@ -0,0 +1,221 @@
"""
代理节点模块
提供海外 VPS 代理节点的注册、心跳、管理功能。
aether-proxy 部署在海外 VPS 上,通过 WebSocket 隧道连接 Aether 转发 API 请求。
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from fastapi import APIRouter
from starlette.routing import BaseRoute
from src.core.modules.base import (
ModuleCategory,
ModuleDefinition,
ModuleHealth,
ModuleMetadata,
)
if TYPE_CHECKING:
from sqlalchemy.orm import Session
_proxy_node_task_coordinator: Any | None = None
_RUST_OWNED_PROXY_NODE_ROUTE_SIGNATURES = frozenset(
{
("GET", "/api/admin/proxy-nodes"),
("GET", "/api/admin/proxy-nodes/"),
("POST", "/api/admin/proxy-nodes/register"),
("POST", "/api/admin/proxy-nodes/heartbeat"),
("POST", "/api/admin/proxy-nodes/unregister"),
("POST", "/api/admin/proxy-nodes/manual"),
("POST", "/api/admin/proxy-nodes/upgrade"),
("POST", "/api/admin/proxy-nodes/test-url"),
("PATCH", "/api/admin/proxy-nodes/{node_id}"),
("DELETE", "/api/admin/proxy-nodes/{node_id}"),
("POST", "/api/admin/proxy-nodes/{node_id}/test"),
("PUT", "/api/admin/proxy-nodes/{node_id}/config"),
("GET", "/api/admin/proxy-nodes/{node_id}/events"),
}
)
def _route_is_rust_owned(route: BaseRoute) -> bool:
path = getattr(route, "path", None)
methods = getattr(route, "methods", None)
if not isinstance(path, str) or not methods:
return False
return any(
(method, path) in _RUST_OWNED_PROXY_NODE_ROUTE_SIGNATURES
for method in methods
if method not in {"HEAD", "OPTIONS"}
)
def _reset_tunnel_connected_on_startup() -> None:
"""服务端启动时将所有 tunnel_connected=True 的节点重置为 False/OFFLINE。
服务端重启后 DB 中可能残留 tunnel_connected=True 的记录。
如果不重置,节点会在 tunnel 状态广播到来前短暂显示 ONLINE。
"""
from datetime import datetime, timezone
from src.core.logger import logger
from src.database import create_session
from src.models.database import ProxyNode, ProxyNodeStatus
db = create_session()
try:
now = datetime.now(timezone.utc)
stale_nodes = (
db.query(ProxyNode)
.filter(
ProxyNode.tunnel_connected == True, # noqa: E712
ProxyNode.is_manual == False, # noqa: E712
)
.all()
)
if stale_nodes:
for node in stale_nodes:
node.tunnel_connected = False
node.tunnel_connected_at = now
node.status = ProxyNodeStatus.OFFLINE
node.updated_at = now
db.commit()
logger.info(
"重置 {} 个节点的残留 tunnel 连接状态 (tunnel_connected -> False)",
len(stale_nodes),
)
except Exception as e:
try:
db.rollback()
except Exception:
pass
logger.warning("重置 tunnel 连接状态失败: {}", e)
finally:
db.close()
def _get_router() -> Any:
"""延迟导入路由"""
from src.api.admin.proxy_nodes import router
filtered_router = APIRouter()
filtered_router.include_router(router)
filtered_router.routes = [
route for route in filtered_router.routes if not _route_is_rust_owned(route)
]
return filtered_router
async def _on_startup() -> None:
"""启动心跳检测调度器"""
import logging
from src.config import config
from src.services.proxy_node.health_scheduler import get_proxy_node_health_scheduler
from src.utils.task_coordinator import StartupTaskCoordinator
logger = logging.getLogger("aether.modules.proxy_nodes")
if config.worker_processes > 1:
logger.info(
"检测到 WEB_CONCURRENCY={},隧道模式允许多 worker 共享 tunnel。",
config.worker_processes,
)
from src.clients import get_redis_client
global _proxy_node_task_coordinator
redis_client = await get_redis_client()
task_coordinator = StartupTaskCoordinator(redis_client)
_proxy_node_task_coordinator = task_coordinator
proxy_node_health_scheduler = get_proxy_node_health_scheduler()
active = await task_coordinator.acquire("proxy_node_health")
if active:
# 仅 leader worker 执行启动重置,避免多 worker 并发启动/重启时
# 把其他 worker 已建立的 tunnel 状态错误重置为 OFFLINE。
_reset_tunnel_connected_on_startup()
else:
logger.info("检测到其他 worker 已运行 ProxyNode 心跳检测,本实例跳过")
if active:
logger.info("启动 ProxyNode 心跳检测调度器...")
await proxy_node_health_scheduler.start()
async def _on_lock_lost(_name: str) -> None:
await _stop_proxy_node_scheduler_on_lock_lost(logger)
task_coordinator.register_lock_lost_callback("proxy_node_health", _on_lock_lost)
async def _on_shutdown() -> None:
"""停止心跳检测调度器"""
import logging
from src.services.proxy_node.health_scheduler import get_proxy_node_health_scheduler
from src.utils.task_coordinator import StartupTaskCoordinator
logger = logging.getLogger("aether.modules.proxy_nodes")
from src.clients import get_redis_client
global _proxy_node_task_coordinator
if _proxy_node_task_coordinator is None:
redis_client = await get_redis_client()
_proxy_node_task_coordinator = StartupTaskCoordinator(redis_client)
scheduler = get_proxy_node_health_scheduler()
if scheduler.running:
logger.info("停止 ProxyNode 心跳检测调度器...")
await scheduler.stop()
await _proxy_node_task_coordinator.release("proxy_node_health")
_proxy_node_task_coordinator = None
async def _stop_proxy_node_scheduler_on_lock_lost(logger: Any) -> None:
from src.services.proxy_node.health_scheduler import get_proxy_node_health_scheduler
scheduler = get_proxy_node_health_scheduler()
if not scheduler.running:
return
logger.warning("检测到 proxy_node_health 的 leader 锁已丢失,停止本实例的 ProxyNode 心跳检测")
await scheduler.stop()
async def _health_check() -> ModuleHealth:
"""健康检查 - 检查是否有在线节点"""
return ModuleHealth.HEALTHY
def _validate_config(db: Session) -> tuple[bool, str]:
"""验证配置tunnel 模式无需额外密钥配置)"""
return True, ""
proxy_nodes_module = ModuleDefinition(
metadata=ModuleMetadata(
name="proxy_nodes",
display_name="代理节点",
description="添加Http/Socket代理节点, 或使用Aether-Proxy自动连接代理节点.",
category=ModuleCategory.INTEGRATION,
env_key="PROXY_NODES_AVAILABLE",
default_available=True,
required_packages=[],
api_prefix="/api/admin/proxy-nodes",
admin_route="/admin/proxy-nodes",
admin_menu_icon="Server",
admin_menu_group="system",
admin_menu_order=60,
),
router_factory=_get_router,
on_startup=_on_startup,
on_shutdown=_on_shutdown,
health_check=_health_check,
validate_config=_validate_config,
)