i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 01:10:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(proxy): 重构 Proxy 节点管理与隧道系统

Browse Source 
- 重构 proxy_nodes 管理端，支持节点注册、心跳、隧道生命周期管理
- 增强 tunnel 嵌入式 hub 和隧道协议
- 重构 aether-proxy 配置、隧道客户端、心跳和调度机制
- 调整 admin OAuth/配额/导入等处理器的参数传递
- 扩展数据迁移模块
- 补充 proxy nodes、OAuth、配额、系统导入等测试
- 更新前端 proxy nodes 视图和 API
This commit is contained in:
fawney19
2026-04-14 22:51:02 +08:00
parent fb31928e44
commit a4e7ac1df6
55 changed files with 3477 additions and 892 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
crates/aether-admin/src/system.rs
View File

@@ -39,7 +39,7 @@ pub struct AdminEmailTemplateUpdate {
}
pub const ADMIN_SYSTEM_CONFIG_EXPORT_VERSION: &str = "2.2";
pub const ADMIN_SYSTEM_CONFIG_SUPPORTED_VERSIONS: &[&str] = &["2.0", "2.1", "2.2"];
pub const ADMIN_SYSTEM_CONFIG_SUPPORTED_VERSIONS: &[&str] = &[ADMIN_SYSTEM_CONFIG_EXPORT_VERSION];
pub const ADMIN_SYSTEM_PROVIDER_OPS_SENSITIVE_CREDENTIAL_FIELDS: &[&str] = &[
"api_key",
"password",
@@ -63,9 +63,7 @@ fn invalid_request(detail: impl Into<String>) -> (http::StatusCode, serde_json::
)
}
fn deserialize_optional_f64_from_number_or_string<'de, D>(
deserializer: D,
) -> Result<Option<f64>, D::Error>
fn deserialize_optional_f64_from_number<'de, D>(deserializer: D) -> Result<Option<f64>, D::Error>
where
D: serde::Deserializer<'de>,
{
@@ -77,16 +75,7 @@ where
.filter(|value| value.is_finite())
.map(Some)
.ok_or_else(|| de::Error::custom("expected a finite number")),
Some(Value::String(raw)) => raw
.trim()
.parse::<f64>()
.ok()
.filter(|value| value.is_finite())
.map(Some)
.ok_or_else(|| de::Error::custom("expected a finite number or numeric string")),
Some(_) => Err(de::Error::custom(
"expected a finite number or numeric string",
)),
Some(_) => Err(de::Error::custom("expected a finite number")),
}
}
@@ -166,10 +155,7 @@ pub struct AdminSystemConfigImportStats {
pub struct AdminSystemConfigGlobalModel {
pub name: String,
pub display_name: String,
#[serde(
default,
deserialize_with = "deserialize_optional_f64_from_number_or_string"
)]
#[serde(default, deserialize_with = "deserialize_optional_f64_from_number")]
pub default_price_per_request: Option<f64>,
#[serde(default)]
pub default_tiered_pricing: Option<Value>,
@@ -258,10 +244,7 @@ pub struct AdminSystemConfigProviderModel {
pub provider_model_name: String,
#[serde(default)]
pub provider_model_mappings: Option<Value>,
#[serde(
default,
deserialize_with = "deserialize_optional_f64_from_number_or_string"
)]
#[serde(default, deserialize_with = "deserialize_optional_f64_from_number")]
pub price_per_request: Option<f64>,
#[serde(default)]
pub tiered_pricing: Option<Value>,
@@ -292,10 +275,7 @@ pub struct AdminSystemConfigProvider {
pub provider_type: Option<String>,
#[serde(default)]
pub billing_type: Option<String>,
#[serde(
default,
deserialize_with = "deserialize_optional_f64_from_number_or_string"
)]
#[serde(default, deserialize_with = "deserialize_optional_f64_from_number")]
pub monthly_quota_usd: Option<f64>,
#[serde(default)]
pub quota_reset_day: Option<u64>,
@@ -311,15 +291,9 @@ pub struct AdminSystemConfigProvider {
pub concurrent_limit: Option<i32>,
#[serde(default)]
pub max_retries: Option<i32>,
#[serde(
default,
deserialize_with = "deserialize_optional_f64_from_number_or_string"
)]
#[serde(default, deserialize_with = "deserialize_optional_f64_from_number")]
pub stream_first_byte_timeout: Option<f64>,
#[serde(
default,
deserialize_with = "deserialize_optional_f64_from_number_or_string"
)]
#[serde(default, deserialize_with = "deserialize_optional_f64_from_number")]
pub request_timeout: Option<f64>,
#[serde(default)]
pub proxy: Option<Value>,
@@ -1888,8 +1862,31 @@ mod tests {
#[test]
fn parse_admin_system_config_import_request_accepts_supported_versions() {
for version in ADMIN_SYSTEM_CONFIG_SUPPORTED_VERSIONS {
let parsed = parse_admin_system_config_import_request(
let parsed = parse_admin_system_config_import_request(
json!({
"version": ADMIN_SYSTEM_CONFIG_EXPORT_VERSION,
"global_models": [],
"providers": [],
})
.to_string()
.as_bytes(),
)
.expect("current version should parse");
assert_eq!(
parsed.request.document.version,
ADMIN_SYSTEM_CONFIG_EXPORT_VERSION
);
assert_eq!(parsed.request.merge_mode, AdminImportMergeMode::Skip);
assert!(parsed.request.document.oauth_providers.is_empty());
assert!(parsed.request.document.system_configs.is_empty());
assert!(parsed.request.document.ldap_config.is_none());
}
#[test]
fn parse_admin_system_config_import_request_rejects_removed_versions() {
for version in ["2.0", "2.1"] {
let err = parse_admin_system_config_import_request(
json!({
"version": version,
"global_models": [],
@@ -1898,13 +1895,16 @@ mod tests {
.to_string()
.as_bytes(),
)
.expect("supported version should parse");
.expect_err("removed versions should fail");
assert_eq!(parsed.request.document.version, *version);
assert_eq!(parsed.request.merge_mode, AdminImportMergeMode::Skip);
assert!(parsed.request.document.oauth_providers.is_empty());
assert!(parsed.request.document.system_configs.is_empty());
assert!(parsed.request.document.ldap_config.is_none());
assert_eq!(err.0, http::StatusCode::BAD_REQUEST);
assert_eq!(
err.1["detail"],
format!(
"不支持的配置版本: {version}，支持的版本: {}",
ADMIN_SYSTEM_CONFIG_SUPPORTED_VERSIONS.join(", ")
)
);
}
}
@@ -1954,8 +1954,8 @@ mod tests {
}
#[test]
fn parse_admin_system_config_import_request_accepts_numeric_string_fields() {
let parsed = parse_admin_system_config_import_request(
fn parse_admin_system_config_import_request_rejects_numeric_string_fields() {
let err = parse_admin_system_config_import_request(
json!({
"version": "2.2",
"global_models": [{
@@ -1978,27 +1978,12 @@ mod tests {
.to_string()
.as_bytes(),
)
.expect("numeric string fields should parse");
.expect_err("numeric string fields should fail");
let global_model = parsed
.request
.document
.global_models
.first()
.expect("global model should exist");
assert_eq!(global_model.default_price_per_request, Some(1.8));
let provider = parsed
.request
.document
.providers
.first()
.expect("provider should exist");
assert_eq!(provider.monthly_quota_usd, Some(12.5));
assert_eq!(provider.stream_first_byte_timeout, Some(60.0));
assert_eq!(provider.request_timeout, Some(120.0));
assert_eq!(provider.models.len(), 1);
assert_eq!(provider.models[0].price_per_request, Some(0.7));
assert_eq!(err.0, http::StatusCode::BAD_REQUEST);
let detail = err.1["detail"].as_str().expect("detail should be a string");
assert!(detail.contains("配置文件格式无效"));
assert!(detail.contains("default_price_per_request"));
}
#[test]

275
crates/aether-data/src/migrate.rs
View File

@@ -11,13 +11,30 @@ static BASELINE_V2_SQL: &str = include_str!("../bootstrap/20260413020000_baselin
const BASELINE_V2_CUTOFF_VERSION: i64 = 20260413030000;
const MIGRATIONS_TABLE_EXISTS_SQL: &str =
"SELECT to_regclass('public._sqlx_migrations') IS NOT NULL";
const EMPTY_DATABASE_USER_TABLE_COUNT_SQL: &str = r#"
const PUBLIC_BASE_TABLE_COUNT_SQL: &str = r#"
SELECT COUNT(*)::BIGINT
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_type = 'BASE TABLE'
AND table_name <> '_sqlx_migrations'
"#;
const AETHER_SCHEMA_FOOTPRINT_TABLE_COUNT_SQL: &str = r#"
SELECT COUNT(*)::BIGINT
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_type = 'BASE TABLE'
AND table_name IN (
'api_key_provider_mappings',
'auth_modules',
'gemini_file_mappings',
'global_models',
'oauth_providers',
'provider_api_keys',
'proxy_nodes',
'usage_routing_snapshots',
'usage_settlement_snapshots'
)
"#;
const INSERT_APPLIED_MIGRATION_SQL: &str = r#"
INSERT INTO _sqlx_migrations (
version,
@@ -248,10 +265,25 @@ async fn should_bootstrap_baseline_v2(conn: &mut PgConnection) -> Result<bool, M
return Ok(false);
}
let user_table_count: i64 = query_scalar(EMPTY_DATABASE_USER_TABLE_COUNT_SQL)
let public_table_count: i64 = query_scalar(PUBLIC_BASE_TABLE_COUNT_SQL)
.fetch_one(&mut *conn)
.await?;
Ok(user_table_count == 0)
if public_table_count == 0 {
return Ok(true);
}
let aether_footprint_table_count: i64 = query_scalar(AETHER_SCHEMA_FOOTPRINT_TABLE_COUNT_SQL)
.fetch_one(&mut *conn)
.await?;
if aether_footprint_table_count == 0 {
info!(
public_table_count,
"no Aether schema footprint detected; allowing baseline bootstrap despite pre-existing public tables"
);
return Ok(true);
}
Ok(false)
}
fn baseline_v2_migrations() -> Result<Vec<&'static sqlx::migrate::Migration>, MigrateError> {
@@ -348,14 +380,174 @@ fn validate_applied_migrations(
#[cfg(test)]
mod tests {
use std::borrow::Cow;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::time::{Duration, Instant};
use sqlx::migrate::AppliedMigration;
use sqlx::{migrate::AppliedMigration, query, query_scalar, Connection, PgConnection, PgPool};
use super::{
all_up_migrations, baseline_v2_migrations, pending_migrations_from_applied,
BASELINE_V2_SQL, MIGRATOR,
prepare_database_for_startup, BASELINE_V2_SQL, MIGRATOR,
};
#[derive(Debug)]
struct ManagedPostgresServer {
child: Option<Child>,
workdir: PathBuf,
database_url: String,
}
impl ManagedPostgresServer {
async fn try_start() -> Result<Option<Self>, Box<dyn std::error::Error>> {
let initdb_bin = std::env::var("AETHER_INITDB_BIN")
.ok()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| "initdb".to_string());
let postgres_bin = std::env::var("AETHER_POSTGRES_BIN")
.ok()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| "postgres".to_string());
if !command_exists(&initdb_bin) || !command_exists(&postgres_bin) {
eprintln!(
"skipping postgres integration test because required binaries are unavailable: initdb={}, postgres={}",
initdb_bin, postgres_bin
);
return Ok(None);
}
Ok(Some(Self::start(initdb_bin, postgres_bin).await?))
}
async fn start(
initdb_bin: String,
postgres_bin: String,
) -> Result<Self, Box<dyn std::error::Error>> {
let port = reserve_local_port()?;
let workdir = std::env::temp_dir().join(format!(
"aether-migrate-tests-{}-{}",
std::process::id(),
port
));
let data_dir = workdir.join("data");
std::fs::create_dir_all(&workdir)?;
let init_output = Command::new(&initdb_bin)
.arg("-D")
.arg(&data_dir)
.arg("-U")
.arg("aether")
.arg("--auth=trust")
.arg("--encoding=UTF8")
.arg("--no-instructions")
.output()?;
if !init_output.status.success() {
return Err(std::io::Error::other(format!(
"initdb failed: {}",
String::from_utf8_lossy(&init_output.stderr)
))
.into());
}
let database_url = format!("postgres://aether@127.0.0.1:{port}/postgres");
let log_path = workdir.join("postgres.log");
let stdout = std::fs::File::create(&log_path)?;
let stderr = stdout.try_clone()?;
let child = Command::new(&postgres_bin)
.arg("-D")
.arg(&data_dir)
.arg("-h")
.arg("127.0.0.1")
.arg("-p")
.arg(port.to_string())
.arg("-F")
.arg("-c")
.arg("fsync=off")
.arg("-c")
.arg("synchronous_commit=off")
.arg("-c")
.arg("full_page_writes=off")
.stdout(Stdio::from(stdout))
.stderr(Stdio::from(stderr))
.spawn()?;
wait_for_postgres(&database_url).await?;
Ok(Self {
child: Some(child),
workdir,
database_url,
})
}
fn database_url(&self) -> &str {
&self.database_url
}
fn stop(&mut self) {
if let Some(mut child) = self.child.take() {
let _ = child.kill();
let _ = child.wait();
}
}
}
impl Drop for ManagedPostgresServer {
fn drop(&mut self) {
self.stop();
let _ = std::fs::remove_dir_all(&self.workdir);
}
}
fn command_exists(bin: &str) -> bool {
if bin.contains(std::path::MAIN_SEPARATOR) {
return Path::new(bin).exists();
}
let Some(paths) = std::env::var_os("PATH") else {
return false;
};
std::env::split_paths(&paths).any(|path| path.join(bin).exists())
}
fn reserve_local_port() -> Result<u16, std::io::Error> {
let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
let port = listener.local_addr()?.port();
drop(listener);
Ok(port)
}
async fn wait_for_postgres(database_url: &str) -> Result<(), Box<dyn std::error::Error>> {
let deadline = Instant::now() + Duration::from_secs(10);
loop {
match PgConnection::connect(database_url).await {
Ok(connection) => {
connection.close().await?;
return Ok(());
}
Err(_) if Instant::now() < deadline => {
tokio::time::sleep(Duration::from_millis(50)).await
}
Err(err) => {
return Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
format!("timed out waiting for local postgres: {err}"),
)
.into())
}
}
}
}
async fn table_exists(pool: &PgPool, table_name: &str) -> Result<bool, sqlx::Error> {
query_scalar::<_, bool>("SELECT to_regclass($1) IS NOT NULL")
.bind(format!("public.{table_name}"))
.fetch_one(pool)
.await
}
#[test]
fn baseline_migration_restores_search_path_for_sqlx_bookkeeping() {
let baseline = MIGRATOR
@@ -517,4 +709,77 @@ mod tests {
"baseline_v2-stamped empty databases should not require a manual migration before first startup"
);
}
#[tokio::test]
async fn prepare_database_for_startup_bootstraps_clean_database() {
let Some(server) = ManagedPostgresServer::try_start()
.await
.expect("postgres bootstrap test should start or skip")
else {
return;
};
let pool = PgPool::connect(server.database_url())
.await
.expect("pool should connect");
let pending = prepare_database_for_startup(&pool)
.await
.expect("clean database bootstrap should succeed");
assert!(
pending.is_empty(),
"fresh databases should not report pending migrations after startup preparation"
);
assert!(table_exists(&pool, "users")
.await
.expect("users lookup should succeed"));
assert!(table_exists(&pool, "usage")
.await
.expect("usage lookup should succeed"));
let applied_count: i64 =
query_scalar("SELECT COUNT(*)::BIGINT FROM public._sqlx_migrations")
.fetch_one(&pool)
.await
.expect("migration count query should succeed");
assert_eq!(
applied_count,
baseline_v2_migrations()
.expect("baseline migrations should resolve")
.len() as i64
);
}
#[tokio::test]
async fn prepare_database_for_startup_bootstraps_when_only_unrelated_public_tables_exist() {
let Some(server) = ManagedPostgresServer::try_start()
.await
.expect("postgres bootstrap test should start or skip")
else {
return;
};
let pool = PgPool::connect(server.database_url())
.await
.expect("pool should connect");
query("CREATE TABLE public.vendor_bootstrap_marker (id integer PRIMARY KEY)")
.execute(&pool)
.await
.expect("fixture table should be created");
let pending = prepare_database_for_startup(&pool)
.await
.expect("startup preparation should tolerate unrelated public tables");
assert!(
pending.is_empty(),
"unrelated public tables should not block baseline bootstrap on first startup"
);
assert!(table_exists(&pool, "vendor_bootstrap_marker")
.await
.expect("fixture table lookup should succeed"));
assert!(table_exists(&pool, "oauth_providers")
.await
.expect("oauth_providers lookup should succeed"));
}
}